diff --git a/outputs/eval/logs/token_t045/babilong.log b/outputs/eval/logs/token_t045/babilong.log new file mode 100644 index 0000000000000000000000000000000000000000..64c93b8ca1b075c697ab1e1556001946446787d9 --- /dev/null +++ b/outputs/eval/logs/token_t045/babilong.log @@ -0,0 +1,312 @@ +2026-07-18:12:26:33 WARNING [config.evaluate_config:281] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT. +2026-07-18:12:26:37 INFO [_cli.run:376] Selected Tasks: ['babilong_longctx'] +2026-07-18:12:26:38 INFO [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234 +2026-07-18:12:26:38 INFO [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'} +2026-07-18:12:26:41 INFO [models.huggingface:161] Using device 'cuda:0' +2026-07-18:12:26:42 INFO [models.huggingface:548] Model type cannot be determined. Using default model type 'causal' +2026-07-18:12:26:42 INFO [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'} +2026-07-18:12:26:44 WARNING [api.task:856] babilong_qa5: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager. +For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme. +2026-07-18:12:26:44 WARNING [api.task:856] babilong_qa4: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager. +For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme. +2026-07-18:12:26:44 WARNING [api.task:856] babilong_qa3: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager. +For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme. +2026-07-18:12:26:44 WARNING [api.task:856] babilong_qa2: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager. +For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme. +2026-07-18:12:26:45 WARNING [api.task:856] babilong_qa1: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager. +For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme. +2026-07-18:12:26:45 INFO [tasks:700] Selected tasks: +2026-07-18:12:26:45 INFO [tasks:703] Group: babilong_longctx +2026-07-18:12:26:45 INFO [tasks:726] ConfigurableGroup(group=babilong_longctx,group_alias=None): {'babilong_qa1': ConfigurableTask(task_name=babilong_qa1,output_type=generate_until,num_fewshot=2,num_samples=1000), 'babilong_qa2': ConfigurableTask(task_name=babilong_qa2,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa3': ConfigurableTask(task_name=babilong_qa3,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa4': ConfigurableTask(task_name=babilong_qa4,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa5': ConfigurableTask(task_name=babilong_qa5,output_type=generate_until,num_fewshot=2,num_samples=999)} +2026-07-18:12:26:45 INFO [evaluator:314] babilong_qa1: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []} +2026-07-18:12:26:45 WARNING [evaluator:333] Overwriting default num_fewshot of babilong_qa1 from 2 to 2 +2026-07-18:12:26:45 INFO [evaluator:314] babilong_qa2: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []} +2026-07-18:12:26:45 WARNING [evaluator:333] Overwriting default num_fewshot of babilong_qa2 from 2 to 2 +2026-07-18:12:26:45 INFO [evaluator:314] babilong_qa3: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []} +2026-07-18:12:26:45 WARNING [evaluator:333] Overwriting default num_fewshot of babilong_qa3 from 2 to 2 +2026-07-18:12:26:45 INFO [evaluator:314] babilong_qa4: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []} +2026-07-18:12:26:45 WARNING [evaluator:333] Overwriting default num_fewshot of babilong_qa4 from 2 to 2 +2026-07-18:12:26:45 INFO [evaluator:314] babilong_qa5: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []} +2026-07-18:12:26:45 WARNING [evaluator:333] Overwriting default num_fewshot of babilong_qa5 from 2 to 2 +2026-07-18:12:26:45 INFO [api.task:311] Building contexts for babilong_qa1 on rank 0... + + 0%| | 0/50 [00:00 dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about positions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location to answer the question.\nAlways return your answer in the following format:\nThe most recent location of 'person' is 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the hallway. Judith come back to the kitchen. Charlie travelled to balcony.", + "question": "Where is Charlie?", + "target": "The most recent location of Charlie is balcony." + }, + { + "input": "Alan moved to the garage. Charlie went to the beach. Alan went to the shop. Rouse travelled to balcony.", + "question": "Where is Alan?", + "target": "The most recent location of Alan is shop." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa2": { + "task": "babilong_qa2", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa2" + }, + "test_split": "qa2", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nThe 'item' is in 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the kitchen. Charlie got a bottle. Charlie moved to the balcony.", + "question": "Where is the bottle?", + "target": "The bottle is in the balcony." + }, + { + "input": "Alan moved to the garage. Alan got a screw driver. Alan moved to the kitchen.", + "question": "Where is the screw driver?", + "target": "The screw driver is in the kitchen." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa3": { + "task": "babilong_qa3", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa3" + }, + "test_split": "qa3", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nBefore the $location_1$ the $item$ was in the $location_2$. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "John journeyed to the bedroom. Mary grabbed the apple. Mary went back to the bathroom. Daniel journeyed to the bedroom. Daniel moved to the garden. Mary travelled to the kitchen.", + "question": "Where was the apple before the kitchen?", + "target": "Before the kitchen the apple was in the bathroom." + }, + { + "input": "John went back to the bedroom. John went back to the garden. John went back to the kitchen. Sandra took the football. Sandra travelled to the garden. Sandra journeyed to the bedroom.", + "question": "Where was the football before the bedroom?", + "target": "Before the bedroom the football was in the garden." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa4": { + "task": "babilong_qa4", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa4" + }, + "test_split": "qa4", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about different people, their location and actions, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - location. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "The hallway is south of the kitchen. The bedroom is north of the kitchen.", + "question": "What is the kitchen south of?", + "target": "bedroom" + }, + { + "input": "The garden is west of the bedroom. The bedroom is west of the kitchen.", + "question": "What is west of the bedroom?", + "target": "garden" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa5": { + "task": "babilong_qa5", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa5" + }, + "test_split": "qa5", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and their relations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word. Do not write anything else after that. Do not explain your answer.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Mary picked up the apple there. Mary gave the apple to Fred. Mary moved to the bedroom. Bill took the milk there.", + "question": "Who did Mary give the apple to?", + "target": "Fred" + }, + { + "input": "Jeff took the football there. Jeff passed the football to Fred. Jeff got the milk there. Bill travelled to the bedroom.", + "question": "Who gave the football?", + "target": "Jeff" + }, + { + "input": "Fred picked up the apple there. Fred handed the apple to Bill. Bill journeyed to the bedroom. Jeff went back to the garden.", + "question": "What did Fred give to Bill?", + "target": "apple" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + } + }, + "versions": { + "babilong_longctx": 0.0, + "babilong_qa1": 0.0, + "babilong_qa2": 0.0, + "babilong_qa3": 0.0, + "babilong_qa4": 0.0, + "babilong_qa5": 0.0 + }, + "n-shot": { + "babilong_qa1": 2, + "babilong_qa2": 2, + "babilong_qa3": 2, + "babilong_qa4": 2, + "babilong_qa5": 2 + }, + "higher_is_better": { + "babilong_longctx": { + "acc": true + }, + "babilong_qa1": { + "acc": true + }, + "babilong_qa2": { + "acc": true + }, + "babilong_qa3": { + "acc": true + }, + "babilong_qa4": { + "acc": true + }, + "babilong_qa5": { + "acc": true + } + }, + "n-samples": { + "babilong_qa1": { + "original": 1000, + "effective": 50 + }, + "babilong_qa2": { + "original": 999, + "effective": 50 + }, + "babilong_qa3": { + "original": 999, + "effective": 50 + }, + "babilong_qa4": { + "original": 999, + "effective": 50 + }, + "babilong_qa5": { + "original": 999, + "effective": 50 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 50.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784377597.4135222, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "babilong_qa1": "bbdcd109bea1bb245e118e814bd0563de6384bc875a21df5218ab4ed75b27380", + "babilong_qa2": "fa4f8d12bdd481e02a1051414304811f648caf7bce8f89dcf03ecbf5b86f1e92", + "babilong_qa3": "737b64396c8b2e5f96da9796bcd5e448f08e5a13e32ae7b610d31a60269b6e48", + "babilong_qa4": "298a25727c69c9babddcd78f438cae1aa21a388e14e7db90b2206eff753c90c0", + "babilong_qa5": "e60afcdcdd1c4b3e24b002ffbf8f87085f5ec4a02fcd72318f042dcdafe230f4" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "307.892340994993" +} \ No newline at end of file diff --git a/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T12-31-42.056499.jsonl b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T12-31-42.056499.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..60af570f4d66628d9b3ba709e60a9f946a68b277 --- /dev/null +++ b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T12-31-42.056499.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bb665d4b030c9de83733515a244cabc2345f83990ccc55d24110b3ec5bfc739 +size 3164084 diff --git a/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T12-31-42.056499.jsonl b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T12-31-42.056499.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..441c1ef2d1488be687eef1ff63ae1304b9e96975 --- /dev/null +++ b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T12-31-42.056499.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a41a461533929734a423ed2f311990f3c1d38a85ada1f9730acc45b1bb70e1be +size 3147502 diff --git a/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T12-31-42.056499.jsonl b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T12-31-42.056499.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..80415f97507707a88d3ede8eecc7f276918e43cd --- /dev/null +++ b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T12-31-42.056499.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71f73bb1d538565321b650de5af906d40c1206e407a27c69cb2554101954b1c2 +size 3126982 diff --git a/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T12-31-42.056499.jsonl b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T12-31-42.056499.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9ff6690a64a48caecf41c9b299b835f3e62e3c1d --- /dev/null +++ b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T12-31-42.056499.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d3469275d1131254499c2c7ffd046c1c3f6014ba45f3580f3fb4149d00c2429 +size 3137614 diff --git a/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T12-31-42.056499.jsonl b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T12-31-42.056499.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ed64d1d36adf610d34ee27289f371dd290e7e539 --- /dev/null +++ b/outputs/eval/token_t045/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T12-31-42.056499.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d6d02c0fb088e4bf897d06df24d9d6969adddcd4873a16dc18a405d2d8e5a38 +size 3135534 diff --git a/outputs/eval/token_t045/helmet.DONE b/outputs/eval/token_t045/helmet.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t045/helmet_icl8k_n50.jsonl b/outputs/eval/token_t045/helmet_icl8k_n50.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3e00d012f11f00e4398536b69e1f629e0e0e4442 --- /dev/null +++ b/outputs/eval/token_t045/helmet_icl8k_n50.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b6ac9eb4e2f34e3969e3385fcaccac5764f931ec526d3167089d68c5d733cc0 +size 6468315 diff --git a/outputs/eval/token_t045/helmet_icl8k_n50.summary.csv b/outputs/eval/token_t045/helmet_icl8k_n50.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..05350094afcb41b9fa744fe180abf31fff21a690 --- /dev/null +++ b/outputs/eval/token_t045/helmet_icl8k_n50.summary.csv @@ -0,0 +1,251 @@ +method,benchmark,config,n,score,prompt_tokens +token_t045,helmet_icl,trec_coarse,1,1.0,6597 +token_t045,helmet_icl,trec_coarse,1,0.0,6549 +token_t045,helmet_icl,trec_coarse,1,0.0,6508 +token_t045,helmet_icl,trec_coarse,1,1.0,6608 +token_t045,helmet_icl,trec_coarse,1,1.0,6497 +token_t045,helmet_icl,trec_coarse,1,0.0,6545 +token_t045,helmet_icl,trec_coarse,1,1.0,6583 +token_t045,helmet_icl,trec_coarse,1,1.0,6455 +token_t045,helmet_icl,trec_coarse,1,1.0,6643 +token_t045,helmet_icl,trec_coarse,1,1.0,6568 +token_t045,helmet_icl,trec_coarse,1,0.0,6518 +token_t045,helmet_icl,trec_coarse,1,1.0,6424 +token_t045,helmet_icl,trec_coarse,1,1.0,6750 +token_t045,helmet_icl,trec_coarse,1,1.0,6642 +token_t045,helmet_icl,trec_coarse,1,1.0,6494 +token_t045,helmet_icl,trec_coarse,1,0.0,6427 +token_t045,helmet_icl,trec_coarse,1,1.0,6547 +token_t045,helmet_icl,trec_coarse,1,0.0,6612 +token_t045,helmet_icl,trec_coarse,1,1.0,6591 +token_t045,helmet_icl,trec_coarse,1,1.0,6485 +token_t045,helmet_icl,trec_coarse,1,1.0,6542 +token_t045,helmet_icl,trec_coarse,1,1.0,6409 +token_t045,helmet_icl,trec_coarse,1,0.0,6482 +token_t045,helmet_icl,trec_coarse,1,1.0,6502 +token_t045,helmet_icl,trec_coarse,1,0.0,6344 +token_t045,helmet_icl,trec_coarse,1,1.0,6479 +token_t045,helmet_icl,trec_coarse,1,1.0,6569 +token_t045,helmet_icl,trec_coarse,1,1.0,6389 +token_t045,helmet_icl,trec_coarse,1,1.0,6479 +token_t045,helmet_icl,trec_coarse,1,0.0,6653 +token_t045,helmet_icl,trec_coarse,1,1.0,6532 +token_t045,helmet_icl,trec_coarse,1,1.0,6491 +token_t045,helmet_icl,trec_coarse,1,1.0,6489 +token_t045,helmet_icl,trec_coarse,1,0.0,6630 +token_t045,helmet_icl,trec_coarse,1,1.0,6488 +token_t045,helmet_icl,trec_coarse,1,1.0,6499 +token_t045,helmet_icl,trec_coarse,1,0.0,6483 +token_t045,helmet_icl,trec_coarse,1,1.0,6660 +token_t045,helmet_icl,trec_coarse,1,0.0,6675 +token_t045,helmet_icl,trec_coarse,1,0.0,6544 +token_t045,helmet_icl,trec_coarse,1,1.0,6520 +token_t045,helmet_icl,trec_coarse,1,1.0,6359 +token_t045,helmet_icl,trec_coarse,1,0.0,6657 +token_t045,helmet_icl,trec_coarse,1,1.0,6546 +token_t045,helmet_icl,trec_coarse,1,1.0,6481 +token_t045,helmet_icl,trec_coarse,1,1.0,6506 +token_t045,helmet_icl,trec_coarse,1,0.0,6621 +token_t045,helmet_icl,trec_coarse,1,0.0,6623 +token_t045,helmet_icl,trec_coarse,1,0.0,6411 +token_t045,helmet_icl,trec_coarse,1,1.0,6577 +token_t045,helmet_icl,trec_fine,1,0.0,6858 +token_t045,helmet_icl,trec_fine,1,1.0,6973 +token_t045,helmet_icl,trec_fine,1,0.0,6977 +token_t045,helmet_icl,trec_fine,1,1.0,6952 +token_t045,helmet_icl,trec_fine,1,0.0,6919 +token_t045,helmet_icl,trec_fine,1,1.0,6979 +token_t045,helmet_icl,trec_fine,1,1.0,6957 +token_t045,helmet_icl,trec_fine,1,0.0,7013 +token_t045,helmet_icl,trec_fine,1,1.0,6824 +token_t045,helmet_icl,trec_fine,1,1.0,6833 +token_t045,helmet_icl,trec_fine,1,0.0,6868 +token_t045,helmet_icl,trec_fine,1,1.0,6914 +token_t045,helmet_icl,trec_fine,1,1.0,6759 +token_t045,helmet_icl,trec_fine,1,1.0,6933 +token_t045,helmet_icl,trec_fine,1,0.0,6868 +token_t045,helmet_icl,trec_fine,1,0.0,6839 +token_t045,helmet_icl,trec_fine,1,0.0,6838 +token_t045,helmet_icl,trec_fine,1,0.0,7000 +token_t045,helmet_icl,trec_fine,1,0.0,6800 +token_t045,helmet_icl,trec_fine,1,0.0,6829 +token_t045,helmet_icl,trec_fine,1,0.0,6971 +token_t045,helmet_icl,trec_fine,1,1.0,6907 +token_t045,helmet_icl,trec_fine,1,0.0,6818 +token_t045,helmet_icl,trec_fine,1,1.0,6902 +token_t045,helmet_icl,trec_fine,1,1.0,6941 +token_t045,helmet_icl,trec_fine,1,0.0,6830 +token_t045,helmet_icl,trec_fine,1,0.0,6934 +token_t045,helmet_icl,trec_fine,1,1.0,6879 +token_t045,helmet_icl,trec_fine,1,1.0,6915 +token_t045,helmet_icl,trec_fine,1,0.0,6953 +token_t045,helmet_icl,trec_fine,1,0.0,6875 +token_t045,helmet_icl,trec_fine,1,1.0,6847 +token_t045,helmet_icl,trec_fine,1,0.0,6847 +token_t045,helmet_icl,trec_fine,1,0.0,6944 +token_t045,helmet_icl,trec_fine,1,0.0,6818 +token_t045,helmet_icl,trec_fine,1,0.0,6868 +token_t045,helmet_icl,trec_fine,1,0.0,7022 +token_t045,helmet_icl,trec_fine,1,0.0,6930 +token_t045,helmet_icl,trec_fine,1,0.0,6863 +token_t045,helmet_icl,trec_fine,1,0.0,6807 +token_t045,helmet_icl,trec_fine,1,0.0,7001 +token_t045,helmet_icl,trec_fine,1,0.0,6890 +token_t045,helmet_icl,trec_fine,1,0.0,6952 +token_t045,helmet_icl,trec_fine,1,1.0,6845 +token_t045,helmet_icl,trec_fine,1,0.0,6902 +token_t045,helmet_icl,trec_fine,1,0.0,6879 +token_t045,helmet_icl,trec_fine,1,1.0,6896 +token_t045,helmet_icl,trec_fine,1,0.0,6770 +token_t045,helmet_icl,trec_fine,1,0.0,6933 +token_t045,helmet_icl,trec_fine,1,1.0,6886 +token_t045,helmet_icl,banking77,1,1.0,6809 +token_t045,helmet_icl,banking77,1,0.0,7162 +token_t045,helmet_icl,banking77,1,1.0,6876 +token_t045,helmet_icl,banking77,1,1.0,7033 +token_t045,helmet_icl,banking77,1,1.0,6647 +token_t045,helmet_icl,banking77,1,1.0,6818 +token_t045,helmet_icl,banking77,1,1.0,6841 +token_t045,helmet_icl,banking77,1,1.0,6944 +token_t045,helmet_icl,banking77,1,1.0,6776 +token_t045,helmet_icl,banking77,1,1.0,7100 +token_t045,helmet_icl,banking77,1,0.0,6967 +token_t045,helmet_icl,banking77,1,1.0,6894 +token_t045,helmet_icl,banking77,1,1.0,6900 +token_t045,helmet_icl,banking77,1,0.0,6964 +token_t045,helmet_icl,banking77,1,1.0,7085 +token_t045,helmet_icl,banking77,1,1.0,6953 +token_t045,helmet_icl,banking77,1,0.0,7406 +token_t045,helmet_icl,banking77,1,0.0,6802 +token_t045,helmet_icl,banking77,1,1.0,7119 +token_t045,helmet_icl,banking77,1,0.0,7202 +token_t045,helmet_icl,banking77,1,0.0,7103 +token_t045,helmet_icl,banking77,1,0.0,7009 +token_t045,helmet_icl,banking77,1,1.0,6940 +token_t045,helmet_icl,banking77,1,0.0,6857 +token_t045,helmet_icl,banking77,1,0.0,6922 +token_t045,helmet_icl,banking77,1,1.0,6988 +token_t045,helmet_icl,banking77,1,1.0,6945 +token_t045,helmet_icl,banking77,1,1.0,6835 +token_t045,helmet_icl,banking77,1,0.0,7035 +token_t045,helmet_icl,banking77,1,1.0,6787 +token_t045,helmet_icl,banking77,1,0.0,6940 +token_t045,helmet_icl,banking77,1,0.0,6661 +token_t045,helmet_icl,banking77,1,1.0,6934 +token_t045,helmet_icl,banking77,1,0.0,7235 +token_t045,helmet_icl,banking77,1,1.0,6829 +token_t045,helmet_icl,banking77,1,1.0,7169 +token_t045,helmet_icl,banking77,1,0.0,7074 +token_t045,helmet_icl,banking77,1,0.0,6797 +token_t045,helmet_icl,banking77,1,0.0,7013 +token_t045,helmet_icl,banking77,1,1.0,6811 +token_t045,helmet_icl,banking77,1,0.0,6983 +token_t045,helmet_icl,banking77,1,0.0,7183 +token_t045,helmet_icl,banking77,1,1.0,6801 +token_t045,helmet_icl,banking77,1,0.0,7174 +token_t045,helmet_icl,banking77,1,1.0,6773 +token_t045,helmet_icl,banking77,1,1.0,7134 +token_t045,helmet_icl,banking77,1,1.0,6907 +token_t045,helmet_icl,banking77,1,1.0,7341 +token_t045,helmet_icl,banking77,1,1.0,7042 +token_t045,helmet_icl,banking77,1,1.0,7304 +token_t045,helmet_icl,clinic150,1,1.0,7241 +token_t045,helmet_icl,clinic150,1,0.0,7282 +token_t045,helmet_icl,clinic150,1,1.0,7261 +token_t045,helmet_icl,clinic150,1,0.0,7292 +token_t045,helmet_icl,clinic150,1,1.0,7217 +token_t045,helmet_icl,clinic150,1,0.0,7280 +token_t045,helmet_icl,clinic150,1,1.0,7293 +token_t045,helmet_icl,clinic150,1,1.0,7250 +token_t045,helmet_icl,clinic150,1,1.0,7264 +token_t045,helmet_icl,clinic150,1,1.0,7238 +token_t045,helmet_icl,clinic150,1,0.0,7243 +token_t045,helmet_icl,clinic150,1,1.0,7191 +token_t045,helmet_icl,clinic150,1,1.0,7333 +token_t045,helmet_icl,clinic150,1,1.0,7299 +token_t045,helmet_icl,clinic150,1,1.0,7225 +token_t045,helmet_icl,clinic150,1,1.0,7409 +token_t045,helmet_icl,clinic150,1,1.0,7277 +token_t045,helmet_icl,clinic150,1,1.0,7376 +token_t045,helmet_icl,clinic150,1,1.0,7383 +token_t045,helmet_icl,clinic150,1,0.0,7270 +token_t045,helmet_icl,clinic150,1,1.0,7173 +token_t045,helmet_icl,clinic150,1,1.0,7106 +token_t045,helmet_icl,clinic150,1,1.0,7413 +token_t045,helmet_icl,clinic150,1,1.0,7253 +token_t045,helmet_icl,clinic150,1,0.0,7224 +token_t045,helmet_icl,clinic150,1,1.0,7307 +token_t045,helmet_icl,clinic150,1,1.0,7272 +token_t045,helmet_icl,clinic150,1,0.0,7215 +token_t045,helmet_icl,clinic150,1,1.0,7344 +token_t045,helmet_icl,clinic150,1,1.0,7224 +token_t045,helmet_icl,clinic150,1,1.0,7310 +token_t045,helmet_icl,clinic150,1,1.0,7222 +token_t045,helmet_icl,clinic150,1,0.0,7260 +token_t045,helmet_icl,clinic150,1,1.0,7351 +token_t045,helmet_icl,clinic150,1,1.0,7301 +token_t045,helmet_icl,clinic150,1,1.0,7155 +token_t045,helmet_icl,clinic150,1,1.0,7253 +token_t045,helmet_icl,clinic150,1,1.0,7248 +token_t045,helmet_icl,clinic150,1,1.0,7128 +token_t045,helmet_icl,clinic150,1,0.0,7282 +token_t045,helmet_icl,clinic150,1,1.0,7301 +token_t045,helmet_icl,clinic150,1,1.0,7388 +token_t045,helmet_icl,clinic150,1,1.0,7244 +token_t045,helmet_icl,clinic150,1,1.0,7186 +token_t045,helmet_icl,clinic150,1,0.0,7309 +token_t045,helmet_icl,clinic150,1,1.0,7241 +token_t045,helmet_icl,clinic150,1,1.0,7214 +token_t045,helmet_icl,clinic150,1,0.0,7264 +token_t045,helmet_icl,clinic150,1,0.0,7219 +token_t045,helmet_icl,clinic150,1,1.0,7228 +token_t045,helmet_icl,nlu,1,1.0,7063 +token_t045,helmet_icl,nlu,1,1.0,6959 +token_t045,helmet_icl,nlu,1,0.0,6985 +token_t045,helmet_icl,nlu,1,1.0,6970 +token_t045,helmet_icl,nlu,1,1.0,7025 +token_t045,helmet_icl,nlu,1,0.0,7034 +token_t045,helmet_icl,nlu,1,1.0,7004 +token_t045,helmet_icl,nlu,1,0.0,7062 +token_t045,helmet_icl,nlu,1,0.0,7062 +token_t045,helmet_icl,nlu,1,0.0,7045 +token_t045,helmet_icl,nlu,1,1.0,7028 +token_t045,helmet_icl,nlu,1,1.0,7018 +token_t045,helmet_icl,nlu,1,1.0,7075 +token_t045,helmet_icl,nlu,1,1.0,7068 +token_t045,helmet_icl,nlu,1,1.0,7025 +token_t045,helmet_icl,nlu,1,1.0,7022 +token_t045,helmet_icl,nlu,1,1.0,7112 +token_t045,helmet_icl,nlu,1,1.0,6967 +token_t045,helmet_icl,nlu,1,0.0,6980 +token_t045,helmet_icl,nlu,1,1.0,7031 +token_t045,helmet_icl,nlu,1,0.0,7093 +token_t045,helmet_icl,nlu,1,0.0,6949 +token_t045,helmet_icl,nlu,1,1.0,7046 +token_t045,helmet_icl,nlu,1,1.0,7039 +token_t045,helmet_icl,nlu,1,0.0,7126 +token_t045,helmet_icl,nlu,1,1.0,7064 +token_t045,helmet_icl,nlu,1,1.0,7057 +token_t045,helmet_icl,nlu,1,0.0,6971 +token_t045,helmet_icl,nlu,1,0.0,7034 +token_t045,helmet_icl,nlu,1,1.0,7094 +token_t045,helmet_icl,nlu,1,1.0,6979 +token_t045,helmet_icl,nlu,1,1.0,7123 +token_t045,helmet_icl,nlu,1,1.0,7144 +token_t045,helmet_icl,nlu,1,1.0,7024 +token_t045,helmet_icl,nlu,1,1.0,6943 +token_t045,helmet_icl,nlu,1,0.0,6961 +token_t045,helmet_icl,nlu,1,1.0,7005 +token_t045,helmet_icl,nlu,1,1.0,7080 +token_t045,helmet_icl,nlu,1,1.0,7179 +token_t045,helmet_icl,nlu,1,1.0,7115 +token_t045,helmet_icl,nlu,1,1.0,6946 +token_t045,helmet_icl,nlu,1,1.0,6985 +token_t045,helmet_icl,nlu,1,1.0,7131 +token_t045,helmet_icl,nlu,1,1.0,7100 +token_t045,helmet_icl,nlu,1,1.0,7190 +token_t045,helmet_icl,nlu,1,1.0,7040 +token_t045,helmet_icl,nlu,1,1.0,7013 +token_t045,helmet_icl,nlu,1,0.0,7080 +token_t045,helmet_icl,nlu,1,0.0,7098 +token_t045,helmet_icl,nlu,1,1.0,6891 diff --git a/outputs/eval/token_t045/mrcr.DONE b/outputs/eval/token_t045/mrcr.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t045/mrcr_8k_2_4_8needle_n10.jsonl b/outputs/eval/token_t045/mrcr_8k_2_4_8needle_n10.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4a701a9882d55fee9fce41a50d0700474197cddb --- /dev/null +++ b/outputs/eval/token_t045/mrcr_8k_2_4_8needle_n10.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d55ec4e8a46dc3a01299c2b51889ade81ec65a00c6e87e003c45673703d90b0c +size 1003949 diff --git a/outputs/eval/token_t045/mrcr_8k_2_4_8needle_n10.summary.csv b/outputs/eval/token_t045/mrcr_8k_2_4_8needle_n10.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..ee9cf6c560a28b8762f8fa56d7127f14a882efdd --- /dev/null +++ b/outputs/eval/token_t045/mrcr_8k_2_4_8needle_n10.summary.csv @@ -0,0 +1,31 @@ +method,benchmark,config,n,score,prompt_tokens +token_t045,mrcr,8k_2needle,1,0.9685194294146582,4852 +token_t045,mrcr,8k_2needle,1,0.983710407239819,7119 +token_t045,mrcr,8k_2needle,1,0.037745098039215684,5756 +token_t045,mrcr,8k_2needle,1,0.05052372150338879,5072 +token_t045,mrcr,8k_2needle,1,0.07633928571428572,5753 +token_t045,mrcr,8k_2needle,1,0.5607594936708861,5481 +token_t045,mrcr,8k_2needle,1,1.0,5143 +token_t045,mrcr,8k_2needle,1,1.0,6399 +token_t045,mrcr,8k_2needle,1,1.0,5987 +token_t045,mrcr,8k_2needle,1,0.0767583006069261,4380 +token_t045,mrcr,8k_4needle,1,0.0678657519006119,5123 +token_t045,mrcr,8k_4needle,1,0.03232242617717478,7908 +token_t045,mrcr,8k_4needle,1,0.02783400809716599,5814 +token_t045,mrcr,8k_4needle,1,0.9595443833464258,4856 +token_t045,mrcr,8k_4needle,1,0.0,7413 +token_t045,mrcr,8k_4needle,1,0.007665505226480836,7764 +token_t045,mrcr,8k_4needle,1,0.11432241490044959,4312 +token_t045,mrcr,8k_4needle,1,0.03953445409200074,4847 +token_t045,mrcr,8k_4needle,1,0.08090698273640815,6287 +token_t045,mrcr,8k_4needle,1,0.3413333333333333,7841 +token_t045,mrcr,8k_8needle,1,0.009378663540445486,4581 +token_t045,mrcr,8k_8needle,1,0.18287107589149648,6357 +token_t045,mrcr,8k_8needle,1,0.0962800875273523,6981 +token_t045,mrcr,8k_8needle,1,0.04247286455875413,6308 +token_t045,mrcr,8k_8needle,1,0.0,6901 +token_t045,mrcr,8k_8needle,1,0.02224694104560623,5407 +token_t045,mrcr,8k_8needle,1,0.1317365269461078,6351 +token_t045,mrcr,8k_8needle,1,0.9895,5657 +token_t045,mrcr,8k_8needle,1,0.9852774631936579,7330 +token_t045,mrcr,8k_8needle,1,0.076951811729548,8028 diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-12-07.465325.json b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-12-07.465325.json new file mode 100644 index 0000000000000000000000000000000000000000..d3ccc1419f213a03adf234e137f1cf904a9ae4df --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-12-07.465325.json @@ -0,0 +1,821 @@ +{ + "results": { + "niah_multikey_2": { + "alias": "niah_multikey_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_multiquery": { + "alias": "niah_multiquery", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.9875, + "8192_stderr,none": "N/A" + }, + "niah_single_1": { + "alias": "niah_single_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_single_3": { + "alias": "niah_single_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.95, + "8192_stderr,none": "N/A" + }, + "ruler_fwe": { + "alias": "ruler_fwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.7666666666666666, + "8192_stderr,none": "N/A" + }, + "ruler_qa_hotpot": { + "alias": "ruler_qa_hotpot", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.4, + "8192_stderr,none": "N/A" + }, + "ruler_vt": { + "alias": "ruler_vt", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.8700000000000003, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_1": [], + "niah_single_3": [], + "niah_multikey_2": [], + "niah_multiquery": [], + "ruler_vt": [], + "ruler_fwe": [], + "ruler_qa_hotpot": [] + }, + "configs": { + "niah_multikey_2": { + "task": "niah_multikey_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multiquery": { + "task": "niah_multiquery", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multiquery(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_q=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_1": { + "task": "niah_single_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"repeat\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"repeat\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_3": { + "task": "niah_single_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_fwe": { + "task": "ruler_fwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def fwe_download(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, max_seq_length=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 50, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_hotpot": { + "task": "ruler_qa_hotpot", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_hotpotqa(**kwargs):\n return get_qa_dataset(\"hotpotqa\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_vt": { + "task": "ruler_vt", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_vt_dataset(**kwargs) -> dict[str, datasets.Dataset]:\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", \"\"))\n df = (\n get_dataset(tokenizer=get_tokenizer(pretrained), seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 30, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_2": 1.0, + "niah_multiquery": 1.0, + "niah_single_1": 1.0, + "niah_single_3": 1.0, + "ruler_fwe": 1.0, + "ruler_qa_hotpot": 1.0, + "ruler_vt": 1.0 + }, + "n-shot": { + "niah_multikey_2": 0, + "niah_multiquery": 0, + "niah_single_1": 0, + "niah_single_3": 0, + "ruler_fwe": 0, + "ruler_qa_hotpot": 0, + "ruler_vt": 0 + }, + "higher_is_better": { + "niah_multikey_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multiquery": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_fwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_hotpot": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_vt": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_hotpot": { + "original": 500, + "effective": 20 + }, + "ruler_fwe": { + "original": 500, + "effective": 20 + }, + "ruler_vt": { + "original": 500, + "effective": 20 + }, + "niah_multiquery": { + "original": 500, + "effective": 20 + }, + "niah_multikey_2": { + "original": 500, + "effective": 20 + }, + "niah_single_3": { + "original": 500, + "effective": 20 + }, + "niah_single_1": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784375919.9357646, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_hotpot": "fd48cb605efc8c5adac7e3771533db4d4c354b71bf91ce00841c814604ba6404", + "ruler_fwe": "c4633bcd0acf3ba4b96f905b5ba269ba94cf45a4c8ca0c13aae1468227cc3a10", + "ruler_vt": "af011da859c8b86f63f57f6256cf59077d3d3d8536084ab67ccf39e18e9a615a", + "niah_multiquery": "f9a36d2854dfd2bb52ffc7a6bf3f53490fb5303ca9da0046256bc28b55c2adb2", + "niah_multikey_2": "86e49b8f875843454b2923143b9124ba82e8a2e23b145f792726cd82f4e49c24", + "niah_single_3": "905e40146a287e5f0c8e2a81f64f082108c45a23862949e6a99bfcbbf561cad2", + "niah_single_1": "426e8138e992480aa416d01e6b9cef5865a68c00f532400e448a0c20ee026cff" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "810.8221090380102" +} \ No newline at end of file diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T12-12-07.465325.jsonl b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T12-12-07.465325.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..833c76aaa47c9dfc1c8e07352d9e1a7da470780d --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T12-12-07.465325.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec7ad717b95e4d5fa923a2627ae2eaf57f1f9db47d459d64eb1ce5f9d6647f8c +size 987526 diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T12-12-07.465325.jsonl b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T12-12-07.465325.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d4a6b11bb69cac66ad7a41cd7c1bba0c817ef8d7 --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T12-12-07.465325.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac9a6008ea1a5889eaeb1f76cacfe65c4c4c5a7abd6f5608b3dadddaa221648 +size 1440697 diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T12-12-07.465325.jsonl b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T12-12-07.465325.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cc1bacc1a428c819b612b3383a89bdecabff9826 --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T12-12-07.465325.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b01ced4cc6722de3a956473b8ae0600525ca6e5485940b5a5e2267633cc3bd5d +size 1231588 diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T12-12-07.465325.jsonl b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T12-12-07.465325.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7a1732a02229abf4ff7e789dde1f79094802bc3f --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T12-12-07.465325.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f20ebd346ce5932c6d4cbcdf5f0330574b9b13b9b52951d58b08d2b974886e0c +size 1435128 diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T12-12-07.465325.jsonl b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T12-12-07.465325.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..58ac9179094a029b84b8a2fe1903060ef5c3ddef --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T12-12-07.465325.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84bda7768fbe2ab66d514e01fa24e4dadff944d9c721e69274765df0bcdd8864 +size 827295 diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T12-12-07.465325.jsonl b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T12-12-07.465325.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..996d0d900711ff4da6e2d9323b6f88f297ce75a6 --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T12-12-07.465325.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2541d5179a144eda9e0c39f423f5ec895fad28e7188eeadae9317754b88954f1 +size 1172252 diff --git a/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T12-12-07.465325.jsonl b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T12-12-07.465325.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8d52a08e8e4ce544800186a2c4b5666420340018 --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T12-12-07.465325.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7c6124651beb57d18609eccd0f1db78d94c9b96851d2e0759422a5477af8a06 +size 1228994 diff --git a/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-26-30.235088.json b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-26-30.235088.json new file mode 100644 index 0000000000000000000000000000000000000000..cece251ff3af6615ed88e9e96ed66a588b7ed69b --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-26-30.235088.json @@ -0,0 +1,714 @@ +{ + "results": { + "niah_multikey_1": { + "alias": "niah_multikey_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_multikey_3": { + "alias": "niah_multikey_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.75, + "8192_stderr,none": "N/A" + }, + "niah_multivalue": { + "alias": "niah_multivalue", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_single_2": { + "alias": "niah_single_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "ruler_cwe": { + "alias": "ruler_cwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.3949999999999999, + "8192_stderr,none": "N/A" + }, + "ruler_qa_squad": { + "alias": "ruler_qa_squad", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.4208333333333334, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_2": [], + "niah_multikey_1": [], + "niah_multikey_3": [], + "niah_multivalue": [], + "ruler_cwe": [], + "ruler_qa_squad": [] + }, + "configs": { + "niah_multikey_1": { + "task": "niah_multikey_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_k=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multikey_3": { + "task": "niah_multikey_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"uuids\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multivalue": { + "task": "niah_multivalue", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multivalue(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_v=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_2": { + "task": "niah_single_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_cwe": { + "task": "ruler_cwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_cw_dataset(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": "\n\n", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": "\n\n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 120, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_squad": { + "task": "ruler_qa_squad", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_squad(**kwargs):\n return get_qa_dataset(\"squad\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_1": 1.0, + "niah_multikey_3": 1.0, + "niah_multivalue": 1.0, + "niah_single_2": 1.0, + "ruler_cwe": 1.0, + "ruler_qa_squad": 1.0 + }, + "n-shot": { + "niah_multikey_1": 0, + "niah_multikey_3": 0, + "niah_multivalue": 0, + "niah_single_2": 0, + "ruler_cwe": 0, + "ruler_qa_squad": 0 + }, + "higher_is_better": { + "niah_multikey_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multikey_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multivalue": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_cwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_squad": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_squad": { + "original": 500, + "effective": 20 + }, + "ruler_cwe": { + "original": 500, + "effective": 20 + }, + "niah_multivalue": { + "original": 500, + "effective": 20 + }, + "niah_multikey_3": { + "original": 500, + "effective": 20 + }, + "niah_multikey_1": { + "original": 500, + "effective": 20 + }, + "niah_single_2": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784376734.1502712, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_squad": "8c658a728ccd3d67ddd3f45e8e76527cf15a1ae4119033d8246f293ecc68cf11", + "ruler_cwe": "f68206129fc7d709951d550ec9269fed05e73bea2fc86ce9f2c758974db5db71", + "niah_multivalue": "29646655c230b4f155a8de1fbc7ddf22c49e43045bd8fdab5770e7cb45026c7c", + "niah_multikey_3": "de2cec52885e38d56e0bda0e344e989909969c71ab5f8048b9368a011dd3616b", + "niah_multikey_1": "338f81e9efa98dbe6979d4f178f7f06cdbd6d367072318667a932b43c5032ab8", + "niah_single_2": "88ba10e924c2fc6c47543eb646b9e6f7487c7ca026c9391d6cd7054fde59eaad" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "859.353213556984" +} \ No newline at end of file diff --git a/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T12-26-30.235088.jsonl b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T12-26-30.235088.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..82db5ece4e560c1047d88374d5aa547ef89cf45c --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T12-26-30.235088.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a33e240ce70e10445a34fea2d42efc0519a02e24a140de317904411fc5f1ef03 +size 1445444 diff --git a/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T12-26-30.235088.jsonl b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T12-26-30.235088.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..85a8720b4ac1a1b65f9cf215bf0a0110703045b1 --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T12-26-30.235088.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d2e7da44df1fc5f05d1a36a2576a831b9b7488a08a2a3749c885bbf1d34d6cd +size 502030 diff --git a/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T12-26-30.235088.jsonl b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T12-26-30.235088.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..81c809da85e934a9c966427baa4b72adc62f0391 --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T12-26-30.235088.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bb6d9fdfc2c27b4b41ba425c1be458b8c3adcc4bf7d376447622c3772e7ffbb +size 1433534 diff --git a/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T12-26-30.235088.jsonl b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T12-26-30.235088.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..39ef7c152fb666623e3a9a2f735623f7bcb8750f --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T12-26-30.235088.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f28fb7ee751ed53efff7df7ca83a1a5924c3ab6903c091ebc104c50ab765d5 +size 1438222 diff --git a/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T12-26-30.235088.jsonl b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T12-26-30.235088.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..235c163cb9ebb92e11555d8d3fd0b2831718062b --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T12-26-30.235088.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a5399a30e42235fe359c466f8b5a56f2ab4da5a81b78fadb4b9c8e353308fb1 +size 635553 diff --git a/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T12-26-30.235088.jsonl b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T12-26-30.235088.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5677585d5a88536fff8ad3d72a357970e369481a --- /dev/null +++ b/outputs/eval/token_t045/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T12-26-30.235088.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c072e0fc458030ab99bf9a9cf9443b4393f6b410ef501be053bdc7be752c60c8 +size 1387270 diff --git a/outputs/eval/token_t045/ruler_a.DONE b/outputs/eval/token_t045/ruler_a.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t045/ruler_b.DONE b/outputs/eval/token_t045/ruler_b.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t045/sparsity_babilong.json b/outputs/eval/token_t045/sparsity_babilong.json new file mode 100644 index 0000000000000000000000000000000000000000..6e62a779197a21ec6ede10a975dc685bfc017e2f --- /dev/null +++ b/outputs/eval/token_t045/sparsity_babilong.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 53161864, + "effective_router_decisions": 425294912, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 24280016, + "total_decisions": 425294912, + "sparsity": 0.057089834171352605, + "full_attention_usage": 0.9429101658286474, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 32, + "total_decisions": 15189104, + "sparsity": 2.106773381761031e-06 + }, + "12": { + "sparse_decisions": 32, + "total_decisions": 15189104, + "sparsity": 2.106773381761031e-06 + }, + "13": { + "sparse_decisions": 2704, + "total_decisions": 15189104, + "sparsity": 0.0001780223507588071 + }, + "14": { + "sparse_decisions": 1744, + "total_decisions": 15189104, + "sparsity": 0.00011481914930597618 + }, + "15": { + "sparse_decisions": 515936, + "total_decisions": 15189104, + "sparsity": 0.0339675072341331 + }, + "16": { + "sparse_decisions": 504, + "total_decisions": 15189104, + "sparsity": 3.3181680762736235e-05 + }, + "17": { + "sparse_decisions": 41672, + "total_decisions": 15189104, + "sparsity": 0.0027435456363983026 + }, + "18": { + "sparse_decisions": 3176, + "total_decisions": 15189104, + "sparsity": 0.0002090972581397823 + }, + "19": { + "sparse_decisions": 5448, + "total_decisions": 15189104, + "sparsity": 0.0003586781682448155 + }, + "20": { + "sparse_decisions": 22504, + "total_decisions": 15189104, + "sparsity": 0.001481588380723445 + }, + "21": { + "sparse_decisions": 66592, + "total_decisions": 15189104, + "sparsity": 0.004384195407444705 + }, + "22": { + "sparse_decisions": 1687440, + "total_decisions": 15189104, + "sparsity": 0.11109542735371356 + }, + "23": { + "sparse_decisions": 4707840, + "total_decisions": 15189104, + "sparsity": 0.3099484999246829 + }, + "24": { + "sparse_decisions": 1144760, + "total_decisions": 15189104, + "sparsity": 0.07536718426577367 + }, + "25": { + "sparse_decisions": 708656, + "total_decisions": 15189104, + "sparsity": 0.04665554992578891 + }, + "26": { + "sparse_decisions": 183888, + "total_decisions": 15189104, + "sparsity": 0.012106573238289764 + }, + "27": { + "sparse_decisions": 15187088, + "total_decisions": 15189104, + "sparsity": 0.999867273276949 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 35696, + "total_decisions": 840000, + "sparsity": 0.04249523809523809, + "full_attention_usage": 0.9575047619047619 + }, + "prefill": { + "sparse_decisions": 24244320, + "total_decisions": 424454912, + "sparsity": 0.057118717005211615, + "full_attention_usage": 0.9428812829947884 + } + } +} diff --git a/outputs/eval/token_t045/sparsity_helmet.json b/outputs/eval/token_t045/sparsity_helmet.json new file mode 100644 index 0000000000000000000000000000000000000000..b606c78a348278a715d9c2cd895ee9bbfa91f7ff --- /dev/null +++ b/outputs/eval/token_t045/sparsity_helmet.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 48728652, + "effective_router_decisions": 389829216, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 22520568, + "total_decisions": 389829216, + "sparsity": 0.057770344231972595, + "full_attention_usage": 0.9422296557680274, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "12": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "13": { + "sparse_decisions": 16, + "total_decisions": 13922472, + "sparsity": 1.1492212015222584e-06 + }, + "14": { + "sparse_decisions": 40, + "total_decisions": 13922472, + "sparsity": 2.873053003805646e-06 + }, + "15": { + "sparse_decisions": 143368, + "total_decisions": 13922472, + "sparsity": 0.010297596576240197 + }, + "16": { + "sparse_decisions": 8, + "total_decisions": 13922472, + "sparsity": 5.746106007611292e-07 + }, + "17": { + "sparse_decisions": 6056, + "total_decisions": 13922472, + "sparsity": 0.0004349802247761748 + }, + "18": { + "sparse_decisions": 2472, + "total_decisions": 13922472, + "sparsity": 0.00017755467563518892 + }, + "19": { + "sparse_decisions": 7728, + "total_decisions": 13922472, + "sparsity": 0.0005550738403352508 + }, + "20": { + "sparse_decisions": 44512, + "total_decisions": 13922472, + "sparsity": 0.003197133382634923 + }, + "21": { + "sparse_decisions": 346960, + "total_decisions": 13922472, + "sparsity": 0.024920861755010175 + }, + "22": { + "sparse_decisions": 1536624, + "total_decisions": 13922472, + "sparsity": 0.11037005497299618 + }, + "23": { + "sparse_decisions": 3912936, + "total_decisions": 13922472, + "sparsity": 0.28105181321248124 + }, + "24": { + "sparse_decisions": 1354304, + "total_decisions": 13922472, + "sparsity": 0.09727467938165003 + }, + "25": { + "sparse_decisions": 1159768, + "total_decisions": 13922472, + "sparsity": 0.08330187340294166 + }, + "26": { + "sparse_decisions": 85304, + "total_decisions": 13922472, + "sparsity": 0.006127072835915921 + }, + "27": { + "sparse_decisions": 13920472, + "total_decisions": 13922472, + "sparsity": 0.9998563473498098 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 47224, + "total_decisions": 1064000, + "sparsity": 0.04438345864661654, + "full_attention_usage": 0.9556165413533835 + }, + "prefill": { + "sparse_decisions": 22473344, + "total_decisions": 388765216, + "sparsity": 0.05780698240245856, + "full_attention_usage": 0.9421930175975415 + } + } +} diff --git a/outputs/eval/token_t045/sparsity_mrcr.json b/outputs/eval/token_t045/sparsity_mrcr.json new file mode 100644 index 0000000000000000000000000000000000000000..eb0736f4fd3f13379fa75ff76fffe318d62735b9 --- /dev/null +++ b/outputs/eval/token_t045/sparsity_mrcr.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 5391064, + "effective_router_decisions": 43128512, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 5716296, + "total_decisions": 43128512, + "sparsity": 0.13254099747285508, + "full_attention_usage": 0.8674590025271449, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "12": { + "sparse_decisions": 0, + "total_decisions": 1540304, + "sparsity": 0.0 + }, + "13": { + "sparse_decisions": 32, + "total_decisions": 1540304, + "sparsity": 2.0775119716627365e-05 + }, + "14": { + "sparse_decisions": 1640, + "total_decisions": 1540304, + "sparsity": 0.0010647248854771525 + }, + "15": { + "sparse_decisions": 402208, + "total_decisions": 1540304, + "sparsity": 0.2611224797182894 + }, + "16": { + "sparse_decisions": 4664, + "total_decisions": 1540304, + "sparsity": 0.0030279736986984387 + }, + "17": { + "sparse_decisions": 287640, + "total_decisions": 1540304, + "sparsity": 0.18674235735283426 + }, + "18": { + "sparse_decisions": 87720, + "total_decisions": 1540304, + "sparsity": 0.05694979692320477 + }, + "19": { + "sparse_decisions": 321632, + "total_decisions": 1540304, + "sparsity": 0.20881072827182168 + }, + "20": { + "sparse_decisions": 374168, + "total_decisions": 1540304, + "sparsity": 0.24291828106659463 + }, + "21": { + "sparse_decisions": 333200, + "total_decisions": 1540304, + "sparsity": 0.21632093404938246 + }, + "22": { + "sparse_decisions": 508832, + "total_decisions": 1540304, + "sparsity": 0.33034517861409174 + }, + "23": { + "sparse_decisions": 857744, + "total_decisions": 1540304, + "sparsity": 0.5568666964443383 + }, + "24": { + "sparse_decisions": 574504, + "total_decisions": 1540304, + "sparsity": 0.3729809180525403 + }, + "25": { + "sparse_decisions": 375984, + "total_decisions": 1540304, + "sparsity": 0.24409726911051324 + }, + "26": { + "sparse_decisions": 46264, + "total_decisions": 1540304, + "sparsity": 0.030035629330314017 + }, + "27": { + "sparse_decisions": 1540064, + "total_decisions": 1540304, + "sparsity": 0.9998441866021253 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 156624, + "total_decisions": 2358720, + "sparsity": 0.0664021164021164, + "full_attention_usage": 0.9335978835978835 + }, + "prefill": { + "sparse_decisions": 5559672, + "total_decisions": 40769792, + "sparsity": 0.1363674359682777, + "full_attention_usage": 0.8636325640317223 + } + } +} diff --git a/outputs/eval/token_t045/sparsity_ruler_a.json b/outputs/eval/token_t045/sparsity_ruler_a.json new file mode 100644 index 0000000000000000000000000000000000000000..4a51dcb84bd2bd551f9637dd0f9d078959453c00 --- /dev/null +++ b/outputs/eval/token_t045/sparsity_ruler_a.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 29979236, + "effective_router_decisions": 239833888, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 18582576, + "total_decisions": 239833888, + "sparsity": 0.07748102720162715, + "full_attention_usage": 0.9225189727983728, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "12": { + "sparse_decisions": 8, + "total_decisions": 8565496, + "sparsity": 9.339797718661009e-07 + }, + "13": { + "sparse_decisions": 3864, + "total_decisions": 8565496, + "sparsity": 0.00045111222981132675 + }, + "14": { + "sparse_decisions": 2736, + "total_decisions": 8565496, + "sparsity": 0.0003194210819782065 + }, + "15": { + "sparse_decisions": 1951992, + "total_decisions": 8565496, + "sparsity": 0.22789013035555675 + }, + "16": { + "sparse_decisions": 6736, + "total_decisions": 8565496, + "sparsity": 0.000786410967911257 + }, + "17": { + "sparse_decisions": 527152, + "total_decisions": 8565496, + "sparsity": 0.06154366308734485 + }, + "18": { + "sparse_decisions": 64216, + "total_decisions": 8565496, + "sparsity": 0.007497055628769192 + }, + "19": { + "sparse_decisions": 76168, + "total_decisions": 8565496, + "sparsity": 0.008892421407937147 + }, + "20": { + "sparse_decisions": 224848, + "total_decisions": 8565496, + "sparsity": 0.02625043546806863 + }, + "21": { + "sparse_decisions": 461296, + "total_decisions": 8565496, + "sparsity": 0.05385514160534311 + }, + "22": { + "sparse_decisions": 1252472, + "total_decisions": 8565496, + "sparsity": 0.1462229391035849 + }, + "23": { + "sparse_decisions": 2972264, + "total_decisions": 8565496, + "sparsity": 0.34700430658072806 + }, + "24": { + "sparse_decisions": 1387288, + "total_decisions": 8565496, + "sparsity": 0.16196236621907242 + }, + "25": { + "sparse_decisions": 1032824, + "total_decisions": 8565496, + "sparsity": 0.12057959048722923 + }, + "26": { + "sparse_decisions": 54360, + "total_decisions": 8565496, + "sparsity": 0.0063463925498301555 + }, + "27": { + "sparse_decisions": 8564352, + "total_decisions": 8565496, + "sparsity": 0.9998664408926231 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 352712, + "total_decisions": 2764160, + "sparsity": 0.12760187543412826, + "full_attention_usage": 0.8723981245658717 + }, + "prefill": { + "sparse_decisions": 18229864, + "total_decisions": 237069728, + "sparsity": 0.07689663355078384, + "full_attention_usage": 0.9231033664492162 + } + } +} diff --git a/outputs/eval/token_t045/sparsity_ruler_b.json b/outputs/eval/token_t045/sparsity_ruler_b.json new file mode 100644 index 0000000000000000000000000000000000000000..2ec4a1abe29b0710e3f0c4146bfeedc4e298f0e0 --- /dev/null +++ b/outputs/eval/token_t045/sparsity_ruler_b.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 25272464, + "effective_router_decisions": 202179712, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 15539856, + "total_decisions": 202179712, + "sparsity": 0.07686159924889002, + "full_attention_usage": 0.92313840075111, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "12": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "13": { + "sparse_decisions": 432, + "total_decisions": 7220704, + "sparsity": 5.982796137329546e-05 + }, + "14": { + "sparse_decisions": 3544, + "total_decisions": 7220704, + "sparsity": 0.000490810868303146 + }, + "15": { + "sparse_decisions": 378032, + "total_decisions": 7220704, + "sparsity": 0.05235389790247599 + }, + "16": { + "sparse_decisions": 1952, + "total_decisions": 7220704, + "sparsity": 0.00027033375139044616 + }, + "17": { + "sparse_decisions": 131768, + "total_decisions": 7220704, + "sparsity": 0.01824863614406573 + }, + "18": { + "sparse_decisions": 33576, + "total_decisions": 7220704, + "sparsity": 0.004649962108957797 + }, + "19": { + "sparse_decisions": 64112, + "total_decisions": 7220704, + "sparsity": 0.008878912637881292 + }, + "20": { + "sparse_decisions": 123288, + "total_decisions": 7220704, + "sparsity": 0.017074235420812153 + }, + "21": { + "sparse_decisions": 287688, + "total_decisions": 7220704, + "sparsity": 0.039842098498982924 + }, + "22": { + "sparse_decisions": 1334864, + "total_decisions": 7220704, + "sparsity": 0.18486618479306172 + }, + "23": { + "sparse_decisions": 3498784, + "total_decisions": 7220704, + "sparsity": 0.48454887501274113 + }, + "24": { + "sparse_decisions": 1545728, + "total_decisions": 7220704, + "sparsity": 0.2140688774944936 + }, + "25": { + "sparse_decisions": 860456, + "total_decisions": 7220704, + "sparsity": 0.11916511187828777 + }, + "26": { + "sparse_decisions": 56296, + "total_decisions": 7220704, + "sparsity": 0.007796469707108892 + }, + "27": { + "sparse_decisions": 7219336, + "total_decisions": 7220704, + "sparsity": 0.9998105447889846 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 270424, + "total_decisions": 2947840, + "sparsity": 0.09173632218844985, + "full_attention_usage": 0.9082636778115502 + }, + "prefill": { + "sparse_decisions": 15269432, + "total_decisions": 199231872, + "sparsity": 0.07664151245840826, + "full_attention_usage": 0.9233584875415918 + } + } +} diff --git a/outputs/eval/token_t0525/babilong.DONE b/outputs/eval/token_t0525/babilong.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-14-07.524190.json b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-14-07.524190.json new file mode 100644 index 0000000000000000000000000000000000000000..af774fe5421dfd485b0d7db013fe35449578f6d9 --- /dev/null +++ b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-14-07.524190.json @@ -0,0 +1,505 @@ +{ + "results": { + "babilong_longctx": { + "acc,none": 0.608, + "acc_stderr,none": 0.029548990797366614, + "alias": "babilong_longctx" + }, + "babilong_qa1": { + "alias": " - babilong_qa1", + "acc,none": 0.74, + "acc_stderr,none": 0.06266203485560373 + }, + "babilong_qa2": { + "alias": " - babilong_qa2", + "acc,none": 0.52, + "acc_stderr,none": 0.0713714056959817 + }, + "babilong_qa3": { + "alias": " - babilong_qa3", + "acc,none": 0.34, + "acc_stderr,none": 0.06767268161329719 + }, + "babilong_qa4": { + "alias": " - babilong_qa4", + "acc,none": 0.72, + "acc_stderr,none": 0.06414269805898185 + }, + "babilong_qa5": { + "alias": " - babilong_qa5", + "acc,none": 0.72, + "acc_stderr,none": 0.06414269805898185 + } + }, + "groups": { + "babilong_longctx": { + "acc,none": 0.608, + "acc_stderr,none": 0.029548990797366614, + "alias": "babilong_longctx" + } + }, + "group_subtasks": { + "babilong_longctx": [ + "babilong_qa1", + "babilong_qa2", + "babilong_qa3", + "babilong_qa4", + "babilong_qa5" + ] + }, + "configs": { + "babilong_qa1": { + "task": "babilong_qa1", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa1" + }, + "test_split": "qa1", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about positions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location to answer the question.\nAlways return your answer in the following format:\nThe most recent location of 'person' is 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the hallway. Judith come back to the kitchen. Charlie travelled to balcony.", + "question": "Where is Charlie?", + "target": "The most recent location of Charlie is balcony." + }, + { + "input": "Alan moved to the garage. Charlie went to the beach. Alan went to the shop. Rouse travelled to balcony.", + "question": "Where is Alan?", + "target": "The most recent location of Alan is shop." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa2": { + "task": "babilong_qa2", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa2" + }, + "test_split": "qa2", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nThe 'item' is in 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the kitchen. Charlie got a bottle. Charlie moved to the balcony.", + "question": "Where is the bottle?", + "target": "The bottle is in the balcony." + }, + { + "input": "Alan moved to the garage. Alan got a screw driver. Alan moved to the kitchen.", + "question": "Where is the screw driver?", + "target": "The screw driver is in the kitchen." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa3": { + "task": "babilong_qa3", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa3" + }, + "test_split": "qa3", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nBefore the $location_1$ the $item$ was in the $location_2$. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "John journeyed to the bedroom. Mary grabbed the apple. Mary went back to the bathroom. Daniel journeyed to the bedroom. Daniel moved to the garden. Mary travelled to the kitchen.", + "question": "Where was the apple before the kitchen?", + "target": "Before the kitchen the apple was in the bathroom." + }, + { + "input": "John went back to the bedroom. John went back to the garden. John went back to the kitchen. Sandra took the football. Sandra travelled to the garden. Sandra journeyed to the bedroom.", + "question": "Where was the football before the bedroom?", + "target": "Before the bedroom the football was in the garden." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa4": { + "task": "babilong_qa4", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa4" + }, + "test_split": "qa4", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about different people, their location and actions, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - location. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "The hallway is south of the kitchen. The bedroom is north of the kitchen.", + "question": "What is the kitchen south of?", + "target": "bedroom" + }, + { + "input": "The garden is west of the bedroom. The bedroom is west of the kitchen.", + "question": "What is west of the bedroom?", + "target": "garden" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa5": { + "task": "babilong_qa5", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa5" + }, + "test_split": "qa5", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and their relations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word. Do not write anything else after that. Do not explain your answer.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Mary picked up the apple there. Mary gave the apple to Fred. Mary moved to the bedroom. Bill took the milk there.", + "question": "Who did Mary give the apple to?", + "target": "Fred" + }, + { + "input": "Jeff took the football there. Jeff passed the football to Fred. Jeff got the milk there. Bill travelled to the bedroom.", + "question": "Who gave the football?", + "target": "Jeff" + }, + { + "input": "Fred picked up the apple there. Fred handed the apple to Bill. Bill journeyed to the bedroom. Jeff went back to the garden.", + "question": "What did Fred give to Bill?", + "target": "apple" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + } + }, + "versions": { + "babilong_longctx": 0.0, + "babilong_qa1": 0.0, + "babilong_qa2": 0.0, + "babilong_qa3": 0.0, + "babilong_qa4": 0.0, + "babilong_qa5": 0.0 + }, + "n-shot": { + "babilong_qa1": 2, + "babilong_qa2": 2, + "babilong_qa3": 2, + "babilong_qa4": 2, + "babilong_qa5": 2 + }, + "higher_is_better": { + "babilong_longctx": { + "acc": true + }, + "babilong_qa1": { + "acc": true + }, + "babilong_qa2": { + "acc": true + }, + "babilong_qa3": { + "acc": true + }, + "babilong_qa4": { + "acc": true + }, + "babilong_qa5": { + "acc": true + } + }, + "n-samples": { + "babilong_qa1": { + "original": 1000, + "effective": 50 + }, + "babilong_qa2": { + "original": 999, + "effective": 50 + }, + "babilong_qa3": { + "original": 999, + "effective": 50 + }, + "babilong_qa4": { + "original": 999, + "effective": 50 + }, + "babilong_qa5": { + "original": 999, + "effective": 50 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 50.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784380148.4561443, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "babilong_qa1": "bbdcd109bea1bb245e118e814bd0563de6384bc875a21df5218ab4ed75b27380", + "babilong_qa2": "fa4f8d12bdd481e02a1051414304811f648caf7bce8f89dcf03ecbf5b86f1e92", + "babilong_qa3": "737b64396c8b2e5f96da9796bcd5e448f08e5a13e32ae7b610d31a60269b6e48", + "babilong_qa4": "298a25727c69c9babddcd78f438cae1aa21a388e14e7db90b2206eff753c90c0", + "babilong_qa5": "e60afcdcdd1c4b3e24b002ffbf8f87085f5ec4a02fcd72318f042dcdafe230f4" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "302.34617991698906" +} \ No newline at end of file diff --git a/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T13-14-07.524190.jsonl b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T13-14-07.524190.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ad4abaececd2313cbcde565dff11de6fb577824e --- /dev/null +++ b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T13-14-07.524190.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:894b7b485f12742da28f607918847453d7bbffaf5c672ac7535694d830422e83 +size 3164092 diff --git a/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T13-14-07.524190.jsonl b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T13-14-07.524190.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b1fa5cd409389d7d15d38a0ac2361ab18ac19e6f --- /dev/null +++ b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T13-14-07.524190.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:954df79f19d6f4037ef1ba1e46baf6dc9c2d30a727f2cf4085997f2ff006dd4e +size 3147516 diff --git a/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T13-14-07.524190.jsonl b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T13-14-07.524190.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e0f9890a70746bab020419b1ee144f35297ea96d --- /dev/null +++ b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T13-14-07.524190.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ef1a161d57d61ed368024971f3a0e608ce70d56782ec8c97b20cc3f3ecfccea +size 3126962 diff --git a/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T13-14-07.524190.jsonl b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T13-14-07.524190.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..33d740e1f80b3c05caed31ddfcb96a2561a0385d --- /dev/null +++ b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T13-14-07.524190.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c40e39f34fff163a0fe5c131806e61937bb4e5d6e7b343c832691b828c3e54f +size 3137620 diff --git a/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T13-14-07.524190.jsonl b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T13-14-07.524190.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1f210e9eaea908877b4658c04cfe703a189c3dd6 --- /dev/null +++ b/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T13-14-07.524190.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dddf97f2f532bec678e21782ffb214a1d795c62adb06c291d394b8f43b287c6 +size 3135600 diff --git a/outputs/eval/token_t0525/helmet.DONE b/outputs/eval/token_t0525/helmet.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0525/helmet_icl8k_n50.jsonl b/outputs/eval/token_t0525/helmet_icl8k_n50.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..550e0b5ca723e92b21268cc3307260c750f5cec5 --- /dev/null +++ b/outputs/eval/token_t0525/helmet_icl8k_n50.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34ddb45d86da451291b6bfc73a63903711eadf296a7574321ad9645af56ef5ce +size 6469232 diff --git a/outputs/eval/token_t0525/helmet_icl8k_n50.summary.csv b/outputs/eval/token_t0525/helmet_icl8k_n50.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..b569502ade202fc8ab8969fcdd262365140d0c20 --- /dev/null +++ b/outputs/eval/token_t0525/helmet_icl8k_n50.summary.csv @@ -0,0 +1,251 @@ +method,benchmark,config,n,score,prompt_tokens +token_t0525,helmet_icl,trec_coarse,1,1.0,6597 +token_t0525,helmet_icl,trec_coarse,1,0.0,6549 +token_t0525,helmet_icl,trec_coarse,1,0.0,6508 +token_t0525,helmet_icl,trec_coarse,1,1.0,6608 +token_t0525,helmet_icl,trec_coarse,1,1.0,6497 +token_t0525,helmet_icl,trec_coarse,1,0.0,6545 +token_t0525,helmet_icl,trec_coarse,1,1.0,6583 +token_t0525,helmet_icl,trec_coarse,1,1.0,6455 +token_t0525,helmet_icl,trec_coarse,1,1.0,6643 +token_t0525,helmet_icl,trec_coarse,1,1.0,6568 +token_t0525,helmet_icl,trec_coarse,1,0.0,6518 +token_t0525,helmet_icl,trec_coarse,1,1.0,6424 +token_t0525,helmet_icl,trec_coarse,1,1.0,6750 +token_t0525,helmet_icl,trec_coarse,1,1.0,6642 +token_t0525,helmet_icl,trec_coarse,1,1.0,6494 +token_t0525,helmet_icl,trec_coarse,1,0.0,6427 +token_t0525,helmet_icl,trec_coarse,1,1.0,6547 +token_t0525,helmet_icl,trec_coarse,1,0.0,6612 +token_t0525,helmet_icl,trec_coarse,1,1.0,6591 +token_t0525,helmet_icl,trec_coarse,1,1.0,6485 +token_t0525,helmet_icl,trec_coarse,1,1.0,6542 +token_t0525,helmet_icl,trec_coarse,1,1.0,6409 +token_t0525,helmet_icl,trec_coarse,1,0.0,6482 +token_t0525,helmet_icl,trec_coarse,1,1.0,6502 +token_t0525,helmet_icl,trec_coarse,1,0.0,6344 +token_t0525,helmet_icl,trec_coarse,1,1.0,6479 +token_t0525,helmet_icl,trec_coarse,1,1.0,6569 +token_t0525,helmet_icl,trec_coarse,1,1.0,6389 +token_t0525,helmet_icl,trec_coarse,1,1.0,6479 +token_t0525,helmet_icl,trec_coarse,1,1.0,6653 +token_t0525,helmet_icl,trec_coarse,1,1.0,6532 +token_t0525,helmet_icl,trec_coarse,1,1.0,6491 +token_t0525,helmet_icl,trec_coarse,1,0.0,6489 +token_t0525,helmet_icl,trec_coarse,1,0.0,6630 +token_t0525,helmet_icl,trec_coarse,1,1.0,6488 +token_t0525,helmet_icl,trec_coarse,1,1.0,6499 +token_t0525,helmet_icl,trec_coarse,1,0.0,6483 +token_t0525,helmet_icl,trec_coarse,1,1.0,6660 +token_t0525,helmet_icl,trec_coarse,1,0.0,6675 +token_t0525,helmet_icl,trec_coarse,1,0.0,6544 +token_t0525,helmet_icl,trec_coarse,1,1.0,6520 +token_t0525,helmet_icl,trec_coarse,1,1.0,6359 +token_t0525,helmet_icl,trec_coarse,1,0.0,6657 +token_t0525,helmet_icl,trec_coarse,1,1.0,6546 +token_t0525,helmet_icl,trec_coarse,1,1.0,6481 +token_t0525,helmet_icl,trec_coarse,1,1.0,6506 +token_t0525,helmet_icl,trec_coarse,1,0.0,6621 +token_t0525,helmet_icl,trec_coarse,1,0.0,6623 +token_t0525,helmet_icl,trec_coarse,1,0.0,6411 +token_t0525,helmet_icl,trec_coarse,1,1.0,6577 +token_t0525,helmet_icl,trec_fine,1,0.0,6858 +token_t0525,helmet_icl,trec_fine,1,1.0,6973 +token_t0525,helmet_icl,trec_fine,1,0.0,6977 +token_t0525,helmet_icl,trec_fine,1,1.0,6952 +token_t0525,helmet_icl,trec_fine,1,0.0,6919 +token_t0525,helmet_icl,trec_fine,1,0.0,6979 +token_t0525,helmet_icl,trec_fine,1,1.0,6957 +token_t0525,helmet_icl,trec_fine,1,0.0,7013 +token_t0525,helmet_icl,trec_fine,1,1.0,6824 +token_t0525,helmet_icl,trec_fine,1,1.0,6833 +token_t0525,helmet_icl,trec_fine,1,0.0,6868 +token_t0525,helmet_icl,trec_fine,1,1.0,6914 +token_t0525,helmet_icl,trec_fine,1,1.0,6759 +token_t0525,helmet_icl,trec_fine,1,1.0,6933 +token_t0525,helmet_icl,trec_fine,1,0.0,6868 +token_t0525,helmet_icl,trec_fine,1,0.0,6839 +token_t0525,helmet_icl,trec_fine,1,0.0,6838 +token_t0525,helmet_icl,trec_fine,1,0.0,7000 +token_t0525,helmet_icl,trec_fine,1,0.0,6800 +token_t0525,helmet_icl,trec_fine,1,0.0,6829 +token_t0525,helmet_icl,trec_fine,1,0.0,6971 +token_t0525,helmet_icl,trec_fine,1,1.0,6907 +token_t0525,helmet_icl,trec_fine,1,0.0,6818 +token_t0525,helmet_icl,trec_fine,1,1.0,6902 +token_t0525,helmet_icl,trec_fine,1,1.0,6941 +token_t0525,helmet_icl,trec_fine,1,0.0,6830 +token_t0525,helmet_icl,trec_fine,1,0.0,6934 +token_t0525,helmet_icl,trec_fine,1,1.0,6879 +token_t0525,helmet_icl,trec_fine,1,1.0,6915 +token_t0525,helmet_icl,trec_fine,1,0.0,6953 +token_t0525,helmet_icl,trec_fine,1,0.0,6875 +token_t0525,helmet_icl,trec_fine,1,1.0,6847 +token_t0525,helmet_icl,trec_fine,1,0.0,6847 +token_t0525,helmet_icl,trec_fine,1,0.0,6944 +token_t0525,helmet_icl,trec_fine,1,0.0,6818 +token_t0525,helmet_icl,trec_fine,1,0.0,6868 +token_t0525,helmet_icl,trec_fine,1,0.0,7022 +token_t0525,helmet_icl,trec_fine,1,0.0,6930 +token_t0525,helmet_icl,trec_fine,1,0.0,6863 +token_t0525,helmet_icl,trec_fine,1,0.0,6807 +token_t0525,helmet_icl,trec_fine,1,0.0,7001 +token_t0525,helmet_icl,trec_fine,1,0.0,6890 +token_t0525,helmet_icl,trec_fine,1,0.0,6952 +token_t0525,helmet_icl,trec_fine,1,1.0,6845 +token_t0525,helmet_icl,trec_fine,1,0.0,6902 +token_t0525,helmet_icl,trec_fine,1,0.0,6879 +token_t0525,helmet_icl,trec_fine,1,1.0,6896 +token_t0525,helmet_icl,trec_fine,1,0.0,6770 +token_t0525,helmet_icl,trec_fine,1,0.0,6933 +token_t0525,helmet_icl,trec_fine,1,1.0,6886 +token_t0525,helmet_icl,banking77,1,1.0,6809 +token_t0525,helmet_icl,banking77,1,0.0,7162 +token_t0525,helmet_icl,banking77,1,1.0,6876 +token_t0525,helmet_icl,banking77,1,1.0,7033 +token_t0525,helmet_icl,banking77,1,1.0,6647 +token_t0525,helmet_icl,banking77,1,1.0,6818 +token_t0525,helmet_icl,banking77,1,1.0,6841 +token_t0525,helmet_icl,banking77,1,1.0,6944 +token_t0525,helmet_icl,banking77,1,1.0,6776 +token_t0525,helmet_icl,banking77,1,1.0,7100 +token_t0525,helmet_icl,banking77,1,0.0,6967 +token_t0525,helmet_icl,banking77,1,1.0,6894 +token_t0525,helmet_icl,banking77,1,1.0,6900 +token_t0525,helmet_icl,banking77,1,0.0,6964 +token_t0525,helmet_icl,banking77,1,1.0,7085 +token_t0525,helmet_icl,banking77,1,1.0,6953 +token_t0525,helmet_icl,banking77,1,0.0,7406 +token_t0525,helmet_icl,banking77,1,0.0,6802 +token_t0525,helmet_icl,banking77,1,1.0,7119 +token_t0525,helmet_icl,banking77,1,0.0,7202 +token_t0525,helmet_icl,banking77,1,0.0,7103 +token_t0525,helmet_icl,banking77,1,0.0,7009 +token_t0525,helmet_icl,banking77,1,1.0,6940 +token_t0525,helmet_icl,banking77,1,0.0,6857 +token_t0525,helmet_icl,banking77,1,0.0,6922 +token_t0525,helmet_icl,banking77,1,1.0,6988 +token_t0525,helmet_icl,banking77,1,1.0,6945 +token_t0525,helmet_icl,banking77,1,1.0,6835 +token_t0525,helmet_icl,banking77,1,0.0,7035 +token_t0525,helmet_icl,banking77,1,1.0,6787 +token_t0525,helmet_icl,banking77,1,0.0,6940 +token_t0525,helmet_icl,banking77,1,0.0,6661 +token_t0525,helmet_icl,banking77,1,1.0,6934 +token_t0525,helmet_icl,banking77,1,0.0,7235 +token_t0525,helmet_icl,banking77,1,1.0,6829 +token_t0525,helmet_icl,banking77,1,1.0,7169 +token_t0525,helmet_icl,banking77,1,0.0,7074 +token_t0525,helmet_icl,banking77,1,0.0,6797 +token_t0525,helmet_icl,banking77,1,0.0,7013 +token_t0525,helmet_icl,banking77,1,1.0,6811 +token_t0525,helmet_icl,banking77,1,0.0,6983 +token_t0525,helmet_icl,banking77,1,0.0,7183 +token_t0525,helmet_icl,banking77,1,1.0,6801 +token_t0525,helmet_icl,banking77,1,0.0,7174 +token_t0525,helmet_icl,banking77,1,1.0,6773 +token_t0525,helmet_icl,banking77,1,1.0,7134 +token_t0525,helmet_icl,banking77,1,1.0,6907 +token_t0525,helmet_icl,banking77,1,1.0,7341 +token_t0525,helmet_icl,banking77,1,1.0,7042 +token_t0525,helmet_icl,banking77,1,1.0,7304 +token_t0525,helmet_icl,clinic150,1,1.0,7241 +token_t0525,helmet_icl,clinic150,1,0.0,7282 +token_t0525,helmet_icl,clinic150,1,1.0,7261 +token_t0525,helmet_icl,clinic150,1,0.0,7292 +token_t0525,helmet_icl,clinic150,1,1.0,7217 +token_t0525,helmet_icl,clinic150,1,0.0,7280 +token_t0525,helmet_icl,clinic150,1,1.0,7293 +token_t0525,helmet_icl,clinic150,1,0.0,7250 +token_t0525,helmet_icl,clinic150,1,1.0,7264 +token_t0525,helmet_icl,clinic150,1,1.0,7238 +token_t0525,helmet_icl,clinic150,1,0.0,7243 +token_t0525,helmet_icl,clinic150,1,1.0,7191 +token_t0525,helmet_icl,clinic150,1,1.0,7333 +token_t0525,helmet_icl,clinic150,1,1.0,7299 +token_t0525,helmet_icl,clinic150,1,1.0,7225 +token_t0525,helmet_icl,clinic150,1,1.0,7409 +token_t0525,helmet_icl,clinic150,1,1.0,7277 +token_t0525,helmet_icl,clinic150,1,1.0,7376 +token_t0525,helmet_icl,clinic150,1,1.0,7383 +token_t0525,helmet_icl,clinic150,1,0.0,7270 +token_t0525,helmet_icl,clinic150,1,1.0,7173 +token_t0525,helmet_icl,clinic150,1,1.0,7106 +token_t0525,helmet_icl,clinic150,1,1.0,7413 +token_t0525,helmet_icl,clinic150,1,1.0,7253 +token_t0525,helmet_icl,clinic150,1,0.0,7224 +token_t0525,helmet_icl,clinic150,1,1.0,7307 +token_t0525,helmet_icl,clinic150,1,1.0,7272 +token_t0525,helmet_icl,clinic150,1,0.0,7215 +token_t0525,helmet_icl,clinic150,1,1.0,7344 +token_t0525,helmet_icl,clinic150,1,1.0,7224 +token_t0525,helmet_icl,clinic150,1,1.0,7310 +token_t0525,helmet_icl,clinic150,1,1.0,7222 +token_t0525,helmet_icl,clinic150,1,0.0,7260 +token_t0525,helmet_icl,clinic150,1,1.0,7351 +token_t0525,helmet_icl,clinic150,1,1.0,7301 +token_t0525,helmet_icl,clinic150,1,1.0,7155 +token_t0525,helmet_icl,clinic150,1,1.0,7253 +token_t0525,helmet_icl,clinic150,1,1.0,7248 +token_t0525,helmet_icl,clinic150,1,1.0,7128 +token_t0525,helmet_icl,clinic150,1,0.0,7282 +token_t0525,helmet_icl,clinic150,1,1.0,7301 +token_t0525,helmet_icl,clinic150,1,1.0,7388 +token_t0525,helmet_icl,clinic150,1,1.0,7244 +token_t0525,helmet_icl,clinic150,1,1.0,7186 +token_t0525,helmet_icl,clinic150,1,0.0,7309 +token_t0525,helmet_icl,clinic150,1,1.0,7241 +token_t0525,helmet_icl,clinic150,1,1.0,7214 +token_t0525,helmet_icl,clinic150,1,0.0,7264 +token_t0525,helmet_icl,clinic150,1,0.0,7219 +token_t0525,helmet_icl,clinic150,1,1.0,7228 +token_t0525,helmet_icl,nlu,1,1.0,7063 +token_t0525,helmet_icl,nlu,1,1.0,6959 +token_t0525,helmet_icl,nlu,1,0.0,6985 +token_t0525,helmet_icl,nlu,1,1.0,6970 +token_t0525,helmet_icl,nlu,1,1.0,7025 +token_t0525,helmet_icl,nlu,1,0.0,7034 +token_t0525,helmet_icl,nlu,1,1.0,7004 +token_t0525,helmet_icl,nlu,1,0.0,7062 +token_t0525,helmet_icl,nlu,1,0.0,7062 +token_t0525,helmet_icl,nlu,1,0.0,7045 +token_t0525,helmet_icl,nlu,1,1.0,7028 +token_t0525,helmet_icl,nlu,1,1.0,7018 +token_t0525,helmet_icl,nlu,1,1.0,7075 +token_t0525,helmet_icl,nlu,1,1.0,7068 +token_t0525,helmet_icl,nlu,1,1.0,7025 +token_t0525,helmet_icl,nlu,1,1.0,7022 +token_t0525,helmet_icl,nlu,1,1.0,7112 +token_t0525,helmet_icl,nlu,1,1.0,6967 +token_t0525,helmet_icl,nlu,1,0.0,6980 +token_t0525,helmet_icl,nlu,1,1.0,7031 +token_t0525,helmet_icl,nlu,1,0.0,7093 +token_t0525,helmet_icl,nlu,1,0.0,6949 +token_t0525,helmet_icl,nlu,1,1.0,7046 +token_t0525,helmet_icl,nlu,1,1.0,7039 +token_t0525,helmet_icl,nlu,1,0.0,7126 +token_t0525,helmet_icl,nlu,1,1.0,7064 +token_t0525,helmet_icl,nlu,1,1.0,7057 +token_t0525,helmet_icl,nlu,1,0.0,6971 +token_t0525,helmet_icl,nlu,1,1.0,7034 +token_t0525,helmet_icl,nlu,1,1.0,7094 +token_t0525,helmet_icl,nlu,1,1.0,6979 +token_t0525,helmet_icl,nlu,1,1.0,7123 +token_t0525,helmet_icl,nlu,1,1.0,7144 +token_t0525,helmet_icl,nlu,1,0.0,7024 +token_t0525,helmet_icl,nlu,1,1.0,6943 +token_t0525,helmet_icl,nlu,1,0.0,6961 +token_t0525,helmet_icl,nlu,1,1.0,7005 +token_t0525,helmet_icl,nlu,1,1.0,7080 +token_t0525,helmet_icl,nlu,1,1.0,7179 +token_t0525,helmet_icl,nlu,1,1.0,7115 +token_t0525,helmet_icl,nlu,1,1.0,6946 +token_t0525,helmet_icl,nlu,1,1.0,6985 +token_t0525,helmet_icl,nlu,1,1.0,7131 +token_t0525,helmet_icl,nlu,1,1.0,7100 +token_t0525,helmet_icl,nlu,1,1.0,7190 +token_t0525,helmet_icl,nlu,1,1.0,7040 +token_t0525,helmet_icl,nlu,1,1.0,7013 +token_t0525,helmet_icl,nlu,1,0.0,7080 +token_t0525,helmet_icl,nlu,1,0.0,7098 +token_t0525,helmet_icl,nlu,1,1.0,6891 diff --git a/outputs/eval/token_t0525/mrcr.DONE b/outputs/eval/token_t0525/mrcr.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0525/mrcr_8k_2_4_8needle_n10.jsonl b/outputs/eval/token_t0525/mrcr_8k_2_4_8needle_n10.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a0812477d431132e9ecb9a76c7e01e1f98df389c --- /dev/null +++ b/outputs/eval/token_t0525/mrcr_8k_2_4_8needle_n10.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f9b7bee9b3f00fe9f09bcf4a84efa9e697ae0050af52e12c54e17fd02fa8941 +size 996950 diff --git a/outputs/eval/token_t0525/mrcr_8k_2_4_8needle_n10.summary.csv b/outputs/eval/token_t0525/mrcr_8k_2_4_8needle_n10.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..368be63b189eb18edcaf7ed041061091abbae224 --- /dev/null +++ b/outputs/eval/token_t0525/mrcr_8k_2_4_8needle_n10.summary.csv @@ -0,0 +1,31 @@ +method,benchmark,config,n,score,prompt_tokens +token_t0525,mrcr,8k_2needle,1,0.07291011942174733,4852 +token_t0525,mrcr,8k_2needle,1,0.983710407239819,7119 +token_t0525,mrcr,8k_2needle,1,0.03769791404875597,5756 +token_t0525,mrcr,8k_2needle,1,0.05052372150338879,5072 +token_t0525,mrcr,8k_2needle,1,0.03801695350629335,5753 +token_t0525,mrcr,8k_2needle,1,0.02821486706456864,5481 +token_t0525,mrcr,8k_2needle,1,1.0,5143 +token_t0525,mrcr,8k_2needle,1,1.0,6399 +token_t0525,mrcr,8k_2needle,1,1.0,5987 +token_t0525,mrcr,8k_2needle,1,0.07798331519767863,4380 +token_t0525,mrcr,8k_4needle,1,0.08286252354048965,5123 +token_t0525,mrcr,8k_4needle,1,0.03232242617717478,7908 +token_t0525,mrcr,8k_4needle,1,0.02734375,5814 +token_t0525,mrcr,8k_4needle,1,0.9595443833464258,4856 +token_t0525,mrcr,8k_4needle,1,0.052568121533638774,7413 +token_t0525,mrcr,8k_4needle,1,0.008365284071104914,7764 +token_t0525,mrcr,8k_4needle,1,0.08731808731808732,4312 +token_t0525,mrcr,8k_4needle,1,0.023166023166023165,4847 +token_t0525,mrcr,8k_4needle,1,0.07769488037046565,6287 +token_t0525,mrcr,8k_4needle,1,0.33827893175074186,7841 +token_t0525,mrcr,8k_8needle,1,0.009603841536614645,4581 +token_t0525,mrcr,8k_8needle,1,0.9426503340757239,6357 +token_t0525,mrcr,8k_8needle,1,0.05405405405405406,6981 +token_t0525,mrcr,8k_8needle,1,0.04523107177974434,6308 +token_t0525,mrcr,8k_8needle,1,0.0,6901 +token_t0525,mrcr,8k_8needle,1,0.02247191011235955,5407 +token_t0525,mrcr,8k_8needle,1,0.1574279379157428,6351 +token_t0525,mrcr,8k_8needle,1,0.1781569965870307,5657 +token_t0525,mrcr,8k_8needle,1,0.9915966386554622,7330 +token_t0525,mrcr,8k_8needle,1,0.076951811729548,8028 diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-56-47.303435.json b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-56-47.303435.json new file mode 100644 index 0000000000000000000000000000000000000000..6224eafe252ae8bfe05a0346f8ed672e193c126c --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T12-56-47.303435.json @@ -0,0 +1,821 @@ +{ + "results": { + "niah_multikey_2": { + "alias": "niah_multikey_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_multiquery": { + "alias": "niah_multiquery", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_single_1": { + "alias": "niah_single_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_single_3": { + "alias": "niah_single_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "ruler_fwe": { + "alias": "ruler_fwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.7666666666666666, + "8192_stderr,none": "N/A" + }, + "ruler_qa_hotpot": { + "alias": "ruler_qa_hotpot", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.4, + "8192_stderr,none": "N/A" + }, + "ruler_vt": { + "alias": "ruler_vt", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.8800000000000002, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_1": [], + "niah_single_3": [], + "niah_multikey_2": [], + "niah_multiquery": [], + "ruler_vt": [], + "ruler_fwe": [], + "ruler_qa_hotpot": [] + }, + "configs": { + "niah_multikey_2": { + "task": "niah_multikey_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multiquery": { + "task": "niah_multiquery", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multiquery(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_q=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_1": { + "task": "niah_single_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"repeat\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"repeat\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_3": { + "task": "niah_single_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_fwe": { + "task": "ruler_fwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def fwe_download(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, max_seq_length=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 50, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_hotpot": { + "task": "ruler_qa_hotpot", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_hotpotqa(**kwargs):\n return get_qa_dataset(\"hotpotqa\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_vt": { + "task": "ruler_vt", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_vt_dataset(**kwargs) -> dict[str, datasets.Dataset]:\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", \"\"))\n df = (\n get_dataset(tokenizer=get_tokenizer(pretrained), seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 30, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_2": 1.0, + "niah_multiquery": 1.0, + "niah_single_1": 1.0, + "niah_single_3": 1.0, + "ruler_fwe": 1.0, + "ruler_qa_hotpot": 1.0, + "ruler_vt": 1.0 + }, + "n-shot": { + "niah_multikey_2": 0, + "niah_multiquery": 0, + "niah_single_1": 0, + "niah_single_3": 0, + "ruler_fwe": 0, + "ruler_qa_hotpot": 0, + "ruler_vt": 0 + }, + "higher_is_better": { + "niah_multikey_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multiquery": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_fwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_hotpot": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_vt": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_hotpot": { + "original": 500, + "effective": 20 + }, + "ruler_fwe": { + "original": 500, + "effective": 20 + }, + "ruler_vt": { + "original": 500, + "effective": 20 + }, + "niah_multiquery": { + "original": 500, + "effective": 20 + }, + "niah_multikey_2": { + "original": 500, + "effective": 20 + }, + "niah_single_3": { + "original": 500, + "effective": 20 + }, + "niah_single_1": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784378672.7943566, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_hotpot": "fd48cb605efc8c5adac7e3771533db4d4c354b71bf91ce00841c814604ba6404", + "ruler_fwe": "c4633bcd0acf3ba4b96f905b5ba269ba94cf45a4c8ca0c13aae1468227cc3a10", + "ruler_vt": "af011da859c8b86f63f57f6256cf59077d3d3d8536084ab67ccf39e18e9a615a", + "niah_multiquery": "f9a36d2854dfd2bb52ffc7a6bf3f53490fb5303ca9da0046256bc28b55c2adb2", + "niah_multikey_2": "86e49b8f875843454b2923143b9124ba82e8a2e23b145f792726cd82f4e49c24", + "niah_single_3": "905e40146a287e5f0c8e2a81f64f082108c45a23862949e6a99bfcbbf561cad2", + "niah_single_1": "426e8138e992480aa416d01e6b9cef5865a68c00f532400e448a0c20ee026cff" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "737.7238195429964" +} \ No newline at end of file diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T12-56-47.303435.jsonl b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T12-56-47.303435.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d3369e03a402a2d8e4b3b3632e34a9b6fe2f965c --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T12-56-47.303435.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1198521d64dbb83c3bcac7e4b000a43afe8c5dc3c6a6261875cba22529f2a61 +size 987952 diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T12-56-47.303435.jsonl b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T12-56-47.303435.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..20369f9528df22c9f588611175dfcc8b225816c0 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T12-56-47.303435.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fda48f7c886c33b97a6e90f648b89cd35e4b95b24d3bd08c1cff5bfd40efb2bc +size 1442196 diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T12-56-47.303435.jsonl b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T12-56-47.303435.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c648b3e48ca8d10c1d7badf44408b476e80a7a5b --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T12-56-47.303435.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b805cb744f2ca51f70ef08f871e84fd7a635d76bd0632761601cf38420eb43b +size 1231408 diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T12-56-47.303435.jsonl b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T12-56-47.303435.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d5a64209ac6bdd48e6fa46bcb2311202fcb91dbd --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T12-56-47.303435.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abf7b2d058ae6464aa9140d934b01d08b30c31804b918a3fb4d99484b0c4a910 +size 1435434 diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T12-56-47.303435.jsonl b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T12-56-47.303435.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d349412e85f41fc04ef88dd36784f723be705e61 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T12-56-47.303435.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b88939d119cb9876450e15d30af248d129e4eaac0f6d6f5daf63508495d5662c +size 826859 diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T12-56-47.303435.jsonl b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T12-56-47.303435.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8332f4fc2be85e86ebdb3e38b7f4b876985b30b9 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T12-56-47.303435.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c489a379e7ad7898390535ead6d1a6d23be6f3e40691c74e3546a15ca8b916d +size 1172240 diff --git a/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T12-56-47.303435.jsonl b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T12-56-47.303435.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f3c52fbb9811ab5e5d7c9754710b464a4ae4c9ef --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T12-56-47.303435.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c13551162068852b669c560c3e9bbc82f84845accd93ed37be9d4123cf1cc1e +size 1228980 diff --git a/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-09-02.096738.json b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-09-02.096738.json new file mode 100644 index 0000000000000000000000000000000000000000..8e9038e51bd7dd64209a4f91baf2c531303aa38b --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-09-02.096738.json @@ -0,0 +1,714 @@ +{ + "results": { + "niah_multikey_1": { + "alias": "niah_multikey_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_multikey_3": { + "alias": "niah_multikey_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.75, + "8192_stderr,none": "N/A" + }, + "niah_multivalue": { + "alias": "niah_multivalue", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_single_2": { + "alias": "niah_single_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "ruler_cwe": { + "alias": "ruler_cwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.40499999999999997, + "8192_stderr,none": "N/A" + }, + "ruler_qa_squad": { + "alias": "ruler_qa_squad", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.4208333333333334, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_2": [], + "niah_multikey_1": [], + "niah_multikey_3": [], + "niah_multivalue": [], + "ruler_cwe": [], + "ruler_qa_squad": [] + }, + "configs": { + "niah_multikey_1": { + "task": "niah_multikey_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_k=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multikey_3": { + "task": "niah_multikey_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"uuids\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multivalue": { + "task": "niah_multivalue", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multivalue(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_v=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_2": { + "task": "niah_single_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_cwe": { + "task": "ruler_cwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_cw_dataset(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": "\n\n", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": "\n\n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 120, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_squad": { + "task": "ruler_qa_squad", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_squad(**kwargs):\n return get_qa_dataset(\"squad\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_1": 1.0, + "niah_multikey_3": 1.0, + "niah_multivalue": 1.0, + "niah_single_2": 1.0, + "ruler_cwe": 1.0, + "ruler_qa_squad": 1.0 + }, + "n-shot": { + "niah_multikey_1": 0, + "niah_multikey_3": 0, + "niah_multivalue": 0, + "niah_single_2": 0, + "ruler_cwe": 0, + "ruler_qa_squad": 0 + }, + "higher_is_better": { + "niah_multikey_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multikey_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multivalue": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_cwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_squad": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_squad": { + "original": 500, + "effective": 20 + }, + "ruler_cwe": { + "original": 500, + "effective": 20 + }, + "niah_multivalue": { + "original": 500, + "effective": 20 + }, + "niah_multikey_3": { + "original": 500, + "effective": 20 + }, + "niah_multikey_1": { + "original": 500, + "effective": 20 + }, + "niah_single_2": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784379413.7114463, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_squad": "8c658a728ccd3d67ddd3f45e8e76527cf15a1ae4119033d8246f293ecc68cf11", + "ruler_cwe": "f68206129fc7d709951d550ec9269fed05e73bea2fc86ce9f2c758974db5db71", + "niah_multivalue": "29646655c230b4f155a8de1fbc7ddf22c49e43045bd8fdab5770e7cb45026c7c", + "niah_multikey_3": "de2cec52885e38d56e0bda0e344e989909969c71ab5f8048b9368a011dd3616b", + "niah_multikey_1": "338f81e9efa98dbe6979d4f178f7f06cdbd6d367072318667a932b43c5032ab8", + "niah_single_2": "88ba10e924c2fc6c47543eb646b9e6f7487c7ca026c9391d6cd7054fde59eaad" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "731.6207718880032" +} \ No newline at end of file diff --git a/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T13-09-02.096738.jsonl b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T13-09-02.096738.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8d7d40cc05a9b9a5bcf36a7ab24e4f89aeb29569 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T13-09-02.096738.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:853ac1c1aecd7005e4d2c3a2442e2f95ab2b6195a181e18dd890522231aa15fd +size 1446074 diff --git a/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T13-09-02.096738.jsonl b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T13-09-02.096738.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..28c80dbd26896fed779bfba3243076621f4c2c13 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T13-09-02.096738.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63b35d8f844bf3281a7c84f9bf92e04c7a9f88dfaadc0b9db2c98c140972b0b7 +size 503288 diff --git a/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T13-09-02.096738.jsonl b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T13-09-02.096738.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9a56b79fd8a174e8caa7932eaa9961b0a74809a5 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T13-09-02.096738.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c47b0de2f9608db51ea260b6b5da89687cf93e04b3104517ee958e5312c24d63 +size 1434824 diff --git a/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T13-09-02.096738.jsonl b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T13-09-02.096738.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d80218bfe6cca84f6944d0471c0f5d55d04fdb0e --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T13-09-02.096738.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83d577e7c704bafad43cc64c986087331cc9b1da48459a533c14cedda1bdcd63 +size 1437596 diff --git a/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T13-09-02.096738.jsonl b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T13-09-02.096738.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2a2a65d9b32832dbc66b99125b1bf6b02c7c1001 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T13-09-02.096738.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9b295568237b75ee448d27bcfd3773b39c837e618a03e9e2b1c74cb81cb0fb2 +size 635633 diff --git a/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T13-09-02.096738.jsonl b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T13-09-02.096738.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3d6feec1f2091f590be8fbd4b8989c15140b1541 --- /dev/null +++ b/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T13-09-02.096738.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c66ae17c903fd34f65a7b5acce334ff5cde436941a2afd501e48c14ca9b2b99 +size 1387354 diff --git a/outputs/eval/token_t0525/ruler_a.DONE b/outputs/eval/token_t0525/ruler_a.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0525/ruler_b.DONE b/outputs/eval/token_t0525/ruler_b.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0525/sparsity_babilong.json b/outputs/eval/token_t0525/sparsity_babilong.json new file mode 100644 index 0000000000000000000000000000000000000000..a3481227feaa12d36c2abbf59a61d91ad2ca31f7 --- /dev/null +++ b/outputs/eval/token_t0525/sparsity_babilong.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 53161864, + "effective_router_decisions": 425294912, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 54488120, + "total_decisions": 425294912, + "sparsity": 0.12811843843549203, + "full_attention_usage": 0.8718815615645079, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 368, + "total_decisions": 15189104, + "sparsity": 2.4227893890251854e-05 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 1088, + "total_decisions": 15189104, + "sparsity": 7.163029497987504e-05 + }, + "12": { + "sparse_decisions": 2664, + "total_decisions": 15189104, + "sparsity": 0.00017538888403160582 + }, + "13": { + "sparse_decisions": 40904, + "total_decisions": 15189104, + "sparsity": 0.002692983075236038 + }, + "14": { + "sparse_decisions": 68952, + "total_decisions": 15189104, + "sparsity": 0.004539569944349581 + }, + "15": { + "sparse_decisions": 2841368, + "total_decisions": 15189104, + "sparsity": 0.18706620219336176 + }, + "16": { + "sparse_decisions": 28696, + "total_decisions": 15189104, + "sparsity": 0.0018892490300942044 + }, + "17": { + "sparse_decisions": 1226768, + "total_decisions": 15189104, + "sparsity": 0.08076631774988176 + }, + "18": { + "sparse_decisions": 403184, + "total_decisions": 15189104, + "sparsity": 0.026544291223498107 + }, + "19": { + "sparse_decisions": 374392, + "total_decisions": 15189104, + "sparsity": 0.02464872187325862 + }, + "20": { + "sparse_decisions": 1019824, + "total_decisions": 15189104, + "sparsity": 0.06714181429003317 + }, + "21": { + "sparse_decisions": 1591304, + "total_decisions": 15189104, + "sparsity": 0.10476615342155798 + }, + "22": { + "sparse_decisions": 8488256, + "total_decisions": 15189104, + "sparsity": 0.5588384936991675 + }, + "23": { + "sparse_decisions": 10478168, + "total_decisions": 15189104, + "sparsity": 0.6898476697506317 + }, + "24": { + "sparse_decisions": 6924024, + "total_decisions": 15189104, + "sparsity": 0.45585467055857937 + }, + "25": { + "sparse_decisions": 4631320, + "total_decisions": 15189104, + "sparsity": 0.3049106780755468 + }, + "26": { + "sparse_decisions": 1179736, + "total_decisions": 15189104, + "sparsity": 0.07766988757203848 + }, + "27": { + "sparse_decisions": 15187104, + "total_decisions": 15189104, + "sparsity": 0.9998683266636399 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 55088, + "total_decisions": 840000, + "sparsity": 0.06558095238095238, + "full_attention_usage": 0.9344190476190476 + }, + "prefill": { + "sparse_decisions": 54433032, + "total_decisions": 424454912, + "sparsity": 0.12824220066983227, + "full_attention_usage": 0.8717577993301677 + } + } +} diff --git a/outputs/eval/token_t0525/sparsity_helmet.json b/outputs/eval/token_t0525/sparsity_helmet.json new file mode 100644 index 0000000000000000000000000000000000000000..91f0e9b84ce552121d149c74e7764f161a17224a --- /dev/null +++ b/outputs/eval/token_t0525/sparsity_helmet.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 48728652, + "effective_router_decisions": 389829216, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 47409952, + "total_decisions": 389829216, + "sparsity": 0.12161723661060848, + "full_attention_usage": 0.8783827633893915, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 56, + "total_decisions": 13922472, + "sparsity": 4.022274205327904e-06 + }, + "12": { + "sparse_decisions": 104, + "total_decisions": 13922472, + "sparsity": 7.46993780989468e-06 + }, + "13": { + "sparse_decisions": 1424, + "total_decisions": 13922472, + "sparsity": 0.000102280686935481 + }, + "14": { + "sparse_decisions": 3816, + "total_decisions": 13922472, + "sparsity": 0.00027408925656305865 + }, + "15": { + "sparse_decisions": 1690896, + "total_decisions": 13922472, + "sparsity": 0.12145084579807379 + }, + "16": { + "sparse_decisions": 4416, + "total_decisions": 13922472, + "sparsity": 0.0003171850516201433 + }, + "17": { + "sparse_decisions": 319072, + "total_decisions": 13922472, + "sparsity": 0.022917769200756877 + }, + "18": { + "sparse_decisions": 353264, + "total_decisions": 13922472, + "sparsity": 0.025373654908409943 + }, + "19": { + "sparse_decisions": 1200496, + "total_decisions": 13922472, + "sparsity": 0.08622721597141657 + }, + "20": { + "sparse_decisions": 2052752, + "total_decisions": 13922472, + "sparsity": 0.1474416324917012 + }, + "21": { + "sparse_decisions": 3338808, + "total_decisions": 13922472, + "sparsity": 0.23981430883825802 + }, + "22": { + "sparse_decisions": 6120168, + "total_decisions": 13922472, + "sparsity": 0.4395891764048798 + }, + "23": { + "sparse_decisions": 7215544, + "total_decisions": 13922472, + "sparsity": 0.5182660090822951 + }, + "24": { + "sparse_decisions": 5079128, + "total_decisions": 13922472, + "sparsity": 0.3648150989278341 + }, + "25": { + "sparse_decisions": 5320344, + "total_decisions": 13922472, + "sparsity": 0.38214075776198364 + }, + "26": { + "sparse_decisions": 789192, + "total_decisions": 13922472, + "sparsity": 0.056684761154484634 + }, + "27": { + "sparse_decisions": 13920472, + "total_decisions": 13922472, + "sparsity": 0.9998563473498098 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 95368, + "total_decisions": 1064000, + "sparsity": 0.08963157894736842, + "full_attention_usage": 0.9103684210526316 + }, + "prefill": { + "sparse_decisions": 47314584, + "total_decisions": 388765216, + "sparsity": 0.12170477720928613, + "full_attention_usage": 0.8782952227907139 + } + } +} diff --git a/outputs/eval/token_t0525/sparsity_mrcr.json b/outputs/eval/token_t0525/sparsity_mrcr.json new file mode 100644 index 0000000000000000000000000000000000000000..c755eb8dd8bb338041930ab062886c02e14c4c00 --- /dev/null +++ b/outputs/eval/token_t0525/sparsity_mrcr.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 5358444, + "effective_router_decisions": 42867552, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 12437064, + "total_decisions": 42867552, + "sparsity": 0.29012769378573333, + "full_attention_usage": 0.7098723062142667, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 16, + "total_decisions": 1530984, + "sparsity": 1.045079504423299e-05 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 1530984, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 1152, + "total_decisions": 1530984, + "sparsity": 0.0007524572431847753 + }, + "12": { + "sparse_decisions": 688, + "total_decisions": 1530984, + "sparsity": 0.0004493841869020186 + }, + "13": { + "sparse_decisions": 12784, + "total_decisions": 1530984, + "sparsity": 0.008350185240342159 + }, + "14": { + "sparse_decisions": 73296, + "total_decisions": 1530984, + "sparsity": 0.04787509209763133 + }, + "15": { + "sparse_decisions": 902304, + "total_decisions": 1530984, + "sparsity": 0.5893621357244753 + }, + "16": { + "sparse_decisions": 230784, + "total_decisions": 1530984, + "sparsity": 0.15074226771801666 + }, + "17": { + "sparse_decisions": 1032720, + "total_decisions": 1530984, + "sparsity": 0.6745465661300183 + }, + "18": { + "sparse_decisions": 752080, + "total_decisions": 1530984, + "sparsity": 0.4912396210541717 + }, + "19": { + "sparse_decisions": 1041256, + "total_decisions": 1530984, + "sparsity": 0.6801220652861166 + }, + "20": { + "sparse_decisions": 1016688, + "total_decisions": 1530984, + "sparsity": 0.6640748694956969 + }, + "21": { + "sparse_decisions": 958376, + "total_decisions": 1530984, + "sparsity": 0.6259869469569898 + }, + "22": { + "sparse_decisions": 1108448, + "total_decisions": 1530984, + "sparsity": 0.724010179074373 + }, + "23": { + "sparse_decisions": 1183312, + "total_decisions": 1530984, + "sparsity": 0.7729094490863393 + }, + "24": { + "sparse_decisions": 1103448, + "total_decisions": 1530984, + "sparsity": 0.7207443056230503 + }, + "25": { + "sparse_decisions": 1032680, + "total_decisions": 1530984, + "sparsity": 0.6745204391424078 + }, + "26": { + "sparse_decisions": 456288, + "total_decisions": 1530984, + "sparsity": 0.2980357730714364 + }, + "27": { + "sparse_decisions": 1530744, + "total_decisions": 1530984, + "sparsity": 0.9998432380743365 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 431888, + "total_decisions": 2097760, + "sparsity": 0.2058805583098162, + "full_attention_usage": 0.7941194416901838 + }, + "prefill": { + "sparse_decisions": 12005176, + "total_decisions": 40769792, + "sparsity": 0.29446252754980945, + "full_attention_usage": 0.7055374724501906 + } + } +} diff --git a/outputs/eval/token_t0525/sparsity_ruler_a.json b/outputs/eval/token_t0525/sparsity_ruler_a.json new file mode 100644 index 0000000000000000000000000000000000000000..9442a323b8ce5d5f7f40b2e7d527442935313e9c --- /dev/null +++ b/outputs/eval/token_t0525/sparsity_ruler_a.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 29979236, + "effective_router_decisions": 239833888, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 40497008, + "total_decisions": 239833888, + "sparsity": 0.16885440309419492, + "full_attention_usage": 0.831145596905805, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 720, + "total_decisions": 8565496, + "sparsity": 8.405817946794909e-05 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 2272, + "total_decisions": 8565496, + "sparsity": 0.00026525025520997264 + }, + "12": { + "sparse_decisions": 520, + "total_decisions": 8565496, + "sparsity": 6.070868517129656e-05 + }, + "13": { + "sparse_decisions": 98320, + "total_decisions": 8565496, + "sparsity": 0.01147861139623438 + }, + "14": { + "sparse_decisions": 105552, + "total_decisions": 8565496, + "sparsity": 0.012322929110001336 + }, + "15": { + "sparse_decisions": 3550232, + "total_decisions": 8565496, + "sparsity": 0.4144806091789664 + }, + "16": { + "sparse_decisions": 140408, + "total_decisions": 8565496, + "sparsity": 0.016392278976021937 + }, + "17": { + "sparse_decisions": 3230520, + "total_decisions": 8565496, + "sparsity": 0.3771550415761095 + }, + "18": { + "sparse_decisions": 987400, + "total_decisions": 8565496, + "sparsity": 0.11527645334257351 + }, + "19": { + "sparse_decisions": 1067592, + "total_decisions": 8565496, + "sparsity": 0.1246386665757593 + }, + "20": { + "sparse_decisions": 3035768, + "total_decisions": 8565496, + "sparsity": 0.35441823800980116 + }, + "21": { + "sparse_decisions": 2289064, + "total_decisions": 8565496, + "sparsity": 0.26724243406336307 + }, + "22": { + "sparse_decisions": 4181680, + "total_decisions": 8565496, + "sparsity": 0.4882005665521296 + }, + "23": { + "sparse_decisions": 5869632, + "total_decisions": 8565496, + "sparsity": 0.6852646945372457 + }, + "24": { + "sparse_decisions": 3762952, + "total_decisions": 8565496, + "sparsity": 0.439315131312886 + }, + "25": { + "sparse_decisions": 3115600, + "total_decisions": 8565496, + "sparsity": 0.363738422153253 + }, + "26": { + "sparse_decisions": 494408, + "total_decisions": 8565496, + "sparsity": 0.057720883881096904 + }, + "27": { + "sparse_decisions": 8564368, + "total_decisions": 8565496, + "sparsity": 0.9998683088521669 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 778856, + "total_decisions": 2764160, + "sparsity": 0.2817695068302848, + "full_attention_usage": 0.7182304931697152 + }, + "prefill": { + "sparse_decisions": 39718152, + "total_decisions": 237069728, + "sparsity": 0.16753784776772512, + "full_attention_usage": 0.8324621522322748 + } + } +} diff --git a/outputs/eval/token_t0525/sparsity_ruler_b.json b/outputs/eval/token_t0525/sparsity_ruler_b.json new file mode 100644 index 0000000000000000000000000000000000000000..0a42e7140aed1e1a4d10c2090ccba1e58b7d4ac9 --- /dev/null +++ b/outputs/eval/token_t0525/sparsity_ruler_b.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 25272464, + "effective_router_decisions": 202179712, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 37034656, + "total_decisions": 202179712, + "sparsity": 0.18317691539693162, + "full_attention_usage": 0.8168230846030684, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 8, + "total_decisions": 7220704, + "sparsity": 1.1079252106165826e-06 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 544, + "total_decisions": 7220704, + "sparsity": 7.533891432192761e-05 + }, + "12": { + "sparse_decisions": 280, + "total_decisions": 7220704, + "sparsity": 3.877738237158039e-05 + }, + "13": { + "sparse_decisions": 11632, + "total_decisions": 7220704, + "sparsity": 0.001610923256236511 + }, + "14": { + "sparse_decisions": 87792, + "total_decisions": 7220704, + "sparsity": 0.012158371261306377 + }, + "15": { + "sparse_decisions": 1535112, + "total_decisions": 7220704, + "sparsity": 0.2125986607400054 + }, + "16": { + "sparse_decisions": 95616, + "total_decisions": 7220704, + "sparsity": 0.013241922117289395 + }, + "17": { + "sparse_decisions": 1435296, + "total_decisions": 7220704, + "sparsity": 0.1987750778871423 + }, + "18": { + "sparse_decisions": 1026864, + "total_decisions": 7220704, + "sparsity": 0.1422110641843233 + }, + "19": { + "sparse_decisions": 1038936, + "total_decisions": 7220704, + "sparsity": 0.14388292332714372 + }, + "20": { + "sparse_decisions": 2143848, + "total_decisions": 7220704, + "sparsity": 0.2969029058662424 + }, + "21": { + "sparse_decisions": 2544592, + "total_decisions": 7220704, + "sparsity": 0.3524022034416589 + }, + "22": { + "sparse_decisions": 4925880, + "total_decisions": 7220704, + "sparsity": 0.6821883295590014 + }, + "23": { + "sparse_decisions": 5586664, + "total_decisions": 7220704, + "sparsity": 0.7737007361055099 + }, + "24": { + "sparse_decisions": 4674040, + "total_decisions": 7220704, + "sparsity": 0.6473108439287915 + }, + "25": { + "sparse_decisions": 3806240, + "total_decisions": 7220704, + "sparsity": 0.5271286567071577 + }, + "26": { + "sparse_decisions": 901568, + "total_decisions": 7220704, + "sparsity": 0.12485873953564638 + }, + "27": { + "sparse_decisions": 7219744, + "total_decisions": 7220704, + "sparsity": 0.999867048974726 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 778736, + "total_decisions": 2947840, + "sparsity": 0.26417173252279635, + "full_attention_usage": 0.7358282674772036 + }, + "prefill": { + "sparse_decisions": 36255920, + "total_decisions": 199231872, + "sparsity": 0.1819785139598548, + "full_attention_usage": 0.8180214860401452 + } + } +} diff --git a/outputs/eval/token_t0575/babilong.DONE b/outputs/eval/token_t0575/babilong.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-02-04.454637.json b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-02-04.454637.json new file mode 100644 index 0000000000000000000000000000000000000000..9604e75b0600c834bcebd83b68c9d40cd3db7f8a --- /dev/null +++ b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-02-04.454637.json @@ -0,0 +1,505 @@ +{ + "results": { + "babilong_longctx": { + "acc,none": 0.592, + "acc_stderr,none": 0.02981302276741847, + "alias": "babilong_longctx" + }, + "babilong_qa1": { + "alias": " - babilong_qa1", + "acc,none": 0.76, + "acc_stderr,none": 0.06101187572589323 + }, + "babilong_qa2": { + "alias": " - babilong_qa2", + "acc,none": 0.5, + "acc_stderr,none": 0.07142857142857142 + }, + "babilong_qa3": { + "alias": " - babilong_qa3", + "acc,none": 0.34, + "acc_stderr,none": 0.06767268161329719 + }, + "babilong_qa4": { + "alias": " - babilong_qa4", + "acc,none": 0.64, + "acc_stderr,none": 0.06857142857142856 + }, + "babilong_qa5": { + "alias": " - babilong_qa5", + "acc,none": 0.72, + "acc_stderr,none": 0.06414269805898185 + } + }, + "groups": { + "babilong_longctx": { + "acc,none": 0.592, + "acc_stderr,none": 0.02981302276741847, + "alias": "babilong_longctx" + } + }, + "group_subtasks": { + "babilong_longctx": [ + "babilong_qa1", + "babilong_qa2", + "babilong_qa3", + "babilong_qa4", + "babilong_qa5" + ] + }, + "configs": { + "babilong_qa1": { + "task": "babilong_qa1", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa1" + }, + "test_split": "qa1", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about positions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location to answer the question.\nAlways return your answer in the following format:\nThe most recent location of 'person' is 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the hallway. Judith come back to the kitchen. Charlie travelled to balcony.", + "question": "Where is Charlie?", + "target": "The most recent location of Charlie is balcony." + }, + { + "input": "Alan moved to the garage. Charlie went to the beach. Alan went to the shop. Rouse travelled to balcony.", + "question": "Where is Alan?", + "target": "The most recent location of Alan is shop." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa2": { + "task": "babilong_qa2", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa2" + }, + "test_split": "qa2", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nThe 'item' is in 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the kitchen. Charlie got a bottle. Charlie moved to the balcony.", + "question": "Where is the bottle?", + "target": "The bottle is in the balcony." + }, + { + "input": "Alan moved to the garage. Alan got a screw driver. Alan moved to the kitchen.", + "question": "Where is the screw driver?", + "target": "The screw driver is in the kitchen." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa3": { + "task": "babilong_qa3", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa3" + }, + "test_split": "qa3", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nBefore the $location_1$ the $item$ was in the $location_2$. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "John journeyed to the bedroom. Mary grabbed the apple. Mary went back to the bathroom. Daniel journeyed to the bedroom. Daniel moved to the garden. Mary travelled to the kitchen.", + "question": "Where was the apple before the kitchen?", + "target": "Before the kitchen the apple was in the bathroom." + }, + { + "input": "John went back to the bedroom. John went back to the garden. John went back to the kitchen. Sandra took the football. Sandra travelled to the garden. Sandra journeyed to the bedroom.", + "question": "Where was the football before the bedroom?", + "target": "Before the bedroom the football was in the garden." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa4": { + "task": "babilong_qa4", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa4" + }, + "test_split": "qa4", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about different people, their location and actions, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - location. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "The hallway is south of the kitchen. The bedroom is north of the kitchen.", + "question": "What is the kitchen south of?", + "target": "bedroom" + }, + { + "input": "The garden is west of the bedroom. The bedroom is west of the kitchen.", + "question": "What is west of the bedroom?", + "target": "garden" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa5": { + "task": "babilong_qa5", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa5" + }, + "test_split": "qa5", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and their relations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word. Do not write anything else after that. Do not explain your answer.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Mary picked up the apple there. Mary gave the apple to Fred. Mary moved to the bedroom. Bill took the milk there.", + "question": "Who did Mary give the apple to?", + "target": "Fred" + }, + { + "input": "Jeff took the football there. Jeff passed the football to Fred. Jeff got the milk there. Bill travelled to the bedroom.", + "question": "Who gave the football?", + "target": "Jeff" + }, + { + "input": "Fred picked up the apple there. Fred handed the apple to Bill. Bill journeyed to the bedroom. Jeff went back to the garden.", + "question": "What did Fred give to Bill?", + "target": "apple" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + } + }, + "versions": { + "babilong_longctx": 0.0, + "babilong_qa1": 0.0, + "babilong_qa2": 0.0, + "babilong_qa3": 0.0, + "babilong_qa4": 0.0, + "babilong_qa5": 0.0 + }, + "n-shot": { + "babilong_qa1": 2, + "babilong_qa2": 2, + "babilong_qa3": 2, + "babilong_qa4": 2, + "babilong_qa5": 2 + }, + "higher_is_better": { + "babilong_longctx": { + "acc": true + }, + "babilong_qa1": { + "acc": true + }, + "babilong_qa2": { + "acc": true + }, + "babilong_qa3": { + "acc": true + }, + "babilong_qa4": { + "acc": true + }, + "babilong_qa5": { + "acc": true + } + }, + "n-samples": { + "babilong_qa1": { + "original": 1000, + "effective": 50 + }, + "babilong_qa2": { + "original": 999, + "effective": 50 + }, + "babilong_qa3": { + "original": 999, + "effective": 50 + }, + "babilong_qa4": { + "original": 999, + "effective": 50 + }, + "babilong_qa5": { + "original": 999, + "effective": 50 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 50.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784382979.542803, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "babilong_qa1": "bbdcd109bea1bb245e118e814bd0563de6384bc875a21df5218ab4ed75b27380", + "babilong_qa2": "fa4f8d12bdd481e02a1051414304811f648caf7bce8f89dcf03ecbf5b86f1e92", + "babilong_qa3": "737b64396c8b2e5f96da9796bcd5e448f08e5a13e32ae7b610d31a60269b6e48", + "babilong_qa4": "298a25727c69c9babddcd78f438cae1aa21a388e14e7db90b2206eff753c90c0", + "babilong_qa5": "e60afcdcdd1c4b3e24b002ffbf8f87085f5ec4a02fcd72318f042dcdafe230f4" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "348.0794764540042" +} \ No newline at end of file diff --git a/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T14-02-04.454637.jsonl b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T14-02-04.454637.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1e3cb7830be21662f604079d362370ff21ba3b9a --- /dev/null +++ b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T14-02-04.454637.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d8cff3684d6ee3ff31dd750e9812e88fc18508db2a918c0b24eed7a624d71d6 +size 3164122 diff --git a/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T14-02-04.454637.jsonl b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T14-02-04.454637.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..96118ea4a0da0f89ec34c49bb2a44cfdebaf5322 --- /dev/null +++ b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T14-02-04.454637.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cea240351e414848beb203e7d358f8ddeaca9fbd5dfb384e41e3af7871e380eb +size 3147482 diff --git a/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T14-02-04.454637.jsonl b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T14-02-04.454637.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b642bfccb804eddd0403b735f1103c13bb38dfe5 --- /dev/null +++ b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T14-02-04.454637.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4d31bf5786e5e0458a04f93f56067cf367d811a74624f8840853ffdaf6b4a02 +size 3126930 diff --git a/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T14-02-04.454637.jsonl b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T14-02-04.454637.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ed02384a9769d2b2675a4e8d1321ff00c4d9b45a --- /dev/null +++ b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T14-02-04.454637.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8db8b94a49cbe14110892a5558c4448fb0d1d5b3a29367a81df5ff3e35ffa307 +size 3137618 diff --git a/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T14-02-04.454637.jsonl b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T14-02-04.454637.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9efb3bfbd3e022af3e83fcaeeb8a95867cbfa6d9 --- /dev/null +++ b/outputs/eval/token_t0575/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T14-02-04.454637.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b701471e4f259f9c80ff79c599ed03b503cdd77cb0967803c5231f4b167990c2 +size 3135738 diff --git a/outputs/eval/token_t0575/helmet.DONE b/outputs/eval/token_t0575/helmet.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0575/helmet_icl8k_n50.jsonl b/outputs/eval/token_t0575/helmet_icl8k_n50.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e1785f449dff88cdd7eb86ea8dbca22e5fb728b1 --- /dev/null +++ b/outputs/eval/token_t0575/helmet_icl8k_n50.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:722b9005b1bc8e3b09d2d2484ba18457ff704478b8ebcff0a436ab8f366dff5d +size 6469002 diff --git a/outputs/eval/token_t0575/helmet_icl8k_n50.summary.csv b/outputs/eval/token_t0575/helmet_icl8k_n50.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..2dfb9985de62f6f5075ce63a7a8e1275f5599170 --- /dev/null +++ b/outputs/eval/token_t0575/helmet_icl8k_n50.summary.csv @@ -0,0 +1,251 @@ +method,benchmark,config,n,score,prompt_tokens +token_t0575,helmet_icl,trec_coarse,1,1.0,6597 +token_t0575,helmet_icl,trec_coarse,1,1.0,6549 +token_t0575,helmet_icl,trec_coarse,1,0.0,6508 +token_t0575,helmet_icl,trec_coarse,1,0.0,6608 +token_t0575,helmet_icl,trec_coarse,1,1.0,6497 +token_t0575,helmet_icl,trec_coarse,1,0.0,6545 +token_t0575,helmet_icl,trec_coarse,1,1.0,6583 +token_t0575,helmet_icl,trec_coarse,1,1.0,6455 +token_t0575,helmet_icl,trec_coarse,1,1.0,6643 +token_t0575,helmet_icl,trec_coarse,1,1.0,6568 +token_t0575,helmet_icl,trec_coarse,1,0.0,6518 +token_t0575,helmet_icl,trec_coarse,1,1.0,6424 +token_t0575,helmet_icl,trec_coarse,1,1.0,6750 +token_t0575,helmet_icl,trec_coarse,1,1.0,6642 +token_t0575,helmet_icl,trec_coarse,1,1.0,6494 +token_t0575,helmet_icl,trec_coarse,1,0.0,6427 +token_t0575,helmet_icl,trec_coarse,1,0.0,6547 +token_t0575,helmet_icl,trec_coarse,1,0.0,6612 +token_t0575,helmet_icl,trec_coarse,1,1.0,6591 +token_t0575,helmet_icl,trec_coarse,1,1.0,6485 +token_t0575,helmet_icl,trec_coarse,1,1.0,6542 +token_t0575,helmet_icl,trec_coarse,1,1.0,6409 +token_t0575,helmet_icl,trec_coarse,1,0.0,6482 +token_t0575,helmet_icl,trec_coarse,1,1.0,6502 +token_t0575,helmet_icl,trec_coarse,1,0.0,6344 +token_t0575,helmet_icl,trec_coarse,1,0.0,6479 +token_t0575,helmet_icl,trec_coarse,1,1.0,6569 +token_t0575,helmet_icl,trec_coarse,1,1.0,6389 +token_t0575,helmet_icl,trec_coarse,1,1.0,6479 +token_t0575,helmet_icl,trec_coarse,1,1.0,6653 +token_t0575,helmet_icl,trec_coarse,1,1.0,6532 +token_t0575,helmet_icl,trec_coarse,1,1.0,6491 +token_t0575,helmet_icl,trec_coarse,1,1.0,6489 +token_t0575,helmet_icl,trec_coarse,1,0.0,6630 +token_t0575,helmet_icl,trec_coarse,1,1.0,6488 +token_t0575,helmet_icl,trec_coarse,1,1.0,6499 +token_t0575,helmet_icl,trec_coarse,1,0.0,6483 +token_t0575,helmet_icl,trec_coarse,1,1.0,6660 +token_t0575,helmet_icl,trec_coarse,1,0.0,6675 +token_t0575,helmet_icl,trec_coarse,1,0.0,6544 +token_t0575,helmet_icl,trec_coarse,1,1.0,6520 +token_t0575,helmet_icl,trec_coarse,1,1.0,6359 +token_t0575,helmet_icl,trec_coarse,1,0.0,6657 +token_t0575,helmet_icl,trec_coarse,1,1.0,6546 +token_t0575,helmet_icl,trec_coarse,1,0.0,6481 +token_t0575,helmet_icl,trec_coarse,1,1.0,6506 +token_t0575,helmet_icl,trec_coarse,1,0.0,6621 +token_t0575,helmet_icl,trec_coarse,1,0.0,6623 +token_t0575,helmet_icl,trec_coarse,1,0.0,6411 +token_t0575,helmet_icl,trec_coarse,1,1.0,6577 +token_t0575,helmet_icl,trec_fine,1,0.0,6858 +token_t0575,helmet_icl,trec_fine,1,1.0,6973 +token_t0575,helmet_icl,trec_fine,1,0.0,6977 +token_t0575,helmet_icl,trec_fine,1,1.0,6952 +token_t0575,helmet_icl,trec_fine,1,0.0,6919 +token_t0575,helmet_icl,trec_fine,1,0.0,6979 +token_t0575,helmet_icl,trec_fine,1,1.0,6957 +token_t0575,helmet_icl,trec_fine,1,0.0,7013 +token_t0575,helmet_icl,trec_fine,1,1.0,6824 +token_t0575,helmet_icl,trec_fine,1,1.0,6833 +token_t0575,helmet_icl,trec_fine,1,1.0,6868 +token_t0575,helmet_icl,trec_fine,1,1.0,6914 +token_t0575,helmet_icl,trec_fine,1,1.0,6759 +token_t0575,helmet_icl,trec_fine,1,1.0,6933 +token_t0575,helmet_icl,trec_fine,1,0.0,6868 +token_t0575,helmet_icl,trec_fine,1,0.0,6839 +token_t0575,helmet_icl,trec_fine,1,0.0,6838 +token_t0575,helmet_icl,trec_fine,1,0.0,7000 +token_t0575,helmet_icl,trec_fine,1,0.0,6800 +token_t0575,helmet_icl,trec_fine,1,1.0,6829 +token_t0575,helmet_icl,trec_fine,1,0.0,6971 +token_t0575,helmet_icl,trec_fine,1,1.0,6907 +token_t0575,helmet_icl,trec_fine,1,0.0,6818 +token_t0575,helmet_icl,trec_fine,1,1.0,6902 +token_t0575,helmet_icl,trec_fine,1,0.0,6941 +token_t0575,helmet_icl,trec_fine,1,0.0,6830 +token_t0575,helmet_icl,trec_fine,1,0.0,6934 +token_t0575,helmet_icl,trec_fine,1,1.0,6879 +token_t0575,helmet_icl,trec_fine,1,1.0,6915 +token_t0575,helmet_icl,trec_fine,1,0.0,6953 +token_t0575,helmet_icl,trec_fine,1,0.0,6875 +token_t0575,helmet_icl,trec_fine,1,1.0,6847 +token_t0575,helmet_icl,trec_fine,1,0.0,6847 +token_t0575,helmet_icl,trec_fine,1,1.0,6944 +token_t0575,helmet_icl,trec_fine,1,0.0,6818 +token_t0575,helmet_icl,trec_fine,1,0.0,6868 +token_t0575,helmet_icl,trec_fine,1,0.0,7022 +token_t0575,helmet_icl,trec_fine,1,0.0,6930 +token_t0575,helmet_icl,trec_fine,1,0.0,6863 +token_t0575,helmet_icl,trec_fine,1,0.0,6807 +token_t0575,helmet_icl,trec_fine,1,0.0,7001 +token_t0575,helmet_icl,trec_fine,1,0.0,6890 +token_t0575,helmet_icl,trec_fine,1,0.0,6952 +token_t0575,helmet_icl,trec_fine,1,1.0,6845 +token_t0575,helmet_icl,trec_fine,1,0.0,6902 +token_t0575,helmet_icl,trec_fine,1,0.0,6879 +token_t0575,helmet_icl,trec_fine,1,1.0,6896 +token_t0575,helmet_icl,trec_fine,1,0.0,6770 +token_t0575,helmet_icl,trec_fine,1,0.0,6933 +token_t0575,helmet_icl,trec_fine,1,0.0,6886 +token_t0575,helmet_icl,banking77,1,0.0,6809 +token_t0575,helmet_icl,banking77,1,0.0,7162 +token_t0575,helmet_icl,banking77,1,1.0,6876 +token_t0575,helmet_icl,banking77,1,1.0,7033 +token_t0575,helmet_icl,banking77,1,1.0,6647 +token_t0575,helmet_icl,banking77,1,1.0,6818 +token_t0575,helmet_icl,banking77,1,0.0,6841 +token_t0575,helmet_icl,banking77,1,1.0,6944 +token_t0575,helmet_icl,banking77,1,1.0,6776 +token_t0575,helmet_icl,banking77,1,1.0,7100 +token_t0575,helmet_icl,banking77,1,0.0,6967 +token_t0575,helmet_icl,banking77,1,1.0,6894 +token_t0575,helmet_icl,banking77,1,1.0,6900 +token_t0575,helmet_icl,banking77,1,0.0,6964 +token_t0575,helmet_icl,banking77,1,1.0,7085 +token_t0575,helmet_icl,banking77,1,1.0,6953 +token_t0575,helmet_icl,banking77,1,0.0,7406 +token_t0575,helmet_icl,banking77,1,0.0,6802 +token_t0575,helmet_icl,banking77,1,1.0,7119 +token_t0575,helmet_icl,banking77,1,0.0,7202 +token_t0575,helmet_icl,banking77,1,0.0,7103 +token_t0575,helmet_icl,banking77,1,0.0,7009 +token_t0575,helmet_icl,banking77,1,1.0,6940 +token_t0575,helmet_icl,banking77,1,0.0,6857 +token_t0575,helmet_icl,banking77,1,0.0,6922 +token_t0575,helmet_icl,banking77,1,1.0,6988 +token_t0575,helmet_icl,banking77,1,1.0,6945 +token_t0575,helmet_icl,banking77,1,1.0,6835 +token_t0575,helmet_icl,banking77,1,0.0,7035 +token_t0575,helmet_icl,banking77,1,1.0,6787 +token_t0575,helmet_icl,banking77,1,0.0,6940 +token_t0575,helmet_icl,banking77,1,0.0,6661 +token_t0575,helmet_icl,banking77,1,1.0,6934 +token_t0575,helmet_icl,banking77,1,0.0,7235 +token_t0575,helmet_icl,banking77,1,1.0,6829 +token_t0575,helmet_icl,banking77,1,1.0,7169 +token_t0575,helmet_icl,banking77,1,0.0,7074 +token_t0575,helmet_icl,banking77,1,0.0,6797 +token_t0575,helmet_icl,banking77,1,0.0,7013 +token_t0575,helmet_icl,banking77,1,1.0,6811 +token_t0575,helmet_icl,banking77,1,0.0,6983 +token_t0575,helmet_icl,banking77,1,0.0,7183 +token_t0575,helmet_icl,banking77,1,1.0,6801 +token_t0575,helmet_icl,banking77,1,0.0,7174 +token_t0575,helmet_icl,banking77,1,1.0,6773 +token_t0575,helmet_icl,banking77,1,1.0,7134 +token_t0575,helmet_icl,banking77,1,1.0,6907 +token_t0575,helmet_icl,banking77,1,1.0,7341 +token_t0575,helmet_icl,banking77,1,1.0,7042 +token_t0575,helmet_icl,banking77,1,1.0,7304 +token_t0575,helmet_icl,clinic150,1,1.0,7241 +token_t0575,helmet_icl,clinic150,1,0.0,7282 +token_t0575,helmet_icl,clinic150,1,1.0,7261 +token_t0575,helmet_icl,clinic150,1,0.0,7292 +token_t0575,helmet_icl,clinic150,1,1.0,7217 +token_t0575,helmet_icl,clinic150,1,0.0,7280 +token_t0575,helmet_icl,clinic150,1,1.0,7293 +token_t0575,helmet_icl,clinic150,1,0.0,7250 +token_t0575,helmet_icl,clinic150,1,1.0,7264 +token_t0575,helmet_icl,clinic150,1,1.0,7238 +token_t0575,helmet_icl,clinic150,1,1.0,7243 +token_t0575,helmet_icl,clinic150,1,1.0,7191 +token_t0575,helmet_icl,clinic150,1,1.0,7333 +token_t0575,helmet_icl,clinic150,1,1.0,7299 +token_t0575,helmet_icl,clinic150,1,1.0,7225 +token_t0575,helmet_icl,clinic150,1,1.0,7409 +token_t0575,helmet_icl,clinic150,1,1.0,7277 +token_t0575,helmet_icl,clinic150,1,1.0,7376 +token_t0575,helmet_icl,clinic150,1,1.0,7383 +token_t0575,helmet_icl,clinic150,1,0.0,7270 +token_t0575,helmet_icl,clinic150,1,1.0,7173 +token_t0575,helmet_icl,clinic150,1,1.0,7106 +token_t0575,helmet_icl,clinic150,1,1.0,7413 +token_t0575,helmet_icl,clinic150,1,1.0,7253 +token_t0575,helmet_icl,clinic150,1,0.0,7224 +token_t0575,helmet_icl,clinic150,1,1.0,7307 +token_t0575,helmet_icl,clinic150,1,1.0,7272 +token_t0575,helmet_icl,clinic150,1,0.0,7215 +token_t0575,helmet_icl,clinic150,1,1.0,7344 +token_t0575,helmet_icl,clinic150,1,1.0,7224 +token_t0575,helmet_icl,clinic150,1,1.0,7310 +token_t0575,helmet_icl,clinic150,1,1.0,7222 +token_t0575,helmet_icl,clinic150,1,0.0,7260 +token_t0575,helmet_icl,clinic150,1,1.0,7351 +token_t0575,helmet_icl,clinic150,1,1.0,7301 +token_t0575,helmet_icl,clinic150,1,1.0,7155 +token_t0575,helmet_icl,clinic150,1,1.0,7253 +token_t0575,helmet_icl,clinic150,1,1.0,7248 +token_t0575,helmet_icl,clinic150,1,1.0,7128 +token_t0575,helmet_icl,clinic150,1,0.0,7282 +token_t0575,helmet_icl,clinic150,1,1.0,7301 +token_t0575,helmet_icl,clinic150,1,1.0,7388 +token_t0575,helmet_icl,clinic150,1,0.0,7244 +token_t0575,helmet_icl,clinic150,1,1.0,7186 +token_t0575,helmet_icl,clinic150,1,0.0,7309 +token_t0575,helmet_icl,clinic150,1,1.0,7241 +token_t0575,helmet_icl,clinic150,1,1.0,7214 +token_t0575,helmet_icl,clinic150,1,0.0,7264 +token_t0575,helmet_icl,clinic150,1,0.0,7219 +token_t0575,helmet_icl,clinic150,1,1.0,7228 +token_t0575,helmet_icl,nlu,1,1.0,7063 +token_t0575,helmet_icl,nlu,1,1.0,6959 +token_t0575,helmet_icl,nlu,1,0.0,6985 +token_t0575,helmet_icl,nlu,1,1.0,6970 +token_t0575,helmet_icl,nlu,1,1.0,7025 +token_t0575,helmet_icl,nlu,1,0.0,7034 +token_t0575,helmet_icl,nlu,1,1.0,7004 +token_t0575,helmet_icl,nlu,1,0.0,7062 +token_t0575,helmet_icl,nlu,1,1.0,7062 +token_t0575,helmet_icl,nlu,1,1.0,7045 +token_t0575,helmet_icl,nlu,1,1.0,7028 +token_t0575,helmet_icl,nlu,1,1.0,7018 +token_t0575,helmet_icl,nlu,1,1.0,7075 +token_t0575,helmet_icl,nlu,1,1.0,7068 +token_t0575,helmet_icl,nlu,1,1.0,7025 +token_t0575,helmet_icl,nlu,1,1.0,7022 +token_t0575,helmet_icl,nlu,1,1.0,7112 +token_t0575,helmet_icl,nlu,1,1.0,6967 +token_t0575,helmet_icl,nlu,1,0.0,6980 +token_t0575,helmet_icl,nlu,1,1.0,7031 +token_t0575,helmet_icl,nlu,1,0.0,7093 +token_t0575,helmet_icl,nlu,1,0.0,6949 +token_t0575,helmet_icl,nlu,1,1.0,7046 +token_t0575,helmet_icl,nlu,1,1.0,7039 +token_t0575,helmet_icl,nlu,1,0.0,7126 +token_t0575,helmet_icl,nlu,1,1.0,7064 +token_t0575,helmet_icl,nlu,1,1.0,7057 +token_t0575,helmet_icl,nlu,1,0.0,6971 +token_t0575,helmet_icl,nlu,1,1.0,7034 +token_t0575,helmet_icl,nlu,1,1.0,7094 +token_t0575,helmet_icl,nlu,1,1.0,6979 +token_t0575,helmet_icl,nlu,1,1.0,7123 +token_t0575,helmet_icl,nlu,1,1.0,7144 +token_t0575,helmet_icl,nlu,1,0.0,7024 +token_t0575,helmet_icl,nlu,1,1.0,6943 +token_t0575,helmet_icl,nlu,1,0.0,6961 +token_t0575,helmet_icl,nlu,1,1.0,7005 +token_t0575,helmet_icl,nlu,1,1.0,7080 +token_t0575,helmet_icl,nlu,1,1.0,7179 +token_t0575,helmet_icl,nlu,1,1.0,7115 +token_t0575,helmet_icl,nlu,1,1.0,6946 +token_t0575,helmet_icl,nlu,1,1.0,6985 +token_t0575,helmet_icl,nlu,1,1.0,7131 +token_t0575,helmet_icl,nlu,1,1.0,7100 +token_t0575,helmet_icl,nlu,1,1.0,7190 +token_t0575,helmet_icl,nlu,1,1.0,7040 +token_t0575,helmet_icl,nlu,1,1.0,7013 +token_t0575,helmet_icl,nlu,1,0.0,7080 +token_t0575,helmet_icl,nlu,1,0.0,7098 +token_t0575,helmet_icl,nlu,1,1.0,6891 diff --git a/outputs/eval/token_t0575/mrcr.DONE b/outputs/eval/token_t0575/mrcr.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0575/mrcr_8k_2_4_8needle_n10.jsonl b/outputs/eval/token_t0575/mrcr_8k_2_4_8needle_n10.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..90519c2bf4bf71a4461aec3432f9476aac650c67 --- /dev/null +++ b/outputs/eval/token_t0575/mrcr_8k_2_4_8needle_n10.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:074ea0837e90cbbde705efaa4c7042c4bdee34c7692d1612f44919acf8d38de0 +size 986952 diff --git a/outputs/eval/token_t0575/mrcr_8k_2_4_8needle_n10.summary.csv b/outputs/eval/token_t0575/mrcr_8k_2_4_8needle_n10.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..adf49a05dc3179a2be0512d6d6d165d26ea145f7 --- /dev/null +++ b/outputs/eval/token_t0575/mrcr_8k_2_4_8needle_n10.summary.csv @@ -0,0 +1,31 @@ +method,benchmark,config,n,score,prompt_tokens +token_t0575,mrcr,8k_2needle,1,0.11641113003975014,4852 +token_t0575,mrcr,8k_2needle,1,0.017407237746220796,7119 +token_t0575,mrcr,8k_2needle,1,0.029922992299229925,5756 +token_t0575,mrcr,8k_2needle,1,0.05036855036855037,5072 +token_t0575,mrcr,8k_2needle,1,0.031995803829006034,5753 +token_t0575,mrcr,8k_2needle,1,0.027874564459930314,5481 +token_t0575,mrcr,8k_2needle,1,0.26126126126126126,5143 +token_t0575,mrcr,8k_2needle,1,0.09808760024676126,6399 +token_t0575,mrcr,8k_2needle,1,0.013729977116704805,5987 +token_t0575,mrcr,8k_2needle,1,0.0008233841086867024,4380 +token_t0575,mrcr,8k_4needle,1,0.028864656831302116,5123 +token_t0575,mrcr,8k_4needle,1,0.009056078021595262,7908 +token_t0575,mrcr,8k_4needle,1,0.032418952618453865,5814 +token_t0575,mrcr,8k_4needle,1,0.03157439446366782,4856 +token_t0575,mrcr,8k_4needle,1,0.024322830292979547,7413 +token_t0575,mrcr,8k_4needle,1,0.017857142857142856,7764 +token_t0575,mrcr,8k_4needle,1,0.06831351021652943,4312 +token_t0575,mrcr,8k_4needle,1,0.023166023166023165,4847 +token_t0575,mrcr,8k_4needle,1,0.04612365063788027,6287 +token_t0575,mrcr,8k_4needle,1,0.19771863117870722,7841 +token_t0575,mrcr,8k_8needle,1,0.009864364981504316,4581 +token_t0575,mrcr,8k_8needle,1,0.3022488755622189,6357 +token_t0575,mrcr,8k_8needle,1,0.05405405405405406,6981 +token_t0575,mrcr,8k_8needle,1,0.041728763040238454,6308 +token_t0575,mrcr,8k_8needle,1,0.038259407189405084,6901 +token_t0575,mrcr,8k_8needle,1,0.008839779005524863,5407 +token_t0575,mrcr,8k_8needle,1,0.01032258064516129,6351 +token_t0575,mrcr,8k_8needle,1,0.3012729844413013,5657 +token_t0575,mrcr,8k_8needle,1,0.0,7330 +token_t0575,mrcr,8k_8needle,1,0.01194152769199094,8028 diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-42-41.625240.json b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-42-41.625240.json new file mode 100644 index 0000000000000000000000000000000000000000..f3b02e1931d4d278f2d9eb78eecd810d236ac7be --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-42-41.625240.json @@ -0,0 +1,821 @@ +{ + "results": { + "niah_multikey_2": { + "alias": "niah_multikey_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.95, + "8192_stderr,none": "N/A" + }, + "niah_multiquery": { + "alias": "niah_multiquery", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.75, + "8192_stderr,none": "N/A" + }, + "niah_single_1": { + "alias": "niah_single_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.05, + "8192_stderr,none": "N/A" + }, + "niah_single_3": { + "alias": "niah_single_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.7, + "8192_stderr,none": "N/A" + }, + "ruler_fwe": { + "alias": "ruler_fwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.7666666666666665, + "8192_stderr,none": "N/A" + }, + "ruler_qa_hotpot": { + "alias": "ruler_qa_hotpot", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.3, + "8192_stderr,none": "N/A" + }, + "ruler_vt": { + "alias": "ruler_vt", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.8300000000000001, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_1": [], + "niah_single_3": [], + "niah_multikey_2": [], + "niah_multiquery": [], + "ruler_vt": [], + "ruler_fwe": [], + "ruler_qa_hotpot": [] + }, + "configs": { + "niah_multikey_2": { + "task": "niah_multikey_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multiquery": { + "task": "niah_multiquery", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multiquery(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_q=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_1": { + "task": "niah_single_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"repeat\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"repeat\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_3": { + "task": "niah_single_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_fwe": { + "task": "ruler_fwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def fwe_download(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, max_seq_length=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 50, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_hotpot": { + "task": "ruler_qa_hotpot", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_hotpotqa(**kwargs):\n return get_qa_dataset(\"hotpotqa\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_vt": { + "task": "ruler_vt", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_vt_dataset(**kwargs) -> dict[str, datasets.Dataset]:\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", \"\"))\n df = (\n get_dataset(tokenizer=get_tokenizer(pretrained), seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 30, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_2": 1.0, + "niah_multiquery": 1.0, + "niah_single_1": 1.0, + "niah_single_3": 1.0, + "ruler_fwe": 1.0, + "ruler_qa_hotpot": 1.0, + "ruler_vt": 1.0 + }, + "n-shot": { + "niah_multikey_2": 0, + "niah_multiquery": 0, + "niah_single_1": 0, + "niah_single_3": 0, + "ruler_fwe": 0, + "ruler_qa_hotpot": 0, + "ruler_vt": 0 + }, + "higher_is_better": { + "niah_multikey_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multiquery": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_fwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_hotpot": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_vt": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_hotpot": { + "original": 500, + "effective": 20 + }, + "ruler_fwe": { + "original": 500, + "effective": 20 + }, + "ruler_vt": { + "original": 500, + "effective": 20 + }, + "niah_multiquery": { + "original": 500, + "effective": 20 + }, + "niah_multikey_2": { + "original": 500, + "effective": 20 + }, + "niah_single_3": { + "original": 500, + "effective": 20 + }, + "niah_single_1": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784381275.8607025, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_hotpot": "fd48cb605efc8c5adac7e3771533db4d4c354b71bf91ce00841c814604ba6404", + "ruler_fwe": "c4633bcd0acf3ba4b96f905b5ba269ba94cf45a4c8ca0c13aae1468227cc3a10", + "ruler_vt": "af011da859c8b86f63f57f6256cf59077d3d3d8536084ab67ccf39e18e9a615a", + "niah_multiquery": "f9a36d2854dfd2bb52ffc7a6bf3f53490fb5303ca9da0046256bc28b55c2adb2", + "niah_multikey_2": "86e49b8f875843454b2923143b9124ba82e8a2e23b145f792726cd82f4e49c24", + "niah_single_3": "905e40146a287e5f0c8e2a81f64f082108c45a23862949e6a99bfcbbf561cad2", + "niah_single_1": "426e8138e992480aa416d01e6b9cef5865a68c00f532400e448a0c20ee026cff" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "889.0543280520069" +} \ No newline at end of file diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T13-42-41.625240.jsonl b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T13-42-41.625240.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6d0bd57d29873146af51ba59c09332c80a504d2d --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T13-42-41.625240.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8fa7fa26812ca76a15eb54a4e4c77f4e66199e5ff8377fc5f9edf98544127b7 +size 987134 diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T13-42-41.625240.jsonl b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T13-42-41.625240.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2e4655d55caa5960be09890f3ac2a30d56138efa --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T13-42-41.625240.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35e72bbd3f17bff700c15671cae4dae5beb4ba9b3de2851f68f2ffa7d05c486e +size 1442420 diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T13-42-41.625240.jsonl b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T13-42-41.625240.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5609b5fe9435c8f32ebe5d9affa9094e7ee688ad --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T13-42-41.625240.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1533e2e6de844cf4846a86d083312a200fae7a44c8175b01f13793a30615fbd8 +size 1231740 diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T13-42-41.625240.jsonl b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T13-42-41.625240.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..18a6d0a5c88e885ef8c23ee27cdaae3746edd5bb --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T13-42-41.625240.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da08c751c9278442069528cd9f542581d49a2b2ca6265a72ccba4598b6b9053d +size 1434518 diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T13-42-41.625240.jsonl b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T13-42-41.625240.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0b64f07e69c913592e5ae2f4e224fd179605273f --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T13-42-41.625240.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eece7e12e3ca0587fac828d954037e81014042ba5436465b5ece9474e58d13d5 +size 827286 diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T13-42-41.625240.jsonl b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T13-42-41.625240.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..64b06b60dfe8132986ce54f7f2b148c425070711 --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T13-42-41.625240.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d59611a59ca8e1016f79880d872637537e5bf807c4634f62fcf0d63870647305 +size 1171976 diff --git a/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T13-42-41.625240.jsonl b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T13-42-41.625240.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d380e9c20d096b67bbdc043adc203ac7d24e0fd8 --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T13-42-41.625240.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bbe07d9c3c6572087a20f6fa88dfa9ef89ad8915e1e374c863ac26b636c188c +size 1229104 diff --git a/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-56-13.180729.json b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-56-13.180729.json new file mode 100644 index 0000000000000000000000000000000000000000..868683687c0e91bc56f6ba4b66fcfda5e0d1ebc6 --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T13-56-13.180729.json @@ -0,0 +1,714 @@ +{ + "results": { + "niah_multikey_1": { + "alias": "niah_multikey_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.8, + "8192_stderr,none": "N/A" + }, + "niah_multikey_3": { + "alias": "niah_multikey_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.65, + "8192_stderr,none": "N/A" + }, + "niah_multivalue": { + "alias": "niah_multivalue", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.4875, + "8192_stderr,none": "N/A" + }, + "niah_single_2": { + "alias": "niah_single_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.2, + "8192_stderr,none": "N/A" + }, + "ruler_cwe": { + "alias": "ruler_cwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.32999999999999996, + "8192_stderr,none": "N/A" + }, + "ruler_qa_squad": { + "alias": "ruler_qa_squad", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.35416666666666663, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_2": [], + "niah_multikey_1": [], + "niah_multikey_3": [], + "niah_multivalue": [], + "ruler_cwe": [], + "ruler_qa_squad": [] + }, + "configs": { + "niah_multikey_1": { + "task": "niah_multikey_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_k=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multikey_3": { + "task": "niah_multikey_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"uuids\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multivalue": { + "task": "niah_multivalue", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multivalue(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_v=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_2": { + "task": "niah_single_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_cwe": { + "task": "ruler_cwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_cw_dataset(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": "\n\n", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": "\n\n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 120, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_squad": { + "task": "ruler_qa_squad", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_squad(**kwargs):\n return get_qa_dataset(\"squad\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_1": 1.0, + "niah_multikey_3": 1.0, + "niah_multivalue": 1.0, + "niah_single_2": 1.0, + "ruler_cwe": 1.0, + "ruler_qa_squad": 1.0 + }, + "n-shot": { + "niah_multikey_1": 0, + "niah_multikey_3": 0, + "niah_multivalue": 0, + "niah_single_2": 0, + "ruler_cwe": 0, + "ruler_qa_squad": 0 + }, + "higher_is_better": { + "niah_multikey_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multikey_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multivalue": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_cwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_squad": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_squad": { + "original": 500, + "effective": 20 + }, + "ruler_cwe": { + "original": 500, + "effective": 20 + }, + "niah_multivalue": { + "original": 500, + "effective": 20 + }, + "niah_multikey_3": { + "original": 500, + "effective": 20 + }, + "niah_multikey_1": { + "original": 500, + "effective": 20 + }, + "niah_single_2": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784382168.2726922, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_squad": "8c658a728ccd3d67ddd3f45e8e76527cf15a1ae4119033d8246f293ecc68cf11", + "ruler_cwe": "f68206129fc7d709951d550ec9269fed05e73bea2fc86ce9f2c758974db5db71", + "niah_multivalue": "29646655c230b4f155a8de1fbc7ddf22c49e43045bd8fdab5770e7cb45026c7c", + "niah_multikey_3": "de2cec52885e38d56e0bda0e344e989909969c71ab5f8048b9368a011dd3616b", + "niah_multikey_1": "338f81e9efa98dbe6979d4f178f7f06cdbd6d367072318667a932b43c5032ab8", + "niah_single_2": "88ba10e924c2fc6c47543eb646b9e6f7487c7ca026c9391d6cd7054fde59eaad" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "808.1070000740001" +} \ No newline at end of file diff --git a/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T13-56-13.180729.jsonl b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T13-56-13.180729.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cc97bb186a1fad009cf042f6ff22b35482e4efca --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T13-56-13.180729.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d16605756a065345efe1016ab312111a3f05f3525b71e84e4e5ec0b06538e4 +size 1445262 diff --git a/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T13-56-13.180729.jsonl b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T13-56-13.180729.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..306e5570f9629210581a9c23394eb42e1df1212f --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T13-56-13.180729.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f76a5a8551dcbc44928dc98794e438af8aee6a8ffc64d00a3f9f48a57250af2c +size 503688 diff --git a/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T13-56-13.180729.jsonl b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T13-56-13.180729.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1e8ca308ed421fa9d67afad81cb027addc69d0b8 --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T13-56-13.180729.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eb9c8ba715f94fa5d8873ba1535d853a5773db57a7e1acc15a693286f3e2489 +size 1435311 diff --git a/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T13-56-13.180729.jsonl b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T13-56-13.180729.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b5b1c66724b3310fb7baef3f8a451ea5601b0450 --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T13-56-13.180729.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5422de5225756b11dfb7b27f2431841bb74f415b00e2f549cf7a8034f7c9ddef +size 1436936 diff --git a/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T13-56-13.180729.jsonl b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T13-56-13.180729.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8c69db5100f9c08460c53df71c0ca2887b9aedda --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T13-56-13.180729.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fddefa5f6dff0969c94c4523388a328bef7c81b26c431a89c0173d40b59f584 +size 636043 diff --git a/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T13-56-13.180729.jsonl b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T13-56-13.180729.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3cc6b9108775bbb6bfb0f54ea244446bcef3028d --- /dev/null +++ b/outputs/eval/token_t0575/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T13-56-13.180729.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:941dac645a1ae83a4fa20e94503db0501193d7c69824ee16d88a0ad797ed9180 +size 1387630 diff --git a/outputs/eval/token_t0575/ruler_a.DONE b/outputs/eval/token_t0575/ruler_a.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0575/ruler_b.DONE b/outputs/eval/token_t0575/ruler_b.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t0575/sparsity_babilong.json b/outputs/eval/token_t0575/sparsity_babilong.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe2ccf9831ff72b8893a867519c5f6d02565dbe --- /dev/null +++ b/outputs/eval/token_t0575/sparsity_babilong.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 53161864, + "effective_router_decisions": 425294912, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 97002416, + "total_decisions": 425294912, + "sparsity": 0.22808270981619455, + "full_attention_usage": 0.7719172901838054, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 7000, + "total_decisions": 15189104, + "sparsity": 0.0004608566772602255 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 10232, + "total_decisions": 15189104, + "sparsity": 0.0006736407888180896 + }, + "12": { + "sparse_decisions": 31080, + "total_decisions": 15189104, + "sparsity": 0.0020462036470354013 + }, + "13": { + "sparse_decisions": 234160, + "total_decisions": 15189104, + "sparsity": 0.015416314221036342 + }, + "14": { + "sparse_decisions": 589968, + "total_decisions": 15189104, + "sparsity": 0.03884152745283725 + }, + "15": { + "sparse_decisions": 6455432, + "total_decisions": 15189104, + "sparsity": 0.4250041345427617 + }, + "16": { + "sparse_decisions": 364448, + "total_decisions": 15189104, + "sparsity": 0.02399404204487638 + }, + "17": { + "sparse_decisions": 5393848, + "total_decisions": 15189104, + "sparsity": 0.35511298098953037 + }, + "18": { + "sparse_decisions": 3350136, + "total_decisions": 15189104, + "sparsity": 0.2205617921899804 + }, + "19": { + "sparse_decisions": 2640848, + "total_decisions": 15189104, + "sparsity": 0.17386463348990172 + }, + "20": { + "sparse_decisions": 4845696, + "total_decisions": 15189104, + "sparsity": 0.3190244796533094 + }, + "21": { + "sparse_decisions": 5868528, + "total_decisions": 15189104, + "sparsity": 0.3863643306412281 + }, + "22": { + "sparse_decisions": 13004184, + "total_decisions": 15189104, + "sparsity": 0.856152146960084 + }, + "23": { + "sparse_decisions": 13248752, + "total_decisions": 15189104, + "sparsity": 0.8722536892235382 + }, + "24": { + "sparse_decisions": 11923056, + "total_decisions": 15189104, + "sparsity": 0.7849742815639421 + }, + "25": { + "sparse_decisions": 10083496, + "total_decisions": 15189104, + "sparsity": 0.6638637802466821 + }, + "26": { + "sparse_decisions": 3764448, + "total_decisions": 15189104, + "sparsity": 0.24783871385698591 + }, + "27": { + "sparse_decisions": 15187104, + "total_decisions": 15189104, + "sparsity": 0.9998683266636399 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 121680, + "total_decisions": 840000, + "sparsity": 0.14485714285714285, + "full_attention_usage": 0.8551428571428572 + }, + "prefill": { + "sparse_decisions": 96880736, + "total_decisions": 424454912, + "sparsity": 0.228247413944405, + "full_attention_usage": 0.7717525860555949 + } + } +} diff --git a/outputs/eval/token_t0575/sparsity_helmet.json b/outputs/eval/token_t0575/sparsity_helmet.json new file mode 100644 index 0000000000000000000000000000000000000000..0466fed68600fac5040bcacbae77024c8cdc6805 --- /dev/null +++ b/outputs/eval/token_t0575/sparsity_helmet.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 48728652, + "effective_router_decisions": 389829216, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 82186480, + "total_decisions": 389829216, + "sparsity": 0.210826886818047, + "full_attention_usage": 0.789173113181953, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 3832, + "total_decisions": 13922472, + "sparsity": 0.0002752384777645809 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 960, + "total_decisions": 13922472, + "sparsity": 6.895327209133551e-05 + }, + "12": { + "sparse_decisions": 1248, + "total_decisions": 13922472, + "sparsity": 8.963925371873615e-05 + }, + "13": { + "sparse_decisions": 80656, + "total_decisions": 13922472, + "sparsity": 0.005793224076873705 + }, + "14": { + "sparse_decisions": 64976, + "total_decisions": 13922472, + "sparsity": 0.004666987299381892 + }, + "15": { + "sparse_decisions": 4870600, + "total_decisions": 13922472, + "sparsity": 0.3498372990083945 + }, + "16": { + "sparse_decisions": 83368, + "total_decisions": 13922472, + "sparsity": 0.005988017070531727 + }, + "17": { + "sparse_decisions": 2393624, + "total_decisions": 13922472, + "sparsity": 0.17192521557953214 + }, + "18": { + "sparse_decisions": 3123720, + "total_decisions": 13922472, + "sparsity": 0.22436532822619432 + }, + "19": { + "sparse_decisions": 5112296, + "total_decisions": 13922472, + "sparsity": 0.3671974344785897 + }, + "20": { + "sparse_decisions": 6704728, + "total_decisions": 13922472, + "sparsity": 0.48157597300249555 + }, + "21": { + "sparse_decisions": 6359344, + "total_decisions": 13922472, + "sparsity": 0.4567683095358353 + }, + "22": { + "sparse_decisions": 8884784, + "total_decisions": 13922472, + "sparsity": 0.6381613839841086 + }, + "23": { + "sparse_decisions": 9502048, + "total_decisions": 13922472, + "sparsity": 0.6824971887176358 + }, + "24": { + "sparse_decisions": 7901272, + "total_decisions": 13922472, + "sparsity": 0.5675193313371361 + }, + "25": { + "sparse_decisions": 10330104, + "total_decisions": 13922472, + "sparsity": 0.741973408170618 + }, + "26": { + "sparse_decisions": 2848448, + "total_decisions": 13922472, + "sparsity": 0.2045935520646046 + }, + "27": { + "sparse_decisions": 13920472, + "total_decisions": 13922472, + "sparsity": 0.9998563473498098 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 192056, + "total_decisions": 1064000, + "sparsity": 0.18050375939849625, + "full_attention_usage": 0.8194962406015037 + }, + "prefill": { + "sparse_decisions": 81994424, + "total_decisions": 388765216, + "sparsity": 0.21090987728696386, + "full_attention_usage": 0.7890901227130361 + } + } +} diff --git a/outputs/eval/token_t0575/sparsity_mrcr.json b/outputs/eval/token_t0575/sparsity_mrcr.json new file mode 100644 index 0000000000000000000000000000000000000000..73e4bbea684226e81658b0d3924a605950f369d2 --- /dev/null +++ b/outputs/eval/token_t0575/sparsity_mrcr.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 5310144, + "effective_router_decisions": 42481152, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 16838640, + "total_decisions": 42481152, + "sparsity": 0.39637908124525434, + "full_attention_usage": 0.6036209187547457, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 432, + "total_decisions": 1517184, + "sparsity": 0.0002847380410022779 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 1517184, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 14208, + "total_decisions": 1517184, + "sparsity": 0.009364717792963807 + }, + "12": { + "sparse_decisions": 16688, + "total_decisions": 1517184, + "sparsity": 0.01099932506538429 + }, + "13": { + "sparse_decisions": 84536, + "total_decisions": 1517184, + "sparsity": 0.0557190162827976 + }, + "14": { + "sparse_decisions": 336968, + "total_decisions": 1517184, + "sparsity": 0.222100944908462 + }, + "15": { + "sparse_decisions": 1182000, + "total_decisions": 1517184, + "sparsity": 0.7790749177423437 + }, + "16": { + "sparse_decisions": 826672, + "total_decisions": 1517184, + "sparsity": 0.544872606091285 + }, + "17": { + "sparse_decisions": 1369128, + "total_decisions": 1517184, + "sparsity": 0.9024139458364971 + }, + "18": { + "sparse_decisions": 1192616, + "total_decisions": 1517184, + "sparsity": 0.7860720914536404 + }, + "19": { + "sparse_decisions": 1342016, + "total_decisions": 1517184, + "sparsity": 0.8845439973002616 + }, + "20": { + "sparse_decisions": 1317912, + "total_decisions": 1517184, + "sparsity": 0.8686566691976715 + }, + "21": { + "sparse_decisions": 1250248, + "total_decisions": 1517184, + "sparsity": 0.8240582552940184 + }, + "22": { + "sparse_decisions": 1342448, + "total_decisions": 1517184, + "sparsity": 0.8848287353412638 + }, + "23": { + "sparse_decisions": 1326744, + "total_decisions": 1517184, + "sparsity": 0.8744779802581625 + }, + "24": { + "sparse_decisions": 1331152, + "total_decisions": 1517184, + "sparsity": 0.8773833628617228 + }, + "25": { + "sparse_decisions": 1331480, + "total_decisions": 1517184, + "sparsity": 0.8775995528558171 + }, + "26": { + "sparse_decisions": 1056448, + "total_decisions": 1517184, + "sparsity": 0.6963216063443853 + }, + "27": { + "sparse_decisions": 1516944, + "total_decisions": 1517184, + "sparsity": 0.9998418121994431 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 729816, + "total_decisions": 1711360, + "sparsity": 0.42645381451009723, + "full_attention_usage": 0.5735461854899028 + }, + "prefill": { + "sparse_decisions": 16108824, + "total_decisions": 40769792, + "sparsity": 0.39511665892237074, + "full_attention_usage": 0.6048833410776293 + } + } +} diff --git a/outputs/eval/token_t0575/sparsity_ruler_a.json b/outputs/eval/token_t0575/sparsity_ruler_a.json new file mode 100644 index 0000000000000000000000000000000000000000..b4dabe6f4e5fa7664c9d959aecd55fc37923f9cd --- /dev/null +++ b/outputs/eval/token_t0575/sparsity_ruler_a.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 29979236, + "effective_router_decisions": 239833888, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 64617208, + "total_decisions": 239833888, + "sparsity": 0.2694248445824303, + "full_attention_usage": 0.7305751554175697, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 155096, + "total_decisions": 8565496, + "sparsity": 0.018107065837168097 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 33416, + "total_decisions": 8565496, + "sparsity": 0.0039012335070847037 + }, + "12": { + "sparse_decisions": 14248, + "total_decisions": 8565496, + "sparsity": 0.0016634179736935257 + }, + "13": { + "sparse_decisions": 723056, + "total_decisions": 8565496, + "sparsity": 0.08441495974080193 + }, + "14": { + "sparse_decisions": 623240, + "total_decisions": 8565496, + "sparsity": 0.07276169412722859 + }, + "15": { + "sparse_decisions": 4935736, + "total_decisions": 8565496, + "sparsity": 0.5762346979089127 + }, + "16": { + "sparse_decisions": 652168, + "total_decisions": 8565496, + "sparsity": 0.07613896498229641 + }, + "17": { + "sparse_decisions": 5110480, + "total_decisions": 8565496, + "sparsity": 0.5966356180657839 + }, + "18": { + "sparse_decisions": 3184560, + "total_decisions": 8565496, + "sparsity": 0.3717893277867388 + }, + "19": { + "sparse_decisions": 3518472, + "total_decisions": 8565496, + "sparsity": 0.410772709484658 + }, + "20": { + "sparse_decisions": 5684152, + "total_decisions": 8565496, + "sparsity": 0.6636103735265302 + }, + "21": { + "sparse_decisions": 4218912, + "total_decisions": 8565496, + "sparsity": 0.49254730841039446 + }, + "22": { + "sparse_decisions": 6686616, + "total_decisions": 8565496, + "sparsity": 0.7806455107795275 + }, + "23": { + "sparse_decisions": 7276744, + "total_decisions": 8565496, + "sparsity": 0.8495414626310024 + }, + "24": { + "sparse_decisions": 6210728, + "total_decisions": 8565496, + "sparsity": 0.7250867900703006 + }, + "25": { + "sparse_decisions": 5428184, + "total_decisions": 8565496, + "sparsity": 0.6337267567459024 + }, + "26": { + "sparse_decisions": 1597024, + "total_decisions": 8565496, + "sparsity": 0.186448513898086 + }, + "27": { + "sparse_decisions": 8564376, + "total_decisions": 8565496, + "sparsity": 0.9998692428319388 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 1078112, + "total_decisions": 2764160, + "sparsity": 0.390032414910859, + "full_attention_usage": 0.6099675850891411 + }, + "prefill": { + "sparse_decisions": 63539096, + "total_decisions": 237069728, + "sparsity": 0.26801859746513057, + "full_attention_usage": 0.7319814025348694 + } + } +} diff --git a/outputs/eval/token_t0575/sparsity_ruler_b.json b/outputs/eval/token_t0575/sparsity_ruler_b.json new file mode 100644 index 0000000000000000000000000000000000000000..356b92f255c211f70da19ac2d2e4927d14a28dda --- /dev/null +++ b/outputs/eval/token_t0575/sparsity_ruler_b.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 25272464, + "effective_router_decisions": 202179712, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 57721400, + "total_decisions": 202179712, + "sparsity": 0.28549551005394647, + "full_attention_usage": 0.7145044899460535, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 1376, + "total_decisions": 7220704, + "sparsity": 0.0001905631362260522 + }, + "10": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "11": { + "sparse_decisions": 3832, + "total_decisions": 7220704, + "sparsity": 0.0005306961758853431 + }, + "12": { + "sparse_decisions": 7864, + "total_decisions": 7220704, + "sparsity": 0.0010890904820361007 + }, + "13": { + "sparse_decisions": 77432, + "total_decisions": 7220704, + "sparsity": 0.010723608113557902 + }, + "14": { + "sparse_decisions": 570040, + "total_decisions": 7220704, + "sparsity": 0.07894521088248459 + }, + "15": { + "sparse_decisions": 2930160, + "total_decisions": 7220704, + "sparsity": 0.40579976689253566 + }, + "16": { + "sparse_decisions": 616352, + "total_decisions": 7220704, + "sparsity": 0.08535898992674398 + }, + "17": { + "sparse_decisions": 3498984, + "total_decisions": 7220704, + "sparsity": 0.48457657314300656 + }, + "18": { + "sparse_decisions": 3091416, + "total_decisions": 7220704, + "sparsity": 0.42813221536293417 + }, + "19": { + "sparse_decisions": 3200488, + "total_decisions": 7220704, + "sparsity": 0.44323766768448064 + }, + "20": { + "sparse_decisions": 4785144, + "total_decisions": 7220704, + "sparsity": 0.6626977092538345 + }, + "21": { + "sparse_decisions": 4743616, + "total_decisions": 7220704, + "sparsity": 0.6569464694855238 + }, + "22": { + "sparse_decisions": 6219872, + "total_decisions": 7220704, + "sparsity": 0.861394124451023 + }, + "23": { + "sparse_decisions": 6321984, + "total_decisions": 7220704, + "sparsity": 0.8755356818393332 + }, + "24": { + "sparse_decisions": 5959760, + "total_decisions": 7220704, + "sparsity": 0.8253710441530355 + }, + "25": { + "sparse_decisions": 5696856, + "total_decisions": 7220704, + "sparsity": 0.7889612979565428 + }, + "26": { + "sparse_decisions": 2776480, + "total_decisions": 7220704, + "sparsity": 0.3845165235965911 + }, + "27": { + "sparse_decisions": 7219744, + "total_decisions": 7220704, + "sparsity": 0.999867048974726 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 1115856, + "total_decisions": 2947840, + "sparsity": 0.3785334346504559, + "full_attention_usage": 0.6214665653495441 + }, + "prefill": { + "sparse_decisions": 56605544, + "total_decisions": 199231872, + "sparsity": 0.28411891848308285, + "full_attention_usage": 0.7158810815169172 + } + } +} diff --git a/outputs/eval/token_t065/babilong.DONE b/outputs/eval/token_t065/babilong.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-45-41.612234.json b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-45-41.612234.json new file mode 100644 index 0000000000000000000000000000000000000000..c9ca642cdddba485103a236beff0e24b14263e75 --- /dev/null +++ b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-45-41.612234.json @@ -0,0 +1,505 @@ +{ + "results": { + "babilong_longctx": { + "acc,none": 0.232, + "acc_stderr,none": 0.026073678963542474, + "alias": "babilong_longctx" + }, + "babilong_qa1": { + "alias": " - babilong_qa1", + "acc,none": 0.28, + "acc_stderr,none": 0.06414269805898186 + }, + "babilong_qa2": { + "alias": " - babilong_qa2", + "acc,none": 0.08, + "acc_stderr,none": 0.03875617133214441 + }, + "babilong_qa3": { + "alias": " - babilong_qa3", + "acc,none": 0.14, + "acc_stderr,none": 0.0495695759225642 + }, + "babilong_qa4": { + "alias": " - babilong_qa4", + "acc,none": 0.28, + "acc_stderr,none": 0.06414269805898185 + }, + "babilong_qa5": { + "alias": " - babilong_qa5", + "acc,none": 0.38, + "acc_stderr,none": 0.06934092056863767 + } + }, + "groups": { + "babilong_longctx": { + "acc,none": 0.232, + "acc_stderr,none": 0.026073678963542474, + "alias": "babilong_longctx" + } + }, + "group_subtasks": { + "babilong_longctx": [ + "babilong_qa1", + "babilong_qa2", + "babilong_qa3", + "babilong_qa4", + "babilong_qa5" + ] + }, + "configs": { + "babilong_qa1": { + "task": "babilong_qa1", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa1" + }, + "test_split": "qa1", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about positions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location to answer the question.\nAlways return your answer in the following format:\nThe most recent location of 'person' is 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the hallway. Judith come back to the kitchen. Charlie travelled to balcony.", + "question": "Where is Charlie?", + "target": "The most recent location of Charlie is balcony." + }, + { + "input": "Alan moved to the garage. Charlie went to the beach. Alan went to the shop. Rouse travelled to balcony.", + "question": "Where is Alan?", + "target": "The most recent location of Alan is shop." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa2": { + "task": "babilong_qa2", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa2" + }, + "test_split": "qa2", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nThe 'item' is in 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the kitchen. Charlie got a bottle. Charlie moved to the balcony.", + "question": "Where is the bottle?", + "target": "The bottle is in the balcony." + }, + { + "input": "Alan moved to the garage. Alan got a screw driver. Alan moved to the kitchen.", + "question": "Where is the screw driver?", + "target": "The screw driver is in the kitchen." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa3": { + "task": "babilong_qa3", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa3" + }, + "test_split": "qa3", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nBefore the $location_1$ the $item$ was in the $location_2$. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "John journeyed to the bedroom. Mary grabbed the apple. Mary went back to the bathroom. Daniel journeyed to the bedroom. Daniel moved to the garden. Mary travelled to the kitchen.", + "question": "Where was the apple before the kitchen?", + "target": "Before the kitchen the apple was in the bathroom." + }, + { + "input": "John went back to the bedroom. John went back to the garden. John went back to the kitchen. Sandra took the football. Sandra travelled to the garden. Sandra journeyed to the bedroom.", + "question": "Where was the football before the bedroom?", + "target": "Before the bedroom the football was in the garden." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa4": { + "task": "babilong_qa4", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa4" + }, + "test_split": "qa4", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about different people, their location and actions, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - location. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "The hallway is south of the kitchen. The bedroom is north of the kitchen.", + "question": "What is the kitchen south of?", + "target": "bedroom" + }, + { + "input": "The garden is west of the bedroom. The bedroom is west of the kitchen.", + "question": "What is west of the bedroom?", + "target": "garden" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa5": { + "task": "babilong_qa5", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa5" + }, + "test_split": "qa5", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and their relations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word. Do not write anything else after that. Do not explain your answer.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Mary picked up the apple there. Mary gave the apple to Fred. Mary moved to the bedroom. Bill took the milk there.", + "question": "Who did Mary give the apple to?", + "target": "Fred" + }, + { + "input": "Jeff took the football there. Jeff passed the football to Fred. Jeff got the milk there. Bill travelled to the bedroom.", + "question": "Who gave the football?", + "target": "Jeff" + }, + { + "input": "Fred picked up the apple there. Fred handed the apple to Bill. Bill journeyed to the bedroom. Jeff went back to the garden.", + "question": "What did Fred give to Bill?", + "target": "apple" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + } + }, + "versions": { + "babilong_longctx": 0.0, + "babilong_qa1": 0.0, + "babilong_qa2": 0.0, + "babilong_qa3": 0.0, + "babilong_qa4": 0.0, + "babilong_qa5": 0.0 + }, + "n-shot": { + "babilong_qa1": 2, + "babilong_qa2": 2, + "babilong_qa3": 2, + "babilong_qa4": 2, + "babilong_qa5": 2 + }, + "higher_is_better": { + "babilong_longctx": { + "acc": true + }, + "babilong_qa1": { + "acc": true + }, + "babilong_qa2": { + "acc": true + }, + "babilong_qa3": { + "acc": true + }, + "babilong_qa4": { + "acc": true + }, + "babilong_qa5": { + "acc": true + } + }, + "n-samples": { + "babilong_qa1": { + "original": 1000, + "effective": 50 + }, + "babilong_qa2": { + "original": 999, + "effective": 50 + }, + "babilong_qa3": { + "original": 999, + "effective": 50 + }, + "babilong_qa4": { + "original": 999, + "effective": 50 + }, + "babilong_qa5": { + "original": 999, + "effective": 50 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 50.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784385596.0906477, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "babilong_qa1": "bbdcd109bea1bb245e118e814bd0563de6384bc875a21df5218ab4ed75b27380", + "babilong_qa2": "fa4f8d12bdd481e02a1051414304811f648caf7bce8f89dcf03ecbf5b86f1e92", + "babilong_qa3": "737b64396c8b2e5f96da9796bcd5e448f08e5a13e32ae7b610d31a60269b6e48", + "babilong_qa4": "298a25727c69c9babddcd78f438cae1aa21a388e14e7db90b2206eff753c90c0", + "babilong_qa5": "e60afcdcdd1c4b3e24b002ffbf8f87085f5ec4a02fcd72318f042dcdafe230f4" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "348.84256471300614" +} \ No newline at end of file diff --git a/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T14-45-41.612234.jsonl b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T14-45-41.612234.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7f29e115dc19e4eb73fbbb7bfd83b232928e4fa8 --- /dev/null +++ b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T14-45-41.612234.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e42bdf6d49aaddb02bf705295dd4b1020411aa4e71475f55fa89eee9c5e7288 +size 3163864 diff --git a/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T14-45-41.612234.jsonl b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T14-45-41.612234.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9755e6031b560616e477b36259a219b89b813cc7 --- /dev/null +++ b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T14-45-41.612234.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45be89d91e4ee28fcfb7d2e4f28dac9b6149fdf779a933b2a8c8d7039022d3d8 +size 3147724 diff --git a/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T14-45-41.612234.jsonl b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T14-45-41.612234.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b88666d19cf46e722b20f4d277ead26dbaed3174 --- /dev/null +++ b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T14-45-41.612234.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:757cc60dec3bf24bdaf9e4a6e84e94790230438a87d1c181e0feef11bd3046ca +size 3126972 diff --git a/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T14-45-41.612234.jsonl b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T14-45-41.612234.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d3c15193c78cbab51c749f16049a2e622cf15491 --- /dev/null +++ b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T14-45-41.612234.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8354e05e52a1ce72fa1d741566e3669cb2d6fee35d91016abab99b9a18f462fd +size 3137736 diff --git a/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T14-45-41.612234.jsonl b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T14-45-41.612234.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..41b653c62e0f940744873492b2f9336e8bea9595 --- /dev/null +++ b/outputs/eval/token_t065/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T14-45-41.612234.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7e5b2f79e10835ec57c1a453093564c7a436bfc05fc9fe92989a91a07c8edd7 +size 3135660 diff --git a/outputs/eval/token_t065/helmet.DONE b/outputs/eval/token_t065/helmet.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t065/helmet_icl8k_n50.jsonl b/outputs/eval/token_t065/helmet_icl8k_n50.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..01906f05542925e545f4405facec129e22bd9f11 --- /dev/null +++ b/outputs/eval/token_t065/helmet_icl8k_n50.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2e11af3e168c9d4afe2858d9496c4e5ed43cc91677eecc00f6943dbfebc49ad +size 6468551 diff --git a/outputs/eval/token_t065/helmet_icl8k_n50.summary.csv b/outputs/eval/token_t065/helmet_icl8k_n50.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..66644ea379446b17f8ab9644c338942216d64fc3 --- /dev/null +++ b/outputs/eval/token_t065/helmet_icl8k_n50.summary.csv @@ -0,0 +1,251 @@ +method,benchmark,config,n,score,prompt_tokens +token_t065,helmet_icl,trec_coarse,1,1.0,6597 +token_t065,helmet_icl,trec_coarse,1,0.0,6549 +token_t065,helmet_icl,trec_coarse,1,1.0,6508 +token_t065,helmet_icl,trec_coarse,1,0.0,6608 +token_t065,helmet_icl,trec_coarse,1,0.0,6497 +token_t065,helmet_icl,trec_coarse,1,0.0,6545 +token_t065,helmet_icl,trec_coarse,1,0.0,6583 +token_t065,helmet_icl,trec_coarse,1,1.0,6455 +token_t065,helmet_icl,trec_coarse,1,0.0,6643 +token_t065,helmet_icl,trec_coarse,1,1.0,6568 +token_t065,helmet_icl,trec_coarse,1,0.0,6518 +token_t065,helmet_icl,trec_coarse,1,1.0,6424 +token_t065,helmet_icl,trec_coarse,1,1.0,6750 +token_t065,helmet_icl,trec_coarse,1,0.0,6642 +token_t065,helmet_icl,trec_coarse,1,0.0,6494 +token_t065,helmet_icl,trec_coarse,1,0.0,6427 +token_t065,helmet_icl,trec_coarse,1,0.0,6547 +token_t065,helmet_icl,trec_coarse,1,0.0,6612 +token_t065,helmet_icl,trec_coarse,1,0.0,6591 +token_t065,helmet_icl,trec_coarse,1,1.0,6485 +token_t065,helmet_icl,trec_coarse,1,1.0,6542 +token_t065,helmet_icl,trec_coarse,1,1.0,6409 +token_t065,helmet_icl,trec_coarse,1,0.0,6482 +token_t065,helmet_icl,trec_coarse,1,1.0,6502 +token_t065,helmet_icl,trec_coarse,1,0.0,6344 +token_t065,helmet_icl,trec_coarse,1,0.0,6479 +token_t065,helmet_icl,trec_coarse,1,1.0,6569 +token_t065,helmet_icl,trec_coarse,1,1.0,6389 +token_t065,helmet_icl,trec_coarse,1,0.0,6479 +token_t065,helmet_icl,trec_coarse,1,1.0,6653 +token_t065,helmet_icl,trec_coarse,1,1.0,6532 +token_t065,helmet_icl,trec_coarse,1,1.0,6491 +token_t065,helmet_icl,trec_coarse,1,0.0,6489 +token_t065,helmet_icl,trec_coarse,1,0.0,6630 +token_t065,helmet_icl,trec_coarse,1,1.0,6488 +token_t065,helmet_icl,trec_coarse,1,1.0,6499 +token_t065,helmet_icl,trec_coarse,1,0.0,6483 +token_t065,helmet_icl,trec_coarse,1,0.0,6660 +token_t065,helmet_icl,trec_coarse,1,0.0,6675 +token_t065,helmet_icl,trec_coarse,1,0.0,6544 +token_t065,helmet_icl,trec_coarse,1,0.0,6520 +token_t065,helmet_icl,trec_coarse,1,0.0,6359 +token_t065,helmet_icl,trec_coarse,1,1.0,6657 +token_t065,helmet_icl,trec_coarse,1,0.0,6546 +token_t065,helmet_icl,trec_coarse,1,0.0,6481 +token_t065,helmet_icl,trec_coarse,1,0.0,6506 +token_t065,helmet_icl,trec_coarse,1,0.0,6621 +token_t065,helmet_icl,trec_coarse,1,0.0,6623 +token_t065,helmet_icl,trec_coarse,1,0.0,6411 +token_t065,helmet_icl,trec_coarse,1,0.0,6577 +token_t065,helmet_icl,trec_fine,1,0.0,6858 +token_t065,helmet_icl,trec_fine,1,0.0,6973 +token_t065,helmet_icl,trec_fine,1,0.0,6977 +token_t065,helmet_icl,trec_fine,1,0.0,6952 +token_t065,helmet_icl,trec_fine,1,0.0,6919 +token_t065,helmet_icl,trec_fine,1,1.0,6979 +token_t065,helmet_icl,trec_fine,1,0.0,6957 +token_t065,helmet_icl,trec_fine,1,0.0,7013 +token_t065,helmet_icl,trec_fine,1,0.0,6824 +token_t065,helmet_icl,trec_fine,1,0.0,6833 +token_t065,helmet_icl,trec_fine,1,0.0,6868 +token_t065,helmet_icl,trec_fine,1,1.0,6914 +token_t065,helmet_icl,trec_fine,1,1.0,6759 +token_t065,helmet_icl,trec_fine,1,1.0,6933 +token_t065,helmet_icl,trec_fine,1,0.0,6868 +token_t065,helmet_icl,trec_fine,1,0.0,6839 +token_t065,helmet_icl,trec_fine,1,0.0,6838 +token_t065,helmet_icl,trec_fine,1,0.0,7000 +token_t065,helmet_icl,trec_fine,1,0.0,6800 +token_t065,helmet_icl,trec_fine,1,0.0,6829 +token_t065,helmet_icl,trec_fine,1,0.0,6971 +token_t065,helmet_icl,trec_fine,1,0.0,6907 +token_t065,helmet_icl,trec_fine,1,0.0,6818 +token_t065,helmet_icl,trec_fine,1,0.0,6902 +token_t065,helmet_icl,trec_fine,1,1.0,6941 +token_t065,helmet_icl,trec_fine,1,0.0,6830 +token_t065,helmet_icl,trec_fine,1,0.0,6934 +token_t065,helmet_icl,trec_fine,1,1.0,6879 +token_t065,helmet_icl,trec_fine,1,1.0,6915 +token_t065,helmet_icl,trec_fine,1,0.0,6953 +token_t065,helmet_icl,trec_fine,1,0.0,6875 +token_t065,helmet_icl,trec_fine,1,0.0,6847 +token_t065,helmet_icl,trec_fine,1,0.0,6847 +token_t065,helmet_icl,trec_fine,1,0.0,6944 +token_t065,helmet_icl,trec_fine,1,0.0,6818 +token_t065,helmet_icl,trec_fine,1,0.0,6868 +token_t065,helmet_icl,trec_fine,1,0.0,7022 +token_t065,helmet_icl,trec_fine,1,0.0,6930 +token_t065,helmet_icl,trec_fine,1,0.0,6863 +token_t065,helmet_icl,trec_fine,1,0.0,6807 +token_t065,helmet_icl,trec_fine,1,0.0,7001 +token_t065,helmet_icl,trec_fine,1,0.0,6890 +token_t065,helmet_icl,trec_fine,1,0.0,6952 +token_t065,helmet_icl,trec_fine,1,1.0,6845 +token_t065,helmet_icl,trec_fine,1,0.0,6902 +token_t065,helmet_icl,trec_fine,1,0.0,6879 +token_t065,helmet_icl,trec_fine,1,0.0,6896 +token_t065,helmet_icl,trec_fine,1,0.0,6770 +token_t065,helmet_icl,trec_fine,1,0.0,6933 +token_t065,helmet_icl,trec_fine,1,0.0,6886 +token_t065,helmet_icl,banking77,1,0.0,6809 +token_t065,helmet_icl,banking77,1,0.0,7162 +token_t065,helmet_icl,banking77,1,1.0,6876 +token_t065,helmet_icl,banking77,1,1.0,7033 +token_t065,helmet_icl,banking77,1,1.0,6647 +token_t065,helmet_icl,banking77,1,1.0,6818 +token_t065,helmet_icl,banking77,1,0.0,6841 +token_t065,helmet_icl,banking77,1,1.0,6944 +token_t065,helmet_icl,banking77,1,1.0,6776 +token_t065,helmet_icl,banking77,1,1.0,7100 +token_t065,helmet_icl,banking77,1,0.0,6967 +token_t065,helmet_icl,banking77,1,0.0,6894 +token_t065,helmet_icl,banking77,1,1.0,6900 +token_t065,helmet_icl,banking77,1,0.0,6964 +token_t065,helmet_icl,banking77,1,1.0,7085 +token_t065,helmet_icl,banking77,1,1.0,6953 +token_t065,helmet_icl,banking77,1,0.0,7406 +token_t065,helmet_icl,banking77,1,1.0,6802 +token_t065,helmet_icl,banking77,1,0.0,7119 +token_t065,helmet_icl,banking77,1,0.0,7202 +token_t065,helmet_icl,banking77,1,0.0,7103 +token_t065,helmet_icl,banking77,1,0.0,7009 +token_t065,helmet_icl,banking77,1,0.0,6940 +token_t065,helmet_icl,banking77,1,0.0,6857 +token_t065,helmet_icl,banking77,1,0.0,6922 +token_t065,helmet_icl,banking77,1,1.0,6988 +token_t065,helmet_icl,banking77,1,1.0,6945 +token_t065,helmet_icl,banking77,1,1.0,6835 +token_t065,helmet_icl,banking77,1,0.0,7035 +token_t065,helmet_icl,banking77,1,1.0,6787 +token_t065,helmet_icl,banking77,1,0.0,6940 +token_t065,helmet_icl,banking77,1,0.0,6661 +token_t065,helmet_icl,banking77,1,1.0,6934 +token_t065,helmet_icl,banking77,1,0.0,7235 +token_t065,helmet_icl,banking77,1,0.0,6829 +token_t065,helmet_icl,banking77,1,1.0,7169 +token_t065,helmet_icl,banking77,1,0.0,7074 +token_t065,helmet_icl,banking77,1,1.0,6797 +token_t065,helmet_icl,banking77,1,0.0,7013 +token_t065,helmet_icl,banking77,1,1.0,6811 +token_t065,helmet_icl,banking77,1,0.0,6983 +token_t065,helmet_icl,banking77,1,0.0,7183 +token_t065,helmet_icl,banking77,1,1.0,6801 +token_t065,helmet_icl,banking77,1,1.0,7174 +token_t065,helmet_icl,banking77,1,1.0,6773 +token_t065,helmet_icl,banking77,1,1.0,7134 +token_t065,helmet_icl,banking77,1,1.0,6907 +token_t065,helmet_icl,banking77,1,1.0,7341 +token_t065,helmet_icl,banking77,1,1.0,7042 +token_t065,helmet_icl,banking77,1,1.0,7304 +token_t065,helmet_icl,clinic150,1,1.0,7241 +token_t065,helmet_icl,clinic150,1,1.0,7282 +token_t065,helmet_icl,clinic150,1,1.0,7261 +token_t065,helmet_icl,clinic150,1,0.0,7292 +token_t065,helmet_icl,clinic150,1,0.0,7217 +token_t065,helmet_icl,clinic150,1,0.0,7280 +token_t065,helmet_icl,clinic150,1,1.0,7293 +token_t065,helmet_icl,clinic150,1,1.0,7250 +token_t065,helmet_icl,clinic150,1,1.0,7264 +token_t065,helmet_icl,clinic150,1,1.0,7238 +token_t065,helmet_icl,clinic150,1,1.0,7243 +token_t065,helmet_icl,clinic150,1,1.0,7191 +token_t065,helmet_icl,clinic150,1,1.0,7333 +token_t065,helmet_icl,clinic150,1,1.0,7299 +token_t065,helmet_icl,clinic150,1,0.0,7225 +token_t065,helmet_icl,clinic150,1,1.0,7409 +token_t065,helmet_icl,clinic150,1,1.0,7277 +token_t065,helmet_icl,clinic150,1,1.0,7376 +token_t065,helmet_icl,clinic150,1,1.0,7383 +token_t065,helmet_icl,clinic150,1,0.0,7270 +token_t065,helmet_icl,clinic150,1,1.0,7173 +token_t065,helmet_icl,clinic150,1,1.0,7106 +token_t065,helmet_icl,clinic150,1,1.0,7413 +token_t065,helmet_icl,clinic150,1,1.0,7253 +token_t065,helmet_icl,clinic150,1,0.0,7224 +token_t065,helmet_icl,clinic150,1,1.0,7307 +token_t065,helmet_icl,clinic150,1,1.0,7272 +token_t065,helmet_icl,clinic150,1,0.0,7215 +token_t065,helmet_icl,clinic150,1,1.0,7344 +token_t065,helmet_icl,clinic150,1,1.0,7224 +token_t065,helmet_icl,clinic150,1,1.0,7310 +token_t065,helmet_icl,clinic150,1,1.0,7222 +token_t065,helmet_icl,clinic150,1,1.0,7260 +token_t065,helmet_icl,clinic150,1,0.0,7351 +token_t065,helmet_icl,clinic150,1,1.0,7301 +token_t065,helmet_icl,clinic150,1,1.0,7155 +token_t065,helmet_icl,clinic150,1,0.0,7253 +token_t065,helmet_icl,clinic150,1,1.0,7248 +token_t065,helmet_icl,clinic150,1,1.0,7128 +token_t065,helmet_icl,clinic150,1,0.0,7282 +token_t065,helmet_icl,clinic150,1,1.0,7301 +token_t065,helmet_icl,clinic150,1,1.0,7388 +token_t065,helmet_icl,clinic150,1,1.0,7244 +token_t065,helmet_icl,clinic150,1,1.0,7186 +token_t065,helmet_icl,clinic150,1,0.0,7309 +token_t065,helmet_icl,clinic150,1,1.0,7241 +token_t065,helmet_icl,clinic150,1,0.0,7214 +token_t065,helmet_icl,clinic150,1,0.0,7264 +token_t065,helmet_icl,clinic150,1,0.0,7219 +token_t065,helmet_icl,clinic150,1,1.0,7228 +token_t065,helmet_icl,nlu,1,1.0,7063 +token_t065,helmet_icl,nlu,1,1.0,6959 +token_t065,helmet_icl,nlu,1,0.0,6985 +token_t065,helmet_icl,nlu,1,1.0,6970 +token_t065,helmet_icl,nlu,1,0.0,7025 +token_t065,helmet_icl,nlu,1,0.0,7034 +token_t065,helmet_icl,nlu,1,0.0,7004 +token_t065,helmet_icl,nlu,1,0.0,7062 +token_t065,helmet_icl,nlu,1,1.0,7062 +token_t065,helmet_icl,nlu,1,1.0,7045 +token_t065,helmet_icl,nlu,1,1.0,7028 +token_t065,helmet_icl,nlu,1,1.0,7018 +token_t065,helmet_icl,nlu,1,1.0,7075 +token_t065,helmet_icl,nlu,1,0.0,7068 +token_t065,helmet_icl,nlu,1,1.0,7025 +token_t065,helmet_icl,nlu,1,1.0,7022 +token_t065,helmet_icl,nlu,1,1.0,7112 +token_t065,helmet_icl,nlu,1,1.0,6967 +token_t065,helmet_icl,nlu,1,0.0,6980 +token_t065,helmet_icl,nlu,1,1.0,7031 +token_t065,helmet_icl,nlu,1,0.0,7093 +token_t065,helmet_icl,nlu,1,0.0,6949 +token_t065,helmet_icl,nlu,1,1.0,7046 +token_t065,helmet_icl,nlu,1,1.0,7039 +token_t065,helmet_icl,nlu,1,0.0,7126 +token_t065,helmet_icl,nlu,1,1.0,7064 +token_t065,helmet_icl,nlu,1,1.0,7057 +token_t065,helmet_icl,nlu,1,0.0,6971 +token_t065,helmet_icl,nlu,1,1.0,7034 +token_t065,helmet_icl,nlu,1,0.0,7094 +token_t065,helmet_icl,nlu,1,1.0,6979 +token_t065,helmet_icl,nlu,1,1.0,7123 +token_t065,helmet_icl,nlu,1,1.0,7144 +token_t065,helmet_icl,nlu,1,0.0,7024 +token_t065,helmet_icl,nlu,1,1.0,6943 +token_t065,helmet_icl,nlu,1,0.0,6961 +token_t065,helmet_icl,nlu,1,1.0,7005 +token_t065,helmet_icl,nlu,1,0.0,7080 +token_t065,helmet_icl,nlu,1,1.0,7179 +token_t065,helmet_icl,nlu,1,1.0,7115 +token_t065,helmet_icl,nlu,1,0.0,6946 +token_t065,helmet_icl,nlu,1,1.0,6985 +token_t065,helmet_icl,nlu,1,1.0,7131 +token_t065,helmet_icl,nlu,1,1.0,7100 +token_t065,helmet_icl,nlu,1,0.0,7190 +token_t065,helmet_icl,nlu,1,1.0,7040 +token_t065,helmet_icl,nlu,1,1.0,7013 +token_t065,helmet_icl,nlu,1,0.0,7080 +token_t065,helmet_icl,nlu,1,0.0,7098 +token_t065,helmet_icl,nlu,1,1.0,6891 diff --git a/outputs/eval/token_t065/mrcr.DONE b/outputs/eval/token_t065/mrcr.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t065/mrcr_8k_2_4_8needle_n10.jsonl b/outputs/eval/token_t065/mrcr_8k_2_4_8needle_n10.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a3559c4faed1d09b9203c1aa818999044776b97a --- /dev/null +++ b/outputs/eval/token_t065/mrcr_8k_2_4_8needle_n10.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e998b56ed98f54dcccae031cc17eb5eca4f00b56486240a2055e4debcd4d1e +size 984174 diff --git a/outputs/eval/token_t065/mrcr_8k_2_4_8needle_n10.summary.csv b/outputs/eval/token_t065/mrcr_8k_2_4_8needle_n10.summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8fce2a8237d069cdd2fc9de2bade953a61849d85 --- /dev/null +++ b/outputs/eval/token_t065/mrcr_8k_2_4_8needle_n10.summary.csv @@ -0,0 +1,31 @@ +method,benchmark,config,n,score,prompt_tokens +token_t065,mrcr,8k_2needle,1,0.0,4852 +token_t065,mrcr,8k_2needle,1,0.0,7119 +token_t065,mrcr,8k_2needle,1,0.029329307742074616,5756 +token_t065,mrcr,8k_2needle,1,0.04926108374384237,5072 +token_t065,mrcr,8k_2needle,1,0.02666666666666667,5753 +token_t065,mrcr,8k_2needle,1,0.015089163237311385,5481 +token_t065,mrcr,8k_2needle,1,0.08731082654249127,5143 +token_t065,mrcr,8k_2needle,1,0.0,6399 +token_t065,mrcr,8k_2needle,1,0.01792573623559539,5987 +token_t065,mrcr,8k_2needle,1,0.016339233612137717,4380 +token_t065,mrcr,8k_4needle,1,0.03290529695024077,5123 +token_t065,mrcr,8k_4needle,1,0.025883175935641833,7908 +token_t065,mrcr,8k_4needle,1,0.015518913676042677,5814 +token_t065,mrcr,8k_4needle,1,0.049269675168955744,4856 +token_t065,mrcr,8k_4needle,1,0.0,7413 +token_t065,mrcr,8k_4needle,1,0.008992805755395683,7764 +token_t065,mrcr,8k_4needle,1,0.0,4312 +token_t065,mrcr,8k_4needle,1,0.0,4847 +token_t065,mrcr,8k_4needle,1,0.02937319695777603,6287 +token_t065,mrcr,8k_4needle,1,0.0,7841 +token_t065,mrcr,8k_8needle,1,0.0,4581 +token_t065,mrcr,8k_8needle,1,0.0,6357 +token_t065,mrcr,8k_8needle,1,0.0,6981 +token_t065,mrcr,8k_8needle,1,0.011609907120743035,6308 +token_t065,mrcr,8k_8needle,1,0.023492560689115115,6901 +token_t065,mrcr,8k_8needle,1,0.0457516339869281,5407 +token_t065,mrcr,8k_8needle,1,0.0,6351 +token_t065,mrcr,8k_8needle,1,0.01167883211678832,5657 +token_t065,mrcr,8k_8needle,1,0.002702702702702703,7330 +token_t065,mrcr,8k_8needle,1,0.026357075619705052,8028 diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-27-38.949809.json b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-27-38.949809.json new file mode 100644 index 0000000000000000000000000000000000000000..afa99cfbfb50cc00a381bdd164de8fd91cc5d527 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-27-38.949809.json @@ -0,0 +1,821 @@ +{ + "results": { + "niah_multikey_2": { + "alias": "niah_multikey_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.15, + "8192_stderr,none": "N/A" + }, + "niah_multiquery": { + "alias": "niah_multiquery", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.1, + "8192_stderr,none": "N/A" + }, + "niah_single_1": { + "alias": "niah_single_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.0, + "8192_stderr,none": "N/A" + }, + "niah_single_3": { + "alias": "niah_single_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.0, + "8192_stderr,none": "N/A" + }, + "ruler_fwe": { + "alias": "ruler_fwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.7999999999999998, + "8192_stderr,none": "N/A" + }, + "ruler_qa_hotpot": { + "alias": "ruler_qa_hotpot", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.2, + "8192_stderr,none": "N/A" + }, + "ruler_vt": { + "alias": "ruler_vt", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.22000000000000006, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_1": [], + "niah_single_3": [], + "niah_multikey_2": [], + "niah_multiquery": [], + "ruler_vt": [], + "ruler_fwe": [], + "ruler_qa_hotpot": [] + }, + "configs": { + "niah_multikey_2": { + "task": "niah_multikey_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multiquery": { + "task": "niah_multiquery", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multiquery(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_q=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_1": { + "task": "niah_single_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"repeat\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"repeat\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_3": { + "task": "niah_single_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_fwe": { + "task": "ruler_fwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def fwe_download(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, max_seq_length=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 50, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_hotpot": { + "task": "ruler_qa_hotpot", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_hotpotqa(**kwargs):\n return get_qa_dataset(\"hotpotqa\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_vt": { + "task": "ruler_vt", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_vt_dataset(**kwargs) -> dict[str, datasets.Dataset]:\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", \"\"))\n df = (\n get_dataset(tokenizer=get_tokenizer(pretrained), seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 30, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_2": 1.0, + "niah_multiquery": 1.0, + "niah_single_1": 1.0, + "niah_single_3": 1.0, + "ruler_fwe": 1.0, + "ruler_qa_hotpot": 1.0, + "ruler_vt": 1.0 + }, + "n-shot": { + "niah_multikey_2": 0, + "niah_multiquery": 0, + "niah_single_1": 0, + "niah_single_3": 0, + "ruler_fwe": 0, + "ruler_qa_hotpot": 0, + "ruler_vt": 0 + }, + "higher_is_better": { + "niah_multikey_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multiquery": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_fwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_hotpot": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_vt": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_hotpot": { + "original": 500, + "effective": 20 + }, + "ruler_fwe": { + "original": 500, + "effective": 20 + }, + "ruler_vt": { + "original": 500, + "effective": 20 + }, + "niah_multiquery": { + "original": 500, + "effective": 20 + }, + "niah_multikey_2": { + "original": 500, + "effective": 20 + }, + "niah_single_3": { + "original": 500, + "effective": 20 + }, + "niah_single_1": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784383966.6311393, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_hotpot": "fd48cb605efc8c5adac7e3771533db4d4c354b71bf91ce00841c814604ba6404", + "ruler_fwe": "c4633bcd0acf3ba4b96f905b5ba269ba94cf45a4c8ca0c13aae1468227cc3a10", + "ruler_vt": "af011da859c8b86f63f57f6256cf59077d3d3d8536084ab67ccf39e18e9a615a", + "niah_multiquery": "f9a36d2854dfd2bb52ffc7a6bf3f53490fb5303ca9da0046256bc28b55c2adb2", + "niah_multikey_2": "86e49b8f875843454b2923143b9124ba82e8a2e23b145f792726cd82f4e49c24", + "niah_single_3": "905e40146a287e5f0c8e2a81f64f082108c45a23862949e6a99bfcbbf561cad2", + "niah_single_1": "426e8138e992480aa416d01e6b9cef5865a68c00f532400e448a0c20ee026cff" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "895.5573387439945" +} \ No newline at end of file diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T14-27-38.949809.jsonl b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T14-27-38.949809.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bf73771103c9a86516d39964ce9e48102c8bd747 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T14-27-38.949809.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15fd523b3afb20ed01672197029e1061f11ac852e507ccc8e45ecfed48a59092 +size 988392 diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T14-27-38.949809.jsonl b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T14-27-38.949809.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ad915edae60e8fdc2880595692af0ba2d0308990 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T14-27-38.949809.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bed8a8da0fe0703bcc466855ecb6af42d94147d7ee5fc3c10f0cca69741ce0bf +size 1443166 diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T14-27-38.949809.jsonl b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T14-27-38.949809.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..96ea5c2cbcc2e99f1a06f658d8814b850fce4c04 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T14-27-38.949809.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd8c0294944a8be78235473083ec6e168a4cd67d6e449d3268531e5ff4185a9 +size 1229180 diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T14-27-38.949809.jsonl b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T14-27-38.949809.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f3a8909146443767d196167c53976a829e59f30d --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T14-27-38.949809.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ab89f9af31dbace863eb904616633970065508c61fe5c27be0667eac1f85821 +size 1428354 diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T14-27-38.949809.jsonl b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T14-27-38.949809.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d1e14d24352309780b199149aee0df4cf90b870f --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T14-27-38.949809.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a87481d43afa708178f5aac43060bdf9321b934b82084400958c132211868e6 +size 827076 diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T14-27-38.949809.jsonl b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T14-27-38.949809.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7bb74e27adc6177e00ff002cbf069a78b4d9cbf8 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T14-27-38.949809.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a817322f2b5ea6d5456bfbf0e4d59aa119391c53e8173346b35d6a25022dc5 +size 1171904 diff --git a/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T14-27-38.949809.jsonl b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T14-27-38.949809.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..87b5ee91fc51ab69f3d8e5c7b7732624a3a8688f --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T14-27-38.949809.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d479bf9bc17862e99dfc630cf603c1ec816d5ba701dd4d8e961d0e802774016a +size 1228542 diff --git a/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-39-49.633113.json b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-39-49.633113.json new file mode 100644 index 0000000000000000000000000000000000000000..e6981c1d9cef6c3d1c8c3dc157436a8ff7b89484 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T14-39-49.633113.json @@ -0,0 +1,714 @@ +{ + "results": { + "niah_multikey_1": { + "alias": "niah_multikey_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.1, + "8192_stderr,none": "N/A" + }, + "niah_multikey_3": { + "alias": "niah_multikey_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.0, + "8192_stderr,none": "N/A" + }, + "niah_multivalue": { + "alias": "niah_multivalue", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.05, + "8192_stderr,none": "N/A" + }, + "niah_single_2": { + "alias": "niah_single_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.1, + "8192_stderr,none": "N/A" + }, + "ruler_cwe": { + "alias": "ruler_cwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.20500000000000002, + "8192_stderr,none": "N/A" + }, + "ruler_qa_squad": { + "alias": "ruler_qa_squad", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.25416666666666665, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_2": [], + "niah_multikey_1": [], + "niah_multikey_3": [], + "niah_multivalue": [], + "ruler_cwe": [], + "ruler_qa_squad": [] + }, + "configs": { + "niah_multikey_1": { + "task": "niah_multikey_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_k=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multikey_3": { + "task": "niah_multikey_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"uuids\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multivalue": { + "task": "niah_multivalue", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multivalue(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_v=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_2": { + "task": "niah_single_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_cwe": { + "task": "ruler_cwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_cw_dataset(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": "\n\n", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": "\n\n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 120, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_squad": { + "task": "ruler_qa_squad", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_squad(**kwargs):\n return get_qa_dataset(\"squad\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_1": 1.0, + "niah_multikey_3": 1.0, + "niah_multivalue": 1.0, + "niah_single_2": 1.0, + "ruler_cwe": 1.0, + "ruler_qa_squad": 1.0 + }, + "n-shot": { + "niah_multikey_1": 0, + "niah_multikey_3": 0, + "niah_multivalue": 0, + "niah_single_2": 0, + "ruler_cwe": 0, + "ruler_qa_squad": 0 + }, + "higher_is_better": { + "niah_multikey_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multikey_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multivalue": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_cwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_squad": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_squad": { + "original": 500, + "effective": 20 + }, + "ruler_cwe": { + "original": 500, + "effective": 20 + }, + "niah_multivalue": { + "original": 500, + "effective": 20 + }, + "niah_multikey_3": { + "original": 500, + "effective": 20 + }, + "niah_multikey_1": { + "original": 500, + "effective": 20 + }, + "niah_single_2": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 2031969308, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784384865.405819, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_squad": "8c658a728ccd3d67ddd3f45e8e76527cf15a1ae4119033d8246f293ecc68cf11", + "ruler_cwe": "f68206129fc7d709951d550ec9269fed05e73bea2fc86ce9f2c758974db5db71", + "niah_multivalue": "29646655c230b4f155a8de1fbc7ddf22c49e43045bd8fdab5770e7cb45026c7c", + "niah_multikey_3": "de2cec52885e38d56e0bda0e344e989909969c71ab5f8048b9368a011dd3616b", + "niah_multikey_1": "338f81e9efa98dbe6979d4f178f7f06cdbd6d367072318667a932b43c5032ab8", + "niah_single_2": "88ba10e924c2fc6c47543eb646b9e6f7487c7ca026c9391d6cd7054fde59eaad" + }, + "model_source": "hf", + "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25", + "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "727.4490673239925" +} \ No newline at end of file diff --git a/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T14-39-49.633113.jsonl b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T14-39-49.633113.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f4d6e50c3a96df3291a100abe44abd3bf7d3a1b5 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T14-39-49.633113.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcf1324d014a9cce5b6f9d569bb89b281f315c830905419ec9d93cae2b273ba8 +size 1442440 diff --git a/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T14-39-49.633113.jsonl b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T14-39-49.633113.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..90c9011efc37ea49f2ae58c602196001889b6685 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T14-39-49.633113.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeed86dbb7991c7c2d3c767423aa936b7e0d80e2bbd129246f9569ca23f74d96 +size 508178 diff --git a/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T14-39-49.633113.jsonl b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T14-39-49.633113.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..518222d3fd864f172cdc5875e7acb94c143e7a75 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T14-39-49.633113.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cb1c46255a1c944757568530ef339174df2d607112966c60361b09764109b41 +size 1434718 diff --git a/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T14-39-49.633113.jsonl b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T14-39-49.633113.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3651075c1217d1a1078b77fa0349db565cfb831a --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T14-39-49.633113.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83412829754b8db8500f3fb87a8716311e2a0b629a8b6480f7437680153e043 +size 1435920 diff --git a/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T14-39-49.633113.jsonl b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T14-39-49.633113.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3e390610c0db32e8c3fc97fed3b6ee52c25070fc --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T14-39-49.633113.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a61fd747ed45ac9170d945720216fc15ec470398e633fcfe5efc5f23e1817e6d +size 636771 diff --git a/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T14-39-49.633113.jsonl b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T14-39-49.633113.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..80fe9d0e9d6872139aa3274e5b6b1cca351b1e92 --- /dev/null +++ b/outputs/eval/token_t065/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T14-39-49.633113.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f0622ed447cc3e1e32661e23f785a0cf26904a93907eedee43464f63a4d8d85 +size 1387824 diff --git a/outputs/eval/token_t065/ruler_a.DONE b/outputs/eval/token_t065/ruler_a.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t065/ruler_b.DONE b/outputs/eval/token_t065/ruler_b.DONE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/outputs/eval/token_t065/sparsity_babilong.json b/outputs/eval/token_t065/sparsity_babilong.json new file mode 100644 index 0000000000000000000000000000000000000000..e5d99b018bfd14c500ea3552304e2f1e3d44b9f0 --- /dev/null +++ b/outputs/eval/token_t065/sparsity_babilong.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 53161864, + "effective_router_decisions": 425294912, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 174318504, + "total_decisions": 425294912, + "sparsity": 0.4098767680531292, + "full_attention_usage": 0.5901232319468708, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 15189104, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 198656, + "total_decisions": 15189104, + "sparsity": 0.013078849153972479 + }, + "10": { + "sparse_decisions": 6688, + "total_decisions": 15189104, + "sparsity": 0.00044031563678805543 + }, + "11": { + "sparse_decisions": 366520, + "total_decisions": 15189104, + "sparsity": 0.024130455621345408 + }, + "12": { + "sparse_decisions": 778896, + "total_decisions": 15189104, + "sparsity": 0.05127991749875437 + }, + "13": { + "sparse_decisions": 2585024, + "total_decisions": 15189104, + "sparsity": 0.1701893673254196 + }, + "14": { + "sparse_decisions": 6056200, + "total_decisions": 15189104, + "sparsity": 0.39872002983191107 + }, + "15": { + "sparse_decisions": 11935840, + "total_decisions": 15189104, + "sparsity": 0.7858159375299557 + }, + "16": { + "sparse_decisions": 5208592, + "total_decisions": 15189104, + "sparsity": 0.34291634318917036 + }, + "17": { + "sparse_decisions": 13174960, + "total_decisions": 15189104, + "sparsity": 0.8673954698051972 + }, + "18": { + "sparse_decisions": 12118600, + "total_decisions": 15189104, + "sparsity": 0.7978482470065383 + }, + "19": { + "sparse_decisions": 11068656, + "total_decisions": 15189104, + "sparsity": 0.7287234322709226 + }, + "20": { + "sparse_decisions": 12698176, + "total_decisions": 15189104, + "sparsity": 0.8360055998036487 + }, + "21": { + "sparse_decisions": 13380232, + "total_decisions": 15189104, + "sparsity": 0.8809098943558488 + }, + "22": { + "sparse_decisions": 14965704, + "total_decisions": 15189104, + "sparsity": 0.9852920883285808 + }, + "23": { + "sparse_decisions": 14819656, + "total_decisions": 15189104, + "sparsity": 0.9756767746142234 + }, + "24": { + "sparse_decisions": 14733136, + "total_decisions": 15189104, + "sparsity": 0.9699805860832871 + }, + "25": { + "sparse_decisions": 14539256, + "total_decisions": 15189104, + "sparsity": 0.9572161728565425 + }, + "26": { + "sparse_decisions": 10496608, + "total_decisions": 15189104, + "sparsity": 0.6910616979118716 + }, + "27": { + "sparse_decisions": 15187104, + "total_decisions": 15189104, + "sparsity": 0.9998683266636399 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 326256, + "total_decisions": 840000, + "sparsity": 0.3884, + "full_attention_usage": 0.6115999999999999 + }, + "prefill": { + "sparse_decisions": 173992248, + "total_decisions": 424454912, + "sparsity": 0.4099192707657957, + "full_attention_usage": 0.5900807292342043 + } + } +} diff --git a/outputs/eval/token_t065/sparsity_helmet.json b/outputs/eval/token_t065/sparsity_helmet.json new file mode 100644 index 0000000000000000000000000000000000000000..a5012690f82428491cccbad43b076faa40a8f9e4 --- /dev/null +++ b/outputs/eval/token_t065/sparsity_helmet.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 48728652, + "effective_router_decisions": 389829216, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 153251272, + "total_decisions": 389829216, + "sparsity": 0.3931241315684251, + "full_attention_usage": 0.6068758684315749, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 13922472, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 2306256, + "total_decisions": 13922472, + "sparsity": 0.16564989320861984 + }, + "10": { + "sparse_decisions": 1080, + "total_decisions": 13922472, + "sparsity": 7.757243110275244e-05 + }, + "11": { + "sparse_decisions": 109752, + "total_decisions": 13922472, + "sparsity": 0.007883082831841932 + }, + "12": { + "sparse_decisions": 574128, + "total_decisions": 13922472, + "sparsity": 0.041237504374223195 + }, + "13": { + "sparse_decisions": 2687560, + "total_decisions": 13922472, + "sparsity": 0.19303755827269756 + }, + "14": { + "sparse_decisions": 2726872, + "total_decisions": 13922472, + "sparsity": 0.19586119476483774 + }, + "15": { + "sparse_decisions": 10561808, + "total_decisions": 13922472, + "sparsity": 0.7586158550004626 + }, + "16": { + "sparse_decisions": 3574672, + "total_decisions": 13922472, + "sparsity": 0.2567555531804984 + }, + "17": { + "sparse_decisions": 9945560, + "total_decisions": 13922472, + "sparsity": 0.714353025813232 + }, + "18": { + "sparse_decisions": 11601984, + "total_decisions": 13922472, + "sparsity": 0.8333278745326261 + }, + "19": { + "sparse_decisions": 10332576, + "total_decisions": 13922472, + "sparsity": 0.7421509628462531 + }, + "20": { + "sparse_decisions": 12615840, + "total_decisions": 13922472, + "sparsity": 0.9061494251882856 + }, + "21": { + "sparse_decisions": 10107032, + "total_decisions": 13922472, + "sparsity": 0.7259509661789947 + }, + "22": { + "sparse_decisions": 12494288, + "total_decisions": 13922472, + "sparsity": 0.8974187917203209 + }, + "23": { + "sparse_decisions": 12785920, + "total_decisions": 13922472, + "sparsity": 0.9183656465604672 + }, + "24": { + "sparse_decisions": 13321176, + "total_decisions": 13922472, + "sparsity": 0.956811118025592 + }, + "25": { + "sparse_decisions": 13898000, + "total_decisions": 13922472, + "sparsity": 0.9982422661722717 + }, + "26": { + "sparse_decisions": 9686296, + "total_decisions": 13922472, + "sparsity": 0.6957310454637653 + }, + "27": { + "sparse_decisions": 13920472, + "total_decisions": 13922472, + "sparsity": 0.9998563473498098 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 422760, + "total_decisions": 1064000, + "sparsity": 0.39733082706766915, + "full_attention_usage": 0.6026691729323308 + }, + "prefill": { + "sparse_decisions": 152828512, + "total_decisions": 388765216, + "sparsity": 0.39311261838816364, + "full_attention_usage": 0.6068873816118363 + } + } +} diff --git a/outputs/eval/token_t065/sparsity_mrcr.json b/outputs/eval/token_t065/sparsity_mrcr.json new file mode 100644 index 0000000000000000000000000000000000000000..aeab40828fa127913f658a8aca113acb4d5442fa --- /dev/null +++ b/outputs/eval/token_t065/sparsity_mrcr.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 5287940, + "effective_router_decisions": 42303520, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 21339288, + "total_decisions": 42303520, + "sparsity": 0.5044329171662311, + "full_attention_usage": 0.49556708283376893, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 1510840, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 25904, + "total_decisions": 1510840, + "sparsity": 0.017145429032855895 + }, + "10": { + "sparse_decisions": 832, + "total_decisions": 1510840, + "sparsity": 0.0005506870350268725 + }, + "11": { + "sparse_decisions": 219792, + "total_decisions": 1510840, + "sparsity": 0.14547668846469514 + }, + "12": { + "sparse_decisions": 219272, + "total_decisions": 1510840, + "sparsity": 0.14513250906780334 + }, + "13": { + "sparse_decisions": 530944, + "total_decisions": 1510840, + "sparsity": 0.35142304942945646 + }, + "14": { + "sparse_decisions": 1153856, + "total_decisions": 1510840, + "sparsity": 0.7637181964999603 + }, + "15": { + "sparse_decisions": 1385488, + "total_decisions": 1510840, + "sparsity": 0.9170315850784994 + }, + "16": { + "sparse_decisions": 1413568, + "total_decisions": 1510840, + "sparsity": 0.9356172725106563 + }, + "17": { + "sparse_decisions": 1476232, + "total_decisions": 1510840, + "sparsity": 0.9770935373699399 + }, + "18": { + "sparse_decisions": 1436416, + "total_decisions": 1510840, + "sparsity": 0.9507399857033174 + }, + "19": { + "sparse_decisions": 1500488, + "total_decisions": 1510840, + "sparsity": 0.993148182468031 + }, + "20": { + "sparse_decisions": 1501016, + "total_decisions": 1510840, + "sparsity": 0.9934976569325673 + }, + "21": { + "sparse_decisions": 1489232, + "total_decisions": 1510840, + "sparsity": 0.9856980222922348 + }, + "22": { + "sparse_decisions": 1506640, + "total_decisions": 1510840, + "sparsity": 0.9972200894866432 + }, + "23": { + "sparse_decisions": 1489368, + "total_decisions": 1510840, + "sparsity": 0.9857880384421911 + }, + "24": { + "sparse_decisions": 1503696, + "total_decisions": 1510840, + "sparsity": 0.9952715045934711 + }, + "25": { + "sparse_decisions": 1499296, + "total_decisions": 1510840, + "sparsity": 0.9923592173890021 + }, + "26": { + "sparse_decisions": 1476648, + "total_decisions": 1510840, + "sparsity": 0.9773688808874533 + }, + "27": { + "sparse_decisions": 1510600, + "total_decisions": 1510840, + "sparsity": 0.9998411479706654 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 795200, + "total_decisions": 1533728, + "sparsity": 0.5184752446326858, + "full_attention_usage": 0.4815247553673142 + }, + "prefill": { + "sparse_decisions": 20544088, + "total_decisions": 40769792, + "sparsity": 0.5039046556823248, + "full_attention_usage": 0.4960953443176752 + } + } +} diff --git a/outputs/eval/token_t065/sparsity_ruler_a.json b/outputs/eval/token_t065/sparsity_ruler_a.json new file mode 100644 index 0000000000000000000000000000000000000000..f1f78d4ad61ac3b8ea11191933027c4518bc2f91 --- /dev/null +++ b/outputs/eval/token_t065/sparsity_ruler_a.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 29979236, + "effective_router_decisions": 239833888, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 108470984, + "total_decisions": 239833888, + "sparsity": 0.45227546826076553, + "full_attention_usage": 0.5477245317392345, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 8565496, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 1870304, + "total_decisions": 8565496, + "sparsity": 0.21835326290503201 + }, + "10": { + "sparse_decisions": 57880, + "total_decisions": 8565496, + "sparsity": 0.00675734364945124 + }, + "11": { + "sparse_decisions": 1415536, + "total_decisions": 8565496, + "sparsity": 0.16526024879353163 + }, + "12": { + "sparse_decisions": 569168, + "total_decisions": 8565496, + "sparsity": 0.06644892484918562 + }, + "13": { + "sparse_decisions": 3463144, + "total_decisions": 8565496, + "sparsity": 0.40431330538243204 + }, + "14": { + "sparse_decisions": 4293288, + "total_decisions": 8565496, + "sparsity": 0.5012305183494336 + }, + "15": { + "sparse_decisions": 7064912, + "total_decisions": 8565496, + "sparsity": 0.8248106122517599 + }, + "16": { + "sparse_decisions": 4284136, + "total_decisions": 8565496, + "sparsity": 0.5001620454904188 + }, + "17": { + "sparse_decisions": 7138128, + "total_decisions": 8565496, + "sparsity": 0.8333583951238784 + }, + "18": { + "sparse_decisions": 6157656, + "total_decisions": 8565496, + "sparsity": 0.718890768263741 + }, + "19": { + "sparse_decisions": 7274328, + "total_decisions": 8565496, + "sparsity": 0.8492594007398988 + }, + "20": { + "sparse_decisions": 8217096, + "total_decisions": 8565496, + "sparsity": 0.9593251809352313 + }, + "21": { + "sparse_decisions": 7756416, + "total_decisions": 8565496, + "sparsity": 0.9055419557723219 + }, + "22": { + "sparse_decisions": 8463664, + "total_decisions": 8565496, + "sparsity": 0.9881113714839164 + }, + "23": { + "sparse_decisions": 8402408, + "total_decisions": 8565496, + "sparsity": 0.9809598883707377 + }, + "24": { + "sparse_decisions": 8333704, + "total_decisions": 8565496, + "sparsity": 0.9729388700899516 + }, + "25": { + "sparse_decisions": 8448408, + "total_decisions": 8565496, + "sparsity": 0.9863302720589677 + }, + "26": { + "sparse_decisions": 6696432, + "total_decisions": 8565496, + "sparsity": 0.7817915039596073 + }, + "27": { + "sparse_decisions": 8564376, + "total_decisions": 8565496, + "sparsity": 0.9998692428319388 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 1402064, + "total_decisions": 2764160, + "sparsity": 0.5072296827969437, + "full_attention_usage": 0.4927703172030563 + }, + "prefill": { + "sparse_decisions": 107068920, + "total_decisions": 237069728, + "sparsity": 0.4516347190477225, + "full_attention_usage": 0.5483652809522774 + } + } +} diff --git a/outputs/eval/token_t065/sparsity_ruler_b.json b/outputs/eval/token_t065/sparsity_ruler_b.json new file mode 100644 index 0000000000000000000000000000000000000000..74c4138ffa982306bb38153643c5b365eb44a1fe --- /dev/null +++ b/outputs/eval/token_t065/sparsity_ruler_b.json @@ -0,0 +1,167 @@ +{ + "definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode", + "router_granularity": "token", + "native_router_decisions": 25272464, + "effective_router_decisions": 202179712, + "native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.", + "sparse_decisions": 86334664, + "total_decisions": 202179712, + "sparsity": 0.42701942319514236, + "full_attention_usage": 0.5729805768048577, + "per_layer": { + "0": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "1": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "2": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "3": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "4": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "5": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "6": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "7": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "8": { + "sparse_decisions": 0, + "total_decisions": 7220704, + "sparsity": 0.0 + }, + "9": { + "sparse_decisions": 79160, + "total_decisions": 7220704, + "sparsity": 0.010962919959051085 + }, + "10": { + "sparse_decisions": 5720, + "total_decisions": 7220704, + "sparsity": 0.0007921665255908565 + }, + "11": { + "sparse_decisions": 130720, + "total_decisions": 7220704, + "sparsity": 0.01810349794147496 + }, + "12": { + "sparse_decisions": 276416, + "total_decisions": 7220704, + "sparsity": 0.03828103187722416 + }, + "13": { + "sparse_decisions": 1201008, + "total_decisions": 7220704, + "sparsity": 0.16632838016902507 + }, + "14": { + "sparse_decisions": 3219680, + "total_decisions": 7220704, + "sparsity": 0.4458955802647498 + }, + "15": { + "sparse_decisions": 5586712, + "total_decisions": 7220704, + "sparsity": 0.7737073836567736 + }, + "16": { + "sparse_decisions": 3334448, + "total_decisions": 7220704, + "sparsity": 0.4617898753362553 + }, + "17": { + "sparse_decisions": 5915808, + "total_decisions": 7220704, + "sparsity": 0.819284103045908 + }, + "18": { + "sparse_decisions": 5246040, + "total_decisions": 7220704, + "sparsity": 0.7265274964878771 + }, + "19": { + "sparse_decisions": 6154752, + "total_decisions": 7220704, + "sparsity": 0.852375613236604 + }, + "20": { + "sparse_decisions": 6963304, + "total_decisions": 7220704, + "sparsity": 0.9643525063484114 + }, + "21": { + "sparse_decisions": 6725584, + "total_decisions": 7220704, + "sparsity": 0.9314305087149397 + }, + "22": { + "sparse_decisions": 7069864, + "total_decisions": 7220704, + "sparsity": 0.9791100701538243 + }, + "23": { + "sparse_decisions": 6992128, + "total_decisions": 7220704, + "sparsity": 0.968344360882263 + }, + "24": { + "sparse_decisions": 7047448, + "total_decisions": 7220704, + "sparsity": 0.9760056637136767 + }, + "25": { + "sparse_decisions": 7035632, + "total_decisions": 7220704, + "sparsity": 0.974369258177596 + }, + "26": { + "sparse_decisions": 6130496, + "total_decisions": 7220704, + "sparsity": 0.8490163839980146 + }, + "27": { + "sparse_decisions": 7219744, + "total_decisions": 7220704, + "sparsity": 0.999867048974726 + } + }, + "by_phase": { + "decode": { + "sparse_decisions": 1527584, + "total_decisions": 2947840, + "sparsity": 0.5182045158488927, + "full_attention_usage": 0.4817954841511073 + }, + "prefill": { + "sparse_decisions": 84807080, + "total_decisions": 199231872, + "sparsity": 0.42567024617426674, + "full_attention_usage": 0.5743297538257333 + } + } +} diff --git a/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/results_2026-07-18T11-49-27.779423.json b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/results_2026-07-18T11-49-27.779423.json new file mode 100644 index 0000000000000000000000000000000000000000..7cec8de8a9ddf64a8b8ac8c19806a559f4cf23b5 --- /dev/null +++ b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/results_2026-07-18T11-49-27.779423.json @@ -0,0 +1,505 @@ +{ + "results": { + "babilong_longctx": { + "acc,none": 0.6, + "acc_stderr,none": 0.02948261334746155, + "alias": "babilong_longctx" + }, + "babilong_qa1": { + "alias": " - babilong_qa1", + "acc,none": 0.68, + "acc_stderr,none": 0.06663945022680343 + }, + "babilong_qa2": { + "alias": " - babilong_qa2", + "acc,none": 0.48, + "acc_stderr,none": 0.07137140569598172 + }, + "babilong_qa3": { + "alias": " - babilong_qa3", + "acc,none": 0.34, + "acc_stderr,none": 0.06767268161329718 + }, + "babilong_qa4": { + "alias": " - babilong_qa4", + "acc,none": 0.72, + "acc_stderr,none": 0.06414269805898185 + }, + "babilong_qa5": { + "alias": " - babilong_qa5", + "acc,none": 0.78, + "acc_stderr,none": 0.059178043363451345 + } + }, + "groups": { + "babilong_longctx": { + "acc,none": 0.6, + "acc_stderr,none": 0.02948261334746155, + "alias": "babilong_longctx" + } + }, + "group_subtasks": { + "babilong_longctx": [ + "babilong_qa1", + "babilong_qa2", + "babilong_qa3", + "babilong_qa4", + "babilong_qa5" + ] + }, + "configs": { + "babilong_qa1": { + "task": "babilong_qa1", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa1" + }, + "test_split": "qa1", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about positions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location to answer the question.\nAlways return your answer in the following format:\nThe most recent location of 'person' is 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the hallway. Judith come back to the kitchen. Charlie travelled to balcony.", + "question": "Where is Charlie?", + "target": "The most recent location of Charlie is balcony." + }, + { + "input": "Alan moved to the garage. Charlie went to the beach. Alan went to the shop. Rouse travelled to balcony.", + "question": "Where is Alan?", + "target": "The most recent location of Alan is shop." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa2": { + "task": "babilong_qa2", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa2" + }, + "test_split": "qa2", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nThe 'item' is in 'location'. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Charlie went to the kitchen. Charlie got a bottle. Charlie moved to the balcony.", + "question": "Where is the bottle?", + "target": "The bottle is in the balcony." + }, + { + "input": "Alan moved to the garage. Alan got a screw driver. Alan moved to the kitchen.", + "question": "Where is the screw driver?", + "target": "The screw driver is in the kitchen." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa3": { + "task": "babilong_qa3", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa3" + }, + "test_split": "qa3", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nBefore the $location_1$ the $item$ was in the $location_2$. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "John journeyed to the bedroom. Mary grabbed the apple. Mary went back to the bathroom. Daniel journeyed to the bedroom. Daniel moved to the garden. Mary travelled to the kitchen.", + "question": "Where was the apple before the kitchen?", + "target": "Before the kitchen the apple was in the bathroom." + }, + { + "input": "John went back to the bedroom. John went back to the garden. John went back to the kitchen. Sandra took the football. Sandra travelled to the garden. Sandra journeyed to the bedroom.", + "question": "Where was the football before the bedroom?", + "target": "Before the bedroom the football was in the garden." + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa4": { + "task": "babilong_qa4", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa4" + }, + "test_split": "qa4", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about different people, their location and actions, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - location. Do not write anything else after that.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "The hallway is south of the kitchen. The bedroom is north of the kitchen.", + "question": "What is the kitchen south of?", + "target": "bedroom" + }, + { + "input": "The garden is west of the bedroom. The bedroom is west of the kitchen.", + "question": "What is west of the bedroom?", + "target": "garden" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + }, + "babilong_qa5": { + "task": "babilong_qa5", + "custom_dataset": "def load_dataset(**kwargs):\n config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n # Get specific qa split\n qa_split = kwargs.get(\"qa_split\")\n\n eval_logger.info(\n f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n )\n dataset = datasets.load_dataset(\n \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n )\n return {qa_split: dataset}\n", + "dataset_path": "RMT-team/babilong-1k-samples", + "dataset_kwargs": { + "qa_split": "qa5" + }, + "test_split": "qa5", + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_target": "{{target}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n pred = postprocess_pred(results)\n target = doc.get(\"target\", \"\").strip()\n\n # String match\n score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n return {\"acc\": score}\n", + "description": "I will give you context with the facts about locations and their relations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word. Do not write anything else after that. Do not explain your answer.\n\n", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": [ + { + "input": "Mary picked up the apple there. Mary gave the apple to Fred. Mary moved to the bedroom. Bill took the milk there.", + "question": "Who did Mary give the apple to?", + "target": "Fred" + }, + { + "input": "Jeff took the football there. Jeff passed the football to Fred. Jeff got the milk there. Bill travelled to the bedroom.", + "question": "Who gave the football?", + "target": "Jeff" + }, + { + "input": "Fred picked up the apple there. Fred handed the apple to Bill. Bill journeyed to the bedroom. Jeff went back to the garden.", + "question": "What did Fred give to Bill?", + "target": "apple" + } + ], + "doc_to_text": "{{input.strip()}}\n{{question.strip()}}", + "doc_to_choice": null, + "doc_to_target": "{{target}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 2, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 16, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": "8k" + } + } + }, + "versions": { + "babilong_longctx": 0.0, + "babilong_qa1": 0.0, + "babilong_qa2": 0.0, + "babilong_qa3": 0.0, + "babilong_qa4": 0.0, + "babilong_qa5": 0.0 + }, + "n-shot": { + "babilong_qa1": 2, + "babilong_qa2": 2, + "babilong_qa3": 2, + "babilong_qa4": 2, + "babilong_qa5": 2 + }, + "higher_is_better": { + "babilong_longctx": { + "acc": true + }, + "babilong_qa1": { + "acc": true + }, + "babilong_qa2": { + "acc": true + }, + "babilong_qa3": { + "acc": true + }, + "babilong_qa4": { + "acc": true + }, + "babilong_qa5": { + "acc": true + } + }, + "n-samples": { + "babilong_qa1": { + "original": 1000, + "effective": 50 + }, + "babilong_qa2": { + "original": 999, + "effective": 50 + }, + "babilong_qa3": { + "original": 999, + "effective": 50 + }, + "babilong_qa4": { + "original": 999, + "effective": 50 + }, + "babilong_qa5": { + "original": 999, + "effective": 50 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 1720574976, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 50.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784375164.8417735, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "babilong_qa1": "bbdcd109bea1bb245e118e814bd0563de6384bc875a21df5218ab4ed75b27380", + "babilong_qa2": "fa4f8d12bdd481e02a1051414304811f648caf7bce8f89dcf03ecbf5b86f1e92", + "babilong_qa3": "737b64396c8b2e5f96da9796bcd5e448f08e5a13e32ae7b610d31a60269b6e48", + "babilong_qa4": "298a25727c69c9babddcd78f438cae1aa21a388e14e7db90b2206eff753c90c0", + "babilong_qa5": "e60afcdcdd1c4b3e24b002ffbf8f87085f5ec4a02fcd72318f042dcdafe230f4" + }, + "model_source": "hf", + "model_name": "/workspace", + "model_name_sanitized": "__workspace", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "206.16118575798464" +} \ No newline at end of file diff --git a/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa1_2026-07-18T11-49-27.779423.jsonl b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa1_2026-07-18T11-49-27.779423.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fa361dce51770a2fb14446b0048068858499f653 --- /dev/null +++ b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa1_2026-07-18T11-49-27.779423.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc0efd1b7fa32f44cdbcbc69a595406a69fad96d7ee9cc70b239280be5e8864 +size 3164110 diff --git a/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa2_2026-07-18T11-49-27.779423.jsonl b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa2_2026-07-18T11-49-27.779423.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e4169238a899a9fc05e23aef822f62439f4b3804 --- /dev/null +++ b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa2_2026-07-18T11-49-27.779423.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe2652bb38bcd261174fbc66a7385060c563dc46c3de1dc631122a72a3c43e1 +size 3147520 diff --git a/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa3_2026-07-18T11-49-27.779423.jsonl b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa3_2026-07-18T11-49-27.779423.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3f53a7d1e6dc2f34b837e3cb3c333cca96a1ba1c --- /dev/null +++ b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa3_2026-07-18T11-49-27.779423.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44c6580367ef0fac22336fd456f584c948466729af4c41206f1e0b2e4285f118 +size 3126960 diff --git a/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa4_2026-07-18T11-49-27.779423.jsonl b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa4_2026-07-18T11-49-27.779423.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5c3b8429bac288651480f0d5bd880766af258871 --- /dev/null +++ b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa4_2026-07-18T11-49-27.779423.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b46c44cd2585ae77c39c98e86fc7029938668286a9bddea9f7c9d4f478e1f181 +size 3137616 diff --git a/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa5_2026-07-18T11-49-27.779423.jsonl b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa5_2026-07-18T11-49-27.779423.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3ee9159b8b23d331840c81da236ee4d02c638052 --- /dev/null +++ b/outputs/eval/vanilla/babilong8k_qa1_qa5_n50/__workspace/samples_babilong_qa5_2026-07-18T11-49-27.779423.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8bdf34f1a7bcc55f1403126e730ea9fc18a4bc56110b95e7d72ad2c16cd5d2 +size 3135608 diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/results_2026-07-18T11-34-54.383274.json b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/results_2026-07-18T11-34-54.383274.json new file mode 100644 index 0000000000000000000000000000000000000000..a8048d41a0cd5aef2477872e5b66ce78a7ddbcb5 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/results_2026-07-18T11-34-54.383274.json @@ -0,0 +1,821 @@ +{ + "results": { + "niah_multikey_2": { + "alias": "niah_multikey_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_multiquery": { + "alias": "niah_multiquery", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.9875, + "8192_stderr,none": "N/A" + }, + "niah_single_1": { + "alias": "niah_single_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_single_3": { + "alias": "niah_single_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "ruler_fwe": { + "alias": "ruler_fwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.7666666666666666, + "8192_stderr,none": "N/A" + }, + "ruler_qa_hotpot": { + "alias": "ruler_qa_hotpot", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.45, + "8192_stderr,none": "N/A" + }, + "ruler_vt": { + "alias": "ruler_vt", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.8600000000000001, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_1": [], + "niah_single_3": [], + "niah_multikey_2": [], + "niah_multiquery": [], + "ruler_vt": [], + "ruler_fwe": [], + "ruler_qa_hotpot": [] + }, + "configs": { + "niah_multikey_2": { + "task": "niah_multikey_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multiquery": { + "task": "niah_multiquery", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multiquery(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_q=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_1": { + "task": "niah_single_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"repeat\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"repeat\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_3": { + "task": "niah_single_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_fwe": { + "task": "ruler_fwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def fwe_download(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, max_seq_length=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 50, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_hotpot": { + "task": "ruler_qa_hotpot", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_hotpotqa(**kwargs):\n return get_qa_dataset(\"hotpotqa\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_vt": { + "task": "ruler_vt", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_vt_dataset(**kwargs) -> dict[str, datasets.Dataset]:\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", \"\"))\n df = (\n get_dataset(tokenizer=get_tokenizer(pretrained), seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 30, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_2": 1.0, + "niah_multiquery": 1.0, + "niah_single_1": 1.0, + "niah_single_3": 1.0, + "ruler_fwe": 1.0, + "ruler_qa_hotpot": 1.0, + "ruler_vt": 1.0 + }, + "n-shot": { + "niah_multikey_2": 0, + "niah_multiquery": 0, + "niah_single_1": 0, + "niah_single_3": 0, + "ruler_fwe": 0, + "ruler_qa_hotpot": 0, + "ruler_vt": 0 + }, + "higher_is_better": { + "niah_multikey_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multiquery": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_fwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_hotpot": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_vt": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_hotpot": { + "original": 500, + "effective": 20 + }, + "ruler_fwe": { + "original": 500, + "effective": 20 + }, + "ruler_vt": { + "original": 500, + "effective": 20 + }, + "niah_multiquery": { + "original": 500, + "effective": 20 + }, + "niah_multikey_2": { + "original": 500, + "effective": 20 + }, + "niah_single_3": { + "original": 500, + "effective": 20 + }, + "niah_single_1": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 1720574976, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784373937.809425, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_hotpot": "fd48cb605efc8c5adac7e3771533db4d4c354b71bf91ce00841c814604ba6404", + "ruler_fwe": "c4633bcd0acf3ba4b96f905b5ba269ba94cf45a4c8ca0c13aae1468227cc3a10", + "ruler_vt": "af011da859c8b86f63f57f6256cf59077d3d3d8536084ab67ccf39e18e9a615a", + "niah_multiquery": "f9a36d2854dfd2bb52ffc7a6bf3f53490fb5303ca9da0046256bc28b55c2adb2", + "niah_multikey_2": "86e49b8f875843454b2923143b9124ba82e8a2e23b145f792726cd82f4e49c24", + "niah_single_3": "905e40146a287e5f0c8e2a81f64f082108c45a23862949e6a99bfcbbf561cad2", + "niah_single_1": "426e8138e992480aa416d01e6b9cef5865a68c00f532400e448a0c20ee026cff" + }, + "model_source": "hf", + "model_name": "/workspace", + "model_name_sanitized": "__workspace", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "559.9003549870104" +} \ No newline at end of file diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_multikey_2_2026-07-18T11-34-54.383274.jsonl b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_multikey_2_2026-07-18T11-34-54.383274.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3737a6ea421ad3163da1940b654014c6bd0af973 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_multikey_2_2026-07-18T11-34-54.383274.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:361b487bcd3e2e312e8e4441fa89adee971d7220e70e4cd5369b258dfe11ebb3 +size 987656 diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_multiquery_2026-07-18T11-34-54.383274.jsonl b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_multiquery_2026-07-18T11-34-54.383274.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1f36cf292cb52af90ce523f8379eed5120b5603f --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_multiquery_2026-07-18T11-34-54.383274.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb22b0f677274eb81acd2a4c5d25588730c15c64831db15e9a8e5118f40be994 +size 1440839 diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_single_1_2026-07-18T11-34-54.383274.jsonl b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_single_1_2026-07-18T11-34-54.383274.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ce88c53d371800bb8b94bffb43f79ef034452d52 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_single_1_2026-07-18T11-34-54.383274.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e4fcb6babd01a97926f269bce4b8a4f4094d0c45e5ee43a0e2c765cdee5d41c +size 1231702 diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_single_3_2026-07-18T11-34-54.383274.jsonl b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_single_3_2026-07-18T11-34-54.383274.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b0d1d5bb2cc8216e742a4e21cbb8277e0a0289ba --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_niah_single_3_2026-07-18T11-34-54.383274.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0af0fa3b3696d5cd6d6146d52ffaa97166940d2a8d880a1ee91ebb119cd5766b +size 1435106 diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_fwe_2026-07-18T11-34-54.383274.jsonl b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_fwe_2026-07-18T11-34-54.383274.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..54c6d9cb2c37257d3957261e4e0ab1803757de15 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_fwe_2026-07-18T11-34-54.383274.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ba6e97fbf67abd0dc6764e64c4887c73bde6f1d6561b76f88453c4aa0c2e008 +size 827107 diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_qa_hotpot_2026-07-18T11-34-54.383274.jsonl b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_qa_hotpot_2026-07-18T11-34-54.383274.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4595b56cb0da1375e7f0b1f299666833608da57e --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_qa_hotpot_2026-07-18T11-34-54.383274.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4034286d389591e80daeda1a9caf84252288fdcd48fdb871faf236566e2e47ea +size 1172370 diff --git a/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_vt_2026-07-18T11-34-54.383274.jsonl b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_vt_2026-07-18T11-34-54.383274.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ed9f2e73207a16ffea15248e6ea0ac75298b9d7c --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/a/lm_eval/__workspace/samples_ruler_vt_2026-07-18T11-34-54.383274.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8b8669268b4a1595f6edcd39938b92155d4aae0453616ca5cf2b531f4244bd3 +size 1228996 diff --git a/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/results_2026-07-18T11-45-58.082714.json b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/results_2026-07-18T11-45-58.082714.json new file mode 100644 index 0000000000000000000000000000000000000000..4ff2269d34a960a366d0c550b0a79296f22f6a98 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/results_2026-07-18T11-45-58.082714.json @@ -0,0 +1,714 @@ +{ + "results": { + "niah_multikey_1": { + "alias": "niah_multikey_1", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_multikey_3": { + "alias": "niah_multikey_3", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.75, + "8192_stderr,none": "N/A" + }, + "niah_multivalue": { + "alias": "niah_multivalue", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "niah_single_2": { + "alias": "niah_single_2", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 1.0, + "8192_stderr,none": "N/A" + }, + "ruler_cwe": { + "alias": "ruler_cwe", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.425, + "8192_stderr,none": "N/A" + }, + "ruler_qa_squad": { + "alias": "ruler_qa_squad", + "4096,none": -1, + "4096_stderr,none": "N/A", + "8192,none": 0.4208333333333334, + "8192_stderr,none": "N/A" + } + }, + "group_subtasks": { + "niah_single_2": [], + "niah_multikey_1": [], + "niah_multikey_3": [], + "niah_multivalue": [], + "ruler_cwe": [], + "ruler_qa_squad": [] + }, + "configs": { + "niah_multikey_1": { + "task": "niah_multikey_1", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_1(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_k=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multikey_3": { + "task": "niah_multikey_3", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multikey_3(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"needle\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"needle\",\n type_needle_k=\"uuids\",\n type_needle_v=\"uuids\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_multivalue": { + "task": "niah_multivalue", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_multivalue(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_needle_v=4,\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "niah_single_2": { + "task": "niah_single_2", + "tag": [ + "longcxt" + ], + "custom_dataset": "def niah_single_2(**kwargs):\n seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n return download_dataset(\n generate_samples(\n get_haystack(type_haystack=\"essay\"),\n max_seq_length=seq,\n template=TEMPLATE,\n type_haystack=\"essay\",\n type_needle_k=\"words\",\n type_needle_v=\"numbers\",\n num_samples=500,\n TOKENIZER=get_tokenizer(**kwargs),\n )\n for seq in seq_lengths\n )\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 128, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_cwe": { + "task": "ruler_cwe", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_cw_dataset(**kwargs):\n pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n df = (\n get_dataset(pretrained, seq=seq)\n for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n )\n\n return {\n \"test\": datasets.Dataset.from_list(\n list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n )\n }\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_all(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": "\n\n", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": "\n\n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 120, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + }, + "ruler_qa_squad": { + "task": "ruler_qa_squad", + "tag": [ + "longcxt" + ], + "custom_dataset": "def get_squad(**kwargs):\n return get_qa_dataset(\"squad\", **kwargs)\n", + "dataset_path": "", + "dataset_name": "", + "test_split": "test", + "doc_to_text": "{{input}}", + "doc_to_target": "{{outputs}}", + "unsafe_code": false, + "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n # hacky: set all other lengths to -1\n metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n input_len = doc[\"max_length\"]\n pred = postprocess_pred(results)\n score = string_match_part(pred, [doc[\"outputs\"]])\n metrics[str(input_len)] = score\n return metrics\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{input}}", + "doc_to_choice": null, + "doc_to_target": "{{outputs}}", + "gen_prefix": "{{gen_prefix}}", + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "4096", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "8192", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "16384", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "32768", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "65536", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + }, + { + "metric": "131072", + "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n res = [x for x in metrics if x != -1]\n if not res:\n # we don't have any samples with this length\n return -1\n return sum(res) / len(res)\n", + "higher_is_better": true + } + ], + "output_type": "generate_until", + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "max_gen_toks": 32, + "until": [] + }, + "repeats": 1, + "should_decontaminate": false, + "gen_prefix": "{{gen_prefix}}", + "metadata": { + "version": 1.0, + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa", + "max_seq_lengths": [ + 8192 + ] + } + } + }, + "versions": { + "niah_multikey_1": 1.0, + "niah_multikey_3": 1.0, + "niah_multivalue": 1.0, + "niah_single_2": 1.0, + "ruler_cwe": 1.0, + "ruler_qa_squad": 1.0 + }, + "n-shot": { + "niah_multikey_1": 0, + "niah_multikey_3": 0, + "niah_multivalue": 0, + "niah_single_2": 0, + "ruler_cwe": 0, + "ruler_qa_squad": 0 + }, + "higher_is_better": { + "niah_multikey_1": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multikey_3": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_multivalue": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "niah_single_2": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_cwe": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + }, + "ruler_qa_squad": { + "4096": true, + "8192": true, + "16384": true, + "32768": true, + "65536": true, + "131072": true + } + }, + "n-samples": { + "ruler_qa_squad": { + "original": 500, + "effective": 20 + }, + "ruler_cwe": { + "original": 500, + "effective": 20 + }, + "niah_multivalue": { + "original": 500, + "effective": 20 + }, + "niah_multikey_3": { + "original": 500, + "effective": 20 + }, + "niah_multikey_1": { + "original": 500, + "effective": 20 + }, + "niah_single_2": { + "original": 500, + "effective": 20 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "/workspace", + "trust_remote_code": true, + "dtype": "bfloat16", + "max_length": 16384, + "attn_implementation": "sdpa" + }, + "model_num_parameters": 1720574976, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": 20.0, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1784374500.9787333, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 24\nSocket(s): 2\nStepping: 7\nBogoMIPS: 6000.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 48 MiB (48 instances)\nL3 cache: 71.5 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23,48-71\nNUMA node1 CPU(s): 24-47,72-95\nVulnerability Gather data sampling: Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Vulnerable\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy 2.3.4 py311h2e04523_0 conda-forge\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] optree 0.17.0 pypi_0 pypi\n[conda] torch 2.9.1+cu128 pypi_0 pypi\n[conda] torchaudio 2.9.1+cu128 pypi_0 pypi\n[conda] torchelastic 0.2.2 pypi_0 pypi\n[conda] torchvision 0.24.1+cu128 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.54.0", + "lm_eval_version": "0.4.11", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151645, + "max_length": 16384, + "task_hashes": { + "ruler_qa_squad": "8c658a728ccd3d67ddd3f45e8e76527cf15a1ae4119033d8246f293ecc68cf11", + "ruler_cwe": "f68206129fc7d709951d550ec9269fed05e73bea2fc86ce9f2c758974db5db71", + "niah_multivalue": "29646655c230b4f155a8de1fbc7ddf22c49e43045bd8fdab5770e7cb45026c7c", + "niah_multikey_3": "de2cec52885e38d56e0bda0e344e989909969c71ab5f8048b9368a011dd3616b", + "niah_multikey_1": "338f81e9efa98dbe6979d4f178f7f06cdbd6d367072318667a932b43c5032ab8", + "niah_single_2": "88ba10e924c2fc6c47543eb646b9e6f7487c7ca026c9391d6cd7054fde59eaad" + }, + "model_source": "hf", + "model_name": "/workspace", + "model_name_sanitized": "__workspace", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "660.3073135850136" +} \ No newline at end of file diff --git a/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multikey_1_2026-07-18T11-45-58.082714.jsonl b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multikey_1_2026-07-18T11-45-58.082714.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..11ee146e9783aa711812ba0f9ad908e59783b7a9 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multikey_1_2026-07-18T11-45-58.082714.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:493f48f51a7983128854573427065eb25400a5133e6677df10d77e8c732f2314 +size 1445254 diff --git a/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multikey_3_2026-07-18T11-45-58.082714.jsonl b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multikey_3_2026-07-18T11-45-58.082714.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3fe9b3896e784f70adfa5b2695b029ae57cf37cf --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multikey_3_2026-07-18T11-45-58.082714.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16640bcab759eca3acd8f774ccd690dd64ba6aca8372f3156953491f05e66628 +size 502636 diff --git a/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multivalue_2026-07-18T11-45-58.082714.jsonl b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multivalue_2026-07-18T11-45-58.082714.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..89fced1ca1a768773cdca16776c51817e53e1a4a --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_multivalue_2026-07-18T11-45-58.082714.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a65a4a964718a8b1cb274ada7752eccc6324315f1eb5e6550147e6e5c09d69b2 +size 1432692 diff --git a/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_single_2_2026-07-18T11-45-58.082714.jsonl b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_single_2_2026-07-18T11-45-58.082714.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d739d56dc4568babdd5752e688b4ab93772e81a9 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_niah_single_2_2026-07-18T11-45-58.082714.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44f68816f94020359ab07c983dd48406b1e4add11067dd0d5e0eb5d03ecc4356 +size 1438132 diff --git a/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_ruler_cwe_2026-07-18T11-45-58.082714.jsonl b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_ruler_cwe_2026-07-18T11-45-58.082714.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..89ad9a37b59e493e237e6d5785f15caf4b998c47 --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_ruler_cwe_2026-07-18T11-45-58.082714.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:076ac6d26f881a75b2849588fa9d29d4f35fff8cb9e0eddc478212dd5f921a01 +size 634811 diff --git a/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_ruler_qa_squad_2026-07-18T11-45-58.082714.jsonl b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_ruler_qa_squad_2026-07-18T11-45-58.082714.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..400a7bc8032edbebb61d56e3a751fcf9a74fa01f --- /dev/null +++ b/outputs/eval/vanilla/ruler8k_splits/b/lm_eval/__workspace/samples_ruler_qa_squad_2026-07-18T11-45-58.082714.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:511e125acd68b2c72221cb76bfe7b130500e426a655225cb703a95f1760a7880 +size 1387264