Evaluate step-114000: BaseBench 1.1 + ArithMark 3 + ARC/HellaSwag/PIQA

evaluations/kaggle-step-114000/arithmark3/_kaggle_working_deep35_eval_resolved_model_arithmark-3_results.json

@@ -0,0 +1,157 @@
+{
+  "model": "/kaggle/working/deep35_eval/resolved_model",
+  "tokenizer": "/kaggle/working/deep35_eval/resolved_model",
+  "params": 35155584,
+  "dataset": {
+    "path": "/kaggle/working/deep35_eval/official_runners/arithmark3/arithmark-3.jsonl",
+    "sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
+    "items": 1000
+  },
+  "runtime": {
+    "device": "cuda",
+    "dtype": "torch.float32",
+    "batch_size": 32,
+    "max_context": 1024,
+    "attention_implementation": "sdpa",
+    "compiled": false,
+    "torch_version": "2.10.0+cu128"
+  },
+  "primary_metric": "acc_norm",
+  "results": {
+    "arithmark-3": {
+      "acc": 33.0,
+      "acc_norm": 33.2,
+      "raw_correct": 330,
+      "norm_correct": 332,
+      "total": 1000,
+      "categories": {
+        "elementary_school_math_continuation::addition::grades_1_2::easy": {
+          "acc": 26.5625,
+          "acc_norm": 26.5625,
+          "raw_correct": 34,
+          "norm_correct": 34,
+          "total": 128
+        },
+        "elementary_school_math_continuation::comparison::grades_2_3::medium": {
+          "acc": 18.181818181818183,
+          "acc_norm": 18.181818181818183,
+          "raw_correct": 8,
+          "norm_correct": 8,
+          "total": 44
+        },
+        "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
+          "acc": 14.583333333333334,
+          "acc_norm": 14.583333333333334,
+          "raw_correct": 7,
+          "norm_correct": 7,
+          "total": 48
+        },
+        "elementary_school_math_continuation::data::grades_2_3::easy": {
+          "acc": 20.930232558139537,
+          "acc_norm": 20.930232558139537,
+          "raw_correct": 9,
+          "norm_correct": 9,
+          "total": 43
+        },
+        "elementary_school_math_continuation::division::grades_3_4::medium": {
+          "acc": 37.03703703703704,
+          "acc_norm": 37.03703703703704,
+          "raw_correct": 20,
+          "norm_correct": 20,
+          "total": 54
+        },
+        "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
+          "acc": 16.0,
+          "acc_norm": 18.0,
+          "raw_correct": 8,
+          "norm_correct": 9,
+          "total": 50
+        },
+        "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
+          "acc": 48.07692307692308,
+          "acc_norm": 50.0,
+          "raw_correct": 25,
+          "norm_correct": 26,
+          "total": 52
+        },
+        "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
+          "acc": 42.22222222222222,
+          "acc_norm": 42.22222222222222,
+          "raw_correct": 19,
+          "norm_correct": 19,
+          "total": 45
+        },
+        "elementary_school_math_continuation::measurement::grades_2_3::easy": {
+          "acc": 40.78947368421053,
+          "acc_norm": 40.78947368421053,
+          "raw_correct": 31,
+          "norm_correct": 31,
+          "total": 76
+        },
+        "elementary_school_math_continuation::money::grades_3_4::medium": {
+          "acc": 31.25,
+          "acc_norm": 31.25,
+          "raw_correct": 20,
+          "norm_correct": 20,
+          "total": 64
+        },
+        "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
+          "acc": 39.189189189189186,
+          "acc_norm": 39.189189189189186,
+          "raw_correct": 29,
+          "norm_correct": 29,
+          "total": 74
+        },
+        "elementary_school_math_continuation::patterns::grades_3_4::medium": {
+          "acc": 22.641509433962266,
+          "acc_norm": 22.641509433962266,
+          "raw_correct": 12,
+          "norm_correct": 12,
+          "total": 53
+        },
+        "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
+          "acc": 24.786324786324787,
+          "acc_norm": 24.786324786324787,
+          "raw_correct": 29,
+          "norm_correct": 29,
+          "total": 117
+        },
+        "elementary_school_math_continuation::time::grades_2_3::easy": {
+          "acc": 100.0,
+          "acc_norm": 100.0,
+          "raw_correct": 55,
+          "norm_correct": 55,
+          "total": 55
+        },
+        "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
+          "acc": 23.91304347826087,
+          "acc_norm": 23.91304347826087,
+          "raw_correct": 11,
+          "norm_correct": 11,
+          "total": 46
+        },
+        "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
+          "acc": 5.263157894736842,
+          "acc_norm": 5.263157894736842,
+          "raw_correct": 1,
+          "norm_correct": 1,
+          "total": 19
+        },
+        "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
+          "acc": 37.5,
+          "acc_norm": 37.5,
+          "raw_correct": 12,
+          "norm_correct": 12,
+          "total": 32
+        }
+      },
+      "timing": {
+        "tokenization_seconds": 0.7077092219999486,
+        "evaluation_seconds": 3.8506093600000213,
+        "examples_per_second": 259.6991557720606
+      },
+      "primary_metric": "acc_norm",
+      "primary_acc": 33.2
+    }
+  }
+}

evaluations/kaggle-step-114000/bananabench_base_1_1/run_config.json

@@ -0,0 +1,12 @@
+{
+  "benchmark": "BananaMind Base Bench 1.1",
+  "model": "/kaggle/working/deep35_eval/resolved_model",
+  "tokenizer": "/kaggle/working/deep35_eval/resolved_model",
+  "model_revision": "main",
+  "dataset_id": "BananaMind/BananaMind-Base-Bench-1.1",
+  "dataset_revision": "main",
+  "dataset_sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21",
+  "score_mode": "mean_logprob",
+  "add_bos": false,
+  "max_context": null
+}

evaluations/kaggle-step-114000/deep35_benchmark_summary.csv

@@ -0,0 +1,7 @@
+benchmark,primary_metric,score,accuracy_pct,acc_norm_pct,extra
+BananaMind Base Bench 1.1,Overall Elo,884.0,34.57142857142857,,weighted_acc=33.55%
+ArithMark 3.0,acc_norm,33.2,33.0,33.2,official AxiomicLabs runner
+ARC Easy,acc_norm,36.91077441077441,40.15151515151515,36.91077441077441,"lm-eval 0.4.12, zero-shot"
+ARC Challenge,acc_norm,22.696245733788395,18.00341296928328,22.696245733788395,"lm-eval 0.4.12, zero-shot"
+HellaSwag,acc_norm,28.440549691296557,27.604062935670186,28.440549691296557,"lm-eval 0.4.12, zero-shot"
+PIQA,acc_norm,59.24918389553863,59.24918389553863,59.24918389553863,"lm-eval 0.4.12, zero-shot"

evaluations/kaggle-step-114000/deep35_benchmark_summary.json

@@ -0,0 +1,67 @@
+{
+  "metadata": {
+    "model_repo": "appvoid/deep35-15l384-1024ffn",
+    "checkpoint_source": "last-checkpoint/",
+    "checkpoint_tag": "step-114000",
+    "global_step": 114000,
+    "parameters": 35155584,
+    "tokenizer_vocab": 32000,
+    "dtype": "float32",
+    "lm_eval_version": "0.4.12",
+    "num_fewshot": 0,
+    "banana_revision": "d4aade51312889e8580963e1ce960c6eaef1a450",
+    "arithmark_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
+    "int_index": 10.026606614387545,
+    "generated_utc": "2026-09-04T04:48:49.909880+00:00"
+  },
+  "results": [
+    {
+      "benchmark": "BananaMind Base Bench 1.1",
+      "primary_metric": "Overall Elo",
+      "score": 884.0,
+      "accuracy_pct": 34.57142857142857,
+      "acc_norm_pct": null,
+      "extra": "weighted_acc=33.55%"
+    },
+    {
+      "benchmark": "ArithMark 3.0",
+      "primary_metric": "acc_norm",
+      "score": 33.2,
+      "accuracy_pct": 33.0,
+      "acc_norm_pct": 33.2,
+      "extra": "official AxiomicLabs runner"
+    },
+    {
+      "benchmark": "ARC Easy",
+      "primary_metric": "acc_norm",
+      "score": 36.91077441077441,
+      "accuracy_pct": 40.15151515151515,
+      "acc_norm_pct": 36.91077441077441,
+      "extra": "lm-eval 0.4.12, zero-shot"
+    },
+    {
+      "benchmark": "ARC Challenge",
+      "primary_metric": "acc_norm",
+      "score": 22.696245733788395,
+      "accuracy_pct": 18.00341296928328,
+      "acc_norm_pct": 22.696245733788395,
+      "extra": "lm-eval 0.4.12, zero-shot"
+    },
+    {
+      "benchmark": "HellaSwag",
+      "primary_metric": "acc_norm",
+      "score": 28.440549691296557,
+      "accuracy_pct": 27.604062935670186,
+      "acc_norm_pct": 28.440549691296557,
+      "extra": "lm-eval 0.4.12, zero-shot"
+    },
+    {
+      "benchmark": "PIQA",
+      "primary_metric": "acc_norm",
+      "score": 59.24918389553863,
+      "accuracy_pct": 59.24918389553863,
+      "acc_norm_pct": 59.24918389553863,
+      "extra": "lm-eval 0.4.12, zero-shot"
+    }
+  ]
+}

evaluations/kaggle-step-114000/lm_eval/__kaggle__working__deep35_eval__resolved_model/results_2026-09-04T04-48-13.703044.json

@@ -0,0 +1,354 @@
+{
+  "results": {
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.4015151515151515,
+      "acc_stderr,none": 0.01005879002075543,
+      "acc_norm,none": 0.3691077441077441,
+      "acc_norm_stderr,none": 0.009901987410242688
+    },
+    "arc_challenge": {
+      "name": "arc_challenge",
+      "alias": "arc_challenge",
+      "sample_len": 1172,
+      "acc,none": 0.18003412969283278,
+      "acc_stderr,none": 0.01122785672905012,
+      "acc_norm,none": 0.22696245733788395,
+      "acc_norm_stderr,none": 0.012240491536132965
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.27604062935670187,
+      "acc_stderr,none": 0.004461235175488133,
+      "acc_norm,none": 0.28440549691296557,
+      "acc_norm_stderr,none": 0.004502088287470376
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.5924918389553863,
+      "acc_stderr,none": 0.01146449062565553,
+      "acc_norm,none": 0.5924918389553863,
+      "acc_norm_stderr,none": 0.01146449062565553
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "arc_challenge": {
+      "task": "arc_challenge",
+      "dataset_path": "allenai/ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{choices.label.index(answerKey)}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices.text}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": "{{choices.text}}",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/kaggle/working/deep35_eval/resolved_model",
+        "dtype": "float32",
+        "attn_implementation": "sdpa",
+        "config_source": "/usr/local/lib/python3.12/dist-packages/lm_eval/tasks/arc/arc_challenge.yaml"
+      }
+    },
+    "arc_easy": {
+      "task": "arc_easy",
+      "dataset_path": "allenai/ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{choices.label.index(answerKey)}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices.text}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": "{{choices.text}}",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/kaggle/working/deep35_eval/resolved_model",
+        "dtype": "float32",
+        "attn_implementation": "sdpa",
+        "config_source": "/usr/local/lib/python3.12/dist-packages/lm_eval/tasks/arc/arc_easy.yaml"
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "dataset_path": "Rowan/hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "unsafe_code": false,
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": "<function process_docs at 0x7d411c3dd9e0>",
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{query}}",
+        "doc_to_choice": "choices",
+        "doc_to_target": "{{label}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/kaggle/working/deep35_eval/resolved_model",
+        "dtype": "float32",
+        "attn_implementation": "sdpa",
+        "config_source": "/usr/local/lib/python3.12/dist-packages/lm_eval/tasks/hellaswag/hellaswag.yaml"
+      }
+    },
+    "piqa": {
+      "task": "piqa",
+      "dataset_path": "baber/piqa",
+      "training_split": "train",
+      "validation_split": "validation",
+      "doc_to_text": "Question: {{goal}}\nAnswer:",
+      "doc_to_target": "label",
+      "unsafe_code": false,
+      "doc_to_choice": "{{[sol1, sol2]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{goal}}\nAnswer:",
+        "doc_to_choice": "{{[sol1, sol2]}}",
+        "doc_to_target": "label",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "goal",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/kaggle/working/deep35_eval/resolved_model",
+        "dtype": "float32",
+        "attn_implementation": "sdpa",
+        "config_source": "/usr/local/lib/python3.12/dist-packages/lm_eval/tasks/piqa/piqa.yaml"
+      }
+    }
+  },
+  "versions": {
+    "arc_challenge": 1.0,
+    "arc_easy": 1.0,
+    "hellaswag": 1.0,
+    "piqa": 1.0
+  },
+  "n-shot": {
+    "arc_challenge": 0,
+    "arc_easy": 0,
+    "hellaswag": 0,
+    "piqa": 0
+  },
+  "higher_is_better": {
+    "arc_challenge": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "arc_easy": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "piqa": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "arc_easy": {
+      "original": 2376,
+      "effective": 2376
+    },
+    "arc_challenge": {
+      "original": 1172,
+      "effective": 1172
+    },
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "piqa": {
+      "original": 1838,
+      "effective": 1838
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "/kaggle/working/deep35_eval/resolved_model",
+      "dtype": "float32",
+      "attn_implementation": "sdpa"
+    },
+    "model_num_parameters": 35155584,
+    "model_dtype": "torch.float32",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "auto:4",
+    "batch_sizes": [
+      121,
+      256,
+      256,
+      256,
+      256
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1788497041.5103345,
+  "pretty_env_info": "PyTorch version: 2.10.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0\nClang version: Could not collect\nCMake version: version 3.31.10\nLibc version: glibc-2.35\n\nPython version: 3.12.13 (main, Mar  4 2026, 09:23:07) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.12.90+-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.8.93\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: Tesla T4\nGPU 1: Tesla T4\n\nNvidia driver version: 580.159.04\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.8.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.8.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.8.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.8.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.8.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.8.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.8.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.8.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           46 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  4\nOn-line CPU(s) list:                     0-3\nVendor ID:                               GenuineIntel\nModel name:                              Intel(R) Xeon(R) CPU @ 2.00GHz\nCPU family:                              6\nModel:                                   85\nThread(s) per core:                      2\nCore(s) per socket:                      2\nSocket(s):                               1\nStepping:                                3\nBogoMIPS:                                4000.28\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat md_clear arch_capabilities\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               64 KiB (2 instances)\nL1i cache:                               64 KiB (2 instances)\nL2 cache:                                2 MiB (2 instances)\nL3 cache:                                38.5 MiB (1 instance)\nNUMA node(s):                            1\nNUMA node0 CPU(s):                       0-3\nVulnerability Gather data sampling:      Not affected\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Mitigation; PTE Inversion\nVulnerability Mds:                       Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:                  Mitigation; PTI\nVulnerability Mmio stale data:           Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Mitigation; IBRS\nVulnerability Spec rstack overflow:      Not affected\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; IBRS; IBPB conditional; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI SW loop, KVM SW loop\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Not affected\nVulnerability Tsx async abort:           Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] intel-cmplr-lib-ur==2025.3.3\n[pip3] intel-openmp==2025.3.3\n[pip3] mkl==2025.3.1\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==2.0.2\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] nvtx==0.2.15\n[pip3] onemkl-license==2025.3.1\n[pip3] onnx==1.22.0\n[pip3] optree==0.19.0\n[pip3] pytorch-ignite==0.5.4\n[pip3] pytorch-lightning==2.6.5\n[pip3] tbb==2022.3.1\n[pip3] tcmlib==1.4.1\n[pip3] torch==2.10.0+cu128\n[pip3] torchao==0.10.0\n[pip3] torchaudio==2.10.0+cu128\n[pip3] torchcodec==0.10.0+cu128\n[pip3] torchdata==0.11.0\n[pip3] torchinfo==1.8.0\n[pip3] torchmetrics==1.9.0\n[pip3] torchsummary==1.5.1\n[pip3] torchtune==0.6.1\n[pip3] torchvision==0.25.0+cu128\n[pip3] triton==3.6.0\n[pip3] umf==1.0.3\n[conda] Could not collect",
+  "transformers_version": "4.57.6",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 1024,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "/kaggle/working/deep35_eval/resolved_model",
+  "model_name_sanitized": "__kaggle__working__deep35_eval__resolved_model",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": null,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "total_evaluation_time_seconds": "261.31383836300006"
+}