Publish full float32 standard-harness evaluation

evaluation/standard_float32/arithmark.json

@@ -0,0 +1,157 @@
+{
+  "model": "solintellegence/sol-pro",
+  "tokenizer": "solintellegence/sol-pro",
+  "params": 138037594,
+  "dataset": {
+    "path": "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/blobs/237840149650455c4c54ff18f02203e2a76c1d75",
+    "sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
+    "items": 1000
+  },
+  "runtime": {
+    "device": "cuda",
+    "dtype": "torch.float32",
+    "batch_size": 32,
+    "max_context": 1024,
+    "attention_implementation": "auto",
+    "compiled": false,
+    "torch_version": "2.11.0+cu130"
+  },
+  "primary_metric": "acc_norm",
+  "results": {
+    "arithmark-3": {
+      "acc": 37.2,
+      "acc_norm": 36.0,
+      "raw_correct": 372,
+      "norm_correct": 360,
+      "total": 1000,
+      "categories": {
+        "elementary_school_math_continuation::addition::grades_1_2::easy": {
+          "acc": 23.4375,
+          "acc_norm": 24.21875,
+          "raw_correct": 30,
+          "norm_correct": 31,
+          "total": 128
+        },
+        "elementary_school_math_continuation::comparison::grades_2_3::medium": {
+          "acc": 20.454545454545457,
+          "acc_norm": 18.181818181818183,
+          "raw_correct": 9,
+          "norm_correct": 8,
+          "total": 44
+        },
+        "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
+          "acc": 37.5,
+          "acc_norm": 35.41666666666667,
+          "raw_correct": 18,
+          "norm_correct": 17,
+          "total": 48
+        },
+        "elementary_school_math_continuation::data::grades_2_3::easy": {
+          "acc": 30.23255813953488,
+          "acc_norm": 37.2093023255814,
+          "raw_correct": 13,
+          "norm_correct": 16,
+          "total": 43
+        },
+        "elementary_school_math_continuation::division::grades_3_4::medium": {
+          "acc": 25.925925925925924,
+          "acc_norm": 25.925925925925924,
+          "raw_correct": 14,
+          "norm_correct": 14,
+          "total": 54
+        },
+        "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
+          "acc": 20.0,
+          "acc_norm": 20.0,
+          "raw_correct": 10,
+          "norm_correct": 10,
+          "total": 50
+        },
+        "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
+          "acc": 57.692307692307686,
+          "acc_norm": 55.769230769230774,
+          "raw_correct": 30,
+          "norm_correct": 29,
+          "total": 52
+        },
+        "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
+          "acc": 62.22222222222222,
+          "acc_norm": 66.66666666666666,
+          "raw_correct": 28,
+          "norm_correct": 30,
+          "total": 45
+        },
+        "elementary_school_math_continuation::measurement::grades_2_3::easy": {
+          "acc": 32.89473684210527,
+          "acc_norm": 31.57894736842105,
+          "raw_correct": 25,
+          "norm_correct": 24,
+          "total": 76
+        },
+        "elementary_school_math_continuation::money::grades_3_4::medium": {
+          "acc": 28.125,
+          "acc_norm": 29.6875,
+          "raw_correct": 18,
+          "norm_correct": 19,
+          "total": 64
+        },
+        "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
+          "acc": 67.56756756756756,
+          "acc_norm": 59.45945945945946,
+          "raw_correct": 50,
+          "norm_correct": 44,
+          "total": 74
+        },
+        "elementary_school_math_continuation::patterns::grades_3_4::medium": {
+          "acc": 33.9622641509434,
+          "acc_norm": 30.18867924528302,
+          "raw_correct": 18,
+          "norm_correct": 16,
+          "total": 53
+        },
+        "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
+          "acc": 28.205128205128204,
+          "acc_norm": 27.350427350427353,
+          "raw_correct": 33,
+          "norm_correct": 32,
+          "total": 117
+        },
+        "elementary_school_math_continuation::time::grades_2_3::easy": {
+          "acc": 100.0,
+          "acc_norm": 87.27272727272727,
+          "raw_correct": 55,
+          "norm_correct": 48,
+          "total": 55
+        },
+        "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
+          "acc": 17.391304347826086,
+          "acc_norm": 19.565217391304348,
+          "raw_correct": 8,
+          "norm_correct": 9,
+          "total": 46
+        },
+        "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
+          "acc": 21.052631578947366,
+          "acc_norm": 21.052631578947366,
+          "raw_correct": 4,
+          "norm_correct": 4,
+          "total": 19
+        },
+        "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
+          "acc": 28.125,
+          "acc_norm": 28.125,
+          "raw_correct": 9,
+          "norm_correct": 9,
+          "total": 32
+        }
+      },
+      "timing": {
+        "tokenization_seconds": 0.2399261619993922,
+        "evaluation_seconds": 49.24065730300026,
+        "examples_per_second": 20.308421023840992
+      },
+      "primary_metric": "acc_norm",
+      "primary_acc": 36.0
+    }
+  }
+}

evaluation/standard_float32/lm_eval.json

@@ -0,0 +1,353 @@
+{
+  "results": {
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.36875124477195775,
+      "acc_stderr,none": 0.004814803098436572,
+      "acc_norm,none": 0.4676359290977893,
+      "acc_norm_stderr,none": 0.004979317515432586
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.5534511784511784,
+      "acc_stderr,none": 0.010200990076245288,
+      "acc_norm,none": 0.5408249158249159,
+      "acc_norm_stderr,none": 0.010225526906982679
+    },
+    "arc_challenge": {
+      "name": "arc_challenge",
+      "alias": "arc_challenge",
+      "sample_len": 1172,
+      "acc,none": 0.2935153583617747,
+      "acc_stderr,none": 0.013307250444941061,
+      "acc_norm,none": 0.3199658703071672,
+      "acc_norm_stderr,none": 0.013631345807016129
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.6926006528835691,
+      "acc_stderr,none": 0.010765602506938959,
+      "acc_norm,none": 0.7094668117519043,
+      "acc_norm_stderr,none": 0.010592765034696548
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "arc_challenge": {
+      "task": "arc_challenge",
+      "dataset_path": "allenai/ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{choices.label.index(answerKey)}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices.text}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": "{{choices.text}}",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "solintellegence/sol-pro",
+        "revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
+        "dtype": "float32",
+        "trust_remote_code": true,
+        "config_source": "/tmp/uv-venv/lib/python3.13/site-packages/lm_eval/tasks/arc/arc_challenge.yaml"
+      }
+    },
+    "arc_easy": {
+      "task": "arc_easy",
+      "dataset_path": "allenai/ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{choices.label.index(answerKey)}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices.text}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": "{{choices.text}}",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "solintellegence/sol-pro",
+        "revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
+        "dtype": "float32",
+        "trust_remote_code": true,
+        "config_source": "/tmp/uv-venv/lib/python3.13/site-packages/lm_eval/tasks/arc/arc_easy.yaml"
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "dataset_path": "Rowan/hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "unsafe_code": false,
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": "<function process_docs at 0x7fec604482c0>",
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{query}}",
+        "doc_to_choice": "choices",
+        "doc_to_target": "{{label}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "solintellegence/sol-pro",
+        "revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
+        "dtype": "float32",
+        "trust_remote_code": true,
+        "config_source": "/tmp/uv-venv/lib/python3.13/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml"
+      }
+    },
+    "piqa": {
+      "task": "piqa",
+      "dataset_path": "baber/piqa",
+      "training_split": "train",
+      "validation_split": "validation",
+      "doc_to_text": "Question: {{goal}}\nAnswer:",
+      "doc_to_target": "label",
+      "unsafe_code": false,
+      "doc_to_choice": "{{[sol1, sol2]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{goal}}\nAnswer:",
+        "doc_to_choice": "{{[sol1, sol2]}}",
+        "doc_to_target": "label",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "goal",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "solintellegence/sol-pro",
+        "revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
+        "dtype": "float32",
+        "trust_remote_code": true,
+        "config_source": "/tmp/uv-venv/lib/python3.13/site-packages/lm_eval/tasks/piqa/piqa.yaml"
+      }
+    }
+  },
+  "versions": {
+    "arc_challenge": 1.0,
+    "arc_easy": 1.0,
+    "hellaswag": 1.0,
+    "piqa": 1.0
+  },
+  "n-shot": {
+    "arc_challenge": 0,
+    "arc_easy": 0,
+    "hellaswag": 0,
+    "piqa": 0
+  },
+  "higher_is_better": {
+    "arc_challenge": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "arc_easy": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "piqa": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "arc_easy": {
+      "original": 2376,
+      "effective": 2376
+    },
+    "arc_challenge": {
+      "original": 1172,
+      "effective": 1172
+    },
+    "piqa": {
+      "original": 1838,
+      "effective": 1838
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "solintellegence/sol-pro",
+      "revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
+      "dtype": "float32",
+      "trust_remote_code": true
+    },
+    "model_num_parameters": 138037594,
+    "model_dtype": "torch.float32",
+    "model_revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
+    "model_sha": "a22f3539b43e0450850748ce445849a06c9c64d6",
+    "batch_size": "64",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1791420621.7781,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu130\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 13 (trixie) (x86_64)\nGCC version: (Debian 14.2.0-19) 14.2.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.41\n\nPython version: 3.13.11 (main, Feb  3 2026, 03:08:54) [GCC 14.2.0] (64-bit runtime)\nPython platform: Linux-4.19.0-gvisor-x86_64-with-glibc2.41\nIs CUDA available: True\nCUDA runtime version: 13.3.73\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA RTX PRO 6000 Blackwell Server Edition\nNvidia driver version: 595.71.05\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:        x86_64\nCPU op-mode(s):      32-bit, 64-bit\nAddress sizes:       46 bits physical, 48 bits virtual\nByte Order:          Little Endian\nCPU(s):              20\nOn-line CPU(s) list: 0-19\nVendor ID:           GenuineIntel\nModel name:          unknown\nCPU family:          6\nModel:               207\nThread(s) per core:  1\nCore(s) per socket:  20\nSocket(s):           1\nStepping:            unknown\nBogoMIPS:            3785.24\nFlags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid cldemote movdiri movdir64b fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities\nVirtualization:      VT-x\n\nVersions of relevant libraries:\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas==13.1.0.3\n[pip3] nvidia-cuda-cupti==13.0.85\n[pip3] nvidia-cuda-nvrtc==13.0.88\n[pip3] nvidia-cuda-runtime==13.0.96\n[pip3] nvidia-cudnn-cu13==9.19.0.56\n[pip3] nvidia-cufft==12.0.0.61\n[pip3] nvidia-curand==10.4.0.35\n[pip3] nvidia-cusolver==12.0.4.66\n[pip3] nvidia-cusparse==12.6.3.3\n[pip3] nvidia-cusparselt-cu13==0.8.0\n[pip3] nvidia-nccl-cu12==2.30.7\n[pip3] nvidia-nccl-cu13==2.28.9\n[pip3] nvidia-nvjitlink==13.0.88\n[pip3] nvidia-nvtx==13.0.85\n[pip3] onnxruntime==1.27.0\n[pip3] optree==0.19.1\n[pip3] torch==2.11.0\n[pip3] torch_c_dlpack_ext==0.1.5\n[pip3] torchao==0.17.0\n[pip3] torchvision==0.26.0\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.14.1",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "3"
+  ],
+  "tokenizer_eos_token": [
+    "<eos>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "solintellegence/sol-pro",
+  "model_name_sanitized": "solintellegence__sol-pro",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": null,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "total_evaluation_time_seconds": "121.92508437099968"
+}

evaluation/standard_float32/summary.json

@@ -0,0 +1,82 @@
+{
+  "model": "solintellegence/sol-pro",
+  "model_revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
+  "parameters": 138037594,
+  "dtype": "float32",
+  "num_fewshot": 0,
+  "metric": "acc_norm",
+  "scores": {
+    "hellaswag": 46.76359290977893,
+    "arc_easy": 54.082491582491585,
+    "arc_challenge": 31.99658703071672,
+    "piqa": 70.94668117519043,
+    "arithmark3": 36.0
+  },
+  "raw_scores": {
+    "hellaswag": 36.875124477195776,
+    "arc_easy": 55.34511784511784,
+    "arc_challenge": 29.351535836177472,
+    "piqa": 69.26006528835691,
+    "arithmark3": 37.2
+  },
+  "intelligence_index": 28.62946264079223,
+  "intelligence_index_raw": 24.12549744306075,
+  "examples": {
+    "hellaswag": 10042,
+    "arc_easy": 2376,
+    "arc_challenge": 1172,
+    "piqa": 1838,
+    "arithmark3": 1000
+  },
+  "versions": {
+    "lm-eval": "0.4.12",
+    "torch": "2.11.0",
+    "transformers": "5.14.1",
+    "datasets": "4.4.1",
+    "huggingface-hub": "1.29.0",
+    "accelerate": "1.15.0"
+  },
+  "arithmark_dataset_sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
+  "arithmark_script_sha256": "09667e1ea05ecd8b91dbfea14a47365b555c72622afed9fc284c127ecd5e1bcd",
+  "arithmark_dataset_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
+  "n_shot": {
+    "arc_challenge": 0,
+    "arc_easy": 0,
+    "hellaswag": 0,
+    "piqa": 0
+  },
+  "model_stage": "supervised task adaptation",
+  "lm_eval_command": [
+    "/tmp/uv-venv/bin/python",
+    "-m",
+    "lm_eval",
+    "--model",
+    "hf",
+    "--model_args",
+    "pretrained=solintellegence/sol-pro,revision=a22f3539b43e0450850748ce445849a06c9c64d6,dtype=float32,trust_remote_code=True",
+    "--tasks",
+    "hellaswag,arc_easy,arc_challenge,piqa",
+    "--num_fewshot",
+    "0",
+    "--batch_size",
+    "64",
+    "--device",
+    "cuda:0",
+    "--output_path",
+    "/marimo/sol_pro_standard_eval/lm_eval"
+  ],
+  "arithmark_command": [
+    "/tmp/uv-venv/bin/python",
+    "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/bencharithmark-3.py",
+    "--model",
+    "solintellegence/sol-pro",
+    "--data-path",
+    "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl",
+    "--device",
+    "cuda",
+    "--dtype",
+    "float32",
+    "--results-dir",
+    "/marimo/sol_pro_standard_eval/arithmark"
+  ]
+}