| { |
| "model": "solintellegence/sol-pro", |
| "model_revision": "a22f3539b43e0450850748ce445849a06c9c64d6", |
| "parameters": 138037594, |
| "dtype": "float32", |
| "num_fewshot": 0, |
| "metric": "acc_norm", |
| "scores": { |
| "hellaswag": 46.76359290977893, |
| "arc_easy": 54.082491582491585, |
| "arc_challenge": 31.99658703071672, |
| "piqa": 70.94668117519043, |
| "arithmark3": 36.0 |
| }, |
| "raw_scores": { |
| "hellaswag": 36.875124477195776, |
| "arc_easy": 55.34511784511784, |
| "arc_challenge": 29.351535836177472, |
| "piqa": 69.26006528835691, |
| "arithmark3": 37.2 |
| }, |
| "intelligence_index": 28.62946264079223, |
| "intelligence_index_raw": 24.12549744306075, |
| "examples": { |
| "hellaswag": 10042, |
| "arc_easy": 2376, |
| "arc_challenge": 1172, |
| "piqa": 1838, |
| "arithmark3": 1000 |
| }, |
| "versions": { |
| "lm-eval": "0.4.12", |
| "torch": "2.11.0", |
| "transformers": "5.14.1", |
| "datasets": "4.4.1", |
| "huggingface-hub": "1.29.0", |
| "accelerate": "1.15.0" |
| }, |
| "arithmark_dataset_sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435", |
| "arithmark_script_sha256": "09667e1ea05ecd8b91dbfea14a47365b555c72622afed9fc284c127ecd5e1bcd", |
| "arithmark_dataset_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746", |
| "n_shot": { |
| "arc_challenge": 0, |
| "arc_easy": 0, |
| "hellaswag": 0, |
| "piqa": 0 |
| }, |
| "model_stage": "supervised task adaptation", |
| "lm_eval_command": [ |
| "/tmp/uv-venv/bin/python", |
| "-m", |
| "lm_eval", |
| "--model", |
| "hf", |
| "--model_args", |
| "pretrained=solintellegence/sol-pro,revision=a22f3539b43e0450850748ce445849a06c9c64d6,dtype=float32,trust_remote_code=True", |
| "--tasks", |
| "hellaswag,arc_easy,arc_challenge,piqa", |
| "--num_fewshot", |
| "0", |
| "--batch_size", |
| "64", |
| "--device", |
| "cuda:0", |
| "--output_path", |
| "/marimo/sol_pro_standard_eval/lm_eval" |
| ], |
| "arithmark_command": [ |
| "/tmp/uv-venv/bin/python", |
| "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/bencharithmark-3.py", |
| "--model", |
| "solintellegence/sol-pro", |
| "--data-path", |
| "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl", |
| "--device", |
| "cuda", |
| "--dtype", |
| "float32", |
| "--results-dir", |
| "/marimo/sol_pro_standard_eval/arithmark" |
| ] |
| } |
