Add head-to-head multi-model benchmark results and scripts (multi_model_head_to_head_benchmark.json)

multi_model_head_to_head_benchmark.json

@@ -0,0 +1,368 @@
+{
+  "timestamp_utc": "2026-11-08T11:31:53Z",
+  "elapsed_sec": 10.31,
+  "leaderboard": [
+    {
+      "model_id": "marsnext/Phobos-1 (Final Post-Trained)",
+      "org": "MarsNext",
+      "total_params_m": 96.26,
+      "active_params_m": 52.22,
+      "train_tokens": "1.56B",
+      "mc20_acc_pct": 35.0,
+      "mc20_correct": "7/20",
+      "gen25_score_pct": 80.0,
+      "gen25_passed": "20/25",
+      "python_pass_at_1": "3/5 (60%)",
+      "sql_synthesis": "4/5 (80%)",
+      "stem_qa": "4/5 (80%)",
+      "math_symbolic": "4/5 (80%)",
+      "think_planning": "5/5 (100%)",
+      "combined_avg_pct": 57.5,
+      "speed_tok_per_s": 17.49
+    },
+    {
+      "model_id": "marsnext/Phobos-1 (Base Step 33,977)",
+      "org": "MarsNext",
+      "total_params_m": 96.26,
+      "active_params_m": 52.22,
+      "train_tokens": "1.56B",
+      "mc20_acc_pct": 35.0,
+      "mc20_correct": "7/20",
+      "gen25_score_pct": 68.0,
+      "gen25_passed": "17/25",
+      "python_pass_at_1": "3/5 (60%)",
+      "sql_synthesis": "4/5 (80%)",
+      "stem_qa": "3/5 (60%)",
+      "math_symbolic": "2/5 (40%)",
+      "think_planning": "5/5 (100%)",
+      "combined_avg_pct": 51.5,
+      "speed_tok_per_s": 18.52
+    },
+    {
+      "model_id": "EleutherAI/pythia-70m-deduped",
+      "org": "EleutherAI",
+      "total_params_m": 70.43,
+      "active_params_m": 70.43,
+      "train_tokens": "300B",
+      "mc20_acc_pct": 30.0,
+      "mc20_correct": "6/20",
+      "gen25_score_pct": 12.0,
+      "gen25_passed": "3/25",
+      "python_pass_at_1": "0/5 (0%)",
+      "sql_synthesis": "0/5 (0%)",
+      "stem_qa": "1/5 (20%)",
+      "math_symbolic": "1/5 (20%)",
+      "think_planning": "1/5 (20%)",
+      "combined_avg_pct": 21.0,
+      "speed_tok_per_s": 101.6
+    },
+    {
+      "model_id": "openai-community/gpt2",
+      "org": "OpenAI",
+      "total_params_m": 124.44,
+      "active_params_m": 124.44,
+      "train_tokens": "40B",
+      "mc20_acc_pct": 40.0,
+      "mc20_correct": "8/20",
+      "gen25_score_pct": 20.0,
+      "gen25_passed": "5/25",
+      "python_pass_at_1": "0/5 (0%)",
+      "sql_synthesis": "1/5 (20%)",
+      "stem_qa": "1/5 (20%)",
+      "math_symbolic": "1/5 (20%)",
+      "think_planning": "2/5 (40%)",
+      "combined_avg_pct": 30.0,
+      "speed_tok_per_s": 70.3
+    },
+    {
+      "model_id": "EleutherAI/pythia-160m-deduped",
+      "org": "EleutherAI",
+      "total_params_m": 162.32,
+      "active_params_m": 162.32,
+      "train_tokens": "300B",
+      "mc20_acc_pct": 25.0,
+      "mc20_correct": "5/20",
+      "gen25_score_pct": 12.0,
+      "gen25_passed": "3/25",
+      "python_pass_at_1": "0/5 (0%)",
+      "sql_synthesis": "0/5 (0%)",
+      "stem_qa": "1/5 (20%)",
+      "math_symbolic": "0/5 (0%)",
+      "think_planning": "2/5 (40%)",
+      "combined_avg_pct": 18.5,
+      "speed_tok_per_s": 67.1
+    },
+    {
+      "model_id": "HuggingFaceTB/SmolLM2-135M",
+      "org": "HuggingFace",
+      "total_params_m": 134.52,
+      "active_params_m": 134.52,
+      "train_tokens": "2,000B",
+      "mc20_acc_pct": 60.0,
+      "mc20_correct": "12/20",
+      "gen25_score_pct": 36.0,
+      "gen25_passed": "9/25",
+      "python_pass_at_1": "0/5 (0%)",
+      "sql_synthesis": "2/5 (40%)",
+      "stem_qa": "4/5 (80%)",
+      "math_symbolic": "1/5 (20%)",
+      "think_planning": "2/5 (40%)",
+      "combined_avg_pct": 48.0,
+      "speed_tok_per_s": 25.8
+    }
+  ],
+  "phobos_mc_final": {
+    "correct": 7,
+    "total": 20,
+    "acc_pct": 35.0,
+    "details": [
+      {
+        "id": "ARC-01",
+        "pred": 0,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "ARC-02",
+        "pred": 2,
+        "ans": 2,
+        "correct": true
+      },
+      {
+        "id": "ARC-03",
+        "pred": 0,
+        "ans": 0,
+        "correct": true
+      },
+      {
+        "id": "ARC-04",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "ARC-05",
+        "pred": 1,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-01",
+        "pred": 1,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-02",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "SCIQ-03",
+        "pred": 1,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-04",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-05",
+        "pred": 2,
+        "ans": 2,
+        "correct": true
+      },
+      {
+        "id": "PIQA-01",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "PIQA-02",
+        "pred": 0,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "PIQA-03",
+        "pred": 3,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "CS-01",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "CS-02",
+        "pred": 0,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "CS-03",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "CS-04",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "MATH-MC1",
+        "pred": 0,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "MATH-MC2",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "MATH-MC3",
+        "pred": 3,
+        "ans": 0,
+        "correct": false
+      }
+    ]
+  },
+  "phobos_mc_base": {
+    "correct": 7,
+    "total": 20,
+    "acc_pct": 35.0,
+    "details": [
+      {
+        "id": "ARC-01",
+        "pred": 0,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "ARC-02",
+        "pred": 2,
+        "ans": 2,
+        "correct": true
+      },
+      {
+        "id": "ARC-03",
+        "pred": 0,
+        "ans": 0,
+        "correct": true
+      },
+      {
+        "id": "ARC-04",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "ARC-05",
+        "pred": 1,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-01",
+        "pred": 1,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-02",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "SCIQ-03",
+        "pred": 1,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-04",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "SCIQ-05",
+        "pred": 2,
+        "ans": 2,
+        "correct": true
+      },
+      {
+        "id": "PIQA-01",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "PIQA-02",
+        "pred": 0,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "PIQA-03",
+        "pred": 3,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "CS-01",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "CS-02",
+        "pred": 0,
+        "ans": 2,
+        "correct": false
+      },
+      {
+        "id": "CS-03",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "CS-04",
+        "pred": 1,
+        "ans": 1,
+        "correct": true
+      },
+      {
+        "id": "MATH-MC1",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "MATH-MC2",
+        "pred": 2,
+        "ans": 1,
+        "correct": false
+      },
+      {
+        "id": "MATH-MC3",
+        "pred": 3,
+        "ans": 0,
+        "correct": false
+      }
+    ]
+  }
+}