Add minimum game count, difficulty multiplier, and updated scoring formula

- Require minimum 5 games for benchmark validity (prevents single-game
lucky wins from topping leaderboard)
- Add difficulty multiplier to composite score (Beginner 0.5x to Hard 1.2x)
- Update scoring formula: 50% win / 20% military / 20% economy / 10% speed
- No-combat military efficiency returns 0.0 instead of 0.5

evaluate.py

@@ -46,6 +46,7 @@ RESULTS_COLUMNS = [
     "agent_name",
     "agent_type",
     "opponent",
+    "difficulty",
     "games",
     "win_rate",
     "score",
@@ -58,6 +59,14 @@ RESULTS_COLUMNS = [
     "replay_url",
 ]
 
+DIFFICULTY_MULTIPLIER = {
+    "Beginner": 0.5,
+    "Easy": 0.7,
+    "Medium": 0.85,
+    "Normal": 1.0,
+    "Hard": 1.2,
+}
+
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
@@ -116,6 +125,9 @@ def parse_args() -> argparse.Namespace:
     )
     args = parser.parse_args()
 
+    if args.games < 5:
+        parser.error("Minimum 5 games required for benchmark validity")
+
     # Auto-detect agent type
     if args.agent_type is None:
         type_map = {"scripted": "Scripted", "llm": "LLM", "mcp": "Scripted", "custom": "RL"}
@@ -205,7 +217,12 @@ async def run_evaluation(args: argparse.Namespace) -> Dict[str, Any]:
         "opponent": args.opponent,
         "games": total,
         "win_rate": round(100.0 * wins / max(total, 1), 1),
-        "score": round(compute_composite_score_from_games(game_results), 1),
+        "score": round(
+            compute_composite_score_from_games(game_results)
+            * DIFFICULTY_MULTIPLIER.get(args.opponent, 1.0),
+            1,
+        ),
+        "difficulty": args.opponent,
         "avg_kills": round(sum(g["kills_cost"] for g in game_results) / max(total, 1)),
         "avg_deaths": round(sum(g["deaths_cost"] for g in game_results) / max(total, 1)),
         "kd_ratio": round(

evaluate_runner.py

@@ -17,6 +17,14 @@ DEFAULT_SERVER = "https://openra-rl-openra-rl.hf.space"
 MAX_STEPS_PER_GAME = 5000
 STEP_TIMEOUT = 60.0
 
+DIFFICULTY_MULTIPLIER = {
+    "Beginner": 0.5,
+    "Easy": 0.7,
+    "Medium": 0.85,
+    "Normal": 1.0,
+    "Hard": 1.2,
+}
+
 
 # ── Scoring (inlined from openra_rl_util/rubrics.py) ────────────────────────
 
@@ -46,7 +54,7 @@ def compute_game_metrics(obs: Dict[str, Any]) -> Dict[str, Any]:
 
 
 def compute_composite_score(game_results: List[Dict[str, Any]]) -> float:
-    """Compute OpenRA-Bench composite score: 50% win + 25% military + 25% economy."""
+    """Compute OpenRA-Bench composite score: 50% win + 20% military + 20% economy + 10% speed."""
     total = len(game_results)
     if total == 0:
         return 0.0
@@ -57,7 +65,7 @@ def compute_composite_score(game_results: List[Dict[str, Any]]) -> float:
     for g in game_results:
         kills, deaths = g["kills_cost"], g["deaths_cost"]
         total_cost = kills + deaths
-        mil_scores.append(kills / total_cost if total_cost > 0 else 0.5)
+        mil_scores.append(kills / total_cost if total_cost > 0 else 0.0)
     avg_mil = sum(mil_scores) / total
 
     econ_scores = []
@@ -66,7 +74,13 @@ def compute_composite_score(game_results: List[Dict[str, Any]]) -> float:
         econ_scores.append(assets / (assets + 10000) if assets >= 0 else 0.0)
     avg_econ = sum(econ_scores) / total
 
-    return 100.0 * (0.5 * win_rate + 0.25 * avg_mil + 0.25 * avg_econ)
+    speed_scores = []
+    for g in game_results:
+        ticks = g.get("ticks", 0)
+        speed_scores.append(1.0 / (1.0 + ticks / 3000))
+    avg_speed = sum(speed_scores) / total
+
+    return 100.0 * (0.5 * win_rate + 0.20 * avg_mil + 0.20 * avg_econ + 0.10 * avg_speed)
 
 
 # ── Server communication ────────────────────────────────────────────────────
@@ -136,6 +150,9 @@ async def run_evaluation(
     Returns:
         Dict with all fields needed for results.csv.
     """
+    if num_games < 5:
+        raise ValueError("Minimum 5 games required for benchmark validity")
+
     game_results: List[Dict[str, Any]] = []
 
     async with httpx.AsyncClient(timeout=STEP_TIMEOUT) as client:
@@ -154,7 +171,12 @@ async def run_evaluation(
         "opponent": opponent,
         "games": total,
         "win_rate": round(100.0 * wins / max(total, 1), 1),
-        "score": round(compute_composite_score(game_results), 1),
+        "score": round(
+            compute_composite_score(game_results)
+            * DIFFICULTY_MULTIPLIER.get(opponent, 1.0),
+            1,
+        ),
+        "difficulty": opponent,
         "avg_kills": round(sum(g["kills_cost"] for g in game_results) / max(total, 1)),
         "avg_deaths": round(sum(g["deaths_cost"] for g in game_results) / max(total, 1)),
         "kd_ratio": round(

tests/test_evaluate.py

@@ -151,7 +151,7 @@ class TestAppendResults:
         assert "score" in RESULTS_COLUMNS
         assert "win_rate" in RESULTS_COLUMNS
         assert "replay_url" in RESULTS_COLUMNS
-        assert len(RESULTS_COLUMNS) == 13
+        assert len(RESULTS_COLUMNS) == 14
 
 
 class TestScoringUsesUtil:
@@ -175,3 +175,51 @@ class TestScoringUsesUtil:
         from rubrics import compute_game_metrics
         from openra_rl_util.rubrics import compute_game_metrics as util_fn
         assert compute_game_metrics is util_fn
+
+
+class TestMinGames:
+    """Minimum game count validation."""
+
+    def test_evaluate_runner_rejects_few_games(self):
+        """run_evaluation should reject num_games < 5."""
+        import pytest
+        from evaluate_runner import run_evaluation
+        with pytest.raises(ValueError, match="Minimum 5 games"):
+            import asyncio
+            asyncio.run(run_evaluation("test", "Normal", 3, "http://fake"))
+
+
+class TestDifficultyMultiplier:
+    """Difficulty scaling for benchmark scores."""
+
+    def test_hard_scores_higher(self):
+        from evaluate_runner import DIFFICULTY_MULTIPLIER
+        assert DIFFICULTY_MULTIPLIER["Hard"] > DIFFICULTY_MULTIPLIER["Normal"]
+
+    def test_beginner_scores_lower(self):
+        from evaluate_runner import DIFFICULTY_MULTIPLIER
+        assert DIFFICULTY_MULTIPLIER["Beginner"] < DIFFICULTY_MULTIPLIER["Normal"]
+
+    def test_normal_is_baseline(self):
+        from evaluate_runner import DIFFICULTY_MULTIPLIER
+        assert DIFFICULTY_MULTIPLIER["Normal"] == 1.0
+
+
+class TestCompositeScoreUpdate:
+    """Updated composite score formula."""
+
+    def test_no_combat_military_is_zero(self):
+        from evaluate_runner import compute_composite_score
+        games = [
+            {"win": True, "kills_cost": 0, "deaths_cost": 0, "assets_value": 10000, "ticks": 2000},
+        ]
+        score = compute_composite_score(games)
+        # 50%*1.0 + 20%*0.0 + 20%*0.5 + 10%*(1/(1+2/3))
+        # = 50 + 0 + 10 + 6.0 = 66.0
+        assert abs(score - 66.0) < 1.0
+
+    def test_speed_bonus_for_fast_games(self):
+        from evaluate_runner import compute_composite_score
+        fast = [{"win": True, "kills_cost": 5000, "deaths_cost": 1000, "assets_value": 10000, "ticks": 500}]
+        slow = [{"win": True, "kills_cost": 5000, "deaths_cost": 1000, "assets_value": 10000, "ticks": 10000}]
+        assert compute_composite_score(fast) > compute_composite_score(slow)