Spaces:
Runtime error
Runtime error
File size: 4,323 Bytes
75f1c59 3ef1409 75f1c59 fd060f9 75f1c59 5721477 75f1c59 fd060f9 75f1c59 fd060f9 75f1c59 3ef1409 fd060f9 75f1c59 fd060f9 75f1c59 fd060f9 75f1c59 fd060f9 75f1c59 fd060f9 75f1c59 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | """Rubric-style task graders for validator compatibility."""
from __future__ import annotations
from typing import Any
try:
from openenv.core.rubric import Rubric as OpenEnvRubric # type: ignore
except Exception:
try:
from openenv.core.rubrics import Rubric as OpenEnvRubric # type: ignore
except Exception:
try:
from openenv.rubric import Rubric as OpenEnvRubric # type: ignore
except Exception:
try:
from openenv.rubrics import Rubric as OpenEnvRubric # type: ignore
except Exception:
class OpenEnvRubric: # pragma: no cover
def reset(self) -> None:
return None
class _RubricMeta(type(OpenEnvRubric)):
"""Allow both RubricClass() and RubricClass(episode_data)."""
def __call__(cls, *args: Any, **kwargs: Any) -> Any:
if args and isinstance(args[0], dict) and len(args) == 1 and not kwargs:
instance = super().__call__()
return instance(args[0])
return super().__call__(*args, **kwargs)
_MAX_REWARD_PER_QUERY: float = 1.5
_MAX_REWARDS: dict[str, float] = {
"task_1": 1.0 * _MAX_REWARD_PER_QUERY,
"task_2": 3.0 * _MAX_REWARD_PER_QUERY,
"task_3": 5.0 * _MAX_REWARD_PER_QUERY,
}
_SCORE_MIN: float = 0.1
_SCORE_MAX: float = 0.99
def _extract_rewards(episode_data: dict[str, Any]) -> list[float]:
rewards = episode_data.get("rewards")
if rewards is None:
rewards = episode_data.get("per_step_rewards")
if rewards is None:
rewards = []
return [float(reward) for reward in rewards]
def _compute_score(task_id: str, rewards: list[float]) -> float:
max_reward = _MAX_REWARDS.get(task_id, _MAX_REWARDS["task_1"])
total_reward = sum(rewards)
raw_score = total_reward / max_reward if max_reward > 0 else 0.0
return float(max(_SCORE_MIN, min(_SCORE_MAX, raw_score)))
def _make_result(task_id: str, score: float, rewards: list[float]) -> dict[str, Any]:
raw_total_reward = round(sum(rewards), 4)
return {
"score": round(score, 4),
"reward": round(score, 4),
"normalized_reward": round(score, 4),
"success": score >= 0.5,
"task_id": task_id,
"total_reward": raw_total_reward,
"raw_total_reward": raw_total_reward,
"steps": len(rewards),
"details": {
"per_step_rewards": [round(reward, 4) for reward in rewards],
"max_possible_reward": _MAX_REWARDS.get(task_id, _MAX_REWARDS["task_1"]),
"score_range": [_SCORE_MIN, _SCORE_MAX],
},
}
class BaseEpisodeRubric(OpenEnvRubric, metaclass=_RubricMeta):
task_id: str = "task_1"
def __init__(self) -> None:
self.reset()
def reset(self) -> None:
self._rewards: list[float] = []
def forward(self, action: Any, observation: Any) -> float:
reward_value = None
if isinstance(observation, dict):
reward_value = observation.get("reward")
elif observation is not None and hasattr(observation, "reward"):
reward_value = getattr(observation, "reward")
if reward_value is not None:
self._rewards.append(float(reward_value))
return _compute_score(self.task_id, self._rewards)
def __call__(self, episode_data: dict[str, Any]) -> dict[str, Any]:
rewards = _extract_rewards(episode_data)
self._rewards = list(rewards)
score = _compute_score(self.task_id, rewards)
return _make_result(self.task_id, score, rewards)
def grade(self, episode_data: dict[str, Any]) -> dict[str, Any]:
return self(episode_data)
def score(self, episode_data: dict[str, Any]) -> float:
return float(self(episode_data)["score"])
class Task1Rubric(BaseEpisodeRubric):
task_id = "task_1"
class Task2Rubric(BaseEpisodeRubric):
task_id = "task_2"
class Task3Rubric(BaseEpisodeRubric):
task_id = "task_3"
Task1Grader = Task1Rubric
Task2Grader = Task2Rubric
Task3Grader = Task3Rubric
GRADERS: dict[str, type[BaseEpisodeRubric]] = {
"task_1": Task1Rubric,
"task_2": Task2Rubric,
"task_3": Task3Rubric,
}
def get_grader(task_id: str) -> BaseEpisodeRubric:
grader_cls = GRADERS.get(task_id, Task1Rubric)
return grader_cls()
|