File size: 4,323 Bytes
75f1c59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3ef1409
 
 
 
 
 
 
 
 
 
75f1c59
 
fd060f9
 
 
75f1c59
5721477
75f1c59
 
 
 
 
 
 
 
 
 
 
 
 
fd060f9
75f1c59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fd060f9
75f1c59
 
 
 
 
3ef1409
fd060f9
75f1c59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fd060f9
75f1c59
 
 
fd060f9
75f1c59
 
 
fd060f9
75f1c59
 
 
 
 
 
 
 
fd060f9
 
 
75f1c59
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
"""Rubric-style task graders for validator compatibility."""

from __future__ import annotations

from typing import Any

try:
    from openenv.core.rubric import Rubric as OpenEnvRubric  # type: ignore
except Exception:
    try:
        from openenv.core.rubrics import Rubric as OpenEnvRubric  # type: ignore
    except Exception:
        try:
            from openenv.rubric import Rubric as OpenEnvRubric  # type: ignore
        except Exception:
            try:
                from openenv.rubrics import Rubric as OpenEnvRubric  # type: ignore
            except Exception:
                class OpenEnvRubric:  # pragma: no cover
                    def reset(self) -> None:
                        return None


class _RubricMeta(type(OpenEnvRubric)):
    """Allow both RubricClass() and RubricClass(episode_data)."""

    def __call__(cls, *args: Any, **kwargs: Any) -> Any:
        if args and isinstance(args[0], dict) and len(args) == 1 and not kwargs:
            instance = super().__call__()
            return instance(args[0])
        return super().__call__(*args, **kwargs)


_MAX_REWARD_PER_QUERY: float = 1.5
_MAX_REWARDS: dict[str, float] = {
    "task_1": 1.0 * _MAX_REWARD_PER_QUERY,
    "task_2": 3.0 * _MAX_REWARD_PER_QUERY,
    "task_3": 5.0 * _MAX_REWARD_PER_QUERY,
}
_SCORE_MIN: float = 0.1
_SCORE_MAX: float = 0.99


def _extract_rewards(episode_data: dict[str, Any]) -> list[float]:
    rewards = episode_data.get("rewards")
    if rewards is None:
        rewards = episode_data.get("per_step_rewards")
    if rewards is None:
        rewards = []
    return [float(reward) for reward in rewards]


def _compute_score(task_id: str, rewards: list[float]) -> float:
    max_reward = _MAX_REWARDS.get(task_id, _MAX_REWARDS["task_1"])
    total_reward = sum(rewards)
    raw_score = total_reward / max_reward if max_reward > 0 else 0.0
    return float(max(_SCORE_MIN, min(_SCORE_MAX, raw_score)))


def _make_result(task_id: str, score: float, rewards: list[float]) -> dict[str, Any]:
    raw_total_reward = round(sum(rewards), 4)
    return {
        "score": round(score, 4),
        "reward": round(score, 4),
        "normalized_reward": round(score, 4),
        "success": score >= 0.5,
        "task_id": task_id,
        "total_reward": raw_total_reward,
        "raw_total_reward": raw_total_reward,
        "steps": len(rewards),
        "details": {
            "per_step_rewards": [round(reward, 4) for reward in rewards],
            "max_possible_reward": _MAX_REWARDS.get(task_id, _MAX_REWARDS["task_1"]),
            "score_range": [_SCORE_MIN, _SCORE_MAX],
        },
    }


class BaseEpisodeRubric(OpenEnvRubric, metaclass=_RubricMeta):
    task_id: str = "task_1"

    def __init__(self) -> None:
        self.reset()

    def reset(self) -> None:
        self._rewards: list[float] = []

    def forward(self, action: Any, observation: Any) -> float:
        reward_value = None
        if isinstance(observation, dict):
            reward_value = observation.get("reward")
        elif observation is not None and hasattr(observation, "reward"):
            reward_value = getattr(observation, "reward")
        if reward_value is not None:
            self._rewards.append(float(reward_value))
        return _compute_score(self.task_id, self._rewards)

    def __call__(self, episode_data: dict[str, Any]) -> dict[str, Any]:
        rewards = _extract_rewards(episode_data)
        self._rewards = list(rewards)
        score = _compute_score(self.task_id, rewards)
        return _make_result(self.task_id, score, rewards)

    def grade(self, episode_data: dict[str, Any]) -> dict[str, Any]:
        return self(episode_data)

    def score(self, episode_data: dict[str, Any]) -> float:
        return float(self(episode_data)["score"])


class Task1Rubric(BaseEpisodeRubric):
    task_id = "task_1"


class Task2Rubric(BaseEpisodeRubric):
    task_id = "task_2"


class Task3Rubric(BaseEpisodeRubric):
    task_id = "task_3"


Task1Grader = Task1Rubric
Task2Grader = Task2Rubric
Task3Grader = Task3Rubric


GRADERS: dict[str, type[BaseEpisodeRubric]] = {
    "task_1": Task1Rubric,
    "task_2": Task2Rubric,
    "task_3": Task3Rubric,
}


def get_grader(task_id: str) -> BaseEpisodeRubric:
    grader_cls = GRADERS.get(task_id, Task1Rubric)
    return grader_cls()