Spaces:
Runtime error
Runtime error
Update tasks.py
Browse files
tasks.py
CHANGED
|
@@ -1,8 +1,7 @@
|
|
| 1 |
-
"""Top-level task exports
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
-
from graders import grade_easy, grade_hard, grade_medium
|
| 6 |
from server.tasks import TASKS as TASK_REGISTRY
|
| 7 |
from server.tasks import get_task
|
| 8 |
|
|
@@ -16,10 +15,8 @@ TASKS = [
|
|
| 16 |
"difficulty": "easy",
|
| 17 |
"description": "Route a single LLM query to the optimal model tier while minimising carbon footprint and latency.",
|
| 18 |
"max_steps": 10,
|
| 19 |
-
"grader":
|
| 20 |
-
"
|
| 21 |
-
"graders": [grade_easy],
|
| 22 |
-
"grader_paths": ["graders:grade_easy"],
|
| 23 |
"reward_range": [0.0, 1.0],
|
| 24 |
},
|
| 25 |
{
|
|
@@ -30,10 +27,8 @@ TASKS = [
|
|
| 30 |
"difficulty": "medium",
|
| 31 |
"description": "Route 3 queries of varying complexity. LARGE model incurs -0.2 penalty per use.",
|
| 32 |
"max_steps": 20,
|
| 33 |
-
"grader":
|
| 34 |
-
"
|
| 35 |
-
"graders": [grade_medium],
|
| 36 |
-
"grader_paths": ["graders:grade_medium"],
|
| 37 |
"reward_range": [0.0, 1.0],
|
| 38 |
},
|
| 39 |
{
|
|
@@ -44,28 +39,25 @@ TASKS = [
|
|
| 44 |
"difficulty": "hard",
|
| 45 |
"description": "5-query episode with caching, KB lookups, cascade strategies, and carbon-aware waiting.",
|
| 46 |
"max_steps": 50,
|
| 47 |
-
"grader":
|
| 48 |
-
"
|
| 49 |
-
"graders": [grade_hard],
|
| 50 |
-
"grader_paths": ["graders:grade_hard"],
|
| 51 |
"reward_range": [0.0, 1.0],
|
| 52 |
},
|
| 53 |
]
|
| 54 |
|
| 55 |
TASK_IDS = [task["task_id"] for task in TASKS]
|
| 56 |
-
|
| 57 |
TASK_MAP = {task["task_id"]: task for task in TASKS}
|
| 58 |
-
easy
|
| 59 |
medium = TASK_MAP["medium"]
|
| 60 |
-
hard
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
__all__ = [
|
| 63 |
-
"TASKS",
|
| 64 |
-
"
|
| 65 |
-
"TASK_MAP",
|
| 66 |
-
"TASK_REGISTRY",
|
| 67 |
-
"easy",
|
| 68 |
-
"medium",
|
| 69 |
-
"hard",
|
| 70 |
-
"get_task",
|
| 71 |
]
|
|
|
|
| 1 |
+
"""Top-level task exports — grader paths point to CLASS names in graders module."""
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
|
|
|
| 5 |
from server.tasks import TASKS as TASK_REGISTRY
|
| 6 |
from server.tasks import get_task
|
| 7 |
|
|
|
|
| 15 |
"difficulty": "easy",
|
| 16 |
"description": "Route a single LLM query to the optimal model tier while minimising carbon footprint and latency.",
|
| 17 |
"max_steps": 10,
|
| 18 |
+
"grader": "graders:EasyGrader",
|
| 19 |
+
"graders": ["graders:EasyGrader"],
|
|
|
|
|
|
|
| 20 |
"reward_range": [0.0, 1.0],
|
| 21 |
},
|
| 22 |
{
|
|
|
|
| 27 |
"difficulty": "medium",
|
| 28 |
"description": "Route 3 queries of varying complexity. LARGE model incurs -0.2 penalty per use.",
|
| 29 |
"max_steps": 20,
|
| 30 |
+
"grader": "graders:MediumGrader",
|
| 31 |
+
"graders": ["graders:MediumGrader"],
|
|
|
|
|
|
|
| 32 |
"reward_range": [0.0, 1.0],
|
| 33 |
},
|
| 34 |
{
|
|
|
|
| 39 |
"difficulty": "hard",
|
| 40 |
"description": "5-query episode with caching, KB lookups, cascade strategies, and carbon-aware waiting.",
|
| 41 |
"max_steps": 50,
|
| 42 |
+
"grader": "graders:HardGrader",
|
| 43 |
+
"graders": ["graders:HardGrader"],
|
|
|
|
|
|
|
| 44 |
"reward_range": [0.0, 1.0],
|
| 45 |
},
|
| 46 |
]
|
| 47 |
|
| 48 |
TASK_IDS = [task["task_id"] for task in TASKS]
|
|
|
|
| 49 |
TASK_MAP = {task["task_id"]: task for task in TASKS}
|
| 50 |
+
easy = TASK_MAP["easy"]
|
| 51 |
medium = TASK_MAP["medium"]
|
| 52 |
+
hard = TASK_MAP["hard"]
|
| 53 |
+
|
| 54 |
+
TASK_GRADER_PAIRS = [
|
| 55 |
+
("easy", "graders:EasyGrader"),
|
| 56 |
+
("medium", "graders:MediumGrader"),
|
| 57 |
+
("hard", "graders:HardGrader"),
|
| 58 |
+
]
|
| 59 |
|
| 60 |
__all__ = [
|
| 61 |
+
"TASKS", "TASK_IDS", "TASK_MAP", "TASK_REGISTRY",
|
| 62 |
+
"TASK_GRADER_PAIRS", "easy", "medium", "hard", "get_task",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
]
|