Spaces:
Sleeping
Sleeping
Commit ·
a3c6db2
1
Parent(s): e093b15
scoring components rebuilt 0-1
Browse files- graders.py +17 -16
- openenv.yaml +9 -6
graders.py
CHANGED
|
@@ -15,7 +15,7 @@ import math
|
|
| 15 |
from typing import Any, Dict, List, Optional, Tuple
|
| 16 |
|
| 17 |
|
| 18 |
-
SCORE_EPSILON = 0.
|
| 19 |
|
| 20 |
|
| 21 |
def _strict_score(score: float) -> float:
|
|
@@ -29,7 +29,7 @@ def _strict_score(score: float) -> float:
|
|
| 29 |
if math.isnan(value):
|
| 30 |
value = SCORE_EPSILON
|
| 31 |
|
| 32 |
-
value = min(max(value,
|
| 33 |
return round(value, 4)
|
| 34 |
|
| 35 |
|
|
@@ -42,7 +42,7 @@ def grade_task(
|
|
| 42 |
return _grade_extraction(episode_state)
|
| 43 |
elif task_id == "task3":
|
| 44 |
return _grade_resolution(episode_state)
|
| 45 |
-
return _strict_score(0.
|
| 46 |
|
| 47 |
|
| 48 |
# ---------------------------------------------------------------------------
|
|
@@ -78,6 +78,7 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
|
|
| 78 |
history = ep.get("action_history", [])
|
| 79 |
|
| 80 |
breakdown: Dict[str, float] = {
|
|
|
|
| 81 |
"category_correct": 0.0,
|
| 82 |
"priority_correct": 0.0,
|
| 83 |
"efficiency": 0.0,
|
|
@@ -89,7 +90,7 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
|
|
| 89 |
|
| 90 |
# Category
|
| 91 |
if _normalize(classify_action.get("category")) == _normalize(gt["category"]):
|
| 92 |
-
breakdown["category_correct"] = 0.
|
| 93 |
|
| 94 |
# Priority
|
| 95 |
if _normalize(classify_action.get("priority")) == _normalize(gt["priority"]):
|
|
@@ -99,9 +100,9 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
|
|
| 99 |
max_steps = ep.get("max_steps", 3)
|
| 100 |
steps_used = ep.get("step_number", max_steps)
|
| 101 |
if steps_used <= 1:
|
| 102 |
-
breakdown["efficiency"] = 0.
|
| 103 |
else:
|
| 104 |
-
breakdown["efficiency"] = round(max(0.0, 0.
|
| 105 |
|
| 106 |
score = _strict_score(sum(breakdown.values()))
|
| 107 |
parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
|
|
@@ -123,15 +124,16 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
|
|
| 123 |
history = ep.get("action_history", [])
|
| 124 |
|
| 125 |
breakdown: Dict[str, float] = {
|
|
|
|
| 126 |
"entity_coverage": 0.0,
|
| 127 |
"action_coverage": 0.0,
|
| 128 |
-
"no_hallucination": 0.
|
| 129 |
}
|
| 130 |
|
| 131 |
extract_action = _last_action_of_type(history, "extract")
|
| 132 |
if extract_action is None:
|
| 133 |
breakdown["no_hallucination"] = 0.0
|
| 134 |
-
return _strict_score(0.
|
| 135 |
|
| 136 |
# --- Entity coverage ---
|
| 137 |
gt_entities: Dict[str, Any] = gt.get("entities", {})
|
|
@@ -143,7 +145,7 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
|
|
| 143 |
pred_val = pred_entities.get(key)
|
| 144 |
if pred_val is not None and _entity_matches(gt_val, pred_val):
|
| 145 |
matched += 1
|
| 146 |
-
breakdown["entity_coverage"] = round(0.
|
| 147 |
|
| 148 |
# --- Action coverage ---
|
| 149 |
gt_actions: List[str] = gt.get("required_actions", [])
|
|
@@ -160,14 +162,12 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
|
|
| 160 |
if pred_entities and gt_entities:
|
| 161 |
extra_keys = set(pred_entities.keys()) - set(gt_entities.keys())
|
| 162 |
if extra_keys:
|
| 163 |
-
penalty = min(len(extra_keys) * 0.02, 0.
|
| 164 |
-
breakdown["no_hallucination"] = round(max(0.0, 0.
|
| 165 |
-
|
| 166 |
score = _strict_score(sum(breakdown.values()))
|
| 167 |
parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
|
| 168 |
return score, breakdown, f"Task 2: {parts}"
|
| 169 |
|
| 170 |
-
|
| 171 |
def _entity_matches(gt_val: Any, pred_val: Any) -> bool:
|
| 172 |
"""Flexible entity comparison — handles strings, lists, and numbers."""
|
| 173 |
if isinstance(gt_val, list) and isinstance(pred_val, list):
|
|
@@ -194,17 +194,18 @@ def _grade_resolution(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
|
|
| 194 |
history = ep.get("action_history", [])
|
| 195 |
|
| 196 |
breakdown: Dict[str, float] = {
|
|
|
|
| 197 |
"keyword_coverage": 0.0,
|
| 198 |
"step_coverage": 0.0,
|
| 199 |
"tone_compliance": 0.0,
|
| 200 |
"length_adequate": 0.0,
|
| 201 |
-
"no_empty_steps": 0.
|
| 202 |
}
|
| 203 |
|
| 204 |
respond_action = _last_action_of_type(history, "respond")
|
| 205 |
if respond_action is None:
|
| 206 |
breakdown["no_empty_steps"] = 0.0
|
| 207 |
-
return _strict_score(0.
|
| 208 |
|
| 209 |
response_text: str = respond_action.get("response_text") or ""
|
| 210 |
resolution_steps: List[str] = respond_action.get("resolution_steps") or []
|
|
@@ -214,7 +215,7 @@ def _grade_resolution(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
|
|
| 214 |
required_keywords: List[str] = gt.get("required_keywords", [])
|
| 215 |
if required_keywords:
|
| 216 |
matched_kw = sum(1 for kw in required_keywords if kw.lower() in response_lower)
|
| 217 |
-
breakdown["keyword_coverage"] = round(0.
|
| 218 |
|
| 219 |
# --- Step coverage ---
|
| 220 |
gt_steps: List[str] = gt.get("required_resolution_steps", [])
|
|
|
|
| 15 |
from typing import Any, Dict, List, Optional, Tuple
|
| 16 |
|
| 17 |
|
| 18 |
+
SCORE_EPSILON = 0.01
|
| 19 |
|
| 20 |
|
| 21 |
def _strict_score(score: float) -> float:
|
|
|
|
| 29 |
if math.isnan(value):
|
| 30 |
value = SCORE_EPSILON
|
| 31 |
|
| 32 |
+
value = min(max(value, 0.0), 1.0)
|
| 33 |
return round(value, 4)
|
| 34 |
|
| 35 |
|
|
|
|
| 42 |
return _grade_extraction(episode_state)
|
| 43 |
elif task_id == "task3":
|
| 44 |
return _grade_resolution(episode_state)
|
| 45 |
+
return _strict_score(0.01), {}, "Unknown task"
|
| 46 |
|
| 47 |
|
| 48 |
# ---------------------------------------------------------------------------
|
|
|
|
| 78 |
history = ep.get("action_history", [])
|
| 79 |
|
| 80 |
breakdown: Dict[str, float] = {
|
| 81 |
+
"baseline": 0.01,
|
| 82 |
"category_correct": 0.0,
|
| 83 |
"priority_correct": 0.0,
|
| 84 |
"efficiency": 0.0,
|
|
|
|
| 90 |
|
| 91 |
# Category
|
| 92 |
if _normalize(classify_action.get("category")) == _normalize(gt["category"]):
|
| 93 |
+
breakdown["category_correct"] = 0.49
|
| 94 |
|
| 95 |
# Priority
|
| 96 |
if _normalize(classify_action.get("priority")) == _normalize(gt["priority"]):
|
|
|
|
| 100 |
max_steps = ep.get("max_steps", 3)
|
| 101 |
steps_used = ep.get("step_number", max_steps)
|
| 102 |
if steps_used <= 1:
|
| 103 |
+
breakdown["efficiency"] = 0.09
|
| 104 |
else:
|
| 105 |
+
breakdown["efficiency"] = round(max(0.0, 0.09 * (1 - (steps_used - 1) / max_steps)), 4)
|
| 106 |
|
| 107 |
score = _strict_score(sum(breakdown.values()))
|
| 108 |
parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
|
|
|
|
| 124 |
history = ep.get("action_history", [])
|
| 125 |
|
| 126 |
breakdown: Dict[str, float] = {
|
| 127 |
+
"baseline": 0.01,
|
| 128 |
"entity_coverage": 0.0,
|
| 129 |
"action_coverage": 0.0,
|
| 130 |
+
"no_hallucination": 0.09, # start with full marks, deduct
|
| 131 |
}
|
| 132 |
|
| 133 |
extract_action = _last_action_of_type(history, "extract")
|
| 134 |
if extract_action is None:
|
| 135 |
breakdown["no_hallucination"] = 0.0
|
| 136 |
+
return _strict_score(0.01), breakdown, "No extract action found."
|
| 137 |
|
| 138 |
# --- Entity coverage ---
|
| 139 |
gt_entities: Dict[str, Any] = gt.get("entities", {})
|
|
|
|
| 145 |
pred_val = pred_entities.get(key)
|
| 146 |
if pred_val is not None and _entity_matches(gt_val, pred_val):
|
| 147 |
matched += 1
|
| 148 |
+
breakdown["entity_coverage"] = round(0.59 * matched / len(gt_entities), 4)
|
| 149 |
|
| 150 |
# --- Action coverage ---
|
| 151 |
gt_actions: List[str] = gt.get("required_actions", [])
|
|
|
|
| 162 |
if pred_entities and gt_entities:
|
| 163 |
extra_keys = set(pred_entities.keys()) - set(gt_entities.keys())
|
| 164 |
if extra_keys:
|
| 165 |
+
penalty = min(len(extra_keys) * 0.02, 0.09)
|
| 166 |
+
breakdown["no_hallucination"] = round(max(0.0, 0.09 - penalty), 4)
|
|
|
|
| 167 |
score = _strict_score(sum(breakdown.values()))
|
| 168 |
parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
|
| 169 |
return score, breakdown, f"Task 2: {parts}"
|
| 170 |
|
|
|
|
| 171 |
def _entity_matches(gt_val: Any, pred_val: Any) -> bool:
|
| 172 |
"""Flexible entity comparison — handles strings, lists, and numbers."""
|
| 173 |
if isinstance(gt_val, list) and isinstance(pred_val, list):
|
|
|
|
| 194 |
history = ep.get("action_history", [])
|
| 195 |
|
| 196 |
breakdown: Dict[str, float] = {
|
| 197 |
+
"baseline": 0.01,
|
| 198 |
"keyword_coverage": 0.0,
|
| 199 |
"step_coverage": 0.0,
|
| 200 |
"tone_compliance": 0.0,
|
| 201 |
"length_adequate": 0.0,
|
| 202 |
+
"no_empty_steps": 0.04, # assume pass unless empty steps found
|
| 203 |
}
|
| 204 |
|
| 205 |
respond_action = _last_action_of_type(history, "respond")
|
| 206 |
if respond_action is None:
|
| 207 |
breakdown["no_empty_steps"] = 0.0
|
| 208 |
+
return _strict_score(0.01), breakdown, "No respond action found."
|
| 209 |
|
| 210 |
response_text: str = respond_action.get("response_text") or ""
|
| 211 |
resolution_steps: List[str] = respond_action.get("resolution_steps") or []
|
|
|
|
| 215 |
required_keywords: List[str] = gt.get("required_keywords", [])
|
| 216 |
if required_keywords:
|
| 217 |
matched_kw = sum(1 for kw in required_keywords if kw.lower() in response_lower)
|
| 218 |
+
breakdown["keyword_coverage"] = round(0.29 * matched_kw / len(required_keywords), 4)
|
| 219 |
|
| 220 |
# --- Step coverage ---
|
| 221 |
gt_steps: List[str] = gt.get("required_resolution_steps", [])
|
openenv.yaml
CHANGED
|
@@ -137,9 +137,10 @@ tasks:
|
|
| 137 |
(billing | technical | account | feature_request | complaint | general)
|
| 138 |
and priority (low | medium | high | critical).
|
| 139 |
scoring:
|
| 140 |
-
|
|
|
|
| 141 |
priority_correct: 0.40
|
| 142 |
-
efficiency: 0.
|
| 143 |
tickets: 5
|
| 144 |
|
| 145 |
task2:
|
|
@@ -150,9 +151,10 @@ tasks:
|
|
| 150 |
Extract structured entities (account IDs, names, amounts, dates, domains)
|
| 151 |
from the ticket body and identify the list of required actions.
|
| 152 |
scoring:
|
| 153 |
-
|
|
|
|
| 154 |
action_coverage: 0.30
|
| 155 |
-
no_hallucination: 0.
|
| 156 |
tickets: 5
|
| 157 |
|
| 158 |
task3:
|
|
@@ -164,11 +166,12 @@ tasks:
|
|
| 164 |
an ordered list of resolution steps. Scored on keyword coverage,
|
| 165 |
step completeness, tone (apology, urgency, timeline), and response length.
|
| 166 |
scoring:
|
| 167 |
-
|
|
|
|
| 168 |
step_coverage: 0.30
|
| 169 |
tone_compliance: 0.25
|
| 170 |
length_adequate: 0.10
|
| 171 |
-
no_empty_steps: 0.
|
| 172 |
tickets: 5
|
| 173 |
|
| 174 |
# -------------------------------------------------
|
|
|
|
| 137 |
(billing | technical | account | feature_request | complaint | general)
|
| 138 |
and priority (low | medium | high | critical).
|
| 139 |
scoring:
|
| 140 |
+
baseline: 0.01
|
| 141 |
+
category_correct: 0.49
|
| 142 |
priority_correct: 0.40
|
| 143 |
+
efficiency: 0.09
|
| 144 |
tickets: 5
|
| 145 |
|
| 146 |
task2:
|
|
|
|
| 151 |
Extract structured entities (account IDs, names, amounts, dates, domains)
|
| 152 |
from the ticket body and identify the list of required actions.
|
| 153 |
scoring:
|
| 154 |
+
baseline: 0.01
|
| 155 |
+
entity_coverage: 0.59
|
| 156 |
action_coverage: 0.30
|
| 157 |
+
no_hallucination: 0.09
|
| 158 |
tickets: 5
|
| 159 |
|
| 160 |
task3:
|
|
|
|
| 166 |
an ordered list of resolution steps. Scored on keyword coverage,
|
| 167 |
step completeness, tone (apology, urgency, timeline), and response length.
|
| 168 |
scoring:
|
| 169 |
+
baseline: 0.01
|
| 170 |
+
keyword_coverage: 0.29
|
| 171 |
step_coverage: 0.30
|
| 172 |
tone_compliance: 0.25
|
| 173 |
length_adequate: 0.10
|
| 174 |
+
no_empty_steps: 0.04
|
| 175 |
tickets: 5
|
| 176 |
|
| 177 |
# -------------------------------------------------
|