exploring-solver commited on
Commit
a3c6db2
·
1 Parent(s): e093b15

scoring components rebuilt 0-1

Browse files
Files changed (2) hide show
  1. graders.py +17 -16
  2. openenv.yaml +9 -6
graders.py CHANGED
@@ -15,7 +15,7 @@ import math
15
  from typing import Any, Dict, List, Optional, Tuple
16
 
17
 
18
- SCORE_EPSILON = 0.0001
19
 
20
 
21
  def _strict_score(score: float) -> float:
@@ -29,7 +29,7 @@ def _strict_score(score: float) -> float:
29
  if math.isnan(value):
30
  value = SCORE_EPSILON
31
 
32
- value = min(max(value, SCORE_EPSILON), 1.0 - SCORE_EPSILON)
33
  return round(value, 4)
34
 
35
 
@@ -42,7 +42,7 @@ def grade_task(
42
  return _grade_extraction(episode_state)
43
  elif task_id == "task3":
44
  return _grade_resolution(episode_state)
45
- return _strict_score(0.0), {}, "Unknown task"
46
 
47
 
48
  # ---------------------------------------------------------------------------
@@ -78,6 +78,7 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
78
  history = ep.get("action_history", [])
79
 
80
  breakdown: Dict[str, float] = {
 
81
  "category_correct": 0.0,
82
  "priority_correct": 0.0,
83
  "efficiency": 0.0,
@@ -89,7 +90,7 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
89
 
90
  # Category
91
  if _normalize(classify_action.get("category")) == _normalize(gt["category"]):
92
- breakdown["category_correct"] = 0.50
93
 
94
  # Priority
95
  if _normalize(classify_action.get("priority")) == _normalize(gt["priority"]):
@@ -99,9 +100,9 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
99
  max_steps = ep.get("max_steps", 3)
100
  steps_used = ep.get("step_number", max_steps)
101
  if steps_used <= 1:
102
- breakdown["efficiency"] = 0.10
103
  else:
104
- breakdown["efficiency"] = round(max(0.0, 0.10 * (1 - (steps_used - 1) / max_steps)), 4)
105
 
106
  score = _strict_score(sum(breakdown.values()))
107
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
@@ -123,15 +124,16 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
123
  history = ep.get("action_history", [])
124
 
125
  breakdown: Dict[str, float] = {
 
126
  "entity_coverage": 0.0,
127
  "action_coverage": 0.0,
128
- "no_hallucination": 0.10, # start with full marks, deduct
129
  }
130
 
131
  extract_action = _last_action_of_type(history, "extract")
132
  if extract_action is None:
133
  breakdown["no_hallucination"] = 0.0
134
- return _strict_score(0.0), breakdown, "No extract action found."
135
 
136
  # --- Entity coverage ---
137
  gt_entities: Dict[str, Any] = gt.get("entities", {})
@@ -143,7 +145,7 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
143
  pred_val = pred_entities.get(key)
144
  if pred_val is not None and _entity_matches(gt_val, pred_val):
145
  matched += 1
146
- breakdown["entity_coverage"] = round(0.60 * matched / len(gt_entities), 4)
147
 
148
  # --- Action coverage ---
149
  gt_actions: List[str] = gt.get("required_actions", [])
@@ -160,14 +162,12 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
160
  if pred_entities and gt_entities:
161
  extra_keys = set(pred_entities.keys()) - set(gt_entities.keys())
162
  if extra_keys:
163
- penalty = min(len(extra_keys) * 0.02, 0.10)
164
- breakdown["no_hallucination"] = round(max(0.0, 0.10 - penalty), 4)
165
-
166
  score = _strict_score(sum(breakdown.values()))
167
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
168
  return score, breakdown, f"Task 2: {parts}"
169
 
170
-
171
  def _entity_matches(gt_val: Any, pred_val: Any) -> bool:
172
  """Flexible entity comparison — handles strings, lists, and numbers."""
173
  if isinstance(gt_val, list) and isinstance(pred_val, list):
@@ -194,17 +194,18 @@ def _grade_resolution(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
194
  history = ep.get("action_history", [])
195
 
196
  breakdown: Dict[str, float] = {
 
197
  "keyword_coverage": 0.0,
198
  "step_coverage": 0.0,
199
  "tone_compliance": 0.0,
200
  "length_adequate": 0.0,
201
- "no_empty_steps": 0.05, # assume pass unless empty steps found
202
  }
203
 
204
  respond_action = _last_action_of_type(history, "respond")
205
  if respond_action is None:
206
  breakdown["no_empty_steps"] = 0.0
207
- return _strict_score(0.0), breakdown, "No respond action found."
208
 
209
  response_text: str = respond_action.get("response_text") or ""
210
  resolution_steps: List[str] = respond_action.get("resolution_steps") or []
@@ -214,7 +215,7 @@ def _grade_resolution(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
214
  required_keywords: List[str] = gt.get("required_keywords", [])
215
  if required_keywords:
216
  matched_kw = sum(1 for kw in required_keywords if kw.lower() in response_lower)
217
- breakdown["keyword_coverage"] = round(0.30 * matched_kw / len(required_keywords), 4)
218
 
219
  # --- Step coverage ---
220
  gt_steps: List[str] = gt.get("required_resolution_steps", [])
 
15
  from typing import Any, Dict, List, Optional, Tuple
16
 
17
 
18
+ SCORE_EPSILON = 0.01
19
 
20
 
21
  def _strict_score(score: float) -> float:
 
29
  if math.isnan(value):
30
  value = SCORE_EPSILON
31
 
32
+ value = min(max(value, 0.0), 1.0)
33
  return round(value, 4)
34
 
35
 
 
42
  return _grade_extraction(episode_state)
43
  elif task_id == "task3":
44
  return _grade_resolution(episode_state)
45
+ return _strict_score(0.01), {}, "Unknown task"
46
 
47
 
48
  # ---------------------------------------------------------------------------
 
78
  history = ep.get("action_history", [])
79
 
80
  breakdown: Dict[str, float] = {
81
+ "baseline": 0.01,
82
  "category_correct": 0.0,
83
  "priority_correct": 0.0,
84
  "efficiency": 0.0,
 
90
 
91
  # Category
92
  if _normalize(classify_action.get("category")) == _normalize(gt["category"]):
93
+ breakdown["category_correct"] = 0.49
94
 
95
  # Priority
96
  if _normalize(classify_action.get("priority")) == _normalize(gt["priority"]):
 
100
  max_steps = ep.get("max_steps", 3)
101
  steps_used = ep.get("step_number", max_steps)
102
  if steps_used <= 1:
103
+ breakdown["efficiency"] = 0.09
104
  else:
105
+ breakdown["efficiency"] = round(max(0.0, 0.09 * (1 - (steps_used - 1) / max_steps)), 4)
106
 
107
  score = _strict_score(sum(breakdown.values()))
108
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
 
124
  history = ep.get("action_history", [])
125
 
126
  breakdown: Dict[str, float] = {
127
+ "baseline": 0.01,
128
  "entity_coverage": 0.0,
129
  "action_coverage": 0.0,
130
+ "no_hallucination": 0.09, # start with full marks, deduct
131
  }
132
 
133
  extract_action = _last_action_of_type(history, "extract")
134
  if extract_action is None:
135
  breakdown["no_hallucination"] = 0.0
136
+ return _strict_score(0.01), breakdown, "No extract action found."
137
 
138
  # --- Entity coverage ---
139
  gt_entities: Dict[str, Any] = gt.get("entities", {})
 
145
  pred_val = pred_entities.get(key)
146
  if pred_val is not None and _entity_matches(gt_val, pred_val):
147
  matched += 1
148
+ breakdown["entity_coverage"] = round(0.59 * matched / len(gt_entities), 4)
149
 
150
  # --- Action coverage ---
151
  gt_actions: List[str] = gt.get("required_actions", [])
 
162
  if pred_entities and gt_entities:
163
  extra_keys = set(pred_entities.keys()) - set(gt_entities.keys())
164
  if extra_keys:
165
+ penalty = min(len(extra_keys) * 0.02, 0.09)
166
+ breakdown["no_hallucination"] = round(max(0.0, 0.09 - penalty), 4)
 
167
  score = _strict_score(sum(breakdown.values()))
168
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
169
  return score, breakdown, f"Task 2: {parts}"
170
 
 
171
  def _entity_matches(gt_val: Any, pred_val: Any) -> bool:
172
  """Flexible entity comparison — handles strings, lists, and numbers."""
173
  if isinstance(gt_val, list) and isinstance(pred_val, list):
 
194
  history = ep.get("action_history", [])
195
 
196
  breakdown: Dict[str, float] = {
197
+ "baseline": 0.01,
198
  "keyword_coverage": 0.0,
199
  "step_coverage": 0.0,
200
  "tone_compliance": 0.0,
201
  "length_adequate": 0.0,
202
+ "no_empty_steps": 0.04, # assume pass unless empty steps found
203
  }
204
 
205
  respond_action = _last_action_of_type(history, "respond")
206
  if respond_action is None:
207
  breakdown["no_empty_steps"] = 0.0
208
+ return _strict_score(0.01), breakdown, "No respond action found."
209
 
210
  response_text: str = respond_action.get("response_text") or ""
211
  resolution_steps: List[str] = respond_action.get("resolution_steps") or []
 
215
  required_keywords: List[str] = gt.get("required_keywords", [])
216
  if required_keywords:
217
  matched_kw = sum(1 for kw in required_keywords if kw.lower() in response_lower)
218
+ breakdown["keyword_coverage"] = round(0.29 * matched_kw / len(required_keywords), 4)
219
 
220
  # --- Step coverage ---
221
  gt_steps: List[str] = gt.get("required_resolution_steps", [])
openenv.yaml CHANGED
@@ -137,9 +137,10 @@ tasks:
137
  (billing | technical | account | feature_request | complaint | general)
138
  and priority (low | medium | high | critical).
139
  scoring:
140
- category_correct: 0.50
 
141
  priority_correct: 0.40
142
- efficiency: 0.10
143
  tickets: 5
144
 
145
  task2:
@@ -150,9 +151,10 @@ tasks:
150
  Extract structured entities (account IDs, names, amounts, dates, domains)
151
  from the ticket body and identify the list of required actions.
152
  scoring:
153
- entity_coverage: 0.60
 
154
  action_coverage: 0.30
155
- no_hallucination: 0.10
156
  tickets: 5
157
 
158
  task3:
@@ -164,11 +166,12 @@ tasks:
164
  an ordered list of resolution steps. Scored on keyword coverage,
165
  step completeness, tone (apology, urgency, timeline), and response length.
166
  scoring:
167
- keyword_coverage: 0.30
 
168
  step_coverage: 0.30
169
  tone_compliance: 0.25
170
  length_adequate: 0.10
171
- no_empty_steps: 0.05
172
  tickets: 5
173
 
174
  # -------------------------------------------------
 
137
  (billing | technical | account | feature_request | complaint | general)
138
  and priority (low | medium | high | critical).
139
  scoring:
140
+ baseline: 0.01
141
+ category_correct: 0.49
142
  priority_correct: 0.40
143
+ efficiency: 0.09
144
  tickets: 5
145
 
146
  task2:
 
151
  Extract structured entities (account IDs, names, amounts, dates, domains)
152
  from the ticket body and identify the list of required actions.
153
  scoring:
154
+ baseline: 0.01
155
+ entity_coverage: 0.59
156
  action_coverage: 0.30
157
+ no_hallucination: 0.09
158
  tickets: 5
159
 
160
  task3:
 
166
  an ordered list of resolution steps. Scored on keyword coverage,
167
  step completeness, tone (apology, urgency, timeline), and response length.
168
  scoring:
169
+ baseline: 0.01
170
+ keyword_coverage: 0.29
171
  step_coverage: 0.30
172
  tone_compliance: 0.25
173
  length_adequate: 0.10
174
+ no_empty_steps: 0.04
175
  tickets: 5
176
 
177
  # -------------------------------------------------