TanujInsane commited on
Commit
9260b23
·
1 Parent(s): a2d30b7

feat: richer grader output object

Browse files
Files changed (1) hide show
  1. grading.py +28 -6
grading.py CHANGED
@@ -15,6 +15,11 @@ from constants import CATEGORY_HIERARCHY
15
  # CATEGORY_HIERARCHY is now in constants.py
16
 
17
  PARTIAL_CREDIT = 0.4 # Score for picking a related category
 
 
 
 
 
18
 
19
 
20
  def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
@@ -151,22 +156,19 @@ class AgentGrader:
151
  avg_processing_time = float(np.mean(all_agent_times)) if all_agent_times else 100.0
152
 
153
  # Generate Confusion Matrix
154
- # Create a unified list of predicted and true labels across all episodes
155
- # Exclude escalations and tools to get true confusion matching
156
  from collections import defaultdict
157
  cm = defaultdict(lambda: defaultdict(int))
158
  for ep_t, ep_p in zip(all_ep_true, all_ep_pred):
159
  for t_label, p_label in zip(ep_t, ep_p):
160
  cm[t_label][p_label] += 1
161
 
162
- # Save confusion matrix to a file as artifact evidence
163
  cm_dict = {k: dict(v) for k, v in cm.items()}
164
  try:
165
  out_path = f"/tmp/confusion_matrix_{self.task_difficulty}.json"
166
  with open(out_path, "w") as f:
167
  json.dump(cm_dict, f, indent=2)
168
  except Exception:
169
- pass # Non-critical — don't crash grading if filesystem is read-only
170
 
171
  # Compute final score based on difficulty
172
  metrics = {
@@ -178,10 +180,30 @@ class AgentGrader:
178
  "escalation_rate": avg_esc_rate,
179
  }
180
 
181
- score = self._compute_score(metrics)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
182
 
183
  return {
184
- "score": min(1.0, max(0.0, score)),
 
 
 
 
 
185
  "accuracy": avg_accuracy,
186
  "hierarchical_accuracy": avg_hier,
187
  "partial_credit_rate": avg_partial,
 
15
  # CATEGORY_HIERARCHY is now in constants.py
16
 
17
  PARTIAL_CREDIT = 0.4 # Score for picking a related category
18
+ TASK_ID_MAP = {
19
+ "easy": 1,
20
+ "medium": 2,
21
+ "hard": 3,
22
+ }
23
 
24
 
25
  def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
 
156
  avg_processing_time = float(np.mean(all_agent_times)) if all_agent_times else 100.0
157
 
158
  # Generate Confusion Matrix
 
 
159
  from collections import defaultdict
160
  cm = defaultdict(lambda: defaultdict(int))
161
  for ep_t, ep_p in zip(all_ep_true, all_ep_pred):
162
  for t_label, p_label in zip(ep_t, ep_p):
163
  cm[t_label][p_label] += 1
164
 
 
165
  cm_dict = {k: dict(v) for k, v in cm.items()}
166
  try:
167
  out_path = f"/tmp/confusion_matrix_{self.task_difficulty}.json"
168
  with open(out_path, "w") as f:
169
  json.dump(cm_dict, f, indent=2)
170
  except Exception:
171
+ pass
172
 
173
  # Compute final score based on difficulty
174
  metrics = {
 
180
  "escalation_rate": avg_esc_rate,
181
  }
182
 
183
+ score_raw = self._compute_score(metrics)
184
+ score_clamped = float(min(1.0, max(0.0, score_raw)))
185
+
186
+ import uuid
187
+ task_id = TASK_ID_MAP.get(self.task_difficulty, 0)
188
+ episode_id = str(uuid.uuid4())
189
+ scenario_id = f"document_classification_{self.task_difficulty}"
190
+
191
+ breakdown = {
192
+ "ACCURACY": avg_accuracy,
193
+ "HIERARCHICAL_ACCURACY": avg_hier,
194
+ "PARTIAL_CREDIT_RATE": avg_partial,
195
+ "AVERAGE_PROCESSING_TIME_MS": avg_processing_time,
196
+ "TOOL_USAGE_RATE": avg_tool_rate,
197
+ "ESCALATION_RATE": avg_esc_rate,
198
+ }
199
 
200
  return {
201
+ "score": score_clamped,
202
+ "task_id": task_id,
203
+ "episode_id": episode_id,
204
+ "scenario_id": scenario_id,
205
+ "breakdown": breakdown,
206
+ "grader_version": "1.0.0",
207
  "accuracy": avg_accuracy,
208
  "hierarchical_accuracy": avg_hier,
209
  "partial_credit_rate": avg_partial,