TanujInsane commited on
Commit ·
9260b23
1
Parent(s): a2d30b7
feat: richer grader output object
Browse files- grading.py +28 -6
grading.py
CHANGED
|
@@ -15,6 +15,11 @@ from constants import CATEGORY_HIERARCHY
|
|
| 15 |
# CATEGORY_HIERARCHY is now in constants.py
|
| 16 |
|
| 17 |
PARTIAL_CREDIT = 0.4 # Score for picking a related category
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
|
|
@@ -151,22 +156,19 @@ class AgentGrader:
|
|
| 151 |
avg_processing_time = float(np.mean(all_agent_times)) if all_agent_times else 100.0
|
| 152 |
|
| 153 |
# Generate Confusion Matrix
|
| 154 |
-
# Create a unified list of predicted and true labels across all episodes
|
| 155 |
-
# Exclude escalations and tools to get true confusion matching
|
| 156 |
from collections import defaultdict
|
| 157 |
cm = defaultdict(lambda: defaultdict(int))
|
| 158 |
for ep_t, ep_p in zip(all_ep_true, all_ep_pred):
|
| 159 |
for t_label, p_label in zip(ep_t, ep_p):
|
| 160 |
cm[t_label][p_label] += 1
|
| 161 |
|
| 162 |
-
# Save confusion matrix to a file as artifact evidence
|
| 163 |
cm_dict = {k: dict(v) for k, v in cm.items()}
|
| 164 |
try:
|
| 165 |
out_path = f"/tmp/confusion_matrix_{self.task_difficulty}.json"
|
| 166 |
with open(out_path, "w") as f:
|
| 167 |
json.dump(cm_dict, f, indent=2)
|
| 168 |
except Exception:
|
| 169 |
-
pass
|
| 170 |
|
| 171 |
# Compute final score based on difficulty
|
| 172 |
metrics = {
|
|
@@ -178,10 +180,30 @@ class AgentGrader:
|
|
| 178 |
"escalation_rate": avg_esc_rate,
|
| 179 |
}
|
| 180 |
|
| 181 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
|
| 183 |
return {
|
| 184 |
-
"score":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
"accuracy": avg_accuracy,
|
| 186 |
"hierarchical_accuracy": avg_hier,
|
| 187 |
"partial_credit_rate": avg_partial,
|
|
|
|
| 15 |
# CATEGORY_HIERARCHY is now in constants.py
|
| 16 |
|
| 17 |
PARTIAL_CREDIT = 0.4 # Score for picking a related category
|
| 18 |
+
TASK_ID_MAP = {
|
| 19 |
+
"easy": 1,
|
| 20 |
+
"medium": 2,
|
| 21 |
+
"hard": 3,
|
| 22 |
+
}
|
| 23 |
|
| 24 |
|
| 25 |
def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
|
|
|
|
| 156 |
avg_processing_time = float(np.mean(all_agent_times)) if all_agent_times else 100.0
|
| 157 |
|
| 158 |
# Generate Confusion Matrix
|
|
|
|
|
|
|
| 159 |
from collections import defaultdict
|
| 160 |
cm = defaultdict(lambda: defaultdict(int))
|
| 161 |
for ep_t, ep_p in zip(all_ep_true, all_ep_pred):
|
| 162 |
for t_label, p_label in zip(ep_t, ep_p):
|
| 163 |
cm[t_label][p_label] += 1
|
| 164 |
|
|
|
|
| 165 |
cm_dict = {k: dict(v) for k, v in cm.items()}
|
| 166 |
try:
|
| 167 |
out_path = f"/tmp/confusion_matrix_{self.task_difficulty}.json"
|
| 168 |
with open(out_path, "w") as f:
|
| 169 |
json.dump(cm_dict, f, indent=2)
|
| 170 |
except Exception:
|
| 171 |
+
pass
|
| 172 |
|
| 173 |
# Compute final score based on difficulty
|
| 174 |
metrics = {
|
|
|
|
| 180 |
"escalation_rate": avg_esc_rate,
|
| 181 |
}
|
| 182 |
|
| 183 |
+
score_raw = self._compute_score(metrics)
|
| 184 |
+
score_clamped = float(min(1.0, max(0.0, score_raw)))
|
| 185 |
+
|
| 186 |
+
import uuid
|
| 187 |
+
task_id = TASK_ID_MAP.get(self.task_difficulty, 0)
|
| 188 |
+
episode_id = str(uuid.uuid4())
|
| 189 |
+
scenario_id = f"document_classification_{self.task_difficulty}"
|
| 190 |
+
|
| 191 |
+
breakdown = {
|
| 192 |
+
"ACCURACY": avg_accuracy,
|
| 193 |
+
"HIERARCHICAL_ACCURACY": avg_hier,
|
| 194 |
+
"PARTIAL_CREDIT_RATE": avg_partial,
|
| 195 |
+
"AVERAGE_PROCESSING_TIME_MS": avg_processing_time,
|
| 196 |
+
"TOOL_USAGE_RATE": avg_tool_rate,
|
| 197 |
+
"ESCALATION_RATE": avg_esc_rate,
|
| 198 |
+
}
|
| 199 |
|
| 200 |
return {
|
| 201 |
+
"score": score_clamped,
|
| 202 |
+
"task_id": task_id,
|
| 203 |
+
"episode_id": episode_id,
|
| 204 |
+
"scenario_id": scenario_id,
|
| 205 |
+
"breakdown": breakdown,
|
| 206 |
+
"grader_version": "1.0.0",
|
| 207 |
"accuracy": avg_accuracy,
|
| 208 |
"hierarchical_accuracy": avg_hier,
|
| 209 |
"partial_credit_rate": avg_partial,
|