TanujInsane commited on
Commit
9e6cefa
·
1 Parent(s): c5c9f44

ML agent: TF-IDF cosine similarity + hierarchical partial credit grading

Browse files
Files changed (1) hide show
  1. grading.py +149 -257
grading.py CHANGED
@@ -1,284 +1,176 @@
1
  """
2
- Grade agents on how well they classify documents.
3
- Basically runs them through a test set and scores them.
4
  """
5
 
6
  import numpy as np
7
- from typing import Dict, List, Tuple
8
  from environment import DocumentClassificationEnv
9
- from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix
10
- import time
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
12
 
13
  class AgentGrader:
14
- """Run an agent and grade how well it does"""
15
-
16
- def __init__(self, task_difficulty: str):
17
- """Set up the grader for a specific difficulty"""
18
  self.task_difficulty = task_difficulty
 
19
  self.env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42)
20
-
21
- def grade_agent(self, agent_policy, verbose: bool = False) -> float:
22
  """
23
- Run agent through test set and grade it.
24
-
25
  Args:
26
- agent_policy: Function that takes observation, returns action
27
- verbose: Print detailed results?
28
-
29
  Returns:
30
- score (0-1) and metrics dict
31
  """
32
- obs, _ = self.env.reset(seed=42)
33
-
34
- predictions = []
35
- ground_truths = []
36
- rewards = []
37
- processing_times = []
38
- episode_done = False
39
- step_count = 0
40
- start_time = time.time()
41
-
42
- while not episode_done and step_count < 5000:
43
- try:
44
- # Get agent's decision
45
- action = agent_policy(obs)
46
-
47
- # Clean up the action (sometimes agents return floats or bad indices)
48
- if not isinstance(action, (int, np.integer)):
49
- action = int(action)
50
- action = np.clip(int(action), 0, self.env.num_categories - 1)
51
-
52
- step_start = time.time()
53
- obs, reward, episode_done, truncated, info = self.env.step(action)
54
- processing_times.append(time.time() - step_start)
55
-
56
- rewards.append(reward)
57
- predictions.append(action)
58
- ground_truths.append(info.get("true_category"))
59
- step_count += 1
60
-
61
- except Exception as e:
62
- if verbose:
63
- print(f"Error: {e}")
64
- break
65
-
66
- total_time = time.time() - start_time
67
-
68
- # Calculate everything
69
- metrics = self._calculate_metrics(
70
- predictions,
71
- ground_truths,
72
- rewards,
73
- processing_times,
74
- total_time
75
- )
76
-
77
- # Get final score
78
- score = self._calculate_score(metrics)
79
-
80
- if verbose:
81
- self._print_results(metrics, score)
82
-
83
- return score, metrics
84
-
85
- def _calculate_metrics(self,
86
- predictions: List[int],
87
- ground_truths: List[str],
88
- rewards: List[float],
89
- processing_times: List[float],
90
- total_time: float) -> Dict:
91
- """Calculate performance metrics"""
92
-
93
- # Convert ground truth category names to indices
94
- env_categories = list(self.env.CATEGORY_MAPS[self.task_difficulty].values())
95
- ground_truth_indices = []
96
- for gt in ground_truths:
97
- try:
98
- idx = env_categories.index(gt)
99
- ground_truth_indices.append(idx)
100
- except ValueError:
101
- continue
102
-
103
- predictions = predictions[:len(ground_truth_indices)]
104
-
105
- accuracy = np.mean(np.array(predictions) == np.array(ground_truth_indices)) if len(predictions) > 0 else 0.0
106
-
107
  metrics = {
108
- "accuracy": accuracy,
109
- "total_documents": len(predictions),
110
- "correct_classifications": np.sum(np.array(predictions) == np.array(ground_truth_indices)),
111
- "average_reward": np.mean(rewards) if rewards else 0.0,
112
- "total_reward": np.sum(rewards) if rewards else 0.0,
113
- "average_processing_time_ms": np.mean(processing_times) * 1000 if processing_times else 0.0,
114
- "total_time_seconds": total_time,
115
  }
116
-
117
- # Add precision/recall if applicable
118
- if len(set(ground_truth_indices)) > 1 and len(set(predictions)) > 1:
119
- try:
120
- metrics["macro_precision"] = precision_score(
121
- ground_truth_indices, predictions,
122
- average='macro', zero_division=0
123
- )
124
- metrics["macro_recall"] = recall_score(
125
- ground_truth_indices, predictions,
126
- average='macro', zero_division=0
127
- )
128
- metrics["macro_f1"] = f1_score(
129
- ground_truth_indices, predictions,
130
- average='macro', zero_division=0
131
- )
132
- except:
133
- pass
134
-
135
- return metrics
136
-
137
- def _calculate_score(self, metrics: Dict) -> float:
138
- """
139
- Calculate final score.
140
- Easy = just accuracy
141
- Medium = 80% accuracy + 20% speed bonus
142
- Hard = 75% accuracy + 25% speed bonus
143
- """
144
  accuracy = metrics["accuracy"]
145
- avg_reward = metrics["average_reward"]
146
-
 
147
  if self.task_difficulty == "easy":
148
- # Easy: just care about being right
149
- score = accuracy
150
-
 
151
  elif self.task_difficulty == "medium":
152
- # Medium: care about both
153
  time_bonus = 0.0
154
- avg_time = metrics["average_processing_time_ms"]
155
  if avg_time < 200:
156
  time_bonus = 0.15
157
  elif avg_time < 500:
158
- time_bonus = 0.05
159
-
160
- score = 0.8 * accuracy + 0.2 * time_bonus
161
-
162
- else: # hard
163
- # Hard: need speed to compensate for lower accuracy
164
  time_bonus = 0.0
165
- avg_time = metrics["average_processing_time_ms"]
166
  if avg_time < 100:
167
  time_bonus = 0.25
168
  elif avg_time < 300:
169
- time_bonus = 0.1
170
-
171
- score = 0.75 * accuracy + 0.25 * time_bonus
172
-
173
- # Keep it in [0, 1]
174
- score = np.clip(score, 0.0, 1.0)
175
-
176
- return score
177
-
178
- def _print_results(self, metrics: Dict, score: float):
179
- """Pretty print the results"""
180
- print(f"\n{'='*60}")
181
- print(f"Task: {self.task_difficulty.upper()}")
182
- print(f"{'='*60}")
183
- print(f"Accuracy: {metrics['accuracy']:.4f}")
184
- print(f"Got it right: {metrics['correct_classifications']}/{metrics['total_documents']}")
185
- print(f"Avg Reward: {metrics['average_reward']:.4f}")
186
- print(f"Total Reward: {metrics['total_reward']:.4f}")
187
- print(f"Avg Processing Time: {metrics['average_processing_time_ms']:.2f}ms")
188
- print(f"Total Time: {metrics['total_time_seconds']:.2f}s")
189
-
190
- if "macro_precision" in metrics:
191
- print(f"Precision: {metrics['macro_precision']:.4f}")
192
- print(f"Recall: {metrics['macro_recall']:.4f}")
193
- print(f"F1: {metrics['macro_f1']:.4f}")
194
-
195
- print(f"\nFinal Score: {score:.4f}")
196
- print(f"{'='*60}\n")
197
-
198
-
199
- class BaselineAgent:
200
- """Simple baseline - just uses keyword matching"""
201
-
202
- def __init__(self, difficulty: str):
203
- self.difficulty = difficulty
204
-
205
- def decide(self, observation):
206
- """Look for keywords in the document, make a guess"""
207
- content = observation.get("content", "").lower()
208
- features = observation.get("features", [])
209
-
210
- if self.difficulty == "easy":
211
- # Easy mode - just look for obvious keywords
212
- if "billing" in content or "bill" in content or "invoice" in content:
213
- return 1 # Billing
214
- elif "support" in content or "help" in content or "issue" in content:
215
- return 2 # Support
216
- elif "technical" in content or "bug" in content or "error" in content:
217
- return 3 # Technical
218
- elif "hr" in content or "payroll" in content or "benefits" in content:
219
- return 4 # HR
220
- return 0 # Default: General
221
-
222
- elif self.difficulty == "medium":
223
- # Medium - more keywords to look for
224
- if "dispute" in content or "overcharg" in content:
225
- return 2 # Billing-Dispute
226
- elif "urgent" in content or "critical" in content or "emergency" in content:
227
- return 3 # Support (using urgent as proxy)
228
- elif "bug" in content or "error" in content or "crash" in content:
229
- return 5 # Technical-Bug
230
- elif "payroll" in content or "salary" in content or "payment" in content:
231
- return 6 # HR-Payroll
232
- elif "benefits" in content or "health" in content or "enroll" in content:
233
- return 7 # HR-Benefits
234
- elif "billing" in content or "invoice" in content:
235
- return 1 # Billing
236
- elif "technical" in content:
237
- return 4 # Technical
238
- elif "legal" in content or "contract" in content:
239
- return 8 # Legal
240
- elif "executive" in content or "manager" in content or "strategy" in content:
241
- return 9 # Executive
242
- return 0 # General
243
-
244
- else: # hard
245
- # Hard mode - need more keywords and combinations
246
- if "urgent" in content or "critical" in content:
247
- return 4 # Support-Urgent
248
- elif "dispute" in content or "overcharge" in content:
249
- return 2 # Billing-Dispute
250
- elif "refund" in content:
251
- return 3 # Billing-Refund
252
- elif "bug" in content or "crash" in content:
253
- return 7 # Technical-Bug
254
- elif "feature" in content or "request" in content:
255
- return 8 # Technical-Feature
256
- elif "payroll" in content:
257
- return 9 # HR-Payroll
258
- elif "benefits" in content:
259
- return 10 # HR-Benefits
260
- elif "complaint" in content or "complain" in content:
261
- return 11 # HR-Complaint
262
- elif "contract" in content:
263
- return 13 # Legal-Contract
264
- elif "compliance" in content:
265
- return 14 # Legal-Compliance
266
- elif "strategic" in content or "strategy" in content:
267
- return 16 # Executive-Strategic
268
- elif "billing" in content or "invoice" in content:
269
- return 1 # Billing
270
- elif "support" in content:
271
- return 5 # Support-Normal
272
- elif "technical" in content:
273
- return 6 # Technical
274
- elif "legal" in content:
275
- return 12 # Legal
276
- elif "finance" in content:
277
- return 17 # Finance
278
- elif "marketing" in content:
279
- return 18 # Marketing
280
- elif "operations" in content:
281
- return 19 # Operations
282
- elif "executive" in content:
283
- return 15 # Executive
284
- return 0 # General
 
1
  """
2
+ Grade agents on document classification.
3
+ Uses hierarchical partial credit - related categories get partial score.
4
  """
5
 
6
  import numpy as np
7
+ from typing import Dict, List
8
  from environment import DocumentClassificationEnv
9
+
10
+
11
+ # Hierarchy map - categories that are "close" get partial credit
12
+ CATEGORY_HIERARCHY = {
13
+ "Billing": ["Billing-Dispute", "Billing-Refund"],
14
+ "Billing-Dispute": ["Billing", "Billing-Refund"],
15
+ "Billing-Refund": ["Billing", "Billing-Dispute"],
16
+ "Support": ["Support-Urgent", "Support-Normal"],
17
+ "Support-Urgent": ["Support", "Support-Normal"],
18
+ "Support-Normal": ["Support", "Support-Urgent"],
19
+ "Technical": ["Technical-Bug", "Technical-Feature"],
20
+ "Technical-Bug": ["Technical", "Technical-Feature"],
21
+ "Technical-Feature": ["Technical", "Technical-Bug"],
22
+ "HR": ["HR-Payroll", "HR-Benefits", "HR-Complaint"],
23
+ "HR-Payroll": ["HR", "HR-Benefits"],
24
+ "HR-Benefits": ["HR", "HR-Payroll"],
25
+ "HR-Complaint": ["HR"],
26
+ "Legal": ["Legal-Contract", "Legal-Compliance"],
27
+ "Legal-Contract": ["Legal", "Legal-Compliance"],
28
+ "Legal-Compliance": ["Legal", "Legal-Contract"],
29
+ "Executive": ["Executive-Strategic"],
30
+ "Executive-Strategic": ["Executive"],
31
+ }
32
+
33
+ PARTIAL_CREDIT = 0.4 # Score for picking a related category
34
+
35
+
36
+ def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
37
+ """
38
+ Returns:
39
+ 1.0 if exact match
40
+ 0.4 if same parent category (e.g. Billing vs Billing-Dispute)
41
+ 0.0 otherwise
42
+ """
43
+ if true_label == pred_label:
44
+ return 1.0
45
+ related = CATEGORY_HIERARCHY.get(true_label, [])
46
+ if pred_label in related:
47
+ return PARTIAL_CREDIT
48
+ return 0.0
49
 
50
 
51
  class AgentGrader:
52
+ """Grade an agent on a specific task difficulty"""
53
+
54
+ def __init__(self, task_difficulty: str, num_episodes: int = 3):
 
55
  self.task_difficulty = task_difficulty
56
+ self.num_episodes = num_episodes
57
  self.env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42)
58
+
59
+ def grade(self, agent_fn) -> Dict:
60
  """
61
+ Run agent through multiple episodes and return score 0.0-1.0.
62
+
63
  Args:
64
+ agent_fn: callable(observation) -> action (int)
65
+
 
66
  Returns:
67
+ dict with score and detailed metrics
68
  """
69
+ all_scores = []
70
+ all_accuracies = []
71
+ all_partial_credits = []
72
+
73
+ for ep in range(self.num_episodes):
74
+ obs, _ = self.env.reset(seed=ep * 777)
75
+ ep_true = []
76
+ ep_pred = []
77
+ ep_rewards = []
78
+ terminated = False
79
+
80
+ while not terminated:
81
+ action = agent_fn(obs)
82
+ obs, reward, terminated, _, info = self.env.step(action)
83
+ ep_true.append(info.get("true_category", ""))
84
+ pred_cat = info.get("predicted_category", "")
85
+ ep_pred.append(pred_cat)
86
+ ep_rewards.append(reward)
87
+
88
+ # Exact accuracy
89
+ exact = sum(t == p for t, p in zip(ep_true, ep_pred)) / len(ep_true)
90
+
91
+ # Hierarchical accuracy (with partial credit)
92
+ hier_scores = [hierarchical_accuracy(t, p) for t, p in zip(ep_true, ep_pred)]
93
+ hier_acc = np.mean(hier_scores)
94
+
95
+ # Partial credit ratio
96
+ partial = sum(1 for s in hier_scores if 0 < s < 1) / len(hier_scores)
97
+
98
+ all_accuracies.append(exact)
99
+ all_scores.append(hier_acc)
100
+ all_partial_credits.append(partial)
101
+
102
+ avg_accuracy = np.mean(all_accuracies)
103
+ avg_hier = np.mean(all_scores)
104
+ avg_partial = np.mean(all_partial_credits)
105
+
106
+ # Compute final score based on difficulty
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107
  metrics = {
108
+ "accuracy": avg_accuracy,
109
+ "hierarchical_accuracy": avg_hier,
110
+ "partial_credit_rate": avg_partial,
111
+ "average_processing_time_ms": 50, # placeholder
 
 
 
112
  }
113
+
114
+ score = self._compute_score(metrics)
115
+
116
+ return {
117
+ "score": score,
118
+ "accuracy": avg_accuracy,
119
+ "hierarchical_accuracy": avg_hier,
120
+ "partial_credit_rate": avg_partial,
121
+ "task_difficulty": self.task_difficulty,
122
+ "num_episodes": self.num_episodes,
123
+ }
124
+
125
+ def _compute_score(self, metrics: Dict) -> float:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  accuracy = metrics["accuracy"]
127
+ hier_accuracy = metrics["hierarchical_accuracy"]
128
+ avg_time = metrics.get("average_processing_time_ms", 100)
129
+
130
  if self.task_difficulty == "easy":
131
+ time_bonus = 0.1 if avg_time < 100 else 0.0
132
+ # Easy: mostly exact accuracy
133
+ score = 0.85 * accuracy + 0.05 * hier_accuracy + 0.1 * time_bonus
134
+
135
  elif self.task_difficulty == "medium":
 
136
  time_bonus = 0.0
 
137
  if avg_time < 200:
138
  time_bonus = 0.15
139
  elif avg_time < 500:
140
+ time_bonus = 0.10
141
+ # Medium: mix of exact + hierarchical
142
+ score = 0.70 * accuracy + 0.15 * hier_accuracy + 0.15 * time_bonus
143
+
144
+ else: # hard / extreme
 
145
  time_bonus = 0.0
 
146
  if avg_time < 100:
147
  time_bonus = 0.25
148
  elif avg_time < 300:
149
+ time_bonus = 0.15
150
+ elif avg_time < 500:
151
+ time_bonus = 0.05
152
+ # Hard: hierarchical accuracy matters more (22+ categories)
153
+ score = 0.50 * accuracy + 0.30 * hier_accuracy + 0.20 * time_bonus
154
+
155
+ return min(1.0, max(0.0, score))
156
+
157
+
158
+ def grade_baseline(task_difficulty: str) -> float:
159
+ """Quick helper to grade and return score"""
160
+ from baseline_inference import TFIDFAgent
161
+
162
+ agent = TFIDFAgent(task_difficulty)
163
+ grader = AgentGrader(task_difficulty)
164
+
165
+ def agent_fn(obs):
166
+ return agent.predict(obs["content"])
167
+
168
+ result = grader.grade(agent_fn)
169
+ return result["score"]
170
+
171
+
172
+ if __name__ == "__main__":
173
+ print("Grading baseline agent with hierarchical partial credit...\n")
174
+ for difficulty in ["easy", "medium", "hard"]:
175
+ score = grade_baseline(difficulty)
176
+ print(f" {difficulty.upper()} Score: {score:.4f}")