TanujInsane
upgraded agent with RL training
1057a24
Raw History Blame Contribute Delete
6.13 kB
"""
Grade agents on document classification.
Uses hierarchical partial credit - related categories get partial score.
"""
import numpy as np
from typing import Dict, List
from environment import DocumentClassificationEnv
# Hierarchy map - categories that are "close" get partial credit
CATEGORY_HIERARCHY = {
"Billing": ["Billing-Dispute", "Billing-Refund"],
"Billing-Dispute": ["Billing", "Billing-Refund"],
"Billing-Refund": ["Billing", "Billing-Dispute"],
"Support": ["Support-Urgent", "Support-Normal"],
"Support-Urgent": ["Support", "Support-Normal"],
"Support-Normal": ["Support", "Support-Urgent"],
"Technical": ["Technical-Bug", "Technical-Feature"],
"Technical-Bug": ["Technical", "Technical-Feature"],
"Technical-Feature": ["Technical", "Technical-Bug"],
"HR": ["HR-Payroll", "HR-Benefits", "HR-Complaint"],
"HR-Payroll": ["HR", "HR-Benefits"],
"HR-Benefits": ["HR", "HR-Payroll"],
"HR-Complaint": ["HR"],
"Legal": ["Legal-Contract", "Legal-Compliance"],
"Legal-Contract": ["Legal", "Legal-Compliance"],
"Legal-Compliance": ["Legal", "Legal-Contract"],
"Executive": ["Executive-Strategic"],
"Executive-Strategic": ["Executive"],
}
PARTIAL_CREDIT = 0.4 # Score for picking a related category
def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
"""
Returns:
1.0 if exact match
0.4 if same parent category (e.g. Billing vs Billing-Dispute)
0.0 otherwise
"""
if true_label == pred_label:
return 1.0
related = CATEGORY_HIERARCHY.get(true_label, [])
if pred_label in related:
return PARTIAL_CREDIT
return 0.0
class AgentGrader:
"""Grade an agent on a specific task difficulty"""
def __init__(self, task_difficulty: str, num_episodes: int = 3):
self.task_difficulty = task_difficulty
self.num_episodes = num_episodes
self.env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42)
def grade(self, agent_fn) -> Dict:
"""
Run agent through multiple episodes and return score 0.0-1.0.
Args:
agent_fn: callable(observation) -> action (int)
Returns:
dict with score and detailed metrics
"""
all_scores = []
all_accuracies = []
all_partial_credits = []
for ep in range(self.num_episodes):
obs, _ = self.env.reset(seed=ep * 777)
ep_true = []
ep_pred = []
ep_rewards = []
terminated = False
while not terminated:
action = agent_fn(obs)
obs, reward, terminated, _, info = self.env.step(action)
ep_true.append(info.get("true_category", ""))
pred_cat = info.get("predicted_category", "")
ep_pred.append(pred_cat)
ep_rewards.append(reward)
# Exact accuracy
exact = sum(t == p for t, p in zip(ep_true, ep_pred)) / len(ep_true)
# Hierarchical accuracy (with partial credit)
hier_scores = [
hierarchical_accuracy(t, p) for t, p in zip(ep_true, ep_pred)
]
hier_acc = np.mean(hier_scores)
# Partial credit ratio
partial = sum(1 for s in hier_scores if 0 < s < 1) / len(hier_scores)
all_accuracies.append(exact)
all_scores.append(hier_acc)
all_partial_credits.append(partial)
avg_accuracy = np.mean(all_accuracies)
avg_hier = np.mean(all_scores)
avg_partial = np.mean(all_partial_credits)
# Compute final score based on difficulty
metrics = {
"accuracy": avg_accuracy,
"hierarchical_accuracy": avg_hier,
"partial_credit_rate": avg_partial,
"average_processing_time_ms": 50, # placeholder
}
score = self._compute_score(metrics)
return {
"score": score,
"accuracy": avg_accuracy,
"hierarchical_accuracy": avg_hier,
"partial_credit_rate": avg_partial,
"task_difficulty": self.task_difficulty,
"num_episodes": self.num_episodes,
}
def _compute_score(self, metrics: Dict) -> float:
accuracy = metrics["accuracy"]
hier_accuracy = metrics["hierarchical_accuracy"]
avg_time = metrics.get("average_processing_time_ms", 100)
if self.task_difficulty == "easy":
time_bonus = 0.1 if avg_time < 100 else 0.0
# Easy: mostly exact accuracy
score = 0.85 * accuracy + 0.05 * hier_accuracy + 0.1 * time_bonus
elif self.task_difficulty == "medium":
time_bonus = 0.0
if avg_time < 200:
time_bonus = 0.15
elif avg_time < 500:
time_bonus = 0.10
# Medium: mix of exact + hierarchical
score = 0.70 * accuracy + 0.15 * hier_accuracy + 0.15 * time_bonus
else: # hard / extreme
time_bonus = 0.0
if avg_time < 100:
time_bonus = 0.25
elif avg_time < 300:
time_bonus = 0.15
elif avg_time < 500:
time_bonus = 0.05
# Hard: hierarchical accuracy matters more (22+ categories)
score = 0.50 * accuracy + 0.30 * hier_accuracy + 0.20 * time_bonus
return min(1.0, max(0.0, score))
def grade_baseline(task_difficulty: str) -> float:
"""Quick helper to grade and return score"""
from baseline_inference import load_or_train
model = load_or_train(task_difficulty)
grader = AgentGrader(task_difficulty)
def agent_fn(obs):
pred = model.predict([obs["content"]])[0]
return int(pred)
result = grader.grade(agent_fn)
return result["score"]
if __name__ == "__main__":
print("Grading baseline agent with hierarchical partial credit...\n")
for difficulty in ["easy", "medium", "hard"]:
score = grade_baseline(difficulty)
print(f" {difficulty.upper()} Score: {score:.4f}")