Spaces:
Sleeping
Sleeping
Download grading.py from TanujInsane/doc-class-fix: direct link, hf CLI and curl.
- Browser
- Download file 6.13 kB
-
https://huggingface.co/spaces/TanujInsane/doc-class-fix/resolve/main/grading.py
- Command line
-
hf download hf://spaces/TanujInsane/doc-class-fix/grading.py
-
curl -L -o grading.py https://huggingface.co/spaces/TanujInsane/doc-class-fix/resolve/main/grading.py
6.13 kB
| """ | |
| Grade agents on document classification. | |
| Uses hierarchical partial credit - related categories get partial score. | |
| """ | |
| import numpy as np | |
| from typing import Dict, List | |
| from environment import DocumentClassificationEnv | |
| # Hierarchy map - categories that are "close" get partial credit | |
| CATEGORY_HIERARCHY = { | |
| "Billing": ["Billing-Dispute", "Billing-Refund"], | |
| "Billing-Dispute": ["Billing", "Billing-Refund"], | |
| "Billing-Refund": ["Billing", "Billing-Dispute"], | |
| "Support": ["Support-Urgent", "Support-Normal"], | |
| "Support-Urgent": ["Support", "Support-Normal"], | |
| "Support-Normal": ["Support", "Support-Urgent"], | |
| "Technical": ["Technical-Bug", "Technical-Feature"], | |
| "Technical-Bug": ["Technical", "Technical-Feature"], | |
| "Technical-Feature": ["Technical", "Technical-Bug"], | |
| "HR": ["HR-Payroll", "HR-Benefits", "HR-Complaint"], | |
| "HR-Payroll": ["HR", "HR-Benefits"], | |
| "HR-Benefits": ["HR", "HR-Payroll"], | |
| "HR-Complaint": ["HR"], | |
| "Legal": ["Legal-Contract", "Legal-Compliance"], | |
| "Legal-Contract": ["Legal", "Legal-Compliance"], | |
| "Legal-Compliance": ["Legal", "Legal-Contract"], | |
| "Executive": ["Executive-Strategic"], | |
| "Executive-Strategic": ["Executive"], | |
| } | |
| PARTIAL_CREDIT = 0.4 # Score for picking a related category | |
| def hierarchical_accuracy(true_label: str, pred_label: str) -> float: | |
| """ | |
| Returns: | |
| 1.0 if exact match | |
| 0.4 if same parent category (e.g. Billing vs Billing-Dispute) | |
| 0.0 otherwise | |
| """ | |
| if true_label == pred_label: | |
| return 1.0 | |
| related = CATEGORY_HIERARCHY.get(true_label, []) | |
| if pred_label in related: | |
| return PARTIAL_CREDIT | |
| return 0.0 | |
| class AgentGrader: | |
| """Grade an agent on a specific task difficulty""" | |
| def __init__(self, task_difficulty: str, num_episodes: int = 3): | |
| self.task_difficulty = task_difficulty | |
| self.num_episodes = num_episodes | |
| self.env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42) | |
| def grade(self, agent_fn) -> Dict: | |
| """ | |
| Run agent through multiple episodes and return score 0.0-1.0. | |
| Args: | |
| agent_fn: callable(observation) -> action (int) | |
| Returns: | |
| dict with score and detailed metrics | |
| """ | |
| all_scores = [] | |
| all_accuracies = [] | |
| all_partial_credits = [] | |
| for ep in range(self.num_episodes): | |
| obs, _ = self.env.reset(seed=ep * 777) | |
| ep_true = [] | |
| ep_pred = [] | |
| ep_rewards = [] | |
| terminated = False | |
| while not terminated: | |
| action = agent_fn(obs) | |
| obs, reward, terminated, _, info = self.env.step(action) | |
| ep_true.append(info.get("true_category", "")) | |
| pred_cat = info.get("predicted_category", "") | |
| ep_pred.append(pred_cat) | |
| ep_rewards.append(reward) | |
| # Exact accuracy | |
| exact = sum(t == p for t, p in zip(ep_true, ep_pred)) / len(ep_true) | |
| # Hierarchical accuracy (with partial credit) | |
| hier_scores = [ | |
| hierarchical_accuracy(t, p) for t, p in zip(ep_true, ep_pred) | |
| ] | |
| hier_acc = np.mean(hier_scores) | |
| # Partial credit ratio | |
| partial = sum(1 for s in hier_scores if 0 < s < 1) / len(hier_scores) | |
| all_accuracies.append(exact) | |
| all_scores.append(hier_acc) | |
| all_partial_credits.append(partial) | |
| avg_accuracy = np.mean(all_accuracies) | |
| avg_hier = np.mean(all_scores) | |
| avg_partial = np.mean(all_partial_credits) | |
| # Compute final score based on difficulty | |
| metrics = { | |
| "accuracy": avg_accuracy, | |
| "hierarchical_accuracy": avg_hier, | |
| "partial_credit_rate": avg_partial, | |
| "average_processing_time_ms": 50, # placeholder | |
| } | |
| score = self._compute_score(metrics) | |
| return { | |
| "score": score, | |
| "accuracy": avg_accuracy, | |
| "hierarchical_accuracy": avg_hier, | |
| "partial_credit_rate": avg_partial, | |
| "task_difficulty": self.task_difficulty, | |
| "num_episodes": self.num_episodes, | |
| } | |
| def _compute_score(self, metrics: Dict) -> float: | |
| accuracy = metrics["accuracy"] | |
| hier_accuracy = metrics["hierarchical_accuracy"] | |
| avg_time = metrics.get("average_processing_time_ms", 100) | |
| if self.task_difficulty == "easy": | |
| time_bonus = 0.1 if avg_time < 100 else 0.0 | |
| # Easy: mostly exact accuracy | |
| score = 0.85 * accuracy + 0.05 * hier_accuracy + 0.1 * time_bonus | |
| elif self.task_difficulty == "medium": | |
| time_bonus = 0.0 | |
| if avg_time < 200: | |
| time_bonus = 0.15 | |
| elif avg_time < 500: | |
| time_bonus = 0.10 | |
| # Medium: mix of exact + hierarchical | |
| score = 0.70 * accuracy + 0.15 * hier_accuracy + 0.15 * time_bonus | |
| else: # hard / extreme | |
| time_bonus = 0.0 | |
| if avg_time < 100: | |
| time_bonus = 0.25 | |
| elif avg_time < 300: | |
| time_bonus = 0.15 | |
| elif avg_time < 500: | |
| time_bonus = 0.05 | |
| # Hard: hierarchical accuracy matters more (22+ categories) | |
| score = 0.50 * accuracy + 0.30 * hier_accuracy + 0.20 * time_bonus | |
| return min(1.0, max(0.0, score)) | |
| def grade_baseline(task_difficulty: str) -> float: | |
| """Quick helper to grade and return score""" | |
| from baseline_inference import load_or_train | |
| model = load_or_train(task_difficulty) | |
| grader = AgentGrader(task_difficulty) | |
| def agent_fn(obs): | |
| pred = model.predict([obs["content"]])[0] | |
| return int(pred) | |
| result = grader.grade(agent_fn) | |
| return result["score"] | |
| if __name__ == "__main__": | |
| print("Grading baseline agent with hierarchical partial credit...\n") | |
| for difficulty in ["easy", "medium", "hard"]: | |
| score = grade_baseline(difficulty) | |
| print(f" {difficulty.upper()} Score: {score:.4f}") | |