Download grading.py from TanujInsane/document-classification-env: direct link, hf CLI and curl.
- Browser
- Download file 9.5 kB
-
https://huggingface.co/spaces/TanujInsane/document-classification-env/resolve/main/grading.py
- Command line
-
hf download hf://spaces/TanujInsane/document-classification-env/grading.py
-
curl -L -o grading.py https://huggingface.co/spaces/TanujInsane/document-classification-env/resolve/main/grading.py
9.5 kB
| """ | |
| Grade agents on document classification. | |
| Uses hierarchical partial credit - related categories get partial score. | |
| Handles tool actions (metadata requests, escalation) gracefully. | |
| """ | |
| import numpy as np | |
| import json | |
| from typing import Dict, List, Callable | |
| from environment import DocumentClassificationEnv | |
| from constants import CATEGORY_HIERARCHY | |
| # Hierarchy map - categories that are "close" get partial credit | |
| # CATEGORY_HIERARCHY is now in constants.py | |
| PARTIAL_CREDIT = 0.4 # Score for picking a related category | |
| TASK_ID_MAP = { | |
| "easy": 1, | |
| "medium": 2, | |
| "hard": 3, | |
| } | |
| def hierarchical_accuracy(true_label: str, pred_label: str) -> float: | |
| """ | |
| Returns: | |
| 1.0 if exact match | |
| 0.4 if same parent category (e.g. Billing vs Billing-Dispute) | |
| 0.0 otherwise (including TOOL_METADATA and ESCALATED) | |
| """ | |
| if pred_label in ("TOOL_METADATA", "ESCALATED"): | |
| return 0.0 | |
| if true_label == pred_label: | |
| return 1.0 | |
| related = CATEGORY_HIERARCHY.get(true_label, []) | |
| if pred_label in related: | |
| return PARTIAL_CREDIT | |
| return 0.0 | |
| class AgentGrader: | |
| """Grade an agent on a specific task difficulty. | |
| Handles the full action space including tool actions: | |
| - Standard classification actions (0..N-1) | |
| - Request Metadata tool (action N) | |
| - Escalate to Human (action N+1) | |
| """ | |
| def __init__(self, task_difficulty: str, num_episodes: int = 3): | |
| self.task_difficulty = task_difficulty | |
| self.num_episodes = num_episodes | |
| self.env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42) | |
| def grade(self, agent_fn: Callable) -> Dict: | |
| """ | |
| Run agent through multiple episodes and return score 0.0-1.0. | |
| Args: | |
| agent_fn: callable(observation) -> action (int) | |
| Returns: | |
| dict with score and detailed metrics | |
| """ | |
| all_scores = [] | |
| all_accuracies = [] | |
| all_partial_credits = [] | |
| all_tool_usage_rates = [] | |
| all_escalation_rates = [] | |
| all_agent_times = [] | |
| all_ep_true = [] | |
| all_ep_pred = [] | |
| import time | |
| for ep in range(self.num_episodes): | |
| obs, _ = self.env.reset(seed=ep * 777) | |
| ep_true = [] | |
| ep_pred = [] | |
| ep_rewards = [] | |
| tool_requests = 0 | |
| escalations = 0 | |
| classifications = 0 | |
| terminated = False | |
| while not terminated: | |
| t0 = time.time() | |
| action = agent_fn(obs) | |
| t1 = time.time() | |
| all_agent_times.append((t1 - t0) * 1000) | |
| obs, reward, terminated, _, info = self.env.step(action) | |
| pred_cat = info.get("predicted_category", "") | |
| true_cat = info.get("true_category", "") | |
| if pred_cat == "TOOL_METADATA": | |
| tool_requests += 1 | |
| # Don't count tool usage as a classification attempt | |
| continue | |
| elif pred_cat == "ESCALATED": | |
| escalations += 1 | |
| ep_true.append(true_cat) | |
| ep_pred.append(pred_cat) | |
| ep_rewards.append(reward) | |
| classifications += 1 | |
| else: | |
| ep_true.append(true_cat) | |
| ep_pred.append(pred_cat) | |
| ep_rewards.append(reward) | |
| classifications += 1 | |
| if len(ep_true) == 0: | |
| all_accuracies.append(0.0) | |
| all_scores.append(0.0) | |
| all_partial_credits.append(0.0) | |
| all_tool_usage_rates.append(0.0) | |
| all_escalation_rates.append(0.0) | |
| continue | |
| # Exact accuracy (only count real classifications, not escalations) | |
| real_classifications = [(t, p) for t, p in zip(ep_true, ep_pred) if p != "ESCALATED"] | |
| if real_classifications: | |
| exact = sum(t == p for t, p in real_classifications) / len(real_classifications) | |
| else: | |
| exact = 0.0 | |
| # Hierarchical accuracy (with partial credit) | |
| hier_scores = [ | |
| hierarchical_accuracy(t, p) for t, p in zip(ep_true, ep_pred) | |
| ] | |
| hier_acc = np.mean(hier_scores) if hier_scores else 0.0 | |
| # Partial credit ratio | |
| partial = sum(1 for s in hier_scores if 0 < s < 1) / len(hier_scores) if hier_scores else 0.0 | |
| # Tool usage metrics | |
| total_actions = classifications + tool_requests | |
| tool_rate = tool_requests / total_actions if total_actions > 0 else 0.0 | |
| esc_rate = escalations / classifications if classifications > 0 else 0.0 | |
| all_accuracies.append(exact) | |
| all_scores.append(hier_acc) | |
| all_partial_credits.append(partial) | |
| all_tool_usage_rates.append(tool_rate) | |
| all_escalation_rates.append(esc_rate) | |
| all_ep_true.append(ep_true) | |
| all_ep_pred.append(ep_pred) | |
| avg_accuracy = float(np.mean(all_accuracies)) | |
| avg_hier = float(np.mean(all_scores)) | |
| avg_partial = float(np.mean(all_partial_credits)) | |
| avg_tool_rate = float(np.mean(all_tool_usage_rates)) | |
| avg_esc_rate = float(np.mean(all_escalation_rates)) | |
| avg_processing_time = float(np.mean(all_agent_times)) if all_agent_times else 100.0 | |
| # Generate Confusion Matrix | |
| from collections import defaultdict | |
| cm = defaultdict(lambda: defaultdict(int)) | |
| for ep_t, ep_p in zip(all_ep_true, all_ep_pred): | |
| for t_label, p_label in zip(ep_t, ep_p): | |
| cm[t_label][p_label] += 1 | |
| cm_dict = {k: dict(v) for k, v in cm.items()} | |
| try: | |
| out_path = f"/tmp/confusion_matrix_{self.task_difficulty}.json" | |
| with open(out_path, "w") as f: | |
| json.dump(cm_dict, f, indent=2) | |
| except Exception: | |
| pass | |
| # Compute final score based on difficulty | |
| metrics = { | |
| "accuracy": avg_accuracy, | |
| "hierarchical_accuracy": avg_hier, | |
| "partial_credit_rate": avg_partial, | |
| "average_processing_time_ms": avg_processing_time, | |
| "tool_usage_rate": avg_tool_rate, | |
| "escalation_rate": avg_esc_rate, | |
| } | |
| score_raw = self._compute_score(metrics) | |
| score_clamped = float(min(1.0, max(0.0, score_raw))) | |
| import uuid | |
| task_id = TASK_ID_MAP.get(self.task_difficulty, 0) | |
| episode_id = str(uuid.uuid4()) | |
| scenario_id = f"document_classification_{self.task_difficulty}" | |
| breakdown = { | |
| "ACCURACY": avg_accuracy, | |
| "HIERARCHICAL_ACCURACY": avg_hier, | |
| "PARTIAL_CREDIT_RATE": avg_partial, | |
| "AVERAGE_PROCESSING_TIME_MS": avg_processing_time, | |
| "TOOL_USAGE_RATE": avg_tool_rate, | |
| "ESCALATION_RATE": avg_esc_rate, | |
| } | |
| return { | |
| "score": score_clamped, | |
| "task_id": task_id, | |
| "episode_id": episode_id, | |
| "scenario_id": scenario_id, | |
| "breakdown": breakdown, | |
| "grader_version": "1.0.0", | |
| "accuracy": avg_accuracy, | |
| "hierarchical_accuracy": avg_hier, | |
| "partial_credit_rate": avg_partial, | |
| "tool_usage_rate": avg_tool_rate, | |
| "escalation_rate": avg_esc_rate, | |
| "task_difficulty": self.task_difficulty, | |
| "num_episodes": self.num_episodes, | |
| } | |
| def _compute_score(self, metrics: Dict) -> float: | |
| accuracy = metrics["accuracy"] | |
| hier_accuracy = metrics["hierarchical_accuracy"] | |
| avg_time = metrics.get("average_processing_time_ms", 100) | |
| if self.task_difficulty == "easy": | |
| time_bonus = 0.1 if avg_time < 100 else 0.0 | |
| # Easy: mostly exact accuracy | |
| score = 0.85 * accuracy + 0.05 * hier_accuracy + 0.1 * time_bonus | |
| elif self.task_difficulty == "medium": | |
| time_bonus = 0.0 | |
| if avg_time < 200: | |
| time_bonus = 0.15 | |
| elif avg_time < 500: | |
| time_bonus = 0.10 | |
| # Medium: mix of exact + hierarchical | |
| score = 0.70 * accuracy + 0.15 * hier_accuracy + 0.15 * time_bonus | |
| else: # hard | |
| time_bonus = 0.0 | |
| if avg_time < 100: | |
| time_bonus = 0.25 | |
| elif avg_time < 300: | |
| time_bonus = 0.15 | |
| elif avg_time < 500: | |
| time_bonus = 0.05 | |
| # Hard: hierarchical accuracy matters more (22 categories) | |
| score = 0.50 * accuracy + 0.30 * hier_accuracy + 0.20 * time_bonus | |
| return min(1.0, max(0.0, score)) | |
| def grade_baseline(task_difficulty: str) -> float: | |
| """Quick helper to grade the TF-IDF baseline and return score.""" | |
| from baseline_inference import load_or_train | |
| model = load_or_train(task_difficulty) | |
| grader = AgentGrader(task_difficulty) | |
| def agent_fn(obs): | |
| pred = model.predict([obs["content"]])[0] | |
| return int(pred) | |
| result = grader.grade(agent_fn) | |
| return result["score"] | |
| if __name__ == "__main__": | |
| print("Grading baseline agent with hierarchical partial credit...\n") | |
| for difficulty in ["easy", "medium", "hard"]: | |
| score = grade_baseline(difficulty) | |
| print(f" {difficulty.upper()} Score: {score:.4f}") | |