File size: 6,128 Bytes
8ddfad6
9e6cefa
 
8ddfad6
 
 
9e6cefa
8ddfad6
9e6cefa
 
 
 
1057a24
 
 
 
 
 
 
 
9e6cefa
1057a24
 
 
 
 
 
 
 
9e6cefa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8ddfad6
 
 
9e6cefa
 
 
8ddfad6
9e6cefa
8ddfad6
9e6cefa
 
8ddfad6
9e6cefa
 
8ddfad6
9e6cefa
 
8ddfad6
9e6cefa
8ddfad6
9e6cefa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1057a24
 
 
9e6cefa
 
 
 
 
 
 
 
 
 
 
 
 
 
8ddfad6
9e6cefa
 
 
 
8ddfad6
9e6cefa
 
 
 
 
 
 
 
 
 
 
 
 
8ddfad6
9e6cefa
 
 
8ddfad6
9e6cefa
 
 
 
8ddfad6
 
 
 
 
9e6cefa
 
 
 
 
8ddfad6
 
ee68d68
8ddfad6
9e6cefa
 
 
 
 
 
 
 
 
 
 
1057a24
9e6cefa
1057a24
9e6cefa
 
 
1057a24
 
9e6cefa
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
"""
Grade agents on document classification.
Uses hierarchical partial credit - related categories get partial score.
"""

import numpy as np
from typing import Dict, List
from environment import DocumentClassificationEnv


# Hierarchy map - categories that are "close" get partial credit
CATEGORY_HIERARCHY = {
    "Billing": ["Billing-Dispute", "Billing-Refund"],
    "Billing-Dispute": ["Billing", "Billing-Refund"],
    "Billing-Refund": ["Billing", "Billing-Dispute"],
    "Support": ["Support-Urgent", "Support-Normal"],
    "Support-Urgent": ["Support", "Support-Normal"],
    "Support-Normal": ["Support", "Support-Urgent"],
    "Technical": ["Technical-Bug", "Technical-Feature"],
    "Technical-Bug": ["Technical", "Technical-Feature"],
    "Technical-Feature": ["Technical", "Technical-Bug"],
    "HR": ["HR-Payroll", "HR-Benefits", "HR-Complaint"],
    "HR-Payroll": ["HR", "HR-Benefits"],
    "HR-Benefits": ["HR", "HR-Payroll"],
    "HR-Complaint": ["HR"],
    "Legal": ["Legal-Contract", "Legal-Compliance"],
    "Legal-Contract": ["Legal", "Legal-Compliance"],
    "Legal-Compliance": ["Legal", "Legal-Contract"],
    "Executive": ["Executive-Strategic"],
    "Executive-Strategic": ["Executive"],
}

PARTIAL_CREDIT = 0.4  # Score for picking a related category


def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
    """
    Returns:
      1.0 if exact match
      0.4 if same parent category (e.g. Billing vs Billing-Dispute)
      0.0 otherwise
    """
    if true_label == pred_label:
        return 1.0
    related = CATEGORY_HIERARCHY.get(true_label, [])
    if pred_label in related:
        return PARTIAL_CREDIT
    return 0.0


class AgentGrader:
    """Grade an agent on a specific task difficulty"""

    def __init__(self, task_difficulty: str, num_episodes: int = 3):
        self.task_difficulty = task_difficulty
        self.num_episodes = num_episodes
        self.env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42)

    def grade(self, agent_fn) -> Dict:
        """
        Run agent through multiple episodes and return score 0.0-1.0.

        Args:
            agent_fn: callable(observation) -> action (int)

        Returns:
            dict with score and detailed metrics
        """
        all_scores = []
        all_accuracies = []
        all_partial_credits = []

        for ep in range(self.num_episodes):
            obs, _ = self.env.reset(seed=ep * 777)
            ep_true = []
            ep_pred = []
            ep_rewards = []
            terminated = False

            while not terminated:
                action = agent_fn(obs)
                obs, reward, terminated, _, info = self.env.step(action)
                ep_true.append(info.get("true_category", ""))
                pred_cat = info.get("predicted_category", "")
                ep_pred.append(pred_cat)
                ep_rewards.append(reward)

            # Exact accuracy
            exact = sum(t == p for t, p in zip(ep_true, ep_pred)) / len(ep_true)

            # Hierarchical accuracy (with partial credit)
            hier_scores = [
                hierarchical_accuracy(t, p) for t, p in zip(ep_true, ep_pred)
            ]
            hier_acc = np.mean(hier_scores)

            # Partial credit ratio
            partial = sum(1 for s in hier_scores if 0 < s < 1) / len(hier_scores)

            all_accuracies.append(exact)
            all_scores.append(hier_acc)
            all_partial_credits.append(partial)

        avg_accuracy = np.mean(all_accuracies)
        avg_hier = np.mean(all_scores)
        avg_partial = np.mean(all_partial_credits)

        # Compute final score based on difficulty
        metrics = {
            "accuracy": avg_accuracy,
            "hierarchical_accuracy": avg_hier,
            "partial_credit_rate": avg_partial,
            "average_processing_time_ms": 50,  # placeholder
        }

        score = self._compute_score(metrics)

        return {
            "score": score,
            "accuracy": avg_accuracy,
            "hierarchical_accuracy": avg_hier,
            "partial_credit_rate": avg_partial,
            "task_difficulty": self.task_difficulty,
            "num_episodes": self.num_episodes,
        }

    def _compute_score(self, metrics: Dict) -> float:
        accuracy = metrics["accuracy"]
        hier_accuracy = metrics["hierarchical_accuracy"]
        avg_time = metrics.get("average_processing_time_ms", 100)

        if self.task_difficulty == "easy":
            time_bonus = 0.1 if avg_time < 100 else 0.0
            # Easy: mostly exact accuracy
            score = 0.85 * accuracy + 0.05 * hier_accuracy + 0.1 * time_bonus

        elif self.task_difficulty == "medium":
            time_bonus = 0.0
            if avg_time < 200:
                time_bonus = 0.15
            elif avg_time < 500:
                time_bonus = 0.10
            # Medium: mix of exact + hierarchical
            score = 0.70 * accuracy + 0.15 * hier_accuracy + 0.15 * time_bonus

        else:  # hard / extreme
            time_bonus = 0.0
            if avg_time < 100:
                time_bonus = 0.25
            elif avg_time < 300:
                time_bonus = 0.15
            elif avg_time < 500:
                time_bonus = 0.05
            # Hard: hierarchical accuracy matters more (22+ categories)
            score = 0.50 * accuracy + 0.30 * hier_accuracy + 0.20 * time_bonus

        return min(1.0, max(0.0, score))


def grade_baseline(task_difficulty: str) -> float:
    """Quick helper to grade and return score"""
    from baseline_inference import load_or_train

    model = load_or_train(task_difficulty)
    grader = AgentGrader(task_difficulty)

    def agent_fn(obs):
        pred = model.predict([obs["content"]])[0]
        return int(pred)

    result = grader.grade(agent_fn)
    return result["score"]


if __name__ == "__main__":
    print("Grading baseline agent with hierarchical partial credit...\n")
    for difficulty in ["easy", "medium", "hard"]:
        score = grade_baseline(difficulty)
        print(f"  {difficulty.upper()} Score: {score:.4f}")