File size: 9,495 Bytes
8ddfad6
9e6cefa
 
49234f8
8ddfad6
 
 
49234f8
 
8ddfad6
49234f8
9e6cefa
 
 
49234f8
9e6cefa
 
9260b23
 
 
 
 
9e6cefa
 
 
 
 
 
 
49234f8
9e6cefa
49234f8
 
9e6cefa
 
 
 
 
 
8ddfad6
 
 
49234f8
 
 
 
 
 
 
9e6cefa
 
8ddfad6
9e6cefa
8ddfad6
9e6cefa
49234f8
8ddfad6
9e6cefa
 
8ddfad6
9e6cefa
 
8ddfad6
9e6cefa
8ddfad6
9e6cefa
 
 
49234f8
 
 
 
 
 
 
 
9e6cefa
 
 
 
 
 
49234f8
 
 
9e6cefa
 
 
49234f8
9e6cefa
49234f8
 
9e6cefa
 
49234f8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9e6cefa
 
1057a24
 
 
49234f8
9e6cefa
 
49234f8
 
 
 
 
 
9e6cefa
 
 
 
49234f8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9260b23
9e6cefa
 
8ddfad6
9e6cefa
 
 
49234f8
 
 
8ddfad6
9e6cefa
9260b23
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9e6cefa
 
9260b23
 
 
 
 
 
9e6cefa
 
 
49234f8
 
9e6cefa
 
 
 
 
8ddfad6
9e6cefa
 
 
8ddfad6
9e6cefa
 
 
 
8ddfad6
 
 
 
 
9e6cefa
 
 
 
49234f8
8ddfad6
 
ee68d68
8ddfad6
9e6cefa
 
 
49234f8
9e6cefa
 
 
 
 
 
49234f8
1057a24
9e6cefa
1057a24
9e6cefa
 
 
1057a24
 
9e6cefa
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
"""
Grade agents on document classification.
Uses hierarchical partial credit - related categories get partial score.
Handles tool actions (metadata requests, escalation) gracefully.
"""

import numpy as np
import json
from typing import Dict, List, Callable
from environment import DocumentClassificationEnv
from constants import CATEGORY_HIERARCHY


# Hierarchy map - categories that are "close" get partial credit
# CATEGORY_HIERARCHY is now in constants.py

PARTIAL_CREDIT = 0.4  # Score for picking a related category
TASK_ID_MAP = {
    "easy": 1,
    "medium": 2,
    "hard": 3,
}


def hierarchical_accuracy(true_label: str, pred_label: str) -> float:
    """
    Returns:
      1.0 if exact match
      0.4 if same parent category (e.g. Billing vs Billing-Dispute)
      0.0 otherwise (including TOOL_METADATA and ESCALATED)
    """
    if pred_label in ("TOOL_METADATA", "ESCALATED"):
        return 0.0
    if true_label == pred_label:
        return 1.0
    related = CATEGORY_HIERARCHY.get(true_label, [])
    if pred_label in related:
        return PARTIAL_CREDIT
    return 0.0


class AgentGrader:
    """Grade an agent on a specific task difficulty.

    Handles the full action space including tool actions:
    - Standard classification actions (0..N-1)
    - Request Metadata tool (action N)
    - Escalate to Human (action N+1)
    """

    def __init__(self, task_difficulty: str, num_episodes: int = 3):
        self.task_difficulty = task_difficulty
        self.num_episodes = num_episodes
        self.env = DocumentClassificationEnv(task_difficulty=task_difficulty, seed=42)

    def grade(self, agent_fn: Callable) -> Dict:
        """
        Run agent through multiple episodes and return score 0.0-1.0.

        Args:
            agent_fn: callable(observation) -> action (int)

        Returns:
            dict with score and detailed metrics
        """
        all_scores = []
        all_accuracies = []
        all_partial_credits = []
        all_tool_usage_rates = []
        all_escalation_rates = []
        all_agent_times = []
        
        all_ep_true = []
        all_ep_pred = []

        import time

        for ep in range(self.num_episodes):
            obs, _ = self.env.reset(seed=ep * 777)
            ep_true = []
            ep_pred = []
            ep_rewards = []
            tool_requests = 0
            escalations = 0
            classifications = 0
            terminated = False

            while not terminated:
                t0 = time.time()
                action = agent_fn(obs)
                t1 = time.time()
                all_agent_times.append((t1 - t0) * 1000)
                obs, reward, terminated, _, info = self.env.step(action)

                pred_cat = info.get("predicted_category", "")
                true_cat = info.get("true_category", "")

                if pred_cat == "TOOL_METADATA":
                    tool_requests += 1
                    # Don't count tool usage as a classification attempt
                    continue
                elif pred_cat == "ESCALATED":
                    escalations += 1
                    ep_true.append(true_cat)
                    ep_pred.append(pred_cat)
                    ep_rewards.append(reward)
                    classifications += 1
                else:
                    ep_true.append(true_cat)
                    ep_pred.append(pred_cat)
                    ep_rewards.append(reward)
                    classifications += 1

            if len(ep_true) == 0:
                all_accuracies.append(0.0)
                all_scores.append(0.0)
                all_partial_credits.append(0.0)
                all_tool_usage_rates.append(0.0)
                all_escalation_rates.append(0.0)
                continue

            # Exact accuracy (only count real classifications, not escalations)
            real_classifications = [(t, p) for t, p in zip(ep_true, ep_pred) if p != "ESCALATED"]
            if real_classifications:
                exact = sum(t == p for t, p in real_classifications) / len(real_classifications)
            else:
                exact = 0.0

            # Hierarchical accuracy (with partial credit)
            hier_scores = [
                hierarchical_accuracy(t, p) for t, p in zip(ep_true, ep_pred)
            ]
            hier_acc = np.mean(hier_scores) if hier_scores else 0.0

            # Partial credit ratio
            partial = sum(1 for s in hier_scores if 0 < s < 1) / len(hier_scores) if hier_scores else 0.0

            # Tool usage metrics
            total_actions = classifications + tool_requests
            tool_rate = tool_requests / total_actions if total_actions > 0 else 0.0
            esc_rate = escalations / classifications if classifications > 0 else 0.0

            all_accuracies.append(exact)
            all_scores.append(hier_acc)
            all_partial_credits.append(partial)
            all_tool_usage_rates.append(tool_rate)
            all_escalation_rates.append(esc_rate)
            all_ep_true.append(ep_true)
            all_ep_pred.append(ep_pred)

        avg_accuracy = float(np.mean(all_accuracies))
        avg_hier = float(np.mean(all_scores))
        avg_partial = float(np.mean(all_partial_credits))
        avg_tool_rate = float(np.mean(all_tool_usage_rates))
        avg_esc_rate = float(np.mean(all_escalation_rates))
        avg_processing_time = float(np.mean(all_agent_times)) if all_agent_times else 100.0

        # Generate Confusion Matrix
        from collections import defaultdict
        cm = defaultdict(lambda: defaultdict(int))
        for ep_t, ep_p in zip(all_ep_true, all_ep_pred):
            for t_label, p_label in zip(ep_t, ep_p):
                cm[t_label][p_label] += 1
        
        cm_dict = {k: dict(v) for k, v in cm.items()}
        try:
            out_path = f"/tmp/confusion_matrix_{self.task_difficulty}.json"
            with open(out_path, "w") as f:
                json.dump(cm_dict, f, indent=2)
        except Exception:
            pass

        # Compute final score based on difficulty
        metrics = {
            "accuracy": avg_accuracy,
            "hierarchical_accuracy": avg_hier,
            "partial_credit_rate": avg_partial,
            "average_processing_time_ms": avg_processing_time,
            "tool_usage_rate": avg_tool_rate,
            "escalation_rate": avg_esc_rate,
        }

        score_raw = self._compute_score(metrics)
        score_clamped = float(min(1.0, max(0.0, score_raw)))

        import uuid
        task_id = TASK_ID_MAP.get(self.task_difficulty, 0)
        episode_id = str(uuid.uuid4())
        scenario_id = f"document_classification_{self.task_difficulty}"

        breakdown = {
            "ACCURACY": avg_accuracy,
            "HIERARCHICAL_ACCURACY": avg_hier,
            "PARTIAL_CREDIT_RATE": avg_partial,
            "AVERAGE_PROCESSING_TIME_MS": avg_processing_time,
            "TOOL_USAGE_RATE": avg_tool_rate,
            "ESCALATION_RATE": avg_esc_rate,
        }

        return {
            "score": score_clamped,
            "task_id": task_id,
            "episode_id": episode_id,
            "scenario_id": scenario_id,
            "breakdown": breakdown,
            "grader_version": "1.0.0",
            "accuracy": avg_accuracy,
            "hierarchical_accuracy": avg_hier,
            "partial_credit_rate": avg_partial,
            "tool_usage_rate": avg_tool_rate,
            "escalation_rate": avg_esc_rate,
            "task_difficulty": self.task_difficulty,
            "num_episodes": self.num_episodes,
        }

    def _compute_score(self, metrics: Dict) -> float:
        accuracy = metrics["accuracy"]
        hier_accuracy = metrics["hierarchical_accuracy"]
        avg_time = metrics.get("average_processing_time_ms", 100)

        if self.task_difficulty == "easy":
            time_bonus = 0.1 if avg_time < 100 else 0.0
            # Easy: mostly exact accuracy
            score = 0.85 * accuracy + 0.05 * hier_accuracy + 0.1 * time_bonus

        elif self.task_difficulty == "medium":
            time_bonus = 0.0
            if avg_time < 200:
                time_bonus = 0.15
            elif avg_time < 500:
                time_bonus = 0.10
            # Medium: mix of exact + hierarchical
            score = 0.70 * accuracy + 0.15 * hier_accuracy + 0.15 * time_bonus

        else:  # hard
            time_bonus = 0.0
            if avg_time < 100:
                time_bonus = 0.25
            elif avg_time < 300:
                time_bonus = 0.15
            elif avg_time < 500:
                time_bonus = 0.05
            # Hard: hierarchical accuracy matters more (22 categories)
            score = 0.50 * accuracy + 0.30 * hier_accuracy + 0.20 * time_bonus

        return min(1.0, max(0.0, score))


def grade_baseline(task_difficulty: str) -> float:
    """Quick helper to grade the TF-IDF baseline and return score."""
    from baseline_inference import load_or_train

    model = load_or_train(task_difficulty)
    grader = AgentGrader(task_difficulty)

    def agent_fn(obs):
        pred = model.predict([obs["content"]])[0]
        return int(pred)

    result = grader.grade(agent_fn)
    return result["score"]


if __name__ == "__main__":
    print("Grading baseline agent with hierarchical partial credit...\n")
    for difficulty in ["easy", "medium", "hard"]:
        score = grade_baseline(difficulty)
        print(f"  {difficulty.upper()} Score: {score:.4f}")