File size: 4,712 Bytes
3a86e09
e263f9c
e7e59dd
e263f9c
e7e59dd
e263f9c
99d66ea
e263f9c
 
 
 
 
 
 
 
 
6cfc309
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e263f9c
6cfc309
 
 
 
 
 
 
 
 
e263f9c
 
 
6cfc309
 
e263f9c
6cfc309
e263f9c
 
 
 
 
 
 
 
 
6cfc309
e263f9c
 
6cfc309
 
e263f9c
 
 
 
 
 
 
6cfc309
 
 
e263f9c
19d3a15
 
6e39a51
19d3a15
e7e59dd
6e39a51
 
 
 
 
 
6cfc309
6e39a51
6cfc309
 
6e39a51
e7e59dd
6e39a51
 
6cfc309
 
 
 
6e39a51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e7e59dd
6e39a51
 
 
 
 
 
 
 
1b64cdd
6cfc309
99d66ea
1b64cdd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
import os
import re
import traceback
from openai import OpenAI
from grading import AgentGrader
from constants import CATEGORIES

# LLM API configuration from environment variables (Grader injected)
API_BASE_URL = os.environ.get("API_BASE_URL")
API_KEY = os.environ.get("API_KEY")

# Initialize LLM client
client = None
if API_BASE_URL and API_KEY:
    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)

# Per-difficulty LLM call budget (to avoid 30-minute timeout)
# After budget is used, we fall back to a fast keyword classifier
MAX_LLM_CALLS_PER_DIFF = 15
llm_budget = {
    "easy": MAX_LLM_CALLS_PER_DIFF,
    "medium": MAX_LLM_CALLS_PER_DIFF,
    "hard": MAX_LLM_CALLS_PER_DIFF,
}

KEYWORD_HINTS = {
    "billing": 1,
    "invoice": 1,
    "payment": 1,
    "charge": 1,
    "refund": 3,
    "dispute": 2,
    "technical": 4,
    "bug": 4,
    "error": 4,
    "support": 3,
    "urgent": 3,
    "hr": 5,
    "payroll": 5,
    "legal": 6,
    "contract": 6,
    "executive": 7,
    "finance": 8,
    "marketing": 9,
    "operations": 10,
}

def keyword_fallback(content, difficulty):
    """Fast keyword-based classification when LLM budget is exhausted."""
    difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"])
    max_valid = max(difficulty_cats.keys())
    content_lower = content.lower()
    for keyword, category_idx in KEYWORD_HINTS.items():
        if keyword in content_lower and category_idx <= max_valid:
            return category_idx
    return 0  # Default to first category

def get_llm_prediction(content, difficulty):
    """Uses LLM to classify document content. Falls back to keyword classifier after budget."""
    global llm_budget

    # Use fast fallback if no client or budget exhausted
    if client is None or llm_budget.get(difficulty, 0) <= 0:
        return keyword_fallback(content, difficulty)

    # Consume one LLM call from budget
    llm_budget[difficulty] -= 1

    difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"])
    category_list = "\n".join([f"- {idx}: {name}" for idx, name in difficulty_cats.items()])

    prompt = f"""You are a document classification expert.
Classify the following customer support document into exactly one of these categories:

{category_list}

Document Content:
{content}

Respond ONLY with the category index (integer). Do not include any other text."""

    try:
        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[{"role": "user", "content": prompt}],
            temperature=0,
            max_tokens=10,
            timeout=10,
        )
        prediction = response.choices[0].message.content.strip()
        match = re.search(r'\d+', prediction)
        if match:
            return int(match.group())
    except Exception as e:
        print(f"LLM Reasoning Error: {e}")

    return keyword_fallback(content, difficulty)


def main():
    difficulties = ["easy", "medium", "hard"]
    scores = {}
    for diff in difficulties:
        try:
            # START line per difficulty
            print(
                f"[START] task={diff} env=DocumentClassification-v1 model=llm_agent",
                flush=True,
            )
            # Create grader for this difficulty
            grader = AgentGrader(task_difficulty=diff, num_episodes=1)
            # Agent function using the LLM
            def agent_fn(obs, _diff=diff):
                return get_llm_prediction(obs["content"], _diff)
            # Run grading
            result = grader.grade(agent_fn)
            score = float(result["score"])
            # Clamp to strictly between 0 and 1
            if score <= 0.0:
                score = 0.05
            elif score >= 1.0:
                score = 0.95
            scores[diff] = score
            # Approximate steps using num_documents from task_config
            steps = grader.env.task_config.get("num_documents", 0)
            # END line per difficulty
            print(
                f"[END] success=true steps={steps} score={score:.4f}",
                flush=True,
            )
        except Exception:
            # Fallback: still produce a valid END line
            scores[diff] = 0.05
            print(
                f"[END] success=false steps=0 score=0.0500",
                flush=True,
            )
            traceback.print_exc()
    # Final summary similar to the example
    print("===== FINAL SCORES =====", flush=True)
    for diff in difficulties:
        if diff in scores:
            print(f"{diff}: {scores[diff]:.4f}", flush=True)
    if scores:
        avg = sum(scores.values()) / len(scores)
        print(f"Average: {avg:.4f}", flush=True)


if __name__ == "__main__":
    main()