File size: 4,712 Bytes
3a86e09 e263f9c e7e59dd e263f9c e7e59dd e263f9c 99d66ea e263f9c 6cfc309 e263f9c 6cfc309 e263f9c 6cfc309 e263f9c 6cfc309 e263f9c 6cfc309 e263f9c 6cfc309 e263f9c 6cfc309 e263f9c 19d3a15 6e39a51 19d3a15 e7e59dd 6e39a51 6cfc309 6e39a51 6cfc309 6e39a51 e7e59dd 6e39a51 6cfc309 6e39a51 e7e59dd 6e39a51 1b64cdd 6cfc309 99d66ea 1b64cdd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | import os
import re
import traceback
from openai import OpenAI
from grading import AgentGrader
from constants import CATEGORIES
# LLM API configuration from environment variables (Grader injected)
API_BASE_URL = os.environ.get("API_BASE_URL")
API_KEY = os.environ.get("API_KEY")
# Initialize LLM client
client = None
if API_BASE_URL and API_KEY:
client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
# Per-difficulty LLM call budget (to avoid 30-minute timeout)
# After budget is used, we fall back to a fast keyword classifier
MAX_LLM_CALLS_PER_DIFF = 15
llm_budget = {
"easy": MAX_LLM_CALLS_PER_DIFF,
"medium": MAX_LLM_CALLS_PER_DIFF,
"hard": MAX_LLM_CALLS_PER_DIFF,
}
KEYWORD_HINTS = {
"billing": 1,
"invoice": 1,
"payment": 1,
"charge": 1,
"refund": 3,
"dispute": 2,
"technical": 4,
"bug": 4,
"error": 4,
"support": 3,
"urgent": 3,
"hr": 5,
"payroll": 5,
"legal": 6,
"contract": 6,
"executive": 7,
"finance": 8,
"marketing": 9,
"operations": 10,
}
def keyword_fallback(content, difficulty):
"""Fast keyword-based classification when LLM budget is exhausted."""
difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"])
max_valid = max(difficulty_cats.keys())
content_lower = content.lower()
for keyword, category_idx in KEYWORD_HINTS.items():
if keyword in content_lower and category_idx <= max_valid:
return category_idx
return 0 # Default to first category
def get_llm_prediction(content, difficulty):
"""Uses LLM to classify document content. Falls back to keyword classifier after budget."""
global llm_budget
# Use fast fallback if no client or budget exhausted
if client is None or llm_budget.get(difficulty, 0) <= 0:
return keyword_fallback(content, difficulty)
# Consume one LLM call from budget
llm_budget[difficulty] -= 1
difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"])
category_list = "\n".join([f"- {idx}: {name}" for idx, name in difficulty_cats.items()])
prompt = f"""You are a document classification expert.
Classify the following customer support document into exactly one of these categories:
{category_list}
Document Content:
{content}
Respond ONLY with the category index (integer). Do not include any other text."""
try:
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": prompt}],
temperature=0,
max_tokens=10,
timeout=10,
)
prediction = response.choices[0].message.content.strip()
match = re.search(r'\d+', prediction)
if match:
return int(match.group())
except Exception as e:
print(f"LLM Reasoning Error: {e}")
return keyword_fallback(content, difficulty)
def main():
difficulties = ["easy", "medium", "hard"]
scores = {}
for diff in difficulties:
try:
# START line per difficulty
print(
f"[START] task={diff} env=DocumentClassification-v1 model=llm_agent",
flush=True,
)
# Create grader for this difficulty
grader = AgentGrader(task_difficulty=diff, num_episodes=1)
# Agent function using the LLM
def agent_fn(obs, _diff=diff):
return get_llm_prediction(obs["content"], _diff)
# Run grading
result = grader.grade(agent_fn)
score = float(result["score"])
# Clamp to strictly between 0 and 1
if score <= 0.0:
score = 0.05
elif score >= 1.0:
score = 0.95
scores[diff] = score
# Approximate steps using num_documents from task_config
steps = grader.env.task_config.get("num_documents", 0)
# END line per difficulty
print(
f"[END] success=true steps={steps} score={score:.4f}",
flush=True,
)
except Exception:
# Fallback: still produce a valid END line
scores[diff] = 0.05
print(
f"[END] success=false steps=0 score=0.0500",
flush=True,
)
traceback.print_exc()
# Final summary similar to the example
print("===== FINAL SCORES =====", flush=True)
for diff in difficulties:
if diff in scores:
print(f"{diff}: {scores[diff]:.4f}", flush=True)
if scores:
avg = sum(scores.values()) / len(scores)
print(f"Average: {avg:.4f}", flush=True)
if __name__ == "__main__":
main()
|