TanujInsane
feat: run easy/medium/hard sequentially with START/END logs
6e39a51
Raw History Blame Contribute Delete
4.71 kB
import os
import re
import traceback
from openai import OpenAI
from grading import AgentGrader
from constants import CATEGORIES
# LLM API configuration from environment variables (Grader injected)
API_BASE_URL = os.environ.get("API_BASE_URL")
API_KEY = os.environ.get("API_KEY")
# Initialize LLM client
client = None
if API_BASE_URL and API_KEY:
client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
# Per-difficulty LLM call budget (to avoid 30-minute timeout)
# After budget is used, we fall back to a fast keyword classifier
MAX_LLM_CALLS_PER_DIFF = 15
llm_budget = {
"easy": MAX_LLM_CALLS_PER_DIFF,
"medium": MAX_LLM_CALLS_PER_DIFF,
"hard": MAX_LLM_CALLS_PER_DIFF,
}
KEYWORD_HINTS = {
"billing": 1,
"invoice": 1,
"payment": 1,
"charge": 1,
"refund": 3,
"dispute": 2,
"technical": 4,
"bug": 4,
"error": 4,
"support": 3,
"urgent": 3,
"hr": 5,
"payroll": 5,
"legal": 6,
"contract": 6,
"executive": 7,
"finance": 8,
"marketing": 9,
"operations": 10,
}
def keyword_fallback(content, difficulty):
"""Fast keyword-based classification when LLM budget is exhausted."""
difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"])
max_valid = max(difficulty_cats.keys())
content_lower = content.lower()
for keyword, category_idx in KEYWORD_HINTS.items():
if keyword in content_lower and category_idx <= max_valid:
return category_idx
return 0 # Default to first category
def get_llm_prediction(content, difficulty):
"""Uses LLM to classify document content. Falls back to keyword classifier after budget."""
global llm_budget
# Use fast fallback if no client or budget exhausted
if client is None or llm_budget.get(difficulty, 0) <= 0:
return keyword_fallback(content, difficulty)
# Consume one LLM call from budget
llm_budget[difficulty] -= 1
difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"])
category_list = "\n".join([f"- {idx}: {name}" for idx, name in difficulty_cats.items()])
prompt = f"""You are a document classification expert.
Classify the following customer support document into exactly one of these categories:
{category_list}
Document Content:
{content}
Respond ONLY with the category index (integer). Do not include any other text."""
try:
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": prompt}],
temperature=0,
max_tokens=10,
timeout=10,
)
prediction = response.choices[0].message.content.strip()
match = re.search(r'\d+', prediction)
if match:
return int(match.group())
except Exception as e:
print(f"LLM Reasoning Error: {e}")
return keyword_fallback(content, difficulty)
def main():
difficulties = ["easy", "medium", "hard"]
scores = {}
for diff in difficulties:
try:
# START line per difficulty
print(
f"[START] task={diff} env=DocumentClassification-v1 model=llm_agent",
flush=True,
)
# Create grader for this difficulty
grader = AgentGrader(task_difficulty=diff, num_episodes=1)
# Agent function using the LLM
def agent_fn(obs, _diff=diff):
return get_llm_prediction(obs["content"], _diff)
# Run grading
result = grader.grade(agent_fn)
score = float(result["score"])
# Clamp to strictly between 0 and 1
if score <= 0.0:
score = 0.05
elif score >= 1.0:
score = 0.95
scores[diff] = score
# Approximate steps using num_documents from task_config
steps = grader.env.task_config.get("num_documents", 0)
# END line per difficulty
print(
f"[END] success=true steps={steps} score={score:.4f}",
flush=True,
)
except Exception:
# Fallback: still produce a valid END line
scores[diff] = 0.05
print(
f"[END] success=false steps=0 score=0.0500",
flush=True,
)
traceback.print_exc()
# Final summary similar to the example
print("===== FINAL SCORES =====", flush=True)
for diff in difficulties:
if diff in scores:
print(f"{diff}: {scores[diff]:.4f}", flush=True)
if scores:
avg = sum(scores.values()) / len(scores)
print(f"Average: {avg:.4f}", flush=True)
if __name__ == "__main__":
main()