import os import re import traceback from openai import OpenAI from grading import AgentGrader from constants import CATEGORIES # LLM API configuration from environment variables (Grader injected) API_BASE_URL = os.environ.get("API_BASE_URL") API_KEY = os.environ.get("API_KEY") # Initialize LLM client client = None if API_BASE_URL and API_KEY: client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY) # Per-difficulty LLM call budget (to avoid 30-minute timeout) # After budget is used, we fall back to a fast keyword classifier MAX_LLM_CALLS_PER_DIFF = 15 llm_budget = { "easy": MAX_LLM_CALLS_PER_DIFF, "medium": MAX_LLM_CALLS_PER_DIFF, "hard": MAX_LLM_CALLS_PER_DIFF, } KEYWORD_HINTS = { "billing": 1, "invoice": 1, "payment": 1, "charge": 1, "refund": 3, "dispute": 2, "technical": 4, "bug": 4, "error": 4, "support": 3, "urgent": 3, "hr": 5, "payroll": 5, "legal": 6, "contract": 6, "executive": 7, "finance": 8, "marketing": 9, "operations": 10, } def keyword_fallback(content, difficulty): """Fast keyword-based classification when LLM budget is exhausted.""" difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"]) max_valid = max(difficulty_cats.keys()) content_lower = content.lower() for keyword, category_idx in KEYWORD_HINTS.items(): if keyword in content_lower and category_idx <= max_valid: return category_idx return 0 # Default to first category def get_llm_prediction(content, difficulty): """Uses LLM to classify document content. Falls back to keyword classifier after budget.""" global llm_budget # Use fast fallback if no client or budget exhausted if client is None or llm_budget.get(difficulty, 0) <= 0: return keyword_fallback(content, difficulty) # Consume one LLM call from budget llm_budget[difficulty] -= 1 difficulty_cats = CATEGORIES.get(difficulty, CATEGORIES["easy"]) category_list = "\n".join([f"- {idx}: {name}" for idx, name in difficulty_cats.items()]) prompt = f"""You are a document classification expert. Classify the following customer support document into exactly one of these categories: {category_list} Document Content: {content} Respond ONLY with the category index (integer). Do not include any other text.""" try: response = client.chat.completions.create( model="gpt-4o", messages=[{"role": "user", "content": prompt}], temperature=0, max_tokens=10, timeout=10, ) prediction = response.choices[0].message.content.strip() match = re.search(r'\d+', prediction) if match: return int(match.group()) except Exception as e: print(f"LLM Reasoning Error: {e}") return keyword_fallback(content, difficulty) def main(): difficulties = ["easy", "medium", "hard"] scores = {} for diff in difficulties: try: # START line per difficulty print( f"[START] task={diff} env=DocumentClassification-v1 model=llm_agent", flush=True, ) # Create grader for this difficulty grader = AgentGrader(task_difficulty=diff, num_episodes=1) # Agent function using the LLM def agent_fn(obs, _diff=diff): return get_llm_prediction(obs["content"], _diff) # Run grading result = grader.grade(agent_fn) score = float(result["score"]) # Clamp to strictly between 0 and 1 if score <= 0.0: score = 0.05 elif score >= 1.0: score = 0.95 scores[diff] = score # Approximate steps using num_documents from task_config steps = grader.env.task_config.get("num_documents", 0) # END line per difficulty print( f"[END] success=true steps={steps} score={score:.4f}", flush=True, ) except Exception: # Fallback: still produce a valid END line scores[diff] = 0.05 print( f"[END] success=false steps=0 score=0.0500", flush=True, ) traceback.print_exc() # Final summary similar to the example print("===== FINAL SCORES =====", flush=True) for diff in difficulties: if diff in scores: print(f"{diff}: {scores[diff]:.4f}", flush=True) if scores: avg = sum(scores.values()) / len(scores) print(f"Average: {avg:.4f}", flush=True) if __name__ == "__main__": main()