"""Baseline inference script — OpenAI API against all 3 tasks.""" import os, json, asyncio import httpx BASE_URL = os.environ.get("ENV_BASE_URL", "http://localhost:7860") OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "") OPENAI_BASE_URL = os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1") MODEL = os.environ.get("BASELINE_MODEL", "gpt-4o-mini") SYSTEM = """You are an expert email manager AI agent. Process emails by: 1. Correct category: urgent/normal/low/spam/internal/external 2. Accurate priority 0.0-1.0 3. Right action: triage/respond/escalate/archive/flag/skip 4. Quality responses when needed RULES: Urgent/security/legal → escalate first. Spam → archive. Premium customers with issues → respond with empathy and resolution. Always output valid JSON only.""" async def call_llm(messages): if not OPENAI_API_KEY: return json.dumps({"action_type": "archive", "email_id": "unknown"}) async with httpx.AsyncClient(timeout=60) as c: r = await c.post(f"{OPENAI_BASE_URL}/chat/completions", headers={"Authorization": f"Bearer {OPENAI_API_KEY}", "Content-Type": "application/json"}, json={"model": MODEL, "messages": messages, "temperature": 0.1, "max_tokens": 500, "response_format": {"type": "json_object"}}) r.raise_for_status() return r.json()["choices"][0]["message"]["content"] def parse_action(text, email_id): try: a = json.loads(text) if "email_id" not in a: a["email_id"] = email_id if "action_type" not in a: a["action_type"] = "archive" return a except: return {"action_type": "archive", "email_id": email_id} def priority_heuristic(email): s = email.get("subject","").lower(); sn = email.get("sender","").lower() score = 0.5 if any(w in s for w in ["critical","p0","down","breach","security","expired","gdpr"]): score=0.95 elif any(w in s for w in ["urgent","asap","today","deadline","final","board","legal","aws bill"]): score=0.80 elif any(w in s for w in ["complaint","dispute","churn","billing","renewal","nps"]): score=0.70 elif any(w in s for w in ["newsletter","lunch","cafeteria","reminder","digest","offer"]): score=0.05 if any(w in sn for w in ["pagerduty","promo","domains-r","crypto","paypa1"]): score=0.0 score += email.get("thread_depth",0)*0.03 return min(1.0,max(0.0,score)) async def run_task(task_id): print(f"\n{'='*55}\nTask: {task_id}\n{'='*55}") if not OPENAI_API_KEY: return {"task_id": task_id, "score": 0.0, "error": "OPENAI_API_KEY not set"} async with httpx.AsyncClient(base_url=BASE_URL, timeout=30) as c: obs = (await c.post(f"/reset?task_id={task_id}")).json()["observation"] print(f"Emails: {len(obs['inbox'])} | Max steps: {obs['context']['max_steps']}") processed = set(); step = 0 while step < obs['context']['max_steps'] + 2: pending = [e for e in obs['inbox'] if e['id'] not in processed] if not pending: break email = sorted(pending, key=lambda e: priority_heuristic(e), reverse=True)[0] body = obs.get('current_email_body','') if obs.get('current_email',{}) and obs['current_email']['id']==email['id'] else email.get('preview','') msg = [{"role":"system","content":SYSTEM}, {"role":"user","content":f"Task: {obs['task_description'][:120]}\n\nEmail:\nID: {email['id']}\nSubject: {email['subject']}\nFrom: {email['sender']}\nBody: {body[:400]}\n\nOutput JSON action for email_id: {email['id']}"}] llm = await call_llm(msg); action = parse_action(llm, email['id']) processed.add(email['id']) sr = await c.post(f"/step?task_id={task_id}", json=action) if sr.status_code != 200: step+=1; continue sd = sr.json(); obs = sd['observation']; step+=1 print(f" [{email['id']}] {action.get('action_type')} cat={action.get('category')} pri={action.get('priority_score')} r={sd['reward']:+.3f}") if sd['done']: break grade = (await c.post(f"/grader?task_id={task_id}")).json() print(f"\n Score: {grade['episode_score']:.4f} | {'PASSED' if grade['passed'] else 'FAILED'} | {grade['summary']}") return {"task_id": task_id, "score": grade['episode_score'], "passed": grade['passed'], "breakdown": grade.get('breakdown',{})} async def run_baseline_all(): if not OPENAI_API_KEY: return {"task_1":{"score":0.0,"error":"No API key"},"task_2":{"score":0.0,"error":"No API key"},"task_3":{"score":0.0,"error":"No API key"},"model":MODEL,"note":"Set OPENAI_API_KEY"} results = {} for tid in ["task_1","task_2","task_3"]: try: results[tid] = await run_task(tid) except Exception as e: results[tid] = {"score":0.0,"error":str(e)} scores = [r["score"] for r in results.values() if "error" not in r] results["average_score"] = round(sum(scores)/len(scores),4) if scores else 0.0 results["model"] = MODEL print(f"\n{'='*55}\nBASELINE SUMMARY\n{'='*55}") for t in ["task_1","task_2","task_3"]: print(f" {t}: {results[t].get('score',0):.4f} {'✅' if results[t].get('passed') else '❌'}") print(f" Average: {results['average_score']:.4f} | Model: {MODEL}") return results if __name__ == "__main__": asyncio.run(run_baseline_all())