Update server/app.py
Browse files- server/app.py +18 -17
server/app.py
CHANGED
|
@@ -18,7 +18,6 @@ app = FastAPI()
|
|
| 18 |
API_BASE_URL = os.environ.get("API_BASE_URL", "https://api.openai.com/v1")
|
| 19 |
API_KEY = os.environ.get("API_KEY", "sk-placeholder-key")
|
| 20 |
MODEL_NAME = os.environ.get("MODEL_NAME", "meta-llama/Llama-3-70b-chat-hf")
|
| 21 |
-
|
| 22 |
client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
|
| 23 |
|
| 24 |
def _classify_with_llm(email: dict) -> np.ndarray:
|
|
@@ -30,13 +29,13 @@ def _classify_with_llm(email: dict) -> np.ndarray:
|
|
| 30 |
return np.array([2, 1, 2], dtype=np.int64) # Security | Tech | Human
|
| 31 |
elif "legal" in desc or "lawsuit" in desc or "threat" in desc:
|
| 32 |
return np.array([2, 2, 2], dtype=np.int64) # Security | Legal | Human
|
| 33 |
-
|
| 34 |
# 2. Billing/Refund Logic
|
| 35 |
elif "refund" in desc or "dispute" in desc:
|
| 36 |
return np.array([1, 2, 2], dtype=np.int64) # Billing | Legal | Human
|
| 37 |
elif "invoice" in desc or "billing" in desc or "overdue" in desc:
|
| 38 |
return np.array([1, 0, 1], dtype=np.int64) # Billing | AI | Draft
|
| 39 |
-
|
| 40 |
# 3. LLM Fallback
|
| 41 |
try:
|
| 42 |
prompt = f"Classify: {desc}. Return 3 numbers (0-2) only."
|
|
@@ -58,33 +57,34 @@ def run_task_demo(task: str) -> str:
|
|
| 58 |
env.reset(seed=42)
|
| 59 |
email_queue = list(env._queue)
|
| 60 |
lines = []
|
| 61 |
-
|
| 62 |
-
|
| 63 |
for i, email in enumerate(email_queue):
|
| 64 |
action = _classify_with_llm(email)
|
| 65 |
_, norm_reward, _, _, info = env.step(action)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
# Exact Match Check
|
| 69 |
raw = info.get("raw_reward", 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
verdict = "✅ EXACT MATCH (+1.0)" if raw >= 0.9 else "❌ MISMATCH"
|
| 71 |
-
|
| 72 |
lines.append(
|
| 73 |
f"#{i+1:02d} [{task.upper()}] {email['description'][:40]}...\n"
|
| 74 |
f" ▶ Agent: {URGENCY_LABELS[action[0]]} | {ROUTING_LABELS[action[1]]} | {RESOLUTION_LABELS[action[2]]}\n"
|
| 75 |
f" 🏆 Status: {verdict}\n" + "-"*40
|
| 76 |
)
|
| 77 |
|
| 78 |
-
# ---
|
| 79 |
total_emails = len(email_queue)
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
lines.append(f"\nTOTAL EPISODE SCORE: {final_score:.3f} / 1.000")
|
| 89 |
return "\n".join(lines)
|
| 90 |
except Exception as e:
|
|
@@ -96,6 +96,7 @@ with gr.Blocks() as demo:
|
|
| 96 |
task_dropdown = gr.Dropdown(choices=["easy", "medium", "hard"], value="easy", label="Select Difficulty")
|
| 97 |
run_btn = gr.Button("Analyze Emails")
|
| 98 |
output_box = gr.Textbox(lines=20, label="Reward Breakdown")
|
|
|
|
| 99 |
run_btn.click(fn=run_task_demo, inputs=task_dropdown, outputs=output_box)
|
| 100 |
|
| 101 |
app = gr.mount_gradio_app(app, demo, path="/")
|
|
|
|
| 18 |
API_BASE_URL = os.environ.get("API_BASE_URL", "https://api.openai.com/v1")
|
| 19 |
API_KEY = os.environ.get("API_KEY", "sk-placeholder-key")
|
| 20 |
MODEL_NAME = os.environ.get("MODEL_NAME", "meta-llama/Llama-3-70b-chat-hf")
|
|
|
|
| 21 |
client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
|
| 22 |
|
| 23 |
def _classify_with_llm(email: dict) -> np.ndarray:
|
|
|
|
| 29 |
return np.array([2, 1, 2], dtype=np.int64) # Security | Tech | Human
|
| 30 |
elif "legal" in desc or "lawsuit" in desc or "threat" in desc:
|
| 31 |
return np.array([2, 2, 2], dtype=np.int64) # Security | Legal | Human
|
| 32 |
+
|
| 33 |
# 2. Billing/Refund Logic
|
| 34 |
elif "refund" in desc or "dispute" in desc:
|
| 35 |
return np.array([1, 2, 2], dtype=np.int64) # Billing | Legal | Human
|
| 36 |
elif "invoice" in desc or "billing" in desc or "overdue" in desc:
|
| 37 |
return np.array([1, 0, 1], dtype=np.int64) # Billing | AI | Draft
|
| 38 |
+
|
| 39 |
# 3. LLM Fallback
|
| 40 |
try:
|
| 41 |
prompt = f"Classify: {desc}. Return 3 numbers (0-2) only."
|
|
|
|
| 57 |
env.reset(seed=42)
|
| 58 |
email_queue = list(env._queue)
|
| 59 |
lines = []
|
| 60 |
+
cumulative_raw_score = 0.0 # Variable renamed for clarity
|
| 61 |
+
|
| 62 |
for i, email in enumerate(email_queue):
|
| 63 |
action = _classify_with_llm(email)
|
| 64 |
_, norm_reward, _, _, info = env.step(action)
|
| 65 |
+
|
|
|
|
| 66 |
# Exact Match Check
|
| 67 |
raw = info.get("raw_reward", 0)
|
| 68 |
+
|
| 69 |
+
# FIX: Adding the raw reward (1.0 for correct) directly
|
| 70 |
+
cumulative_raw_score += raw
|
| 71 |
+
|
| 72 |
verdict = "✅ EXACT MATCH (+1.0)" if raw >= 0.9 else "❌ MISMATCH"
|
|
|
|
| 73 |
lines.append(
|
| 74 |
f"#{i+1:02d} [{task.upper()}] {email['description'][:40]}...\n"
|
| 75 |
f" ▶ Agent: {URGENCY_LABELS[action[0]]} | {ROUTING_LABELS[action[1]]} | {RESOLUTION_LABELS[action[2]]}\n"
|
| 76 |
f" 🏆 Status: {verdict}\n" + "-"*40
|
| 77 |
)
|
| 78 |
|
| 79 |
+
# --- CORRECTED SCORING FIX ---
|
| 80 |
total_emails = len(email_queue)
|
| 81 |
+
|
| 82 |
+
# Perfect Arithmetic (e.g., 3.0 / 3 = 1.0)
|
| 83 |
+
final_score = cumulative_raw_score / total_emails
|
| 84 |
+
|
| 85 |
+
# Safety fallback to keep score strictly between 0.0 and 1.0
|
| 86 |
+
final_score = min(1.0, max(0.0, final_score))
|
| 87 |
+
|
|
|
|
| 88 |
lines.append(f"\nTOTAL EPISODE SCORE: {final_score:.3f} / 1.000")
|
| 89 |
return "\n".join(lines)
|
| 90 |
except Exception as e:
|
|
|
|
| 96 |
task_dropdown = gr.Dropdown(choices=["easy", "medium", "hard"], value="easy", label="Select Difficulty")
|
| 97 |
run_btn = gr.Button("Analyze Emails")
|
| 98 |
output_box = gr.Textbox(lines=20, label="Reward Breakdown")
|
| 99 |
+
|
| 100 |
run_btn.click(fn=run_task_demo, inputs=task_dropdown, outputs=output_box)
|
| 101 |
|
| 102 |
app = gr.mount_gradio_app(app, demo, path="/")
|