TanujInsane commited on
Commit ยท
a739312
1
Parent(s): f83b25e
fix syntax errors
Browse files
app.py
CHANGED
|
@@ -1,232 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import gradio as gr
|
| 2 |
-
import
|
| 3 |
-
import
|
| 4 |
-
import pickle
|
| 5 |
-
from flask import Flask, request, jsonify
|
| 6 |
from environment import DocumentClassificationEnv
|
| 7 |
-
from baseline_inference import
|
| 8 |
-
from agent import TicketAgent
|
| 9 |
-
|
| 10 |
-
print("Pre-training models...")
|
| 11 |
-
models = {}
|
| 12 |
-
for d in ["easy", "medium", "hard"]:
|
| 13 |
-
models[d] = load_or_train(d)
|
| 14 |
-
print(f" {d} model ready!")
|
| 15 |
-
print("All models ready!")
|
| 16 |
-
|
| 17 |
-
agent = TicketAgent(model_dir=".")
|
| 18 |
-
print("Agent ready!")
|
| 19 |
-
|
| 20 |
-
CATEGORIES = {
|
| 21 |
-
"easy": ["General","Billing","Support","Technical","HR"],
|
| 22 |
-
"medium": ["General","Billing","Support","Technical","HR","Legal","Sales","Marketing","Operations","Complaints"],
|
| 23 |
-
"hard": [f"Cat_{i}" for i in range(22)]
|
| 24 |
-
}
|
| 25 |
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def api_step():
|
| 41 |
-
data = request.json or {}
|
| 42 |
-
action = data.get("action", 0)
|
| 43 |
-
env = api_envs.get("current")
|
| 44 |
-
if env is None:
|
| 45 |
-
return jsonify({"error": "Call /api/reset first"}), 400
|
| 46 |
-
obs, reward, done, _, info = env.step(int(action))
|
| 47 |
-
return jsonify({
|
| 48 |
-
"observation": {k: v.tolist() if hasattr(v, 'tolist') else v for k, v in obs.items()},
|
| 49 |
-
"reward": reward, "done": done, "info": info
|
| 50 |
-
})
|
| 51 |
-
|
| 52 |
-
@api.route("/api/run", methods=["POST"])
|
| 53 |
-
def api_run():
|
| 54 |
-
data = request.json or {}
|
| 55 |
-
difficulty = data.get("difficulty", "easy")
|
| 56 |
-
score, t = run_task(difficulty)
|
| 57 |
-
return jsonify({"difficulty": difficulty, "score": score, "time": t})
|
| 58 |
-
|
| 59 |
-
@api.route("/api/agent/process", methods=["POST"])
|
| 60 |
-
def api_agent_process():
|
| 61 |
-
data = request.json or {}
|
| 62 |
-
text = data.get("text", "")
|
| 63 |
-
difficulty = data.get("difficulty", "easy")
|
| 64 |
-
ticket_id = data.get("ticket_id", None)
|
| 65 |
-
if not text:
|
| 66 |
-
return jsonify({"error": "text field required"}), 400
|
| 67 |
-
result = agent.process_ticket(text, difficulty=difficulty, ticket_id=ticket_id)
|
| 68 |
-
return jsonify(result)
|
| 69 |
-
|
| 70 |
-
@api.route("/api/agent/classify", methods=["POST"])
|
| 71 |
-
def api_agent_classify():
|
| 72 |
-
data = request.json or {}
|
| 73 |
-
text = data.get("text", "")
|
| 74 |
-
difficulty = data.get("difficulty", "easy")
|
| 75 |
-
category, scores = agent.classify(text, difficulty)
|
| 76 |
-
priority = agent.get_priority(text)
|
| 77 |
-
return jsonify({"category": category, "priority": priority, "scores": scores})
|
| 78 |
-
|
| 79 |
-
@api.route("/health", methods=["GET"])
|
| 80 |
-
def health():
|
| 81 |
-
return jsonify({"status": "ok", "models_loaded": list(models.keys())})
|
| 82 |
-
|
| 83 |
-
def run_flask():
|
| 84 |
-
api.run(host="0.0.0.0", port=7861, debug=False, use_reloader=False, threaded=True)
|
| 85 |
-
|
| 86 |
-
env_state = {"env": None, "difficulty": "easy"}
|
| 87 |
-
|
| 88 |
-
def create_env(difficulty):
|
| 89 |
-
e = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
|
| 90 |
-
obs, _ = e.reset()
|
| 91 |
-
env_state["env"] = e
|
| 92 |
-
env_state["difficulty"] = difficulty
|
| 93 |
-
model = models[difficulty]
|
| 94 |
-
cats = CATEGORIES[difficulty]
|
| 95 |
-
proba = model.predict_proba([obs["content"]])[0]
|
| 96 |
-
top3 = sorted(enumerate(proba), key=lambda x: -x[1])[:3]
|
| 97 |
-
explain = "\n".join([f" {cats[i] if i < len(cats) else f'Cat_{i}'}: {p*100:.1f}%" for i, p in top3])
|
| 98 |
-
pred_idx = int(model.predict([obs["content"]])[0])
|
| 99 |
-
pred_cat = cats[pred_idx] if pred_idx < len(cats) else f"Cat_{pred_idx}"
|
| 100 |
-
return (
|
| 101 |
-
f"โ
Environment created! Difficulty: {difficulty}",
|
| 102 |
-
obs["content"],
|
| 103 |
-
f"๐ค Model predicts: **{pred_cat}**\n\nTop 3 predictions:\n{explain}",
|
| 104 |
-
""
|
| 105 |
-
)
|
| 106 |
-
|
| 107 |
-
def classify_doc(category_idx):
|
| 108 |
-
e = env_state["env"]
|
| 109 |
-
if e is None:
|
| 110 |
-
return "Create environment first!", "", "", ""
|
| 111 |
-
obs, reward, done, _, info = e.step(int(category_idx))
|
| 112 |
-
result = f"Reward: {reward:.2f} | Correct: {info.get('is_correct', False)}"
|
| 113 |
-
if done:
|
| 114 |
-
acc = info.get('episode_accuracy', 0)
|
| 115 |
-
result += f"\n๐ Episode done! Final Accuracy: {acc:.2%}"
|
| 116 |
-
return result, "Episode complete โ create new environment", "", str(info)
|
| 117 |
-
difficulty = env_state["difficulty"]
|
| 118 |
-
model = models[difficulty]
|
| 119 |
-
cats = CATEGORIES[difficulty]
|
| 120 |
-
proba = model.predict_proba([obs["content"]])[0]
|
| 121 |
-
top3 = sorted(enumerate(proba), key=lambda x: -x[1])[:3]
|
| 122 |
-
explain = "\n".join([f" {cats[i] if i < len(cats) else f'Cat_{i}'}: {p*100:.1f}%" for i, p in top3])
|
| 123 |
-
pred_idx = int(model.predict([obs["content"]])[0])
|
| 124 |
-
pred_cat = cats[pred_idx] if pred_idx < len(cats) else f"Cat_{pred_idx}"
|
| 125 |
-
return (
|
| 126 |
-
result,
|
| 127 |
-
obs["content"],
|
| 128 |
-
f"๐ค Model predicts: **{pred_cat}**\n\nTop 3 predictions:\n{explain}",
|
| 129 |
-
str(info)
|
| 130 |
-
)
|
| 131 |
-
|
| 132 |
-
def run_baseline_all():
|
| 133 |
-
rows = []
|
| 134 |
-
for d in ["easy", "medium", "hard"]:
|
| 135 |
-
score, t = run_task(d)
|
| 136 |
-
rows.append([d.upper(), f"{score:.4f}", f"{score*100:.1f}%", f"{t:.1f}s"])
|
| 137 |
-
return rows
|
| 138 |
-
|
| 139 |
-
def process_ticket_ui(ticket_text, difficulty):
|
| 140 |
-
if not ticket_text.strip():
|
| 141 |
-
return "โ Please enter ticket text!", "", "", "", "", ""
|
| 142 |
-
result = agent.process_ticket(ticket_text, difficulty=difficulty)
|
| 143 |
-
category_out = f"๐ท๏ธ {result['category']} (confidence: {result['confidence']*100:.1f}%)"
|
| 144 |
-
priority_out = f"๐จ {result['priority'].upper()}"
|
| 145 |
-
dept_out = f"๐ข {result['department']}\n๐ง {result['email']}\nโฑ๏ธ SLA: {result['sla']}"
|
| 146 |
-
top3_out = "\n".join([f" {c}: {p*100:.1f}%" for c, p in result['top3']])
|
| 147 |
-
ref_out = f"๐ซ {result['ref_id']} | โฐ {result['timestamp'][:19]}"
|
| 148 |
-
reply_out = result['reply']
|
| 149 |
-
return category_out, priority_out, dept_out, top3_out, ref_out, reply_out
|
| 150 |
|
| 151 |
SAMPLE_TICKETS = {
|
| 152 |
-
"Billing
|
| 153 |
-
"
|
| 154 |
-
"
|
| 155 |
-
"
|
| 156 |
-
"
|
|
|
|
| 157 |
}
|
| 158 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
def load_sample(sample_name):
|
| 160 |
return SAMPLE_TICKETS.get(sample_name, "")
|
| 161 |
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
)
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
)
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
placeholder="Paste email, chat message, bug report, HR query..."
|
| 199 |
)
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
with gr.
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Gradio UI for Document Classification Environment
|
| 3 |
+
Interactive demo for judges + baseline evaluation
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
import gradio as gr
|
| 7 |
+
import json
|
| 8 |
+
import time
|
|
|
|
|
|
|
| 9 |
from environment import DocumentClassificationEnv
|
| 10 |
+
from baseline_inference import keyword_agent
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
+
# Global state
|
| 13 |
+
current_env = None
|
| 14 |
+
current_obs = None
|
| 15 |
+
current_difficulty = "easy"
|
| 16 |
+
|
| 17 |
+
CATEGORY_COLORS = {
|
| 18 |
+
"General": "๐ต", "Billing": "๐ฐ", "Billing-Dispute": "โ ๏ธ", "Billing-Refund": "๐ธ",
|
| 19 |
+
"Support": "๐", "Support-Urgent": "๐จ", "Support-Normal": "๐ฌ",
|
| 20 |
+
"Technical": "โ๏ธ", "Technical-Bug": "๐", "Technical-Feature": "โจ",
|
| 21 |
+
"HR": "๐ฅ", "HR-Payroll": "๐ต", "HR-Benefits": "๐ฅ", "HR-Complaint": "๐ข",
|
| 22 |
+
"Legal": "โ๏ธ", "Legal-Contract": "๐", "Legal-Compliance": "๐",
|
| 23 |
+
"Executive": "๐", "Executive-Strategic": "๐ฏ",
|
| 24 |
+
"Finance": "๐", "Marketing": "๐ฃ", "Operations": "๐ง"
|
| 25 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
|
| 27 |
SAMPLE_TICKETS = {
|
| 28 |
+
"Billing Issue": "I was overcharged on my last invoice. The amount shows $150 but it should be $100. Please review and correct this billing error immediately.",
|
| 29 |
+
"Urgent Bug": "URGENT: Our production system is completely down! The application crashes every time users try to login. This is affecting all customers. Critical issue needs immediate attention!",
|
| 30 |
+
"Feature Request": "I would like to request a new feature for the dashboard. Can you add functionality for exporting data to CSV format? This would greatly improve our workflow.",
|
| 31 |
+
"HR Complaint": "I would like to file a formal complaint about a workplace issue. A colleague has been creating a hostile work environment and I need to report this serious HR matter.",
|
| 32 |
+
"Legal Contract": "I need a review of this contract before signing. The contract terms need clarification, specifically regarding the liability clauses and termination provisions.",
|
| 33 |
+
"Refund Request": "I would like to request a refund for my recent purchase. The product did not meet the specifications described and I need to return it and get refunded.",
|
| 34 |
}
|
| 35 |
|
| 36 |
+
|
| 37 |
+
def start_episode(difficulty):
|
| 38 |
+
global current_env, current_obs, current_difficulty
|
| 39 |
+
current_difficulty = difficulty
|
| 40 |
+
current_env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
|
| 41 |
+
current_obs, _ = current_env.reset()
|
| 42 |
+
|
| 43 |
+
categories = list(current_env.CATEGORY_MAPS[difficulty].values())
|
| 44 |
+
doc_content = current_obs["content"]
|
| 45 |
+
word_count = int(current_obs["word_count"][0])
|
| 46 |
+
doc_id = current_obs["document_id"]
|
| 47 |
+
total = int(current_obs["total_documents"][0])
|
| 48 |
+
|
| 49 |
+
info_text = f"**Document ID:** {doc_id} | **Words:** {word_count} | **Total Docs:** {total}"
|
| 50 |
+
return doc_content, info_text, gr.update(choices=categories, value=None), "Episode started! Classify the document above.", ""
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def classify_document(selected_category):
|
| 54 |
+
global current_env, current_obs, current_difficulty
|
| 55 |
+
|
| 56 |
+
if current_env is None:
|
| 57 |
+
return "Please start an episode first!", "", "", gr.update(choices=[])
|
| 58 |
+
if selected_category is None:
|
| 59 |
+
return current_obs["content"], "", "Please select a category!", ""
|
| 60 |
+
|
| 61 |
+
categories = list(current_env.CATEGORY_MAPS[current_difficulty].values())
|
| 62 |
+
action = categories.index(selected_category)
|
| 63 |
+
|
| 64 |
+
obs, reward, terminated, _, info = current_env.step(action)
|
| 65 |
+
current_obs = obs
|
| 66 |
+
|
| 67 |
+
is_correct = info.get("is_correct", False)
|
| 68 |
+
true_cat = info.get("true_category", "?")
|
| 69 |
+
pred_cat = info.get("predicted_category", "?")
|
| 70 |
+
accuracy = info.get("episode_accuracy", 0)
|
| 71 |
+
|
| 72 |
+
true_icon = CATEGORY_COLORS.get(true_cat, "๐")
|
| 73 |
+
pred_icon = CATEGORY_COLORS.get(pred_cat, "๐")
|
| 74 |
+
|
| 75 |
+
result = f"{'โ
CORRECT' if is_correct else 'โ INCORRECT'}\n"
|
| 76 |
+
result += f"Your pick: {pred_icon} {pred_cat}\n"
|
| 77 |
+
result += f"True category: {true_icon} {true_cat}\n"
|
| 78 |
+
result += f"Reward: {reward:.3f} | Accuracy so far: {accuracy:.1%}"
|
| 79 |
+
|
| 80 |
+
if terminated:
|
| 81 |
+
summary = info.get("episode_summary", {})
|
| 82 |
+
final_acc = summary.get("accuracy", 0)
|
| 83 |
+
total_reward = summary.get("total_reward", 0)
|
| 84 |
+
next_doc = "๐ Episode Complete!"
|
| 85 |
+
result += f"\n\n**FINAL: Accuracy={final_acc:.1%} | Total Reward={total_reward:.2f}**"
|
| 86 |
+
return "Episode complete! Click 'Start Episode' to play again.", "", result, gr.update(choices=categories)
|
| 87 |
+
|
| 88 |
+
next_doc = obs["content"]
|
| 89 |
+
next_id = obs["document_id"]
|
| 90 |
+
next_words = int(obs["word_count"][0])
|
| 91 |
+
next_idx = int(obs["document_index"][0])
|
| 92 |
+
total = int(obs["total_documents"][0])
|
| 93 |
+
info_text = f"**Document ID:** {next_id} | **Words:** {next_words} | **Progress:** {next_idx}/{total}"
|
| 94 |
+
|
| 95 |
+
return next_doc, info_text, result, gr.update(choices=categories, value=None)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def auto_classify():
|
| 99 |
+
global current_env, current_obs, current_difficulty
|
| 100 |
+
if current_env is None or current_obs is None:
|
| 101 |
+
return "Please start an episode first!", "", "No episode running.", ""
|
| 102 |
+
|
| 103 |
+
action = keyword_agent(current_obs, current_difficulty)
|
| 104 |
+
categories = list(current_env.CATEGORY_MAPS[current_difficulty].values())
|
| 105 |
+
selected = categories[action]
|
| 106 |
+
return classify_document(selected)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
def load_sample(sample_name):
|
| 110 |
return SAMPLE_TICKETS.get(sample_name, "")
|
| 111 |
|
| 112 |
+
|
| 113 |
+
def run_baseline_eval(difficulty):
|
| 114 |
+
env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
|
| 115 |
+
obs, _ = env.reset()
|
| 116 |
+
correct = 0
|
| 117 |
+
total = 0
|
| 118 |
+
terminated = False
|
| 119 |
+
results = []
|
| 120 |
+
|
| 121 |
+
while not terminated:
|
| 122 |
+
action = keyword_agent(obs, difficulty)
|
| 123 |
+
obs, reward, terminated, _, info = env.step(action)
|
| 124 |
+
correct += int(info.get("is_correct", False))
|
| 125 |
+
total += 1
|
| 126 |
+
if total <= 10:
|
| 127 |
+
true_cat = info.get("true_category", "?")
|
| 128 |
+
pred_cat = info.get("predicted_category", "?")
|
| 129 |
+
icon = "โ
" if info.get("is_correct") else "โ"
|
| 130 |
+
results.append(f"{icon} Predicted: {pred_cat} | True: {true_cat}")
|
| 131 |
+
|
| 132 |
+
accuracy = correct / total if total > 0 else 0
|
| 133 |
+
output = f"## Baseline Results โ {difficulty.upper()}\n\n"
|
| 134 |
+
output += f"**Accuracy: {accuracy:.1%}** ({correct}/{total} correct)\n\n"
|
| 135 |
+
output += "### Sample Predictions (first 10):\n"
|
| 136 |
+
output += "\n".join(results)
|
| 137 |
+
return output
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def create_interface():
|
| 141 |
+
with gr.Blocks(title="Document Classification OpenEnv", theme=gr.themes.Soft()) as demo:
|
| 142 |
+
gr.Markdown("""
|
| 143 |
+
# ๐ Document Classification Environment
|
| 144 |
+
### OpenEnv โ Meta x PyTorch Hackathon Submission
|
| 145 |
+
An interactive environment for training AI agents to classify and route support tickets.
|
| 146 |
+
""")
|
| 147 |
+
|
| 148 |
+
with gr.Tabs():
|
| 149 |
+
# Tab 1: Interactive Demo
|
| 150 |
+
with gr.TabItem("๐ฎ Interactive Demo"):
|
| 151 |
+
gr.Markdown("### Try classifying documents yourself!")
|
| 152 |
+
with gr.Row():
|
| 153 |
+
difficulty_select = gr.Radio(
|
| 154 |
+
choices=["easy", "medium", "hard"],
|
| 155 |
+
value="easy",
|
| 156 |
+
label="Difficulty Level",
|
| 157 |
+
info="Easy=5 categories, Medium=10, Hard=22"
|
| 158 |
+
)
|
| 159 |
+
start_btn = gr.Button("โถ Start Episode", variant="primary", scale=1)
|
| 160 |
+
|
| 161 |
+
with gr.Row():
|
| 162 |
+
with gr.Column(scale=2):
|
| 163 |
+
doc_display = gr.Textbox(
|
| 164 |
+
label="๐จ Document to Classify",
|
| 165 |
+
lines=6,
|
| 166 |
+
placeholder="Click 'Start Episode' to begin..."
|
| 167 |
+
)
|
| 168 |
+
doc_info = gr.Markdown("")
|
| 169 |
+
|
| 170 |
+
with gr.Column(scale=1):
|
| 171 |
+
category_radio = gr.Radio(choices=[], label="๐ท๏ธ Select Category")
|
| 172 |
+
with gr.Row():
|
| 173 |
+
classify_btn = gr.Button("โ
Classify", variant="primary")
|
| 174 |
+
auto_btn = gr.Button("๐ค Auto (AI)", variant="secondary")
|
| 175 |
+
|
| 176 |
+
result_display = gr.Textbox(label="๐ Result", lines=5)
|
| 177 |
+
|
| 178 |
+
# Sample tickets
|
| 179 |
+
gr.Markdown("### ๐ Try Sample Tickets")
|
| 180 |
+
with gr.Row():
|
| 181 |
+
for name in SAMPLE_TICKETS:
|
| 182 |
+
gr.Button(name).click(
|
| 183 |
+
fn=lambda n=name: load_sample(n),
|
| 184 |
+
outputs=doc_display
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
start_btn.click(
|
| 188 |
+
fn=start_episode,
|
| 189 |
+
inputs=difficulty_select,
|
| 190 |
+
outputs=[doc_display, doc_info, category_radio, result_display, gr.Textbox(visible=False)]
|
| 191 |
+
)
|
| 192 |
+
classify_btn.click(
|
| 193 |
+
fn=classify_document,
|
| 194 |
+
inputs=category_radio,
|
| 195 |
+
outputs=[doc_display, doc_info, result_display, category_radio]
|
| 196 |
)
|
| 197 |
+
auto_btn.click(
|
| 198 |
+
fn=auto_classify,
|
| 199 |
+
outputs=[doc_display, doc_info, result_display, category_radio]
|
|
|
|
| 200 |
)
|
| 201 |
+
|
| 202 |
+
# Tab 2: Baseline Evaluation
|
| 203 |
+
with gr.TabItem("๐ Baseline Evaluation"):
|
| 204 |
+
gr.Markdown("### Run the keyword-based baseline agent and see scores")
|
| 205 |
+
with gr.Row():
|
| 206 |
+
eval_difficulty = gr.Radio(
|
| 207 |
+
choices=["easy", "medium", "hard"],
|
| 208 |
+
value="easy",
|
| 209 |
+
label="Select Difficulty"
|
| 210 |
+
)
|
| 211 |
+
eval_btn = gr.Button("๐ Run Evaluation", variant="primary")
|
| 212 |
+
|
| 213 |
+
eval_output = gr.Markdown("Click 'Run Evaluation' to start...")
|
| 214 |
+
eval_btn.click(fn=run_baseline_eval, inputs=eval_difficulty, outputs=eval_output)
|
| 215 |
+
|
| 216 |
+
# Tab 3: Environment Info
|
| 217 |
+
with gr.TabItem("๐ Environment Info"):
|
| 218 |
+
gr.Markdown("""
|
| 219 |
+
## About This Environment
|
| 220 |
+
|
| 221 |
+
### Task: Document Classification & Routing
|
| 222 |
+
An agent receives customer support tickets/documents and must classify them into the correct department category.
|
| 223 |
+
|
| 224 |
+
### Difficulty Levels
|
| 225 |
+
| Level | Categories | Documents | Time Limit |
|
| 226 |
+
|-------|-----------|-----------|------------|
|
| 227 |
+
| Easy | 5 | 100 | None |
|
| 228 |
+
| Medium| 10 | 500 | 2 seconds |
|
| 229 |
+
| Hard | 22 | 1000 | 1 second |
|
| 230 |
+
|
| 231 |
+
### Reward Structure
|
| 232 |
+
- โ
Correct classification: **+1.0**
|
| 233 |
+
- โ Wrong classification: **-0.5**
|
| 234 |
+
- โก Speed bonus (fast response): **+0.1 to +0.25**
|
| 235 |
+
|
| 236 |
+
### API Usage
|
| 237 |
+
```python
|
| 238 |
+
from environment import DocumentClassificationEnv
|
| 239 |
+
|
| 240 |
+
env = DocumentClassificationEnv(task_difficulty="hard", seed=42)
|
| 241 |
+
obs, info = env.reset()
|
| 242 |
+
|
| 243 |
+
while True:
|
| 244 |
+
action = your_agent(obs) # 0 to N-1
|
| 245 |
+
obs, reward, done, truncated, info = env.step(action)
|
| 246 |
+
if done:
|
| 247 |
+
print(info["episode_summary"])
|
| 248 |
+
break
|
| 249 |
+
```
|
| 250 |
+
|
| 251 |
+
### Baseline Scores (keyword agent)
|
| 252 |
+
- **EASY:** 0.77
|
| 253 |
+
- **MEDIUM:** 0.89
|
| 254 |
+
- **HARD:** 0.165
|
| 255 |
+
|
| 256 |
+
### Categories (Hard Mode โ 22 total)
|
| 257 |
+
General, Billing, Billing-Dispute, Billing-Refund, Support, Support-Urgent, Support-Normal,
|
| 258 |
+
Technical, Technical-Bug, Technical-Feature, HR, HR-Payroll, HR-Benefits, HR-Complaint,
|
| 259 |
+
Legal, Legal-Contract, Legal-Compliance, Executive, Executive-Strategic, Finance, Marketing, Operations
|
| 260 |
+
""")
|
| 261 |
+
|
| 262 |
+
return demo
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
if __name__ == "__main__":
|
| 266 |
+
demo = create_interface()
|
| 267 |
+
demo.launch(server_name="0.0.0.0", server_port=7860)
|