TanujInsane commited on
Commit ·
345bccf
1
Parent(s): 38bb3c4
real ML classifier TF-IDF with trained models
Browse files- agent.py +148 -0
- app.py +0 -267
- baseline_inference.py +52 -18
agent.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import time
|
| 3 |
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
| 4 |
+
from sklearn.metrics.pairwise import cosine_similarity
|
| 5 |
+
from sklearn.preprocessing import normalize
|
| 6 |
+
|
| 7 |
+
CATEGORY_TEMPLATES = {
|
| 8 |
+
"General": ["general inquiry about services", "question about company", "general information request", "policies question"],
|
| 9 |
+
"Billing": ["billing invoice payment", "account charges subscription", "payment status bill", "billing address update"],
|
| 10 |
+
"Billing-Dispute": ["overcharged incorrect invoice", "dispute charge billing error", "wrong amount charged", "charge should not appear"],
|
| 11 |
+
"Billing-Refund": ["request refund money back", "process refund return", "refund my payment", "need refund overpayment"],
|
| 12 |
+
"Support": ["technical assistance help", "customer support question", "help with issue product", "contact support"],
|
| 13 |
+
"Support-Urgent": ["urgent critical system down", "emergency production outage", "immediate help critical", "everything broken urgent"],
|
| 14 |
+
"Support-Normal": ["basic feature help", "how to use product", "walk me through steps", "understanding functionality"],
|
| 15 |
+
"Technical": ["technical issue software", "API integration documentation", "system technical problem", "software technical help"],
|
| 16 |
+
"Technical-Bug": ["bug crash error exception", "software not working defect", "application crashing broken", "bug report error found"],
|
| 17 |
+
"Technical-Feature": ["feature request enhancement", "new feature suggestion", "add functionality improvement", "product feature idea"],
|
| 18 |
+
"HR": ["HR department human resources", "employee workforce staff", "HR policy question", "human resources inquiry"],
|
| 19 |
+
"HR-Payroll": ["paycheck salary payroll", "wage compensation pay stub", "payroll department salary", "incorrect pay payroll"],
|
| 20 |
+
"HR-Benefits": ["benefits insurance enrollment", "health dental coverage", "employee benefits package", "401k vacation benefits"],
|
| 21 |
+
"HR-Complaint": ["complaint harassment workplace", "misconduct grievance hostile", "discrimination complaint report", "hostile environment complaint"],
|
| 22 |
+
"Legal": ["legal assistance attorney", "legal department advice", "law contract legal question", "legal liability counsel"],
|
| 23 |
+
"Legal-Contract": ["contract review agreement", "signing contract terms clause", "contract dispute legal", "vendor agreement review"],
|
| 24 |
+
"Legal-Compliance": ["compliance regulation GDPR", "regulatory audit requirement", "HIPAA compliance legal", "regulatory requirement compliance"],
|
| 25 |
+
"Executive": ["executive management leadership", "senior team escalation", "executive inquiry question", "management leadership team"],
|
| 26 |
+
"Executive-Strategic": ["strategic planning business", "executive strategy roadmap", "business vision direction", "strategic initiative planning"],
|
| 27 |
+
"Finance": ["finance budget accounting", "revenue fiscal financial", "financial report budget", "accounting finance question"],
|
| 28 |
+
"Marketing": ["marketing campaign promotion", "advertising brand marketing", "campaign strategy promotion", "marketing question brand"],
|
| 29 |
+
"Operations": ["operations logistics workflow", "supply chain operational", "process management operations", "operational question logistics"],
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
class SmartClassificationAgent:
|
| 33 |
+
def __init__(self, difficulty):
|
| 34 |
+
self.difficulty = difficulty
|
| 35 |
+
self.vectorizer = None
|
| 36 |
+
self.template_matrix = None
|
| 37 |
+
self.template_labels = []
|
| 38 |
+
self.categories = []
|
| 39 |
+
self._trained = False
|
| 40 |
+
|
| 41 |
+
def train(self, categories):
|
| 42 |
+
self.categories = categories
|
| 43 |
+
texts, labels = [], []
|
| 44 |
+
for cat in categories:
|
| 45 |
+
if cat in CATEGORY_TEMPLATES:
|
| 46 |
+
for t in CATEGORY_TEMPLATES[cat]:
|
| 47 |
+
texts.append(t)
|
| 48 |
+
labels.append(cat)
|
| 49 |
+
else:
|
| 50 |
+
texts.append(cat.lower().replace("-", " "))
|
| 51 |
+
labels.append(cat)
|
| 52 |
+
self.vectorizer = TfidfVectorizer(ngram_range=(1, 3), sublinear_tf=True, max_features=10000)
|
| 53 |
+
self.template_matrix = self.vectorizer.fit_transform(texts)
|
| 54 |
+
self.template_labels = labels
|
| 55 |
+
self._trained = True
|
| 56 |
+
|
| 57 |
+
def predict(self, obs):
|
| 58 |
+
if not self._trained:
|
| 59 |
+
return 0, 0.0
|
| 60 |
+
|
| 61 |
+
content = obs.get("content", "").lower()
|
| 62 |
+
features = np.array(obs.get("features", []))
|
| 63 |
+
word_count = obs.get("word_count", 0)
|
| 64 |
+
has_urgency = obs.get("has_urgency_markers", False)
|
| 65 |
+
|
| 66 |
+
# TF-IDF content similarity
|
| 67 |
+
content_vec = self.vectorizer.transform([content])
|
| 68 |
+
sims = cosine_similarity(content_vec, self.template_matrix)[0]
|
| 69 |
+
best_idx = int(np.argmax(sims))
|
| 70 |
+
best_cat = self.template_labels[best_idx]
|
| 71 |
+
content_conf = float(sims[best_idx])
|
| 72 |
+
|
| 73 |
+
# Feature vector signal
|
| 74 |
+
feat_cat = None
|
| 75 |
+
feat_conf = 0.0
|
| 76 |
+
if len(features) > 0 and np.max(features) > -0.5:
|
| 77 |
+
valid = [(i, v) for i, v in enumerate(features) if v > -0.5]
|
| 78 |
+
if valid:
|
| 79 |
+
best_feat_idx = max(valid, key=lambda x: x[1])[0]
|
| 80 |
+
if best_feat_idx < len(self.categories):
|
| 81 |
+
feat_cat = self.categories[best_feat_idx]
|
| 82 |
+
feat_conf = float(features[best_feat_idx])
|
| 83 |
+
|
| 84 |
+
# Urgency boost
|
| 85 |
+
if has_urgency and "Support" in best_cat and "Urgent" not in best_cat:
|
| 86 |
+
if "Support-Urgent" in self.categories:
|
| 87 |
+
best_cat = "Support-Urgent"
|
| 88 |
+
content_conf = min(content_conf + 0.2, 1.0)
|
| 89 |
+
|
| 90 |
+
# Combine signals
|
| 91 |
+
final_cat = best_cat
|
| 92 |
+
if feat_cat and feat_cat in self.categories:
|
| 93 |
+
if feat_conf > content_conf * 0.8:
|
| 94 |
+
final_cat = feat_cat
|
| 95 |
+
|
| 96 |
+
try:
|
| 97 |
+
action = self.categories.index(final_cat)
|
| 98 |
+
confidence = max(content_conf, feat_conf)
|
| 99 |
+
except ValueError:
|
| 100 |
+
action = 0
|
| 101 |
+
confidence = 0.1
|
| 102 |
+
|
| 103 |
+
return action, confidence
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def run_episode(difficulty, seed=42):
|
| 107 |
+
from environment import DocumentClassificationEnv
|
| 108 |
+
env = DocumentClassificationEnv(difficulty)
|
| 109 |
+
cats_map = env.CATEGORY_MAPS[difficulty]
|
| 110 |
+
cats_list = [cats_map[i] for i in sorted(cats_map.keys())]
|
| 111 |
+
|
| 112 |
+
agent = SmartClassificationAgent(difficulty)
|
| 113 |
+
agent.train(cats_list)
|
| 114 |
+
|
| 115 |
+
obs, info = env.reset(seed=seed)
|
| 116 |
+
results = []
|
| 117 |
+
total_reward = 0
|
| 118 |
+
step = 0
|
| 119 |
+
|
| 120 |
+
while True:
|
| 121 |
+
t0 = time.time()
|
| 122 |
+
action, confidence = agent.predict(obs)
|
| 123 |
+
elapsed_ms = (time.time() - t0) * 1000
|
| 124 |
+
|
| 125 |
+
obs_next, reward, done, truncated, info = env.step(action)
|
| 126 |
+
total_reward += reward
|
| 127 |
+
step += 1
|
| 128 |
+
|
| 129 |
+
results.append({
|
| 130 |
+
"step": step,
|
| 131 |
+
"predicted": cats_map.get(action, "Unknown"),
|
| 132 |
+
"reward": reward,
|
| 133 |
+
"confidence": confidence,
|
| 134 |
+
"time_ms": elapsed_ms,
|
| 135 |
+
})
|
| 136 |
+
|
| 137 |
+
if done or truncated:
|
| 138 |
+
break
|
| 139 |
+
obs = obs_next
|
| 140 |
+
|
| 141 |
+
accuracy = total_reward / step if step > 0 else 0
|
| 142 |
+
return results, accuracy, info
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
if __name__ == "__main__":
|
| 146 |
+
for diff in ["easy", "medium", "hard"]:
|
| 147 |
+
results, acc, info = run_episode(diff)
|
| 148 |
+
print(f"{diff.upper()}: accuracy={acc:.4f}, steps={len(results)}")
|
app.py
CHANGED
|
@@ -1,267 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Gradio UI for Document Classification Environment
|
| 3 |
-
Interactive demo for judges + baseline evaluation
|
| 4 |
-
"""
|
| 5 |
-
|
| 6 |
-
import gradio as gr
|
| 7 |
-
import json
|
| 8 |
-
import time
|
| 9 |
-
from environment import DocumentClassificationEnv
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
# Global state
|
| 13 |
-
current_env = None
|
| 14 |
-
current_obs = None
|
| 15 |
-
current_difficulty = "easy"
|
| 16 |
-
|
| 17 |
-
CATEGORY_COLORS = {
|
| 18 |
-
"General": "🔵", "Billing": "💰", "Billing-Dispute": "⚠️", "Billing-Refund": "💸",
|
| 19 |
-
"Support": "🆘", "Support-Urgent": "🚨", "Support-Normal": "💬",
|
| 20 |
-
"Technical": "⚙️", "Technical-Bug": "🐛", "Technical-Feature": "✨",
|
| 21 |
-
"HR": "👥", "HR-Payroll": "💵", "HR-Benefits": "🏥", "HR-Complaint": "📢",
|
| 22 |
-
"Legal": "⚖️", "Legal-Contract": "📝", "Legal-Compliance": "📋",
|
| 23 |
-
"Executive": "👔", "Executive-Strategic": "🎯",
|
| 24 |
-
"Finance": "📊", "Marketing": "📣", "Operations": "🔧"
|
| 25 |
-
}
|
| 26 |
-
|
| 27 |
-
SAMPLE_TICKETS = {
|
| 28 |
-
"Billing Issue": "I was overcharged on my last invoice. The amount shows $150 but it should be $100. Please review and correct this billing error immediately.",
|
| 29 |
-
"Urgent Bug": "URGENT: Our production system is completely down! The application crashes every time users try to login. This is affecting all customers. Critical issue needs immediate attention!",
|
| 30 |
-
"Feature Request": "I would like to request a new feature for the dashboard. Can you add functionality for exporting data to CSV format? This would greatly improve our workflow.",
|
| 31 |
-
"HR Complaint": "I would like to file a formal complaint about a workplace issue. A colleague has been creating a hostile work environment and I need to report this serious HR matter.",
|
| 32 |
-
"Legal Contract": "I need a review of this contract before signing. The contract terms need clarification, specifically regarding the liability clauses and termination provisions.",
|
| 33 |
-
"Refund Request": "I would like to request a refund for my recent purchase. The product did not meet the specifications described and I need to return it and get refunded.",
|
| 34 |
-
}
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
def start_episode(difficulty):
|
| 38 |
-
global current_env, current_obs, current_difficulty
|
| 39 |
-
current_difficulty = difficulty
|
| 40 |
-
current_env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
|
| 41 |
-
current_obs, _ = current_env.reset()
|
| 42 |
-
|
| 43 |
-
categories = list(current_env.CATEGORY_MAPS[difficulty].values())
|
| 44 |
-
doc_content = current_obs["content"]
|
| 45 |
-
word_count = int(current_obs["word_count"][0])
|
| 46 |
-
doc_id = current_obs["document_id"]
|
| 47 |
-
total = int(current_obs["total_documents"][0])
|
| 48 |
-
|
| 49 |
-
info_text = f"**Document ID:** {doc_id} | **Words:** {word_count} | **Total Docs:** {total}"
|
| 50 |
-
return doc_content, info_text, gr.update(choices=categories, value=None), "Episode started! Classify the document above.", ""
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
def classify_document(selected_category):
|
| 54 |
-
global current_env, current_obs, current_difficulty
|
| 55 |
-
|
| 56 |
-
if current_env is None:
|
| 57 |
-
return "Please start an episode first!", "", "", gr.update(choices=[])
|
| 58 |
-
if selected_category is None:
|
| 59 |
-
return current_obs["content"], "", "Please select a category!", ""
|
| 60 |
-
|
| 61 |
-
categories = list(current_env.CATEGORY_MAPS[current_difficulty].values())
|
| 62 |
-
action = categories.index(selected_category)
|
| 63 |
-
|
| 64 |
-
obs, reward, terminated, _, info = current_env.step(action)
|
| 65 |
-
current_obs = obs
|
| 66 |
-
|
| 67 |
-
is_correct = info.get("is_correct", False)
|
| 68 |
-
true_cat = info.get("true_category", "?")
|
| 69 |
-
pred_cat = info.get("predicted_category", "?")
|
| 70 |
-
accuracy = info.get("episode_accuracy", 0)
|
| 71 |
-
|
| 72 |
-
true_icon = CATEGORY_COLORS.get(true_cat, "📄")
|
| 73 |
-
pred_icon = CATEGORY_COLORS.get(pred_cat, "📄")
|
| 74 |
-
|
| 75 |
-
result = f"{'✅ CORRECT' if is_correct else '❌ INCORRECT'}\n"
|
| 76 |
-
result += f"Your pick: {pred_icon} {pred_cat}\n"
|
| 77 |
-
result += f"True category: {true_icon} {true_cat}\n"
|
| 78 |
-
result += f"Reward: {reward:.3f} | Accuracy so far: {accuracy:.1%}"
|
| 79 |
-
|
| 80 |
-
if terminated:
|
| 81 |
-
summary = info.get("episode_summary", {})
|
| 82 |
-
final_acc = summary.get("accuracy", 0)
|
| 83 |
-
total_reward = summary.get("total_reward", 0)
|
| 84 |
-
next_doc = "🎉 Episode Complete!"
|
| 85 |
-
result += f"\n\n**FINAL: Accuracy={final_acc:.1%} | Total Reward={total_reward:.2f}**"
|
| 86 |
-
return "Episode complete! Click 'Start Episode' to play again.", "", result, gr.update(choices=categories)
|
| 87 |
-
|
| 88 |
-
next_doc = obs["content"]
|
| 89 |
-
next_id = obs["document_id"]
|
| 90 |
-
next_words = int(obs["word_count"][0])
|
| 91 |
-
next_idx = int(obs["document_index"][0])
|
| 92 |
-
total = int(obs["total_documents"][0])
|
| 93 |
-
info_text = f"**Document ID:** {next_id} | **Words:** {next_words} | **Progress:** {next_idx}/{total}"
|
| 94 |
-
|
| 95 |
-
return next_doc, info_text, result, gr.update(choices=categories, value=None)
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
def auto_classify():
|
| 99 |
-
global current_env, current_obs, current_difficulty
|
| 100 |
-
if current_env is None or current_obs is None:
|
| 101 |
-
return "Please start an episode first!", "", "No episode running.", ""
|
| 102 |
-
|
| 103 |
-
action = keyword_agent(current_obs, current_difficulty)
|
| 104 |
-
categories = list(current_env.CATEGORY_MAPS[current_difficulty].values())
|
| 105 |
-
selected = categories[action]
|
| 106 |
-
return classify_document(selected)
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
def load_sample(sample_name):
|
| 110 |
-
return SAMPLE_TICKETS.get(sample_name, "")
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
def run_baseline_eval(difficulty):
|
| 114 |
-
env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
|
| 115 |
-
obs, _ = env.reset()
|
| 116 |
-
correct = 0
|
| 117 |
-
total = 0
|
| 118 |
-
terminated = False
|
| 119 |
-
results = []
|
| 120 |
-
|
| 121 |
-
while not terminated:
|
| 122 |
-
action = keyword_agent(obs, difficulty)
|
| 123 |
-
obs, reward, terminated, _, info = env.step(action)
|
| 124 |
-
correct += int(info.get("is_correct", False))
|
| 125 |
-
total += 1
|
| 126 |
-
if total <= 10:
|
| 127 |
-
true_cat = info.get("true_category", "?")
|
| 128 |
-
pred_cat = info.get("predicted_category", "?")
|
| 129 |
-
icon = "✅" if info.get("is_correct") else "❌"
|
| 130 |
-
results.append(f"{icon} Predicted: {pred_cat} | True: {true_cat}")
|
| 131 |
-
|
| 132 |
-
accuracy = correct / total if total > 0 else 0
|
| 133 |
-
output = f"## Baseline Results — {difficulty.upper()}\n\n"
|
| 134 |
-
output += f"**Accuracy: {accuracy:.1%}** ({correct}/{total} correct)\n\n"
|
| 135 |
-
output += "### Sample Predictions (first 10):\n"
|
| 136 |
-
output += "\n".join(results)
|
| 137 |
-
return output
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
def create_interface():
|
| 141 |
-
with gr.Blocks(title="Document Classification OpenEnv", theme=gr.themes.Soft()) as demo:
|
| 142 |
-
gr.Markdown("""
|
| 143 |
-
# 📄 Document Classification Environment
|
| 144 |
-
### OpenEnv — Meta x PyTorch Hackathon Submission
|
| 145 |
-
An interactive environment for training AI agents to classify and route support tickets.
|
| 146 |
-
""")
|
| 147 |
-
|
| 148 |
-
with gr.Tabs():
|
| 149 |
-
# Tab 1: Interactive Demo
|
| 150 |
-
with gr.TabItem("🎮 Interactive Demo"):
|
| 151 |
-
gr.Markdown("### Try classifying documents yourself!")
|
| 152 |
-
with gr.Row():
|
| 153 |
-
difficulty_select = gr.Radio(
|
| 154 |
-
choices=["easy", "medium", "hard"],
|
| 155 |
-
value="easy",
|
| 156 |
-
label="Difficulty Level",
|
| 157 |
-
info="Easy=5 categories, Medium=10, Hard=22"
|
| 158 |
-
)
|
| 159 |
-
start_btn = gr.Button("▶ Start Episode", variant="primary", scale=1)
|
| 160 |
-
|
| 161 |
-
with gr.Row():
|
| 162 |
-
with gr.Column(scale=2):
|
| 163 |
-
doc_display = gr.Textbox(
|
| 164 |
-
label="📨 Document to Classify",
|
| 165 |
-
lines=6,
|
| 166 |
-
placeholder="Click 'Start Episode' to begin..."
|
| 167 |
-
)
|
| 168 |
-
doc_info = gr.Markdown("")
|
| 169 |
-
|
| 170 |
-
with gr.Column(scale=1):
|
| 171 |
-
category_radio = gr.Radio(choices=[], label="🏷️ Select Category")
|
| 172 |
-
with gr.Row():
|
| 173 |
-
classify_btn = gr.Button("✅ Classify", variant="primary")
|
| 174 |
-
auto_btn = gr.Button("🤖 Auto (AI)", variant="secondary")
|
| 175 |
-
|
| 176 |
-
result_display = gr.Textbox(label="📊 Result", lines=5)
|
| 177 |
-
|
| 178 |
-
# Sample tickets
|
| 179 |
-
gr.Markdown("### 📋 Try Sample Tickets")
|
| 180 |
-
with gr.Row():
|
| 181 |
-
for name in SAMPLE_TICKETS:
|
| 182 |
-
gr.Button(name).click(
|
| 183 |
-
fn=lambda n=name: load_sample(n),
|
| 184 |
-
outputs=doc_display
|
| 185 |
-
)
|
| 186 |
-
|
| 187 |
-
start_btn.click(
|
| 188 |
-
fn=start_episode,
|
| 189 |
-
inputs=difficulty_select,
|
| 190 |
-
outputs=[doc_display, doc_info, category_radio, result_display, gr.Textbox(visible=False)]
|
| 191 |
-
)
|
| 192 |
-
classify_btn.click(
|
| 193 |
-
fn=classify_document,
|
| 194 |
-
inputs=category_radio,
|
| 195 |
-
outputs=[doc_display, doc_info, result_display, category_radio]
|
| 196 |
-
)
|
| 197 |
-
auto_btn.click(
|
| 198 |
-
fn=auto_classify,
|
| 199 |
-
outputs=[doc_display, doc_info, result_display, category_radio]
|
| 200 |
-
)
|
| 201 |
-
|
| 202 |
-
# Tab 2: Baseline Evaluation
|
| 203 |
-
with gr.TabItem("📊 Baseline Evaluation"):
|
| 204 |
-
gr.Markdown("### Run the keyword-based baseline agent and see scores")
|
| 205 |
-
with gr.Row():
|
| 206 |
-
eval_difficulty = gr.Radio(
|
| 207 |
-
choices=["easy", "medium", "hard"],
|
| 208 |
-
value="easy",
|
| 209 |
-
label="Select Difficulty"
|
| 210 |
-
)
|
| 211 |
-
eval_btn = gr.Button("🚀 Run Evaluation", variant="primary")
|
| 212 |
-
|
| 213 |
-
eval_output = gr.Markdown("Click 'Run Evaluation' to start...")
|
| 214 |
-
eval_btn.click(fn=run_baseline_eval, inputs=eval_difficulty, outputs=eval_output)
|
| 215 |
-
|
| 216 |
-
# Tab 3: Environment Info
|
| 217 |
-
with gr.TabItem("📖 Environment Info"):
|
| 218 |
-
gr.Markdown("""
|
| 219 |
-
## About This Environment
|
| 220 |
-
|
| 221 |
-
### Task: Document Classification & Routing
|
| 222 |
-
An agent receives customer support tickets/documents and must classify them into the correct department category.
|
| 223 |
-
|
| 224 |
-
### Difficulty Levels
|
| 225 |
-
| Level | Categories | Documents | Time Limit |
|
| 226 |
-
|-------|-----------|-----------|------------|
|
| 227 |
-
| Easy | 5 | 100 | None |
|
| 228 |
-
| Medium| 10 | 500 | 2 seconds |
|
| 229 |
-
| Hard | 22 | 1000 | 1 second |
|
| 230 |
-
|
| 231 |
-
### Reward Structure
|
| 232 |
-
- ✅ Correct classification: **+1.0**
|
| 233 |
-
- ❌ Wrong classification: **-0.5**
|
| 234 |
-
- ⚡ Speed bonus (fast response): **+0.1 to +0.25**
|
| 235 |
-
|
| 236 |
-
### API Usage
|
| 237 |
-
```python
|
| 238 |
-
from environment import DocumentClassificationEnv
|
| 239 |
-
|
| 240 |
-
env = DocumentClassificationEnv(task_difficulty="hard", seed=42)
|
| 241 |
-
obs, info = env.reset()
|
| 242 |
-
|
| 243 |
-
while True:
|
| 244 |
-
action = your_agent(obs) # 0 to N-1
|
| 245 |
-
obs, reward, done, truncated, info = env.step(action)
|
| 246 |
-
if done:
|
| 247 |
-
print(info["episode_summary"])
|
| 248 |
-
break
|
| 249 |
-
```
|
| 250 |
-
|
| 251 |
-
### Baseline Scores (keyword agent)
|
| 252 |
-
- **EASY:** 0.77
|
| 253 |
-
- **MEDIUM:** 0.89
|
| 254 |
-
- **HARD:** 0.165
|
| 255 |
-
|
| 256 |
-
### Categories (Hard Mode — 22 total)
|
| 257 |
-
General, Billing, Billing-Dispute, Billing-Refund, Support, Support-Urgent, Support-Normal,
|
| 258 |
-
Technical, Technical-Bug, Technical-Feature, HR, HR-Payroll, HR-Benefits, HR-Complaint,
|
| 259 |
-
Legal, Legal-Contract, Legal-Compliance, Executive, Executive-Strategic, Finance, Marketing, Operations
|
| 260 |
-
""")
|
| 261 |
-
|
| 262 |
-
return demo
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
if __name__ == "__main__":
|
| 266 |
-
demo = create_interface()
|
| 267 |
-
demo.launch(server_name="0.0.0.0", server_port=7860)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
baseline_inference.py
CHANGED
|
@@ -1,21 +1,56 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
from tasks import TaskDataGenerator
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
def run_task(difficulty):
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
t0 = time.time()
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
if pred == true_label:
|
| 17 |
-
correct += 1
|
| 18 |
-
return correct/len(docs), time.time()-t0
|
| 19 |
|
| 20 |
def main():
|
| 21 |
parser = argparse.ArgumentParser()
|
|
@@ -33,8 +68,7 @@ def main():
|
|
| 33 |
for t,r in results.items():
|
| 34 |
print(f" {t.upper()} Score: {r['score']:.4f}")
|
| 35 |
with open(args.output,"w") as f:
|
| 36 |
-
json.dump(results,
|
| 37 |
-
print(f"\nResults saved to {args.output}")
|
| 38 |
|
| 39 |
if __name__ == "__main__":
|
| 40 |
-
main()
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import json
|
| 3 |
+
import time
|
| 4 |
+
import pickle
|
| 5 |
+
import os
|
| 6 |
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
| 7 |
+
from sklearn.linear_model import LogisticRegression
|
| 8 |
+
from sklearn.pipeline import Pipeline
|
| 9 |
from tasks import TaskDataGenerator
|
| 10 |
+
from environment import DocumentClassificationEnv
|
| 11 |
+
|
| 12 |
+
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 13 |
+
|
| 14 |
+
def get_model_path(difficulty):
|
| 15 |
+
return os.path.join(MODEL_DIR, f"model_{difficulty}.pkl")
|
| 16 |
+
|
| 17 |
+
def train_model(difficulty):
|
| 18 |
+
print(f"Training {difficulty} model...")
|
| 19 |
+
all_texts, all_labels = [], []
|
| 20 |
+
for seed in range(10):
|
| 21 |
+
gen = TaskDataGenerator(difficulty, seed=seed)
|
| 22 |
+
docs, labels = gen.generate_task_data()
|
| 23 |
+
for doc, label in zip(docs, labels):
|
| 24 |
+
all_texts.append(doc["content"])
|
| 25 |
+
all_labels.append(label)
|
| 26 |
+
pipeline = Pipeline([
|
| 27 |
+
("tfidf", TfidfVectorizer(ngram_range=(1,2), max_features=5000, sublinear_tf=True)),
|
| 28 |
+
("clf", LogisticRegression(max_iter=1000, C=5.0, solver="lbfgs", multi_class="multinomial"))
|
| 29 |
+
])
|
| 30 |
+
pipeline.fit(all_texts, all_labels)
|
| 31 |
+
with open(get_model_path(difficulty), "wb") as f:
|
| 32 |
+
pickle.dump(pipeline, f)
|
| 33 |
+
print(f" Saved!")
|
| 34 |
+
return pipeline
|
| 35 |
+
|
| 36 |
+
def load_or_train(difficulty):
|
| 37 |
+
path = get_model_path(difficulty)
|
| 38 |
+
if os.path.exists(path):
|
| 39 |
+
with open(path, "rb") as f:
|
| 40 |
+
return pickle.load(f)
|
| 41 |
+
return train_model(difficulty)
|
| 42 |
|
| 43 |
def run_task(difficulty):
|
| 44 |
+
model = load_or_train(difficulty)
|
| 45 |
+
env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
|
| 46 |
+
obs, _ = env.reset()
|
| 47 |
+
correct, total, t0, terminated = 0, 0, time.time(), False
|
| 48 |
+
while not terminated:
|
| 49 |
+
action = int(model.predict([obs["content"]])[0])
|
| 50 |
+
obs, reward, terminated, _, info = env.step(action)
|
| 51 |
+
correct += int(info.get("is_correct", False))
|
| 52 |
+
total += 1
|
| 53 |
+
return correct/total if total > 0 else 0, time.time()-t0
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
def main():
|
| 56 |
parser = argparse.ArgumentParser()
|
|
|
|
| 68 |
for t,r in results.items():
|
| 69 |
print(f" {t.upper()} Score: {r['score']:.4f}")
|
| 70 |
with open(args.output,"w") as f:
|
| 71 |
+
json.dump(results,f,indent=2)
|
|
|
|
| 72 |
|
| 73 |
if __name__ == "__main__":
|
| 74 |
+
main()
|