TanujInsane
fix: deploy updates, add inference.py and /reset /step endpoints
49234f8
Raw History Blame Contribute Delete
7.07 kB
"""
Generate realistic synthetic customer support and internal escalation documents.
Features multi-format data (JSON, Slack, Logs, Emails), ambiguity, and VIP handling.
"""
import numpy as np
import random
import string
import json
from typing import Tuple, List, Dict
from sklearn.feature_extraction.text import TfidfVectorizer
from constants import CATEGORIES
def get_task_config(difficulty: str) -> Dict:
configs = {
"easy": {
"num_documents": 100,
"num_categories": 5,
"feature_dim": 100,
"max_word_count": 200,
"time_limit_per_action": None,
"time_limit_per_episode": None,
"ambiguity_rate": 0.0,
"sla_seconds": None,
},
"medium": {
"num_documents": 500,
"num_categories": 10,
"feature_dim": 100,
"max_word_count": 500,
"time_limit_per_action": 2.0,
"time_limit_per_episode": 1800,
"ambiguity_rate": 0.15,
"sla_seconds": 120.0,
},
"hard": {
"num_documents": 1000,
"num_categories": 22,
"feature_dim": 100,
"max_word_count": 1000,
"time_limit_per_action": 1.0,
"time_limit_per_episode": 1200,
"ambiguity_rate": 0.30,
"sla_seconds": 60.0,
}
}
return configs[difficulty]
FIRST_NAMES = ["James", "Sarah", "Michael", "Emily", "David", "Jessica", "Robert", "Wei", "Fatima", "Carlos"]
LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Chen", "Al-Rashid"]
COMPANIES = ["Acme Corp", "TechForward Inc", "Global Solutions Ltd", "Stripe", "AWS", "Datadog"]
def _ctx():
name = f"{random.choice(FIRST_NAMES)} {random.choice(LAST_NAMES)}"
is_vip = random.random() < 0.15 # 15% chance of being a VIP enterprise client
return {
"name": name,
"email": f"{name.lower().replace(' ', '.')}@{random.choice(['gmail.com', 'company.com', 'enterprise.io'])}",
"company": random.choice(COMPANIES),
"order_id": f"ORD-{random.randint(100000, 999999)}",
"amount": f"${random.randint(50, 9999)}.{random.randint(0, 99):02d}",
"date": f"2025-{random.randint(1,12):02d}-{random.randint(1,28):02d}",
"is_vip": is_vip,
"urgency_level": "CRITICAL" if is_vip else random.choice(["LOW", "MEDIUM", "HIGH"])
}
class TaskDataGenerator:
def __init__(self, difficulty: str, seed: int = None):
self.difficulty = difficulty
self.config = get_task_config(difficulty)
if seed is not None:
np.random.seed(seed)
random.seed(seed)
self.categories = list(CATEGORIES[difficulty].values())
# Simple vocabulary for TF-IDF training
self.vectorizer = TfidfVectorizer(max_features=self.config["feature_dim"], lowercase=True, stop_words='english')
self.vectorizer.fit(["invoice billing refund", "technical bug crash", "legal contract compliance", "hr payroll benefits complaint", "marketing operations finance strategy tax social"])
def _get_corpus_text(self) -> str:
if not hasattr(self, "_hf_corpus"):
try:
from datasets import load_dataset
ds = load_dataset("ag_news", split="test")
self._hf_corpus = [x["text"] for x in ds]
except Exception:
self._hf_corpus = ["Please assist with process.", "Error encountered in system."]
return random.choice(self._hf_corpus)
def _generate_format(self, category: str, ctx: dict) -> str:
"""Generates content in various formats (JSON, Slack, Log, Email) with Real Corpus Injection"""
fmt = random.choice(["json", "slack", "log", "email"])
cat_lower = category.lower()
real_text = self._get_corpus_text()
content = ""
if fmt == "json":
payload = {
"event_id": f"evt_{random.randint(1000,9999)}",
"source": "webhook_listener",
"timestamp": ctx["date"],
"customer": {"id": ctx["email"], "vip_tier": "Enterprise" if ctx["is_vip"] else "Standard"},
"payload": f"Category: {cat_lower}. Order {ctx['order_id']} value {ctx['amount']}. Extract: {real_text}",
"metadata": {"raw_category_hint": category, "severity": ctx["urgency_level"]}
}
content = json.dumps(payload, indent=2)
elif fmt == "slack":
content = f"**Slack Transcript - Channel: #triage-incoming**\n"
content += f"[{ctx['date']} 09:12 AM] @system_bot: New ticket from {ctx['name']} ({ctx['company']}). VIP: {ctx['is_vip']}\n"
content += f"[{ctx['date']} 09:13 AM] {ctx['name']}: Hey team, we have an issue regarding {cat_lower}.\n"
content += f"[{ctx['date']} 09:14 AM] {ctx['name']}: For context: {real_text}"
elif fmt == "log":
content = f"2025-01-01T12:00:00Z WARN [req_id={random.randint(100,999)}] subsystem={category.split('-')[0]} action=PROCESS userId={ctx['email']} msg=\"{real_text}\""
else: # Email
vip_tag = "[VIP - URGENT] " if ctx['is_vip'] else ""
content = f"Subject: {vip_tag}Inquiry regarding {cat_lower}\nFrom: {ctx['email']}\n\nTo whom it may concern,\nWe are reaching out about order {ctx['order_id']} value {ctx['amount']}.\n\nAdditional details: {real_text}\n\nRegards,\n{ctx['name']}"
return content
def generate_task_data(self) -> Tuple[List[Dict], np.ndarray]:
documents = []
labels = []
num_docs = self.config["num_documents"]
ambiguity_rate = self.config.get("ambiguity_rate", 0.0)
for i in range(num_docs):
ctx = _ctx()
if random.random() < ambiguity_rate:
# Deliberate heavy ambiguity: Mention multiple departments
cat1 = random.choice(self.categories)
cat2 = random.choice(self.categories)
true_cat = random.choice([cat1, cat2])
content = self._generate_format(f"{cat1} and {cat2}", ctx)
else:
true_cat = random.choice(self.categories)
content = self._generate_format(true_cat, ctx)
features = self.vectorizer.transform([content]).toarray()[0]
if np.max(features) > 0: features = 2 * features / (np.max(features) + 1e-8) - 1
doc = {
"id": f"doc_{i:06d}",
"content": content,
"word_count": len(content.split()),
"has_urgency_markers": ctx["is_vip"], # VIP flag passed via urgency
"features": features.tolist(),
"true_category": true_cat,
"is_vip": ctx["is_vip"]
}
documents.append(doc)
labels.append(self.categories.index(true_cat))
return documents, np.array(labels)