File size: 7,068 Bytes
8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | """
Generate realistic synthetic customer support and internal escalation documents.
Features multi-format data (JSON, Slack, Logs, Emails), ambiguity, and VIP handling.
"""
import numpy as np
import random
import string
import json
from typing import Tuple, List, Dict
from sklearn.feature_extraction.text import TfidfVectorizer
from constants import CATEGORIES
def get_task_config(difficulty: str) -> Dict:
configs = {
"easy": {
"num_documents": 100,
"num_categories": 5,
"feature_dim": 100,
"max_word_count": 200,
"time_limit_per_action": None,
"time_limit_per_episode": None,
"ambiguity_rate": 0.0,
"sla_seconds": None,
},
"medium": {
"num_documents": 500,
"num_categories": 10,
"feature_dim": 100,
"max_word_count": 500,
"time_limit_per_action": 2.0,
"time_limit_per_episode": 1800,
"ambiguity_rate": 0.15,
"sla_seconds": 120.0,
},
"hard": {
"num_documents": 1000,
"num_categories": 22,
"feature_dim": 100,
"max_word_count": 1000,
"time_limit_per_action": 1.0,
"time_limit_per_episode": 1200,
"ambiguity_rate": 0.30,
"sla_seconds": 60.0,
}
}
return configs[difficulty]
FIRST_NAMES = ["James", "Sarah", "Michael", "Emily", "David", "Jessica", "Robert", "Wei", "Fatima", "Carlos"]
LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Chen", "Al-Rashid"]
COMPANIES = ["Acme Corp", "TechForward Inc", "Global Solutions Ltd", "Stripe", "AWS", "Datadog"]
def _ctx():
name = f"{random.choice(FIRST_NAMES)} {random.choice(LAST_NAMES)}"
is_vip = random.random() < 0.15 # 15% chance of being a VIP enterprise client
return {
"name": name,
"email": f"{name.lower().replace(' ', '.')}@{random.choice(['gmail.com', 'company.com', 'enterprise.io'])}",
"company": random.choice(COMPANIES),
"order_id": f"ORD-{random.randint(100000, 999999)}",
"amount": f"${random.randint(50, 9999)}.{random.randint(0, 99):02d}",
"date": f"2025-{random.randint(1,12):02d}-{random.randint(1,28):02d}",
"is_vip": is_vip,
"urgency_level": "CRITICAL" if is_vip else random.choice(["LOW", "MEDIUM", "HIGH"])
}
class TaskDataGenerator:
def __init__(self, difficulty: str, seed: int = None):
self.difficulty = difficulty
self.config = get_task_config(difficulty)
if seed is not None:
np.random.seed(seed)
random.seed(seed)
self.categories = list(CATEGORIES[difficulty].values())
# Simple vocabulary for TF-IDF training
self.vectorizer = TfidfVectorizer(max_features=self.config["feature_dim"], lowercase=True, stop_words='english')
self.vectorizer.fit(["invoice billing refund", "technical bug crash", "legal contract compliance", "hr payroll benefits complaint", "marketing operations finance strategy tax social"])
def _get_corpus_text(self) -> str:
if not hasattr(self, "_hf_corpus"):
try:
from datasets import load_dataset
ds = load_dataset("ag_news", split="test")
self._hf_corpus = [x["text"] for x in ds]
except Exception:
self._hf_corpus = ["Please assist with process.", "Error encountered in system."]
return random.choice(self._hf_corpus)
def _generate_format(self, category: str, ctx: dict) -> str:
"""Generates content in various formats (JSON, Slack, Log, Email) with Real Corpus Injection"""
fmt = random.choice(["json", "slack", "log", "email"])
cat_lower = category.lower()
real_text = self._get_corpus_text()
content = ""
if fmt == "json":
payload = {
"event_id": f"evt_{random.randint(1000,9999)}",
"source": "webhook_listener",
"timestamp": ctx["date"],
"customer": {"id": ctx["email"], "vip_tier": "Enterprise" if ctx["is_vip"] else "Standard"},
"payload": f"Category: {cat_lower}. Order {ctx['order_id']} value {ctx['amount']}. Extract: {real_text}",
"metadata": {"raw_category_hint": category, "severity": ctx["urgency_level"]}
}
content = json.dumps(payload, indent=2)
elif fmt == "slack":
content = f"**Slack Transcript - Channel: #triage-incoming**\n"
content += f"[{ctx['date']} 09:12 AM] @system_bot: New ticket from {ctx['name']} ({ctx['company']}). VIP: {ctx['is_vip']}\n"
content += f"[{ctx['date']} 09:13 AM] {ctx['name']}: Hey team, we have an issue regarding {cat_lower}.\n"
content += f"[{ctx['date']} 09:14 AM] {ctx['name']}: For context: {real_text}"
elif fmt == "log":
content = f"2025-01-01T12:00:00Z WARN [req_id={random.randint(100,999)}] subsystem={category.split('-')[0]} action=PROCESS userId={ctx['email']} msg=\"{real_text}\""
else: # Email
vip_tag = "[VIP - URGENT] " if ctx['is_vip'] else ""
content = f"Subject: {vip_tag}Inquiry regarding {cat_lower}\nFrom: {ctx['email']}\n\nTo whom it may concern,\nWe are reaching out about order {ctx['order_id']} value {ctx['amount']}.\n\nAdditional details: {real_text}\n\nRegards,\n{ctx['name']}"
return content
def generate_task_data(self) -> Tuple[List[Dict], np.ndarray]:
documents = []
labels = []
num_docs = self.config["num_documents"]
ambiguity_rate = self.config.get("ambiguity_rate", 0.0)
for i in range(num_docs):
ctx = _ctx()
if random.random() < ambiguity_rate:
# Deliberate heavy ambiguity: Mention multiple departments
cat1 = random.choice(self.categories)
cat2 = random.choice(self.categories)
true_cat = random.choice([cat1, cat2])
content = self._generate_format(f"{cat1} and {cat2}", ctx)
else:
true_cat = random.choice(self.categories)
content = self._generate_format(true_cat, ctx)
features = self.vectorizer.transform([content]).toarray()[0]
if np.max(features) > 0: features = 2 * features / (np.max(features) + 1e-8) - 1
doc = {
"id": f"doc_{i:06d}",
"content": content,
"word_count": len(content.split()),
"has_urgency_markers": ctx["is_vip"], # VIP flag passed via urgency
"features": features.tolist(),
"true_category": true_cat,
"is_vip": ctx["is_vip"]
}
documents.append(doc)
labels.append(self.categories.index(true_cat))
return documents, np.array(labels)
|