TanujInsane
initial commit
8ddfad6
Raw History Blame Contribute Delete
11.9 kB
"""
Generate fake documents for training.
Templates for each category, then we mess with them a bit to make variations.
"""
import numpy as np
import random
from typing import Tuple, List, Dict
from sklearn.feature_extraction.text import TfidfVectorizer
def get_task_config(difficulty: str) -> Dict:
"""Just return the config for a difficulty level"""
configs = {
"easy": {
"num_documents": 100,
"num_categories": 5,
"feature_dim": 100,
"max_word_count": 200,
"time_limit": None, # Can take as long as you want
},
"medium": {
"num_documents": 500,
"num_categories": 10,
"feature_dim": 100,
"max_word_count": 500,
"time_limit": 2.0, # 2 seconds per doc
},
"hard": {
"num_documents": 1000,
"num_categories": 20,
"feature_dim": 100,
"max_word_count": 1000,
"time_limit": 1.0, # 1 second, get moving
}
}
return configs[difficulty]
class TaskDataGenerator:
"""Generate synthetic documents for the task"""
DOCUMENT_TEMPLATES = {
# Basic category - just regular stuff
"General": [
"This is a general inquiry about our services.",
"I would like to know more about your company.",
"Could you provide general information?",
"General question regarding policies.",
],
# Billing stuff
"Billing": [
"My invoice shows an incorrect amount. Please review.",
"I need to update my billing address.",
"What is the status of my payment?",
"I have questions about my bill.",
],
# People complaining about being overcharged
"Billing-Dispute": [
"I was overcharged on my last invoice.",
"This charge should not appear on my account.",
"I dispute this billing error.",
"The pricing on this invoice is incorrect.",
],
# Refund requests
"Billing-Refund": [
"I would like to request a refund.",
"Please process a refund for my order.",
"When will my refund be processed?",
"I need to return and get refunded.",
],
# Basic support
"Support": [
"I need technical assistance with your product.",
"How can I contact customer support?",
"I have a question about your services.",
"Can you help me with this issue?",
],
# EVERYTHING'S ON FIRE type support
"Support-Urgent": [
"URGENT: My system is down and I need immediate help!",
"CRITICAL: This is affecting production!",
"Emergency support needed immediately!",
"This is urgent and needs immediate attention!",
],
# Normal support that can wait
"Support-Normal": [
"I am having trouble with a basic feature.",
"Can you walk me through the process?",
"I need help understanding this feature.",
"How do I use this functionality?",
],
# Technical stuff
"Technical": [
"I am experiencing a technical issue with your software.",
"There seems to be a bug in the system.",
"Technical support needed for integration.",
"I need help with API documentation.",
],
# Code broken
"Technical-Bug": [
"I found a bug in the system that crashes the app.",
"The system throws an error when I try to login.",
"Error: Function X is not working correctly.",
"Bug report: System crashes on this operation.",
],
# New features people want
"Technical-Feature": [
"I would like to request a new feature.",
"Can you add functionality for X?",
"Feature request: Please add support for this.",
"Would it be possible to implement this capability?",
],
# HR stuff
"HR": [
"I have an HR-related question.",
"I need assistance with HR matters.",
"Can you help with HR policies?",
"I have a question about HR.",
],
# Money questions
"HR-Payroll": [
"I have a question about my paycheck.",
"My salary appears to be incorrect.",
"When is the next payroll cycle?",
"Can you explain my payroll deductions?",
],
# Health insurance etc
"HR-Benefits": [
"I need to enroll in health benefits.",
"Can I update my benefits information?",
"What benefits am I eligible for?",
"I have questions about the benefits plan.",
],
# Someone's upset
"HR-Complaint": [
"I would like to file a formal complaint.",
"I need to report a workplace issue.",
"This is a serious HR matter that needs attention.",
"I am filing an official complaint.",
],
# Legal stuff
"Legal": [
"I need legal advice regarding a contract.",
"I have legal questions about terms.",
"Can you help with legal documentation?",
"I need assistance with legal matters.",
],
# Contracts
"Legal-Contract": [
"I need review of this contract before signing.",
"The contract terms need clarification.",
"I have questions about contract obligations.",
"Can you explain these contract provisions?",
],
# Compliance - boring but important
"Legal-Compliance": [
"We need to ensure compliance with regulations.",
"I have questions about legal compliance.",
"Are we meeting compliance requirements?",
"I need guidance on regulatory compliance.",
],
# Executive stuff
"Executive": [
"I would like to schedule a meeting with management.",
"This is an executive-level inquiry.",
"Can I speak with a senior manager?",
"I need executive attention for this matter.",
],
# Big picture strategy
"Executive-Strategic": [
"Strategic partnership opportunity for discussion.",
"I would like to discuss corporate strategy.",
"This requires strategic-level decision making.",
"Executive strategic initiative to discuss.",
],
# Finance
"Finance": [
"I have questions about financial matters.",
"Can you provide financial statements?",
"I need to review financial records.",
"Financial assistance or information needed.",
],
# Marketing
"Marketing": [
"I am interested in marketing partnerships.",
"Can we discuss marketing opportunities?",
"I have a marketing proposal for you.",
"Marketing collaboration inquiry.",
],
# Operations
"Operations": [
"I have operational questions.",
"Operations scheduling needs adjustment.",
"Operational efficiency improvement proposal.",
"I need to discuss operational procedures.",
],
}
def __init__(self, difficulty: str, seed: int = None):
self.difficulty = difficulty
self.config = get_task_config(difficulty)
if seed is not None:
np.random.seed(seed)
random.seed(seed)
# Generate category list based on difficulty
if difficulty == "easy":
self.categories = ["General", "Billing", "Support", "Technical", "HR"]
elif difficulty == "medium":
self.categories = [
"General", "Billing", "Billing-Dispute", "Support",
"Technical", "Technical-Bug", "HR-Payroll", "HR-Benefits",
"Legal", "Executive"
]
else: # hard
self.categories = list(self.DOCUMENT_TEMPLATES.keys())
# Initialize TF-IDF vectorizer
all_docs = []
for templates in self.DOCUMENT_TEMPLATES.values():
all_docs.extend(templates)
self.vectorizer = TfidfVectorizer(
max_features=self.config["feature_dim"],
lowercase=True,
stop_words='english'
)
self.vectorizer.fit(all_docs)
def generate_task_data(self) -> Tuple[List[Dict], np.ndarray]:
"""
Create a batch of documents.
Pick random categories, grab templates, add some noise, extract features.
"""
documents = []
labels = []
num_docs = self.config["num_documents"]
for i in range(num_docs):
# Pick a random category for this doc
category = random.choice(self.categories)
category_idx = self.categories.index(category)
# Get a template from this category
template = random.choice(self.DOCUMENT_TEMPLATES[category])
# Add some variation so it's not too repetitive
variations = self._add_variation(template)
# Extract features (TF-IDF)
features = self.vectorizer.transform([variations]).toarray()[0]
# Normalize to [-1, 1] range
features = 2 * features / (np.max(features) + 1e-8) - 1
# Create document dict
doc = {
"id": f"doc_{i:06d}",
"content": variations,
"word_count": len(variations.split()),
"has_urgency_markers": self._has_urgency(variations),
"features": features.tolist(),
"true_category": category,
}
documents.append(doc)
labels.append(category_idx)
return documents, np.array(labels)
def _add_variation(self, template: str) -> str:
"""Add some noise/variation to make it feel less templated"""
words = template.split()
# Sometimes add filler words
if random.random() < 0.3:
filler = random.choice([
"I think", "Regarding", "In addition", "Furthermore",
"Also", "Additionally", "Moreover", "Please note"
])
words.insert(random.randint(0, len(words)), filler)
# Maybe add some detail
if random.random() < 0.2:
details = random.choice([
" regarding order #12345.",
" It's urgent.",
" This needs immediate attention.",
" Thank you for your help.",
" Please let me know ASAP.",
])
words.append(details)
return " ".join(words)
def _has_urgency(self, text: str) -> bool:
"""Check if text sounds urgent/important"""
urgency_words = ["urgent", "emergency", "critical", "immediate", "asap", "urgent!"]
text_lower = text.lower()
return any(word in text_lower for word in urgency_words)
# For testing
if __name__ == "__main__":
# Test data generation
for difficulty in ["easy", "medium", "hard"]:
gen = TaskDataGenerator(difficulty, seed=42)
docs, labels = gen.generate_task_data()
print(f"\n{difficulty.upper()} Task:")
print(f" Documents: {len(docs)}")
print(f" Categories: {len(set(labels))}")
print(f" Sample doc: {docs[0]['id']}")
print(f" Sample content: {docs[0]['content'][:50]}...")
print(f" Features shape: {len(docs[0]['features'])}")