Download tasks.py from TanujInsane/document-classification-env: direct link, hf CLI and curl.
- Browser
- Download file 7.07 kB
-
https://huggingface.co/spaces/TanujInsane/document-classification-env/resolve/main/tasks.py
- Command line
-
hf download hf://spaces/TanujInsane/document-classification-env/tasks.py
-
curl -L -o tasks.py https://huggingface.co/spaces/TanujInsane/document-classification-env/resolve/main/tasks.py
7.07 kB
| """ | |
| Generate realistic synthetic customer support and internal escalation documents. | |
| Features multi-format data (JSON, Slack, Logs, Emails), ambiguity, and VIP handling. | |
| """ | |
| import numpy as np | |
| import random | |
| import string | |
| import json | |
| from typing import Tuple, List, Dict | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from constants import CATEGORIES | |
| def get_task_config(difficulty: str) -> Dict: | |
| configs = { | |
| "easy": { | |
| "num_documents": 100, | |
| "num_categories": 5, | |
| "feature_dim": 100, | |
| "max_word_count": 200, | |
| "time_limit_per_action": None, | |
| "time_limit_per_episode": None, | |
| "ambiguity_rate": 0.0, | |
| "sla_seconds": None, | |
| }, | |
| "medium": { | |
| "num_documents": 500, | |
| "num_categories": 10, | |
| "feature_dim": 100, | |
| "max_word_count": 500, | |
| "time_limit_per_action": 2.0, | |
| "time_limit_per_episode": 1800, | |
| "ambiguity_rate": 0.15, | |
| "sla_seconds": 120.0, | |
| }, | |
| "hard": { | |
| "num_documents": 1000, | |
| "num_categories": 22, | |
| "feature_dim": 100, | |
| "max_word_count": 1000, | |
| "time_limit_per_action": 1.0, | |
| "time_limit_per_episode": 1200, | |
| "ambiguity_rate": 0.30, | |
| "sla_seconds": 60.0, | |
| } | |
| } | |
| return configs[difficulty] | |
| FIRST_NAMES = ["James", "Sarah", "Michael", "Emily", "David", "Jessica", "Robert", "Wei", "Fatima", "Carlos"] | |
| LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Chen", "Al-Rashid"] | |
| COMPANIES = ["Acme Corp", "TechForward Inc", "Global Solutions Ltd", "Stripe", "AWS", "Datadog"] | |
| def _ctx(): | |
| name = f"{random.choice(FIRST_NAMES)} {random.choice(LAST_NAMES)}" | |
| is_vip = random.random() < 0.15 # 15% chance of being a VIP enterprise client | |
| return { | |
| "name": name, | |
| "email": f"{name.lower().replace(' ', '.')}@{random.choice(['gmail.com', 'company.com', 'enterprise.io'])}", | |
| "company": random.choice(COMPANIES), | |
| "order_id": f"ORD-{random.randint(100000, 999999)}", | |
| "amount": f"${random.randint(50, 9999)}.{random.randint(0, 99):02d}", | |
| "date": f"2025-{random.randint(1,12):02d}-{random.randint(1,28):02d}", | |
| "is_vip": is_vip, | |
| "urgency_level": "CRITICAL" if is_vip else random.choice(["LOW", "MEDIUM", "HIGH"]) | |
| } | |
| class TaskDataGenerator: | |
| def __init__(self, difficulty: str, seed: int = None): | |
| self.difficulty = difficulty | |
| self.config = get_task_config(difficulty) | |
| if seed is not None: | |
| np.random.seed(seed) | |
| random.seed(seed) | |
| self.categories = list(CATEGORIES[difficulty].values()) | |
| # Simple vocabulary for TF-IDF training | |
| self.vectorizer = TfidfVectorizer(max_features=self.config["feature_dim"], lowercase=True, stop_words='english') | |
| self.vectorizer.fit(["invoice billing refund", "technical bug crash", "legal contract compliance", "hr payroll benefits complaint", "marketing operations finance strategy tax social"]) | |
| def _get_corpus_text(self) -> str: | |
| if not hasattr(self, "_hf_corpus"): | |
| try: | |
| from datasets import load_dataset | |
| ds = load_dataset("ag_news", split="test") | |
| self._hf_corpus = [x["text"] for x in ds] | |
| except Exception: | |
| self._hf_corpus = ["Please assist with process.", "Error encountered in system."] | |
| return random.choice(self._hf_corpus) | |
| def _generate_format(self, category: str, ctx: dict) -> str: | |
| """Generates content in various formats (JSON, Slack, Log, Email) with Real Corpus Injection""" | |
| fmt = random.choice(["json", "slack", "log", "email"]) | |
| cat_lower = category.lower() | |
| real_text = self._get_corpus_text() | |
| content = "" | |
| if fmt == "json": | |
| payload = { | |
| "event_id": f"evt_{random.randint(1000,9999)}", | |
| "source": "webhook_listener", | |
| "timestamp": ctx["date"], | |
| "customer": {"id": ctx["email"], "vip_tier": "Enterprise" if ctx["is_vip"] else "Standard"}, | |
| "payload": f"Category: {cat_lower}. Order {ctx['order_id']} value {ctx['amount']}. Extract: {real_text}", | |
| "metadata": {"raw_category_hint": category, "severity": ctx["urgency_level"]} | |
| } | |
| content = json.dumps(payload, indent=2) | |
| elif fmt == "slack": | |
| content = f"**Slack Transcript - Channel: #triage-incoming**\n" | |
| content += f"[{ctx['date']} 09:12 AM] @system_bot: New ticket from {ctx['name']} ({ctx['company']}). VIP: {ctx['is_vip']}\n" | |
| content += f"[{ctx['date']} 09:13 AM] {ctx['name']}: Hey team, we have an issue regarding {cat_lower}.\n" | |
| content += f"[{ctx['date']} 09:14 AM] {ctx['name']}: For context: {real_text}" | |
| elif fmt == "log": | |
| content = f"2025-01-01T12:00:00Z WARN [req_id={random.randint(100,999)}] subsystem={category.split('-')[0]} action=PROCESS userId={ctx['email']} msg=\"{real_text}\"" | |
| else: # Email | |
| vip_tag = "[VIP - URGENT] " if ctx['is_vip'] else "" | |
| content = f"Subject: {vip_tag}Inquiry regarding {cat_lower}\nFrom: {ctx['email']}\n\nTo whom it may concern,\nWe are reaching out about order {ctx['order_id']} value {ctx['amount']}.\n\nAdditional details: {real_text}\n\nRegards,\n{ctx['name']}" | |
| return content | |
| def generate_task_data(self) -> Tuple[List[Dict], np.ndarray]: | |
| documents = [] | |
| labels = [] | |
| num_docs = self.config["num_documents"] | |
| ambiguity_rate = self.config.get("ambiguity_rate", 0.0) | |
| for i in range(num_docs): | |
| ctx = _ctx() | |
| if random.random() < ambiguity_rate: | |
| # Deliberate heavy ambiguity: Mention multiple departments | |
| cat1 = random.choice(self.categories) | |
| cat2 = random.choice(self.categories) | |
| true_cat = random.choice([cat1, cat2]) | |
| content = self._generate_format(f"{cat1} and {cat2}", ctx) | |
| else: | |
| true_cat = random.choice(self.categories) | |
| content = self._generate_format(true_cat, ctx) | |
| features = self.vectorizer.transform([content]).toarray()[0] | |
| if np.max(features) > 0: features = 2 * features / (np.max(features) + 1e-8) - 1 | |
| doc = { | |
| "id": f"doc_{i:06d}", | |
| "content": content, | |
| "word_count": len(content.split()), | |
| "has_urgency_markers": ctx["is_vip"], # VIP flag passed via urgency | |
| "features": features.tolist(), | |
| "true_category": true_cat, | |
| "is_vip": ctx["is_vip"] | |
| } | |
| documents.append(doc) | |
| labels.append(self.categories.index(true_cat)) | |
| return documents, np.array(labels) | |