Spaces:
Sleeping
Sleeping
Download tasks.py from TanujInsane/doc-class-fix: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/TanujInsane/doc-class-fix/resolve/main/tasks.py
- Command line
-
hf download hf://spaces/TanujInsane/doc-class-fix/tasks.py
-
curl -L -o tasks.py https://huggingface.co/spaces/TanujInsane/doc-class-fix/resolve/main/tasks.py
11.9 kB
| """ | |
| Generate fake documents for training. | |
| Templates for each category, then we mess with them a bit to make variations. | |
| """ | |
| import numpy as np | |
| import random | |
| from typing import Tuple, List, Dict | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| def get_task_config(difficulty: str) -> Dict: | |
| """Just return the config for a difficulty level""" | |
| configs = { | |
| "easy": { | |
| "num_documents": 100, | |
| "num_categories": 5, | |
| "feature_dim": 100, | |
| "max_word_count": 200, | |
| "time_limit": None, # Can take as long as you want | |
| }, | |
| "medium": { | |
| "num_documents": 500, | |
| "num_categories": 10, | |
| "feature_dim": 100, | |
| "max_word_count": 500, | |
| "time_limit": 2.0, # 2 seconds per doc | |
| }, | |
| "hard": { | |
| "num_documents": 1000, | |
| "num_categories": 20, | |
| "feature_dim": 100, | |
| "max_word_count": 1000, | |
| "time_limit": 1.0, # 1 second, get moving | |
| } | |
| } | |
| return configs[difficulty] | |
| class TaskDataGenerator: | |
| """Generate synthetic documents for the task""" | |
| DOCUMENT_TEMPLATES = { | |
| # Basic category - just regular stuff | |
| "General": [ | |
| "This is a general inquiry about our services.", | |
| "I would like to know more about your company.", | |
| "Could you provide general information?", | |
| "General question regarding policies.", | |
| ], | |
| # Billing stuff | |
| "Billing": [ | |
| "My invoice shows an incorrect amount. Please review.", | |
| "I need to update my billing address.", | |
| "What is the status of my payment?", | |
| "I have questions about my bill.", | |
| ], | |
| # People complaining about being overcharged | |
| "Billing-Dispute": [ | |
| "I was overcharged on my last invoice.", | |
| "This charge should not appear on my account.", | |
| "I dispute this billing error.", | |
| "The pricing on this invoice is incorrect.", | |
| ], | |
| # Refund requests | |
| "Billing-Refund": [ | |
| "I would like to request a refund.", | |
| "Please process a refund for my order.", | |
| "When will my refund be processed?", | |
| "I need to return and get refunded.", | |
| ], | |
| # Basic support | |
| "Support": [ | |
| "I need technical assistance with your product.", | |
| "How can I contact customer support?", | |
| "I have a question about your services.", | |
| "Can you help me with this issue?", | |
| ], | |
| # EVERYTHING'S ON FIRE type support | |
| "Support-Urgent": [ | |
| "URGENT: My system is down and I need immediate help!", | |
| "CRITICAL: This is affecting production!", | |
| "Emergency support needed immediately!", | |
| "This is urgent and needs immediate attention!", | |
| ], | |
| # Normal support that can wait | |
| "Support-Normal": [ | |
| "I am having trouble with a basic feature.", | |
| "Can you walk me through the process?", | |
| "I need help understanding this feature.", | |
| "How do I use this functionality?", | |
| ], | |
| # Technical stuff | |
| "Technical": [ | |
| "I am experiencing a technical issue with your software.", | |
| "There seems to be a bug in the system.", | |
| "Technical support needed for integration.", | |
| "I need help with API documentation.", | |
| ], | |
| # Code broken | |
| "Technical-Bug": [ | |
| "I found a bug in the system that crashes the app.", | |
| "The system throws an error when I try to login.", | |
| "Error: Function X is not working correctly.", | |
| "Bug report: System crashes on this operation.", | |
| ], | |
| # New features people want | |
| "Technical-Feature": [ | |
| "I would like to request a new feature.", | |
| "Can you add functionality for X?", | |
| "Feature request: Please add support for this.", | |
| "Would it be possible to implement this capability?", | |
| ], | |
| # HR stuff | |
| "HR": [ | |
| "I have an HR-related question.", | |
| "I need assistance with HR matters.", | |
| "Can you help with HR policies?", | |
| "I have a question about HR.", | |
| ], | |
| # Money questions | |
| "HR-Payroll": [ | |
| "I have a question about my paycheck.", | |
| "My salary appears to be incorrect.", | |
| "When is the next payroll cycle?", | |
| "Can you explain my payroll deductions?", | |
| ], | |
| # Health insurance etc | |
| "HR-Benefits": [ | |
| "I need to enroll in health benefits.", | |
| "Can I update my benefits information?", | |
| "What benefits am I eligible for?", | |
| "I have questions about the benefits plan.", | |
| ], | |
| # Someone's upset | |
| "HR-Complaint": [ | |
| "I would like to file a formal complaint.", | |
| "I need to report a workplace issue.", | |
| "This is a serious HR matter that needs attention.", | |
| "I am filing an official complaint.", | |
| ], | |
| # Legal stuff | |
| "Legal": [ | |
| "I need legal advice regarding a contract.", | |
| "I have legal questions about terms.", | |
| "Can you help with legal documentation?", | |
| "I need assistance with legal matters.", | |
| ], | |
| # Contracts | |
| "Legal-Contract": [ | |
| "I need review of this contract before signing.", | |
| "The contract terms need clarification.", | |
| "I have questions about contract obligations.", | |
| "Can you explain these contract provisions?", | |
| ], | |
| # Compliance - boring but important | |
| "Legal-Compliance": [ | |
| "We need to ensure compliance with regulations.", | |
| "I have questions about legal compliance.", | |
| "Are we meeting compliance requirements?", | |
| "I need guidance on regulatory compliance.", | |
| ], | |
| # Executive stuff | |
| "Executive": [ | |
| "I would like to schedule a meeting with management.", | |
| "This is an executive-level inquiry.", | |
| "Can I speak with a senior manager?", | |
| "I need executive attention for this matter.", | |
| ], | |
| # Big picture strategy | |
| "Executive-Strategic": [ | |
| "Strategic partnership opportunity for discussion.", | |
| "I would like to discuss corporate strategy.", | |
| "This requires strategic-level decision making.", | |
| "Executive strategic initiative to discuss.", | |
| ], | |
| # Finance | |
| "Finance": [ | |
| "I have questions about financial matters.", | |
| "Can you provide financial statements?", | |
| "I need to review financial records.", | |
| "Financial assistance or information needed.", | |
| ], | |
| # Marketing | |
| "Marketing": [ | |
| "I am interested in marketing partnerships.", | |
| "Can we discuss marketing opportunities?", | |
| "I have a marketing proposal for you.", | |
| "Marketing collaboration inquiry.", | |
| ], | |
| # Operations | |
| "Operations": [ | |
| "I have operational questions.", | |
| "Operations scheduling needs adjustment.", | |
| "Operational efficiency improvement proposal.", | |
| "I need to discuss operational procedures.", | |
| ], | |
| } | |
| def __init__(self, difficulty: str, seed: int = None): | |
| self.difficulty = difficulty | |
| self.config = get_task_config(difficulty) | |
| if seed is not None: | |
| np.random.seed(seed) | |
| random.seed(seed) | |
| # Generate category list based on difficulty | |
| if difficulty == "easy": | |
| self.categories = ["General", "Billing", "Support", "Technical", "HR"] | |
| elif difficulty == "medium": | |
| self.categories = [ | |
| "General", "Billing", "Billing-Dispute", "Support", | |
| "Technical", "Technical-Bug", "HR-Payroll", "HR-Benefits", | |
| "Legal", "Executive" | |
| ] | |
| else: # hard | |
| self.categories = list(self.DOCUMENT_TEMPLATES.keys()) | |
| # Initialize TF-IDF vectorizer | |
| all_docs = [] | |
| for templates in self.DOCUMENT_TEMPLATES.values(): | |
| all_docs.extend(templates) | |
| self.vectorizer = TfidfVectorizer( | |
| max_features=self.config["feature_dim"], | |
| lowercase=True, | |
| stop_words='english' | |
| ) | |
| self.vectorizer.fit(all_docs) | |
| def generate_task_data(self) -> Tuple[List[Dict], np.ndarray]: | |
| """ | |
| Create a batch of documents. | |
| Pick random categories, grab templates, add some noise, extract features. | |
| """ | |
| documents = [] | |
| labels = [] | |
| num_docs = self.config["num_documents"] | |
| for i in range(num_docs): | |
| # Pick a random category for this doc | |
| category = random.choice(self.categories) | |
| category_idx = self.categories.index(category) | |
| # Get a template from this category | |
| template = random.choice(self.DOCUMENT_TEMPLATES[category]) | |
| # Add some variation so it's not too repetitive | |
| variations = self._add_variation(template) | |
| # Extract features (TF-IDF) | |
| features = self.vectorizer.transform([variations]).toarray()[0] | |
| # Normalize to [-1, 1] range | |
| features = 2 * features / (np.max(features) + 1e-8) - 1 | |
| # Create document dict | |
| doc = { | |
| "id": f"doc_{i:06d}", | |
| "content": variations, | |
| "word_count": len(variations.split()), | |
| "has_urgency_markers": self._has_urgency(variations), | |
| "features": features.tolist(), | |
| "true_category": category, | |
| } | |
| documents.append(doc) | |
| labels.append(category_idx) | |
| return documents, np.array(labels) | |
| def _add_variation(self, template: str) -> str: | |
| """Add some noise/variation to make it feel less templated""" | |
| words = template.split() | |
| # Sometimes add filler words | |
| if random.random() < 0.3: | |
| filler = random.choice([ | |
| "I think", "Regarding", "In addition", "Furthermore", | |
| "Also", "Additionally", "Moreover", "Please note" | |
| ]) | |
| words.insert(random.randint(0, len(words)), filler) | |
| # Maybe add some detail | |
| if random.random() < 0.2: | |
| details = random.choice([ | |
| " regarding order #12345.", | |
| " It's urgent.", | |
| " This needs immediate attention.", | |
| " Thank you for your help.", | |
| " Please let me know ASAP.", | |
| ]) | |
| words.append(details) | |
| return " ".join(words) | |
| def _has_urgency(self, text: str) -> bool: | |
| """Check if text sounds urgent/important""" | |
| urgency_words = ["urgent", "emergency", "critical", "immediate", "asap", "urgent!"] | |
| text_lower = text.lower() | |
| return any(word in text_lower for word in urgency_words) | |
| # For testing | |
| if __name__ == "__main__": | |
| # Test data generation | |
| for difficulty in ["easy", "medium", "hard"]: | |
| gen = TaskDataGenerator(difficulty, seed=42) | |
| docs, labels = gen.generate_task_data() | |
| print(f"\n{difficulty.upper()} Task:") | |
| print(f" Documents: {len(docs)}") | |
| print(f" Categories: {len(set(labels))}") | |
| print(f" Sample doc: {docs[0]['id']}") | |
| print(f" Sample content: {docs[0]['content'][:50]}...") | |
| print(f" Features shape: {len(docs[0]['features'])}") | |