""" generate_hinglish_augmented.py ============================== Generates a large augmented Hinglish hate/toxic dataset using the hindi150swearwords.csv seed file. Strategy: - Load 150 Hinglish swear words - Embed them into diverse sentence templates per label category - Generate 10,500+ realistic toxic comments covering: profanity, personal_attack, sexual_harassment, threat, bullying, hate_speech - Save to data/processed/hinglish_augmented.csv Usage: python preprocessing/generate_hinglish_augmented.py """ import logging import random from pathlib import Path import pandas as pd logging.basicConfig( level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s", ) logger = logging.getLogger(__name__) PROJECT_ROOT = Path(__file__).resolve().parent.parent RAW_DIR = PROJECT_ROOT / "data" / "raw" PROCESSED_DIR = PROJECT_ROOT / "data" / "processed" PROCESSED_DIR.mkdir(parents=True, exist_ok=True) random.seed(42) # --------------------------------------------------------------------------- # Load swear word seed # --------------------------------------------------------------------------- def load_swear_words(path: Path) -> list: """Load the hinglish swear words (first column only = romanised form).""" df = pd.read_csv(path, header=None, names=["hinglish", "hindi"]) words = ( df["hinglish"] .dropna() .str.strip() .str.lower() .unique() .tolist() ) words = [w for w in words if w] logger.info(f"Loaded {len(words)} unique swear words") return words # --------------------------------------------------------------------------- # Template banks per label # --------------------------------------------------------------------------- TARGETS = [ "tu", "tum", "yeh banda", "woh aadmi", "woh ladki", "yaar", "bhai", "teri maa", "tera baap", "teri behen", "tere ghar wale", "teri family", "yeh sab log", "tumhare jaiso ko", "is group ka banda", "is page ka admin", ] INTENSIFIERS = [ "saala", "sala", "saali", "ekdum", "sach mein", "ek number ka", "full on", "bilkul", "totally", "arey yaar", "abe", "are", "sun", "teri kasam", "pakka", "poora", "asli", "real mein", ] FILLER_PHRASES = [ "bahut bura insaan hai", "kuch nahi karta", "bas bakwaas karta hai", "dimag kharab kar deta hai", "samajh mein nahi aata isko", "barbad kar deta hai sab kuch", "hamesha aisa karta hai", "kabhi nahi sudhrega", "tere jaisa nahi dekha", "aisa andha kya hoga", "pagalon ka raja hai yeh", "sharm nahi aati isko", "ganda insaan hai", "kisi kaam ka nahi", "faltu hai yeh", ] CLOSING_PHRASES = [ "nikal yahan se", "door reh mujhse", "meri zindagi se ja", "koi fark nahi padta tujhse", "bas kar ab", "chup ho ja", "band kar ye bakwaas", "mat aa dobara", "jaa yahin se", "dekh lena baad mein", "yaad rakhna", ] PROFANITY_TEMPLATES = [ "{sw}", "{sw} saala", "{sw} {target}", "{sw} kahi ka", "arey {sw}", "tu toh {sw} hai", "yeh banda {sw} hai", "{intens} {sw}", "{intens} {sw} hai yeh", "{target} ek dum {sw} hai", "{sw} mat ban yaar", "{sw} ki aulaad", "ek number ka {sw}", "{sw} he {sw} hai tu", "sun {sw}", "oye {sw}", "{sw} nikal yahan se", "{target} ko {sw} bol do", "tu {sw} hai bhai seriously", "yaar tu toh pura {sw} nikla", "arey {intens} {sw}", "{sw} bol tujhe kya lagta hai", "kya {sw} hota hai tu", "bhai tu toh {sw} ka baap hai", "{sw} log kahin ke", "chal {sw}", "chup kar {sw}", "teri {sw} ho", "saala {sw} insaan", "aisa {sw} pehli baar dekha", "{target} toh {sw} hai hi", "sun oye {sw}", "yeh {sw} kya samjha hai apne aap ko", "tu hi {sw} hai bhai", "sab jaante hain tu {sw} hai", ] PERSONAL_ATTACK_TEMPLATES = [ "tu bewakoof hai aur {sw} bhi", "{target} bilkul {sw} hai", "tera koi kaam ka nahi tu {sw} hai", "tu {sw} jaise hi rehta hai hamesha", "{sw} jaise insaan se baat nahi karni", "itna bada {sw} pehli baar dekha", "tere jaisa {sw} nahi dekha", "{target} ek dum gadha aur {sw} hai", "tu pagal hai aur upar se {sw}", "koi kuch bhi bol le tu toh {sw} hi rahega", "{sw} hai tu kuch nahi banega", "dimag nahi hai tera ekdum {sw} level ka", "faltu insaan {sw} kahi ka", "tere se kya umeed rakhni tu toh {sw} hai", "arey yaar tu toh pura {sw} nikla", "{intens} {sw} hai yeh banda", "nalayak {sw} ki tarah kar raha hai", "{target} ki shakal hi {sw} jaise hai", "yeh {sw} hamesha aisa hi karega", "kab sudhrega tu {sw}", "tu ek number ka {sw} hai", "{sw} hai tu isliye koi respect nahi karta", "tujhe dekhke hi pata chalta hai {sw} hai", "yaar tu toh {sw} ki definition hai", "{target} se baat karna matlab {sw} se baat karna", "itna bada {sw} kahin nahi dekha yaar", "{sw} mat ban zindagi mein kuch ban", "teri soch hi {sw} jaise hai", "tu {sw} tha hai aur rahega", "bas ek hi kaam aata hai tujhe {sw} banna", ] SEXUAL_HARASSMENT_TEMPLATES = [ "teri {sw} karenge", "{sw} lagane layak bhi nahi tu", "{target} ko {sw} kar denge", "{sw} wala kaam karta hai tu", "roz {sw} karte rehte ho", "{sw} wala banda hai yeh", "chal {sw} karte hain", "{sw} ka kaam karta hai tu bas", "{intens} {sw} waali baat kar raha hai", "tu {sw} karne layak bhi nahi", "{target} ke saath {sw} ho gaya", "{sw} ki duniya mein rehta hai yeh", "kya {sw} wala kaam hai yeh tera", "itna {sw} kyun karta hai", "{sw} wala content mat daal yahan", "yeh {sw} post kyun kiya", "aisa {sw} kyun likhta hai", "bas {sw} hi sujhti hai isko", "{sw} ki baatein karta hai", "{target} {sw} wala insaan hai", "yeh {sw} waali cheezein mat kar public mein", "teri {sw} sab suna do kya", "{sw} se bhi neeche hai teri soch", "arey {sw} karte rehte ho sab baar", "yeh {sw} wale comment mat karo", ] THREAT_TEMPLATES = [ "teri {sw} kar dunga", "tujhe {sw} kar dunga", "{target} ko {sw} kar denge", "sun {sw} karke dikha dunga tujhe", "maar dunga {sw}", "tod dunga teri {sw}", "dekh lena baad mein teri {sw}", "ek baar mil tu {sw} waala haal kar dunga", "abhi nikal warna {sw} waali haalat kar dunga", "{sw} karke ghar bhejunga tujhe", "aaj teri {sw} pakki hai", "teri {sw} aaj guaranteed hai", "tu aa samne phir teri {sw}", "jaa warna {sw} ho jayegi teri", "nikal yahan se warna {sw} waala scene hoga", "ek baar mil {sw} waala maza dikha dunga", "aaj {sw} waali raat hai teri", "mat kar aisa warna {sw} kar dunga", "teri {sw} aane wali hai sun", "abhi bahut achha lagega teri {sw} karke", "tujhe {sw} karke rahenge", "ek baar dekh lena {sw} waala haal hoga", "yaad rakhna teri {sw} hogi", "{target} ki {sw} kar denge aaj", "aaj {sw} waali party teri hai", ] BULLYING_TEMPLATES = [ "saara group jaanta hai tu {sw} hai", "log hasenge tere pe tu {sw} lagta hai", "{target} waise bhi {sw} hai sab jaante hain", "ek dum {sw} dikh raha hai tu", "hamesha {sw} hi rahega tu kuch nahi banega", "sab ko pata hai tu {sw} hai", "teri aukaat hi {sw} hai", "tere jaisa {sw} koi nahi hai yahan", "bas {sw} hi banta hai teri taraf se", "kabhi nahi sudhrega {sw} ki aulaad", "yeh {sw} wahi karega jo hamesha karta hai", "{target} toh {sw} hai sabke liye", "group mein {sw} ki jagah nahi hai", "nikal yahan se {sw}", "teri wajah se sab {sw} ban rahe hain", "tu toh {sw} certified hai", "{intens} {sw} kaha tha sab ne", "pura mohalla jaanta hai yeh {sw} hai", "social media pe bhi {sw} wali harkat ki tune", "tujhe hata do is group se {sw} hai yeh", "arey sab dekh rahe hain tu {sw} hai", "class mein bhi tu {sw} tha bahar bhi", "teri izzat {sw} jaise bhi nahi", "is group ka {sw} hai yeh banda", "{target} ko nikaalo {sw} hai yeh", ] HATE_SPEECH_TEMPLATES = [ "yeh log sab {sw} hote hain", "is community ke sab {sw} hain", "inki nasal hi {sw} hai", "yeh qaum hi {sw} wali hai", "tumhare jaise log {sw} hote hain", "is religion ke log {sw} hain sab", "yeh jaat hi {sw} hoti hai", "yeh sab {sw} logon ka group hai", "inhe bahar nikalo yeh {sw} hain", "yeh caste ke log {sw} hote hain", "is dharm ke log {sw} hain", "tumhari community {sw} logon se bhari hai", "yeh {sw} log kab sudhreenge", "yeh pradesh wale sab {sw} hote hain", "inke ghar mein hi {sw} sikhate hain", "aisi {sw} kaum ko bahar nikalo", "tumhare desh ke log {sw} hain", "yeh {sw} logon ko yahan nahi chahiye", "sab {sw} hain is biradari ke", "inhe dekhte hi pata chalta hai {sw} log hain", "yeh {sw} nasl ke log hain sab", "is majhab ke log {sw} hote hain always", "tumhare sample se pata chala sab {sw} hain wahan", "yeh {sw} qaum kabhi nahi sudhri", "inhe nikalo is desh se {sw} sab ke sab", ] TEMPLATES_BY_LABEL = { "profanity": PROFANITY_TEMPLATES, "personal_attack": PERSONAL_ATTACK_TEMPLATES, "sexual_harassment": SEXUAL_HARASSMENT_TEMPLATES, "threat": THREAT_TEMPLATES, "bullying": BULLYING_TEMPLATES, "hate_speech": HATE_SPEECH_TEMPLATES, } # Target counts per label (total = 100,000) TARGET_COUNTS = { "profanity": 25000, "personal_attack": 20000, "sexual_harassment": 15000, "threat": 15000, "bullying": 15000, "hate_speech": 15000, } # --------------------------------------------------------------------------- # Generation logic # --------------------------------------------------------------------------- def generate_for_label( label: str, templates: list, swear_words: list, count: int, ) -> list: """Generate `count` unique sentences for a given label.""" rows = [] seen = set() attempts = 0 max_attempts = count * 30 while len(rows) < count and attempts < max_attempts: attempts += 1 template = random.choice(templates) sw = random.choice(swear_words) target = random.choice(TARGETS) intens = random.choice(INTENSIFIERS) text = ( template .replace("{sw}", sw) .replace("{target}", target) .replace("{intens}", intens) .strip() ) # Optionally append a closing phrase (~25% chance) if random.random() < 0.25: text = text + " " + random.choice(CLOSING_PHRASES) # Optionally prefix with a filler phrase (~15% chance) if random.random() < 0.15: text = random.choice(FILLER_PHRASES) + ", " + text text = text.strip() if text not in seen and len(text) >= 5: seen.add(text) rows.append({"text": text, "label": label}) logger.info(f" {label}: generated {len(rows):,} samples (target {count:,})") return rows # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def main(): swear_path = RAW_DIR / "hindi150swearwords.csv" if not swear_path.exists(): logger.error(f"hindi150swearwords.csv not found at {swear_path}") return swear_words = load_swear_words(swear_path) all_rows = [] logger.info("Generating augmented Hinglish toxic dataset...") for label, templates in TEMPLATES_BY_LABEL.items(): count = TARGET_COUNTS[label] rows = generate_for_label(label, templates, swear_words, count) all_rows.extend(rows) df = pd.DataFrame(all_rows) df = df.sample(frac=1, random_state=42).reset_index(drop=True) output_path = PROCESSED_DIR / "hinglish_augmented.csv" df.to_csv(output_path, index=False) logger.info("=" * 60) logger.info(f"Total augmented rows: {len(df):,}") logger.info("Label distribution:") for label, cnt in df["label"].value_counts().items(): logger.info(f" {label:25s}: {cnt:>6,}") logger.info(f"Saved to: {output_path}") logger.info("=" * 60) if __name__ == "__main__": main()