import os import sys import re import joblib from src.preprocess import clean_text try: from src.bert_model import MultilingualBertEngine except ImportError: try: from bert_model import MultilingualBertEngine except ImportError: MultilingualBertEngine = None if hasattr(sys.stdout, 'reconfigure'): try: sys.stdout.reconfigure(encoding='utf-8') except Exception: pass CONTEXT_WORDS = { 'ghanta': { 'safe_keywords': ['mandir', 'temple', 'pooja', 'aarti', 'bajana', 'baj', 'bajao', 'bajaya', 'bajta', 'bajte', 'bajate', 'church', 'school', 'awaaz', 'bell', 'sundar', 'ek', 'do', 'teen', 'char', 'paanch', 'chhe', 'saat', 'aath', 'nau', 'das', '1', '2', '3', '4', '5', 'lagega', 'lagenge', 'samay', 'waqt', 'hour', 'hours', 'der', 'aadha', 'kitne', 'kitna'], 'safe_label': 'Normal', 'safe_reason': '🔔 "Ghanta" detected in religious/bell/time context (temple bell or time duration) — classified as Normal.', 'offensive_reason': '⚠️ "Ghanta" detected as vulgar slang (dismissive/offensive usage).' }, 'ghante': { 'safe_keywords': ['mandir', 'temple', 'pooja', 'aarti', 'bajana', 'baj', 'bajao', 'bajaya', 'bajta', 'bajte', 'bajate', 'church', 'school', 'awaaz', 'bell', 'ek', 'do', 'teen', 'char', 'paanch', 'chhe', 'saat', 'aath', 'nau', 'das', '1', '2', '3', '4', '5', 'lagega', 'lagenge', 'samay', 'waqt', 'hour', 'hours', 'der', 'aadha', 'kitne', 'kitna', 'baad', 'pehle'], 'safe_label': 'Normal', 'safe_reason': '⏱️ "Ghante" detected in time/duration context — classified as Normal.', 'offensive_reason': '⚠️ "Ghante" detected in offensive/vulgar context.' }, 'घंटा': { 'safe_keywords': ['मंदिर', 'पूजा', 'आरती', 'बजाना', 'बज', 'बजाओ', 'बजाया', 'बजता', 'बजते', 'स्कूल', 'आवाज', 'सुंदर', 'घंटी', 'एक', 'दो', 'तीन', 'चार', 'पांच', 'समय', 'वक्त', 'लगेगा', 'लगेगी', 'लगेंगे', 'आधा', 'देर', 'कितने', 'कितना'], 'safe_label': 'Normal', 'safe_reason': '🔔 "घंटा" religious/bell/time context (घंटी या समय) mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ "घंटा" vulgar slang ke roop mein paya gaya.' }, 'घंटे': { 'safe_keywords': ['मंदिर', 'पूजा', 'आरती', 'बजाना', 'बज', 'बजाओ', 'बजाया', 'बजता', 'बजते', 'स्कूल', 'आवाज', 'सुंदर', 'घंटी', 'एक', 'दो', 'तीन', 'चार', 'पांच', 'समय', 'वक्त', 'लगेगा', 'लगेगी', 'लगेंगे', 'आधा', 'देर', 'कितने', 'बाद', 'पहले'], 'safe_label': 'Normal', 'safe_reason': '⏱️ "घंटे" time/duration context (समय) mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ "घंटे" vulgar slang ke roop mein paya gaya.' }, 'kutta': { 'safe_keywords': ['mera', 'pyaara', 'ghar', 'rakhwala', 'doctor', 'desi', 'samajhdaar', 'mohalle', 'khel', 'street', 'khana', 'bacche', 'pet', 'paaltu', 'cute'], 'safe_label': 'Normal', 'safe_reason': '🐕 "Kutta" detected in pet/animal context (talking about a dog) — classified as Normal.', 'offensive_reason': '⚠️ "Kutta" detected as targeted insult (calling someone a dog).' }, 'कुत्ता': { 'safe_keywords': ['मेरा', 'प्यारा', 'घर', 'रखवाला', 'डॉक्टर', 'देसी', 'समझदार', 'मोहल्ले', 'खेल', 'खाना', 'बच्चे', 'पालतू'], 'safe_label': 'Normal', 'safe_reason': '🐕 "कुत्ता" pet/animal context mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ "कुत्ता" insult ke roop mein use hua.' }, 'kutte': { 'safe_keywords': ['mera', 'pyaara', 'ghar', 'rakhwala', 'doctor', 'desi', 'samajhdaar', 'mohalle', 'khel', 'street', 'khana', 'bacche', 'pet', 'paaltu'], 'safe_label': 'Normal', 'safe_reason': '🐕 "Kutte" detected in pet/animal context — classified as Normal.', 'offensive_reason': '⚠️ "Kutte" detected as targeted insult.' }, 'कुत्ते': { 'safe_keywords': ['मेरा', 'प्यारा', 'घर', 'रखवाला', 'डॉक्टर', 'देसी', 'समझदार', 'मोहल्ले', 'खेल', 'खाना', 'बच्चे', 'पालतू'], 'safe_label': 'Normal', 'safe_reason': '🐕 "कुत्ते" animal context mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ "कुत्ते" insult ke roop mein use hua.' }, 'saala': { 'safe_keywords': ['mera', 'behen', 'wife', 'bhai', 'shaadi', 'ghar', 'family', 'rishtedaar', 'sahab', 'milke', 'accha', 'insaan', 'help', 'matlab', 'rehta'], 'safe_label': 'Normal', 'safe_reason': '👨‍👩‍👦 "Saala" detected in family context (brother-in-law) — classified as Normal.', 'offensive_reason': '⚠️ "Saala" detected as abusive slang (targeted insult).' }, 'साला': { 'safe_keywords': ['मेरा', 'बहन', 'पत्नी', 'भाई', 'शादी', 'घर', 'परिवार', 'रिश्तेदार', 'साहब', 'अच्छा'], 'safe_label': 'Normal', 'safe_reason': '👨‍👩‍👦 "साला" family relation (brother-in-law) mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ "साला" gaali ke roop mein use hua.' }, 'saale': { 'safe_keywords': ['mera', 'behen', 'wife', 'bhai', 'shaadi', 'ghar', 'family', 'rishtedaar', 'sahab', 'milke', 'accha', 'help'], 'safe_label': 'Normal', 'safe_reason': '👨‍👩‍👦 "Saale" detected in family context (brother-in-law) — classified as Normal.', 'offensive_reason': '⚠️ "Saale" detected as abusive slang.' }, 'साले': { 'safe_keywords': ['मेरा', 'बहन', 'पत्नी', 'भाई', 'शादी', 'घर', 'परिवार', 'रिश्तेदार', 'साहब'], 'safe_label': 'Normal', 'safe_reason': '👨‍👩‍👦 "साले" family relation context mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ "साले" abusive context mein paya gaya.' }, 'gadha': { 'safe_keywords': ['gaon', 'jaanwar', 'animal', 'sawari', 'village', 'mehnati', 'baccho', 'farm', 'khet'], 'safe_label': 'Normal', 'safe_reason': '🫏 "Gadha" detected in animal context (donkey) — classified as Normal.', 'offensive_reason': '⚠️ "Gadha" used as insult.' }, 'गधा': { 'safe_keywords': ['गांव', 'जानवर', 'सवारी', 'मेहनती', 'खेत', 'जंगल'], 'safe_label': 'Normal', 'safe_reason': '🫏 "गधा" animal context mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ "गधा" insult ke roop mein use hua.' }, 'ullu': { 'safe_keywords': ['raat', 'ped', 'tree', 'pakshi', 'bird', 'wildlife', 'jungle', 'forest', 'aakhein', 'nature'], 'safe_label': 'Normal', 'safe_reason': '🦉 "Ullu" detected in nocturnal bird/wildlife context (owl) — classified as Normal.', 'offensive_reason': '⚠️ "Ullu" detected as slang insult (calling someone a fool).' }, 'उल्लू': { 'safe_keywords': ['रात', 'पेड़', 'पक्षी', 'चिड़िया', 'जंगल', 'वन्यजीव', 'प्रकृति', 'आंखें'], 'safe_label': 'Normal', 'safe_reason': '🦉 "उल्लू" पक्षी / वन्यजीव के संदर्भ में पाया गया — classified as Normal.', 'offensive_reason': '⚠️ "उल्लू" मूर्ख / बेवकूफ बनाने के स्लैंग में प्रयोग हुआ।' }, 'pagal': { 'safe_keywords': ['cricket', 'padhai', 'shauk', 'passion', 'hospital', 'doctor', 'ilaj', 'mehnat', 'deewana', 'geet', 'khel'], 'safe_label': 'Normal', 'safe_reason': '🧠 "Pagal" detected in passion/enthusiasm/medical context — classified as Normal.', 'offensive_reason': '⚠️ "Pagal" detected as derogatory mental insult.' }, 'पागल': { 'safe_keywords': ['क्रिकेट', 'पढ़ाई', 'शौक', 'जुनून', 'अस्पताल', 'डॉक्टर', 'इलाज', 'मेहनत', 'दीवाना', 'गीत', 'खेल'], 'safe_label': 'Normal', 'safe_reason': '🧠 "पागल" जुनून या मेडिकल संदर्भ में पाया गया — classified as Normal.', 'offensive_reason': '⚠️ "पागल" अपमानजनक मानसिक कटाक्ष के रूप में प्रयुक्त हुआ।' }, 'kamina': { 'safe_keywords': ['yaar', 'dost', 'friend', 'kitne din', 'masti', 'bhai', 'kaisa', 'college', 'school', 'bachpan'], 'safe_label': 'Normal', 'safe_reason': '🤝 "Kamina" detected in informal friend banter/camaraderie — classified as Normal.', 'offensive_reason': '⚠️ "Kamina" detected as malicious character insult.' }, 'कमीना': { 'safe_keywords': ['यार', 'दोस्त', 'भाई', 'कितने दिन', 'मस्ती', 'कॉलेज', 'स्कूल', 'बचपन', 'कैसा'], 'safe_label': 'Normal', 'safe_reason': '🤝 "कमीना" दोस्तों के बीच अनौपचारिक दोस्ताना हंसी-मजाक में पाया गया — classified as Normal.', 'offensive_reason': '⚠️ "कमीना" व्यक्तिगत द्वेष या गाली के रूप में प्रयुक्त हुआ।' }, 'billi': { 'safe_keywords': ['pet', 'paaltu', 'doodh', 'ghar', 'cute', 'animal', 'kitten', 'white', 'black', 'meow'], 'safe_label': 'Normal', 'safe_reason': '🐱 "Billi" detected in domestic cat context — classified as Normal.', 'offensive_reason': '⚠️ "Billi" detected in derogatory context.' }, 'बिल्ली': { 'safe_keywords': ['पालतू', 'दूध', 'घर', 'सफेद', 'काली', 'प्यारी', 'जानवर'], 'safe_label': 'Normal', 'safe_reason': '🐱 "बिल्ली" पालतू जानवर के संदर्भ में पाया गया — classified as Normal.', 'offensive_reason': '⚠️ "बिल्ली" आपत्तिजनक रूप में प्रयुक्त हुआ।' }, # English Context Words Disambiguation 'bitch': { 'safe_keywords': ['female', 'dog', 'puppy', 'puppies', 'litter', 'breeder', 'breeding', 'canine', 'animal', 'vet', 'veterinary'], 'safe_label': 'Normal', 'safe_reason': '🐕 "Bitch" detected in biological canine / breeding context (female dog) — classified as Normal.', 'offensive_reason': '⚠️ "Bitch" detected as misogynistic / targeted insult.' }, 'kill': { 'safe_keywords': ['stage', 'performance', 'show', 'song', 'dance', 'laughing', 'time', 'joke', 'awesome', 'crushed', 'vibe', 'killing'], 'safe_label': 'Normal', 'safe_reason': '⚡ "Kill" detected as colloquial performance praise / positive metaphor ("killing it") — classified as Normal.', 'offensive_reason': '🚨 "Kill" detected in violent threat context.' }, 'shoot': { 'safe_keywords': ['video', 'photo', 'film', 'camera', 'scene', 'movie', 'hoops', 'basketball', 'goal', 'picture', 'commercial'], 'safe_label': 'Normal', 'safe_reason': '🎥 "Shoot" detected in camera / video production or sports context — classified as Normal.', 'offensive_reason': '🚨 "Shoot" detected in violent firearm threat context.' }, 'pig': { 'safe_keywords': ['farm', 'animal', 'swine', 'piglet', 'piglets', 'mud', 'barn', 'agriculture', 'livestock'], 'safe_label': 'Normal', 'safe_reason': '🐖 "Pig" detected in farm animal / livestock context — classified as Normal.', 'offensive_reason': '⚠️ "Pig" detected as derogatory insult.' }, 'bomb': { 'safe_keywords': ['the', 'movie', 'party', 'track', 'album', 'awesome', 'cool', 'song'], 'safe_label': 'Normal', 'safe_reason': '💥 "The bomb" detected as informal slang for great / impressive — classified as Normal.', 'offensive_reason': '🚨 "Bomb" detected in explosive terror threat context.' }, 'sick': { 'safe_keywords': ['trick', 'move', 'skate', 'beat', 'music', 'game', 'cool', 'awesome', 'style', 'guitar', 'drop'], 'safe_label': 'Normal', 'safe_reason': '🛹 "Sick" detected as modern slang for impressive / awesome — classified as Normal.', 'offensive_reason': '⚠️ "Sick" detected in derogatory / harassment context.' }, # Telugu Context Words Disambiguation (Telugu Script & Romanized Tenglish) 'kukka': { 'safe_keywords': ['maa', 'intlo', 'chala', 'baguntundi', 'pet', 'cute', 'pillalu', 'pedda', 'chinna', 'illu', 'intiki', 'pempudu', 'prema'], 'safe_label': 'Normal', 'safe_reason': '🐕 "Kukka" detected in domestic pet context in Telugu (కుక్క / Pet Dog) — classified as Normal.', 'offensive_reason': '⚠️ "Kukka" detected as targeted abusive insult in Telugu.' }, 'కుక్క': { 'safe_keywords': ['మా', 'ఇంట్లో', 'చాలా', 'బాగుంటుంది', 'పెట్', 'ముద్దుగా', 'పిల్లలు', 'పెద్ద', 'చిన్న', 'రక్షణ', 'పెంపుడు', 'ఇంటి'], 'safe_label': 'Normal', 'safe_reason': '🐕 "కుక్క" domestic pet context (పెంపుడు కుక్క) లో కనుగొనబడింది — classified as Normal.', 'offensive_reason': '⚠️ "కుక్క" దూషణ / అవమానకరమైన పదంగా ఉపయోగించబడింది.' }, 'pichi': { 'safe_keywords': ['cricket', 'cinema', 'songs', 'chala', 'istam', 'prema', 'movie', 'abhimanam', 'hero'], 'safe_label': 'Normal', 'safe_reason': '🏏 "Pichi" detected in passionate fandom / hobby context in Telugu (cricket/cinema craze) — Normal.', 'offensive_reason': '⚠️ "Pichi" detected as derogatory mental insult in Telugu.' }, 'పిచ్చి': { 'safe_keywords': ['క్రికెట్', 'సినిమా', 'పాటలు', 'చాలా', 'ఇష్టం', 'ప్రేమ', 'అభిమానం', 'హీరో'], 'safe_label': 'Normal', 'safe_reason': '🏏 "పిచ్చి" క్రీడాభిమానం లేదా హాబీ సందర్భంలో ఉపయోగించబడింది — classified as Normal.', 'offensive_reason': '⚠️ "పిచ్చి" మానసిక అవమానకరమైన పదంగా ఉపయోగించబడింది.' }, 'donga': { 'safe_keywords': ['papa', 'babu', 'chinna', 'navvu', 'muddhu', 'pillalu', 'kanna', 'bujji'], 'safe_label': 'Normal', 'safe_reason': '👶 "Donga" detected as affectionate playful teasing of a child in Telugu — classified as Normal.', 'offensive_reason': '⚠️ "Donga" detected as criminal thief / derogatory insult in Telugu.' }, 'దొంగ': { 'safe_keywords': ['పాప', 'బాబు', 'చిన్న', 'నవ్వు', 'ముద్దు', 'పిల్లలు', 'కన్నా', 'బుజ్జి'], 'safe_label': 'Normal', 'safe_reason': '👶 "దొంగ" చిన్న పిల్లల ముద్దు పేచీ లేదా నవ్వు సందర్భంలో వాడబడింది — classified as Normal.', 'offensive_reason': '⚠️ "దొంగ" నేరపూరిత దూషణగా వాడబడింది.' }, 'gadida': { 'safe_keywords': ['polam', 'pashuvu', 'pani', 'baruvu', 'janthuvu', 'kasta', 'rythu'], 'safe_label': 'Normal', 'safe_reason': '🫏 "Gadida" detected in hardworking farm animal context in Telugu — classified as Normal.', 'offensive_reason': '⚠️ "Gadida" detected as intellectual ridicule / insult in Telugu.' }, 'padhna': { 'safe_keywords': ['padhna', 'padhai', 'padh', 'tu', 'tum', 'sher', 'kyun', 'kyon', 'kitab', 'school', 'exam', 'aaj', 'kal', 'toh', 'to', 'karo', 'kar', 'liya', 'nahi'], 'safe_label': 'Normal', 'safe_reason': '📚 "Padhna / Padhai" (Study & Education) context detected — classified as Normal & Encouraging.', 'offensive_reason': '⚠️ Context review.' }, 'पढ़ना': { 'safe_keywords': ['पढ़ना', 'पढ़', 'पढ़ाई', 'तू', 'तुम', 'शेर', 'क्यों', 'किताब', 'स्कूल', 'परीक्षा', 'एग्जाम', 'आज', 'कल', 'तो', 'करो', 'कर', 'लिया', 'नहीं', 'है'], 'safe_label': 'Normal', 'safe_reason': '📚 पढ़ाई / अध्ययन (Study & Education) के संदर्भ में पाया गया — classified as Normal & Encouraging.', 'offensive_reason': '⚠️ Context review.' }, 'पढ़': { 'safe_keywords': ['पढ़ना', 'पढ़', 'पढ़ाई', 'तू', 'तुम', 'शेर', 'क्यों', 'किताब', 'स्कूल', 'परीक्षा', 'एग्जाम', 'आज', 'कल', 'तो', 'करो', 'कर', 'लिया', 'नहीं', 'है'], 'safe_label': 'Normal', 'safe_reason': '📚 पढ़ाई / अध्ययन (Study & Education) के संदर्भ में पाया गया — classified as Normal & Encouraging.', 'offensive_reason': '⚠️ Context review.' }, 'शेर': { 'safe_keywords': ['तू', 'तुम', 'आयुष', 'बेटा', 'भाई', 'दोस्त', 'जंगल', 'पशु', 'जानवर', 'सर्कस', 'बहादुर', 'शाबाश', 'पढ़ना', 'काम', 'तो'], 'safe_label': 'Normal', 'safe_reason': '🦁 "शेर" bravery / affection / encouragement context (शेर बेटा / बहादुर) mein paya gaya — classified as Normal.', 'offensive_reason': '⚠️ Context review.' }, 'sher': { 'safe_keywords': ['tu', 'tum', 'ayush', 'bhai', 'dost', 'jungle', 'animal', 'lion', 'brave', 'padhna', 'kaam', 'shabash', 'toh'], 'safe_label': 'Normal', 'safe_reason': '🦁 "Sher" detected as praise / brave / affection ("sher beta") — classified as Normal.', 'offensive_reason': '⚠️ Context review.' } } # ============================================================ # BENIGN EVERYDAY VOCABULARY GUARD (Prevents False Positives) # ============================================================ BENIGN_DAILY_WORDS = { # Education & Study 'पढ़ना', 'पढ़', 'पढ़ाई', 'पढ़ता', 'पढ़ती', 'पढ़ते', 'पढ़ाना', 'किताब', 'किताबें', 'स्कूल', 'कॉलेज', 'परीक्षा', 'एग्जाम', 'होमवर्क', 'padhna', 'padhai', 'padh', 'study', 'studying', 'studies', 'school', 'college', 'exam', 'book', 'books', 'homework', 'learn', 'learning', # Encouragement / Praise 'शेर', 'बहादुर', 'शाबाश', 'प्यारा', 'प्यारी', 'दोस्त', 'भाई', 'sher', 'brave', 'proud', 'champion', 'shabash', 'accha', # Daily activities & questions 'खाना', 'सोना', 'काम', 'जाना', 'आना', 'बात', 'गाना', 'खेलना', 'खेल', 'मदद', 'समय', 'वक्त', 'kaam', 'khana', 'soch', 'baat', 'khel', 'help', 'time', 'क्यों', 'कहाँ', 'कब', 'कैसे', 'क्या', 'kyon', 'kyun', 'kahan', 'kaise', 'kya', 'why', 'how', 'when', 'what', 'where' } # ============================================================ # TIER 1: SEVERE HATE SPEECH & VIOLENT THREATS DATABASE (Multilingual) # ============================================================ EXPLICIT_HATE_SPEECH = { # Devanagari Violent & Lethal Threats (Hindi/Marathi/Phonetic English) "मार दूंगा", "मार डालूंगा", "जान से मार", "खत्म कर दूंगा", "गोली मार", "काट दूंगा", "उड़ा दूंगा", "ज़िंदा जला", "जिंदा जला", "चीर दूंगा", "फाड़ दूंगा", "गला काट", "तबाह कर दूंगा", "मारून टाकीन", "जीव घेईन", "मारना चाहता", "मारना चाहता हूं", "मारना चाहता हु", "मारना चाहती", "मारना चाहते", "मार डालना चाहता", "पीट दूंगा", "जान लेना चाहता", "खत्म करना चाहता", "मारने आया हूं", "मारने आया", "हत्या कर दूंगा", # Phonetic English in Devanagari (from speech-to-text or typing) "आई वांट टू किल यू", "वांट टू किल यू", "आई विल किल यू", "विल किल यू", "किल यू", "शूट यू", "मर्डर यू", "किल कर दूंगा", # Roman / Hinglish Violent Threats (with single & double 'a') "maar dunga", "mar dunga", "maar dalunga", "mar dalunga", "jaan se maar", "jaan se mar", "jaan se maar dunga", "jaan se mar dunga", "tujhe maar dunga", "tujhe mar dunga", "marna chahta", "marna chahta hoon", "marna chahti", "mar dalna chahta", "marne aaya", "marne aaya hu", "jaan lena chahta", "khatam karna chahta", "khatam kar dunga", "khatm kar dunga", "goli maar", "goli mar", "marunga", "maarunga", "kaat dunga", "uda dunga", "zinda jala", "cheer dunga", "tabah kar dunga", # Telugu Violent Threats (Telugu Script & Tenglish) "చంపేస్తా", "చంపేస్తాను", "నిన్ను చంపేస్తా", "నరికేస్తా", "నరికేస్తాను", "ప్రాణం తీస్తా", "ఖతం చేస్తా", "రక్తం తాగుతా", "champesta", "champestanu", "ninnu champesta", "narikesta", "narikestanu", "pranam teesta", "katham chesta", "raktam taagutha", # Tamil & Bengali Violent Threats "கொன்னுடுவேன்", "kondruven", "মেরে ফেলবো", "mere phelbo", # English Violent Threats & Hate "i will kill you", "will kill you", "i want to kill you", "want to kill you", "kill you scum", "kill you", "murder you", "shoot you", "cut your throat", "burn you", "burn to death", "die you absolute trash", "kill yourself", "die you scum", # Communal / Identity / Terror Hate Markers "आतंकवादी", "जिहादी", "गद्दार", "देशद्रोही", "काफिर", "chakka", "hijra", "katue", "terrorist", "nazi", "faggot", "faggots", "kill all" } # ============================================================ # TIER 2: CASUAL PROFANITY & OFFENSIVE SLANG DATABASE (Multi-Script) # ============================================================ EXPLICIT_OFFENSIVE = { # Devanagari Hindi Profanities & Informal Slang "हरामज़ादे", "हरामजादे", "हरामजादा", "हरामज़ादा", "हरामी", "हरामखोर", "भोसड़ीके", "भोसड़ीके", "भोसडीके", "भोसडे", "भोसड़ा", "भोसुडी", "भोस्डी", "भोसड़ी", "मादरचोद", "मदरचोद", "मादरजात", "बहनचोद", "बहेनचोद", "बेहेनचोद", "चूतिया", "चूतिये", "चूतियापा", "चूत", "कमीना", "कमीने", "कमीनी", "लौड़े", "लौंडे", "लंड", "लौड़ा", "लवड़े", "लवड़ा", "लावडे", "लावडा", "गांडू", "गांड", "रांडी", "रंडी", "झांटू", "झांट", "भाड़वे", "भड़वे", "भड़वा", "सूअर", "चोदू", "चोमू", "दलाल", "रंडवा", # Roman / Hinglish Profanities & Slangs "haramzade", "haramzada", "haramzadi", "harami", "haramkhor", "bhosdike", "bhosdika", "bhosadi", "bhosadike", "bsdk", "bhosdk", "bhosdi", "boshde", "madarchod", "maderchod", "mc", "behenchod", "bhenchod", "bc", "bhenkelaude", "chutiya", "chutiye", "chut", "chutiyapa", "gaand", "gandu", "kamina", "kamine", "laude", "lodu", "loda", "lund", "lavde", "lavda", "randi", "jhantu", "jhant", "bhadwe", # Telugu Profanities & Slangs (Telugu Script & Tenglish) "లంచోడు", "దొంగ నా కొడకా", "దవడ పగలగొడతా", "వెధవ", "నీ యమ్మ", "నీ అమ్మ", "ముండ", "బోకే", "లంజ", "గుద్ద", "vedhava", "naa kodaka", "donga na kodaka", "lanja", "lanjamunda", "mundamopi", "boke", "nee amma", "gudha", "dengey", # English Slurs / Profanities "fuck", "fucking", "fcker", "bitch", "bastard", "asshole", "whore", "slut", "cunt", "moron", "retarded", "retarted" } class HateSpeechPredictor: def __init__(self, models_dir='models'): model_path = os.path.join(models_dir, 'model.pkl') vectorizer_path = os.path.join(models_dir, 'vectorizer.pkl') self.model = None self.vectorizer = None if os.path.exists(model_path) and os.path.exists(vectorizer_path): try: self.model = joblib.load(model_path) self.vectorizer = joblib.load(vectorizer_path) except Exception as e: print(f"ℹ️ [Predictor] Classical model load skipped ({e}). Relying on fine-tuned Transformer.") self.bert = MultilingualBertEngine() if MultilingualBertEngine else None def detect_context(self, text): text_lower = text.lower() words_in_text = set(re.findall(r'[\w\u0900-\u097F\u0C00-\u0C7F\u0B80-\u0BFF\u0980-\u09FF]+', text_lower)) for context_word, config in CONTEXT_WORDS.items(): if context_word in words_in_text or context_word in text_lower: safe_hits = words_in_text.intersection(set(config['safe_keywords'])) if safe_hits: return { 'found_word': context_word, 'is_safe': True, 'override_label': config['safe_label'], 'reason': config['safe_reason'], 'context_keywords': list(safe_hits) } else: return { 'found_word': context_word, 'is_safe': False, 'override_label': None, 'reason': config['offensive_reason'], 'context_keywords': [] } return None def detect_negated_threats(self, text): """ Detects statements where violent threats are explicitly negated, questioned, or denied. e.g. "I don't want to kill you", "why would I kill you", "mai tumhe nahi marna chahta" """ text_lower = text.lower() negation_patterns = [ r"(?:don'?t|do\s+not|never|wouldn'?t|won'?t|not|no\s+intention\s+to|why\s+would\s+i|refuse\s+to|didn'?t)\s+(?:even\s+)?(?:want\s+to\s+|try\s+to\s+|going\s+to\s+|plan\s+to\s+)?(?:kill|murder|harm|shoot|hurt|attack|burn|destroy)", r"(?:nahi|nhi|na|kabhi\s+nahi)\s+(?:marna|maarna|jaan\s+se\s+marna|khatam\s+karna|thokna)", r"(?:marna|maarna|jaan\s+se\s+marna)\s+(?:nahi|nhi)\s+chahta", r"मारना\s+नहीं\s+चाहता|मारना\s+नहीं\s+चाहती|नहीं\s+मारूंगा|नहीं\s+मारना|जान\s+से\s+नहीं\s+मारना|क्यों\s+मारूंगा|मारना\s+नहीं\s+है" ] for pat in negation_patterns: m = re.search(pat, text_lower, re.IGNORECASE) if m: return m.group(0) return None def detect_hate_threats(self, text): text_lower = text.lower() # If statement contains an explicit negation/denial of threat, it is NOT a threat if self.detect_negated_threats(text): return [] matched = [] for threat in EXPLICIT_HATE_SPEECH: if threat in text_lower: matched.append(threat) # Robust Regex Patterns for flexible Hinglish, Hindi, Telugu, and English death threats if not matched: patterns = [ r'(?:tujhe|tumhe|tere\s*ko|tera)?\s*(?:jaan\s*s[e|y]\s*)?m[a]+r\s*(?:d[u|o]ng[a|e|i]|d[a]*lung[a|e|i])', r'm[a]+rn[a|e]\s*ch[a]+ht[a|e|i]', r'मारना\s*चाहता|मारना\s*चाहती|मारना\s*चाहते|मार\s*डालना\s*चाहता|जान\s*लेना\s*चाहता|खत्म\s*करना\s*चाहता', r'm[a]+rung[a|e|i]', r'khat[a]*m\s*kar\s*d[u|o]ng[a|e|i]', r'goli\s*m[a]+r', r'uda\s*d[u|o]ng[a|e|i]', r'मौत|तेरी\s*मौत|maut|teri\s*maut', r'कर\s*डालूंगा|मार\s*डालूंगा|काट\s*डालूंगा|फाड़\s*डालूंगा|उड़ा\s*डालूंगा|चीर\s*डालूंगा', r'(?:kar|mar|maar|kaat|faad|khatam|uda)\s*(?:d[a]*lung[a|e|i]|d[u|o]ng[a|e|i])', r'(?:i\s+)?(?:want\s+to|will|gonna|going\s+to|wanna)\s+kill\s+you', r'kill\s+you', r'आई\s*वांट\s*टू\s*किल\s*यू|वांट\s*टू\s*किल|किल\s*यू|आई\s*विल\s*किल\s*यू|शूट\s*यू', r'(?:ninnu\s+)?champ[e|a]st[a|u]|narik[e|a]st[a|u]|pranam\s*teest[a|u]', r'చంపేస్తా|నరికేస్తా|ప్రాణం\s*తీస్తా' ] for pat in patterns: m = re.search(pat, text_lower, re.IGNORECASE) if m: matched.append(m.group(0)) break return matched def detect_offensive_slurs(self, text): text_lower = text.lower() matched = [] for slur in EXPLICIT_OFFENSIVE: if slur in text_lower: matched.append(slur) return matched def predict(self, text): cleaned_text = clean_text(text) if self.bert and (self.bert.is_custom_finetuned or self.bert._is_ready): try: bert_res = self.bert.predict(text) label = bert_res['prediction'] except Exception: label = 'Normal' elif self.model and self.vectorizer: vectorized_text = self.vectorizer.transform([cleaned_text]) prediction = self.model.predict(vectorized_text)[0] label = str(prediction) elif self.bert: bert_res = self.bert.predict(text) label = bert_res['prediction'] else: label = 'Normal' # 0. Check Explicit Negation / Peaceful Denial of Violence (e.g. "I don't want to kill you", "nahi marna chahta") negated_phrase = self.detect_negated_threats(text) if negated_phrase: return { 'original_text': text, 'cleaned_text': cleaned_text, 'prediction': 'Normal', 'context': { 'found_word': negated_phrase, 'is_safe': True, 'category': 'Peaceful Denial / Negation', 'reason': f"🕊️ Peaceful Negation detected: '{negated_phrase}' — explicit denial of violent intent. Classified as Normal & Safe." } } # 1. Check Explicit Severe Hate Speech & Violent Threats (Top Priority: Threats are NEVER excused) matched_hate = self.detect_hate_threats(text) if matched_hate: return { 'original_text': text, 'cleaned_text': cleaned_text, 'prediction': 'Hate Speech', 'context': { 'found_word': matched_hate[0], 'is_safe': False, 'category': 'Hate Speech', 'reason': f"🚨 Violent Threat / Hate Speech detected: '{matched_hate[0]}'" } } # 2. Check Context Overrides (Religious/Animal/Study/Encouragement relation) context_info = self.detect_context(text) if context_info and context_info.get('is_safe'): return { 'original_text': text, 'cleaned_text': cleaned_text, 'prediction': 'Normal', 'context': context_info } # 3. Check Explicit Casual Abusive / Profanities (e.g. हरामजादे, चूतिया) matched_offensive = self.detect_offensive_slurs(text) if matched_offensive: return { 'original_text': text, 'cleaned_text': cleaned_text, 'prediction': 'Offensive', 'context': { 'found_word': matched_offensive[0], 'is_safe': False, 'category': 'Offensive Language', 'reason': f"⚠️ Casual / Abusive Slang detected: '{matched_offensive[0]}' (Informal profanity, not hate speech)" } } # 4. Context words in non-safe context (e.g. 'ghanta' without bell context) if context_info and not context_info.get('is_safe'): return { 'original_text': text, 'cleaned_text': cleaned_text, 'prediction': 'Offensive', 'context': context_info } # 5. Respect Neural Transformer prediction return { 'original_text': text, 'cleaned_text': cleaned_text, 'prediction': label, 'context': { 'found_word': 'Contextual Neural Inference', 'is_safe': label == 'Normal', 'category': label, 'reason': f"🧠 Neural Model classified as {label} based on semantic context." } } if __name__ == "__main__": predictor = HateSpeechPredictor() tests = [ "आयुष हरामजादे पढ़ लिया कर यार पढ़ लिया", "Mandir mein ghanta baj raha hai", "Ghanta farak padta hai tujhe", "Mera kutta bahut pyaara hai", "Kutte kamine jaan se maar dunga tujhe", "नमस्ते, आप कैसे हैं?" ] for t in tests: result = predictor.predict(t) ctx = result.get('context') ctx_info = f" | Context: {ctx['reason']}" if ctx else "" print(f"Text: {t} -> {result['prediction']}{ctx_info}")