Spaces:
Sleeping
Sleeping
File size: 7,832 Bytes
77c5607 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | """
utils/multilingual_loader.py
-----------------------------
Loads Hindi and Hinglish test datasets for multilingual evaluation.
Used ONLY for evaluation — not added to training data.
"""
import pandas as pd
import numpy as np
import re
from datasets import load_dataset as hf_load
def detect_language(text: str) -> str:
"""
Detect if a message is:
- 'hindi' : contains Devanagari Unicode characters (pure Hindi)
- 'hinglish': Roman script with Hindi words mixed with English
- 'english' : standard English
"""
# Check for Devanagari Unicode range (0900-097F)
devanagari_pattern = re.compile(r'[\u0900-\u097F]')
if devanagari_pattern.search(text):
return 'hindi'
# Common Hinglish words that appear in Roman script
hinglish_markers = {
'aapka', 'aapke', 'abhi', 'karo', 'kare', 'kiya', 'nahi', 'nai',
'hai', 'hain', 'ho', 'ka', 'ki', 'ke', 'ko', 'se', 'mein', 'par',
'aur', 'bhi', 'yeh', 'woh', 'ap', 'apka', 'jana', 'karna', 'dena',
'lena', 'milega', 'milegi', 'paisa', 'rupaye', 'khata', 'band',
'verify', 'kijiye', 'dijiye', 'lijiye', 'phir', 'agar', 'toh',
'jaldi', 'turant', 'abhi', 'sirf', 'bahut', 'accha', 'theek'
}
words = set(text.lower().split())
if len(words.intersection(hinglish_markers)) >= 2:
return 'hinglish'
return 'english'
def load_hindi_test_data() -> pd.DataFrame:
"""
Load Hindi (Devanagari) test data from HuggingFace multilingual dataset.
Returns DataFrame with columns: message, label, language
"""
print("Loading Hindi test data from HuggingFace...")
try:
ds = hf_load("dbarbedillo/SMS_Spam_Multilingual_Collection_Dataset")
dfs = []
for split_name in ds.keys():
split_df = ds[split_name].to_pandas()
dfs.append(split_df)
df = pd.concat(dfs, ignore_index=True)
# This dataset has columns for each language
# Find the Hindi column
hindi_col = None
for col in df.columns:
if 'hindi' in col.lower() or 'hi' == col.lower():
hindi_col = col
break
if hindi_col is None:
# Try text_hi column (actual name in this dataset)
if 'text_hi' in df.columns:
hindi_col = 'text_hi'
else:
# Auto-detect by Devanagari content
skip = {'label', 'labels', 'v1', 'v2'}
for col in df.columns:
if col in skip:
continue
sample = df[col].dropna().head(10).astype(str)
devanagari = re.compile(r'[\u0900-\u097F]')
if any(devanagari.search(s) for s in sample):
hindi_col = col
break
if hindi_col is None:
print(" Warning: Could not find Hindi column")
return pd.DataFrame(columns=['message', 'label', 'language'])
# Get label column — could be 'v1', 'label', or 'labels' (case-insensitive)
col_lower_map = {c.lower().strip(): c for c in df.columns}
label_col = next((col_lower_map[c] for c in ['v1', 'label', 'labels'] if c in col_lower_map), None)
if label_col is None:
print(f" Warning: No label column found. Got: {df.columns.tolist()}")
return pd.DataFrame(columns=['message', 'label', 'language'])
result_df = pd.DataFrame({
'message': df[hindi_col].astype(str),
'label': df[label_col]
})
# Standardise labels
label_map = {'ham': 0, 'spam': 1, 0: 0, 1: 1, '0': 0, '1': 1}
result_df['label'] = result_df['label'].map(label_map)
result_df = result_df.dropna(subset=['message', 'label'])
result_df['label'] = result_df['label'].astype(int)
result_df['language'] = 'hindi'
print(f" Hindi test data: {len(result_df)} messages "
f"({result_df['label'].sum()} spam, "
f"{(result_df['label']==0).sum()} ham)")
return result_df.reset_index(drop=True)
except Exception as e:
print(f" Warning: Failed to load Hindi data: {e}")
return pd.DataFrame(columns=['message', 'label', 'language'])
def load_hinglish_test_data() -> pd.DataFrame:
"""
Load Hinglish test data from Kaggle multilingual spam dataset.
Returns DataFrame with columns: message, label, language
"""
print("Loading Hinglish test data from Kaggle...")
try:
import kagglehub
path = kagglehub.dataset_download("rajnathpatel/multilingual-spam-data")
import os
csv_files = [f for f in os.listdir(path) if f.endswith('.csv')]
if not csv_files:
print(" Warning: No CSV files in Kaggle multilingual dataset")
return pd.DataFrame(columns=['message', 'label', 'language'])
dfs = []
for csv_file in csv_files:
try:
df = pd.read_csv(f"{path}/{csv_file}", encoding='latin-1')
dfs.append(df)
except Exception:
continue
if not dfs:
return pd.DataFrame(columns=['message', 'label', 'language'])
df = pd.concat(dfs, ignore_index=True)
# Standardise column names
col_lower = {c: c.lower().strip() for c in df.columns}
df = df.rename(columns=col_lower)
# Find message and label columns
msg_col = next((c for c in df.columns if c in ['text', 'message', 'sms', 'v2']), None)
lbl_col = next((c for c in ['label', 'labels', 'v1', 'category', 'class'] if c in df.columns), None)
if not msg_col or not lbl_col:
print(f" Warning: Could not identify columns. Found: {df.columns.tolist()}")
return pd.DataFrame(columns=['message', 'label', 'language'])
result_df = pd.DataFrame({
'message': df[msg_col].astype(str),
'label': df[lbl_col]
})
label_map = {'ham': 0, 'spam': 1, 0: 0, 1: 1, '0': 0, '1': 1}
result_df['label'] = result_df['label'].map(label_map)
result_df = result_df.dropna(subset=['message', 'label'])
result_df['label'] = result_df['label'].astype(int)
# Detect language per message and keep only Hindi/Hinglish
result_df['language'] = result_df['message'].apply(detect_language)
result_df = result_df[result_df['language'].isin(['hindi', 'hinglish'])]
print(f" Hinglish/Hindi test data: {len(result_df)} messages "
f"({result_df['label'].sum()} spam, "
f"{(result_df['label']==0).sum()} ham)")
return result_df.reset_index(drop=True)
except Exception as e:
print(f" Warning: Failed to load Hinglish data: {e}")
return pd.DataFrame(columns=['message', 'label', 'language'])
def load_all_multilingual_test_data() -> dict[str, pd.DataFrame]:
"""
Loads all multilingual test data.
Returns dict with keys: 'hindi', 'hinglish', 'combined'
"""
hindi_df = load_hindi_test_data()
hinglish_df = load_hinglish_test_data()
# Separate hinglish from hindi in the combined df
pure_hindi = pd.concat([
hindi_df,
hinglish_df[hinglish_df['language'] == 'hindi']
], ignore_index=True)
pure_hinglish = hinglish_df[hinglish_df['language'] == 'hinglish'].reset_index(drop=True)
combined = pd.concat([pure_hindi, pure_hinglish], ignore_index=True)
print(f"\nTotal multilingual test data:")
print(f" Pure Hindi: {len(pure_hindi)} messages")
print(f" Hinglish: {len(pure_hinglish)} messages")
print(f" Combined: {len(combined)} messages")
return {
'hindi': pure_hindi,
'hinglish': pure_hinglish,
'combined': combined
}
|