|
|
| import os |
| import re |
| import sys |
| import requests |
| import pandas as pd |
| from datasets import load_dataset |
| from sklearn .model_selection import train_test_split |
|
|
| DATA_DIR =os .path .dirname (os .path .abspath (__file__ )) |
|
|
| def separator (title ): |
| print ("\n"+"="*60 ) |
| print (f" {title }") |
| print ("="*60 ) |
|
|
| def clean_text (text ): |
| if not isinstance (text ,str ): |
| return "" |
| text =re .sub (r'http\S+','',text ) |
| text =re .sub (r'@\w+','',text ) |
| text =re .sub (r'#\w+','',text ) |
| text =re .sub (r'\s+',' ',text ) |
| return text .strip () |
|
|
| def split_and_save (df ,name ,text_col ,label_col ): |
| df =df [[text_col ,label_col ]].rename (columns ={text_col :'text',label_col :'label'}) |
| df =df .dropna (subset =['text','label']) |
| df ['text']=df ['text'].apply (clean_text ) |
| df =df [df ['text'].str .len ()>2 ].reset_index (drop =True ) |
|
|
| train ,temp =train_test_split (df ,test_size =0.20 ,random_state =42 ,stratify =df ['label']) |
| val ,test =train_test_split (temp ,test_size =0.50 ,random_state =42 ,stratify =temp ['label']) |
|
|
| for split ,data in [('train',train ),('val',val ),('test',test )]: |
| path =os .path .join (DATA_DIR ,f'{name }_{split }.csv') |
| data .to_csv (path ,index =False ,encoding ='utf-8-sig') |
| print (f" Saved {path } ({len (data )} rows)") |
|
|
| return train ,val ,test |
|
|
| separator ("DATASET 1: Roman Urdu Sentiment (HuggingFace)") |
| try : |
| ds1 =load_dataset ('community-datasets/roman_urdu',trust_remote_code =True ) |
| print ("Available splits:",list (ds1 .keys ())) |
| df1 =ds1 ['train'].to_pandas () |
| print (f"Columns : {df1 .columns .tolist ()}") |
| print (f"Shape : {df1 .shape }") |
| print (f"Sample row :\n{df1 .iloc [0 ]}") |
| print (f"\nLabel dist. :\n{df1 .iloc [:,-1 ].value_counts ()}") |
|
|
| text_col =df1 .columns [0 ] |
| label_col =df1 .columns [-1 ] |
| print (f"\nUsing text='{text_col }' label='{label_col }'") |
| split_and_save (df1 ,'roman_urdu_sentiment',text_col ,label_col ) |
| print ("Dataset 1 DONE") |
| except Exception as e : |
| print (f"ERROR loading Dataset 1: {e }") |
|
|
| separator ("DATASET 2: SemEval 2018 Task 1 - Emotion (HuggingFace)") |
| try : |
| ds2 =load_dataset ('SemEvalWorkshop/sem_eval_2018_task_1','subtask5.english',trust_remote_code =True ) |
| print ("Available splits:",list (ds2 .keys ())) |
|
|
| frames =[] |
| for split_name in ds2 .keys (): |
| tmp =ds2 [split_name ].to_pandas () |
| frames .append (tmp ) |
| df2_all =pd .concat (frames ,ignore_index =True ) |
|
|
| print (f"Columns : {df2_all .columns .tolist ()}") |
| print (f"Shape : {df2_all .shape }") |
| print (f"Sample :\n{df2_all .iloc [0 ]}") |
|
|
| emotion_cols =['anger','anticipation','disgust','fear','joy', |
| 'love','optimism','pessimism','sadness','surprise','trust'] |
| target_emotions =['joy','anger','fear','sadness'] |
| available =[c for c in target_emotions if c in df2_all .columns ] |
|
|
| if available : |
| df2_all ['label']=df2_all [available ].idxmax (axis =1 ) |
| df2_all ['max_score']=df2_all [available ].max (axis =1 ) |
| df2_all =df2_all [df2_all ['max_score']>0 ] |
| text_col2 ='Tweet'if 'Tweet'in df2_all .columns else df2_all .columns [0 ] |
| print (f"\nEmotion dist.:\n{df2_all ['label'].value_counts ()}") |
| split_and_save (df2_all ,'semeval_emotion',text_col2 ,'label') |
| print ("Dataset 2 DONE") |
| else : |
| print (f"Emotion columns not found. Available: {df2_all .columns .tolist ()}") |
| except Exception as e : |
| print (f"ERROR loading Dataset 2: {e }") |
| print ("Trying alternate config...") |
| try : |
| print ("Available configs:") |
| from datasets import get_dataset_config_names |
| configs =get_dataset_config_names ('SemEvalWorkshop/sem_eval_2018_task_1') |
| print (configs ) |
| except Exception as e2 : |
| print (f"Could not list configs: {e2 }") |
|
|
| separator ("DATASET 3: mirfan899 Urdu Sentiment TSV (GitHub Download)") |
| tar_path =os .path .join (DATA_DIR ,'urdu.tsv.tar.gz') |
| tsv_path =os .path .join (DATA_DIR ,'urdu_v1.tsv') |
| try : |
| if not os .path .exists (tsv_path ): |
| url ='https://raw.githubusercontent.com/mirfan899/Urdu/master/sentiment/urdu.tsv.tar.gz' |
| print (f"Downloading from: {url }") |
| r =requests .get (url ,timeout =30 ) |
| r .raise_for_status () |
| with open (tar_path ,'wb')as f : |
| f .write (r .content ) |
| print (f"Downloaded successfully ({len (r .content )} bytes)") |
|
|
| import tarfile |
| with tarfile .open (tar_path ,"r:gz")as tar : |
| tar .extractall (path =DATA_DIR ) |
| print ("Extracted urdu_v1.tsv") |
|
|
| for sep in ['\t',',',';']: |
| try : |
| df3 =pd .read_csv (tsv_path ,sep =sep ,header =0 ,encoding ='utf-8') |
| if df3 .shape [1 ]>=2 : |
| break |
| except : |
| continue |
|
|
| print (f"Columns : {df3 .columns .tolist ()}") |
| print (f"Shape : {df3 .shape }") |
| print (f"\nLabel dist.:\n{df3 .iloc [:,-1 ].value_counts ()}") |
|
|
| split_and_save (df3 ,'mirfan_urdu_sentiment',df3 .columns [0 ],df3 .columns [-1 ]) |
| print ("Dataset 3 DONE") |
| except Exception as e : |
| print (f"ERROR loading Dataset 3: {e }") |
|
|
| separator ("DATASET 4: Urdu Sentiment Corpus (GitHub Download)") |
| tsv4_path =os .path .join (DATA_DIR ,'urdu-sentiment-corpus-v1.tsv') |
| try : |
| if not os .path .exists (tsv4_path ): |
| url4 ='https://raw.githubusercontent.com/MuhammadYaseenKhan/Urdu-Sentiment-Corpus/master/urdu-sentiment-corpus-v1.tsv' |
| print (f"Downloading from: {url4 }") |
| r4 =requests .get (url4 ,timeout =30 ) |
| r4 .raise_for_status () |
| with open (tsv4_path ,'wb')as f : |
| f .write (r4 .content ) |
| print (f"Downloaded successfully ({len (r4 .content )} bytes)") |
|
|
| for enc in ['utf-8','utf-8-sig','cp1252','latin-1']: |
| try : |
| df4 =pd .read_csv (tsv4_path ,sep ='\t',encoding =enc ) |
| break |
| except : |
| continue |
|
|
| print (f"Columns : {df4 .columns .tolist ()}") |
| print (f"Shape : {df4 .shape }") |
|
|
| text_col4 =df4 .columns [0 ] |
| label_col4 =df4 .columns [-1 ] |
| print (f"\nLabel dist.:\n{df4 [label_col4 ].value_counts ()}") |
| split_and_save (df4 ,'urdu_sentiment_corpus',text_col4 ,label_col4 ) |
| print ("Dataset 4 DONE") |
| except Exception as e : |
| print (f"ERROR loading Dataset 4: {e }") |
|
|
| separator ("PHASE 2 COMPLETE — Summary of saved files") |
| all_files =[f for f in os .listdir (DATA_DIR )if f .endswith ('.csv')] |
| print (f"{'File':<45} {'Rows':>6}") |
| print ("-"*55 ) |
| for f in sorted (all_files ): |
| path =os .path .join (DATA_DIR ,f ) |
| try : |
| n =len (pd .read_csv (path ,encoding ='utf-8-sig')) |
| print (f"{f :<45} {n :>6}") |
| except : |
| print (f"{f :<45} (error reading)") |
| print ("\nAll datasets collected, cleaned, and split. Ready for Phase 3.") |
|
|
