Download scripts/01_prep_data.py from AbhijithNusamNu/scam_dectect_sheid: direct link, hf CLI and curl.
- Browser
- Download file 7.11 kB
-
https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/01_prep_data.py
- Command line
-
hf download hf://spaces/AbhijithNusamNu/scam_dectect_sheid/scripts/01_prep_data.py
-
curl -L -o 01_prep_data.py https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/01_prep_data.py
7.11 kB
| """ | |
| Prepares the final dataset by explicitly constructing a balanced Train and Test set. | |
| Train Set: 100% of LLM data + Teeconnie + Legacy (2550 rows, perfectly balanced) | |
| Test Set: Completely unseen Teeconnie + Legacy (4000 rows, perfectly balanced) | |
| """ | |
| import os | |
| import json | |
| import random | |
| import re | |
| import glob | |
| import zipfile | |
| import pandas as pd | |
| import requests | |
| import dagshub | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| SEED = 42 | |
| random.seed(SEED) | |
| def clean_text(text): | |
| text = str(text) | |
| text = re.sub(r'(?i)(innocent|suspect):\s*', '', text) | |
| text = re.sub(r'\[.*?\]', '', text) | |
| text = re.sub(r'\s+', ' ', text).strip() | |
| return text.lower() | |
| def main(): | |
| repo_owner = os.getenv("DAGSHUB_REPO_OWNER") | |
| repo_name = os.getenv("DAGSHUB_REPO_NAME") | |
| username = os.getenv("MLFLOW_TRACKING_USERNAME") | |
| password = os.getenv("MLFLOW_TRACKING_PASSWORD") | |
| if not (repo_owner and repo_name): | |
| print("ERROR: DAGSHUB_REPO_OWNER and DAGSHUB_REPO_NAME must be set in .env") | |
| return | |
| # Map MLflow credentials to DagsHub authentication | |
| if username and password: | |
| os.environ["DAGSHUB_USER"] = username | |
| os.environ["DAGSHUB_TOKEN"] = password | |
| dagshub.auth.add_app_token(password) | |
| # Initialize DagsHub S3 client | |
| repo_id = f"{repo_owner}/{repo_name}" | |
| s3_client = dagshub.get_repo_bucket_client(repo_id) | |
| # 1. Load Synthesized JSON data (LLM Data) | |
| print("Loading synthesized JSON data from DagsHub...") | |
| json_dir = "data/raw_jsons" | |
| os.makedirs(json_dir, exist_ok=True) | |
| json_files = ["scam_call_hard_examples_250.json", "scam_call_transcripts_250_combined.json"] | |
| synth_data = [] | |
| for fname in json_files: | |
| path = os.path.join(json_dir, fname) | |
| s3_key = f"data/raw_jsons/{fname}" | |
| try: | |
| s3_client.download_file(repo_name, s3_key, path) | |
| except Exception as e: | |
| pass # ignore if exists locally | |
| if os.path.exists(path): | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| synth_data.extend(data) | |
| df_synth = pd.DataFrame(synth_data) | |
| df_synth["text"] = df_synth["text"].apply(clean_text) | |
| synth_scams = df_synth[df_synth["label"] == 1] | |
| synth_legits = df_synth[df_synth["label"] == 0] | |
| print(f"Loaded LLM Data: {len(synth_scams)} Scams, {len(synth_legits)} Legits.") | |
| # 2. Download and Load Teeconnie dataset | |
| print("\nDownloading teeconnie dataset from DagsHub S3...") | |
| teeconnie_zip = "data/raw_teeconnie/teeconnie_dataset.zip" | |
| os.makedirs("data/raw_teeconnie", exist_ok=True) | |
| s3_key = "data/raw_teeconnie/teeconnie_dataset.zip" | |
| try: | |
| s3_client.download_file(repo_name, s3_key, teeconnie_zip) | |
| except Exception as e: | |
| pass | |
| with zipfile.ZipFile(teeconnie_zip, 'r') as zipf: | |
| zipf.extractall("data/raw_teeconnie/") | |
| teeconnie_files = glob.glob("data/raw_teeconnie/**/*", recursive=True) | |
| nonscam_txt = [f for f in teeconnie_files if f.lower().endswith(".txt") and "non" in f.lower() and "scam" in f.lower()] | |
| entries = [] | |
| if nonscam_txt: | |
| with open(nonscam_txt[0], "r", encoding="utf-8", errors="ignore") as f: | |
| content = f.read() | |
| entries = [e.strip() for e in content.split("\n\n") if e.strip()] | |
| if len(entries) < 5: | |
| entries = [e.strip() for e in content.split("\n") if e.strip()] | |
| random.shuffle(entries) | |
| df_teeconnie_full = pd.DataFrame({ | |
| "text": entries, | |
| "label": [0] * len(entries) | |
| }) | |
| df_teeconnie_full["text"] = df_teeconnie_full["text"].apply(clean_text) | |
| # We need 350 for train, 1000 for test | |
| teeconnie_train = df_teeconnie_full.iloc[:350] | |
| teeconnie_test = df_teeconnie_full.iloc[350:1350] | |
| print(f"Sampled Teeconnie Data: {len(teeconnie_train)} Train Legits, {len(teeconnie_test)} Test Legits.") | |
| # 3. Download and Load Legacy Kaggle Dataset | |
| print("\nDownloading Legacy Kaggle composite dataset from DagsHub S3...") | |
| csv1 = "data/legacy_composite/composite_train.csv" | |
| csv2 = "data/legacy_composite/composite_test.csv" | |
| os.makedirs("data/legacy_composite", exist_ok=True) | |
| try: | |
| s3_client.download_file(repo_name, csv1, csv1) | |
| s3_client.download_file(repo_name, csv2, csv2) | |
| except Exception as e: | |
| pass | |
| df1 = pd.read_csv(csv1) | |
| df2 = pd.read_csv(csv2) | |
| df_legacy_full = pd.concat([df1, df2], ignore_index=True) | |
| df_legacy_full = df_legacy_full.dropna(subset=["text", "label"]) | |
| df_legacy_full["text"] = df_legacy_full["text"].apply(clean_text) | |
| legacy_scams_full = df_legacy_full[df_legacy_full["label"] == 1].sample(frac=1, random_state=SEED) | |
| legacy_legits_full = df_legacy_full[df_legacy_full["label"] == 0].sample(frac=1, random_state=SEED) | |
| # We need 850 Scams / 850 Legits for Train | |
| legacy_train_scams = legacy_scams_full.iloc[:850] | |
| legacy_train_legits = legacy_legits_full.iloc[:850] | |
| # We need 2000 Scams / (2000 - len(teeconnie_test)) Legits for Test (to perfectly balance 2000 scams) | |
| needed_legacy_legits_for_test = 2000 - len(teeconnie_test) | |
| legacy_test_scams = legacy_scams_full.iloc[850:2850] | |
| legacy_test_legits = legacy_legits_full.iloc[850:850+needed_legacy_legits_for_test] | |
| print(f"Sampled Legacy Train Data: {len(legacy_train_scams)} Scams, {len(legacy_train_legits)} Legits.") | |
| print(f"Sampled Legacy Test Data: {len(legacy_test_scams)} Scams, {len(legacy_test_legits)} Legits.") | |
| # 4. Construct Explicit Train and Test Datasets | |
| print("\nConstructing Explicit Train and Test Datasets...") | |
| train_df = pd.concat([synth_scams, synth_legits, teeconnie_train, legacy_train_scams, legacy_train_legits], ignore_index=True) | |
| test_df = pd.concat([legacy_test_scams, legacy_test_legits, teeconnie_test], ignore_index=True) | |
| # Shuffle the datasets thoroughly | |
| train_df = train_df.sample(frac=1, random_state=SEED).reset_index(drop=True) | |
| test_df = test_df.sample(frac=1, random_state=SEED).reset_index(drop=True) | |
| print(f"\nFINAL TRAIN DATASET: {len(train_df)} rows") | |
| print(train_df["label"].value_counts().to_dict()) | |
| print(f"FINAL TEST DATASET: {len(test_df)} rows") | |
| print(test_df["label"].value_counts().to_dict()) | |
| # 5. Save and Upload | |
| os.makedirs("data/phase1", exist_ok=True) | |
| train_df.to_csv("data/phase1/train.csv", index=False) | |
| test_df.to_csv("data/phase1/test.csv", index=False) | |
| print("\nSaved locally.") | |
| print(f"\nUploading datasets directly to DagsHub ({repo_id})...") | |
| try: | |
| dagshub.upload_files(repo_id, local_path="data/phase1/train.csv", remote_path="data/phase1/train.csv", bucket=True) | |
| dagshub.upload_files(repo_id, local_path="data/phase1/test.csv", remote_path="data/phase1/test.csv", bucket=True) | |
| print("Successfully uploaded to DagsHub Storage Bucket!") | |
| except Exception as e: | |
| print(f"Failed to upload to DagsHub: {e}") | |
| if __name__ == "__main__": | |
| main() |