File size: 7,111 Bytes
ae77244 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | """
Prepares the final dataset by explicitly constructing a balanced Train and Test set.
Train Set: 100% of LLM data + Teeconnie + Legacy (2550 rows, perfectly balanced)
Test Set: Completely unseen Teeconnie + Legacy (4000 rows, perfectly balanced)
"""
import os
import json
import random
import re
import glob
import zipfile
import pandas as pd
import requests
import dagshub
from dotenv import load_dotenv
load_dotenv()
SEED = 42
random.seed(SEED)
def clean_text(text):
text = str(text)
text = re.sub(r'(?i)(innocent|suspect):\s*', '', text)
text = re.sub(r'\[.*?\]', '', text)
text = re.sub(r'\s+', ' ', text).strip()
return text.lower()
def main():
repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
repo_name = os.getenv("DAGSHUB_REPO_NAME")
username = os.getenv("MLFLOW_TRACKING_USERNAME")
password = os.getenv("MLFLOW_TRACKING_PASSWORD")
if not (repo_owner and repo_name):
print("ERROR: DAGSHUB_REPO_OWNER and DAGSHUB_REPO_NAME must be set in .env")
return
# Map MLflow credentials to DagsHub authentication
if username and password:
os.environ["DAGSHUB_USER"] = username
os.environ["DAGSHUB_TOKEN"] = password
dagshub.auth.add_app_token(password)
# Initialize DagsHub S3 client
repo_id = f"{repo_owner}/{repo_name}"
s3_client = dagshub.get_repo_bucket_client(repo_id)
# 1. Load Synthesized JSON data (LLM Data)
print("Loading synthesized JSON data from DagsHub...")
json_dir = "data/raw_jsons"
os.makedirs(json_dir, exist_ok=True)
json_files = ["scam_call_hard_examples_250.json", "scam_call_transcripts_250_combined.json"]
synth_data = []
for fname in json_files:
path = os.path.join(json_dir, fname)
s3_key = f"data/raw_jsons/{fname}"
try:
s3_client.download_file(repo_name, s3_key, path)
except Exception as e:
pass # ignore if exists locally
if os.path.exists(path):
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
synth_data.extend(data)
df_synth = pd.DataFrame(synth_data)
df_synth["text"] = df_synth["text"].apply(clean_text)
synth_scams = df_synth[df_synth["label"] == 1]
synth_legits = df_synth[df_synth["label"] == 0]
print(f"Loaded LLM Data: {len(synth_scams)} Scams, {len(synth_legits)} Legits.")
# 2. Download and Load Teeconnie dataset
print("\nDownloading teeconnie dataset from DagsHub S3...")
teeconnie_zip = "data/raw_teeconnie/teeconnie_dataset.zip"
os.makedirs("data/raw_teeconnie", exist_ok=True)
s3_key = "data/raw_teeconnie/teeconnie_dataset.zip"
try:
s3_client.download_file(repo_name, s3_key, teeconnie_zip)
except Exception as e:
pass
with zipfile.ZipFile(teeconnie_zip, 'r') as zipf:
zipf.extractall("data/raw_teeconnie/")
teeconnie_files = glob.glob("data/raw_teeconnie/**/*", recursive=True)
nonscam_txt = [f for f in teeconnie_files if f.lower().endswith(".txt") and "non" in f.lower() and "scam" in f.lower()]
entries = []
if nonscam_txt:
with open(nonscam_txt[0], "r", encoding="utf-8", errors="ignore") as f:
content = f.read()
entries = [e.strip() for e in content.split("\n\n") if e.strip()]
if len(entries) < 5:
entries = [e.strip() for e in content.split("\n") if e.strip()]
random.shuffle(entries)
df_teeconnie_full = pd.DataFrame({
"text": entries,
"label": [0] * len(entries)
})
df_teeconnie_full["text"] = df_teeconnie_full["text"].apply(clean_text)
# We need 350 for train, 1000 for test
teeconnie_train = df_teeconnie_full.iloc[:350]
teeconnie_test = df_teeconnie_full.iloc[350:1350]
print(f"Sampled Teeconnie Data: {len(teeconnie_train)} Train Legits, {len(teeconnie_test)} Test Legits.")
# 3. Download and Load Legacy Kaggle Dataset
print("\nDownloading Legacy Kaggle composite dataset from DagsHub S3...")
csv1 = "data/legacy_composite/composite_train.csv"
csv2 = "data/legacy_composite/composite_test.csv"
os.makedirs("data/legacy_composite", exist_ok=True)
try:
s3_client.download_file(repo_name, csv1, csv1)
s3_client.download_file(repo_name, csv2, csv2)
except Exception as e:
pass
df1 = pd.read_csv(csv1)
df2 = pd.read_csv(csv2)
df_legacy_full = pd.concat([df1, df2], ignore_index=True)
df_legacy_full = df_legacy_full.dropna(subset=["text", "label"])
df_legacy_full["text"] = df_legacy_full["text"].apply(clean_text)
legacy_scams_full = df_legacy_full[df_legacy_full["label"] == 1].sample(frac=1, random_state=SEED)
legacy_legits_full = df_legacy_full[df_legacy_full["label"] == 0].sample(frac=1, random_state=SEED)
# We need 850 Scams / 850 Legits for Train
legacy_train_scams = legacy_scams_full.iloc[:850]
legacy_train_legits = legacy_legits_full.iloc[:850]
# We need 2000 Scams / (2000 - len(teeconnie_test)) Legits for Test (to perfectly balance 2000 scams)
needed_legacy_legits_for_test = 2000 - len(teeconnie_test)
legacy_test_scams = legacy_scams_full.iloc[850:2850]
legacy_test_legits = legacy_legits_full.iloc[850:850+needed_legacy_legits_for_test]
print(f"Sampled Legacy Train Data: {len(legacy_train_scams)} Scams, {len(legacy_train_legits)} Legits.")
print(f"Sampled Legacy Test Data: {len(legacy_test_scams)} Scams, {len(legacy_test_legits)} Legits.")
# 4. Construct Explicit Train and Test Datasets
print("\nConstructing Explicit Train and Test Datasets...")
train_df = pd.concat([synth_scams, synth_legits, teeconnie_train, legacy_train_scams, legacy_train_legits], ignore_index=True)
test_df = pd.concat([legacy_test_scams, legacy_test_legits, teeconnie_test], ignore_index=True)
# Shuffle the datasets thoroughly
train_df = train_df.sample(frac=1, random_state=SEED).reset_index(drop=True)
test_df = test_df.sample(frac=1, random_state=SEED).reset_index(drop=True)
print(f"\nFINAL TRAIN DATASET: {len(train_df)} rows")
print(train_df["label"].value_counts().to_dict())
print(f"FINAL TEST DATASET: {len(test_df)} rows")
print(test_df["label"].value_counts().to_dict())
# 5. Save and Upload
os.makedirs("data/phase1", exist_ok=True)
train_df.to_csv("data/phase1/train.csv", index=False)
test_df.to_csv("data/phase1/test.csv", index=False)
print("\nSaved locally.")
print(f"\nUploading datasets directly to DagsHub ({repo_id})...")
try:
dagshub.upload_files(repo_id, local_path="data/phase1/train.csv", remote_path="data/phase1/train.csv", bucket=True)
dagshub.upload_files(repo_id, local_path="data/phase1/test.csv", remote_path="data/phase1/test.csv", bucket=True)
print("Successfully uploaded to DagsHub Storage Bucket!")
except Exception as e:
print(f"Failed to upload to DagsHub: {e}")
if __name__ == "__main__":
main() |