File size: 8,473 Bytes
ae77244 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | """
Phase 2: Hybrid ASR Data Acquisition
- Scams (Label 1): Fetches pre-transcribed (Whisper) robocall metadata from NCSU github.
- Legit (Label 0): Fetches 'english_transcription' from Hugging Face PolyAI/minds14 dataset.
Aggregates, balances (1:1), and splits into Train/Val/Test/PTQ.
"""
import os
import io
import json
import subprocess
from pathlib import Path
import requests
import pandas as pd
import dagshub
from dotenv import load_dotenv
from datasets import load_dataset
from sklearn.model_selection import train_test_split
REPO_ROOT = Path(__file__).resolve().parents[1]
BRANCH_STAGE_MAP = {
"feature/phase-2-audio-asr": "feature/phase-2-audio-asr",
"feature/phase-3-serving-quantization": "feature/phase-3-serving-quantization",
"feature/phase-1.5-ultimate-dataset": "model-modernbert-universal",
"model-long-context": "model-modernbert-universal",
"model-distilbert": "model-distilbert",
"main": "main",
}
def detect_branch(default="main"):
env_branch = os.getenv("DAGSHUB_BRANCH") or os.getenv("GIT_BRANCH")
if env_branch:
return env_branch.strip()
result = subprocess.run(
["git", "branch", "--show-current"],
cwd=REPO_ROOT,
capture_output=True,
text=True,
check=False,
)
branch = result.stdout.strip()
return branch or default
def stage_for_branch(branch):
return BRANCH_STAGE_MAP.get(branch, branch.replace("/", "-") or "main")
def fetch_ncsu_scam_data():
"""
Downloads the metadata.csv from the NCSU Robocall dataset repository.
Extracts the 'transcript' column as Label 1 (Scam).
"""
print("Fetching NCSU Robocall Dataset (Scams)...")
url = "https://raw.githubusercontent.com/wspr-ncsu/robocall-audio-dataset/main/metadata.csv"
try:
response = requests.get(url)
response.raise_for_status()
# Read CSV from string
df = pd.read_csv(io.StringIO(response.text))
# We only need the transcription
transcripts = df['transcript'].dropna().tolist()
results = []
for text in transcripts:
# Filter extremely short artifacts
if len(str(text).split()) > 10:
results.append({
"source": "ncsu_robocall",
"text": str(text),
"label": 1
})
print(f" [SUCCESS] Extracted {len(results)} scam transcripts.")
return results
except Exception as e:
print(f" [FAILED] Could not fetch NCSU dataset: {e}")
return []
def fetch_hf_legit_data():
"""
Fetches the PolyAI/minds14 dataset from Hugging Face (en-US, en-GB, en-AU).
Drops the audio column immediately to prevent decoding errors.
Returns a list of dictionaries with text and label 0.
"""
results = []
print(f"\nFetching PolyAI/minds14 Dataset (Legit)...")
locales = ["en-US", "en-GB", "en-AU"]
for loc in locales:
try:
print(f" Loading subset: {loc}...")
# Load dataset and immediately drop the 'audio' column to prevent torchcodec decoding crashes
ds = load_dataset("PolyAI/minds14", name=loc, split="train", trust_remote_code=True)
ds = ds.remove_columns(["audio", "path", "intent_class", "lang_id"])
for row in ds:
text = row.get("english_transcription", "")
if text and len(str(text).split()) > 5:
results.append({
"source": f"minds14_{loc}",
"text": str(text),
"label": 0
})
print(f" [SUCCESS] Extracted transcripts from {loc}.")
except Exception as e:
print(f" [FAILED] {loc}: {e}")
print(f" [SUCCESS] Total legit transcripts extracted: {len(results)}")
return results
def split_and_upload(df, output_dir="data/phase2_asr", stage="feature/phase-2-audio-asr"):
"""
Splits the data into Train(60), Val(10), Test(20), PTQ(10) and uploads to S3.
"""
print("\n--- Splitting Dataset ---")
if len(df) < 4:
print("ERROR: Not enough data to split into 4 buckets. Skipping split.")
return
train_df, rest_df = train_test_split(df, test_size=0.4, stratify=df["label"], random_state=42)
test_df, val_ptq_df = train_test_split(rest_df, test_size=0.5, stratify=rest_df["label"], random_state=42)
val_df, ptq_df = train_test_split(val_ptq_df, test_size=0.5, stratify=val_ptq_df["label"], random_state=42)
print("\n=========================================")
print(" DATASET SPLIT SUMMARY STATS")
print("=========================================")
def print_stats(name, dataframe):
total = len(dataframe)
scams = len(dataframe[dataframe['label'] == 1])
legits = len(dataframe[dataframe['label'] == 0])
print(f"--- {name} ---")
print(f"Total Rows: {total}")
if total > 0:
print(f"Class Balance: {scams} Scams ({scams/total:.1%}) | {legits} Legits ({legits/total:.1%})")
print("")
print_stats("Train (Fine-Tuning, 60%)", train_df)
print_stats("Test (Evaluation, 20%)", test_df)
print_stats("Validation (10%)", val_df)
print_stats("PTQ Calibration (10%)", ptq_df)
print("=========================================\n")
# Save locally
os.makedirs(output_dir, exist_ok=True)
train_path = os.path.join(output_dir, "train.csv")
val_path = os.path.join(output_dir, "val.csv")
test_path = os.path.join(output_dir, "test.csv")
ptq_path = os.path.join(output_dir, "ptq_calibration.csv")
train_df.to_csv(train_path, index=False)
val_df.to_csv(val_path, index=False)
test_df.to_csv(test_path, index=False)
ptq_df.to_csv(ptq_path, index=False)
manifest = {
"stage": stage,
"train_rows": len(train_df),
"val_rows": len(val_df),
"test_rows": len(test_df),
"ptq_rows": len(ptq_df),
"source": "ncsu_robocall + PolyAI/minds14",
}
manifest_path = os.path.join(output_dir, "manifest.json")
with open(manifest_path, "w", encoding="utf-8") as f:
json.dump(manifest, f, indent=2)
# Upload to DagsHub S3
print("\n--- Uploading to DagsHub S3 ---")
load_dotenv()
repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
repo_name = os.getenv("DAGSHUB_REPO_NAME")
if repo_owner and repo_name:
try:
dagshub.auth.add_app_token(os.getenv("MLFLOW_TRACKING_PASSWORD"))
s3_client = dagshub.get_repo_bucket_client(f"{repo_owner}/{repo_name}")
remote_dir = f"data/{stage}/phase2_asr"
for file_path in [train_path, val_path, test_path, ptq_path, manifest_path]:
print(f"Uploading {file_path} to S3...")
remote_path = f"{remote_dir}/{os.path.basename(file_path)}"
s3_client.upload_file(file_path, repo_name, remote_path)
print("Successfully backed up Phase 2 datasets to DagsHub S3!")
except Exception as e:
print(f"Failed to upload to S3: {e}")
else:
print("Skipping DagsHub S3 upload (missing env vars).")
def main():
print("Initializing Phase 2 Hybrid ASR Data Acquisition...\n")
branch = detect_branch()
stage = stage_for_branch(branch)
# 1. Fetch 1400+ Scams from NCSU
scam_data = fetch_ncsu_scam_data()
# 2. Fetch Legit Transcripts from Hugging Face
legit_data = fetch_hf_legit_data()
# 3. Balance Dataset (Downsample majority class)
min_size = min(len(scam_data), len(legit_data))
print(f"\nBalancing Dataset: Downsampling to {min_size} files per class.")
if min_size == 0:
print("ERROR: One of the classes has 0 transcripts. Cannot proceed.")
return
balanced_data = scam_data[:min_size] + legit_data[:min_size]
df = pd.DataFrame(balanced_data)
# Save the raw aggregated dataset
output_dir = "data/phase2_asr"
os.makedirs(output_dir, exist_ok=True)
raw_path = os.path.join(output_dir, "raw_asr_transcripts.csv")
df.to_csv(raw_path, index=False)
print(f"Saved {len(df)} raw transcripts to {raw_path}")
# Generate splits and upload
split_and_upload(df, stage=stage)
if __name__ == "__main__":
main()
|