File size: 3,491 Bytes
ae77244 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | import os
import requests
import kagglehub
import glob
import shutil
import zipfile
from dotenv import load_dotenv
def download_hf_json(url, output_path):
print(f"Downloading {url}...")
os.makedirs(os.path.dirname(output_path), exist_ok=True)
# Convert github/hf blob URLs to raw download URLs
url = url.replace("/blob/", "/resolve/")
resp = requests.get(url)
if resp.status_code == 200:
with open(output_path, "wb") as f:
f.write(resp.content)
print(f"Saved to {output_path}")
return True
else:
print(f"Failed to download from HF (Status {resp.status_code})")
return False
def main():
load_dotenv()
repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
repo_name = os.getenv("DAGSHUB_REPO_NAME")
if not repo_owner or not repo_name:
print("ERROR: DAGSHUB_REPO_OWNER or DAGSHUB_REPO_NAME missing from .env")
return
print("--- 1. Downloading historical data from web ---")
# 1. LLM JSONs (from Hugging Face)
json1 = "data/raw_jsons/scam_call_hard_examples_250.json"
json2 = "data/raw_jsons/scam_call_transcripts_250_combined.json"
download_hf_json("https://huggingface.co/datasets/tanu011235/spam/resolve/main/scam_call_hard_examples_250.json", json1)
download_hf_json("https://huggingface.co/datasets/tanu011235/spam/resolve/main/scam_call_transcripts_250_combined.json", json2)
# 2. Legacy Composite (from Kaggle)
print("\nDownloading Kaggle Composite dataset...")
kaggle_path = kagglehub.dataset_download("ibrahimbagwan12/composite-scam-transcript-dataset")
csv1 = "data/legacy_composite/composite_train.csv"
csv2 = "data/legacy_composite/composite_test.csv"
os.makedirs("data/legacy_composite", exist_ok=True)
for f in glob.glob(kaggle_path + "/*.csv"):
if "train" in f.lower():
shutil.copy(f, csv1)
elif "test" in f.lower():
shutil.copy(f, csv2)
# 3. Teeconnie Kaggle Dataset
print("\nDownloading Kaggle teeconnie dataset...")
teeconnie_path = kagglehub.dataset_download("teeconnie/scam-and-non-scam-call-conversation-dataset")
teeconnie_zip = "data/raw_teeconnie/teeconnie_dataset.zip"
os.makedirs("data/raw_teeconnie", exist_ok=True)
print(f"Zipping teeconnie dataset to {teeconnie_zip}...")
with zipfile.ZipFile(teeconnie_zip, 'w', zipfile.ZIP_DEFLATED) as zipf:
for root, _, files in os.walk(teeconnie_path):
for file in files:
file_path = os.path.join(root, file)
# Keep the folder structure inside the zip
arcname = os.path.relpath(file_path, teeconnie_path)
zipf.write(file_path, arcname)
print("\n--- 2. Uploading to DagsHub Data Storage (Bucket) ---")
try:
import dagshub
repo_id = f"{repo_owner}/{repo_name}"
for file in [json1, json2, csv1, csv2, teeconnie_zip]:
if os.path.exists(file):
print(f"Uploading {file} to DagsHub storage bucket...")
dagshub.upload_files(
repo_id,
local_path=file,
remote_path=file,
bucket=True
)
print("Upload complete! All historical datasets are now centralized in DagsHub Storage.")
except Exception as e:
print(f"Failed to upload to DagsHub: {e}")
if __name__ == "__main__":
main()
|