Download scripts/utils/migrate_all_data.py from AbhijithNusamNu/scam_dectect_sheid: direct link, hf CLI and curl.
- Browser
- Download file 3.49 kB
-
https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/utils/migrate_all_data.py
- Command line
-
hf download hf://spaces/AbhijithNusamNu/scam_dectect_sheid/scripts/utils/migrate_all_data.py
-
curl -L -o migrate_all_data.py https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/utils/migrate_all_data.py
3.49 kB
| import os | |
| import requests | |
| import kagglehub | |
| import glob | |
| import shutil | |
| import zipfile | |
| from dotenv import load_dotenv | |
| def download_hf_json(url, output_path): | |
| print(f"Downloading {url}...") | |
| os.makedirs(os.path.dirname(output_path), exist_ok=True) | |
| # Convert github/hf blob URLs to raw download URLs | |
| url = url.replace("/blob/", "/resolve/") | |
| resp = requests.get(url) | |
| if resp.status_code == 200: | |
| with open(output_path, "wb") as f: | |
| f.write(resp.content) | |
| print(f"Saved to {output_path}") | |
| return True | |
| else: | |
| print(f"Failed to download from HF (Status {resp.status_code})") | |
| return False | |
| def main(): | |
| load_dotenv() | |
| repo_owner = os.getenv("DAGSHUB_REPO_OWNER") | |
| repo_name = os.getenv("DAGSHUB_REPO_NAME") | |
| if not repo_owner or not repo_name: | |
| print("ERROR: DAGSHUB_REPO_OWNER or DAGSHUB_REPO_NAME missing from .env") | |
| return | |
| print("--- 1. Downloading historical data from web ---") | |
| # 1. LLM JSONs (from Hugging Face) | |
| json1 = "data/raw_jsons/scam_call_hard_examples_250.json" | |
| json2 = "data/raw_jsons/scam_call_transcripts_250_combined.json" | |
| download_hf_json("https://huggingface.co/datasets/tanu011235/spam/resolve/main/scam_call_hard_examples_250.json", json1) | |
| download_hf_json("https://huggingface.co/datasets/tanu011235/spam/resolve/main/scam_call_transcripts_250_combined.json", json2) | |
| # 2. Legacy Composite (from Kaggle) | |
| print("\nDownloading Kaggle Composite dataset...") | |
| kaggle_path = kagglehub.dataset_download("ibrahimbagwan12/composite-scam-transcript-dataset") | |
| csv1 = "data/legacy_composite/composite_train.csv" | |
| csv2 = "data/legacy_composite/composite_test.csv" | |
| os.makedirs("data/legacy_composite", exist_ok=True) | |
| for f in glob.glob(kaggle_path + "/*.csv"): | |
| if "train" in f.lower(): | |
| shutil.copy(f, csv1) | |
| elif "test" in f.lower(): | |
| shutil.copy(f, csv2) | |
| # 3. Teeconnie Kaggle Dataset | |
| print("\nDownloading Kaggle teeconnie dataset...") | |
| teeconnie_path = kagglehub.dataset_download("teeconnie/scam-and-non-scam-call-conversation-dataset") | |
| teeconnie_zip = "data/raw_teeconnie/teeconnie_dataset.zip" | |
| os.makedirs("data/raw_teeconnie", exist_ok=True) | |
| print(f"Zipping teeconnie dataset to {teeconnie_zip}...") | |
| with zipfile.ZipFile(teeconnie_zip, 'w', zipfile.ZIP_DEFLATED) as zipf: | |
| for root, _, files in os.walk(teeconnie_path): | |
| for file in files: | |
| file_path = os.path.join(root, file) | |
| # Keep the folder structure inside the zip | |
| arcname = os.path.relpath(file_path, teeconnie_path) | |
| zipf.write(file_path, arcname) | |
| print("\n--- 2. Uploading to DagsHub Data Storage (Bucket) ---") | |
| try: | |
| import dagshub | |
| repo_id = f"{repo_owner}/{repo_name}" | |
| for file in [json1, json2, csv1, csv2, teeconnie_zip]: | |
| if os.path.exists(file): | |
| print(f"Uploading {file} to DagsHub storage bucket...") | |
| dagshub.upload_files( | |
| repo_id, | |
| local_path=file, | |
| remote_path=file, | |
| bucket=True | |
| ) | |
| print("Upload complete! All historical datasets are now centralized in DagsHub Storage.") | |
| except Exception as e: | |
| print(f"Failed to upload to DagsHub: {e}") | |
| if __name__ == "__main__": | |
| main() | |