File size: 3,491 Bytes
ae77244
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
import os
import requests
import kagglehub
import glob
import shutil
import zipfile
from dotenv import load_dotenv

def download_hf_json(url, output_path):
    print(f"Downloading {url}...")
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    # Convert github/hf blob URLs to raw download URLs
    url = url.replace("/blob/", "/resolve/")
    resp = requests.get(url)
    if resp.status_code == 200:
        with open(output_path, "wb") as f:
            f.write(resp.content)
        print(f"Saved to {output_path}")
        return True
    else:
        print(f"Failed to download from HF (Status {resp.status_code})")
        return False

def main():
    load_dotenv()
    
    repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
    repo_name = os.getenv("DAGSHUB_REPO_NAME")
    
    if not repo_owner or not repo_name:
        print("ERROR: DAGSHUB_REPO_OWNER or DAGSHUB_REPO_NAME missing from .env")
        return

    print("--- 1. Downloading historical data from web ---")
    
    # 1. LLM JSONs (from Hugging Face)
    json1 = "data/raw_jsons/scam_call_hard_examples_250.json"
    json2 = "data/raw_jsons/scam_call_transcripts_250_combined.json"
    download_hf_json("https://huggingface.co/datasets/tanu011235/spam/resolve/main/scam_call_hard_examples_250.json", json1)
    download_hf_json("https://huggingface.co/datasets/tanu011235/spam/resolve/main/scam_call_transcripts_250_combined.json", json2)
    
    # 2. Legacy Composite (from Kaggle)
    print("\nDownloading Kaggle Composite dataset...")
    kaggle_path = kagglehub.dataset_download("ibrahimbagwan12/composite-scam-transcript-dataset")
    csv1 = "data/legacy_composite/composite_train.csv"
    csv2 = "data/legacy_composite/composite_test.csv"
    os.makedirs("data/legacy_composite", exist_ok=True)
    for f in glob.glob(kaggle_path + "/*.csv"):
        if "train" in f.lower():
            shutil.copy(f, csv1)
        elif "test" in f.lower():
            shutil.copy(f, csv2)

    # 3. Teeconnie Kaggle Dataset
    print("\nDownloading Kaggle teeconnie dataset...")
    teeconnie_path = kagglehub.dataset_download("teeconnie/scam-and-non-scam-call-conversation-dataset")
    teeconnie_zip = "data/raw_teeconnie/teeconnie_dataset.zip"
    os.makedirs("data/raw_teeconnie", exist_ok=True)
    
    print(f"Zipping teeconnie dataset to {teeconnie_zip}...")
    with zipfile.ZipFile(teeconnie_zip, 'w', zipfile.ZIP_DEFLATED) as zipf:
        for root, _, files in os.walk(teeconnie_path):
            for file in files:
                file_path = os.path.join(root, file)
                # Keep the folder structure inside the zip
                arcname = os.path.relpath(file_path, teeconnie_path)
                zipf.write(file_path, arcname)

    print("\n--- 2. Uploading to DagsHub Data Storage (Bucket) ---")
    try:
        import dagshub
        repo_id = f"{repo_owner}/{repo_name}"
        
        for file in [json1, json2, csv1, csv2, teeconnie_zip]:
            if os.path.exists(file):
                print(f"Uploading {file} to DagsHub storage bucket...")
                dagshub.upload_files(
                    repo_id, 
                    local_path=file, 
                    remote_path=file, 
                    bucket=True
                )
                
        print("Upload complete! All historical datasets are now centralized in DagsHub Storage.")
    except Exception as e:
        print(f"Failed to upload to DagsHub: {e}")

if __name__ == "__main__":
    main()