sync.py

import os
import json
import time
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, unquote
from huggingface_hub import HfApi

# Configuration from Environment Variables (Secrets)
ONION_URL = os.getenv("ONION_URL")  # Must be: http://6qqz6m3b6htudohg2mlf5gdcalonxy3sh5g4dix4mpyirjcgelqqufad.onion/bankofbaroda.bank.in/
HF_TOKEN = os.getenv("HF_TOKEN")
DATASET_REPO_ID = os.getenv("DATASET_REPO_ID")  # username/dataset-name

HISTORY_FILE = "history.json"
MAX_LOCAL_STORAGE_BYTES = 35 * 1024 * 1024 * 1024  # Keep below 40GB limit (35GB threshold)

# Configure requests session to route through Tor SOCKS5 proxy
session = requests.Session()
session.proxies = {
    'http': 'socks5h://127.0.0.1:9050',
    'https': 'socks5h://127.0.0.1:9050'
}

def load_history():
    if os.path.exists(HISTORY_FILE):
        with open(HISTORY_FILE, "r") as f:
            return set(json.load(f))
    return set()

def save_history(completed_files):
    with open(HISTORY_FILE, "w") as f:
        json.dump(list(completed_files), f)

def get_dir_size(start_path='.'):
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(start_path):
        for f in filenames:
            if f == HISTORY_FILE:
                continue
            fp = os.path.join(dirpath, f)
            if os.path.exists(fp):
                total_size += os.path.getsize(fp)
    return total_size

def crawl_and_collect(url, relative_path=""):
    """Recursively discover all files from the web directory listing."""
    print(f"Scanning: {url}")
    files_to_download = []
    try:
        response = session.get(url, timeout=30)
        if response.status_code != 200:
            print(f"Failed to fetch {url}: Status {response.status_code}")
            return files_to_download
        
        soup = BeautifulSoup(response.text, 'html.parser')
        for link in soup.find_all('a'):
            href = link.get('href')
            if not href or href.startswith('?') or href in ['../', './', '..', '.']:
                continue
            
            clean_href = unquote(href).strip('/')
            if not clean_href:
                continue

            full_url = urljoin(url, href)
            target_rel_path = os.path.join(relative_path, clean_href)

            # Check if it's a directory (ends with / in href or has trailing slash)
            if href.endswith('/') or link.text.endswith('/'):
                files_to_download.extend(crawl_and_collect(full_url, target_rel_path))
            else:
                files_to_download.append((full_url, target_rel_path))
                
    except Exception as e:
        print(f"Error crawling {url}: {e}")
        
    return files_to_download

def main():
    if not all([ONION_URL, HF_TOKEN, DATASET_REPO_ID]):
        raise ValueError("Please set ONION_URL, HF_TOKEN, and DATASET_REPO_ID secrets.")

    api = HfApi(token=HF_TOKEN)
    completed_files = load_history()
    
    print("Waiting for Tor circuits to finalize...")
    time.sleep(10)

    print("Discovering file structure from .onion root source...")
    all_files = crawl_and_collect(ONION_URL)
    print(f"Total files discovered: {len(all_files)}")

    for file_url, rel_path in all_files:
        if rel_path in completed_files:
            continue

        # Check local space utilization before downloading
        while get_dir_size() > MAX_LOCAL_STORAGE_BYTES:
            print("Storage threshold reached (35GB). Waiting/Cleaning...")
            time.sleep(10)

        print(f"Downloading: {rel_path}")
        local_file_path = os.path.join("downloads", rel_path)
        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)

        try:
            # Stream download to handle large files efficiently
            with session.get(file_url, stream=True, timeout=60) as r:
                r.raise_for_status()
                with open(local_file_path, 'wb') as f:
                    for chunk in r.iter_content(chunk_size=8192):
                        if chunk:
                            f.write(chunk)

            # Upload immediately to Hugging Face Dataset Repo
            print(f"Uploading {rel_path} to HF dataset...")
            api.upload_file(
                path_or_fileobj=local_file_path,
                path_in_repo=rel_path,
                repo_id=DATASET_REPO_ID,
                repo_type="dataset"
            )

            # Mark as completed and remove locally to free up space
            completed_files.add(rel_path)
            save_history(completed_files)
            
            if os.path.exists(local_file_path):
                os.remove(local_file_path)

        except Exception as e:
            print(f"Error processing {rel_path}: {e}")
            time.sleep(5)  # Backoff on error

    print("Synchronization complete!")

if __name__ == "__main__":
    main()