Buckets:
| """ | |
| download_processed_data.py | |
| Autonomous download script for preprocessed EEG seizure dataset from Hugging Face bucket. | |
| Target folder: processed/10s | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import time | |
| import urllib.request | |
| import urllib.error | |
| from datetime import datetime | |
| BUCKET_NAME = "DLProjectCNNLSTM/eeg-seizure-project" | |
| TREE_API_URL = f"https://huggingface.co/api/buckets/{BUCKET_NAME}/tree/processed/10s" | |
| RESOLVE_BASE_URL = f"https://huggingface.co/buckets/{BUCKET_NAME}/resolve" | |
| LOCAL_TARGET_DIR = os.path.join("data", "processed", "10s") | |
| MANIFEST_PATH = os.path.join(LOCAL_TARGET_DIR, "download_manifest.json") | |
| # Files to download (exclude raw EDF files or unrelated root files) | |
| TARGET_FILENAMES = [ | |
| "eeg_windows.npz", | |
| "window_metadata_full.csv", | |
| "processing_manifest.json", | |
| "window_metadata.csv" | |
| ] | |
| def fetch_remote_inventory(): | |
| """Fetch file inventory from Hugging Face bucket API.""" | |
| print(f"Fetching remote inventory from {TREE_API_URL}...") | |
| req = urllib.request.Request( | |
| TREE_API_URL, | |
| headers={"User-Agent": "Autonomous-EEG-Downloader/1.0"} | |
| ) | |
| with urllib.request.urlopen(req, timeout=30) as resp: | |
| data = json.loads(resp.read().decode("utf-8")) | |
| inventory = {} | |
| for item in data: | |
| path = item.get("path", "") | |
| filename = os.path.basename(path) | |
| inventory[filename] = { | |
| "remote_path": path, | |
| "size": item.get("size", 0), | |
| "xetHash": item.get("xetHash", ""), | |
| "uploadedAt": item.get("uploadedAt", ""), | |
| "download_url": f"{RESOLVE_BASE_URL}/{path}" | |
| } | |
| return inventory | |
| def download_file(url, local_path, expected_size, max_retries=5, chunk_size=4*1024*1024): | |
| """ | |
| Download a file with resume support (HTTP Range) and retries. | |
| """ | |
| os.makedirs(os.path.dirname(local_path), exist_ok=True) | |
| # Check if already fully downloaded | |
| if os.path.exists(local_path): | |
| current_size = os.path.getsize(local_path) | |
| if expected_size and current_size == expected_size: | |
| print(f"File {local_path} already complete ({current_size:,} bytes). Skipping download.") | |
| return True, "already_complete", 0.0 | |
| elif current_size > expected_size: | |
| print(f"Local file larger than expected ({current_size} > {expected_size}). Resetting...") | |
| os.remove(local_path) | |
| for attempt in range(1, max_retries + 1): | |
| try: | |
| start_time = time.time() | |
| existing_size = os.path.getsize(local_path) if os.path.exists(local_path) else 0 | |
| headers = {"User-Agent": "Autonomous-EEG-Downloader/1.0"} | |
| if existing_size > 0: | |
| headers["Range"] = f"bytes={existing_size}-" | |
| print(f"Resuming {os.path.basename(local_path)} from byte {existing_size:,}/{expected_size:,}...") | |
| else: | |
| print(f"Starting download of {os.path.basename(local_path)} ({expected_size:,} bytes)...") | |
| req = urllib.request.Request(url, headers=headers) | |
| with urllib.request.urlopen(req, timeout=60) as resp: | |
| mode = "ab" if existing_size > 0 and resp.status == 206 else "wb" | |
| if mode == "wb": | |
| existing_size = 0 | |
| downloaded = existing_size | |
| last_report = time.time() | |
| bytes_since_report = 0 | |
| with open(local_path, mode) as f: | |
| while True: | |
| chunk = resp.read(chunk_size) | |
| if not chunk: | |
| break | |
| f.write(chunk) | |
| downloaded += len(chunk) | |
| bytes_since_report += len(chunk) | |
| now = time.time() | |
| if now - last_report >= 5.0 or (expected_size and downloaded >= expected_size): | |
| speed = bytes_since_report / (now - last_report) / (1024 * 1024) | |
| pct = (downloaded / expected_size * 100) if expected_size else 0 | |
| print(f" [{os.path.basename(local_path)}] {downloaded:,}/{expected_size:,} bytes ({pct:.1f}%) - Speed: {speed:.2f} MB/s") | |
| last_report = now | |
| bytes_since_report = 0 | |
| final_size = os.path.getsize(local_path) | |
| duration = time.time() - start_time | |
| if expected_size and final_size != expected_size: | |
| raise ValueError(f"Size mismatch: got {final_size}, expected {expected_size}") | |
| print(f"Successfully downloaded {os.path.basename(local_path)} ({final_size:,} bytes in {duration:.1f}s)") | |
| return True, "downloaded", duration | |
| except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError, IOError) as e: | |
| print(f"Attempt {attempt}/{max_retries} failed for {os.path.basename(local_path)}: {e}") | |
| if attempt < max_retries: | |
| wait_sec = min(2 ** attempt * 2, 60) | |
| print(f"Retrying in {wait_sec}s...") | |
| time.sleep(wait_sec) | |
| else: | |
| return False, f"failed: {e}", 0.0 | |
| def main(): | |
| start_total = time.time() | |
| os.makedirs(LOCAL_TARGET_DIR, exist_ok=True) | |
| # 1. Fetch remote inventory | |
| inventory = fetch_remote_inventory() | |
| print(f"Discovered {len(inventory)} files in processed/10s on Hugging Face.") | |
| # 2. Process downloads | |
| manifest = { | |
| "timestamp": datetime.utcnow().isoformat() + "Z", | |
| "bucket": BUCKET_NAME, | |
| "remote_folder": "processed/10s", | |
| "local_folder": LOCAL_TARGET_DIR, | |
| "files": {} | |
| } | |
| all_success = True | |
| for filename in TARGET_FILENAMES: | |
| if filename not in inventory: | |
| print(f"WARNING: Target file '{filename}' was not found in remote inventory!") | |
| manifest["files"][filename] = { | |
| "status": "not_found_on_remote", | |
| "error": "File does not exist in bucket" | |
| } | |
| if filename in ["eeg_windows.npz", "window_metadata_full.csv"]: | |
| all_success = False | |
| continue | |
| info = inventory[filename] | |
| local_path = os.path.join(LOCAL_TARGET_DIR, filename) | |
| expected_size = info["size"] | |
| success, status, duration = download_file( | |
| url=info["download_url"], | |
| local_path=local_path, | |
| expected_size=expected_size | |
| ) | |
| actual_size = os.path.getsize(local_path) if os.path.exists(local_path) else 0 | |
| manifest["files"][filename] = { | |
| "remote_path": info["remote_path"], | |
| "local_path": local_path, | |
| "remote_size_bytes": expected_size, | |
| "local_size_bytes": actual_size, | |
| "xet_hash": info["xetHash"], | |
| "download_status": status, | |
| "duration_seconds": round(duration, 2), | |
| "size_verified": (actual_size == expected_size) | |
| } | |
| if not success: | |
| all_success = False | |
| manifest["total_duration_seconds"] = round(time.time() - start_total, 2) | |
| manifest["all_downloads_successful"] = all_success | |
| with open(MANIFEST_PATH, "w") as f: | |
| json.dump(manifest, f, indent=2) | |
| print(f"\nDownload manifest written to {MANIFEST_PATH}") | |
| if not all_success: | |
| print("ERROR: One or more critical files failed to download.") | |
| sys.exit(1) | |
| else: | |
| print("All downloads completed and verified successfully!") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 7.7 kB
- Xet hash:
- 6bc1d525a34d888114c9041af6338e22ea2811e7f4f917ce9b08f60d74c71079
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.