import os import sys import time import requests from astropy.coordinates import SkyCoord import astropy.units as u from astroquery.ipac.irsa import Irsa from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Lock RA_DEG = 127.69444 DEC_DEG = -39.17760 RADIUS_DEG = 0.05 COLLECTION = "spherex_qr2" BAND_FILTER = None MAX_IMAGES = None MAX_WORKERS = 16 MAX_RETRIES = 3 CHUNK_SIZE = 1024 * 1024 TIMEOUT = 180 DATA_DIR = "spherex_data" FORCE_REDOWNLOAD = False progress_lock = Lock() progress = {"done": 0, "failed": 0, "skipped": 0, "total": 0} def log_progress(tag): with progress_lock: progress[tag] += 1 completed = progress["done"] + progress["failed"] + progress["skipped"] print( f"[{completed}/{progress['total']}] " f"done={progress['done']} skipped={progress['skipped']} failed={progress['failed']}", end="\r", ) def get_records(): pos = SkyCoord(ra=RA_DEG * u.deg, dec=DEC_DEG * u.deg) print(f"Querying IRSA SIA: RA={RA_DEG}, Dec={DEC_DEG}, radius={RADIUS_DEG} deg ...") result = Irsa.query_sia(pos=(pos, RADIUS_DEG * u.deg), collection=COLLECTION) print(f"Found {len(result)} raw results") urls = [ uri if str(uri).startswith("http") else f"https://irsa.ipac.caltech.edu/{uri}" for uri in result["access_url"] ] bands = ( [str(b) for b in result["energy_bandpassname"]] if "energy_bandpassname" in result.colnames else [""] * len(urls) ) records = list(zip(urls, bands)) if BAND_FILTER: records = [(u_, b) for u_, b in records if BAND_FILTER in b] print(f"Filtered to band '{BAND_FILTER}': {len(records)} images") if MAX_IMAGES: records = records[:MAX_IMAGES] print(f"Limited to first {MAX_IMAGES} images for this run") return records def band_folder(band): safe = band.replace("/", "_").strip() if band else "unknown_band" folder = os.path.join(DATA_DIR, safe or "unknown_band") os.makedirs(folder, exist_ok=True) return folder def download_one(url, band): filename = os.path.basename(url) dest_folder = band_folder(band) dest_path = os.path.join(dest_folder, filename) if not FORCE_REDOWNLOAD and os.path.exists(dest_path): try: head = requests.head(url, timeout=30) remote_size = int(head.headers.get("Content-Length", 0)) local_size = os.path.getsize(dest_path) if remote_size and local_size == remote_size: log_progress("skipped") return dest_path except Exception: log_progress("skipped") return dest_path tmp_path = dest_path + ".part" last_err = None for attempt in range(1, MAX_RETRIES + 1): try: with requests.get(url, stream=True, timeout=TIMEOUT) as r: r.raise_for_status() with open(tmp_path, "wb") as f: for chunk in r.iter_content(chunk_size=CHUNK_SIZE): if chunk: f.write(chunk) os.replace(tmp_path, dest_path) log_progress("done") return dest_path except Exception as e: last_err = e if os.path.exists(tmp_path): os.remove(tmp_path) time.sleep(2 * attempt) log_progress("failed") print(f"\nFAILED after {MAX_RETRIES} attempts: {url}\n reason: {last_err}") return None def main(): os.makedirs(DATA_DIR, exist_ok=True) if FORCE_REDOWNLOAD: print("FORCE_REDOWNLOAD is True — existing files will be overwritten from scratch.\n") records = get_records() if not records: print("No images found for this query. Try widening RADIUS_DEG.") sys.exit(1) progress["total"] = len(records) print(f"\nDownloading {len(records)} .fits files with {MAX_WORKERS} parallel workers...\n") downloaded_paths = [] with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: futures = {executor.submit(download_one, url, band): (url, band) for url, band in records} for future in as_completed(futures): path = future.result() if path: downloaded_paths.append(path) print(f"\n\nDownload complete: {len(downloaded_paths)}/{len(records)} .fits files saved.") print(f"All files kept as raw .fits under: {os.path.abspath(DATA_DIR)}") if __name__ == "__main__": main()