AkshithSurya's picture
Deploy SPHEREx Data Explorer (Static)
057a205 verified
Raw History Blame Contribute Delete
4.54 kB
import os
import sys
import time
import requests
from astropy.coordinates import SkyCoord
import astropy.units as u
from astroquery.ipac.irsa import Irsa
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock
RA_DEG = 127.69444
DEC_DEG = -39.17760
RADIUS_DEG = 0.05
COLLECTION = "spherex_qr2"
BAND_FILTER = None
MAX_IMAGES = None
MAX_WORKERS = 16
MAX_RETRIES = 3
CHUNK_SIZE = 1024 * 1024
TIMEOUT = 180
DATA_DIR = "spherex_data"
FORCE_REDOWNLOAD = False
progress_lock = Lock()
progress = {"done": 0, "failed": 0, "skipped": 0, "total": 0}
def log_progress(tag):
with progress_lock:
progress[tag] += 1
completed = progress["done"] + progress["failed"] + progress["skipped"]
print(
f"[{completed}/{progress['total']}] "
f"done={progress['done']} skipped={progress['skipped']} failed={progress['failed']}",
end="\r",
)
def get_records():
pos = SkyCoord(ra=RA_DEG * u.deg, dec=DEC_DEG * u.deg)
print(f"Querying IRSA SIA: RA={RA_DEG}, Dec={DEC_DEG}, radius={RADIUS_DEG} deg ...")
result = Irsa.query_sia(pos=(pos, RADIUS_DEG * u.deg), collection=COLLECTION)
print(f"Found {len(result)} raw results")
urls = [
uri if str(uri).startswith("http") else f"https://irsa.ipac.caltech.edu/{uri}"
for uri in result["access_url"]
]
bands = (
[str(b) for b in result["energy_bandpassname"]]
if "energy_bandpassname" in result.colnames
else [""] * len(urls)
)
records = list(zip(urls, bands))
if BAND_FILTER:
records = [(u_, b) for u_, b in records if BAND_FILTER in b]
print(f"Filtered to band '{BAND_FILTER}': {len(records)} images")
if MAX_IMAGES:
records = records[:MAX_IMAGES]
print(f"Limited to first {MAX_IMAGES} images for this run")
return records
def band_folder(band):
safe = band.replace("/", "_").strip() if band else "unknown_band"
folder = os.path.join(DATA_DIR, safe or "unknown_band")
os.makedirs(folder, exist_ok=True)
return folder
def download_one(url, band):
filename = os.path.basename(url)
dest_folder = band_folder(band)
dest_path = os.path.join(dest_folder, filename)
if not FORCE_REDOWNLOAD and os.path.exists(dest_path):
try:
head = requests.head(url, timeout=30)
remote_size = int(head.headers.get("Content-Length", 0))
local_size = os.path.getsize(dest_path)
if remote_size and local_size == remote_size:
log_progress("skipped")
return dest_path
except Exception:
log_progress("skipped")
return dest_path
tmp_path = dest_path + ".part"
last_err = None
for attempt in range(1, MAX_RETRIES + 1):
try:
with requests.get(url, stream=True, timeout=TIMEOUT) as r:
r.raise_for_status()
with open(tmp_path, "wb") as f:
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
if chunk:
f.write(chunk)
os.replace(tmp_path, dest_path)
log_progress("done")
return dest_path
except Exception as e:
last_err = e
if os.path.exists(tmp_path):
os.remove(tmp_path)
time.sleep(2 * attempt)
log_progress("failed")
print(f"\nFAILED after {MAX_RETRIES} attempts: {url}\n reason: {last_err}")
return None
def main():
os.makedirs(DATA_DIR, exist_ok=True)
if FORCE_REDOWNLOAD:
print("FORCE_REDOWNLOAD is True — existing files will be overwritten from scratch.\n")
records = get_records()
if not records:
print("No images found for this query. Try widening RADIUS_DEG.")
sys.exit(1)
progress["total"] = len(records)
print(f"\nDownloading {len(records)} .fits files with {MAX_WORKERS} parallel workers...\n")
downloaded_paths = []
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
futures = {executor.submit(download_one, url, band): (url, band) for url, band in records}
for future in as_completed(futures):
path = future.result()
if path:
downloaded_paths.append(path)
print(f"\n\nDownload complete: {len(downloaded_paths)}/{len(records)} .fits files saved.")
print(f"All files kept as raw .fits under: {os.path.abspath(DATA_DIR)}")
if __name__ == "__main__":
main()