Spaces:
Running
Running
File size: 4,538 Bytes
057a205 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | import os
import sys
import time
import requests
from astropy.coordinates import SkyCoord
import astropy.units as u
from astroquery.ipac.irsa import Irsa
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock
RA_DEG = 127.69444
DEC_DEG = -39.17760
RADIUS_DEG = 0.05
COLLECTION = "spherex_qr2"
BAND_FILTER = None
MAX_IMAGES = None
MAX_WORKERS = 16
MAX_RETRIES = 3
CHUNK_SIZE = 1024 * 1024
TIMEOUT = 180
DATA_DIR = "spherex_data"
FORCE_REDOWNLOAD = False
progress_lock = Lock()
progress = {"done": 0, "failed": 0, "skipped": 0, "total": 0}
def log_progress(tag):
with progress_lock:
progress[tag] += 1
completed = progress["done"] + progress["failed"] + progress["skipped"]
print(
f"[{completed}/{progress['total']}] "
f"done={progress['done']} skipped={progress['skipped']} failed={progress['failed']}",
end="\r",
)
def get_records():
pos = SkyCoord(ra=RA_DEG * u.deg, dec=DEC_DEG * u.deg)
print(f"Querying IRSA SIA: RA={RA_DEG}, Dec={DEC_DEG}, radius={RADIUS_DEG} deg ...")
result = Irsa.query_sia(pos=(pos, RADIUS_DEG * u.deg), collection=COLLECTION)
print(f"Found {len(result)} raw results")
urls = [
uri if str(uri).startswith("http") else f"https://irsa.ipac.caltech.edu/{uri}"
for uri in result["access_url"]
]
bands = (
[str(b) for b in result["energy_bandpassname"]]
if "energy_bandpassname" in result.colnames
else [""] * len(urls)
)
records = list(zip(urls, bands))
if BAND_FILTER:
records = [(u_, b) for u_, b in records if BAND_FILTER in b]
print(f"Filtered to band '{BAND_FILTER}': {len(records)} images")
if MAX_IMAGES:
records = records[:MAX_IMAGES]
print(f"Limited to first {MAX_IMAGES} images for this run")
return records
def band_folder(band):
safe = band.replace("/", "_").strip() if band else "unknown_band"
folder = os.path.join(DATA_DIR, safe or "unknown_band")
os.makedirs(folder, exist_ok=True)
return folder
def download_one(url, band):
filename = os.path.basename(url)
dest_folder = band_folder(band)
dest_path = os.path.join(dest_folder, filename)
if not FORCE_REDOWNLOAD and os.path.exists(dest_path):
try:
head = requests.head(url, timeout=30)
remote_size = int(head.headers.get("Content-Length", 0))
local_size = os.path.getsize(dest_path)
if remote_size and local_size == remote_size:
log_progress("skipped")
return dest_path
except Exception:
log_progress("skipped")
return dest_path
tmp_path = dest_path + ".part"
last_err = None
for attempt in range(1, MAX_RETRIES + 1):
try:
with requests.get(url, stream=True, timeout=TIMEOUT) as r:
r.raise_for_status()
with open(tmp_path, "wb") as f:
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
if chunk:
f.write(chunk)
os.replace(tmp_path, dest_path)
log_progress("done")
return dest_path
except Exception as e:
last_err = e
if os.path.exists(tmp_path):
os.remove(tmp_path)
time.sleep(2 * attempt)
log_progress("failed")
print(f"\nFAILED after {MAX_RETRIES} attempts: {url}\n reason: {last_err}")
return None
def main():
os.makedirs(DATA_DIR, exist_ok=True)
if FORCE_REDOWNLOAD:
print("FORCE_REDOWNLOAD is True — existing files will be overwritten from scratch.\n")
records = get_records()
if not records:
print("No images found for this query. Try widening RADIUS_DEG.")
sys.exit(1)
progress["total"] = len(records)
print(f"\nDownloading {len(records)} .fits files with {MAX_WORKERS} parallel workers...\n")
downloaded_paths = []
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
futures = {executor.submit(download_one, url, band): (url, band) for url, band in records}
for future in as_completed(futures):
path = future.result()
if path:
downloaded_paths.append(path)
print(f"\n\nDownload complete: {len(downloaded_paths)}/{len(records)} .fits files saved.")
print(f"All files kept as raw .fits under: {os.path.abspath(DATA_DIR)}")
if __name__ == "__main__":
main()
|