File size: 4,538 Bytes
057a205
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
import os
import sys
import time

import requests
from astropy.coordinates import SkyCoord
import astropy.units as u
from astroquery.ipac.irsa import Irsa

from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock

RA_DEG = 127.69444
DEC_DEG = -39.17760
RADIUS_DEG = 0.05
COLLECTION = "spherex_qr2"
BAND_FILTER = None
MAX_IMAGES = None

MAX_WORKERS = 16
MAX_RETRIES = 3
CHUNK_SIZE = 1024 * 1024
TIMEOUT = 180

DATA_DIR = "spherex_data"

FORCE_REDOWNLOAD = False

progress_lock = Lock()
progress = {"done": 0, "failed": 0, "skipped": 0, "total": 0}

def log_progress(tag):
    with progress_lock:
        progress[tag] += 1
        completed = progress["done"] + progress["failed"] + progress["skipped"]
        print(
            f"[{completed}/{progress['total']}] "
            f"done={progress['done']} skipped={progress['skipped']} failed={progress['failed']}",
            end="\r",
        )

def get_records():
    pos = SkyCoord(ra=RA_DEG * u.deg, dec=DEC_DEG * u.deg)
    print(f"Querying IRSA SIA: RA={RA_DEG}, Dec={DEC_DEG}, radius={RADIUS_DEG} deg ...")
    result = Irsa.query_sia(pos=(pos, RADIUS_DEG * u.deg), collection=COLLECTION)
    print(f"Found {len(result)} raw results")

    urls = [
        uri if str(uri).startswith("http") else f"https://irsa.ipac.caltech.edu/{uri}"
        for uri in result["access_url"]
    ]
    bands = (
        [str(b) for b in result["energy_bandpassname"]]
        if "energy_bandpassname" in result.colnames
        else [""] * len(urls)
    )

    records = list(zip(urls, bands))

    if BAND_FILTER:
        records = [(u_, b) for u_, b in records if BAND_FILTER in b]
        print(f"Filtered to band '{BAND_FILTER}': {len(records)} images")

    if MAX_IMAGES:
        records = records[:MAX_IMAGES]
        print(f"Limited to first {MAX_IMAGES} images for this run")

    return records

def band_folder(band):
    safe = band.replace("/", "_").strip() if band else "unknown_band"
    folder = os.path.join(DATA_DIR, safe or "unknown_band")
    os.makedirs(folder, exist_ok=True)
    return folder

def download_one(url, band):
    filename = os.path.basename(url)
    dest_folder = band_folder(band)
    dest_path = os.path.join(dest_folder, filename)

    if not FORCE_REDOWNLOAD and os.path.exists(dest_path):
        try:
            head = requests.head(url, timeout=30)
            remote_size = int(head.headers.get("Content-Length", 0))
            local_size = os.path.getsize(dest_path)
            if remote_size and local_size == remote_size:
                log_progress("skipped")
                return dest_path
        except Exception:
            log_progress("skipped")
            return dest_path

    tmp_path = dest_path + ".part"
    last_err = None
    for attempt in range(1, MAX_RETRIES + 1):
        try:
            with requests.get(url, stream=True, timeout=TIMEOUT) as r:
                r.raise_for_status()
                with open(tmp_path, "wb") as f:
                    for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
                        if chunk:
                            f.write(chunk)
            os.replace(tmp_path, dest_path)
            log_progress("done")
            return dest_path
        except Exception as e:
            last_err = e
            if os.path.exists(tmp_path):
                os.remove(tmp_path)
            time.sleep(2 * attempt)

    log_progress("failed")
    print(f"\nFAILED after {MAX_RETRIES} attempts: {url}\n  reason: {last_err}")
    return None

def main():
    os.makedirs(DATA_DIR, exist_ok=True)

    if FORCE_REDOWNLOAD:
        print("FORCE_REDOWNLOAD is True — existing files will be overwritten from scratch.\n")

    records = get_records()
    if not records:
        print("No images found for this query. Try widening RADIUS_DEG.")
        sys.exit(1)

    progress["total"] = len(records)
    print(f"\nDownloading {len(records)} .fits files with {MAX_WORKERS} parallel workers...\n")

    downloaded_paths = []
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(download_one, url, band): (url, band) for url, band in records}
        for future in as_completed(futures):
            path = future.result()
            if path:
                downloaded_paths.append(path)

    print(f"\n\nDownload complete: {len(downloaded_paths)}/{len(records)} .fits files saved.")
    print(f"All files kept as raw .fits under: {os.path.abspath(DATA_DIR)}")

if __name__ == "__main__":
    main()