Download training_code/scripts/10_download.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/10_download.py
- Command line
-
hf download hf://TrendeX90/expected-greenness@refs/pr/1/training_code/scripts/10_download.py
-
curl -L -o 10_download.py https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/10_download.py
10.2 kB
| """Bulk download of every NASA layer for the cube. | |
| python scripts/10_download.py submit # AppEEARS: one area task per product, ids saved to JSON | |
| python scripts/10_download.py fetch # AppEEARS: poll tasks, download finished bundles (re-runnable) | |
| python scripts/10_download.py gesdisc # GES DISC: IMERG + GLDAS monthly files (re-runnable) | |
| Credentials come from ~/.netrc (urs.earthdata.nasa.gov). Every step skips work already done, | |
| so it is safe to re-run after a crash or a lost SSH session. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import netrc | |
| import sys | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import requests | |
| APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api" | |
| ROOT = Path("/root/trendex-ml/data") | |
| AOI = ROOT / "aoi" / "aoi_request.geojson" | |
| RAW = ROOT / "raw" | |
| TASK_IDS = RAW / "appeears_task_ids.json" | |
| LAST_YEAR = 2025 | |
| def full(start: str, end: str = f"12-31-{LAST_YEAR}") -> dict: | |
| return {"startDate": start, "endDate": end} | |
| def months_each_year(start_mmdd: str, end_mmdd: str, first_year: int) -> dict: | |
| """Recurring window, e.g. Mar-Sep of every year.""" | |
| return {"startDate": start_mmdd, "endDate": end_mmdd, "recurring": True, | |
| "yearRange": [first_year, LAST_YEAR]} | |
| # product -> (date window, layers). LST and snow are only needed Mar-Sep (model features). | |
| PRODUCTS = { | |
| "MOD13A3": (full("02-01-2000"), [("MOD13A3.061", "_1_km_monthly_NDVI"), | |
| ("MOD13A3.061", "_1_km_monthly_pixel_reliability")]), | |
| "MYD13A3": (full("07-01-2002"), [("MYD13A3.061", "_1_km_monthly_NDVI"), | |
| ("MYD13A3.061", "_1_km_monthly_pixel_reliability")]), | |
| "MOD11A2": (months_each_year("03-01", "09-30", 2000), [("MOD11A2.061", "LST_Day_1km"), | |
| ("MOD11A2.061", "LST_Night_1km"), | |
| ("MOD11A2.061", "QC_Day"), | |
| ("MOD11A2.061", "QC_Night")]), | |
| "MOD10A2": (months_each_year("03-01", "09-30", 2000), [("MOD10A2.061", "Maximum_Snow_Extent")]), | |
| "MCD12Q1": (full("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1"), | |
| ("MCD12Q1.061", "LW")]), | |
| "NASADEM": (full("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]), | |
| } | |
| ORDER = ["MOD13A3", "NASADEM", "MCD12Q1", "MOD11A2", "MOD10A2", "MYD13A3"] # what the formula needs first | |
| LPDAAC = { # fallback: raw HDF4 tiles straight from LP DAAC / NSIDC if the AppEEARS queue is too slow | |
| "MOD13A3": ("MOD13A3", "061", "2000-02-01"), | |
| "MYD13A3": ("MYD13A3", "061", "2002-07-01"), | |
| "MOD11A2": ("MOD11A2", "061", "2000-02-18"), | |
| "MOD10A2": ("MOD10A2", "61", "2000-02-24"), | |
| "MCD12Q1": ("MCD12Q1", "061", "2020-01-01"), | |
| "NASADEM": ("NASADEM_HGT", "001", "2000-02-11"), | |
| } | |
| GESDISC = { # short_name: (version, first month) | |
| "GPM_3IMERGM": ("07", "2000-06-01"), | |
| "GLDAS_NOAH025_M": ("2.1", "2000-01-01"), | |
| } | |
| def headers() -> dict: | |
| auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov") | |
| if not auth: | |
| sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc") | |
| r = requests.post(f"{APPEEARS}/login", auth=(auth[0], auth[2]), timeout=60) | |
| r.raise_for_status() | |
| return {"Authorization": f"Bearer {r.json()['token']}"} | |
| def submit() -> None: | |
| RAW.mkdir(parents=True, exist_ok=True) | |
| ids = json.loads(TASK_IDS.read_text()) if TASK_IDS.exists() else {} | |
| geo = json.loads(AOI.read_text()) | |
| h = headers() | |
| for name in ORDER: | |
| if name in ids: | |
| print(f"[submit] {name}: already submitted {ids[name]}") | |
| continue | |
| window, layers = PRODUCTS[name] | |
| body = {"task_type": "area", "task_name": f"trendex-{name}", | |
| "params": {"dates": [window], | |
| "layers": [{"product": p, "layer": l} for p, l in layers], | |
| "geo": geo, | |
| "output": {"format": {"type": "geotiff"}, "projection": "native"}}} | |
| r = requests.post(f"{APPEEARS}/task", json=body, headers=h, timeout=120) | |
| if r.status_code >= 400: | |
| print(f"[submit] {name}: FAILED {r.status_code} {r.text[:400]}") | |
| continue | |
| ids[name] = r.json()["task_id"] | |
| TASK_IDS.write_text(json.dumps(ids, indent=2)) | |
| print(f"[submit] {name}: {ids[name]}") | |
| class Session: | |
| """AppEEARS client: one login, re-login on 401, polite back-off on 429/5xx (it rate-limits hard).""" | |
| def __init__(self) -> None: | |
| self.h = headers() | |
| def get(self, url: str, stream: bool = False, tries: int = 8) -> requests.Response: | |
| for i in range(tries): | |
| try: | |
| r = requests.get(url, headers=self.h, stream=stream, allow_redirects=True, timeout=300) | |
| except requests.RequestException as exc: | |
| print(f" network error ({exc}); retry in {20 * (i + 1)}s", flush=True) | |
| time.sleep(20 * (i + 1)) | |
| continue | |
| if r.status_code == 401: | |
| self.h = headers() | |
| continue | |
| if r.status_code == 429 or r.status_code >= 500: | |
| wait = int(r.headers.get("Retry-After", 0) or 60 * (i + 1)) | |
| print(f" HTTP {r.status_code}; waiting {wait}s", flush=True) | |
| r.close() | |
| time.sleep(wait) | |
| continue | |
| r.raise_for_status() | |
| return r | |
| raise RuntimeError(f"giving up on {url}") | |
| def _download(s: Session, url: str, path: Path, size: int | None) -> None: | |
| if path.exists() and (size is None or path.stat().st_size == size): | |
| return | |
| tmp = path.with_suffix(path.suffix + ".part") | |
| with s.get(url, stream=True) as r: | |
| with open(tmp, "wb") as fh: | |
| for chunk in r.iter_content(1 << 20): | |
| fh.write(chunk) | |
| tmp.rename(path) | |
| def fetch(poll_s: int, max_hours: float) -> None: | |
| ids = json.loads(TASK_IDS.read_text()) | |
| done_marker = RAW / "appeears_done.json" | |
| done = json.loads(done_marker.read_text()) if done_marker.exists() else {} | |
| deadline = time.time() + max_hours * 3600 | |
| s = Session() | |
| while time.time() < deadline: | |
| pending = [n for n in ids if n not in done] | |
| if not pending: | |
| break | |
| # one request for every task's status instead of one per task | |
| statuses = {t["task_id"]: t.get("status") for t in s.get(f"{APPEEARS}/task?limit=100").json()} | |
| for name in pending: | |
| tid = ids[name] | |
| status = statuses.get(tid) | |
| if status in ("error", "expired", "deleted"): | |
| print(f"[fetch] {name}: task {status}; resubmit with `submit` after removing it from {TASK_IDS.name}") | |
| done[name] = status | |
| continue | |
| if status != "done": | |
| continue | |
| files = s.get(f"{APPEEARS}/bundle/{tid}").json()["files"] | |
| dest = RAW / "appeears" / name | |
| dest.mkdir(parents=True, exist_ok=True) | |
| jobs = [(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", dest / Path(f["file_name"]).name, | |
| f.get("file_size")) for f in files] | |
| with ThreadPoolExecutor(6) as pool: | |
| list(pool.map(lambda j: _download(s, j[0], j[1], j[2]), jobs)) | |
| done[name] = "done" | |
| done_marker.write_text(json.dumps(done, indent=2)) | |
| print(f"[fetch] {name}: downloaded {len(jobs)} files", flush=True) | |
| left = [n for n in ids if n not in done] | |
| if left: | |
| shown = {n: statuses.get(ids[n]) for n in left} | |
| print(f"[fetch] waiting on {shown} ({time.strftime('%H:%M:%S')})", flush=True) | |
| time.sleep(poll_s) | |
| print("FETCH_COMPLETE" if all(done.get(n) == "done" for n in ids) else f"FETCH_STATUS {done}") | |
| def aoi_bbox() -> tuple[float, float, float, float]: | |
| geo = json.loads(AOI.read_text()) | |
| xs = [c[0] for f in geo["features"] for c in f["geometry"]["coordinates"][0]] | |
| ys = [c[1] for f in geo["features"] for c in f["geometry"]["coordinates"][0]] | |
| return (min(xs), min(ys), max(xs), max(ys)) | |
| def lpdaac(products: list[str]) -> None: | |
| import earthaccess | |
| earthaccess.login(strategy="netrc") | |
| bbox = aoi_bbox() | |
| for name in products: | |
| short, ver, first = LPDAAC[name] | |
| last = "2020-12-31" if name == "MCD12Q1" else f"{LAST_YEAR}-12-31" | |
| found = earthaccess.search_data(short_name=short, version=ver, temporal=(first, last), bounding_box=bbox) | |
| dest = RAW / "earthaccess" / name | |
| dest.mkdir(parents=True, exist_ok=True) | |
| print(f"[lpdaac] {name}: {len(found)} granules", flush=True) | |
| earthaccess.download(found, local_path=str(dest), threads=16) | |
| print(f"[lpdaac] {name}: {len(list(dest.iterdir()))} files on disk", flush=True) | |
| print("LPDAAC_COMPLETE", products) | |
| def gesdisc() -> None: | |
| import earthaccess | |
| earthaccess.login(strategy="netrc") | |
| bbox = aoi_bbox() | |
| for short, (ver, first) in GESDISC.items(): | |
| found = earthaccess.search_data(short_name=short, version=ver, | |
| temporal=(first, f"{LAST_YEAR}-12-31"), bounding_box=bbox) | |
| dest = RAW / "gesdisc" / short | |
| dest.mkdir(parents=True, exist_ok=True) | |
| print(f"[gesdisc] {short} v{ver}: {len(found)} monthly granules", flush=True) | |
| earthaccess.download(found, local_path=str(dest), threads=16) | |
| print(f"[gesdisc] {short}: {len(list(dest.iterdir()))} files on disk", flush=True) | |
| print("GESDISC_COMPLETE") | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("step", choices=["submit", "fetch", "gesdisc", "lpdaac"]) | |
| ap.add_argument("--products", nargs="*", default=["MOD13A3"]) | |
| ap.add_argument("--poll-s", type=int, default=60) | |
| ap.add_argument("--max-hours", type=float, default=6) | |
| a = ap.parse_args() | |
| {"submit": submit, "gesdisc": gesdisc, "lpdaac": lambda: lpdaac(a.products), | |
| "fetch": lambda: fetch(a.poll_s, a.max_hours)}[a.step]() | |
| if __name__ == "__main__": | |
| main() | |