"""Smoke test: prove every NASA data source opens before any bulk request. Needs ~/.netrc with an entry for urs.earthdata.nasa.gov (the user creates it on the box). python scripts/00_smoke.py --out /root/smoke # submit + wait python scripts/00_smoke.py --out /root/smoke --resume # re-poll tasks saved earlier """ from __future__ import annotations import argparse import json import netrc import sys import time from pathlib import Path import requests APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api" # ~20 x 20 km box at Son-Kul: the lake plus the summer pastures around it. BBOX = (75.00, 41.75, 75.25, 41.93) # lon_min, lat_min, lon_max, lat_max JULY_2020 = ("07-01-2020", "07-31-2020") # One AppEEARS task per product (native projection differs between MODIS and NASADEM). TASKS = { "MOD13A3": (JULY_2020, [("MOD13A3.061", "_1_km_monthly_NDVI"), ("MOD13A3.061", "_1_km_monthly_pixel_reliability")]), "MYD13A3": (JULY_2020, [("MYD13A3.061", "_1_km_monthly_NDVI")]), "MOD11A2": (JULY_2020, [("MOD11A2.061", "LST_Day_1km"), ("MOD11A2.061", "LST_Night_1km"), ("MOD11A2.061", "QC_Day")]), "MOD10A2": (JULY_2020, [("MOD10A2.061", "Maximum_Snow_Extent")]), "MCD12Q1": (("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1")]), "NASADEM": (("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]), } GESDISC = [ # (short_name, version, variables to print) ("GPM_3IMERGM", "07", ["precipitation"]), ("GLDAS_NOAH025_M", "2.1", ["Rainf_f_tavg", "Tair_f_inst", "SoilMoi0_10cm_inst", "SWE_inst"]), ] def earthdata_credentials() -> tuple[str, str]: auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov") if not auth: sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc") return auth[0], auth[2] def appeears_token() -> str: user, password = earthdata_credentials() r = requests.post(f"{APPEEARS}/login", auth=(user, password), timeout=60) r.raise_for_status() return r.json()["token"] def polygon() -> dict: x0, y0, x1, y1 = BBOX ring = [[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]] return {"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Polygon", "coordinates": [ring]}}]} def submit(headers: dict) -> dict[str, str]: ids = {} for name, ((start, end), layers) in TASKS.items(): body = {"task_type": "area", "task_name": f"trendex-smoke-{name}", "params": {"dates": [{"startDate": start, "endDate": end}], "layers": [{"product": p, "layer": l} for p, l in layers], "geo": polygon(), "output": {"format": {"type": "geotiff"}, "projection": "native"}}} r = requests.post(f"{APPEEARS}/task", json=body, headers=headers, timeout=60) if r.status_code >= 400: print(f"[appeears] {name}: submit FAILED {r.status_code} {r.text[:300]}") continue ids[name] = r.json()["task_id"] print(f"[appeears] {name}: submitted {ids[name]}") return ids def wait_and_download(ids: dict[str, str], headers: dict, out: Path, timeout_min: int) -> dict: results, pending = {}, dict(ids) deadline = time.time() + timeout_min * 60 while pending and time.time() < deadline: for name, tid in list(pending.items()): status = requests.get(f"{APPEEARS}/task/{tid}", headers=headers, timeout=60).json().get("status") if status == "done": results[name] = download_bundle(name, tid, headers, out) pending.pop(name) elif status in ("error", "expired", "deleted"): print(f"[appeears] {name}: {status}") results[name] = {"status": status} pending.pop(name) if pending: print(f"[appeears] waiting on {sorted(pending)} ...", flush=True) time.sleep(30) for name in pending: results[name] = {"status": "still_queued"} return results def download_bundle(name: str, tid: str, headers: dict, out: Path) -> dict: files = requests.get(f"{APPEEARS}/bundle/{tid}", headers=headers, timeout=60).json()["files"] dest = out / "appeears" / name dest.mkdir(parents=True, exist_ok=True) tifs = [] for f in files: if not f["file_name"].endswith(".tif"): continue path = dest / Path(f["file_name"]).name with requests.get(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", headers=headers, stream=True, allow_redirects=True, timeout=300) as r: r.raise_for_status() with open(path, "wb") as fh: for chunk in r.iter_content(1 << 20): fh.write(chunk) tifs.append(path) return {"status": "done", "files": [describe_tif(p) for p in tifs]} def describe_tif(path: Path) -> dict: import numpy as np import rasterio with rasterio.open(path) as src: a = src.read(1, masked=True) valid = a.compressed() info = {"file": path.name, "crs": str(src.crs)[:60], "shape": a.shape, "res": [round(v, 6) for v in src.res], "nodata": src.nodata, "valid_share": round(valid.size / a.size, 3) if a.size else 0} if valid.size: info.update(min=float(np.min(valid)), max=float(np.max(valid))) print(f" {info}") return info def check_gesdisc(out: Path) -> dict: import earthaccess import xarray as xr earthaccess.login(strategy="netrc") lon = (BBOX[0] + BBOX[2]) / 2 lat = (BBOX[1] + BBOX[3]) / 2 results = {} for short, ver, variables in GESDISC: try: found = earthaccess.search_data(short_name=short, version=ver, temporal=("2020-07-01", "2020-07-31"), bounding_box=BBOX) print(f"[gesdisc] {short} v{ver}: {len(found)} granule(s) for July 2020") path = Path(earthaccess.download(found[:1], local_path=str(out / "gesdisc" / short))[0]) kw = {"group": "Grid", "engine": "h5netcdf"} if short.startswith("GPM") else {} ds = xr.open_dataset(path, **kw) vals = {} for v in variables: if v in ds: pt = ds[v].sel(lon=lon, lat=lat, method="nearest").squeeze() vals[v] = f"{float(pt.values):.4g} {ds[v].attrs.get('units', '')}".strip() print(f"[gesdisc] {short}: file={path.name} at Son-Kul: {vals}") results[short] = {"status": "done", "file": path.name, "values": vals} except Exception as exc: # report and keep going: one failure must not hide the others print(f"[gesdisc] {short}: FAILED {type(exc).__name__}: {exc}") results[short] = {"status": "failed", "error": str(exc)[:300]} return results def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--out", type=Path, default=Path("/root/smoke")) ap.add_argument("--resume", action="store_true", help="re-poll task ids saved by an earlier run") ap.add_argument("--timeout-min", type=int, default=40) args = ap.parse_args() args.out.mkdir(parents=True, exist_ok=True) ids_file = args.out / "appeears_task_ids.json" report = {"gesdisc": check_gesdisc(args.out)} headers = {"Authorization": f"Bearer {appeears_token()}"} if args.resume and ids_file.exists(): ids = json.loads(ids_file.read_text()) else: ids = submit(headers) ids_file.write_text(json.dumps(ids, indent=2)) report["appeears"] = wait_and_download(ids, headers, args.out, args.timeout_min) (args.out / "smoke_report.json").write_text(json.dumps(report, indent=2, default=str)) bad = [k for sec in report.values() for k, v in sec.items() if v.get("status") != "done"] print("SMOKE_OK" if not bad else f"SMOKE_INCOMPLETE: {bad}") if __name__ == "__main__": main()