Download training_code/scripts/00_smoke.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 8.11 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/00_smoke.py
- Command line
-
hf download hf://TrendeX90/expected-greenness@refs/pr/1/training_code/scripts/00_smoke.py
-
curl -L -o 00_smoke.py https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/00_smoke.py
8.11 kB
| """Smoke test: prove every NASA data source opens before any bulk request. | |
| Needs ~/.netrc with an entry for urs.earthdata.nasa.gov (the user creates it on the box). | |
| python scripts/00_smoke.py --out /root/smoke # submit + wait | |
| python scripts/00_smoke.py --out /root/smoke --resume # re-poll tasks saved earlier | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import netrc | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import requests | |
| APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api" | |
| # ~20 x 20 km box at Son-Kul: the lake plus the summer pastures around it. | |
| BBOX = (75.00, 41.75, 75.25, 41.93) # lon_min, lat_min, lon_max, lat_max | |
| JULY_2020 = ("07-01-2020", "07-31-2020") | |
| # One AppEEARS task per product (native projection differs between MODIS and NASADEM). | |
| TASKS = { | |
| "MOD13A3": (JULY_2020, [("MOD13A3.061", "_1_km_monthly_NDVI"), | |
| ("MOD13A3.061", "_1_km_monthly_pixel_reliability")]), | |
| "MYD13A3": (JULY_2020, [("MYD13A3.061", "_1_km_monthly_NDVI")]), | |
| "MOD11A2": (JULY_2020, [("MOD11A2.061", "LST_Day_1km"), | |
| ("MOD11A2.061", "LST_Night_1km"), | |
| ("MOD11A2.061", "QC_Day")]), | |
| "MOD10A2": (JULY_2020, [("MOD10A2.061", "Maximum_Snow_Extent")]), | |
| "MCD12Q1": (("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1")]), | |
| "NASADEM": (("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]), | |
| } | |
| GESDISC = [ # (short_name, version, variables to print) | |
| ("GPM_3IMERGM", "07", ["precipitation"]), | |
| ("GLDAS_NOAH025_M", "2.1", ["Rainf_f_tavg", "Tair_f_inst", "SoilMoi0_10cm_inst", "SWE_inst"]), | |
| ] | |
| def earthdata_credentials() -> tuple[str, str]: | |
| auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov") | |
| if not auth: | |
| sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc") | |
| return auth[0], auth[2] | |
| def appeears_token() -> str: | |
| user, password = earthdata_credentials() | |
| r = requests.post(f"{APPEEARS}/login", auth=(user, password), timeout=60) | |
| r.raise_for_status() | |
| return r.json()["token"] | |
| def polygon() -> dict: | |
| x0, y0, x1, y1 = BBOX | |
| ring = [[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]] | |
| return {"type": "FeatureCollection", | |
| "features": [{"type": "Feature", "properties": {}, | |
| "geometry": {"type": "Polygon", "coordinates": [ring]}}]} | |
| def submit(headers: dict) -> dict[str, str]: | |
| ids = {} | |
| for name, ((start, end), layers) in TASKS.items(): | |
| body = {"task_type": "area", "task_name": f"trendex-smoke-{name}", | |
| "params": {"dates": [{"startDate": start, "endDate": end}], | |
| "layers": [{"product": p, "layer": l} for p, l in layers], | |
| "geo": polygon(), | |
| "output": {"format": {"type": "geotiff"}, "projection": "native"}}} | |
| r = requests.post(f"{APPEEARS}/task", json=body, headers=headers, timeout=60) | |
| if r.status_code >= 400: | |
| print(f"[appeears] {name}: submit FAILED {r.status_code} {r.text[:300]}") | |
| continue | |
| ids[name] = r.json()["task_id"] | |
| print(f"[appeears] {name}: submitted {ids[name]}") | |
| return ids | |
| def wait_and_download(ids: dict[str, str], headers: dict, out: Path, timeout_min: int) -> dict: | |
| results, pending = {}, dict(ids) | |
| deadline = time.time() + timeout_min * 60 | |
| while pending and time.time() < deadline: | |
| for name, tid in list(pending.items()): | |
| status = requests.get(f"{APPEEARS}/task/{tid}", headers=headers, timeout=60).json().get("status") | |
| if status == "done": | |
| results[name] = download_bundle(name, tid, headers, out) | |
| pending.pop(name) | |
| elif status in ("error", "expired", "deleted"): | |
| print(f"[appeears] {name}: {status}") | |
| results[name] = {"status": status} | |
| pending.pop(name) | |
| if pending: | |
| print(f"[appeears] waiting on {sorted(pending)} ...", flush=True) | |
| time.sleep(30) | |
| for name in pending: | |
| results[name] = {"status": "still_queued"} | |
| return results | |
| def download_bundle(name: str, tid: str, headers: dict, out: Path) -> dict: | |
| files = requests.get(f"{APPEEARS}/bundle/{tid}", headers=headers, timeout=60).json()["files"] | |
| dest = out / "appeears" / name | |
| dest.mkdir(parents=True, exist_ok=True) | |
| tifs = [] | |
| for f in files: | |
| if not f["file_name"].endswith(".tif"): | |
| continue | |
| path = dest / Path(f["file_name"]).name | |
| with requests.get(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", headers=headers, | |
| stream=True, allow_redirects=True, timeout=300) as r: | |
| r.raise_for_status() | |
| with open(path, "wb") as fh: | |
| for chunk in r.iter_content(1 << 20): | |
| fh.write(chunk) | |
| tifs.append(path) | |
| return {"status": "done", "files": [describe_tif(p) for p in tifs]} | |
| def describe_tif(path: Path) -> dict: | |
| import numpy as np | |
| import rasterio | |
| with rasterio.open(path) as src: | |
| a = src.read(1, masked=True) | |
| valid = a.compressed() | |
| info = {"file": path.name, "crs": str(src.crs)[:60], "shape": a.shape, | |
| "res": [round(v, 6) for v in src.res], "nodata": src.nodata, | |
| "valid_share": round(valid.size / a.size, 3) if a.size else 0} | |
| if valid.size: | |
| info.update(min=float(np.min(valid)), max=float(np.max(valid))) | |
| print(f" {info}") | |
| return info | |
| def check_gesdisc(out: Path) -> dict: | |
| import earthaccess | |
| import xarray as xr | |
| earthaccess.login(strategy="netrc") | |
| lon = (BBOX[0] + BBOX[2]) / 2 | |
| lat = (BBOX[1] + BBOX[3]) / 2 | |
| results = {} | |
| for short, ver, variables in GESDISC: | |
| try: | |
| found = earthaccess.search_data(short_name=short, version=ver, | |
| temporal=("2020-07-01", "2020-07-31"), bounding_box=BBOX) | |
| print(f"[gesdisc] {short} v{ver}: {len(found)} granule(s) for July 2020") | |
| path = Path(earthaccess.download(found[:1], local_path=str(out / "gesdisc" / short))[0]) | |
| kw = {"group": "Grid", "engine": "h5netcdf"} if short.startswith("GPM") else {} | |
| ds = xr.open_dataset(path, **kw) | |
| vals = {} | |
| for v in variables: | |
| if v in ds: | |
| pt = ds[v].sel(lon=lon, lat=lat, method="nearest").squeeze() | |
| vals[v] = f"{float(pt.values):.4g} {ds[v].attrs.get('units', '')}".strip() | |
| print(f"[gesdisc] {short}: file={path.name} at Son-Kul: {vals}") | |
| results[short] = {"status": "done", "file": path.name, "values": vals} | |
| except Exception as exc: # report and keep going: one failure must not hide the others | |
| print(f"[gesdisc] {short}: FAILED {type(exc).__name__}: {exc}") | |
| results[short] = {"status": "failed", "error": str(exc)[:300]} | |
| return results | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--out", type=Path, default=Path("/root/smoke")) | |
| ap.add_argument("--resume", action="store_true", help="re-poll task ids saved by an earlier run") | |
| ap.add_argument("--timeout-min", type=int, default=40) | |
| args = ap.parse_args() | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| ids_file = args.out / "appeears_task_ids.json" | |
| report = {"gesdisc": check_gesdisc(args.out)} | |
| headers = {"Authorization": f"Bearer {appeears_token()}"} | |
| if args.resume and ids_file.exists(): | |
| ids = json.loads(ids_file.read_text()) | |
| else: | |
| ids = submit(headers) | |
| ids_file.write_text(json.dumps(ids, indent=2)) | |
| report["appeears"] = wait_and_download(ids, headers, args.out, args.timeout_min) | |
| (args.out / "smoke_report.json").write_text(json.dumps(report, indent=2, default=str)) | |
| bad = [k for sec in report.values() for k, v in sec.items() if v.get("status") != "done"] | |
| print("SMOKE_OK" if not bad else f"SMOKE_INCOMPLETE: {bad}") | |
| if __name__ == "__main__": | |
| main() | |