Konuktor92's picture
Add full training code (open source) for model v1
d7c810d verified
Raw History Blame
8.11 kB
"""Smoke test: prove every NASA data source opens before any bulk request.
Needs ~/.netrc with an entry for urs.earthdata.nasa.gov (the user creates it on the box).
python scripts/00_smoke.py --out /root/smoke # submit + wait
python scripts/00_smoke.py --out /root/smoke --resume # re-poll tasks saved earlier
"""
from __future__ import annotations
import argparse
import json
import netrc
import sys
import time
from pathlib import Path
import requests
APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api"
# ~20 x 20 km box at Son-Kul: the lake plus the summer pastures around it.
BBOX = (75.00, 41.75, 75.25, 41.93) # lon_min, lat_min, lon_max, lat_max
JULY_2020 = ("07-01-2020", "07-31-2020")
# One AppEEARS task per product (native projection differs between MODIS and NASADEM).
TASKS = {
"MOD13A3": (JULY_2020, [("MOD13A3.061", "_1_km_monthly_NDVI"),
("MOD13A3.061", "_1_km_monthly_pixel_reliability")]),
"MYD13A3": (JULY_2020, [("MYD13A3.061", "_1_km_monthly_NDVI")]),
"MOD11A2": (JULY_2020, [("MOD11A2.061", "LST_Day_1km"),
("MOD11A2.061", "LST_Night_1km"),
("MOD11A2.061", "QC_Day")]),
"MOD10A2": (JULY_2020, [("MOD10A2.061", "Maximum_Snow_Extent")]),
"MCD12Q1": (("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1")]),
"NASADEM": (("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]),
}
GESDISC = [ # (short_name, version, variables to print)
("GPM_3IMERGM", "07", ["precipitation"]),
("GLDAS_NOAH025_M", "2.1", ["Rainf_f_tavg", "Tair_f_inst", "SoilMoi0_10cm_inst", "SWE_inst"]),
]
def earthdata_credentials() -> tuple[str, str]:
auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov")
if not auth:
sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc")
return auth[0], auth[2]
def appeears_token() -> str:
user, password = earthdata_credentials()
r = requests.post(f"{APPEEARS}/login", auth=(user, password), timeout=60)
r.raise_for_status()
return r.json()["token"]
def polygon() -> dict:
x0, y0, x1, y1 = BBOX
ring = [[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]
return {"type": "FeatureCollection",
"features": [{"type": "Feature", "properties": {},
"geometry": {"type": "Polygon", "coordinates": [ring]}}]}
def submit(headers: dict) -> dict[str, str]:
ids = {}
for name, ((start, end), layers) in TASKS.items():
body = {"task_type": "area", "task_name": f"trendex-smoke-{name}",
"params": {"dates": [{"startDate": start, "endDate": end}],
"layers": [{"product": p, "layer": l} for p, l in layers],
"geo": polygon(),
"output": {"format": {"type": "geotiff"}, "projection": "native"}}}
r = requests.post(f"{APPEEARS}/task", json=body, headers=headers, timeout=60)
if r.status_code >= 400:
print(f"[appeears] {name}: submit FAILED {r.status_code} {r.text[:300]}")
continue
ids[name] = r.json()["task_id"]
print(f"[appeears] {name}: submitted {ids[name]}")
return ids
def wait_and_download(ids: dict[str, str], headers: dict, out: Path, timeout_min: int) -> dict:
results, pending = {}, dict(ids)
deadline = time.time() + timeout_min * 60
while pending and time.time() < deadline:
for name, tid in list(pending.items()):
status = requests.get(f"{APPEEARS}/task/{tid}", headers=headers, timeout=60).json().get("status")
if status == "done":
results[name] = download_bundle(name, tid, headers, out)
pending.pop(name)
elif status in ("error", "expired", "deleted"):
print(f"[appeears] {name}: {status}")
results[name] = {"status": status}
pending.pop(name)
if pending:
print(f"[appeears] waiting on {sorted(pending)} ...", flush=True)
time.sleep(30)
for name in pending:
results[name] = {"status": "still_queued"}
return results
def download_bundle(name: str, tid: str, headers: dict, out: Path) -> dict:
files = requests.get(f"{APPEEARS}/bundle/{tid}", headers=headers, timeout=60).json()["files"]
dest = out / "appeears" / name
dest.mkdir(parents=True, exist_ok=True)
tifs = []
for f in files:
if not f["file_name"].endswith(".tif"):
continue
path = dest / Path(f["file_name"]).name
with requests.get(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", headers=headers,
stream=True, allow_redirects=True, timeout=300) as r:
r.raise_for_status()
with open(path, "wb") as fh:
for chunk in r.iter_content(1 << 20):
fh.write(chunk)
tifs.append(path)
return {"status": "done", "files": [describe_tif(p) for p in tifs]}
def describe_tif(path: Path) -> dict:
import numpy as np
import rasterio
with rasterio.open(path) as src:
a = src.read(1, masked=True)
valid = a.compressed()
info = {"file": path.name, "crs": str(src.crs)[:60], "shape": a.shape,
"res": [round(v, 6) for v in src.res], "nodata": src.nodata,
"valid_share": round(valid.size / a.size, 3) if a.size else 0}
if valid.size:
info.update(min=float(np.min(valid)), max=float(np.max(valid)))
print(f" {info}")
return info
def check_gesdisc(out: Path) -> dict:
import earthaccess
import xarray as xr
earthaccess.login(strategy="netrc")
lon = (BBOX[0] + BBOX[2]) / 2
lat = (BBOX[1] + BBOX[3]) / 2
results = {}
for short, ver, variables in GESDISC:
try:
found = earthaccess.search_data(short_name=short, version=ver,
temporal=("2020-07-01", "2020-07-31"), bounding_box=BBOX)
print(f"[gesdisc] {short} v{ver}: {len(found)} granule(s) for July 2020")
path = Path(earthaccess.download(found[:1], local_path=str(out / "gesdisc" / short))[0])
kw = {"group": "Grid", "engine": "h5netcdf"} if short.startswith("GPM") else {}
ds = xr.open_dataset(path, **kw)
vals = {}
for v in variables:
if v in ds:
pt = ds[v].sel(lon=lon, lat=lat, method="nearest").squeeze()
vals[v] = f"{float(pt.values):.4g} {ds[v].attrs.get('units', '')}".strip()
print(f"[gesdisc] {short}: file={path.name} at Son-Kul: {vals}")
results[short] = {"status": "done", "file": path.name, "values": vals}
except Exception as exc: # report and keep going: one failure must not hide the others
print(f"[gesdisc] {short}: FAILED {type(exc).__name__}: {exc}")
results[short] = {"status": "failed", "error": str(exc)[:300]}
return results
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", type=Path, default=Path("/root/smoke"))
ap.add_argument("--resume", action="store_true", help="re-poll task ids saved by an earlier run")
ap.add_argument("--timeout-min", type=int, default=40)
args = ap.parse_args()
args.out.mkdir(parents=True, exist_ok=True)
ids_file = args.out / "appeears_task_ids.json"
report = {"gesdisc": check_gesdisc(args.out)}
headers = {"Authorization": f"Bearer {appeears_token()}"}
if args.resume and ids_file.exists():
ids = json.loads(ids_file.read_text())
else:
ids = submit(headers)
ids_file.write_text(json.dumps(ids, indent=2))
report["appeears"] = wait_and_download(ids, headers, args.out, args.timeout_min)
(args.out / "smoke_report.json").write_text(json.dumps(report, indent=2, default=str))
bad = [k for sec in report.values() for k, v in sec.items() if v.get("status") != "done"]
print("SMOKE_OK" if not bad else f"SMOKE_INCOMPLETE: {bad}")
if __name__ == "__main__":
main()