Download backend/environmental.py from m4az19/saef-api: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/environmental.py
- Command line
-
hf download hf://spaces/m4az19/saef-api/backend/environmental.py
-
curl -L -o environmental.py https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/environmental.py
10.2 kB
| """Bounded, credential-aware satellite adapters. Failures never become synthetic data.""" | |
| import os | |
| import re | |
| from datetime import datetime, timedelta, timezone | |
| from pathlib import Path | |
| import hashlib | |
| import httpx | |
| import numpy as np | |
| import rasterio | |
| from rasterio.io import MemoryFile | |
| from rasterio.transform import from_bounds | |
| from rasterio.windows import from_bounds as window_from_bounds | |
| from . import store, locations | |
| from .observations import digest, utc | |
| CDSE='https://sh.dataspace.copernicus.eu' | |
| PPS='https://jsimpsonhttps.pps.eosdis.nasa.gov' | |
| def cdse_vegetation(lon,lat): | |
| locations.require_uae(lon,lat) | |
| client_id=os.getenv('CDSE_CLIENT_ID');secret=os.getenv('CDSE_CLIENT_SECRET') | |
| if not client_id or not secret:raise ValueError('Copernicus Data Space OAuth client credentials not configured; existing Sentinel COG crops remain usable') | |
| end=utc();start=end-timedelta(days=45);bbox=[lon-.025,lat-.025,lon+.025,lat+.025] | |
| with httpx.Client(timeout=45) as client: | |
| r=client.post('https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token', | |
| data={'grant_type':'client_credentials','client_id':client_id,'client_secret':secret}) | |
| # Do not propagate response/token/request bodies into logs. | |
| if r.status_code!=200:raise ValueError(f'CDSE authentication failed (HTTP {r.status_code})') | |
| headers={'Authorization':'Bearer '+r.json()['access_token']} | |
| r=client.post(CDSE+'/api/v1/catalog/1.0.0/search',headers=headers,json={ | |
| 'collections':['sentinel-2-l2a'],'bbox':bbox,'datetime':f'{start.isoformat()}/{end.isoformat()}', | |
| 'limit':30,'filter':'eo:cloud_cover < 30','filter-lang':'cql2-text'}) | |
| r.raise_for_status();items=r.json().get('features',[]) | |
| if not items:raise ValueError('No recent Sentinel-2 L2A scene in CDSE search') | |
| item=max(items,key=lambda i:i['properties']['datetime']);stamp=utc(item['properties']['datetime']) | |
| key=digest(['cdse-red-nir-scl-1',item['id'],bbox]) | |
| try:return store.get('assets',key) | |
| except KeyError:pass | |
| script='''//VERSION=3 | |
| function setup(){return {input:[{bands:["B04","B08","SCL","dataMask"],units:["REFLECTANCE","REFLECTANCE","DN","DN"]}],output:{bands:4,sampleType:"FLOAT32"}};} | |
| function evaluatePixel(s){return [s.B04,s.B08,s.SCL,s.dataMask];}''' | |
| r=client.post(CDSE+'/api/v1/process',headers=headers,json={ | |
| 'input':{'bounds':{'bbox':bbox,'properties':{'crs':'http://www.opengis.net/def/crs/EPSG/0/4326'}}, | |
| 'data':[{'type':'sentinel-2-l2a','dataFilter':{'timeRange':{'from':(stamp-timedelta(seconds=1)).isoformat(),'to':(stamp+timedelta(seconds=1)).isoformat()},'mosaickingOrder':'mostRecent'}}]}, | |
| 'output':{'width':256,'height':256,'responses':[{'identifier':'default','format':{'type':'image/tiff'}}]},'evalscript':script}) | |
| r.raise_for_status() | |
| if len(r.content)>5_000_000:raise ValueError('CDSE crop exceeds bounded response limit') | |
| with MemoryFile(r.content) as mem,mem.open() as src: | |
| if src.count!=4 or src.width!=256 or src.height!=256:raise ValueError('Unexpected CDSE band/grid contract') | |
| a=src.read();transform=from_bounds(*bbox,256,256) | |
| valid=np.isfinite(a).all(0)&(a[3]>0)&np.isin(a[2],[4,5]) # Vegetated/non-vegetated land, exclude cloud/water/shadow/uncertain. | |
| np.savez_compressed(store.path_for(key,'vegetation.npz'),red=a[0],nir=a[1],valid=valid,transform=list(transform)[:6],crs='EPSG:4326') | |
| return store.put('assets',{'kind':'vegetation_image','name':item['id'],'source':'COPERNICUS DATA SPACE', | |
| 'source_url':CDSE+'/api/v1/process','product_id':item['id'],'acquisition_time':stamp.isoformat(), | |
| 'retrieved_at':store.now(),'bbox':bbox,'crs':'EPSG:4326','resolution':[.05/256,.05/256], | |
| 'license':'Copernicus Sentinel data terms','qa':'SCL land classes 4/5 and dataMask; cloud/shadow/snow/water/uncertain excluded', | |
| 'warnings':['Historical satellite observation, not live vegetation','256x256 resampling; source bands 10/20 m']},key) | |
| def imerg_import(path,lon,lat,filename=None): | |
| """NASA PPS 30-minute GIS accumulations: stored integer x0.1 mm, not mm/h.""" | |
| locations.require_uae(lon,lat);name=filename or Path(path).name | |
| match=re.search(r'(\d{8})-S(\d{6})-E(\d{6}).*\.30min\.tif$',name,re.I) | |
| if not match:raise ValueError('IMERG import requires the original NASA .30min.tif filename with interval') | |
| start=datetime.strptime(match[1]+match[2],'%Y%m%d%H%M%S').replace(tzinfo=timezone.utc) | |
| end=start+timedelta(minutes=30) | |
| if end>utc()+timedelta(minutes=5):raise ValueError('IMERG observation interval is in the future') | |
| with rasterio.open(path) as src: | |
| if str(src.crs)!='EPSG:4326' or src.count!=1:raise ValueError('Expected NASA GIS geographic precipitation raster') | |
| if not .09<=abs(src.transform.a)<=.11 or not .09<=abs(src.transform.e)<=.11:raise ValueError('Expected IMERG 0.1 degree grid') | |
| win=window_from_bounds(lon-.15,lat-.15,lon+.15,lat+.15,src.transform).round_offsets().round_lengths() | |
| data=src.read(1,window=win,masked=True).astype('float32');t=src.window_transform(win) | |
| data=np.ma.masked_where((data<0)|(data>=29999),data)*.1 | |
| if not data.count():raise ValueError('No valid IMERG observations for study area') | |
| grid=[] | |
| for y,x in zip(*np.where(~np.ma.getmaskarray(data))): | |
| point=t*(int(x)+.5,int(y)+.5) | |
| if locations.within_uae(*point):grid.append({'lon':point[0],'lat':point[1],'mm':float(data[y,x])}) | |
| if not grid:raise ValueError('No valid IMERG cells inside UAE') | |
| identity=digest(['imerg-gis-30min-1',name,lon,lat]) | |
| return store.put('precipitation_observations',{'source':'NASA GPM IMERG Early/Late GIS','source_url':'https://gpm.nasa.gov/data/imerg', | |
| 'product_id':name,'category':'external_observation','retrieved_at':store.now(),'valid_start':start.isoformat(),'valid_end':end.isoformat(), | |
| 'resolution_deg':.1,'unit':'mm','interval_minutes':30,'longitude':lon,'latitude':lat,'grid':grid, | |
| 'checksum':hashlib.sha256(Path(path).read_bytes()).hexdigest(), | |
| 'limitations':['Satellite precipitation estimate; not a ground rain gauge','Coarse 0.1 degree support; 30-minute accumulation scaled by 0.1']},identity) | |
| def fetch_imerg(lon,lat): | |
| locations.require_uae(lon,lat) | |
| user=os.getenv('NASA_PPS_USERNAME');password=os.getenv('NASA_PPS_PASSWORD') | |
| if not user or not password:raise ValueError('NASA PPS credentials not configured; IMERG GeoTIFF import remains available') | |
| day=utc()-timedelta(hours=6) | |
| directory=PPS+f'/imerg/gis/early/{day:%Y/%m}/' | |
| with httpx.Client(auth=(user,password),timeout=45,follow_redirects=True) as client: | |
| r=client.get(directory) | |
| if r.status_code!=200:raise ValueError(f'NASA PPS directory unavailable (HTTP {r.status_code})') | |
| names=re.findall(r'href=["\']([^"\'/]+\.30min\.tif)["\']',r.text,re.I) | |
| names=[n for n in names if re.fullmatch(r'[A-Za-z0-9_.-]+',n)] | |
| if not names:raise ValueError('No NASA 30-minute GIS GeoTIFF listed; provider layout/access may have changed') | |
| name=max(names) | |
| existing=next((v for v in store.all_records('precipitation_observations') if v['product_id']==name and v['longitude']==lon and v['latitude']==lat),None) | |
| if existing:return existing | |
| # One compressed global GIS image bounded at 40 MB; only the small AOI is processed. | |
| target=store.path_for(digest([name]),'imerg.tif') | |
| try: | |
| with client.stream('GET',directory+name) as response: | |
| response.raise_for_status();size=0 | |
| with target.open('wb') as f: | |
| for chunk in response.iter_bytes(65536): | |
| size+=len(chunk) | |
| if size>40_000_000:raise ValueError('IMERG GIS download exceeds 40 MB limit') | |
| f.write(chunk) | |
| return imerg_import(target,lon,lat,name) | |
| finally:target.unlink(missing_ok=True) | |
| def evaluate_rainfall(wadi): | |
| """Archive point/grid comparisons only for two contiguous half-hour observations. | |
| Choose the latest forecast acquired BEFORE the observed hour started; no hindsight | |
| selection or overlap between an hourly forecast and a single half-hour target. | |
| """ | |
| from .intelligence import area | |
| w=area(wadi);observed=[r for r in store.all_records('precipitation_observations') if r.get('unit')=='mm' and r.get('interval_minutes')==30 and abs(r['longitude']-w['lon'])<.01 and abs(r['latitude']-w['lat'])<.01] | |
| results=[] | |
| for a in observed: | |
| end=utc(a['valid_end']) | |
| if end.minute!=0:continue | |
| b=next((r for r in observed if r['valid_end']==a['valid_start']),None) | |
| if not b:continue | |
| start=end-timedelta(hours=1) | |
| weather=[r for r in store.all_records('weather') if r.get('provenance')=='EXTERNAL FORECAST' and r.get('units',{}).get('precipitation')=='mm' and utc(r['retrieved_at'])<=start and abs(r.get('location',{}).get('requested_lon',999)-w['lon'])<.01 and abs(r.get('location',{}).get('requested_lat',999)-w['lat'])<.01] | |
| for f in sorted(weather,key=lambda r:r['retrieved_at'],reverse=True): | |
| pairs=list(zip(f['hourly']['time'],f['hourly'].get('precipitation',[]))) | |
| value=next((v for t,v in pairs if utc(t if t.endswith('Z') or '+' in t[10:] else t+'+00:00')==end),None) | |
| if value is None:continue | |
| nearest=lambda r:min(r['grid'],key=lambda p:(p['lon']-w['lon'])**2+(p['lat']-w['lat'])**2) | |
| ca,cb=nearest(a),nearest(b) | |
| if abs(ca['lon']-cb['lon'])>1e-5 or abs(ca['lat']-cb['lat'])>1e-5:continue | |
| total=ca['mm']+cb['mm'];key=digest([f['id'],a['id'],b['id']]) | |
| results.append(store.put('rainfall_evaluations',{'wadi':wadi,'forecast_id':f['id'],'observation_ids':[a['id'],b['id']], | |
| 'valid_end':end.isoformat(),'forecast_mm':value,'observed_estimate_mm':total,'error_mm':value-total, | |
| 'absolute_error_mm':abs(value-total),'scope':'Single coarse IMERG grid-cell vs forecast point; not gauge validation'},key));break | |
| return results | |