"""Bounded, credential-aware satellite adapters. Failures never become synthetic data.""" import os import re from datetime import datetime, timedelta, timezone from pathlib import Path import hashlib import httpx import numpy as np import rasterio from rasterio.io import MemoryFile from rasterio.transform import from_bounds from rasterio.windows import from_bounds as window_from_bounds from . import store, locations from .observations import digest, utc CDSE='https://sh.dataspace.copernicus.eu' PPS='https://jsimpsonhttps.pps.eosdis.nasa.gov' def cdse_vegetation(lon,lat): locations.require_uae(lon,lat) client_id=os.getenv('CDSE_CLIENT_ID');secret=os.getenv('CDSE_CLIENT_SECRET') if not client_id or not secret:raise ValueError('Copernicus Data Space OAuth client credentials not configured; existing Sentinel COG crops remain usable') end=utc();start=end-timedelta(days=45);bbox=[lon-.025,lat-.025,lon+.025,lat+.025] with httpx.Client(timeout=45) as client: r=client.post('https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token', data={'grant_type':'client_credentials','client_id':client_id,'client_secret':secret}) # Do not propagate response/token/request bodies into logs. if r.status_code!=200:raise ValueError(f'CDSE authentication failed (HTTP {r.status_code})') headers={'Authorization':'Bearer '+r.json()['access_token']} r=client.post(CDSE+'/api/v1/catalog/1.0.0/search',headers=headers,json={ 'collections':['sentinel-2-l2a'],'bbox':bbox,'datetime':f'{start.isoformat()}/{end.isoformat()}', 'limit':30,'filter':'eo:cloud_cover < 30','filter-lang':'cql2-text'}) r.raise_for_status();items=r.json().get('features',[]) if not items:raise ValueError('No recent Sentinel-2 L2A scene in CDSE search') item=max(items,key=lambda i:i['properties']['datetime']);stamp=utc(item['properties']['datetime']) key=digest(['cdse-red-nir-scl-1',item['id'],bbox]) try:return store.get('assets',key) except KeyError:pass script='''//VERSION=3 function setup(){return {input:[{bands:["B04","B08","SCL","dataMask"],units:["REFLECTANCE","REFLECTANCE","DN","DN"]}],output:{bands:4,sampleType:"FLOAT32"}};} function evaluatePixel(s){return [s.B04,s.B08,s.SCL,s.dataMask];}''' r=client.post(CDSE+'/api/v1/process',headers=headers,json={ 'input':{'bounds':{'bbox':bbox,'properties':{'crs':'http://www.opengis.net/def/crs/EPSG/0/4326'}}, 'data':[{'type':'sentinel-2-l2a','dataFilter':{'timeRange':{'from':(stamp-timedelta(seconds=1)).isoformat(),'to':(stamp+timedelta(seconds=1)).isoformat()},'mosaickingOrder':'mostRecent'}}]}, 'output':{'width':256,'height':256,'responses':[{'identifier':'default','format':{'type':'image/tiff'}}]},'evalscript':script}) r.raise_for_status() if len(r.content)>5_000_000:raise ValueError('CDSE crop exceeds bounded response limit') with MemoryFile(r.content) as mem,mem.open() as src: if src.count!=4 or src.width!=256 or src.height!=256:raise ValueError('Unexpected CDSE band/grid contract') a=src.read();transform=from_bounds(*bbox,256,256) valid=np.isfinite(a).all(0)&(a[3]>0)&np.isin(a[2],[4,5]) # Vegetated/non-vegetated land, exclude cloud/water/shadow/uncertain. np.savez_compressed(store.path_for(key,'vegetation.npz'),red=a[0],nir=a[1],valid=valid,transform=list(transform)[:6],crs='EPSG:4326') return store.put('assets',{'kind':'vegetation_image','name':item['id'],'source':'COPERNICUS DATA SPACE', 'source_url':CDSE+'/api/v1/process','product_id':item['id'],'acquisition_time':stamp.isoformat(), 'retrieved_at':store.now(),'bbox':bbox,'crs':'EPSG:4326','resolution':[.05/256,.05/256], 'license':'Copernicus Sentinel data terms','qa':'SCL land classes 4/5 and dataMask; cloud/shadow/snow/water/uncertain excluded', 'warnings':['Historical satellite observation, not live vegetation','256x256 resampling; source bands 10/20 m']},key) def imerg_import(path,lon,lat,filename=None): """NASA PPS 30-minute GIS accumulations: stored integer x0.1 mm, not mm/h.""" locations.require_uae(lon,lat);name=filename or Path(path).name match=re.search(r'(\d{8})-S(\d{6})-E(\d{6}).*\.30min\.tif$',name,re.I) if not match:raise ValueError('IMERG import requires the original NASA .30min.tif filename with interval') start=datetime.strptime(match[1]+match[2],'%Y%m%d%H%M%S').replace(tzinfo=timezone.utc) end=start+timedelta(minutes=30) if end>utc()+timedelta(minutes=5):raise ValueError('IMERG observation interval is in the future') with rasterio.open(path) as src: if str(src.crs)!='EPSG:4326' or src.count!=1:raise ValueError('Expected NASA GIS geographic precipitation raster') if not .09<=abs(src.transform.a)<=.11 or not .09<=abs(src.transform.e)<=.11:raise ValueError('Expected IMERG 0.1 degree grid') win=window_from_bounds(lon-.15,lat-.15,lon+.15,lat+.15,src.transform).round_offsets().round_lengths() data=src.read(1,window=win,masked=True).astype('float32');t=src.window_transform(win) data=np.ma.masked_where((data<0)|(data>=29999),data)*.1 if not data.count():raise ValueError('No valid IMERG observations for study area') grid=[] for y,x in zip(*np.where(~np.ma.getmaskarray(data))): point=t*(int(x)+.5,int(y)+.5) if locations.within_uae(*point):grid.append({'lon':point[0],'lat':point[1],'mm':float(data[y,x])}) if not grid:raise ValueError('No valid IMERG cells inside UAE') identity=digest(['imerg-gis-30min-1',name,lon,lat]) return store.put('precipitation_observations',{'source':'NASA GPM IMERG Early/Late GIS','source_url':'https://gpm.nasa.gov/data/imerg', 'product_id':name,'category':'external_observation','retrieved_at':store.now(),'valid_start':start.isoformat(),'valid_end':end.isoformat(), 'resolution_deg':.1,'unit':'mm','interval_minutes':30,'longitude':lon,'latitude':lat,'grid':grid, 'checksum':hashlib.sha256(Path(path).read_bytes()).hexdigest(), 'limitations':['Satellite precipitation estimate; not a ground rain gauge','Coarse 0.1 degree support; 30-minute accumulation scaled by 0.1']},identity) def fetch_imerg(lon,lat): locations.require_uae(lon,lat) user=os.getenv('NASA_PPS_USERNAME');password=os.getenv('NASA_PPS_PASSWORD') if not user or not password:raise ValueError('NASA PPS credentials not configured; IMERG GeoTIFF import remains available') day=utc()-timedelta(hours=6) directory=PPS+f'/imerg/gis/early/{day:%Y/%m}/' with httpx.Client(auth=(user,password),timeout=45,follow_redirects=True) as client: r=client.get(directory) if r.status_code!=200:raise ValueError(f'NASA PPS directory unavailable (HTTP {r.status_code})') names=re.findall(r'href=["\']([^"\'/]+\.30min\.tif)["\']',r.text,re.I) names=[n for n in names if re.fullmatch(r'[A-Za-z0-9_.-]+',n)] if not names:raise ValueError('No NASA 30-minute GIS GeoTIFF listed; provider layout/access may have changed') name=max(names) existing=next((v for v in store.all_records('precipitation_observations') if v['product_id']==name and v['longitude']==lon and v['latitude']==lat),None) if existing:return existing # One compressed global GIS image bounded at 40 MB; only the small AOI is processed. target=store.path_for(digest([name]),'imerg.tif') try: with client.stream('GET',directory+name) as response: response.raise_for_status();size=0 with target.open('wb') as f: for chunk in response.iter_bytes(65536): size+=len(chunk) if size>40_000_000:raise ValueError('IMERG GIS download exceeds 40 MB limit') f.write(chunk) return imerg_import(target,lon,lat,name) finally:target.unlink(missing_ok=True) def evaluate_rainfall(wadi): """Archive point/grid comparisons only for two contiguous half-hour observations. Choose the latest forecast acquired BEFORE the observed hour started; no hindsight selection or overlap between an hourly forecast and a single half-hour target. """ from .intelligence import area w=area(wadi);observed=[r for r in store.all_records('precipitation_observations') if r.get('unit')=='mm' and r.get('interval_minutes')==30 and abs(r['longitude']-w['lon'])<.01 and abs(r['latitude']-w['lat'])<.01] results=[] for a in observed: end=utc(a['valid_end']) if end.minute!=0:continue b=next((r for r in observed if r['valid_end']==a['valid_start']),None) if not b:continue start=end-timedelta(hours=1) weather=[r for r in store.all_records('weather') if r.get('provenance')=='EXTERNAL FORECAST' and r.get('units',{}).get('precipitation')=='mm' and utc(r['retrieved_at'])<=start and abs(r.get('location',{}).get('requested_lon',999)-w['lon'])<.01 and abs(r.get('location',{}).get('requested_lat',999)-w['lat'])<.01] for f in sorted(weather,key=lambda r:r['retrieved_at'],reverse=True): pairs=list(zip(f['hourly']['time'],f['hourly'].get('precipitation',[]))) value=next((v for t,v in pairs if utc(t if t.endswith('Z') or '+' in t[10:] else t+'+00:00')==end),None) if value is None:continue nearest=lambda r:min(r['grid'],key=lambda p:(p['lon']-w['lon'])**2+(p['lat']-w['lat'])**2) ca,cb=nearest(a),nearest(b) if abs(ca['lon']-cb['lon'])>1e-5 or abs(ca['lat']-cb['lat'])>1e-5:continue total=ca['mm']+cb['mm'];key=digest([f['id'],a['id'],b['id']]) results.append(store.put('rainfall_evaluations',{'wadi':wadi,'forecast_id':f['id'],'observation_ids':[a['id'],b['id']], 'valid_end':end.isoformat(),'forecast_mm':value,'observed_estimate_mm':total,'error_mm':value-total, 'absolute_error_mm':abs(value-total),'scope':'Single coarse IMERG grid-cell vs forecast point; not gauge validation'},key));break return results