"""Fixed-provider adapters. No arbitrary user URL or filesystem downloads.""" import hashlib import json import math from pathlib import Path import httpx import rasterio import numpy as np from rasterio.windows import from_bounds as window_from_bounds from rasterio.transform import from_bounds from rasterio.warp import reproject, transform_bounds, Resampling from .store import ROOT, DATA, path_for, asset, put, now, all_records from .geo import ingest_dem, validate_geojson from .locations import require_uae MODEL_REPO='giswqs/s2-water-unetplusplus-efficientnet-b4' MODEL_REV='efb6f636480188f48779b41f2a94bb941fd7b900' MODEL_DIR=ROOT/'models'/'s2-water' HEADERS={'User-Agent':'Satforge-Research-Prototype/1.0 (educational geospatial application)'} def download(url,path,max_bytes=150_000_000): path=Path(path); path.parent.mkdir(parents=True,exist_ok=True) temporary=path.with_suffix(path.suffix+'.part') count=0 try: with httpx.stream('GET',url,timeout=120,follow_redirects=True,headers=HEADERS) as r: r.raise_for_status() with temporary.open('wb') as f: for chunk in r.iter_bytes(65536): count+=len(chunk) if count>max_bytes: raise ValueError('Provider file exceeds download size limit') f.write(chunk) temporary.replace(path) finally: temporary.unlink(missing_ok=True) return hashlib.sha256(path.read_bytes()).hexdigest() def acquire_dem(lon,lat,radius=.035): require_uae(lon,lat) # Crop one tile. Reject crossing tile boundaries rather than silently lose coverage. bbox=[max(math.floor(lon)+.001,lon-radius),max(math.floor(lat)+.001,lat-radius),min(math.floor(lon)+.999,lon+radius),min(math.floor(lat)+.999,lat+radius)] tile=f'Copernicus_DSM_COG_30_N{math.floor(lat):02d}_00_E{math.floor(lon):03d}_00_DEM' url=f'https://copernicus-dem-90m.s3.eu-central-1.amazonaws.com/{tile}/{tile}.tif' target=DATA/f'{tile}.tif' if not target.exists(): download(url,target,40_000_000) return ingest_dem(target,f'Copernicus GLO-90 · {lat:.3f}, {lon:.3f}','COPERNICUS DEM',bbox, {'source_url':url,'publication_time':'2021 release','license':'Copernicus DEM free public licence; attribution required', 'license_url':'https://registry.opendata.aws/copernicus-dem/','native_resolution_m':90,'sha256':hashlib.sha256(target.read_bytes()).hexdigest(), 'acquisition_time':None}) def acquire_infrastructure(lon,lat,radius=.035): require_uae(lon,lat) bbox=f'{lat-radius},{lon-radius},{lat+radius},{lon+radius}' query=f'[out:json][timeout:40];(way[highway]({bbox});way[building]({bbox});node[amenity~"hospital|school|fire_station"]({bbox}););out geom;' with httpx.Client(timeout=55,headers=HEADERS) as client: for endpoint in ('https://overpass-api.de/api/interpreter','https://overpass.private.coffee/api/interpreter'): try: r=client.post(endpoint,data={'data':query});r.raise_for_status();data=r.json() break except httpx.HTTPError: if endpoint.endswith('overpass.private.coffee/api/interpreter'):raise fs=[] for el in data.get('elements',[]): tags=el.get('tags',{}); category='road' if 'highway' in tags else 'building' if 'building' in tags else 'facility' if el['type']=='node': geom={'type':'Point','coordinates':[el['lon'],el['lat']]} else: coords=[[p['lon'],p['lat']] for p in el.get('geometry',[])] if len(coords)<2: continue closed=category=='building' and len(coords)>=4 and coords[0]==coords[-1] geom={'type':'Polygon' if closed else 'LineString','coordinates':[coords] if closed else coords} fs.append({'type':'Feature','id':str(el['id']),'geometry':geom,'properties':{**tags,'category':category,'name':tags.get('name',tags.get('highway',category))}}) fc=validate_geojson({'type':'FeatureCollection','features':fs}) return asset('infrastructure','OpenStreetMap roads & buildings','OPENSTREETMAP',geojson=fc,source_url=endpoint,license='ODbL 1.0; © OpenStreetMap contributors',acquisition_time=data.get('osm3s',{}).get('timestamp_osm_base'),bbox=[lon-radius,lat-radius,lon+radius,lat+radius],warnings=['Community mapping may be incomplete','Intersection is not verified damage']) def fetch_weather(lon,lat,days=3,historical_start=None,historical_end=None): require_uae(lon,lat) historical=historical_start is not None base='https://archive-api.open-meteo.com/v1/archive' if historical else 'https://api.open-meteo.com/v1/forecast' variables='temperature_2m,relative_humidity_2m,precipitation,surface_pressure,cloud_cover,wind_speed_10m,wind_direction_10m' params={'latitude':lat,'longitude':lon,'hourly':variables,'timezone':'UTC','wind_speed_unit':'ms'} if historical: params.update(start_date=historical_start,end_date=historical_end) else: params.update(forecast_days=days) key=json.dumps(params,sort_keys=True) from datetime import datetime,timezone for old in all_records('weather'): if old.get('cache_key')==key and (datetime.now(timezone.utc)-datetime.fromisoformat(old['retrieved_at'])).total_seconds()<1800: return {**old,'cached':True} r=httpx.get(base,params=params,headers=HEADERS,timeout=30);r.raise_for_status();data=r.json() if 'hourly' not in data or not data['hourly'].get('time'): raise ValueError('Provider returned no hourly weather') record=put('weather',{'source':'Open-Meteo','source_url':str(r.url),'license':'CC BY 4.0','retrieved_at':now(),'cache_key':key, 'provenance':'REANALYSIS' if historical else 'EXTERNAL FORECAST','location':{'requested_lon':lon,'requested_lat':lat,'grid_lon':data['longitude'],'grid_lat':data['latitude']}, 'hourly':data['hourly'],'units':data.get('hourly_units',{}),'cached':False, 'warnings':['Model grid-point data, not catchment-wide measured rainfall','No cloud microphysics or validated seeding suitability']}) return record def install_model(): MODEL_DIR.mkdir(parents=True,exist_ok=True) manifest={} for name in ['config.json','README.md','model.pth']: p=MODEL_DIR/name url=f'https://huggingface.co/{MODEL_REPO}/resolve/{MODEL_REV}/{name}' sha=hashlib.sha256(p.read_bytes()).hexdigest() if p.exists() else download(url,p) manifest[name]=sha (MODEL_DIR/'manifest.json').write_text(json.dumps({'repo':MODEL_REPO,'revision':MODEL_REV,'sha256':manifest,'downloaded_at':now(),'license':'CC BY 4.0'},indent=2)) return {'installed':True,'repo':MODEL_REPO,'revision':MODEL_REV,'sha256':manifest} def acquire_sentinel(lon,lat): """Small real six-band Sentinel-2 crop, with product ID and acquisition timestamp.""" from datetime import datetime,timezone,timedelta require_uae(lon,lat) end=datetime.now(timezone.utc);start=end-timedelta(days=365) bbox=[lon-.025,lat-.025,lon+.025,lat+.025] query={'collections':['sentinel-2-l2a'],'bbox':bbox,'datetime':f'{start:%Y-%m-%dT00:00:00Z}/{end:%Y-%m-%dT23:59:59Z}','limit':20, 'query':{'eo:cloud_cover':{'lt':20}}} r=httpx.post('https://earth-search.aws.element84.com/v1/search',json=query,timeout=45);r.raise_for_status() items=r.json().get('features',[]) if not items: raise ValueError('No Sentinel-2 scene with <20% scene cloud cover in the last year; import imagery instead') item=max(items,key=lambda i:i['properties']['datetime']); props=item['properties'] existing=next((a for a in all_records('assets') if a.get('product_id')==item['id'] and a.get('bbox') and a['bbox'][0]<=lon<=a['bbox'][2] and a['bbox'][1]<=lat<=a['bbox'][3]),None) if existing:return existing bands=['blue','green','red','nir','swir16','swir22'] arrays=[]; dst_t=None; dst_crs=None; valid=None # COG window reads avoid downloading complete satellite scenes. with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',CPL_VSIL_CURL_ALLOWED_EXTENSIONS='.tif',GDAL_HTTP_TIMEOUT='60'): for band in bands: url=item['assets'][band]['href'] with rasterio.open(url) as src: bb=transform_bounds('EPSG:4326',src.crs,*bbox) if dst_t is None: dst_crs=src.crs; dst_t=from_bounds(*bb,256,256) arr=np.full((256,256),np.nan,dtype='float32') win=window_from_bounds(*bb,src.transform).round_offsets().round_lengths() crop=src.read(1,window=win,boundless=True,fill_value=src.nodata or 0) reproject(crop,arr,src_transform=src.window_transform(win),src_crs=src.crs,src_nodata=src.nodata or 0,dst_transform=dst_t,dst_crs=dst_crs,dst_nodata=np.nan,resampling=Resampling.bilinear) # Harmonize physical reflectance using provider scale/offset; store reflectance*10000. rb=item['assets'][band].get('raster:bands',[{}])[0] scale=rb.get('scale',.0001); offset=rb.get('offset',0.) arr=(arr*scale+offset)*10000 arrays.append(arr) scl=item['assets'].get('scl') cloud_mask=None if scl: with rasterio.open(scl['href']) as src: bb=transform_bounds('EPSG:4326',src.crs,*bbox);win=window_from_bounds(*bb,src.transform).round_offsets().round_lengths() crop=src.read(1,window=win,boundless=True,fill_value=0);classes=np.zeros((256,256),dtype='uint8') reproject(crop,classes,src_transform=src.window_transform(win),src_crs=src.crs,dst_transform=dst_t,dst_crs=dst_crs,resampling=Resampling.nearest) cloud_mask=np.isin(classes,[0,1,3,8,9,10,11]) from .imagery import ingest_image import uuid tmp=DATA/f'sentinel_{uuid.uuid4().hex}.tif' data=np.stack(arrays);valid=np.isfinite(data).all(axis=0) if cloud_mask is not None: valid&=~cloud_mask data[:,~valid]=-9999 with rasterio.open(tmp,'w',driver='GTiff',width=256,height=256,count=6,dtype='float32',crs=dst_crs,transform=dst_t,nodata=-9999) as dst: dst.write(data);dst.descriptions=('B2','B3','B4','B8','B11','B12') rec=ingest_image(tmp,f'Sentinel-2 · {item["id"]}',metadata={'source':'COPERNICUS SENTINEL-2','source_url':item.get('links',[{}])[0].get('href','https://earth-search.aws.element84.com/v1'), 'product_id':item['id'],'acquisition_time':props['datetime'],'scene_cloud_cover_pct':props.get('eo:cloud_cover'),'license':'Copernicus Sentinel data terms; free and open', 'band_contract':'S2_L2A_6B_DN10000','qa_available':cloud_mask is not None,'processing':['COG crop','Resampled to 256×256','Provider scale/offset to reflectance ×10000','SCL cloud/shadow/snow/nodata exclusion'], 'warnings':['Acquisition date is historical, not live','Scene cloud percentage is not local cloud percentage','Mixed 10/20m source bands; output resolution recorded']}) tmp.unlink(missing_ok=True) return rec