saef-api / backend /sources.py
m4az19's picture
Deploy SAHAB-1 backend and pretrained AI
124816b verified
Raw History Blame Contribute Delete
11.2 kB
"""Fixed-provider adapters. No arbitrary user URL or filesystem downloads."""
import hashlib
import json
import math
from pathlib import Path
import httpx
import rasterio
import numpy as np
from rasterio.windows import from_bounds as window_from_bounds
from rasterio.transform import from_bounds
from rasterio.warp import reproject, transform_bounds, Resampling
from .store import ROOT, DATA, path_for, asset, put, now, all_records
from .geo import ingest_dem, validate_geojson
from .locations import require_uae
MODEL_REPO='giswqs/s2-water-unetplusplus-efficientnet-b4'
MODEL_REV='efb6f636480188f48779b41f2a94bb941fd7b900'
MODEL_DIR=ROOT/'models'/'s2-water'
HEADERS={'User-Agent':'Satforge-Research-Prototype/1.0 (educational geospatial application)'}
def download(url,path,max_bytes=150_000_000):
path=Path(path); path.parent.mkdir(parents=True,exist_ok=True)
temporary=path.with_suffix(path.suffix+'.part')
count=0
try:
with httpx.stream('GET',url,timeout=120,follow_redirects=True,headers=HEADERS) as r:
r.raise_for_status()
with temporary.open('wb') as f:
for chunk in r.iter_bytes(65536):
count+=len(chunk)
if count>max_bytes: raise ValueError('Provider file exceeds download size limit')
f.write(chunk)
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)
return hashlib.sha256(path.read_bytes()).hexdigest()
def acquire_dem(lon,lat,radius=.035):
require_uae(lon,lat)
# Crop one tile. Reject crossing tile boundaries rather than silently lose coverage.
bbox=[max(math.floor(lon)+.001,lon-radius),max(math.floor(lat)+.001,lat-radius),min(math.floor(lon)+.999,lon+radius),min(math.floor(lat)+.999,lat+radius)]
tile=f'Copernicus_DSM_COG_30_N{math.floor(lat):02d}_00_E{math.floor(lon):03d}_00_DEM'
url=f'https://copernicus-dem-90m.s3.eu-central-1.amazonaws.com/{tile}/{tile}.tif'
target=DATA/f'{tile}.tif'
if not target.exists(): download(url,target,40_000_000)
return ingest_dem(target,f'Copernicus GLO-90 · {lat:.3f}, {lon:.3f}','COPERNICUS DEM',bbox,
{'source_url':url,'publication_time':'2021 release','license':'Copernicus DEM free public licence; attribution required',
'license_url':'https://registry.opendata.aws/copernicus-dem/','native_resolution_m':90,'sha256':hashlib.sha256(target.read_bytes()).hexdigest(),
'acquisition_time':None})
def acquire_infrastructure(lon,lat,radius=.035):
require_uae(lon,lat)
bbox=f'{lat-radius},{lon-radius},{lat+radius},{lon+radius}'
query=f'[out:json][timeout:40];(way[highway]({bbox});way[building]({bbox});node[amenity~"hospital|school|fire_station"]({bbox}););out geom;'
with httpx.Client(timeout=55,headers=HEADERS) as client:
for endpoint in ('https://overpass-api.de/api/interpreter','https://overpass.private.coffee/api/interpreter'):
try:
r=client.post(endpoint,data={'data':query});r.raise_for_status();data=r.json()
break
except httpx.HTTPError:
if endpoint.endswith('overpass.private.coffee/api/interpreter'):raise
fs=[]
for el in data.get('elements',[]):
tags=el.get('tags',{}); category='road' if 'highway' in tags else 'building' if 'building' in tags else 'facility'
if el['type']=='node': geom={'type':'Point','coordinates':[el['lon'],el['lat']]}
else:
coords=[[p['lon'],p['lat']] for p in el.get('geometry',[])]
if len(coords)<2: continue
closed=category=='building' and len(coords)>=4 and coords[0]==coords[-1]
geom={'type':'Polygon' if closed else 'LineString','coordinates':[coords] if closed else coords}
fs.append({'type':'Feature','id':str(el['id']),'geometry':geom,'properties':{**tags,'category':category,'name':tags.get('name',tags.get('highway',category))}})
fc=validate_geojson({'type':'FeatureCollection','features':fs})
return asset('infrastructure','OpenStreetMap roads & buildings','OPENSTREETMAP',geojson=fc,source_url=endpoint,license='ODbL 1.0; © OpenStreetMap contributors',acquisition_time=data.get('osm3s',{}).get('timestamp_osm_base'),bbox=[lon-radius,lat-radius,lon+radius,lat+radius],warnings=['Community mapping may be incomplete','Intersection is not verified damage'])
def fetch_weather(lon,lat,days=3,historical_start=None,historical_end=None):
require_uae(lon,lat)
historical=historical_start is not None
base='https://archive-api.open-meteo.com/v1/archive' if historical else 'https://api.open-meteo.com/v1/forecast'
variables='temperature_2m,relative_humidity_2m,precipitation,surface_pressure,cloud_cover,wind_speed_10m,wind_direction_10m'
params={'latitude':lat,'longitude':lon,'hourly':variables,'timezone':'UTC','wind_speed_unit':'ms'}
if historical: params.update(start_date=historical_start,end_date=historical_end)
else: params.update(forecast_days=days)
key=json.dumps(params,sort_keys=True)
from datetime import datetime,timezone
for old in all_records('weather'):
if old.get('cache_key')==key and (datetime.now(timezone.utc)-datetime.fromisoformat(old['retrieved_at'])).total_seconds()<1800:
return {**old,'cached':True}
r=httpx.get(base,params=params,headers=HEADERS,timeout=30);r.raise_for_status();data=r.json()
if 'hourly' not in data or not data['hourly'].get('time'): raise ValueError('Provider returned no hourly weather')
record=put('weather',{'source':'Open-Meteo','source_url':str(r.url),'license':'CC BY 4.0','retrieved_at':now(),'cache_key':key,
'provenance':'REANALYSIS' if historical else 'EXTERNAL FORECAST','location':{'requested_lon':lon,'requested_lat':lat,'grid_lon':data['longitude'],'grid_lat':data['latitude']},
'hourly':data['hourly'],'units':data.get('hourly_units',{}),'cached':False,
'warnings':['Model grid-point data, not catchment-wide measured rainfall','No cloud microphysics or validated seeding suitability']})
return record
def install_model():
MODEL_DIR.mkdir(parents=True,exist_ok=True)
manifest={}
for name in ['config.json','README.md','model.pth']:
p=MODEL_DIR/name
url=f'https://huggingface.co/{MODEL_REPO}/resolve/{MODEL_REV}/{name}'
sha=hashlib.sha256(p.read_bytes()).hexdigest() if p.exists() else download(url,p)
manifest[name]=sha
(MODEL_DIR/'manifest.json').write_text(json.dumps({'repo':MODEL_REPO,'revision':MODEL_REV,'sha256':manifest,'downloaded_at':now(),'license':'CC BY 4.0'},indent=2))
return {'installed':True,'repo':MODEL_REPO,'revision':MODEL_REV,'sha256':manifest}
def acquire_sentinel(lon,lat):
"""Small real six-band Sentinel-2 crop, with product ID and acquisition timestamp."""
from datetime import datetime,timezone,timedelta
require_uae(lon,lat)
end=datetime.now(timezone.utc);start=end-timedelta(days=365)
bbox=[lon-.025,lat-.025,lon+.025,lat+.025]
query={'collections':['sentinel-2-l2a'],'bbox':bbox,'datetime':f'{start:%Y-%m-%dT00:00:00Z}/{end:%Y-%m-%dT23:59:59Z}','limit':20,
'query':{'eo:cloud_cover':{'lt':20}}}
r=httpx.post('https://earth-search.aws.element84.com/v1/search',json=query,timeout=45);r.raise_for_status()
items=r.json().get('features',[])
if not items: raise ValueError('No Sentinel-2 scene with <20% scene cloud cover in the last year; import imagery instead')
item=max(items,key=lambda i:i['properties']['datetime']); props=item['properties']
existing=next((a for a in all_records('assets') if a.get('product_id')==item['id'] and a.get('bbox') and
a['bbox'][0]<=lon<=a['bbox'][2] and a['bbox'][1]<=lat<=a['bbox'][3]),None)
if existing:return existing
bands=['blue','green','red','nir','swir16','swir22']
arrays=[]; dst_t=None; dst_crs=None; valid=None
# COG window reads avoid downloading complete satellite scenes.
with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',CPL_VSIL_CURL_ALLOWED_EXTENSIONS='.tif',GDAL_HTTP_TIMEOUT='60'):
for band in bands:
url=item['assets'][band]['href']
with rasterio.open(url) as src:
bb=transform_bounds('EPSG:4326',src.crs,*bbox)
if dst_t is None:
dst_crs=src.crs; dst_t=from_bounds(*bb,256,256)
arr=np.full((256,256),np.nan,dtype='float32')
win=window_from_bounds(*bb,src.transform).round_offsets().round_lengths()
crop=src.read(1,window=win,boundless=True,fill_value=src.nodata or 0)
reproject(crop,arr,src_transform=src.window_transform(win),src_crs=src.crs,src_nodata=src.nodata or 0,dst_transform=dst_t,dst_crs=dst_crs,dst_nodata=np.nan,resampling=Resampling.bilinear)
# Harmonize physical reflectance using provider scale/offset; store reflectance*10000.
rb=item['assets'][band].get('raster:bands',[{}])[0]
scale=rb.get('scale',.0001); offset=rb.get('offset',0.)
arr=(arr*scale+offset)*10000
arrays.append(arr)
scl=item['assets'].get('scl')
cloud_mask=None
if scl:
with rasterio.open(scl['href']) as src:
bb=transform_bounds('EPSG:4326',src.crs,*bbox);win=window_from_bounds(*bb,src.transform).round_offsets().round_lengths()
crop=src.read(1,window=win,boundless=True,fill_value=0);classes=np.zeros((256,256),dtype='uint8')
reproject(crop,classes,src_transform=src.window_transform(win),src_crs=src.crs,dst_transform=dst_t,dst_crs=dst_crs,resampling=Resampling.nearest)
cloud_mask=np.isin(classes,[0,1,3,8,9,10,11])
from .imagery import ingest_image
import uuid
tmp=DATA/f'sentinel_{uuid.uuid4().hex}.tif'
data=np.stack(arrays);valid=np.isfinite(data).all(axis=0)
if cloud_mask is not None: valid&=~cloud_mask
data[:,~valid]=-9999
with rasterio.open(tmp,'w',driver='GTiff',width=256,height=256,count=6,dtype='float32',crs=dst_crs,transform=dst_t,nodata=-9999) as dst:
dst.write(data);dst.descriptions=('B2','B3','B4','B8','B11','B12')
rec=ingest_image(tmp,f'Sentinel-2 · {item["id"]}',metadata={'source':'COPERNICUS SENTINEL-2','source_url':item.get('links',[{}])[0].get('href','https://earth-search.aws.element84.com/v1'),
'product_id':item['id'],'acquisition_time':props['datetime'],'scene_cloud_cover_pct':props.get('eo:cloud_cover'),'license':'Copernicus Sentinel data terms; free and open',
'band_contract':'S2_L2A_6B_DN10000','qa_available':cloud_mask is not None,'processing':['COG crop','Resampled to 256×256','Provider scale/offset to reflectance ×10000','SCL cloud/shadow/snow/nodata exclusion'],
'warnings':['Acquisition date is historical, not live','Scene cloud percentage is not local cloud percentage','Mixed 10/20m source bands; output resolution recorded']})
tmp.unlink(missing_ok=True)
return rec