Download backend/sources.py from m4az19/saef-api: direct link, hf CLI and curl.
- Browser
- Download file 11.2 kB
-
https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/sources.py
- Command line
-
hf download hf://spaces/m4az19/saef-api/backend/sources.py
-
curl -L -o sources.py https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/sources.py
11.2 kB
| """Fixed-provider adapters. No arbitrary user URL or filesystem downloads.""" | |
| import hashlib | |
| import json | |
| import math | |
| from pathlib import Path | |
| import httpx | |
| import rasterio | |
| import numpy as np | |
| from rasterio.windows import from_bounds as window_from_bounds | |
| from rasterio.transform import from_bounds | |
| from rasterio.warp import reproject, transform_bounds, Resampling | |
| from .store import ROOT, DATA, path_for, asset, put, now, all_records | |
| from .geo import ingest_dem, validate_geojson | |
| from .locations import require_uae | |
| MODEL_REPO='giswqs/s2-water-unetplusplus-efficientnet-b4' | |
| MODEL_REV='efb6f636480188f48779b41f2a94bb941fd7b900' | |
| MODEL_DIR=ROOT/'models'/'s2-water' | |
| HEADERS={'User-Agent':'Satforge-Research-Prototype/1.0 (educational geospatial application)'} | |
| def download(url,path,max_bytes=150_000_000): | |
| path=Path(path); path.parent.mkdir(parents=True,exist_ok=True) | |
| temporary=path.with_suffix(path.suffix+'.part') | |
| count=0 | |
| try: | |
| with httpx.stream('GET',url,timeout=120,follow_redirects=True,headers=HEADERS) as r: | |
| r.raise_for_status() | |
| with temporary.open('wb') as f: | |
| for chunk in r.iter_bytes(65536): | |
| count+=len(chunk) | |
| if count>max_bytes: raise ValueError('Provider file exceeds download size limit') | |
| f.write(chunk) | |
| temporary.replace(path) | |
| finally: | |
| temporary.unlink(missing_ok=True) | |
| return hashlib.sha256(path.read_bytes()).hexdigest() | |
| def acquire_dem(lon,lat,radius=.035): | |
| require_uae(lon,lat) | |
| # Crop one tile. Reject crossing tile boundaries rather than silently lose coverage. | |
| bbox=[max(math.floor(lon)+.001,lon-radius),max(math.floor(lat)+.001,lat-radius),min(math.floor(lon)+.999,lon+radius),min(math.floor(lat)+.999,lat+radius)] | |
| tile=f'Copernicus_DSM_COG_30_N{math.floor(lat):02d}_00_E{math.floor(lon):03d}_00_DEM' | |
| url=f'https://copernicus-dem-90m.s3.eu-central-1.amazonaws.com/{tile}/{tile}.tif' | |
| target=DATA/f'{tile}.tif' | |
| if not target.exists(): download(url,target,40_000_000) | |
| return ingest_dem(target,f'Copernicus GLO-90 · {lat:.3f}, {lon:.3f}','COPERNICUS DEM',bbox, | |
| {'source_url':url,'publication_time':'2021 release','license':'Copernicus DEM free public licence; attribution required', | |
| 'license_url':'https://registry.opendata.aws/copernicus-dem/','native_resolution_m':90,'sha256':hashlib.sha256(target.read_bytes()).hexdigest(), | |
| 'acquisition_time':None}) | |
| def acquire_infrastructure(lon,lat,radius=.035): | |
| require_uae(lon,lat) | |
| bbox=f'{lat-radius},{lon-radius},{lat+radius},{lon+radius}' | |
| query=f'[out:json][timeout:40];(way[highway]({bbox});way[building]({bbox});node[amenity~"hospital|school|fire_station"]({bbox}););out geom;' | |
| with httpx.Client(timeout=55,headers=HEADERS) as client: | |
| for endpoint in ('https://overpass-api.de/api/interpreter','https://overpass.private.coffee/api/interpreter'): | |
| try: | |
| r=client.post(endpoint,data={'data':query});r.raise_for_status();data=r.json() | |
| break | |
| except httpx.HTTPError: | |
| if endpoint.endswith('overpass.private.coffee/api/interpreter'):raise | |
| fs=[] | |
| for el in data.get('elements',[]): | |
| tags=el.get('tags',{}); category='road' if 'highway' in tags else 'building' if 'building' in tags else 'facility' | |
| if el['type']=='node': geom={'type':'Point','coordinates':[el['lon'],el['lat']]} | |
| else: | |
| coords=[[p['lon'],p['lat']] for p in el.get('geometry',[])] | |
| if len(coords)<2: continue | |
| closed=category=='building' and len(coords)>=4 and coords[0]==coords[-1] | |
| geom={'type':'Polygon' if closed else 'LineString','coordinates':[coords] if closed else coords} | |
| fs.append({'type':'Feature','id':str(el['id']),'geometry':geom,'properties':{**tags,'category':category,'name':tags.get('name',tags.get('highway',category))}}) | |
| fc=validate_geojson({'type':'FeatureCollection','features':fs}) | |
| return asset('infrastructure','OpenStreetMap roads & buildings','OPENSTREETMAP',geojson=fc,source_url=endpoint,license='ODbL 1.0; © OpenStreetMap contributors',acquisition_time=data.get('osm3s',{}).get('timestamp_osm_base'),bbox=[lon-radius,lat-radius,lon+radius,lat+radius],warnings=['Community mapping may be incomplete','Intersection is not verified damage']) | |
| def fetch_weather(lon,lat,days=3,historical_start=None,historical_end=None): | |
| require_uae(lon,lat) | |
| historical=historical_start is not None | |
| base='https://archive-api.open-meteo.com/v1/archive' if historical else 'https://api.open-meteo.com/v1/forecast' | |
| variables='temperature_2m,relative_humidity_2m,precipitation,surface_pressure,cloud_cover,wind_speed_10m,wind_direction_10m' | |
| params={'latitude':lat,'longitude':lon,'hourly':variables,'timezone':'UTC','wind_speed_unit':'ms'} | |
| if historical: params.update(start_date=historical_start,end_date=historical_end) | |
| else: params.update(forecast_days=days) | |
| key=json.dumps(params,sort_keys=True) | |
| from datetime import datetime,timezone | |
| for old in all_records('weather'): | |
| if old.get('cache_key')==key and (datetime.now(timezone.utc)-datetime.fromisoformat(old['retrieved_at'])).total_seconds()<1800: | |
| return {**old,'cached':True} | |
| r=httpx.get(base,params=params,headers=HEADERS,timeout=30);r.raise_for_status();data=r.json() | |
| if 'hourly' not in data or not data['hourly'].get('time'): raise ValueError('Provider returned no hourly weather') | |
| record=put('weather',{'source':'Open-Meteo','source_url':str(r.url),'license':'CC BY 4.0','retrieved_at':now(),'cache_key':key, | |
| 'provenance':'REANALYSIS' if historical else 'EXTERNAL FORECAST','location':{'requested_lon':lon,'requested_lat':lat,'grid_lon':data['longitude'],'grid_lat':data['latitude']}, | |
| 'hourly':data['hourly'],'units':data.get('hourly_units',{}),'cached':False, | |
| 'warnings':['Model grid-point data, not catchment-wide measured rainfall','No cloud microphysics or validated seeding suitability']}) | |
| return record | |
| def install_model(): | |
| MODEL_DIR.mkdir(parents=True,exist_ok=True) | |
| manifest={} | |
| for name in ['config.json','README.md','model.pth']: | |
| p=MODEL_DIR/name | |
| url=f'https://huggingface.co/{MODEL_REPO}/resolve/{MODEL_REV}/{name}' | |
| sha=hashlib.sha256(p.read_bytes()).hexdigest() if p.exists() else download(url,p) | |
| manifest[name]=sha | |
| (MODEL_DIR/'manifest.json').write_text(json.dumps({'repo':MODEL_REPO,'revision':MODEL_REV,'sha256':manifest,'downloaded_at':now(),'license':'CC BY 4.0'},indent=2)) | |
| return {'installed':True,'repo':MODEL_REPO,'revision':MODEL_REV,'sha256':manifest} | |
| def acquire_sentinel(lon,lat): | |
| """Small real six-band Sentinel-2 crop, with product ID and acquisition timestamp.""" | |
| from datetime import datetime,timezone,timedelta | |
| require_uae(lon,lat) | |
| end=datetime.now(timezone.utc);start=end-timedelta(days=365) | |
| bbox=[lon-.025,lat-.025,lon+.025,lat+.025] | |
| query={'collections':['sentinel-2-l2a'],'bbox':bbox,'datetime':f'{start:%Y-%m-%dT00:00:00Z}/{end:%Y-%m-%dT23:59:59Z}','limit':20, | |
| 'query':{'eo:cloud_cover':{'lt':20}}} | |
| r=httpx.post('https://earth-search.aws.element84.com/v1/search',json=query,timeout=45);r.raise_for_status() | |
| items=r.json().get('features',[]) | |
| if not items: raise ValueError('No Sentinel-2 scene with <20% scene cloud cover in the last year; import imagery instead') | |
| item=max(items,key=lambda i:i['properties']['datetime']); props=item['properties'] | |
| existing=next((a for a in all_records('assets') if a.get('product_id')==item['id'] and a.get('bbox') and | |
| a['bbox'][0]<=lon<=a['bbox'][2] and a['bbox'][1]<=lat<=a['bbox'][3]),None) | |
| if existing:return existing | |
| bands=['blue','green','red','nir','swir16','swir22'] | |
| arrays=[]; dst_t=None; dst_crs=None; valid=None | |
| # COG window reads avoid downloading complete satellite scenes. | |
| with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',CPL_VSIL_CURL_ALLOWED_EXTENSIONS='.tif',GDAL_HTTP_TIMEOUT='60'): | |
| for band in bands: | |
| url=item['assets'][band]['href'] | |
| with rasterio.open(url) as src: | |
| bb=transform_bounds('EPSG:4326',src.crs,*bbox) | |
| if dst_t is None: | |
| dst_crs=src.crs; dst_t=from_bounds(*bb,256,256) | |
| arr=np.full((256,256),np.nan,dtype='float32') | |
| win=window_from_bounds(*bb,src.transform).round_offsets().round_lengths() | |
| crop=src.read(1,window=win,boundless=True,fill_value=src.nodata or 0) | |
| reproject(crop,arr,src_transform=src.window_transform(win),src_crs=src.crs,src_nodata=src.nodata or 0,dst_transform=dst_t,dst_crs=dst_crs,dst_nodata=np.nan,resampling=Resampling.bilinear) | |
| # Harmonize physical reflectance using provider scale/offset; store reflectance*10000. | |
| rb=item['assets'][band].get('raster:bands',[{}])[0] | |
| scale=rb.get('scale',.0001); offset=rb.get('offset',0.) | |
| arr=(arr*scale+offset)*10000 | |
| arrays.append(arr) | |
| scl=item['assets'].get('scl') | |
| cloud_mask=None | |
| if scl: | |
| with rasterio.open(scl['href']) as src: | |
| bb=transform_bounds('EPSG:4326',src.crs,*bbox);win=window_from_bounds(*bb,src.transform).round_offsets().round_lengths() | |
| crop=src.read(1,window=win,boundless=True,fill_value=0);classes=np.zeros((256,256),dtype='uint8') | |
| reproject(crop,classes,src_transform=src.window_transform(win),src_crs=src.crs,dst_transform=dst_t,dst_crs=dst_crs,resampling=Resampling.nearest) | |
| cloud_mask=np.isin(classes,[0,1,3,8,9,10,11]) | |
| from .imagery import ingest_image | |
| import uuid | |
| tmp=DATA/f'sentinel_{uuid.uuid4().hex}.tif' | |
| data=np.stack(arrays);valid=np.isfinite(data).all(axis=0) | |
| if cloud_mask is not None: valid&=~cloud_mask | |
| data[:,~valid]=-9999 | |
| with rasterio.open(tmp,'w',driver='GTiff',width=256,height=256,count=6,dtype='float32',crs=dst_crs,transform=dst_t,nodata=-9999) as dst: | |
| dst.write(data);dst.descriptions=('B2','B3','B4','B8','B11','B12') | |
| rec=ingest_image(tmp,f'Sentinel-2 · {item["id"]}',metadata={'source':'COPERNICUS SENTINEL-2','source_url':item.get('links',[{}])[0].get('href','https://earth-search.aws.element84.com/v1'), | |
| 'product_id':item['id'],'acquisition_time':props['datetime'],'scene_cloud_cover_pct':props.get('eo:cloud_cover'),'license':'Copernicus Sentinel data terms; free and open', | |
| 'band_contract':'S2_L2A_6B_DN10000','qa_available':cloud_mask is not None,'processing':['COG crop','Resampled to 256×256','Provider scale/offset to reflectance ×10000','SCL cloud/shadow/snow/nodata exclusion'], | |
| 'warnings':['Acquisition date is historical, not live','Scene cloud percentage is not local cloud percentage','Mixed 10/20m source bands; output resolution recorded']}) | |
| tmp.unlink(missing_ok=True) | |
| return rec | |