saef-api / backend /app.py
m4az19's picture
Deploy SAHAB-1 backend and pretrained AI
20b0a08 verified
Raw History Blame Contribute Delete
23.7 kB
import csv
import io
import json
import os
import tempfile
import uuid
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from datetime import datetime
from pathlib import Path
from typing import Literal,Optional
import httpx
import numpy as np
from fastapi import FastAPI,UploadFile,File,Form,HTTPException,Header,Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse,JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel,Field,model_validator
from PIL import UnidentifiedImageError
from . import store,geo,imagery,sources,operations,pipeline,observations,locations
from .simulation import Scenario,simulate
pool=ThreadPoolExecutor(max_workers=2)
@asynccontextmanager
async def lifespan(app):
for j in store.all_records('jobs'):
if j['status'] in ('queued','running'):store.put('jobs',{**j,'status':'failed','error':'Server restarted; resubmit job'},j['id'])
pipeline.start()
try:yield
finally:pipeline.stop()
app=FastAPI(title='SAHAB-1 | satforge 2026',version='2.0.0',lifespan=lifespan,
description='Research prototype. Real source data, explicit simulated scenarios. No emergency alerts or cloud-seeding commands.')
app.add_middleware(CORSMiddleware,allow_origins=['http://127.0.0.1:5173','http://localhost:5173','http://127.0.0.1:8000'],allow_methods=['GET','POST','PUT'],allow_headers=['Content-Type','X-Device-Key'])
@app.middleware('http')
async def dashboard_gateway(request,call_next):
import secrets
configured=os.getenv('SATFORGE_DASHBOARD_KEY')
device_routes=('/api/v2/telemetry','/api/hardware/','/api/observations','/api/import/imerg')
if configured and request.url.path.startswith('/api/') and request.url.path!='/api/health':
# Device writers authenticate with the existing device key. Dashboard
# traffic arrives through the Vercel server gateway, never a browser key.
device_write=request.method=='POST' and any(request.url.path.startswith(p) for p in device_routes)
if not device_write and not secrets.compare_digest(request.headers.get('X-Dashboard-Key',''),configured):
return JSONResponse(status_code=401,content={'detail':'Dashboard gateway authentication required'})
return await call_next(request)
@app.exception_handler(KeyError)
async def missing(request,exc):return JSONResponse(status_code=404,content={'detail':str(exc)})
@app.exception_handler(ValueError)
async def invalid(request,exc):return JSONResponse(status_code=422,content={'detail':str(exc)})
@app.exception_handler(httpx.HTTPError)
async def remote(request,exc):return JSONResponse(status_code=502,content={'detail':f'External source unavailable: {type(exc).__name__}. Cached local data remains usable.'})
def job(kind,fn,*args,**kwargs):
j=store.put('jobs',{'kind':kind,'status':'queued','created_at':store.now()})
def run():
store.put('jobs',{**j,'status':'running'},j['id'])
try:
result=fn(*args,**kwargs)
store.put('jobs',{**j,'status':'complete','completed_at':store.now(),'result':result},j['id'])
store.event(f'{kind} completed',details_id=result.get('id') if isinstance(result,dict) else None)
if kind in ('Weather retrieval','Real DEM acquisition','OSM acquisition','Sentinel-2 acquisition'):pipeline.wake()
except Exception as exc:
store.put('jobs',{**j,'status':'failed','error':str(exc)[:1000]},j['id'])
store.event(f'{kind} failed: {str(exc)[:200]}')
pool.submit(run)
return j
from .intelligence_api import router as intelligence_router
app.include_router(intelligence_router)
from .telemetry_v2 import router as telemetry_router
app.include_router(telemetry_router)
from fastapi.exceptions import RequestValidationError
@app.exception_handler(RequestValidationError)
async def validation_error(request,exc):
errors=[{'loc':list(e['loc']),'type':e['type'],'msg':e['msg']} for e in exc.errors()]
if request.url.path in ('/api/observations','/api/telemetry') or request.url.path.startswith('/api/v2/telemetry'):
store.put('validation_issues',{'at':store.now(),'source':request.url.path,'message':'Malformed incoming payload rejected','errors':errors})
return JSONResponse(status_code=422,content={'detail':errors})
@app.get('/api/health')
def health():return {'status':'ok','mission':'SAHAB-1','version':'2.0.0','training':'Pretrained satellite and explanation models; no training required'}
@app.get('/api/context')
def monitor_context():
try:config=store.get('config','project')
except KeyError:config={'mode':'real','wadi':'shawka'}
return {'config':config,'wadis':locations.all_areas()}
@app.get('/api/state')
def state():
return {**monitor_context(),'assets':store.all_records('assets'),'events':store.all_records('events')[:50],
'geometries':store.all_records('geometries'),'segmentations':store.all_records('segmentations'),
'simulations':[{k:v for k,v in s.items() if k!='frames'} for s in store.all_records('simulations')],
'weather':store.all_records('weather')[:10], 'model':{'installed':(sources.MODEL_DIR/'model.pth').exists(),'repo':sources.MODEL_REPO,'revision':sources.MODEL_REV,
'license':'CC BY 4.0','reported_metrics':{'iou':.9635,'dice':.9814},'metrics_label':'Publisher-reported validation; not independently reproduced or UAE accuracy'}}
class Config(BaseModel):
mode:Literal['real','demo']='real'
wadi:str=Field(default='shawka',min_length=1,max_length=64)
@app.put('/api/config')
def config(cfg:Config):
locations.get_area(cfg.wadi)
result=store.put('config',cfg.model_dump(),'project');pipeline.wake();return result
@app.get('/api/jobs/{id}')
def get_job(id:str):return store.get('jobs',id)
class Location(BaseModel):
lon:float=Field(ge=-180,le=180)
lat:float=Field(ge=-80,le=80)
@model_validator(mode='after')
def coverage(self):
locations.require_uae(self.lon,self.lat);return self
@app.post('/api/sources/terrain')
def download_dem(loc:Location):return job('Real DEM acquisition',sources.acquire_dem,loc.lon,loc.lat)
@app.post('/api/sources/infrastructure')
def download_infra(loc:Location):return job('OSM acquisition',sources.acquire_infrastructure,loc.lon,loc.lat)
@app.post('/api/sources/sentinel')
def download_sentinel(loc:Location):return job('Sentinel-2 acquisition',sources.acquire_sentinel,loc.lon,loc.lat)
@app.post('/api/models/install')
def download_model():return job('Pretrained model installation',sources.install_model)
async def save_upload(upload):
suffix=Path(upload.filename or '').suffix.lower()
path=store.path_for(uuid.uuid4().hex,'upload'+suffix)
size=0
try:
with path.open('wb') as f:
while chunk:=await upload.read(1024*1024):
size+=len(chunk)
if size>100_000_000: raise ValueError('Uploads limited to 100 MB; crop large scenes first')
f.write(chunk)
return path
except Exception:
path.unlink(missing_ok=True);raise
@app.post('/api/import/image')
async def import_image(file:UploadFile=File(...),source:str=Form('USER IMPORT'),acquisition_time:str=Form(''),license:str=Form('User must verify')):
path=await save_upload(file)
try:
result=imagery.ingest_image(path,file.filename,{'source':source,'acquisition_time':acquisition_time or None,'license':license})
store.event('Image imported',source,asset_id=result['id']);return result
except (UnidentifiedImageError,OSError) as e:raise HTTPException(422,f'Cannot read image: {str(e)[:200]}')
finally:path.unlink(missing_ok=True)
@app.post('/api/import/terrain')
async def import_terrain(file:UploadFile=File(...),source:str=Form('USER IMPORT'),license:str=Form('User must verify')):
path=await save_upload(file)
def process():
try:return geo.ingest_dem(path,file.filename,source,metadata={'license':license})
finally:path.unlink(missing_ok=True)
return job('DEM processing',process)
@app.post('/api/import/imerg')
async def import_imerg(file:UploadFile=File(...),lon:float=Form(...),lat:float=Form(...),x_device_key:Optional[str]=Header(None)):
from .intelligence_api import device_auth
from .environmental import imerg_import
device_auth(x_device_key);locations.require_uae(lon,lat)
path=await save_upload(file)
try:
result=imerg_import(path,lon,lat,file.filename);pipeline.wake();return result
finally:path.unlink(missing_ok=True)
@app.post('/api/import/infrastructure')
async def import_infrastructure(file:UploadFile=File(...),source:str=Form('USER IMPORT')):
content=await file.read(10_000_001)
if len(content)>10_000_000:raise ValueError('GeoJSON limit is 10 MB')
fc=geo.validate_geojson(json.loads(content))
return store.asset('infrastructure',file.filename,source,geojson=fc,crs='EPSG:4326')
@app.get('/api/terrain/{id}')
def terrain(id:str):return geo.mesh(id)
@app.get('/api/terrain/{id}/sample')
def terrain_sample(id:str,row:int|None=None,col:int|None=None,lon:float|None=None,lat:float|None=None,simulation_id:str|None=None,frame:int=0):
from pyproj import Transformer
a,t,crs=geo.load_grid(id);z=a['z'];n,m=z.shape
if lon is not None and lat is not None:
if not np.isfinite([lon,lat]).all() or not (-180<=lon<=180 and -90<=lat<=90):raise ValueError('Invalid geographic coordinates')
x,y=Transformer.from_crs(4326,crs,always_xy=True).transform(lon,lat)
cc,rr=(~t)*(x,y);row=int(np.floor(rr));col=int(np.floor(cc))
if row is None or col is None:raise ValueError('Provide a row and column or longitude and latitude')
if not(0<=row<n and 0<=col<m):return {'available':False,'reason':'This point is outside the acquired terrain footprint'}
if not bool(a['valid'][row,col]):return {'available':False,'reason':'No valid elevation at this terrain cell'}
x,y=t*(col+.5,row+.5);lng,latitude=Transformer.from_crs(crs,4326,always_xy=True).transform(x,y)
depth=None;time_min=None;threshold=None
if simulation_id:
sim=store.get('simulations',simulation_id)
if sim['config']['terrain_id']!=id:raise ValueError('Simulation does not match this terrain')
if not 0<=frame<len(sim['frames']):raise ValueError('Invalid simulation frame')
with np.load(store.path_for(simulation_id,'depth.npz')) as data:depth=float(data['depth'][frame,row,col])
time_min=sim['frames'][frame]['time_min'];threshold=sim['config']['depth_threshold_m']
return {'available':True,'row':row,'col':col,'longitude':lng,'latitude':latitude,
'elevation_m':float(z[row,col]),'slope_deg':float(a['slope'][row,col]),
'upstream_cells':int(a['accumulation'][row,col]),
'upstream_area_km2':float(a['accumulation'][row,col])*abs(t.a*t.e)/1e6,
'simulation_depth_m':depth,'simulation_time_min':time_min,'extent_threshold_m':threshold,
'cell_size_m':[abs(t.a),abs(t.e)],'source':'DEM sample; depth is a simulated model state; upstream area is limited to this DEM footprint'}
@app.post('/api/terrain/{id}/overlays')
def terrain_overlays(id:str,fc:dict):
from pyproj import Transformer
geo.validate_geojson(fc)
a,t,crs=geo.load_grid(id);z=a['z'];n,m=z.shape
project=Transformer.from_crs(4326,crs,always_xy=True)
scale=max((m-1)*abs(t.a),(n-1)*abs(t.e))/2;low=float(z[a['valid']].min());lines=[];points=[]
def pos(p):
x,y=project.transform(p[0],p[1]);c,r=(~t)*(x,y);c-=.5;r-=.5
if not (0<=r<n and 0<=c<m):return None
elev=float(z[min(n-1,int(round(r))),min(m-1,int(round(c)))])
return [(c-(m-1)/2)*abs(t.a)/scale,(elev-low)/scale+.002,(r-(n-1)/2)*abs(t.e)/scale]
def walk(g,color):
kind=g['type'];cs=g['coordinates']
if kind=='Point':
p=pos(cs)
if p:points.append({'point':p,'color':color})
elif kind in ('LineString','Polygon','MultiLineString','MultiPolygon'):
rings=[cs] if kind=='LineString' else [r for poly in cs for r in poly] if kind=='MultiPolygon' else cs
for ring in rings:
for p,q in zip(ring,ring[1:]):
u,v=pos(p),pos(q)
if u and v:lines.append({'points':u+v,'color':color})
for f in fc['features']:walk(f['geometry'],f.get('properties',{}).get('display_color','#f0c47d'))
# Drape classified water on the acquired DEM grid, rather than flattening
# large polygons above valleys. Only declared water evidence is rasterized.
from rasterio.features import rasterize
from rasterio.warp import transform_geom
water_shapes=[transform_geom('EPSG:4326',crs,f['geometry']) for f in fc['features']
if f.get('properties',{}).get('display_role')=='observed_water' and f['geometry']['type'] in ('Polygon','MultiPolygon')]
mask=rasterize([(g,1) for g in water_shapes],out_shape=z.shape,transform=t,fill=0).astype(bool)&a['valid'] if water_shapes else None
return {'lines':lines,'points':points,'observed_water':mask.tolist() if mask is not None else None}
@app.post('/api/segmentation')
def segment(req:imagery.SegmentRequest):return job('Water segmentation',imagery.segment,req)
class Threshold(BaseModel):
value:float=Field(ge=.01,le=.99)
@app.put('/api/segmentation/{id}/threshold')
def threshold(id:str,req:Threshold):return imagery.threshold_result(id,req.value)
class Geometry(BaseModel):
name:str=Field(max_length=120)
geojson:dict
original_id:Optional[str]=None
parent_id:Optional[str]=None
@app.post('/api/geometries')
def geometry(req:Geometry):
fc=geo.validate_geojson(req.geojson,True)
rec=store.put('geometries',{**req.model_dump(),'created_at':store.now(),'provenance':'MANUALLY EDITED',**geo.measurements(fc)})
store.event('Reviewed geometry saved','MANUALLY EDITED',geometry_id=rec['id'],parent_id=req.parent_id)
return rec
@app.get('/api/geometries/{id}/export')
def export_geometry(id:str):return JSONResponse(store.get('geometries',id)['geojson'],headers={'Content-Disposition':'attachment; filename="reviewed-observation.geojson"'})
@app.post('/api/simulations')
def simulation(cfg:Scenario):return job('Flood simulation',simulate,cfg)
@app.get('/api/simulations/{id}')
def simulation_get(id:str):return store.get('simulations',id)
@app.get('/api/simulations/{id}/depth/{index}')
def depth(id:str,index:int):
a=np.load(store.path_for(id,'depth.npz'))['depth']
if index<0 or index>=len(a):raise ValueError('Invalid simulation frame')
return {'depth':a[index].round(4).tolist()}
class WeatherLocation(Location):
days:int=Field(default=3,ge=1,le=7)
historical_start:Optional[str]=None
historical_end:Optional[str]=None
@app.post('/api/weather')
def weather(loc:WeatherLocation):
if loc.historical_start:
start=datetime.fromisoformat(loc.historical_start);end=datetime.fromisoformat(loc.historical_end or '')
if not 0<=(end-start).days<=31:raise ValueError('Historical interval must be 0–31 days')
return job('Weather retrieval',sources.fetch_weather,loc.lon,loc.lat,loc.days,loc.historical_start,loc.historical_end)
@app.post('/api/import/weather')
async def weather_csv(file:UploadFile=File(...),source:str=Form('Offline CSV')):
raw=await file.read(2_000_001)
if len(raw)>2_000_000:raise ValueError('CSV limit is 2 MB')
rows=list(csv.DictReader(io.StringIO(raw.decode('utf-8-sig'))))
keys=['temperature_2m','relative_humidity_2m','precipitation','surface_pressure','cloud_cover','wind_speed_10m','wind_direction_10m']
hourly={'time':[] ,**{k:[] for k in keys}}
for row in rows:
datetime.fromisoformat(row['time']);hourly['time'].append(row['time'])
for k in keys:
value=float(row[k]) if row.get(k) else None
if value is not None and not np.isfinite(value):raise ValueError('Weather values must be finite')
hourly[k].append(value)
return store.put('weather',{'source':source,'retrieved_at':store.now(),'hourly':hourly,'units':dict(zip(keys,['°C','%','mm','hPa','%','m/s','°'])),
'provenance':'USER IMPORT','warnings':['User-supplied units and location must be verified']})
class Assessment(BaseModel):
weather_id:Optional[str]=None
baseline_id:Optional[str]=None
enhancement_pct:float=Field(default=20,ge=0,le=100)
@app.post('/api/cloud-assessment')
def assess(req:Assessment):
def run():
w=store.get('weather',req.weather_id) if req.weather_id else None
b=store.get('simulations',req.baseline_id) if req.baseline_id else None
e=None
if b:
config={**b['config'],'rainfall_mm_h':b['config']['rainfall_mm_h']*(1+req.enhancement_pct/100),'name':'Hypothetical rainfall enhancement'}
if config.get('rainfall_series_mm_h'):config['rainfall_series_mm_h']=[x*(1+req.enhancement_pct/100) for x in config['rainfall_series_mm_h']]
e=simulate(Scenario(**config))
result=operations.cloud_assessment(w,b,e)
if e:result['enhanced']=e
return store.put('assessments',result)
return job('Rainfall scenario comparison',run)
@app.post('/api/telemetry')
def telemetry(t:operations.Telemetry,x_device_key:Optional[str]=Header(None)):
configured=os.environ.get('SATFORGE_DEVICE_KEY')
if not configured:raise HTTPException(503,'Set SATFORGE_DEVICE_KEY before accepting hardware telemetry')
import secrets
if not secrets.compare_digest(x_device_key or '',configured):raise HTTPException(401,'Invalid device key')
if t.provenance!='MEASURED' or t.soc_method=='simulated':raise ValueError('Hardware endpoint accepts measured telemetry only')
d=observations.telemetry_ingest(t);pipeline.wake();return d
@app.get('/api/telemetry')
def telemetry_get():return observations.mission_status()
@app.post('/api/hardware/image')
async def hardware_image(request:Request,x_device_key:Optional[str]=Header(None),x_device_id:str=Header('esp32'),x_captured_at:str=Header(...)):
import secrets
configured=os.environ.get('SATFORGE_DEVICE_KEY')
if not configured or not secrets.compare_digest(x_device_key or '',configured):raise HTTPException(401,'Invalid device key')
captured=datetime.fromisoformat(x_captured_at.replace('Z','+00:00'))
if captured.tzinfo is None:raise ValueError('Camera timestamp needs a timezone')
data=bytearray()
async for part in request.stream():
data.extend(part)
if len(data)>2_000_000:raise ValueError('Camera image limit is 2 MB')
path=store.path_for(uuid.uuid4().hex,'jpg');path.write_bytes(data)
try:
result=imagery.ingest_image(path,f'ESP32 ground camera · {x_device_id[:64]}',{'source':'ESP32 GROUND CAMERA','acquisition_time':captured.isoformat(),'warnings':['Ground demonstration photograph; not an orbital observation']})
store.event('Ground camera image received','MEASURED',asset_id=result['id']);return result
except (UnidentifiedImageError,OSError):raise ValueError('Invalid camera JPEG')
finally:path.unlink(missing_ok=True)
@app.get('/api/energy/demo')
def demo_energy(low:bool=False):return operations.demo_energy(low)
@app.post('/api/energy/policy')
def energy_policy(policy:operations.EnergyPolicy,low:bool=False,demo:bool=False):
mission=observations.mission_status()
latest=operations.demo_energy(low)['latest'] if demo else (mission['samples'][0] if mission['status']=='RECENT GROUND TELEMETRY' else {})
store.put('config',policy.model_dump(),'energy-policy')
return operations.schedule(latest,policy)
@app.get('/api/queue')
def queue():return store.all_records('queue')
@app.post('/api/queue')
def queue_add(item:operations.QueueItem):return operations.enqueue(item)
class QueueAction(BaseModel):
action:Literal['pause','resume','retry','priority']
priority:int=Field(default=50,ge=0,le=100)
@app.put('/api/queue/{id}')
def queue_edit(id:str,req:QueueAction):
j=store.get('queue',id)
if req.action=='priority':j['priority']=req.priority
elif req.action=='pause' and j['state']!='sent':j['state']='paused'
elif req.action in ('resume','retry') and j['state']!='sent':j['state']='queued';j['retries']+=int(req.action=='retry')
return store.put('queue',j,id)
class LinkStep(BaseModel):
seconds:int=Field(default=30,ge=1,le=600)
rate_kbps:float=Field(default=16,gt=0,le=10000)
link_open:bool=True
energy_allowed:bool=True
demo:bool=False
@app.post('/api/queue/advance')
def link_step(req:LinkStep):
if not req.demo:raise ValueError('Simulated communications require explicit demo mode')
return operations.transmit(req.seconds,req.rate_kbps,req.link_open,req.energy_allowed)
@app.post('/api/demo/{wadi}')
def demo(wadi:str):
terrain=geo.demo_terrain(wadi);w=locations.get_area(wadi)
fs=[]
for i in range(9):
fs.append({'type':'Feature','id':f'demo-{i}','properties':{'name':f'Demo road {i+1}','category':'road'},
'geometry':{'type':'LineString','coordinates':[[w['lon']-.012,w['lat']-.012+i*.003],[w['lon']+.012,w['lat']-.012+i*.003]]}})
infra=store.asset('infrastructure','Synthetic infrastructure','SYNTHETIC DEMO',wadi=wadi,geojson={'type':'FeatureCollection','features':fs},license='CC0')
store.event('Reproducible demo loaded','SYNTHETIC DEMO')
return {'terrain':terrain,'infrastructure':infra}
@app.get('/api/files/{id}/{suffix}')
def file_get(id:str,suffix:str):
if suffix not in ('png','mask.png','mask.tif','prob.tif','geojson','tif','image'):raise HTTPException(404)
p=store.path_for(id,suffix)
if not p.exists():raise HTTPException(404,'Export unavailable for this asset')
return FileResponse(p,filename=p.name if suffix not in ('png','mask.png') else None)
@app.get('/api/report/{id}')
def report(id:str):
s=store.get('simulations',id)
return JSONResponse({'title':'SAHAB-1 research experiment','subtitle':'satforge 2026','exported_at':store.now(),'simulation':s,'assessments':store.all_records('assessments'),
'validation':store.all_records('validation'),'warning':'SIMULATED SCENARIO. Not an operational warning.'},headers={'Content-Disposition':f'attachment; filename="satforge-{id}.json"'})
@app.post('/api/validation/{id}')
async def validation(id:str,file:UploadFile=File(...),scene_id:str=Form(...)):
from PIL import Image
seg=store.get('segmentations',id);a=np.load(store.path_for(id,'prob.npz'));path=await save_upload(file)
try:
with Image.open(path) as im:truth=np.asarray(im.convert('L'))>127
result=imagery.metrics(a['p']>=seg['threshold'],truth,a['valid'])
return store.put('validation',{'segmentation_id':id,'scene_id':scene_id,'metrics':result,'threshold':seg['threshold'],'created_at':store.now(),
'warning':'User-supplied aligned truth; independent held-out status is not verified'})
finally:path.unlink(missing_ok=True)
@app.get('/api/validation')
def validations():return store.all_records('validation')
# Built dashboard served by the same process for the single-command launcher.
dist=store.ROOT/'frontend'/'dist'
if dist.exists():app.mount('/',StaticFiles(directory=dist,html=True),name='dashboard')