saef-api / backend /intelligence.py
m4az19's picture
Deploy SAHAB-1 backend and pretrained AI
124816b verified
Raw History Blame Contribute Delete
16.4 kB
"""Versioned rainfall screening, scenario comparisons, and internal research alerts.
No trained flood-risk model is available. These outputs are deliberately not
probabilities, arrival forecasts, confirmed flooding, or public-safety warnings.
"""
from datetime import timedelta
from threading import RLock
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
from . import store, geo
from .observations import utc, quality, digest
from .simulation import Scenario, simulate
MODEL_VERSION = 'rainfall-screening-1.0'
FEATURE_VERSION = 'hourly-grid-rainfall-1.1'
ALERT_LOCK = RLock()
TIERS = {'insufficient': -1, 'below_threshold': 0, 'watch': 1, 'elevated': 2, 'critical': 3}
class PipelineSettings(BaseModel):
model_config = ConfigDict(extra='forbid', allow_inf_nan=False)
enabled: bool = True
external_weather: bool = True
auto_scenarios: bool = True
auto_segmentation: bool = True
auto_acquire: bool = True
imagery_refresh_hours: int = Field(default=24,ge=6,le=168)
poll_seconds: int = Field(default=60, ge=15, le=3600)
weather_refresh_minutes: int = Field(default=60, ge=30, le=1440)
forecast_max_age_hours: float = Field(default=6, ge=.5, le=24)
watch_mm: float = Field(default=10, gt=0, le=500)
elevated_mm: float = Field(default=25, gt=0, le=500)
critical_mm: float = Field(default=50, gt=0, le=500)
infiltration_mm_h: float = Field(default=8, ge=0, le=200)
runoff: float = Field(default=.65, ge=0, le=1)
@model_validator(mode='after')
def ordered(self):
if not self.watch_mm < self.elevated_mm < self.critical_mm:
raise ValueError('Rainfall screening thresholds must strictly increase')
return self
def settings():
import os
try:
return PipelineSettings(**{k:v for k,v in store.get('config','pipeline').items() if k!='id'})
except KeyError:
return PipelineSettings(enabled=os.environ.get('SATFORGE_PIPELINE_ENABLED','1')=='1',
external_weather=os.environ.get('SATFORGE_AUTO_WEATHER','1')=='1')
def area(wadi):
from .locations import get_area
return get_area(wadi)
def in_area(record, wadi):
w = area(wadi)
return abs(record['longitude']-w['lon']) <= .06 and abs(record['latitude']-w['lat']) <= .06
def assets(wadi, mode):
w = area(wadi)
selected = {}
for a in store.all_records('assets'):
demo = a['source']=='SYNTHETIC DEMO'
if demo != (mode=='demo'):
continue
b = a.get('bbox')
if not (a.get('wadi')==wadi or b and b[0]<=w['lon']<=b[2] and b[1]<=w['lat']<=b[3]):
continue
selected.setdefault(a['kind'], a)
return selected
def features(wadi, mode, config, now=None):
now = utc(now)
# Use six complete future hours; provider values are interval-ending totals.
start = now.replace(minute=0,second=0,microsecond=0)
if now>start:
start += timedelta(hours=1)
times = [start+timedelta(hours=i) for i in range(1,7)]
relevant = [r for r in store.all_records('observations') if in_area(r,wadi) and
(r['category']=='simulation')==(mode=='demo')]
forecasts = [r for r in relevant if r['product']=='forecast' and r['measurement']=='precipitation_mm' and r['interval_minutes']==60]
# Never combine different forecast runs or providers into one artificial series.
groups = {}
for r in forecasts:
run_id=r.get('metadata',{}).get('weather_id') or r.get('metadata',{}).get('forecast_run_id')
groups.setdefault((r['source_id'],r['instrument_id'],r['acquired_at'],run_id),[]).append(r)
latest = max(groups.values(), key=lambda rs:(utc(rs[0]['acquired_at']),utc(rs[0]['ingested_at'])), default=[])
points=[];missing=[];ids=[]
for t in times:
rs=[r for r in latest if utc(r['valid_at'])==t]
r=rs[0] if len(rs)==1 else None
status=quality(r,now,config.forecast_max_age_hours) if r else 'missing'
valid=r is not None and status=='valid'
if not valid:missing.append(f'{t.isoformat()}: {status}')
points.append({'end_at':t.isoformat(),'value_mm':r['value'] if valid else None,'quality':status,'observation_id':r['id'] if r else None})
if r:ids.append(r['id'])
complete=not missing
values=[p['value_mm'] for p in points]
observed=[{**r,'current_quality':quality(r,now)} for r in relevant if r['product']=='observation']
# Disagreement is flagged, never averaged into spurious confidence.
conflicts=[]
for i,r in enumerate(observed[:100]):
if r['current_quality']!='valid':continue
for other in observed[:i]:
if other['current_quality']=='valid' and r['source_id']!=other['source_id'] and r['measurement']==other['measurement'] and r['interval_minutes']==other['interval_minutes'] and abs((utc(r['valid_at'])-utc(other['valid_at'])).total_seconds())<300 and abs(r['value']-other['value'])>max(5,abs(r['value'])*.5):
conflicts.append({'observation_ids':[r['id'],other['id']],'reason':'Nearby-time sources disagree; spatial support and calibration need review'})
return {'version':FEATURE_VERSION,'window_start':start.isoformat(),'window_end':times[-1].isoformat(),'points':points,
'complete':complete,'forecast_total_mm':sum(values) if complete else None,'forecast_peak_mm_h':max(values) if complete else None,
'forecast_acquired_at':latest[0]['acquired_at'] if latest else None,'source_id':latest[0]['source_id'] if latest else None,
'weather_id':latest[0].get('metadata',{}).get('weather_id') if latest else None,
'observation_ids':ids,'missing':missing,'ground_observations':observed[:30],'conflicts':conflicts}
def tier(total, config):
if total is None:return 'insufficient'
for name in ('critical','elevated','watch'):
if total>=getattr(config,name+'_mm'):return name
return 'below_threshold'
def assess(wadi, mode, config=None, now=None):
config=config or settings();now=utc(now);f=features(wadi,mode,config,now)
selected=assets(wadi,mode);level=tier(f['forecast_total_mm'],config)
input_signature={k:v for k,v in f.items() if k not in ('ground_observations',)}
input_signature['ground_observations']=[(r['id'],r['current_quality']) for r in f['ground_observations']]
signature=digest({'mode':mode,'wadi':wadi,'features':input_signature,'model':MODEL_VERSION,
'config':config.model_dump(),'routing_summary_version':'1.1','assets':{k:v['id'] for k,v in selected.items()}})
for old in store.all_records('predictions'):
if old.get('signature')==signature:return old,False
limits=['No calibrated or independently validated flood-risk model; no flood probability or arrival time is estimated.',
'Six-hour rainfall thresholds are configurable research settings, not locally calibrated warning levels.',
'A single forecast grid point is not catchment-average rainfall. No verified ground rain gauge or water-level network is connected.',
'Image water classifications and modeled inundation are not confirmed flooding.']
if level=='insufficient':
explanation='The six-hour forecast is missing, incomplete, rejected, or stale. Flood conditions cannot be assessed from the available evidence.'
action='Restore a complete fresh rainfall forecast and review source quality. Do not interpret missing data as low risk.'
else:
explanation=f"{f['source_id']} provides {f['forecast_total_mm']:.1f} mm over six complete future hours, with an hourly peak of {f['forecast_peak_mm_h']:.1f} mm. "
explanation+=f"This exceeds the configured {getattr(config,level+'_mm'):g} mm {level} screening threshold." if level!='below_threshold' else 'This is below the configured research screening thresholds; flooding is not ruled out.'
action='Review the forecast evidence, inspect modeled infrastructure intersections, and check authoritative local weather guidance.' if level!='below_threshold' else 'Continue monitoring and verify any field reports. Below-threshold rainfall is not an all-clear.'
if f['conflicts']:limits.append('Ground sources disagree; reconcile the flagged observations before operational interpretation.')
if 'terrain' not in selected:limits.append('No terrain dataset covers this area; inundation scenarios are unavailable.')
if 'infrastructure' not in selected:limits.append('No infrastructure dataset covers this area; affected assets cannot be enumerated.')
record=store.put('predictions',{'signature':signature,'created_at':now.isoformat(),'wadi':wadi,'area':area(wadi),'mode':mode,
'category':'simulation' if mode=='demo' else 'external','product':'EXPERIMENTAL RAINFALL SCREENING','model_type':'transparent rule baseline',
'model_version':MODEL_VERSION,'feature_version':FEATURE_VERSION,'features':f,'risk_level':level,'probability':None,'onset_at':None,
'confidence':'LIMITED / UNVALIDATED','explanation':explanation,'recommended_action':action,'limitations':limits,
'settings':config.model_dump(),'input_ids':f['observation_ids'],'asset_ids':{k:v['id'] for k,v in selected.items()},
'scenario_status':'pending' if f['complete'] and 'terrain' in selected and config.auto_scenarios else 'unavailable', 'scenarios':[]})
store.event(f'{area(wadi)["name"]}: {level.replace("_"," ")} rainfall screening',record['category'],prediction_id=record['id'],wadi=wadi,mode=mode)
for id in f['observation_ids']:
o=store.get('observations',id);store.put('observations',{**o,'processing_status':'assessed','latest_prediction_id':record['id']},id)
return record,True
def generate_scenarios(prediction):
if prediction['scenario_status']!='pending':return prediction
f=prediction['features'];cfg=prediction['settings'];comparisons=[]
prediction={**prediction,'scenario_status':'running'};store.put('predictions',prediction,prediction['id'])
completed={run.get('rainfall_multiplier'):run for run in store.all_records('simulations') if run.get('automatic_prediction_id')==prediction['id']}
try:
for label,multiplier in [('Forecast baseline',1.),('Elevated rainfall assumption',1.25),('Severe rainfall stress test',2.)]:
rates=[p['value_mm']*multiplier for p in f['points']]
if max(rates)>300:raise ValueError('Forecast stress intensity exceeds the routing model input limit; no clipping was applied')
run=completed.get(multiplier) or simulate(Scenario(terrain_id=prediction['asset_ids']['terrain'],infrastructure_id=prediction['asset_ids'].get('infrastructure'),
name=label, rainfall_mm_h=sum(rates)/6,rainfall_series_mm_h=rates,rain_duration_min=360,duration_min=360,
output_step_min=30,infiltration_mm_h=cfg['infiltration_mm_h'],runoff=cfg['runoff'],upstream_m3_s=0,initial_depth_m=0))
run.update(automatic_prediction_id=prediction['id'],mode=prediction['mode'],wadi=prediction['wadi'],
forecast_window_start=f['window_start'],rainfall_multiplier=multiplier)
store.put('simulations',run,run['id'])
final=run['frames'][-1];peak=max(run['frames'],key=lambda frame:frame['area_km2'])
comparisons.append({'id':run['id'],'label':label,'multiplier':multiplier,'category':'simulation',
'final_area_km2':final['area_km2'],'peak_area_km2':peak['area_km2'],'max_depth_m':max(x['max_depth_m'] for x in run['frames']),
'exposed_features':max(x['exposed_count'] for x in run['frames']) if run['config']['infrastructure_id'] else None,
'affected':[e for e in final['exposure'] if e['first_exposure_min'] is not None],
'difference_final_area_km2':final['area_km2']-(comparisons[0]['final_area_km2'] if comparisons else final['area_km2']),
'difference_exposed_features':(max(x['exposed_count'] for x in run['frames'])-(comparisons[0]['exposed_features'] if comparisons else max(x['exposed_count'] for x in run['frames']))) if run['config']['infrastructure_id'] else None})
prediction.update(scenario_status='complete',scenarios=comparisons)
except Exception as exc:
prediction.update(scenario_status='failed',scenario_error=str(exc)[:500],scenarios=comparisons)
store.event('Automatic scenario generation failed',prediction['category'],prediction_id=prediction['id'],error=str(exc)[:500])
return store.put('predictions',prediction,prediction['id'])
def notify_internal(alert, event):
"""Notification adapter contract. No external transport is configured or invoked."""
return store.put('notifications',{'alert_id':alert['id'],'event':event,'created_at':store.now(),
'channel':'internal','delivery_status':'available in alert centre','external_delivery':'not configured'})
def evaluate_alert(prediction):
key=f"{prediction['mode']}:{prediction['wadi']}:rainfall"
with ALERT_LOCK:
existing=next((a for a in store.all_records('alerts') if a['dedup_key']==key and a['status']!='resolved'),None)
level=prediction['risk_level'];now=store.now()
if level=='insufficient':
if existing:
return store.put('alerts',{**existing,'data_status':'insufficient; condition not proven resolved','last_checked_at':now},existing['id'])
return None
if level=='below_threshold':
if not existing:return None
existing.update(status='resolved',resolved_at=now,data_status='fresh',prediction_id=prediction['id'])
existing['history'].append({'at':now,'event':'resolved','reason':'Fresh complete forecast fell below the configured watch threshold'})
result=store.put('alerts',existing,existing['id']);notify_internal(result,'resolved');return result
if existing:
changed=level!=existing['severity'];event='escalated' if TIERS[level]>TIERS[existing['severity']] else 'de-escalated'
existing.update(prediction_id=prediction['id'],evidence=prediction['explanation'],horizon={'start':prediction['features']['window_start'],'end':prediction['features']['window_end']},
last_checked_at=now,data_status='fresh',recommended_action=prediction['recommended_action'],limitations=prediction['limitations'])
if changed:
existing.update(severity=level,status='open' if event=='escalated' else existing['status'])
existing['history'].append({'at':now,'event':event,'severity':level})
result=store.put('alerts',existing,existing['id'])
if changed:notify_internal(result,event)
return result
alert=store.put('alerts',{'dedup_key':key,'wadi':prediction['wadi'],'area':prediction['area'],'mode':prediction['mode'],
'severity':level,'status':'open','detected_at':now,'last_checked_at':now,'prediction_id':prediction['id'],
'evidence':prediction['explanation'],'horizon':{'start':prediction['features']['window_start'],'end':prediction['features']['window_end']},
'confidence':prediction['confidence'],'limitations':prediction['limitations'],'recommended_action':prediction['recommended_action'],
'classification':'EXPERIMENTAL INTERNAL ALERT — not a public-safety warning','data_status':'fresh','history':[{'at':now,'event':'opened','severity':level}]})
notify_internal(alert,'opened');store.event('Experimental '+level+' rainfall alert',prediction['category'],alert_id=alert['id'],mode=prediction['mode'],wadi=prediction['wadi'])
return alert
def acknowledge(id, operator, note):
with ALERT_LOCK:
a=store.get('alerts',id)
if a['status']=='resolved':raise ValueError('Resolved alerts cannot be acknowledged')
if a['status']=='acknowledged':return a
a['status']='acknowledged';a['history'].append({'at':store.now(),'event':'acknowledged','operator':operator,'note':note})
return store.put('alerts',a,id)
def current(wadi,mode):
return next((p for p in store.all_records('predictions') if p['wadi']==wadi and p['mode']==mode),None)