File size: 16,415 Bytes
124816b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 | """Versioned rainfall screening, scenario comparisons, and internal research alerts.
No trained flood-risk model is available. These outputs are deliberately not
probabilities, arrival forecasts, confirmed flooding, or public-safety warnings.
"""
from datetime import timedelta
from threading import RLock
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
from . import store, geo
from .observations import utc, quality, digest
from .simulation import Scenario, simulate
MODEL_VERSION = 'rainfall-screening-1.0'
FEATURE_VERSION = 'hourly-grid-rainfall-1.1'
ALERT_LOCK = RLock()
TIERS = {'insufficient': -1, 'below_threshold': 0, 'watch': 1, 'elevated': 2, 'critical': 3}
class PipelineSettings(BaseModel):
model_config = ConfigDict(extra='forbid', allow_inf_nan=False)
enabled: bool = True
external_weather: bool = True
auto_scenarios: bool = True
auto_segmentation: bool = True
auto_acquire: bool = True
imagery_refresh_hours: int = Field(default=24,ge=6,le=168)
poll_seconds: int = Field(default=60, ge=15, le=3600)
weather_refresh_minutes: int = Field(default=60, ge=30, le=1440)
forecast_max_age_hours: float = Field(default=6, ge=.5, le=24)
watch_mm: float = Field(default=10, gt=0, le=500)
elevated_mm: float = Field(default=25, gt=0, le=500)
critical_mm: float = Field(default=50, gt=0, le=500)
infiltration_mm_h: float = Field(default=8, ge=0, le=200)
runoff: float = Field(default=.65, ge=0, le=1)
@model_validator(mode='after')
def ordered(self):
if not self.watch_mm < self.elevated_mm < self.critical_mm:
raise ValueError('Rainfall screening thresholds must strictly increase')
return self
def settings():
import os
try:
return PipelineSettings(**{k:v for k,v in store.get('config','pipeline').items() if k!='id'})
except KeyError:
return PipelineSettings(enabled=os.environ.get('SATFORGE_PIPELINE_ENABLED','1')=='1',
external_weather=os.environ.get('SATFORGE_AUTO_WEATHER','1')=='1')
def area(wadi):
from .locations import get_area
return get_area(wadi)
def in_area(record, wadi):
w = area(wadi)
return abs(record['longitude']-w['lon']) <= .06 and abs(record['latitude']-w['lat']) <= .06
def assets(wadi, mode):
w = area(wadi)
selected = {}
for a in store.all_records('assets'):
demo = a['source']=='SYNTHETIC DEMO'
if demo != (mode=='demo'):
continue
b = a.get('bbox')
if not (a.get('wadi')==wadi or b and b[0]<=w['lon']<=b[2] and b[1]<=w['lat']<=b[3]):
continue
selected.setdefault(a['kind'], a)
return selected
def features(wadi, mode, config, now=None):
now = utc(now)
# Use six complete future hours; provider values are interval-ending totals.
start = now.replace(minute=0,second=0,microsecond=0)
if now>start:
start += timedelta(hours=1)
times = [start+timedelta(hours=i) for i in range(1,7)]
relevant = [r for r in store.all_records('observations') if in_area(r,wadi) and
(r['category']=='simulation')==(mode=='demo')]
forecasts = [r for r in relevant if r['product']=='forecast' and r['measurement']=='precipitation_mm' and r['interval_minutes']==60]
# Never combine different forecast runs or providers into one artificial series.
groups = {}
for r in forecasts:
run_id=r.get('metadata',{}).get('weather_id') or r.get('metadata',{}).get('forecast_run_id')
groups.setdefault((r['source_id'],r['instrument_id'],r['acquired_at'],run_id),[]).append(r)
latest = max(groups.values(), key=lambda rs:(utc(rs[0]['acquired_at']),utc(rs[0]['ingested_at'])), default=[])
points=[];missing=[];ids=[]
for t in times:
rs=[r for r in latest if utc(r['valid_at'])==t]
r=rs[0] if len(rs)==1 else None
status=quality(r,now,config.forecast_max_age_hours) if r else 'missing'
valid=r is not None and status=='valid'
if not valid:missing.append(f'{t.isoformat()}: {status}')
points.append({'end_at':t.isoformat(),'value_mm':r['value'] if valid else None,'quality':status,'observation_id':r['id'] if r else None})
if r:ids.append(r['id'])
complete=not missing
values=[p['value_mm'] for p in points]
observed=[{**r,'current_quality':quality(r,now)} for r in relevant if r['product']=='observation']
# Disagreement is flagged, never averaged into spurious confidence.
conflicts=[]
for i,r in enumerate(observed[:100]):
if r['current_quality']!='valid':continue
for other in observed[:i]:
if other['current_quality']=='valid' and r['source_id']!=other['source_id'] and r['measurement']==other['measurement'] and r['interval_minutes']==other['interval_minutes'] and abs((utc(r['valid_at'])-utc(other['valid_at'])).total_seconds())<300 and abs(r['value']-other['value'])>max(5,abs(r['value'])*.5):
conflicts.append({'observation_ids':[r['id'],other['id']],'reason':'Nearby-time sources disagree; spatial support and calibration need review'})
return {'version':FEATURE_VERSION,'window_start':start.isoformat(),'window_end':times[-1].isoformat(),'points':points,
'complete':complete,'forecast_total_mm':sum(values) if complete else None,'forecast_peak_mm_h':max(values) if complete else None,
'forecast_acquired_at':latest[0]['acquired_at'] if latest else None,'source_id':latest[0]['source_id'] if latest else None,
'weather_id':latest[0].get('metadata',{}).get('weather_id') if latest else None,
'observation_ids':ids,'missing':missing,'ground_observations':observed[:30],'conflicts':conflicts}
def tier(total, config):
if total is None:return 'insufficient'
for name in ('critical','elevated','watch'):
if total>=getattr(config,name+'_mm'):return name
return 'below_threshold'
def assess(wadi, mode, config=None, now=None):
config=config or settings();now=utc(now);f=features(wadi,mode,config,now)
selected=assets(wadi,mode);level=tier(f['forecast_total_mm'],config)
input_signature={k:v for k,v in f.items() if k not in ('ground_observations',)}
input_signature['ground_observations']=[(r['id'],r['current_quality']) for r in f['ground_observations']]
signature=digest({'mode':mode,'wadi':wadi,'features':input_signature,'model':MODEL_VERSION,
'config':config.model_dump(),'routing_summary_version':'1.1','assets':{k:v['id'] for k,v in selected.items()}})
for old in store.all_records('predictions'):
if old.get('signature')==signature:return old,False
limits=['No calibrated or independently validated flood-risk model; no flood probability or arrival time is estimated.',
'Six-hour rainfall thresholds are configurable research settings, not locally calibrated warning levels.',
'A single forecast grid point is not catchment-average rainfall. No verified ground rain gauge or water-level network is connected.',
'Image water classifications and modeled inundation are not confirmed flooding.']
if level=='insufficient':
explanation='The six-hour forecast is missing, incomplete, rejected, or stale. Flood conditions cannot be assessed from the available evidence.'
action='Restore a complete fresh rainfall forecast and review source quality. Do not interpret missing data as low risk.'
else:
explanation=f"{f['source_id']} provides {f['forecast_total_mm']:.1f} mm over six complete future hours, with an hourly peak of {f['forecast_peak_mm_h']:.1f} mm. "
explanation+=f"This exceeds the configured {getattr(config,level+'_mm'):g} mm {level} screening threshold." if level!='below_threshold' else 'This is below the configured research screening thresholds; flooding is not ruled out.'
action='Review the forecast evidence, inspect modeled infrastructure intersections, and check authoritative local weather guidance.' if level!='below_threshold' else 'Continue monitoring and verify any field reports. Below-threshold rainfall is not an all-clear.'
if f['conflicts']:limits.append('Ground sources disagree; reconcile the flagged observations before operational interpretation.')
if 'terrain' not in selected:limits.append('No terrain dataset covers this area; inundation scenarios are unavailable.')
if 'infrastructure' not in selected:limits.append('No infrastructure dataset covers this area; affected assets cannot be enumerated.')
record=store.put('predictions',{'signature':signature,'created_at':now.isoformat(),'wadi':wadi,'area':area(wadi),'mode':mode,
'category':'simulation' if mode=='demo' else 'external','product':'EXPERIMENTAL RAINFALL SCREENING','model_type':'transparent rule baseline',
'model_version':MODEL_VERSION,'feature_version':FEATURE_VERSION,'features':f,'risk_level':level,'probability':None,'onset_at':None,
'confidence':'LIMITED / UNVALIDATED','explanation':explanation,'recommended_action':action,'limitations':limits,
'settings':config.model_dump(),'input_ids':f['observation_ids'],'asset_ids':{k:v['id'] for k,v in selected.items()},
'scenario_status':'pending' if f['complete'] and 'terrain' in selected and config.auto_scenarios else 'unavailable', 'scenarios':[]})
store.event(f'{area(wadi)["name"]}: {level.replace("_"," ")} rainfall screening',record['category'],prediction_id=record['id'],wadi=wadi,mode=mode)
for id in f['observation_ids']:
o=store.get('observations',id);store.put('observations',{**o,'processing_status':'assessed','latest_prediction_id':record['id']},id)
return record,True
def generate_scenarios(prediction):
if prediction['scenario_status']!='pending':return prediction
f=prediction['features'];cfg=prediction['settings'];comparisons=[]
prediction={**prediction,'scenario_status':'running'};store.put('predictions',prediction,prediction['id'])
completed={run.get('rainfall_multiplier'):run for run in store.all_records('simulations') if run.get('automatic_prediction_id')==prediction['id']}
try:
for label,multiplier in [('Forecast baseline',1.),('Elevated rainfall assumption',1.25),('Severe rainfall stress test',2.)]:
rates=[p['value_mm']*multiplier for p in f['points']]
if max(rates)>300:raise ValueError('Forecast stress intensity exceeds the routing model input limit; no clipping was applied')
run=completed.get(multiplier) or simulate(Scenario(terrain_id=prediction['asset_ids']['terrain'],infrastructure_id=prediction['asset_ids'].get('infrastructure'),
name=label, rainfall_mm_h=sum(rates)/6,rainfall_series_mm_h=rates,rain_duration_min=360,duration_min=360,
output_step_min=30,infiltration_mm_h=cfg['infiltration_mm_h'],runoff=cfg['runoff'],upstream_m3_s=0,initial_depth_m=0))
run.update(automatic_prediction_id=prediction['id'],mode=prediction['mode'],wadi=prediction['wadi'],
forecast_window_start=f['window_start'],rainfall_multiplier=multiplier)
store.put('simulations',run,run['id'])
final=run['frames'][-1];peak=max(run['frames'],key=lambda frame:frame['area_km2'])
comparisons.append({'id':run['id'],'label':label,'multiplier':multiplier,'category':'simulation',
'final_area_km2':final['area_km2'],'peak_area_km2':peak['area_km2'],'max_depth_m':max(x['max_depth_m'] for x in run['frames']),
'exposed_features':max(x['exposed_count'] for x in run['frames']) if run['config']['infrastructure_id'] else None,
'affected':[e for e in final['exposure'] if e['first_exposure_min'] is not None],
'difference_final_area_km2':final['area_km2']-(comparisons[0]['final_area_km2'] if comparisons else final['area_km2']),
'difference_exposed_features':(max(x['exposed_count'] for x in run['frames'])-(comparisons[0]['exposed_features'] if comparisons else max(x['exposed_count'] for x in run['frames']))) if run['config']['infrastructure_id'] else None})
prediction.update(scenario_status='complete',scenarios=comparisons)
except Exception as exc:
prediction.update(scenario_status='failed',scenario_error=str(exc)[:500],scenarios=comparisons)
store.event('Automatic scenario generation failed',prediction['category'],prediction_id=prediction['id'],error=str(exc)[:500])
return store.put('predictions',prediction,prediction['id'])
def notify_internal(alert, event):
"""Notification adapter contract. No external transport is configured or invoked."""
return store.put('notifications',{'alert_id':alert['id'],'event':event,'created_at':store.now(),
'channel':'internal','delivery_status':'available in alert centre','external_delivery':'not configured'})
def evaluate_alert(prediction):
key=f"{prediction['mode']}:{prediction['wadi']}:rainfall"
with ALERT_LOCK:
existing=next((a for a in store.all_records('alerts') if a['dedup_key']==key and a['status']!='resolved'),None)
level=prediction['risk_level'];now=store.now()
if level=='insufficient':
if existing:
return store.put('alerts',{**existing,'data_status':'insufficient; condition not proven resolved','last_checked_at':now},existing['id'])
return None
if level=='below_threshold':
if not existing:return None
existing.update(status='resolved',resolved_at=now,data_status='fresh',prediction_id=prediction['id'])
existing['history'].append({'at':now,'event':'resolved','reason':'Fresh complete forecast fell below the configured watch threshold'})
result=store.put('alerts',existing,existing['id']);notify_internal(result,'resolved');return result
if existing:
changed=level!=existing['severity'];event='escalated' if TIERS[level]>TIERS[existing['severity']] else 'de-escalated'
existing.update(prediction_id=prediction['id'],evidence=prediction['explanation'],horizon={'start':prediction['features']['window_start'],'end':prediction['features']['window_end']},
last_checked_at=now,data_status='fresh',recommended_action=prediction['recommended_action'],limitations=prediction['limitations'])
if changed:
existing.update(severity=level,status='open' if event=='escalated' else existing['status'])
existing['history'].append({'at':now,'event':event,'severity':level})
result=store.put('alerts',existing,existing['id'])
if changed:notify_internal(result,event)
return result
alert=store.put('alerts',{'dedup_key':key,'wadi':prediction['wadi'],'area':prediction['area'],'mode':prediction['mode'],
'severity':level,'status':'open','detected_at':now,'last_checked_at':now,'prediction_id':prediction['id'],
'evidence':prediction['explanation'],'horizon':{'start':prediction['features']['window_start'],'end':prediction['features']['window_end']},
'confidence':prediction['confidence'],'limitations':prediction['limitations'],'recommended_action':prediction['recommended_action'],
'classification':'EXPERIMENTAL INTERNAL ALERT — not a public-safety warning','data_status':'fresh','history':[{'at':now,'event':'opened','severity':level}]})
notify_internal(alert,'opened');store.event('Experimental '+level+' rainfall alert',prediction['category'],alert_id=alert['id'],mode=prediction['mode'],wadi=prediction['wadi'])
return alert
def acknowledge(id, operator, note):
with ALERT_LOCK:
a=store.get('alerts',id)
if a['status']=='resolved':raise ValueError('Resolved alerts cannot be acknowledged')
if a['status']=='acknowledged':return a
a['status']='acknowledged';a['history'].append({'at':store.now(),'event':'acknowledged','operator':operator,'note':note})
return store.put('alerts',a,id)
def current(wadi,mode):
return next((p for p in store.all_records('predictions') if p['wadi']==wadi and p['mode']==mode),None)
|