File size: 16,415 Bytes
124816b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
"""Versioned rainfall screening, scenario comparisons, and internal research alerts.



No trained flood-risk model is available. These outputs are deliberately not

probabilities, arrival forecasts, confirmed flooding, or public-safety warnings.

"""
from datetime import timedelta
from threading import RLock
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
from . import store, geo
from .observations import utc, quality, digest
from .simulation import Scenario, simulate

MODEL_VERSION = 'rainfall-screening-1.0'
FEATURE_VERSION = 'hourly-grid-rainfall-1.1'
ALERT_LOCK = RLock()
TIERS = {'insufficient': -1, 'below_threshold': 0, 'watch': 1, 'elevated': 2, 'critical': 3}

class PipelineSettings(BaseModel):
    model_config = ConfigDict(extra='forbid', allow_inf_nan=False)
    enabled: bool = True
    external_weather: bool = True
    auto_scenarios: bool = True
    auto_segmentation: bool = True
    auto_acquire: bool = True
    imagery_refresh_hours: int = Field(default=24,ge=6,le=168)
    poll_seconds: int = Field(default=60, ge=15, le=3600)
    weather_refresh_minutes: int = Field(default=60, ge=30, le=1440)
    forecast_max_age_hours: float = Field(default=6, ge=.5, le=24)
    watch_mm: float = Field(default=10, gt=0, le=500)
    elevated_mm: float = Field(default=25, gt=0, le=500)
    critical_mm: float = Field(default=50, gt=0, le=500)
    infiltration_mm_h: float = Field(default=8, ge=0, le=200)
    runoff: float = Field(default=.65, ge=0, le=1)

    @model_validator(mode='after')
    def ordered(self):
        if not self.watch_mm < self.elevated_mm < self.critical_mm:
            raise ValueError('Rainfall screening thresholds must strictly increase')
        return self

def settings():
    import os
    try:
        return PipelineSettings(**{k:v for k,v in store.get('config','pipeline').items() if k!='id'})
    except KeyError:
        return PipelineSettings(enabled=os.environ.get('SATFORGE_PIPELINE_ENABLED','1')=='1',
                                external_weather=os.environ.get('SATFORGE_AUTO_WEATHER','1')=='1')

def area(wadi):
    from .locations import get_area
    return get_area(wadi)

def in_area(record, wadi):
    w = area(wadi)
    return abs(record['longitude']-w['lon']) <= .06 and abs(record['latitude']-w['lat']) <= .06

def assets(wadi, mode):
    w = area(wadi)
    selected = {}
    for a in store.all_records('assets'):
        demo = a['source']=='SYNTHETIC DEMO'
        if demo != (mode=='demo'):
            continue
        b = a.get('bbox')
        if not (a.get('wadi')==wadi or b and b[0]<=w['lon']<=b[2] and b[1]<=w['lat']<=b[3]):
            continue
        selected.setdefault(a['kind'], a)
    return selected

def features(wadi, mode, config, now=None):
    now = utc(now)
    # Use six complete future hours; provider values are interval-ending totals.
    start = now.replace(minute=0,second=0,microsecond=0)
    if now>start:
        start += timedelta(hours=1)
    times = [start+timedelta(hours=i) for i in range(1,7)]
    relevant = [r for r in store.all_records('observations') if in_area(r,wadi) and
                (r['category']=='simulation')==(mode=='demo')]
    forecasts = [r for r in relevant if r['product']=='forecast' and r['measurement']=='precipitation_mm' and r['interval_minutes']==60]
    # Never combine different forecast runs or providers into one artificial series.
    groups = {}
    for r in forecasts:
        run_id=r.get('metadata',{}).get('weather_id') or r.get('metadata',{}).get('forecast_run_id')
        groups.setdefault((r['source_id'],r['instrument_id'],r['acquired_at'],run_id),[]).append(r)
    latest = max(groups.values(), key=lambda rs:(utc(rs[0]['acquired_at']),utc(rs[0]['ingested_at'])), default=[])
    points=[];missing=[];ids=[]
    for t in times:
        rs=[r for r in latest if utc(r['valid_at'])==t]
        r=rs[0] if len(rs)==1 else None
        status=quality(r,now,config.forecast_max_age_hours) if r else 'missing'
        valid=r is not None and status=='valid'
        if not valid:missing.append(f'{t.isoformat()}: {status}')
        points.append({'end_at':t.isoformat(),'value_mm':r['value'] if valid else None,'quality':status,'observation_id':r['id'] if r else None})
        if r:ids.append(r['id'])
    complete=not missing
    values=[p['value_mm'] for p in points]
    observed=[{**r,'current_quality':quality(r,now)} for r in relevant if r['product']=='observation']
    # Disagreement is flagged, never averaged into spurious confidence.
    conflicts=[]
    for i,r in enumerate(observed[:100]):
        if r['current_quality']!='valid':continue
        for other in observed[:i]:
            if other['current_quality']=='valid' and r['source_id']!=other['source_id'] and r['measurement']==other['measurement'] and r['interval_minutes']==other['interval_minutes'] and abs((utc(r['valid_at'])-utc(other['valid_at'])).total_seconds())<300 and abs(r['value']-other['value'])>max(5,abs(r['value'])*.5):
                conflicts.append({'observation_ids':[r['id'],other['id']],'reason':'Nearby-time sources disagree; spatial support and calibration need review'})
    return {'version':FEATURE_VERSION,'window_start':start.isoformat(),'window_end':times[-1].isoformat(),'points':points,
        'complete':complete,'forecast_total_mm':sum(values) if complete else None,'forecast_peak_mm_h':max(values) if complete else None,
        'forecast_acquired_at':latest[0]['acquired_at'] if latest else None,'source_id':latest[0]['source_id'] if latest else None,
        'weather_id':latest[0].get('metadata',{}).get('weather_id') if latest else None,
        'observation_ids':ids,'missing':missing,'ground_observations':observed[:30],'conflicts':conflicts}

def tier(total, config):
    if total is None:return 'insufficient'
    for name in ('critical','elevated','watch'):
        if total>=getattr(config,name+'_mm'):return name
    return 'below_threshold'

def assess(wadi, mode, config=None, now=None):
    config=config or settings();now=utc(now);f=features(wadi,mode,config,now)
    selected=assets(wadi,mode);level=tier(f['forecast_total_mm'],config)
    input_signature={k:v for k,v in f.items() if k not in ('ground_observations',)}
    input_signature['ground_observations']=[(r['id'],r['current_quality']) for r in f['ground_observations']]
    signature=digest({'mode':mode,'wadi':wadi,'features':input_signature,'model':MODEL_VERSION,
        'config':config.model_dump(),'routing_summary_version':'1.1','assets':{k:v['id'] for k,v in selected.items()}})
    for old in store.all_records('predictions'):
        if old.get('signature')==signature:return old,False
    limits=['No calibrated or independently validated flood-risk model; no flood probability or arrival time is estimated.',
        'Six-hour rainfall thresholds are configurable research settings, not locally calibrated warning levels.',
        'A single forecast grid point is not catchment-average rainfall. No verified ground rain gauge or water-level network is connected.',
        'Image water classifications and modeled inundation are not confirmed flooding.']
    if level=='insufficient':
        explanation='The six-hour forecast is missing, incomplete, rejected, or stale. Flood conditions cannot be assessed from the available evidence.'
        action='Restore a complete fresh rainfall forecast and review source quality. Do not interpret missing data as low risk.'
    else:
        explanation=f"{f['source_id']} provides {f['forecast_total_mm']:.1f} mm over six complete future hours, with an hourly peak of {f['forecast_peak_mm_h']:.1f} mm. "
        explanation+=f"This exceeds the configured {getattr(config,level+'_mm'):g} mm {level} screening threshold." if level!='below_threshold' else 'This is below the configured research screening thresholds; flooding is not ruled out.'
        action='Review the forecast evidence, inspect modeled infrastructure intersections, and check authoritative local weather guidance.' if level!='below_threshold' else 'Continue monitoring and verify any field reports. Below-threshold rainfall is not an all-clear.'
    if f['conflicts']:limits.append('Ground sources disagree; reconcile the flagged observations before operational interpretation.')
    if 'terrain' not in selected:limits.append('No terrain dataset covers this area; inundation scenarios are unavailable.')
    if 'infrastructure' not in selected:limits.append('No infrastructure dataset covers this area; affected assets cannot be enumerated.')
    record=store.put('predictions',{'signature':signature,'created_at':now.isoformat(),'wadi':wadi,'area':area(wadi),'mode':mode,
        'category':'simulation' if mode=='demo' else 'external','product':'EXPERIMENTAL RAINFALL SCREENING','model_type':'transparent rule baseline',
        'model_version':MODEL_VERSION,'feature_version':FEATURE_VERSION,'features':f,'risk_level':level,'probability':None,'onset_at':None,
        'confidence':'LIMITED / UNVALIDATED','explanation':explanation,'recommended_action':action,'limitations':limits,
        'settings':config.model_dump(),'input_ids':f['observation_ids'],'asset_ids':{k:v['id'] for k,v in selected.items()},
        'scenario_status':'pending' if f['complete'] and 'terrain' in selected and config.auto_scenarios else 'unavailable', 'scenarios':[]})
    store.event(f'{area(wadi)["name"]}: {level.replace("_"," ")} rainfall screening',record['category'],prediction_id=record['id'],wadi=wadi,mode=mode)
    for id in f['observation_ids']:
        o=store.get('observations',id);store.put('observations',{**o,'processing_status':'assessed','latest_prediction_id':record['id']},id)
    return record,True

def generate_scenarios(prediction):
    if prediction['scenario_status']!='pending':return prediction
    f=prediction['features'];cfg=prediction['settings'];comparisons=[]
    prediction={**prediction,'scenario_status':'running'};store.put('predictions',prediction,prediction['id'])
    completed={run.get('rainfall_multiplier'):run for run in store.all_records('simulations') if run.get('automatic_prediction_id')==prediction['id']}
    try:
        for label,multiplier in [('Forecast baseline',1.),('Elevated rainfall assumption',1.25),('Severe rainfall stress test',2.)]:
            rates=[p['value_mm']*multiplier for p in f['points']]
            if max(rates)>300:raise ValueError('Forecast stress intensity exceeds the routing model input limit; no clipping was applied')
            run=completed.get(multiplier) or simulate(Scenario(terrain_id=prediction['asset_ids']['terrain'],infrastructure_id=prediction['asset_ids'].get('infrastructure'),
                name=label, rainfall_mm_h=sum(rates)/6,rainfall_series_mm_h=rates,rain_duration_min=360,duration_min=360,
                output_step_min=30,infiltration_mm_h=cfg['infiltration_mm_h'],runoff=cfg['runoff'],upstream_m3_s=0,initial_depth_m=0))
            run.update(automatic_prediction_id=prediction['id'],mode=prediction['mode'],wadi=prediction['wadi'],
                       forecast_window_start=f['window_start'],rainfall_multiplier=multiplier)
            store.put('simulations',run,run['id'])
            final=run['frames'][-1];peak=max(run['frames'],key=lambda frame:frame['area_km2'])
            comparisons.append({'id':run['id'],'label':label,'multiplier':multiplier,'category':'simulation',
                'final_area_km2':final['area_km2'],'peak_area_km2':peak['area_km2'],'max_depth_m':max(x['max_depth_m'] for x in run['frames']),
                'exposed_features':max(x['exposed_count'] for x in run['frames']) if run['config']['infrastructure_id'] else None,
                'affected':[e for e in final['exposure'] if e['first_exposure_min'] is not None],
                'difference_final_area_km2':final['area_km2']-(comparisons[0]['final_area_km2'] if comparisons else final['area_km2']),
                'difference_exposed_features':(max(x['exposed_count'] for x in run['frames'])-(comparisons[0]['exposed_features'] if comparisons else max(x['exposed_count'] for x in run['frames']))) if run['config']['infrastructure_id'] else None})
        prediction.update(scenario_status='complete',scenarios=comparisons)
    except Exception as exc:
        prediction.update(scenario_status='failed',scenario_error=str(exc)[:500],scenarios=comparisons)
        store.event('Automatic scenario generation failed',prediction['category'],prediction_id=prediction['id'],error=str(exc)[:500])
    return store.put('predictions',prediction,prediction['id'])

def notify_internal(alert, event):
    """Notification adapter contract. No external transport is configured or invoked."""
    return store.put('notifications',{'alert_id':alert['id'],'event':event,'created_at':store.now(),
        'channel':'internal','delivery_status':'available in alert centre','external_delivery':'not configured'})

def evaluate_alert(prediction):
    key=f"{prediction['mode']}:{prediction['wadi']}:rainfall"
    with ALERT_LOCK:
        existing=next((a for a in store.all_records('alerts') if a['dedup_key']==key and a['status']!='resolved'),None)
        level=prediction['risk_level'];now=store.now()
        if level=='insufficient':
            if existing:
                return store.put('alerts',{**existing,'data_status':'insufficient; condition not proven resolved','last_checked_at':now},existing['id'])
            return None
        if level=='below_threshold':
            if not existing:return None
            existing.update(status='resolved',resolved_at=now,data_status='fresh',prediction_id=prediction['id'])
            existing['history'].append({'at':now,'event':'resolved','reason':'Fresh complete forecast fell below the configured watch threshold'})
            result=store.put('alerts',existing,existing['id']);notify_internal(result,'resolved');return result
        if existing:
            changed=level!=existing['severity'];event='escalated' if TIERS[level]>TIERS[existing['severity']] else 'de-escalated'
            existing.update(prediction_id=prediction['id'],evidence=prediction['explanation'],horizon={'start':prediction['features']['window_start'],'end':prediction['features']['window_end']},
                            last_checked_at=now,data_status='fresh',recommended_action=prediction['recommended_action'],limitations=prediction['limitations'])
            if changed:
                existing.update(severity=level,status='open' if event=='escalated' else existing['status'])
                existing['history'].append({'at':now,'event':event,'severity':level})
            result=store.put('alerts',existing,existing['id'])
            if changed:notify_internal(result,event)
            return result
        alert=store.put('alerts',{'dedup_key':key,'wadi':prediction['wadi'],'area':prediction['area'],'mode':prediction['mode'],
            'severity':level,'status':'open','detected_at':now,'last_checked_at':now,'prediction_id':prediction['id'],
            'evidence':prediction['explanation'],'horizon':{'start':prediction['features']['window_start'],'end':prediction['features']['window_end']},
            'confidence':prediction['confidence'],'limitations':prediction['limitations'],'recommended_action':prediction['recommended_action'],
            'classification':'EXPERIMENTAL INTERNAL ALERT — not a public-safety warning','data_status':'fresh','history':[{'at':now,'event':'opened','severity':level}]})
        notify_internal(alert,'opened');store.event('Experimental '+level+' rainfall alert',prediction['category'],alert_id=alert['id'],mode=prediction['mode'],wadi=prediction['wadi'])
        return alert

def acknowledge(id, operator, note):
    with ALERT_LOCK:
        a=store.get('alerts',id)
        if a['status']=='resolved':raise ValueError('Resolved alerts cannot be acknowledged')
        if a['status']=='acknowledged':return a
        a['status']='acknowledged';a['history'].append({'at':store.now(),'event':'acknowledged','operator':operator,'note':note})
        return store.put('alerts',a,id)

def current(wadi,mode):
    return next((p for p in store.all_records('predictions') if p['wadi']==wadi and p['mode']==mode),None)