Download backend/intelligence.py from m4az19/saef-api: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/intelligence.py
- Command line
-
hf download hf://spaces/m4az19/saef-api/backend/intelligence.py
-
curl -L -o intelligence.py https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/intelligence.py
16.4 kB
| """Versioned rainfall screening, scenario comparisons, and internal research alerts. | |
| No trained flood-risk model is available. These outputs are deliberately not | |
| probabilities, arrival forecasts, confirmed flooding, or public-safety warnings. | |
| """ | |
| from datetime import timedelta | |
| from threading import RLock | |
| from typing import Literal | |
| from pydantic import BaseModel, ConfigDict, Field, model_validator | |
| from . import store, geo | |
| from .observations import utc, quality, digest | |
| from .simulation import Scenario, simulate | |
| MODEL_VERSION = 'rainfall-screening-1.0' | |
| FEATURE_VERSION = 'hourly-grid-rainfall-1.1' | |
| ALERT_LOCK = RLock() | |
| TIERS = {'insufficient': -1, 'below_threshold': 0, 'watch': 1, 'elevated': 2, 'critical': 3} | |
| class PipelineSettings(BaseModel): | |
| model_config = ConfigDict(extra='forbid', allow_inf_nan=False) | |
| enabled: bool = True | |
| external_weather: bool = True | |
| auto_scenarios: bool = True | |
| auto_segmentation: bool = True | |
| auto_acquire: bool = True | |
| imagery_refresh_hours: int = Field(default=24,ge=6,le=168) | |
| poll_seconds: int = Field(default=60, ge=15, le=3600) | |
| weather_refresh_minutes: int = Field(default=60, ge=30, le=1440) | |
| forecast_max_age_hours: float = Field(default=6, ge=.5, le=24) | |
| watch_mm: float = Field(default=10, gt=0, le=500) | |
| elevated_mm: float = Field(default=25, gt=0, le=500) | |
| critical_mm: float = Field(default=50, gt=0, le=500) | |
| infiltration_mm_h: float = Field(default=8, ge=0, le=200) | |
| runoff: float = Field(default=.65, ge=0, le=1) | |
| def ordered(self): | |
| if not self.watch_mm < self.elevated_mm < self.critical_mm: | |
| raise ValueError('Rainfall screening thresholds must strictly increase') | |
| return self | |
| def settings(): | |
| import os | |
| try: | |
| return PipelineSettings(**{k:v for k,v in store.get('config','pipeline').items() if k!='id'}) | |
| except KeyError: | |
| return PipelineSettings(enabled=os.environ.get('SATFORGE_PIPELINE_ENABLED','1')=='1', | |
| external_weather=os.environ.get('SATFORGE_AUTO_WEATHER','1')=='1') | |
| def area(wadi): | |
| from .locations import get_area | |
| return get_area(wadi) | |
| def in_area(record, wadi): | |
| w = area(wadi) | |
| return abs(record['longitude']-w['lon']) <= .06 and abs(record['latitude']-w['lat']) <= .06 | |
| def assets(wadi, mode): | |
| w = area(wadi) | |
| selected = {} | |
| for a in store.all_records('assets'): | |
| demo = a['source']=='SYNTHETIC DEMO' | |
| if demo != (mode=='demo'): | |
| continue | |
| b = a.get('bbox') | |
| if not (a.get('wadi')==wadi or b and b[0]<=w['lon']<=b[2] and b[1]<=w['lat']<=b[3]): | |
| continue | |
| selected.setdefault(a['kind'], a) | |
| return selected | |
| def features(wadi, mode, config, now=None): | |
| now = utc(now) | |
| # Use six complete future hours; provider values are interval-ending totals. | |
| start = now.replace(minute=0,second=0,microsecond=0) | |
| if now>start: | |
| start += timedelta(hours=1) | |
| times = [start+timedelta(hours=i) for i in range(1,7)] | |
| relevant = [r for r in store.all_records('observations') if in_area(r,wadi) and | |
| (r['category']=='simulation')==(mode=='demo')] | |
| forecasts = [r for r in relevant if r['product']=='forecast' and r['measurement']=='precipitation_mm' and r['interval_minutes']==60] | |
| # Never combine different forecast runs or providers into one artificial series. | |
| groups = {} | |
| for r in forecasts: | |
| run_id=r.get('metadata',{}).get('weather_id') or r.get('metadata',{}).get('forecast_run_id') | |
| groups.setdefault((r['source_id'],r['instrument_id'],r['acquired_at'],run_id),[]).append(r) | |
| latest = max(groups.values(), key=lambda rs:(utc(rs[0]['acquired_at']),utc(rs[0]['ingested_at'])), default=[]) | |
| points=[];missing=[];ids=[] | |
| for t in times: | |
| rs=[r for r in latest if utc(r['valid_at'])==t] | |
| r=rs[0] if len(rs)==1 else None | |
| status=quality(r,now,config.forecast_max_age_hours) if r else 'missing' | |
| valid=r is not None and status=='valid' | |
| if not valid:missing.append(f'{t.isoformat()}: {status}') | |
| points.append({'end_at':t.isoformat(),'value_mm':r['value'] if valid else None,'quality':status,'observation_id':r['id'] if r else None}) | |
| if r:ids.append(r['id']) | |
| complete=not missing | |
| values=[p['value_mm'] for p in points] | |
| observed=[{**r,'current_quality':quality(r,now)} for r in relevant if r['product']=='observation'] | |
| # Disagreement is flagged, never averaged into spurious confidence. | |
| conflicts=[] | |
| for i,r in enumerate(observed[:100]): | |
| if r['current_quality']!='valid':continue | |
| for other in observed[:i]: | |
| if other['current_quality']=='valid' and r['source_id']!=other['source_id'] and r['measurement']==other['measurement'] and r['interval_minutes']==other['interval_minutes'] and abs((utc(r['valid_at'])-utc(other['valid_at'])).total_seconds())<300 and abs(r['value']-other['value'])>max(5,abs(r['value'])*.5): | |
| conflicts.append({'observation_ids':[r['id'],other['id']],'reason':'Nearby-time sources disagree; spatial support and calibration need review'}) | |
| return {'version':FEATURE_VERSION,'window_start':start.isoformat(),'window_end':times[-1].isoformat(),'points':points, | |
| 'complete':complete,'forecast_total_mm':sum(values) if complete else None,'forecast_peak_mm_h':max(values) if complete else None, | |
| 'forecast_acquired_at':latest[0]['acquired_at'] if latest else None,'source_id':latest[0]['source_id'] if latest else None, | |
| 'weather_id':latest[0].get('metadata',{}).get('weather_id') if latest else None, | |
| 'observation_ids':ids,'missing':missing,'ground_observations':observed[:30],'conflicts':conflicts} | |
| def tier(total, config): | |
| if total is None:return 'insufficient' | |
| for name in ('critical','elevated','watch'): | |
| if total>=getattr(config,name+'_mm'):return name | |
| return 'below_threshold' | |
| def assess(wadi, mode, config=None, now=None): | |
| config=config or settings();now=utc(now);f=features(wadi,mode,config,now) | |
| selected=assets(wadi,mode);level=tier(f['forecast_total_mm'],config) | |
| input_signature={k:v for k,v in f.items() if k not in ('ground_observations',)} | |
| input_signature['ground_observations']=[(r['id'],r['current_quality']) for r in f['ground_observations']] | |
| signature=digest({'mode':mode,'wadi':wadi,'features':input_signature,'model':MODEL_VERSION, | |
| 'config':config.model_dump(),'routing_summary_version':'1.1','assets':{k:v['id'] for k,v in selected.items()}}) | |
| for old in store.all_records('predictions'): | |
| if old.get('signature')==signature:return old,False | |
| limits=['No calibrated or independently validated flood-risk model; no flood probability or arrival time is estimated.', | |
| 'Six-hour rainfall thresholds are configurable research settings, not locally calibrated warning levels.', | |
| 'A single forecast grid point is not catchment-average rainfall. No verified ground rain gauge or water-level network is connected.', | |
| 'Image water classifications and modeled inundation are not confirmed flooding.'] | |
| if level=='insufficient': | |
| explanation='The six-hour forecast is missing, incomplete, rejected, or stale. Flood conditions cannot be assessed from the available evidence.' | |
| action='Restore a complete fresh rainfall forecast and review source quality. Do not interpret missing data as low risk.' | |
| else: | |
| explanation=f"{f['source_id']} provides {f['forecast_total_mm']:.1f} mm over six complete future hours, with an hourly peak of {f['forecast_peak_mm_h']:.1f} mm. " | |
| explanation+=f"This exceeds the configured {getattr(config,level+'_mm'):g} mm {level} screening threshold." if level!='below_threshold' else 'This is below the configured research screening thresholds; flooding is not ruled out.' | |
| action='Review the forecast evidence, inspect modeled infrastructure intersections, and check authoritative local weather guidance.' if level!='below_threshold' else 'Continue monitoring and verify any field reports. Below-threshold rainfall is not an all-clear.' | |
| if f['conflicts']:limits.append('Ground sources disagree; reconcile the flagged observations before operational interpretation.') | |
| if 'terrain' not in selected:limits.append('No terrain dataset covers this area; inundation scenarios are unavailable.') | |
| if 'infrastructure' not in selected:limits.append('No infrastructure dataset covers this area; affected assets cannot be enumerated.') | |
| record=store.put('predictions',{'signature':signature,'created_at':now.isoformat(),'wadi':wadi,'area':area(wadi),'mode':mode, | |
| 'category':'simulation' if mode=='demo' else 'external','product':'EXPERIMENTAL RAINFALL SCREENING','model_type':'transparent rule baseline', | |
| 'model_version':MODEL_VERSION,'feature_version':FEATURE_VERSION,'features':f,'risk_level':level,'probability':None,'onset_at':None, | |
| 'confidence':'LIMITED / UNVALIDATED','explanation':explanation,'recommended_action':action,'limitations':limits, | |
| 'settings':config.model_dump(),'input_ids':f['observation_ids'],'asset_ids':{k:v['id'] for k,v in selected.items()}, | |
| 'scenario_status':'pending' if f['complete'] and 'terrain' in selected and config.auto_scenarios else 'unavailable', 'scenarios':[]}) | |
| store.event(f'{area(wadi)["name"]}: {level.replace("_"," ")} rainfall screening',record['category'],prediction_id=record['id'],wadi=wadi,mode=mode) | |
| for id in f['observation_ids']: | |
| o=store.get('observations',id);store.put('observations',{**o,'processing_status':'assessed','latest_prediction_id':record['id']},id) | |
| return record,True | |
| def generate_scenarios(prediction): | |
| if prediction['scenario_status']!='pending':return prediction | |
| f=prediction['features'];cfg=prediction['settings'];comparisons=[] | |
| prediction={**prediction,'scenario_status':'running'};store.put('predictions',prediction,prediction['id']) | |
| completed={run.get('rainfall_multiplier'):run for run in store.all_records('simulations') if run.get('automatic_prediction_id')==prediction['id']} | |
| try: | |
| for label,multiplier in [('Forecast baseline',1.),('Elevated rainfall assumption',1.25),('Severe rainfall stress test',2.)]: | |
| rates=[p['value_mm']*multiplier for p in f['points']] | |
| if max(rates)>300:raise ValueError('Forecast stress intensity exceeds the routing model input limit; no clipping was applied') | |
| run=completed.get(multiplier) or simulate(Scenario(terrain_id=prediction['asset_ids']['terrain'],infrastructure_id=prediction['asset_ids'].get('infrastructure'), | |
| name=label, rainfall_mm_h=sum(rates)/6,rainfall_series_mm_h=rates,rain_duration_min=360,duration_min=360, | |
| output_step_min=30,infiltration_mm_h=cfg['infiltration_mm_h'],runoff=cfg['runoff'],upstream_m3_s=0,initial_depth_m=0)) | |
| run.update(automatic_prediction_id=prediction['id'],mode=prediction['mode'],wadi=prediction['wadi'], | |
| forecast_window_start=f['window_start'],rainfall_multiplier=multiplier) | |
| store.put('simulations',run,run['id']) | |
| final=run['frames'][-1];peak=max(run['frames'],key=lambda frame:frame['area_km2']) | |
| comparisons.append({'id':run['id'],'label':label,'multiplier':multiplier,'category':'simulation', | |
| 'final_area_km2':final['area_km2'],'peak_area_km2':peak['area_km2'],'max_depth_m':max(x['max_depth_m'] for x in run['frames']), | |
| 'exposed_features':max(x['exposed_count'] for x in run['frames']) if run['config']['infrastructure_id'] else None, | |
| 'affected':[e for e in final['exposure'] if e['first_exposure_min'] is not None], | |
| 'difference_final_area_km2':final['area_km2']-(comparisons[0]['final_area_km2'] if comparisons else final['area_km2']), | |
| 'difference_exposed_features':(max(x['exposed_count'] for x in run['frames'])-(comparisons[0]['exposed_features'] if comparisons else max(x['exposed_count'] for x in run['frames']))) if run['config']['infrastructure_id'] else None}) | |
| prediction.update(scenario_status='complete',scenarios=comparisons) | |
| except Exception as exc: | |
| prediction.update(scenario_status='failed',scenario_error=str(exc)[:500],scenarios=comparisons) | |
| store.event('Automatic scenario generation failed',prediction['category'],prediction_id=prediction['id'],error=str(exc)[:500]) | |
| return store.put('predictions',prediction,prediction['id']) | |
| def notify_internal(alert, event): | |
| """Notification adapter contract. No external transport is configured or invoked.""" | |
| return store.put('notifications',{'alert_id':alert['id'],'event':event,'created_at':store.now(), | |
| 'channel':'internal','delivery_status':'available in alert centre','external_delivery':'not configured'}) | |
| def evaluate_alert(prediction): | |
| key=f"{prediction['mode']}:{prediction['wadi']}:rainfall" | |
| with ALERT_LOCK: | |
| existing=next((a for a in store.all_records('alerts') if a['dedup_key']==key and a['status']!='resolved'),None) | |
| level=prediction['risk_level'];now=store.now() | |
| if level=='insufficient': | |
| if existing: | |
| return store.put('alerts',{**existing,'data_status':'insufficient; condition not proven resolved','last_checked_at':now},existing['id']) | |
| return None | |
| if level=='below_threshold': | |
| if not existing:return None | |
| existing.update(status='resolved',resolved_at=now,data_status='fresh',prediction_id=prediction['id']) | |
| existing['history'].append({'at':now,'event':'resolved','reason':'Fresh complete forecast fell below the configured watch threshold'}) | |
| result=store.put('alerts',existing,existing['id']);notify_internal(result,'resolved');return result | |
| if existing: | |
| changed=level!=existing['severity'];event='escalated' if TIERS[level]>TIERS[existing['severity']] else 'de-escalated' | |
| existing.update(prediction_id=prediction['id'],evidence=prediction['explanation'],horizon={'start':prediction['features']['window_start'],'end':prediction['features']['window_end']}, | |
| last_checked_at=now,data_status='fresh',recommended_action=prediction['recommended_action'],limitations=prediction['limitations']) | |
| if changed: | |
| existing.update(severity=level,status='open' if event=='escalated' else existing['status']) | |
| existing['history'].append({'at':now,'event':event,'severity':level}) | |
| result=store.put('alerts',existing,existing['id']) | |
| if changed:notify_internal(result,event) | |
| return result | |
| alert=store.put('alerts',{'dedup_key':key,'wadi':prediction['wadi'],'area':prediction['area'],'mode':prediction['mode'], | |
| 'severity':level,'status':'open','detected_at':now,'last_checked_at':now,'prediction_id':prediction['id'], | |
| 'evidence':prediction['explanation'],'horizon':{'start':prediction['features']['window_start'],'end':prediction['features']['window_end']}, | |
| 'confidence':prediction['confidence'],'limitations':prediction['limitations'],'recommended_action':prediction['recommended_action'], | |
| 'classification':'EXPERIMENTAL INTERNAL ALERT — not a public-safety warning','data_status':'fresh','history':[{'at':now,'event':'opened','severity':level}]}) | |
| notify_internal(alert,'opened');store.event('Experimental '+level+' rainfall alert',prediction['category'],alert_id=alert['id'],mode=prediction['mode'],wadi=prediction['wadi']) | |
| return alert | |
| def acknowledge(id, operator, note): | |
| with ALERT_LOCK: | |
| a=store.get('alerts',id) | |
| if a['status']=='resolved':raise ValueError('Resolved alerts cannot be acknowledged') | |
| if a['status']=='acknowledged':return a | |
| a['status']='acknowledged';a['history'].append({'at':store.now(),'event':'acknowledged','operator':operator,'note':note}) | |
| return store.put('alerts',a,id) | |
| def current(wadi,mode): | |
| return next((p for p in store.all_records('predictions') if p['wadi']==wadi and p['mode']==mode),None) | |