File size: 13,849 Bytes
124816b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
"""Traceable vegetation/rainfall opportunity screening; not a trained greening model."""
from datetime import timedelta
import math
import numpy as np
from affine import Affine
from rasterio.transform import from_bounds, array_bounds
from rasterio.warp import reproject, Resampling, transform_geom
from shapely.geometry import box, shape, mapping, Point
from . import store, geo, intelligence, locations, operations
from .observations import utc, digest

VERSION='cloud-to-green-1.1'
EMPTY={'type':'FeatureCollection','features':[]}

def ndvi(red,nir,valid):
    red=np.asarray(red,dtype='float32');nir=np.asarray(nir,dtype='float32')
    mask=np.asarray(valid,bool)&np.isfinite(red)&np.isfinite(nir)&(red>=0)&(nir>=0)&(red+nir>1e-6)
    result=np.full(red.shape,np.nan,np.float32)
    np.divide(nir-red,nir+red,out=result,where=mask)
    return np.clip(result,-1,1),mask

def vegetation(asset):
    if asset['kind']=='vegetation_image':
        with np.load(store.path_for(asset['id'],'vegetation.npz')) as a:
            return (*ndvi(a['red'],a['nir'],a['valid']),Affine(*a['transform']),str(a['crs']))
    if asset.get('band_contract')!='S2_L2A_6B_DN10000' or not asset.get('georeferenced'):
        raise ValueError('NDVI requires georeferenced, calibrated Sentinel-2 red/NIR bands')
    if not asset.get('qa_available',any('SCL' in p for p in asset.get('processing',[]))):
        raise ValueError('Satellite cloud/quality mask unavailable; NDVI inference withheld')
    with np.load(store.path_for(asset['id'],'image.npz')) as a:
        data=a['data'];valid=a['valid']
        values,mask=ndvi(data[2]/10000,data[3]/10000,valid)
    return values,mask,Affine(*asset['transform']),asset['crs']

def align(values,valid,transform,crs,target_transform,target_crs,size=8):
    """Area averages and separately averaged valid fraction, never filling cloud gaps."""
    dst=np.full((size,size),np.nan,np.float32);coverage=np.zeros_like(dst)
    src=np.where(valid,values,np.nan).astype('float32')
    reproject(src,dst,src_transform=transform,src_crs=crs,src_nodata=np.nan,
              dst_transform=target_transform,dst_crs=target_crs,dst_nodata=np.nan,resampling=Resampling.average)
    reproject(valid.astype('float32'),coverage,src_transform=transform,src_crs=crs,
              dst_transform=target_transform,dst_crs=target_crs,dst_nodata=0,resampling=Resampling.average)
    # GDAL averages only the intersecting source footprint. A tiny sliver of a
    # valid image must not become "100% valid" for a much larger edge cell.
    h,w=values.shape
    corners=[transform*(0,0),transform*(w,0),transform*(w,h),transform*(0,h),transform*(0,0)]
    footprint=shape(transform_geom(crs,target_crs,{'type':'Polygon','coordinates':[corners]}))
    for row in range(size):
        for col in range(size):
            x,y=target_transform*(col,row);xx,yy=target_transform*(col+1,row+1)
            cell=box(min(x,xx),min(y,yy),max(x,xx),max(y,yy))
            coverage[row,col]*=min(1.,footprint.intersection(cell).area/cell.area)
    return dst,(coverage>=.5)&np.isfinite(dst),coverage

def score_factors(value,rain_mm,slope,baseline=None):
    # NDVI>=.2 means a vegetation candidate only, not proof of species, stress or drought.
    eligible=value>=.2 and value<.65
    need=max(0,min(1,(baseline-value)/.3)) if baseline is not None else max(0,min(1,(.65-value)/.45))
    rainfall=max(0,min(1,rain_mm/10))
    terrain=max(0,1-min(slope/30,1))
    return {'eligible_vegetation':bool(eligible),'water_need_proxy':float(need),'rainfall_overlap':float(rainfall),
        'gentle_terrain_factor':float(terrain),'score':float(100*need*rainfall*(.5+.5*terrain)) if eligible else 0.}

def _polygon(t,crs,row,col):
    x,y=t*(col,row);xx,yy=t*(col+1,row+1)
    geom=transform_geom(crs,'EPSG:4326',mapping(box(min(x,xx),min(y,yy),max(x,xx),max(y,yy))))
    return shape(geom).intersection(locations.coverage())

def rainfall_layer(prediction,wadi):
    if not prediction:return {**EMPTY,'source_type':'weather_forecast','limitations':['Forecast unavailable']}
    f=prediction['features'];w=intelligence.area(wadi)
    footprint=box(w['lon']-.025,w['lat']-.025,w['lon']+.025,w['lat']+.025).intersection(locations.coverage())
    return {'type':'FeatureCollection','features':[] if footprint.is_empty else [{'type':'Feature','geometry':mapping(footprint),
        'properties':{'name':'Six-hour forecast support area','rainfall_mm':f['forecast_total_mm'],'valid_start':f['window_start'],
        'valid_end':f['window_end'],'acquired_at':f['forecast_acquired_at'],'weather_id':f['weather_id'],
        'source_type':'SIMULATED FORECAST' if prediction['mode']=='demo' else 'EXTERNAL WEATHER FORECAST',
        'spatial_support':'One provider grid point applied uniformly to study footprint; no sub-grid skill implied'}}],
        'timeline':f['points'],'resolution':'Hourly; native model grid resolution not supplied by automatic provider selection',
        'limitations':['Study footprint is visualization support, not the native forecast pixel','No custom rainfall-prediction AI was trained']}

def process(wadi,mode,prediction,weather=None):
    # Demo can reuse real historical satellite/DEM inputs, explicitly combined with
    # synthetic forecast. It never promotes simulated readings to real observations.
    selected=intelligence.assets(wadi,'real');terrain=selected.get('terrain')
    w=intelligence.area(wadi);point=Point(w['lon'],w['lat'])
    candidates=[a for a in store.all_records('assets') if a['kind'] in ('image','vegetation_image') and a.get('source')!='SYNTHETIC DEMO' and a.get('acquisition_time') and a.get('bbox') and box(*a['bbox']).covers(point)]
    candidates.sort(key=lambda a:a['acquisition_time'],reverse=True)
    image=next((a for a in candidates if a['kind']=='vegetation_image' or a.get('band_contract')=='S2_L2A_6B_DN10000'),None)
    # Include time quality in identity so fresh results become stale without new downloads.
    now=utc();f=prediction['features'] if prediction else {}
    image_age=(now-utc(image['acquisition_time'])).total_seconds()/86400 if image else None
    forecast_fresh=bool(f.get('complete') and f.get('forecast_acquired_at') and (now-utc(f['forecast_acquired_at'])).total_seconds()<=intelligence.settings().forecast_max_age_hours*3600 and now<utc(f['window_start'])+timedelta(hours=1))
    baseline_candidates=[]
    if image:
        image_time=utc(image['acquisition_time'])
        for a in candidates:
            dt=utc(a['acquisition_time']);season=abs(dt.timetuple().tm_yday-image_time.timetuple().tm_yday)
            if (image_time-dt).days>=180 and min(season,366-season)<=45:baseline_candidates.append(a)
    signature=digest([VERSION,wadi,mode,prediction['id'] if prediction else None,terrain['id'] if terrain else None,
        image['id'] if image else None,[a['id'] for a in baseline_candidates],forecast_fresh,image_age is not None and 0<=image_age<=45])
    try:return store.get('cloud_green',signature)
    except KeyError:pass
    result={'wadi':wadi,'mode':mode,'created_at':store.now(),'version':VERSION,'status':'insufficient_data',
        'source_type':'CALCULATED ENVIRONMENTAL SCREENING' if mode=='real' else 'SIMULATION / REPLAY',
        'rainfall':rainfall_layer(prediction,wadi),'ndvi':EMPTY,'rankings':EMPTY,'candidate_count':0,
        'input_ids':{'prediction':prediction['id'] if prediction else None,'image':image['id'] if image else None,'terrain':terrain['id'] if terrain else None},
        'image_acquired_at':image.get('acquisition_time') if image else None,'image_age_days':image_age,
        'forecast_period':{'start':f.get('window_start'),'end':f.get('window_end')},
        'missing':[],'baseline_count':0,'uncertainty':'high; no calibrated outcome probabilities',
        'method':'100 * water_need_proxy * min(forecast_mm/10,1) * (0.5 + 0.5 * max(0,1-slope_degrees/30))',
        'limitations':['Potential rainfall benefit only; vegetation recovery is not guaranteed',
            'Low NDVI can reflect soil, species, season, harvest or urban surfaces; it is not a drought diagnosis',
            'Rain is a uniform provider grid-point forecast over the study footprint',
            'NDVI is averaged to the coarse analysis grid; cells with <50% valid coverage excluded',
            'No soil moisture or ground vegetation labels connected; ranking is transparent research screening']}
    if not image:result['missing'].append('Cloud-masked red/NIR satellite observation')
    if not terrain:result['missing'].append('Georeferenced DEM')
    if not forecast_fresh:result['missing'].append('Complete fresh rainfall forecast')
    if image and not 0<=image_age<=45:
        if mode=='real':result['missing'].append('Satellite observation no older than 45 days')
        else:result['limitations'].append('Historical demonstration imagery; rankings are not current opportunities')
    if mode=='demo':result['limitations'].append('Demo combines invented rainfall with identified real historical satellite/DEM inputs when available; synthetic terrain is not used for vegetation ranking')
    if image and terrain:
        try:
            values,valid,src_t,src_crs=vegetation(image)
            a,t,crs=geo.load_grid(terrain['id']);bounds=array_bounds(*a['z'].shape,t);target=from_bounds(*bounds,8,8)
            v,mask,coverage=align(values,valid,src_t,src_crs,target,crs)
            slope,slope_valid,_=align(a['slope'],a['valid'],t,crs,target,crs)
            mask &= slope_valid
            baselines=[];dates=set()
            for old in baseline_candidates:
                day=old['acquisition_time'][:10]
                if day in dates:continue
                try:
                    ov,om,ot,oc=vegetation(old);aligned,bm,_=align(ov,om,ot,oc,target,crs)
                    baselines.append(np.where(bm,aligned,np.nan));dates.add(day)
                except (ValueError,OSError,KeyError):continue
            baseline=None;baseline_n=None
            if len(baselines)>=3:
                stack=np.stack(baselines);baseline_n=np.sum(np.isfinite(stack),axis=0)
                import warnings
                with warnings.catch_warnings():
                    warnings.simplefilter('ignore',RuntimeWarning);baseline=np.nanmedian(stack,axis=0)
                result['baseline_count']=len(baselines)
            else:result['limitations'].append('No >=3 independent seasonal baseline dates; water need uses sparse vegetation proxy, not stress')
            ndvi_features=[];rankings=[];rain_geom=shape(result['rainfall']['features'][0]['geometry']) if result['rainfall']['features'] else None
            for row,col in zip(*np.where(mask)):
                polygon=_polygon(target,crs,int(row),int(col))
                if rain_geom is not None:polygon=polygon.intersection(rain_geom)
                if polygon.is_empty:continue
                value=float(v[row,col]);base=float(baseline[row,col]) if baseline is not None and baseline_n[row,col]>=3 and np.isfinite(baseline[row,col]) else None
                props={'name':f'Vegetation cell {row+1}/{col+1}','ndvi':value,'valid_fraction':float(coverage[row,col]),
                       'slope_deg':float(slope[row,col]),'seasonal_baseline_ndvi':base,'acquired_at':image['acquisition_time'],
                       'image_id':image['id'],'historical':bool(image_age>45),'source_type':'EXTERNAL SATELLITE OBSERVATION'}
                ndvi_features.append({'type':'Feature','geometry':mapping(polygon),'properties':props})
                factors=score_factors(value,f.get('forecast_total_mm') or 0,float(slope[row,col]),base)
                if not result['missing'] and factors['eligible_vegetation'] and factors['score']>0:
                    rankings.append({'type':'Feature','geometry':mapping(polygon),'properties':{**props,**factors,
                        'rainfall_mm':f['forecast_total_mm'],'valid_start':f['window_start'],'valid_end':f['window_end'],
                        'need_basis':'seasonal NDVI decline' if base is not None else 'sparse vegetation proxy; stress unconfirmed',
                        'uncertainty':'high','source_type':'CALCULATED OPPORTUNITY; NOT GUARANTEED GREENING','display_color':'#8be6a2'}})
            rankings.sort(key=lambda f:f['properties']['score'],reverse=True)
            for i,item in enumerate(rankings):item['properties']['rank']=i+1
            result.update(ndvi={'type':'FeatureCollection','features':ndvi_features},rankings={'type':'FeatureCollection','features':rankings},candidate_count=len(rankings),
                valid_cell_count=len(ndvi_features),grid={'crs':crs,'transform':list(target)[:6],'shape':[8,8],'cell_m':[abs(target.a),abs(target.e)],
                    'ndvi':np.where(mask,v,0).tolist(),'valid':mask.tolist()})
            if not ndvi_features:result['missing'].append('Valid satellite/terrain overlap inside UAE study area')
        except (ValueError,OSError,KeyError) as exc:result['missing'].append(str(exc)[:250])
    if not result['missing']:result['status']='opportunities' if result['candidate_count'] else 'no_candidates'
    return store.put('cloud_green',result,signature)

def cloud_evidence(weather):
    assessment=operations.cloud_assessment(weather)
    age=(utc()-utc(weather['retrieved_at'])).total_seconds()/3600 if weather else None
    assessment.update(category='insufficient_data',source_type=weather.get('provenance') if weather else 'UNAVAILABLE',
        weather_id=weather.get('id') if weather else None,acquired_at=weather.get('retrieved_at') if weather else None,
        freshness='missing' if age is None else 'stale' if age>6 else 'recent',confidence=None,
        explanation='Surface forecasts cannot establish seedable clouds. Vertical profiles, liquid water and microphysics are missing. No causal seeding benefit or actuation is inferred.')
    return assessment