Spaces:
Sleeping
Sleeping
Upload realtime_pipeline.py
Browse files- realtime_pipeline.py +96 -51
realtime_pipeline.py
CHANGED
|
@@ -14,6 +14,8 @@ import calendar
|
|
| 14 |
from datetime import timedelta
|
| 15 |
import json
|
| 16 |
from pathlib import Path
|
|
|
|
|
|
|
| 17 |
|
| 18 |
# ==================== CONFIGURATION ====================
|
| 19 |
|
|
@@ -30,7 +32,7 @@ class CoordinateRequest(BaseModel):
|
|
| 30 |
class HistoricalFeaturesRequest(BaseModel):
|
| 31 |
latitude: float
|
| 32 |
longitude: float
|
| 33 |
-
#
|
| 34 |
|
| 35 |
class HistoricalFeaturesResponse(BaseModel):
|
| 36 |
success: bool
|
|
@@ -97,7 +99,7 @@ class MLFeaturesResponse(BaseModel):
|
|
| 97 |
# ==================== TEMPORAL DATA HANDLER ====================
|
| 98 |
|
| 99 |
class TemporalDataManager:
|
| 100 |
-
"""
|
| 101 |
|
| 102 |
def __init__(self):
|
| 103 |
self.logger = logging.getLogger(self.__class__.__name__)
|
|
@@ -106,6 +108,10 @@ class TemporalDataManager:
|
|
| 106 |
self.cache_dir = Path("/tmp/temporal_cache")
|
| 107 |
self.cache_dir.mkdir(exist_ok=True)
|
| 108 |
self.soil_cache_file = self.cache_dir / "soil_cache.json"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
self.soil_cache = self._load_soil_cache()
|
| 110 |
|
| 111 |
# Soil versions mapping
|
|
@@ -116,17 +122,19 @@ class TemporalDataManager:
|
|
| 116 |
}
|
| 117 |
|
| 118 |
def _load_soil_cache(self):
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
|
|
|
| 126 |
|
| 127 |
def _save_soil_cache(self):
|
| 128 |
-
with
|
| 129 |
-
|
|
|
|
| 130 |
|
| 131 |
def get_soil_version(self, year: int):
|
| 132 |
for (start, end), info in self.soil_versions.items():
|
|
@@ -137,9 +145,10 @@ class TemporalDataManager:
|
|
| 137 |
def get_historical_soil_data(self, lat: float, lon: float, year: int) -> Dict:
|
| 138 |
cache_key = f"{lat:.3f}_{lon:.3f}_{year}"
|
| 139 |
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
|
|
|
| 143 |
|
| 144 |
try:
|
| 145 |
point = ee.Geometry.Point([lon, lat])
|
|
@@ -157,6 +166,7 @@ class TemporalDataManager:
|
|
| 157 |
|
| 158 |
for scale in [250, 500, 1000]:
|
| 159 |
try:
|
|
|
|
| 160 |
soil_data = soil_image.reduceRegion(
|
| 161 |
reducer=ee.Reducer.mean(),
|
| 162 |
geometry=point,
|
|
@@ -164,6 +174,7 @@ class TemporalDataManager:
|
|
| 164 |
bestEffort=True,
|
| 165 |
maxPixels=1e9
|
| 166 |
).getInfo()
|
|
|
|
| 167 |
|
| 168 |
if soil_data.get('sand') is not None:
|
| 169 |
result = {
|
|
@@ -178,11 +189,14 @@ class TemporalDataManager:
|
|
| 178 |
'_collection': version['collection']
|
| 179 |
}
|
| 180 |
|
| 181 |
-
self.
|
| 182 |
-
|
|
|
|
| 183 |
|
| 184 |
self.logger.info(f"✅ Fetched temporal soil data for {year} (scale: {scale}m)")
|
| 185 |
return result
|
|
|
|
|
|
|
| 186 |
|
| 187 |
except Exception as e:
|
| 188 |
self.logger.warning(f"⚠️ Soil scale {scale}m failed: {str(e)[:50]}")
|
|
@@ -269,8 +283,12 @@ class TemporalDataManager:
|
|
| 269 |
'weather_source': 'ERA5_LAND_Historical'
|
| 270 |
}
|
| 271 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
self.logger.info(f"✅ ERA5 climate for {date.year}-{date.month:02d}: "
|
| 273 |
-
f"Temp={
|
| 274 |
|
| 275 |
return climate_data
|
| 276 |
|
|
@@ -279,12 +297,12 @@ class TemporalDataManager:
|
|
| 279 |
return self._get_climate_fallback(lat, date.month)
|
| 280 |
|
| 281 |
def get_historical_bdod_data(self, lat: float, lon: float, year: int) -> Optional[float]:
|
| 282 |
-
"""
|
| 283 |
try:
|
| 284 |
point = ee.Geometry.Point([lon, lat])
|
| 285 |
version = self.get_soil_version(year)
|
| 286 |
|
| 287 |
-
# BDOD
|
| 288 |
bdod = ee.Image(f"{version['collection']}/bdod_mean").select("bdod_0-5cm_mean")
|
| 289 |
|
| 290 |
for scale in [250, 500, 1000]:
|
|
@@ -299,8 +317,8 @@ class TemporalDataManager:
|
|
| 299 |
|
| 300 |
bdod_value = bdod_data.get('bdod_0-5cm_mean')
|
| 301 |
if bdod_value is not None:
|
| 302 |
-
bdod_kg_m3 = bdod_value * 10 #
|
| 303 |
-
bdod_g_cm3 = bdod_kg_m3 / 1000.0 #
|
| 304 |
self.logger.info(f"✅ BDOD for {year}: {bdod_g_cm3:.2f} g/cm³ (scale: {scale}m)")
|
| 305 |
return bdod_g_cm3
|
| 306 |
|
|
@@ -447,7 +465,7 @@ class RealTimeFeatureExtractor:
|
|
| 447 |
self.logger.error(f"❌ Earth Engine failed: {e}")
|
| 448 |
raise
|
| 449 |
|
| 450 |
-
# ====================
|
| 451 |
|
| 452 |
def compute_dewpoint(self, t_celsius, rh_percent):
|
| 453 |
try:
|
|
@@ -652,7 +670,7 @@ class RealTimeFeatureExtractor:
|
|
| 652 |
self.logger.warning(f"⚠️ Soil buffer {buffer_size}m failed: {str(e)[:50]}")
|
| 653 |
continue
|
| 654 |
|
| 655 |
-
self.logger.
|
| 656 |
return {
|
| 657 |
'sand': None, 'silt': None, 'clay': None, 'soc': None, 'ph': None, 'cec': None,
|
| 658 |
'nitrogen_percent': 0.15, 'phosphorus_mg_kg': 12.0, 'potassium_mg_kg': 150.0,
|
|
@@ -929,49 +947,56 @@ class RealTimeFeatureExtractor:
|
|
| 929 |
]
|
| 930 |
return (sum(scores) / len(scores)) * 100
|
| 931 |
|
| 932 |
-
# ====================
|
| 933 |
|
| 934 |
def extract_12_months_history(self, latitude: float, longitude: float) -> Dict:
|
| 935 |
"""
|
| 936 |
-
|
|
|
|
| 937 |
"""
|
| 938 |
self.logger.info(f"📅 Extracting 12 months historical data for ({latitude:.4f}, {longitude:.4f})")
|
| 939 |
|
| 940 |
-
features_list = []
|
| 941 |
current_date = datetime.now()
|
| 942 |
|
| 943 |
-
# 🆕
|
|
|
|
| 944 |
weather_data = self.get_realtime_weather(latitude, longitude)
|
| 945 |
location_name = weather_data.get('location_name', 'Unknown') if weather_data else 'Unknown'
|
|
|
|
| 946 |
|
| 947 |
-
#
|
|
|
|
| 948 |
soil_year = current_date.year
|
| 949 |
soil_data = self.temporal_manager.get_historical_soil_data(latitude, longitude, soil_year)
|
|
|
|
| 950 |
|
| 951 |
bdod_data = self.temporal_manager.get_historical_bdod_data(latitude, longitude, soil_year)
|
| 952 |
|
| 953 |
-
#
|
| 954 |
npk_data = self._estimate_npk_nutrients(soil_data)
|
| 955 |
|
| 956 |
-
#
|
|
|
|
| 957 |
lc_type = self.get_landcover_type(latitude, longitude)
|
| 958 |
|
| 959 |
-
#
|
| 960 |
-
|
|
|
|
|
|
|
|
|
|
| 961 |
target_date = current_date.replace(day=15)
|
| 962 |
target_date = target_date - timedelta(days=30.44 * month_offset)
|
| 963 |
|
| 964 |
year = target_date.year
|
| 965 |
month = target_date.month
|
| 966 |
|
| 967 |
-
#
|
| 968 |
climate_data = self.temporal_manager.get_historical_climate(latitude, longitude, target_date)
|
| 969 |
|
| 970 |
-
#
|
| 971 |
ndvi_data = self.temporal_manager.get_historical_ndvi(latitude, longitude, target_date)
|
| 972 |
|
| 973 |
-
|
| 974 |
-
month_features = {
|
| 975 |
'timestamp': target_date.isoformat(),
|
| 976 |
'year': year,
|
| 977 |
'month': month,
|
|
@@ -979,7 +1004,7 @@ class RealTimeFeatureExtractor:
|
|
| 979 |
'longitude': longitude,
|
| 980 |
'location_name': location_name,
|
| 981 |
|
| 982 |
-
#
|
| 983 |
'sand': round(soil_data.get('sand'), 1) if soil_data.get('sand') else None,
|
| 984 |
'silt': round(soil_data.get('silt'), 1) if soil_data.get('silt') else None,
|
| 985 |
'clay': round(soil_data.get('clay'), 1) if soil_data.get('clay') else None,
|
|
@@ -989,17 +1014,17 @@ class RealTimeFeatureExtractor:
|
|
| 989 |
'soil_version': soil_data.get('_version_year'),
|
| 990 |
'bdod': round(bdod_data, 2) if bdod_data else None,
|
| 991 |
|
| 992 |
-
# NPK (
|
| 993 |
'nitrogen_percent': npk_data['nitrogen_percent'],
|
| 994 |
'phosphorus_mg_kg': npk_data['phosphorus_mg_kg'],
|
| 995 |
'potassium_mg_kg': npk_data['potassium_mg_kg'],
|
| 996 |
'npk_confidence': npk_data['npk_confidence'],
|
| 997 |
|
| 998 |
-
#
|
| 999 |
'ndvi': round(ndvi_data.get('ndvi'), 4) if ndvi_data.get('ndvi') else None,
|
| 1000 |
'ndvi_source': ndvi_data.get('source'),
|
| 1001 |
|
| 1002 |
-
#
|
| 1003 |
'temperature_c': round(climate_data.get('t2m_c'), 1) if climate_data.get('t2m_c') else None,
|
| 1004 |
'td2m_c': round(climate_data.get('td2m_c'), 1) if climate_data.get('td2m_c') else None,
|
| 1005 |
'relative_humidity_percent': round(climate_data.get('rh_pct'), 1) if climate_data.get('rh_pct') else None,
|
|
@@ -1007,27 +1032,47 @@ class RealTimeFeatureExtractor:
|
|
| 1007 |
'solar_radiation_j_m2': round(climate_data.get('ssrd_jm2'), 1) if climate_data.get('ssrd_jm2') else None,
|
| 1008 |
'weather_source': climate_data.get('weather_source'),
|
| 1009 |
|
| 1010 |
-
# Land Cover (
|
| 1011 |
'lc_type1': lc_type,
|
| 1012 |
'lc_source': 'ESA_WorldCover_v100_2020',
|
| 1013 |
|
| 1014 |
-
#
|
| 1015 |
'data_quality': 85.0,
|
| 1016 |
'soil_retrieval_method': 'Temporal_SoilGrids',
|
| 1017 |
'soil_buffer_used_m': soil_data.get('_scale_used')
|
| 1018 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1019 |
|
| 1020 |
-
|
| 1021 |
-
|
|
|
|
|
|
|
| 1022 |
|
| 1023 |
-
#
|
| 1024 |
if features_list:
|
| 1025 |
start_date = features_list[-1]['timestamp']
|
| 1026 |
end_date = features_list[0]['timestamp']
|
| 1027 |
else:
|
| 1028 |
start_date = end_date = current_date.isoformat()
|
| 1029 |
|
| 1030 |
-
self.logger.info(f"✅ Extracted {len(features_list)} months of historical data (fixed 12 months)")
|
| 1031 |
|
| 1032 |
return {
|
| 1033 |
'success': True,
|
|
@@ -1041,7 +1086,7 @@ class RealTimeFeatureExtractor:
|
|
| 1041 |
'end_date': end_date,
|
| 1042 |
'months_count': len(features_list),
|
| 1043 |
'note': 'Fixed 12 months: Current month + 11 previous months',
|
| 1044 |
-
'location': location_name # 🆕
|
| 1045 |
}
|
| 1046 |
}
|
| 1047 |
# ==================== FASTAPI APP ====================
|
|
@@ -1149,8 +1194,8 @@ async def extract_ml_features(request: MLFeaturesRequest):
|
|
| 1149 |
@app.post("/12-months-history", response_model=HistoricalFeaturesResponse)
|
| 1150 |
async def get_12_months_history(request: HistoricalFeaturesRequest):
|
| 1151 |
"""
|
| 1152 |
-
🎯 ENDPOINT
|
| 1153 |
-
|
| 1154 |
"""
|
| 1155 |
try:
|
| 1156 |
if not extractor:
|
|
@@ -1169,8 +1214,8 @@ async def get_12_months_history(request: HistoricalFeaturesRequest):
|
|
| 1169 |
@app.post("/12-months-history-ml")
|
| 1170 |
async def get_12_months_history_ml(request: HistoricalFeaturesRequest):
|
| 1171 |
"""
|
| 1172 |
-
🎯 ENDPOINT
|
| 1173 |
-
12
|
| 1174 |
"""
|
| 1175 |
try:
|
| 1176 |
if not extractor:
|
|
|
|
| 14 |
from datetime import timedelta
|
| 15 |
import json
|
| 16 |
from pathlib import Path
|
| 17 |
+
import threading
|
| 18 |
+
import concurrent.futures
|
| 19 |
|
| 20 |
# ==================== CONFIGURATION ====================
|
| 21 |
|
|
|
|
| 32 |
class HistoricalFeaturesRequest(BaseModel):
|
| 33 |
latitude: float
|
| 34 |
longitude: float
|
| 35 |
+
# Removed months_back because it is now fixed = 12 months
|
| 36 |
|
| 37 |
class HistoricalFeaturesResponse(BaseModel):
|
| 38 |
success: bool
|
|
|
|
| 99 |
# ==================== TEMPORAL DATA HANDLER ====================
|
| 100 |
|
| 101 |
class TemporalDataManager:
|
| 102 |
+
"""Historical Data Manager"""
|
| 103 |
|
| 104 |
def __init__(self):
|
| 105 |
self.logger = logging.getLogger(self.__class__.__name__)
|
|
|
|
| 108 |
self.cache_dir = Path("/tmp/temporal_cache")
|
| 109 |
self.cache_dir.mkdir(exist_ok=True)
|
| 110 |
self.soil_cache_file = self.cache_dir / "soil_cache.json"
|
| 111 |
+
|
| 112 |
+
# Thread safety
|
| 113 |
+
self._lock = threading.Lock()
|
| 114 |
+
|
| 115 |
self.soil_cache = self._load_soil_cache()
|
| 116 |
|
| 117 |
# Soil versions mapping
|
|
|
|
| 122 |
}
|
| 123 |
|
| 124 |
def _load_soil_cache(self):
|
| 125 |
+
with self._lock:
|
| 126 |
+
if self.soil_cache_file.exists():
|
| 127 |
+
try:
|
| 128 |
+
with open(self.soil_cache_file, 'r') as f:
|
| 129 |
+
return json.load(f)
|
| 130 |
+
except:
|
| 131 |
+
pass
|
| 132 |
+
return {}
|
| 133 |
|
| 134 |
def _save_soil_cache(self):
|
| 135 |
+
with self._lock:
|
| 136 |
+
with open(self.soil_cache_file, 'w') as f:
|
| 137 |
+
json.dump(self.soil_cache, f, indent=2)
|
| 138 |
|
| 139 |
def get_soil_version(self, year: int):
|
| 140 |
for (start, end), info in self.soil_versions.items():
|
|
|
|
| 145 |
def get_historical_soil_data(self, lat: float, lon: float, year: int) -> Dict:
|
| 146 |
cache_key = f"{lat:.3f}_{lon:.3f}_{year}"
|
| 147 |
|
| 148 |
+
with self._lock:
|
| 149 |
+
if cache_key in self.soil_cache:
|
| 150 |
+
self.logger.info(f"📦 Using cached temporal soil data for {year}")
|
| 151 |
+
return self.soil_cache[cache_key]
|
| 152 |
|
| 153 |
try:
|
| 154 |
point = ee.Geometry.Point([lon, lat])
|
|
|
|
| 166 |
|
| 167 |
for scale in [250, 500, 1000]:
|
| 168 |
try:
|
| 169 |
+
self.logger.info(f"⏳ Soil GEE Request: scale={scale}m")
|
| 170 |
soil_data = soil_image.reduceRegion(
|
| 171 |
reducer=ee.Reducer.mean(),
|
| 172 |
geometry=point,
|
|
|
|
| 174 |
bestEffort=True,
|
| 175 |
maxPixels=1e9
|
| 176 |
).getInfo()
|
| 177 |
+
self.logger.info(f"✅ Soil GEE Response: scale={scale}m received")
|
| 178 |
|
| 179 |
if soil_data.get('sand') is not None:
|
| 180 |
result = {
|
|
|
|
| 189 |
'_collection': version['collection']
|
| 190 |
}
|
| 191 |
|
| 192 |
+
with self._lock:
|
| 193 |
+
self.soil_cache[cache_key] = result
|
| 194 |
+
self._save_soil_cache()
|
| 195 |
|
| 196 |
self.logger.info(f"✅ Fetched temporal soil data for {year} (scale: {scale}m)")
|
| 197 |
return result
|
| 198 |
+
else:
|
| 199 |
+
self.logger.warning(f"⚠️ Soil data None for scale {scale}m")
|
| 200 |
|
| 201 |
except Exception as e:
|
| 202 |
self.logger.warning(f"⚠️ Soil scale {scale}m failed: {str(e)[:50]}")
|
|
|
|
| 283 |
'weather_source': 'ERA5_LAND_Historical'
|
| 284 |
}
|
| 285 |
|
| 286 |
+
# Format logger safely to avoid "NoneType" error
|
| 287 |
+
temp_log = f"{climate_data['t2m_c']:.1f}" if climate_data['t2m_c'] is not None else "None"
|
| 288 |
+
rh_log = f"{climate_data['rh_pct']:.1f}" if climate_data['rh_pct'] is not None else "None"
|
| 289 |
+
|
| 290 |
self.logger.info(f"✅ ERA5 climate for {date.year}-{date.month:02d}: "
|
| 291 |
+
f"Temp={temp_log}°C, RH={rh_log}%")
|
| 292 |
|
| 293 |
return climate_data
|
| 294 |
|
|
|
|
| 297 |
return self._get_climate_fallback(lat, date.month)
|
| 298 |
|
| 299 |
def get_historical_bdod_data(self, lat: float, lon: float, year: int) -> Optional[float]:
|
| 300 |
+
"""Fetch historical BDOD data"""
|
| 301 |
try:
|
| 302 |
point = ee.Geometry.Point([lon, lat])
|
| 303 |
version = self.get_soil_version(year)
|
| 304 |
|
| 305 |
+
# BDOD is in the same collection
|
| 306 |
bdod = ee.Image(f"{version['collection']}/bdod_mean").select("bdod_0-5cm_mean")
|
| 307 |
|
| 308 |
for scale in [250, 500, 1000]:
|
|
|
|
| 317 |
|
| 318 |
bdod_value = bdod_data.get('bdod_0-5cm_mean')
|
| 319 |
if bdod_value is not None:
|
| 320 |
+
bdod_kg_m3 = bdod_value * 10 # Conversion from cg/cm³ to kg/m³
|
| 321 |
+
bdod_g_cm3 = bdod_kg_m3 / 1000.0 # Conversion to g/cm³
|
| 322 |
self.logger.info(f"✅ BDOD for {year}: {bdod_g_cm3:.2f} g/cm³ (scale: {scale}m)")
|
| 323 |
return bdod_g_cm3
|
| 324 |
|
|
|
|
| 465 |
self.logger.error(f"❌ Earth Engine failed: {e}")
|
| 466 |
raise
|
| 467 |
|
| 468 |
+
# ==================== Current Functions (As Is) ====================
|
| 469 |
|
| 470 |
def compute_dewpoint(self, t_celsius, rh_percent):
|
| 471 |
try:
|
|
|
|
| 670 |
self.logger.warning(f"⚠️ Soil buffer {buffer_size}m failed: {str(e)[:50]}")
|
| 671 |
continue
|
| 672 |
|
| 673 |
+
self.logger.info("ℹ️ All soil retrieval methods failed (likely water body/ocean)")
|
| 674 |
return {
|
| 675 |
'sand': None, 'silt': None, 'clay': None, 'soc': None, 'ph': None, 'cec': None,
|
| 676 |
'nitrogen_percent': 0.15, 'phosphorus_mg_kg': 12.0, 'potassium_mg_kg': 150.0,
|
|
|
|
| 947 |
]
|
| 948 |
return (sum(scores) / len(scores)) * 100
|
| 949 |
|
| 950 |
+
# ==================== New Historical Data Functions (Fixed 12 Months) ====================
|
| 951 |
|
| 952 |
def extract_12_months_history(self, latitude: float, longitude: float) -> Dict:
|
| 953 |
"""
|
| 954 |
+
Get exactly 12 months: current month + 11 previous months
|
| 955 |
+
Parallelized for performance.
|
| 956 |
"""
|
| 957 |
self.logger.info(f"📅 Extracting 12 months historical data for ({latitude:.4f}, {longitude:.4f})")
|
| 958 |
|
|
|
|
| 959 |
current_date = datetime.now()
|
| 960 |
|
| 961 |
+
# 🆕 Get location name from current weather data
|
| 962 |
+
self.logger.info("⏳ M1: Fetching current weather for location name...")
|
| 963 |
weather_data = self.get_realtime_weather(latitude, longitude)
|
| 964 |
location_name = weather_data.get('location_name', 'Unknown') if weather_data else 'Unknown'
|
| 965 |
+
self.logger.info(f"✅ M1: Location found: {location_name}")
|
| 966 |
|
| 967 |
+
# Get soil data once (since it is fixed)
|
| 968 |
+
self.logger.info("⏳ M2: Fetching historical soil data...")
|
| 969 |
soil_year = current_date.year
|
| 970 |
soil_data = self.temporal_manager.get_historical_soil_data(latitude, longitude, soil_year)
|
| 971 |
+
self.logger.info("✅ M2: Historical soil fetched")
|
| 972 |
|
| 973 |
bdod_data = self.temporal_manager.get_historical_bdod_data(latitude, longitude, soil_year)
|
| 974 |
|
| 975 |
+
# Get NPK from soil
|
| 976 |
npk_data = self._estimate_npk_nutrients(soil_data)
|
| 977 |
|
| 978 |
+
# Get land cover once
|
| 979 |
+
self.logger.info("⏳ M3: Fetching land scale...")
|
| 980 |
lc_type = self.get_landcover_type(latitude, longitude)
|
| 981 |
|
| 982 |
+
# Define helper for parallel execution
|
| 983 |
+
def _fetch_single_month(month_offset):
|
| 984 |
+
# Log start of thread
|
| 985 |
+
# self.logger.info(f"🧵 Thread start: Offset {month_offset}")
|
| 986 |
+
|
| 987 |
target_date = current_date.replace(day=15)
|
| 988 |
target_date = target_date - timedelta(days=30.44 * month_offset)
|
| 989 |
|
| 990 |
year = target_date.year
|
| 991 |
month = target_date.month
|
| 992 |
|
| 993 |
+
# Get historical climate
|
| 994 |
climate_data = self.temporal_manager.get_historical_climate(latitude, longitude, target_date)
|
| 995 |
|
| 996 |
+
# Get historical NDVI
|
| 997 |
ndvi_data = self.temporal_manager.get_historical_ndvi(latitude, longitude, target_date)
|
| 998 |
|
| 999 |
+
return {
|
|
|
|
| 1000 |
'timestamp': target_date.isoformat(),
|
| 1001 |
'year': year,
|
| 1002 |
'month': month,
|
|
|
|
| 1004 |
'longitude': longitude,
|
| 1005 |
'location_name': location_name,
|
| 1006 |
|
| 1007 |
+
# Soil (same for all months)
|
| 1008 |
'sand': round(soil_data.get('sand'), 1) if soil_data.get('sand') else None,
|
| 1009 |
'silt': round(soil_data.get('silt'), 1) if soil_data.get('silt') else None,
|
| 1010 |
'clay': round(soil_data.get('clay'), 1) if soil_data.get('clay') else None,
|
|
|
|
| 1014 |
'soil_version': soil_data.get('_version_year'),
|
| 1015 |
'bdod': round(bdod_data, 2) if bdod_data else None,
|
| 1016 |
|
| 1017 |
+
# NPK (same for all months)
|
| 1018 |
'nitrogen_percent': npk_data['nitrogen_percent'],
|
| 1019 |
'phosphorus_mg_kg': npk_data['phosphorus_mg_kg'],
|
| 1020 |
'potassium_mg_kg': npk_data['potassium_mg_kg'],
|
| 1021 |
'npk_confidence': npk_data['npk_confidence'],
|
| 1022 |
|
| 1023 |
+
# Vegetation (variable)
|
| 1024 |
'ndvi': round(ndvi_data.get('ndvi'), 4) if ndvi_data.get('ndvi') else None,
|
| 1025 |
'ndvi_source': ndvi_data.get('source'),
|
| 1026 |
|
| 1027 |
+
# Climate (variable)
|
| 1028 |
'temperature_c': round(climate_data.get('t2m_c'), 1) if climate_data.get('t2m_c') else None,
|
| 1029 |
'td2m_c': round(climate_data.get('td2m_c'), 1) if climate_data.get('td2m_c') else None,
|
| 1030 |
'relative_humidity_percent': round(climate_data.get('rh_pct'), 1) if climate_data.get('rh_pct') else None,
|
|
|
|
| 1032 |
'solar_radiation_j_m2': round(climate_data.get('ssrd_jm2'), 1) if climate_data.get('ssrd_jm2') else None,
|
| 1033 |
'weather_source': climate_data.get('weather_source'),
|
| 1034 |
|
| 1035 |
+
# Land Cover (same for all months)
|
| 1036 |
'lc_type1': lc_type,
|
| 1037 |
'lc_source': 'ESA_WorldCover_v100_2020',
|
| 1038 |
|
| 1039 |
+
# Quality
|
| 1040 |
'data_quality': 85.0,
|
| 1041 |
'soil_retrieval_method': 'Temporal_SoilGrids',
|
| 1042 |
'soil_buffer_used_m': soil_data.get('_scale_used')
|
| 1043 |
}
|
| 1044 |
+
|
| 1045 |
+
# ✅ Execute in parallel using ThreadPoolExecutor (Reduced workers to 4 for stability)
|
| 1046 |
+
self.logger.info("🚀 Starting parallel extraction (4 workers)...")
|
| 1047 |
+
features_list = []
|
| 1048 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
| 1049 |
+
# Create tasks for 0 to 11
|
| 1050 |
+
future_to_offset = {executor.submit(_fetch_single_month, offset): offset for offset in range(12)}
|
| 1051 |
+
|
| 1052 |
+
# Collect results as they complete (but we want to order them locally)
|
| 1053 |
+
results_map = {}
|
| 1054 |
+
for future in concurrent.futures.as_completed(future_to_offset):
|
| 1055 |
+
offset = future_to_offset[future]
|
| 1056 |
+
try:
|
| 1057 |
+
data = future.result()
|
| 1058 |
+
results_map[offset] = data
|
| 1059 |
+
self.logger.info(f"✅ Extracted month {data['year']}-{data['month']:02d} (Offset {offset})")
|
| 1060 |
+
except Exception as e:
|
| 1061 |
+
self.logger.error(f"❌ Failed processing month offset {offset}: {e}")
|
| 1062 |
|
| 1063 |
+
# Reassemble in order (0 to 11)
|
| 1064 |
+
for offset in range(12):
|
| 1065 |
+
if offset in results_map:
|
| 1066 |
+
features_list.append(results_map[offset])
|
| 1067 |
|
| 1068 |
+
# Calculate timeframe
|
| 1069 |
if features_list:
|
| 1070 |
start_date = features_list[-1]['timestamp']
|
| 1071 |
end_date = features_list[0]['timestamp']
|
| 1072 |
else:
|
| 1073 |
start_date = end_date = current_date.isoformat()
|
| 1074 |
|
| 1075 |
+
self.logger.info(f"✅ Extracted {len(features_list)} months of historical data (fixed 12 months) in parallel")
|
| 1076 |
|
| 1077 |
return {
|
| 1078 |
'success': True,
|
|
|
|
| 1086 |
'end_date': end_date,
|
| 1087 |
'months_count': len(features_list),
|
| 1088 |
'note': 'Fixed 12 months: Current month + 11 previous months',
|
| 1089 |
+
'location': location_name # 🆕 Added here to be in metadata
|
| 1090 |
}
|
| 1091 |
}
|
| 1092 |
# ==================== FASTAPI APP ====================
|
|
|
|
| 1194 |
@app.post("/12-months-history", response_model=HistoricalFeaturesResponse)
|
| 1195 |
async def get_12_months_history(request: HistoricalFeaturesRequest):
|
| 1196 |
"""
|
| 1197 |
+
🎯 ENDPOINT for Historical Data - Fixed 12 Months
|
| 1198 |
+
Returns current month + 11 previous months
|
| 1199 |
"""
|
| 1200 |
try:
|
| 1201 |
if not extractor:
|
|
|
|
| 1214 |
@app.post("/12-months-history-ml")
|
| 1215 |
async def get_12_months_history_ml(request: HistoricalFeaturesRequest):
|
| 1216 |
"""
|
| 1217 |
+
🎯 ENDPOINT for ML Ready Historical Data
|
| 1218 |
+
Fixed 12 months
|
| 1219 |
"""
|
| 1220 |
try:
|
| 1221 |
if not extractor:
|