Spaces:
Sleeping
Sleeping
Upload realtime_pipeline.py
Browse files- realtime_pipeline.py +107 -16
realtime_pipeline.py
CHANGED
|
@@ -1,7 +1,4 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
import ee
|
| 4 |
-
from datetime import datetime
|
| 5 |
import math
|
| 6 |
import requests
|
| 7 |
import logging
|
|
@@ -12,6 +9,8 @@ import uvicorn
|
|
| 12 |
from contextlib import asynccontextmanager
|
| 13 |
from ee import ServiceAccountCredentials
|
| 14 |
import os
|
|
|
|
|
|
|
| 15 |
# ==================== CONFIGURATION ====================
|
| 16 |
|
| 17 |
class Config:
|
|
@@ -284,7 +283,74 @@ class RealTimeFeatureExtractor:
|
|
| 284 |
except Exception as e:
|
| 285 |
self.logger.error(f"❌ Soil extraction failed: {e}")
|
| 286 |
return None
|
| 287 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
def get_ndvi_data(self, latitude, longitude):
|
| 289 |
try:
|
| 290 |
point = ee.Geometry.Point([longitude, latitude])
|
|
@@ -316,10 +382,16 @@ class RealTimeFeatureExtractor:
|
|
| 316 |
|
| 317 |
query_time = datetime.now()
|
| 318 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
# Get all data sources
|
| 320 |
weather_data = self.get_realtime_weather(latitude, longitude)
|
| 321 |
soil_data = self.get_soil_data_with_buffer(latitude, longitude)
|
| 322 |
ndvi_data = self.get_ndvi_data(latitude, longitude)
|
|
|
|
|
|
|
| 323 |
|
| 324 |
# Calculate data quality
|
| 325 |
data_quality_score = self._calculate_data_quality(soil_data, ndvi_data, weather_data)
|
|
@@ -332,11 +404,17 @@ class RealTimeFeatureExtractor:
|
|
| 332 |
'query_timestamp': query_time.isoformat(),
|
| 333 |
'data_quality_score': data_quality_score,
|
| 334 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
# Soil Properties
|
| 336 |
'sand': soil_data.get('sand'),
|
|
|
|
| 337 |
'clay': soil_data.get('clay'),
|
| 338 |
'soc': soil_data.get('soc'),
|
| 339 |
'ph': soil_data.get('ph'),
|
|
|
|
| 340 |
'cec': soil_data.get('cec'),
|
| 341 |
|
| 342 |
# NPK Nutrients
|
|
@@ -351,10 +429,14 @@ class RealTimeFeatureExtractor:
|
|
| 351 |
|
| 352 |
# Climate
|
| 353 |
'temperature_c': weather_data.get('t2m_c') if weather_data else None,
|
|
|
|
| 354 |
'relative_humidity_percent': weather_data.get('rh_pct') if weather_data else None,
|
| 355 |
'precipitation_m': weather_data.get('tp_m') if weather_data else None,
|
| 356 |
'solar_radiation_j_m2': weather_data.get('ssrd_jm2') if weather_data else None,
|
| 357 |
|
|
|
|
|
|
|
|
|
|
| 358 |
# Metadata
|
| 359 |
'soil_retrieval_method': soil_data.get('soil_retrieval_method'),
|
| 360 |
'soil_buffer_used_m': soil_data.get('soil_buffer_used_m'),
|
|
@@ -363,7 +445,7 @@ class RealTimeFeatureExtractor:
|
|
| 363 |
|
| 364 |
self.logger.info("✅ Feature extraction completed")
|
| 365 |
return features
|
| 366 |
-
|
| 367 |
def _calculate_data_quality(self, soil_data, ndvi_data, weather_data) -> float:
|
| 368 |
scores = [
|
| 369 |
1.0 if soil_data and soil_data.get('sand') is not None else 0.0,
|
|
@@ -434,28 +516,38 @@ async def extract_ml_features(request: MLFeaturesRequest):
|
|
| 434 |
longitude=request.longitude
|
| 435 |
)
|
| 436 |
|
| 437 |
-
# Format specifically for ML models
|
| 438 |
ml_ready_features = {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 439 |
# Soil Features
|
| 440 |
'sand': features.get('sand'),
|
|
|
|
| 441 |
'clay': features.get('clay'),
|
| 442 |
'soc': features.get('soc'),
|
| 443 |
'ph': features.get('ph'),
|
|
|
|
| 444 |
'cec': features.get('cec'),
|
| 445 |
|
| 446 |
-
# NPK Nutrients
|
| 447 |
-
'nitrogen': features.get('nitrogen_percent'),
|
| 448 |
-
'phosphorus': features.get('phosphorus_mg_kg'),
|
| 449 |
-
'potassium': features.get('potassium_mg_kg'),
|
| 450 |
-
|
| 451 |
# Vegetation
|
| 452 |
'ndvi': features.get('ndvi'),
|
| 453 |
|
| 454 |
# Climate
|
| 455 |
-
'
|
| 456 |
-
'
|
| 457 |
-
'
|
| 458 |
-
'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 459 |
|
| 460 |
# Location
|
| 461 |
'latitude': features.get('latitude'),
|
|
@@ -476,7 +568,6 @@ async def extract_ml_features(request: MLFeaturesRequest):
|
|
| 476 |
|
| 477 |
except Exception as e:
|
| 478 |
raise HTTPException(status_code=500, detail=str(e))
|
| 479 |
-
|
| 480 |
@app.get("/health")
|
| 481 |
async def health_check():
|
| 482 |
return {
|
|
|
|
|
|
|
|
|
|
| 1 |
import ee
|
|
|
|
| 2 |
import math
|
| 3 |
import requests
|
| 4 |
import logging
|
|
|
|
| 9 |
from contextlib import asynccontextmanager
|
| 10 |
from ee import ServiceAccountCredentials
|
| 11 |
import os
|
| 12 |
+
from datetime import datetime, timedelta
|
| 13 |
+
import calendar
|
| 14 |
# ==================== CONFIGURATION ====================
|
| 15 |
|
| 16 |
class Config:
|
|
|
|
| 283 |
except Exception as e:
|
| 284 |
self.logger.error(f"❌ Soil extraction failed: {e}")
|
| 285 |
return None
|
| 286 |
+
def get_bdod_data(self, latitude, longitude):
|
| 287 |
+
"""Get soil bulk density"""
|
| 288 |
+
try:
|
| 289 |
+
point = ee.Geometry.Point([longitude, latitude])
|
| 290 |
+
bdod = ee.Image("projects/soilgrids-isric/bdod_mean").select("bdod_0-5cm_mean")
|
| 291 |
+
|
| 292 |
+
bdod_data = bdod.reduceRegion(
|
| 293 |
+
reducer=ee.Reducer.mean(),
|
| 294 |
+
geometry=point,
|
| 295 |
+
scale=250,
|
| 296 |
+
bestEffort=True
|
| 297 |
+
)
|
| 298 |
+
|
| 299 |
+
bdod_value = bdod_data.get('bdod_0-5cm_mean').getInfo()
|
| 300 |
+
if bdod_value:
|
| 301 |
+
# Convert from cg/cm³ to kg/m³
|
| 302 |
+
bdod_kg_m3 = bdod_value * 10
|
| 303 |
+
self.logger.info(f"✅ BDOD: {bdod_kg_m3:.1f} kg/m³")
|
| 304 |
+
return bdod_kg_m3
|
| 305 |
+
return None
|
| 306 |
+
except Exception as e:
|
| 307 |
+
self.logger.error(f"❌ BDOD failed: {e}")
|
| 308 |
+
return None
|
| 309 |
+
def get_landcover_type(self, latitude, longitude):
|
| 310 |
+
"""Get MODIS land cover classification"""
|
| 311 |
+
try:
|
| 312 |
+
point = ee.Geometry.Point([longitude, latitude])
|
| 313 |
+
|
| 314 |
+
# MODIS Land Cover Type 1 (IGBP classification)
|
| 315 |
+
lc_col = ee.ImageCollection('MODIS/061/MCD12Q1').select('LC_Type1')
|
| 316 |
+
|
| 317 |
+
# Get latest available year
|
| 318 |
+
lc_latest = lc_col.sort('system:time_start', False).first()
|
| 319 |
+
|
| 320 |
+
lc_value = lc_latest.reduceRegion(
|
| 321 |
+
reducer=ee.Reducer.mode(),
|
| 322 |
+
geometry=point,
|
| 323 |
+
scale=500
|
| 324 |
+
).get('LC_Type1').getInfo()
|
| 325 |
+
|
| 326 |
+
# IGBP classification codes
|
| 327 |
+
lc_classes = {
|
| 328 |
+
1: "Evergreen Needleleaf Forest",
|
| 329 |
+
2: "Evergreen Broadleaf Forest",
|
| 330 |
+
3: "Deciduous Needleleaf Forest",
|
| 331 |
+
4: "Deciduous Broadleaf Forest",
|
| 332 |
+
5: "Mixed Forests",
|
| 333 |
+
6: "Closed Shrublands",
|
| 334 |
+
7: "Open Shrublands",
|
| 335 |
+
8: "Woody Savannas",
|
| 336 |
+
9: "Savannas",
|
| 337 |
+
10: "Grasslands",
|
| 338 |
+
11: "Permanent Wetlands",
|
| 339 |
+
12: "Croplands",
|
| 340 |
+
13: "Urban and Built-up",
|
| 341 |
+
14: "Cropland/Natural Vegetation Mosaic",
|
| 342 |
+
15: "Snow and Ice",
|
| 343 |
+
16: "Barren or Sparsely Vegetated",
|
| 344 |
+
17: "Water Bodies"
|
| 345 |
+
}
|
| 346 |
+
|
| 347 |
+
if lc_value in lc_classes:
|
| 348 |
+
self.logger.info(f"✅ Land Cover: {lc_classes[lc_value]}")
|
| 349 |
+
return lc_value # Return the code number
|
| 350 |
+
return 16 # Default: Barren or Sparsely Vegetated
|
| 351 |
+
except Exception as e:
|
| 352 |
+
self.logger.error(f"❌ Land cover failed: {e}")
|
| 353 |
+
return 16 # Default
|
| 354 |
def get_ndvi_data(self, latitude, longitude):
|
| 355 |
try:
|
| 356 |
point = ee.Geometry.Point([longitude, latitude])
|
|
|
|
| 382 |
|
| 383 |
query_time = datetime.now()
|
| 384 |
|
| 385 |
+
# Get current year and month
|
| 386 |
+
year = query_time.year
|
| 387 |
+
month = query_time.month
|
| 388 |
+
|
| 389 |
# Get all data sources
|
| 390 |
weather_data = self.get_realtime_weather(latitude, longitude)
|
| 391 |
soil_data = self.get_soil_data_with_buffer(latitude, longitude)
|
| 392 |
ndvi_data = self.get_ndvi_data(latitude, longitude)
|
| 393 |
+
bdod_data = self.get_bdod_data(latitude, longitude)
|
| 394 |
+
lc_type = self.get_landcover_type(latitude, longitude)
|
| 395 |
|
| 396 |
# Calculate data quality
|
| 397 |
data_quality_score = self._calculate_data_quality(soil_data, ndvi_data, weather_data)
|
|
|
|
| 404 |
'query_timestamp': query_time.isoformat(),
|
| 405 |
'data_quality_score': data_quality_score,
|
| 406 |
|
| 407 |
+
# Year and Month
|
| 408 |
+
'year': year,
|
| 409 |
+
'month': month,
|
| 410 |
+
|
| 411 |
# Soil Properties
|
| 412 |
'sand': soil_data.get('sand'),
|
| 413 |
+
'silt': soil_data.get('silt'), # Add silt
|
| 414 |
'clay': soil_data.get('clay'),
|
| 415 |
'soc': soil_data.get('soc'),
|
| 416 |
'ph': soil_data.get('ph'),
|
| 417 |
+
'bdod': bdod_data, # Add bdod
|
| 418 |
'cec': soil_data.get('cec'),
|
| 419 |
|
| 420 |
# NPK Nutrients
|
|
|
|
| 429 |
|
| 430 |
# Climate
|
| 431 |
'temperature_c': weather_data.get('t2m_c') if weather_data else None,
|
| 432 |
+
'td2m_c': weather_data.get('td2m_c') if weather_data else None, # Add dew point
|
| 433 |
'relative_humidity_percent': weather_data.get('rh_pct') if weather_data else None,
|
| 434 |
'precipitation_m': weather_data.get('tp_m') if weather_data else None,
|
| 435 |
'solar_radiation_j_m2': weather_data.get('ssrd_jm2') if weather_data else None,
|
| 436 |
|
| 437 |
+
# Land Cover
|
| 438 |
+
'lc_type1': lc_type, # Add land cover type
|
| 439 |
+
|
| 440 |
# Metadata
|
| 441 |
'soil_retrieval_method': soil_data.get('soil_retrieval_method'),
|
| 442 |
'soil_buffer_used_m': soil_data.get('soil_buffer_used_m'),
|
|
|
|
| 445 |
|
| 446 |
self.logger.info("✅ Feature extraction completed")
|
| 447 |
return features
|
| 448 |
+
|
| 449 |
def _calculate_data_quality(self, soil_data, ndvi_data, weather_data) -> float:
|
| 450 |
scores = [
|
| 451 |
1.0 if soil_data and soil_data.get('sand') is not None else 0.0,
|
|
|
|
| 516 |
longitude=request.longitude
|
| 517 |
)
|
| 518 |
|
| 519 |
+
# Format specifically for ML models
|
| 520 |
ml_ready_features = {
|
| 521 |
+
# Time
|
| 522 |
+
'year': features.get('year'),
|
| 523 |
+
'month': features.get('month'),
|
| 524 |
+
|
| 525 |
# Soil Features
|
| 526 |
'sand': features.get('sand'),
|
| 527 |
+
'silt': features.get('silt'), # Add silt
|
| 528 |
'clay': features.get('clay'),
|
| 529 |
'soc': features.get('soc'),
|
| 530 |
'ph': features.get('ph'),
|
| 531 |
+
'bdod': features.get('bdod'), # Add bdod
|
| 532 |
'cec': features.get('cec'),
|
| 533 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 534 |
# Vegetation
|
| 535 |
'ndvi': features.get('ndvi'),
|
| 536 |
|
| 537 |
# Climate
|
| 538 |
+
't2m_c': features.get('temperature_c'),
|
| 539 |
+
'td2m_c': features.get('td2m_c'), # Add dew point
|
| 540 |
+
'rh_pct': features.get('relative_humidity_percent'),
|
| 541 |
+
'tp_m': features.get('precipitation_m'),
|
| 542 |
+
'ssrd_jm2': features.get('solar_radiation_j_m2'),
|
| 543 |
+
|
| 544 |
+
# Land Cover
|
| 545 |
+
'lc_type1': features.get('lc_type1'), # Add land cover
|
| 546 |
+
|
| 547 |
+
# NPK Nutrients
|
| 548 |
+
'nitrogen': features.get('nitrogen_percent'),
|
| 549 |
+
'phosphorus': features.get('phosphorus_mg_kg'),
|
| 550 |
+
'potassium': features.get('potassium_mg_kg'),
|
| 551 |
|
| 552 |
# Location
|
| 553 |
'latitude': features.get('latitude'),
|
|
|
|
| 568 |
|
| 569 |
except Exception as e:
|
| 570 |
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
| 571 |
@app.get("/health")
|
| 572 |
async def health_check():
|
| 573 |
return {
|