radwaamr1 commited on
Commit
7367b80
·
verified ·
1 Parent(s): 340e2cc

Upload realtime_pipeline.py

Browse files
Files changed (1) hide show
  1. realtime_pipeline.py +107 -16
realtime_pipeline.py CHANGED
@@ -1,7 +1,4 @@
1
-
2
-
3
  import ee
4
- from datetime import datetime
5
  import math
6
  import requests
7
  import logging
@@ -12,6 +9,8 @@ import uvicorn
12
  from contextlib import asynccontextmanager
13
  from ee import ServiceAccountCredentials
14
  import os
 
 
15
  # ==================== CONFIGURATION ====================
16
 
17
  class Config:
@@ -284,7 +283,74 @@ class RealTimeFeatureExtractor:
284
  except Exception as e:
285
  self.logger.error(f"❌ Soil extraction failed: {e}")
286
  return None
287
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
288
  def get_ndvi_data(self, latitude, longitude):
289
  try:
290
  point = ee.Geometry.Point([longitude, latitude])
@@ -316,10 +382,16 @@ class RealTimeFeatureExtractor:
316
 
317
  query_time = datetime.now()
318
 
 
 
 
 
319
  # Get all data sources
320
  weather_data = self.get_realtime_weather(latitude, longitude)
321
  soil_data = self.get_soil_data_with_buffer(latitude, longitude)
322
  ndvi_data = self.get_ndvi_data(latitude, longitude)
 
 
323
 
324
  # Calculate data quality
325
  data_quality_score = self._calculate_data_quality(soil_data, ndvi_data, weather_data)
@@ -332,11 +404,17 @@ class RealTimeFeatureExtractor:
332
  'query_timestamp': query_time.isoformat(),
333
  'data_quality_score': data_quality_score,
334
 
 
 
 
 
335
  # Soil Properties
336
  'sand': soil_data.get('sand'),
 
337
  'clay': soil_data.get('clay'),
338
  'soc': soil_data.get('soc'),
339
  'ph': soil_data.get('ph'),
 
340
  'cec': soil_data.get('cec'),
341
 
342
  # NPK Nutrients
@@ -351,10 +429,14 @@ class RealTimeFeatureExtractor:
351
 
352
  # Climate
353
  'temperature_c': weather_data.get('t2m_c') if weather_data else None,
 
354
  'relative_humidity_percent': weather_data.get('rh_pct') if weather_data else None,
355
  'precipitation_m': weather_data.get('tp_m') if weather_data else None,
356
  'solar_radiation_j_m2': weather_data.get('ssrd_jm2') if weather_data else None,
357
 
 
 
 
358
  # Metadata
359
  'soil_retrieval_method': soil_data.get('soil_retrieval_method'),
360
  'soil_buffer_used_m': soil_data.get('soil_buffer_used_m'),
@@ -363,7 +445,7 @@ class RealTimeFeatureExtractor:
363
 
364
  self.logger.info("✅ Feature extraction completed")
365
  return features
366
-
367
  def _calculate_data_quality(self, soil_data, ndvi_data, weather_data) -> float:
368
  scores = [
369
  1.0 if soil_data and soil_data.get('sand') is not None else 0.0,
@@ -434,28 +516,38 @@ async def extract_ml_features(request: MLFeaturesRequest):
434
  longitude=request.longitude
435
  )
436
 
437
- # Format specifically for ML models
438
  ml_ready_features = {
 
 
 
 
439
  # Soil Features
440
  'sand': features.get('sand'),
 
441
  'clay': features.get('clay'),
442
  'soc': features.get('soc'),
443
  'ph': features.get('ph'),
 
444
  'cec': features.get('cec'),
445
 
446
- # NPK Nutrients
447
- 'nitrogen': features.get('nitrogen_percent'),
448
- 'phosphorus': features.get('phosphorus_mg_kg'),
449
- 'potassium': features.get('potassium_mg_kg'),
450
-
451
  # Vegetation
452
  'ndvi': features.get('ndvi'),
453
 
454
  # Climate
455
- 'temperature': features.get('temperature_c'),
456
- 'humidity': features.get('relative_humidity_percent'),
457
- 'precipitation': features.get('precipitation_m'),
458
- 'solar_radiation': features.get('solar_radiation_j_m2'),
 
 
 
 
 
 
 
 
 
459
 
460
  # Location
461
  'latitude': features.get('latitude'),
@@ -476,7 +568,6 @@ async def extract_ml_features(request: MLFeaturesRequest):
476
 
477
  except Exception as e:
478
  raise HTTPException(status_code=500, detail=str(e))
479
-
480
  @app.get("/health")
481
  async def health_check():
482
  return {
 
 
 
1
  import ee
 
2
  import math
3
  import requests
4
  import logging
 
9
  from contextlib import asynccontextmanager
10
  from ee import ServiceAccountCredentials
11
  import os
12
+ from datetime import datetime, timedelta
13
+ import calendar
14
  # ==================== CONFIGURATION ====================
15
 
16
  class Config:
 
283
  except Exception as e:
284
  self.logger.error(f"❌ Soil extraction failed: {e}")
285
  return None
286
+ def get_bdod_data(self, latitude, longitude):
287
+ """Get soil bulk density"""
288
+ try:
289
+ point = ee.Geometry.Point([longitude, latitude])
290
+ bdod = ee.Image("projects/soilgrids-isric/bdod_mean").select("bdod_0-5cm_mean")
291
+
292
+ bdod_data = bdod.reduceRegion(
293
+ reducer=ee.Reducer.mean(),
294
+ geometry=point,
295
+ scale=250,
296
+ bestEffort=True
297
+ )
298
+
299
+ bdod_value = bdod_data.get('bdod_0-5cm_mean').getInfo()
300
+ if bdod_value:
301
+ # Convert from cg/cm³ to kg/m³
302
+ bdod_kg_m3 = bdod_value * 10
303
+ self.logger.info(f"✅ BDOD: {bdod_kg_m3:.1f} kg/m³")
304
+ return bdod_kg_m3
305
+ return None
306
+ except Exception as e:
307
+ self.logger.error(f"❌ BDOD failed: {e}")
308
+ return None
309
+ def get_landcover_type(self, latitude, longitude):
310
+ """Get MODIS land cover classification"""
311
+ try:
312
+ point = ee.Geometry.Point([longitude, latitude])
313
+
314
+ # MODIS Land Cover Type 1 (IGBP classification)
315
+ lc_col = ee.ImageCollection('MODIS/061/MCD12Q1').select('LC_Type1')
316
+
317
+ # Get latest available year
318
+ lc_latest = lc_col.sort('system:time_start', False).first()
319
+
320
+ lc_value = lc_latest.reduceRegion(
321
+ reducer=ee.Reducer.mode(),
322
+ geometry=point,
323
+ scale=500
324
+ ).get('LC_Type1').getInfo()
325
+
326
+ # IGBP classification codes
327
+ lc_classes = {
328
+ 1: "Evergreen Needleleaf Forest",
329
+ 2: "Evergreen Broadleaf Forest",
330
+ 3: "Deciduous Needleleaf Forest",
331
+ 4: "Deciduous Broadleaf Forest",
332
+ 5: "Mixed Forests",
333
+ 6: "Closed Shrublands",
334
+ 7: "Open Shrublands",
335
+ 8: "Woody Savannas",
336
+ 9: "Savannas",
337
+ 10: "Grasslands",
338
+ 11: "Permanent Wetlands",
339
+ 12: "Croplands",
340
+ 13: "Urban and Built-up",
341
+ 14: "Cropland/Natural Vegetation Mosaic",
342
+ 15: "Snow and Ice",
343
+ 16: "Barren or Sparsely Vegetated",
344
+ 17: "Water Bodies"
345
+ }
346
+
347
+ if lc_value in lc_classes:
348
+ self.logger.info(f"✅ Land Cover: {lc_classes[lc_value]}")
349
+ return lc_value # Return the code number
350
+ return 16 # Default: Barren or Sparsely Vegetated
351
+ except Exception as e:
352
+ self.logger.error(f"❌ Land cover failed: {e}")
353
+ return 16 # Default
354
  def get_ndvi_data(self, latitude, longitude):
355
  try:
356
  point = ee.Geometry.Point([longitude, latitude])
 
382
 
383
  query_time = datetime.now()
384
 
385
+ # Get current year and month
386
+ year = query_time.year
387
+ month = query_time.month
388
+
389
  # Get all data sources
390
  weather_data = self.get_realtime_weather(latitude, longitude)
391
  soil_data = self.get_soil_data_with_buffer(latitude, longitude)
392
  ndvi_data = self.get_ndvi_data(latitude, longitude)
393
+ bdod_data = self.get_bdod_data(latitude, longitude)
394
+ lc_type = self.get_landcover_type(latitude, longitude)
395
 
396
  # Calculate data quality
397
  data_quality_score = self._calculate_data_quality(soil_data, ndvi_data, weather_data)
 
404
  'query_timestamp': query_time.isoformat(),
405
  'data_quality_score': data_quality_score,
406
 
407
+ # Year and Month
408
+ 'year': year,
409
+ 'month': month,
410
+
411
  # Soil Properties
412
  'sand': soil_data.get('sand'),
413
+ 'silt': soil_data.get('silt'), # Add silt
414
  'clay': soil_data.get('clay'),
415
  'soc': soil_data.get('soc'),
416
  'ph': soil_data.get('ph'),
417
+ 'bdod': bdod_data, # Add bdod
418
  'cec': soil_data.get('cec'),
419
 
420
  # NPK Nutrients
 
429
 
430
  # Climate
431
  'temperature_c': weather_data.get('t2m_c') if weather_data else None,
432
+ 'td2m_c': weather_data.get('td2m_c') if weather_data else None, # Add dew point
433
  'relative_humidity_percent': weather_data.get('rh_pct') if weather_data else None,
434
  'precipitation_m': weather_data.get('tp_m') if weather_data else None,
435
  'solar_radiation_j_m2': weather_data.get('ssrd_jm2') if weather_data else None,
436
 
437
+ # Land Cover
438
+ 'lc_type1': lc_type, # Add land cover type
439
+
440
  # Metadata
441
  'soil_retrieval_method': soil_data.get('soil_retrieval_method'),
442
  'soil_buffer_used_m': soil_data.get('soil_buffer_used_m'),
 
445
 
446
  self.logger.info("✅ Feature extraction completed")
447
  return features
448
+
449
  def _calculate_data_quality(self, soil_data, ndvi_data, weather_data) -> float:
450
  scores = [
451
  1.0 if soil_data and soil_data.get('sand') is not None else 0.0,
 
516
  longitude=request.longitude
517
  )
518
 
519
+ # Format specifically for ML models
520
  ml_ready_features = {
521
+ # Time
522
+ 'year': features.get('year'),
523
+ 'month': features.get('month'),
524
+
525
  # Soil Features
526
  'sand': features.get('sand'),
527
+ 'silt': features.get('silt'), # Add silt
528
  'clay': features.get('clay'),
529
  'soc': features.get('soc'),
530
  'ph': features.get('ph'),
531
+ 'bdod': features.get('bdod'), # Add bdod
532
  'cec': features.get('cec'),
533
 
 
 
 
 
 
534
  # Vegetation
535
  'ndvi': features.get('ndvi'),
536
 
537
  # Climate
538
+ 't2m_c': features.get('temperature_c'),
539
+ 'td2m_c': features.get('td2m_c'), # Add dew point
540
+ 'rh_pct': features.get('relative_humidity_percent'),
541
+ 'tp_m': features.get('precipitation_m'),
542
+ 'ssrd_jm2': features.get('solar_radiation_j_m2'),
543
+
544
+ # Land Cover
545
+ 'lc_type1': features.get('lc_type1'), # Add land cover
546
+
547
+ # NPK Nutrients
548
+ 'nitrogen': features.get('nitrogen_percent'),
549
+ 'phosphorus': features.get('phosphorus_mg_kg'),
550
+ 'potassium': features.get('potassium_mg_kg'),
551
 
552
  # Location
553
  'latitude': features.get('latitude'),
 
568
 
569
  except Exception as e:
570
  raise HTTPException(status_code=500, detail=str(e))
 
571
  @app.get("/health")
572
  async def health_check():
573
  return {