radwaamr1 commited on
Commit
fb4a645
·
verified ·
1 Parent(s): 663f5c5

Upload realtime_pipeline.py

Browse files
Files changed (1) hide show
  1. realtime_pipeline.py +69 -123
realtime_pipeline.py CHANGED
@@ -1,5 +1,3 @@
1
-
2
-
3
  import ee
4
  from datetime import datetime
5
  import math
@@ -9,9 +7,10 @@ from typing import Dict, Optional
9
  from fastapi import FastAPI, HTTPException
10
  from pydantic import BaseModel
11
  import uvicorn
12
- from contextlib import asynccontextmanager
13
  from ee import ServiceAccountCredentials
14
  import os
 
 
15
  # ==================== CONFIGURATION ====================
16
 
17
  class Config:
@@ -33,11 +32,11 @@ class FeatureResponse(BaseModel):
33
  data_quality_score: float
34
 
35
  # Soil Properties
36
- sand: Optional[float] # g/kg
37
- clay: Optional[float] # g/kg
38
- soc: Optional[float] # cg/kg
39
- ph: Optional[float] # pH * 10
40
- cec: Optional[float] # mmol(c)/kg
41
 
42
  # NPK Nutrients
43
  nitrogen_percent: Optional[float]
@@ -60,8 +59,6 @@ class FeatureResponse(BaseModel):
60
  soil_buffer_used_m: Optional[int]
61
  weather_source: Optional[str]
62
 
63
- # ==================== ML ENGINEERS MODELS ====================
64
-
65
  class MLFeaturesRequest(BaseModel):
66
  latitude: float
67
  longitude: float
@@ -73,22 +70,6 @@ class MLFeaturesResponse(BaseModel):
73
 
74
  # ==================== REAL-TIME EXTRACTOR ====================
75
 
76
- import os
77
- import base64
78
- import ee
79
- from ee import ServiceAccountCredentials
80
-
81
- # Load GEE key from Hugging Face secret
82
- if "GEE_KEY_BASE64" in os.environ:
83
- key_b64 = os.environ["GEE_KEY_BASE64"]
84
- key_json = base64.b64decode(key_b64)
85
- key_path = "/tmp/gee-key.json"
86
- with open(key_path, "wb") as f:
87
- f.write(key_json)
88
- os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = key_path
89
-
90
-
91
-
92
  class RealTimeFeatureExtractor:
93
  def __init__(self):
94
  self.logger = self._setup_logging()
@@ -103,14 +84,12 @@ class RealTimeFeatureExtractor:
103
  if "GEE_KEY_BASE64" not in os.environ:
104
  raise Exception("GEE_KEY_BASE64 secret not found")
105
 
106
- # Decode secret and save as temporary JSON
107
  key_b64 = os.environ["GEE_KEY_BASE64"]
108
  key_json = base64.b64decode(key_b64)
109
  key_path = "/tmp/gee-key.json"
110
  with open(key_path, "wb") as f:
111
  f.write(key_json)
112
 
113
- # Use ServiceAccountCredentials
114
  credentials = ServiceAccountCredentials(None, key_file=key_path)
115
  ee.Initialize(credentials, project=Config.GEE_PROJECT_ID)
116
 
@@ -119,6 +98,7 @@ class RealTimeFeatureExtractor:
119
  self.logger.error(f"❌ Earth Engine failed: {e}")
120
  raise
121
 
 
122
  def compute_dewpoint(self, t_celsius, rh_percent):
123
  try:
124
  if t_celsius is None or rh_percent is None: return None
@@ -374,120 +354,86 @@ class RealTimeFeatureExtractor:
374
 
375
  # ==================== FASTAPI APP ====================
376
 
377
- extractor = None
378
-
379
- @asynccontextmanager
380
- async def lifespan(app: FastAPI):
381
- # Startup
382
- global extractor
383
- extractor = RealTimeFeatureExtractor()
384
- yield
385
- # Shutdown - cleanup if needed
386
-
387
  app = FastAPI(
388
  title="Real-Time Feature Extraction API",
389
  version="1.0.0",
390
  docs_url="/docs",
391
  redoc_url="/redoc",
392
- lifespan=lifespan
393
  )
394
 
395
- @app.get("/")
396
- async def root():
 
 
 
397
  return {
398
- "message": "Real-Time Feature Extraction API",
399
- "status": "running",
400
- "endpoints": {
401
- "health": "/health",
402
- "extract_features": "/extract-features",
403
- "ml_features": "/extract-ml-features"
404
- }
405
  }
406
 
407
  @app.post("/extract-features", response_model=FeatureResponse)
408
  async def extract_features(request: CoordinateRequest):
409
- """Extract real-time features for coordinates"""
410
- try:
411
- if not extractor:
412
- raise HTTPException(status_code=500, detail="Service not initialized")
413
-
414
- features = extractor.extract_features(
415
- latitude=request.latitude,
416
- longitude=request.longitude
417
- )
418
- return FeatureResponse(**features)
419
- except Exception as e:
420
- raise HTTPException(status_code=500, detail=str(e))
421
 
422
  @app.post("/extract-ml-features", response_model=MLFeaturesResponse)
423
  async def extract_ml_features(request: MLFeaturesRequest):
424
- """
425
- 🎯 YOUR MAIN ENDPOINT for ML Engineers
426
- This is what ML engineers will call to get features for their models
427
- """
428
- try:
429
- if not extractor:
430
- raise HTTPException(status_code=500, detail="Service not initialized")
431
-
432
- features = extractor.extract_features(
433
- latitude=request.latitude,
434
- longitude=request.longitude
435
- )
436
-
437
- # Format specifically for ML models
438
- ml_ready_features = {
439
- # Soil Features
440
- 'sand': features.get('sand'),
441
- 'clay': features.get('clay'),
442
- 'soc': features.get('soc'),
443
- 'ph': features.get('ph'),
444
- 'cec': features.get('cec'),
445
-
446
- # NPK Nutrients
447
- 'nitrogen': features.get('nitrogen_percent'),
448
- 'phosphorus': features.get('phosphorus_mg_kg'),
449
- 'potassium': features.get('potassium_mg_kg'),
450
-
451
- # Vegetation
452
- 'ndvi': features.get('ndvi'),
453
-
454
- # Climate
455
- 'temperature': features.get('temperature_c'),
456
- 'humidity': features.get('relative_humidity_percent'),
457
- 'precipitation': features.get('precipitation_m'),
458
- 'solar_radiation': features.get('solar_radiation_j_m2'),
459
-
460
- # Location
461
- 'latitude': features.get('latitude'),
462
- 'longitude': features.get('longitude')
463
- }
464
-
465
- return MLFeaturesResponse(
466
- success=True,
467
- features=ml_ready_features,
468
- metadata={
469
- 'location_name': features.get('location_name'),
470
- 'query_timestamp': features.get('query_timestamp'),
471
- 'data_quality': features.get('data_quality_score'),
472
- 'soil_retrieval_method': features.get('soil_retrieval_method'),
473
- 'npk_confidence': features.get('npk_confidence')
474
- }
475
- )
476
-
477
- except Exception as e:
478
- raise HTTPException(status_code=500, detail=str(e))
479
-
480
- @app.get("/health")
481
- async def health_check():
482
- return {
483
- "status": "healthy",
484
- "timestamp": datetime.now().isoformat(),
485
- "service": "feature-extraction"
486
  }
 
 
 
 
 
 
 
 
 
 
 
 
487
 
488
  if __name__ == "__main__":
489
  uvicorn.run(
490
  "realtime_pipeline:app",
491
  host="0.0.0.0",
492
  port=8080,
493
- )
 
 
 
1
  import ee
2
  from datetime import datetime
3
  import math
 
7
  from fastapi import FastAPI, HTTPException
8
  from pydantic import BaseModel
9
  import uvicorn
 
10
  from ee import ServiceAccountCredentials
11
  import os
12
+ import base64
13
+
14
  # ==================== CONFIGURATION ====================
15
 
16
  class Config:
 
32
  data_quality_score: float
33
 
34
  # Soil Properties
35
+ sand: Optional[float]
36
+ clay: Optional[float]
37
+ soc: Optional[float]
38
+ ph: Optional[float]
39
+ cec: Optional[float]
40
 
41
  # NPK Nutrients
42
  nitrogen_percent: Optional[float]
 
59
  soil_buffer_used_m: Optional[int]
60
  weather_source: Optional[str]
61
 
 
 
62
  class MLFeaturesRequest(BaseModel):
63
  latitude: float
64
  longitude: float
 
70
 
71
  # ==================== REAL-TIME EXTRACTOR ====================
72
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  class RealTimeFeatureExtractor:
74
  def __init__(self):
75
  self.logger = self._setup_logging()
 
84
  if "GEE_KEY_BASE64" not in os.environ:
85
  raise Exception("GEE_KEY_BASE64 secret not found")
86
 
 
87
  key_b64 = os.environ["GEE_KEY_BASE64"]
88
  key_json = base64.b64decode(key_b64)
89
  key_path = "/tmp/gee-key.json"
90
  with open(key_path, "wb") as f:
91
  f.write(key_json)
92
 
 
93
  credentials = ServiceAccountCredentials(None, key_file=key_path)
94
  ee.Initialize(credentials, project=Config.GEE_PROJECT_ID)
95
 
 
98
  self.logger.error(f"❌ Earth Engine failed: {e}")
99
  raise
100
 
101
+
102
  def compute_dewpoint(self, t_celsius, rh_percent):
103
  try:
104
  if t_celsius is None or rh_percent is None: return None
 
354
 
355
  # ==================== FASTAPI APP ====================
356
 
 
 
 
 
 
 
 
 
 
 
357
  app = FastAPI(
358
  title="Real-Time Feature Extraction API",
359
  version="1.0.0",
360
  docs_url="/docs",
361
  redoc_url="/redoc",
 
362
  )
363
 
364
+ # Lazy initialization
365
+ extractor: Optional[RealTimeFeatureExtractor] = None
366
+
367
+ @app.get("/health")
368
+ async def health_check():
369
  return {
370
+ "status": "healthy",
371
+ "ee_initialized": extractor is not None,
372
+ "timestamp": datetime.now().isoformat()
 
 
 
 
373
  }
374
 
375
  @app.post("/extract-features", response_model=FeatureResponse)
376
  async def extract_features(request: CoordinateRequest):
377
+ global extractor
378
+ if extractor is None:
379
+ try:
380
+ extractor = RealTimeFeatureExtractor()
381
+ except Exception as e:
382
+ raise HTTPException(status_code=500, detail=f"EE initialization failed: {e}")
383
+
384
+ features = extractor.extract_features(
385
+ latitude=request.latitude,
386
+ longitude=request.longitude
387
+ )
388
+ return FeatureResponse(**features)
389
 
390
  @app.post("/extract-ml-features", response_model=MLFeaturesResponse)
391
  async def extract_ml_features(request: MLFeaturesRequest):
392
+ global extractor
393
+ if extractor is None:
394
+ try:
395
+ extractor = RealTimeFeatureExtractor()
396
+ except Exception as e:
397
+ raise HTTPException(status_code=500, detail=f"EE initialization failed: {e}")
398
+
399
+ features = extractor.extract_features(
400
+ latitude=request.latitude,
401
+ longitude=request.longitude
402
+ )
403
+
404
+ ml_ready_features = {
405
+ 'sand': features.get('sand'),
406
+ 'clay': features.get('clay'),
407
+ 'soc': features.get('soc'),
408
+ 'ph': features.get('ph'),
409
+ 'cec': features.get('cec'),
410
+ 'nitrogen': features.get('nitrogen_percent'),
411
+ 'phosphorus': features.get('phosphorus_mg_kg'),
412
+ 'potassium': features.get('potassium_mg_kg'),
413
+ 'ndvi': features.get('ndvi'),
414
+ 'temperature': features.get('temperature_c'),
415
+ 'humidity': features.get('relative_humidity_percent'),
416
+ 'precipitation': features.get('precipitation_m'),
417
+ 'solar_radiation': features.get('solar_radiation_j_m2'),
418
+ 'latitude': features.get('latitude'),
419
+ 'longitude': features.get('longitude')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
420
  }
421
+
422
+ return MLFeaturesResponse(
423
+ success=True,
424
+ features=ml_ready_features,
425
+ metadata={
426
+ 'location_name': features.get('location_name'),
427
+ 'query_timestamp': features.get('query_timestamp'),
428
+ 'data_quality': features.get('data_quality_score'),
429
+ 'soil_retrieval_method': features.get('soil_retrieval_method'),
430
+ 'npk_confidence': features.get('npk_confidence')
431
+ }
432
+ )
433
 
434
  if __name__ == "__main__":
435
  uvicorn.run(
436
  "realtime_pipeline:app",
437
  host="0.0.0.0",
438
  port=8080,
439
+ )