radwaamr1 commited on
Commit
1ddc8fa
·
verified ·
1 Parent(s): 754028d

Upload realtime_pipeline.py

Browse files
Files changed (1) hide show
  1. realtime_pipeline.py +5 -10
realtime_pipeline.py CHANGED
@@ -413,12 +413,6 @@ if "GEE_KEY_BASE64" in os.environ:
413
 
414
  class RealTimeFeatureExtractor:
415
  def __init__(self):
416
- # ⚡ Global Patch: Increase requests connection pool size
417
- # This fixes "Connection pool is full" caused by parallel execution
418
- # We modify the class attributes directly so ANY new Session (including GEE's) uses these defaults
419
- requests.adapters.HTTPAdapter.DEFAULT_POOLSIZE = 20
420
- requests.adapters.HTTPAdapter.DEFAULT_RETRIES = 3
421
-
422
  self.logger = self._setup_logging()
423
  self._initialize_ee()
424
  self.temporal_manager = TemporalDataManager()
@@ -438,6 +432,9 @@ class RealTimeFeatureExtractor:
438
  with open(key_path, "wb") as f:
439
  f.write(key_json)
440
 
 
 
 
441
  self.logger.info("✅ Earth Engine initialized with Service Account")
442
  except Exception as e:
443
  self.logger.error(f"❌ Earth Engine failed: {e}")
@@ -1012,11 +1009,9 @@ class RealTimeFeatureExtractor:
1012
  'soil_buffer_used_m': soil_data.get('_scale_used')
1013
  }
1014
 
1015
- # ✅ Execute in parallel using ThreadPoolExecutor (Maximized to 12 for single-batch processing)
1016
  features_list = []
1017
- # Since we have exactly 12 months, using 12 workers allows processing them all in ONE batch.
1018
- # This is the theoretical maximum speed (limited only by GEE quota).
1019
- with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
1020
  # Create tasks for 0 to 11
1021
  future_to_offset = {executor.submit(_fetch_single_month, offset): offset for offset in range(12)}
1022
 
 
413
 
414
  class RealTimeFeatureExtractor:
415
  def __init__(self):
 
 
 
 
 
 
416
  self.logger = self._setup_logging()
417
  self._initialize_ee()
418
  self.temporal_manager = TemporalDataManager()
 
432
  with open(key_path, "wb") as f:
433
  f.write(key_json)
434
 
435
+ credentials = ServiceAccountCredentials(None, key_file=key_path)
436
+ ee.Initialize(credentials, project=Config.GEE_PROJECT_ID)
437
+
438
  self.logger.info("✅ Earth Engine initialized with Service Account")
439
  except Exception as e:
440
  self.logger.error(f"❌ Earth Engine failed: {e}")
 
1009
  'soil_buffer_used_m': soil_data.get('_scale_used')
1010
  }
1011
 
1012
+ # ✅ Execute in parallel using ThreadPoolExecutor (Set to 8 as per user request)
1013
  features_list = []
1014
+ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
 
 
1015
  # Create tasks for 0 to 11
1016
  future_to_offset = {executor.submit(_fetch_single_month, offset): offset for offset in range(12)}
1017