radwaamr1 commited on
Commit
754028d
·
verified ·
1 Parent(s): 3350a41

Upload realtime_pipeline.py

Browse files
Files changed (1) hide show
  1. realtime_pipeline.py +7 -24
realtime_pipeline.py CHANGED
@@ -413,6 +413,12 @@ if "GEE_KEY_BASE64" in os.environ:
413
 
414
  class RealTimeFeatureExtractor:
415
  def __init__(self):
 
 
 
 
 
 
416
  self.logger = self._setup_logging()
417
  self._initialize_ee()
418
  self.temporal_manager = TemporalDataManager()
@@ -432,30 +438,7 @@ class RealTimeFeatureExtractor:
432
  with open(key_path, "wb") as f:
433
  f.write(key_json)
434
 
435
- # Use ServiceAccountCredentials with increased connection pool
436
- import google.auth.transport.requests
437
- from google.oauth2 import service_account
438
-
439
- # Create a session with a larger pool size
440
- session = requests.Session()
441
- adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=20)
442
- session.mount('https://', adapter)
443
-
444
- # Load credentials
445
- credentials = service_account.Credentials.from_service_account_file(
446
- key_path,
447
- scopes=['https://www.googleapis.com/auth/earthengine']
448
- )
449
-
450
- # Create authorized session with the custom transport
451
- authorized_session = google.auth.transport.requests.AuthorizedSession(
452
- credentials, auth_request=google.auth.transport.requests.Request(session=session)
453
- )
454
-
455
- # Initialize Earth Engine with the custom http_transport
456
- ee.Initialize(credentials, project=Config.GEE_PROJECT_ID, http_transport=authorized_session)
457
-
458
- self.logger.info("✅ Earth Engine initialized with Service Account (Pool Size: 20)")
459
  except Exception as e:
460
  self.logger.error(f"❌ Earth Engine failed: {e}")
461
  raise
 
413
 
414
  class RealTimeFeatureExtractor:
415
  def __init__(self):
416
+ # ⚡ Global Patch: Increase requests connection pool size
417
+ # This fixes "Connection pool is full" caused by parallel execution
418
+ # We modify the class attributes directly so ANY new Session (including GEE's) uses these defaults
419
+ requests.adapters.HTTPAdapter.DEFAULT_POOLSIZE = 20
420
+ requests.adapters.HTTPAdapter.DEFAULT_RETRIES = 3
421
+
422
  self.logger = self._setup_logging()
423
  self._initialize_ee()
424
  self.temporal_manager = TemporalDataManager()
 
438
  with open(key_path, "wb") as f:
439
  f.write(key_json)
440
 
441
+ self.logger.info("✅ Earth Engine initialized with Service Account")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
442
  except Exception as e:
443
  self.logger.error(f"❌ Earth Engine failed: {e}")
444
  raise