Spaces:
Sleeping
Sleeping
Upload realtime_pipeline.py
Browse files- realtime_pipeline.py +5 -10
realtime_pipeline.py
CHANGED
|
@@ -413,12 +413,6 @@ if "GEE_KEY_BASE64" in os.environ:
|
|
| 413 |
|
| 414 |
class RealTimeFeatureExtractor:
|
| 415 |
def __init__(self):
|
| 416 |
-
# ⚡ Global Patch: Increase requests connection pool size
|
| 417 |
-
# This fixes "Connection pool is full" caused by parallel execution
|
| 418 |
-
# We modify the class attributes directly so ANY new Session (including GEE's) uses these defaults
|
| 419 |
-
requests.adapters.HTTPAdapter.DEFAULT_POOLSIZE = 20
|
| 420 |
-
requests.adapters.HTTPAdapter.DEFAULT_RETRIES = 3
|
| 421 |
-
|
| 422 |
self.logger = self._setup_logging()
|
| 423 |
self._initialize_ee()
|
| 424 |
self.temporal_manager = TemporalDataManager()
|
|
@@ -438,6 +432,9 @@ class RealTimeFeatureExtractor:
|
|
| 438 |
with open(key_path, "wb") as f:
|
| 439 |
f.write(key_json)
|
| 440 |
|
|
|
|
|
|
|
|
|
|
| 441 |
self.logger.info("✅ Earth Engine initialized with Service Account")
|
| 442 |
except Exception as e:
|
| 443 |
self.logger.error(f"❌ Earth Engine failed: {e}")
|
|
@@ -1012,11 +1009,9 @@ class RealTimeFeatureExtractor:
|
|
| 1012 |
'soil_buffer_used_m': soil_data.get('_scale_used')
|
| 1013 |
}
|
| 1014 |
|
| 1015 |
-
# ✅ Execute in parallel using ThreadPoolExecutor (
|
| 1016 |
features_list = []
|
| 1017 |
-
|
| 1018 |
-
# This is the theoretical maximum speed (limited only by GEE quota).
|
| 1019 |
-
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
|
| 1020 |
# Create tasks for 0 to 11
|
| 1021 |
future_to_offset = {executor.submit(_fetch_single_month, offset): offset for offset in range(12)}
|
| 1022 |
|
|
|
|
| 413 |
|
| 414 |
class RealTimeFeatureExtractor:
|
| 415 |
def __init__(self):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
self.logger = self._setup_logging()
|
| 417 |
self._initialize_ee()
|
| 418 |
self.temporal_manager = TemporalDataManager()
|
|
|
|
| 432 |
with open(key_path, "wb") as f:
|
| 433 |
f.write(key_json)
|
| 434 |
|
| 435 |
+
credentials = ServiceAccountCredentials(None, key_file=key_path)
|
| 436 |
+
ee.Initialize(credentials, project=Config.GEE_PROJECT_ID)
|
| 437 |
+
|
| 438 |
self.logger.info("✅ Earth Engine initialized with Service Account")
|
| 439 |
except Exception as e:
|
| 440 |
self.logger.error(f"❌ Earth Engine failed: {e}")
|
|
|
|
| 1009 |
'soil_buffer_used_m': soil_data.get('_scale_used')
|
| 1010 |
}
|
| 1011 |
|
| 1012 |
+
# ✅ Execute in parallel using ThreadPoolExecutor (Set to 8 as per user request)
|
| 1013 |
features_list = []
|
| 1014 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
|
|
|
|
|
|
| 1015 |
# Create tasks for 0 to 11
|
| 1016 |
future_to_offset = {executor.submit(_fetch_single_month, offset): offset for offset in range(12)}
|
| 1017 |
|