Spaces:
Sleeping
Sleeping
Upload realtime_pipeline.py
Browse files- realtime_pipeline.py +7 -24
realtime_pipeline.py
CHANGED
|
@@ -413,6 +413,12 @@ if "GEE_KEY_BASE64" in os.environ:
|
|
| 413 |
|
| 414 |
class RealTimeFeatureExtractor:
|
| 415 |
def __init__(self):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
self.logger = self._setup_logging()
|
| 417 |
self._initialize_ee()
|
| 418 |
self.temporal_manager = TemporalDataManager()
|
|
@@ -432,30 +438,7 @@ class RealTimeFeatureExtractor:
|
|
| 432 |
with open(key_path, "wb") as f:
|
| 433 |
f.write(key_json)
|
| 434 |
|
| 435 |
-
|
| 436 |
-
import google.auth.transport.requests
|
| 437 |
-
from google.oauth2 import service_account
|
| 438 |
-
|
| 439 |
-
# Create a session with a larger pool size
|
| 440 |
-
session = requests.Session()
|
| 441 |
-
adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=20)
|
| 442 |
-
session.mount('https://', adapter)
|
| 443 |
-
|
| 444 |
-
# Load credentials
|
| 445 |
-
credentials = service_account.Credentials.from_service_account_file(
|
| 446 |
-
key_path,
|
| 447 |
-
scopes=['https://www.googleapis.com/auth/earthengine']
|
| 448 |
-
)
|
| 449 |
-
|
| 450 |
-
# Create authorized session with the custom transport
|
| 451 |
-
authorized_session = google.auth.transport.requests.AuthorizedSession(
|
| 452 |
-
credentials, auth_request=google.auth.transport.requests.Request(session=session)
|
| 453 |
-
)
|
| 454 |
-
|
| 455 |
-
# Initialize Earth Engine with the custom http_transport
|
| 456 |
-
ee.Initialize(credentials, project=Config.GEE_PROJECT_ID, http_transport=authorized_session)
|
| 457 |
-
|
| 458 |
-
self.logger.info("✅ Earth Engine initialized with Service Account (Pool Size: 20)")
|
| 459 |
except Exception as e:
|
| 460 |
self.logger.error(f"❌ Earth Engine failed: {e}")
|
| 461 |
raise
|
|
|
|
| 413 |
|
| 414 |
class RealTimeFeatureExtractor:
|
| 415 |
def __init__(self):
|
| 416 |
+
# ⚡ Global Patch: Increase requests connection pool size
|
| 417 |
+
# This fixes "Connection pool is full" caused by parallel execution
|
| 418 |
+
# We modify the class attributes directly so ANY new Session (including GEE's) uses these defaults
|
| 419 |
+
requests.adapters.HTTPAdapter.DEFAULT_POOLSIZE = 20
|
| 420 |
+
requests.adapters.HTTPAdapter.DEFAULT_RETRIES = 3
|
| 421 |
+
|
| 422 |
self.logger = self._setup_logging()
|
| 423 |
self._initialize_ee()
|
| 424 |
self.temporal_manager = TemporalDataManager()
|
|
|
|
| 438 |
with open(key_path, "wb") as f:
|
| 439 |
f.write(key_json)
|
| 440 |
|
| 441 |
+
self.logger.info("✅ Earth Engine initialized with Service Account")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 442 |
except Exception as e:
|
| 443 |
self.logger.error(f"❌ Earth Engine failed: {e}")
|
| 444 |
raise
|