Spaces:
Runtime error
Runtime error
fix: use connection context instead of LazyEngine directly for read_sql/to_sql
Browse files
src/similarity_model/sync_projects.py
CHANGED
|
@@ -44,12 +44,14 @@ def sync_projects():
|
|
| 44 |
)
|
| 45 |
"""
|
| 46 |
logger.info("Fetching raw active projects from 'Projects' table...")
|
| 47 |
-
|
|
|
|
| 48 |
logger.info(f"Loaded {len(projects_df)} active projects from database.")
|
| 49 |
|
| 50 |
# 2. Fetch existing preprocessed projects
|
| 51 |
logger.info("Fetching existing records from 'PreProcessed_Projects'...")
|
| 52 |
-
|
|
|
|
| 53 |
|
| 54 |
allowed_ids = set(projects_df["Id"].tolist())
|
| 55 |
processed_ids = set(existing_df["id"].tolist())
|
|
@@ -118,12 +120,13 @@ def sync_projects():
|
|
| 118 |
|
| 119 |
# Insert into database
|
| 120 |
logger.info("Uploading preprocessed records to database...")
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
|
|
|
| 127 |
logger.info(f"Successfully processed and inserted {len(processed_df)} projects.")
|
| 128 |
changed = True
|
| 129 |
else:
|
|
|
|
| 44 |
)
|
| 45 |
"""
|
| 46 |
logger.info("Fetching raw active projects from 'Projects' table...")
|
| 47 |
+
with engine.connect() as conn:
|
| 48 |
+
projects_df = pd.read_sql(projects_query, conn)
|
| 49 |
logger.info(f"Loaded {len(projects_df)} active projects from database.")
|
| 50 |
|
| 51 |
# 2. Fetch existing preprocessed projects
|
| 52 |
logger.info("Fetching existing records from 'PreProcessed_Projects'...")
|
| 53 |
+
with engine.connect() as conn:
|
| 54 |
+
existing_df = pd.read_sql("SELECT id FROM PreProcessed_Projects", conn)
|
| 55 |
|
| 56 |
allowed_ids = set(projects_df["Id"].tolist())
|
| 57 |
processed_ids = set(existing_df["id"].tolist())
|
|
|
|
| 120 |
|
| 121 |
# Insert into database
|
| 122 |
logger.info("Uploading preprocessed records to database...")
|
| 123 |
+
with engine.begin() as conn:
|
| 124 |
+
processed_df.to_sql(
|
| 125 |
+
"PreProcessed_Projects",
|
| 126 |
+
conn,
|
| 127 |
+
if_exists="append",
|
| 128 |
+
index=False
|
| 129 |
+
)
|
| 130 |
logger.info(f"Successfully processed and inserted {len(processed_df)} projects.")
|
| 131 |
changed = True
|
| 132 |
else:
|