bat-6 commited on
Commit
2e8649c
·
1 Parent(s): d2f9ca7

fix: use connection context instead of LazyEngine directly for read_sql/to_sql

Browse files
src/similarity_model/sync_projects.py CHANGED
@@ -44,12 +44,14 @@ def sync_projects():
44
  )
45
  """
46
  logger.info("Fetching raw active projects from 'Projects' table...")
47
- projects_df = pd.read_sql(projects_query, engine)
 
48
  logger.info(f"Loaded {len(projects_df)} active projects from database.")
49
 
50
  # 2. Fetch existing preprocessed projects
51
  logger.info("Fetching existing records from 'PreProcessed_Projects'...")
52
- existing_df = pd.read_sql("SELECT id FROM PreProcessed_Projects", engine)
 
53
 
54
  allowed_ids = set(projects_df["Id"].tolist())
55
  processed_ids = set(existing_df["id"].tolist())
@@ -118,12 +120,13 @@ def sync_projects():
118
 
119
  # Insert into database
120
  logger.info("Uploading preprocessed records to database...")
121
- processed_df.to_sql(
122
- "PreProcessed_Projects",
123
- engine,
124
- if_exists="append",
125
- index=False
126
- )
 
127
  logger.info(f"Successfully processed and inserted {len(processed_df)} projects.")
128
  changed = True
129
  else:
 
44
  )
45
  """
46
  logger.info("Fetching raw active projects from 'Projects' table...")
47
+ with engine.connect() as conn:
48
+ projects_df = pd.read_sql(projects_query, conn)
49
  logger.info(f"Loaded {len(projects_df)} active projects from database.")
50
 
51
  # 2. Fetch existing preprocessed projects
52
  logger.info("Fetching existing records from 'PreProcessed_Projects'...")
53
+ with engine.connect() as conn:
54
+ existing_df = pd.read_sql("SELECT id FROM PreProcessed_Projects", conn)
55
 
56
  allowed_ids = set(projects_df["Id"].tolist())
57
  processed_ids = set(existing_df["id"].tolist())
 
120
 
121
  # Insert into database
122
  logger.info("Uploading preprocessed records to database...")
123
+ with engine.begin() as conn:
124
+ processed_df.to_sql(
125
+ "PreProcessed_Projects",
126
+ conn,
127
+ if_exists="append",
128
+ index=False
129
+ )
130
  logger.info(f"Successfully processed and inserted {len(processed_df)} projects.")
131
  changed = True
132
  else: