Senum2001 commited on
Commit
5de9e01
·
1 Parent(s): 01d0daa

Fix error handling in feedback pipeline for model versioning

Browse files
Files changed (1) hide show
  1. scripts/feedback_learning_pipeline.py +28 -21
scripts/feedback_learning_pipeline.py CHANGED
@@ -269,9 +269,13 @@ class FeedbackLearningPipeline:
269
  # Capture model state BEFORE training
270
  before_state = None
271
  if self.version_tracker:
272
- before_state = self.version_tracker.get_current_model_state()
273
- self.version_tracker.log_model_version(before_state)
274
- print(f"[Model Versioning] Captured state before training: {before_state['version_id'][:8]}...")
 
 
 
 
275
 
276
  # Fetch new feedback
277
  feedback_logs = self.fetch_new_feedback(limit=1000)
@@ -314,22 +318,25 @@ class FeedbackLearningPipeline:
314
  # Capture model state AFTER training
315
  after_state = None
316
  training_cycle_id = None
317
- if self.version_tracker:
318
- after_state = self.version_tracker.get_current_model_state()
319
- self.version_tracker.log_model_version(after_state)
320
- print(f"[Model Versioning] Captured state after training: {after_state['version_id'][:8]}...")
321
-
322
- # Log the training cycle with before/after comparison
323
- training_cycle_id = self.version_tracker.log_training_cycle(
324
- before_state=before_state,
325
- after_state=after_state,
326
- feedback_count=len(corrections),
327
- patterns=patterns,
328
- performance_metrics=None # TODO: Calculate actual metrics
329
- )
330
-
331
- if training_cycle_id:
332
- print(f"[Training History] Logged training cycle: {training_cycle_id[:8]}...")
 
 
 
333
 
334
  print(f"[Feedback Pipeline] Training cycle completed successfully")
335
  print(f"[Feedback Pipeline] Total feedback processed: {self.training_state['total_feedback_processed']}")
@@ -339,8 +346,8 @@ class FeedbackLearningPipeline:
339
  "corrections_processed": len(corrections),
340
  "patterns": patterns,
341
  "total_feedback_processed": self.training_state["total_feedback_processed"],
342
- "before_version_id": before_state["version_id"] if before_state else None,
343
- "after_version_id": after_state["version_id"] if after_state else None,
344
  "training_cycle_id": training_cycle_id
345
  }
346
 
 
269
  # Capture model state BEFORE training
270
  before_state = None
271
  if self.version_tracker:
272
+ try:
273
+ before_state = self.version_tracker.get_current_model_state()
274
+ self.version_tracker.log_model_version(before_state)
275
+ print(f"[Model Versioning] Captured state before training: {before_state['version_id'][:8]}...")
276
+ except Exception as e:
277
+ print(f"[Model Versioning] Warning: Could not capture before state: {e}")
278
+ self.version_tracker = None # Disable versioning for this run
279
 
280
  # Fetch new feedback
281
  feedback_logs = self.fetch_new_feedback(limit=1000)
 
318
  # Capture model state AFTER training
319
  after_state = None
320
  training_cycle_id = None
321
+ if self.version_tracker and before_state:
322
+ try:
323
+ after_state = self.version_tracker.get_current_model_state()
324
+ self.version_tracker.log_model_version(after_state)
325
+ print(f"[Model Versioning] Captured state after training: {after_state['version_id'][:8]}...")
326
+
327
+ # Log the training cycle with before/after comparison
328
+ training_cycle_id = self.version_tracker.log_training_cycle(
329
+ before_state=before_state,
330
+ after_state=after_state,
331
+ feedback_count=len(corrections),
332
+ patterns=patterns,
333
+ performance_metrics=None # TODO: Calculate actual metrics
334
+ )
335
+
336
+ if training_cycle_id:
337
+ print(f"[Training History] Logged training cycle: {training_cycle_id[:8]}...")
338
+ except Exception as e:
339
+ print(f"[Model Versioning] Error logging version: {e}")
340
 
341
  print(f"[Feedback Pipeline] Training cycle completed successfully")
342
  print(f"[Feedback Pipeline] Total feedback processed: {self.training_state['total_feedback_processed']}")
 
346
  "corrections_processed": len(corrections),
347
  "patterns": patterns,
348
  "total_feedback_processed": self.training_state["total_feedback_processed"],
349
+ "before_version_id": before_state.get("version_id") if before_state else None,
350
+ "after_version_id": after_state.get("version_id") if after_state else None,
351
  "training_cycle_id": training_cycle_id
352
  }
353