natmin322 commited on
Commit
3a19926
·
1 Parent(s): c2c5218

fix: SVD CPU fallback for ill-conditioned matrices

Browse files
root_gainlora/src/cl_trainer_gainlora_inflora.py CHANGED
@@ -191,16 +191,12 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer):
191
  try:
192
  U, S, V = torch.linalg.svd(cur_trans_matrix)
193
  except Exception:
194
- try:
195
- # Retry with more robust LAPACK driver
196
- U, S, V = torch.linalg.svd(cur_trans_matrix, driver='gesvd')
197
- except Exception:
198
- # Last resort: add regularization and retry
199
- reg = 1e-3 * torch.eye(
200
- cur_trans_matrix.shape[0], cur_trans_matrix.shape[1],
201
- device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype
202
- )
203
- U, S, V = torch.linalg.svd(cur_trans_matrix + reg, driver='gesvd')
204
  module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T)
205
  # ipdb.set_trace()
206
  module.matrix_trans_1[index].zero_()
 
191
  try:
192
  U, S, V = torch.linalg.svd(cur_trans_matrix)
193
  except Exception:
194
+ # CUDA SVD may fail on ill-conditioned matrices; fall back to CPU
195
+ cpu_mat = cur_trans_matrix.detach().cpu().float()
196
+ U, S, V = torch.linalg.svd(cpu_mat)
197
+ U = U.to(device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype)
198
+ S = S.to(device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype)
199
+ V = V.to(device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype)
 
 
 
 
200
  module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T)
201
  # ipdb.set_trace()
202
  module.matrix_trans_1[index].zero_()