fix: SVD CPU fallback for ill-conditioned matrices
Browse files
root_gainlora/src/cl_trainer_gainlora_inflora.py
CHANGED
|
@@ -191,16 +191,12 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer):
|
|
| 191 |
try:
|
| 192 |
U, S, V = torch.linalg.svd(cur_trans_matrix)
|
| 193 |
except Exception:
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
cur_trans_matrix.shape[0], cur_trans_matrix.shape[1],
|
| 201 |
-
device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype
|
| 202 |
-
)
|
| 203 |
-
U, S, V = torch.linalg.svd(cur_trans_matrix + reg, driver='gesvd')
|
| 204 |
module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T)
|
| 205 |
# ipdb.set_trace()
|
| 206 |
module.matrix_trans_1[index].zero_()
|
|
|
|
| 191 |
try:
|
| 192 |
U, S, V = torch.linalg.svd(cur_trans_matrix)
|
| 193 |
except Exception:
|
| 194 |
+
# CUDA SVD may fail on ill-conditioned matrices; fall back to CPU
|
| 195 |
+
cpu_mat = cur_trans_matrix.detach().cpu().float()
|
| 196 |
+
U, S, V = torch.linalg.svd(cpu_mat)
|
| 197 |
+
U = U.to(device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype)
|
| 198 |
+
S = S.to(device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype)
|
| 199 |
+
V = V.to(device=cur_trans_matrix.device, dtype=cur_trans_matrix.dtype)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T)
|
| 201 |
# ipdb.set_trace()
|
| 202 |
module.matrix_trans_1[index].zero_()
|