fix(job_train): force CUDA_VISIBLE_DEVICES=0 to dodge TRL multi-GPU entropy bug
Browse files- scripts/job_train.sh +7 -0
scripts/job_train.sh
CHANGED
|
@@ -5,6 +5,13 @@ set -e
|
|
| 5 |
cd /w
|
| 6 |
export PYTHONPATH="/w:${PYTHONPATH}"
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
# Install huggingface_hub first so we can download data files (parity with job_smoke.sh)
|
| 9 |
pip install -q 'huggingface_hub>=0.28'
|
| 10 |
|
|
|
|
| 5 |
cd /w
|
| 6 |
export PYTHONPATH="/w:${PYTHONPATH}"
|
| 7 |
|
| 8 |
+
# Force single-GPU training. The 0.5B (and 1.5B) 4-bit + LoRA model fits on
|
| 9 |
+
# one A10G easily, and TRL's SFTTrainer entropy logging path on multi-GPU
|
| 10 |
+
# (a10g-largex2) crashes with `RuntimeError: tensor a (2) must match tensor
|
| 11 |
+
# b (4) at non-singleton dimension 0` because the per-rank logits get mixed
|
| 12 |
+
# with the full-batch attention mask. Hide the second GPU completely.
|
| 13 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 14 |
+
|
| 15 |
# Install huggingface_hub first so we can download data files (parity with job_smoke.sh)
|
| 16 |
pip install -q 'huggingface_hub>=0.28'
|
| 17 |
|