Pratham-math commited on
Commit
28d7ac6
·
verified ·
1 Parent(s): 7f150a7

fix(job_train): force CUDA_VISIBLE_DEVICES=0 to dodge TRL multi-GPU entropy bug

Browse files
Files changed (1) hide show
  1. scripts/job_train.sh +7 -0
scripts/job_train.sh CHANGED
@@ -5,6 +5,13 @@ set -e
5
  cd /w
6
  export PYTHONPATH="/w:${PYTHONPATH}"
7
 
 
 
 
 
 
 
 
8
  # Install huggingface_hub first so we can download data files (parity with job_smoke.sh)
9
  pip install -q 'huggingface_hub>=0.28'
10
 
 
5
  cd /w
6
  export PYTHONPATH="/w:${PYTHONPATH}"
7
 
8
+ # Force single-GPU training. The 0.5B (and 1.5B) 4-bit + LoRA model fits on
9
+ # one A10G easily, and TRL's SFTTrainer entropy logging path on multi-GPU
10
+ # (a10g-largex2) crashes with `RuntimeError: tensor a (2) must match tensor
11
+ # b (4) at non-singleton dimension 0` because the per-rank logits get mixed
12
+ # with the full-batch attention mask. Hide the second GPU completely.
13
+ export CUDA_VISIBLE_DEVICES=0
14
+
15
  # Install huggingface_hub first so we can download data files (parity with job_smoke.sh)
16
  pip install -q 'huggingface_hub>=0.28'
17