natmin322 commited on
Commit
3443fa9
Β·
1 Parent(s): 6ebc983
improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute.sh CHANGED
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
22
  exit 1
23
  fi
24
 
25
- # Determine GPU type
26
- if [ "$GPU_MEM" -lt 20000 ]; then
27
- IS_T4=1
28
- echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
 
 
 
 
29
  else
30
- IS_T4=0
31
- echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
32
  fi
33
 
34
- # Determine parallelism strategy
35
- if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
36
  GPU_MODE="t4_2gpu"
37
  GPU_IDS="0,1"
38
- FP16_FLAG=""
39
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
40
- elif [ "$IS_T4" -eq 1 ]; then
41
  GPU_MODE="t4_1gpu"
42
  GPU_IDS="${1:-0}"
43
- FP16_FLAG=""
44
- echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
 
 
 
 
 
45
  else
46
  GPU_MODE="a100"
47
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
48
  FP16_FLAG=""
49
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
50
  fi
51
 
52
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
@@ -54,11 +68,13 @@ echo "============================================================"
54
  echo ""
55
 
56
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
57
- BSZ=16; GA=1; EVAL_BSZ=256
58
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
59
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
60
  else
61
- BSZ=64; GA=1; EVAL_BSZ=512
62
  fi
63
 
64
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -107,11 +123,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp
107
  sleep 5
108
 
109
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
110
- BSZ=16; GA=1; EVAL_BSZ=256
111
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
112
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
113
  else
114
- BSZ=64; GA=1; EVAL_BSZ=512
115
  fi
116
 
117
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -161,11 +179,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amaz
161
  sleep 5
162
 
163
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
164
- BSZ=16; GA=1; EVAL_BSZ=256
165
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
166
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
167
  else
168
- BSZ=64; GA=1; EVAL_BSZ=512
169
  fi
170
 
171
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -215,11 +235,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli
215
  sleep 5
216
 
217
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
218
- BSZ=16; GA=1; EVAL_BSZ=256
219
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
220
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
221
  else
222
- BSZ=64; GA=1; EVAL_BSZ=512
223
  fi
224
 
225
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -269,11 +291,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/c
269
  sleep 5
270
 
271
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
272
- BSZ=16; GA=1; EVAL_BSZ=256
273
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
274
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
275
  else
276
- BSZ=64; GA=1; EVAL_BSZ=512
277
  fi
278
 
279
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -323,11 +347,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa
323
  sleep 5
324
 
325
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
326
- BSZ=16; GA=1; EVAL_BSZ=256
327
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
328
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
329
  else
330
- BSZ=64; GA=1; EVAL_BSZ=512
331
  fi
332
 
333
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -377,11 +403,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/
377
  sleep 5
378
 
379
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
380
- BSZ=16; GA=1; EVAL_BSZ=256
381
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
382
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
383
  else
384
- BSZ=64; GA=1; EVAL_BSZ=512
385
  fi
386
 
387
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -431,11 +459,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/
431
  sleep 5
432
 
433
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
434
- BSZ=16; GA=1; EVAL_BSZ=256
435
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
436
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
437
  else
438
- BSZ=64; GA=1; EVAL_BSZ=512
439
  fi
440
 
441
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -485,11 +515,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb
485
  sleep 5
486
 
487
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
488
- BSZ=16; GA=1; EVAL_BSZ=256
489
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
490
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
491
  else
492
- BSZ=64; GA=1; EVAL_BSZ=512
493
  fi
494
 
495
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -539,11 +571,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2
539
  sleep 5
540
 
541
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
542
- BSZ=16; GA=1; EVAL_BSZ=256
543
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
544
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
545
  else
546
- BSZ=64; GA=1; EVAL_BSZ=512
547
  fi
548
 
549
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -593,11 +627,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbp
593
  sleep 5
594
 
595
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
596
- BSZ=16; GA=1; EVAL_BSZ=256
597
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
598
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
599
  else
600
- BSZ=64; GA=1; EVAL_BSZ=512
601
  fi
602
 
603
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -647,11 +683,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agn
647
  sleep 5
648
 
649
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
650
- BSZ=16; GA=1; EVAL_BSZ=256
651
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
652
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
653
  else
654
- BSZ=64; GA=1; EVAL_BSZ=512
655
  fi
656
 
657
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -701,11 +739,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/12-yah
701
  sleep 5
702
 
703
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
704
- BSZ=16; GA=1; EVAL_BSZ=256
705
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
706
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
707
  else
708
- BSZ=64; GA=1; EVAL_BSZ=512
709
  fi
710
 
711
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -755,11 +795,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/13-mul
755
  sleep 5
756
 
757
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
758
- BSZ=16; GA=1; EVAL_BSZ=256
759
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
760
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
761
  else
762
- BSZ=64; GA=1; EVAL_BSZ=512
763
  fi
764
 
765
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -809,11 +851,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/14-boo
809
  sleep 5
810
 
811
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
812
- BSZ=16; GA=1; EVAL_BSZ=256
813
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
814
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
815
  else
816
- BSZ=64; GA=1; EVAL_BSZ=512
817
  fi
818
 
819
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
22
  exit 1
23
  fi
24
 
25
+ # GPU type detection
26
+ # T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
27
+ if [ "$GPU_MEM" -lt 15500 ]; then
28
+ GPU_TYPE="t4"
29
+ echo "[GPU] Detected T4 (${GPU_MEM}MB)"
30
+ elif [ "$GPU_MEM" -le 17000 ]; then
31
+ GPU_TYPE="p100"
32
+ echo "[GPU] Detected P100 (${GPU_MEM}MB)"
33
  else
34
+ GPU_TYPE="highvram"
35
+ echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
36
  fi
37
 
38
+ # Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
39
+ if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
40
  GPU_MODE="t4_2gpu"
41
  GPU_IDS="0,1"
42
+ FP16_FLAG="--gradient_checkpointing"
43
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
44
+ elif [ "$GPU_TYPE" = "t4" ]; then
45
  GPU_MODE="t4_1gpu"
46
  GPU_IDS="${1:-0}"
47
+ FP16_FLAG="--gradient_checkpointing"
48
+ echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
49
+ elif [ "$GPU_TYPE" = "p100" ]; then
50
+ GPU_MODE="p100"
51
+ GPU_IDS="${1:-0}"
52
+ FP16_FLAG="--gradient_checkpointing"
53
+ echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
54
  else
55
  GPU_MODE="a100"
56
+ if [ "$NUM_GPUS" -ge 2 ]; then
57
+ GPU_IDS="0,1"
58
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
59
+ else
60
+ GPU_IDS="${1:-0}"
61
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
62
+ fi
63
  FP16_FLAG=""
 
64
  fi
65
 
66
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
68
  echo ""
69
 
70
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
71
+ BSZ=8; GA=2; EVAL_BSZ=64
72
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
73
+ BSZ=8; GA=2; EVAL_BSZ=32
74
+ elif [ "$GPU_MODE" = "p100" ]; then
75
+ BSZ=16; GA=2; EVAL_BSZ=32
76
  else
77
+ BSZ=64; GA=1; EVAL_BSZ=128
78
  fi
79
 
80
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
123
  sleep 5
124
 
125
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
126
+ BSZ=8; GA=2; EVAL_BSZ=64
127
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
128
+ BSZ=8; GA=2; EVAL_BSZ=32
129
+ elif [ "$GPU_MODE" = "p100" ]; then
130
+ BSZ=16; GA=2; EVAL_BSZ=32
131
  else
132
+ BSZ=64; GA=1; EVAL_BSZ=128
133
  fi
134
 
135
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
179
  sleep 5
180
 
181
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
182
+ BSZ=8; GA=2; EVAL_BSZ=64
183
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
184
+ BSZ=8; GA=2; EVAL_BSZ=32
185
+ elif [ "$GPU_MODE" = "p100" ]; then
186
+ BSZ=16; GA=2; EVAL_BSZ=32
187
  else
188
+ BSZ=64; GA=1; EVAL_BSZ=128
189
  fi
190
 
191
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
235
  sleep 5
236
 
237
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
238
+ BSZ=8; GA=2; EVAL_BSZ=64
239
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
240
+ BSZ=8; GA=2; EVAL_BSZ=32
241
+ elif [ "$GPU_MODE" = "p100" ]; then
242
+ BSZ=16; GA=2; EVAL_BSZ=32
243
  else
244
+ BSZ=64; GA=1; EVAL_BSZ=128
245
  fi
246
 
247
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
291
  sleep 5
292
 
293
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
294
+ BSZ=8; GA=2; EVAL_BSZ=64
295
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
296
+ BSZ=8; GA=2; EVAL_BSZ=32
297
+ elif [ "$GPU_MODE" = "p100" ]; then
298
+ BSZ=16; GA=2; EVAL_BSZ=32
299
  else
300
+ BSZ=64; GA=1; EVAL_BSZ=128
301
  fi
302
 
303
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
347
  sleep 5
348
 
349
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
350
+ BSZ=8; GA=2; EVAL_BSZ=64
351
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
352
+ BSZ=8; GA=2; EVAL_BSZ=32
353
+ elif [ "$GPU_MODE" = "p100" ]; then
354
+ BSZ=16; GA=2; EVAL_BSZ=32
355
  else
356
+ BSZ=64; GA=1; EVAL_BSZ=128
357
  fi
358
 
359
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
403
  sleep 5
404
 
405
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
406
+ BSZ=8; GA=2; EVAL_BSZ=64
407
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
408
+ BSZ=8; GA=2; EVAL_BSZ=32
409
+ elif [ "$GPU_MODE" = "p100" ]; then
410
+ BSZ=16; GA=2; EVAL_BSZ=32
411
  else
412
+ BSZ=64; GA=1; EVAL_BSZ=128
413
  fi
414
 
415
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
459
  sleep 5
460
 
461
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
462
+ BSZ=8; GA=2; EVAL_BSZ=64
463
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
464
+ BSZ=8; GA=2; EVAL_BSZ=32
465
+ elif [ "$GPU_MODE" = "p100" ]; then
466
+ BSZ=16; GA=2; EVAL_BSZ=32
467
  else
468
+ BSZ=64; GA=1; EVAL_BSZ=128
469
  fi
470
 
471
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
515
  sleep 5
516
 
517
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
518
+ BSZ=8; GA=2; EVAL_BSZ=64
519
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
520
+ BSZ=8; GA=2; EVAL_BSZ=32
521
+ elif [ "$GPU_MODE" = "p100" ]; then
522
+ BSZ=16; GA=2; EVAL_BSZ=32
523
  else
524
+ BSZ=64; GA=1; EVAL_BSZ=128
525
  fi
526
 
527
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
571
  sleep 5
572
 
573
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
574
+ BSZ=8; GA=2; EVAL_BSZ=64
575
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
576
+ BSZ=8; GA=2; EVAL_BSZ=32
577
+ elif [ "$GPU_MODE" = "p100" ]; then
578
+ BSZ=16; GA=2; EVAL_BSZ=32
579
  else
580
+ BSZ=64; GA=1; EVAL_BSZ=128
581
  fi
582
 
583
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
627
  sleep 5
628
 
629
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
630
+ BSZ=8; GA=2; EVAL_BSZ=64
631
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
632
+ BSZ=8; GA=2; EVAL_BSZ=32
633
+ elif [ "$GPU_MODE" = "p100" ]; then
634
+ BSZ=16; GA=2; EVAL_BSZ=32
635
  else
636
+ BSZ=64; GA=1; EVAL_BSZ=128
637
  fi
638
 
639
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
683
  sleep 5
684
 
685
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
686
+ BSZ=8; GA=2; EVAL_BSZ=64
687
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
688
+ BSZ=8; GA=2; EVAL_BSZ=32
689
+ elif [ "$GPU_MODE" = "p100" ]; then
690
+ BSZ=16; GA=2; EVAL_BSZ=32
691
  else
692
+ BSZ=64; GA=1; EVAL_BSZ=128
693
  fi
694
 
695
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
739
  sleep 5
740
 
741
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
742
+ BSZ=8; GA=2; EVAL_BSZ=64
743
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
744
+ BSZ=8; GA=2; EVAL_BSZ=32
745
+ elif [ "$GPU_MODE" = "p100" ]; then
746
+ BSZ=16; GA=2; EVAL_BSZ=32
747
  else
748
+ BSZ=64; GA=1; EVAL_BSZ=128
749
  fi
750
 
751
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
795
  sleep 5
796
 
797
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
798
+ BSZ=8; GA=2; EVAL_BSZ=64
799
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
800
+ BSZ=8; GA=2; EVAL_BSZ=32
801
+ elif [ "$GPU_MODE" = "p100" ]; then
802
+ BSZ=16; GA=2; EVAL_BSZ=32
803
  else
804
+ BSZ=64; GA=1; EVAL_BSZ=128
805
  fi
806
 
807
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
851
  sleep 5
852
 
853
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
854
+ BSZ=8; GA=2; EVAL_BSZ=64
855
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
856
+ BSZ=8; GA=2; EVAL_BSZ=32
857
+ elif [ "$GPU_MODE" = "p100" ]; then
858
+ BSZ=16; GA=2; EVAL_BSZ=32
859
  else
860
+ BSZ=64; GA=1; EVAL_BSZ=128
861
  fi
862
 
863
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute_v10a.sh CHANGED
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
22
  exit 1
23
  fi
24
 
25
- # Determine GPU type
26
- if [ "$GPU_MEM" -lt 20000 ]; then
27
- IS_T4=1
28
- echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
 
 
 
 
29
  else
30
- IS_T4=0
31
- echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
32
  fi
33
 
34
- # Determine parallelism strategy
35
- if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
36
  GPU_MODE="t4_2gpu"
37
  GPU_IDS="0,1"
38
- FP16_FLAG=""
39
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
40
- elif [ "$IS_T4" -eq 1 ]; then
41
  GPU_MODE="t4_1gpu"
42
  GPU_IDS="${1:-0}"
43
- FP16_FLAG=""
44
- echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
 
 
 
 
 
45
  else
46
  GPU_MODE="a100"
47
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
48
  FP16_FLAG=""
49
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
50
  fi
51
 
52
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
@@ -54,11 +68,13 @@ echo "============================================================"
54
  echo ""
55
 
56
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
57
- BSZ=16; GA=1; EVAL_BSZ=256
58
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
59
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
60
  else
61
- BSZ=64; GA=1; EVAL_BSZ=512
62
  fi
63
 
64
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -108,11 +124,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
108
  sleep 5
109
 
110
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
111
- BSZ=16; GA=1; EVAL_BSZ=256
112
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
113
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
114
  else
115
- BSZ=64; GA=1; EVAL_BSZ=512
116
  fi
117
 
118
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -165,11 +183,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/2
165
  sleep 5
166
 
167
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
168
- BSZ=16; GA=1; EVAL_BSZ=256
169
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
170
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
171
  else
172
- BSZ=64; GA=1; EVAL_BSZ=512
173
  fi
174
 
175
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -222,11 +242,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/3
222
  sleep 5
223
 
224
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
225
- BSZ=16; GA=1; EVAL_BSZ=256
226
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
227
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
228
  else
229
- BSZ=64; GA=1; EVAL_BSZ=512
230
  fi
231
 
232
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -279,11 +301,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/4
279
  sleep 5
280
 
281
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
282
- BSZ=16; GA=1; EVAL_BSZ=256
283
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
284
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
285
  else
286
- BSZ=64; GA=1; EVAL_BSZ=512
287
  fi
288
 
289
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -336,11 +360,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/5
336
  sleep 5
337
 
338
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
339
- BSZ=16; GA=1; EVAL_BSZ=256
340
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
341
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
342
  else
343
- BSZ=64; GA=1; EVAL_BSZ=512
344
  fi
345
 
346
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -393,11 +419,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/6
393
  sleep 5
394
 
395
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
396
- BSZ=16; GA=1; EVAL_BSZ=256
397
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
398
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
399
  else
400
- BSZ=64; GA=1; EVAL_BSZ=512
401
  fi
402
 
403
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -450,11 +478,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/7
450
  sleep 5
451
 
452
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
453
- BSZ=16; GA=1; EVAL_BSZ=256
454
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
455
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
456
  else
457
- BSZ=64; GA=1; EVAL_BSZ=512
458
  fi
459
 
460
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -507,11 +537,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/8
507
  sleep 5
508
 
509
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
510
- BSZ=16; GA=1; EVAL_BSZ=256
511
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
512
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
513
  else
514
- BSZ=64; GA=1; EVAL_BSZ=512
515
  fi
516
 
517
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -564,11 +596,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/9
564
  sleep 5
565
 
566
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
567
- BSZ=16; GA=1; EVAL_BSZ=256
568
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
569
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
570
  else
571
- BSZ=64; GA=1; EVAL_BSZ=512
572
  fi
573
 
574
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -621,11 +655,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
621
  sleep 5
622
 
623
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
624
- BSZ=16; GA=1; EVAL_BSZ=256
625
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
626
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
627
  else
628
- BSZ=64; GA=1; EVAL_BSZ=512
629
  fi
630
 
631
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -678,11 +714,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
678
  sleep 5
679
 
680
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
681
- BSZ=16; GA=1; EVAL_BSZ=256
682
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
683
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
684
  else
685
- BSZ=64; GA=1; EVAL_BSZ=512
686
  fi
687
 
688
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -735,11 +773,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
735
  sleep 5
736
 
737
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
738
- BSZ=16; GA=1; EVAL_BSZ=256
739
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
740
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
741
  else
742
- BSZ=64; GA=1; EVAL_BSZ=512
743
  fi
744
 
745
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -792,11 +832,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
792
  sleep 5
793
 
794
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
795
- BSZ=16; GA=1; EVAL_BSZ=256
796
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
797
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
798
  else
799
- BSZ=64; GA=1; EVAL_BSZ=512
800
  fi
801
 
802
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -849,11 +891,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
849
  sleep 5
850
 
851
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
852
- BSZ=16; GA=1; EVAL_BSZ=256
853
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
854
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
855
  else
856
- BSZ=64; GA=1; EVAL_BSZ=512
857
  fi
858
 
859
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
22
  exit 1
23
  fi
24
 
25
+ # GPU type detection
26
+ # T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
27
+ if [ "$GPU_MEM" -lt 15500 ]; then
28
+ GPU_TYPE="t4"
29
+ echo "[GPU] Detected T4 (${GPU_MEM}MB)"
30
+ elif [ "$GPU_MEM" -le 17000 ]; then
31
+ GPU_TYPE="p100"
32
+ echo "[GPU] Detected P100 (${GPU_MEM}MB)"
33
  else
34
+ GPU_TYPE="highvram"
35
+ echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
36
  fi
37
 
38
+ # Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
39
+ if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
40
  GPU_MODE="t4_2gpu"
41
  GPU_IDS="0,1"
42
+ FP16_FLAG="--gradient_checkpointing"
43
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
44
+ elif [ "$GPU_TYPE" = "t4" ]; then
45
  GPU_MODE="t4_1gpu"
46
  GPU_IDS="${1:-0}"
47
+ FP16_FLAG="--gradient_checkpointing"
48
+ echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
49
+ elif [ "$GPU_TYPE" = "p100" ]; then
50
+ GPU_MODE="p100"
51
+ GPU_IDS="${1:-0}"
52
+ FP16_FLAG="--gradient_checkpointing"
53
+ echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
54
  else
55
  GPU_MODE="a100"
56
+ if [ "$NUM_GPUS" -ge 2 ]; then
57
+ GPU_IDS="0,1"
58
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
59
+ else
60
+ GPU_IDS="${1:-0}"
61
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
62
+ fi
63
  FP16_FLAG=""
 
64
  fi
65
 
66
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
68
  echo ""
69
 
70
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
71
+ BSZ=8; GA=2; EVAL_BSZ=64
72
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
73
+ BSZ=8; GA=2; EVAL_BSZ=32
74
+ elif [ "$GPU_MODE" = "p100" ]; then
75
+ BSZ=16; GA=2; EVAL_BSZ=32
76
  else
77
+ BSZ=64; GA=1; EVAL_BSZ=128
78
  fi
79
 
80
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
124
  sleep 5
125
 
126
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
127
+ BSZ=8; GA=2; EVAL_BSZ=64
128
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
129
+ BSZ=8; GA=2; EVAL_BSZ=32
130
+ elif [ "$GPU_MODE" = "p100" ]; then
131
+ BSZ=16; GA=2; EVAL_BSZ=32
132
  else
133
+ BSZ=64; GA=1; EVAL_BSZ=128
134
  fi
135
 
136
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
183
  sleep 5
184
 
185
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
186
+ BSZ=8; GA=2; EVAL_BSZ=64
187
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
188
+ BSZ=8; GA=2; EVAL_BSZ=32
189
+ elif [ "$GPU_MODE" = "p100" ]; then
190
+ BSZ=16; GA=2; EVAL_BSZ=32
191
  else
192
+ BSZ=64; GA=1; EVAL_BSZ=128
193
  fi
194
 
195
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
242
  sleep 5
243
 
244
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
245
+ BSZ=8; GA=2; EVAL_BSZ=64
246
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
247
+ BSZ=8; GA=2; EVAL_BSZ=32
248
+ elif [ "$GPU_MODE" = "p100" ]; then
249
+ BSZ=16; GA=2; EVAL_BSZ=32
250
  else
251
+ BSZ=64; GA=1; EVAL_BSZ=128
252
  fi
253
 
254
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
301
  sleep 5
302
 
303
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
304
+ BSZ=8; GA=2; EVAL_BSZ=64
305
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
306
+ BSZ=8; GA=2; EVAL_BSZ=32
307
+ elif [ "$GPU_MODE" = "p100" ]; then
308
+ BSZ=16; GA=2; EVAL_BSZ=32
309
  else
310
+ BSZ=64; GA=1; EVAL_BSZ=128
311
  fi
312
 
313
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
360
  sleep 5
361
 
362
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
363
+ BSZ=8; GA=2; EVAL_BSZ=64
364
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
365
+ BSZ=8; GA=2; EVAL_BSZ=32
366
+ elif [ "$GPU_MODE" = "p100" ]; then
367
+ BSZ=16; GA=2; EVAL_BSZ=32
368
  else
369
+ BSZ=64; GA=1; EVAL_BSZ=128
370
  fi
371
 
372
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
419
  sleep 5
420
 
421
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
422
+ BSZ=8; GA=2; EVAL_BSZ=64
423
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
424
+ BSZ=8; GA=2; EVAL_BSZ=32
425
+ elif [ "$GPU_MODE" = "p100" ]; then
426
+ BSZ=16; GA=2; EVAL_BSZ=32
427
  else
428
+ BSZ=64; GA=1; EVAL_BSZ=128
429
  fi
430
 
431
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
478
  sleep 5
479
 
480
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
481
+ BSZ=8; GA=2; EVAL_BSZ=64
482
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
483
+ BSZ=8; GA=2; EVAL_BSZ=32
484
+ elif [ "$GPU_MODE" = "p100" ]; then
485
+ BSZ=16; GA=2; EVAL_BSZ=32
486
  else
487
+ BSZ=64; GA=1; EVAL_BSZ=128
488
  fi
489
 
490
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
537
  sleep 5
538
 
539
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
540
+ BSZ=8; GA=2; EVAL_BSZ=64
541
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
542
+ BSZ=8; GA=2; EVAL_BSZ=32
543
+ elif [ "$GPU_MODE" = "p100" ]; then
544
+ BSZ=16; GA=2; EVAL_BSZ=32
545
  else
546
+ BSZ=64; GA=1; EVAL_BSZ=128
547
  fi
548
 
549
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
596
  sleep 5
597
 
598
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
599
+ BSZ=8; GA=2; EVAL_BSZ=64
600
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
601
+ BSZ=8; GA=2; EVAL_BSZ=32
602
+ elif [ "$GPU_MODE" = "p100" ]; then
603
+ BSZ=16; GA=2; EVAL_BSZ=32
604
  else
605
+ BSZ=64; GA=1; EVAL_BSZ=128
606
  fi
607
 
608
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
655
  sleep 5
656
 
657
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
658
+ BSZ=8; GA=2; EVAL_BSZ=64
659
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
660
+ BSZ=8; GA=2; EVAL_BSZ=32
661
+ elif [ "$GPU_MODE" = "p100" ]; then
662
+ BSZ=16; GA=2; EVAL_BSZ=32
663
  else
664
+ BSZ=64; GA=1; EVAL_BSZ=128
665
  fi
666
 
667
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
714
  sleep 5
715
 
716
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
717
+ BSZ=8; GA=2; EVAL_BSZ=64
718
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
719
+ BSZ=8; GA=2; EVAL_BSZ=32
720
+ elif [ "$GPU_MODE" = "p100" ]; then
721
+ BSZ=16; GA=2; EVAL_BSZ=32
722
  else
723
+ BSZ=64; GA=1; EVAL_BSZ=128
724
  fi
725
 
726
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
773
  sleep 5
774
 
775
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
776
+ BSZ=8; GA=2; EVAL_BSZ=64
777
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
778
+ BSZ=8; GA=2; EVAL_BSZ=32
779
+ elif [ "$GPU_MODE" = "p100" ]; then
780
+ BSZ=16; GA=2; EVAL_BSZ=32
781
  else
782
+ BSZ=64; GA=1; EVAL_BSZ=128
783
  fi
784
 
785
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
832
  sleep 5
833
 
834
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
835
+ BSZ=8; GA=2; EVAL_BSZ=64
836
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
837
+ BSZ=8; GA=2; EVAL_BSZ=32
838
+ elif [ "$GPU_MODE" = "p100" ]; then
839
+ BSZ=16; GA=2; EVAL_BSZ=32
840
  else
841
+ BSZ=64; GA=1; EVAL_BSZ=128
842
  fi
843
 
844
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
891
  sleep 5
892
 
893
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
894
+ BSZ=8; GA=2; EVAL_BSZ=64
895
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
896
+ BSZ=8; GA=2; EVAL_BSZ=32
897
+ elif [ "$GPU_MODE" = "p100" ]; then
898
+ BSZ=16; GA=2; EVAL_BSZ=32
899
  else
900
+ BSZ=64; GA=1; EVAL_BSZ=128
901
  fi
902
 
903
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute_v10b.sh CHANGED
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
22
  exit 1
23
  fi
24
 
25
- # Determine GPU type
26
- if [ "$GPU_MEM" -lt 20000 ]; then
27
- IS_T4=1
28
- echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
 
 
 
 
29
  else
30
- IS_T4=0
31
- echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
32
  fi
33
 
34
- # Determine parallelism strategy
35
- if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
36
  GPU_MODE="t4_2gpu"
37
  GPU_IDS="0,1"
38
- FP16_FLAG=""
39
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
40
- elif [ "$IS_T4" -eq 1 ]; then
41
  GPU_MODE="t4_1gpu"
42
  GPU_IDS="${1:-0}"
43
- FP16_FLAG=""
44
- echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
 
 
 
 
 
45
  else
46
  GPU_MODE="a100"
47
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
48
  FP16_FLAG=""
49
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
50
  fi
51
 
52
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
@@ -54,11 +68,13 @@ echo "============================================================"
54
  echo ""
55
 
56
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
57
- BSZ=16; GA=1; EVAL_BSZ=256
58
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
59
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
60
  else
61
- BSZ=64; GA=1; EVAL_BSZ=512
62
  fi
63
 
64
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -108,11 +124,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
108
  sleep 5
109
 
110
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
111
- BSZ=16; GA=1; EVAL_BSZ=256
112
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
113
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
114
  else
115
- BSZ=64; GA=1; EVAL_BSZ=512
116
  fi
117
 
118
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -163,11 +181,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/2
163
  sleep 5
164
 
165
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
166
- BSZ=16; GA=1; EVAL_BSZ=256
167
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
168
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
169
  else
170
- BSZ=64; GA=1; EVAL_BSZ=512
171
  fi
172
 
173
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -218,11 +238,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/3
218
  sleep 5
219
 
220
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
221
- BSZ=16; GA=1; EVAL_BSZ=256
222
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
223
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
224
  else
225
- BSZ=64; GA=1; EVAL_BSZ=512
226
  fi
227
 
228
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -273,11 +295,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/4
273
  sleep 5
274
 
275
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
276
- BSZ=16; GA=1; EVAL_BSZ=256
277
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
278
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
279
  else
280
- BSZ=64; GA=1; EVAL_BSZ=512
281
  fi
282
 
283
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -328,11 +352,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/5
328
  sleep 5
329
 
330
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
331
- BSZ=16; GA=1; EVAL_BSZ=256
332
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
333
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
334
  else
335
- BSZ=64; GA=1; EVAL_BSZ=512
336
  fi
337
 
338
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -383,11 +409,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/6
383
  sleep 5
384
 
385
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
386
- BSZ=16; GA=1; EVAL_BSZ=256
387
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
388
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
389
  else
390
- BSZ=64; GA=1; EVAL_BSZ=512
391
  fi
392
 
393
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -438,11 +466,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/7
438
  sleep 5
439
 
440
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
441
- BSZ=16; GA=1; EVAL_BSZ=256
442
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
443
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
444
  else
445
- BSZ=64; GA=1; EVAL_BSZ=512
446
  fi
447
 
448
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -493,11 +523,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/8
493
  sleep 5
494
 
495
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
496
- BSZ=16; GA=1; EVAL_BSZ=256
497
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
498
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
499
  else
500
- BSZ=64; GA=1; EVAL_BSZ=512
501
  fi
502
 
503
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -548,11 +580,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/9
548
  sleep 5
549
 
550
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
551
- BSZ=16; GA=1; EVAL_BSZ=256
552
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
553
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
554
  else
555
- BSZ=64; GA=1; EVAL_BSZ=512
556
  fi
557
 
558
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -603,11 +637,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
603
  sleep 5
604
 
605
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
606
- BSZ=16; GA=1; EVAL_BSZ=256
607
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
608
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
609
  else
610
- BSZ=64; GA=1; EVAL_BSZ=512
611
  fi
612
 
613
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -658,11 +694,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
658
  sleep 5
659
 
660
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
661
- BSZ=16; GA=1; EVAL_BSZ=256
662
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
663
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
664
  else
665
- BSZ=64; GA=1; EVAL_BSZ=512
666
  fi
667
 
668
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -713,11 +751,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
713
  sleep 5
714
 
715
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
716
- BSZ=16; GA=1; EVAL_BSZ=256
717
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
718
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
719
  else
720
- BSZ=64; GA=1; EVAL_BSZ=512
721
  fi
722
 
723
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -768,11 +808,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
768
  sleep 5
769
 
770
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
771
- BSZ=16; GA=1; EVAL_BSZ=256
772
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
773
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
774
  else
775
- BSZ=64; GA=1; EVAL_BSZ=512
776
  fi
777
 
778
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -823,11 +865,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
823
  sleep 5
824
 
825
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
826
- BSZ=16; GA=1; EVAL_BSZ=256
827
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
828
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
829
  else
830
- BSZ=64; GA=1; EVAL_BSZ=512
831
  fi
832
 
833
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
22
  exit 1
23
  fi
24
 
25
+ # GPU type detection
26
+ # T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
27
+ if [ "$GPU_MEM" -lt 15500 ]; then
28
+ GPU_TYPE="t4"
29
+ echo "[GPU] Detected T4 (${GPU_MEM}MB)"
30
+ elif [ "$GPU_MEM" -le 17000 ]; then
31
+ GPU_TYPE="p100"
32
+ echo "[GPU] Detected P100 (${GPU_MEM}MB)"
33
  else
34
+ GPU_TYPE="highvram"
35
+ echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
36
  fi
37
 
38
+ # Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
39
+ if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
40
  GPU_MODE="t4_2gpu"
41
  GPU_IDS="0,1"
42
+ FP16_FLAG="--gradient_checkpointing"
43
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
44
+ elif [ "$GPU_TYPE" = "t4" ]; then
45
  GPU_MODE="t4_1gpu"
46
  GPU_IDS="${1:-0}"
47
+ FP16_FLAG="--gradient_checkpointing"
48
+ echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
49
+ elif [ "$GPU_TYPE" = "p100" ]; then
50
+ GPU_MODE="p100"
51
+ GPU_IDS="${1:-0}"
52
+ FP16_FLAG="--gradient_checkpointing"
53
+ echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
54
  else
55
  GPU_MODE="a100"
56
+ if [ "$NUM_GPUS" -ge 2 ]; then
57
+ GPU_IDS="0,1"
58
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
59
+ else
60
+ GPU_IDS="${1:-0}"
61
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
62
+ fi
63
  FP16_FLAG=""
 
64
  fi
65
 
66
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
68
  echo ""
69
 
70
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
71
+ BSZ=8; GA=2; EVAL_BSZ=64
72
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
73
+ BSZ=8; GA=2; EVAL_BSZ=32
74
+ elif [ "$GPU_MODE" = "p100" ]; then
75
+ BSZ=16; GA=2; EVAL_BSZ=32
76
  else
77
+ BSZ=64; GA=1; EVAL_BSZ=128
78
  fi
79
 
80
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
124
  sleep 5
125
 
126
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
127
+ BSZ=8; GA=2; EVAL_BSZ=64
128
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
129
+ BSZ=8; GA=2; EVAL_BSZ=32
130
+ elif [ "$GPU_MODE" = "p100" ]; then
131
+ BSZ=16; GA=2; EVAL_BSZ=32
132
  else
133
+ BSZ=64; GA=1; EVAL_BSZ=128
134
  fi
135
 
136
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
181
  sleep 5
182
 
183
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
184
+ BSZ=8; GA=2; EVAL_BSZ=64
185
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
186
+ BSZ=8; GA=2; EVAL_BSZ=32
187
+ elif [ "$GPU_MODE" = "p100" ]; then
188
+ BSZ=16; GA=2; EVAL_BSZ=32
189
  else
190
+ BSZ=64; GA=1; EVAL_BSZ=128
191
  fi
192
 
193
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
238
  sleep 5
239
 
240
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
241
+ BSZ=8; GA=2; EVAL_BSZ=64
242
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
243
+ BSZ=8; GA=2; EVAL_BSZ=32
244
+ elif [ "$GPU_MODE" = "p100" ]; then
245
+ BSZ=16; GA=2; EVAL_BSZ=32
246
  else
247
+ BSZ=64; GA=1; EVAL_BSZ=128
248
  fi
249
 
250
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
295
  sleep 5
296
 
297
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
298
+ BSZ=8; GA=2; EVAL_BSZ=64
299
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
300
+ BSZ=8; GA=2; EVAL_BSZ=32
301
+ elif [ "$GPU_MODE" = "p100" ]; then
302
+ BSZ=16; GA=2; EVAL_BSZ=32
303
  else
304
+ BSZ=64; GA=1; EVAL_BSZ=128
305
  fi
306
 
307
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
352
  sleep 5
353
 
354
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
355
+ BSZ=8; GA=2; EVAL_BSZ=64
356
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
357
+ BSZ=8; GA=2; EVAL_BSZ=32
358
+ elif [ "$GPU_MODE" = "p100" ]; then
359
+ BSZ=16; GA=2; EVAL_BSZ=32
360
  else
361
+ BSZ=64; GA=1; EVAL_BSZ=128
362
  fi
363
 
364
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
409
  sleep 5
410
 
411
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
412
+ BSZ=8; GA=2; EVAL_BSZ=64
413
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
414
+ BSZ=8; GA=2; EVAL_BSZ=32
415
+ elif [ "$GPU_MODE" = "p100" ]; then
416
+ BSZ=16; GA=2; EVAL_BSZ=32
417
  else
418
+ BSZ=64; GA=1; EVAL_BSZ=128
419
  fi
420
 
421
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
466
  sleep 5
467
 
468
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
469
+ BSZ=8; GA=2; EVAL_BSZ=64
470
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
471
+ BSZ=8; GA=2; EVAL_BSZ=32
472
+ elif [ "$GPU_MODE" = "p100" ]; then
473
+ BSZ=16; GA=2; EVAL_BSZ=32
474
  else
475
+ BSZ=64; GA=1; EVAL_BSZ=128
476
  fi
477
 
478
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
523
  sleep 5
524
 
525
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
526
+ BSZ=8; GA=2; EVAL_BSZ=64
527
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
528
+ BSZ=8; GA=2; EVAL_BSZ=32
529
+ elif [ "$GPU_MODE" = "p100" ]; then
530
+ BSZ=16; GA=2; EVAL_BSZ=32
531
  else
532
+ BSZ=64; GA=1; EVAL_BSZ=128
533
  fi
534
 
535
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
580
  sleep 5
581
 
582
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
583
+ BSZ=8; GA=2; EVAL_BSZ=64
584
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
585
+ BSZ=8; GA=2; EVAL_BSZ=32
586
+ elif [ "$GPU_MODE" = "p100" ]; then
587
+ BSZ=16; GA=2; EVAL_BSZ=32
588
  else
589
+ BSZ=64; GA=1; EVAL_BSZ=128
590
  fi
591
 
592
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
637
  sleep 5
638
 
639
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
640
+ BSZ=8; GA=2; EVAL_BSZ=64
641
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
642
+ BSZ=8; GA=2; EVAL_BSZ=32
643
+ elif [ "$GPU_MODE" = "p100" ]; then
644
+ BSZ=16; GA=2; EVAL_BSZ=32
645
  else
646
+ BSZ=64; GA=1; EVAL_BSZ=128
647
  fi
648
 
649
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
694
  sleep 5
695
 
696
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
697
+ BSZ=8; GA=2; EVAL_BSZ=64
698
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
699
+ BSZ=8; GA=2; EVAL_BSZ=32
700
+ elif [ "$GPU_MODE" = "p100" ]; then
701
+ BSZ=16; GA=2; EVAL_BSZ=32
702
  else
703
+ BSZ=64; GA=1; EVAL_BSZ=128
704
  fi
705
 
706
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
751
  sleep 5
752
 
753
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
754
+ BSZ=8; GA=2; EVAL_BSZ=64
755
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
756
+ BSZ=8; GA=2; EVAL_BSZ=32
757
+ elif [ "$GPU_MODE" = "p100" ]; then
758
+ BSZ=16; GA=2; EVAL_BSZ=32
759
  else
760
+ BSZ=64; GA=1; EVAL_BSZ=128
761
  fi
762
 
763
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
808
  sleep 5
809
 
810
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
811
+ BSZ=8; GA=2; EVAL_BSZ=64
812
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
813
+ BSZ=8; GA=2; EVAL_BSZ=32
814
+ elif [ "$GPU_MODE" = "p100" ]; then
815
+ BSZ=16; GA=2; EVAL_BSZ=32
816
  else
817
+ BSZ=64; GA=1; EVAL_BSZ=128
818
  fi
819
 
820
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
865
  sleep 5
866
 
867
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
868
+ BSZ=8; GA=2; EVAL_BSZ=64
869
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
870
+ BSZ=8; GA=2; EVAL_BSZ=32
871
+ elif [ "$GPU_MODE" = "p100" ]; then
872
+ BSZ=16; GA=2; EVAL_BSZ=32
873
  else
874
+ BSZ=64; GA=1; EVAL_BSZ=128
875
  fi
876
 
877
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
improve_gainlora/T5_small/gen_script_long_order4_t5_small_specroute.sh CHANGED
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
22
  exit 1
23
  fi
24
 
25
- # Determine GPU type
26
- if [ "$GPU_MEM" -lt 20000 ]; then
27
- IS_T4=1
28
- echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
 
 
 
 
29
  else
30
- IS_T4=0
31
- echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
32
  fi
33
 
34
- # Determine parallelism strategy
35
- if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
36
  GPU_MODE="t4_2gpu"
37
  GPU_IDS="0,1"
38
- FP16_FLAG=""
39
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
40
- elif [ "$IS_T4" -eq 1 ]; then
41
  GPU_MODE="t4_1gpu"
42
  GPU_IDS="${1:-0}"
43
- FP16_FLAG=""
44
- echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
 
 
 
 
 
45
  else
46
  GPU_MODE="a100"
47
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
48
  FP16_FLAG=""
49
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
50
  fi
51
 
52
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
@@ -54,11 +68,13 @@ echo "============================================================"
54
  echo ""
55
 
56
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
57
- BSZ=16; GA=1; EVAL_BSZ=256
58
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
59
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
60
  else
61
- BSZ=64; GA=1; EVAL_BSZ=512
62
  fi
63
 
64
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -107,11 +123,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli
107
  sleep 5
108
 
109
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
110
- BSZ=16; GA=1; EVAL_BSZ=256
111
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
112
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
113
  else
114
- BSZ=64; GA=1; EVAL_BSZ=512
115
  fi
116
 
117
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -161,11 +179,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/c
161
  sleep 5
162
 
163
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
164
- BSZ=16; GA=1; EVAL_BSZ=256
165
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
166
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
167
  else
168
- BSZ=64; GA=1; EVAL_BSZ=512
169
  fi
170
 
171
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -215,11 +235,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/
215
  sleep 5
216
 
217
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
218
- BSZ=16; GA=1; EVAL_BSZ=256
219
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
220
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
221
  else
222
- BSZ=64; GA=1; EVAL_BSZ=512
223
  fi
224
 
225
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -269,11 +291,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa
269
  sleep 5
270
 
271
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
272
- BSZ=16; GA=1; EVAL_BSZ=256
273
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
274
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
275
  else
276
- BSZ=64; GA=1; EVAL_BSZ=512
277
  fi
278
 
279
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -323,11 +347,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/
323
  sleep 5
324
 
325
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
326
- BSZ=16; GA=1; EVAL_BSZ=256
327
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
328
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
329
  else
330
- BSZ=64; GA=1; EVAL_BSZ=512
331
  fi
332
 
333
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -377,11 +403,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-bool
377
  sleep 5
378
 
379
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
380
- BSZ=16; GA=1; EVAL_BSZ=256
381
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
382
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
383
  else
384
- BSZ=64; GA=1; EVAL_BSZ=512
385
  fi
386
 
387
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -431,11 +459,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/
431
  sleep 5
432
 
433
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
434
- BSZ=16; GA=1; EVAL_BSZ=256
435
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
436
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
437
  else
438
- BSZ=64; GA=1; EVAL_BSZ=512
439
  fi
440
 
441
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -485,11 +515,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb
485
  sleep 5
486
 
487
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
488
- BSZ=16; GA=1; EVAL_BSZ=256
489
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
490
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
491
  else
492
- BSZ=64; GA=1; EVAL_BSZ=512
493
  fi
494
 
495
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -539,11 +571,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp
539
  sleep 5
540
 
541
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
542
- BSZ=16; GA=1; EVAL_BSZ=256
543
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
544
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
545
  else
546
- BSZ=64; GA=1; EVAL_BSZ=512
547
  fi
548
 
549
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -593,11 +627,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-ama
593
  sleep 5
594
 
595
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
596
- BSZ=16; GA=1; EVAL_BSZ=256
597
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
598
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
599
  else
600
- BSZ=64; GA=1; EVAL_BSZ=512
601
  fi
602
 
603
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -647,11 +683,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst
647
  sleep 5
648
 
649
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
650
- BSZ=16; GA=1; EVAL_BSZ=256
651
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
652
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
653
  else
654
- BSZ=64; GA=1; EVAL_BSZ=512
655
  fi
656
 
657
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -701,11 +739,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/12-dbp
701
  sleep 5
702
 
703
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
704
- BSZ=16; GA=1; EVAL_BSZ=256
705
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
706
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
707
  else
708
- BSZ=64; GA=1; EVAL_BSZ=512
709
  fi
710
 
711
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -755,11 +795,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/13-agn
755
  sleep 5
756
 
757
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
758
- BSZ=16; GA=1; EVAL_BSZ=256
759
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
760
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
761
  else
762
- BSZ=64; GA=1; EVAL_BSZ=512
763
  fi
764
 
765
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
@@ -809,11 +851,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/14-mul
809
  sleep 5
810
 
811
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
812
- BSZ=16; GA=1; EVAL_BSZ=256
813
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
814
- BSZ=32; GA=1; EVAL_BSZ=256
 
 
815
  else
816
- BSZ=64; GA=1; EVAL_BSZ=512
817
  fi
818
 
819
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
22
  exit 1
23
  fi
24
 
25
+ # GPU type detection
26
+ # T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
27
+ if [ "$GPU_MEM" -lt 15500 ]; then
28
+ GPU_TYPE="t4"
29
+ echo "[GPU] Detected T4 (${GPU_MEM}MB)"
30
+ elif [ "$GPU_MEM" -le 17000 ]; then
31
+ GPU_TYPE="p100"
32
+ echo "[GPU] Detected P100 (${GPU_MEM}MB)"
33
  else
34
+ GPU_TYPE="highvram"
35
+ echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
36
  fi
37
 
38
+ # Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
39
+ if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
40
  GPU_MODE="t4_2gpu"
41
  GPU_IDS="0,1"
42
+ FP16_FLAG="--gradient_checkpointing"
43
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
44
+ elif [ "$GPU_TYPE" = "t4" ]; then
45
  GPU_MODE="t4_1gpu"
46
  GPU_IDS="${1:-0}"
47
+ FP16_FLAG="--gradient_checkpointing"
48
+ echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
49
+ elif [ "$GPU_TYPE" = "p100" ]; then
50
+ GPU_MODE="p100"
51
+ GPU_IDS="${1:-0}"
52
+ FP16_FLAG="--gradient_checkpointing"
53
+ echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
54
  else
55
  GPU_MODE="a100"
56
+ if [ "$NUM_GPUS" -ge 2 ]; then
57
+ GPU_IDS="0,1"
58
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
59
+ else
60
+ GPU_IDS="${1:-0}"
61
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
62
+ fi
63
  FP16_FLAG=""
 
64
  fi
65
 
66
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
68
  echo ""
69
 
70
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
71
+ BSZ=8; GA=2; EVAL_BSZ=64
72
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
73
+ BSZ=8; GA=2; EVAL_BSZ=32
74
+ elif [ "$GPU_MODE" = "p100" ]; then
75
+ BSZ=16; GA=2; EVAL_BSZ=32
76
  else
77
+ BSZ=64; GA=1; EVAL_BSZ=128
78
  fi
79
 
80
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
123
  sleep 5
124
 
125
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
126
+ BSZ=8; GA=2; EVAL_BSZ=64
127
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
128
+ BSZ=8; GA=2; EVAL_BSZ=32
129
+ elif [ "$GPU_MODE" = "p100" ]; then
130
+ BSZ=16; GA=2; EVAL_BSZ=32
131
  else
132
+ BSZ=64; GA=1; EVAL_BSZ=128
133
  fi
134
 
135
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
179
  sleep 5
180
 
181
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
182
+ BSZ=8; GA=2; EVAL_BSZ=64
183
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
184
+ BSZ=8; GA=2; EVAL_BSZ=32
185
+ elif [ "$GPU_MODE" = "p100" ]; then
186
+ BSZ=16; GA=2; EVAL_BSZ=32
187
  else
188
+ BSZ=64; GA=1; EVAL_BSZ=128
189
  fi
190
 
191
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
235
  sleep 5
236
 
237
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
238
+ BSZ=8; GA=2; EVAL_BSZ=64
239
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
240
+ BSZ=8; GA=2; EVAL_BSZ=32
241
+ elif [ "$GPU_MODE" = "p100" ]; then
242
+ BSZ=16; GA=2; EVAL_BSZ=32
243
  else
244
+ BSZ=64; GA=1; EVAL_BSZ=128
245
  fi
246
 
247
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
291
  sleep 5
292
 
293
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
294
+ BSZ=8; GA=2; EVAL_BSZ=64
295
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
296
+ BSZ=8; GA=2; EVAL_BSZ=32
297
+ elif [ "$GPU_MODE" = "p100" ]; then
298
+ BSZ=16; GA=2; EVAL_BSZ=32
299
  else
300
+ BSZ=64; GA=1; EVAL_BSZ=128
301
  fi
302
 
303
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
347
  sleep 5
348
 
349
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
350
+ BSZ=8; GA=2; EVAL_BSZ=64
351
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
352
+ BSZ=8; GA=2; EVAL_BSZ=32
353
+ elif [ "$GPU_MODE" = "p100" ]; then
354
+ BSZ=16; GA=2; EVAL_BSZ=32
355
  else
356
+ BSZ=64; GA=1; EVAL_BSZ=128
357
  fi
358
 
359
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
403
  sleep 5
404
 
405
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
406
+ BSZ=8; GA=2; EVAL_BSZ=64
407
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
408
+ BSZ=8; GA=2; EVAL_BSZ=32
409
+ elif [ "$GPU_MODE" = "p100" ]; then
410
+ BSZ=16; GA=2; EVAL_BSZ=32
411
  else
412
+ BSZ=64; GA=1; EVAL_BSZ=128
413
  fi
414
 
415
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
459
  sleep 5
460
 
461
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
462
+ BSZ=8; GA=2; EVAL_BSZ=64
463
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
464
+ BSZ=8; GA=2; EVAL_BSZ=32
465
+ elif [ "$GPU_MODE" = "p100" ]; then
466
+ BSZ=16; GA=2; EVAL_BSZ=32
467
  else
468
+ BSZ=64; GA=1; EVAL_BSZ=128
469
  fi
470
 
471
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
515
  sleep 5
516
 
517
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
518
+ BSZ=8; GA=2; EVAL_BSZ=64
519
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
520
+ BSZ=8; GA=2; EVAL_BSZ=32
521
+ elif [ "$GPU_MODE" = "p100" ]; then
522
+ BSZ=16; GA=2; EVAL_BSZ=32
523
  else
524
+ BSZ=64; GA=1; EVAL_BSZ=128
525
  fi
526
 
527
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
571
  sleep 5
572
 
573
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
574
+ BSZ=8; GA=2; EVAL_BSZ=64
575
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
576
+ BSZ=8; GA=2; EVAL_BSZ=32
577
+ elif [ "$GPU_MODE" = "p100" ]; then
578
+ BSZ=16; GA=2; EVAL_BSZ=32
579
  else
580
+ BSZ=64; GA=1; EVAL_BSZ=128
581
  fi
582
 
583
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
627
  sleep 5
628
 
629
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
630
+ BSZ=8; GA=2; EVAL_BSZ=64
631
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
632
+ BSZ=8; GA=2; EVAL_BSZ=32
633
+ elif [ "$GPU_MODE" = "p100" ]; then
634
+ BSZ=16; GA=2; EVAL_BSZ=32
635
  else
636
+ BSZ=64; GA=1; EVAL_BSZ=128
637
  fi
638
 
639
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
683
  sleep 5
684
 
685
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
686
+ BSZ=8; GA=2; EVAL_BSZ=64
687
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
688
+ BSZ=8; GA=2; EVAL_BSZ=32
689
+ elif [ "$GPU_MODE" = "p100" ]; then
690
+ BSZ=16; GA=2; EVAL_BSZ=32
691
  else
692
+ BSZ=64; GA=1; EVAL_BSZ=128
693
  fi
694
 
695
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
739
  sleep 5
740
 
741
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
742
+ BSZ=8; GA=2; EVAL_BSZ=64
743
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
744
+ BSZ=8; GA=2; EVAL_BSZ=32
745
+ elif [ "$GPU_MODE" = "p100" ]; then
746
+ BSZ=16; GA=2; EVAL_BSZ=32
747
  else
748
+ BSZ=64; GA=1; EVAL_BSZ=128
749
  fi
750
 
751
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
795
  sleep 5
796
 
797
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
798
+ BSZ=8; GA=2; EVAL_BSZ=64
799
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
800
+ BSZ=8; GA=2; EVAL_BSZ=32
801
+ elif [ "$GPU_MODE" = "p100" ]; then
802
+ BSZ=16; GA=2; EVAL_BSZ=32
803
  else
804
+ BSZ=64; GA=1; EVAL_BSZ=128
805
  fi
806
 
807
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
 
851
  sleep 5
852
 
853
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
854
+ BSZ=8; GA=2; EVAL_BSZ=64
855
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
856
+ BSZ=8; GA=2; EVAL_BSZ=32
857
+ elif [ "$GPU_MODE" = "p100" ]; then
858
+ BSZ=16; GA=2; EVAL_BSZ=32
859
  else
860
+ BSZ=64; GA=1; EVAL_BSZ=128
861
  fi
862
 
863
  CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
improve_gainlora/T5_small/gen_script_superni_order1_t5_small_specroute.sh CHANGED
@@ -32,23 +32,45 @@ else
32
  fi
33
 
34
  # Determine parallelism strategy
35
- # NOTE: T5 models trained in bfloat16 produce NaN with fp16 (overflow).
36
- # T4 GPUs do not support bf16. Use fp32 + gradient_checkpointing instead.
37
- if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
 
 
 
 
 
 
 
 
 
 
 
 
38
  GPU_MODE="t4_2gpu"
39
  GPU_IDS="0,1"
40
- FP16_FLAG=""
41
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
42
- elif [ "$IS_T4" -eq 1 ]; then
43
  GPU_MODE="t4_1gpu"
44
  GPU_IDS="${1:-0}"
45
- FP16_FLAG=""
46
- echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
 
 
 
 
 
47
  else
48
  GPU_MODE="a100"
49
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
50
  FP16_FLAG=""
51
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
52
  fi
53
 
54
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
@@ -58,6 +80,8 @@ echo ""
58
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
59
  BSZ=8; GA=2; EVAL_BSZ=16
60
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
61
  BSZ=16; GA=2; EVAL_BSZ=16
62
  else
63
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -108,6 +132,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
108
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
109
  BSZ=8; GA=2; EVAL_BSZ=16
110
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
111
  BSZ=16; GA=2; EVAL_BSZ=16
112
  else
113
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -159,6 +185,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
159
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
160
  BSZ=8; GA=2; EVAL_BSZ=16
161
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
162
  BSZ=16; GA=2; EVAL_BSZ=16
163
  else
164
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -210,6 +238,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
210
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
211
  BSZ=8; GA=2; EVAL_BSZ=16
212
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
213
  BSZ=16; GA=2; EVAL_BSZ=16
214
  else
215
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -261,6 +291,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
261
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
262
  BSZ=8; GA=2; EVAL_BSZ=16
263
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
264
  BSZ=16; GA=2; EVAL_BSZ=16
265
  else
266
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -312,6 +344,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
312
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
313
  BSZ=8; GA=2; EVAL_BSZ=16
314
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
315
  BSZ=16; GA=2; EVAL_BSZ=16
316
  else
317
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -363,6 +397,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
363
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
364
  BSZ=8; GA=2; EVAL_BSZ=16
365
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
366
  BSZ=16; GA=2; EVAL_BSZ=16
367
  else
368
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -414,6 +450,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
414
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
415
  BSZ=8; GA=2; EVAL_BSZ=16
416
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
417
  BSZ=16; GA=2; EVAL_BSZ=16
418
  else
419
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -465,6 +503,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
465
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
466
  BSZ=8; GA=2; EVAL_BSZ=16
467
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
468
  BSZ=16; GA=2; EVAL_BSZ=16
469
  else
470
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -516,6 +556,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
516
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
517
  BSZ=8; GA=2; EVAL_BSZ=16
518
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
519
  BSZ=16; GA=2; EVAL_BSZ=16
520
  else
521
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -567,6 +609,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
567
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
568
  BSZ=8; GA=2; EVAL_BSZ=16
569
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
570
  BSZ=16; GA=2; EVAL_BSZ=16
571
  else
572
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -618,6 +662,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
618
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
619
  BSZ=8; GA=2; EVAL_BSZ=16
620
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
621
  BSZ=16; GA=2; EVAL_BSZ=16
622
  else
623
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -669,6 +715,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
669
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
670
  BSZ=8; GA=2; EVAL_BSZ=16
671
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
672
  BSZ=16; GA=2; EVAL_BSZ=16
673
  else
674
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -720,6 +768,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
720
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
721
  BSZ=8; GA=2; EVAL_BSZ=16
722
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
723
  BSZ=16; GA=2; EVAL_BSZ=16
724
  else
725
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -771,6 +821,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
771
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
772
  BSZ=8; GA=2; EVAL_BSZ=16
773
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
774
  BSZ=16; GA=2; EVAL_BSZ=16
775
  else
776
  BSZ=32; GA=1; EVAL_BSZ=32
 
32
  fi
33
 
34
  # Determine parallelism strategy
35
+ # GPU type detection
36
+ # T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
37
+ if [ "$GPU_MEM" -lt 15500 ]; then
38
+ GPU_TYPE="t4"
39
+ echo "[GPU] Detected T4 (${GPU_MEM}MB)"
40
+ elif [ "$GPU_MEM" -le 17000 ]; then
41
+ GPU_TYPE="p100"
42
+ echo "[GPU] Detected P100 (${GPU_MEM}MB)"
43
+ else
44
+ GPU_TYPE="highvram"
45
+ echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
46
+ fi
47
+
48
+ # Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
49
+ if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
50
  GPU_MODE="t4_2gpu"
51
  GPU_IDS="0,1"
52
+ FP16_FLAG="--gradient_checkpointing"
53
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
54
+ elif [ "$GPU_TYPE" = "t4" ]; then
55
  GPU_MODE="t4_1gpu"
56
  GPU_IDS="${1:-0}"
57
+ FP16_FLAG="--gradient_checkpointing"
58
+ echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
59
+ elif [ "$GPU_TYPE" = "p100" ]; then
60
+ GPU_MODE="p100"
61
+ GPU_IDS="${1:-0}"
62
+ FP16_FLAG="--gradient_checkpointing"
63
+ echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
64
  else
65
  GPU_MODE="a100"
66
+ if [ "$NUM_GPUS" -ge 2 ]; then
67
+ GPU_IDS="0,1"
68
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
69
+ else
70
+ GPU_IDS="${1:-0}"
71
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
72
+ fi
73
  FP16_FLAG=""
 
74
  fi
75
 
76
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
80
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
81
  BSZ=8; GA=2; EVAL_BSZ=16
82
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
83
+ BSZ=8; GA=2; EVAL_BSZ=16
84
+ elif [ "$GPU_MODE" = "p100" ]; then
85
  BSZ=16; GA=2; EVAL_BSZ=16
86
  else
87
  BSZ=32; GA=1; EVAL_BSZ=32
 
132
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
133
  BSZ=8; GA=2; EVAL_BSZ=16
134
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
135
+ BSZ=8; GA=2; EVAL_BSZ=16
136
+ elif [ "$GPU_MODE" = "p100" ]; then
137
  BSZ=16; GA=2; EVAL_BSZ=16
138
  else
139
  BSZ=32; GA=1; EVAL_BSZ=32
 
185
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
186
  BSZ=8; GA=2; EVAL_BSZ=16
187
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
188
+ BSZ=8; GA=2; EVAL_BSZ=16
189
+ elif [ "$GPU_MODE" = "p100" ]; then
190
  BSZ=16; GA=2; EVAL_BSZ=16
191
  else
192
  BSZ=32; GA=1; EVAL_BSZ=32
 
238
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
239
  BSZ=8; GA=2; EVAL_BSZ=16
240
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
241
+ BSZ=8; GA=2; EVAL_BSZ=16
242
+ elif [ "$GPU_MODE" = "p100" ]; then
243
  BSZ=16; GA=2; EVAL_BSZ=16
244
  else
245
  BSZ=32; GA=1; EVAL_BSZ=32
 
291
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
292
  BSZ=8; GA=2; EVAL_BSZ=16
293
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
294
+ BSZ=8; GA=2; EVAL_BSZ=16
295
+ elif [ "$GPU_MODE" = "p100" ]; then
296
  BSZ=16; GA=2; EVAL_BSZ=16
297
  else
298
  BSZ=32; GA=1; EVAL_BSZ=32
 
344
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
345
  BSZ=8; GA=2; EVAL_BSZ=16
346
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
347
+ BSZ=8; GA=2; EVAL_BSZ=16
348
+ elif [ "$GPU_MODE" = "p100" ]; then
349
  BSZ=16; GA=2; EVAL_BSZ=16
350
  else
351
  BSZ=32; GA=1; EVAL_BSZ=32
 
397
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
398
  BSZ=8; GA=2; EVAL_BSZ=16
399
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
400
+ BSZ=8; GA=2; EVAL_BSZ=16
401
+ elif [ "$GPU_MODE" = "p100" ]; then
402
  BSZ=16; GA=2; EVAL_BSZ=16
403
  else
404
  BSZ=32; GA=1; EVAL_BSZ=32
 
450
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
451
  BSZ=8; GA=2; EVAL_BSZ=16
452
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
453
+ BSZ=8; GA=2; EVAL_BSZ=16
454
+ elif [ "$GPU_MODE" = "p100" ]; then
455
  BSZ=16; GA=2; EVAL_BSZ=16
456
  else
457
  BSZ=32; GA=1; EVAL_BSZ=32
 
503
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
504
  BSZ=8; GA=2; EVAL_BSZ=16
505
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
506
+ BSZ=8; GA=2; EVAL_BSZ=16
507
+ elif [ "$GPU_MODE" = "p100" ]; then
508
  BSZ=16; GA=2; EVAL_BSZ=16
509
  else
510
  BSZ=32; GA=1; EVAL_BSZ=32
 
556
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
557
  BSZ=8; GA=2; EVAL_BSZ=16
558
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
559
+ BSZ=8; GA=2; EVAL_BSZ=16
560
+ elif [ "$GPU_MODE" = "p100" ]; then
561
  BSZ=16; GA=2; EVAL_BSZ=16
562
  else
563
  BSZ=32; GA=1; EVAL_BSZ=32
 
609
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
610
  BSZ=8; GA=2; EVAL_BSZ=16
611
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
612
+ BSZ=8; GA=2; EVAL_BSZ=16
613
+ elif [ "$GPU_MODE" = "p100" ]; then
614
  BSZ=16; GA=2; EVAL_BSZ=16
615
  else
616
  BSZ=32; GA=1; EVAL_BSZ=32
 
662
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
663
  BSZ=8; GA=2; EVAL_BSZ=16
664
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
665
+ BSZ=8; GA=2; EVAL_BSZ=16
666
+ elif [ "$GPU_MODE" = "p100" ]; then
667
  BSZ=16; GA=2; EVAL_BSZ=16
668
  else
669
  BSZ=32; GA=1; EVAL_BSZ=32
 
715
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
716
  BSZ=8; GA=2; EVAL_BSZ=16
717
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
718
+ BSZ=8; GA=2; EVAL_BSZ=16
719
+ elif [ "$GPU_MODE" = "p100" ]; then
720
  BSZ=16; GA=2; EVAL_BSZ=16
721
  else
722
  BSZ=32; GA=1; EVAL_BSZ=32
 
768
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
769
  BSZ=8; GA=2; EVAL_BSZ=16
770
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
771
+ BSZ=8; GA=2; EVAL_BSZ=16
772
+ elif [ "$GPU_MODE" = "p100" ]; then
773
  BSZ=16; GA=2; EVAL_BSZ=16
774
  else
775
  BSZ=32; GA=1; EVAL_BSZ=32
 
821
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
822
  BSZ=8; GA=2; EVAL_BSZ=16
823
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
824
+ BSZ=8; GA=2; EVAL_BSZ=16
825
+ elif [ "$GPU_MODE" = "p100" ]; then
826
  BSZ=16; GA=2; EVAL_BSZ=16
827
  else
828
  BSZ=32; GA=1; EVAL_BSZ=32
improve_gainlora/T5_small/gen_script_superni_order2_t5_small_specroute.sh CHANGED
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
22
  exit 1
23
  fi
24
 
25
- # Determine GPU type
26
- if [ "$GPU_MEM" -lt 20000 ]; then
27
- IS_T4=1
28
- echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
 
 
 
 
29
  else
30
- IS_T4=0
31
- echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
32
  fi
33
 
34
- # Determine parallelism strategy
35
- if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
36
  GPU_MODE="t4_2gpu"
37
  GPU_IDS="0,1"
38
- FP16_FLAG=""
39
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
40
- elif [ "$IS_T4" -eq 1 ]; then
41
  GPU_MODE="t4_1gpu"
42
  GPU_IDS="${1:-0}"
43
- FP16_FLAG=""
44
- echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
 
 
 
 
 
45
  else
46
  GPU_MODE="a100"
47
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
48
  FP16_FLAG=""
49
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
50
  fi
51
 
52
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
@@ -56,6 +70,8 @@ echo ""
56
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
57
  BSZ=8; GA=2; EVAL_BSZ=16
58
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
59
  BSZ=16; GA=2; EVAL_BSZ=16
60
  else
61
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -105,6 +121,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
105
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
106
  BSZ=8; GA=2; EVAL_BSZ=16
107
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
108
  BSZ=16; GA=2; EVAL_BSZ=16
109
  else
110
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -155,6 +173,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
155
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
156
  BSZ=8; GA=2; EVAL_BSZ=16
157
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
158
  BSZ=16; GA=2; EVAL_BSZ=16
159
  else
160
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -205,6 +225,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
205
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
206
  BSZ=8; GA=2; EVAL_BSZ=16
207
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
208
  BSZ=16; GA=2; EVAL_BSZ=16
209
  else
210
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -255,6 +277,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
255
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
256
  BSZ=8; GA=2; EVAL_BSZ=16
257
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
258
  BSZ=16; GA=2; EVAL_BSZ=16
259
  else
260
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -305,6 +329,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
305
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
306
  BSZ=8; GA=2; EVAL_BSZ=16
307
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
308
  BSZ=16; GA=2; EVAL_BSZ=16
309
  else
310
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -355,6 +381,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
355
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
356
  BSZ=8; GA=2; EVAL_BSZ=16
357
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
358
  BSZ=16; GA=2; EVAL_BSZ=16
359
  else
360
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -405,6 +433,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
405
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
406
  BSZ=8; GA=2; EVAL_BSZ=16
407
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
408
  BSZ=16; GA=2; EVAL_BSZ=16
409
  else
410
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -455,6 +485,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
455
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
456
  BSZ=8; GA=2; EVAL_BSZ=16
457
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
458
  BSZ=16; GA=2; EVAL_BSZ=16
459
  else
460
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -505,6 +537,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
505
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
506
  BSZ=8; GA=2; EVAL_BSZ=16
507
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
508
  BSZ=16; GA=2; EVAL_BSZ=16
509
  else
510
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -555,6 +589,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
555
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
556
  BSZ=8; GA=2; EVAL_BSZ=16
557
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
558
  BSZ=16; GA=2; EVAL_BSZ=16
559
  else
560
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -605,6 +641,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
605
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
606
  BSZ=8; GA=2; EVAL_BSZ=16
607
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
608
  BSZ=16; GA=2; EVAL_BSZ=16
609
  else
610
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -655,6 +693,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
655
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
656
  BSZ=8; GA=2; EVAL_BSZ=16
657
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
658
  BSZ=16; GA=2; EVAL_BSZ=16
659
  else
660
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -705,6 +745,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
705
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
706
  BSZ=8; GA=2; EVAL_BSZ=16
707
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
708
  BSZ=16; GA=2; EVAL_BSZ=16
709
  else
710
  BSZ=32; GA=1; EVAL_BSZ=32
@@ -755,6 +797,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
755
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
756
  BSZ=8; GA=2; EVAL_BSZ=16
757
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
 
 
758
  BSZ=16; GA=2; EVAL_BSZ=16
759
  else
760
  BSZ=32; GA=1; EVAL_BSZ=32
 
22
  exit 1
23
  fi
24
 
25
+ # GPU type detection
26
+ # T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
27
+ if [ "$GPU_MEM" -lt 15500 ]; then
28
+ GPU_TYPE="t4"
29
+ echo "[GPU] Detected T4 (${GPU_MEM}MB)"
30
+ elif [ "$GPU_MEM" -le 17000 ]; then
31
+ GPU_TYPE="p100"
32
+ echo "[GPU] Detected P100 (${GPU_MEM}MB)"
33
  else
34
+ GPU_TYPE="highvram"
35
+ echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
36
  fi
37
 
38
+ # Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
39
+ if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
40
  GPU_MODE="t4_2gpu"
41
  GPU_IDS="0,1"
42
+ FP16_FLAG="--gradient_checkpointing"
43
  echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
44
+ elif [ "$GPU_TYPE" = "t4" ]; then
45
  GPU_MODE="t4_1gpu"
46
  GPU_IDS="${1:-0}"
47
+ FP16_FLAG="--gradient_checkpointing"
48
+ echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
49
+ elif [ "$GPU_TYPE" = "p100" ]; then
50
+ GPU_MODE="p100"
51
+ GPU_IDS="${1:-0}"
52
+ FP16_FLAG="--gradient_checkpointing"
53
+ echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
54
  else
55
  GPU_MODE="a100"
56
+ if [ "$NUM_GPUS" -ge 2 ]; then
57
+ GPU_IDS="0,1"
58
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
59
+ else
60
+ GPU_IDS="${1:-0}"
61
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
62
+ fi
63
  FP16_FLAG=""
 
64
  fi
65
 
66
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
70
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
71
  BSZ=8; GA=2; EVAL_BSZ=16
72
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
73
+ BSZ=8; GA=2; EVAL_BSZ=16
74
+ elif [ "$GPU_MODE" = "p100" ]; then
75
  BSZ=16; GA=2; EVAL_BSZ=16
76
  else
77
  BSZ=32; GA=1; EVAL_BSZ=32
 
121
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
122
  BSZ=8; GA=2; EVAL_BSZ=16
123
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
124
+ BSZ=8; GA=2; EVAL_BSZ=16
125
+ elif [ "$GPU_MODE" = "p100" ]; then
126
  BSZ=16; GA=2; EVAL_BSZ=16
127
  else
128
  BSZ=32; GA=1; EVAL_BSZ=32
 
173
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
174
  BSZ=8; GA=2; EVAL_BSZ=16
175
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
176
+ BSZ=8; GA=2; EVAL_BSZ=16
177
+ elif [ "$GPU_MODE" = "p100" ]; then
178
  BSZ=16; GA=2; EVAL_BSZ=16
179
  else
180
  BSZ=32; GA=1; EVAL_BSZ=32
 
225
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
226
  BSZ=8; GA=2; EVAL_BSZ=16
227
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
228
+ BSZ=8; GA=2; EVAL_BSZ=16
229
+ elif [ "$GPU_MODE" = "p100" ]; then
230
  BSZ=16; GA=2; EVAL_BSZ=16
231
  else
232
  BSZ=32; GA=1; EVAL_BSZ=32
 
277
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
278
  BSZ=8; GA=2; EVAL_BSZ=16
279
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
280
+ BSZ=8; GA=2; EVAL_BSZ=16
281
+ elif [ "$GPU_MODE" = "p100" ]; then
282
  BSZ=16; GA=2; EVAL_BSZ=16
283
  else
284
  BSZ=32; GA=1; EVAL_BSZ=32
 
329
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
330
  BSZ=8; GA=2; EVAL_BSZ=16
331
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
332
+ BSZ=8; GA=2; EVAL_BSZ=16
333
+ elif [ "$GPU_MODE" = "p100" ]; then
334
  BSZ=16; GA=2; EVAL_BSZ=16
335
  else
336
  BSZ=32; GA=1; EVAL_BSZ=32
 
381
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
382
  BSZ=8; GA=2; EVAL_BSZ=16
383
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
384
+ BSZ=8; GA=2; EVAL_BSZ=16
385
+ elif [ "$GPU_MODE" = "p100" ]; then
386
  BSZ=16; GA=2; EVAL_BSZ=16
387
  else
388
  BSZ=32; GA=1; EVAL_BSZ=32
 
433
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
434
  BSZ=8; GA=2; EVAL_BSZ=16
435
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
436
+ BSZ=8; GA=2; EVAL_BSZ=16
437
+ elif [ "$GPU_MODE" = "p100" ]; then
438
  BSZ=16; GA=2; EVAL_BSZ=16
439
  else
440
  BSZ=32; GA=1; EVAL_BSZ=32
 
485
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
486
  BSZ=8; GA=2; EVAL_BSZ=16
487
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
488
+ BSZ=8; GA=2; EVAL_BSZ=16
489
+ elif [ "$GPU_MODE" = "p100" ]; then
490
  BSZ=16; GA=2; EVAL_BSZ=16
491
  else
492
  BSZ=32; GA=1; EVAL_BSZ=32
 
537
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
538
  BSZ=8; GA=2; EVAL_BSZ=16
539
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
540
+ BSZ=8; GA=2; EVAL_BSZ=16
541
+ elif [ "$GPU_MODE" = "p100" ]; then
542
  BSZ=16; GA=2; EVAL_BSZ=16
543
  else
544
  BSZ=32; GA=1; EVAL_BSZ=32
 
589
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
590
  BSZ=8; GA=2; EVAL_BSZ=16
591
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
592
+ BSZ=8; GA=2; EVAL_BSZ=16
593
+ elif [ "$GPU_MODE" = "p100" ]; then
594
  BSZ=16; GA=2; EVAL_BSZ=16
595
  else
596
  BSZ=32; GA=1; EVAL_BSZ=32
 
641
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
642
  BSZ=8; GA=2; EVAL_BSZ=16
643
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
644
+ BSZ=8; GA=2; EVAL_BSZ=16
645
+ elif [ "$GPU_MODE" = "p100" ]; then
646
  BSZ=16; GA=2; EVAL_BSZ=16
647
  else
648
  BSZ=32; GA=1; EVAL_BSZ=32
 
693
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
694
  BSZ=8; GA=2; EVAL_BSZ=16
695
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
696
+ BSZ=8; GA=2; EVAL_BSZ=16
697
+ elif [ "$GPU_MODE" = "p100" ]; then
698
  BSZ=16; GA=2; EVAL_BSZ=16
699
  else
700
  BSZ=32; GA=1; EVAL_BSZ=32
 
745
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
746
  BSZ=8; GA=2; EVAL_BSZ=16
747
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
748
+ BSZ=8; GA=2; EVAL_BSZ=16
749
+ elif [ "$GPU_MODE" = "p100" ]; then
750
  BSZ=16; GA=2; EVAL_BSZ=16
751
  else
752
  BSZ=32; GA=1; EVAL_BSZ=32
 
797
  if [ "$GPU_MODE" = "t4_2gpu" ]; then
798
  BSZ=8; GA=2; EVAL_BSZ=16
799
  elif [ "$GPU_MODE" = "t4_1gpu" ]; then
800
+ BSZ=8; GA=2; EVAL_BSZ=16
801
+ elif [ "$GPU_MODE" = "p100" ]; then
802
  BSZ=16; GA=2; EVAL_BSZ=16
803
  else
804
  BSZ=32; GA=1; EVAL_BSZ=32
improve_gainlora/gen_script_long_order3_t5_specroute.sh CHANGED
@@ -49,9 +49,14 @@ elif [ "$GPU_MEM" -le 17000 ]; then
49
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
50
  else
51
  GPU_MODE="a100"
52
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
53
  FP16_FLAG=""
54
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
55
  fi
56
 
57
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
49
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
50
  else
51
  GPU_MODE="a100"
52
+ if [ "$NUM_GPUS" -ge 2 ]; then
53
+ GPU_IDS="0,1"
54
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
55
+ else
56
+ GPU_IDS="${1:-0}"
57
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
58
+ fi
59
  FP16_FLAG=""
 
60
  fi
61
 
62
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
improve_gainlora/gen_script_long_order4_t5_specroute.sh CHANGED
@@ -49,9 +49,14 @@ elif [ "$GPU_MEM" -le 17000 ]; then
49
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
50
  else
51
  GPU_MODE="a100"
52
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
53
  FP16_FLAG=""
54
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
55
  fi
56
 
57
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
49
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
50
  else
51
  GPU_MODE="a100"
52
+ if [ "$NUM_GPUS" -ge 2 ]; then
53
+ GPU_IDS="0,1"
54
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
55
+ else
56
+ GPU_IDS="${1:-0}"
57
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
58
+ fi
59
  FP16_FLAG=""
 
60
  fi
61
 
62
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
improve_gainlora/gen_script_superni_order1_llama_specroute.sh CHANGED
@@ -8,9 +8,35 @@
8
  #SBATCH --gres=gpu:a100-sxm4-80gb:1
9
 
10
  export CUDA_DEVICE_ORDER="PCI_BUS_ID"
11
-
12
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  --do_train \
 
14
  --do_predict \
15
  --predict_with_generate \
16
  --model_name_or_path $2 \
@@ -20,7 +46,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
20
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary \
21
  --per_device_train_batch_size 2 \
22
  --per_device_eval_batch_size 8 \
23
- --gradient_accumulation_steps 16 \
24
  --learning_rate 5e-05 \
25
  --num_train_epochs 50 \
26
  --bf16 \
@@ -55,8 +81,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task
55
  sleep 5
56
 
57
 
58
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
59
  --do_train \
 
60
  --do_predict \
61
  --predict_with_generate \
62
  --model_name_or_path $2 \
@@ -69,7 +96,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
69
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification \
70
  --per_device_train_batch_size 16 \
71
  --per_device_eval_batch_size 8 \
72
- --gradient_accumulation_steps 2 \
73
  --learning_rate 5e-05 \
74
  --num_train_epochs 50 \
75
  --bf16 \
@@ -104,8 +131,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task
104
  sleep 5
105
 
106
 
107
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
108
  --do_train \
 
109
  --do_predict \
110
  --predict_with_generate \
111
  --model_name_or_path $2 \
@@ -118,7 +146,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
118
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization \
119
  --per_device_train_batch_size 1 \
120
  --per_device_eval_batch_size 8 \
121
- --gradient_accumulation_steps 32 \
122
  --learning_rate 5e-05 \
123
  --num_train_epochs 50 \
124
  --bf16 \
@@ -153,8 +181,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task
153
  sleep 5
154
 
155
 
156
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
157
  --do_train \
 
158
  --do_predict \
159
  --predict_with_generate \
160
  --model_name_or_path $2 \
@@ -167,7 +196,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
167
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction \
168
  --per_device_train_batch_size 4 \
169
  --per_device_eval_batch_size 8 \
170
- --gradient_accumulation_steps 8 \
171
  --learning_rate 5e-05 \
172
  --num_train_epochs 50 \
173
  --bf16 \
@@ -202,8 +231,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task
202
  sleep 5
203
 
204
 
205
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
206
  --do_train \
 
207
  --do_predict \
208
  --predict_with_generate \
209
  --model_name_or_path $2 \
@@ -216,7 +246,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
216
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation \
217
  --per_device_train_batch_size 1 \
218
  --per_device_eval_batch_size 8 \
219
- --gradient_accumulation_steps 32 \
220
  --learning_rate 5e-05 \
221
  --num_train_epochs 50 \
222
  --bf16 \
@@ -251,8 +281,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task
251
  sleep 5
252
 
253
 
254
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
255
  --do_train \
 
256
  --do_predict \
257
  --predict_with_generate \
258
  --model_name_or_path $2 \
@@ -265,7 +296,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
265
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction \
266
  --per_device_train_batch_size 16 \
267
  --per_device_eval_batch_size 8 \
268
- --gradient_accumulation_steps 2 \
269
  --learning_rate 5e-05 \
270
  --num_train_epochs 50 \
271
  --bf16 \
@@ -300,8 +331,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task
300
  sleep 5
301
 
302
 
303
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
304
  --do_train \
 
305
  --do_predict \
306
  --predict_with_generate \
307
  --model_name_or_path $2 \
@@ -314,7 +346,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
314
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation \
315
  --per_device_train_batch_size 16 \
316
  --per_device_eval_batch_size 8 \
317
- --gradient_accumulation_steps 2 \
318
  --learning_rate 5e-05 \
319
  --num_train_epochs 50 \
320
  --bf16 \
@@ -349,8 +381,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task
349
  sleep 5
350
 
351
 
352
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
353
  --do_train \
 
354
  --do_predict \
355
  --predict_with_generate \
356
  --model_name_or_path $2 \
@@ -363,7 +396,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
363
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next \
364
  --per_device_train_batch_size 2 \
365
  --per_device_eval_batch_size 8 \
366
- --gradient_accumulation_steps 16 \
367
  --learning_rate 5e-05 \
368
  --num_train_epochs 50 \
369
  --bf16 \
@@ -398,8 +431,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task
398
  sleep 5
399
 
400
 
401
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
402
  --do_train \
 
403
  --do_predict \
404
  --predict_with_generate \
405
  --model_name_or_path $2 \
@@ -412,7 +446,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
412
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation \
413
  --per_device_train_batch_size 8 \
414
  --per_device_eval_batch_size 8 \
415
- --gradient_accumulation_steps 4 \
416
  --learning_rate 5e-05 \
417
  --num_train_epochs 50 \
418
  --bf16 \
@@ -447,8 +481,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task
447
  sleep 5
448
 
449
 
450
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
451
  --do_train \
 
452
  --do_predict \
453
  --predict_with_generate \
454
  --model_name_or_path $2 \
@@ -461,7 +496,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
461
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation \
462
  --per_device_train_batch_size 2 \
463
  --per_device_eval_batch_size 8 \
464
- --gradient_accumulation_steps 16 \
465
  --learning_rate 5e-05 \
466
  --num_train_epochs 50 \
467
  --bf16 \
@@ -496,8 +531,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-tas
496
  sleep 5
497
 
498
 
499
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
500
  --do_train \
 
501
  --do_predict \
502
  --predict_with_generate \
503
  --model_name_or_path $2 \
@@ -510,7 +546,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
510
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection \
511
  --per_device_train_batch_size 4 \
512
  --per_device_eval_batch_size 8 \
513
- --gradient_accumulation_steps 8 \
514
  --learning_rate 5e-05 \
515
  --num_train_epochs 50 \
516
  --bf16 \
@@ -545,8 +581,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-tas
545
  sleep 5
546
 
547
 
548
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
549
  --do_train \
 
550
  --do_predict \
551
  --predict_with_generate \
552
  --model_name_or_path $2 \
@@ -559,7 +596,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
559
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
560
  --per_device_train_batch_size 1 \
561
  --per_device_eval_batch_size 8 \
562
- --gradient_accumulation_steps 32 \
563
  --learning_rate 5e-05 \
564
  --num_train_epochs 50 \
565
  --bf16 \
@@ -594,8 +631,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-tas
594
  sleep 5
595
 
596
 
597
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
598
  --do_train \
 
599
  --do_predict \
600
  --predict_with_generate \
601
  --model_name_or_path $2 \
@@ -608,7 +646,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
608
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation \
609
  --per_device_train_batch_size 8 \
610
  --per_device_eval_batch_size 8 \
611
- --gradient_accumulation_steps 4 \
612
  --learning_rate 5e-05 \
613
  --num_train_epochs 50 \
614
  --bf16 \
@@ -643,8 +681,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-tas
643
  sleep 5
644
 
645
 
646
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
647
  --do_train \
 
648
  --do_predict \
649
  --predict_with_generate \
650
  --model_name_or_path $2 \
@@ -657,7 +696,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
657
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification \
658
  --per_device_train_batch_size 8 \
659
  --per_device_eval_batch_size 8 \
660
- --gradient_accumulation_steps 4 \
661
  --learning_rate 5e-05 \
662
  --num_train_epochs 50 \
663
  --bf16 \
@@ -692,8 +731,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-tas
692
  sleep 5
693
 
694
 
695
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
696
  --do_train \
 
697
  --do_predict \
698
  --predict_with_generate \
699
  --model_name_or_path $2 \
@@ -706,7 +746,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
706
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification \
707
  --per_device_train_batch_size 4 \
708
  --per_device_eval_batch_size 8 \
709
- --gradient_accumulation_steps 8 \
710
  --learning_rate 5e-05 \
711
  --num_train_epochs 50 \
712
  --bf16 \
 
8
  #SBATCH --gres=gpu:a100-sxm4-80gb:1
9
 
10
  export CUDA_DEVICE_ORDER="PCI_BUS_ID"
11
+ # ── GPU auto-detection ─────────────────────────────────────────────────────
12
+ NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
13
+ GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
14
+
15
+ if [ -z "$GPU_MEM" ]; then
16
+ echo "[ERROR] No GPU detected. Exiting."
17
+ exit 1
18
+ fi
19
+
20
+ if [ "$NUM_GPUS" -ge 2 ] && [ "$GPU_MEM" -ge 20000 ]; then
21
+ # 2Γ— 3090/4090 (24 GB+): run DDP on both GPUs
22
+ DS_INCLUDE="localhost:0,1"
23
+ echo "[GPU] 2Γ— GPU detected (${GPU_MEM} MB each) β†’ DDP on localhost:0,1"
24
+ elif [ "$GPU_MEM" -lt 20000 ]; then
25
+ # P100/V100 16 GB: use the dedicated P100 script instead
26
+ echo "[GPU] WARNING: 16 GB GPU detected. Use the _p100.sh script instead."
27
+ echo " This script requires β‰₯ 20 GB VRAM per GPU."
28
+ exit 1
29
+ else
30
+ # Single high-memory GPU (A100/H100 or single 3090)
31
+ DS_INCLUDE="localhost:${1:-0}"
32
+ echo "[GPU] Single GPU detected (${GPU_MEM} MB) β†’ localhost:${1:-0}"
33
+ fi
34
+ # ──────────────────────────────────────────────────────────────────────────
35
+
36
+
37
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
38
  --do_train \
39
+ --gradient_checkpointing \
40
  --do_predict \
41
  --predict_with_generate \
42
  --model_name_or_path $2 \
 
46
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary \
47
  --per_device_train_batch_size 2 \
48
  --per_device_eval_batch_size 8 \
49
+ --gradient_accumulation_steps 8 \
50
  --learning_rate 5e-05 \
51
  --num_train_epochs 50 \
52
  --bf16 \
 
81
  sleep 5
82
 
83
 
84
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
85
  --do_train \
86
+ --gradient_checkpointing \
87
  --do_predict \
88
  --predict_with_generate \
89
  --model_name_or_path $2 \
 
96
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification \
97
  --per_device_train_batch_size 16 \
98
  --per_device_eval_batch_size 8 \
99
+ --gradient_accumulation_steps 1 \
100
  --learning_rate 5e-05 \
101
  --num_train_epochs 50 \
102
  --bf16 \
 
131
  sleep 5
132
 
133
 
134
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
135
  --do_train \
136
+ --gradient_checkpointing \
137
  --do_predict \
138
  --predict_with_generate \
139
  --model_name_or_path $2 \
 
146
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization \
147
  --per_device_train_batch_size 1 \
148
  --per_device_eval_batch_size 8 \
149
+ --gradient_accumulation_steps 16 \
150
  --learning_rate 5e-05 \
151
  --num_train_epochs 50 \
152
  --bf16 \
 
181
  sleep 5
182
 
183
 
184
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
185
  --do_train \
186
+ --gradient_checkpointing \
187
  --do_predict \
188
  --predict_with_generate \
189
  --model_name_or_path $2 \
 
196
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction \
197
  --per_device_train_batch_size 4 \
198
  --per_device_eval_batch_size 8 \
199
+ --gradient_accumulation_steps 4 \
200
  --learning_rate 5e-05 \
201
  --num_train_epochs 50 \
202
  --bf16 \
 
231
  sleep 5
232
 
233
 
234
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
235
  --do_train \
236
+ --gradient_checkpointing \
237
  --do_predict \
238
  --predict_with_generate \
239
  --model_name_or_path $2 \
 
246
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation \
247
  --per_device_train_batch_size 1 \
248
  --per_device_eval_batch_size 8 \
249
+ --gradient_accumulation_steps 16 \
250
  --learning_rate 5e-05 \
251
  --num_train_epochs 50 \
252
  --bf16 \
 
281
  sleep 5
282
 
283
 
284
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
285
  --do_train \
286
+ --gradient_checkpointing \
287
  --do_predict \
288
  --predict_with_generate \
289
  --model_name_or_path $2 \
 
296
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction \
297
  --per_device_train_batch_size 16 \
298
  --per_device_eval_batch_size 8 \
299
+ --gradient_accumulation_steps 1 \
300
  --learning_rate 5e-05 \
301
  --num_train_epochs 50 \
302
  --bf16 \
 
331
  sleep 5
332
 
333
 
334
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
335
  --do_train \
336
+ --gradient_checkpointing \
337
  --do_predict \
338
  --predict_with_generate \
339
  --model_name_or_path $2 \
 
346
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation \
347
  --per_device_train_batch_size 16 \
348
  --per_device_eval_batch_size 8 \
349
+ --gradient_accumulation_steps 1 \
350
  --learning_rate 5e-05 \
351
  --num_train_epochs 50 \
352
  --bf16 \
 
381
  sleep 5
382
 
383
 
384
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
385
  --do_train \
386
+ --gradient_checkpointing \
387
  --do_predict \
388
  --predict_with_generate \
389
  --model_name_or_path $2 \
 
396
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next \
397
  --per_device_train_batch_size 2 \
398
  --per_device_eval_batch_size 8 \
399
+ --gradient_accumulation_steps 8 \
400
  --learning_rate 5e-05 \
401
  --num_train_epochs 50 \
402
  --bf16 \
 
431
  sleep 5
432
 
433
 
434
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
435
  --do_train \
436
+ --gradient_checkpointing \
437
  --do_predict \
438
  --predict_with_generate \
439
  --model_name_or_path $2 \
 
446
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation \
447
  --per_device_train_batch_size 8 \
448
  --per_device_eval_batch_size 8 \
449
+ --gradient_accumulation_steps 2 \
450
  --learning_rate 5e-05 \
451
  --num_train_epochs 50 \
452
  --bf16 \
 
481
  sleep 5
482
 
483
 
484
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
485
  --do_train \
486
+ --gradient_checkpointing \
487
  --do_predict \
488
  --predict_with_generate \
489
  --model_name_or_path $2 \
 
496
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation \
497
  --per_device_train_batch_size 2 \
498
  --per_device_eval_batch_size 8 \
499
+ --gradient_accumulation_steps 8 \
500
  --learning_rate 5e-05 \
501
  --num_train_epochs 50 \
502
  --bf16 \
 
531
  sleep 5
532
 
533
 
534
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
535
  --do_train \
536
+ --gradient_checkpointing \
537
  --do_predict \
538
  --predict_with_generate \
539
  --model_name_or_path $2 \
 
546
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection \
547
  --per_device_train_batch_size 4 \
548
  --per_device_eval_batch_size 8 \
549
+ --gradient_accumulation_steps 4 \
550
  --learning_rate 5e-05 \
551
  --num_train_epochs 50 \
552
  --bf16 \
 
581
  sleep 5
582
 
583
 
584
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
585
  --do_train \
586
+ --gradient_checkpointing \
587
  --do_predict \
588
  --predict_with_generate \
589
  --model_name_or_path $2 \
 
596
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
597
  --per_device_train_batch_size 1 \
598
  --per_device_eval_batch_size 8 \
599
+ --gradient_accumulation_steps 16 \
600
  --learning_rate 5e-05 \
601
  --num_train_epochs 50 \
602
  --bf16 \
 
631
  sleep 5
632
 
633
 
634
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
635
  --do_train \
636
+ --gradient_checkpointing \
637
  --do_predict \
638
  --predict_with_generate \
639
  --model_name_or_path $2 \
 
646
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation \
647
  --per_device_train_batch_size 8 \
648
  --per_device_eval_batch_size 8 \
649
+ --gradient_accumulation_steps 2 \
650
  --learning_rate 5e-05 \
651
  --num_train_epochs 50 \
652
  --bf16 \
 
681
  sleep 5
682
 
683
 
684
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
685
  --do_train \
686
+ --gradient_checkpointing \
687
  --do_predict \
688
  --predict_with_generate \
689
  --model_name_or_path $2 \
 
696
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification \
697
  --per_device_train_batch_size 8 \
698
  --per_device_eval_batch_size 8 \
699
+ --gradient_accumulation_steps 2 \
700
  --learning_rate 5e-05 \
701
  --num_train_epochs 50 \
702
  --bf16 \
 
731
  sleep 5
732
 
733
 
734
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
735
  --do_train \
736
+ --gradient_checkpointing \
737
  --do_predict \
738
  --predict_with_generate \
739
  --model_name_or_path $2 \
 
746
  --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification \
747
  --per_device_train_batch_size 4 \
748
  --per_device_eval_batch_size 8 \
749
+ --gradient_accumulation_steps 4 \
750
  --learning_rate 5e-05 \
751
  --num_train_epochs 50 \
752
  --bf16 \
improve_gainlora/gen_script_superni_order1_llama_specroute_p100.sh ADDED
@@ -0,0 +1,761 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # P100 (16 GB) variant β€” auto-generated by gen_p100_3090_scripts.py
3
+ # Changes vs H100 original:
4
+ # * bf16 β†’ fp16 (P100 has no bf16 hardware support)
5
+ # * DeepSpeed stage2_fp16.config (fp16 forced + ZeRO-2 CPU-optimizer offload)
6
+ # * BSZ=1, GA=original_BSZ Γ— original_GA (effective batch unchanged)
7
+ # * max_source_length 1024 β†’ 512 (halves activation memory)
8
+ # * per_device_eval_batch_size 8 β†’ 4
9
+ # * --gradient_checkpointing added
10
+ # * deepspeed --num_gpus 1 (single GPU, no include needed)
11
+
12
+ export CUDA_DEVICE_ORDER="PCI_BUS_ID"
13
+
14
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
15
+ --do_train \
16
+ --gradient_checkpointing \
17
+ --do_predict \
18
+ --predict_with_generate \
19
+ --model_name_or_path $2 \
20
+ --data_dir CL_Benchmark \
21
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
22
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary \
23
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary \
24
+ --per_device_train_batch_size 1 \
25
+ --per_device_eval_batch_size 4 \
26
+ --gradient_accumulation_steps 32 \
27
+ --learning_rate 5e-05 \
28
+ --num_train_epochs 50 \
29
+ --fp16 \
30
+ --deepspeed configs/ds_configs/stage2_fp16.config \
31
+ --run_name gen_script_superni_order1_llama_specroute \
32
+ --max_source_length 512 \
33
+ --max_target_length 50 \
34
+ --generation_max_length 50 \
35
+ --add_task_name False \
36
+ --add_dataset_name False \
37
+ --overwrite_output_dir \
38
+ --overwrite_cache \
39
+ --lr_scheduler_type constant \
40
+ --warmup_steps 0 \
41
+ --logging_strategy steps \
42
+ --logging_steps 10 \
43
+ --metric_for_best_model eval_rougeL \
44
+ --evaluation_strategy steps \
45
+ --save_strategy steps \
46
+ --save_total_limit 1 \
47
+ --load_best_model_at_end \
48
+ --lora_r 4 \
49
+ --lora_alpha 32 \
50
+ --lora_dropout 0.0 \
51
+ --data_replay_freq -1 \
52
+ --chunk 4 \
53
+ --model_name specroute \
54
+ --threshold 0.995
55
+
56
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/checkpoint*
57
+
58
+ sleep 5
59
+
60
+
61
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
62
+ --do_train \
63
+ --gradient_checkpointing \
64
+ --do_predict \
65
+ --predict_with_generate \
66
+ --model_name_or_path $2 \
67
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights \
68
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights/spectral_signatures.pt \
69
+ --data_dir CL_Benchmark \
70
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
71
+ --gen_data_dir generated_data/lora_gen_superni_llama \
72
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification \
73
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification \
74
+ --per_device_train_batch_size 1 \
75
+ --per_device_eval_batch_size 4 \
76
+ --gradient_accumulation_steps 32 \
77
+ --learning_rate 5e-05 \
78
+ --num_train_epochs 50 \
79
+ --fp16 \
80
+ --deepspeed configs/ds_configs/stage2_fp16.config \
81
+ --run_name gen_script_superni_order1_llama_specroute \
82
+ --max_source_length 512 \
83
+ --max_target_length 50 \
84
+ --generation_max_length 50 \
85
+ --add_task_name False \
86
+ --add_dataset_name False \
87
+ --overwrite_output_dir \
88
+ --overwrite_cache \
89
+ --lr_scheduler_type constant \
90
+ --warmup_steps 0 \
91
+ --logging_strategy steps \
92
+ --logging_steps 10 \
93
+ --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
94
+ --evaluation_strategy steps \
95
+ --save_strategy steps \
96
+ --save_total_limit 1 \
97
+ --load_best_model_at_end \
98
+ --lora_r 4 \
99
+ --lora_alpha 32 \
100
+ --lora_dropout 0.0 \
101
+ --data_replay_freq -1 \
102
+ --chunk 4 \
103
+ --model_name specroute \
104
+ --threshold 0.995
105
+
106
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/checkpoint*
107
+
108
+ sleep 5
109
+
110
+
111
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
112
+ --do_train \
113
+ --gradient_checkpointing \
114
+ --do_predict \
115
+ --predict_with_generate \
116
+ --model_name_or_path $2 \
117
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights \
118
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights/spectral_signatures.pt \
119
+ --data_dir CL_Benchmark \
120
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
121
+ --gen_data_dir generated_data/lora_gen_superni_llama \
122
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization \
123
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization \
124
+ --per_device_train_batch_size 1 \
125
+ --per_device_eval_batch_size 4 \
126
+ --gradient_accumulation_steps 32 \
127
+ --learning_rate 5e-05 \
128
+ --num_train_epochs 50 \
129
+ --fp16 \
130
+ --deepspeed configs/ds_configs/stage2_fp16.config \
131
+ --run_name gen_script_superni_order1_llama_specroute \
132
+ --max_source_length 512 \
133
+ --max_target_length 50 \
134
+ --generation_max_length 50 \
135
+ --add_task_name False \
136
+ --add_dataset_name False \
137
+ --overwrite_output_dir \
138
+ --overwrite_cache \
139
+ --lr_scheduler_type constant \
140
+ --warmup_steps 0 \
141
+ --logging_strategy steps \
142
+ --logging_steps 10 \
143
+ --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
144
+ --evaluation_strategy steps \
145
+ --save_strategy steps \
146
+ --save_total_limit 1 \
147
+ --load_best_model_at_end \
148
+ --lora_r 4 \
149
+ --lora_alpha 32 \
150
+ --lora_dropout 0.0 \
151
+ --data_replay_freq -1 \
152
+ --chunk 4 \
153
+ --model_name specroute \
154
+ --threshold 0.995
155
+
156
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/checkpoint*
157
+
158
+ sleep 5
159
+
160
+
161
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
162
+ --do_train \
163
+ --gradient_checkpointing \
164
+ --do_predict \
165
+ --predict_with_generate \
166
+ --model_name_or_path $2 \
167
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights \
168
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights/spectral_signatures.pt \
169
+ --data_dir CL_Benchmark \
170
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
171
+ --gen_data_dir generated_data/lora_gen_superni_llama \
172
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction \
173
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction \
174
+ --per_device_train_batch_size 1 \
175
+ --per_device_eval_batch_size 4 \
176
+ --gradient_accumulation_steps 32 \
177
+ --learning_rate 5e-05 \
178
+ --num_train_epochs 50 \
179
+ --fp16 \
180
+ --deepspeed configs/ds_configs/stage2_fp16.config \
181
+ --run_name gen_script_superni_order1_llama_specroute \
182
+ --max_source_length 512 \
183
+ --max_target_length 50 \
184
+ --generation_max_length 50 \
185
+ --add_task_name False \
186
+ --add_dataset_name False \
187
+ --overwrite_output_dir \
188
+ --overwrite_cache \
189
+ --lr_scheduler_type constant \
190
+ --warmup_steps 0 \
191
+ --logging_strategy steps \
192
+ --logging_steps 10 \
193
+ --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
194
+ --evaluation_strategy steps \
195
+ --save_strategy steps \
196
+ --save_total_limit 1 \
197
+ --load_best_model_at_end \
198
+ --lora_r 4 \
199
+ --lora_alpha 32 \
200
+ --lora_dropout 0.0 \
201
+ --data_replay_freq -1 \
202
+ --chunk 4 \
203
+ --model_name specroute \
204
+ --threshold 0.995
205
+
206
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/checkpoint*
207
+
208
+ sleep 5
209
+
210
+
211
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
212
+ --do_train \
213
+ --gradient_checkpointing \
214
+ --do_predict \
215
+ --predict_with_generate \
216
+ --model_name_or_path $2 \
217
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights \
218
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights/spectral_signatures.pt \
219
+ --data_dir CL_Benchmark \
220
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
221
+ --gen_data_dir generated_data/lora_gen_superni_llama \
222
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation \
223
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation \
224
+ --per_device_train_batch_size 1 \
225
+ --per_device_eval_batch_size 4 \
226
+ --gradient_accumulation_steps 32 \
227
+ --learning_rate 5e-05 \
228
+ --num_train_epochs 50 \
229
+ --fp16 \
230
+ --deepspeed configs/ds_configs/stage2_fp16.config \
231
+ --run_name gen_script_superni_order1_llama_specroute \
232
+ --max_source_length 512 \
233
+ --max_target_length 50 \
234
+ --generation_max_length 50 \
235
+ --add_task_name False \
236
+ --add_dataset_name False \
237
+ --overwrite_output_dir \
238
+ --overwrite_cache \
239
+ --lr_scheduler_type constant \
240
+ --warmup_steps 0 \
241
+ --logging_strategy steps \
242
+ --logging_steps 10 \
243
+ --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
244
+ --evaluation_strategy steps \
245
+ --save_strategy steps \
246
+ --save_total_limit 1 \
247
+ --load_best_model_at_end \
248
+ --lora_r 4 \
249
+ --lora_alpha 32 \
250
+ --lora_dropout 0.0 \
251
+ --data_replay_freq -1 \
252
+ --chunk 4 \
253
+ --model_name specroute \
254
+ --threshold 0.995
255
+
256
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/checkpoint*
257
+
258
+ sleep 5
259
+
260
+
261
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
262
+ --do_train \
263
+ --gradient_checkpointing \
264
+ --do_predict \
265
+ --predict_with_generate \
266
+ --model_name_or_path $2 \
267
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights \
268
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights/spectral_signatures.pt \
269
+ --data_dir CL_Benchmark \
270
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
271
+ --gen_data_dir generated_data/lora_gen_superni_llama \
272
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction \
273
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction \
274
+ --per_device_train_batch_size 1 \
275
+ --per_device_eval_batch_size 4 \
276
+ --gradient_accumulation_steps 32 \
277
+ --learning_rate 5e-05 \
278
+ --num_train_epochs 50 \
279
+ --fp16 \
280
+ --deepspeed configs/ds_configs/stage2_fp16.config \
281
+ --run_name gen_script_superni_order1_llama_specroute \
282
+ --max_source_length 512 \
283
+ --max_target_length 50 \
284
+ --generation_max_length 50 \
285
+ --add_task_name False \
286
+ --add_dataset_name False \
287
+ --overwrite_output_dir \
288
+ --overwrite_cache \
289
+ --lr_scheduler_type constant \
290
+ --warmup_steps 0 \
291
+ --logging_strategy steps \
292
+ --logging_steps 10 \
293
+ --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
294
+ --evaluation_strategy steps \
295
+ --save_strategy steps \
296
+ --save_total_limit 1 \
297
+ --load_best_model_at_end \
298
+ --lora_r 4 \
299
+ --lora_alpha 32 \
300
+ --lora_dropout 0.0 \
301
+ --data_replay_freq -1 \
302
+ --chunk 4 \
303
+ --model_name specroute \
304
+ --threshold 0.995
305
+
306
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/checkpoint*
307
+
308
+ sleep 5
309
+
310
+
311
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
312
+ --do_train \
313
+ --gradient_checkpointing \
314
+ --do_predict \
315
+ --predict_with_generate \
316
+ --model_name_or_path $2 \
317
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights \
318
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights/spectral_signatures.pt \
319
+ --data_dir CL_Benchmark \
320
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
321
+ --gen_data_dir generated_data/lora_gen_superni_llama \
322
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation \
323
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation \
324
+ --per_device_train_batch_size 1 \
325
+ --per_device_eval_batch_size 4 \
326
+ --gradient_accumulation_steps 32 \
327
+ --learning_rate 5e-05 \
328
+ --num_train_epochs 50 \
329
+ --fp16 \
330
+ --deepspeed configs/ds_configs/stage2_fp16.config \
331
+ --run_name gen_script_superni_order1_llama_specroute \
332
+ --max_source_length 512 \
333
+ --max_target_length 50 \
334
+ --generation_max_length 50 \
335
+ --add_task_name False \
336
+ --add_dataset_name False \
337
+ --overwrite_output_dir \
338
+ --overwrite_cache \
339
+ --lr_scheduler_type constant \
340
+ --warmup_steps 0 \
341
+ --logging_strategy steps \
342
+ --logging_steps 10 \
343
+ --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
344
+ --evaluation_strategy steps \
345
+ --save_strategy steps \
346
+ --save_total_limit 1 \
347
+ --load_best_model_at_end \
348
+ --lora_r 4 \
349
+ --lora_alpha 32 \
350
+ --lora_dropout 0.0 \
351
+ --data_replay_freq -1 \
352
+ --chunk 4 \
353
+ --model_name specroute \
354
+ --threshold 0.995
355
+
356
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/checkpoint*
357
+
358
+ sleep 5
359
+
360
+
361
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
362
+ --do_train \
363
+ --gradient_checkpointing \
364
+ --do_predict \
365
+ --predict_with_generate \
366
+ --model_name_or_path $2 \
367
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights \
368
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/spectral_signatures.pt \
369
+ --data_dir CL_Benchmark \
370
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
371
+ --gen_data_dir generated_data/lora_gen_superni_llama \
372
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next \
373
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next \
374
+ --per_device_train_batch_size 1 \
375
+ --per_device_eval_batch_size 4 \
376
+ --gradient_accumulation_steps 32 \
377
+ --learning_rate 5e-05 \
378
+ --num_train_epochs 50 \
379
+ --fp16 \
380
+ --deepspeed configs/ds_configs/stage2_fp16.config \
381
+ --run_name gen_script_superni_order1_llama_specroute \
382
+ --max_source_length 512 \
383
+ --max_target_length 50 \
384
+ --generation_max_length 50 \
385
+ --add_task_name False \
386
+ --add_dataset_name False \
387
+ --overwrite_output_dir \
388
+ --overwrite_cache \
389
+ --lr_scheduler_type constant \
390
+ --warmup_steps 0 \
391
+ --logging_strategy steps \
392
+ --logging_steps 10 \
393
+ --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
394
+ --evaluation_strategy steps \
395
+ --save_strategy steps \
396
+ --save_total_limit 1 \
397
+ --load_best_model_at_end \
398
+ --lora_r 4 \
399
+ --lora_alpha 32 \
400
+ --lora_dropout 0.0 \
401
+ --data_replay_freq -1 \
402
+ --chunk 4 \
403
+ --model_name specroute \
404
+ --threshold 0.995
405
+
406
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/checkpoint*
407
+
408
+ sleep 5
409
+
410
+
411
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
412
+ --do_train \
413
+ --gradient_checkpointing \
414
+ --do_predict \
415
+ --predict_with_generate \
416
+ --model_name_or_path $2 \
417
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights \
418
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights/spectral_signatures.pt \
419
+ --data_dir CL_Benchmark \
420
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
421
+ --gen_data_dir generated_data/lora_gen_superni_llama \
422
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation \
423
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation \
424
+ --per_device_train_batch_size 1 \
425
+ --per_device_eval_batch_size 4 \
426
+ --gradient_accumulation_steps 32 \
427
+ --learning_rate 5e-05 \
428
+ --num_train_epochs 50 \
429
+ --fp16 \
430
+ --deepspeed configs/ds_configs/stage2_fp16.config \
431
+ --run_name gen_script_superni_order1_llama_specroute \
432
+ --max_source_length 512 \
433
+ --max_target_length 50 \
434
+ --generation_max_length 50 \
435
+ --add_task_name False \
436
+ --add_dataset_name False \
437
+ --overwrite_output_dir \
438
+ --overwrite_cache \
439
+ --lr_scheduler_type constant \
440
+ --warmup_steps 0 \
441
+ --logging_strategy steps \
442
+ --logging_steps 10 \
443
+ --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
444
+ --evaluation_strategy steps \
445
+ --save_strategy steps \
446
+ --save_total_limit 1 \
447
+ --load_best_model_at_end \
448
+ --lora_r 4 \
449
+ --lora_alpha 32 \
450
+ --lora_dropout 0.0 \
451
+ --data_replay_freq -1 \
452
+ --chunk 4 \
453
+ --model_name specroute \
454
+ --threshold 0.995
455
+
456
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/checkpoint*
457
+
458
+ sleep 5
459
+
460
+
461
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
462
+ --do_train \
463
+ --gradient_checkpointing \
464
+ --do_predict \
465
+ --predict_with_generate \
466
+ --model_name_or_path $2 \
467
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights \
468
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/spectral_signatures.pt \
469
+ --data_dir CL_Benchmark \
470
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
471
+ --gen_data_dir generated_data/lora_gen_superni_llama \
472
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation \
473
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation \
474
+ --per_device_train_batch_size 1 \
475
+ --per_device_eval_batch_size 4 \
476
+ --gradient_accumulation_steps 32 \
477
+ --learning_rate 5e-05 \
478
+ --num_train_epochs 50 \
479
+ --fp16 \
480
+ --deepspeed configs/ds_configs/stage2_fp16.config \
481
+ --run_name gen_script_superni_order1_llama_specroute \
482
+ --max_source_length 512 \
483
+ --max_target_length 50 \
484
+ --generation_max_length 50 \
485
+ --add_task_name False \
486
+ --add_dataset_name False \
487
+ --overwrite_output_dir \
488
+ --overwrite_cache \
489
+ --lr_scheduler_type constant \
490
+ --warmup_steps 0 \
491
+ --logging_strategy steps \
492
+ --logging_steps 10 \
493
+ --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
494
+ --evaluation_strategy steps \
495
+ --save_strategy steps \
496
+ --save_total_limit 1 \
497
+ --load_best_model_at_end \
498
+ --lora_r 4 \
499
+ --lora_alpha 32 \
500
+ --lora_dropout 0.0 \
501
+ --data_replay_freq -1 \
502
+ --chunk 4 \
503
+ --model_name specroute \
504
+ --threshold 0.995
505
+
506
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/checkpoint*
507
+
508
+ sleep 5
509
+
510
+
511
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
512
+ --do_train \
513
+ --gradient_checkpointing \
514
+ --do_predict \
515
+ --predict_with_generate \
516
+ --model_name_or_path $2 \
517
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights \
518
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights/spectral_signatures.pt \
519
+ --data_dir CL_Benchmark \
520
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
521
+ --gen_data_dir generated_data/lora_gen_superni_llama \
522
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection \
523
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection \
524
+ --per_device_train_batch_size 1 \
525
+ --per_device_eval_batch_size 4 \
526
+ --gradient_accumulation_steps 32 \
527
+ --learning_rate 5e-05 \
528
+ --num_train_epochs 50 \
529
+ --fp16 \
530
+ --deepspeed configs/ds_configs/stage2_fp16.config \
531
+ --run_name gen_script_superni_order1_llama_specroute \
532
+ --max_source_length 512 \
533
+ --max_target_length 50 \
534
+ --generation_max_length 50 \
535
+ --add_task_name False \
536
+ --add_dataset_name False \
537
+ --overwrite_output_dir \
538
+ --overwrite_cache \
539
+ --lr_scheduler_type constant \
540
+ --warmup_steps 0 \
541
+ --logging_strategy steps \
542
+ --logging_steps 10 \
543
+ --metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \
544
+ --evaluation_strategy steps \
545
+ --save_strategy steps \
546
+ --save_total_limit 1 \
547
+ --load_best_model_at_end \
548
+ --lora_r 4 \
549
+ --lora_alpha 32 \
550
+ --lora_dropout 0.0 \
551
+ --data_replay_freq -1 \
552
+ --chunk 4 \
553
+ --model_name specroute \
554
+ --threshold 0.995
555
+
556
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/checkpoint*
557
+
558
+ sleep 5
559
+
560
+
561
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
562
+ --do_train \
563
+ --gradient_checkpointing \
564
+ --do_predict \
565
+ --predict_with_generate \
566
+ --model_name_or_path $2 \
567
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights \
568
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/spectral_signatures.pt \
569
+ --data_dir CL_Benchmark \
570
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
571
+ --gen_data_dir generated_data/lora_gen_superni_llama \
572
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization \
573
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
574
+ --per_device_train_batch_size 1 \
575
+ --per_device_eval_batch_size 4 \
576
+ --gradient_accumulation_steps 32 \
577
+ --learning_rate 5e-05 \
578
+ --num_train_epochs 50 \
579
+ --fp16 \
580
+ --deepspeed configs/ds_configs/stage2_fp16.config \
581
+ --run_name gen_script_superni_order1_llama_specroute \
582
+ --max_source_length 512 \
583
+ --max_target_length 50 \
584
+ --generation_max_length 50 \
585
+ --add_task_name False \
586
+ --add_dataset_name False \
587
+ --overwrite_output_dir \
588
+ --overwrite_cache \
589
+ --lr_scheduler_type constant \
590
+ --warmup_steps 0 \
591
+ --logging_strategy steps \
592
+ --logging_steps 10 \
593
+ --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
594
+ --evaluation_strategy steps \
595
+ --save_strategy steps \
596
+ --save_total_limit 1 \
597
+ --load_best_model_at_end \
598
+ --lora_r 4 \
599
+ --lora_alpha 32 \
600
+ --lora_dropout 0.0 \
601
+ --data_replay_freq -1 \
602
+ --chunk 4 \
603
+ --model_name specroute \
604
+ --threshold 0.995
605
+
606
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/checkpoint*
607
+
608
+ sleep 5
609
+
610
+
611
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
612
+ --do_train \
613
+ --gradient_checkpointing \
614
+ --do_predict \
615
+ --predict_with_generate \
616
+ --model_name_or_path $2 \
617
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
618
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/spectral_signatures.pt \
619
+ --data_dir CL_Benchmark \
620
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
621
+ --gen_data_dir generated_data/lora_gen_superni_llama \
622
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation \
623
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation \
624
+ --per_device_train_batch_size 1 \
625
+ --per_device_eval_batch_size 4 \
626
+ --gradient_accumulation_steps 32 \
627
+ --learning_rate 5e-05 \
628
+ --num_train_epochs 50 \
629
+ --fp16 \
630
+ --deepspeed configs/ds_configs/stage2_fp16.config \
631
+ --run_name gen_script_superni_order1_llama_specroute \
632
+ --max_source_length 512 \
633
+ --max_target_length 50 \
634
+ --generation_max_length 50 \
635
+ --add_task_name False \
636
+ --add_dataset_name False \
637
+ --overwrite_output_dir \
638
+ --overwrite_cache \
639
+ --lr_scheduler_type constant \
640
+ --warmup_steps 0 \
641
+ --logging_strategy steps \
642
+ --logging_steps 10 \
643
+ --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
644
+ --evaluation_strategy steps \
645
+ --save_strategy steps \
646
+ --save_total_limit 1 \
647
+ --load_best_model_at_end \
648
+ --lora_r 4 \
649
+ --lora_alpha 32 \
650
+ --lora_dropout 0.0 \
651
+ --data_replay_freq -1 \
652
+ --chunk 4 \
653
+ --model_name specroute \
654
+ --threshold 0.995
655
+
656
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/checkpoint*
657
+
658
+ sleep 5
659
+
660
+
661
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
662
+ --do_train \
663
+ --gradient_checkpointing \
664
+ --do_predict \
665
+ --predict_with_generate \
666
+ --model_name_or_path $2 \
667
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/saved_weights \
668
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/saved_weights/spectral_signatures.pt \
669
+ --data_dir CL_Benchmark \
670
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
671
+ --gen_data_dir generated_data/lora_gen_superni_llama \
672
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification \
673
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification \
674
+ --per_device_train_batch_size 1 \
675
+ --per_device_eval_batch_size 4 \
676
+ --gradient_accumulation_steps 32 \
677
+ --learning_rate 5e-05 \
678
+ --num_train_epochs 50 \
679
+ --fp16 \
680
+ --deepspeed configs/ds_configs/stage2_fp16.config \
681
+ --run_name gen_script_superni_order1_llama_specroute \
682
+ --max_source_length 512 \
683
+ --max_target_length 50 \
684
+ --generation_max_length 50 \
685
+ --add_task_name False \
686
+ --add_dataset_name False \
687
+ --overwrite_output_dir \
688
+ --overwrite_cache \
689
+ --lr_scheduler_type constant \
690
+ --warmup_steps 0 \
691
+ --logging_strategy steps \
692
+ --logging_steps 10 \
693
+ --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
694
+ --evaluation_strategy steps \
695
+ --save_strategy steps \
696
+ --save_total_limit 1 \
697
+ --load_best_model_at_end \
698
+ --lora_r 4 \
699
+ --lora_alpha 32 \
700
+ --lora_dropout 0.0 \
701
+ --data_replay_freq -1 \
702
+ --chunk 4 \
703
+ --model_name specroute \
704
+ --threshold 0.995
705
+
706
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification/checkpoint*
707
+
708
+ sleep 5
709
+
710
+
711
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
712
+ --do_train \
713
+ --gradient_checkpointing \
714
+ --do_predict \
715
+ --predict_with_generate \
716
+ --model_name_or_path $2 \
717
+ --previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification/saved_weights \
718
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification/saved_weights/spectral_signatures.pt \
719
+ --data_dir CL_Benchmark \
720
+ --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
721
+ --gen_data_dir generated_data/lora_gen_superni_llama \
722
+ --task_config_dir configs/gen_script_superni_order1_llama_configs/task875_emotion_classification \
723
+ --output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification \
724
+ --per_device_train_batch_size 1 \
725
+ --per_device_eval_batch_size 4 \
726
+ --gradient_accumulation_steps 32 \
727
+ --learning_rate 5e-05 \
728
+ --num_train_epochs 50 \
729
+ --fp16 \
730
+ --deepspeed configs/ds_configs/stage2_fp16.config \
731
+ --run_name gen_script_superni_order1_llama_specroute \
732
+ --max_source_length 512 \
733
+ --max_target_length 50 \
734
+ --generation_max_length 50 \
735
+ --add_task_name False \
736
+ --add_dataset_name False \
737
+ --overwrite_output_dir \
738
+ --overwrite_cache \
739
+ --lr_scheduler_type constant \
740
+ --warmup_steps 0 \
741
+ --logging_strategy steps \
742
+ --logging_steps 10 \
743
+ --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
744
+ --evaluation_strategy steps \
745
+ --save_strategy steps \
746
+ --save_total_limit 1 \
747
+ --load_best_model_at_end \
748
+ --lora_r 4 \
749
+ --lora_alpha 32 \
750
+ --lora_dropout 0.0 \
751
+ --data_replay_freq -1 \
752
+ --chunk 4 \
753
+ --model_name specroute \
754
+ --threshold 0.995
755
+
756
+ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification/checkpoint*
757
+
758
+ sleep 5
759
+
760
+
761
+ python score.py gen_script_superni_order1_llama_specroute gen_script_superni_order1_llama_specroute
improve_gainlora/gen_script_superni_order1_t5_specroute.sh CHANGED
@@ -51,9 +51,14 @@ elif [ "$GPU_MEM" -gt 16000 ]; then
51
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
52
  else
53
  GPU_MODE="a100"
54
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
55
  FP16_FLAG=""
56
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
57
  fi
58
 
59
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
51
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
52
  else
53
  GPU_MODE="a100"
54
+ if [ "$NUM_GPUS" -ge 2 ]; then
55
+ GPU_IDS="0,1"
56
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
57
+ else
58
+ GPU_IDS="${1:-0}"
59
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
60
+ fi
61
  FP16_FLAG=""
 
62
  fi
63
 
64
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
improve_gainlora/gen_script_superni_order2_llama_specroute.sh CHANGED
@@ -8,9 +8,35 @@
8
  #SBATCH --gres=gpu:a100-sxm4-80gb:1
9
 
10
  export CUDA_DEVICE_ORDER="PCI_BUS_ID"
11
-
12
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  --do_train \
 
14
  --do_predict \
15
  --predict_with_generate \
16
  --model_name_or_path $2 \
@@ -20,7 +46,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
20
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection \
21
  --per_device_train_batch_size 4 \
22
  --per_device_eval_batch_size 8 \
23
- --gradient_accumulation_steps 8 \
24
  --learning_rate 5e-05 \
25
  --num_train_epochs 50 \
26
  --bf16 \
@@ -55,8 +81,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task
55
  sleep 5
56
 
57
 
58
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
59
  --do_train \
 
60
  --do_predict \
61
  --predict_with_generate \
62
  --model_name_or_path $2 \
@@ -69,7 +96,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
69
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation \
70
  --per_device_train_batch_size 8 \
71
  --per_device_eval_batch_size 8 \
72
- --gradient_accumulation_steps 4 \
73
  --learning_rate 5e-05 \
74
  --num_train_epochs 50 \
75
  --bf16 \
@@ -104,8 +131,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task
104
  sleep 5
105
 
106
 
107
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
108
  --do_train \
 
109
  --do_predict \
110
  --predict_with_generate \
111
  --model_name_or_path $2 \
@@ -118,7 +146,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
118
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation \
119
  --per_device_train_batch_size 2 \
120
  --per_device_eval_batch_size 8 \
121
- --gradient_accumulation_steps 16 \
122
  --learning_rate 5e-05 \
123
  --num_train_epochs 50 \
124
  --bf16 \
@@ -153,8 +181,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task
153
  sleep 5
154
 
155
 
156
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
157
  --do_train \
 
158
  --do_predict \
159
  --predict_with_generate \
160
  --model_name_or_path $2 \
@@ -167,7 +196,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
167
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation \
168
  --per_device_train_batch_size 16 \
169
  --per_device_eval_batch_size 8 \
170
- --gradient_accumulation_steps 2 \
171
  --learning_rate 5e-05 \
172
  --num_train_epochs 50 \
173
  --bf16 \
@@ -202,8 +231,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task
202
  sleep 5
203
 
204
 
205
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
206
  --do_train \
 
207
  --do_predict \
208
  --predict_with_generate \
209
  --model_name_or_path $2 \
@@ -216,7 +246,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
216
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary \
217
  --per_device_train_batch_size 2 \
218
  --per_device_eval_batch_size 8 \
219
- --gradient_accumulation_steps 16 \
220
  --learning_rate 5e-05 \
221
  --num_train_epochs 50 \
222
  --bf16 \
@@ -251,8 +281,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task
251
  sleep 5
252
 
253
 
254
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
255
  --do_train \
 
256
  --do_predict \
257
  --predict_with_generate \
258
  --model_name_or_path $2 \
@@ -265,7 +296,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
265
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification \
266
  --per_device_train_batch_size 8 \
267
  --per_device_eval_batch_size 8 \
268
- --gradient_accumulation_steps 4 \
269
  --learning_rate 5e-05 \
270
  --num_train_epochs 50 \
271
  --bf16 \
@@ -300,8 +331,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task
300
  sleep 5
301
 
302
 
303
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
304
  --do_train \
 
305
  --do_predict \
306
  --predict_with_generate \
307
  --model_name_or_path $2 \
@@ -314,7 +346,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
314
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation \
315
  --per_device_train_batch_size 8 \
316
  --per_device_eval_batch_size 8 \
317
- --gradient_accumulation_steps 4 \
318
  --learning_rate 5e-05 \
319
  --num_train_epochs 50 \
320
  --bf16 \
@@ -349,8 +381,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task
349
  sleep 5
350
 
351
 
352
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
353
  --do_train \
 
354
  --do_predict \
355
  --predict_with_generate \
356
  --model_name_or_path $2 \
@@ -363,7 +396,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
363
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification \
364
  --per_device_train_batch_size 16 \
365
  --per_device_eval_batch_size 8 \
366
- --gradient_accumulation_steps 2 \
367
  --learning_rate 5e-05 \
368
  --num_train_epochs 50 \
369
  --bf16 \
@@ -398,8 +431,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task
398
  sleep 5
399
 
400
 
401
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
402
  --do_train \
 
403
  --do_predict \
404
  --predict_with_generate \
405
  --model_name_or_path $2 \
@@ -412,7 +446,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
412
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction \
413
  --per_device_train_batch_size 16 \
414
  --per_device_eval_batch_size 8 \
415
- --gradient_accumulation_steps 2 \
416
  --learning_rate 5e-05 \
417
  --num_train_epochs 50 \
418
  --bf16 \
@@ -447,8 +481,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task
447
  sleep 5
448
 
449
 
450
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
451
  --do_train \
 
452
  --do_predict \
453
  --predict_with_generate \
454
  --model_name_or_path $2 \
@@ -461,7 +496,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
461
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next \
462
  --per_device_train_batch_size 2 \
463
  --per_device_eval_batch_size 8 \
464
- --gradient_accumulation_steps 16 \
465
  --learning_rate 5e-05 \
466
  --num_train_epochs 50 \
467
  --bf16 \
@@ -496,8 +531,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-tas
496
  sleep 5
497
 
498
 
499
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
500
  --do_train \
 
501
  --do_predict \
502
  --predict_with_generate \
503
  --model_name_or_path $2 \
@@ -510,7 +546,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
510
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction \
511
  --per_device_train_batch_size 4 \
512
  --per_device_eval_batch_size 8 \
513
- --gradient_accumulation_steps 8 \
514
  --learning_rate 5e-05 \
515
  --num_train_epochs 50 \
516
  --bf16 \
@@ -545,8 +581,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-tas
545
  sleep 5
546
 
547
 
548
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
549
  --do_train \
 
550
  --do_predict \
551
  --predict_with_generate \
552
  --model_name_or_path $2 \
@@ -559,7 +596,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
559
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
560
  --per_device_train_batch_size 1 \
561
  --per_device_eval_batch_size 8 \
562
- --gradient_accumulation_steps 32 \
563
  --learning_rate 5e-05 \
564
  --num_train_epochs 50 \
565
  --bf16 \
@@ -594,8 +631,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-tas
594
  sleep 5
595
 
596
 
597
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
598
  --do_train \
 
599
  --do_predict \
600
  --predict_with_generate \
601
  --model_name_or_path $2 \
@@ -608,7 +646,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
608
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation \
609
  --per_device_train_batch_size 1 \
610
  --per_device_eval_batch_size 8 \
611
- --gradient_accumulation_steps 32 \
612
  --learning_rate 5e-05 \
613
  --num_train_epochs 50 \
614
  --bf16 \
@@ -643,8 +681,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-tas
643
  sleep 5
644
 
645
 
646
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
647
  --do_train \
 
648
  --do_predict \
649
  --predict_with_generate \
650
  --model_name_or_path $2 \
@@ -657,7 +696,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
657
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization \
658
  --per_device_train_batch_size 1 \
659
  --per_device_eval_batch_size 8 \
660
- --gradient_accumulation_steps 32 \
661
  --learning_rate 5e-05 \
662
  --num_train_epochs 50 \
663
  --bf16 \
@@ -692,8 +731,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-tas
692
  sleep 5
693
 
694
 
695
- deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
696
  --do_train \
 
697
  --do_predict \
698
  --predict_with_generate \
699
  --model_name_or_path $2 \
@@ -706,7 +746,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
706
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification \
707
  --per_device_train_batch_size 4 \
708
  --per_device_eval_batch_size 8 \
709
- --gradient_accumulation_steps 8 \
710
  --learning_rate 5e-05 \
711
  --num_train_epochs 50 \
712
  --bf16 \
 
8
  #SBATCH --gres=gpu:a100-sxm4-80gb:1
9
 
10
  export CUDA_DEVICE_ORDER="PCI_BUS_ID"
11
+ # ── GPU auto-detection ─────────────────────────────────────────────────────
12
+ NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
13
+ GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
14
+
15
+ if [ -z "$GPU_MEM" ]; then
16
+ echo "[ERROR] No GPU detected. Exiting."
17
+ exit 1
18
+ fi
19
+
20
+ if [ "$NUM_GPUS" -ge 2 ] && [ "$GPU_MEM" -ge 20000 ]; then
21
+ # 2Γ— 3090/4090 (24 GB+): run DDP on both GPUs
22
+ DS_INCLUDE="localhost:0,1"
23
+ echo "[GPU] 2Γ— GPU detected (${GPU_MEM} MB each) β†’ DDP on localhost:0,1"
24
+ elif [ "$GPU_MEM" -lt 20000 ]; then
25
+ # P100/V100 16 GB: use the dedicated P100 script instead
26
+ echo "[GPU] WARNING: 16 GB GPU detected. Use the _p100.sh script instead."
27
+ echo " This script requires β‰₯ 20 GB VRAM per GPU."
28
+ exit 1
29
+ else
30
+ # Single high-memory GPU (A100/H100 or single 3090)
31
+ DS_INCLUDE="localhost:${1:-0}"
32
+ echo "[GPU] Single GPU detected (${GPU_MEM} MB) β†’ localhost:${1:-0}"
33
+ fi
34
+ # ──────────────────────────────────────────────────────────────────────────
35
+
36
+
37
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
38
  --do_train \
39
+ --gradient_checkpointing \
40
  --do_predict \
41
  --predict_with_generate \
42
  --model_name_or_path $2 \
 
46
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection \
47
  --per_device_train_batch_size 4 \
48
  --per_device_eval_batch_size 8 \
49
+ --gradient_accumulation_steps 4 \
50
  --learning_rate 5e-05 \
51
  --num_train_epochs 50 \
52
  --bf16 \
 
81
  sleep 5
82
 
83
 
84
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
85
  --do_train \
86
+ --gradient_checkpointing \
87
  --do_predict \
88
  --predict_with_generate \
89
  --model_name_or_path $2 \
 
96
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation \
97
  --per_device_train_batch_size 8 \
98
  --per_device_eval_batch_size 8 \
99
+ --gradient_accumulation_steps 2 \
100
  --learning_rate 5e-05 \
101
  --num_train_epochs 50 \
102
  --bf16 \
 
131
  sleep 5
132
 
133
 
134
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
135
  --do_train \
136
+ --gradient_checkpointing \
137
  --do_predict \
138
  --predict_with_generate \
139
  --model_name_or_path $2 \
 
146
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation \
147
  --per_device_train_batch_size 2 \
148
  --per_device_eval_batch_size 8 \
149
+ --gradient_accumulation_steps 8 \
150
  --learning_rate 5e-05 \
151
  --num_train_epochs 50 \
152
  --bf16 \
 
181
  sleep 5
182
 
183
 
184
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
185
  --do_train \
186
+ --gradient_checkpointing \
187
  --do_predict \
188
  --predict_with_generate \
189
  --model_name_or_path $2 \
 
196
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation \
197
  --per_device_train_batch_size 16 \
198
  --per_device_eval_batch_size 8 \
199
+ --gradient_accumulation_steps 1 \
200
  --learning_rate 5e-05 \
201
  --num_train_epochs 50 \
202
  --bf16 \
 
231
  sleep 5
232
 
233
 
234
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
235
  --do_train \
236
+ --gradient_checkpointing \
237
  --do_predict \
238
  --predict_with_generate \
239
  --model_name_or_path $2 \
 
246
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary \
247
  --per_device_train_batch_size 2 \
248
  --per_device_eval_batch_size 8 \
249
+ --gradient_accumulation_steps 8 \
250
  --learning_rate 5e-05 \
251
  --num_train_epochs 50 \
252
  --bf16 \
 
281
  sleep 5
282
 
283
 
284
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
285
  --do_train \
286
+ --gradient_checkpointing \
287
  --do_predict \
288
  --predict_with_generate \
289
  --model_name_or_path $2 \
 
296
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification \
297
  --per_device_train_batch_size 8 \
298
  --per_device_eval_batch_size 8 \
299
+ --gradient_accumulation_steps 2 \
300
  --learning_rate 5e-05 \
301
  --num_train_epochs 50 \
302
  --bf16 \
 
331
  sleep 5
332
 
333
 
334
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
335
  --do_train \
336
+ --gradient_checkpointing \
337
  --do_predict \
338
  --predict_with_generate \
339
  --model_name_or_path $2 \
 
346
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation \
347
  --per_device_train_batch_size 8 \
348
  --per_device_eval_batch_size 8 \
349
+ --gradient_accumulation_steps 2 \
350
  --learning_rate 5e-05 \
351
  --num_train_epochs 50 \
352
  --bf16 \
 
381
  sleep 5
382
 
383
 
384
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
385
  --do_train \
386
+ --gradient_checkpointing \
387
  --do_predict \
388
  --predict_with_generate \
389
  --model_name_or_path $2 \
 
396
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification \
397
  --per_device_train_batch_size 16 \
398
  --per_device_eval_batch_size 8 \
399
+ --gradient_accumulation_steps 1 \
400
  --learning_rate 5e-05 \
401
  --num_train_epochs 50 \
402
  --bf16 \
 
431
  sleep 5
432
 
433
 
434
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
435
  --do_train \
436
+ --gradient_checkpointing \
437
  --do_predict \
438
  --predict_with_generate \
439
  --model_name_or_path $2 \
 
446
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction \
447
  --per_device_train_batch_size 16 \
448
  --per_device_eval_batch_size 8 \
449
+ --gradient_accumulation_steps 1 \
450
  --learning_rate 5e-05 \
451
  --num_train_epochs 50 \
452
  --bf16 \
 
481
  sleep 5
482
 
483
 
484
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
485
  --do_train \
486
+ --gradient_checkpointing \
487
  --do_predict \
488
  --predict_with_generate \
489
  --model_name_or_path $2 \
 
496
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next \
497
  --per_device_train_batch_size 2 \
498
  --per_device_eval_batch_size 8 \
499
+ --gradient_accumulation_steps 8 \
500
  --learning_rate 5e-05 \
501
  --num_train_epochs 50 \
502
  --bf16 \
 
531
  sleep 5
532
 
533
 
534
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
535
  --do_train \
536
+ --gradient_checkpointing \
537
  --do_predict \
538
  --predict_with_generate \
539
  --model_name_or_path $2 \
 
546
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction \
547
  --per_device_train_batch_size 4 \
548
  --per_device_eval_batch_size 8 \
549
+ --gradient_accumulation_steps 4 \
550
  --learning_rate 5e-05 \
551
  --num_train_epochs 50 \
552
  --bf16 \
 
581
  sleep 5
582
 
583
 
584
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
585
  --do_train \
586
+ --gradient_checkpointing \
587
  --do_predict \
588
  --predict_with_generate \
589
  --model_name_or_path $2 \
 
596
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
597
  --per_device_train_batch_size 1 \
598
  --per_device_eval_batch_size 8 \
599
+ --gradient_accumulation_steps 16 \
600
  --learning_rate 5e-05 \
601
  --num_train_epochs 50 \
602
  --bf16 \
 
631
  sleep 5
632
 
633
 
634
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
635
  --do_train \
636
+ --gradient_checkpointing \
637
  --do_predict \
638
  --predict_with_generate \
639
  --model_name_or_path $2 \
 
646
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation \
647
  --per_device_train_batch_size 1 \
648
  --per_device_eval_batch_size 8 \
649
+ --gradient_accumulation_steps 16 \
650
  --learning_rate 5e-05 \
651
  --num_train_epochs 50 \
652
  --bf16 \
 
681
  sleep 5
682
 
683
 
684
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
685
  --do_train \
686
+ --gradient_checkpointing \
687
  --do_predict \
688
  --predict_with_generate \
689
  --model_name_or_path $2 \
 
696
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization \
697
  --per_device_train_batch_size 1 \
698
  --per_device_eval_batch_size 8 \
699
+ --gradient_accumulation_steps 16 \
700
  --learning_rate 5e-05 \
701
  --num_train_epochs 50 \
702
  --bf16 \
 
731
  sleep 5
732
 
733
 
734
+ deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
735
  --do_train \
736
+ --gradient_checkpointing \
737
  --do_predict \
738
  --predict_with_generate \
739
  --model_name_or_path $2 \
 
746
  --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification \
747
  --per_device_train_batch_size 4 \
748
  --per_device_eval_batch_size 8 \
749
+ --gradient_accumulation_steps 4 \
750
  --learning_rate 5e-05 \
751
  --num_train_epochs 50 \
752
  --bf16 \
improve_gainlora/gen_script_superni_order2_llama_specroute_p100.sh ADDED
@@ -0,0 +1,761 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # P100 (16 GB) variant β€” auto-generated by gen_p100_3090_scripts.py
3
+ # Changes vs H100 original:
4
+ # * bf16 β†’ fp16 (P100 has no bf16 hardware support)
5
+ # * DeepSpeed stage2_fp16.config (fp16 forced + ZeRO-2 CPU-optimizer offload)
6
+ # * BSZ=1, GA=original_BSZ Γ— original_GA (effective batch unchanged)
7
+ # * max_source_length 1024 β†’ 512 (halves activation memory)
8
+ # * per_device_eval_batch_size 8 β†’ 4
9
+ # * --gradient_checkpointing added
10
+ # * deepspeed --num_gpus 1 (single GPU, no include needed)
11
+
12
+ export CUDA_DEVICE_ORDER="PCI_BUS_ID"
13
+
14
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
15
+ --do_train \
16
+ --gradient_checkpointing \
17
+ --do_predict \
18
+ --predict_with_generate \
19
+ --model_name_or_path $2 \
20
+ --data_dir CL_Benchmark \
21
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
22
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection \
23
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection \
24
+ --per_device_train_batch_size 1 \
25
+ --per_device_eval_batch_size 4 \
26
+ --gradient_accumulation_steps 32 \
27
+ --learning_rate 5e-05 \
28
+ --num_train_epochs 50 \
29
+ --fp16 \
30
+ --deepspeed configs/ds_configs/stage2_fp16.config \
31
+ --run_name gen_script_superni_order2_llama_specroute \
32
+ --max_source_length 512 \
33
+ --max_target_length 50 \
34
+ --generation_max_length 50 \
35
+ --add_task_name False \
36
+ --add_dataset_name False \
37
+ --overwrite_output_dir \
38
+ --overwrite_cache \
39
+ --lr_scheduler_type constant \
40
+ --warmup_steps 0 \
41
+ --logging_strategy steps \
42
+ --logging_steps 10 \
43
+ --metric_for_best_model eval_rougeL \
44
+ --evaluation_strategy steps \
45
+ --save_strategy steps \
46
+ --save_total_limit 1 \
47
+ --load_best_model_at_end \
48
+ --lora_r 4 \
49
+ --lora_alpha 32 \
50
+ --lora_dropout 0.0 \
51
+ --data_replay_freq -1 \
52
+ --chunk 4 \
53
+ --model_name specroute \
54
+ --threshold 0.995
55
+
56
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/checkpoint*
57
+
58
+ sleep 5
59
+
60
+
61
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
62
+ --do_train \
63
+ --gradient_checkpointing \
64
+ --do_predict \
65
+ --predict_with_generate \
66
+ --model_name_or_path $2 \
67
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights \
68
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/spectral_signatures.pt \
69
+ --data_dir CL_Benchmark \
70
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
71
+ --gen_data_dir generated_data/lora_gen_superni_llama \
72
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation \
73
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation \
74
+ --per_device_train_batch_size 1 \
75
+ --per_device_eval_batch_size 4 \
76
+ --gradient_accumulation_steps 32 \
77
+ --learning_rate 5e-05 \
78
+ --num_train_epochs 50 \
79
+ --fp16 \
80
+ --deepspeed configs/ds_configs/stage2_fp16.config \
81
+ --run_name gen_script_superni_order2_llama_specroute \
82
+ --max_source_length 512 \
83
+ --max_target_length 50 \
84
+ --generation_max_length 50 \
85
+ --add_task_name False \
86
+ --add_dataset_name False \
87
+ --overwrite_output_dir \
88
+ --overwrite_cache \
89
+ --lr_scheduler_type constant \
90
+ --warmup_steps 0 \
91
+ --logging_strategy steps \
92
+ --logging_steps 10 \
93
+ --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
94
+ --evaluation_strategy steps \
95
+ --save_strategy steps \
96
+ --save_total_limit 1 \
97
+ --load_best_model_at_end \
98
+ --lora_r 4 \
99
+ --lora_alpha 32 \
100
+ --lora_dropout 0.0 \
101
+ --data_replay_freq -1 \
102
+ --chunk 4 \
103
+ --model_name specroute \
104
+ --threshold 0.995
105
+
106
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/checkpoint*
107
+
108
+ sleep 5
109
+
110
+
111
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
112
+ --do_train \
113
+ --gradient_checkpointing \
114
+ --do_predict \
115
+ --predict_with_generate \
116
+ --model_name_or_path $2 \
117
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights \
118
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/spectral_signatures.pt \
119
+ --data_dir CL_Benchmark \
120
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
121
+ --gen_data_dir generated_data/lora_gen_superni_llama \
122
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation \
123
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation \
124
+ --per_device_train_batch_size 1 \
125
+ --per_device_eval_batch_size 4 \
126
+ --gradient_accumulation_steps 32 \
127
+ --learning_rate 5e-05 \
128
+ --num_train_epochs 50 \
129
+ --fp16 \
130
+ --deepspeed configs/ds_configs/stage2_fp16.config \
131
+ --run_name gen_script_superni_order2_llama_specroute \
132
+ --max_source_length 512 \
133
+ --max_target_length 50 \
134
+ --generation_max_length 50 \
135
+ --add_task_name False \
136
+ --add_dataset_name False \
137
+ --overwrite_output_dir \
138
+ --overwrite_cache \
139
+ --lr_scheduler_type constant \
140
+ --warmup_steps 0 \
141
+ --logging_strategy steps \
142
+ --logging_steps 10 \
143
+ --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
144
+ --evaluation_strategy steps \
145
+ --save_strategy steps \
146
+ --save_total_limit 1 \
147
+ --load_best_model_at_end \
148
+ --lora_r 4 \
149
+ --lora_alpha 32 \
150
+ --lora_dropout 0.0 \
151
+ --data_replay_freq -1 \
152
+ --chunk 4 \
153
+ --model_name specroute \
154
+ --threshold 0.995
155
+
156
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/checkpoint*
157
+
158
+ sleep 5
159
+
160
+
161
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
162
+ --do_train \
163
+ --gradient_checkpointing \
164
+ --do_predict \
165
+ --predict_with_generate \
166
+ --model_name_or_path $2 \
167
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights \
168
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights/spectral_signatures.pt \
169
+ --data_dir CL_Benchmark \
170
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
171
+ --gen_data_dir generated_data/lora_gen_superni_llama \
172
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation \
173
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation \
174
+ --per_device_train_batch_size 1 \
175
+ --per_device_eval_batch_size 4 \
176
+ --gradient_accumulation_steps 32 \
177
+ --learning_rate 5e-05 \
178
+ --num_train_epochs 50 \
179
+ --fp16 \
180
+ --deepspeed configs/ds_configs/stage2_fp16.config \
181
+ --run_name gen_script_superni_order2_llama_specroute \
182
+ --max_source_length 512 \
183
+ --max_target_length 50 \
184
+ --generation_max_length 50 \
185
+ --add_task_name False \
186
+ --add_dataset_name False \
187
+ --overwrite_output_dir \
188
+ --overwrite_cache \
189
+ --lr_scheduler_type constant \
190
+ --warmup_steps 0 \
191
+ --logging_strategy steps \
192
+ --logging_steps 10 \
193
+ --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
194
+ --evaluation_strategy steps \
195
+ --save_strategy steps \
196
+ --save_total_limit 1 \
197
+ --load_best_model_at_end \
198
+ --lora_r 4 \
199
+ --lora_alpha 32 \
200
+ --lora_dropout 0.0 \
201
+ --data_replay_freq -1 \
202
+ --chunk 4 \
203
+ --model_name specroute \
204
+ --threshold 0.995
205
+
206
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/checkpoint*
207
+
208
+ sleep 5
209
+
210
+
211
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
212
+ --do_train \
213
+ --gradient_checkpointing \
214
+ --do_predict \
215
+ --predict_with_generate \
216
+ --model_name_or_path $2 \
217
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights \
218
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/spectral_signatures.pt \
219
+ --data_dir CL_Benchmark \
220
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
221
+ --gen_data_dir generated_data/lora_gen_superni_llama \
222
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary \
223
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary \
224
+ --per_device_train_batch_size 1 \
225
+ --per_device_eval_batch_size 4 \
226
+ --gradient_accumulation_steps 32 \
227
+ --learning_rate 5e-05 \
228
+ --num_train_epochs 50 \
229
+ --fp16 \
230
+ --deepspeed configs/ds_configs/stage2_fp16.config \
231
+ --run_name gen_script_superni_order2_llama_specroute \
232
+ --max_source_length 512 \
233
+ --max_target_length 50 \
234
+ --generation_max_length 50 \
235
+ --add_task_name False \
236
+ --add_dataset_name False \
237
+ --overwrite_output_dir \
238
+ --overwrite_cache \
239
+ --lr_scheduler_type constant \
240
+ --warmup_steps 0 \
241
+ --logging_strategy steps \
242
+ --logging_steps 10 \
243
+ --metric_for_best_model eval_rougeL_for_task1572_samsum_summary \
244
+ --evaluation_strategy steps \
245
+ --save_strategy steps \
246
+ --save_total_limit 1 \
247
+ --load_best_model_at_end \
248
+ --lora_r 4 \
249
+ --lora_alpha 32 \
250
+ --lora_dropout 0.0 \
251
+ --data_replay_freq -1 \
252
+ --chunk 4 \
253
+ --model_name specroute \
254
+ --threshold 0.995
255
+
256
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/checkpoint*
257
+
258
+ sleep 5
259
+
260
+
261
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
262
+ --do_train \
263
+ --gradient_checkpointing \
264
+ --do_predict \
265
+ --predict_with_generate \
266
+ --model_name_or_path $2 \
267
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights \
268
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights/spectral_signatures.pt \
269
+ --data_dir CL_Benchmark \
270
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
271
+ --gen_data_dir generated_data/lora_gen_superni_llama \
272
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification \
273
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification \
274
+ --per_device_train_batch_size 1 \
275
+ --per_device_eval_batch_size 4 \
276
+ --gradient_accumulation_steps 32 \
277
+ --learning_rate 5e-05 \
278
+ --num_train_epochs 50 \
279
+ --fp16 \
280
+ --deepspeed configs/ds_configs/stage2_fp16.config \
281
+ --run_name gen_script_superni_order2_llama_specroute \
282
+ --max_source_length 512 \
283
+ --max_target_length 50 \
284
+ --generation_max_length 50 \
285
+ --add_task_name False \
286
+ --add_dataset_name False \
287
+ --overwrite_output_dir \
288
+ --overwrite_cache \
289
+ --lr_scheduler_type constant \
290
+ --warmup_steps 0 \
291
+ --logging_strategy steps \
292
+ --logging_steps 10 \
293
+ --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
294
+ --evaluation_strategy steps \
295
+ --save_strategy steps \
296
+ --save_total_limit 1 \
297
+ --load_best_model_at_end \
298
+ --lora_r 4 \
299
+ --lora_alpha 32 \
300
+ --lora_dropout 0.0 \
301
+ --data_replay_freq -1 \
302
+ --chunk 4 \
303
+ --model_name specroute \
304
+ --threshold 0.995
305
+
306
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/checkpoint*
307
+
308
+ sleep 5
309
+
310
+
311
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
312
+ --do_train \
313
+ --gradient_checkpointing \
314
+ --do_predict \
315
+ --predict_with_generate \
316
+ --model_name_or_path $2 \
317
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights \
318
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights/spectral_signatures.pt \
319
+ --data_dir CL_Benchmark \
320
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
321
+ --gen_data_dir generated_data/lora_gen_superni_llama \
322
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation \
323
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation \
324
+ --per_device_train_batch_size 1 \
325
+ --per_device_eval_batch_size 4 \
326
+ --gradient_accumulation_steps 32 \
327
+ --learning_rate 5e-05 \
328
+ --num_train_epochs 50 \
329
+ --fp16 \
330
+ --deepspeed configs/ds_configs/stage2_fp16.config \
331
+ --run_name gen_script_superni_order2_llama_specroute \
332
+ --max_source_length 512 \
333
+ --max_target_length 50 \
334
+ --generation_max_length 50 \
335
+ --add_task_name False \
336
+ --add_dataset_name False \
337
+ --overwrite_output_dir \
338
+ --overwrite_cache \
339
+ --lr_scheduler_type constant \
340
+ --warmup_steps 0 \
341
+ --logging_strategy steps \
342
+ --logging_steps 10 \
343
+ --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
344
+ --evaluation_strategy steps \
345
+ --save_strategy steps \
346
+ --save_total_limit 1 \
347
+ --load_best_model_at_end \
348
+ --lora_r 4 \
349
+ --lora_alpha 32 \
350
+ --lora_dropout 0.0 \
351
+ --data_replay_freq -1 \
352
+ --chunk 4 \
353
+ --model_name specroute \
354
+ --threshold 0.995
355
+
356
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/checkpoint*
357
+
358
+ sleep 5
359
+
360
+
361
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
362
+ --do_train \
363
+ --gradient_checkpointing \
364
+ --do_predict \
365
+ --predict_with_generate \
366
+ --model_name_or_path $2 \
367
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights \
368
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights/spectral_signatures.pt \
369
+ --data_dir CL_Benchmark \
370
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
371
+ --gen_data_dir generated_data/lora_gen_superni_llama \
372
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification \
373
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification \
374
+ --per_device_train_batch_size 1 \
375
+ --per_device_eval_batch_size 4 \
376
+ --gradient_accumulation_steps 32 \
377
+ --learning_rate 5e-05 \
378
+ --num_train_epochs 50 \
379
+ --fp16 \
380
+ --deepspeed configs/ds_configs/stage2_fp16.config \
381
+ --run_name gen_script_superni_order2_llama_specroute \
382
+ --max_source_length 512 \
383
+ --max_target_length 50 \
384
+ --generation_max_length 50 \
385
+ --add_task_name False \
386
+ --add_dataset_name False \
387
+ --overwrite_output_dir \
388
+ --overwrite_cache \
389
+ --lr_scheduler_type constant \
390
+ --warmup_steps 0 \
391
+ --logging_strategy steps \
392
+ --logging_steps 10 \
393
+ --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
394
+ --evaluation_strategy steps \
395
+ --save_strategy steps \
396
+ --save_total_limit 1 \
397
+ --load_best_model_at_end \
398
+ --lora_r 4 \
399
+ --lora_alpha 32 \
400
+ --lora_dropout 0.0 \
401
+ --data_replay_freq -1 \
402
+ --chunk 4 \
403
+ --model_name specroute \
404
+ --threshold 0.995
405
+
406
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/checkpoint*
407
+
408
+ sleep 5
409
+
410
+
411
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
412
+ --do_train \
413
+ --gradient_checkpointing \
414
+ --do_predict \
415
+ --predict_with_generate \
416
+ --model_name_or_path $2 \
417
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights \
418
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights/spectral_signatures.pt \
419
+ --data_dir CL_Benchmark \
420
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
421
+ --gen_data_dir generated_data/lora_gen_superni_llama \
422
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction \
423
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction \
424
+ --per_device_train_batch_size 1 \
425
+ --per_device_eval_batch_size 4 \
426
+ --gradient_accumulation_steps 32 \
427
+ --learning_rate 5e-05 \
428
+ --num_train_epochs 50 \
429
+ --fp16 \
430
+ --deepspeed configs/ds_configs/stage2_fp16.config \
431
+ --run_name gen_script_superni_order2_llama_specroute \
432
+ --max_source_length 512 \
433
+ --max_target_length 50 \
434
+ --generation_max_length 50 \
435
+ --add_task_name False \
436
+ --add_dataset_name False \
437
+ --overwrite_output_dir \
438
+ --overwrite_cache \
439
+ --lr_scheduler_type constant \
440
+ --warmup_steps 0 \
441
+ --logging_strategy steps \
442
+ --logging_steps 10 \
443
+ --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
444
+ --evaluation_strategy steps \
445
+ --save_strategy steps \
446
+ --save_total_limit 1 \
447
+ --load_best_model_at_end \
448
+ --lora_r 4 \
449
+ --lora_alpha 32 \
450
+ --lora_dropout 0.0 \
451
+ --data_replay_freq -1 \
452
+ --chunk 4 \
453
+ --model_name specroute \
454
+ --threshold 0.995
455
+
456
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/checkpoint*
457
+
458
+ sleep 5
459
+
460
+
461
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
462
+ --do_train \
463
+ --gradient_checkpointing \
464
+ --do_predict \
465
+ --predict_with_generate \
466
+ --model_name_or_path $2 \
467
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights \
468
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights/spectral_signatures.pt \
469
+ --data_dir CL_Benchmark \
470
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
471
+ --gen_data_dir generated_data/lora_gen_superni_llama \
472
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next \
473
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next \
474
+ --per_device_train_batch_size 1 \
475
+ --per_device_eval_batch_size 4 \
476
+ --gradient_accumulation_steps 32 \
477
+ --learning_rate 5e-05 \
478
+ --num_train_epochs 50 \
479
+ --fp16 \
480
+ --deepspeed configs/ds_configs/stage2_fp16.config \
481
+ --run_name gen_script_superni_order2_llama_specroute \
482
+ --max_source_length 512 \
483
+ --max_target_length 50 \
484
+ --generation_max_length 50 \
485
+ --add_task_name False \
486
+ --add_dataset_name False \
487
+ --overwrite_output_dir \
488
+ --overwrite_cache \
489
+ --lr_scheduler_type constant \
490
+ --warmup_steps 0 \
491
+ --logging_strategy steps \
492
+ --logging_steps 10 \
493
+ --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
494
+ --evaluation_strategy steps \
495
+ --save_strategy steps \
496
+ --save_total_limit 1 \
497
+ --load_best_model_at_end \
498
+ --lora_r 4 \
499
+ --lora_alpha 32 \
500
+ --lora_dropout 0.0 \
501
+ --data_replay_freq -1 \
502
+ --chunk 4 \
503
+ --model_name specroute \
504
+ --threshold 0.995
505
+
506
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/checkpoint*
507
+
508
+ sleep 5
509
+
510
+
511
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
512
+ --do_train \
513
+ --gradient_checkpointing \
514
+ --do_predict \
515
+ --predict_with_generate \
516
+ --model_name_or_path $2 \
517
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights \
518
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights/spectral_signatures.pt \
519
+ --data_dir CL_Benchmark \
520
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
521
+ --gen_data_dir generated_data/lora_gen_superni_llama \
522
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction \
523
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction \
524
+ --per_device_train_batch_size 1 \
525
+ --per_device_eval_batch_size 4 \
526
+ --gradient_accumulation_steps 32 \
527
+ --learning_rate 5e-05 \
528
+ --num_train_epochs 50 \
529
+ --fp16 \
530
+ --deepspeed configs/ds_configs/stage2_fp16.config \
531
+ --run_name gen_script_superni_order2_llama_specroute \
532
+ --max_source_length 512 \
533
+ --max_target_length 50 \
534
+ --generation_max_length 50 \
535
+ --add_task_name False \
536
+ --add_dataset_name False \
537
+ --overwrite_output_dir \
538
+ --overwrite_cache \
539
+ --lr_scheduler_type constant \
540
+ --warmup_steps 0 \
541
+ --logging_strategy steps \
542
+ --logging_steps 10 \
543
+ --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
544
+ --evaluation_strategy steps \
545
+ --save_strategy steps \
546
+ --save_total_limit 1 \
547
+ --load_best_model_at_end \
548
+ --lora_r 4 \
549
+ --lora_alpha 32 \
550
+ --lora_dropout 0.0 \
551
+ --data_replay_freq -1 \
552
+ --chunk 4 \
553
+ --model_name specroute \
554
+ --threshold 0.995
555
+
556
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/checkpoint*
557
+
558
+ sleep 5
559
+
560
+
561
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
562
+ --do_train \
563
+ --gradient_checkpointing \
564
+ --do_predict \
565
+ --predict_with_generate \
566
+ --model_name_or_path $2 \
567
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights \
568
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights/spectral_signatures.pt \
569
+ --data_dir CL_Benchmark \
570
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
571
+ --gen_data_dir generated_data/lora_gen_superni_llama \
572
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization \
573
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
574
+ --per_device_train_batch_size 1 \
575
+ --per_device_eval_batch_size 4 \
576
+ --gradient_accumulation_steps 32 \
577
+ --learning_rate 5e-05 \
578
+ --num_train_epochs 50 \
579
+ --fp16 \
580
+ --deepspeed configs/ds_configs/stage2_fp16.config \
581
+ --run_name gen_script_superni_order2_llama_specroute \
582
+ --max_source_length 512 \
583
+ --max_target_length 50 \
584
+ --generation_max_length 50 \
585
+ --add_task_name False \
586
+ --add_dataset_name False \
587
+ --overwrite_output_dir \
588
+ --overwrite_cache \
589
+ --lr_scheduler_type constant \
590
+ --warmup_steps 0 \
591
+ --logging_strategy steps \
592
+ --logging_steps 10 \
593
+ --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
594
+ --evaluation_strategy steps \
595
+ --save_strategy steps \
596
+ --save_total_limit 1 \
597
+ --load_best_model_at_end \
598
+ --lora_r 4 \
599
+ --lora_alpha 32 \
600
+ --lora_dropout 0.0 \
601
+ --data_replay_freq -1 \
602
+ --chunk 4 \
603
+ --model_name specroute \
604
+ --threshold 0.995
605
+
606
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/checkpoint*
607
+
608
+ sleep 5
609
+
610
+
611
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
612
+ --do_train \
613
+ --gradient_checkpointing \
614
+ --do_predict \
615
+ --predict_with_generate \
616
+ --model_name_or_path $2 \
617
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
618
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/spectral_signatures.pt \
619
+ --data_dir CL_Benchmark \
620
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
621
+ --gen_data_dir generated_data/lora_gen_superni_llama \
622
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation \
623
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation \
624
+ --per_device_train_batch_size 1 \
625
+ --per_device_eval_batch_size 4 \
626
+ --gradient_accumulation_steps 32 \
627
+ --learning_rate 5e-05 \
628
+ --num_train_epochs 50 \
629
+ --fp16 \
630
+ --deepspeed configs/ds_configs/stage2_fp16.config \
631
+ --run_name gen_script_superni_order2_llama_specroute \
632
+ --max_source_length 512 \
633
+ --max_target_length 50 \
634
+ --generation_max_length 50 \
635
+ --add_task_name False \
636
+ --add_dataset_name False \
637
+ --overwrite_output_dir \
638
+ --overwrite_cache \
639
+ --lr_scheduler_type constant \
640
+ --warmup_steps 0 \
641
+ --logging_strategy steps \
642
+ --logging_steps 10 \
643
+ --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
644
+ --evaluation_strategy steps \
645
+ --save_strategy steps \
646
+ --save_total_limit 1 \
647
+ --load_best_model_at_end \
648
+ --lora_r 4 \
649
+ --lora_alpha 32 \
650
+ --lora_dropout 0.0 \
651
+ --data_replay_freq -1 \
652
+ --chunk 4 \
653
+ --model_name specroute \
654
+ --threshold 0.995
655
+
656
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/checkpoint*
657
+
658
+ sleep 5
659
+
660
+
661
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
662
+ --do_train \
663
+ --gradient_checkpointing \
664
+ --do_predict \
665
+ --predict_with_generate \
666
+ --model_name_or_path $2 \
667
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/saved_weights \
668
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/saved_weights/spectral_signatures.pt \
669
+ --data_dir CL_Benchmark \
670
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
671
+ --gen_data_dir generated_data/lora_gen_superni_llama \
672
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization \
673
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization \
674
+ --per_device_train_batch_size 1 \
675
+ --per_device_eval_batch_size 4 \
676
+ --gradient_accumulation_steps 32 \
677
+ --learning_rate 5e-05 \
678
+ --num_train_epochs 50 \
679
+ --fp16 \
680
+ --deepspeed configs/ds_configs/stage2_fp16.config \
681
+ --run_name gen_script_superni_order2_llama_specroute \
682
+ --max_source_length 512 \
683
+ --max_target_length 50 \
684
+ --generation_max_length 50 \
685
+ --add_task_name False \
686
+ --add_dataset_name False \
687
+ --overwrite_output_dir \
688
+ --overwrite_cache \
689
+ --lr_scheduler_type constant \
690
+ --warmup_steps 0 \
691
+ --logging_strategy steps \
692
+ --logging_steps 10 \
693
+ --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
694
+ --evaluation_strategy steps \
695
+ --save_strategy steps \
696
+ --save_total_limit 1 \
697
+ --load_best_model_at_end \
698
+ --lora_r 4 \
699
+ --lora_alpha 32 \
700
+ --lora_dropout 0.0 \
701
+ --data_replay_freq -1 \
702
+ --chunk 4 \
703
+ --model_name specroute \
704
+ --threshold 0.995
705
+
706
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization/checkpoint*
707
+
708
+ sleep 5
709
+
710
+
711
+ deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
712
+ --do_train \
713
+ --gradient_checkpointing \
714
+ --do_predict \
715
+ --predict_with_generate \
716
+ --model_name_or_path $2 \
717
+ --previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization/saved_weights \
718
+ --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization/saved_weights/spectral_signatures.pt \
719
+ --data_dir CL_Benchmark \
720
+ --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
721
+ --gen_data_dir generated_data/lora_gen_superni_llama \
722
+ --task_config_dir configs/gen_script_superni_order2_llama_configs/task875_emotion_classification \
723
+ --output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification \
724
+ --per_device_train_batch_size 1 \
725
+ --per_device_eval_batch_size 4 \
726
+ --gradient_accumulation_steps 32 \
727
+ --learning_rate 5e-05 \
728
+ --num_train_epochs 50 \
729
+ --fp16 \
730
+ --deepspeed configs/ds_configs/stage2_fp16.config \
731
+ --run_name gen_script_superni_order2_llama_specroute \
732
+ --max_source_length 512 \
733
+ --max_target_length 50 \
734
+ --generation_max_length 50 \
735
+ --add_task_name False \
736
+ --add_dataset_name False \
737
+ --overwrite_output_dir \
738
+ --overwrite_cache \
739
+ --lr_scheduler_type constant \
740
+ --warmup_steps 0 \
741
+ --logging_strategy steps \
742
+ --logging_steps 10 \
743
+ --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
744
+ --evaluation_strategy steps \
745
+ --save_strategy steps \
746
+ --save_total_limit 1 \
747
+ --load_best_model_at_end \
748
+ --lora_r 4 \
749
+ --lora_alpha 32 \
750
+ --lora_dropout 0.0 \
751
+ --data_replay_freq -1 \
752
+ --chunk 4 \
753
+ --model_name specroute \
754
+ --threshold 0.995
755
+
756
+ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification/checkpoint*
757
+
758
+ sleep 5
759
+
760
+
761
+ python score.py gen_script_superni_order2_llama_specroute gen_script_superni_order2_llama_specroute
improve_gainlora/gen_script_superni_order2_t5_specroute.sh CHANGED
@@ -49,9 +49,14 @@ elif [ "$GPU_MEM" -gt 16000 ]; then
49
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
50
  else
51
  GPU_MODE="a100"
52
- GPU_IDS="${1:-0}"
 
 
 
 
 
 
53
  FP16_FLAG=""
54
- echo "[GPU] Strategy: A100 (single GPU, fp32)"
55
  fi
56
 
57
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
 
49
  echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
50
  else
51
  GPU_MODE="a100"
52
+ if [ "$NUM_GPUS" -ge 2 ]; then
53
+ GPU_IDS="0,1"
54
+ echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
55
+ else
56
+ GPU_IDS="${1:-0}"
57
+ echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
58
+ fi
59
  FP16_FLAG=""
 
60
  fi
61
 
62
  echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
improve_gainlora/patch_t5_scripts.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Patches all T5 SpecRoute shell scripts to:
4
+ 1. T5_small scripts: Replace GPU detection block (add P100 + 3090 2-GPU + gradient_checkpointing)
5
+ 2. T5_small Long scripts: Reduce BSZ for P100/T4 (EVAL_BSZ 256->32, add p100 branch)
6
+ 3. T5_small SuperNI scripts: Add p100 BSZ branch
7
+ 4. Root T5 scripts: Add 3090 2-GPU support to a100 else branch
8
+ """
9
+ import os
10
+
11
+ BASE = "/Users/nnminh322/Desktop/personal/Continual/improve_gainlora"
12
+
13
+ # ── Replacements for T5_small scripts ────────────────────────────────────────
14
+
15
+ # GPU detection block β€” Long scripts (no NOTE comment)
16
+ OLD_DETECT_LONG = (
17
+ "# Determine GPU type\n"
18
+ 'if [ "$GPU_MEM" -lt 20000 ]; then\n'
19
+ " IS_T4=1\n"
20
+ ' echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"\n'
21
+ "else\n"
22
+ " IS_T4=0\n"
23
+ ' echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"\n'
24
+ "fi\n"
25
+ "\n"
26
+ "# Determine parallelism strategy\n"
27
+ 'if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then\n'
28
+ ' GPU_MODE="t4_2gpu"\n'
29
+ ' GPU_IDS="0,1"\n'
30
+ ' FP16_FLAG=""\n'
31
+ ' echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"\n'
32
+ 'elif [ "$IS_T4" -eq 1 ]; then\n'
33
+ ' GPU_MODE="t4_1gpu"\n'
34
+ ' GPU_IDS="${1:-0}"\n'
35
+ ' FP16_FLAG=""\n'
36
+ ' echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"\n'
37
+ "else\n"
38
+ ' GPU_MODE="a100"\n'
39
+ ' GPU_IDS="${1:-0}"\n'
40
+ ' FP16_FLAG=""\n'
41
+ ' echo "[GPU] Strategy: A100 (single GPU, fp32)"\n'
42
+ "fi"
43
+ )
44
+
45
+ # SuperNI T5_small scripts have this NOTE comment block instead
46
+ OLD_DETECT_SUPERNI = (
47
+ "# NOTE: T5 models trained in bfloat16 produce NaN with fp16 (overflow).\n"
48
+ "# T4 GPUs do not support bf16. Use fp32 + gradient_checkpointing instead.\n"
49
+ 'if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then\n'
50
+ ' GPU_MODE="t4_2gpu"\n'
51
+ ' GPU_IDS="0,1"\n'
52
+ ' FP16_FLAG=""\n'
53
+ ' echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"\n'
54
+ 'elif [ "$IS_T4" -eq 1 ]; then\n'
55
+ ' GPU_MODE="t4_1gpu"\n'
56
+ ' GPU_IDS="${1:-0}"\n'
57
+ ' FP16_FLAG=""\n'
58
+ ' echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"\n'
59
+ "else\n"
60
+ ' GPU_MODE="a100"\n'
61
+ ' GPU_IDS="${1:-0}"\n'
62
+ ' FP16_FLAG=""\n'
63
+ ' echo "[GPU] Strategy: A100 (single GPU, fp32)"\n'
64
+ "fi"
65
+ )
66
+
67
+ # New unified GPU detection block for all T5_small scripts
68
+ NEW_DETECT = (
69
+ "# GPU type detection\n"
70
+ "# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000\n"
71
+ 'if [ "$GPU_MEM" -lt 15500 ]; then\n'
72
+ ' GPU_TYPE="t4"\n'
73
+ ' echo "[GPU] Detected T4 (${GPU_MEM}MB)"\n'
74
+ 'elif [ "$GPU_MEM" -le 17000 ]; then\n'
75
+ ' GPU_TYPE="p100"\n'
76
+ ' echo "[GPU] Detected P100 (${GPU_MEM}MB)"\n'
77
+ "else\n"
78
+ ' GPU_TYPE="highvram"\n'
79
+ ' echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"\n'
80
+ "fi\n"
81
+ "\n"
82
+ "# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs\n"
83
+ 'if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then\n'
84
+ ' GPU_MODE="t4_2gpu"\n'
85
+ ' GPU_IDS="0,1"\n'
86
+ ' FP16_FLAG="--gradient_checkpointing"\n'
87
+ ' echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"\n'
88
+ 'elif [ "$GPU_TYPE" = "t4" ]; then\n'
89
+ ' GPU_MODE="t4_1gpu"\n'
90
+ ' GPU_IDS="${1:-0}"\n'
91
+ ' FP16_FLAG="--gradient_checkpointing"\n'
92
+ ' echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"\n'
93
+ 'elif [ "$GPU_TYPE" = "p100" ]; then\n'
94
+ ' GPU_MODE="p100"\n'
95
+ ' GPU_IDS="${1:-0}"\n'
96
+ ' FP16_FLAG="--gradient_checkpointing"\n'
97
+ ' echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"\n'
98
+ "else\n"
99
+ ' GPU_MODE="a100"\n'
100
+ ' if [ "$NUM_GPUS" -ge 2 ]; then\n'
101
+ ' GPU_IDS="0,1"\n'
102
+ ' echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"\n'
103
+ " else\n"
104
+ ' GPU_IDS="${1:-0}"\n'
105
+ ' echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"\n'
106
+ " fi\n"
107
+ ' FP16_FLAG=""\n'
108
+ "fi"
109
+ )
110
+
111
+ # ── Per-task BSZ blocks ───────────────────────────────────────────────────────
112
+
113
+ # Long T5_small scripts: old BSZ block (with space before EVAL_BSZ=256)
114
+ OLD_BSZ_LONG_A = (
115
+ 'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
116
+ " BSZ=16; GA=1; EVAL_BSZ=256\n"
117
+ 'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
118
+ " BSZ=32; GA=1; EVAL_BSZ=256\n"
119
+ "else\n"
120
+ " BSZ=64; GA=1; EVAL_BSZ=512\n"
121
+ "fi"
122
+ )
123
+
124
+ NEW_BSZ_LONG = (
125
+ 'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
126
+ " BSZ=8; GA=2; EVAL_BSZ=64\n"
127
+ 'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
128
+ " BSZ=8; GA=2; EVAL_BSZ=32\n"
129
+ 'elif [ "$GPU_MODE" = "p100" ]; then\n'
130
+ " BSZ=16; GA=2; EVAL_BSZ=32\n"
131
+ "else\n"
132
+ " BSZ=64; GA=1; EVAL_BSZ=128\n"
133
+ "fi"
134
+ )
135
+
136
+ # SuperNI T5_small: old BSZ block (2 space variants)
137
+ OLD_BSZ_SUPERNI_A = (
138
+ 'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
139
+ " BSZ=8; GA=2; EVAL_BSZ=16\n"
140
+ 'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
141
+ " BSZ=16; GA=2; EVAL_BSZ=16\n"
142
+ "else\n"
143
+ " BSZ=32; GA=1; EVAL_BSZ=32\n"
144
+ "fi"
145
+ )
146
+
147
+ OLD_BSZ_SUPERNI_B = (
148
+ 'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
149
+ " BSZ=8; GA=2; EVAL_BSZ=16\n"
150
+ 'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
151
+ " BSZ=16; GA=2; EVAL_BSZ=16\n"
152
+ "else\n"
153
+ " BSZ=32; GA=1; EVAL_BSZ=32\n"
154
+ "fi"
155
+ )
156
+
157
+ NEW_BSZ_SUPERNI = (
158
+ 'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
159
+ " BSZ=8; GA=2; EVAL_BSZ=16\n"
160
+ 'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
161
+ " BSZ=8; GA=2; EVAL_BSZ=16\n"
162
+ 'elif [ "$GPU_MODE" = "p100" ]; then\n'
163
+ " BSZ=16; GA=2; EVAL_BSZ=16\n"
164
+ "else\n"
165
+ " BSZ=32; GA=1; EVAL_BSZ=32\n"
166
+ "fi"
167
+ )
168
+
169
+ # ── Root T5 scripts: add 3090 2-GPU to a100 branch ───────────────────────────
170
+
171
+ OLD_A100_ROOT = (
172
+ "else\n"
173
+ ' GPU_MODE="a100"\n'
174
+ ' GPU_IDS="${1:-0}"\n'
175
+ ' FP16_FLAG=""\n'
176
+ ' echo "[GPU] Strategy: A100 (single GPU, fp32)"\n'
177
+ "fi"
178
+ )
179
+
180
+ NEW_A100_ROOT = (
181
+ "else\n"
182
+ ' GPU_MODE="a100"\n'
183
+ ' if [ "$NUM_GPUS" -ge 2 ]; then\n'
184
+ ' GPU_IDS="0,1"\n'
185
+ ' echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"\n'
186
+ " else\n"
187
+ ' GPU_IDS="${1:-0}"\n'
188
+ ' echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"\n'
189
+ " fi\n"
190
+ ' FP16_FLAG=""\n'
191
+ "fi"
192
+ )
193
+
194
+
195
+ def patch(path, replacements):
196
+ full = os.path.join(BASE, path)
197
+ if not os.path.exists(full):
198
+ print(f" SKIP (not found): {path}")
199
+ return
200
+ with open(full) as f:
201
+ content = f.read()
202
+ original = content
203
+ for old, new in replacements:
204
+ n = content.count(old)
205
+ if n > 0:
206
+ content = content.replace(old, new)
207
+ print(f" {path}: replaced {n}x '{old[:50].strip()}...'")
208
+ if content != original:
209
+ with open(full, "w") as f:
210
+ f.write(content)
211
+ else:
212
+ print(f" {path}: no changes (already patched or pattern mismatch)")
213
+
214
+
215
+ T5_SMALL_LONG = [
216
+ "T5_small/gen_script_long_order3_t5_small_specroute.sh",
217
+ "T5_small/gen_script_long_order3_t5_small_specroute_v10a.sh",
218
+ "T5_small/gen_script_long_order3_t5_small_specroute_v10b.sh",
219
+ "T5_small/gen_script_long_order4_t5_small_specroute.sh",
220
+ ]
221
+
222
+ T5_SMALL_SUPERNI = [
223
+ "T5_small/gen_script_superni_order1_t5_small_specroute.sh",
224
+ "T5_small/gen_script_superni_order2_t5_small_specroute.sh",
225
+ ]
226
+
227
+ ROOT_T5 = [
228
+ "gen_script_long_order3_t5_specroute.sh",
229
+ "gen_script_long_order4_t5_specroute.sh",
230
+ "gen_script_superni_order1_t5_specroute.sh",
231
+ "gen_script_superni_order2_t5_specroute.sh",
232
+ ]
233
+
234
+ print("=== T5_small Long scripts ===")
235
+ for p in T5_SMALL_LONG:
236
+ patch(p, [
237
+ (OLD_DETECT_LONG, NEW_DETECT),
238
+ (OLD_BSZ_LONG_A, NEW_BSZ_LONG),
239
+ ])
240
+
241
+ print("\n=== T5_small SuperNI scripts ===")
242
+ for p in T5_SMALL_SUPERNI:
243
+ patch(p, [
244
+ (OLD_DETECT_SUPERNI, NEW_DETECT),
245
+ (OLD_DETECT_LONG, NEW_DETECT),
246
+ (OLD_BSZ_SUPERNI_A, NEW_BSZ_SUPERNI),
247
+ (OLD_BSZ_SUPERNI_B, NEW_BSZ_SUPERNI),
248
+ ])
249
+
250
+ print("\n=== Root T5 scripts (add 3090 2-GPU to a100 branch) ===")
251
+ for p in ROOT_T5:
252
+ patch(p, [(OLD_A100_ROOT, NEW_A100_ROOT)])
253
+
254
+ print("\nDone.")