v10a
Browse files- improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute.sh +103 -59
- improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute_v10a.sh +103 -59
- improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute_v10b.sh +103 -59
- improve_gainlora/T5_small/gen_script_long_order4_t5_small_specroute.sh +103 -59
- improve_gainlora/T5_small/gen_script_superni_order1_t5_small_specroute.sh +61 -9
- improve_gainlora/T5_small/gen_script_superni_order2_t5_small_specroute.sh +58 -14
- improve_gainlora/gen_script_long_order3_t5_specroute.sh +7 -2
- improve_gainlora/gen_script_long_order4_t5_specroute.sh +7 -2
- improve_gainlora/gen_script_superni_order1_llama_specroute.sh +71 -31
- improve_gainlora/gen_script_superni_order1_llama_specroute_p100.sh +761 -0
- improve_gainlora/gen_script_superni_order1_t5_specroute.sh +7 -2
- improve_gainlora/gen_script_superni_order2_llama_specroute.sh +71 -31
- improve_gainlora/gen_script_superni_order2_llama_specroute_p100.sh +761 -0
- improve_gainlora/gen_script_superni_order2_t5_specroute.sh +7 -2
- improve_gainlora/patch_t5_scripts.py +254 -0
improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute.sh
CHANGED
|
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
-
#
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
else
|
| 30 |
-
|
| 31 |
-
echo "[GPU] Detected high-
|
| 32 |
fi
|
| 33 |
|
| 34 |
-
#
|
| 35 |
-
if [ "$
|
| 36 |
GPU_MODE="t4_2gpu"
|
| 37 |
GPU_IDS="0,1"
|
| 38 |
-
FP16_FLAG=""
|
| 39 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 40 |
-
elif [ "$
|
| 41 |
GPU_MODE="t4_1gpu"
|
| 42 |
GPU_IDS="${1:-0}"
|
| 43 |
-
FP16_FLAG=""
|
| 44 |
-
echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
else
|
| 46 |
GPU_MODE="a100"
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
FP16_FLAG=""
|
| 49 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 50 |
fi
|
| 51 |
|
| 52 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
@@ -54,11 +68,13 @@ echo "============================================================"
|
|
| 54 |
echo ""
|
| 55 |
|
| 56 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 57 |
-
BSZ=
|
| 58 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 59 |
-
BSZ=
|
|
|
|
|
|
|
| 60 |
else
|
| 61 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 62 |
fi
|
| 63 |
|
| 64 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -107,11 +123,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp
|
|
| 107 |
sleep 5
|
| 108 |
|
| 109 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 110 |
-
BSZ=
|
| 111 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 112 |
-
BSZ=
|
|
|
|
|
|
|
| 113 |
else
|
| 114 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 115 |
fi
|
| 116 |
|
| 117 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -161,11 +179,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amaz
|
|
| 161 |
sleep 5
|
| 162 |
|
| 163 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 164 |
-
BSZ=
|
| 165 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 166 |
-
BSZ=
|
|
|
|
|
|
|
| 167 |
else
|
| 168 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 169 |
fi
|
| 170 |
|
| 171 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -215,11 +235,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli
|
|
| 215 |
sleep 5
|
| 216 |
|
| 217 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 218 |
-
BSZ=
|
| 219 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 220 |
-
BSZ=
|
|
|
|
|
|
|
| 221 |
else
|
| 222 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 223 |
fi
|
| 224 |
|
| 225 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -269,11 +291,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/c
|
|
| 269 |
sleep 5
|
| 270 |
|
| 271 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 272 |
-
BSZ=
|
| 273 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 274 |
-
BSZ=
|
|
|
|
|
|
|
| 275 |
else
|
| 276 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 277 |
fi
|
| 278 |
|
| 279 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -323,11 +347,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa
|
|
| 323 |
sleep 5
|
| 324 |
|
| 325 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 326 |
-
BSZ=
|
| 327 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 328 |
-
BSZ=
|
|
|
|
|
|
|
| 329 |
else
|
| 330 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 331 |
fi
|
| 332 |
|
| 333 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -377,11 +403,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/
|
|
| 377 |
sleep 5
|
| 378 |
|
| 379 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 380 |
-
BSZ=
|
| 381 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 382 |
-
BSZ=
|
|
|
|
|
|
|
| 383 |
else
|
| 384 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 385 |
fi
|
| 386 |
|
| 387 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -431,11 +459,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/
|
|
| 431 |
sleep 5
|
| 432 |
|
| 433 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 434 |
-
BSZ=
|
| 435 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 436 |
-
BSZ=
|
|
|
|
|
|
|
| 437 |
else
|
| 438 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 439 |
fi
|
| 440 |
|
| 441 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -485,11 +515,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb
|
|
| 485 |
sleep 5
|
| 486 |
|
| 487 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 488 |
-
BSZ=
|
| 489 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 490 |
-
BSZ=
|
|
|
|
|
|
|
| 491 |
else
|
| 492 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 493 |
fi
|
| 494 |
|
| 495 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -539,11 +571,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2
|
|
| 539 |
sleep 5
|
| 540 |
|
| 541 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 542 |
-
BSZ=
|
| 543 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 544 |
-
BSZ=
|
|
|
|
|
|
|
| 545 |
else
|
| 546 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 547 |
fi
|
| 548 |
|
| 549 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -593,11 +627,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbp
|
|
| 593 |
sleep 5
|
| 594 |
|
| 595 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 596 |
-
BSZ=
|
| 597 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 598 |
-
BSZ=
|
|
|
|
|
|
|
| 599 |
else
|
| 600 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 601 |
fi
|
| 602 |
|
| 603 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -647,11 +683,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agn
|
|
| 647 |
sleep 5
|
| 648 |
|
| 649 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 650 |
-
BSZ=
|
| 651 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 652 |
-
BSZ=
|
|
|
|
|
|
|
| 653 |
else
|
| 654 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 655 |
fi
|
| 656 |
|
| 657 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -701,11 +739,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/12-yah
|
|
| 701 |
sleep 5
|
| 702 |
|
| 703 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 704 |
-
BSZ=
|
| 705 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 706 |
-
BSZ=
|
|
|
|
|
|
|
| 707 |
else
|
| 708 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 709 |
fi
|
| 710 |
|
| 711 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -755,11 +795,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/13-mul
|
|
| 755 |
sleep 5
|
| 756 |
|
| 757 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 758 |
-
BSZ=
|
| 759 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 760 |
-
BSZ=
|
|
|
|
|
|
|
| 761 |
else
|
| 762 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 763 |
fi
|
| 764 |
|
| 765 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -809,11 +851,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/14-boo
|
|
| 809 |
sleep 5
|
| 810 |
|
| 811 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 812 |
-
BSZ=
|
| 813 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 814 |
-
BSZ=
|
|
|
|
|
|
|
| 815 |
else
|
| 816 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 817 |
fi
|
| 818 |
|
| 819 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
+
# GPU type detection
|
| 26 |
+
# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
|
| 27 |
+
if [ "$GPU_MEM" -lt 15500 ]; then
|
| 28 |
+
GPU_TYPE="t4"
|
| 29 |
+
echo "[GPU] Detected T4 (${GPU_MEM}MB)"
|
| 30 |
+
elif [ "$GPU_MEM" -le 17000 ]; then
|
| 31 |
+
GPU_TYPE="p100"
|
| 32 |
+
echo "[GPU] Detected P100 (${GPU_MEM}MB)"
|
| 33 |
else
|
| 34 |
+
GPU_TYPE="highvram"
|
| 35 |
+
echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
|
| 36 |
fi
|
| 37 |
|
| 38 |
+
# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
|
| 39 |
+
if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
|
| 40 |
GPU_MODE="t4_2gpu"
|
| 41 |
GPU_IDS="0,1"
|
| 42 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 43 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 44 |
+
elif [ "$GPU_TYPE" = "t4" ]; then
|
| 45 |
GPU_MODE="t4_1gpu"
|
| 46 |
GPU_IDS="${1:-0}"
|
| 47 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 48 |
+
echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
|
| 49 |
+
elif [ "$GPU_TYPE" = "p100" ]; then
|
| 50 |
+
GPU_MODE="p100"
|
| 51 |
+
GPU_IDS="${1:-0}"
|
| 52 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 53 |
+
echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
|
| 54 |
else
|
| 55 |
GPU_MODE="a100"
|
| 56 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 57 |
+
GPU_IDS="0,1"
|
| 58 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 59 |
+
else
|
| 60 |
+
GPU_IDS="${1:-0}"
|
| 61 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 62 |
+
fi
|
| 63 |
FP16_FLAG=""
|
|
|
|
| 64 |
fi
|
| 65 |
|
| 66 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 68 |
echo ""
|
| 69 |
|
| 70 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 71 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 72 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 73 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 74 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 75 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 76 |
else
|
| 77 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 78 |
fi
|
| 79 |
|
| 80 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 123 |
sleep 5
|
| 124 |
|
| 125 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 126 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 127 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 128 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 129 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 130 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 131 |
else
|
| 132 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 133 |
fi
|
| 134 |
|
| 135 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 179 |
sleep 5
|
| 180 |
|
| 181 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 182 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 183 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 184 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 185 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 186 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 187 |
else
|
| 188 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 189 |
fi
|
| 190 |
|
| 191 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 235 |
sleep 5
|
| 236 |
|
| 237 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 238 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 239 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 240 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 241 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 242 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 243 |
else
|
| 244 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 245 |
fi
|
| 246 |
|
| 247 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 291 |
sleep 5
|
| 292 |
|
| 293 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 294 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 295 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 296 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 297 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 298 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 299 |
else
|
| 300 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 301 |
fi
|
| 302 |
|
| 303 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 347 |
sleep 5
|
| 348 |
|
| 349 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 350 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 351 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 352 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 353 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 354 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 355 |
else
|
| 356 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 357 |
fi
|
| 358 |
|
| 359 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 403 |
sleep 5
|
| 404 |
|
| 405 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 406 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 407 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 408 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 409 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 410 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 411 |
else
|
| 412 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 413 |
fi
|
| 414 |
|
| 415 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 459 |
sleep 5
|
| 460 |
|
| 461 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 462 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 463 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 464 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 465 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 466 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 467 |
else
|
| 468 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 469 |
fi
|
| 470 |
|
| 471 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 515 |
sleep 5
|
| 516 |
|
| 517 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 518 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 519 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 520 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 521 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 522 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 523 |
else
|
| 524 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 525 |
fi
|
| 526 |
|
| 527 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 571 |
sleep 5
|
| 572 |
|
| 573 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 574 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 575 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 576 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 577 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 578 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 579 |
else
|
| 580 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 581 |
fi
|
| 582 |
|
| 583 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 627 |
sleep 5
|
| 628 |
|
| 629 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 630 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 631 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 632 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 633 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 634 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 635 |
else
|
| 636 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 637 |
fi
|
| 638 |
|
| 639 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 683 |
sleep 5
|
| 684 |
|
| 685 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 686 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 687 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 688 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 689 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 690 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 691 |
else
|
| 692 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 693 |
fi
|
| 694 |
|
| 695 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 739 |
sleep 5
|
| 740 |
|
| 741 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 742 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 743 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 744 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 745 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 746 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 747 |
else
|
| 748 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 749 |
fi
|
| 750 |
|
| 751 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 795 |
sleep 5
|
| 796 |
|
| 797 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 798 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 799 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 800 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 801 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 802 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 803 |
else
|
| 804 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 805 |
fi
|
| 806 |
|
| 807 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 851 |
sleep 5
|
| 852 |
|
| 853 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 854 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 855 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 856 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 857 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 858 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 859 |
else
|
| 860 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 861 |
fi
|
| 862 |
|
| 863 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute_v10a.sh
CHANGED
|
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
-
#
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
else
|
| 30 |
-
|
| 31 |
-
echo "[GPU] Detected high-
|
| 32 |
fi
|
| 33 |
|
| 34 |
-
#
|
| 35 |
-
if [ "$
|
| 36 |
GPU_MODE="t4_2gpu"
|
| 37 |
GPU_IDS="0,1"
|
| 38 |
-
FP16_FLAG=""
|
| 39 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 40 |
-
elif [ "$
|
| 41 |
GPU_MODE="t4_1gpu"
|
| 42 |
GPU_IDS="${1:-0}"
|
| 43 |
-
FP16_FLAG=""
|
| 44 |
-
echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
else
|
| 46 |
GPU_MODE="a100"
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
FP16_FLAG=""
|
| 49 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 50 |
fi
|
| 51 |
|
| 52 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
@@ -54,11 +68,13 @@ echo "============================================================"
|
|
| 54 |
echo ""
|
| 55 |
|
| 56 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 57 |
-
BSZ=
|
| 58 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 59 |
-
BSZ=
|
|
|
|
|
|
|
| 60 |
else
|
| 61 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 62 |
fi
|
| 63 |
|
| 64 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -108,11 +124,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
|
|
| 108 |
sleep 5
|
| 109 |
|
| 110 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 111 |
-
BSZ=
|
| 112 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 113 |
-
BSZ=
|
|
|
|
|
|
|
| 114 |
else
|
| 115 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 116 |
fi
|
| 117 |
|
| 118 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -165,11 +183,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/2
|
|
| 165 |
sleep 5
|
| 166 |
|
| 167 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 168 |
-
BSZ=
|
| 169 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 170 |
-
BSZ=
|
|
|
|
|
|
|
| 171 |
else
|
| 172 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 173 |
fi
|
| 174 |
|
| 175 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -222,11 +242,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/3
|
|
| 222 |
sleep 5
|
| 223 |
|
| 224 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 225 |
-
BSZ=
|
| 226 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 227 |
-
BSZ=
|
|
|
|
|
|
|
| 228 |
else
|
| 229 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 230 |
fi
|
| 231 |
|
| 232 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -279,11 +301,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/4
|
|
| 279 |
sleep 5
|
| 280 |
|
| 281 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 282 |
-
BSZ=
|
| 283 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 284 |
-
BSZ=
|
|
|
|
|
|
|
| 285 |
else
|
| 286 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 287 |
fi
|
| 288 |
|
| 289 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -336,11 +360,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/5
|
|
| 336 |
sleep 5
|
| 337 |
|
| 338 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 339 |
-
BSZ=
|
| 340 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 341 |
-
BSZ=
|
|
|
|
|
|
|
| 342 |
else
|
| 343 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 344 |
fi
|
| 345 |
|
| 346 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -393,11 +419,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/6
|
|
| 393 |
sleep 5
|
| 394 |
|
| 395 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 396 |
-
BSZ=
|
| 397 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 398 |
-
BSZ=
|
|
|
|
|
|
|
| 399 |
else
|
| 400 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 401 |
fi
|
| 402 |
|
| 403 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -450,11 +478,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/7
|
|
| 450 |
sleep 5
|
| 451 |
|
| 452 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 453 |
-
BSZ=
|
| 454 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 455 |
-
BSZ=
|
|
|
|
|
|
|
| 456 |
else
|
| 457 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 458 |
fi
|
| 459 |
|
| 460 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -507,11 +537,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/8
|
|
| 507 |
sleep 5
|
| 508 |
|
| 509 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 510 |
-
BSZ=
|
| 511 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 512 |
-
BSZ=
|
|
|
|
|
|
|
| 513 |
else
|
| 514 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 515 |
fi
|
| 516 |
|
| 517 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -564,11 +596,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/9
|
|
| 564 |
sleep 5
|
| 565 |
|
| 566 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 567 |
-
BSZ=
|
| 568 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 569 |
-
BSZ=
|
|
|
|
|
|
|
| 570 |
else
|
| 571 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 572 |
fi
|
| 573 |
|
| 574 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -621,11 +655,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
|
|
| 621 |
sleep 5
|
| 622 |
|
| 623 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 624 |
-
BSZ=
|
| 625 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 626 |
-
BSZ=
|
|
|
|
|
|
|
| 627 |
else
|
| 628 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 629 |
fi
|
| 630 |
|
| 631 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -678,11 +714,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
|
|
| 678 |
sleep 5
|
| 679 |
|
| 680 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 681 |
-
BSZ=
|
| 682 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 683 |
-
BSZ=
|
|
|
|
|
|
|
| 684 |
else
|
| 685 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 686 |
fi
|
| 687 |
|
| 688 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -735,11 +773,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
|
|
| 735 |
sleep 5
|
| 736 |
|
| 737 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 738 |
-
BSZ=
|
| 739 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 740 |
-
BSZ=
|
|
|
|
|
|
|
| 741 |
else
|
| 742 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 743 |
fi
|
| 744 |
|
| 745 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -792,11 +832,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
|
|
| 792 |
sleep 5
|
| 793 |
|
| 794 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 795 |
-
BSZ=
|
| 796 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 797 |
-
BSZ=
|
|
|
|
|
|
|
| 798 |
else
|
| 799 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 800 |
fi
|
| 801 |
|
| 802 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -849,11 +891,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10a/outputs/1
|
|
| 849 |
sleep 5
|
| 850 |
|
| 851 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 852 |
-
BSZ=
|
| 853 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 854 |
-
BSZ=
|
|
|
|
|
|
|
| 855 |
else
|
| 856 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 857 |
fi
|
| 858 |
|
| 859 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
+
# GPU type detection
|
| 26 |
+
# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
|
| 27 |
+
if [ "$GPU_MEM" -lt 15500 ]; then
|
| 28 |
+
GPU_TYPE="t4"
|
| 29 |
+
echo "[GPU] Detected T4 (${GPU_MEM}MB)"
|
| 30 |
+
elif [ "$GPU_MEM" -le 17000 ]; then
|
| 31 |
+
GPU_TYPE="p100"
|
| 32 |
+
echo "[GPU] Detected P100 (${GPU_MEM}MB)"
|
| 33 |
else
|
| 34 |
+
GPU_TYPE="highvram"
|
| 35 |
+
echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
|
| 36 |
fi
|
| 37 |
|
| 38 |
+
# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
|
| 39 |
+
if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
|
| 40 |
GPU_MODE="t4_2gpu"
|
| 41 |
GPU_IDS="0,1"
|
| 42 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 43 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 44 |
+
elif [ "$GPU_TYPE" = "t4" ]; then
|
| 45 |
GPU_MODE="t4_1gpu"
|
| 46 |
GPU_IDS="${1:-0}"
|
| 47 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 48 |
+
echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
|
| 49 |
+
elif [ "$GPU_TYPE" = "p100" ]; then
|
| 50 |
+
GPU_MODE="p100"
|
| 51 |
+
GPU_IDS="${1:-0}"
|
| 52 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 53 |
+
echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
|
| 54 |
else
|
| 55 |
GPU_MODE="a100"
|
| 56 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 57 |
+
GPU_IDS="0,1"
|
| 58 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 59 |
+
else
|
| 60 |
+
GPU_IDS="${1:-0}"
|
| 61 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 62 |
+
fi
|
| 63 |
FP16_FLAG=""
|
|
|
|
| 64 |
fi
|
| 65 |
|
| 66 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 68 |
echo ""
|
| 69 |
|
| 70 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 71 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 72 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 73 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 74 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 75 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 76 |
else
|
| 77 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 78 |
fi
|
| 79 |
|
| 80 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 124 |
sleep 5
|
| 125 |
|
| 126 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 127 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 128 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 129 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 130 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 131 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 132 |
else
|
| 133 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 134 |
fi
|
| 135 |
|
| 136 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 183 |
sleep 5
|
| 184 |
|
| 185 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 186 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 187 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 188 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 189 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 190 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 191 |
else
|
| 192 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 193 |
fi
|
| 194 |
|
| 195 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 242 |
sleep 5
|
| 243 |
|
| 244 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 245 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 246 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 247 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 248 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 249 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 250 |
else
|
| 251 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 252 |
fi
|
| 253 |
|
| 254 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 301 |
sleep 5
|
| 302 |
|
| 303 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 304 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 305 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 306 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 307 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 308 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 309 |
else
|
| 310 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 311 |
fi
|
| 312 |
|
| 313 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 360 |
sleep 5
|
| 361 |
|
| 362 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 363 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 364 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 365 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 366 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 367 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 368 |
else
|
| 369 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 370 |
fi
|
| 371 |
|
| 372 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 419 |
sleep 5
|
| 420 |
|
| 421 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 422 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 423 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 424 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 425 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 426 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 427 |
else
|
| 428 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 429 |
fi
|
| 430 |
|
| 431 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 478 |
sleep 5
|
| 479 |
|
| 480 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 481 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 482 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 483 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 484 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 485 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 486 |
else
|
| 487 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 488 |
fi
|
| 489 |
|
| 490 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 537 |
sleep 5
|
| 538 |
|
| 539 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 540 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 541 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 542 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 543 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 544 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 545 |
else
|
| 546 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 547 |
fi
|
| 548 |
|
| 549 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 596 |
sleep 5
|
| 597 |
|
| 598 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 599 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 600 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 601 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 602 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 603 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 604 |
else
|
| 605 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 606 |
fi
|
| 607 |
|
| 608 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 655 |
sleep 5
|
| 656 |
|
| 657 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 658 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 659 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 660 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 661 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 662 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 663 |
else
|
| 664 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 665 |
fi
|
| 666 |
|
| 667 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 714 |
sleep 5
|
| 715 |
|
| 716 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 717 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 718 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 719 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 720 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 721 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 722 |
else
|
| 723 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 724 |
fi
|
| 725 |
|
| 726 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 773 |
sleep 5
|
| 774 |
|
| 775 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 776 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 777 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 778 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 779 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 780 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 781 |
else
|
| 782 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 783 |
fi
|
| 784 |
|
| 785 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 832 |
sleep 5
|
| 833 |
|
| 834 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 835 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 836 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 837 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 838 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 839 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 840 |
else
|
| 841 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 842 |
fi
|
| 843 |
|
| 844 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 891 |
sleep 5
|
| 892 |
|
| 893 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 894 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 895 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 896 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 897 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 898 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 899 |
else
|
| 900 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 901 |
fi
|
| 902 |
|
| 903 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute_v10b.sh
CHANGED
|
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
-
#
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
else
|
| 30 |
-
|
| 31 |
-
echo "[GPU] Detected high-
|
| 32 |
fi
|
| 33 |
|
| 34 |
-
#
|
| 35 |
-
if [ "$
|
| 36 |
GPU_MODE="t4_2gpu"
|
| 37 |
GPU_IDS="0,1"
|
| 38 |
-
FP16_FLAG=""
|
| 39 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 40 |
-
elif [ "$
|
| 41 |
GPU_MODE="t4_1gpu"
|
| 42 |
GPU_IDS="${1:-0}"
|
| 43 |
-
FP16_FLAG=""
|
| 44 |
-
echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
else
|
| 46 |
GPU_MODE="a100"
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
FP16_FLAG=""
|
| 49 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 50 |
fi
|
| 51 |
|
| 52 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
@@ -54,11 +68,13 @@ echo "============================================================"
|
|
| 54 |
echo ""
|
| 55 |
|
| 56 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 57 |
-
BSZ=
|
| 58 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 59 |
-
BSZ=
|
|
|
|
|
|
|
| 60 |
else
|
| 61 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 62 |
fi
|
| 63 |
|
| 64 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -108,11 +124,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
|
|
| 108 |
sleep 5
|
| 109 |
|
| 110 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 111 |
-
BSZ=
|
| 112 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 113 |
-
BSZ=
|
|
|
|
|
|
|
| 114 |
else
|
| 115 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 116 |
fi
|
| 117 |
|
| 118 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -163,11 +181,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/2
|
|
| 163 |
sleep 5
|
| 164 |
|
| 165 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 166 |
-
BSZ=
|
| 167 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 168 |
-
BSZ=
|
|
|
|
|
|
|
| 169 |
else
|
| 170 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 171 |
fi
|
| 172 |
|
| 173 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -218,11 +238,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/3
|
|
| 218 |
sleep 5
|
| 219 |
|
| 220 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 221 |
-
BSZ=
|
| 222 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 223 |
-
BSZ=
|
|
|
|
|
|
|
| 224 |
else
|
| 225 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 226 |
fi
|
| 227 |
|
| 228 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -273,11 +295,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/4
|
|
| 273 |
sleep 5
|
| 274 |
|
| 275 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 276 |
-
BSZ=
|
| 277 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 278 |
-
BSZ=
|
|
|
|
|
|
|
| 279 |
else
|
| 280 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 281 |
fi
|
| 282 |
|
| 283 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -328,11 +352,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/5
|
|
| 328 |
sleep 5
|
| 329 |
|
| 330 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 331 |
-
BSZ=
|
| 332 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 333 |
-
BSZ=
|
|
|
|
|
|
|
| 334 |
else
|
| 335 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 336 |
fi
|
| 337 |
|
| 338 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -383,11 +409,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/6
|
|
| 383 |
sleep 5
|
| 384 |
|
| 385 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 386 |
-
BSZ=
|
| 387 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 388 |
-
BSZ=
|
|
|
|
|
|
|
| 389 |
else
|
| 390 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 391 |
fi
|
| 392 |
|
| 393 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -438,11 +466,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/7
|
|
| 438 |
sleep 5
|
| 439 |
|
| 440 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 441 |
-
BSZ=
|
| 442 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 443 |
-
BSZ=
|
|
|
|
|
|
|
| 444 |
else
|
| 445 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 446 |
fi
|
| 447 |
|
| 448 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -493,11 +523,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/8
|
|
| 493 |
sleep 5
|
| 494 |
|
| 495 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 496 |
-
BSZ=
|
| 497 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 498 |
-
BSZ=
|
|
|
|
|
|
|
| 499 |
else
|
| 500 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 501 |
fi
|
| 502 |
|
| 503 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -548,11 +580,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/9
|
|
| 548 |
sleep 5
|
| 549 |
|
| 550 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 551 |
-
BSZ=
|
| 552 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 553 |
-
BSZ=
|
|
|
|
|
|
|
| 554 |
else
|
| 555 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 556 |
fi
|
| 557 |
|
| 558 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -603,11 +637,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
|
|
| 603 |
sleep 5
|
| 604 |
|
| 605 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 606 |
-
BSZ=
|
| 607 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 608 |
-
BSZ=
|
|
|
|
|
|
|
| 609 |
else
|
| 610 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 611 |
fi
|
| 612 |
|
| 613 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -658,11 +694,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
|
|
| 658 |
sleep 5
|
| 659 |
|
| 660 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 661 |
-
BSZ=
|
| 662 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 663 |
-
BSZ=
|
|
|
|
|
|
|
| 664 |
else
|
| 665 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 666 |
fi
|
| 667 |
|
| 668 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -713,11 +751,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
|
|
| 713 |
sleep 5
|
| 714 |
|
| 715 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 716 |
-
BSZ=
|
| 717 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 718 |
-
BSZ=
|
|
|
|
|
|
|
| 719 |
else
|
| 720 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 721 |
fi
|
| 722 |
|
| 723 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -768,11 +808,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
|
|
| 768 |
sleep 5
|
| 769 |
|
| 770 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 771 |
-
BSZ=
|
| 772 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 773 |
-
BSZ=
|
|
|
|
|
|
|
| 774 |
else
|
| 775 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 776 |
fi
|
| 777 |
|
| 778 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -823,11 +865,13 @@ rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute_v10b/outputs/1
|
|
| 823 |
sleep 5
|
| 824 |
|
| 825 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 826 |
-
BSZ=
|
| 827 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 828 |
-
BSZ=
|
|
|
|
|
|
|
| 829 |
else
|
| 830 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 831 |
fi
|
| 832 |
|
| 833 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
+
# GPU type detection
|
| 26 |
+
# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
|
| 27 |
+
if [ "$GPU_MEM" -lt 15500 ]; then
|
| 28 |
+
GPU_TYPE="t4"
|
| 29 |
+
echo "[GPU] Detected T4 (${GPU_MEM}MB)"
|
| 30 |
+
elif [ "$GPU_MEM" -le 17000 ]; then
|
| 31 |
+
GPU_TYPE="p100"
|
| 32 |
+
echo "[GPU] Detected P100 (${GPU_MEM}MB)"
|
| 33 |
else
|
| 34 |
+
GPU_TYPE="highvram"
|
| 35 |
+
echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
|
| 36 |
fi
|
| 37 |
|
| 38 |
+
# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
|
| 39 |
+
if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
|
| 40 |
GPU_MODE="t4_2gpu"
|
| 41 |
GPU_IDS="0,1"
|
| 42 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 43 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 44 |
+
elif [ "$GPU_TYPE" = "t4" ]; then
|
| 45 |
GPU_MODE="t4_1gpu"
|
| 46 |
GPU_IDS="${1:-0}"
|
| 47 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 48 |
+
echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
|
| 49 |
+
elif [ "$GPU_TYPE" = "p100" ]; then
|
| 50 |
+
GPU_MODE="p100"
|
| 51 |
+
GPU_IDS="${1:-0}"
|
| 52 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 53 |
+
echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
|
| 54 |
else
|
| 55 |
GPU_MODE="a100"
|
| 56 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 57 |
+
GPU_IDS="0,1"
|
| 58 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 59 |
+
else
|
| 60 |
+
GPU_IDS="${1:-0}"
|
| 61 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 62 |
+
fi
|
| 63 |
FP16_FLAG=""
|
|
|
|
| 64 |
fi
|
| 65 |
|
| 66 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 68 |
echo ""
|
| 69 |
|
| 70 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 71 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 72 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 73 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 74 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 75 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 76 |
else
|
| 77 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 78 |
fi
|
| 79 |
|
| 80 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 124 |
sleep 5
|
| 125 |
|
| 126 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 127 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 128 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 129 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 130 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 131 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 132 |
else
|
| 133 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 134 |
fi
|
| 135 |
|
| 136 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 181 |
sleep 5
|
| 182 |
|
| 183 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 184 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 185 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 186 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 187 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 188 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 189 |
else
|
| 190 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 191 |
fi
|
| 192 |
|
| 193 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 238 |
sleep 5
|
| 239 |
|
| 240 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 241 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 242 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 243 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 244 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 245 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 246 |
else
|
| 247 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 248 |
fi
|
| 249 |
|
| 250 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 295 |
sleep 5
|
| 296 |
|
| 297 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 298 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 299 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 300 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 301 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 302 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 303 |
else
|
| 304 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 305 |
fi
|
| 306 |
|
| 307 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 352 |
sleep 5
|
| 353 |
|
| 354 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 355 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 356 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 357 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 358 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 359 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 360 |
else
|
| 361 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 362 |
fi
|
| 363 |
|
| 364 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 409 |
sleep 5
|
| 410 |
|
| 411 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 412 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 413 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 414 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 415 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 416 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 417 |
else
|
| 418 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 419 |
fi
|
| 420 |
|
| 421 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 466 |
sleep 5
|
| 467 |
|
| 468 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 469 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 470 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 471 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 472 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 473 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 474 |
else
|
| 475 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 476 |
fi
|
| 477 |
|
| 478 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 523 |
sleep 5
|
| 524 |
|
| 525 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 526 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 527 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 528 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 529 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 530 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 531 |
else
|
| 532 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 533 |
fi
|
| 534 |
|
| 535 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 580 |
sleep 5
|
| 581 |
|
| 582 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 583 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 584 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 585 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 586 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 587 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 588 |
else
|
| 589 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 590 |
fi
|
| 591 |
|
| 592 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 637 |
sleep 5
|
| 638 |
|
| 639 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 640 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 641 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 642 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 643 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 644 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 645 |
else
|
| 646 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 647 |
fi
|
| 648 |
|
| 649 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 694 |
sleep 5
|
| 695 |
|
| 696 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 697 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 698 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 699 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 700 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 701 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 702 |
else
|
| 703 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 704 |
fi
|
| 705 |
|
| 706 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 751 |
sleep 5
|
| 752 |
|
| 753 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 754 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 755 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 756 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 757 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 758 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 759 |
else
|
| 760 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 761 |
fi
|
| 762 |
|
| 763 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 808 |
sleep 5
|
| 809 |
|
| 810 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 811 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 812 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 813 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 814 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 815 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 816 |
else
|
| 817 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 818 |
fi
|
| 819 |
|
| 820 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 865 |
sleep 5
|
| 866 |
|
| 867 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 868 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 869 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 870 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 871 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 872 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 873 |
else
|
| 874 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 875 |
fi
|
| 876 |
|
| 877 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
improve_gainlora/T5_small/gen_script_long_order4_t5_small_specroute.sh
CHANGED
|
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
-
#
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
else
|
| 30 |
-
|
| 31 |
-
echo "[GPU] Detected high-
|
| 32 |
fi
|
| 33 |
|
| 34 |
-
#
|
| 35 |
-
if [ "$
|
| 36 |
GPU_MODE="t4_2gpu"
|
| 37 |
GPU_IDS="0,1"
|
| 38 |
-
FP16_FLAG=""
|
| 39 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 40 |
-
elif [ "$
|
| 41 |
GPU_MODE="t4_1gpu"
|
| 42 |
GPU_IDS="${1:-0}"
|
| 43 |
-
FP16_FLAG=""
|
| 44 |
-
echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
else
|
| 46 |
GPU_MODE="a100"
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
FP16_FLAG=""
|
| 49 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 50 |
fi
|
| 51 |
|
| 52 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
@@ -54,11 +68,13 @@ echo "============================================================"
|
|
| 54 |
echo ""
|
| 55 |
|
| 56 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 57 |
-
BSZ=
|
| 58 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 59 |
-
BSZ=
|
|
|
|
|
|
|
| 60 |
else
|
| 61 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 62 |
fi
|
| 63 |
|
| 64 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -107,11 +123,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli
|
|
| 107 |
sleep 5
|
| 108 |
|
| 109 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 110 |
-
BSZ=
|
| 111 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 112 |
-
BSZ=
|
|
|
|
|
|
|
| 113 |
else
|
| 114 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 115 |
fi
|
| 116 |
|
| 117 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -161,11 +179,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/c
|
|
| 161 |
sleep 5
|
| 162 |
|
| 163 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 164 |
-
BSZ=
|
| 165 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 166 |
-
BSZ=
|
|
|
|
|
|
|
| 167 |
else
|
| 168 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 169 |
fi
|
| 170 |
|
| 171 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -215,11 +235,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/
|
|
| 215 |
sleep 5
|
| 216 |
|
| 217 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 218 |
-
BSZ=
|
| 219 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 220 |
-
BSZ=
|
|
|
|
|
|
|
| 221 |
else
|
| 222 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 223 |
fi
|
| 224 |
|
| 225 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -269,11 +291,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa
|
|
| 269 |
sleep 5
|
| 270 |
|
| 271 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 272 |
-
BSZ=
|
| 273 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 274 |
-
BSZ=
|
|
|
|
|
|
|
| 275 |
else
|
| 276 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 277 |
fi
|
| 278 |
|
| 279 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -323,11 +347,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/
|
|
| 323 |
sleep 5
|
| 324 |
|
| 325 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 326 |
-
BSZ=
|
| 327 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 328 |
-
BSZ=
|
|
|
|
|
|
|
| 329 |
else
|
| 330 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 331 |
fi
|
| 332 |
|
| 333 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -377,11 +403,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-bool
|
|
| 377 |
sleep 5
|
| 378 |
|
| 379 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 380 |
-
BSZ=
|
| 381 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 382 |
-
BSZ=
|
|
|
|
|
|
|
| 383 |
else
|
| 384 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 385 |
fi
|
| 386 |
|
| 387 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -431,11 +459,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/
|
|
| 431 |
sleep 5
|
| 432 |
|
| 433 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 434 |
-
BSZ=
|
| 435 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 436 |
-
BSZ=
|
|
|
|
|
|
|
| 437 |
else
|
| 438 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 439 |
fi
|
| 440 |
|
| 441 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -485,11 +515,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb
|
|
| 485 |
sleep 5
|
| 486 |
|
| 487 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 488 |
-
BSZ=
|
| 489 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 490 |
-
BSZ=
|
|
|
|
|
|
|
| 491 |
else
|
| 492 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 493 |
fi
|
| 494 |
|
| 495 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -539,11 +571,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp
|
|
| 539 |
sleep 5
|
| 540 |
|
| 541 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 542 |
-
BSZ=
|
| 543 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 544 |
-
BSZ=
|
|
|
|
|
|
|
| 545 |
else
|
| 546 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 547 |
fi
|
| 548 |
|
| 549 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -593,11 +627,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-ama
|
|
| 593 |
sleep 5
|
| 594 |
|
| 595 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 596 |
-
BSZ=
|
| 597 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 598 |
-
BSZ=
|
|
|
|
|
|
|
| 599 |
else
|
| 600 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 601 |
fi
|
| 602 |
|
| 603 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -647,11 +683,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst
|
|
| 647 |
sleep 5
|
| 648 |
|
| 649 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 650 |
-
BSZ=
|
| 651 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 652 |
-
BSZ=
|
|
|
|
|
|
|
| 653 |
else
|
| 654 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 655 |
fi
|
| 656 |
|
| 657 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -701,11 +739,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/12-dbp
|
|
| 701 |
sleep 5
|
| 702 |
|
| 703 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 704 |
-
BSZ=
|
| 705 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 706 |
-
BSZ=
|
|
|
|
|
|
|
| 707 |
else
|
| 708 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 709 |
fi
|
| 710 |
|
| 711 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -755,11 +795,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/13-agn
|
|
| 755 |
sleep 5
|
| 756 |
|
| 757 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 758 |
-
BSZ=
|
| 759 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 760 |
-
BSZ=
|
|
|
|
|
|
|
| 761 |
else
|
| 762 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 763 |
fi
|
| 764 |
|
| 765 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
@@ -809,11 +851,13 @@ rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/14-mul
|
|
| 809 |
sleep 5
|
| 810 |
|
| 811 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 812 |
-
BSZ=
|
| 813 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 814 |
-
BSZ=
|
|
|
|
|
|
|
| 815 |
else
|
| 816 |
-
BSZ=64; GA=1; EVAL_BSZ=
|
| 817 |
fi
|
| 818 |
|
| 819 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
+
# GPU type detection
|
| 26 |
+
# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
|
| 27 |
+
if [ "$GPU_MEM" -lt 15500 ]; then
|
| 28 |
+
GPU_TYPE="t4"
|
| 29 |
+
echo "[GPU] Detected T4 (${GPU_MEM}MB)"
|
| 30 |
+
elif [ "$GPU_MEM" -le 17000 ]; then
|
| 31 |
+
GPU_TYPE="p100"
|
| 32 |
+
echo "[GPU] Detected P100 (${GPU_MEM}MB)"
|
| 33 |
else
|
| 34 |
+
GPU_TYPE="highvram"
|
| 35 |
+
echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
|
| 36 |
fi
|
| 37 |
|
| 38 |
+
# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
|
| 39 |
+
if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
|
| 40 |
GPU_MODE="t4_2gpu"
|
| 41 |
GPU_IDS="0,1"
|
| 42 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 43 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 44 |
+
elif [ "$GPU_TYPE" = "t4" ]; then
|
| 45 |
GPU_MODE="t4_1gpu"
|
| 46 |
GPU_IDS="${1:-0}"
|
| 47 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 48 |
+
echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
|
| 49 |
+
elif [ "$GPU_TYPE" = "p100" ]; then
|
| 50 |
+
GPU_MODE="p100"
|
| 51 |
+
GPU_IDS="${1:-0}"
|
| 52 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 53 |
+
echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
|
| 54 |
else
|
| 55 |
GPU_MODE="a100"
|
| 56 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 57 |
+
GPU_IDS="0,1"
|
| 58 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 59 |
+
else
|
| 60 |
+
GPU_IDS="${1:-0}"
|
| 61 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 62 |
+
fi
|
| 63 |
FP16_FLAG=""
|
|
|
|
| 64 |
fi
|
| 65 |
|
| 66 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 68 |
echo ""
|
| 69 |
|
| 70 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 71 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 72 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 73 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 74 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 75 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 76 |
else
|
| 77 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 78 |
fi
|
| 79 |
|
| 80 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 123 |
sleep 5
|
| 124 |
|
| 125 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 126 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 127 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 128 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 129 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 130 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 131 |
else
|
| 132 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 133 |
fi
|
| 134 |
|
| 135 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 179 |
sleep 5
|
| 180 |
|
| 181 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 182 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 183 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 184 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 185 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 186 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 187 |
else
|
| 188 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 189 |
fi
|
| 190 |
|
| 191 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 235 |
sleep 5
|
| 236 |
|
| 237 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 238 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 239 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 240 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 241 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 242 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 243 |
else
|
| 244 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 245 |
fi
|
| 246 |
|
| 247 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 291 |
sleep 5
|
| 292 |
|
| 293 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 294 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 295 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 296 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 297 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 298 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 299 |
else
|
| 300 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 301 |
fi
|
| 302 |
|
| 303 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 347 |
sleep 5
|
| 348 |
|
| 349 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 350 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 351 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 352 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 353 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 354 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 355 |
else
|
| 356 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 357 |
fi
|
| 358 |
|
| 359 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 403 |
sleep 5
|
| 404 |
|
| 405 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 406 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 407 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 408 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 409 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 410 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 411 |
else
|
| 412 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 413 |
fi
|
| 414 |
|
| 415 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 459 |
sleep 5
|
| 460 |
|
| 461 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 462 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 463 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 464 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 465 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 466 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 467 |
else
|
| 468 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 469 |
fi
|
| 470 |
|
| 471 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 515 |
sleep 5
|
| 516 |
|
| 517 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 518 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 519 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 520 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 521 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 522 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 523 |
else
|
| 524 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 525 |
fi
|
| 526 |
|
| 527 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 571 |
sleep 5
|
| 572 |
|
| 573 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 574 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 575 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 576 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 577 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 578 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 579 |
else
|
| 580 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 581 |
fi
|
| 582 |
|
| 583 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 627 |
sleep 5
|
| 628 |
|
| 629 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 630 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 631 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 632 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 633 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 634 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 635 |
else
|
| 636 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 637 |
fi
|
| 638 |
|
| 639 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 683 |
sleep 5
|
| 684 |
|
| 685 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 686 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 687 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 688 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 689 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 690 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 691 |
else
|
| 692 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 693 |
fi
|
| 694 |
|
| 695 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 739 |
sleep 5
|
| 740 |
|
| 741 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 742 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 743 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 744 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 745 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 746 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 747 |
else
|
| 748 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 749 |
fi
|
| 750 |
|
| 751 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 795 |
sleep 5
|
| 796 |
|
| 797 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 798 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 799 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 800 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 801 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 802 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 803 |
else
|
| 804 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 805 |
fi
|
| 806 |
|
| 807 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
|
|
| 851 |
sleep 5
|
| 852 |
|
| 853 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 854 |
+
BSZ=8; GA=2; EVAL_BSZ=64
|
| 855 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 856 |
+
BSZ=8; GA=2; EVAL_BSZ=32
|
| 857 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 858 |
+
BSZ=16; GA=2; EVAL_BSZ=32
|
| 859 |
else
|
| 860 |
+
BSZ=64; GA=1; EVAL_BSZ=128
|
| 861 |
fi
|
| 862 |
|
| 863 |
CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
improve_gainlora/T5_small/gen_script_superni_order1_t5_small_specroute.sh
CHANGED
|
@@ -32,23 +32,45 @@ else
|
|
| 32 |
fi
|
| 33 |
|
| 34 |
# Determine parallelism strategy
|
| 35 |
-
#
|
| 36 |
-
# T4
|
| 37 |
-
if [ "$
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
GPU_MODE="t4_2gpu"
|
| 39 |
GPU_IDS="0,1"
|
| 40 |
-
FP16_FLAG=""
|
| 41 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 42 |
-
elif [ "$
|
| 43 |
GPU_MODE="t4_1gpu"
|
| 44 |
GPU_IDS="${1:-0}"
|
| 45 |
-
FP16_FLAG=""
|
| 46 |
-
echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
else
|
| 48 |
GPU_MODE="a100"
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
FP16_FLAG=""
|
| 51 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 52 |
fi
|
| 53 |
|
| 54 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
@@ -58,6 +80,8 @@ echo ""
|
|
| 58 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 59 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 60 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 61 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 62 |
else
|
| 63 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -108,6 +132,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 108 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 109 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 110 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 111 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 112 |
else
|
| 113 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -159,6 +185,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 159 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 160 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 161 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 162 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 163 |
else
|
| 164 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -210,6 +238,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 210 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 211 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 212 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 213 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 214 |
else
|
| 215 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -261,6 +291,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 261 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 262 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 263 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 264 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 265 |
else
|
| 266 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -312,6 +344,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 312 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 313 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 314 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 315 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 316 |
else
|
| 317 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -363,6 +397,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 363 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 364 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 365 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 366 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 367 |
else
|
| 368 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -414,6 +450,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 414 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 415 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 416 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 417 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 418 |
else
|
| 419 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -465,6 +503,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 465 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 466 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 467 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 468 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 469 |
else
|
| 470 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -516,6 +556,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 516 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 517 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 518 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 519 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 520 |
else
|
| 521 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -567,6 +609,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 567 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 568 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 569 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 570 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 571 |
else
|
| 572 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -618,6 +662,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 618 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 619 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 620 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 621 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 622 |
else
|
| 623 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -669,6 +715,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 669 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 670 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 671 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 672 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 673 |
else
|
| 674 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -720,6 +768,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 720 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 721 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 722 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 723 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 724 |
else
|
| 725 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -771,6 +821,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 771 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 772 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 773 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 774 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 775 |
else
|
| 776 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 32 |
fi
|
| 33 |
|
| 34 |
# Determine parallelism strategy
|
| 35 |
+
# GPU type detection
|
| 36 |
+
# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
|
| 37 |
+
if [ "$GPU_MEM" -lt 15500 ]; then
|
| 38 |
+
GPU_TYPE="t4"
|
| 39 |
+
echo "[GPU] Detected T4 (${GPU_MEM}MB)"
|
| 40 |
+
elif [ "$GPU_MEM" -le 17000 ]; then
|
| 41 |
+
GPU_TYPE="p100"
|
| 42 |
+
echo "[GPU] Detected P100 (${GPU_MEM}MB)"
|
| 43 |
+
else
|
| 44 |
+
GPU_TYPE="highvram"
|
| 45 |
+
echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
|
| 46 |
+
fi
|
| 47 |
+
|
| 48 |
+
# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
|
| 49 |
+
if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
|
| 50 |
GPU_MODE="t4_2gpu"
|
| 51 |
GPU_IDS="0,1"
|
| 52 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 53 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 54 |
+
elif [ "$GPU_TYPE" = "t4" ]; then
|
| 55 |
GPU_MODE="t4_1gpu"
|
| 56 |
GPU_IDS="${1:-0}"
|
| 57 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 58 |
+
echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
|
| 59 |
+
elif [ "$GPU_TYPE" = "p100" ]; then
|
| 60 |
+
GPU_MODE="p100"
|
| 61 |
+
GPU_IDS="${1:-0}"
|
| 62 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 63 |
+
echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
|
| 64 |
else
|
| 65 |
GPU_MODE="a100"
|
| 66 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 67 |
+
GPU_IDS="0,1"
|
| 68 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 69 |
+
else
|
| 70 |
+
GPU_IDS="${1:-0}"
|
| 71 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 72 |
+
fi
|
| 73 |
FP16_FLAG=""
|
|
|
|
| 74 |
fi
|
| 75 |
|
| 76 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 80 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 81 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 82 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 83 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 84 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 85 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 86 |
else
|
| 87 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 132 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 133 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 134 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 135 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 136 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 137 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 138 |
else
|
| 139 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 185 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 186 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 187 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 188 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 189 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 190 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 191 |
else
|
| 192 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 238 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 239 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 240 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 241 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 242 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 243 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 244 |
else
|
| 245 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 291 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 292 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 293 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 294 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 295 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 296 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 297 |
else
|
| 298 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 344 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 345 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 346 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 347 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 348 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 349 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 350 |
else
|
| 351 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 397 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 398 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 399 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 400 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 401 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 402 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 403 |
else
|
| 404 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 450 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 451 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 452 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 453 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 454 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 455 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 456 |
else
|
| 457 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 503 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 504 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 505 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 506 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 507 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 508 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 509 |
else
|
| 510 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 556 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 557 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 558 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 559 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 560 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 561 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 562 |
else
|
| 563 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 609 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 610 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 611 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 612 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 613 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 614 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 615 |
else
|
| 616 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 662 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 663 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 664 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 665 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 666 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 667 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 668 |
else
|
| 669 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 715 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 716 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 717 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 718 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 719 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 720 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 721 |
else
|
| 722 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 768 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 769 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 770 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 771 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 772 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 773 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 774 |
else
|
| 775 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 821 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 822 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 823 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 824 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 825 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 826 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 827 |
else
|
| 828 |
BSZ=32; GA=1; EVAL_BSZ=32
|
improve_gainlora/T5_small/gen_script_superni_order2_t5_small_specroute.sh
CHANGED
|
@@ -22,31 +22,45 @@ if [ -z "$GPU_MEM" ]; then
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
-
#
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
else
|
| 30 |
-
|
| 31 |
-
echo "[GPU] Detected high-
|
| 32 |
fi
|
| 33 |
|
| 34 |
-
#
|
| 35 |
-
if [ "$
|
| 36 |
GPU_MODE="t4_2gpu"
|
| 37 |
GPU_IDS="0,1"
|
| 38 |
-
FP16_FLAG=""
|
| 39 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 40 |
-
elif [ "$
|
| 41 |
GPU_MODE="t4_1gpu"
|
| 42 |
GPU_IDS="${1:-0}"
|
| 43 |
-
FP16_FLAG=""
|
| 44 |
-
echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
else
|
| 46 |
GPU_MODE="a100"
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
FP16_FLAG=""
|
| 49 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 50 |
fi
|
| 51 |
|
| 52 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
@@ -56,6 +70,8 @@ echo ""
|
|
| 56 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 57 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 58 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 59 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 60 |
else
|
| 61 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -105,6 +121,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 105 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 106 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 107 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 108 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 109 |
else
|
| 110 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -155,6 +173,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 155 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 156 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 157 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 158 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 159 |
else
|
| 160 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -205,6 +225,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 205 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 206 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 207 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 208 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 209 |
else
|
| 210 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -255,6 +277,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 255 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 256 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 257 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 258 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 259 |
else
|
| 260 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -305,6 +329,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 305 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 306 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 307 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 308 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 309 |
else
|
| 310 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -355,6 +381,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 355 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 356 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 357 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 358 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 359 |
else
|
| 360 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -405,6 +433,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 405 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 406 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 407 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 408 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 409 |
else
|
| 410 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -455,6 +485,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 455 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 456 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 457 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 458 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 459 |
else
|
| 460 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -505,6 +537,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 505 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 506 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 507 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 508 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 509 |
else
|
| 510 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -555,6 +589,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 555 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 556 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 557 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 558 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 559 |
else
|
| 560 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -605,6 +641,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 605 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 606 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 607 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 608 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 609 |
else
|
| 610 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -655,6 +693,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 655 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 656 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 657 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 658 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 659 |
else
|
| 660 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -705,6 +745,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 705 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 706 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 707 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 708 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 709 |
else
|
| 710 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
@@ -755,6 +797,8 @@ CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
|
|
| 755 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 756 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 757 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
|
|
|
|
|
|
| 758 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 759 |
else
|
| 760 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 22 |
exit 1
|
| 23 |
fi
|
| 24 |
|
| 25 |
+
# GPU type detection
|
| 26 |
+
# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000
|
| 27 |
+
if [ "$GPU_MEM" -lt 15500 ]; then
|
| 28 |
+
GPU_TYPE="t4"
|
| 29 |
+
echo "[GPU] Detected T4 (${GPU_MEM}MB)"
|
| 30 |
+
elif [ "$GPU_MEM" -le 17000 ]; then
|
| 31 |
+
GPU_TYPE="p100"
|
| 32 |
+
echo "[GPU] Detected P100 (${GPU_MEM}MB)"
|
| 33 |
else
|
| 34 |
+
GPU_TYPE="highvram"
|
| 35 |
+
echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"
|
| 36 |
fi
|
| 37 |
|
| 38 |
+
# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs
|
| 39 |
+
if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then
|
| 40 |
GPU_MODE="t4_2gpu"
|
| 41 |
GPU_IDS="0,1"
|
| 42 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 43 |
echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
|
| 44 |
+
elif [ "$GPU_TYPE" = "t4" ]; then
|
| 45 |
GPU_MODE="t4_1gpu"
|
| 46 |
GPU_IDS="${1:-0}"
|
| 47 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 48 |
+
echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"
|
| 49 |
+
elif [ "$GPU_TYPE" = "p100" ]; then
|
| 50 |
+
GPU_MODE="p100"
|
| 51 |
+
GPU_IDS="${1:-0}"
|
| 52 |
+
FP16_FLAG="--gradient_checkpointing"
|
| 53 |
+
echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"
|
| 54 |
else
|
| 55 |
GPU_MODE="a100"
|
| 56 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 57 |
+
GPU_IDS="0,1"
|
| 58 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 59 |
+
else
|
| 60 |
+
GPU_IDS="${1:-0}"
|
| 61 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 62 |
+
fi
|
| 63 |
FP16_FLAG=""
|
|
|
|
| 64 |
fi
|
| 65 |
|
| 66 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 70 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 71 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 72 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 73 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 74 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 75 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 76 |
else
|
| 77 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 121 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 122 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 123 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 124 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 125 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 126 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 127 |
else
|
| 128 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 173 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 174 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 175 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 176 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 177 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 178 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 179 |
else
|
| 180 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 225 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 226 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 227 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 228 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 229 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 230 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 231 |
else
|
| 232 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 277 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 278 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 279 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 280 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 281 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 282 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 283 |
else
|
| 284 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 329 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 330 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 331 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 332 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 333 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 334 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 335 |
else
|
| 336 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 381 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 382 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 383 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 384 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 385 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 386 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 387 |
else
|
| 388 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 433 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 434 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 435 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 436 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 437 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 438 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 439 |
else
|
| 440 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 485 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 486 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 487 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 488 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 489 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 490 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 491 |
else
|
| 492 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 537 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 538 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 539 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 540 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 541 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 542 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 543 |
else
|
| 544 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 589 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 590 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 591 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 592 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 593 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 594 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 595 |
else
|
| 596 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 641 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 642 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 643 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 644 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 645 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 646 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 647 |
else
|
| 648 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 693 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 694 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 695 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 696 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 697 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 698 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 699 |
else
|
| 700 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 745 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 746 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 747 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 748 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 749 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 750 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 751 |
else
|
| 752 |
BSZ=32; GA=1; EVAL_BSZ=32
|
|
|
|
| 797 |
if [ "$GPU_MODE" = "t4_2gpu" ]; then
|
| 798 |
BSZ=8; GA=2; EVAL_BSZ=16
|
| 799 |
elif [ "$GPU_MODE" = "t4_1gpu" ]; then
|
| 800 |
+
BSZ=8; GA=2; EVAL_BSZ=16
|
| 801 |
+
elif [ "$GPU_MODE" = "p100" ]; then
|
| 802 |
BSZ=16; GA=2; EVAL_BSZ=16
|
| 803 |
else
|
| 804 |
BSZ=32; GA=1; EVAL_BSZ=32
|
improve_gainlora/gen_script_long_order3_t5_specroute.sh
CHANGED
|
@@ -49,9 +49,14 @@ elif [ "$GPU_MEM" -le 17000 ]; then
|
|
| 49 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 50 |
else
|
| 51 |
GPU_MODE="a100"
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
FP16_FLAG=""
|
| 54 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 55 |
fi
|
| 56 |
|
| 57 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 49 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 50 |
else
|
| 51 |
GPU_MODE="a100"
|
| 52 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 53 |
+
GPU_IDS="0,1"
|
| 54 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 55 |
+
else
|
| 56 |
+
GPU_IDS="${1:-0}"
|
| 57 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 58 |
+
fi
|
| 59 |
FP16_FLAG=""
|
|
|
|
| 60 |
fi
|
| 61 |
|
| 62 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
improve_gainlora/gen_script_long_order4_t5_specroute.sh
CHANGED
|
@@ -49,9 +49,14 @@ elif [ "$GPU_MEM" -le 17000 ]; then
|
|
| 49 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 50 |
else
|
| 51 |
GPU_MODE="a100"
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
FP16_FLAG=""
|
| 54 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 55 |
fi
|
| 56 |
|
| 57 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 49 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 50 |
else
|
| 51 |
GPU_MODE="a100"
|
| 52 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 53 |
+
GPU_IDS="0,1"
|
| 54 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 55 |
+
else
|
| 56 |
+
GPU_IDS="${1:-0}"
|
| 57 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 58 |
+
fi
|
| 59 |
FP16_FLAG=""
|
|
|
|
| 60 |
fi
|
| 61 |
|
| 62 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
improve_gainlora/gen_script_superni_order1_llama_specroute.sh
CHANGED
|
@@ -8,9 +8,35 @@
|
|
| 8 |
#SBATCH --gres=gpu:a100-sxm4-80gb:1
|
| 9 |
|
| 10 |
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
--do_train \
|
|
|
|
| 14 |
--do_predict \
|
| 15 |
--predict_with_generate \
|
| 16 |
--model_name_or_path $2 \
|
|
@@ -20,7 +46,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 20 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary \
|
| 21 |
--per_device_train_batch_size 2 \
|
| 22 |
--per_device_eval_batch_size 8 \
|
| 23 |
-
--gradient_accumulation_steps
|
| 24 |
--learning_rate 5e-05 \
|
| 25 |
--num_train_epochs 50 \
|
| 26 |
--bf16 \
|
|
@@ -55,8 +81,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task
|
|
| 55 |
sleep 5
|
| 56 |
|
| 57 |
|
| 58 |
-
deepspeed --include
|
| 59 |
--do_train \
|
|
|
|
| 60 |
--do_predict \
|
| 61 |
--predict_with_generate \
|
| 62 |
--model_name_or_path $2 \
|
|
@@ -69,7 +96,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 69 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification \
|
| 70 |
--per_device_train_batch_size 16 \
|
| 71 |
--per_device_eval_batch_size 8 \
|
| 72 |
-
--gradient_accumulation_steps
|
| 73 |
--learning_rate 5e-05 \
|
| 74 |
--num_train_epochs 50 \
|
| 75 |
--bf16 \
|
|
@@ -104,8 +131,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task
|
|
| 104 |
sleep 5
|
| 105 |
|
| 106 |
|
| 107 |
-
deepspeed --include
|
| 108 |
--do_train \
|
|
|
|
| 109 |
--do_predict \
|
| 110 |
--predict_with_generate \
|
| 111 |
--model_name_or_path $2 \
|
|
@@ -118,7 +146,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 118 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization \
|
| 119 |
--per_device_train_batch_size 1 \
|
| 120 |
--per_device_eval_batch_size 8 \
|
| 121 |
-
--gradient_accumulation_steps
|
| 122 |
--learning_rate 5e-05 \
|
| 123 |
--num_train_epochs 50 \
|
| 124 |
--bf16 \
|
|
@@ -153,8 +181,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task
|
|
| 153 |
sleep 5
|
| 154 |
|
| 155 |
|
| 156 |
-
deepspeed --include
|
| 157 |
--do_train \
|
|
|
|
| 158 |
--do_predict \
|
| 159 |
--predict_with_generate \
|
| 160 |
--model_name_or_path $2 \
|
|
@@ -167,7 +196,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 167 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction \
|
| 168 |
--per_device_train_batch_size 4 \
|
| 169 |
--per_device_eval_batch_size 8 \
|
| 170 |
-
--gradient_accumulation_steps
|
| 171 |
--learning_rate 5e-05 \
|
| 172 |
--num_train_epochs 50 \
|
| 173 |
--bf16 \
|
|
@@ -202,8 +231,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task
|
|
| 202 |
sleep 5
|
| 203 |
|
| 204 |
|
| 205 |
-
deepspeed --include
|
| 206 |
--do_train \
|
|
|
|
| 207 |
--do_predict \
|
| 208 |
--predict_with_generate \
|
| 209 |
--model_name_or_path $2 \
|
|
@@ -216,7 +246,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 216 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation \
|
| 217 |
--per_device_train_batch_size 1 \
|
| 218 |
--per_device_eval_batch_size 8 \
|
| 219 |
-
--gradient_accumulation_steps
|
| 220 |
--learning_rate 5e-05 \
|
| 221 |
--num_train_epochs 50 \
|
| 222 |
--bf16 \
|
|
@@ -251,8 +281,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task
|
|
| 251 |
sleep 5
|
| 252 |
|
| 253 |
|
| 254 |
-
deepspeed --include
|
| 255 |
--do_train \
|
|
|
|
| 256 |
--do_predict \
|
| 257 |
--predict_with_generate \
|
| 258 |
--model_name_or_path $2 \
|
|
@@ -265,7 +296,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 265 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction \
|
| 266 |
--per_device_train_batch_size 16 \
|
| 267 |
--per_device_eval_batch_size 8 \
|
| 268 |
-
--gradient_accumulation_steps
|
| 269 |
--learning_rate 5e-05 \
|
| 270 |
--num_train_epochs 50 \
|
| 271 |
--bf16 \
|
|
@@ -300,8 +331,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task
|
|
| 300 |
sleep 5
|
| 301 |
|
| 302 |
|
| 303 |
-
deepspeed --include
|
| 304 |
--do_train \
|
|
|
|
| 305 |
--do_predict \
|
| 306 |
--predict_with_generate \
|
| 307 |
--model_name_or_path $2 \
|
|
@@ -314,7 +346,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 314 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation \
|
| 315 |
--per_device_train_batch_size 16 \
|
| 316 |
--per_device_eval_batch_size 8 \
|
| 317 |
-
--gradient_accumulation_steps
|
| 318 |
--learning_rate 5e-05 \
|
| 319 |
--num_train_epochs 50 \
|
| 320 |
--bf16 \
|
|
@@ -349,8 +381,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task
|
|
| 349 |
sleep 5
|
| 350 |
|
| 351 |
|
| 352 |
-
deepspeed --include
|
| 353 |
--do_train \
|
|
|
|
| 354 |
--do_predict \
|
| 355 |
--predict_with_generate \
|
| 356 |
--model_name_or_path $2 \
|
|
@@ -363,7 +396,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 363 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next \
|
| 364 |
--per_device_train_batch_size 2 \
|
| 365 |
--per_device_eval_batch_size 8 \
|
| 366 |
-
--gradient_accumulation_steps
|
| 367 |
--learning_rate 5e-05 \
|
| 368 |
--num_train_epochs 50 \
|
| 369 |
--bf16 \
|
|
@@ -398,8 +431,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task
|
|
| 398 |
sleep 5
|
| 399 |
|
| 400 |
|
| 401 |
-
deepspeed --include
|
| 402 |
--do_train \
|
|
|
|
| 403 |
--do_predict \
|
| 404 |
--predict_with_generate \
|
| 405 |
--model_name_or_path $2 \
|
|
@@ -412,7 +446,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 412 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation \
|
| 413 |
--per_device_train_batch_size 8 \
|
| 414 |
--per_device_eval_batch_size 8 \
|
| 415 |
-
--gradient_accumulation_steps
|
| 416 |
--learning_rate 5e-05 \
|
| 417 |
--num_train_epochs 50 \
|
| 418 |
--bf16 \
|
|
@@ -447,8 +481,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task
|
|
| 447 |
sleep 5
|
| 448 |
|
| 449 |
|
| 450 |
-
deepspeed --include
|
| 451 |
--do_train \
|
|
|
|
| 452 |
--do_predict \
|
| 453 |
--predict_with_generate \
|
| 454 |
--model_name_or_path $2 \
|
|
@@ -461,7 +496,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 461 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation \
|
| 462 |
--per_device_train_batch_size 2 \
|
| 463 |
--per_device_eval_batch_size 8 \
|
| 464 |
-
--gradient_accumulation_steps
|
| 465 |
--learning_rate 5e-05 \
|
| 466 |
--num_train_epochs 50 \
|
| 467 |
--bf16 \
|
|
@@ -496,8 +531,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-tas
|
|
| 496 |
sleep 5
|
| 497 |
|
| 498 |
|
| 499 |
-
deepspeed --include
|
| 500 |
--do_train \
|
|
|
|
| 501 |
--do_predict \
|
| 502 |
--predict_with_generate \
|
| 503 |
--model_name_or_path $2 \
|
|
@@ -510,7 +546,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 510 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection \
|
| 511 |
--per_device_train_batch_size 4 \
|
| 512 |
--per_device_eval_batch_size 8 \
|
| 513 |
-
--gradient_accumulation_steps
|
| 514 |
--learning_rate 5e-05 \
|
| 515 |
--num_train_epochs 50 \
|
| 516 |
--bf16 \
|
|
@@ -545,8 +581,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-tas
|
|
| 545 |
sleep 5
|
| 546 |
|
| 547 |
|
| 548 |
-
deepspeed --include
|
| 549 |
--do_train \
|
|
|
|
| 550 |
--do_predict \
|
| 551 |
--predict_with_generate \
|
| 552 |
--model_name_or_path $2 \
|
|
@@ -559,7 +596,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 559 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
|
| 560 |
--per_device_train_batch_size 1 \
|
| 561 |
--per_device_eval_batch_size 8 \
|
| 562 |
-
--gradient_accumulation_steps
|
| 563 |
--learning_rate 5e-05 \
|
| 564 |
--num_train_epochs 50 \
|
| 565 |
--bf16 \
|
|
@@ -594,8 +631,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-tas
|
|
| 594 |
sleep 5
|
| 595 |
|
| 596 |
|
| 597 |
-
deepspeed --include
|
| 598 |
--do_train \
|
|
|
|
| 599 |
--do_predict \
|
| 600 |
--predict_with_generate \
|
| 601 |
--model_name_or_path $2 \
|
|
@@ -608,7 +646,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 608 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation \
|
| 609 |
--per_device_train_batch_size 8 \
|
| 610 |
--per_device_eval_batch_size 8 \
|
| 611 |
-
--gradient_accumulation_steps
|
| 612 |
--learning_rate 5e-05 \
|
| 613 |
--num_train_epochs 50 \
|
| 614 |
--bf16 \
|
|
@@ -643,8 +681,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-tas
|
|
| 643 |
sleep 5
|
| 644 |
|
| 645 |
|
| 646 |
-
deepspeed --include
|
| 647 |
--do_train \
|
|
|
|
| 648 |
--do_predict \
|
| 649 |
--predict_with_generate \
|
| 650 |
--model_name_or_path $2 \
|
|
@@ -657,7 +696,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 657 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification \
|
| 658 |
--per_device_train_batch_size 8 \
|
| 659 |
--per_device_eval_batch_size 8 \
|
| 660 |
-
--gradient_accumulation_steps
|
| 661 |
--learning_rate 5e-05 \
|
| 662 |
--num_train_epochs 50 \
|
| 663 |
--bf16 \
|
|
@@ -692,8 +731,9 @@ rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-tas
|
|
| 692 |
sleep 5
|
| 693 |
|
| 694 |
|
| 695 |
-
deepspeed --include
|
| 696 |
--do_train \
|
|
|
|
| 697 |
--do_predict \
|
| 698 |
--predict_with_generate \
|
| 699 |
--model_name_or_path $2 \
|
|
@@ -706,7 +746,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 706 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification \
|
| 707 |
--per_device_train_batch_size 4 \
|
| 708 |
--per_device_eval_batch_size 8 \
|
| 709 |
-
--gradient_accumulation_steps
|
| 710 |
--learning_rate 5e-05 \
|
| 711 |
--num_train_epochs 50 \
|
| 712 |
--bf16 \
|
|
|
|
| 8 |
#SBATCH --gres=gpu:a100-sxm4-80gb:1
|
| 9 |
|
| 10 |
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
|
| 11 |
+
# ββ GPU auto-detection βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 12 |
+
NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
|
| 13 |
+
GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
|
| 14 |
+
|
| 15 |
+
if [ -z "$GPU_MEM" ]; then
|
| 16 |
+
echo "[ERROR] No GPU detected. Exiting."
|
| 17 |
+
exit 1
|
| 18 |
+
fi
|
| 19 |
+
|
| 20 |
+
if [ "$NUM_GPUS" -ge 2 ] && [ "$GPU_MEM" -ge 20000 ]; then
|
| 21 |
+
# 2Γ 3090/4090 (24 GB+): run DDP on both GPUs
|
| 22 |
+
DS_INCLUDE="localhost:0,1"
|
| 23 |
+
echo "[GPU] 2Γ GPU detected (${GPU_MEM} MB each) β DDP on localhost:0,1"
|
| 24 |
+
elif [ "$GPU_MEM" -lt 20000 ]; then
|
| 25 |
+
# P100/V100 16 GB: use the dedicated P100 script instead
|
| 26 |
+
echo "[GPU] WARNING: 16 GB GPU detected. Use the _p100.sh script instead."
|
| 27 |
+
echo " This script requires β₯ 20 GB VRAM per GPU."
|
| 28 |
+
exit 1
|
| 29 |
+
else
|
| 30 |
+
# Single high-memory GPU (A100/H100 or single 3090)
|
| 31 |
+
DS_INCLUDE="localhost:${1:-0}"
|
| 32 |
+
echo "[GPU] Single GPU detected (${GPU_MEM} MB) β localhost:${1:-0}"
|
| 33 |
+
fi
|
| 34 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 38 |
--do_train \
|
| 39 |
+
--gradient_checkpointing \
|
| 40 |
--do_predict \
|
| 41 |
--predict_with_generate \
|
| 42 |
--model_name_or_path $2 \
|
|
|
|
| 46 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary \
|
| 47 |
--per_device_train_batch_size 2 \
|
| 48 |
--per_device_eval_batch_size 8 \
|
| 49 |
+
--gradient_accumulation_steps 8 \
|
| 50 |
--learning_rate 5e-05 \
|
| 51 |
--num_train_epochs 50 \
|
| 52 |
--bf16 \
|
|
|
|
| 81 |
sleep 5
|
| 82 |
|
| 83 |
|
| 84 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 85 |
--do_train \
|
| 86 |
+
--gradient_checkpointing \
|
| 87 |
--do_predict \
|
| 88 |
--predict_with_generate \
|
| 89 |
--model_name_or_path $2 \
|
|
|
|
| 96 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification \
|
| 97 |
--per_device_train_batch_size 16 \
|
| 98 |
--per_device_eval_batch_size 8 \
|
| 99 |
+
--gradient_accumulation_steps 1 \
|
| 100 |
--learning_rate 5e-05 \
|
| 101 |
--num_train_epochs 50 \
|
| 102 |
--bf16 \
|
|
|
|
| 131 |
sleep 5
|
| 132 |
|
| 133 |
|
| 134 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 135 |
--do_train \
|
| 136 |
+
--gradient_checkpointing \
|
| 137 |
--do_predict \
|
| 138 |
--predict_with_generate \
|
| 139 |
--model_name_or_path $2 \
|
|
|
|
| 146 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization \
|
| 147 |
--per_device_train_batch_size 1 \
|
| 148 |
--per_device_eval_batch_size 8 \
|
| 149 |
+
--gradient_accumulation_steps 16 \
|
| 150 |
--learning_rate 5e-05 \
|
| 151 |
--num_train_epochs 50 \
|
| 152 |
--bf16 \
|
|
|
|
| 181 |
sleep 5
|
| 182 |
|
| 183 |
|
| 184 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 185 |
--do_train \
|
| 186 |
+
--gradient_checkpointing \
|
| 187 |
--do_predict \
|
| 188 |
--predict_with_generate \
|
| 189 |
--model_name_or_path $2 \
|
|
|
|
| 196 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction \
|
| 197 |
--per_device_train_batch_size 4 \
|
| 198 |
--per_device_eval_batch_size 8 \
|
| 199 |
+
--gradient_accumulation_steps 4 \
|
| 200 |
--learning_rate 5e-05 \
|
| 201 |
--num_train_epochs 50 \
|
| 202 |
--bf16 \
|
|
|
|
| 231 |
sleep 5
|
| 232 |
|
| 233 |
|
| 234 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 235 |
--do_train \
|
| 236 |
+
--gradient_checkpointing \
|
| 237 |
--do_predict \
|
| 238 |
--predict_with_generate \
|
| 239 |
--model_name_or_path $2 \
|
|
|
|
| 246 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation \
|
| 247 |
--per_device_train_batch_size 1 \
|
| 248 |
--per_device_eval_batch_size 8 \
|
| 249 |
+
--gradient_accumulation_steps 16 \
|
| 250 |
--learning_rate 5e-05 \
|
| 251 |
--num_train_epochs 50 \
|
| 252 |
--bf16 \
|
|
|
|
| 281 |
sleep 5
|
| 282 |
|
| 283 |
|
| 284 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 285 |
--do_train \
|
| 286 |
+
--gradient_checkpointing \
|
| 287 |
--do_predict \
|
| 288 |
--predict_with_generate \
|
| 289 |
--model_name_or_path $2 \
|
|
|
|
| 296 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction \
|
| 297 |
--per_device_train_batch_size 16 \
|
| 298 |
--per_device_eval_batch_size 8 \
|
| 299 |
+
--gradient_accumulation_steps 1 \
|
| 300 |
--learning_rate 5e-05 \
|
| 301 |
--num_train_epochs 50 \
|
| 302 |
--bf16 \
|
|
|
|
| 331 |
sleep 5
|
| 332 |
|
| 333 |
|
| 334 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 335 |
--do_train \
|
| 336 |
+
--gradient_checkpointing \
|
| 337 |
--do_predict \
|
| 338 |
--predict_with_generate \
|
| 339 |
--model_name_or_path $2 \
|
|
|
|
| 346 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation \
|
| 347 |
--per_device_train_batch_size 16 \
|
| 348 |
--per_device_eval_batch_size 8 \
|
| 349 |
+
--gradient_accumulation_steps 1 \
|
| 350 |
--learning_rate 5e-05 \
|
| 351 |
--num_train_epochs 50 \
|
| 352 |
--bf16 \
|
|
|
|
| 381 |
sleep 5
|
| 382 |
|
| 383 |
|
| 384 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 385 |
--do_train \
|
| 386 |
+
--gradient_checkpointing \
|
| 387 |
--do_predict \
|
| 388 |
--predict_with_generate \
|
| 389 |
--model_name_or_path $2 \
|
|
|
|
| 396 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next \
|
| 397 |
--per_device_train_batch_size 2 \
|
| 398 |
--per_device_eval_batch_size 8 \
|
| 399 |
+
--gradient_accumulation_steps 8 \
|
| 400 |
--learning_rate 5e-05 \
|
| 401 |
--num_train_epochs 50 \
|
| 402 |
--bf16 \
|
|
|
|
| 431 |
sleep 5
|
| 432 |
|
| 433 |
|
| 434 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 435 |
--do_train \
|
| 436 |
+
--gradient_checkpointing \
|
| 437 |
--do_predict \
|
| 438 |
--predict_with_generate \
|
| 439 |
--model_name_or_path $2 \
|
|
|
|
| 446 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation \
|
| 447 |
--per_device_train_batch_size 8 \
|
| 448 |
--per_device_eval_batch_size 8 \
|
| 449 |
+
--gradient_accumulation_steps 2 \
|
| 450 |
--learning_rate 5e-05 \
|
| 451 |
--num_train_epochs 50 \
|
| 452 |
--bf16 \
|
|
|
|
| 481 |
sleep 5
|
| 482 |
|
| 483 |
|
| 484 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 485 |
--do_train \
|
| 486 |
+
--gradient_checkpointing \
|
| 487 |
--do_predict \
|
| 488 |
--predict_with_generate \
|
| 489 |
--model_name_or_path $2 \
|
|
|
|
| 496 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation \
|
| 497 |
--per_device_train_batch_size 2 \
|
| 498 |
--per_device_eval_batch_size 8 \
|
| 499 |
+
--gradient_accumulation_steps 8 \
|
| 500 |
--learning_rate 5e-05 \
|
| 501 |
--num_train_epochs 50 \
|
| 502 |
--bf16 \
|
|
|
|
| 531 |
sleep 5
|
| 532 |
|
| 533 |
|
| 534 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 535 |
--do_train \
|
| 536 |
+
--gradient_checkpointing \
|
| 537 |
--do_predict \
|
| 538 |
--predict_with_generate \
|
| 539 |
--model_name_or_path $2 \
|
|
|
|
| 546 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection \
|
| 547 |
--per_device_train_batch_size 4 \
|
| 548 |
--per_device_eval_batch_size 8 \
|
| 549 |
+
--gradient_accumulation_steps 4 \
|
| 550 |
--learning_rate 5e-05 \
|
| 551 |
--num_train_epochs 50 \
|
| 552 |
--bf16 \
|
|
|
|
| 581 |
sleep 5
|
| 582 |
|
| 583 |
|
| 584 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 585 |
--do_train \
|
| 586 |
+
--gradient_checkpointing \
|
| 587 |
--do_predict \
|
| 588 |
--predict_with_generate \
|
| 589 |
--model_name_or_path $2 \
|
|
|
|
| 596 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
|
| 597 |
--per_device_train_batch_size 1 \
|
| 598 |
--per_device_eval_batch_size 8 \
|
| 599 |
+
--gradient_accumulation_steps 16 \
|
| 600 |
--learning_rate 5e-05 \
|
| 601 |
--num_train_epochs 50 \
|
| 602 |
--bf16 \
|
|
|
|
| 631 |
sleep 5
|
| 632 |
|
| 633 |
|
| 634 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 635 |
--do_train \
|
| 636 |
+
--gradient_checkpointing \
|
| 637 |
--do_predict \
|
| 638 |
--predict_with_generate \
|
| 639 |
--model_name_or_path $2 \
|
|
|
|
| 646 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation \
|
| 647 |
--per_device_train_batch_size 8 \
|
| 648 |
--per_device_eval_batch_size 8 \
|
| 649 |
+
--gradient_accumulation_steps 2 \
|
| 650 |
--learning_rate 5e-05 \
|
| 651 |
--num_train_epochs 50 \
|
| 652 |
--bf16 \
|
|
|
|
| 681 |
sleep 5
|
| 682 |
|
| 683 |
|
| 684 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 685 |
--do_train \
|
| 686 |
+
--gradient_checkpointing \
|
| 687 |
--do_predict \
|
| 688 |
--predict_with_generate \
|
| 689 |
--model_name_or_path $2 \
|
|
|
|
| 696 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification \
|
| 697 |
--per_device_train_batch_size 8 \
|
| 698 |
--per_device_eval_batch_size 8 \
|
| 699 |
+
--gradient_accumulation_steps 2 \
|
| 700 |
--learning_rate 5e-05 \
|
| 701 |
--num_train_epochs 50 \
|
| 702 |
--bf16 \
|
|
|
|
| 731 |
sleep 5
|
| 732 |
|
| 733 |
|
| 734 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 735 |
--do_train \
|
| 736 |
+
--gradient_checkpointing \
|
| 737 |
--do_predict \
|
| 738 |
--predict_with_generate \
|
| 739 |
--model_name_or_path $2 \
|
|
|
|
| 746 |
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification \
|
| 747 |
--per_device_train_batch_size 4 \
|
| 748 |
--per_device_eval_batch_size 8 \
|
| 749 |
+
--gradient_accumulation_steps 4 \
|
| 750 |
--learning_rate 5e-05 \
|
| 751 |
--num_train_epochs 50 \
|
| 752 |
--bf16 \
|
improve_gainlora/gen_script_superni_order1_llama_specroute_p100.sh
ADDED
|
@@ -0,0 +1,761 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# P100 (16 GB) variant β auto-generated by gen_p100_3090_scripts.py
|
| 3 |
+
# Changes vs H100 original:
|
| 4 |
+
# * bf16 β fp16 (P100 has no bf16 hardware support)
|
| 5 |
+
# * DeepSpeed stage2_fp16.config (fp16 forced + ZeRO-2 CPU-optimizer offload)
|
| 6 |
+
# * BSZ=1, GA=original_BSZ Γ original_GA (effective batch unchanged)
|
| 7 |
+
# * max_source_length 1024 β 512 (halves activation memory)
|
| 8 |
+
# * per_device_eval_batch_size 8 β 4
|
| 9 |
+
# * --gradient_checkpointing added
|
| 10 |
+
# * deepspeed --num_gpus 1 (single GPU, no include needed)
|
| 11 |
+
|
| 12 |
+
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
|
| 13 |
+
|
| 14 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 15 |
+
--do_train \
|
| 16 |
+
--gradient_checkpointing \
|
| 17 |
+
--do_predict \
|
| 18 |
+
--predict_with_generate \
|
| 19 |
+
--model_name_or_path $2 \
|
| 20 |
+
--data_dir CL_Benchmark \
|
| 21 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 22 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary \
|
| 23 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary \
|
| 24 |
+
--per_device_train_batch_size 1 \
|
| 25 |
+
--per_device_eval_batch_size 4 \
|
| 26 |
+
--gradient_accumulation_steps 32 \
|
| 27 |
+
--learning_rate 5e-05 \
|
| 28 |
+
--num_train_epochs 50 \
|
| 29 |
+
--fp16 \
|
| 30 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 31 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 32 |
+
--max_source_length 512 \
|
| 33 |
+
--max_target_length 50 \
|
| 34 |
+
--generation_max_length 50 \
|
| 35 |
+
--add_task_name False \
|
| 36 |
+
--add_dataset_name False \
|
| 37 |
+
--overwrite_output_dir \
|
| 38 |
+
--overwrite_cache \
|
| 39 |
+
--lr_scheduler_type constant \
|
| 40 |
+
--warmup_steps 0 \
|
| 41 |
+
--logging_strategy steps \
|
| 42 |
+
--logging_steps 10 \
|
| 43 |
+
--metric_for_best_model eval_rougeL \
|
| 44 |
+
--evaluation_strategy steps \
|
| 45 |
+
--save_strategy steps \
|
| 46 |
+
--save_total_limit 1 \
|
| 47 |
+
--load_best_model_at_end \
|
| 48 |
+
--lora_r 4 \
|
| 49 |
+
--lora_alpha 32 \
|
| 50 |
+
--lora_dropout 0.0 \
|
| 51 |
+
--data_replay_freq -1 \
|
| 52 |
+
--chunk 4 \
|
| 53 |
+
--model_name specroute \
|
| 54 |
+
--threshold 0.995
|
| 55 |
+
|
| 56 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/checkpoint*
|
| 57 |
+
|
| 58 |
+
sleep 5
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 62 |
+
--do_train \
|
| 63 |
+
--gradient_checkpointing \
|
| 64 |
+
--do_predict \
|
| 65 |
+
--predict_with_generate \
|
| 66 |
+
--model_name_or_path $2 \
|
| 67 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights \
|
| 68 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights/spectral_signatures.pt \
|
| 69 |
+
--data_dir CL_Benchmark \
|
| 70 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 71 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 72 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification \
|
| 73 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification \
|
| 74 |
+
--per_device_train_batch_size 1 \
|
| 75 |
+
--per_device_eval_batch_size 4 \
|
| 76 |
+
--gradient_accumulation_steps 32 \
|
| 77 |
+
--learning_rate 5e-05 \
|
| 78 |
+
--num_train_epochs 50 \
|
| 79 |
+
--fp16 \
|
| 80 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 81 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 82 |
+
--max_source_length 512 \
|
| 83 |
+
--max_target_length 50 \
|
| 84 |
+
--generation_max_length 50 \
|
| 85 |
+
--add_task_name False \
|
| 86 |
+
--add_dataset_name False \
|
| 87 |
+
--overwrite_output_dir \
|
| 88 |
+
--overwrite_cache \
|
| 89 |
+
--lr_scheduler_type constant \
|
| 90 |
+
--warmup_steps 0 \
|
| 91 |
+
--logging_strategy steps \
|
| 92 |
+
--logging_steps 10 \
|
| 93 |
+
--metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
|
| 94 |
+
--evaluation_strategy steps \
|
| 95 |
+
--save_strategy steps \
|
| 96 |
+
--save_total_limit 1 \
|
| 97 |
+
--load_best_model_at_end \
|
| 98 |
+
--lora_r 4 \
|
| 99 |
+
--lora_alpha 32 \
|
| 100 |
+
--lora_dropout 0.0 \
|
| 101 |
+
--data_replay_freq -1 \
|
| 102 |
+
--chunk 4 \
|
| 103 |
+
--model_name specroute \
|
| 104 |
+
--threshold 0.995
|
| 105 |
+
|
| 106 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/checkpoint*
|
| 107 |
+
|
| 108 |
+
sleep 5
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 112 |
+
--do_train \
|
| 113 |
+
--gradient_checkpointing \
|
| 114 |
+
--do_predict \
|
| 115 |
+
--predict_with_generate \
|
| 116 |
+
--model_name_or_path $2 \
|
| 117 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights \
|
| 118 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights/spectral_signatures.pt \
|
| 119 |
+
--data_dir CL_Benchmark \
|
| 120 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 121 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 122 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization \
|
| 123 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization \
|
| 124 |
+
--per_device_train_batch_size 1 \
|
| 125 |
+
--per_device_eval_batch_size 4 \
|
| 126 |
+
--gradient_accumulation_steps 32 \
|
| 127 |
+
--learning_rate 5e-05 \
|
| 128 |
+
--num_train_epochs 50 \
|
| 129 |
+
--fp16 \
|
| 130 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 131 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 132 |
+
--max_source_length 512 \
|
| 133 |
+
--max_target_length 50 \
|
| 134 |
+
--generation_max_length 50 \
|
| 135 |
+
--add_task_name False \
|
| 136 |
+
--add_dataset_name False \
|
| 137 |
+
--overwrite_output_dir \
|
| 138 |
+
--overwrite_cache \
|
| 139 |
+
--lr_scheduler_type constant \
|
| 140 |
+
--warmup_steps 0 \
|
| 141 |
+
--logging_strategy steps \
|
| 142 |
+
--logging_steps 10 \
|
| 143 |
+
--metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
|
| 144 |
+
--evaluation_strategy steps \
|
| 145 |
+
--save_strategy steps \
|
| 146 |
+
--save_total_limit 1 \
|
| 147 |
+
--load_best_model_at_end \
|
| 148 |
+
--lora_r 4 \
|
| 149 |
+
--lora_alpha 32 \
|
| 150 |
+
--lora_dropout 0.0 \
|
| 151 |
+
--data_replay_freq -1 \
|
| 152 |
+
--chunk 4 \
|
| 153 |
+
--model_name specroute \
|
| 154 |
+
--threshold 0.995
|
| 155 |
+
|
| 156 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/checkpoint*
|
| 157 |
+
|
| 158 |
+
sleep 5
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 162 |
+
--do_train \
|
| 163 |
+
--gradient_checkpointing \
|
| 164 |
+
--do_predict \
|
| 165 |
+
--predict_with_generate \
|
| 166 |
+
--model_name_or_path $2 \
|
| 167 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights \
|
| 168 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights/spectral_signatures.pt \
|
| 169 |
+
--data_dir CL_Benchmark \
|
| 170 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 171 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 172 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction \
|
| 173 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction \
|
| 174 |
+
--per_device_train_batch_size 1 \
|
| 175 |
+
--per_device_eval_batch_size 4 \
|
| 176 |
+
--gradient_accumulation_steps 32 \
|
| 177 |
+
--learning_rate 5e-05 \
|
| 178 |
+
--num_train_epochs 50 \
|
| 179 |
+
--fp16 \
|
| 180 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 181 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 182 |
+
--max_source_length 512 \
|
| 183 |
+
--max_target_length 50 \
|
| 184 |
+
--generation_max_length 50 \
|
| 185 |
+
--add_task_name False \
|
| 186 |
+
--add_dataset_name False \
|
| 187 |
+
--overwrite_output_dir \
|
| 188 |
+
--overwrite_cache \
|
| 189 |
+
--lr_scheduler_type constant \
|
| 190 |
+
--warmup_steps 0 \
|
| 191 |
+
--logging_strategy steps \
|
| 192 |
+
--logging_steps 10 \
|
| 193 |
+
--metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
|
| 194 |
+
--evaluation_strategy steps \
|
| 195 |
+
--save_strategy steps \
|
| 196 |
+
--save_total_limit 1 \
|
| 197 |
+
--load_best_model_at_end \
|
| 198 |
+
--lora_r 4 \
|
| 199 |
+
--lora_alpha 32 \
|
| 200 |
+
--lora_dropout 0.0 \
|
| 201 |
+
--data_replay_freq -1 \
|
| 202 |
+
--chunk 4 \
|
| 203 |
+
--model_name specroute \
|
| 204 |
+
--threshold 0.995
|
| 205 |
+
|
| 206 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/checkpoint*
|
| 207 |
+
|
| 208 |
+
sleep 5
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 212 |
+
--do_train \
|
| 213 |
+
--gradient_checkpointing \
|
| 214 |
+
--do_predict \
|
| 215 |
+
--predict_with_generate \
|
| 216 |
+
--model_name_or_path $2 \
|
| 217 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights \
|
| 218 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights/spectral_signatures.pt \
|
| 219 |
+
--data_dir CL_Benchmark \
|
| 220 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 221 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 222 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation \
|
| 223 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation \
|
| 224 |
+
--per_device_train_batch_size 1 \
|
| 225 |
+
--per_device_eval_batch_size 4 \
|
| 226 |
+
--gradient_accumulation_steps 32 \
|
| 227 |
+
--learning_rate 5e-05 \
|
| 228 |
+
--num_train_epochs 50 \
|
| 229 |
+
--fp16 \
|
| 230 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 231 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 232 |
+
--max_source_length 512 \
|
| 233 |
+
--max_target_length 50 \
|
| 234 |
+
--generation_max_length 50 \
|
| 235 |
+
--add_task_name False \
|
| 236 |
+
--add_dataset_name False \
|
| 237 |
+
--overwrite_output_dir \
|
| 238 |
+
--overwrite_cache \
|
| 239 |
+
--lr_scheduler_type constant \
|
| 240 |
+
--warmup_steps 0 \
|
| 241 |
+
--logging_strategy steps \
|
| 242 |
+
--logging_steps 10 \
|
| 243 |
+
--metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
|
| 244 |
+
--evaluation_strategy steps \
|
| 245 |
+
--save_strategy steps \
|
| 246 |
+
--save_total_limit 1 \
|
| 247 |
+
--load_best_model_at_end \
|
| 248 |
+
--lora_r 4 \
|
| 249 |
+
--lora_alpha 32 \
|
| 250 |
+
--lora_dropout 0.0 \
|
| 251 |
+
--data_replay_freq -1 \
|
| 252 |
+
--chunk 4 \
|
| 253 |
+
--model_name specroute \
|
| 254 |
+
--threshold 0.995
|
| 255 |
+
|
| 256 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/checkpoint*
|
| 257 |
+
|
| 258 |
+
sleep 5
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 262 |
+
--do_train \
|
| 263 |
+
--gradient_checkpointing \
|
| 264 |
+
--do_predict \
|
| 265 |
+
--predict_with_generate \
|
| 266 |
+
--model_name_or_path $2 \
|
| 267 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights \
|
| 268 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights/spectral_signatures.pt \
|
| 269 |
+
--data_dir CL_Benchmark \
|
| 270 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 271 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 272 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction \
|
| 273 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction \
|
| 274 |
+
--per_device_train_batch_size 1 \
|
| 275 |
+
--per_device_eval_batch_size 4 \
|
| 276 |
+
--gradient_accumulation_steps 32 \
|
| 277 |
+
--learning_rate 5e-05 \
|
| 278 |
+
--num_train_epochs 50 \
|
| 279 |
+
--fp16 \
|
| 280 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 281 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 282 |
+
--max_source_length 512 \
|
| 283 |
+
--max_target_length 50 \
|
| 284 |
+
--generation_max_length 50 \
|
| 285 |
+
--add_task_name False \
|
| 286 |
+
--add_dataset_name False \
|
| 287 |
+
--overwrite_output_dir \
|
| 288 |
+
--overwrite_cache \
|
| 289 |
+
--lr_scheduler_type constant \
|
| 290 |
+
--warmup_steps 0 \
|
| 291 |
+
--logging_strategy steps \
|
| 292 |
+
--logging_steps 10 \
|
| 293 |
+
--metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
|
| 294 |
+
--evaluation_strategy steps \
|
| 295 |
+
--save_strategy steps \
|
| 296 |
+
--save_total_limit 1 \
|
| 297 |
+
--load_best_model_at_end \
|
| 298 |
+
--lora_r 4 \
|
| 299 |
+
--lora_alpha 32 \
|
| 300 |
+
--lora_dropout 0.0 \
|
| 301 |
+
--data_replay_freq -1 \
|
| 302 |
+
--chunk 4 \
|
| 303 |
+
--model_name specroute \
|
| 304 |
+
--threshold 0.995
|
| 305 |
+
|
| 306 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/checkpoint*
|
| 307 |
+
|
| 308 |
+
sleep 5
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 312 |
+
--do_train \
|
| 313 |
+
--gradient_checkpointing \
|
| 314 |
+
--do_predict \
|
| 315 |
+
--predict_with_generate \
|
| 316 |
+
--model_name_or_path $2 \
|
| 317 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights \
|
| 318 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights/spectral_signatures.pt \
|
| 319 |
+
--data_dir CL_Benchmark \
|
| 320 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 321 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 322 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation \
|
| 323 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation \
|
| 324 |
+
--per_device_train_batch_size 1 \
|
| 325 |
+
--per_device_eval_batch_size 4 \
|
| 326 |
+
--gradient_accumulation_steps 32 \
|
| 327 |
+
--learning_rate 5e-05 \
|
| 328 |
+
--num_train_epochs 50 \
|
| 329 |
+
--fp16 \
|
| 330 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 331 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 332 |
+
--max_source_length 512 \
|
| 333 |
+
--max_target_length 50 \
|
| 334 |
+
--generation_max_length 50 \
|
| 335 |
+
--add_task_name False \
|
| 336 |
+
--add_dataset_name False \
|
| 337 |
+
--overwrite_output_dir \
|
| 338 |
+
--overwrite_cache \
|
| 339 |
+
--lr_scheduler_type constant \
|
| 340 |
+
--warmup_steps 0 \
|
| 341 |
+
--logging_strategy steps \
|
| 342 |
+
--logging_steps 10 \
|
| 343 |
+
--metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
|
| 344 |
+
--evaluation_strategy steps \
|
| 345 |
+
--save_strategy steps \
|
| 346 |
+
--save_total_limit 1 \
|
| 347 |
+
--load_best_model_at_end \
|
| 348 |
+
--lora_r 4 \
|
| 349 |
+
--lora_alpha 32 \
|
| 350 |
+
--lora_dropout 0.0 \
|
| 351 |
+
--data_replay_freq -1 \
|
| 352 |
+
--chunk 4 \
|
| 353 |
+
--model_name specroute \
|
| 354 |
+
--threshold 0.995
|
| 355 |
+
|
| 356 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/checkpoint*
|
| 357 |
+
|
| 358 |
+
sleep 5
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 362 |
+
--do_train \
|
| 363 |
+
--gradient_checkpointing \
|
| 364 |
+
--do_predict \
|
| 365 |
+
--predict_with_generate \
|
| 366 |
+
--model_name_or_path $2 \
|
| 367 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights \
|
| 368 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/spectral_signatures.pt \
|
| 369 |
+
--data_dir CL_Benchmark \
|
| 370 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 371 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 372 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next \
|
| 373 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next \
|
| 374 |
+
--per_device_train_batch_size 1 \
|
| 375 |
+
--per_device_eval_batch_size 4 \
|
| 376 |
+
--gradient_accumulation_steps 32 \
|
| 377 |
+
--learning_rate 5e-05 \
|
| 378 |
+
--num_train_epochs 50 \
|
| 379 |
+
--fp16 \
|
| 380 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 381 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 382 |
+
--max_source_length 512 \
|
| 383 |
+
--max_target_length 50 \
|
| 384 |
+
--generation_max_length 50 \
|
| 385 |
+
--add_task_name False \
|
| 386 |
+
--add_dataset_name False \
|
| 387 |
+
--overwrite_output_dir \
|
| 388 |
+
--overwrite_cache \
|
| 389 |
+
--lr_scheduler_type constant \
|
| 390 |
+
--warmup_steps 0 \
|
| 391 |
+
--logging_strategy steps \
|
| 392 |
+
--logging_steps 10 \
|
| 393 |
+
--metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
|
| 394 |
+
--evaluation_strategy steps \
|
| 395 |
+
--save_strategy steps \
|
| 396 |
+
--save_total_limit 1 \
|
| 397 |
+
--load_best_model_at_end \
|
| 398 |
+
--lora_r 4 \
|
| 399 |
+
--lora_alpha 32 \
|
| 400 |
+
--lora_dropout 0.0 \
|
| 401 |
+
--data_replay_freq -1 \
|
| 402 |
+
--chunk 4 \
|
| 403 |
+
--model_name specroute \
|
| 404 |
+
--threshold 0.995
|
| 405 |
+
|
| 406 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/checkpoint*
|
| 407 |
+
|
| 408 |
+
sleep 5
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 412 |
+
--do_train \
|
| 413 |
+
--gradient_checkpointing \
|
| 414 |
+
--do_predict \
|
| 415 |
+
--predict_with_generate \
|
| 416 |
+
--model_name_or_path $2 \
|
| 417 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights \
|
| 418 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights/spectral_signatures.pt \
|
| 419 |
+
--data_dir CL_Benchmark \
|
| 420 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 421 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 422 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation \
|
| 423 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation \
|
| 424 |
+
--per_device_train_batch_size 1 \
|
| 425 |
+
--per_device_eval_batch_size 4 \
|
| 426 |
+
--gradient_accumulation_steps 32 \
|
| 427 |
+
--learning_rate 5e-05 \
|
| 428 |
+
--num_train_epochs 50 \
|
| 429 |
+
--fp16 \
|
| 430 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 431 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 432 |
+
--max_source_length 512 \
|
| 433 |
+
--max_target_length 50 \
|
| 434 |
+
--generation_max_length 50 \
|
| 435 |
+
--add_task_name False \
|
| 436 |
+
--add_dataset_name False \
|
| 437 |
+
--overwrite_output_dir \
|
| 438 |
+
--overwrite_cache \
|
| 439 |
+
--lr_scheduler_type constant \
|
| 440 |
+
--warmup_steps 0 \
|
| 441 |
+
--logging_strategy steps \
|
| 442 |
+
--logging_steps 10 \
|
| 443 |
+
--metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
|
| 444 |
+
--evaluation_strategy steps \
|
| 445 |
+
--save_strategy steps \
|
| 446 |
+
--save_total_limit 1 \
|
| 447 |
+
--load_best_model_at_end \
|
| 448 |
+
--lora_r 4 \
|
| 449 |
+
--lora_alpha 32 \
|
| 450 |
+
--lora_dropout 0.0 \
|
| 451 |
+
--data_replay_freq -1 \
|
| 452 |
+
--chunk 4 \
|
| 453 |
+
--model_name specroute \
|
| 454 |
+
--threshold 0.995
|
| 455 |
+
|
| 456 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/checkpoint*
|
| 457 |
+
|
| 458 |
+
sleep 5
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 462 |
+
--do_train \
|
| 463 |
+
--gradient_checkpointing \
|
| 464 |
+
--do_predict \
|
| 465 |
+
--predict_with_generate \
|
| 466 |
+
--model_name_or_path $2 \
|
| 467 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights \
|
| 468 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/spectral_signatures.pt \
|
| 469 |
+
--data_dir CL_Benchmark \
|
| 470 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 471 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 472 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation \
|
| 473 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation \
|
| 474 |
+
--per_device_train_batch_size 1 \
|
| 475 |
+
--per_device_eval_batch_size 4 \
|
| 476 |
+
--gradient_accumulation_steps 32 \
|
| 477 |
+
--learning_rate 5e-05 \
|
| 478 |
+
--num_train_epochs 50 \
|
| 479 |
+
--fp16 \
|
| 480 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 481 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 482 |
+
--max_source_length 512 \
|
| 483 |
+
--max_target_length 50 \
|
| 484 |
+
--generation_max_length 50 \
|
| 485 |
+
--add_task_name False \
|
| 486 |
+
--add_dataset_name False \
|
| 487 |
+
--overwrite_output_dir \
|
| 488 |
+
--overwrite_cache \
|
| 489 |
+
--lr_scheduler_type constant \
|
| 490 |
+
--warmup_steps 0 \
|
| 491 |
+
--logging_strategy steps \
|
| 492 |
+
--logging_steps 10 \
|
| 493 |
+
--metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
|
| 494 |
+
--evaluation_strategy steps \
|
| 495 |
+
--save_strategy steps \
|
| 496 |
+
--save_total_limit 1 \
|
| 497 |
+
--load_best_model_at_end \
|
| 498 |
+
--lora_r 4 \
|
| 499 |
+
--lora_alpha 32 \
|
| 500 |
+
--lora_dropout 0.0 \
|
| 501 |
+
--data_replay_freq -1 \
|
| 502 |
+
--chunk 4 \
|
| 503 |
+
--model_name specroute \
|
| 504 |
+
--threshold 0.995
|
| 505 |
+
|
| 506 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/checkpoint*
|
| 507 |
+
|
| 508 |
+
sleep 5
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 512 |
+
--do_train \
|
| 513 |
+
--gradient_checkpointing \
|
| 514 |
+
--do_predict \
|
| 515 |
+
--predict_with_generate \
|
| 516 |
+
--model_name_or_path $2 \
|
| 517 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights \
|
| 518 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights/spectral_signatures.pt \
|
| 519 |
+
--data_dir CL_Benchmark \
|
| 520 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 521 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 522 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection \
|
| 523 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection \
|
| 524 |
+
--per_device_train_batch_size 1 \
|
| 525 |
+
--per_device_eval_batch_size 4 \
|
| 526 |
+
--gradient_accumulation_steps 32 \
|
| 527 |
+
--learning_rate 5e-05 \
|
| 528 |
+
--num_train_epochs 50 \
|
| 529 |
+
--fp16 \
|
| 530 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 531 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 532 |
+
--max_source_length 512 \
|
| 533 |
+
--max_target_length 50 \
|
| 534 |
+
--generation_max_length 50 \
|
| 535 |
+
--add_task_name False \
|
| 536 |
+
--add_dataset_name False \
|
| 537 |
+
--overwrite_output_dir \
|
| 538 |
+
--overwrite_cache \
|
| 539 |
+
--lr_scheduler_type constant \
|
| 540 |
+
--warmup_steps 0 \
|
| 541 |
+
--logging_strategy steps \
|
| 542 |
+
--logging_steps 10 \
|
| 543 |
+
--metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \
|
| 544 |
+
--evaluation_strategy steps \
|
| 545 |
+
--save_strategy steps \
|
| 546 |
+
--save_total_limit 1 \
|
| 547 |
+
--load_best_model_at_end \
|
| 548 |
+
--lora_r 4 \
|
| 549 |
+
--lora_alpha 32 \
|
| 550 |
+
--lora_dropout 0.0 \
|
| 551 |
+
--data_replay_freq -1 \
|
| 552 |
+
--chunk 4 \
|
| 553 |
+
--model_name specroute \
|
| 554 |
+
--threshold 0.995
|
| 555 |
+
|
| 556 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/checkpoint*
|
| 557 |
+
|
| 558 |
+
sleep 5
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 562 |
+
--do_train \
|
| 563 |
+
--gradient_checkpointing \
|
| 564 |
+
--do_predict \
|
| 565 |
+
--predict_with_generate \
|
| 566 |
+
--model_name_or_path $2 \
|
| 567 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights \
|
| 568 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/spectral_signatures.pt \
|
| 569 |
+
--data_dir CL_Benchmark \
|
| 570 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 571 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 572 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization \
|
| 573 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
|
| 574 |
+
--per_device_train_batch_size 1 \
|
| 575 |
+
--per_device_eval_batch_size 4 \
|
| 576 |
+
--gradient_accumulation_steps 32 \
|
| 577 |
+
--learning_rate 5e-05 \
|
| 578 |
+
--num_train_epochs 50 \
|
| 579 |
+
--fp16 \
|
| 580 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 581 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 582 |
+
--max_source_length 512 \
|
| 583 |
+
--max_target_length 50 \
|
| 584 |
+
--generation_max_length 50 \
|
| 585 |
+
--add_task_name False \
|
| 586 |
+
--add_dataset_name False \
|
| 587 |
+
--overwrite_output_dir \
|
| 588 |
+
--overwrite_cache \
|
| 589 |
+
--lr_scheduler_type constant \
|
| 590 |
+
--warmup_steps 0 \
|
| 591 |
+
--logging_strategy steps \
|
| 592 |
+
--logging_steps 10 \
|
| 593 |
+
--metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
|
| 594 |
+
--evaluation_strategy steps \
|
| 595 |
+
--save_strategy steps \
|
| 596 |
+
--save_total_limit 1 \
|
| 597 |
+
--load_best_model_at_end \
|
| 598 |
+
--lora_r 4 \
|
| 599 |
+
--lora_alpha 32 \
|
| 600 |
+
--lora_dropout 0.0 \
|
| 601 |
+
--data_replay_freq -1 \
|
| 602 |
+
--chunk 4 \
|
| 603 |
+
--model_name specroute \
|
| 604 |
+
--threshold 0.995
|
| 605 |
+
|
| 606 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/checkpoint*
|
| 607 |
+
|
| 608 |
+
sleep 5
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 612 |
+
--do_train \
|
| 613 |
+
--gradient_checkpointing \
|
| 614 |
+
--do_predict \
|
| 615 |
+
--predict_with_generate \
|
| 616 |
+
--model_name_or_path $2 \
|
| 617 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
|
| 618 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/spectral_signatures.pt \
|
| 619 |
+
--data_dir CL_Benchmark \
|
| 620 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 621 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 622 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation \
|
| 623 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation \
|
| 624 |
+
--per_device_train_batch_size 1 \
|
| 625 |
+
--per_device_eval_batch_size 4 \
|
| 626 |
+
--gradient_accumulation_steps 32 \
|
| 627 |
+
--learning_rate 5e-05 \
|
| 628 |
+
--num_train_epochs 50 \
|
| 629 |
+
--fp16 \
|
| 630 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 631 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 632 |
+
--max_source_length 512 \
|
| 633 |
+
--max_target_length 50 \
|
| 634 |
+
--generation_max_length 50 \
|
| 635 |
+
--add_task_name False \
|
| 636 |
+
--add_dataset_name False \
|
| 637 |
+
--overwrite_output_dir \
|
| 638 |
+
--overwrite_cache \
|
| 639 |
+
--lr_scheduler_type constant \
|
| 640 |
+
--warmup_steps 0 \
|
| 641 |
+
--logging_strategy steps \
|
| 642 |
+
--logging_steps 10 \
|
| 643 |
+
--metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
|
| 644 |
+
--evaluation_strategy steps \
|
| 645 |
+
--save_strategy steps \
|
| 646 |
+
--save_total_limit 1 \
|
| 647 |
+
--load_best_model_at_end \
|
| 648 |
+
--lora_r 4 \
|
| 649 |
+
--lora_alpha 32 \
|
| 650 |
+
--lora_dropout 0.0 \
|
| 651 |
+
--data_replay_freq -1 \
|
| 652 |
+
--chunk 4 \
|
| 653 |
+
--model_name specroute \
|
| 654 |
+
--threshold 0.995
|
| 655 |
+
|
| 656 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/checkpoint*
|
| 657 |
+
|
| 658 |
+
sleep 5
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 662 |
+
--do_train \
|
| 663 |
+
--gradient_checkpointing \
|
| 664 |
+
--do_predict \
|
| 665 |
+
--predict_with_generate \
|
| 666 |
+
--model_name_or_path $2 \
|
| 667 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/saved_weights \
|
| 668 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/saved_weights/spectral_signatures.pt \
|
| 669 |
+
--data_dir CL_Benchmark \
|
| 670 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 671 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 672 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification \
|
| 673 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification \
|
| 674 |
+
--per_device_train_batch_size 1 \
|
| 675 |
+
--per_device_eval_batch_size 4 \
|
| 676 |
+
--gradient_accumulation_steps 32 \
|
| 677 |
+
--learning_rate 5e-05 \
|
| 678 |
+
--num_train_epochs 50 \
|
| 679 |
+
--fp16 \
|
| 680 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 681 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 682 |
+
--max_source_length 512 \
|
| 683 |
+
--max_target_length 50 \
|
| 684 |
+
--generation_max_length 50 \
|
| 685 |
+
--add_task_name False \
|
| 686 |
+
--add_dataset_name False \
|
| 687 |
+
--overwrite_output_dir \
|
| 688 |
+
--overwrite_cache \
|
| 689 |
+
--lr_scheduler_type constant \
|
| 690 |
+
--warmup_steps 0 \
|
| 691 |
+
--logging_strategy steps \
|
| 692 |
+
--logging_steps 10 \
|
| 693 |
+
--metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
|
| 694 |
+
--evaluation_strategy steps \
|
| 695 |
+
--save_strategy steps \
|
| 696 |
+
--save_total_limit 1 \
|
| 697 |
+
--load_best_model_at_end \
|
| 698 |
+
--lora_r 4 \
|
| 699 |
+
--lora_alpha 32 \
|
| 700 |
+
--lora_dropout 0.0 \
|
| 701 |
+
--data_replay_freq -1 \
|
| 702 |
+
--chunk 4 \
|
| 703 |
+
--model_name specroute \
|
| 704 |
+
--threshold 0.995
|
| 705 |
+
|
| 706 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification/checkpoint*
|
| 707 |
+
|
| 708 |
+
sleep 5
|
| 709 |
+
|
| 710 |
+
|
| 711 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 712 |
+
--do_train \
|
| 713 |
+
--gradient_checkpointing \
|
| 714 |
+
--do_predict \
|
| 715 |
+
--predict_with_generate \
|
| 716 |
+
--model_name_or_path $2 \
|
| 717 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/13-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification/saved_weights \
|
| 718 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/14-task1687_sentiment140_classification/saved_weights/spectral_signatures.pt \
|
| 719 |
+
--data_dir CL_Benchmark \
|
| 720 |
+
--task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
|
| 721 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 722 |
+
--task_config_dir configs/gen_script_superni_order1_llama_configs/task875_emotion_classification \
|
| 723 |
+
--output_dir logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification \
|
| 724 |
+
--per_device_train_batch_size 1 \
|
| 725 |
+
--per_device_eval_batch_size 4 \
|
| 726 |
+
--gradient_accumulation_steps 32 \
|
| 727 |
+
--learning_rate 5e-05 \
|
| 728 |
+
--num_train_epochs 50 \
|
| 729 |
+
--fp16 \
|
| 730 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 731 |
+
--run_name gen_script_superni_order1_llama_specroute \
|
| 732 |
+
--max_source_length 512 \
|
| 733 |
+
--max_target_length 50 \
|
| 734 |
+
--generation_max_length 50 \
|
| 735 |
+
--add_task_name False \
|
| 736 |
+
--add_dataset_name False \
|
| 737 |
+
--overwrite_output_dir \
|
| 738 |
+
--overwrite_cache \
|
| 739 |
+
--lr_scheduler_type constant \
|
| 740 |
+
--warmup_steps 0 \
|
| 741 |
+
--logging_strategy steps \
|
| 742 |
+
--logging_steps 10 \
|
| 743 |
+
--metric_for_best_model eval_rougeL_for_task875_emotion_classification \
|
| 744 |
+
--evaluation_strategy steps \
|
| 745 |
+
--save_strategy steps \
|
| 746 |
+
--save_total_limit 1 \
|
| 747 |
+
--load_best_model_at_end \
|
| 748 |
+
--lora_r 4 \
|
| 749 |
+
--lora_alpha 32 \
|
| 750 |
+
--lora_dropout 0.0 \
|
| 751 |
+
--data_replay_freq -1 \
|
| 752 |
+
--chunk 4 \
|
| 753 |
+
--model_name specroute \
|
| 754 |
+
--threshold 0.995
|
| 755 |
+
|
| 756 |
+
rm -rf logs_and_outputs/gen_script_superni_order1_llama_specroute/outputs/15-task875_emotion_classification/checkpoint*
|
| 757 |
+
|
| 758 |
+
sleep 5
|
| 759 |
+
|
| 760 |
+
|
| 761 |
+
python score.py gen_script_superni_order1_llama_specroute gen_script_superni_order1_llama_specroute
|
improve_gainlora/gen_script_superni_order1_t5_specroute.sh
CHANGED
|
@@ -51,9 +51,14 @@ elif [ "$GPU_MEM" -gt 16000 ]; then
|
|
| 51 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 52 |
else
|
| 53 |
GPU_MODE="a100"
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
FP16_FLAG=""
|
| 56 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 57 |
fi
|
| 58 |
|
| 59 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 51 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 52 |
else
|
| 53 |
GPU_MODE="a100"
|
| 54 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 55 |
+
GPU_IDS="0,1"
|
| 56 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 57 |
+
else
|
| 58 |
+
GPU_IDS="${1:-0}"
|
| 59 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 60 |
+
fi
|
| 61 |
FP16_FLAG=""
|
|
|
|
| 62 |
fi
|
| 63 |
|
| 64 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
improve_gainlora/gen_script_superni_order2_llama_specroute.sh
CHANGED
|
@@ -8,9 +8,35 @@
|
|
| 8 |
#SBATCH --gres=gpu:a100-sxm4-80gb:1
|
| 9 |
|
| 10 |
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
--do_train \
|
|
|
|
| 14 |
--do_predict \
|
| 15 |
--predict_with_generate \
|
| 16 |
--model_name_or_path $2 \
|
|
@@ -20,7 +46,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 20 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection \
|
| 21 |
--per_device_train_batch_size 4 \
|
| 22 |
--per_device_eval_batch_size 8 \
|
| 23 |
-
--gradient_accumulation_steps
|
| 24 |
--learning_rate 5e-05 \
|
| 25 |
--num_train_epochs 50 \
|
| 26 |
--bf16 \
|
|
@@ -55,8 +81,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task
|
|
| 55 |
sleep 5
|
| 56 |
|
| 57 |
|
| 58 |
-
deepspeed --include
|
| 59 |
--do_train \
|
|
|
|
| 60 |
--do_predict \
|
| 61 |
--predict_with_generate \
|
| 62 |
--model_name_or_path $2 \
|
|
@@ -69,7 +96,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 69 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation \
|
| 70 |
--per_device_train_batch_size 8 \
|
| 71 |
--per_device_eval_batch_size 8 \
|
| 72 |
-
--gradient_accumulation_steps
|
| 73 |
--learning_rate 5e-05 \
|
| 74 |
--num_train_epochs 50 \
|
| 75 |
--bf16 \
|
|
@@ -104,8 +131,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task
|
|
| 104 |
sleep 5
|
| 105 |
|
| 106 |
|
| 107 |
-
deepspeed --include
|
| 108 |
--do_train \
|
|
|
|
| 109 |
--do_predict \
|
| 110 |
--predict_with_generate \
|
| 111 |
--model_name_or_path $2 \
|
|
@@ -118,7 +146,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 118 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation \
|
| 119 |
--per_device_train_batch_size 2 \
|
| 120 |
--per_device_eval_batch_size 8 \
|
| 121 |
-
--gradient_accumulation_steps
|
| 122 |
--learning_rate 5e-05 \
|
| 123 |
--num_train_epochs 50 \
|
| 124 |
--bf16 \
|
|
@@ -153,8 +181,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task
|
|
| 153 |
sleep 5
|
| 154 |
|
| 155 |
|
| 156 |
-
deepspeed --include
|
| 157 |
--do_train \
|
|
|
|
| 158 |
--do_predict \
|
| 159 |
--predict_with_generate \
|
| 160 |
--model_name_or_path $2 \
|
|
@@ -167,7 +196,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 167 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation \
|
| 168 |
--per_device_train_batch_size 16 \
|
| 169 |
--per_device_eval_batch_size 8 \
|
| 170 |
-
--gradient_accumulation_steps
|
| 171 |
--learning_rate 5e-05 \
|
| 172 |
--num_train_epochs 50 \
|
| 173 |
--bf16 \
|
|
@@ -202,8 +231,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task
|
|
| 202 |
sleep 5
|
| 203 |
|
| 204 |
|
| 205 |
-
deepspeed --include
|
| 206 |
--do_train \
|
|
|
|
| 207 |
--do_predict \
|
| 208 |
--predict_with_generate \
|
| 209 |
--model_name_or_path $2 \
|
|
@@ -216,7 +246,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 216 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary \
|
| 217 |
--per_device_train_batch_size 2 \
|
| 218 |
--per_device_eval_batch_size 8 \
|
| 219 |
-
--gradient_accumulation_steps
|
| 220 |
--learning_rate 5e-05 \
|
| 221 |
--num_train_epochs 50 \
|
| 222 |
--bf16 \
|
|
@@ -251,8 +281,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task
|
|
| 251 |
sleep 5
|
| 252 |
|
| 253 |
|
| 254 |
-
deepspeed --include
|
| 255 |
--do_train \
|
|
|
|
| 256 |
--do_predict \
|
| 257 |
--predict_with_generate \
|
| 258 |
--model_name_or_path $2 \
|
|
@@ -265,7 +296,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 265 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification \
|
| 266 |
--per_device_train_batch_size 8 \
|
| 267 |
--per_device_eval_batch_size 8 \
|
| 268 |
-
--gradient_accumulation_steps
|
| 269 |
--learning_rate 5e-05 \
|
| 270 |
--num_train_epochs 50 \
|
| 271 |
--bf16 \
|
|
@@ -300,8 +331,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task
|
|
| 300 |
sleep 5
|
| 301 |
|
| 302 |
|
| 303 |
-
deepspeed --include
|
| 304 |
--do_train \
|
|
|
|
| 305 |
--do_predict \
|
| 306 |
--predict_with_generate \
|
| 307 |
--model_name_or_path $2 \
|
|
@@ -314,7 +346,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 314 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation \
|
| 315 |
--per_device_train_batch_size 8 \
|
| 316 |
--per_device_eval_batch_size 8 \
|
| 317 |
-
--gradient_accumulation_steps
|
| 318 |
--learning_rate 5e-05 \
|
| 319 |
--num_train_epochs 50 \
|
| 320 |
--bf16 \
|
|
@@ -349,8 +381,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task
|
|
| 349 |
sleep 5
|
| 350 |
|
| 351 |
|
| 352 |
-
deepspeed --include
|
| 353 |
--do_train \
|
|
|
|
| 354 |
--do_predict \
|
| 355 |
--predict_with_generate \
|
| 356 |
--model_name_or_path $2 \
|
|
@@ -363,7 +396,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 363 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification \
|
| 364 |
--per_device_train_batch_size 16 \
|
| 365 |
--per_device_eval_batch_size 8 \
|
| 366 |
-
--gradient_accumulation_steps
|
| 367 |
--learning_rate 5e-05 \
|
| 368 |
--num_train_epochs 50 \
|
| 369 |
--bf16 \
|
|
@@ -398,8 +431,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task
|
|
| 398 |
sleep 5
|
| 399 |
|
| 400 |
|
| 401 |
-
deepspeed --include
|
| 402 |
--do_train \
|
|
|
|
| 403 |
--do_predict \
|
| 404 |
--predict_with_generate \
|
| 405 |
--model_name_or_path $2 \
|
|
@@ -412,7 +446,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 412 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction \
|
| 413 |
--per_device_train_batch_size 16 \
|
| 414 |
--per_device_eval_batch_size 8 \
|
| 415 |
-
--gradient_accumulation_steps
|
| 416 |
--learning_rate 5e-05 \
|
| 417 |
--num_train_epochs 50 \
|
| 418 |
--bf16 \
|
|
@@ -447,8 +481,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task
|
|
| 447 |
sleep 5
|
| 448 |
|
| 449 |
|
| 450 |
-
deepspeed --include
|
| 451 |
--do_train \
|
|
|
|
| 452 |
--do_predict \
|
| 453 |
--predict_with_generate \
|
| 454 |
--model_name_or_path $2 \
|
|
@@ -461,7 +496,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 461 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next \
|
| 462 |
--per_device_train_batch_size 2 \
|
| 463 |
--per_device_eval_batch_size 8 \
|
| 464 |
-
--gradient_accumulation_steps
|
| 465 |
--learning_rate 5e-05 \
|
| 466 |
--num_train_epochs 50 \
|
| 467 |
--bf16 \
|
|
@@ -496,8 +531,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-tas
|
|
| 496 |
sleep 5
|
| 497 |
|
| 498 |
|
| 499 |
-
deepspeed --include
|
| 500 |
--do_train \
|
|
|
|
| 501 |
--do_predict \
|
| 502 |
--predict_with_generate \
|
| 503 |
--model_name_or_path $2 \
|
|
@@ -510,7 +546,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 510 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction \
|
| 511 |
--per_device_train_batch_size 4 \
|
| 512 |
--per_device_eval_batch_size 8 \
|
| 513 |
-
--gradient_accumulation_steps
|
| 514 |
--learning_rate 5e-05 \
|
| 515 |
--num_train_epochs 50 \
|
| 516 |
--bf16 \
|
|
@@ -545,8 +581,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-tas
|
|
| 545 |
sleep 5
|
| 546 |
|
| 547 |
|
| 548 |
-
deepspeed --include
|
| 549 |
--do_train \
|
|
|
|
| 550 |
--do_predict \
|
| 551 |
--predict_with_generate \
|
| 552 |
--model_name_or_path $2 \
|
|
@@ -559,7 +596,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 559 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
|
| 560 |
--per_device_train_batch_size 1 \
|
| 561 |
--per_device_eval_batch_size 8 \
|
| 562 |
-
--gradient_accumulation_steps
|
| 563 |
--learning_rate 5e-05 \
|
| 564 |
--num_train_epochs 50 \
|
| 565 |
--bf16 \
|
|
@@ -594,8 +631,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-tas
|
|
| 594 |
sleep 5
|
| 595 |
|
| 596 |
|
| 597 |
-
deepspeed --include
|
| 598 |
--do_train \
|
|
|
|
| 599 |
--do_predict \
|
| 600 |
--predict_with_generate \
|
| 601 |
--model_name_or_path $2 \
|
|
@@ -608,7 +646,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 608 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation \
|
| 609 |
--per_device_train_batch_size 1 \
|
| 610 |
--per_device_eval_batch_size 8 \
|
| 611 |
-
--gradient_accumulation_steps
|
| 612 |
--learning_rate 5e-05 \
|
| 613 |
--num_train_epochs 50 \
|
| 614 |
--bf16 \
|
|
@@ -643,8 +681,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-tas
|
|
| 643 |
sleep 5
|
| 644 |
|
| 645 |
|
| 646 |
-
deepspeed --include
|
| 647 |
--do_train \
|
|
|
|
| 648 |
--do_predict \
|
| 649 |
--predict_with_generate \
|
| 650 |
--model_name_or_path $2 \
|
|
@@ -657,7 +696,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 657 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization \
|
| 658 |
--per_device_train_batch_size 1 \
|
| 659 |
--per_device_eval_batch_size 8 \
|
| 660 |
-
--gradient_accumulation_steps
|
| 661 |
--learning_rate 5e-05 \
|
| 662 |
--num_train_epochs 50 \
|
| 663 |
--bf16 \
|
|
@@ -692,8 +731,9 @@ rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-tas
|
|
| 692 |
sleep 5
|
| 693 |
|
| 694 |
|
| 695 |
-
deepspeed --include
|
| 696 |
--do_train \
|
|
|
|
| 697 |
--do_predict \
|
| 698 |
--predict_with_generate \
|
| 699 |
--model_name_or_path $2 \
|
|
@@ -706,7 +746,7 @@ deepspeed --include localhost:${1} --master_port 49500 src/run_llama.py \
|
|
| 706 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification \
|
| 707 |
--per_device_train_batch_size 4 \
|
| 708 |
--per_device_eval_batch_size 8 \
|
| 709 |
-
--gradient_accumulation_steps
|
| 710 |
--learning_rate 5e-05 \
|
| 711 |
--num_train_epochs 50 \
|
| 712 |
--bf16 \
|
|
|
|
| 8 |
#SBATCH --gres=gpu:a100-sxm4-80gb:1
|
| 9 |
|
| 10 |
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
|
| 11 |
+
# ββ GPU auto-detection βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 12 |
+
NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
|
| 13 |
+
GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
|
| 14 |
+
|
| 15 |
+
if [ -z "$GPU_MEM" ]; then
|
| 16 |
+
echo "[ERROR] No GPU detected. Exiting."
|
| 17 |
+
exit 1
|
| 18 |
+
fi
|
| 19 |
+
|
| 20 |
+
if [ "$NUM_GPUS" -ge 2 ] && [ "$GPU_MEM" -ge 20000 ]; then
|
| 21 |
+
# 2Γ 3090/4090 (24 GB+): run DDP on both GPUs
|
| 22 |
+
DS_INCLUDE="localhost:0,1"
|
| 23 |
+
echo "[GPU] 2Γ GPU detected (${GPU_MEM} MB each) β DDP on localhost:0,1"
|
| 24 |
+
elif [ "$GPU_MEM" -lt 20000 ]; then
|
| 25 |
+
# P100/V100 16 GB: use the dedicated P100 script instead
|
| 26 |
+
echo "[GPU] WARNING: 16 GB GPU detected. Use the _p100.sh script instead."
|
| 27 |
+
echo " This script requires β₯ 20 GB VRAM per GPU."
|
| 28 |
+
exit 1
|
| 29 |
+
else
|
| 30 |
+
# Single high-memory GPU (A100/H100 or single 3090)
|
| 31 |
+
DS_INCLUDE="localhost:${1:-0}"
|
| 32 |
+
echo "[GPU] Single GPU detected (${GPU_MEM} MB) β localhost:${1:-0}"
|
| 33 |
+
fi
|
| 34 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 38 |
--do_train \
|
| 39 |
+
--gradient_checkpointing \
|
| 40 |
--do_predict \
|
| 41 |
--predict_with_generate \
|
| 42 |
--model_name_or_path $2 \
|
|
|
|
| 46 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection \
|
| 47 |
--per_device_train_batch_size 4 \
|
| 48 |
--per_device_eval_batch_size 8 \
|
| 49 |
+
--gradient_accumulation_steps 4 \
|
| 50 |
--learning_rate 5e-05 \
|
| 51 |
--num_train_epochs 50 \
|
| 52 |
--bf16 \
|
|
|
|
| 81 |
sleep 5
|
| 82 |
|
| 83 |
|
| 84 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 85 |
--do_train \
|
| 86 |
+
--gradient_checkpointing \
|
| 87 |
--do_predict \
|
| 88 |
--predict_with_generate \
|
| 89 |
--model_name_or_path $2 \
|
|
|
|
| 96 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation \
|
| 97 |
--per_device_train_batch_size 8 \
|
| 98 |
--per_device_eval_batch_size 8 \
|
| 99 |
+
--gradient_accumulation_steps 2 \
|
| 100 |
--learning_rate 5e-05 \
|
| 101 |
--num_train_epochs 50 \
|
| 102 |
--bf16 \
|
|
|
|
| 131 |
sleep 5
|
| 132 |
|
| 133 |
|
| 134 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 135 |
--do_train \
|
| 136 |
+
--gradient_checkpointing \
|
| 137 |
--do_predict \
|
| 138 |
--predict_with_generate \
|
| 139 |
--model_name_or_path $2 \
|
|
|
|
| 146 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation \
|
| 147 |
--per_device_train_batch_size 2 \
|
| 148 |
--per_device_eval_batch_size 8 \
|
| 149 |
+
--gradient_accumulation_steps 8 \
|
| 150 |
--learning_rate 5e-05 \
|
| 151 |
--num_train_epochs 50 \
|
| 152 |
--bf16 \
|
|
|
|
| 181 |
sleep 5
|
| 182 |
|
| 183 |
|
| 184 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 185 |
--do_train \
|
| 186 |
+
--gradient_checkpointing \
|
| 187 |
--do_predict \
|
| 188 |
--predict_with_generate \
|
| 189 |
--model_name_or_path $2 \
|
|
|
|
| 196 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation \
|
| 197 |
--per_device_train_batch_size 16 \
|
| 198 |
--per_device_eval_batch_size 8 \
|
| 199 |
+
--gradient_accumulation_steps 1 \
|
| 200 |
--learning_rate 5e-05 \
|
| 201 |
--num_train_epochs 50 \
|
| 202 |
--bf16 \
|
|
|
|
| 231 |
sleep 5
|
| 232 |
|
| 233 |
|
| 234 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 235 |
--do_train \
|
| 236 |
+
--gradient_checkpointing \
|
| 237 |
--do_predict \
|
| 238 |
--predict_with_generate \
|
| 239 |
--model_name_or_path $2 \
|
|
|
|
| 246 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary \
|
| 247 |
--per_device_train_batch_size 2 \
|
| 248 |
--per_device_eval_batch_size 8 \
|
| 249 |
+
--gradient_accumulation_steps 8 \
|
| 250 |
--learning_rate 5e-05 \
|
| 251 |
--num_train_epochs 50 \
|
| 252 |
--bf16 \
|
|
|
|
| 281 |
sleep 5
|
| 282 |
|
| 283 |
|
| 284 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 285 |
--do_train \
|
| 286 |
+
--gradient_checkpointing \
|
| 287 |
--do_predict \
|
| 288 |
--predict_with_generate \
|
| 289 |
--model_name_or_path $2 \
|
|
|
|
| 296 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification \
|
| 297 |
--per_device_train_batch_size 8 \
|
| 298 |
--per_device_eval_batch_size 8 \
|
| 299 |
+
--gradient_accumulation_steps 2 \
|
| 300 |
--learning_rate 5e-05 \
|
| 301 |
--num_train_epochs 50 \
|
| 302 |
--bf16 \
|
|
|
|
| 331 |
sleep 5
|
| 332 |
|
| 333 |
|
| 334 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 335 |
--do_train \
|
| 336 |
+
--gradient_checkpointing \
|
| 337 |
--do_predict \
|
| 338 |
--predict_with_generate \
|
| 339 |
--model_name_or_path $2 \
|
|
|
|
| 346 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation \
|
| 347 |
--per_device_train_batch_size 8 \
|
| 348 |
--per_device_eval_batch_size 8 \
|
| 349 |
+
--gradient_accumulation_steps 2 \
|
| 350 |
--learning_rate 5e-05 \
|
| 351 |
--num_train_epochs 50 \
|
| 352 |
--bf16 \
|
|
|
|
| 381 |
sleep 5
|
| 382 |
|
| 383 |
|
| 384 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 385 |
--do_train \
|
| 386 |
+
--gradient_checkpointing \
|
| 387 |
--do_predict \
|
| 388 |
--predict_with_generate \
|
| 389 |
--model_name_or_path $2 \
|
|
|
|
| 396 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification \
|
| 397 |
--per_device_train_batch_size 16 \
|
| 398 |
--per_device_eval_batch_size 8 \
|
| 399 |
+
--gradient_accumulation_steps 1 \
|
| 400 |
--learning_rate 5e-05 \
|
| 401 |
--num_train_epochs 50 \
|
| 402 |
--bf16 \
|
|
|
|
| 431 |
sleep 5
|
| 432 |
|
| 433 |
|
| 434 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 435 |
--do_train \
|
| 436 |
+
--gradient_checkpointing \
|
| 437 |
--do_predict \
|
| 438 |
--predict_with_generate \
|
| 439 |
--model_name_or_path $2 \
|
|
|
|
| 446 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction \
|
| 447 |
--per_device_train_batch_size 16 \
|
| 448 |
--per_device_eval_batch_size 8 \
|
| 449 |
+
--gradient_accumulation_steps 1 \
|
| 450 |
--learning_rate 5e-05 \
|
| 451 |
--num_train_epochs 50 \
|
| 452 |
--bf16 \
|
|
|
|
| 481 |
sleep 5
|
| 482 |
|
| 483 |
|
| 484 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 485 |
--do_train \
|
| 486 |
+
--gradient_checkpointing \
|
| 487 |
--do_predict \
|
| 488 |
--predict_with_generate \
|
| 489 |
--model_name_or_path $2 \
|
|
|
|
| 496 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next \
|
| 497 |
--per_device_train_batch_size 2 \
|
| 498 |
--per_device_eval_batch_size 8 \
|
| 499 |
+
--gradient_accumulation_steps 8 \
|
| 500 |
--learning_rate 5e-05 \
|
| 501 |
--num_train_epochs 50 \
|
| 502 |
--bf16 \
|
|
|
|
| 531 |
sleep 5
|
| 532 |
|
| 533 |
|
| 534 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 535 |
--do_train \
|
| 536 |
+
--gradient_checkpointing \
|
| 537 |
--do_predict \
|
| 538 |
--predict_with_generate \
|
| 539 |
--model_name_or_path $2 \
|
|
|
|
| 546 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction \
|
| 547 |
--per_device_train_batch_size 4 \
|
| 548 |
--per_device_eval_batch_size 8 \
|
| 549 |
+
--gradient_accumulation_steps 4 \
|
| 550 |
--learning_rate 5e-05 \
|
| 551 |
--num_train_epochs 50 \
|
| 552 |
--bf16 \
|
|
|
|
| 581 |
sleep 5
|
| 582 |
|
| 583 |
|
| 584 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 585 |
--do_train \
|
| 586 |
+
--gradient_checkpointing \
|
| 587 |
--do_predict \
|
| 588 |
--predict_with_generate \
|
| 589 |
--model_name_or_path $2 \
|
|
|
|
| 596 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
|
| 597 |
--per_device_train_batch_size 1 \
|
| 598 |
--per_device_eval_batch_size 8 \
|
| 599 |
+
--gradient_accumulation_steps 16 \
|
| 600 |
--learning_rate 5e-05 \
|
| 601 |
--num_train_epochs 50 \
|
| 602 |
--bf16 \
|
|
|
|
| 631 |
sleep 5
|
| 632 |
|
| 633 |
|
| 634 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 635 |
--do_train \
|
| 636 |
+
--gradient_checkpointing \
|
| 637 |
--do_predict \
|
| 638 |
--predict_with_generate \
|
| 639 |
--model_name_or_path $2 \
|
|
|
|
| 646 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation \
|
| 647 |
--per_device_train_batch_size 1 \
|
| 648 |
--per_device_eval_batch_size 8 \
|
| 649 |
+
--gradient_accumulation_steps 16 \
|
| 650 |
--learning_rate 5e-05 \
|
| 651 |
--num_train_epochs 50 \
|
| 652 |
--bf16 \
|
|
|
|
| 681 |
sleep 5
|
| 682 |
|
| 683 |
|
| 684 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 685 |
--do_train \
|
| 686 |
+
--gradient_checkpointing \
|
| 687 |
--do_predict \
|
| 688 |
--predict_with_generate \
|
| 689 |
--model_name_or_path $2 \
|
|
|
|
| 696 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization \
|
| 697 |
--per_device_train_batch_size 1 \
|
| 698 |
--per_device_eval_batch_size 8 \
|
| 699 |
+
--gradient_accumulation_steps 16 \
|
| 700 |
--learning_rate 5e-05 \
|
| 701 |
--num_train_epochs 50 \
|
| 702 |
--bf16 \
|
|
|
|
| 731 |
sleep 5
|
| 732 |
|
| 733 |
|
| 734 |
+
deepspeed --include $DS_INCLUDE --master_port 49500 src/run_llama.py \
|
| 735 |
--do_train \
|
| 736 |
+
--gradient_checkpointing \
|
| 737 |
--do_predict \
|
| 738 |
--predict_with_generate \
|
| 739 |
--model_name_or_path $2 \
|
|
|
|
| 746 |
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification \
|
| 747 |
--per_device_train_batch_size 4 \
|
| 748 |
--per_device_eval_batch_size 8 \
|
| 749 |
+
--gradient_accumulation_steps 4 \
|
| 750 |
--learning_rate 5e-05 \
|
| 751 |
--num_train_epochs 50 \
|
| 752 |
--bf16 \
|
improve_gainlora/gen_script_superni_order2_llama_specroute_p100.sh
ADDED
|
@@ -0,0 +1,761 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# P100 (16 GB) variant β auto-generated by gen_p100_3090_scripts.py
|
| 3 |
+
# Changes vs H100 original:
|
| 4 |
+
# * bf16 β fp16 (P100 has no bf16 hardware support)
|
| 5 |
+
# * DeepSpeed stage2_fp16.config (fp16 forced + ZeRO-2 CPU-optimizer offload)
|
| 6 |
+
# * BSZ=1, GA=original_BSZ Γ original_GA (effective batch unchanged)
|
| 7 |
+
# * max_source_length 1024 β 512 (halves activation memory)
|
| 8 |
+
# * per_device_eval_batch_size 8 β 4
|
| 9 |
+
# * --gradient_checkpointing added
|
| 10 |
+
# * deepspeed --num_gpus 1 (single GPU, no include needed)
|
| 11 |
+
|
| 12 |
+
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
|
| 13 |
+
|
| 14 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 15 |
+
--do_train \
|
| 16 |
+
--gradient_checkpointing \
|
| 17 |
+
--do_predict \
|
| 18 |
+
--predict_with_generate \
|
| 19 |
+
--model_name_or_path $2 \
|
| 20 |
+
--data_dir CL_Benchmark \
|
| 21 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 22 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection \
|
| 23 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection \
|
| 24 |
+
--per_device_train_batch_size 1 \
|
| 25 |
+
--per_device_eval_batch_size 4 \
|
| 26 |
+
--gradient_accumulation_steps 32 \
|
| 27 |
+
--learning_rate 5e-05 \
|
| 28 |
+
--num_train_epochs 50 \
|
| 29 |
+
--fp16 \
|
| 30 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 31 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 32 |
+
--max_source_length 512 \
|
| 33 |
+
--max_target_length 50 \
|
| 34 |
+
--generation_max_length 50 \
|
| 35 |
+
--add_task_name False \
|
| 36 |
+
--add_dataset_name False \
|
| 37 |
+
--overwrite_output_dir \
|
| 38 |
+
--overwrite_cache \
|
| 39 |
+
--lr_scheduler_type constant \
|
| 40 |
+
--warmup_steps 0 \
|
| 41 |
+
--logging_strategy steps \
|
| 42 |
+
--logging_steps 10 \
|
| 43 |
+
--metric_for_best_model eval_rougeL \
|
| 44 |
+
--evaluation_strategy steps \
|
| 45 |
+
--save_strategy steps \
|
| 46 |
+
--save_total_limit 1 \
|
| 47 |
+
--load_best_model_at_end \
|
| 48 |
+
--lora_r 4 \
|
| 49 |
+
--lora_alpha 32 \
|
| 50 |
+
--lora_dropout 0.0 \
|
| 51 |
+
--data_replay_freq -1 \
|
| 52 |
+
--chunk 4 \
|
| 53 |
+
--model_name specroute \
|
| 54 |
+
--threshold 0.995
|
| 55 |
+
|
| 56 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/checkpoint*
|
| 57 |
+
|
| 58 |
+
sleep 5
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 62 |
+
--do_train \
|
| 63 |
+
--gradient_checkpointing \
|
| 64 |
+
--do_predict \
|
| 65 |
+
--predict_with_generate \
|
| 66 |
+
--model_name_or_path $2 \
|
| 67 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights \
|
| 68 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/spectral_signatures.pt \
|
| 69 |
+
--data_dir CL_Benchmark \
|
| 70 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 71 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 72 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation \
|
| 73 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation \
|
| 74 |
+
--per_device_train_batch_size 1 \
|
| 75 |
+
--per_device_eval_batch_size 4 \
|
| 76 |
+
--gradient_accumulation_steps 32 \
|
| 77 |
+
--learning_rate 5e-05 \
|
| 78 |
+
--num_train_epochs 50 \
|
| 79 |
+
--fp16 \
|
| 80 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 81 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 82 |
+
--max_source_length 512 \
|
| 83 |
+
--max_target_length 50 \
|
| 84 |
+
--generation_max_length 50 \
|
| 85 |
+
--add_task_name False \
|
| 86 |
+
--add_dataset_name False \
|
| 87 |
+
--overwrite_output_dir \
|
| 88 |
+
--overwrite_cache \
|
| 89 |
+
--lr_scheduler_type constant \
|
| 90 |
+
--warmup_steps 0 \
|
| 91 |
+
--logging_strategy steps \
|
| 92 |
+
--logging_steps 10 \
|
| 93 |
+
--metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
|
| 94 |
+
--evaluation_strategy steps \
|
| 95 |
+
--save_strategy steps \
|
| 96 |
+
--save_total_limit 1 \
|
| 97 |
+
--load_best_model_at_end \
|
| 98 |
+
--lora_r 4 \
|
| 99 |
+
--lora_alpha 32 \
|
| 100 |
+
--lora_dropout 0.0 \
|
| 101 |
+
--data_replay_freq -1 \
|
| 102 |
+
--chunk 4 \
|
| 103 |
+
--model_name specroute \
|
| 104 |
+
--threshold 0.995
|
| 105 |
+
|
| 106 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/checkpoint*
|
| 107 |
+
|
| 108 |
+
sleep 5
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 112 |
+
--do_train \
|
| 113 |
+
--gradient_checkpointing \
|
| 114 |
+
--do_predict \
|
| 115 |
+
--predict_with_generate \
|
| 116 |
+
--model_name_or_path $2 \
|
| 117 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights \
|
| 118 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/spectral_signatures.pt \
|
| 119 |
+
--data_dir CL_Benchmark \
|
| 120 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 121 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 122 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation \
|
| 123 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation \
|
| 124 |
+
--per_device_train_batch_size 1 \
|
| 125 |
+
--per_device_eval_batch_size 4 \
|
| 126 |
+
--gradient_accumulation_steps 32 \
|
| 127 |
+
--learning_rate 5e-05 \
|
| 128 |
+
--num_train_epochs 50 \
|
| 129 |
+
--fp16 \
|
| 130 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 131 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 132 |
+
--max_source_length 512 \
|
| 133 |
+
--max_target_length 50 \
|
| 134 |
+
--generation_max_length 50 \
|
| 135 |
+
--add_task_name False \
|
| 136 |
+
--add_dataset_name False \
|
| 137 |
+
--overwrite_output_dir \
|
| 138 |
+
--overwrite_cache \
|
| 139 |
+
--lr_scheduler_type constant \
|
| 140 |
+
--warmup_steps 0 \
|
| 141 |
+
--logging_strategy steps \
|
| 142 |
+
--logging_steps 10 \
|
| 143 |
+
--metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
|
| 144 |
+
--evaluation_strategy steps \
|
| 145 |
+
--save_strategy steps \
|
| 146 |
+
--save_total_limit 1 \
|
| 147 |
+
--load_best_model_at_end \
|
| 148 |
+
--lora_r 4 \
|
| 149 |
+
--lora_alpha 32 \
|
| 150 |
+
--lora_dropout 0.0 \
|
| 151 |
+
--data_replay_freq -1 \
|
| 152 |
+
--chunk 4 \
|
| 153 |
+
--model_name specroute \
|
| 154 |
+
--threshold 0.995
|
| 155 |
+
|
| 156 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/checkpoint*
|
| 157 |
+
|
| 158 |
+
sleep 5
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 162 |
+
--do_train \
|
| 163 |
+
--gradient_checkpointing \
|
| 164 |
+
--do_predict \
|
| 165 |
+
--predict_with_generate \
|
| 166 |
+
--model_name_or_path $2 \
|
| 167 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights \
|
| 168 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights/spectral_signatures.pt \
|
| 169 |
+
--data_dir CL_Benchmark \
|
| 170 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 171 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 172 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation \
|
| 173 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation \
|
| 174 |
+
--per_device_train_batch_size 1 \
|
| 175 |
+
--per_device_eval_batch_size 4 \
|
| 176 |
+
--gradient_accumulation_steps 32 \
|
| 177 |
+
--learning_rate 5e-05 \
|
| 178 |
+
--num_train_epochs 50 \
|
| 179 |
+
--fp16 \
|
| 180 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 181 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 182 |
+
--max_source_length 512 \
|
| 183 |
+
--max_target_length 50 \
|
| 184 |
+
--generation_max_length 50 \
|
| 185 |
+
--add_task_name False \
|
| 186 |
+
--add_dataset_name False \
|
| 187 |
+
--overwrite_output_dir \
|
| 188 |
+
--overwrite_cache \
|
| 189 |
+
--lr_scheduler_type constant \
|
| 190 |
+
--warmup_steps 0 \
|
| 191 |
+
--logging_strategy steps \
|
| 192 |
+
--logging_steps 10 \
|
| 193 |
+
--metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
|
| 194 |
+
--evaluation_strategy steps \
|
| 195 |
+
--save_strategy steps \
|
| 196 |
+
--save_total_limit 1 \
|
| 197 |
+
--load_best_model_at_end \
|
| 198 |
+
--lora_r 4 \
|
| 199 |
+
--lora_alpha 32 \
|
| 200 |
+
--lora_dropout 0.0 \
|
| 201 |
+
--data_replay_freq -1 \
|
| 202 |
+
--chunk 4 \
|
| 203 |
+
--model_name specroute \
|
| 204 |
+
--threshold 0.995
|
| 205 |
+
|
| 206 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/checkpoint*
|
| 207 |
+
|
| 208 |
+
sleep 5
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 212 |
+
--do_train \
|
| 213 |
+
--gradient_checkpointing \
|
| 214 |
+
--do_predict \
|
| 215 |
+
--predict_with_generate \
|
| 216 |
+
--model_name_or_path $2 \
|
| 217 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights \
|
| 218 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/spectral_signatures.pt \
|
| 219 |
+
--data_dir CL_Benchmark \
|
| 220 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 221 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 222 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary \
|
| 223 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary \
|
| 224 |
+
--per_device_train_batch_size 1 \
|
| 225 |
+
--per_device_eval_batch_size 4 \
|
| 226 |
+
--gradient_accumulation_steps 32 \
|
| 227 |
+
--learning_rate 5e-05 \
|
| 228 |
+
--num_train_epochs 50 \
|
| 229 |
+
--fp16 \
|
| 230 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 231 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 232 |
+
--max_source_length 512 \
|
| 233 |
+
--max_target_length 50 \
|
| 234 |
+
--generation_max_length 50 \
|
| 235 |
+
--add_task_name False \
|
| 236 |
+
--add_dataset_name False \
|
| 237 |
+
--overwrite_output_dir \
|
| 238 |
+
--overwrite_cache \
|
| 239 |
+
--lr_scheduler_type constant \
|
| 240 |
+
--warmup_steps 0 \
|
| 241 |
+
--logging_strategy steps \
|
| 242 |
+
--logging_steps 10 \
|
| 243 |
+
--metric_for_best_model eval_rougeL_for_task1572_samsum_summary \
|
| 244 |
+
--evaluation_strategy steps \
|
| 245 |
+
--save_strategy steps \
|
| 246 |
+
--save_total_limit 1 \
|
| 247 |
+
--load_best_model_at_end \
|
| 248 |
+
--lora_r 4 \
|
| 249 |
+
--lora_alpha 32 \
|
| 250 |
+
--lora_dropout 0.0 \
|
| 251 |
+
--data_replay_freq -1 \
|
| 252 |
+
--chunk 4 \
|
| 253 |
+
--model_name specroute \
|
| 254 |
+
--threshold 0.995
|
| 255 |
+
|
| 256 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/checkpoint*
|
| 257 |
+
|
| 258 |
+
sleep 5
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 262 |
+
--do_train \
|
| 263 |
+
--gradient_checkpointing \
|
| 264 |
+
--do_predict \
|
| 265 |
+
--predict_with_generate \
|
| 266 |
+
--model_name_or_path $2 \
|
| 267 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights \
|
| 268 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights/spectral_signatures.pt \
|
| 269 |
+
--data_dir CL_Benchmark \
|
| 270 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 271 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 272 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification \
|
| 273 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification \
|
| 274 |
+
--per_device_train_batch_size 1 \
|
| 275 |
+
--per_device_eval_batch_size 4 \
|
| 276 |
+
--gradient_accumulation_steps 32 \
|
| 277 |
+
--learning_rate 5e-05 \
|
| 278 |
+
--num_train_epochs 50 \
|
| 279 |
+
--fp16 \
|
| 280 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 281 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 282 |
+
--max_source_length 512 \
|
| 283 |
+
--max_target_length 50 \
|
| 284 |
+
--generation_max_length 50 \
|
| 285 |
+
--add_task_name False \
|
| 286 |
+
--add_dataset_name False \
|
| 287 |
+
--overwrite_output_dir \
|
| 288 |
+
--overwrite_cache \
|
| 289 |
+
--lr_scheduler_type constant \
|
| 290 |
+
--warmup_steps 0 \
|
| 291 |
+
--logging_strategy steps \
|
| 292 |
+
--logging_steps 10 \
|
| 293 |
+
--metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
|
| 294 |
+
--evaluation_strategy steps \
|
| 295 |
+
--save_strategy steps \
|
| 296 |
+
--save_total_limit 1 \
|
| 297 |
+
--load_best_model_at_end \
|
| 298 |
+
--lora_r 4 \
|
| 299 |
+
--lora_alpha 32 \
|
| 300 |
+
--lora_dropout 0.0 \
|
| 301 |
+
--data_replay_freq -1 \
|
| 302 |
+
--chunk 4 \
|
| 303 |
+
--model_name specroute \
|
| 304 |
+
--threshold 0.995
|
| 305 |
+
|
| 306 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/checkpoint*
|
| 307 |
+
|
| 308 |
+
sleep 5
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 312 |
+
--do_train \
|
| 313 |
+
--gradient_checkpointing \
|
| 314 |
+
--do_predict \
|
| 315 |
+
--predict_with_generate \
|
| 316 |
+
--model_name_or_path $2 \
|
| 317 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights \
|
| 318 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights/spectral_signatures.pt \
|
| 319 |
+
--data_dir CL_Benchmark \
|
| 320 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 321 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 322 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation \
|
| 323 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation \
|
| 324 |
+
--per_device_train_batch_size 1 \
|
| 325 |
+
--per_device_eval_batch_size 4 \
|
| 326 |
+
--gradient_accumulation_steps 32 \
|
| 327 |
+
--learning_rate 5e-05 \
|
| 328 |
+
--num_train_epochs 50 \
|
| 329 |
+
--fp16 \
|
| 330 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 331 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 332 |
+
--max_source_length 512 \
|
| 333 |
+
--max_target_length 50 \
|
| 334 |
+
--generation_max_length 50 \
|
| 335 |
+
--add_task_name False \
|
| 336 |
+
--add_dataset_name False \
|
| 337 |
+
--overwrite_output_dir \
|
| 338 |
+
--overwrite_cache \
|
| 339 |
+
--lr_scheduler_type constant \
|
| 340 |
+
--warmup_steps 0 \
|
| 341 |
+
--logging_strategy steps \
|
| 342 |
+
--logging_steps 10 \
|
| 343 |
+
--metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
|
| 344 |
+
--evaluation_strategy steps \
|
| 345 |
+
--save_strategy steps \
|
| 346 |
+
--save_total_limit 1 \
|
| 347 |
+
--load_best_model_at_end \
|
| 348 |
+
--lora_r 4 \
|
| 349 |
+
--lora_alpha 32 \
|
| 350 |
+
--lora_dropout 0.0 \
|
| 351 |
+
--data_replay_freq -1 \
|
| 352 |
+
--chunk 4 \
|
| 353 |
+
--model_name specroute \
|
| 354 |
+
--threshold 0.995
|
| 355 |
+
|
| 356 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/checkpoint*
|
| 357 |
+
|
| 358 |
+
sleep 5
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 362 |
+
--do_train \
|
| 363 |
+
--gradient_checkpointing \
|
| 364 |
+
--do_predict \
|
| 365 |
+
--predict_with_generate \
|
| 366 |
+
--model_name_or_path $2 \
|
| 367 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights \
|
| 368 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights/spectral_signatures.pt \
|
| 369 |
+
--data_dir CL_Benchmark \
|
| 370 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 371 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 372 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification \
|
| 373 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification \
|
| 374 |
+
--per_device_train_batch_size 1 \
|
| 375 |
+
--per_device_eval_batch_size 4 \
|
| 376 |
+
--gradient_accumulation_steps 32 \
|
| 377 |
+
--learning_rate 5e-05 \
|
| 378 |
+
--num_train_epochs 50 \
|
| 379 |
+
--fp16 \
|
| 380 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 381 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 382 |
+
--max_source_length 512 \
|
| 383 |
+
--max_target_length 50 \
|
| 384 |
+
--generation_max_length 50 \
|
| 385 |
+
--add_task_name False \
|
| 386 |
+
--add_dataset_name False \
|
| 387 |
+
--overwrite_output_dir \
|
| 388 |
+
--overwrite_cache \
|
| 389 |
+
--lr_scheduler_type constant \
|
| 390 |
+
--warmup_steps 0 \
|
| 391 |
+
--logging_strategy steps \
|
| 392 |
+
--logging_steps 10 \
|
| 393 |
+
--metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
|
| 394 |
+
--evaluation_strategy steps \
|
| 395 |
+
--save_strategy steps \
|
| 396 |
+
--save_total_limit 1 \
|
| 397 |
+
--load_best_model_at_end \
|
| 398 |
+
--lora_r 4 \
|
| 399 |
+
--lora_alpha 32 \
|
| 400 |
+
--lora_dropout 0.0 \
|
| 401 |
+
--data_replay_freq -1 \
|
| 402 |
+
--chunk 4 \
|
| 403 |
+
--model_name specroute \
|
| 404 |
+
--threshold 0.995
|
| 405 |
+
|
| 406 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/checkpoint*
|
| 407 |
+
|
| 408 |
+
sleep 5
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 412 |
+
--do_train \
|
| 413 |
+
--gradient_checkpointing \
|
| 414 |
+
--do_predict \
|
| 415 |
+
--predict_with_generate \
|
| 416 |
+
--model_name_or_path $2 \
|
| 417 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights \
|
| 418 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights/spectral_signatures.pt \
|
| 419 |
+
--data_dir CL_Benchmark \
|
| 420 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 421 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 422 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction \
|
| 423 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction \
|
| 424 |
+
--per_device_train_batch_size 1 \
|
| 425 |
+
--per_device_eval_batch_size 4 \
|
| 426 |
+
--gradient_accumulation_steps 32 \
|
| 427 |
+
--learning_rate 5e-05 \
|
| 428 |
+
--num_train_epochs 50 \
|
| 429 |
+
--fp16 \
|
| 430 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 431 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 432 |
+
--max_source_length 512 \
|
| 433 |
+
--max_target_length 50 \
|
| 434 |
+
--generation_max_length 50 \
|
| 435 |
+
--add_task_name False \
|
| 436 |
+
--add_dataset_name False \
|
| 437 |
+
--overwrite_output_dir \
|
| 438 |
+
--overwrite_cache \
|
| 439 |
+
--lr_scheduler_type constant \
|
| 440 |
+
--warmup_steps 0 \
|
| 441 |
+
--logging_strategy steps \
|
| 442 |
+
--logging_steps 10 \
|
| 443 |
+
--metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
|
| 444 |
+
--evaluation_strategy steps \
|
| 445 |
+
--save_strategy steps \
|
| 446 |
+
--save_total_limit 1 \
|
| 447 |
+
--load_best_model_at_end \
|
| 448 |
+
--lora_r 4 \
|
| 449 |
+
--lora_alpha 32 \
|
| 450 |
+
--lora_dropout 0.0 \
|
| 451 |
+
--data_replay_freq -1 \
|
| 452 |
+
--chunk 4 \
|
| 453 |
+
--model_name specroute \
|
| 454 |
+
--threshold 0.995
|
| 455 |
+
|
| 456 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/checkpoint*
|
| 457 |
+
|
| 458 |
+
sleep 5
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 462 |
+
--do_train \
|
| 463 |
+
--gradient_checkpointing \
|
| 464 |
+
--do_predict \
|
| 465 |
+
--predict_with_generate \
|
| 466 |
+
--model_name_or_path $2 \
|
| 467 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights \
|
| 468 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights/spectral_signatures.pt \
|
| 469 |
+
--data_dir CL_Benchmark \
|
| 470 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 471 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 472 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next \
|
| 473 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next \
|
| 474 |
+
--per_device_train_batch_size 1 \
|
| 475 |
+
--per_device_eval_batch_size 4 \
|
| 476 |
+
--gradient_accumulation_steps 32 \
|
| 477 |
+
--learning_rate 5e-05 \
|
| 478 |
+
--num_train_epochs 50 \
|
| 479 |
+
--fp16 \
|
| 480 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 481 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 482 |
+
--max_source_length 512 \
|
| 483 |
+
--max_target_length 50 \
|
| 484 |
+
--generation_max_length 50 \
|
| 485 |
+
--add_task_name False \
|
| 486 |
+
--add_dataset_name False \
|
| 487 |
+
--overwrite_output_dir \
|
| 488 |
+
--overwrite_cache \
|
| 489 |
+
--lr_scheduler_type constant \
|
| 490 |
+
--warmup_steps 0 \
|
| 491 |
+
--logging_strategy steps \
|
| 492 |
+
--logging_steps 10 \
|
| 493 |
+
--metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
|
| 494 |
+
--evaluation_strategy steps \
|
| 495 |
+
--save_strategy steps \
|
| 496 |
+
--save_total_limit 1 \
|
| 497 |
+
--load_best_model_at_end \
|
| 498 |
+
--lora_r 4 \
|
| 499 |
+
--lora_alpha 32 \
|
| 500 |
+
--lora_dropout 0.0 \
|
| 501 |
+
--data_replay_freq -1 \
|
| 502 |
+
--chunk 4 \
|
| 503 |
+
--model_name specroute \
|
| 504 |
+
--threshold 0.995
|
| 505 |
+
|
| 506 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/checkpoint*
|
| 507 |
+
|
| 508 |
+
sleep 5
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 512 |
+
--do_train \
|
| 513 |
+
--gradient_checkpointing \
|
| 514 |
+
--do_predict \
|
| 515 |
+
--predict_with_generate \
|
| 516 |
+
--model_name_or_path $2 \
|
| 517 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights \
|
| 518 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights/spectral_signatures.pt \
|
| 519 |
+
--data_dir CL_Benchmark \
|
| 520 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 521 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 522 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction \
|
| 523 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction \
|
| 524 |
+
--per_device_train_batch_size 1 \
|
| 525 |
+
--per_device_eval_batch_size 4 \
|
| 526 |
+
--gradient_accumulation_steps 32 \
|
| 527 |
+
--learning_rate 5e-05 \
|
| 528 |
+
--num_train_epochs 50 \
|
| 529 |
+
--fp16 \
|
| 530 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 531 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 532 |
+
--max_source_length 512 \
|
| 533 |
+
--max_target_length 50 \
|
| 534 |
+
--generation_max_length 50 \
|
| 535 |
+
--add_task_name False \
|
| 536 |
+
--add_dataset_name False \
|
| 537 |
+
--overwrite_output_dir \
|
| 538 |
+
--overwrite_cache \
|
| 539 |
+
--lr_scheduler_type constant \
|
| 540 |
+
--warmup_steps 0 \
|
| 541 |
+
--logging_strategy steps \
|
| 542 |
+
--logging_steps 10 \
|
| 543 |
+
--metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
|
| 544 |
+
--evaluation_strategy steps \
|
| 545 |
+
--save_strategy steps \
|
| 546 |
+
--save_total_limit 1 \
|
| 547 |
+
--load_best_model_at_end \
|
| 548 |
+
--lora_r 4 \
|
| 549 |
+
--lora_alpha 32 \
|
| 550 |
+
--lora_dropout 0.0 \
|
| 551 |
+
--data_replay_freq -1 \
|
| 552 |
+
--chunk 4 \
|
| 553 |
+
--model_name specroute \
|
| 554 |
+
--threshold 0.995
|
| 555 |
+
|
| 556 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/checkpoint*
|
| 557 |
+
|
| 558 |
+
sleep 5
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 562 |
+
--do_train \
|
| 563 |
+
--gradient_checkpointing \
|
| 564 |
+
--do_predict \
|
| 565 |
+
--predict_with_generate \
|
| 566 |
+
--model_name_or_path $2 \
|
| 567 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights \
|
| 568 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights/spectral_signatures.pt \
|
| 569 |
+
--data_dir CL_Benchmark \
|
| 570 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 571 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 572 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization \
|
| 573 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
|
| 574 |
+
--per_device_train_batch_size 1 \
|
| 575 |
+
--per_device_eval_batch_size 4 \
|
| 576 |
+
--gradient_accumulation_steps 32 \
|
| 577 |
+
--learning_rate 5e-05 \
|
| 578 |
+
--num_train_epochs 50 \
|
| 579 |
+
--fp16 \
|
| 580 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 581 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 582 |
+
--max_source_length 512 \
|
| 583 |
+
--max_target_length 50 \
|
| 584 |
+
--generation_max_length 50 \
|
| 585 |
+
--add_task_name False \
|
| 586 |
+
--add_dataset_name False \
|
| 587 |
+
--overwrite_output_dir \
|
| 588 |
+
--overwrite_cache \
|
| 589 |
+
--lr_scheduler_type constant \
|
| 590 |
+
--warmup_steps 0 \
|
| 591 |
+
--logging_strategy steps \
|
| 592 |
+
--logging_steps 10 \
|
| 593 |
+
--metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
|
| 594 |
+
--evaluation_strategy steps \
|
| 595 |
+
--save_strategy steps \
|
| 596 |
+
--save_total_limit 1 \
|
| 597 |
+
--load_best_model_at_end \
|
| 598 |
+
--lora_r 4 \
|
| 599 |
+
--lora_alpha 32 \
|
| 600 |
+
--lora_dropout 0.0 \
|
| 601 |
+
--data_replay_freq -1 \
|
| 602 |
+
--chunk 4 \
|
| 603 |
+
--model_name specroute \
|
| 604 |
+
--threshold 0.995
|
| 605 |
+
|
| 606 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/checkpoint*
|
| 607 |
+
|
| 608 |
+
sleep 5
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 612 |
+
--do_train \
|
| 613 |
+
--gradient_checkpointing \
|
| 614 |
+
--do_predict \
|
| 615 |
+
--predict_with_generate \
|
| 616 |
+
--model_name_or_path $2 \
|
| 617 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
|
| 618 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/spectral_signatures.pt \
|
| 619 |
+
--data_dir CL_Benchmark \
|
| 620 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 621 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 622 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation \
|
| 623 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation \
|
| 624 |
+
--per_device_train_batch_size 1 \
|
| 625 |
+
--per_device_eval_batch_size 4 \
|
| 626 |
+
--gradient_accumulation_steps 32 \
|
| 627 |
+
--learning_rate 5e-05 \
|
| 628 |
+
--num_train_epochs 50 \
|
| 629 |
+
--fp16 \
|
| 630 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 631 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 632 |
+
--max_source_length 512 \
|
| 633 |
+
--max_target_length 50 \
|
| 634 |
+
--generation_max_length 50 \
|
| 635 |
+
--add_task_name False \
|
| 636 |
+
--add_dataset_name False \
|
| 637 |
+
--overwrite_output_dir \
|
| 638 |
+
--overwrite_cache \
|
| 639 |
+
--lr_scheduler_type constant \
|
| 640 |
+
--warmup_steps 0 \
|
| 641 |
+
--logging_strategy steps \
|
| 642 |
+
--logging_steps 10 \
|
| 643 |
+
--metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
|
| 644 |
+
--evaluation_strategy steps \
|
| 645 |
+
--save_strategy steps \
|
| 646 |
+
--save_total_limit 1 \
|
| 647 |
+
--load_best_model_at_end \
|
| 648 |
+
--lora_r 4 \
|
| 649 |
+
--lora_alpha 32 \
|
| 650 |
+
--lora_dropout 0.0 \
|
| 651 |
+
--data_replay_freq -1 \
|
| 652 |
+
--chunk 4 \
|
| 653 |
+
--model_name specroute \
|
| 654 |
+
--threshold 0.995
|
| 655 |
+
|
| 656 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/checkpoint*
|
| 657 |
+
|
| 658 |
+
sleep 5
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 662 |
+
--do_train \
|
| 663 |
+
--gradient_checkpointing \
|
| 664 |
+
--do_predict \
|
| 665 |
+
--predict_with_generate \
|
| 666 |
+
--model_name_or_path $2 \
|
| 667 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/saved_weights \
|
| 668 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/saved_weights/spectral_signatures.pt \
|
| 669 |
+
--data_dir CL_Benchmark \
|
| 670 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 671 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 672 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization \
|
| 673 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization \
|
| 674 |
+
--per_device_train_batch_size 1 \
|
| 675 |
+
--per_device_eval_batch_size 4 \
|
| 676 |
+
--gradient_accumulation_steps 32 \
|
| 677 |
+
--learning_rate 5e-05 \
|
| 678 |
+
--num_train_epochs 50 \
|
| 679 |
+
--fp16 \
|
| 680 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 681 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 682 |
+
--max_source_length 512 \
|
| 683 |
+
--max_target_length 50 \
|
| 684 |
+
--generation_max_length 50 \
|
| 685 |
+
--add_task_name False \
|
| 686 |
+
--add_dataset_name False \
|
| 687 |
+
--overwrite_output_dir \
|
| 688 |
+
--overwrite_cache \
|
| 689 |
+
--lr_scheduler_type constant \
|
| 690 |
+
--warmup_steps 0 \
|
| 691 |
+
--logging_strategy steps \
|
| 692 |
+
--logging_steps 10 \
|
| 693 |
+
--metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
|
| 694 |
+
--evaluation_strategy steps \
|
| 695 |
+
--save_strategy steps \
|
| 696 |
+
--save_total_limit 1 \
|
| 697 |
+
--load_best_model_at_end \
|
| 698 |
+
--lora_r 4 \
|
| 699 |
+
--lora_alpha 32 \
|
| 700 |
+
--lora_dropout 0.0 \
|
| 701 |
+
--data_replay_freq -1 \
|
| 702 |
+
--chunk 4 \
|
| 703 |
+
--model_name specroute \
|
| 704 |
+
--threshold 0.995
|
| 705 |
+
|
| 706 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization/checkpoint*
|
| 707 |
+
|
| 708 |
+
sleep 5
|
| 709 |
+
|
| 710 |
+
|
| 711 |
+
deepspeed --num_gpus 1 --master_port 49500 src/run_llama.py \
|
| 712 |
+
--do_train \
|
| 713 |
+
--gradient_checkpointing \
|
| 714 |
+
--do_predict \
|
| 715 |
+
--predict_with_generate \
|
| 716 |
+
--model_name_or_path $2 \
|
| 717 |
+
--previous_lora_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/13-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization/saved_weights \
|
| 718 |
+
--previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/14-task1290_xsum_summarization/saved_weights/spectral_signatures.pt \
|
| 719 |
+
--data_dir CL_Benchmark \
|
| 720 |
+
--task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
|
| 721 |
+
--gen_data_dir generated_data/lora_gen_superni_llama \
|
| 722 |
+
--task_config_dir configs/gen_script_superni_order2_llama_configs/task875_emotion_classification \
|
| 723 |
+
--output_dir logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification \
|
| 724 |
+
--per_device_train_batch_size 1 \
|
| 725 |
+
--per_device_eval_batch_size 4 \
|
| 726 |
+
--gradient_accumulation_steps 32 \
|
| 727 |
+
--learning_rate 5e-05 \
|
| 728 |
+
--num_train_epochs 50 \
|
| 729 |
+
--fp16 \
|
| 730 |
+
--deepspeed configs/ds_configs/stage2_fp16.config \
|
| 731 |
+
--run_name gen_script_superni_order2_llama_specroute \
|
| 732 |
+
--max_source_length 512 \
|
| 733 |
+
--max_target_length 50 \
|
| 734 |
+
--generation_max_length 50 \
|
| 735 |
+
--add_task_name False \
|
| 736 |
+
--add_dataset_name False \
|
| 737 |
+
--overwrite_output_dir \
|
| 738 |
+
--overwrite_cache \
|
| 739 |
+
--lr_scheduler_type constant \
|
| 740 |
+
--warmup_steps 0 \
|
| 741 |
+
--logging_strategy steps \
|
| 742 |
+
--logging_steps 10 \
|
| 743 |
+
--metric_for_best_model eval_rougeL_for_task875_emotion_classification \
|
| 744 |
+
--evaluation_strategy steps \
|
| 745 |
+
--save_strategy steps \
|
| 746 |
+
--save_total_limit 1 \
|
| 747 |
+
--load_best_model_at_end \
|
| 748 |
+
--lora_r 4 \
|
| 749 |
+
--lora_alpha 32 \
|
| 750 |
+
--lora_dropout 0.0 \
|
| 751 |
+
--data_replay_freq -1 \
|
| 752 |
+
--chunk 4 \
|
| 753 |
+
--model_name specroute \
|
| 754 |
+
--threshold 0.995
|
| 755 |
+
|
| 756 |
+
rm -rf logs_and_outputs/gen_script_superni_order2_llama_specroute/outputs/15-task875_emotion_classification/checkpoint*
|
| 757 |
+
|
| 758 |
+
sleep 5
|
| 759 |
+
|
| 760 |
+
|
| 761 |
+
python score.py gen_script_superni_order2_llama_specroute gen_script_superni_order2_llama_specroute
|
improve_gainlora/gen_script_superni_order2_t5_specroute.sh
CHANGED
|
@@ -49,9 +49,14 @@ elif [ "$GPU_MEM" -gt 16000 ]; then
|
|
| 49 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 50 |
else
|
| 51 |
GPU_MODE="a100"
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
FP16_FLAG=""
|
| 54 |
-
echo "[GPU] Strategy: A100 (single GPU, fp32)"
|
| 55 |
fi
|
| 56 |
|
| 57 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
|
|
|
| 49 |
echo "[GPU] Strategy: P100 16GB (fp32 + gradient_checkpointing)"
|
| 50 |
else
|
| 51 |
GPU_MODE="a100"
|
| 52 |
+
if [ "$NUM_GPUS" -ge 2 ]; then
|
| 53 |
+
GPU_IDS="0,1"
|
| 54 |
+
echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"
|
| 55 |
+
else
|
| 56 |
+
GPU_IDS="${1:-0}"
|
| 57 |
+
echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"
|
| 58 |
+
fi
|
| 59 |
FP16_FLAG=""
|
|
|
|
| 60 |
fi
|
| 61 |
|
| 62 |
echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
|
improve_gainlora/patch_t5_scripts.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Patches all T5 SpecRoute shell scripts to:
|
| 4 |
+
1. T5_small scripts: Replace GPU detection block (add P100 + 3090 2-GPU + gradient_checkpointing)
|
| 5 |
+
2. T5_small Long scripts: Reduce BSZ for P100/T4 (EVAL_BSZ 256->32, add p100 branch)
|
| 6 |
+
3. T5_small SuperNI scripts: Add p100 BSZ branch
|
| 7 |
+
4. Root T5 scripts: Add 3090 2-GPU support to a100 else branch
|
| 8 |
+
"""
|
| 9 |
+
import os
|
| 10 |
+
|
| 11 |
+
BASE = "/Users/nnminh322/Desktop/personal/Continual/improve_gainlora"
|
| 12 |
+
|
| 13 |
+
# ββ Replacements for T5_small scripts ββββββββββββββββββββββββββββββββββββββββ
|
| 14 |
+
|
| 15 |
+
# GPU detection block β Long scripts (no NOTE comment)
|
| 16 |
+
OLD_DETECT_LONG = (
|
| 17 |
+
"# Determine GPU type\n"
|
| 18 |
+
'if [ "$GPU_MEM" -lt 20000 ]; then\n'
|
| 19 |
+
" IS_T4=1\n"
|
| 20 |
+
' echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"\n'
|
| 21 |
+
"else\n"
|
| 22 |
+
" IS_T4=0\n"
|
| 23 |
+
' echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"\n'
|
| 24 |
+
"fi\n"
|
| 25 |
+
"\n"
|
| 26 |
+
"# Determine parallelism strategy\n"
|
| 27 |
+
'if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then\n'
|
| 28 |
+
' GPU_MODE="t4_2gpu"\n'
|
| 29 |
+
' GPU_IDS="0,1"\n'
|
| 30 |
+
' FP16_FLAG=""\n'
|
| 31 |
+
' echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"\n'
|
| 32 |
+
'elif [ "$IS_T4" -eq 1 ]; then\n'
|
| 33 |
+
' GPU_MODE="t4_1gpu"\n'
|
| 34 |
+
' GPU_IDS="${1:-0}"\n'
|
| 35 |
+
' FP16_FLAG=""\n'
|
| 36 |
+
' echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"\n'
|
| 37 |
+
"else\n"
|
| 38 |
+
' GPU_MODE="a100"\n'
|
| 39 |
+
' GPU_IDS="${1:-0}"\n'
|
| 40 |
+
' FP16_FLAG=""\n'
|
| 41 |
+
' echo "[GPU] Strategy: A100 (single GPU, fp32)"\n'
|
| 42 |
+
"fi"
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
# SuperNI T5_small scripts have this NOTE comment block instead
|
| 46 |
+
OLD_DETECT_SUPERNI = (
|
| 47 |
+
"# NOTE: T5 models trained in bfloat16 produce NaN with fp16 (overflow).\n"
|
| 48 |
+
"# T4 GPUs do not support bf16. Use fp32 + gradient_checkpointing instead.\n"
|
| 49 |
+
'if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then\n'
|
| 50 |
+
' GPU_MODE="t4_2gpu"\n'
|
| 51 |
+
' GPU_IDS="0,1"\n'
|
| 52 |
+
' FP16_FLAG=""\n'
|
| 53 |
+
' echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"\n'
|
| 54 |
+
'elif [ "$IS_T4" -eq 1 ]; then\n'
|
| 55 |
+
' GPU_MODE="t4_1gpu"\n'
|
| 56 |
+
' GPU_IDS="${1:-0}"\n'
|
| 57 |
+
' FP16_FLAG=""\n'
|
| 58 |
+
' echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"\n'
|
| 59 |
+
"else\n"
|
| 60 |
+
' GPU_MODE="a100"\n'
|
| 61 |
+
' GPU_IDS="${1:-0}"\n'
|
| 62 |
+
' FP16_FLAG=""\n'
|
| 63 |
+
' echo "[GPU] Strategy: A100 (single GPU, fp32)"\n'
|
| 64 |
+
"fi"
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
# New unified GPU detection block for all T5_small scripts
|
| 68 |
+
NEW_DETECT = (
|
| 69 |
+
"# GPU type detection\n"
|
| 70 |
+
"# T4 <15500 MB | P100 15500-17000 MB | RTX3090 ~24576 | A100 40000 | H100 80000\n"
|
| 71 |
+
'if [ "$GPU_MEM" -lt 15500 ]; then\n'
|
| 72 |
+
' GPU_TYPE="t4"\n'
|
| 73 |
+
' echo "[GPU] Detected T4 (${GPU_MEM}MB)"\n'
|
| 74 |
+
'elif [ "$GPU_MEM" -le 17000 ]; then\n'
|
| 75 |
+
' GPU_TYPE="p100"\n'
|
| 76 |
+
' echo "[GPU] Detected P100 (${GPU_MEM}MB)"\n'
|
| 77 |
+
"else\n"
|
| 78 |
+
' GPU_TYPE="highvram"\n'
|
| 79 |
+
' echo "[GPU] Detected high-VRAM GPU (${GPU_MEM}MB)"\n'
|
| 80 |
+
"fi\n"
|
| 81 |
+
"\n"
|
| 82 |
+
"# Parallelism: T4/P100 use gradient_checkpointing (16 GB fp32); highvram uses DataParallel if 2+ GPUs\n"
|
| 83 |
+
'if [ "$GPU_TYPE" = "t4" ] && [ "$NUM_GPUS" -ge 2 ]; then\n'
|
| 84 |
+
' GPU_MODE="t4_2gpu"\n'
|
| 85 |
+
' GPU_IDS="0,1"\n'
|
| 86 |
+
' FP16_FLAG="--gradient_checkpointing"\n'
|
| 87 |
+
' echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"\n'
|
| 88 |
+
'elif [ "$GPU_TYPE" = "t4" ]; then\n'
|
| 89 |
+
' GPU_MODE="t4_1gpu"\n'
|
| 90 |
+
' GPU_IDS="${1:-0}"\n'
|
| 91 |
+
' FP16_FLAG="--gradient_checkpointing"\n'
|
| 92 |
+
' echo "[GPU] Strategy: 1x T4 (${GPU_MEM}MB) + fp32 + gradient_checkpointing"\n'
|
| 93 |
+
'elif [ "$GPU_TYPE" = "p100" ]; then\n'
|
| 94 |
+
' GPU_MODE="p100"\n'
|
| 95 |
+
' GPU_IDS="${1:-0}"\n'
|
| 96 |
+
' FP16_FLAG="--gradient_checkpointing"\n'
|
| 97 |
+
' echo "[GPU] Strategy: P100 16GB + fp32 + gradient_checkpointing"\n'
|
| 98 |
+
"else\n"
|
| 99 |
+
' GPU_MODE="a100"\n'
|
| 100 |
+
' if [ "$NUM_GPUS" -ge 2 ]; then\n'
|
| 101 |
+
' GPU_IDS="0,1"\n'
|
| 102 |
+
' echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"\n'
|
| 103 |
+
" else\n"
|
| 104 |
+
' GPU_IDS="${1:-0}"\n'
|
| 105 |
+
' echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"\n'
|
| 106 |
+
" fi\n"
|
| 107 |
+
' FP16_FLAG=""\n'
|
| 108 |
+
"fi"
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
# ββ Per-task BSZ blocks βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 112 |
+
|
| 113 |
+
# Long T5_small scripts: old BSZ block (with space before EVAL_BSZ=256)
|
| 114 |
+
OLD_BSZ_LONG_A = (
|
| 115 |
+
'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
|
| 116 |
+
" BSZ=16; GA=1; EVAL_BSZ=256\n"
|
| 117 |
+
'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
|
| 118 |
+
" BSZ=32; GA=1; EVAL_BSZ=256\n"
|
| 119 |
+
"else\n"
|
| 120 |
+
" BSZ=64; GA=1; EVAL_BSZ=512\n"
|
| 121 |
+
"fi"
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
NEW_BSZ_LONG = (
|
| 125 |
+
'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
|
| 126 |
+
" BSZ=8; GA=2; EVAL_BSZ=64\n"
|
| 127 |
+
'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
|
| 128 |
+
" BSZ=8; GA=2; EVAL_BSZ=32\n"
|
| 129 |
+
'elif [ "$GPU_MODE" = "p100" ]; then\n'
|
| 130 |
+
" BSZ=16; GA=2; EVAL_BSZ=32\n"
|
| 131 |
+
"else\n"
|
| 132 |
+
" BSZ=64; GA=1; EVAL_BSZ=128\n"
|
| 133 |
+
"fi"
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
# SuperNI T5_small: old BSZ block (2 space variants)
|
| 137 |
+
OLD_BSZ_SUPERNI_A = (
|
| 138 |
+
'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
|
| 139 |
+
" BSZ=8; GA=2; EVAL_BSZ=16\n"
|
| 140 |
+
'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
|
| 141 |
+
" BSZ=16; GA=2; EVAL_BSZ=16\n"
|
| 142 |
+
"else\n"
|
| 143 |
+
" BSZ=32; GA=1; EVAL_BSZ=32\n"
|
| 144 |
+
"fi"
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
OLD_BSZ_SUPERNI_B = (
|
| 148 |
+
'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
|
| 149 |
+
" BSZ=8; GA=2; EVAL_BSZ=16\n"
|
| 150 |
+
'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
|
| 151 |
+
" BSZ=16; GA=2; EVAL_BSZ=16\n"
|
| 152 |
+
"else\n"
|
| 153 |
+
" BSZ=32; GA=1; EVAL_BSZ=32\n"
|
| 154 |
+
"fi"
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
NEW_BSZ_SUPERNI = (
|
| 158 |
+
'if [ "$GPU_MODE" = "t4_2gpu" ]; then\n'
|
| 159 |
+
" BSZ=8; GA=2; EVAL_BSZ=16\n"
|
| 160 |
+
'elif [ "$GPU_MODE" = "t4_1gpu" ]; then\n'
|
| 161 |
+
" BSZ=8; GA=2; EVAL_BSZ=16\n"
|
| 162 |
+
'elif [ "$GPU_MODE" = "p100" ]; then\n'
|
| 163 |
+
" BSZ=16; GA=2; EVAL_BSZ=16\n"
|
| 164 |
+
"else\n"
|
| 165 |
+
" BSZ=32; GA=1; EVAL_BSZ=32\n"
|
| 166 |
+
"fi"
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
# ββ Root T5 scripts: add 3090 2-GPU to a100 branch βββββββββββββββββββββββββββ
|
| 170 |
+
|
| 171 |
+
OLD_A100_ROOT = (
|
| 172 |
+
"else\n"
|
| 173 |
+
' GPU_MODE="a100"\n'
|
| 174 |
+
' GPU_IDS="${1:-0}"\n'
|
| 175 |
+
' FP16_FLAG=""\n'
|
| 176 |
+
' echo "[GPU] Strategy: A100 (single GPU, fp32)"\n'
|
| 177 |
+
"fi"
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
NEW_A100_ROOT = (
|
| 181 |
+
"else\n"
|
| 182 |
+
' GPU_MODE="a100"\n'
|
| 183 |
+
' if [ "$NUM_GPUS" -ge 2 ]; then\n'
|
| 184 |
+
' GPU_IDS="0,1"\n'
|
| 185 |
+
' echo "[GPU] Strategy: ${NUM_GPUS}x ${GPU_MEM}MB DataParallel (RTX3090/A100, fp32)"\n'
|
| 186 |
+
" else\n"
|
| 187 |
+
' GPU_IDS="${1:-0}"\n'
|
| 188 |
+
' echo "[GPU] Strategy: 1x ${GPU_MEM}MB GPU (fp32)"\n'
|
| 189 |
+
" fi\n"
|
| 190 |
+
' FP16_FLAG=""\n'
|
| 191 |
+
"fi"
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def patch(path, replacements):
|
| 196 |
+
full = os.path.join(BASE, path)
|
| 197 |
+
if not os.path.exists(full):
|
| 198 |
+
print(f" SKIP (not found): {path}")
|
| 199 |
+
return
|
| 200 |
+
with open(full) as f:
|
| 201 |
+
content = f.read()
|
| 202 |
+
original = content
|
| 203 |
+
for old, new in replacements:
|
| 204 |
+
n = content.count(old)
|
| 205 |
+
if n > 0:
|
| 206 |
+
content = content.replace(old, new)
|
| 207 |
+
print(f" {path}: replaced {n}x '{old[:50].strip()}...'")
|
| 208 |
+
if content != original:
|
| 209 |
+
with open(full, "w") as f:
|
| 210 |
+
f.write(content)
|
| 211 |
+
else:
|
| 212 |
+
print(f" {path}: no changes (already patched or pattern mismatch)")
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
T5_SMALL_LONG = [
|
| 216 |
+
"T5_small/gen_script_long_order3_t5_small_specroute.sh",
|
| 217 |
+
"T5_small/gen_script_long_order3_t5_small_specroute_v10a.sh",
|
| 218 |
+
"T5_small/gen_script_long_order3_t5_small_specroute_v10b.sh",
|
| 219 |
+
"T5_small/gen_script_long_order4_t5_small_specroute.sh",
|
| 220 |
+
]
|
| 221 |
+
|
| 222 |
+
T5_SMALL_SUPERNI = [
|
| 223 |
+
"T5_small/gen_script_superni_order1_t5_small_specroute.sh",
|
| 224 |
+
"T5_small/gen_script_superni_order2_t5_small_specroute.sh",
|
| 225 |
+
]
|
| 226 |
+
|
| 227 |
+
ROOT_T5 = [
|
| 228 |
+
"gen_script_long_order3_t5_specroute.sh",
|
| 229 |
+
"gen_script_long_order4_t5_specroute.sh",
|
| 230 |
+
"gen_script_superni_order1_t5_specroute.sh",
|
| 231 |
+
"gen_script_superni_order2_t5_specroute.sh",
|
| 232 |
+
]
|
| 233 |
+
|
| 234 |
+
print("=== T5_small Long scripts ===")
|
| 235 |
+
for p in T5_SMALL_LONG:
|
| 236 |
+
patch(p, [
|
| 237 |
+
(OLD_DETECT_LONG, NEW_DETECT),
|
| 238 |
+
(OLD_BSZ_LONG_A, NEW_BSZ_LONG),
|
| 239 |
+
])
|
| 240 |
+
|
| 241 |
+
print("\n=== T5_small SuperNI scripts ===")
|
| 242 |
+
for p in T5_SMALL_SUPERNI:
|
| 243 |
+
patch(p, [
|
| 244 |
+
(OLD_DETECT_SUPERNI, NEW_DETECT),
|
| 245 |
+
(OLD_DETECT_LONG, NEW_DETECT),
|
| 246 |
+
(OLD_BSZ_SUPERNI_A, NEW_BSZ_SUPERNI),
|
| 247 |
+
(OLD_BSZ_SUPERNI_B, NEW_BSZ_SUPERNI),
|
| 248 |
+
])
|
| 249 |
+
|
| 250 |
+
print("\n=== Root T5 scripts (add 3090 2-GPU to a100 branch) ===")
|
| 251 |
+
for p in ROOT_T5:
|
| 252 |
+
patch(p, [(OLD_A100_ROOT, NEW_A100_ROOT)])
|
| 253 |
+
|
| 254 |
+
print("\nDone.")
|