Hugging Face's logo

jasonfan
/
FST_code

Safetensors
Model card Files
xet
 Community
FST_code
File size: 3,950 Bytes
3b2d368
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#!/bin/bash

# Optional arguments [MODEL_NAME] [NUM_GPUS] [NUM_NODES]
NUM_GPUS=${1:-4}                # default 4 GPUs per node
NUM_NODES=${2:-1}               # default 1 node

# Clear first 2 args to avoid hydra issues
shift 2

# CPUs and memory automatically determined 
CPUS_PER_TASK=8                             # 8 CPUs per GPU
TOTAL_CPUS=$((NUM_GPUS * CPUS_PER_TASK))
TOTAL_MEM=$((NUM_GPUS * 32))                # 32gb ram per GPU

# Extract model name from hydra overrides
MODEL_NAME="transformer"
for arg in "$@"; do
  if [[ "$arg" == model=* ]]; then
    MODEL_NAME="${arg#model=}"
    break
  fi
done

JOB_NAME="${MODEL_NAME}_train_medium"

DATE=$(date +%F)

echo "Launching training job: $JOB_NAME"
echo "Nodes: ${NUM_NODES}, GPUs per node: ${NUM_GPUS}"
echo "Resources per node: ${TOTAL_CPUS} CPUs, ${TOTAL_MEM} GB memory"

SBATCH_ARGS="--job-name=${JOB_NAME} \
             --nodes=${NUM_NODES} \
             --gres=gpu:h200:${NUM_GPUS} \
             --ntasks-per-node=${NUM_GPUS} \
             --cpus-per-task=${CPUS_PER_TASK} \
             --mem=${TOTAL_MEM}G \
             --output=../../logs/${MODEL_NAME}/${DATE}/%x-%j.out \
             --error=../../logs/${MODEL_NAME}/${DATE}/%x-%j.err"

# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_1_3b.sh \
    # size=1_3b \
    # training=1_3b \
    # dataset=tinygsm_resume \
    # "$@"

# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_353M.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@" \
#     model=transformer_1_3B


# cp /work/jf381/code/lm-research/main_1_2.py /work/jf381/code/lm-research/main.py
bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_fst_353M_prediction.sh \
    size=medium \
    training=medium \
    dataset=slimpajama_60b \
    "$@" \
    model=fst_353M_prediction


# ork/jf381/code/lm-research/main_1_2.py /work/jf381/code/lm-research/main.py
# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_353M_bert.sh \
#     size=medium \
#     training=medium \
#     dataset=slimpajama_6b \
#     "$@" \
#     model=transformer_353M_bert

# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_1_3B.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@" \
#     model=transformer_1_3B

# cp /work/jf381/code/lm-research/main_1_7.py /work/jf381/code/lm-research/main.py
# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_1_3B.sh \
#     size=1_3b \
#     training=1_3b \
#     dataset=tinygsm_resume \
#     "$@" \
#     model=fst_1_3B


# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_353M.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@" \
#     model=fst_353M

# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_353M.sh \
#     size=1_3b \
#     training=1_3b \
#     dataset=tinygsm_resume \
#     "$@" \
#     model=fst_1_3B

# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_353M.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@" \
#     model=transformer_353M
# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_1_3b.sh \
#     size=1_3b \
#     training=1_3b \
#     dataset=tinygsm_resume \
#     "$@"

# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_generate.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@"
  ## left_small 02-
  ## 04 big-
  ## 03 fst big

# # cat << 'EOF' > 1_3b_transformer.txt
# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_1_3b_generate.sh \
#     size=1_3b \
#     training=1_3b \
#     dataset=tinygsm_resume \
#     "$@"
# EOF