Hugging Face's logo

jasonfan
/
FST_code

Safetensors
Model card Files
xet
 Community
FST_code
File size: 2,882 Bytes
3b2d368
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
#!/bin/bash

# Optional arguments [MODEL_NAME] [NUM_GPUS] [NUM_NODES]
NUM_GPUS=${1:-4}                # default 4 GPUs per node
NUM_NODES=${2:-1}               # default 1 node

# Clear first 2 args to avoid hydra issues
shift 2

# CPUs and memory automatically determined 
CPUS_PER_TASK=8                             # 8 CPUs per GPU
TOTAL_CPUS=$((NUM_GPUS * CPUS_PER_TASK))
TOTAL_MEM=$((NUM_GPUS * 32))                # 32gb ram per GPU

# Extract model name from hydra overrides
MODEL_NAME="transformer"
for arg in "$@"; do
  if [[ "$arg" == model=* ]]; then
    MODEL_NAME="${arg#model=}"
    break
  fi
done

JOB_NAME="${MODEL_NAME}_train_medium"

DATE=$(date +%F)

echo "Launching training job: $JOB_NAME"
echo "Nodes: ${NUM_NODES}, GPUs per node: ${NUM_GPUS}"
echo "Resources per node: ${TOTAL_CPUS} CPUs, ${TOTAL_MEM} GB memory"

SBATCH_ARGS="--job-name=${JOB_NAME} \
             --nodes=${NUM_NODES} \
             --gres=gpu:h200:${NUM_GPUS} \
             --ntasks-per-node=${NUM_GPUS} \
             --cpus-per-task=${CPUS_PER_TASK} \
             --mem=${TOTAL_MEM}G \
             --output=../../logs/${MODEL_NAME}/${DATE}/%x-%j.out \
             --error=../../logs/${MODEL_NAME}/${DATE}/%x-%j.err"

# bash /work/jf381/code/lm-research/scripts/training/train_medium_bash_resume.sh model=fst;bash /work/jf381/code/lm-research/scripts/training/train_medium_bash_resume_transformer.sh model=transformer;

# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@"

# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_1_3b_trial.sh \
#     size=1_3b \
#     training=1_3b \
#     dataset=tinygsm_resume \
#     "$@" \
#     model=fst_1_3B
bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_transformer_1_3b_trial.sh \
    size=medium \
    training=medium \
    dataset=tinygsm_resume \
    "$@" \
    model=fst_1_3B



# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_353M.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@" 

# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_353M.sh \
#     size=1_3b \
#     training=1_3b \
#     dataset=tinygsm_resume \
#     "$@" 

# bash  /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_generate.sh \
#     size=medium \
#     training=medium \
#     dataset=tinygsm_resume \
#     "$@"

# bash /work/jf381/code/lm-research/scripts/jobs/run_job_h200_bash_resume_1_3b_generate.sh \
#     size=1_3b \
#     training=1_3b \
#     dataset=tinygsm_resume \
#     "$@"

# bash /work/jf381/code/lm-research/scripts/training/train_medium_bash_resume.sh model=fst;

# bash /work/jf381/code/lm-research/scripts/training/train_medium_bash_resume.sh model=transformer;