File size: 2,720 Bytes
c21820d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | wandb:
entity: null
resume: 'auto'
experiment:
project: "rl_sdar_code" # need to be same of this file name
function: "train"
start_from_scratch: True
total_step: 80
save_every: 10
eval_every: 10
current_epoch: 1
deepspeed_file: "1_node_8_gpus_deepspeed_zero3"
num_node: 1
node_index: 0
model:
pretrained_model: "/abs/path/of/model" # absolute path of your model
optimized_name: "optimized" # the output name for your optimized model, will be saved under sft_dream/ckpt
model_base: "sdar" # set sdar for TraDo and SDAR
dataset:
train_dataset: "PrimeIntellect" # "MATH_train""PrimeIntellect"
optimization_data: "rl_data" # name of the rollout data output
data_type: "code" # "math" "code"
rollout:
tensor_parallel_size: 1
max_active: 256
num_task_per_step: 64
num_response_per_task: 16
temperature: 1.0
max_token: 2000
block_size: 4
denoising_steps_per_block: 4
top_p: 1.0
top_k: 0
remasking_strategy: "low_confidence_dynamic" #"low_confidence_static""low_confidence_dynamic"
dynamic_threshold: 0.9 # no use for "low_confidence_static"
start_with_think: False
execute:
num_chunk: 128 # batch size of executing codes in coding eval tasks
training:
gradient_checkpointing_enable: True
gradient_accumulation_steps: 32
batch_size_lm: 1
mixed_precision: "bf16"
enable_tf32: True
seed: 10086
num_train_epochs: 1
max_grad_norm: 1.0
method: "TraceRL" # "random_masking" "TraceRL" "coupled"
block_size: 4
shrink: 1
post_num: 0
max_gen_length: 2000
max_prompt_len: 784
lower_p: 0.1
upper_p: 0.9
eps: 0.20
beta: 0.01
use_kl_estimator_k3: True
optimizer:
name: adamw
params: # default adamw params
learning_rate: 1e-6
scale_lr: False # scale learning rate by total batch size
beta1: 0.9
beta2: 0.999
weight_decay: 0.0
epsilon: 1e-8
lr_scheduler:
scheduler: "cosine"
params:
learning_rate: ${optimizer.params.learning_rate}
warmup_steps: 0
min_lr_scale: 1.0
evaluation:
eval_dataset: "LiveCodeBench" # "MATH500" "LiveCodeBench"
data_type: "code" #"math"
tensor_parallel_size: 1
max_active: 256
num_response_per_task: 3
temperature: 1.0
max_token: 2000
block_size: 4
denoising_steps_per_block: 4
top_p: 1.0
top_k: [0, 1]
remasking_strategy: ["low_confidence_dynamic", "low_confidence_static"] #"low_confidence_static""low_confidence_dynamic"
dynamic_threshold: 0.9 # no use for "low_confidence_static"
start_with_think: False
# len(top_k) must == len(remasking_strategy)
|