File size: 892 Bytes
5b39202 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | {
"starting_checkpoint": "../checkpoints/pretrained/ckpt.pt",
"epochs": 3,
"effective_batch_size": 32,
"learning_rate": 6.25e-05,
"warmup_fraction": 0.002,
"schedule": "linear warmup then linear decay, matching GPT-1 task adaptation",
"lm_coefficient": 0.5,
"classifier_dropout": 0.1,
"weight_decay": 0.01,
"betas": [
0.9,
0.999
],
"epsilon": 1e-08,
"grad_clip": 1.0,
"max_length": 512,
"seed": 42,
"cpu_threads": 2,
"desktop_reserve_gib": 2.5,
"max_process_gib": 9.0,
"checkpoint_interval": 500,
"log_interval": 25,
"max_consecutive_nonfinite": 8,
"tasks": {
"sst2": {
"microbatch_size": 16,
"gradient_checkpointing": false,
"eval_interval": 1000
},
"race_middle": {
"microbatch_size": 2,
"gradient_checkpointing": true,
"eval_interval": 400
}
}
}
|