SmallJerry/config.json

{
  "vocab_size": 32768,
  "n_embd": 768,
  "n_head": 12,
  "n_layer": 12,
  "intermediate_size": 2048,
  "block_size": 1024,
  "rope_theta": 500000.0,
  "norm_eps": 1e-05,
  "dropout": 0.0,
  "max_iters": 150000,
  "learning_rate": 0.0006,
  "min_lr": 6e-05,
  "warmup_iters": 3000,
  "weight_decay": 0.01,
  "beta1": 0.9,
  "beta2": 0.98,
  "grad_clip": 1.0,
  "batch_size": 4,
  "gradient_accumulation_steps": 8,
  "mask_prob": 0.2,
  "log_interval": 50,
  "eval_interval": 500,
  "eval_iters": 20,
  "save_interval": 200,
  "hf_repo": "Amogh1221/bellhart_training",
  "device": "cuda",
  "dtype": "bfloat16",
  "tf32": true,
  "use_8bit_optimizer": false
}