| { |
| "train_loss": [], |
| "val_loss": [], |
| "val_pp": [], |
| "val_acc": [], |
| "args": { |
| "seed": 0, |
| "data_seed": 1337, |
| "device": "cuda:0", |
| "dtype": "bfloat16", |
| "compile": true, |
| "vocab_size": 50257, |
| "block_size": 1024, |
| "n_layer": 12, |
| "n_embd": 768, |
| "n_head": 12, |
| "kv_lora_rank": 192, |
| "q_lora_rank": 256, |
| "rope_dim": 64, |
| "n_experts": 32, |
| "n_experts_per_token": 2, |
| "expert_intermediate_size": 296, |
| "shared_expert_intermediate_size": 296, |
| "mtp_num_heads": 0, |
| "dropout": 0.0, |
| "bias": true, |
| "gate_bias": true, |
| "gate_score_func": "softmax", |
| "aux_loss_weight": 0.0, |
| "mtp_loss_weight": 0.3, |
| "opt": "adamw", |
| "lr": 0.0003, |
| "weight_decay": 0.1, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "grad_clip": 1.0, |
| "batch_size": 32, |
| "acc_steps": 8, |
| "scheduler": "cos", |
| "iterations": 12800, |
| "warmup_steps": 1280, |
| "min_lr_ratio": 0.1, |
| "wsd_fract_decay": 0.1, |
| "wsd_final_lr_scale": 0.0, |
| "decay_type": "cosine", |
| "eval_interval": 200, |
| "eval_batches": 32, |
| "full_eval_at": [], |
| "log_interval": 50, |
| "diag_interval": 200, |
| "diag_batches": 4, |
| "permanent_ckpt_interval": 3200, |
| "latest_ckpt_interval": 500, |
| "resume_from": null, |
| "auto_resume": true, |
| "wandb": true |
| } |
| } |
