| { | |
| "optimizer": "Mars", | |
| "max_grad_norm": "1.0", | |
| "weight_decay": "0.1, 0.0", | |
| "betas": "(0.9, 0.95)", | |
| "gamma": "0.025", | |
| "eps": "1e-08", | |
| "lr": "0.0003", | |
| "param_groups": "2", | |
| "trainable_params": "10391808", | |
| "schedule": "linear warmup 2% then cosine to 0% of peak", | |
| "batch_size": "32", | |
| "grad_accum": "1", | |
| "tokens_per_step": "8192", | |
| "n_ctx": "256", | |
| "total_steps": "5137", | |
| "warmup_steps": "102", | |
| "trained_tokens": "42090496", | |
| "chinchilla_ratio": "0.3342", | |
| "precision": "fp16", | |
| "seed": "26" | |
| } |