| { | |
| "model": "amor_gdn", | |
| "scale": "440m", | |
| "backbone_config": { | |
| "d_model": 1024, | |
| "n_layer": 24, | |
| "d_ff": 1984, | |
| "d_state": 128, | |
| "expand": 2, | |
| "head_dim": 64, | |
| "d_conv": 4, | |
| "n_groups": 1, | |
| "peak_lr": 0.0003 | |
| }, | |
| "seq_len": 3072, | |
| "batch_size": 4, | |
| "grad_accum": 40, | |
| "world_size": 1, | |
| "total_tokens": 8973009920, | |
| "total_steps": 18255, | |
| "warmup_steps": 2000, | |
| "peak_lr": 0.0003, | |
| "min_lr": 1e-05, | |
| "chinchilla_multiplier": 20, | |
| "seed": 42, | |
| "vocab_size": 128256, | |
| "tokenizer": "teknium/Llama-3.1-AlternateTokenizer", | |
| "n_params": 442133056, | |
| "amor": { | |
| "lr_alpha": 0.003, | |
| "init_alpha": 0.0, | |
| "init_threshold": 0.3, | |
| "gate_offset": 0.03, | |
| "offset_mode": "adaptive", | |
| "offset_k": 0.2 | |
| } | |
| } | |