File size: 760 Bytes
145c3b7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | {
"model": "amor_gdn",
"scale": "1.5b",
"backbone_config": {
"d_model": 2048,
"n_layer": 24,
"d_ff": 4096,
"d_state": 128,
"expand": 2,
"head_dim": 64,
"d_conv": 4,
"n_groups": 1,
"peak_lr": 0.00015
},
"seq_len": 3072,
"batch_size": 2,
"grad_accum": 80,
"world_size": 1,
"total_tokens": 30748518400,
"total_steps": 62558,
"warmup_steps": 2000,
"peak_lr": 0.00015,
"min_lr": 1e-05,
"chinchilla_multiplier": 20,
"seed": 42,
"vocab_size": 128256,
"tokenizer": "teknium/Llama-3.1-AlternateTokenizer",
"n_params": 1524025472,
"amor": {
"lr_alpha": 0.003,
"init_alpha": 0.0,
"init_threshold": 0.3,
"gate_offset": 0.03,
"offset_mode": "adaptive",
"offset_k": 0.2
}
}
|