intent-0.1-0.5b / config.json
sripkunda's picture
Publish model checkpoint
20935c3 verified
Raw History Blame Contribute Delete
788 Bytes
{
"model": {
"vocab_size": 32000,
"sequence_len": 2048,
"n_layer": 24,
"n_head": 9,
"n_kv_head": 9,
"n_embd": 1152,
"mlp_ratio": 4.0,
"rope_base": 10000.0,
"dropout": 0.0,
"use_qk_norm": true,
"use_liger": true,
"attention_backend": "auto"
},
"tokenizer": "meta-llama/Llama-2-7b-chat-hf",
"target_tokens": 5000000000,
"micro_batch_size": 28,
"gradient_accumulation_steps": 1,
"dtype": "bfloat16",
"compile": true,
"compile_mode": "default",
"zero_optimizer": true,
"verify_replicas": true,
"matrix_lr": 0.005,
"adam_lr": 5e-05,
"weight_decay": 0.1,
"warmup_steps": 500,
"checkpoint_interval_tokens": 1000000000,
"latest_interval_tokens": 500000000,
"log_interval": 10,
"grad_clip": 1.0,
"seed": 42
}