{ "model": { "vocab_size": 32000, "sequence_len": 2048, "n_layer": 24, "n_head": 9, "n_kv_head": 9, "n_embd": 1152, "mlp_ratio": 4.0, "rope_base": 10000.0, "dropout": 0.0, "use_qk_norm": true, "use_liger": true, "attention_backend": "auto" }, "tokenizer": "meta-llama/Llama-2-7b-chat-hf", "target_tokens": 5000000000, "micro_batch_size": 28, "gradient_accumulation_steps": 1, "dtype": "bfloat16", "compile": true, "compile_mode": "default", "zero_optimizer": true, "verify_replicas": true, "matrix_lr": 0.005, "adam_lr": 5e-05, "weight_decay": 0.1, "warmup_steps": 500, "checkpoint_interval_tokens": 1000000000, "latest_interval_tokens": 500000000, "log_interval": 10, "grad_clip": 1.0, "seed": 42 }