File size: 1,248 Bytes
bad775e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | {
"step": 300000,
"recipe": {
"vocab_size": 8192,
"num_hidden_layers": 12,
"hidden_size": 128,
"hidden_size_per_layer_input": 16,
"intermediate_size": 384,
"num_attention_heads": 4,
"num_key_value_heads": 1,
"head_dim": 32,
"global_head_dim": 32,
"sliding_window": 128,
"max_position_embeddings": 2048,
"full_attention_every": 4
},
"args": {
"output_dir": "runs\\tiny-gemma4-v3-muon-corrected",
"cache_dir": "runs\\tiny-gemma4-v3\\cache",
"init_from": null,
"data_dir": "data",
"tokenizer_dir": "runs\\tiny-gemma4-v3\\tokenizer",
"recipe": "v3",
"data_mode": null,
"max_steps": 300000,
"batch_size": 32,
"block_size": 256,
"gradient_accumulation_steps": 1,
"learning_rate": 0.00035,
"min_learning_rate": 3.5e-05,
"warmup_steps": 1000,
"schedule_start_step": 0,
"optimizer": "muon",
"weight_decay": 0.1,
"grad_clip": 1.0,
"save_every": 10000,
"eval_every": 1000,
"eval_batches": 50,
"train_char_limit": null,
"valid_char_limit": null,
"num_workers": 0,
"seed": 1337,
"resume": false,
"compile": false
},
"time": "2026-07-22 21:53:23"
} |