| init: | |
| init_type: mup | |
| std_embed: 0.02 | |
| std_o: 0.01 | |
| std_qkv: 0.02 | |
| loss: | |
| label_smoothing: 0.0 | |
| type: next_token_ce | |
| model: | |
| attn_scale: 0.35 | |
| attn_type: quadratic | |
| d_model: 16 | |
| n_ctx: 16 | |
| n_head: 2 | |
| n_layers: 2 | |
| norm_places: | |
| - pre_unembed | |
| norm_type: none | |
| rope_base: 10000 | |
| use_bias_qk: false | |
| use_rmsnorm_qk: false | |
| vocab_size: 16 | |
| name: big_experiment | |
| seed: 42 | |
| train: | |
| batch_size: 128 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| checkpoint_every: 60 | |
| checkpoint_linear_count: 20 | |
| checkpoint_log_count: 0 | |
| checkpoint_schedule: interval | |
| debug: true | |
| dtype: bfloat16 | |
| eval_every: 100 | |
| grad_clip: 1.0 | |
| lr: 0.0003 | |
| lr_decay_frac: 1.0 | |
| max_steps: 6000 | |
| muon_lr: 0.03 | |
| use_muon: true | |
| warmup_steps: 500 | |
| weight_decay: 0.01 | |