| { |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "adam_epsilon": 1e-08, |
| "cpu_prefetch_factor": 2, |
| "cpu_workers": 2, |
| "dataset_manifest": "/home/nvidia/llm-data/haiku/tokenized/sft-all-v3/manifest.json", |
| "ema_inv_gamma": 1.0, |
| "ema_max_decay": 0.9999, |
| "ema_power": 0.75, |
| "enable_gradient_checkpointing": true, |
| "exclude_components": [ |
| "nemotron-cascade-2" |
| ], |
| "fp32_master_weights": true, |
| "full_shuffle": true, |
| "gradient_accumulation_steps": 16, |
| "gradient_checkpointing_layers": 22, |
| "gradient_clip_norm": 1.0, |
| "initial_run_dir": "/media/nvidia/4TB2/llm-runs/senryu/pretrain-v1", |
| "initial_weights": "ema", |
| "learning_rate": 0.0004, |
| "log_every_optimizer_steps": 1, |
| "logit_z_loss_coefficient": 0.0001, |
| "maximum_checkpoints": 5, |
| "maximum_optimizer_steps": null, |
| "micro_batch_size": 1, |
| "minimum_learning_rate_ratio": 0.1, |
| "muon_learning_rate": null, |
| "muon_lr_adjustment": "match_rms_adamw", |
| "muon_momentum": 0.95, |
| "optimizer": "hybrid_muon", |
| "output_dir": "/media/nvidia/4TB2/llm-runs/senryu/sft-v1", |
| "require_complete_pretraining": false, |
| "require_complete_sft_source": false, |
| "rewrite_empty_think": true, |
| "save_every_optimizer_steps": 100, |
| "seed": 20261008, |
| "sequence_length": 32768, |
| "shuffle_block_sequences": 2048, |
| "stream_download_threads": 8, |
| "stream_english_share": null, |
| "stream_open_languages": null, |
| "stream_pairs_per_row": 1, |
| "tokenizer": null, |
| "unclosed_think": "mask_end", |
| "validation_batches": 16, |
| "warmup_ratio": 0.0, |
| "warmup_steps": 200, |
| "weight_decay": 0.1 |
| } |
|
|
