sft_config.json

{
  "adam_beta1": 0.9,
  "adam_beta2": 0.95,
  "adam_epsilon": 1e-08,
  "cpu_prefetch_factor": 4,
  "cpu_workers": 2,
  "dataset_manifest": "data/tokenized/smoltalk2-sft/manifest.json",
  "ema_inv_gamma": 1.0,
  "ema_max_decay": 0.9999,
  "ema_power": 0.75,
  "enable_gradient_checkpointing": true,
  "full_shuffle": true,
  "gradient_accumulation_steps": 16,
  "gradient_clip_norm": 1.0,
  "initial_run_dir": "runs/tercet-r-sft",
  "initial_weights": "ema",
  "learning_rate": 5e-05,
  "log_every_optimizer_steps": 1,
  "logit_z_loss_coefficient": 0.0001,
  "maximum_checkpoints": 5,
  "maximum_optimizer_steps": null,
  "micro_batch_size": 1,
  "minimum_learning_rate_ratio": 0.1,
  "muon_learning_rate": 0.01,
  "muon_momentum": 0.95,
  "optimizer": "adamw",
  "output_dir": "runs/tercet-r-instruct",
  "require_complete_pretraining": false,
  "require_complete_sft_source": false,
  "save_every_optimizer_steps": 100,
  "seed": 20260830,
  "sequence_length": 16384,
  "shuffle_block_sequences": 4096,
  "validation_batches": 16,
  "warmup_ratio": 0.01,
  "warmup_steps": 200,
  "weight_decay": 0.0
}