training_config.json

{
  "step": 15000,
  "source_tokens": 3932160000,
  "dataset_files": "FineWeb-Edu sample-10BT local parquet shards",
  "args": {
    "model": "diffusion",
    "max_steps": 15000,
    "stop_after_step": 0,
    "batch_size": 8,
    "grad_accum": 16,
    "seq_len": 2048,
    "lr": 0.001,
    "min_lr_frac": 0.1,
    "weight_decay": 0.1,
    "warmup_frac": 0.01,
    "betas": [
      0.9,
      0.95
    ],
    "grad_clip": 1.0,
    "dtype": "bf16",
    "device": "auto",
    "seed": 2026,
    "text_file": null,
    "dataset_name": "parquet",
    "log_every": 20,
    "no_tensorboard": false,
    "save_steps": 100,
    "max_host_memory_gib": 12.0,
    "shuffle_buffer": 100000,
    "compile": true,
    "compile_layers": false,
    "non_blocking_transfer": true,
    "cuda_memory_fraction": 0.9,
    "ul2": false,
    "diffusion_time_conditioning": "none",
    "ce_chunk": 0,
    "grad_ckpt": false,
    "resume": false
  }
}