File size: 925 Bytes
80aea5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | {
"step": 15000,
"source_tokens": 3932160000,
"dataset_files": "FineWeb-Edu sample-10BT local parquet shards",
"args": {
"model": "diffusion",
"max_steps": 15000,
"stop_after_step": 0,
"batch_size": 8,
"grad_accum": 16,
"seq_len": 2048,
"lr": 0.001,
"min_lr_frac": 0.1,
"weight_decay": 0.1,
"warmup_frac": 0.01,
"betas": [
0.9,
0.95
],
"grad_clip": 1.0,
"dtype": "bf16",
"device": "auto",
"seed": 2026,
"text_file": null,
"dataset_name": "parquet",
"log_every": 20,
"no_tensorboard": false,
"save_steps": 100,
"max_host_memory_gib": 12.0,
"shuffle_buffer": 100000,
"compile": true,
"compile_layers": false,
"non_blocking_transfer": true,
"cuda_memory_fraction": 0.9,
"ul2": false,
"diffusion_time_conditioning": "none",
"ce_chunk": 0,
"grad_ckpt": false,
"resume": false
}
}
|