train_args.json

{
  "data": "cosmopedia_v2_3B_8k_digits.uint16.bin",
  "base": "DynamicMind-Mini",
  "revision": "",
  "tokenizer": "",
  "out": "checkpoints/DynamicMind-v3-9M-CosmopediaV2-3B-1024",
  "seq_len": 1024,
  "batch_size": 72,
  "grad_accum": 16,
  "max_steps": 2613,
  "lr": 0.0001,
  "min_lr": 1e-05,
  "warmup_steps": 200,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "val_tokens": 5000000,
  "eval_every": 250,
  "eval_batches": 50,
  "save_every": 250,
  "device": "cuda",
  "dtype": "auto",
  "compile": true,
  "compile_mode": "default",
  "seed": 1337,
  "resume": ""
}