| { | |
| "size": "100m", | |
| "data_path": "data\\bootstrap\\demo_corpus.txt", | |
| "output_dir": "checkpoints/pixel_100m", | |
| "batch_size": 1, | |
| "grad_accumulation_steps": 2, | |
| "learning_rate": 0.0003, | |
| "min_learning_rate": 3e-05, | |
| "warmup_steps": 4, | |
| "total_steps": 10, | |
| "weight_decay": 0.1, | |
| "max_grad_norm": 1.0, | |
| "eval_every": 10, | |
| "save_every": 10, | |
| "seed": 42, | |
| "gradient_checkpointing": false, | |
| "mode": "pretrain", | |
| "sequence_length": 32, | |
| "num_workers": 0 | |
| } |