| { | |
| "model": { | |
| "width": 256, | |
| "heads": 8, | |
| "layers": 6, | |
| "feed_forward_multiplier": 4, | |
| "context_length": 768, | |
| "prompt_bytes": 126, | |
| "codebooks": 4, | |
| "codebook_size": 1024 | |
| }, | |
| "optimizer": { | |
| "peak_learning_rate": 0.0006, | |
| "final_learning_rate": 0.00006, | |
| "warmup_steps": 500, | |
| "decay_steps": 30000, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "epsilon": 1e-8, | |
| "weight_decay": 0.1, | |
| "max_gradient_norm": 1.0 | |
| }, | |
| "batch_size": 8, | |
| "sequence_length": 768, | |
| "steps": 30000, | |
| "log_every": 25, | |
| "validate_every": 500, | |
| "validation_batches": 16, | |
| "checkpoint_every": 500, | |
| "seed": 1234 | |
| } | |