| { | |
| "run_name": "model1_8m", | |
| "results_tag": "canonical", | |
| "hf_repo_id": "FAIRC/token-averaging-model1_8m", | |
| "checkpoint_format": { | |
| "keys": [ | |
| "step", | |
| "tokens_seen", | |
| "cumulative_flops", | |
| "model", | |
| "optimizer", | |
| "scheduler" | |
| ], | |
| "note": "Raw torch.save dict from experiments/chinchilla/train.py. Load with torch.load(..., map_location='cpu', weights_only=False) and take state['model']. Not a transformers AutoModel checkpoint." | |
| }, | |
| "model_config": { | |
| "name": "model1_8m", | |
| "d_model": 128, | |
| "n_heads": 4, | |
| "n_layers": 6, | |
| "context_len": 512, | |
| "averaging_k": 1, | |
| "method_name": null, | |
| "multi_token_phase_ratio": 0.0, | |
| "tie_embeddings": true, | |
| "grad_checkpoint": false, | |
| "color": "#58a6ff", | |
| "label": "~8M standard (n=512)", | |
| "target_tokens": 300000000, | |
| "lr": 0.0004, | |
| "warmup_steps": 500, | |
| "n_params_approx": 7612544 | |
| } | |
| } | |