| { | |
| "variant": "stage3-e3", | |
| "step": 2800, | |
| "n_params": 39582595, | |
| "trainer_dev_metrics_NOT_comparable": { | |
| "first_tok_top1": 0.30078125, | |
| "first_tok_top5": 0.7080078125, | |
| "first_tok_ce": 2.5095736607909203 | |
| }, | |
| "arch": { | |
| "vocab_size": 4096, | |
| "block_size": 768, | |
| "n_layer": 8, | |
| "n_head": 10, | |
| "n_embd": 640, | |
| "intermediate_size": 1536, | |
| "dropout": 0.1, | |
| "rope_base": 10000.0, | |
| "norm_eps": 1e-05, | |
| "policy_rank": 128, | |
| "value_hidden": 128, | |
| "value_mid": 32, | |
| "n_value_out": 3, | |
| "tie_embeddings": true, | |
| "depth_rope": false | |
| } | |
| } |