| { |
| "params_total": 1095845888, |
| "n_tensors": 143, |
| "dtype": "bf16", |
| "vocab_size": 32000, |
| "d_model": 2048, |
| "n_blocks": 20, |
| "tensor_keys_sample": [ |
| "blocks.0.attn.out_proj.weight", |
| "blocks.0.attn.qkv.weight", |
| "blocks.0.attn_norm.scale", |
| "blocks.0.ffn.down.weight", |
| "blocks.0.ffn.gate.weight", |
| "blocks.0.ffn.up.weight", |
| "blocks.0.ffn_norm.scale", |
| "blocks.1.attn.out_proj.weight" |
| ], |
| "source_checkpoint": "<workspace>/1B WSM/checkpoints_wsm_1b/merged/checkpoint.pt", |
| "source_checkpoint_relative": "checkpoints_wsm_1b/merged/checkpoint.pt", |
| "source_keys": [ |
| "optimizer", |
| "scheduler", |
| "config", |
| "training_state", |
| "timestamp", |
| "model" |
| ], |
| "source_config": { |
| "vocab_size": 32000, |
| "d_model": 2048, |
| "num_layers": 20, |
| "num_heads": 16, |
| "d_ff": 5120, |
| "max_seq_len": 1024, |
| "rope_theta": 500000.0, |
| "dropout_rate": 0.0, |
| "use_fp8": true |
| }, |
| "config_note": "use_fp8 was true in the raw checkpoint config (FP8 training kernels); written false here because the released weights are plain bf16 and ModelConfig(use_fp8=True) cannot run on CPU.", |
| "checkpoint_step": 630000, |
| "checkpoint_timestamp": "2026-10-10T11:20:03.217270", |
| "training_state": { |
| "step": 630000, |
| "total_tokens": 30977679360, |
| "epoch": 0, |
| "best_val_loss": 1.1991855552196502, |
| "max_tokens": 31013881856, |
| "round_start_tokens": 0, |
| "data_position": { |
| "source_type": "bin", |
| "file_index": 0, |
| "files_total": 0, |
| "rows_skipped": 0, |
| "bin_path": "data\\all\\train.bin", |
| "token_offset": 30977679360 |
| } |
| }, |
| "nonfinite_after_cast": [], |
| "created_utc": "2026-10-23T10:57:39Z" |
| } |
