File size: 1,698 Bytes
845526b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | {
"params_total": 1095845888,
"n_tensors": 143,
"dtype": "bf16",
"vocab_size": 32000,
"d_model": 2048,
"n_blocks": 20,
"tensor_keys_sample": [
"blocks.0.attn.out_proj.weight",
"blocks.0.attn.qkv.weight",
"blocks.0.attn_norm.scale",
"blocks.0.ffn.down.weight",
"blocks.0.ffn.gate.weight",
"blocks.0.ffn.up.weight",
"blocks.0.ffn_norm.scale",
"blocks.1.attn.out_proj.weight"
],
"source_checkpoint": "<workspace>/1B WSM/checkpoints_wsm_1b/merged/checkpoint.pt",
"source_checkpoint_relative": "checkpoints_wsm_1b/merged/checkpoint.pt",
"source_keys": [
"optimizer",
"scheduler",
"config",
"training_state",
"timestamp",
"model"
],
"source_config": {
"vocab_size": 32000,
"d_model": 2048,
"num_layers": 20,
"num_heads": 16,
"d_ff": 5120,
"max_seq_len": 1024,
"rope_theta": 500000.0,
"dropout_rate": 0.0,
"use_fp8": true
},
"config_note": "use_fp8 was true in the raw checkpoint config (FP8 training kernels); written false here because the released weights are plain bf16 and ModelConfig(use_fp8=True) cannot run on CPU.",
"checkpoint_step": 630000,
"checkpoint_timestamp": "2026-10-10T11:20:03.217270",
"training_state": {
"step": 630000,
"total_tokens": 30977679360,
"epoch": 0,
"best_val_loss": 1.1991855552196502,
"max_tokens": 31013881856,
"round_start_tokens": 0,
"data_position": {
"source_type": "bin",
"file_index": 0,
"files_total": 0,
"rows_skipped": 0,
"bin_path": "data\\all\\train.bin",
"token_offset": 30977679360
}
},
"nonfinite_after_cast": [],
"created_utc": "2026-10-23T10:57:39Z"
} |