{
  "params_total": 1095845888,
  "n_tensors": 143,
  "dtype": "bf16",
  "vocab_size": 32000,
  "d_model": 2048,
  "n_blocks": 20,
  "tensor_keys_sample": [
    "blocks.0.attn.out_proj.weight",
    "blocks.0.attn.qkv.weight",
    "blocks.0.attn_norm.scale",
    "blocks.0.ffn.down.weight",
    "blocks.0.ffn.gate.weight",
    "blocks.0.ffn.up.weight",
    "blocks.0.ffn_norm.scale",
    "blocks.1.attn.out_proj.weight"
  ],
  "source_checkpoint": "<workspace>/1B WSM/checkpoints_wsm_1b/merged/checkpoint.pt",
  "source_checkpoint_relative": "checkpoints_wsm_1b/merged/checkpoint.pt",
  "source_keys": [
    "optimizer",
    "scheduler",
    "config",
    "training_state",
    "timestamp",
    "model"
  ],
  "source_config": {
    "vocab_size": 32000,
    "d_model": 2048,
    "num_layers": 20,
    "num_heads": 16,
    "d_ff": 5120,
    "max_seq_len": 1024,
    "rope_theta": 500000.0,
    "dropout_rate": 0.0,
    "use_fp8": true
  },
  "config_note": "use_fp8 was true in the raw checkpoint config (FP8 training kernels); written false here because the released weights are plain bf16 and ModelConfig(use_fp8=True) cannot run on CPU.",
  "checkpoint_step": 630000,
  "checkpoint_timestamp": "2026-10-10T11:20:03.217270",
  "training_state": {
    "step": 630000,
    "total_tokens": 30977679360,
    "epoch": 0,
    "best_val_loss": 1.1991855552196502,
    "max_tokens": 31013881856,
    "round_start_tokens": 0,
    "data_position": {
      "source_type": "bin",
      "file_index": 0,
      "files_total": 0,
      "rows_skipped": 0,
      "bin_path": "data\\all\\train.bin",
      "token_offset": 30977679360
    }
  },
  "nonfinite_after_cast": [],
  "created_utc": "2026-10-23T10:57:39Z"
}