TightX commited on
Commit
845526b
·
verified ·
1 Parent(s): fcb35ee

add arch_facts.json

Browse files
Files changed (1) hide show
  1. arch_facts.json +60 -0
arch_facts.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "params_total": 1095845888,
3
+ "n_tensors": 143,
4
+ "dtype": "bf16",
5
+ "vocab_size": 32000,
6
+ "d_model": 2048,
7
+ "n_blocks": 20,
8
+ "tensor_keys_sample": [
9
+ "blocks.0.attn.out_proj.weight",
10
+ "blocks.0.attn.qkv.weight",
11
+ "blocks.0.attn_norm.scale",
12
+ "blocks.0.ffn.down.weight",
13
+ "blocks.0.ffn.gate.weight",
14
+ "blocks.0.ffn.up.weight",
15
+ "blocks.0.ffn_norm.scale",
16
+ "blocks.1.attn.out_proj.weight"
17
+ ],
18
+ "source_checkpoint": "<workspace>/1B WSM/checkpoints_wsm_1b/merged/checkpoint.pt",
19
+ "source_checkpoint_relative": "checkpoints_wsm_1b/merged/checkpoint.pt",
20
+ "source_keys": [
21
+ "optimizer",
22
+ "scheduler",
23
+ "config",
24
+ "training_state",
25
+ "timestamp",
26
+ "model"
27
+ ],
28
+ "source_config": {
29
+ "vocab_size": 32000,
30
+ "d_model": 2048,
31
+ "num_layers": 20,
32
+ "num_heads": 16,
33
+ "d_ff": 5120,
34
+ "max_seq_len": 1024,
35
+ "rope_theta": 500000.0,
36
+ "dropout_rate": 0.0,
37
+ "use_fp8": true
38
+ },
39
+ "config_note": "use_fp8 was true in the raw checkpoint config (FP8 training kernels); written false here because the released weights are plain bf16 and ModelConfig(use_fp8=True) cannot run on CPU.",
40
+ "checkpoint_step": 630000,
41
+ "checkpoint_timestamp": "2026-10-10T11:20:03.217270",
42
+ "training_state": {
43
+ "step": 630000,
44
+ "total_tokens": 30977679360,
45
+ "epoch": 0,
46
+ "best_val_loss": 1.1991855552196502,
47
+ "max_tokens": 31013881856,
48
+ "round_start_tokens": 0,
49
+ "data_position": {
50
+ "source_type": "bin",
51
+ "file_index": 0,
52
+ "files_total": 0,
53
+ "rows_skipped": 0,
54
+ "bin_path": "data\\all\\train.bin",
55
+ "token_offset": 30977679360
56
+ }
57
+ },
58
+ "nonfinite_after_cast": [],
59
+ "created_utc": "2026-10-23T10:57:39Z"
60
+ }