weilan55 commited on
Commit
c544f4a
·
verified ·
1 Parent(s): 01f91d1

g3c: final checkpoint + run config

Browse files
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ g3c/model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
37
+ g3c/model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
38
+ g3c/model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
39
+ g3c/model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
g3c/FINAL_ITERATION ADDED
@@ -0,0 +1 @@
 
 
1
+ 6001
g3c/README.md ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # g3c — ALL-TURN corpus control.
2
+
3
+ **Final checkpoint: iteration 6001** (from `latest_checkpointed_iteration.txt`).
4
+
5
+ ## The variable
6
+
7
+ Trains on the all-turn corpus (`v3ctl_allturn_train.parquet`) instead of the v3 blend that every later arm uses. It is the only g3 arm whose corpus differs, so it is NOT single-variable against g3d.
8
+
9
+ ## Architecture / run settings (read from this run's own saved config)
10
+
11
+ | key | value |
12
+ |---|---|
13
+ | `fc_norm` | `false` |
14
+ | `block_size` | 3 |
15
+ | `target_layer_ids` | [1, 29, 57] |
16
+ | `inference_num_gpus` | 16 |
17
+ | `train_data_path` | `/data/corpus/v3ctl_allturn_train.parquet` |
18
+ | draft config | `kimi_k3_flash_dspark_mla_g3c.json` |
19
+
20
+ - target: `moonshotai/Kimi-K3-Flash` (61 text layers)
21
+ - draft: 3 layers, MLA, `enable_confidence_head: true`, `markov_head_type: vanilla`
22
+ - `max_seq_length: 65536`, `last_turn_loss_only: true`, `micro_batch_size: 1`
23
+ - cluster: GB300 (aarch64, 4 GPUs/node), 2x4 trainer ranks + producer engines
24
+ - `num_train_steps: null` — the schedule is driven by the corpus, not a step cap
25
+
26
+ ## Files
27
+
28
+ | file | what it is |
29
+ |---|---|
30
+ | `model/` | the final iteration's distcp weight shards — **this is the checkpoint** |
31
+ | `draft_model_config.json` | draft architecture; the file the trainer actually pointed at |
32
+ | `train_config.yaml` | the full run config as saved next to the checkpoints |
33
+ | `ckpt_meta.json` | trainer metadata (`global_step`, `world_size`, timestamp) |
34
+ | `FINAL_ITERATION` | the iteration number, for scripted selection |
35
+
36
+ `optimizer/` is **not** included: it is ~80% of the on-disk bytes and is only needed
37
+ to resume training. For export + evaluation the `model/` shards are sufficient.
38
+
39
+ ## Converting to a servable HF checkpoint
40
+
41
+ These are TorchSpec distcp shards, not an HF folder. Convert with the repo's own
42
+ exporter (`arthur/scripts/export_k3flash_dspark_vllm.sh`, which wraps TorchSpec's
43
+ `tools/convert_to_hf.py --vllm`), then serve the result as the draft in
44
+ `--speculative-config`. Two gates worth keeping, both of which fail silently if
45
+ skipped:
46
+
47
+ 1. **`num_speculative_tokens` must equal this checkpoint's `block_size` (3).**
48
+ A mismatch drafts a different width and reads as low acceptance, not as an error.
49
+ 2. **The target's depth must match `target_num_hidden_layers: 61`.** Pointed at a
50
+ different-depth target, vLLM RESCALES the aux capture ids and logs only a WARNING,
51
+ after which the draft reads layers it never trained on.
g3c/ckpt_meta.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "global_step": 6000,
3
+ "inference_id": 6000,
4
+ "iteration": 6001,
5
+ "next_inference_id": 6001,
6
+ "next_step": 6001,
7
+ "step": 6000,
8
+ "timestamp": 1790867069.8654246,
9
+ "world_size": 8
10
+ }
g3c/draft_model_config.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "K3DSparkModel"
4
+ ],
5
+ "model_type": "k3_dspark",
6
+ "hidden_size": 7168,
7
+ "intermediate_size": 18432,
8
+ "num_hidden_layers": 3,
9
+ "num_attention_heads": 64,
10
+ "num_key_value_heads": 64,
11
+ "q_lora_rank": 1536,
12
+ "kv_lora_rank": 512,
13
+ "qk_nope_head_dim": 128,
14
+ "qk_rope_head_dim": 64,
15
+ "v_head_dim": 128,
16
+ "mla_use_output_gate": false,
17
+ "vocab_size": 163840,
18
+ "draft_vocab_size": 163840,
19
+ "rms_norm_eps": 1e-05,
20
+ "max_position_embeddings": 1048576,
21
+ "rope_theta": 1000000.0,
22
+ "num_target_layers": 3,
23
+ "target_hidden_size": 7168,
24
+ "target_num_hidden_layers": 61,
25
+ "target_layer_ids": [
26
+ 1,
27
+ 29,
28
+ 57
29
+ ],
30
+ "fc_norm": false,
31
+ "mask_token_id": 163837,
32
+ "bos_token_id": 163584,
33
+ "eos_token_id": 163586,
34
+ "pad_token_id": 163839,
35
+ "markov_rank": 256,
36
+ "markov_head_type": "vanilla",
37
+ "enable_confidence_head": true,
38
+ "confidence_head_with_markov": true,
39
+ "sample_from_anchor": true,
40
+ "block_size": 3,
41
+ "tie_word_embeddings": false,
42
+ "_torchspec_version": "0.1.0",
43
+ "torch_dtype": "bfloat16",
44
+ "rope_parameters": {
45
+ "rope_theta": 1000000.0,
46
+ "rope_type": "default"
47
+ }
48
+ }
g3c/model/.metadata ADDED
Binary file (75.4 kB). View file
 
g3c/model/__0_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05970a74b62ed2123677c598376bbc77bed2870dee0eb0afd75c9f751a9723b5
3
+ size 726320908
g3c/model/__1_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7597f0e603ea32f4ea5239ed21f928204d444275529bba404f9b3eb5fef74436
3
+ size 726302906
g3c/model/__2_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75c5f6f6ab8f5d977cfe3ca5a71fbcaaa201fa3051ab51f0d3abc5fe7ed62339
3
+ size 726304419
g3c/model/__3_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff52b873a8e2247108d1baa009c3686110ed1fcc38c0cc18a11a7eaa6214bb87
3
+ size 726304419
g3c/train_config.yaml ADDED
@@ -0,0 +1,220 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ chat_template: null
3
+ defer_tokenization: false
4
+ eval_data_path: null
5
+ eval_interval: 500
6
+ eval_micro_batch_size: null
7
+ eval_prompt_key: conversations
8
+ last_turn_loss_only: true
9
+ length_group_size: 1
10
+ min_loss_tokens: 16
11
+ prompt_key: conversations
12
+ renderer: kimi-k3
13
+ shuffle_dataset: true
14
+ train_data_path: /data/corpus/v3ctl_allturn_train.parquet
15
+ debug:
16
+ enable_perf_metrics: true
17
+ max_dump_steps: 5
18
+ memory_recorder: torch
19
+ memory_snapshot_dir: .
20
+ memory_snapshot_num_steps: null
21
+ memory_snapshot_path: ''
22
+ profile_dir_name: /tmp/torchspec_profiles
23
+ profile_step_end: 0
24
+ profile_step_start: 0
25
+ profile_target:
26
+ - train_overall
27
+ record_memory_history: false
28
+ save_debug_train_data: null
29
+ use_pytorch_profiler: false
30
+ decode:
31
+ cuda_graph_max_bs: null
32
+ max_new_tokens: 512
33
+ min_new_tokens: 2
34
+ stop_token_ids: null
35
+ max_running_requests: null
36
+ speculative_algorithm: null
37
+ speculative_draft_model_path: null
38
+ speculative_eagle_topk: null
39
+ speculative_num_draft_tokens: null
40
+ speculative_num_steps: null
41
+ temperature: 1.0
42
+ top_k: -1
43
+ top_p: 1.0
44
+ weight_sync_enabled: false
45
+ weight_sync_interval: 500
46
+ inference:
47
+ aux_hidden_states_layers:
48
+ - 1
49
+ - 29
50
+ - 57
51
+ inference_batch_size: 4
52
+ inference_buffer_threshold: 8
53
+ inference_engine_type: vllm
54
+ inference_fetch_batch: 8
55
+ inference_num_gpus: 16
56
+ inference_num_gpus_per_engine: 4
57
+ inference_num_gpus_per_node: 4
58
+ inference_node_ips:
59
+ - 10.0.3.33
60
+ - 10.0.3.34
61
+ - 10.0.3.60
62
+ - 10.0.3.61
63
+ inference_node_selectors: null
64
+ last_hidden_states_prenorm: true
65
+ max_sample_pool_size: 64
66
+ store_last_hidden_states: true
67
+ offline:
68
+ data_path: null
69
+ num_engines: 1
70
+ sglang:
71
+ tp_size: 8
72
+ pp_size: 1
73
+ nnodes: 1
74
+ mem_fraction_static: 0.8
75
+ enable_metrics: false
76
+ enable_multimodal: false
77
+ dist_init_addr: null
78
+ dist_timeout: 60
79
+ init_timeout: 300
80
+ extra_args: {}
81
+ vllm:
82
+ tp_size: 4
83
+ pp_size: 1
84
+ nnodes: 1
85
+ mem_fraction_static: 0.85
86
+ enable_metrics: false
87
+ enable_multimodal: false
88
+ dist_init_addr: null
89
+ dist_timeout: 1800
90
+ init_timeout: 3600
91
+ extra_args:
92
+ max_model_len: 65536
93
+ max_num_batched_tokens: 32768
94
+ enable_chunked_prefill: true
95
+ enforce_eager: true
96
+ block_size: 64
97
+ kv_cache_dtype: fp8_ds_mla
98
+ tokenizer_mode: kimi_k3
99
+ kv_cache_memory_bytes: 17179869184
100
+ async_scheduling: true
101
+ load_format: fastsafetensors
102
+ trtllm:
103
+ tp_size: 8
104
+ pp_size: 1
105
+ nnodes: 1
106
+ mem_fraction_static: 0.8
107
+ init_timeout: 600
108
+ extra_args: {}
109
+ tokenspeed:
110
+ tp_size: 1
111
+ nnodes: 1
112
+ mem_fraction_static: 0.8
113
+ init_timeout: 600
114
+ extra_args: {}
115
+ logging:
116
+ report_to: none
117
+ use_tensorboard: false
118
+ use_wandb: false
119
+ wandb_dir: null
120
+ wandb_group: null
121
+ wandb_host: null
122
+ wandb_key: null
123
+ wandb_mode: online
124
+ wandb_project: train_kimi_k3_flash_dspark
125
+ wandb_random_suffix: true
126
+ wandb_run_id: null
127
+ wandb_team: null
128
+ model:
129
+ draft_model_config: configs/draft_models/kimi_k3_flash_dspark_mla_g3c.json
130
+ initial_draft_model_path: null
131
+ keep_initial_vocab_mapping: false
132
+ embedding_key: language_model.model.embed_tokens.weight
133
+ freeze_lm_head: false
134
+ lm_head_key: language_model.lm_head.weight
135
+ norm_key: language_model.model.norm.weight
136
+ target_model_backend: vllm
137
+ target_model_path: /data/models/moonshotai/Kimi-K3-Flash
138
+ trust_remote_code: true
139
+ mooncake:
140
+ master_server_address: 10.0.3.101:51661
141
+ metadata_server: null
142
+ metadata_port: 8653
143
+ protocol: rdma
144
+ device_name: mlx5_bond_0
145
+ enable_gpu_direct: false
146
+ hidden_dim: 7168
147
+ enable_hard_pin: true
148
+ kv_lease_ttl_s: 180.0
149
+ get_retry_max_wait_seconds: 180.0
150
+ global_segment_size: 128GB
151
+ local_buffer_size: 16GB
152
+ max_seq_len: 65536
153
+ training:
154
+ attention_backend: flex_attention
155
+ colocate: false
156
+ continual_training: false
157
+ distributed_backend: nccl
158
+ distributed_timeout_minutes: 60
159
+ draft_accumulation_steps: 16
160
+ fsdp_reduce_dtype: float32
161
+ fsdp_strategy: FULL_SHARD
162
+ placement_strategy: custom
163
+ training_node_ips:
164
+ - 10.0.3.101
165
+ - 10.0.3.31
166
+ training_node_selectors: null
167
+ compile_model: false
168
+ sp_ring_size: 1
169
+ sp_ulysses_size: 1
170
+ gradient_checkpointing: true
171
+ learning_rate: 5.0e-05
172
+ lk_eta: 3.0
173
+ load_path: /data/output/ckpts/checkpoints
174
+ loss_type: forward_kl
175
+ lr_decay_style: cosine
176
+ lr_wsd_decay_ratio: 0.2
177
+ lr_wsd_decay_style: cosine
178
+ lr_total_steps: 6412
179
+ max_concurrent_batches: 1
180
+ max_grad_norm: 1.0
181
+ max_seq_length: 65536
182
+ min_lr: 0.0
183
+ weight_decay: 0.0
184
+ num_epochs: 2
185
+ num_train_steps: null
186
+ micro_batch_size: 1
187
+ prefetch_depth: 2
188
+ save_interval: 250
189
+ save_per_epoch: true
190
+ max_checkpoints: 3
191
+ seed: 42
192
+ train_backend: fsdp
193
+ train_env_vars: '{}'
194
+ train_with_decode: false
195
+ training_num_gpus_per_node: 4
196
+ training_num_nodes: 2
197
+ ttt_length: 7
198
+ ploss_weights: null
199
+ warmup_ratio: 0.04
200
+ wsd_decay_ratio: 0.2
201
+ wsd_decay_style: null
202
+ dflash_block_size: 3
203
+ dflash_dpace_alpha: 0.5
204
+ dflash_loss_decay_gamma: 7.0
205
+ dflash_loss_objective: decay
206
+ dflash_ce_loss_alpha: 1.0
207
+ dflash_l1_loss_alpha: 0.0
208
+ dflash_num_anchors: 512
209
+ dflash_num_target_layers: 5
210
+ dflash2_selector_loss_alpha: 1.0
211
+ dspark_num_anchors: 512
212
+ dspark_num_target_layers: 3
213
+ dspark_loss_decay_gamma: 4.0
214
+ dspark_ce_loss_alpha: 0.1
215
+ dspark_l1_loss_alpha: 0.9
216
+ dspark_confidence_head_alpha: 1.0
217
+ cache_dir: /data/output/cache
218
+ cache_key: null
219
+ model_download_dir: null
220
+ output_dir: /data/output/ckpts