g3c: final checkpoint + run config
Browse files- .gitattributes +4 -0
- g3c/FINAL_ITERATION +1 -0
- g3c/README.md +51 -0
- g3c/ckpt_meta.json +10 -0
- g3c/draft_model_config.json +48 -0
- g3c/model/.metadata +0 -0
- g3c/model/__0_0.distcp +3 -0
- g3c/model/__1_0.distcp +3 -0
- g3c/model/__2_0.distcp +3 -0
- g3c/model/__3_0.distcp +3 -0
- g3c/train_config.yaml +220 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
g3c/model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
g3c/model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
g3c/model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
g3c/model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
|
g3c/FINAL_ITERATION
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
6001
|
g3c/README.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# g3c — ALL-TURN corpus control.
|
| 2 |
+
|
| 3 |
+
**Final checkpoint: iteration 6001** (from `latest_checkpointed_iteration.txt`).
|
| 4 |
+
|
| 5 |
+
## The variable
|
| 6 |
+
|
| 7 |
+
Trains on the all-turn corpus (`v3ctl_allturn_train.parquet`) instead of the v3 blend that every later arm uses. It is the only g3 arm whose corpus differs, so it is NOT single-variable against g3d.
|
| 8 |
+
|
| 9 |
+
## Architecture / run settings (read from this run's own saved config)
|
| 10 |
+
|
| 11 |
+
| key | value |
|
| 12 |
+
|---|---|
|
| 13 |
+
| `fc_norm` | `false` |
|
| 14 |
+
| `block_size` | 3 |
|
| 15 |
+
| `target_layer_ids` | [1, 29, 57] |
|
| 16 |
+
| `inference_num_gpus` | 16 |
|
| 17 |
+
| `train_data_path` | `/data/corpus/v3ctl_allturn_train.parquet` |
|
| 18 |
+
| draft config | `kimi_k3_flash_dspark_mla_g3c.json` |
|
| 19 |
+
|
| 20 |
+
- target: `moonshotai/Kimi-K3-Flash` (61 text layers)
|
| 21 |
+
- draft: 3 layers, MLA, `enable_confidence_head: true`, `markov_head_type: vanilla`
|
| 22 |
+
- `max_seq_length: 65536`, `last_turn_loss_only: true`, `micro_batch_size: 1`
|
| 23 |
+
- cluster: GB300 (aarch64, 4 GPUs/node), 2x4 trainer ranks + producer engines
|
| 24 |
+
- `num_train_steps: null` — the schedule is driven by the corpus, not a step cap
|
| 25 |
+
|
| 26 |
+
## Files
|
| 27 |
+
|
| 28 |
+
| file | what it is |
|
| 29 |
+
|---|---|
|
| 30 |
+
| `model/` | the final iteration's distcp weight shards — **this is the checkpoint** |
|
| 31 |
+
| `draft_model_config.json` | draft architecture; the file the trainer actually pointed at |
|
| 32 |
+
| `train_config.yaml` | the full run config as saved next to the checkpoints |
|
| 33 |
+
| `ckpt_meta.json` | trainer metadata (`global_step`, `world_size`, timestamp) |
|
| 34 |
+
| `FINAL_ITERATION` | the iteration number, for scripted selection |
|
| 35 |
+
|
| 36 |
+
`optimizer/` is **not** included: it is ~80% of the on-disk bytes and is only needed
|
| 37 |
+
to resume training. For export + evaluation the `model/` shards are sufficient.
|
| 38 |
+
|
| 39 |
+
## Converting to a servable HF checkpoint
|
| 40 |
+
|
| 41 |
+
These are TorchSpec distcp shards, not an HF folder. Convert with the repo's own
|
| 42 |
+
exporter (`arthur/scripts/export_k3flash_dspark_vllm.sh`, which wraps TorchSpec's
|
| 43 |
+
`tools/convert_to_hf.py --vllm`), then serve the result as the draft in
|
| 44 |
+
`--speculative-config`. Two gates worth keeping, both of which fail silently if
|
| 45 |
+
skipped:
|
| 46 |
+
|
| 47 |
+
1. **`num_speculative_tokens` must equal this checkpoint's `block_size` (3).**
|
| 48 |
+
A mismatch drafts a different width and reads as low acceptance, not as an error.
|
| 49 |
+
2. **The target's depth must match `target_num_hidden_layers: 61`.** Pointed at a
|
| 50 |
+
different-depth target, vLLM RESCALES the aux capture ids and logs only a WARNING,
|
| 51 |
+
after which the draft reads layers it never trained on.
|
g3c/ckpt_meta.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"global_step": 6000,
|
| 3 |
+
"inference_id": 6000,
|
| 4 |
+
"iteration": 6001,
|
| 5 |
+
"next_inference_id": 6001,
|
| 6 |
+
"next_step": 6001,
|
| 7 |
+
"step": 6000,
|
| 8 |
+
"timestamp": 1790867069.8654246,
|
| 9 |
+
"world_size": 8
|
| 10 |
+
}
|
g3c/draft_model_config.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"K3DSparkModel"
|
| 4 |
+
],
|
| 5 |
+
"model_type": "k3_dspark",
|
| 6 |
+
"hidden_size": 7168,
|
| 7 |
+
"intermediate_size": 18432,
|
| 8 |
+
"num_hidden_layers": 3,
|
| 9 |
+
"num_attention_heads": 64,
|
| 10 |
+
"num_key_value_heads": 64,
|
| 11 |
+
"q_lora_rank": 1536,
|
| 12 |
+
"kv_lora_rank": 512,
|
| 13 |
+
"qk_nope_head_dim": 128,
|
| 14 |
+
"qk_rope_head_dim": 64,
|
| 15 |
+
"v_head_dim": 128,
|
| 16 |
+
"mla_use_output_gate": false,
|
| 17 |
+
"vocab_size": 163840,
|
| 18 |
+
"draft_vocab_size": 163840,
|
| 19 |
+
"rms_norm_eps": 1e-05,
|
| 20 |
+
"max_position_embeddings": 1048576,
|
| 21 |
+
"rope_theta": 1000000.0,
|
| 22 |
+
"num_target_layers": 3,
|
| 23 |
+
"target_hidden_size": 7168,
|
| 24 |
+
"target_num_hidden_layers": 61,
|
| 25 |
+
"target_layer_ids": [
|
| 26 |
+
1,
|
| 27 |
+
29,
|
| 28 |
+
57
|
| 29 |
+
],
|
| 30 |
+
"fc_norm": false,
|
| 31 |
+
"mask_token_id": 163837,
|
| 32 |
+
"bos_token_id": 163584,
|
| 33 |
+
"eos_token_id": 163586,
|
| 34 |
+
"pad_token_id": 163839,
|
| 35 |
+
"markov_rank": 256,
|
| 36 |
+
"markov_head_type": "vanilla",
|
| 37 |
+
"enable_confidence_head": true,
|
| 38 |
+
"confidence_head_with_markov": true,
|
| 39 |
+
"sample_from_anchor": true,
|
| 40 |
+
"block_size": 3,
|
| 41 |
+
"tie_word_embeddings": false,
|
| 42 |
+
"_torchspec_version": "0.1.0",
|
| 43 |
+
"torch_dtype": "bfloat16",
|
| 44 |
+
"rope_parameters": {
|
| 45 |
+
"rope_theta": 1000000.0,
|
| 46 |
+
"rope_type": "default"
|
| 47 |
+
}
|
| 48 |
+
}
|
g3c/model/.metadata
ADDED
|
Binary file (75.4 kB). View file
|
|
|
g3c/model/__0_0.distcp
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05970a74b62ed2123677c598376bbc77bed2870dee0eb0afd75c9f751a9723b5
|
| 3 |
+
size 726320908
|
g3c/model/__1_0.distcp
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7597f0e603ea32f4ea5239ed21f928204d444275529bba404f9b3eb5fef74436
|
| 3 |
+
size 726302906
|
g3c/model/__2_0.distcp
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:75c5f6f6ab8f5d977cfe3ca5a71fbcaaa201fa3051ab51f0d3abc5fe7ed62339
|
| 3 |
+
size 726304419
|
g3c/model/__3_0.distcp
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ff52b873a8e2247108d1baa009c3686110ed1fcc38c0cc18a11a7eaa6214bb87
|
| 3 |
+
size 726304419
|
g3c/train_config.yaml
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset:
|
| 2 |
+
chat_template: null
|
| 3 |
+
defer_tokenization: false
|
| 4 |
+
eval_data_path: null
|
| 5 |
+
eval_interval: 500
|
| 6 |
+
eval_micro_batch_size: null
|
| 7 |
+
eval_prompt_key: conversations
|
| 8 |
+
last_turn_loss_only: true
|
| 9 |
+
length_group_size: 1
|
| 10 |
+
min_loss_tokens: 16
|
| 11 |
+
prompt_key: conversations
|
| 12 |
+
renderer: kimi-k3
|
| 13 |
+
shuffle_dataset: true
|
| 14 |
+
train_data_path: /data/corpus/v3ctl_allturn_train.parquet
|
| 15 |
+
debug:
|
| 16 |
+
enable_perf_metrics: true
|
| 17 |
+
max_dump_steps: 5
|
| 18 |
+
memory_recorder: torch
|
| 19 |
+
memory_snapshot_dir: .
|
| 20 |
+
memory_snapshot_num_steps: null
|
| 21 |
+
memory_snapshot_path: ''
|
| 22 |
+
profile_dir_name: /tmp/torchspec_profiles
|
| 23 |
+
profile_step_end: 0
|
| 24 |
+
profile_step_start: 0
|
| 25 |
+
profile_target:
|
| 26 |
+
- train_overall
|
| 27 |
+
record_memory_history: false
|
| 28 |
+
save_debug_train_data: null
|
| 29 |
+
use_pytorch_profiler: false
|
| 30 |
+
decode:
|
| 31 |
+
cuda_graph_max_bs: null
|
| 32 |
+
max_new_tokens: 512
|
| 33 |
+
min_new_tokens: 2
|
| 34 |
+
stop_token_ids: null
|
| 35 |
+
max_running_requests: null
|
| 36 |
+
speculative_algorithm: null
|
| 37 |
+
speculative_draft_model_path: null
|
| 38 |
+
speculative_eagle_topk: null
|
| 39 |
+
speculative_num_draft_tokens: null
|
| 40 |
+
speculative_num_steps: null
|
| 41 |
+
temperature: 1.0
|
| 42 |
+
top_k: -1
|
| 43 |
+
top_p: 1.0
|
| 44 |
+
weight_sync_enabled: false
|
| 45 |
+
weight_sync_interval: 500
|
| 46 |
+
inference:
|
| 47 |
+
aux_hidden_states_layers:
|
| 48 |
+
- 1
|
| 49 |
+
- 29
|
| 50 |
+
- 57
|
| 51 |
+
inference_batch_size: 4
|
| 52 |
+
inference_buffer_threshold: 8
|
| 53 |
+
inference_engine_type: vllm
|
| 54 |
+
inference_fetch_batch: 8
|
| 55 |
+
inference_num_gpus: 16
|
| 56 |
+
inference_num_gpus_per_engine: 4
|
| 57 |
+
inference_num_gpus_per_node: 4
|
| 58 |
+
inference_node_ips:
|
| 59 |
+
- 10.0.3.33
|
| 60 |
+
- 10.0.3.34
|
| 61 |
+
- 10.0.3.60
|
| 62 |
+
- 10.0.3.61
|
| 63 |
+
inference_node_selectors: null
|
| 64 |
+
last_hidden_states_prenorm: true
|
| 65 |
+
max_sample_pool_size: 64
|
| 66 |
+
store_last_hidden_states: true
|
| 67 |
+
offline:
|
| 68 |
+
data_path: null
|
| 69 |
+
num_engines: 1
|
| 70 |
+
sglang:
|
| 71 |
+
tp_size: 8
|
| 72 |
+
pp_size: 1
|
| 73 |
+
nnodes: 1
|
| 74 |
+
mem_fraction_static: 0.8
|
| 75 |
+
enable_metrics: false
|
| 76 |
+
enable_multimodal: false
|
| 77 |
+
dist_init_addr: null
|
| 78 |
+
dist_timeout: 60
|
| 79 |
+
init_timeout: 300
|
| 80 |
+
extra_args: {}
|
| 81 |
+
vllm:
|
| 82 |
+
tp_size: 4
|
| 83 |
+
pp_size: 1
|
| 84 |
+
nnodes: 1
|
| 85 |
+
mem_fraction_static: 0.85
|
| 86 |
+
enable_metrics: false
|
| 87 |
+
enable_multimodal: false
|
| 88 |
+
dist_init_addr: null
|
| 89 |
+
dist_timeout: 1800
|
| 90 |
+
init_timeout: 3600
|
| 91 |
+
extra_args:
|
| 92 |
+
max_model_len: 65536
|
| 93 |
+
max_num_batched_tokens: 32768
|
| 94 |
+
enable_chunked_prefill: true
|
| 95 |
+
enforce_eager: true
|
| 96 |
+
block_size: 64
|
| 97 |
+
kv_cache_dtype: fp8_ds_mla
|
| 98 |
+
tokenizer_mode: kimi_k3
|
| 99 |
+
kv_cache_memory_bytes: 17179869184
|
| 100 |
+
async_scheduling: true
|
| 101 |
+
load_format: fastsafetensors
|
| 102 |
+
trtllm:
|
| 103 |
+
tp_size: 8
|
| 104 |
+
pp_size: 1
|
| 105 |
+
nnodes: 1
|
| 106 |
+
mem_fraction_static: 0.8
|
| 107 |
+
init_timeout: 600
|
| 108 |
+
extra_args: {}
|
| 109 |
+
tokenspeed:
|
| 110 |
+
tp_size: 1
|
| 111 |
+
nnodes: 1
|
| 112 |
+
mem_fraction_static: 0.8
|
| 113 |
+
init_timeout: 600
|
| 114 |
+
extra_args: {}
|
| 115 |
+
logging:
|
| 116 |
+
report_to: none
|
| 117 |
+
use_tensorboard: false
|
| 118 |
+
use_wandb: false
|
| 119 |
+
wandb_dir: null
|
| 120 |
+
wandb_group: null
|
| 121 |
+
wandb_host: null
|
| 122 |
+
wandb_key: null
|
| 123 |
+
wandb_mode: online
|
| 124 |
+
wandb_project: train_kimi_k3_flash_dspark
|
| 125 |
+
wandb_random_suffix: true
|
| 126 |
+
wandb_run_id: null
|
| 127 |
+
wandb_team: null
|
| 128 |
+
model:
|
| 129 |
+
draft_model_config: configs/draft_models/kimi_k3_flash_dspark_mla_g3c.json
|
| 130 |
+
initial_draft_model_path: null
|
| 131 |
+
keep_initial_vocab_mapping: false
|
| 132 |
+
embedding_key: language_model.model.embed_tokens.weight
|
| 133 |
+
freeze_lm_head: false
|
| 134 |
+
lm_head_key: language_model.lm_head.weight
|
| 135 |
+
norm_key: language_model.model.norm.weight
|
| 136 |
+
target_model_backend: vllm
|
| 137 |
+
target_model_path: /data/models/moonshotai/Kimi-K3-Flash
|
| 138 |
+
trust_remote_code: true
|
| 139 |
+
mooncake:
|
| 140 |
+
master_server_address: 10.0.3.101:51661
|
| 141 |
+
metadata_server: null
|
| 142 |
+
metadata_port: 8653
|
| 143 |
+
protocol: rdma
|
| 144 |
+
device_name: mlx5_bond_0
|
| 145 |
+
enable_gpu_direct: false
|
| 146 |
+
hidden_dim: 7168
|
| 147 |
+
enable_hard_pin: true
|
| 148 |
+
kv_lease_ttl_s: 180.0
|
| 149 |
+
get_retry_max_wait_seconds: 180.0
|
| 150 |
+
global_segment_size: 128GB
|
| 151 |
+
local_buffer_size: 16GB
|
| 152 |
+
max_seq_len: 65536
|
| 153 |
+
training:
|
| 154 |
+
attention_backend: flex_attention
|
| 155 |
+
colocate: false
|
| 156 |
+
continual_training: false
|
| 157 |
+
distributed_backend: nccl
|
| 158 |
+
distributed_timeout_minutes: 60
|
| 159 |
+
draft_accumulation_steps: 16
|
| 160 |
+
fsdp_reduce_dtype: float32
|
| 161 |
+
fsdp_strategy: FULL_SHARD
|
| 162 |
+
placement_strategy: custom
|
| 163 |
+
training_node_ips:
|
| 164 |
+
- 10.0.3.101
|
| 165 |
+
- 10.0.3.31
|
| 166 |
+
training_node_selectors: null
|
| 167 |
+
compile_model: false
|
| 168 |
+
sp_ring_size: 1
|
| 169 |
+
sp_ulysses_size: 1
|
| 170 |
+
gradient_checkpointing: true
|
| 171 |
+
learning_rate: 5.0e-05
|
| 172 |
+
lk_eta: 3.0
|
| 173 |
+
load_path: /data/output/ckpts/checkpoints
|
| 174 |
+
loss_type: forward_kl
|
| 175 |
+
lr_decay_style: cosine
|
| 176 |
+
lr_wsd_decay_ratio: 0.2
|
| 177 |
+
lr_wsd_decay_style: cosine
|
| 178 |
+
lr_total_steps: 6412
|
| 179 |
+
max_concurrent_batches: 1
|
| 180 |
+
max_grad_norm: 1.0
|
| 181 |
+
max_seq_length: 65536
|
| 182 |
+
min_lr: 0.0
|
| 183 |
+
weight_decay: 0.0
|
| 184 |
+
num_epochs: 2
|
| 185 |
+
num_train_steps: null
|
| 186 |
+
micro_batch_size: 1
|
| 187 |
+
prefetch_depth: 2
|
| 188 |
+
save_interval: 250
|
| 189 |
+
save_per_epoch: true
|
| 190 |
+
max_checkpoints: 3
|
| 191 |
+
seed: 42
|
| 192 |
+
train_backend: fsdp
|
| 193 |
+
train_env_vars: '{}'
|
| 194 |
+
train_with_decode: false
|
| 195 |
+
training_num_gpus_per_node: 4
|
| 196 |
+
training_num_nodes: 2
|
| 197 |
+
ttt_length: 7
|
| 198 |
+
ploss_weights: null
|
| 199 |
+
warmup_ratio: 0.04
|
| 200 |
+
wsd_decay_ratio: 0.2
|
| 201 |
+
wsd_decay_style: null
|
| 202 |
+
dflash_block_size: 3
|
| 203 |
+
dflash_dpace_alpha: 0.5
|
| 204 |
+
dflash_loss_decay_gamma: 7.0
|
| 205 |
+
dflash_loss_objective: decay
|
| 206 |
+
dflash_ce_loss_alpha: 1.0
|
| 207 |
+
dflash_l1_loss_alpha: 0.0
|
| 208 |
+
dflash_num_anchors: 512
|
| 209 |
+
dflash_num_target_layers: 5
|
| 210 |
+
dflash2_selector_loss_alpha: 1.0
|
| 211 |
+
dspark_num_anchors: 512
|
| 212 |
+
dspark_num_target_layers: 3
|
| 213 |
+
dspark_loss_decay_gamma: 4.0
|
| 214 |
+
dspark_ce_loss_alpha: 0.1
|
| 215 |
+
dspark_l1_loss_alpha: 0.9
|
| 216 |
+
dspark_confidence_head_alpha: 1.0
|
| 217 |
+
cache_dir: /data/output/cache
|
| 218 |
+
cache_key: null
|
| 219 |
+
model_download_dir: null
|
| 220 |
+
output_dir: /data/output/ckpts
|