yiqichen01 commited on
Commit
58169cd
·
verified ·
1 Parent(s): 1b24b13
Files changed (43) hide show
  1. .gitattributes +4 -0
  2. LICENSE +245 -0
  3. README.md +314 -0
  4. README_CN.md +317 -0
  5. assets/benchmark.png +3 -0
  6. assets/blind-evaluation-en.png +0 -0
  7. assets/blind-evaluation-zh.png +0 -0
  8. assets/dfly2.png +3 -0
  9. assets/logo-en.png +0 -0
  10. assets/logo-zh.png +0 -0
  11. assets/performance-cost-open-benchmarks.png +3 -0
  12. assets/performance-cost-workindex.png +3 -0
  13. assets/workindex-cost.png +0 -0
  14. finetune/README.md +584 -0
  15. finetune/README_CN.md +584 -0
  16. finetune/data/example_data.jsonl +8 -0
  17. finetune/data/example_vl_data.jsonl +4 -0
  18. finetune/deepspeed_support/ds_zero2_no_offload.json +28 -0
  19. finetune/deepspeed_support/ds_zero2_offload.json +34 -0
  20. finetune/deepspeed_support/ds_zero3_no_offload.json +41 -0
  21. finetune/deepspeed_support/ds_zero3_offload.json +41 -0
  22. finetune/deepspeed_support/train_llm.sh +114 -0
  23. finetune/deepspeed_support/train_vl.py +872 -0
  24. finetune/deepspeed_support/train_vl.sh +111 -0
  25. finetune/llama_factory_support/dataset_info.json +32 -0
  26. finetune/llama_factory_support/hy_v4_flash_full_sft.yaml +56 -0
  27. finetune/llama_factory_support/hy_v4_flash_lora_sft.yaml +69 -0
  28. finetune/llama_factory_support/hy_v4_flash_vl_full_sft.yaml +54 -0
  29. finetune/llama_factory_support/hy_v4_flash_vl_lora_sft.yaml +66 -0
  30. finetune/llama_factory_support/hy_v4_patches.py +732 -0
  31. finetune/llama_factory_support/hy_v4_template.py +151 -0
  32. finetune/llama_factory_support/train_hy_v4.py +214 -0
  33. finetune/llama_factory_support/train_lf.sh +166 -0
  34. finetune/llama_factory_support/train_lf_llm.sh +166 -0
  35. finetune/llama_factory_support/train_lf_lora.sh +166 -0
  36. finetune/llama_factory_support/train_lf_lora_llm.sh +166 -0
  37. finetune/ms_swift_support/fsdp_hy_v4_flash.json +20 -0
  38. finetune/ms_swift_support/hy_v4_flash_swift_patches.py +595 -0
  39. finetune/ms_swift_support/sft_train.sh +195 -0
  40. finetune/ms_swift_support/sft_train_lora.sh +188 -0
  41. finetune/ms_swift_support/sft_train_vl.sh +200 -0
  42. finetune/ms_swift_support/sft_train_vl_lora.sh +205 -0
  43. finetune/requirements.txt +11 -0
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ assets/benchmark.png filter=lfs diff=lfs merge=lfs -text
37
+ assets/dfly2.png filter=lfs diff=lfs merge=lfs -text
38
+ assets/performance-cost-open-benchmarks.png filter=lfs diff=lfs merge=lfs -text
39
+ assets/performance-cost-workindex.png filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Tencent Hy License, Version 1.0
2
+
3
+ This License is derived from the Apache License, Version 2.0
4
+ (http://www.apache.org/licenses/LICENSE-2.0), with modifications, mainly
5
+ the additional commercial terms for large Model-as-a-Service ("MaaS")
6
+ businesses in Section 10. This License is not the Apache License,
7
+ Version 2.0, and is not endorsed by or affiliated with the Apache
8
+ Software Foundation.
9
+
10
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
11
+
12
+ 1. Definitions.
13
+
14
+ "License" shall mean the terms and conditions for use, reproduction,
15
+ and distribution as defined by Sections 1 through 10 of this document.
16
+
17
+ "Licensor" shall mean the copyright owner or entity authorized by
18
+ the copyright owner that is granting the License.
19
+
20
+ "Legal Entity" shall mean the union of the acting entity and all
21
+ other entities that control, are controlled by, or are under common
22
+ control with that entity. For the purposes of this definition,
23
+ "control" means (i) the power, direct or indirect, to cause the
24
+ direction or management of such entity, whether by contract or
25
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
26
+ outstanding shares, or (iii) beneficial ownership of such entity.
27
+
28
+ "You" (or "Your") shall mean an individual or Legal Entity
29
+ exercising permissions granted by this License.
30
+
31
+ "Source" form shall mean the preferred form for making modifications,
32
+ including but not limited to software source code, documentation
33
+ source, and configuration files.
34
+
35
+ "Object" form shall mean any form resulting from mechanical
36
+ transformation or translation of a Source form, including but
37
+ not limited to compiled object code, generated documentation,
38
+ and conversions to other media types.
39
+
40
+ "Work" shall mean the work of authorship, whether in Source or
41
+ Object form, made available under the License, as indicated by a
42
+ copyright notice that is included in or attached to the work
43
+ (an example is provided in the Appendix below). For the avoidance
44
+ of doubt, the Work includes the model weights, parameters,
45
+ configuration files, inference and training code, and documentation
46
+ made available by Licensor under this License.
47
+
48
+ "Derivative Works" shall mean any work, whether in Source or Object
49
+ form, that is based on (or derived from) the Work and for which the
50
+ editorial revisions, annotations, elaborations, or other modifications
51
+ represent, as a whole, an original work of authorship. For the purposes
52
+ of this License, Derivative Works shall not include works that remain
53
+ separable from, or merely link (or bind by name) to the interfaces of,
54
+ the Work and Derivative Works thereof. For the avoidance of doubt,
55
+ Derivative Works include any fine-tuned, quantized, distilled, merged,
56
+ pruned, or otherwise modified versions of the model weights.
57
+
58
+ "Contribution" shall mean any work of authorship, including
59
+ the original version of the Work and any modifications or additions
60
+ to that Work or Derivative Works thereof, that is intentionally
61
+ submitted to Licensor for inclusion in the Work by the copyright owner
62
+ or by an individual or Legal Entity authorized to submit on behalf of
63
+ the copyright owner. For the purposes of this definition, "submitted"
64
+ means any form of electronic, verbal, or written communication sent
65
+ to the Licensor or its representatives, including but not limited to
66
+ communication on electronic mailing lists, source code control systems,
67
+ and issue tracking systems that are managed by, or on behalf of, the
68
+ Licensor for the purpose of discussing and improving the Work, but
69
+ excluding communication that is conspicuously marked or otherwise
70
+ designated in writing by the copyright owner as "Not a Contribution."
71
+
72
+ "Contributor" shall mean Licensor and any individual or Legal Entity
73
+ on behalf of whom a Contribution has been received by Licensor and
74
+ subsequently incorporated within the Work.
75
+
76
+ 2. Grant of Copyright License. Subject to the terms and conditions of
77
+ this License, each Contributor hereby grants to You a perpetual,
78
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
79
+ copyright license to reproduce, prepare Derivative Works of,
80
+ publicly display, publicly perform, sublicense, and distribute the
81
+ Work and such Derivative Works in Source or Object form.
82
+
83
+ 3. Grant of Patent License. Subject to the terms and conditions of
84
+ this License, each Contributor hereby grants to You a perpetual,
85
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
86
+ (except as stated in this section) patent license to make, have made,
87
+ use, offer to sell, sell, import, and otherwise transfer the Work,
88
+ where such license applies only to those patent claims licensable
89
+ by such Contributor that are necessarily infringed by their
90
+ Contribution(s) alone or by combination of their Contribution(s)
91
+ with the Work to which such Contribution(s) was submitted. If You
92
+ institute patent litigation against any entity (including a
93
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
94
+ or a Contribution incorporated within the Work constitutes direct
95
+ or contributory patent infringement, then any patent licenses
96
+ granted to You under this License for that Work shall terminate
97
+ as of the date such litigation is filed.
98
+
99
+ 4. Redistribution. You may reproduce and distribute copies of the
100
+ Work or Derivative Works thereof in any medium, with or without
101
+ modifications, and in Source or Object form, provided that You
102
+ meet the following conditions:
103
+
104
+ (a) You must give any other recipients of the Work or
105
+ Derivative Works a copy of this License; and
106
+
107
+ (b) You must cause any modified files to carry prominent notices
108
+ stating that You changed the files; and
109
+
110
+ (c) You must retain, in the Source form of any Derivative Works
111
+ that You distribute, all copyright, patent, trademark, and
112
+ attribution notices from the Source form of the Work,
113
+ excluding those notices that do not pertain to any part of
114
+ the Derivative Works; and
115
+
116
+ (d) If the Work includes a "NOTICE" text file as part of its
117
+ distribution, then any Derivative Works that You distribute must
118
+ include a readable copy of the attribution notices contained
119
+ within such NOTICE file, excluding those notices that do not
120
+ pertain to any part of the Derivative Works, in at least one
121
+ of the following places: within a NOTICE text file distributed
122
+ as part of the Derivative Works; within the Source form or
123
+ documentation, if provided along with the Derivative Works; or,
124
+ within a display generated by the Derivative Works, if and
125
+ wherever such third-party notices normally appear. The contents
126
+ of the NOTICE file are for informational purposes only and
127
+ do not modify the License. You may add Your own attribution
128
+ notices within Derivative Works that You distribute, alongside
129
+ or as an addendum to the NOTICE text from the Work, provided
130
+ that such additional attribution notices cannot be construed
131
+ as modifying the License.
132
+
133
+ You may add Your own copyright statement to Your modifications and
134
+ may provide additional or different license terms and conditions
135
+ for use, reproduction, or distribution of Your modifications, or
136
+ for any such Derivative Works as a whole, provided Your use,
137
+ reproduction, and distribution of the Work otherwise complies with
138
+ the conditions stated in this License. Notwithstanding the
139
+ foregoing, any license terms You provide for the Work or any
140
+ Derivative Works must not remove, limit, or conflict with Section 10,
141
+ and Section 10 shall continue to apply to every recipient of the
142
+ Work or any Derivative Works.
143
+
144
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
145
+ any Contribution intentionally submitted for inclusion in the Work
146
+ by You to the Licensor shall be under the terms and conditions of
147
+ this License, without any additional terms or conditions.
148
+ Notwithstanding the above, nothing herein shall supersede or modify
149
+ the terms of any separate license agreement you may have executed
150
+ with Licensor regarding such Contributions.
151
+
152
+ 6. Trademarks. This License does not grant permission to use the trade
153
+ names, trademarks, service marks, or product names of the Licensor,
154
+ except as required for reasonable and customary use in describing the
155
+ origin of the Work and reproducing the content of the NOTICE file.
156
+
157
+ 7. Disclaimer of Warranty. Unless required by applicable law or
158
+ agreed to in writing, Licensor provides the Work (and each
159
+ Contributor provides its Contributions) on an "AS IS" BASIS,
160
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
161
+ implied, including, without limitation, any warranties or conditions
162
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
163
+ PARTICULAR PURPOSE. You are solely responsible for determining the
164
+ appropriateness of using or redistributing the Work and assume any
165
+ risks associated with Your exercise of permissions under this License.
166
+
167
+ 8. Limitation of Liability. In no event and under no legal theory,
168
+ whether in tort (including negligence), contract, or otherwise,
169
+ unless required by applicable law (such as deliberate and grossly
170
+ negligent acts) or agreed to in writing, shall any Contributor be
171
+ liable to You for damages, including any direct, indirect, special,
172
+ incidental, or consequential damages of any character arising as a
173
+ result of this License or out of the use or inability to use the
174
+ Work (including but not limited to damages for loss of goodwill,
175
+ work stoppage, computer failure or malfunction, or any and all
176
+ other commercial damages or losses), even if such Contributor
177
+ has been advised of the possibility of such damages.
178
+
179
+ 9. Accepting Warranty or Additional Liability. While redistributing
180
+ the Work or Derivative Works thereof, You may choose to offer,
181
+ and charge a fee for, acceptance of support, warranty, indemnity,
182
+ or other liability obligations and/or rights consistent with this
183
+ License. However, in accepting such obligations, You may act only
184
+ on Your own behalf and on Your sole responsibility, not on behalf
185
+ of any other Contributor, and only if You agree to indemnify,
186
+ defend, and hold each Contributor harmless for any liability
187
+ incurred by, or claims asserted against, such Contributor by reason
188
+ of your accepting any such warranty or additional liability.
189
+
190
+ 10. Additional Commercial Terms.
191
+
192
+ 10.1 MaaS. "MaaS" means giving third parties remote access to model
193
+ inference or fine-tuning (e.g., via API), allowing them to
194
+ submit inputs or specify parameters or training data. This
195
+ excludes (i) end-user products with model capabilities confined
196
+ to specific features or workflows, without general-purpose
197
+ model access, or (ii) merely relaying requests to an
198
+ independently operated third-party model service.
199
+
200
+ 10.2 Separate License Required. Notwithstanding Sections 2 and 3,
201
+ and except as provided in Section 10.3, if You operate a MaaS
202
+ business, You must obtain a separate written license to use the
203
+ Work or any Derivative Works from Licensor (including by email),
204
+ in either of the following cases:
205
+
206
+ (a) if Your aggregate worldwide gross revenue exceeds
207
+ US$50 million (or the equivalent in other currencies) in
208
+ any consecutive 12-month period, whether or not such
209
+ revenue is derived from the Work or any Derivative Works,
210
+ before using (or continuing to use) the Work or any
211
+ Derivative Works; or
212
+
213
+ (b) before using the Work or any Derivative Works in or for any
214
+ product or service provided by You that has more than
215
+ 100 million monthly active users in any calendar month.
216
+
217
+ 10.3 Internal Use. Section 10.2 does not apply to Your use of the
218
+ Work or any Derivative Works solely for internal purposes,
219
+ provided that neither the Work, any Derivative Works, nor their
220
+ outputs are made available to third parties.
221
+
222
+ 10.4 Contact. For licensing requests or questions, please contact
223
+ hy_partners@tencent.com.
224
+
225
+ END OF TERMS AND CONDITIONS
226
+
227
+ APPENDIX: How to apply the Tencent Hy License to your work.
228
+
229
+ To apply this License to a Derivative Work, attach the following
230
+ notice, with the fields enclosed by brackets "[]" replaced with your
231
+ own identifying information. (Don't include the brackets!)
232
+
233
+ Copyright [yyyy] [name of copyright owner]
234
+
235
+ Licensed under the Tencent Hy License, Version 1.0 (the "License"),
236
+ which is based on the Apache License, Version 2.0, with additional
237
+ commercial terms. You may not use this file except in compliance
238
+ with the License. You may obtain a copy of the License in the
239
+ LICENSE file distributed with this work.
240
+
241
+ Unless required by applicable law or agreed to in writing, software
242
+ distributed under the License is distributed on an "AS IS" BASIS,
243
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
244
+ implied. See the License for the specific language governing
245
+ permissions and limitations under the License.
README.md ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <p align="left">
2
+ <a href="README_CN.md">中文</a>&nbsp;|&nbsp;English
3
+ </p>
4
+ <br>
5
+
6
+ <p align="center">
7
+ <img src="assets/logo-en.png" alt="Tencent Hy" width="400"/>
8
+ </p>
9
+
10
+ <div align="center" style="line-height: 1;">
11
+
12
+ [![License](https://img.shields.io/badge/License-Tencent%20Hy%20License-blue)](#license)
13
+ &nbsp;&nbsp;
14
+ [![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Tencent%20Hy-ffc107?color=ffc107&logoColor=white)](https://huggingface.co/tencent/Hy4-flash)
15
+ &nbsp;&nbsp;
16
+ [![ModelScope](https://img.shields.io/badge/ModelScope-Tencent%20Hy-624aff)](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash)
17
+ &nbsp;&nbsp;
18
+ [![GitCode](https://img.shields.io/badge/GitCode-Tencent%20Hy-red?logoColor=white)](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash)
19
+
20
+ </div>
21
+
22
+ <p align="center">
23
+ <a href="https://hy.tencent.ai/research/hy4-flash"><b>Official Website</b></a>&nbsp;&nbsp;|&nbsp;&nbsp;
24
+ <a href="https://github.com/Tencent-Hunyuan/Hy4-flash"><b>GitHub</b></a>&nbsp;&nbsp;|&nbsp;&nbsp;
25
+ <a href="https://huggingface.co/AngelSlim/Hy4-flash-GGUF"><b>GGUF</b></a>
26
+ </p>
27
+
28
+ ---
29
+
30
+ ## Table of Contents
31
+
32
+ - [Model Introduction](#model-introduction)
33
+ - [Practicality and Cost Efficiency](#practicality-and-cost-efficiency)
34
+ - [Optimization Approach](#optimization-approach)
35
+ - [Task Performance and Cost](#task-performance-and-cost)
36
+ - [User Blind Evaluation](#user-blind-evaluation)
37
+ - [Appendix: Benchmark results](#appendix-benchmark-results)
38
+ - [News](#news)
39
+ - [Model Links](#model-links)
40
+ - [Quickstart](#quickstart)
41
+ - [Transformers](#transformers)
42
+ - [OpenAI-compatible API](#openai-compatible-api)
43
+ - [Deployment](#deployment)
44
+ - [DFly2 for Speculative Decoding](#dfly2-for-speculative-decoding)
45
+ - [Quantization](#quantization)
46
+ - [Inference Frameworks](#inference-frameworks)
47
+ - [vLLM](#vllm)
48
+ - [SGLang](#sglang)
49
+ - [Finetuning](#finetuning)
50
+ - [License](#license)
51
+ - [Contact Us](#contact-us)
52
+
53
+ ## Model Introduction
54
+
55
+ **Hy4 flash** is our most cost-effective Hy model to date, featuring 220B total parameters (including 64B Engram parameters) with just 12B active parameters per token. It natively accepts text and image inputs, generates text, and supports a context window of up to 1M tokens.
56
+
57
+ Its 42-layer language backbone consists of one dense feed-forward layer and 41 MoE layers, with identity Hyper-Connections (iHC) supporting inter-layer information flow. A vision encoder (adapted from SigLIP2-So400m-Patch16-Res512) is connected to the backbone through an MLP projector. In addition to the backbone, a built-in 5-layer drafter, [DFly2](#dfly2-for-speculative-decoding) (0.85B parameters), reuses the target model's KV cache and sparse indices to support long-context speculative decoding at a low, constant cost.
58
+
59
+ Compared with Hy4, Hy4 Flash introduces three notable architectural changes:
60
+
61
+ 1. **Hybrid attention.** Sparse attention (IndexPool) and sliding-window attention (SWA) are interleaved at a 1:3 ratio.
62
+ 2. **LatentMoE.** Routed experts operate in a lower-dimensional latent space. Each token activates 16 of 512 routed experts, alongside one shared expert.
63
+ 3. **N-gram embedding (Engram).** The backbone is augmented with learned bigram and trigram representations, retrieved via multi-head hash lookups.
64
+
65
+ | Property | Value |
66
+ | --- | --- |
67
+ | **Overview** | |
68
+ | Architecture | Mixture-of-Experts (MoE) |
69
+ | Total Parameters (excluding Engram and ViT) | 155.5B |
70
+ | Engram Parameters | 63.5B |
71
+ | ViT Parameters | 0.43B |
72
+ | Activated Parameters per Token | Text 11.3B + Vision 0.43B |
73
+ | Context Length | 1M (1,048,576 tokens) |
74
+ | **Language Backbone** | |
75
+ | Backbone Layers | 42 — 1 dense layer and 41 MoE layers |
76
+ | Hidden Size | 3,072 |
77
+ | Vocabulary Size | 120,832 |
78
+ | Residual Connections | iHC (identity Hyper-Connections) with 4 residual streams |
79
+ | **Attention — Hybrid Layout** | |
80
+ | Attention Architecture | Sparse attention + sliding-window attention (SWA) |
81
+ | Backbone Layer Pattern | 2 × Sparse → 10 × (3 × SWA → Sparse) |
82
+ | Attention Formulation | Multi-head Latent Attention (MLA) |
83
+ | Multimodal Position Encoding | Interleaved MRoPE |
84
+ | **Attention — Layers** | **Sparse / SWA** |
85
+ | Backbone Layers | 12 / 30 |
86
+ | Attention Heads | 32 / 64 |
87
+ | MLA q_lora_rank | 1,024 / 1,024 |
88
+ | MLA kv_lora_rank | 512 / 512 |
89
+ | QK Head Dimension (non-RoPE + RoPE) | 192 + 64 / 192 + 64 |
90
+ | Value Head Dimension | 256 / 256 |
91
+ | RoPE Base | 10,000,000 / 10,000 |
92
+ | Sparse Context Selection | Up to 2,048 tokens via group-level selection |
93
+ | SWA Context Window | 512-token causal window (511 preceding + current) |
94
+ | **Feed-Forward Networks** | |
95
+ | Dense FFN Intermediate Size | 13,824 |
96
+ | Routed Experts per MoE Layer | 512 |
97
+ | Shared Experts per MoE Layer | 1 |
98
+ | Activated Routed Experts per Token | 16 |
99
+ | MoE Latent Dimension | 1,536 |
100
+ | Expert Intermediate Size | 1,536 |
101
+ | **N-gram Embedding** | |
102
+ | N-gram Orders | Bigrams and trigrams |
103
+ | Backbone Layer Indices (starting from 0) | 1, 11, 21, 31 |
104
+ | Hash Heads per N-gram Order | 8 |
105
+ | Hash Vocabulary Size per Head | 10.3M |
106
+ | Embedding Dimension per N-gram Order | 768 (8 heads × 96 dimensions) |
107
+ | Embedding Table Parameters | 63.4B |
108
+ | **Vision Encoder** | |
109
+ | Vision Encoder Layers | 27 |
110
+ | Vision Encoder Hidden Size | 1,152 |
111
+ | Vision Encoder Intermediate Size | 4,304 |
112
+ | Vision Encoder Attention Heads | 16 |
113
+ | Vision Patch Size | 16 × 16 |
114
+ | Vision Position Embedding Size | 128 × 128 |
115
+ | Visual Feature Merging | 2 x 2 |
116
+
117
+ ## Practicality and Cost Efficiency
118
+
119
+ ### Optimization Approach
120
+
121
+ Across training, evaluation, and inference, we aggressively optimized for cost efficiency while preserving intelligence:
122
+
123
+ - **Pre-training**: Hy4 flash integrates hybrid attention, LatentMoE, and Engram into its architecture—reducing KV cache size to just **1/24 of Hy3** and **1/6 of Hy4**, with extensive scaling ladder experiments ensuring performance meets expectations.
124
+ - **Post-training**: Self-distillation from Hy4 establishes strong general intelligence, while reinforcement learning cuts average end-to-end output tokens by **50%**.
125
+ - **Evaluation**: We built WorkIndex, an internal benchmark grounded in typical WorkBuddy tasks, and paired it with human evaluation—focusing on **deliverable quality, single-turn output length, and task turnaround time** rather than public benchmark scores detached from everyday work.
126
+ - **Inference**: Combining megakernels, 4-bit quantization, and [DFly2](#dfly2-for-speculative-decoding) pushes decoding speed past **150 tokens/s**, enabling low-cost serving at scale.
127
+
128
+ ### Task Performance and Cost
129
+
130
+ **Hy4 flash delivers overall task performance slightly above Hy4 preview at a much lower cost per task.** On WorkIndex, it scores **42.1**, compared with **39.7** for Hy4 preview, while its average cost per task is just **1/13 of Hy4 preview**, **1/20 of Hy4**, and **1/8 of DeepSeek-V4.1-Flash-1017** (Max tier, peak-hour pricing).
131
+
132
+ <p align="center">
133
+ <a href="assets/performance-cost-workindex.png"><img src="assets/performance-cost-workindex.png" alt="Performance and cost per task on WorkIndex for Hy4 flash and comparison models" width="49%"/></a>
134
+ <a href="assets/performance-cost-open-benchmarks.png"><img src="assets/performance-cost-open-benchmarks.png" alt="Performance and cost per task on Open Benchmarks for Hy4 flash and comparison models" width="49%"/></a>
135
+ </p>
136
+ <p align="center"><sub>Figure 1: Performance and cost across internal and external tasks. WorkIndex (internal) comprises 8 evaluation sets covering slide creation, slide editing, human-AI slide co-writing, Markdown human-AI co-writing, document editing, spreadsheet repair, audio/video processing, and visual understanding.<br/>Open Benchmarks includes 7 public agent benchmarks: DeepSWE, ProgramBench, Terminal-Bench 4.0, APEX-Agents, AutomationBench, HLE, and OSWorld 2.1.</sub></p>
137
+
138
+ <p align="center">
139
+ <a href="assets/workindex-cost.png"><img src="assets/workindex-cost.png" alt="Average cost per task on WorkIndex for Hy4 flash, Hy4 preview, and Hy4" width="100%"/></a>
140
+ </p>
141
+ <p align="center"><sub>Figure 2: Average cost per task on WorkIndex. Task scores are 42.1 for Hy4 flash, 39.7 for Hy4 preview, and 51.4 for Hy4.</sub></p>
142
+
143
+ We recommend Hy4 flash for most everyday productivity tasks, including **deep search, document processing, slide creation, simple frontend demos, and computer use**. For complex tasks such as long-horizon software engineering, we recommend Hy4 for its higher intelligence ceiling.
144
+
145
+ ### User Blind Evaluation
146
+
147
+ In the blind evaluation, **Hy4 flash achieved user satisfaction close to Claude Haiku 5.5, while recording the lowest cost per task and shortest median end-to-end completion time among the evaluated models**. Its user satisfaction score was **2.79/4** (Claude Haiku 5.5: **2.81/4**), with a median end-to-end completion time of **6.7 minutes** and a cost of **RMB 0.26 per task**.
148
+
149
+ <p align="center">
150
+ <a href="assets/blind-evaluation-en.png"><img src="assets/blind-evaluation-en.png" alt="Blind evaluation comparing user satisfaction, median end-to-end completion time, and cost per task" width="100%"/></a>
151
+ </p>
152
+ <p align="center"><sub>Figure 3: Blind evaluation with 85 internal WorkBuddy users across 259 tasks spanning 27 categories.</sub></p>
153
+
154
+ ## Appendix: Benchmark results
155
+
156
+ <p align="center">
157
+ <a href="assets/benchmark.png"><img src="assets/benchmark.png" alt="Full benchmark scores, task costs, and evaluation notes for Hy4 flash and comparison models" width="100%"/></a>
158
+ </p>
159
+
160
+ ## News
161
+
162
+ * We release **Hy4 flash**, **Hy4 flash-FP8**, and the pre-trained **Hy4 flash-base** as open-weight models on [Hugging Face](https://huggingface.co/tencent/Hy4-flash), [ModelScope](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash), and [GitCode](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash).
163
+
164
+ ## Model Links
165
+
166
+ | Model Name | Description | Hugging Face | ModelScope | GitCode |
167
+ |:---|:---|:---:|:---:|:---:|
168
+ | Hy4 flash | Instruct model | [Model](https://huggingface.co/tencent/Hy4-flash) | [Model](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash) | [Model](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash) |
169
+ | Hy4 flash-FP8 | FP8 quantized instruct model | [Model](https://huggingface.co/tencent/Hy4-flash-FP8) | [Model](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash-FP8) | [Model](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash-FP8) |
170
+ | Hy4 flash-base | Pre-trained base model | [Model](https://huggingface.co/tencent/Hy4-flash-base) | [Model](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash-base) | [Model](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash-base) |
171
+
172
+ Hy4 flash-base is a pre-trained base model, not an instruction-tuned chat model. The chat and reasoning examples below apply to Hy4 flash and Hy4 flash-FP8.
173
+
174
+ ## Quickstart
175
+
176
+ Use [Transformers](#transformers) to load the model for local inference, or deploy a server and access it through the [OpenAI-compatible API](#openai-compatible-api).
177
+
178
+ > **Reasoning mode (instruct models)**: `reasoning_effort` supports `"high"` (default, thinking enabled) and `"no_think"` (direct responses). Both examples below use `"no_think"`.
179
+ >
180
+ > **Recommended sampling parameters**: `temperature=0.9`, `top_p=1.0`.
181
+
182
+ ### Transformers
183
+
184
+ > Support for Hy4 flash will be merged into the `transformers` main branch soon. Until then, install from the commit below.
185
+
186
+ ```bash
187
+ pip install "transformers[torch,vision] @ git+https://github.com/huggingface/transformers.git@refs/pull/50903/head"
188
+ ```
189
+
190
+ The following example generates a brief image description:
191
+
192
+ ```python
193
+ from transformers import AutoModelForImageTextToText, AutoProcessor
194
+
195
+ model_id = "tencent/Hy4-flash"
196
+ processor = AutoProcessor.from_pretrained(model_id)
197
+ model = AutoModelForImageTextToText.from_pretrained(model_id, device_map="auto")
198
+
199
+ messages = [
200
+ {
201
+ "role": "user",
202
+ "content": [
203
+ {
204
+ "type": "image",
205
+ "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg",
206
+ },
207
+ {"type": "text", "text": "Describe this image in one sentence."},
208
+ ],
209
+ }
210
+ ]
211
+
212
+ inputs = processor.apply_chat_template(
213
+ messages,
214
+ add_generation_prompt=True,
215
+ tokenize=True,
216
+ return_dict=True,
217
+ return_tensors="pt",
218
+ reasoning_effort="no_think",
219
+ ).to(model.device)
220
+
221
+ outputs = model.generate(**inputs, max_new_tokens=256)
222
+ print(processor.decode(outputs[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True))
223
+ ```
224
+
225
+ ### OpenAI-compatible API
226
+
227
+ For API access, start a server using the [vLLM deployment instructions](#vllm), then run the following client example:
228
+
229
+ ```python
230
+ from openai import OpenAI
231
+
232
+ client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="EMPTY")
233
+
234
+ response = client.chat.completions.create(
235
+ model="hy4-flash",
236
+ messages=[
237
+ {"role": "user", "content": "Hello! Can you briefly introduce yourself?"},
238
+ ],
239
+ temperature=0.9,
240
+ top_p=1.0,
241
+ extra_body={"chat_template_kwargs": {"reasoning_effort": "no_think"}},
242
+ )
243
+ print(response.choices[0].message.content)
244
+ ```
245
+
246
+ ## Deployment
247
+
248
+ ### DFly2 for Speculative Decoding
249
+
250
+ DFly2 is Hy4 flash's speculative-decoding drafter, trained with [AngelSpec](https://github.com/Tencent/AngelSpec) and built directly on the target model's **Top-k Sparse KV**: each draft layer shares the KV cache (**KV Share**) and the compressed indexer's selection of up to 2,048 positions (**Index Share**) with its corresponding sparse attention layer. This yields three advantages:
251
+
252
+ - **Constant cost**: each draft token attends to at most ~2K keys, regardless of context length.
253
+ - **Zero extra KV memory**: no separate KV cache is maintained for the drafter.
254
+ - **PD-disaggregation ready**: not involved in prefill; the prefill-to-decode transfer adds only one last-layer hidden state.
255
+
256
+ <p align="center">
257
+ <a href="assets/dfly2.png"><img src="assets/dfly2.png" alt="DFly2 architecture: each draft layer reuses the KV cache and the indexer-selected positions of a sparse attention layer of Hy4 flash" width="40%"/></a>
258
+ </p>
259
+
260
+ ### Quantization
261
+
262
+ We provide [AngelSlim](https://github.com/tencent/AngelSlim), a more accessible, comprehensive, and efficient toolkit for large model compression. AngelSlim supports a comprehensive suite of compression tools for large-scale multimodal models, including common quantization algorithms, low-bit quantization, and speculative sampling.
263
+
264
+ For this model, we have provided a [GGUF](https://huggingface.co/AngelSlim/Hy4-flash-GGUF) compressed version for end-to-end testing purposes.
265
+
266
+ ### Inference Frameworks
267
+
268
+ For production serving, we recommend using [vLLM](https://github.com/vllm-project/vllm) or [SGLang](https://docs.sglang.io/). Please refer to the recipes:
269
+ - [Hy4 flash vLLM Recipe](https://recipes.vllm.ai/tencent/Hy4-flash)
270
+ - [Hy4 flash SGLang Cookbook (Coming Soon)](https://lmsysorg.mintlify.app/cookbook/autoregressive/Tencent/Hy4-flash)
271
+
272
+ #### vLLM
273
+
274
+ Use official prebuilt image `vllm/vllm-openai:hy4-flash`:
275
+
276
+ ```bash
277
+ docker run --gpus all \
278
+ -p 8000:8000 \
279
+ --ipc=host \
280
+ -v ~/.cache/huggingface:/root/.cache/huggingface \
281
+ vllm/vllm-openai:hy4-flash tencent/Hy4-flash-FP8 \
282
+ --tensor-parallel-size 8 \
283
+ --speculative-config '{"num_speculative_tokens":5,"method":"dfly"}' \
284
+ --attention-backend FLASHMLA_SPARSE \
285
+ --tool-call-parser hy_v4 \
286
+ --reasoning-parser hy_v4 \
287
+ --enable-auto-tool-choice \
288
+ --port 8000 \
289
+ --served-model-name hy4-flash
290
+ ```
291
+
292
+ #### SGLang
293
+
294
+ The official SGLang prebuilt image is coming soon.
295
+
296
+ ## Finetuning
297
+
298
+ Hy4 flash provides a complete model finetuning pipeline. For detailed documentation, please refer to: [Finetuning Guide](./finetune/README.md)
299
+
300
+ ## License
301
+
302
+ Hy4 flash is released under the **Tencent Hy License**. See [LICENSE](./LICENSE) for details.
303
+
304
+ ## Contact Us
305
+
306
+ If you have any questions or suggestions, feel free to reach out to our R&D and product teams via email:
307
+
308
+ 📧 **hunyuan_opensource@tencent.com**
309
+
310
+ ---
311
+
312
+ <p align="center">
313
+ <i>Hy4 flash is developed by the Tencent Hy Team.</i>
314
+ </p>
README_CN.md ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <p align="left">
2
+ <a href="README.md">English</a>&nbsp;|&nbsp;中文
3
+ </p>
4
+ <br>
5
+
6
+ <p align="center">
7
+ <img src="assets/logo-zh.png" alt="腾讯混元" width="400"/>
8
+ </p>
9
+
10
+ <div align="center" style="line-height: 1;">
11
+
12
+ [![License](https://img.shields.io/badge/License-Tencent%20Hy%20License-blue)](#许可证)
13
+ &nbsp;&nbsp;
14
+ [![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Tencent%20Hy-ffc107?color=ffc107&logoColor=white)](https://huggingface.co/tencent/Hy4-flash)
15
+ &nbsp;&nbsp;
16
+ [![ModelScope](https://img.shields.io/badge/ModelScope-Tencent%20Hy-624aff)](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash)
17
+ &nbsp;&nbsp;
18
+ [![GitCode](https://img.shields.io/badge/GitCode-Tencent%20Hy-red?logoColor=white)](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash)
19
+
20
+ </div>
21
+
22
+ <p align="center">
23
+ <a href="https://hy.tencent.com/research/hy4-flash?langVersion=zh"><b>官方网站</b></a>&nbsp;&nbsp;|&nbsp;&nbsp;
24
+ <a href="https://github.com/Tencent-Hunyuan/Hy4-flash"><b>GitHub</b></a>&nbsp;&nbsp;|&nbsp;&nbsp;
25
+ <a href="https://huggingface.co/AngelSlim/Hy4-flash-GGUF"><b>GGUF</b></a>
26
+ </p>
27
+
28
+ ---
29
+
30
+ ## 目录
31
+
32
+ - [模型介绍](#模型介绍)
33
+ - [实用性与成本效率](#实用性与成本效率)
34
+ - [优化思路](#优化思路)
35
+ - [任务效果与成本](#任务效果与成本)
36
+ - [用户盲测](#用户盲测)
37
+ - [Benchmark 附录](#benchmark-附录)
38
+ - [新闻](#新闻)
39
+ - [模型链接](#模型链接)
40
+ - [快速开始](#快速开始)
41
+ - [Transformers](#transformers)
42
+ - [OpenAI 兼容 API](#openai-兼容-api)
43
+ - [推理和部署](#推理和部署)
44
+ - [DFly2 投机解码](#dfly2-投机解码)
45
+ - [量化工具](#量化工具)
46
+ - [推理框架](#推理框架)
47
+ - [vLLM](#vllm)
48
+ - [SGLang](#sglang)
49
+ - [模型微调](#模型微调)
50
+ - [许可证](#许可证)
51
+ - [联系我们](#联系我们)
52
+
53
+ ## 模型介绍
54
+
55
+ **Hy4 flash** 是混元系列迄今性价比最高的多模态模型,原生支持文本和图像输入,可生成文本,上下文长度可达 1M token。模型总参数量约为 2200 亿,其中 Engram 约占 640 亿。处理每个 token 时,只需激活约 120 亿个参数。
56
+
57
+ 语言模型主干由 1 层稠密前馈网络和 41 层混合专家网络(MoE)组成,共 42 层。各层通过 identity Hyper-Connections(iHC)传递残差信息。视觉编码器由 SigLIP2-So400m-Patch16-Res512 训练而来,提取的图像特征经 MLP 投影后送入语言模型主干。主干之外还内置了 5 层草稿模型 [DFly2](#dfly2-投机解码)(约 8.5 亿参数),直接复用主模型的 KV cache 与稀疏索引,以固定的低开销支持长上下文投机解码。
58
+
59
+ 相较于 Hy4,Hy4 flash 在架构上有三处主要变化。
60
+
61
+ 1. **混合注意力(Hybrid attention)** 用滑动窗口注意力(SWA)捕捉相邻 token 之间的联系,用 IndexPool 稀疏注意力从更长的上下文中选取相关片段。稀疏注意力与 SWA 以 1:3 的模式交替组合。
62
+ 2. **LatentMoE** 将 token 的表示从 3072 维投影到 1536 维,让路由专家在更低维的空间里完成计算。每个 MoE 层会为当前 token 从 512 个路由专家中选出 16 个,另有 1 个共享专家参与每个 token 的计算。
63
+ 3. **Engram(N-gram embedding)** 为模型引入了按需查表的记忆机制。模型根据最近两个或三个 token 的组合,通过多头哈希取出训练中学到的片段表示,并将其融入当前 token 的隐藏状态。
64
+
65
+ | 规格项 | 取值 |
66
+ | --- | --- |
67
+ | **模型概览** | |
68
+ | Architecture | Mixture-of-Experts (MoE) |
69
+ | Total Parameters (excluding Engram and ViT) | 155.5B |
70
+ | Engram Parameters | 63.5B |
71
+ | ViT Parameters | 0.43B |
72
+ | Activated Parameters per Token | Text 11.3B + Vision 0.43B |
73
+ | Context Length | 1M (1,048,576 tokens) |
74
+ | **语言模型主干** | |
75
+ | Backbone Layers | 42 — 1 dense layer and 41 MoE layers |
76
+ | Hidden Size | 3,072 |
77
+ | Vocabulary Size | 120,832 |
78
+ | Residual Connections | iHC (identity Hyper-Connections) with 4 residual streams |
79
+ | **混合注意力布局** | |
80
+ | Attention Architecture | Sparse attention + sliding-window attention (SWA) |
81
+ | Backbone Layer Pattern | 2 × Sparse → 10 × (3 × SWA → Sparse) |
82
+ | Attention Formulation | Multi-head Latent Attention (MLA) |
83
+ | Multimodal Position Encoding | Interleaved MRoPE |
84
+ | **注意力层配置** | **稀疏注意力 / 滑动窗口注意力** |
85
+ | Backbone Layers | 12 / 30 |
86
+ | Attention Heads | 32 / 64 |
87
+ | MLA q_lora_rank | 1,024 / 1,024 |
88
+ | MLA kv_lora_rank | 512 / 512 |
89
+ | QK Head Dimension (non-RoPE + RoPE) | 192 + 64 / 192 + 64 |
90
+ | Value Head Dimension | 256 / 256 |
91
+ | RoPE Base | 10,000,000 / 10,000 |
92
+ | Sparse Context Selection | Up to 2,048 tokens via group-level selection |
93
+ | SWA Context Window | 512-token causal window (511 preceding + current) |
94
+ | **前馈网络** | |
95
+ | Dense FFN Intermediate Size | 13,824 |
96
+ | Routed Experts per MoE Layer | 512 |
97
+ | Shared Experts per MoE Layer | 1 |
98
+ | Activated Routed Experts per Token | 16 |
99
+ | MoE Latent Dimension | 1,536 |
100
+ | Expert Intermediate Size | 1,536 |
101
+ | **N-gram 嵌入(Engram)** | |
102
+ | N-gram Orders | Bigrams and trigrams |
103
+ | Backbone Layer Indices (starting from 0) | 1, 11, 21, 31 |
104
+ | Hash Heads per N-gram Order | 8 |
105
+ | Hash Vocabulary Size per Head | 10.3M |
106
+ | Embedding Dimension per N-gram Order | 768 (8 heads × 96 dimensions) |
107
+ | Embedding Table Parameters | 63.4B |
108
+ | **视觉编码器** | |
109
+ | Vision Encoder Layers | 27 |
110
+ | Vision Encoder Hidden Size | 1,152 |
111
+ | Vision Encoder Intermediate Size | 4,304 |
112
+ | Vision Encoder Attention Heads | 16 |
113
+ | Vision Patch Size | 16 × 16 |
114
+ | Vision Position Embedding Size | 128 × 128 |
115
+ | Visual Feature Merging | 2 x 2 |
116
+
117
+ ## 实用性与成本效率
118
+
119
+ ### 优化思路
120
+
121
+ 我们通过一系列工作在极致优化成本的同时兼顾智能水平和实用性:
122
+
123
+ - 在预训练阶段,Hy4 flash 加入了混合注意力、LatentMoE、Engram 等架构特性,KV cache 大小仅为 Hy3 的 **1/24**、Hy4 的 **1/6**,且通过大量 scaling ladder 实验确保效果符合预期;
124
+ - 在后训练阶段,我们通过自蒸馏 Hy4 获得了较强的综合智能水平,再通过强化学习使端到端任务输出长度降低 **50%**;
125
+ - 在评估阶段,我们构建基于 WorkBuddy 典型任务的内部评测集 WorkIndex,并结合人评,聚焦日常工作的**产物效果、单轮输出长度、端到端用时**,而非远离日常工作的公开榜单分数;
126
+ - 在推理阶段,我们结合 megakernel、4bit 量化、[DFly2](#dfly2-投机解码) 等手段极致优化,使 Hy4 flash 解码速度超过 **150 tps**,且可以大规模低成本部署运营。
127
+
128
+ ### 任务效果与成本
129
+
130
+ **Hy4 flash 的综合任务效果略优于 Hy4 preview,同时显著降低了完成任务的成本。** 在 WorkIndex 上,其得分为 **42.1**,高于 Hy4 preview 的 **39.7**;平均单任务成本仅为 **Hy4 preview 的 1/13、Hy4 的 1/20、DeepSeek-V4.1-Flash-1017 的 1/8**(DeepSeek 按 `max` 档位、忙时价格计算)。
131
+
132
+ <p align="center">
133
+ <a href="assets/performance-cost-workindex.png"><img src="assets/performance-cost-workindex.png" alt="Hy4 flash 与对比模型在 WorkIndex 上的得分和单任务成本" width="49%"/></a>
134
+ <a href="assets/performance-cost-open-benchmarks.png"><img src="assets/performance-cost-open-benchmarks.png" alt="Hy4 flash 与对比模型在公开评测上的得分和单任务成本" width="49%"/></a>
135
+ </p>
136
+ <p align="center"><sub>图 1:内外部任务上的效果与成本。WorkIndex (internal) 包含 8 个内部评测集:PPT 制作、PPT 修改、PPT 人机双写、Markdown 人机双写、文档编辑、表格修复、音视频处理和视觉理解。<br/>Open Benchmarks 包含 7 个公开 Agent 评测集:DeepSWE、ProgramBench、Terminal-Bench 4.0、APEX-Agents、AutomationBench、HLE 和 OSWorld 2.1。</sub></p>
137
+
138
+ <p align="center">
139
+ <a href="assets/workindex-cost.png"><img src="assets/workindex-cost.png" alt="Hy4 flash、Hy4 preview 和 Hy4 的 WorkIndex 平均单任务成本与得分" width="100%"/></a>
140
+ </p>
141
+ <p align="center"><sub>图 2:WorkIndex 平均单任务成本对比;Hy4 flash、Hy4 preview、Hy4 的任务得分分别为 42.1、39.7、51.4。</sub></p>
142
+
143
+ 我们推荐在**深度搜索、文档处理、PPT 制作、简单前端展示和 computer use** 等大部分日常任务中使用 Hy4 flash;对于长程软件工程等复杂任务,仍推荐智能上限更高的 Hy4。
144
+
145
+ ### 用户盲测
146
+
147
+ **在真实工作任务盲测中,Hy4 flash 的满意度接近 Claude Haiku 5.5,同时在参评模型中实现了最低的单任务成本和最短的任务耗时中位数。** Hy4 flash 的用户满意度为 **2.79/4**(Claude Haiku 5.5 为 **2.81/4**),端到端任务时长中位数为 **6.7 分钟**,单任务成本为 **0.26 元**。
148
+
149
+ <p align="center">
150
+ <a href="assets/blind-evaluation-zh.png"><img src="assets/blind-evaluation-zh.png" alt="用户盲测中的满意度、端到端任务时长中位数和单任务成本对比" width="100%"/></a>
151
+ </p>
152
+ <p align="center"><sub>图 3:85 位 WorkBuddy 内部用户在 27 类、259 个日常工作任务上的盲测结果。</sub></p>
153
+
154
+ ## Benchmark 附录
155
+
156
+ <p align="center">
157
+ <a href="assets/benchmark.png"><img src="assets/benchmark.png" alt="Hy4 flash 与对比模型的完整评测得分、单任务成本和测试说明" width="100%"/></a>
158
+ </p>
159
+
160
+ ## 新闻
161
+
162
+ * 我们在 [Hugging Face](https://huggingface.co/tencent/Hy4-flash)、[ModelScope](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash) 和 [GitCode](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash) 开源 **Hy4 flash**、**Hy4 flash-FP8** 和预训练模型 **Hy4 flash-base** 的权重。
163
+
164
+ ## 模型链接
165
+
166
+ | 模型名 | 简介 | Hugging Face | ModelScope | GitCode |
167
+ |:---|:---|:---:|:---:|:---:|
168
+ | Hy4 flash | Instruct 模型 | [Model](https://huggingface.co/tencent/Hy4-flash) | [Model](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash) | [Model](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash) |
169
+ | Hy4 flash-FP8 | FP8 量化 Instruct 模型 | [Model](https://huggingface.co/tencent/Hy4-flash-FP8) | [Model](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash-FP8) | [Model](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash-FP8) |
170
+ | Hy4 flash-base | 预训练模型 | [Model](https://huggingface.co/tencent/Hy4-flash-base) | [Model](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-flash-base) | [Model](https://ai.gitcode.com/tencent_hunyuan/Hy4-flash-base) |
171
+
172
+ Hy4 flash-base 为预训练底座,未经过指令微调;下方对话与推理模式示例适用于 Hy4 flash 和 Hy4 flash-FP8。
173
+
174
+ ## 快速开始
175
+
176
+ 可使用 [Transformers](#transformers) 直接加载模型进行本地推理,或部署服务后通过 [OpenAI 兼容 API](#openai-兼容-api) 调用。
177
+
178
+ > **推理模式(Instruct 模型)**:`reasoning_effort` 支持 `"high"`(默认,启用思考)和 `"no_think"`(直接回复)。以下两个示例均使用 `"no_think"`。
179
+ >
180
+ > **推荐采样参数**:`temperature=0.9`,`top_p=1.0`。
181
+
182
+ ### Transformers
183
+
184
+ > Hy4 flash 的支持即将合入 `transformers` 主分支,在此之前请从下面的提交安装。
185
+
186
+ ```bash
187
+ pip install "transformers[torch,vision] @ git+https://github.com/huggingface/transformers.git@refs/pull/50903/head"
188
+ ```
189
+
190
+ 下面以图片描述为例:
191
+
192
+ ```python
193
+ from transformers import AutoModelForImageTextToText, AutoProcessor
194
+
195
+ model_id = "tencent/Hy4-flash"
196
+ processor = AutoProcessor.from_pretrained(model_id)
197
+ model = AutoModelForImageTextToText.from_pretrained(model_id, device_map="auto")
198
+
199
+ messages = [
200
+ {
201
+ "role": "user",
202
+ "content": [
203
+ {
204
+ "type": "image",
205
+ "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg",
206
+ },
207
+ {"type": "text", "text": "Describe this image in one sentence."},
208
+ ],
209
+ }
210
+ ]
211
+
212
+ inputs = processor.apply_chat_template(
213
+ messages,
214
+ add_generation_prompt=True,
215
+ tokenize=True,
216
+ return_dict=True,
217
+ return_tensors="pt",
218
+ reasoning_effort="no_think",
219
+ ).to(model.device)
220
+
221
+ outputs = model.generate(**inputs, max_new_tokens=256)
222
+ print(processor.decode(outputs[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True))
223
+ ```
224
+
225
+ ### OpenAI 兼容 API
226
+
227
+ 选择 API 调用时,先按 [vLLM 部署说明](#vllm) 启动服务,再运行以下客户端示例:
228
+
229
+ ```python
230
+ from openai import OpenAI
231
+
232
+ client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="EMPTY")
233
+
234
+ response = client.chat.completions.create(
235
+ model="hy4-flash",
236
+ messages=[
237
+ {"role": "user", "content": "你好!请简单介绍一下你自己。"},
238
+ ],
239
+ temperature=0.9,
240
+ top_p=1.0,
241
+ extra_body={"chat_template_kwargs": {"reasoning_effort": "no_think"}},
242
+ )
243
+ print(response.choices[0].message.content)
244
+ ```
245
+
246
+ ## 推理和部署
247
+
248
+ ### DFly2 投机解码
249
+
250
+ DFly2 是基于 [AngelSpec](https://github.com/Tencent/AngelSpec) 框架训练的 Hy4 flash 投机解码草稿模型,直接复用主模型的 **Top-k Sparse KV**:每个 draft 层与主模型对应的稀疏注意力层共享 KV cache(**KV Share**)及压缩索引器选出的最多 2,048 个位置(**Index Share**)。由此带来三点优势:
251
+
252
+ - **计算开销固定**:每个草稿 token 至多关注约 2K 个 key,注意力复杂度与上下文长度无关。
253
+ - **零额外 KV 显存**:无需维护草稿模型独立的 KV cache。
254
+ - **适配 PD 分离**:不参与 prefill,P→D 仅额外传输一个末层隐状态。
255
+
256
+ <p align="center">
257
+ <a href="assets/dfly2.png"><img src="assets/dfly2.png" alt="DFly2 结构图:每个 draft 层复用 Hy4 flash 对应稀疏注意力层的 KV cache 与压缩索引器选出的位置" width="40%"/></a>
258
+ </p>
259
+
260
+ ### 量化工具
261
+
262
+ 我们提供了 [AngelSlim](https://github.com/tencent/AngelSlim)——一套易用、全面、高效的大模型压缩工具包,涵盖常用量化算法、低比特量化和投机采样等能力。
263
+
264
+ 对于本模型我们提供了[GGUF](https://huggingface.co/AngelSlim/Hy4-flash-GGUF) 压缩版本以供端测使用。
265
+
266
+ ### 推理框架
267
+
268
+ 对于生产环境部署,我们建议使用 [vLLM](https://github.com/vllm-project/vllm) 或 [SGLang](https://docs.sglang.io/)。请参考部署方案:
269
+ - [Hy4 flash vLLM Recipe](https://recipes.vllm.ai/tencent/Hy4-flash)
270
+ - [Hy4 flash SGLang Cookbook(敬请期待)](https://lmsysorg.mintlify.app/cookbook/autoregressive/Tencent/Hy4-flash)
271
+
272
+
273
+ #### vLLM
274
+
275
+ 使用社区官方镜像 `vllm/vllm-openai:hy4-flash` 部署:
276
+
277
+ ```bash
278
+ docker run --gpus all \
279
+ -p 8000:8000 \
280
+ --ipc=host \
281
+ -v ~/.cache/huggingface:/root/.cache/huggingface \
282
+ vllm/vllm-openai:hy4-flash tencent/Hy4-flash-FP8 \
283
+ --tensor-parallel-size 8 \
284
+ --speculative-config '{"num_speculative_tokens":5,"method":"dfly"}' \
285
+ --attention-backend FLASHMLA_SPARSE \
286
+ --tool-call-parser hy_v4 \
287
+ --reasoning-parser hy_v4 \
288
+ --enable-auto-tool-choice \
289
+ --port 8000 \
290
+ --served-model-name hy4-flash
291
+ ```
292
+
293
+
294
+ #### SGLang
295
+
296
+ SGLang 社区官方镜像稍后提供,敬请期待。
297
+
298
+
299
+ ## 模��微调
300
+
301
+ Hy4 flash 提供了完整的模型微调流程,详细的微调文档请参考:[模型微调指南](./finetune/README_CN.md)
302
+
303
+ ## 许可证
304
+
305
+ Hy4 flash 基于 **Tencent Hy License** 发布。详情请参阅 [LICENSE](./LICENSE)。
306
+
307
+ ## 联系我们
308
+
309
+ 如有问题或建议,欢迎通过邮件联系我们的研发和产品团队:
310
+
311
+ 📧 **hunyuan_opensource@tencent.com**
312
+
313
+ ---
314
+
315
+ <p align="center">
316
+ <i>Hy4 flash 由腾讯混元团队研发。</i>
317
+ </p>
assets/benchmark.png ADDED

Git LFS Details

  • SHA256: 2aab42905fa871f41086bdbd474a7d850970b5614cd18e7eadb4656606ae113f
  • Pointer size: 131 Bytes
  • Size of remote file: 938 kB
assets/blind-evaluation-en.png ADDED
assets/blind-evaluation-zh.png ADDED
assets/dfly2.png ADDED

Git LFS Details

  • SHA256: 1a38561973c67c7834cbacf44df02d7d6ea9e569e50514b6bd58ac28a6e94ae9
  • Pointer size: 131 Bytes
  • Size of remote file: 906 kB
assets/logo-en.png ADDED
assets/logo-zh.png ADDED
assets/performance-cost-open-benchmarks.png ADDED

Git LFS Details

  • SHA256: 825c7f7994b6c16f69232a277d34ee6e51c7db57d4b8f1a8ef3ef8e45858182a
  • Pointer size: 131 Bytes
  • Size of remote file: 119 kB
assets/performance-cost-workindex.png ADDED

Git LFS Details

  • SHA256: 74ab28aff592ad549606d26ae40295135a5873908b76c67eab938e948f2a9552
  • Pointer size: 131 Bytes
  • Size of remote file: 115 kB
assets/workindex-cost.png ADDED
finetune/README.md ADDED
@@ -0,0 +1,584 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <p align="left">
2
+ <a href="README_CN.md">中文</a> | English
3
+ </p>
4
+
5
+ # Model Fine-tuning
6
+
7
+ Hy4 flash provides processes related to model fine-tuning. This section details how to process training data for model fine-tuning purposes.
8
+
9
+ ## Training Data Format and Processing
10
+
11
+ **Hy4 flash supports two thinking modes, controlled by the `reasoning_effort` parameter (only `no_think` and `high` are accepted; any other value raises an error). The available options are**:
12
+
13
+ | `reasoning_effort` | Mode | Description |
14
+ |---|---|---|
15
+ | `no_think` | Fast thinking | No reasoning process; the model directly outputs the answer. |
16
+ | `high` | Deep thinking | Thorough reasoning with a detailed thinking process. |
17
+
18
+ The training data should be formatted as a list of messages shown below. By default, the system prompt for both training and inference is empty; customize it as needed.
19
+
20
+ For `high` mode, the assistant's reasoning process can be provided in the `reasoning_content` field (the `chat_template` also accepts a `reasoning` field, which takes precedence); in `no_think` mode the template always emits empty `<think></think>` tags, so no reasoning field is needed.
21
+
22
+ ```jsonl
23
+ # Fast thinking pattern (no_think) — no reasoning field
24
+ {"reasoning_effort": "no_think", "messages": [{"content": "You are a helpful assistant.\nThe current time is 2026-01-01 13:26:12 Thursday", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2"}]}
25
+
26
+ # Deep thinking pattern (high) — with reasoning content
27
+ {"reasoning_effort": "high", "messages": [{"content": "You are a helpful assistant.\nThe current time is 2026-01-01 13:26:12 Thursday", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2", "reasoning_content": "The user is asking for the result of 1 + 1. In basic decimal arithmetic, 1 + 1 equals 2."}]}
28
+ ```
29
+
30
+ > **`reasoning_effort` default values:**
31
+ > - **During training**: If `reasoning_effort` is not specified in the training data, the training script defaults to `no_think` (fast thinking). This is because samples without a reasoning field are treated as non-thinking-mode data by the training script.
32
+ > - **During inference**: If `reasoning_effort` is not passed when calling `apply_chat_template`, the `chat_template` defaults to `high` (deep thinking).
33
+
34
+ Example of using `apply_chat_template` to tokenize:
35
+
36
+ ```python
37
+ from transformers import AutoTokenizer
38
+ tokenizer = AutoTokenizer.from_pretrained("./models", use_fast=False, trust_remote_code=True)
39
+
40
+ messages = [
41
+ {"content": "You are a helpful assistant.", "role": "system"},
42
+ {"content": "1+1=?", "role": "user"},
43
+ {"role": "assistant", "content": "1+1=2"}
44
+ ]
45
+ ids = tokenizer.apply_chat_template(messages, tokenize=True, reasoning_effort="no_think")
46
+ ```
47
+
48
+ ### Multimodal Data Format (Image / Video)
49
+
50
+ Hy4 flash supports mixed training with images, videos, and pure text. The data format is JSONL, with one sample per line.
51
+
52
+ #### Image Data
53
+
54
+ Use the `<image>` tag in the user message to mark the image position, and provide the image path list in the `images` field:
55
+
56
+ ```json
57
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "<image>Please describe this image in detail."}, {"role": "assistant", "content": "The image shows a beautiful sunset over the ocean."}], "images": ["test_images/sunset.jpg"]}
58
+ ```
59
+
60
+ Multiple images are supported — use multiple `<image>` tags and provide the corresponding paths in the `images` list (in order):
61
+
62
+ ```json
63
+ {"messages": [{"role": "user", "content": "<image>Compare the first image with the second one.<image>"}], "images": ["img1.jpg", "img2.jpg"]}
64
+ ```
65
+
66
+ #### Video Data
67
+
68
+ Use the `<video>` tag in the user message to mark the video position, and provide the video path list in the `videos` field:
69
+
70
+ ```json
71
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "<video>Please describe what is happening in this video."}, {"role": "assistant", "content": "The video shows a cat playing with a ball of yarn on a hardwood floor."}], "videos": ["test_videos/cat.mp4"]}
72
+ ```
73
+
74
+ #### Pure Text Data (Mixed Training)
75
+
76
+ Samples without `<image>` or `<video>` tags are treated as pure text and can be mixed in the same JSONL file:
77
+
78
+ ```json
79
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris."}]}
80
+ ```
81
+
82
+ #### Notes
83
+
84
+ - Image/video paths can be **absolute paths** or **relative paths** (relative to the JSONL file location or the working directory, depending on the training framework).
85
+ - The number of `<image>` / `<video>` tags in the message must match the number of paths in the `images` / `videos` list.
86
+ - A single sample can contain both `images` and `videos` fields simultaneously.
87
+ - Supported image formats depend on the [Pillow](https://pillow.readthedocs.io/) library (common formats such as JPEG, PNG, etc. are supported). Supported video formats depend on the video decoding backend (default is [PyAV](https://pyav.org/), which is based on FFmpeg; common formats such as MP4 are supported).
88
+ - Pure text example data is provided in `data/example_data.jsonl`, and multimodal example data in `data/example_vl_data.jsonl`.
89
+
90
+ ## Fine-tuning Process
91
+
92
+ ### Hardware Requirements
93
+
94
+ Based on testing, the minimum resource configuration is as follows:
95
+
96
+ - **LoRA Fine-tuning**: At least 8 machines with 64 GPUs (at least 96GB GPU memory per GPU, at least 2TB CPU memory per machine).
97
+ - **Full Fine-tuning**: At least 16 machines with 128 GPUs (at least 96GB GPU memory per GPU, at least 2TB CPU memory per machine).
98
+
99
+ > Note: The above are minimum resource configurations; actual requirements increase with `max_seq_length`, batch size, image resolution, number of video frames, etc. Multimodal data (especially high-resolution images and videos) consumes significantly more memory than pure text.
100
+
101
+ ### Configure Passwordless SSH Login Between Machines (Multi-Machine Training)
102
+
103
+ > If you only use single-machine training, you can skip this section.
104
+
105
+ The following instructions use two machines as an example, with their IPs denoted as `${ip1}` and `${ip2}`. All steps should be performed inside the Docker container.
106
+
107
+ First, configure passwordless SSH for each container on every machine:
108
+
109
+ ```sh
110
+ ssh-keygen # Generate id_rsa and id_rsa.pub for passwordless login
111
+ ssh-keygen -t rsa -A # Generate /etc/ssh/ssh_host_rsa_key and ssh_host_ecdsa_key for SSH listening
112
+ /usr/sbin/sshd -p 36005 -o ListenAddress=0.0.0.0 # Start SSH listening
113
+ echo "Port 36005" > ~/.ssh/config # Set SSH connection port to 36005
114
+ passwd root # Set the root password to avoid monitoring platform alerts
115
+ ```
116
+
117
+ Note: `36005` is an example port. You may use any available port, but ensure it is **open** and **not occupied by other processes**.
118
+
119
+ Next, in each machine's container, execute:
120
+
121
+ ```sh
122
+ cat ~/.ssh/id_rsa.pub
123
+ ```
124
+
125
+ **Copy the output SSH public key and paste it into the `~/.ssh/authorized_keys` file, one key per line. This must be done on every machine.** In the end, the `~/.ssh/authorized_keys` file on each machine should be identical and contain the public keys of all machines.
126
+
127
+ Please note that for multi-node training, the code executed on each node must be identical. It is recommended to mount a shared network drive. If this is not possible, you must manually copy the dataset, scripts, and code to the same directory on each machine.
128
+
129
+ ### Launch Methods
130
+
131
+ This repository provides three fine-tuning approaches:
132
+
133
+ - **DeepSpeed Native Fine-tuning** (based on HuggingFace Transformers Trainer): Located in the `deepspeed_support` directory
134
+ - **LLaMA-Factory Fine-tuning**: Located in the `llama_factory_support` directory
135
+ - **ms-swift Fine-tuning**: Located in the `ms_swift_support` directory
136
+
137
+ #### DeepSpeed Native Fine-tuning
138
+
139
+ Reference: [HuggingFace Transformers Trainer](https://huggingface.co/docs/transformers/main/en/main_classes/trainer)
140
+
141
+ The `deepspeed_support` directory provides the following launch scripts:
142
+
143
+ | Scenario | Launch Script | Training Script | Description |
144
+ |---|---|---|---|
145
+ | VL Fine-tuning (image/video/text) | `bash train_vl.sh` | `train_vl.py` | Multimodal fine-tuning (ViT frozen) |
146
+ | Pure Text Fine-tuning | `bash train_llm.sh` | `train_vl.py` | Pure text training using VL checkpoint (ViT frozen) |
147
+
148
+ ##### Single-Machine Fine-tuning
149
+
150
+ First, install dependencies (`requirements.txt` is located in the `finetune` root directory):
151
+
152
+ ```sh
153
+ pip install -r ../requirements.txt
154
+ ```
155
+
156
+ Then, in the `deepspeed_support` directory, execute:
157
+
158
+ ```sh
159
+ # Pure text fine-tuning
160
+ bash train_llm.sh
161
+
162
+ # VL multimodal fine-tuning
163
+ bash train_vl.sh
164
+ ```
165
+
166
+ ##### Multi-Machine Fine-tuning
167
+
168
+ To launch fine-tuning across multiple machines, please first complete the configuration in [Configure Passwordless SSH Login Between Machines](#configure-passwordless-ssh-login-between-machines-multi-machine-training), and ensure all machines are within the same cluster.
169
+
170
+ Confirm that dependencies are installed (if not, run `pip install -r ../requirements.txt`), then modify the following configuration at the beginning of the corresponding launch script (e.g., `train_llm.sh` or `train_vl.sh`):
171
+
172
+ ```sh
173
+ export HOST_GPU_NUM=8
174
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
175
+ IP_LIST=${IP_LIST:-"127.0.0.1"}
176
+ ```
177
+
178
+ Note: If the `IP_LIST` environment variable is not set, replace `IP_LIST` with the IP list! The format is:
179
+ ```text
180
+ For a single IP:
181
+ IP_LIST=${ip_1}
182
+
183
+ For multiple IPs:
184
+ IP_LIST=${ip_1},${ip_2}
185
+
186
+ ```
187
+
188
+ Replace `${ip_1}` and `${ip_2}` with the actual IP addresses.
189
+
190
+ Then, on the machine with `${ip1}`, execute the corresponding launch script in the `deepspeed_support/` directory (e.g., `bash train_llm.sh` or `bash train_vl.sh`). On first launch, you may see the following output:
191
+
192
+ ```text
193
+ The authenticity of host '[ip]:36005 ([ip]:36005)' can't be established.
194
+ ECDSA key fingerprint is xxxxxx.
195
+ ECDSA key fingerprint is MD5:xxxxxx.
196
+ Are you sure you want to continue connecting (yes/no)?
197
+ ```
198
+
199
+ Type `yes` to continue.
200
+
201
+ ##### Key Parameters
202
+
203
+ The key parameters in the script are as follows:
204
+
205
+ - `--deepspeed`: Path to the DeepSpeed configuration file. Four default DeepSpeed configuration files are provided in the `deepspeed_support` folder: `ds_zero2_no_offload.json`, `ds_zero2_offload.json`, `ds_zero3_no_offload.json`, and `ds_zero3_offload.json`, with different ZeRO stages (ZeRO-2 / ZeRO-3) and offload strategies selectable based on available GPU memory and communication constraints.
206
+ - `--model_name_or_path`: Path to the Hy4 flash HF pre-trained model weights to load, otherwise loading will fail.
207
+ - `--tokenizer_name_or_path`: Path to the tokenizer folder, otherwise loading will fail.
208
+ - `--train_data_file`: Path to the training file, which should be a jsonl file.
209
+ - `--output_dir`: Output directory where logs, tensorboard files, and model weights will be stored.
210
+ - `--per_device_train_batch_size`: Batch size per GPU.
211
+ - `--gradient_accumulation_steps`: Number of gradient accumulation steps. The global batch size is `per_device_train_batch_size * gradient_accumulation_steps * dp_size`.
212
+ - `--max_steps`: Total number of training steps.
213
+ - `--save_steps`: Number of steps between saving checkpoints.
214
+ - `--gradient_checkpointing`: Enable gradient checkpointing.
215
+ - `--learning_rate`: Maximum learning rate during training.
216
+ - `--min_lr`: Minimum learning rate during training (absolute value, e.g. `1e-6`; if not specified, defaults to `0.01` and is computed as `learning_rate * min_lr` ratio).
217
+ - `--use_flash_attn`: Enable flash-attention for accelerated training.
218
+
219
+ ##### What if GPU Memory is Insufficient?
220
+
221
+ Reference: [DeepSpeed Configuration](https://www.deepspeed.ai/docs/config-json/)
222
+
223
+ You can try modifying the DeepSpeed configuration by removing the `auto` attribute from the following parameters and reducing their values:
224
+
225
+ - `stage3_param_persistence_threshold`
226
+ - `stage3_prefetch_bucket_size`
227
+ - `stage3_max_reuse_distance`
228
+
229
+ ##### Multimodal (VL) Fine-tuning
230
+
231
+ `train_vl.sh` supports three types of data simultaneously: image + text, video + text, and pure text. The data format is described in [Multimodal Data Format](#multimodal-data-format-image--video).
232
+
233
+ VL-specific parameters (in addition to the parameters listed above):
234
+
235
+ - `--freeze_vit`: Freeze the ViT vision encoder during training (enabled by default, i.e. `default=True`). This prevents the vision encoder weights from being updated, focusing training on the LLM and projector. To train the vision encoder, explicitly set `--freeze_vit false`.
236
+ - `--train_data_file`: Path to the JSONL training file containing multimodal data (with `images` / `videos` fields).
237
+ - `--image_base_dir`: Base directory for resolving relative image/video paths in the data file. If not specified, paths are resolved relative to the working directory.
238
+
239
+ **Image processing parameters** (aligned with HuggingFace processor's `images_kwargs` API):
240
+
241
+ - `--image_min_pixels`: Minimum pixel count for image resizing. Maps to `size["shortest_edge"]` in the HF image processor. Default `None` (uses the value from `preprocessor_config.json`, default 4096).
242
+ - `--image_max_pixels`: Maximum pixel count for image resizing. Maps to `size["longest_edge"]` in the HF image processor. Default `None` (uses the value from `preprocessor_config.json`, default 4194304).
243
+
244
+ **Video processing parameters** (aligned with HuggingFace processor's `videos_kwargs` API):
245
+
246
+ - `--video_max_pixels`: Total pixel budget for all video frames. Maps to `size["longest_edge"]` in the HF video processor. Default `None` (uses the value from `video_preprocessor_config.json`). When set, the per-frame pixel budget is `video_max_pixels / num_frames`, which controls the number of tokens generated by the video.
247
+ - `--video_max_frames`: Maximum number of frames to sample from a video. Maps to `max_frames` in the HF video processor. Default `None`.
248
+ - `--video_fps`: Target frame rate for video sampling (frames per second). Maps to `fps` in the HF video processor. Default `None`.
249
+
250
+ > **Video parameters:** Videos generate significantly more tokens than images. You should set video parameters appropriately based on `--max_seq_length` to ensure video tokens do not exceed the sequence length limit.
251
+
252
+ #### LLaMA-Factory Fine-tuning
253
+
254
+ If you are familiar with LLaMA-Factory, you can use it for fine-tuning. All scripts, code, and configuration files are located in the `llama_factory_support` directory. Unless otherwise specified, all files mentioned below are located in this directory.
255
+
256
+ ##### Installation
257
+
258
+ You can install LLaMA-Factory by downloading the source code from https://github.com/hiyouga/LLaMA-Factory/tree/main and following the instructions on the website.
259
+
260
+ ##### Configuration Files
261
+
262
+ We provide sample LLaMA-Factory fine-tuning configuration files for different scenarios:
263
+
264
+ | Configuration File | Scenario |
265
+ |---|---|
266
+ | `hy_v4_flash_full_sft.yaml` | LLM Full Fine-tuning |
267
+ | `hy_v4_flash_lora_sft.yaml` | LLM LoRA Fine-tuning |
268
+ | `hy_v4_flash_vl_full_sft.yaml` | VL Full Fine-tuning |
269
+ | `hy_v4_flash_vl_lora_sft.yaml` | VL LoRA Fine-tuning |
270
+
271
+ Key parameters in the configuration files are as follows:
272
+
273
+ **Model:**
274
+
275
+ - `model_name_or_path`: Path to the Hy4 flash HF format pre-trained model weights
276
+ - `trust_remote_code`: Whether to trust remote code; Hy4 flash requires this to be set to `true`
277
+
278
+ **Training Method:**
279
+
280
+ - `stage`: Training stage, currently `sft` (supervised fine-tuning)
281
+ - `finetuning_type`: Fine-tuning type, either `full` (full fine-tuning) or `lora` (LoRA fine-tuning)
282
+ - `deepspeed`: DeepSpeed configuration file path; `../deepspeed_support/ds_zero3_offload.json` is recommended for full fine-tuning
283
+ - `fsdp` + `fsdp_config`: FSDP distributed strategy; recommended for LoRA fine-tuning (configuration is built into `hy_v4_flash_lora_sft.yaml`); mutually exclusive with DeepSpeed
284
+
285
+ > **Distributed Strategy Recommendations:**
286
+ > - **FSDP**: Recommended for LoRA fine-tuning, good compatibility and simple configuration
287
+ > - **DeepSpeed ZeRO-3 + Offload**: Recommended for full fine-tuning or memory-constrained scenarios
288
+
289
+ **LoRA Parameters (only effective during LoRA fine-tuning):**
290
+
291
+ - `lora_rank`: LoRA rank, default `64`
292
+ - `lora_alpha`: LoRA alpha coefficient, default `128`
293
+ - `lora_dropout`: LoRA dropout ratio, default `0.05`
294
+ - `lora_target`: Target modules for LoRA, default `q_a_proj,q_b_proj,kv_a_proj_with_mqa,kv_b_proj,o_proj`
295
+
296
+ **Dataset:**
297
+
298
+ - `dataset_dir`: Dataset directory path
299
+ - `dataset`: Dataset name, must be registered in `dataset_info.json` under `dataset_dir`
300
+ - `template`: Chat template; Hy4 flash uses `hy_v4_flash`
301
+ - `cutoff_len`: Maximum sequence length; sequences exceeding this will be truncated. For LoRA fine-tuning, a smaller value is recommended to save memory
302
+ - `max_samples`: Maximum number of samples per dataset
303
+ - `overwrite_cache`: Whether to overwrite cached preprocessed datasets
304
+
305
+ **Output:**
306
+
307
+ - `output_dir`: Output directory where logs, TensorBoard files, and weights will be stored
308
+ - `logging_steps`: Number of steps between logging
309
+ - `save_steps`: Number of steps between saving checkpoints
310
+ - `plot_loss`: Whether to plot the training loss curve
311
+ - `overwrite_output_dir`: Whether to overwrite the existing output directory
312
+ - `save_only_model`: Whether to save only model weights (excluding optimizer states, etc.)
313
+ - `report_to`: Logging tool, options: `none`, `wandb`, `tensorboard`, `swanlab`, `mlflow`
314
+
315
+ **Training Hyperparameters:**
316
+
317
+ - `per_device_train_batch_size`: Batch size per GPU
318
+ - `gradient_accumulation_steps`: Gradient accumulation steps; `per_device_train_batch_size * gradient_accumulation_steps * dp_size` equals the global batch size
319
+ - `learning_rate`: Maximum learning rate; `1.0e-5` recommended for full fine-tuning, `2.0e-4` for LoRA fine-tuning
320
+ - `num_train_epochs`: Number of training epochs
321
+ - `lr_scheduler_type`: Learning rate scheduler type; `cosine_with_min_lr` is recommended
322
+ - `lr_scheduler_kwargs.min_lr_rate`: Ratio of minimum to maximum learning rate; e.g., `0.1` means the minimum learning rate is 10% of the maximum
323
+ - `warmup_steps`: Number of warmup steps
324
+ - `bf16`: Whether to use BFloat16 mixed precision training
325
+ - `gradient_checkpointing`: Whether to enable gradient checkpointing to save memory
326
+ - `ddp_timeout`: Distributed training timeout (milliseconds)
327
+ - `flash_attn`: Attention implementation; options include `auto` (automatic selection), `sdpa`, `disabled`, etc. The actual default depends on the configuration file (e.g. LLM full fine-tuning uses `disabled`, LLM LoRA uses `auto`, VL full fine-tuning uses `auto`, VL LoRA uses `disabled`)
328
+ - `resume_from_checkpoint`: Resume training from a specified checkpoint path; set to `null` to start from scratch
329
+
330
+ **Image/Video Processing Parameters (effective for VL fine-tuning):**
331
+
332
+ The following parameters are determined by the model's built-in processor configuration (see `hy_v4_template.py`), with defaults listed below. To adjust, you may override them in the YAML configuration file (if supported by your LLaMA-Factory version) or modify the processor configuration:
333
+
334
+ - `image_max_pixels`: Maximum pixel count for image resizing, default `4194304` (2048×2048)
335
+ - `image_min_pixels`: Minimum pixel count for image resizing, default `4096` (64×64)
336
+ - `video_max_pixels`: Maximum pixel count per video frame, default `65536` (256×256)
337
+ - `video_min_pixels`: Minimum pixel count per video frame, default `256` (16×16)
338
+ - `video_fps`: Video sampling frame rate, default `2.0`
339
+ - `video_maxlen`: Maximum number of sampled video frames, default `128`
340
+
341
+ > **Video parameter note:** Videos produce significantly more tokens than images. Set video parameters appropriately based on `cutoff_len` to ensure video tokens do not exceed the sequence length limit. Lowering `video_max_pixels` or `video_maxlen` can significantly reduce GPU memory usage.
342
+
343
+ ##### Launch Scripts
344
+
345
+ We provide dedicated launch scripts for different scenarios:
346
+
347
+ | Scenario | Launch Script | Default Configuration |
348
+ |---|---|---|
349
+ | VL Full Fine-tuning | `bash train_lf.sh` | `hy_v4_flash_vl_full_sft.yaml` |
350
+ | VL LoRA Fine-tuning | `bash train_lf_lora.sh` | `hy_v4_flash_vl_lora_sft.yaml` |
351
+ | LLM Full Fine-tuning | `bash train_lf_llm.sh` | `hy_v4_flash_full_sft.yaml` |
352
+ | LLM LoRA Fine-tuning | `bash train_lf_lora_llm.sh` | `hy_v4_flash_lora_sft.yaml` |
353
+
354
+ Each script supports overriding the default configuration file via the `YAML_FILE` environment variable.
355
+
356
+ ##### Launch Fine-tuning
357
+
358
+ For multi-machine fine-tuning, please first complete the configuration in [Configure Passwordless SSH Login Between Machines](#configure-passwordless-ssh-login-between-machines-multi-machine-training) (single-machine fine-tuning can skip this step).
359
+
360
+ Modify the following configuration at the beginning of the corresponding launch script:
361
+
362
+ ```sh
363
+ export HOST_GPU_NUM=8
364
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
365
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
366
+ ```
367
+
368
+ Note:
369
+ 1. If the `IP_LIST` environment variable is not set, replace `IP_LIST` with the IP list! The format is:
370
+ ```text
371
+ For a single IP:
372
+ IP_LIST=${ip_1}
373
+
374
+ For multiple IPs:
375
+ IP_LIST=${ip_1},${ip_2}
376
+
377
+ ```
378
+ Replace `${ip_1}` and `${ip_2}` with the actual IP addresses.
379
+
380
+ 2. To specify a fine-tuning configuration file, set the `YAML_FILE` environment variable (absolute paths are recommended, or `cd` into the `llama_factory_support/` directory before running). For example:
381
+ ```sh
382
+ export YAML_FILE=/path/to/llama_factory_support/hy_v4_flash_lora_sft.yaml
383
+ ```
384
+
385
+ For multi-machine training, each machine needs to run a `torchrun` process. The launch script has built-in `pdsh` support for one-click distribution — you only need to execute it on one machine and it will automatically distribute to all nodes:
386
+
387
+ ```sh
388
+ # VL full fine-tuning
389
+ bash train_lf.sh
390
+
391
+ # LLM full fine-tuning
392
+ bash train_lf_llm.sh
393
+ ```
394
+
395
+ > **Note:** If your environment does not support `pdsh`, or you need a custom launch method, please manually execute the launch script on each machine separately. When doing so, you must set `_WORKER_MODE=1` to skip the pdsh distribution branch, for example:
396
+ > ```bash
397
+ > # On each machine:
398
+ > export IP_LIST="10.0.0.1,10.0.0.2"
399
+ > export _WORKER_MODE=1
400
+ > bash train_lf.sh
401
+ > ```
402
+
403
+ ##### Multimodal (VL) Fine-tuning
404
+
405
+ Key differences between VL and pure text configuration:
406
+
407
+ - `template`: Set to `hy_v4_flash_vl` (instead of `hy_v4_flash`)
408
+ - `dataset`: Use a VL dataset registered in `dataset_info.json`, which must include an `images` column; to also support video, include a `videos` column
409
+ - `freeze_vision_tower`: Set to `true` to freeze the ViT vision encoder during training (recommended)
410
+
411
+ To register a custom VL dataset, add an entry to `dataset_info.json`:
412
+
413
+ ```json
414
+ {
415
+ "my_vl_dataset": {
416
+ "file_name": "/path/to/my_vl_data.jsonl",
417
+ "formatting": "sharegpt",
418
+ "columns": {
419
+ "messages": "messages",
420
+ "images": "images",
421
+ "videos": "videos"
422
+ },
423
+ "tags": {
424
+ "role_tag": "role",
425
+ "content_tag": "content",
426
+ "user_tag": "user",
427
+ "assistant_tag": "assistant",
428
+ "system_tag": "system"
429
+ }
430
+ }
431
+ }
432
+ ```
433
+
434
+ #### ms-swift Fine-tuning
435
+
436
+ If you are familiar with ms-swift, you can use it for fine-tuning. All scripts and code are located in the `ms_swift_support` directory. Unless otherwise specified, all files mentioned below are located in this directory.
437
+
438
+ ##### Installation
439
+
440
+ You can install ms-swift via pip:
441
+
442
+ ```sh
443
+ pip install ms-swift
444
+ ```
445
+
446
+ Or install from source: https://github.com/modelscope/ms-swift
447
+
448
+ ##### Fine-tuning Scripts
449
+
450
+ | Scenario | Fine-tuning Method | Launch Script |
451
+ |---|---|---|
452
+ | Pure Text (LLM) | Full Fine-tuning | `bash sft_train.sh` |
453
+ | Pure Text (LLM) | LoRA Fine-tuning | `bash sft_train_lora.sh` |
454
+ | Multimodal (VL) | Full Fine-tuning | `bash sft_train_vl.sh` |
455
+ | Multimodal (VL) | LoRA Fine-tuning | `bash sft_train_vl_lora.sh` |
456
+
457
+ > **Note:** Training parameters for ms-swift are configured directly as CLI arguments in the launch scripts. To modify parameters, edit the corresponding launch script directly.
458
+
459
+ ##### Key Parameters
460
+
461
+ Key parameters in the launch scripts are as follows:
462
+
463
+ **Model:**
464
+
465
+ - `model`: Model path, can be a HuggingFace Hub ID or a local path
466
+ - `model_type`: Model type, set to `hy_v4_flash_vl`
467
+ - `template`: Chat template, set to `hy_v4_flash_vl`
468
+ - `torch_dtype`: Data type, `bfloat16` is recommended
469
+ - `attn_impl`: Attention implementation, default `eager`
470
+
471
+ **Training Method:**
472
+
473
+ - `tuner_type`: Fine-tuning type; set to `full` for full fine-tuning, `lora` for LoRA fine-tuning
474
+ - `lora_rank`: LoRA rank, default `64`
475
+ - `lora_alpha`: LoRA alpha coefficient, default `128`
476
+ - `lora_dropout`: LoRA dropout ratio, default `0.05`
477
+ - `target_modules`: Target modules for LoRA, default MLA projection layers: `q_a_proj`, `q_b_proj`, `kv_a_proj_with_mqa`, `kv_b_proj`, `o_proj`
478
+
479
+ **Dataset:**
480
+
481
+ - `dataset`: Dataset path, supports local jsonl files (sharegpt format)
482
+ - `max_length`: Maximum sequence length; sequences exceeding this will be truncated
483
+ - `lazy_tokenize`: Whether to use lazy tokenization, `true` is recommended
484
+
485
+ **Output:**
486
+
487
+ - `output_dir`: Output directory
488
+ - `save_steps`: Number of steps between saving checkpoints
489
+ - `save_total_limit`: Maximum number of checkpoints to keep
490
+ - `logging_steps`: Number of steps between logging
491
+ - `report_to`: Logging tool, options: `none`, `wandb`, `tensorboard`, `swanlab`, `mlflow`
492
+
493
+ **Training Hyperparameters:**
494
+
495
+ - `per_device_train_batch_size`: Batch size per GPU
496
+ - `gradient_accumulation_steps`: Gradient accumulation steps
497
+ - `learning_rate`: Maximum learning rate; `1.0e-5` recommended for full fine-tuning, `2.0e-4` for LoRA fine-tuning
498
+ - `num_train_epochs`: Number of training epochs
499
+ - `lr_scheduler_type`: Learning rate scheduler type, `cosine` is recommended
500
+ - `warmup_steps`: Number of warmup steps
501
+ - `bf16`: Whether to use BFloat16 mixed precision training
502
+
503
+ **Distributed Strategy / Optimization:**
504
+
505
+ - `deepspeed`: DeepSpeed configuration file path; the scripts default to `../deepspeed_support/ds_zero3_offload.json`; ZeRO-3 + Offload is recommended for full fine-tuning
506
+ - `fsdp`: FSDP configuration file path (JSON); recommended for LoRA fine-tuning; mutually exclusive with DeepSpeed
507
+ - `gradient_checkpointing`: Whether to enable gradient checkpointing
508
+ - `max_grad_norm`: Gradient clipping threshold
509
+
510
+ > **Distributed Strategy Recommendations:**
511
+ > - **FSDP**: Recommended for LoRA fine-tuning, good compatibility and simple configuration
512
+ > - **DeepSpeed ZeRO-3 + Offload**: Recommended for full fine-tuning or memory-constrained scenarios
513
+
514
+ **Other:**
515
+
516
+ - `ddp_timeout`: Distributed training timeout (milliseconds)
517
+ - `seed`: Random seed
518
+ - `resume_from_checkpoint`: Resume training from a specified checkpoint path
519
+
520
+ **Image/Video Processing Parameters (effective for VL fine-tuning):**
521
+
522
+ Image/video processing in ms-swift is controlled by the model's built-in image processor and video processor. Parameter defaults come from `preprocessor_config.json` and `video_preprocessor_config.json` in the model directory (the actual values depend on the specific checkpoint). To adjust these values, directly modify the corresponding JSON configuration files in the model directory.
523
+
524
+ Image processing parameters (`preprocessor_config.json`):
525
+
526
+ - `min_pixels`: Minimum pixel count for image resizing, default `4096`
527
+ - `max_pixels`: Maximum pixel count for image resizing, default `4194304` (2048×2048)
528
+
529
+ Video processing parameters (`video_preprocessor_config.json`):
530
+
531
+ - `size.longest_edge`: Total pixel budget for all video frames, default `1572864`
532
+ - `max_image_size.longest_edge`: Maximum pixel count per single video frame, default `262144`
533
+ - `max_frames`: Maximum number of sampled video frames, default `2`
534
+ - `fps`: Target frame rate for video sampling (frames per second), default `2`
535
+ - `do_sample_frames`: Whether to enable frame sampling, default `true`
536
+
537
+ > **Video parameter note:** Videos produce significantly more tokens than images. Set video parameters appropriately based on `max_length` to ensure video tokens do not exceed the sequence length limit. Lowering `size.longest_edge`, `max_image_size.longest_edge`, or `max_frames` can significantly reduce GPU memory usage.
538
+
539
+ ##### Launch Fine-tuning
540
+
541
+ For multi-machine fine-tuning, please first complete the configuration in [Configure Passwordless SSH Login Between Machines](#configure-passwordless-ssh-login-between-machines-multi-machine-training) (single-machine fine-tuning can skip this step).
542
+
543
+ Modify the following configuration at the beginning of the corresponding launch script:
544
+
545
+ ```sh
546
+ export HOST_GPU_NUM=8
547
+ # IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
548
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
549
+ ```
550
+
551
+ For multi-machine training, each machine needs to run a `torchrun` process. The launch script has built-in `pdsh` support for one-click distribution — you only need to execute it on one machine and it will automatically distribute to all nodes:
552
+
553
+ ```sh
554
+ # Pure text full fine-tuning
555
+ bash sft_train.sh
556
+
557
+ # Pure text LoRA fine-tuning
558
+ bash sft_train_lora.sh
559
+
560
+ # VL full fine-tuning with DeepSpeed ZeRO-3
561
+ bash sft_train_vl.sh
562
+
563
+ # VL LoRA fine-tuning with FSDP
564
+ bash sft_train_vl_lora.sh
565
+
566
+ # Multi-machine training (one-click distribution via pdsh, execute on one machine only)
567
+ IP_LIST="10.0.0.1,10.0.0.2" bash sft_train.sh
568
+ ```
569
+
570
+ > **Note:** If your environment does not support `pdsh`, or you need a custom launch method, please manually execute the launch script on each machine separately. When doing so, you must set `_WORKER_MODE=1` to skip the pdsh distribution branch, for example:
571
+ > ```bash
572
+ > # On each machine:
573
+ > export IP_LIST="10.0.0.1,10.0.0.2"
574
+ > export _WORKER_MODE=1
575
+ > bash sft_train.sh
576
+ > ```
577
+
578
+ ##### Multimodal (VL) Fine-tuning
579
+
580
+ The key differences between VL and pure text scripts are:
581
+
582
+ - `model_type`: Set to `hy_v4_flash_vl` (instead of `hy_v4_flash`)
583
+ - `template`: Set to `hy_v4_flash_vl` (instead of `hy_v4_flash`)
584
+ - `dataset`: Point to a JSONL file containing multimodal data (with `images` / `videos` fields)
finetune/README_CN.md ADDED
@@ -0,0 +1,584 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <p align="left">
2
+ <a href="README.md">English</a> | 中文
3
+ </p>
4
+
5
+ # 模型微调
6
+
7
+ Hy4 flash 提供了模型微调相关流程,您可以在此章节对训练数据格式进行处理以供模型微调使用。
8
+
9
+ ## 训练数据格式及处理
10
+
11
+ **Hy4 flash 支持两种思考模式,可通过 `reasoning_effort` 参数控制(仅支持 `no_think` 与 `high`,传入其他值会报错)**:
12
+
13
+ | `reasoning_effort` | 模式 | 说明 |
14
+ |---|---|---|
15
+ | `no_think` | 快思考 | 不进行推理过程,模型直接输出答案。 |
16
+ | `high` | 深度思考 | 充分推理,思考过程详细。 |
17
+
18
+ 训练数据按照以下形式处理为 messages 格式,训练和推理的默认 system prompt 为空,可以根据自己的需求进行设定。
19
+
20
+ 对于 `high` 模式,assistant 的推理过程可放在 `reasoning_content` 字段中(`chat_template` 也支持 `reasoning` 字段且优先级更高);`no_think` 模式下模板始终输出空的 `<think></think>`,无需提供推理字段。
21
+
22
+ ```jsonl
23
+ # 快思考模式 (no_think) — 无推理字段
24
+ {"reasoning_effort": "no_think", "messages": [{"content": "你是一个有用的人工智能助手。\n现在的时间是2026-01-01 13:26:12 周四", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2"}]}
25
+
26
+ # 深度思考模式 (high) — 包含推理过程
27
+ {"reasoning_effort": "high", "messages": [{"content": "你是一个有用的人工智能助手。\n现在的时间是2026-01-01 13:26:12 周四", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2", "reasoning_content": "用户问的是1+1等于多少。在基本的十进制算术中,1+1等于2。"}]}
28
+ ```
29
+
30
+ > **`reasoning_effort` 默认值说明:**
31
+ > - **训练时**:如果训练数据中未指定 `reasoning_effort`,训练脚本默认使用 `no_think`(快思考)。这是因为训练脚本会将不含推理字段的样本视为非思考模式数据。
32
+ > - **推理时**:如果调用 `apply_chat_template` 时未传入 `reasoning_effort` 参数,`chat_template` 默认使用 `high`(深度思考)。
33
+
34
+ 使用 `apply_chat_template` 进行 tokenize 的示例:
35
+
36
+ ```python
37
+ from transformers import AutoTokenizer
38
+ tokenizer = AutoTokenizer.from_pretrained("./models", use_fast=False, trust_remote_code=True)
39
+
40
+ messages = [
41
+ {"content": "你是一个有用的人工智能助手。", "role": "system"},
42
+ {"content": "1+1=?", "role": "user"},
43
+ {"role": "assistant", "content": "1+1=2"}
44
+ ]
45
+ ids = tokenizer.apply_chat_template(messages, tokenize=True, reasoning_effort="no_think")
46
+ ```
47
+
48
+ ### 多模态数据格式(图片 / 视频)
49
+
50
+ Hy4 flash 支持图片、视频和纯文本的混合训练。数据格式为 JSONL,每行一个样本。
51
+
52
+ #### 图片数据
53
+
54
+ 在 user 消息中使用 `<image>` 标签标记图片位置,并在 `images` 字段中提供图片路径列表:
55
+
56
+ ```json
57
+ {"messages": [{"role": "system", "content": "你是一个有用的人工智能助手。"}, {"role": "user", "content": "<image>请详细描述这张图片。"}, {"role": "assistant", "content": "图片展示了一幅美丽的海上日落景象。"}], "images": ["test_images/sunset.jpg"]}
58
+ ```
59
+
60
+ 支持多张图片 —— 使用多个 `<image>` 标签,并在 `images` 列表中按顺序提供对应的路径:
61
+
62
+ ```json
63
+ {"messages": [{"role": "user", "content": "<image>请比较第一张图片和第二张图片。<image>"}], "images": ["img1.jpg", "img2.jpg"]}
64
+ ```
65
+
66
+ #### 视频数据
67
+
68
+ 在 user 消息中使用 `<video>` 标签标记视频位置,并在 `videos` 字段中提供视频路径列表:
69
+
70
+ ```json
71
+ {"messages": [{"role": "system", "content": "你是一个有用的人工智能助手。"}, {"role": "user", "content": "<video>请描述这个视频中发生了什么。"}, {"role": "assistant", "content": "视频展示了一只猫在硬木地板上玩毛线球。"}], "videos": ["test_videos/cat.mp4"]}
72
+ ```
73
+
74
+ #### 纯文本数据(混合训练)
75
+
76
+ 不包含 `<image>` 或 `<video>` 标签的样本会被当作纯文本处理,可以与多模态数据混合在同一个 JSONL 文件中:
77
+
78
+ ```json
79
+ {"messages": [{"role": "system", "content": "你是一个有用的人工智能助手。"}, {"role": "user", "content": "法国的首都是哪里?"}, {"role": "assistant", "content": "法国的首都是巴黎。"}]}
80
+ ```
81
+
82
+ #### 注意事项
83
+
84
+ - 图片/视频路径可以是**绝对路径**或**相对路径**(相对于 JSONL 文件所在目录或工作目录,具体取决于训练框架)。
85
+ - 消息中的 `<image>` / `<video>` 标签数量必须与 `images` / `videos` 列表中的路径数量一致。
86
+ - 单个样本可以同时包含 `images` 和 `videos` 字段。
87
+ - 支持的图片格式取决于 [Pillow](https://pillow.readthedocs.io/) 库(常见格式如 JPEG、PNG 等均支持)。支持的视频格式取决于视频解码后端(默认为 [PyAV](https://pyav.org/),基于 FFmpeg,常见格式如 MP4 等均支持)。
88
+ - 纯文本示例数据位于 `data/example_data.jsonl`,多模态示例数据位于 `data/example_vl_data.jsonl`。
89
+
90
+ ## 微调流程
91
+
92
+ ### 硬件需求
93
+
94
+ 经过测试,最小的资源配置如下:
95
+
96
+ - **LoRA 微调**:最少需要 8 机 64 卡,每卡显存至少 96GB,每机 CPU 内存至少 2TB。
97
+ - **全量微调**:最少需要 16 机 128 卡,每卡显存至少 96GB,每机 CPU 内存至少 2TB。
98
+
99
+ > 注:以上为最小资源配置,实际所需资源会随 max_seq_length、batch size、图片分辨率、视频帧数等因素相应增加。多模态数据(尤其是高分辨率图片和视频)的显存消耗显著高于纯文本。
100
+
101
+ ### 配置机器间免密 ssh 登录(多机训练)
102
+
103
+ > 如果只使用单机训练,可跳过本节。
104
+
105
+ 以下操作以两个机器为例,两台机器的 ip 分别以`${ip1}`和`${ip2}`标识,以下操作均在 docker container 内执行。
106
+
107
+ 首先,配置多机container免密,在每台机器上执行。
108
+
109
+ ```sh
110
+ ssh-keygen # 生成id_rsa和id_rsa.pub,用于免密登录
111
+ ssh-keygen -t rsa -A # 生成/etc/ssh/ssh_host_rsa_key和ssh_host_ecdsa_key, 用于后面启动ssh listen
112
+ /usr/sbin/sshd -p 36005 -o ListenAddress=0.0.0.0 # 启动 SSH 监听
113
+ echo "Port 36005" > ~/.ssh/config # ssh 连接端口修改为 36005
114
+ passwd root # 需要配置root密码,否则监测平台会报警
115
+ ```
116
+
117
+ 注意:这里的`36005`是一个示例端口,可以选用任意端口,但需要保证使用的端口**开放**且**不被其他的进程占用**。
118
+
119
+ 接下来,在每台机器的 container 内,执行:
120
+
121
+ ```sh
122
+ cat ~/.ssh/id_rsa.pub
123
+ ```
124
+
125
+ **将输出的 ssh 公钥复制并粘贴到`~/.ssh/authorized_keys`文件中,每行一个公钥,每台机器上都要做这个操作**。最终每台机器上的`~/.ssh/authorized_keys`文件内容应当是一致的,并且包含了所有机器的公钥。
126
+
127
+ 需要注意,多节点训练时,每个节点上执行的代码都得一致,建议挂载一个共享的网络盘,如果无法挂载共享网盘,则需要手动将数据集、脚本、代码复制在多台机器的相同目录下。
128
+
129
+ ### 启动方式
130
+
131
+ 本项目提供三种微调方式,您可以根据需求选择:
132
+
133
+ - **DeepSpeed 原生微调**(基于 HuggingFace Transformers Trainer):位于 `deepspeed_support` 目录下
134
+ - **LLaMA-Factory 微调**:位于 `llama_factory_support` 目录下
135
+ - **ms-swift 微调**:位于 `ms_swift_support` 目录下
136
+
137
+ #### DeepSpeed 原生微调
138
+
139
+ 参考:[HuggingFace Transformers Trainer](https://huggingface.co/docs/transformers/main/en/main_classes/trainer)
140
+
141
+ `deepspeed_support` 目录下提供了以下启动脚本:
142
+
143
+ | 场景 | 启动脚本 | 训练脚本 | 说明 |
144
+ |---|---|---|---|
145
+ | VL 微调(图片/视频/文本) | `bash train_vl.sh` | `train_vl.py` | 多模态微调(冻结 ViT) |
146
+ | 纯文本微调 | `bash train_llm.sh` | `train_vl.py` | 使用 VL checkpoint 进行纯文本训练(冻结 ViT) |
147
+
148
+ ##### 单机启动微调
149
+
150
+ 首先安装依赖(`requirements.txt` 位于 `finetune` 根目录下):
151
+
152
+ ```sh
153
+ pip install -r ../requirements.txt
154
+ ```
155
+
156
+ 然后在 `deepspeed_support` 目录下,执行:
157
+
158
+ ```sh
159
+ # 纯文本微调
160
+ bash train_llm.sh
161
+
162
+ # VL 多模态微调
163
+ bash train_vl.sh
164
+ ```
165
+
166
+ ##### 多机启动微调
167
+
168
+ 如果要用多台机器启动微调,请先完成 [配置机器间免密 ssh 登录](#配置机器间免密-ssh-登录多机训练) 中的配置,并保证多台机器在一个集群内。
169
+
170
+ 确认依赖已经安装完成(如未安装,请执行 `pip install -r ../requirements.txt` 安装),然后修改对应启动脚本(如 `train_llm.sh` 或 `train_vl.sh`)中开头的以下配置:
171
+
172
+ ```sh
173
+ export HOST_GPU_NUM=8
174
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
175
+ IP_LIST=${IP_LIST:-"127.0.0.1"}
176
+ ```
177
+
178
+ 注意:如果`IP_LIST`环境变量未设置,则将`IP_LIST`替换为IP列表!格式为:
179
+ ```text
180
+ 如果只有一个IP:
181
+ IP_LIST=${ip_1}
182
+
183
+ 如果有多个IP:
184
+ IP_LIST=${ip_1},${ip_2}
185
+
186
+ ```
187
+
188
+ 请将`${ip_1}`和`${ip_2}`替换为真实的IP地址。
189
+
190
+ 然后,在`${ip1}`的机器上,在`deepspeed_support/`目录下,执行对应的启动脚本即可(如 `bash train_llm.sh` 或 `bash train_vl.sh`)。注意第一次启动时可能会看见以下的输出:
191
+
192
+ ```text
193
+ The authenticity of host '[ip]:36005 ([ip]:36005)' can't be established.
194
+ ECDSA key fingerprint is xxxxxx.
195
+ ECDSA key fingerprint is MD5:xxxxxx.
196
+ Are you sure you want to continue connecting (yes/no)?
197
+ ```
198
+
199
+ 此时输入`yes`即可继续。
200
+
201
+ ##### 关键参数
202
+
203
+ 脚本中的关键参数如下:
204
+
205
+ - `--deepspeed`: 此参数应当指向一个 deepspeed 的配置文件,`deepspeed_support`文件夹下提供了四种 DeepSpeed 的默认配置文件:`ds_zero2_no_offload.json`, `ds_zero2_offload.json`, `ds_zero3_no_offload.json`, `ds_zero3_offload.json`,可根据显存与通信情况选择不同的 ZeRO 阶段(ZeRO-2 / ZeRO-3)与 offload 策略
206
+ - `--model_name_or_path`: 要加载的 Hy4 flash 的 HF 预训练模型权重路径,否则无法加载
207
+ - `--tokenizer_name_or_path`: tokenizer 文件夹路径,否则无法加载
208
+ - `--train_data_file`: 训练文件路径,应该为一个 jsonl 文件
209
+ - `--output_dir`: 输出文件夹,log、tensorboard 和权重都会存储在这个路径下
210
+ - `--per_device_train_batch_size`: 每张卡上的 batch size
211
+ - `--gradient_accumulation_steps`: 梯度累计次数,`per_device_train_batch_size * gradient_accumulation_steps * dp_size`为 global_batch_size
212
+ - `--max_steps`: 训练的总步数
213
+ - `--save_steps`: 每多少个 step 存储一个 checkpoint
214
+ - `--gradient_checkpointing`: 开启梯度检查点
215
+ - `--learning_rate`: 训练时的最大学习率
216
+ - `--min_lr`: 训练时的最小学习率(绝对值,如 `1e-6`;不传时默认 `0.01`,此时按 `learning_rate * min_lr` 比例计算)
217
+ - `--use_flash_attn`: 开启 flash-attention 进行训练加速
218
+
219
+ ##### 显存不足怎么办?
220
+
221
+ 参考:[DeepSpeed Configuration](https://www.deepspeed.ai/docs/config-json/)
222
+
223
+ 可以尝试修改 ds config,去掉这几个参数的 auto 属性,改小试试看:
224
+
225
+ - `stage3_param_persistence_threshold`
226
+ - `stage3_prefetch_bucket_size`
227
+ - `stage3_max_reuse_distance`
228
+
229
+ ##### 多模态(VL)微调
230
+
231
+ `train_vl.sh` 同时支持三种数据:图片+文本、视频+文本、纯文本。数据格式详见 [多模态数据格式](#多模态数据格式图片--视频)。
232
+
233
+ VL 特有参数(除上述参数外):
234
+
235
+ - `--freeze_vit`: 冻结 ViT 视觉编码器(默认开启,即 `default=True`)。冻结后视觉编码器权重不会被更新,训练集中在 LLM 和投影层上。如需训练视觉编码器,请显式设置 `--freeze_vit false`。
236
+ - `--train_data_file`: 包含多模态数据的 JSONL 训练文件路径(含 `images` / `videos` 字段)。
237
+ - `--image_base_dir`: 用于解析数据文件中相对图片/视频路径的基础目录。如未指定,路径相对于工作目录解析。
238
+
239
+ **图片处理参数**(对齐 HuggingFace processor 的 `images_kwargs` API):
240
+
241
+ - `--image_min_pixels`: 图片缩放的最小像素数。对应 HF image processor 中 `size["shortest_edge"]`。默认值 `None`(使用 `preprocessor_config.json` 中的配置,默认 4096)。
242
+ - `--image_max_pixels`: 图片缩放的最大像素数。对应 HF image processor 中 `size["longest_edge"]`。默认值 `None`(使用 `preprocessor_config.json` 中的配置,默认 4194304)。
243
+
244
+ **视频处理参数**(对齐 HuggingFace processor 的 `videos_kwargs` API):
245
+
246
+ - `--video_max_pixels`: 视频所有帧的总像素预算。对应 HF video processor 中 `size["longest_edge"]`。默认值 `None`(使用 `video_preprocessor_config.json` 中的配置)。设置此参数后,每帧的像素预算为 `video_max_pixels / num_frames`,从而控制视频产生的 token 数量。
247
+ - `--video_max_frames`: 从视频中采样的最大帧数。对应 HF video processor 中 `max_frames`。默认值 `None`。
248
+ - `--video_fps`: 视频采样的目标帧率(每秒采样帧数)。对应 HF video processor 中 `fps`。默认值 `None`。
249
+
250
+ > **视频参数说明:** 视频产生的 token 数量远大于图片。应根据 `--max_seq_length` 合理设置视频参数,确保视频 token 不超出序列长度限制。
251
+
252
+ #### LLaMA-Factory 微调
253
+
254
+ 如果对 LLaMA-Factory 较为熟悉,可使用 LLaMA-Factory 进行微调。脚本、代码以及配置文件都归档在 `llama_factory_support` 目录下。如果没有特别说明,接下来我们提到的文件都是该目录下的文件。
255
+
256
+ ##### 安装
257
+
258
+ 可以通过下载源码 https://github.com/hiyouga/LLaMA-Factory/tree/main ,根据网站的指引进行安装。
259
+
260
+ ##### 配置文件
261
+
262
+ 我们提供了 llama-factory 的微调示例配置文件,分别对应不同的微调场景:
263
+
264
+ | 配置文件 | 场景 |
265
+ |---|---|
266
+ | `hy_v4_flash_full_sft.yaml` | LLM 全量微调 |
267
+ | `hy_v4_flash_lora_sft.yaml` | LLM LoRA 微调 |
268
+ | `hy_v4_flash_vl_full_sft.yaml` | VL 全量微调 |
269
+ | `hy_v4_flash_vl_lora_sft.yaml` | VL LoRA 微调 |
270
+
271
+ 脚本中的关键参数如下:
272
+
273
+ **模型相关:**
274
+
275
+ - `model_name_or_path`: Hy4 flash HF 格式预训练模型权重路径
276
+ - `trust_remote_code`: 是否信任远程代码,Hy4 flash 需要设置为 `true`
277
+
278
+ **训练方法:**
279
+
280
+ - `stage`: 训练阶段,当前为 `sft`(监督微调)
281
+ - `finetuning_type`: 微调类型,可选 `full`(全量微调)或 `lora`(LoRA 微调)
282
+ - `deepspeed`: DeepSpeed 配置文件路径,全量微调推荐 `../deepspeed_support/ds_zero3_offload.json`
283
+ - `fsdp` + `fsdp_config`: FSDP 分布式策略,LoRA 微调推荐使用 FSDP(配置已内置在 `hy_v4_flash_lora_sft.yaml` 中);与 DeepSpeed 二选一
284
+
285
+ > **分布式策略选择建议:**
286
+ > - **FSDP**:推荐用于 LoRA 微调,兼容性好,配置简单
287
+ > - **DeepSpeed ZeRO-3 + Offload**:推荐用于全量微调或显存紧张的场景
288
+
289
+ **LoRA 参数(仅 LoRA 微调时生效):**
290
+
291
+ - `lora_rank`: LoRA 秩,默认 `64`
292
+ - `lora_alpha`: LoRA alpha 系数,默认 `128`
293
+ - `lora_dropout`: LoRA dropout ���率,默认 `0.05`
294
+ - `lora_target`: LoRA 应用的目标模块,默认为 `q_a_proj,q_b_proj,kv_a_proj_with_mqa,kv_b_proj,o_proj`
295
+
296
+ **数据集:**
297
+
298
+ - `dataset_dir`: 数据集目录路径
299
+ - `dataset`: 数据集名称,需要在 `dataset_dir` 下的 `dataset_info.json` 中注册
300
+ - `template`: 对话模板,Hy4 flash 使用 `hy_v4_flash`
301
+ - `cutoff_len`: 最大序列长度,超出部分会被截断;LoRA 微调建议适当减小以节省显存
302
+ - `max_samples`: 每个数据集最多使用的样本数
303
+ - `overwrite_cache`: 是否覆盖已缓存的预处理数据集
304
+
305
+ **输出:**
306
+
307
+ - `output_dir`: 输出目录,日志、TensorBoard 和权重都会存储在此路径下
308
+ - `logging_steps`: 每多少步记录一次日志
309
+ - `save_steps`: 每多少步保存一次 checkpoint
310
+ - `plot_loss`: 是否绘制训练 loss 曲线
311
+ - `overwrite_output_dir`: 是否覆盖已有的输出目录
312
+ - `save_only_model`: 是否只保存模型权重(不保存优化器状态等)
313
+ - `report_to`: 日志上报工具,可选 `none`、`wandb`、`tensorboard`、`swanlab`、`mlflow`
314
+
315
+ **训练超参数:**
316
+
317
+ - `per_device_train_batch_size`: 每张卡上的 batch size
318
+ - `gradient_accumulation_steps`: 梯度累积步数,`per_device_train_batch_size * gradient_accumulation_steps * dp_size` 为 global batch size
319
+ - `learning_rate`: 最大学习率,全量微调推荐 `1.0e-5`,LoRA 微调推荐 `2.0e-4`
320
+ - `num_train_epochs`: 训练轮数
321
+ - `lr_scheduler_type`: 学习率调度器类型,推荐使用 `cosine_with_min_lr`
322
+ - `lr_scheduler_kwargs.min_lr_rate`: 最小学习率与最大学习率的比值,例如 `0.1` 表示最小学习率为最大学习率的 10%
323
+ - `warmup_steps`: 预热步数
324
+ - `bf16`: 是否使用 BFloat16 混合精度训练
325
+ - `gradient_checkpointing`: 是否开启梯度重计算以节省显存
326
+ - `ddp_timeout`: 分布式训练超时时间(毫秒)
327
+ - `flash_attn`: 注意力实现方式,可选 `auto`(自动选择)、`sdpa`、`disabled`(禁用)等。具体默认值视配置文件而定(如 LLM 全量微调为 `disabled`,LLM LoRA 为 `auto`,VL 全量微调为 `auto`,VL LoRA 为 `disabled`)
328
+ - `resume_from_checkpoint`: 从指定 checkpoint 路径恢复训练,设为 `null` 表示从头开始训练
329
+
330
+ **图片/视频处理参数(VL 微调时生效):**
331
+
332
+ 以下参数由模型自带的 processor 配置决定(见 `hy_v4_template.py`),默认值如下。如需调整,可在 YAML 配置文件中覆盖(需确认 LLaMA-Factory 版本支持)或修改 processor 配置:
333
+
334
+ - `image_max_pixels`: 图片缩放的最大像素数,默认 `4194304`(2048×2048)
335
+ - `image_min_pixels`: 图片缩放的最小像素数,默认 `4096`(64×64)
336
+ - `video_max_pixels`: 视频每帧的最大像素数,默认 `65536`(256×256)
337
+ - `video_min_pixels`: 视频每帧的最小像素数,默认 `256`(16×16)
338
+ - `video_fps`: 视频采样帧率,默认 `2.0`
339
+ - `video_maxlen`: 视频最大采样帧数,默认 `128`
340
+
341
+ > **视频参数说明:** 视频产生的 token 数量远大于图片。应根据 `cutoff_len` 合理设置视频参数,确保视频 token 不超出序列长度限制。降低 `video_max_pixels` 或 `video_maxlen` 可显著减少显存占用。
342
+
343
+ ##### 启动脚本
344
+
345
+ 我们为不同场景提供了对应的启动脚本:
346
+
347
+ | 场景 | 启动脚本 | 默认配置文件 |
348
+ |---|---|---|
349
+ | VL 全量微调 | `bash train_lf.sh` | `hy_v4_flash_vl_full_sft.yaml` |
350
+ | VL LoRA 微调 | `bash train_lf_lora.sh` | `hy_v4_flash_vl_lora_sft.yaml` |
351
+ | LLM 全量微调 | `bash train_lf_llm.sh` | `hy_v4_flash_full_sft.yaml` |
352
+ | LLM LoRA 微调 | `bash train_lf_lora_llm.sh` | `hy_v4_flash_lora_sft.yaml` |
353
+
354
+ 每个脚本均支持通过 `YAML_FILE` 环境变量覆盖默认配置文件。
355
+
356
+ ##### 启动微调
357
+
358
+ 如需多机训练,请先完成 [配置机器间免密 ssh 登录](#配置机器间免密-ssh-登录多机训练) 中的配置(单机训练可跳过此步骤)。
359
+
360
+ 修改对应启动脚本中开头的以下配置:
361
+
362
+ ```sh
363
+ export HOST_GPU_NUM=8
364
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
365
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
366
+ ```
367
+
368
+ 注意:
369
+ 1. 如果 `IP_LIST` 环境变量未设置,则将 `IP_LIST` 替换为 IP 列表!格式为:
370
+ ```text
371
+ 如果只有一个IP:
372
+ IP_LIST=${ip_1}
373
+
374
+ 如果有多个IP:
375
+ IP_LIST=${ip_1},${ip_2}
376
+
377
+ ```
378
+ 请将 `${ip_1}` 和 `${ip_2}` 替换为真实的 IP 地址。
379
+
380
+ 2. 如需指定微调配置文件,可通过 `YAML_FILE` 环境变量设置(建议使用绝对路径,或先 `cd` 到 `llama_factory_support/` 目录后再执行)。例如:
381
+ ```sh
382
+ export YAML_FILE=/path/to/llama_factory_support/hy_v4_flash_lora_sft.yaml
383
+ ```
384
+
385
+ 多机训练时,每台机器上都需要启动 `torchrun` 进程。启动脚本中内置了 `pdsh` 一键分发功能,只需在一台机器上执行即可自动分发到所有节点:
386
+
387
+ ```sh
388
+ # VL 全量微调
389
+ bash train_lf.sh
390
+
391
+ # LLM 全量微调
392
+ bash train_lf_llm.sh
393
+ ```
394
+
395
+ > **注意:** 如果您的环境不支持 `pdsh`,或需要自��义启动方式,请在每台机器上分别手动执行启动脚本。手动执行时必须设置 `_WORKER_MODE=1` 以跳过 pdsh 分发分支,例如:
396
+ > ```bash
397
+ > # 在每台机器上分别执行:
398
+ > export IP_LIST="10.0.0.1,10.0.0.2"
399
+ > export _WORKER_MODE=1
400
+ > bash train_lf.sh
401
+ > ```
402
+
403
+ ##### 多模态(VL)微调
404
+
405
+ VL 微调与纯文本配置的主要区别:
406
+
407
+ - `template`: 设为 `hy_v4_flash_vl`(而非 `hy_v4_flash`)
408
+ - `dataset`: 使用在 `dataset_info.json` 中注册的 VL 数据集,需包含 `images` 列;如需同时支持视频,还需包含 `videos` 列
409
+ - `freeze_vision_tower`: 设为 `true` 以冻结 ViT 视觉编码器(推荐)
410
+
411
+ 如需注册自定义 VL 数据集,在 `dataset_info.json` 中添加条目:
412
+
413
+ ```json
414
+ {
415
+ "my_vl_dataset": {
416
+ "file_name": "/path/to/my_vl_data.jsonl",
417
+ "formatting": "sharegpt",
418
+ "columns": {
419
+ "messages": "messages",
420
+ "images": "images",
421
+ "videos": "videos"
422
+ },
423
+ "tags": {
424
+ "role_tag": "role",
425
+ "content_tag": "content",
426
+ "user_tag": "user",
427
+ "assistant_tag": "assistant",
428
+ "system_tag": "system"
429
+ }
430
+ }
431
+ }
432
+ ```
433
+
434
+ #### ms-swift 微调
435
+
436
+ 如果对 ms-swift 较为熟悉,可使用 ms-swift 进行微调。脚本和代码都归档在 `ms_swift_support` 目录下。如果没有特别说明,接下来我们提到的文件都是该目录下的文件。
437
+
438
+ ##### 安装
439
+
440
+ 可以通过 pip 安装 ms-swift:
441
+
442
+ ```sh
443
+ pip install ms-swift
444
+ ```
445
+
446
+ 或从源码安装:https://github.com/modelscope/ms-swift
447
+
448
+ ##### 微调脚本
449
+
450
+ | 场景 | 微调方式 | 启动脚本 |
451
+ |---|---|---|
452
+ | 纯文本(LLM) | 全量微调 | `bash sft_train.sh` |
453
+ | 纯文本(LLM) | LoRA 微调 | `bash sft_train_lora.sh` |
454
+ | 多模态(VL) | 全量微调 | `bash sft_train_vl.sh` |
455
+ | 多模态(VL) | LoRA 微调 | `bash sft_train_vl_lora.sh` |
456
+
457
+ > **注意:** ms-swift 的训练参数直接在启动脚本中以 CLI 参数形式配置,如需修改请直接编辑对应的启动脚本。
458
+
459
+ ##### 关键参数
460
+
461
+ 启动脚本中的关键参数如下:
462
+
463
+ **模型相关:**
464
+
465
+ - `model`: 模型路径,可以是 HuggingFace Hub ID 或本地路径
466
+ - `model_type`: 模型类型,设为 `hy_v4_flash_vl`
467
+ - `template`: 对话模板,设为 `hy_v4_flash_vl`
468
+ - `torch_dtype`: 数据类型,推荐 `bfloat16`
469
+ - `attn_impl`: 注意力实现,默认 `eager`
470
+
471
+ **训练方法:**
472
+
473
+ - `tuner_type`: 微调类型,全量微调设为 `full`,LoRA 微调设为 `lora`
474
+ - `lora_rank`: LoRA 秩,默认 `64`
475
+ - `lora_alpha`: LoRA alpha 系数,默认 `128`
476
+ - `lora_dropout`: LoRA dropout 比率,默认 `0.05`
477
+ - `target_modules`: LoRA 应用的目标模块,默认为 MLA 投影层:`q_a_proj`, `q_b_proj`, `kv_a_proj_with_mqa`, `kv_b_proj`, `o_proj`
478
+
479
+ **数据集:**
480
+
481
+ - `dataset`: 数据集路径,支持本地 jsonl 文件(sharegpt 格式)
482
+ - `max_length`: 最大序列长度,超出部分会被截断
483
+ - `lazy_tokenize`: 是否延迟 tokenize,推荐 `true`
484
+
485
+ **输出:**
486
+
487
+ - `output_dir`: 输出目录
488
+ - `save_steps`: 每多少步保存一次 checkpoint
489
+ - `save_total_limit`: 最多保留的 checkpoint 数量
490
+ - `logging_steps`: 每多少步记录一次日志
491
+ - `report_to`: 日志上报工具,可选 `none`, `wandb`, `tensorboard`, `swanlab`, `mlflow`
492
+
493
+ **训练超参数:**
494
+
495
+ - `per_device_train_batch_size`: 每张卡上的 batch size
496
+ - `gradient_accumulation_steps`: 梯度累积步数
497
+ - `learning_rate`: 最大学习率,全量微调推荐 `1.0e-5`,LoRA 微调推荐 `2.0e-4`
498
+ - `num_train_epochs`: 训练轮数
499
+ - `lr_scheduler_type`: 学习率调度器类型,推荐 `cosine`
500
+ - `warmup_steps`: 预热步数
501
+ - `bf16`: 是否使用 BFloat16 混合精度训练
502
+
503
+ **分布式策略 / 优化:**
504
+
505
+ - `deepspeed`: DeepSpeed 配置文件路径,脚本中默认使用 `../deepspeed_support/ds_zero3_offload.json`;全量微调推荐 ZeRO-3 + Offload
506
+ - `fsdp`: FSDP 配置文件路径(JSON),LoRA 微调推荐使用 FSDP;与 DeepSpeed 二选一
507
+ - `gradient_checkpointing`: 是否开启梯度重计算
508
+ - `max_grad_norm`: 梯度裁剪阈值
509
+
510
+ > **分布式策略选择建议:**
511
+ > - **FSDP**:推荐用于 LoRA 微调,兼容性好,配置简单
512
+ > - **DeepSpeed ZeRO-3 + Offload**:推荐用于全量微调或显存紧张的场景
513
+
514
+ **其他:**
515
+
516
+ - `ddp_timeout`: 分布式训练超时时间(毫秒)
517
+ - `seed`: 随机种子
518
+ - `resume_from_checkpoint`: 从指定 checkpoint 路径恢复训练
519
+
520
+ **图片/视频处理参数(VL 微调时生效):**
521
+
522
+ ms-swift 的图片/视频处理由模型自带的 image processor 和 video processor 控制,参数默认值来自模型目录下的 `preprocessor_config.json` 和 `video_preprocessor_config.json`(以实际 ckpt 内的 JSON 配置为准)。如需调整,可直接修改模型目录下对应的 JSON 配置文件。
523
+
524
+ 图片处理参数(`preprocessor_config.json`):
525
+
526
+ - `min_pixels`: 图片缩放的最小像素数,默认 `4096`
527
+ - `max_pixels`: 图片缩放的最大像素数,默认 `4194304`(2048×2048)
528
+
529
+ 视频处理参数(`video_preprocessor_config.json`):
530
+
531
+ - `size.longest_edge`: 视频所有帧的总像素预算,默认 `1572864`
532
+ - `max_image_size.longest_edge`: 单帧视频的最大像素数,默认 `262144`
533
+ - `max_frames`: 视频最大采样帧数,默认 `2`
534
+ - `fps`: 视频采样的目标帧率(每秒采样帧数),默认 `2`
535
+ - `do_sample_frames`: 是否启用帧采样,默认 `true`
536
+
537
+ > **视频参数说明:** 视频产生的 token 数量远大于图片。应根据 `max_length` 合理设置视频参数,确保视频 token 不超出序列长度限制。降低 `size.longest_edge`、`max_image_size.longest_edge` 或 `max_frames` 可显著减少显存占用。
538
+
539
+ ##### 启动微调
540
+
541
+ 如需多机训练,请先完成 [配置机器间免密 ssh 登录](#配置机器间免密-ssh-登录多机训练) 中的配置(单机训练可跳过此步骤)。
542
+
543
+ 修改对应启动脚本中开头的以下配置:
544
+
545
+ ```sh
546
+ export HOST_GPU_NUM=8
547
+ # IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
548
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
549
+ ```
550
+
551
+ 多机训练时,每台机器上都需要启动 `torchrun` 进程。启动脚本中内置了 `pdsh` 一键分发功能,只需在一台机器上执行即可自动分发到所有节点:
552
+
553
+ ```sh
554
+ # 纯文本全量微调
555
+ bash sft_train.sh
556
+
557
+ # 纯文本 LoRA 微调
558
+ bash sft_train_lora.sh
559
+
560
+ # VL 全量微调 + DeepSpeed ZeRO-3
561
+ bash sft_train_vl.sh
562
+
563
+ # VL LoRA 微调 + FSDP
564
+ bash sft_train_vl_lora.sh
565
+
566
+ # 多机训练(使用 pdsh 一键分发,只需在一台机器上执行)
567
+ IP_LIST="10.0.0.1,10.0.0.2" bash sft_train.sh
568
+ ```
569
+
570
+ > **注意:** 如果您的环境不支持 `pdsh`,或需要自定义启动方式,请在每台机器上分别手动执行启动脚本。手动执行时必须设置 `_WORKER_MODE=1` 以跳过 pdsh 分发分支,例如:
571
+ > ```bash
572
+ > # 在每台机器上分别执行:
573
+ > export IP_LIST="10.0.0.1,10.0.0.2"
574
+ > export _WORKER_MODE=1
575
+ > bash sft_train.sh
576
+ > ```
577
+
578
+ ##### 多模态(VL)微调
579
+
580
+ VL 微调与纯文本脚本的主要区别如下:
581
+
582
+ - `model_type`: 设为 `hy_v4_flash_vl`(而非 `hy_v4_flash`)
583
+ - `template`: 设为 `hy_v4_flash_vl`(而非 `hy_v4_flash`)
584
+ - `dataset`: 指向包含多模态数据的 JSONL 文件(含 `images` / `videos` 字段)
finetune/data/example_data.jsonl ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "能否帮我撰写一个关于环保的议论文开头部分?\n主题:环保\n立场:重要性和我们的责任"}, {"role": "assistant", "content": "在我们的生活中,环保已经成为一个不可忽视的议题。随着科技的发展和人口的增长,我们的环境正在遭受前所未有的压力。森林被砍伐,河流被污染,动植物种类正在消失,全球气候正在发生变化。这些都是我们无法忽视的事实。然而,环保并不仅仅是一个环境问题,它更是一个关乎我们生存和发展的问题。环保的重要性已经超越了任何一个单一的领域,它关乎我们的健康、经济、社会和文化。因此,我们每一个人都有责任和义务去保护我们的环境,这不仅是为了我们自己,也是为了我们的后代和整个地球。"}]}
2
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请描述小红书APP的用户画像,并用表格的形式展示结果。"}, {"role": "assistant", "content": "| 用户特征 | 描述 |\n| -------- | ------------------------------ |\n| 年龄段 | 18-35岁 |\n| 性别 | 女性用户较多(约占70%) |\n| 地域 | 一线城市和二线城市用户占比较高 |\n| 职业 | 白领、创业者、自由职业者等 |\n| 爱好 | 美容、时尚、旅行、美食、健康、生活方式等 |\n| 消费能力 | 中高消费能力用户 |\n| 使用目的 | 阅读内容,分享购物心得,结识相似兴趣的朋友 |\n\n注:以上用户画像仅供参考,实际情况可能会有所不同。"}]}
3
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请根据以下两个句子的描述,判断它们是否对同一情况进行了描述。句子1:“外面下着大雨,人们纷纷打起了伞。” 句子2:“由于雨量较大,人们都撑起了雨伞。”"}, {"role": "assistant", "content": "是的,这两个句子描述的是同一情况。"}]}
4
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "你认为游戏开发者应该如何处理游戏中的社交网络集成?"}, {"role": "assistant", "content": "游戏开发者在处理游戏中的社交网络集成时可以考虑以下几点:\n\n1. 以用户为中心:确保集成社交网络功能是为了提高用户体验,让玩家更容易地与朋友互动、分享成就和游戏进度。\n\n2. 保护隐私:在集成社交网络时,务必保护用户的隐私。在收集和使用用户数据时,需要遵循相关法规和用户协议,同时提供明确的隐私政策。\n\n3. 适度推送:在推送游戏动态、邀请和通知时要适度,避免过度打扰用户或侵犯用户隐私。可提供自定义设置,让玩家选择性地接收通知。\n\n4. 社交功能设计:在游戏中加入社交功能,如聊天、组队、比赛等,以增进玩家之间的互动。同时,确保这些功能易用且不影响游戏体验。\n\n5. 社区建设:鼓励玩家在社交平台上建立游戏相关的社区,分享游戏心得、攻略和经验。这将有助于增加游戏的知名度和吸引力。\n\n6. 奖励机制:设定奖励机制,鼓励玩家在社交网络上分享游戏内容、邀请好友等。但要确保奖励机制不会破坏游戏平衡。\n\n7. 持续更新和优化:根据玩家的反馈,不断优化社交网络集成功能,确保其与游戏的其他方面保持一致。\n\n总之,游戏开发者应该在保护用户隐私的前提下,提供便捷的社交网络集成功能,以提高游戏的互动性和吸引力。同时,要关注玩家的反馈,不断优化功能。"}]}
5
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "将这段过去进行时的文字改为现在完成进行时:“她们在公园里跳舞。”"}, {"role": "assistant", "content": "她们已经在公园里跳舞了。"}]}
6
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "在godaddy上买了一台服务器怎么更改root账户"}, {"role": "assistant", "content": "在GoDaddy上购买的服务器可能是Linux系统的VPS或者是专用服务器。要更改root账户,你可以通过SSH连接到服务器并修改密码。以下是具体步骤:\n\n1. 打开终端(Terminal)或使用SSH客户端(如PuTTY)。\n2. 使用SSH连接到服务器。在终端中输入以下命令:\n\n ```\n ssh root@your_server_ip_address\n ```\n\n 将`your_server_ip_address`替换为你的服务器IP地址。如果需要,你可能还需要输入服务器的SSH端口号。\n \n3. 当系统提示输入密码时,输入你的当前root密码。如果连接成功,你将看到服务器的命令提示符。\n\n4. 要更改root密码,请输入以下命令:\n\n ```\n passwd\n ```\n\n5. 系统将提示你输入新的UNIX密码。输入你想要设置的新密码并按回车键。\n\n6. 系统将要求你重新输入新密码以进行确认。再次输入新密码并按回车键。\n\n7. 如果密码更改成功,系统将显示“密码已成功更新”的消息。\n\n8. 使用`exit`命令断开与服务器的连接。\n\n现在你已经成功更改了root账户的密码。下次使用SSH连接到服务器时,请使用新的密码进行身份验证。"}]}
7
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "PHP算法逻辑:如何计算年龄?"}, {"role": "assistant", "content": "要计算年龄,您可以使用以下PHP算法。此算法考虑了生日、当前日期以及闰年的计算。\n\n```php\nfunction calculateAge($birthdate) {\n // 格式为YYYY-MM-DD的生日\n $birthdate = strtotime($birthdate);\n \n // 获取当前日期\n $now = time();\n\n // 计算年龄\n $age = date('Y', $now) - date('Y', $birthdate);\n \n // 如果还没有过生日,减去一年\n if (date('m-d', $now) < date('m-d', $birthdate)) {\n $age--;\n }\n\n return $age;\n}\n\n// 使用\n$birthdate = \"2000-01-01\";\necho \"年龄是: \" . calculateAge($birthdate) . \" 岁\";\n```\n\n这个函数将生日作为输入,输出用户的年龄。首先,它将生日转换为时间戳,然后获取当前的时间戳。接下来,它计算年份之间的差异。如果当前日期仍未超过这一年的生日,这个函数将年龄减少1。\n\n在使用函数时,只需传入出生日期即可,函数会返回相应的年龄。"}]}
8
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "给我解释一下什么是胰岛素抵抗? 请详细描述胰岛素抵抗的症状、原因与治疗方法。 胰岛素抵抗会影响一个人的日常生活吗? 有哪些生活习惯改变可以帮助改善胰岛素抵抗?"}, {"role": "assistant", "content": "胰岛素抵抗是指身体对胰岛素的敏感性降低,胰岛素不能有效地发挥作用,导致血糖无法正常进入身体细胞。胰岛素是一种激素,由胰腺产生,其主要作用是帮助身体细胞吸收血糖并将其转化为能量。当胰岛素抵抗发生时,胰腺会分泌更多的胰岛素以维持正常的血糖水平,但长期的胰岛素抵抗可能导致2型糖尿病和其他健康问题的发生。\n\n胰岛素抵抗的症状可能包括:\n\n1. 高血糖:由于胰岛素不能正常工作,血糖无法进入细胞,导致血糖升高。\n2. 疲劳:由于身体无法有效利用血糖产生能量,患者可能会感到疲劳无力。\n3. 肥胖:胰岛素抵抗可能导致过量的胰岛素分泌,使得脂肪存储增加,进而导致体重增加。\n4. 高血压:胰岛素抵抗与高血压之间存在关联,可能导致血压升高。\n5. 黑棘皮病:皮肤出现黑色、粗糙的皮疹,通常出现在颈部、腋窝等皮肤皱褶处。\n\n胰岛素抵抗的原因可能包括:\n\n1. 遗传因素:遗传易感基因可能会增加一个人患胰岛素抵抗的风险。\n2. 肥胖:过重或肥胖会增加患胰岛素抵抗的风险。\n3. 缺乏运动:缺乏运动可能会导致肌肉对胰岛素的敏感性降低。\n4. 高糖饮食:长期摄入高糖食品可能导致胰岛素抵抗。\n5. 睡眠不足:长期睡眠不足可能会影响胰岛素的正常功能。\n\n治疗胰岛素抵抗的方法主要包括生活方式的改变和药物治疗:\n\n1. 减轻体重:通过改善饮食和增加运动来减轻体重,有助于提高身体对胰岛素的敏感性。\n2. 增加运动:增加有氧运动和力量训练,有助于提高肌肉对胰岛素的敏感性。\n3. 改善饮食:摄入低糖、高纤维、富含全谷物和蔬菜水果的食物,有助于控制血糖和减轻体重。\n4. 药物治疗:在必要时,医生可能会开处降糖药物,如二甲双胍,以帮助提高胰岛素敏感性。\n\n胰岛素抵抗会影响一个人的日常生活,包括疲劳、体重增加、血糖波动等问题。通过改变生活习惯,如减轻体重、增加运动、改善饮食和保证充足睡眠,可以帮助改善胰岛素抵抗,降低患2型糖尿病的风险。"}]}
finetune/data/example_vl_data.jsonl ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "<image>Please describe this image in detail."}, {"role": "assistant", "content": "The image shows a beautiful sunset over the ocean. The sky is painted in shades of orange, pink, and purple, with wispy clouds catching the last rays of sunlight. The calm water reflects the vibrant colors, creating a mirror-like effect on the surface."}], "images": ["test_images/sunset.jpg"]}
2
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "<image>What objects can you see in this picture?"}, {"role": "assistant", "content": "In this picture, I can see a wooden desk with a laptop computer on it, a coffee mug to the right side, and a small potted plant near the window. There are also some books stacked on the left corner of the desk."}], "images": ["test_images/desk.jpg"]}
3
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "<video>Please describe what is happening in this video."}, {"role": "assistant", "content": "The video shows a cat playing with a ball of yarn on a hardwood floor. The cat bats the yarn around with its paws, occasionally pouncing on it and rolling over."}], "videos": ["test_videos/cat.mp4"]}
4
+ {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris."}]}
finetune/deepspeed_support/ds_zero2_no_offload.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "zero_optimization": {
14
+ "stage": 2,
15
+ "allgather_partitions": true,
16
+ "allgather_bucket_size": 1e8,
17
+ "overlap_comm": true,
18
+ "reduce_scatter": true,
19
+ "reduce_bucket_size": 1e8,
20
+ "contiguous_gradients": true
21
+ },
22
+ "gradient_accumulation_steps": "auto",
23
+ "gradient_clipping": "auto",
24
+ "steps_per_print": 10,
25
+ "train_batch_size": "auto",
26
+ "train_micro_batch_size_per_gpu": "auto",
27
+ "wall_clock_breakdown": false
28
+ }
finetune/deepspeed_support/ds_zero2_offload.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+
14
+ "zero_optimization": {
15
+ "stage": 2,
16
+ "offload_optimizer": {
17
+ "device": "cpu",
18
+ "pin_memory": false
19
+ },
20
+ "allgather_partitions": true,
21
+ "allgather_bucket_size": 5e8,
22
+ "overlap_comm": true,
23
+ "reduce_scatter": true,
24
+ "reduce_bucket_size": 5e8,
25
+ "contiguous_gradients": true
26
+ },
27
+
28
+ "gradient_accumulation_steps": "auto",
29
+ "gradient_clipping": "auto",
30
+ "steps_per_print": 10,
31
+ "train_batch_size": "auto",
32
+ "train_micro_batch_size_per_gpu": "auto",
33
+ "wall_clock_breakdown": false
34
+ }
finetune/deepspeed_support/ds_zero3_no_offload.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+
14
+ "zero_optimization": {
15
+ "stage": 3,
16
+ "offload_optimizer": {
17
+ "device": "none",
18
+ "pin_memory": false
19
+ },
20
+ "offload_param": {
21
+ "device": "none",
22
+ "pin_memory": false
23
+ },
24
+ "overlap_comm": true,
25
+ "contiguous_gradients": true,
26
+ "sub_group_size": 1e9,
27
+ "reduce_bucket_size": "auto",
28
+ "stage3_prefetch_bucket_size": "auto",
29
+ "stage3_param_persistence_threshold": 0,
30
+ "stage3_max_live_parameters": 1e9,
31
+ "stage3_max_reuse_distance": 1e9,
32
+ "stage3_gather_16bit_weights_on_model_save": false
33
+ },
34
+
35
+ "gradient_accumulation_steps": "auto",
36
+ "gradient_clipping": "auto",
37
+ "steps_per_print": 10,
38
+ "train_batch_size": "auto",
39
+ "train_micro_batch_size_per_gpu": "auto",
40
+ "wall_clock_breakdown": false
41
+ }
finetune/deepspeed_support/ds_zero3_offload.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+
14
+ "zero_optimization": {
15
+ "stage": 3,
16
+ "offload_optimizer": {
17
+ "device": "cpu",
18
+ "pin_memory": false
19
+ },
20
+ "offload_param": {
21
+ "device": "cpu",
22
+ "pin_memory": false
23
+ },
24
+ "overlap_comm": true,
25
+ "contiguous_gradients": true,
26
+ "sub_group_size": 1e9,
27
+ "reduce_bucket_size": "auto",
28
+ "stage3_prefetch_bucket_size": "auto",
29
+ "stage3_param_persistence_threshold": 0,
30
+ "stage3_max_live_parameters": 1e9,
31
+ "stage3_max_reuse_distance": 1e9,
32
+ "stage3_gather_16bit_weights_on_model_save": false
33
+ },
34
+
35
+ "gradient_accumulation_steps": "auto",
36
+ "gradient_clipping": "auto",
37
+ "steps_per_print": 10,
38
+ "train_batch_size": "auto",
39
+ "train_micro_batch_size_per_gpu": "auto",
40
+ "wall_clock_breakdown": false
41
+ }
finetune/deepspeed_support/train_llm.sh ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # HY4-Flash LLM SFT training launch script (using VL checkpoint, only training LLM)
4
+ #
5
+ # This script loads the full VL checkpoint but freezes the ViT encoder,
6
+ # effectively performing pure LLM SFT. The pure-text training data does not
7
+ # contain <image>/<video> tags, so the ViT is never invoked during forward.
8
+
9
+ NET_TYPE="high"
10
+ export NCCL_DEBUG=WARN
11
+ export NCCL_P2P_LEVEL=NVL
12
+ export NCCL_IB_TIMEOUT=24
13
+ export NCCL_NVLS_ENABLE=0
14
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
15
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
16
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
17
+ if [[ "${NET_TYPE}" = "low" ]]; then
18
+ export NCCL_SOCKET_IFNAME=eth1
19
+ export NCCL_IB_GID_INDEX=3
20
+ export NCCL_IB_HCA=mlx5_2:1
21
+ export NCCL_IB_SL=3
22
+ export NCCL_CHECK_DISABLE=1
23
+ export NCCL_P2P_DISABLE=0
24
+ export NCCL_LL_THRESHOLD=16384
25
+ export NCCL_IB_CUDA_SUPPORT=1
26
+ else
27
+ export NCCL_IB_GID_INDEX=3
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_IB_DISABLE=0
32
+ export NCCL_LL_THRESHOLD=16384
33
+ export NCCL_IB_CUDA_SUPPORT=1
34
+ export NCCL_SOCKET_IFNAME=bond1
35
+ export UCX_NET_DEVICES=bond1
36
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
37
+ export NCCL_COLLNET_ENABLE=0
38
+ export SHARP_COLL_ENABLE_SAT=0
39
+ export NCCL_NET_GDR_LEVEL=2
40
+ export NCCL_IB_QPS_PER_CONNECTION=4
41
+ export NCCL_IB_TC=160
42
+ export NCCL_PXN_DISABLE=1
43
+ fi
44
+
45
+ export HOST_GPU_NUM=8
46
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
47
+ IP_LIST=${IP_LIST:-"127.0.0.1"}
48
+
49
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
50
+ export NODES=${#IP_ARRAY[@]}
51
+ export LOCAL_IP=${IP_ARRAY[0]}
52
+ NODE_IP_LIST=""
53
+ for ip in "${IP_ARRAY[@]}"; do
54
+ if [ -n "$NODE_IP_LIST" ]; then
55
+ NODE_IP_LIST="${NODE_IP_LIST},"
56
+ fi
57
+ NODE_IP_LIST="${NODE_IP_LIST}${ip}:${HOST_GPU_NUM}"
58
+ done
59
+ export NODE_IP_LIST
60
+ export NODE_NUM=$((${NODES} * ${HOST_GPU_NUM}))
61
+
62
+ # ============================================================================
63
+ # LLM SFT paths (using VL checkpoint + pure text data)
64
+ # ============================================================================
65
+ model_path=/path/to/hy4_ckpt
66
+ tokenizer_path=/path/to/hy4_ckpt
67
+ train_data_file=../data/example_data.jsonl
68
+
69
+ ds_config_file=ds_zero3_offload.json
70
+
71
+ output_path=./output_llm
72
+
73
+ mkdir -p ${output_path}
74
+
75
+ current_time=$(date "+%Y.%m.%d-%H.%M.%S")
76
+ log_file=${output_path}/"log_${current_time}.txt"
77
+
78
+ echo $NODE_IP_LIST > env.txt 2>&1
79
+ sed "s/:/ slots=/g" env.txt | sed "s/,/\n/g" > "hostfile"
80
+ sed "s/:.//g" env.txt | sed "s/,/\n/g" > "pssh.hosts"
81
+ export CHIEF_IP=$LOCAL_IP
82
+
83
+ if [ ${NODES} -gt 1 ]; then
84
+ HOST_PATH=hostfile
85
+ DS_ARGS="--hostfile=${HOST_PATH} --master_addr ${CHIEF_IP}"
86
+ else
87
+ DS_ARGS=""
88
+ fi
89
+
90
+ echo "NODES: ${NODES}, LOCAL_IP: ${LOCAL_IP}, NODE_IP_LIST: ${NODE_IP_LIST}"
91
+
92
+ deepspeed ${DS_ARGS} \
93
+ train_vl.py \
94
+ --do_train \
95
+ --model_name_or_path ${model_path} \
96
+ --tokenizer_name_or_path ${tokenizer_path} \
97
+ --train_data_file ${train_data_file} \
98
+ --freeze_vit \
99
+ --deepspeed ${ds_config_file} \
100
+ --output_dir ${output_path} \
101
+ --per_device_train_batch_size 1 \
102
+ --gradient_accumulation_steps 1 \
103
+ --gradient_checkpointing \
104
+ --lr_scheduler_type cosine_with_min_lr \
105
+ --logging_steps 1 \
106
+ --max_steps 10 \
107
+ --save_steps 10 \
108
+ --learning_rate 1e-5 \
109
+ --min_lr 1e-6 \
110
+ --warmup_steps 0 \
111
+ --save_strategy steps \
112
+ --bf16 \
113
+ --max_grad_norm 0 \
114
+ --max_seq_length 1024 | tee ${log_file}
finetune/deepspeed_support/train_vl.py ADDED
@@ -0,0 +1,872 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2024 Tencent Inc. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """
16
+ HY4-VL SFT training script with DeepSpeed.
17
+
18
+ Supports:
19
+ - Image + text multimodal SFT
20
+ - Video + text multimodal SFT
21
+ - Pure text SFT (backward compatible)
22
+ - LoRA and full fine-tuning
23
+ - DeepSpeed ZeRO-2/3 with optional CPU offload
24
+
25
+ Data format (jsonl):
26
+ # Image example
27
+ {"messages": [...], "images": ["/path/to/img.jpg"]}
28
+ # Video example
29
+ {"messages": [...], "videos": ["/path/to/video.mp4"]}
30
+ # Pure text example
31
+ {"messages": [...]}
32
+ """
33
+
34
+ import os
35
+ import sys
36
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
37
+ import json
38
+ import torch
39
+ import shutil
40
+ import logging
41
+ from dataclasses import dataclass, field
42
+ from typing import Optional, Dict, List, Any
43
+
44
+ import transformers
45
+ from torch.utils.data import Dataset
46
+ from transformers import Trainer, TrainerCallback
47
+ from peft import LoraConfig, get_peft_model
48
+ from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
49
+ from PIL import Image
50
+
51
+ logger = logging.getLogger(__name__)
52
+
53
+
54
+ def print_args(args, name='arguments'):
55
+ """Print arguments."""
56
+ if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0:
57
+ print(f'------------------------ {name} ------------------------', flush=True)
58
+ str_list = []
59
+ for arg in vars(args):
60
+ dots = '.' * (48 - len(arg))
61
+ str_list.append(' {} {} {}'.format(arg, dots, getattr(args, arg)))
62
+ for arg in sorted(str_list, key=lambda x: x.lower()):
63
+ print(arg, flush=True)
64
+ print(f'-------------------- end of {name} ---------------------', flush=True)
65
+
66
+
67
+ @dataclass
68
+ class ModelArguments:
69
+ use_flash_attn: bool = field(
70
+ default=False,
71
+ metadata={"help": "Enable FlashAttention-2 for faster training."}
72
+ )
73
+ use_lora: bool = field(default=False, metadata={"help": "Enable LoRA for efficient training."})
74
+ lora_rank: int = field(default=64, metadata={"help": "The rank of LoRA."})
75
+ lora_alpha: int = field(default=8, metadata={"help": "LoRA alpha"})
76
+ lora_dropout: float = field(default=0.0, metadata={"help": "LoRA dropout"})
77
+ freeze_vit: bool = field(
78
+ default=True,
79
+ metadata={"help": "Freeze the Vision Transformer (ViT) encoder during training."}
80
+ )
81
+
82
+
83
+ @dataclass
84
+ class DataArguments:
85
+ train_data_file: str = field(default=None, metadata={"help": "Path to the training data (jsonl)."})
86
+ max_seq_length: int = field(
87
+ default=2048,
88
+ metadata={"help": "The max sequence length of the model inputs after tokenization."}
89
+ )
90
+ image_base_dir: Optional[str] = field(
91
+ default=None,
92
+ metadata={"help": "Base directory for resolving relative image/video paths in the data."}
93
+ )
94
+ use_dummy_data: bool = field(default=False, metadata={"help": "Use dummy data for testing."})
95
+ # Image processing overrides (align with HF processor's images_kwargs API)
96
+ image_min_pixels: Optional[int] = field(
97
+ default=None,
98
+ metadata={"help": "Min pixels for image resizing. Maps to size['shortest_edge'] in HF image processor."}
99
+ )
100
+ image_max_pixels: Optional[int] = field(
101
+ default=None,
102
+ metadata={"help": "Max pixels for image resizing. Maps to size['longest_edge'] in HF image processor."}
103
+ )
104
+ # Video processing overrides (align with HF processor's videos_kwargs API)
105
+ video_max_pixels: Optional[int] = field(
106
+ default=None,
107
+ metadata={"help": "Max total pixels for video frames. Maps to size['longest_edge'] in HF video processor."}
108
+ )
109
+ video_max_frames: Optional[int] = field(
110
+ default=None,
111
+ metadata={"help": "Max number of frames to sample from video. Maps to max_frames in HF video processor."}
112
+ )
113
+ video_fps: Optional[int] = field(
114
+ default=None,
115
+ metadata={"help": "Target frames per second for video sampling. Maps to fps in HF video processor."}
116
+ )
117
+
118
+
119
+ @dataclass
120
+ class TrainingArguments(transformers.TrainingArguments):
121
+ cache_dir: Optional[str] = field(default=None)
122
+ optim: str = field(default="adamw_torch")
123
+ tokenizer_name_or_path: Optional[str] = field(default=None)
124
+ model_name_or_path: Optional[str] = field(default=None)
125
+ min_lr: float = field(
126
+ default=0.01,
127
+ metadata={"help": "The final learning rate at the end of the decay will be learning_rate * min_lr"}
128
+ )
129
+
130
+
131
+ IGNORE_INDEX = -100
132
+
133
+ HY_START_ID = 120000
134
+ HY_MIDDLE_ID = 120001
135
+ HY_END_ID = 120025
136
+
137
+
138
+ # ============================================================================
139
+ # Dummy Dataset (for testing without real data)
140
+ # ============================================================================
141
+
142
+ class VLDummyDataset(Dataset):
143
+ """Dummy dataset that generates random tokens for VL pipeline testing."""
144
+
145
+ def __init__(self, processor, max_seq_length=512, length=100):
146
+ self.processor = processor
147
+ self.tokenizer = processor.tokenizer
148
+ self.max_seq_length = max_seq_length
149
+ self.length = length
150
+
151
+ def __len__(self):
152
+ return self.length
153
+
154
+ def __getitem__(self, index):
155
+ tokens = torch.randint(0, self.tokenizer.vocab_size, (self.max_seq_length,))
156
+ # For dummy data, use 1D position_ids (no MRoPE)
157
+ position_ids = torch.arange(self.max_seq_length).unsqueeze(0).expand(3, -1)
158
+ return {
159
+ 'input_ids': tokens,
160
+ 'labels': tokens,
161
+ 'attention_mask': torch.ones(self.max_seq_length, dtype=torch.bool),
162
+ 'position_ids': position_ids,
163
+ 'mm_token_type_ids': torch.zeros(self.max_seq_length, dtype=torch.long),
164
+ }
165
+
166
+
167
+ # ============================================================================
168
+ # VL SFT Dataset
169
+ # ============================================================================
170
+
171
+ class VLSFTDataset(Dataset):
172
+ """
173
+ Vision-Language SFT dataset.
174
+
175
+ Supports three data types:
176
+ - Image + text: {"messages": [...], "images": ["path/to/img.jpg"]}
177
+ - Video + text: {"messages": [...], "videos": ["path/to/video.mp4"]}
178
+ - Pure text: {"messages": [...]}
179
+
180
+ The processor handles:
181
+ - Image/video loading and preprocessing (pixel_values, grid_thw)
182
+ - Token expansion (replacing <image>/<video> with correct number of tokens)
183
+ - 3D MRoPE position_ids construction [t, h, w]
184
+ """
185
+
186
+ def __init__(self, data_file, processor, max_seq_length=2048, image_base_dir=None,
187
+ image_min_pixels=None, image_max_pixels=None,
188
+ video_max_pixels=None, video_max_frames=None, video_fps=None):
189
+ self.processor = processor
190
+ self.tokenizer = processor.tokenizer
191
+ self.max_seq_length = max_seq_length
192
+ self.image_base_dir = image_base_dir
193
+ # Image/video processing overrides
194
+ self.image_min_pixels = image_min_pixels
195
+ self.image_max_pixels = image_max_pixels
196
+ self.video_max_pixels = video_max_pixels
197
+ self.video_max_frames = video_max_frames
198
+ self.video_fps = video_fps
199
+
200
+ # Pre-compute special token IDs for loss masking
201
+ self.hy_start_id = HY_START_ID
202
+ self.hy_middle_id = HY_MIDDLE_ID
203
+ self.hy_end_id = HY_END_ID
204
+ self.assistant_bpe_ids = self.tokenizer.encode('assistant', add_special_tokens=False)
205
+ self.pad_token_id = self.tokenizer.pad_token_id
206
+
207
+ self.data_list = self._load_data(data_file)
208
+
209
+ def __len__(self):
210
+ return len(self.data_list)
211
+
212
+ def _load_data(self, data_file):
213
+ logger.info(f'Loading data: {data_file}')
214
+ with open(data_file, 'r', encoding='utf8') as f:
215
+ data_list = f.readlines()
216
+ logger.info(f"Loaded {len(data_list)} samples")
217
+ return data_list
218
+
219
+ def _resolve_path(self, path):
220
+ """Resolve relative paths using image_base_dir."""
221
+ if os.path.isabs(path):
222
+ return path
223
+ if self.image_base_dir:
224
+ return os.path.join(self.image_base_dir, path)
225
+ return path
226
+
227
+ def _load_images(self, image_paths):
228
+ """Load images from paths, return list of PIL Images."""
229
+ images = []
230
+ for p in image_paths:
231
+ full_path = self._resolve_path(p)
232
+ try:
233
+ img = Image.open(full_path).convert("RGB")
234
+ images.append(img)
235
+ except Exception as e:
236
+ logger.warning(f"Failed to load image {full_path}: {e}")
237
+ # Create a small placeholder image
238
+ images.append(Image.new("RGB", (224, 224), color=(128, 128, 128)))
239
+ return images
240
+
241
+ def _find_assistant_turn_boundaries(self, token_ids):
242
+ """Find assistant turn boundaries for loss masking.
243
+
244
+ Locates assistant turns by finding <hy_start> + "assistant" BPE tokens pattern,
245
+ then marks loss from the <hy_middle> token (exclusive) to <hy_end> (inclusive).
246
+ """
247
+ boundaries = []
248
+ assistant_len = len(self.assistant_bpe_ids)
249
+ ids_list = token_ids.tolist() if isinstance(token_ids, torch.Tensor) else token_ids
250
+ n = len(ids_list)
251
+
252
+ i = 0
253
+ while i < n:
254
+ if ids_list[i] == self.hy_start_id:
255
+ role_start = i + 1
256
+ role_end = role_start + assistant_len
257
+ if role_end <= n and ids_list[role_start:role_end] == self.assistant_bpe_ids:
258
+ if role_end < n and ids_list[role_end] == self.hy_middle_id:
259
+ middle_pos = role_end
260
+ end_pos = None
261
+ for j in range(middle_pos + 1, n):
262
+ if ids_list[j] == self.hy_end_id:
263
+ end_pos = j
264
+ break
265
+ if end_pos is not None:
266
+ boundaries.append((middle_pos, end_pos))
267
+ i = end_pos + 1
268
+ continue
269
+ i += 1
270
+
271
+ return boundaries
272
+
273
+ @staticmethod
274
+ def _normalize_messages(messages):
275
+ """Convert legacy <image>/<video> string placeholders to multimodal content list format.
276
+
277
+ Legacy format: {"role": "user", "content": "<image>Describe this image."}
278
+ Normalized: {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "Describe this image."}]}
279
+
280
+ This ensures chat_template's render_content() correctly emits image/video pad tokens.
281
+ Messages already using list-of-dict content are left unchanged.
282
+ """
283
+ IMAGE_PLACEHOLDER = "<image>"
284
+ VIDEO_PLACEHOLDER = "<video>"
285
+ normalized = []
286
+ for msg in messages:
287
+ content = msg.get("content", "")
288
+ # Only transform string content that contains placeholders
289
+ if isinstance(content, str) and (IMAGE_PLACEHOLDER in content or VIDEO_PLACEHOLDER in content):
290
+ parts = []
291
+ remaining = content
292
+ while remaining:
293
+ img_idx = remaining.find(IMAGE_PLACEHOLDER)
294
+ vid_idx = remaining.find(VIDEO_PLACEHOLDER)
295
+ # Find the earliest placeholder
296
+ candidates = []
297
+ if img_idx != -1:
298
+ candidates.append((img_idx, IMAGE_PLACEHOLDER, "image"))
299
+ if vid_idx != -1:
300
+ candidates.append((vid_idx, VIDEO_PLACEHOLDER, "video"))
301
+ if not candidates:
302
+ # No more placeholders, add remaining text
303
+ text_part = remaining.strip()
304
+ if text_part:
305
+ parts.append({"type": "text", "text": text_part})
306
+ break
307
+ # Pick the earliest one
308
+ candidates.sort(key=lambda x: x[0])
309
+ idx, placeholder, media_type = candidates[0]
310
+ # Add text before the placeholder
311
+ before = remaining[:idx].strip()
312
+ if before:
313
+ parts.append({"type": "text", "text": before})
314
+ parts.append({"type": media_type})
315
+ remaining = remaining[idx + len(placeholder):]
316
+ normalized.append({**msg, "content": parts})
317
+ else:
318
+ normalized.append(msg)
319
+ return normalized
320
+
321
+ def _build_text_from_messages(self, messages, reasoning_effort='no_think'):
322
+ """Build text string from messages, applying chat template."""
323
+ try:
324
+ text = self.tokenizer.apply_chat_template(
325
+ messages, tokenize=False,
326
+ reasoning_effort=reasoning_effort
327
+ )
328
+ except Exception as e:
329
+ logger.warning(f"apply_chat_template failed: {e}, messages: {messages}")
330
+ text = ""
331
+ return text
332
+
333
+ def __getitem__(self, index):
334
+ data = json.loads(self.data_list[index])
335
+ messages = data.get('messages', [])
336
+ image_paths = data.get('images', [])
337
+ video_paths = data.get('videos', [])
338
+
339
+ # Read reasoning_effort from data; default to 'no_think' for data without reasoning_content
340
+ reasoning_effort = data.get('reasoning_effort', 'no_think')
341
+
342
+ # Normalize legacy <image>/<video> string placeholders to multimodal content format
343
+ # so that chat_template correctly emits image/video pad tokens
344
+ messages = self._normalize_messages(messages)
345
+
346
+ # Build the text with chat template (not tokenized yet)
347
+ text = self._build_text_from_messages(messages, reasoning_effort=reasoning_effort)
348
+
349
+ if not text:
350
+ return self._make_placeholder()
351
+
352
+ # Determine modality and call processor
353
+ try:
354
+ processor_kwargs = {"text": text, "return_tensors": "pt"}
355
+
356
+ if image_paths:
357
+ images = self._load_images(image_paths)
358
+ processor_kwargs["images"] = images
359
+ # Override image processor defaults via HF's images_kwargs API
360
+ ik = {}
361
+ if self.image_min_pixels is not None or self.image_max_pixels is not None:
362
+ ik["size"] = dict(self.processor.image_processor.size)
363
+ if self.image_min_pixels is not None:
364
+ ik["size"]["shortest_edge"] = self.image_min_pixels
365
+ if self.image_max_pixels is not None:
366
+ ik["size"]["longest_edge"] = self.image_max_pixels
367
+ if ik:
368
+ processor_kwargs["images_kwargs"] = ik
369
+ if video_paths:
370
+ # Video loading is handled by the processor's video_processor
371
+ resolved_videos = [self._resolve_path(p) for p in video_paths]
372
+ processor_kwargs["videos"] = resolved_videos
373
+ # Override video processor defaults via HF's videos_kwargs API
374
+ vk = {}
375
+ if self.video_max_pixels is not None:
376
+ vk["size"] = {"shortest_edge": 4096, "longest_edge": self.video_max_pixels}
377
+ if self.video_max_frames is not None:
378
+ vk["max_frames"] = self.video_max_frames
379
+ if self.video_fps is not None:
380
+ vk["fps"] = self.video_fps
381
+ # Enable frame sampling so that max_frames/fps take effect
382
+ if self.video_max_frames is not None or self.video_fps is not None:
383
+ vk["do_sample_frames"] = True
384
+ if vk:
385
+ processor_kwargs["videos_kwargs"] = vk
386
+
387
+ inputs = self.processor(**processor_kwargs)
388
+ except Exception as e:
389
+ logger.warning(f"Processor failed for sample {index}: {e}")
390
+ return self._make_placeholder()
391
+
392
+ # Extract tensors (processor returns batch dim, squeeze it)
393
+ input_ids = inputs["input_ids"].squeeze(0) # [seq_len]
394
+ attention_mask = inputs["attention_mask"].squeeze(0) # [seq_len]
395
+ # position_ids: optional — old processor (transformers-dev) returns it,
396
+ # new processor (transformers-hy4vl) does not (model computes it internally)
397
+ position_ids = inputs["position_ids"].squeeze(0) if "position_ids" in inputs else None
398
+
399
+ # Truncate to max_seq_length
400
+ seq_len = min(input_ids.shape[0], self.max_seq_length)
401
+ input_ids = input_ids[:seq_len]
402
+ attention_mask = attention_mask[:seq_len]
403
+ if position_ids is not None:
404
+ position_ids = position_ids[:, :seq_len]
405
+
406
+ # Build labels with loss masking (only compute loss on assistant turns)
407
+ labels = torch.full_like(input_ids, IGNORE_INDEX)
408
+ boundaries = self._find_assistant_turn_boundaries(input_ids)
409
+ for middle_pos, end_pos in boundaries:
410
+ end_pos = min(end_pos, seq_len - 1)
411
+ if middle_pos + 1 < seq_len:
412
+ labels[middle_pos + 1:end_pos + 1] = input_ids[middle_pos + 1:end_pos + 1]
413
+
414
+ result = {
415
+ "input_ids": input_ids,
416
+ "labels": labels,
417
+ "attention_mask": attention_mask,
418
+ }
419
+ if position_ids is not None:
420
+ result["position_ids"] = position_ids
421
+
422
+ # Add pixel_values and grid_thw if present (for vision inputs)
423
+ if "pixel_values" in inputs:
424
+ result["pixel_values"] = inputs["pixel_values"]
425
+ # pixel_values: [num_patches, C, H, W] or similar - keep as-is
426
+ if "image_grid_thw" in inputs:
427
+ result["image_grid_thw"] = inputs["image_grid_thw"]
428
+ if "pixel_values_videos" in inputs:
429
+ result["pixel_values_videos"] = inputs["pixel_values_videos"]
430
+ if "video_grid_thw" in inputs:
431
+ result["video_grid_thw"] = inputs["video_grid_thw"]
432
+
433
+ # mm_token_type_ids: per-token modality indicator (0=text, 1=image, 2=video)
434
+ # Required by the opensource model's get_rope_index() for 3D MRoPE position_ids
435
+ if "mm_token_type_ids" in inputs:
436
+ mm_token_type_ids = inputs["mm_token_type_ids"].squeeze(0)[:seq_len]
437
+ result["mm_token_type_ids"] = mm_token_type_ids
438
+
439
+ return result
440
+
441
+ def _make_placeholder(self):
442
+ """Return a minimal valid sample when data processing fails."""
443
+ seq_len = min(2, self.max_seq_length)
444
+ input_ids = torch.tensor([self.hy_start_id, self.hy_end_id], dtype=torch.long)[:seq_len]
445
+ labels = torch.full((seq_len,), IGNORE_INDEX, dtype=torch.long)
446
+ attention_mask = torch.ones(seq_len, dtype=torch.bool)
447
+ return {
448
+ "input_ids": input_ids,
449
+ "labels": labels,
450
+ "attention_mask": attention_mask,
451
+ "mm_token_type_ids": torch.zeros(seq_len, dtype=torch.long),
452
+ }
453
+
454
+
455
+ # ============================================================================
456
+ # VL Data Collator
457
+ # ============================================================================
458
+
459
+ @dataclass
460
+ class VLDataCollator:
461
+ """Collate examples for VL supervised fine-tuning.
462
+
463
+ Handles:
464
+ - Padding input_ids, labels, attention_mask to the same length
465
+ - Padding 3D position_ids [3, seq_len] to the same length (if present)
466
+ - Concatenating pixel_values and grid_thw across the batch
467
+ """
468
+
469
+ pad_token_id: int
470
+ max_seq_length: int = 2048
471
+
472
+ def __call__(self, instances: List[Dict[str, Any]]) -> Dict[str, Any]:
473
+ # Pad text tensors
474
+ input_ids = [inst['input_ids'] for inst in instances]
475
+ labels = [inst['labels'] for inst in instances]
476
+ attention_mask = [inst['attention_mask'] for inst in instances]
477
+
478
+ # Pad input_ids and labels
479
+ input_ids = torch.nn.utils.rnn.pad_sequence(
480
+ input_ids, batch_first=True, padding_value=self.pad_token_id
481
+ )
482
+ labels = torch.nn.utils.rnn.pad_sequence(
483
+ labels, batch_first=True, padding_value=IGNORE_INDEX
484
+ )
485
+ attention_mask = input_ids.ne(self.pad_token_id)
486
+
487
+ batch = {
488
+ "input_ids": input_ids,
489
+ "labels": labels,
490
+ "attention_mask": attention_mask,
491
+ }
492
+
493
+ # Pad position_ids if present (old processor provides them;
494
+ # new processor omits them and the model computes internally)
495
+ position_ids_list = [inst.get('position_ids') for inst in instances]
496
+ if any(p is not None for p in position_ids_list):
497
+ max_len = input_ids.shape[1]
498
+ padded_position_ids = []
499
+ for pos_ids in position_ids_list:
500
+ if pos_ids is None:
501
+ # Fallback: simple incremental position ids
502
+ pos_ids = torch.arange(max_len).unsqueeze(0).expand(3, -1)
503
+ # pos_ids shape: [3, seq_len_i]
504
+ seq_len_i = pos_ids.shape[1]
505
+ if seq_len_i < max_len:
506
+ pad = torch.zeros(3, max_len - seq_len_i, dtype=pos_ids.dtype)
507
+ pos_ids = torch.cat([pos_ids, pad], dim=1)
508
+ padded_position_ids.append(pos_ids)
509
+ batch["position_ids"] = torch.stack(padded_position_ids, dim=0) # [B, 3, max_len]
510
+
511
+ # Concatenate pixel_values across the batch (variable number of patches per sample)
512
+ all_pixel_values = [inst["pixel_values"] for inst in instances if "pixel_values" in inst]
513
+ if all_pixel_values:
514
+ # Each pixel_values may have different number of patches, concat along dim 0
515
+ batch["pixel_values"] = torch.cat(all_pixel_values, dim=0)
516
+
517
+ all_image_grid_thw = [inst["image_grid_thw"] for inst in instances if "image_grid_thw" in inst]
518
+ if all_image_grid_thw:
519
+ batch["image_grid_thw"] = torch.cat(all_image_grid_thw, dim=0)
520
+
521
+ all_pixel_values_videos = [inst["pixel_values_videos"] for inst in instances if "pixel_values_videos" in inst]
522
+ if all_pixel_values_videos:
523
+ batch["pixel_values_videos"] = torch.cat(all_pixel_values_videos, dim=0)
524
+
525
+ all_video_grid_thw = [inst["video_grid_thw"] for inst in instances if "video_grid_thw" in inst]
526
+ if all_video_grid_thw:
527
+ batch["video_grid_thw"] = torch.cat(all_video_grid_thw, dim=0)
528
+
529
+ # mm_token_type_ids: per-token modality indicator for 3D MRoPE
530
+ # Pad with 0 (text type) to match the padded sequence length
531
+ all_mm_token_type_ids = [inst.get("mm_token_type_ids") for inst in instances]
532
+ if any(v is not None for v in all_mm_token_type_ids):
533
+ mm_list = []
534
+ for i, mm_ids in enumerate(all_mm_token_type_ids):
535
+ if mm_ids is None:
536
+ # Fallback: all-text for samples without mm_token_type_ids
537
+ mm_ids = torch.zeros(instances[i]["input_ids"].shape[0], dtype=torch.long)
538
+ mm_list.append(mm_ids)
539
+ batch["mm_token_type_ids"] = torch.nn.utils.rnn.pad_sequence(
540
+ mm_list, batch_first=True, padding_value=0
541
+ )
542
+
543
+ return batch
544
+
545
+
546
+ # ============================================================================
547
+ # Data Module
548
+ # ============================================================================
549
+
550
+ def make_vl_data_module(processor, data_args) -> Dict:
551
+ """Make dataset and collator for VL supervised fine-tuning."""
552
+ if data_args.use_dummy_data:
553
+ train_dataset = VLDummyDataset(processor, data_args.max_seq_length)
554
+ else:
555
+ train_dataset = VLSFTDataset(
556
+ data_file=data_args.train_data_file,
557
+ processor=processor,
558
+ max_seq_length=data_args.max_seq_length,
559
+ image_base_dir=data_args.image_base_dir,
560
+ image_min_pixels=data_args.image_min_pixels,
561
+ image_max_pixels=data_args.image_max_pixels,
562
+ video_max_pixels=data_args.video_max_pixels,
563
+ video_max_frames=data_args.video_max_frames,
564
+ video_fps=data_args.video_fps,
565
+ )
566
+ data_collator = VLDataCollator(
567
+ pad_token_id=processor.tokenizer.pad_token_id,
568
+ max_seq_length=data_args.max_seq_length,
569
+ )
570
+ return dict(train_dataset=train_dataset, eval_dataset=None, data_collator=data_collator)
571
+
572
+
573
+ # ============================================================================
574
+ # Checkpoint Save Callback
575
+ # ============================================================================
576
+
577
+ # Files to copy to each checkpoint directory
578
+ _CHECKPOINT_FILES = [
579
+ 'config.json',
580
+ 'generation_config.json',
581
+ 'tokenizer_config.json',
582
+ 'tokenizer.json',
583
+ 'chat_template.jinja',
584
+ 'preprocessor_config.json',
585
+ 'video_preprocessor_config.json',
586
+ 'hy.tiktoken',
587
+ 'tokenization_hy.py',
588
+ 'special_tokens_map.json',
589
+ ]
590
+
591
+
592
+ class CustomSaveCallback(TrainerCallback):
593
+ """Copy tokenizer and config files when saving checkpoints."""
594
+
595
+ def on_save(self, args, state, control, **kwargs):
596
+ if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0:
597
+ output_dir = os.path.join(
598
+ args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}"
599
+ )
600
+ for fname in _CHECKPOINT_FILES:
601
+ src = os.path.join(args.tokenizer_name_or_path, fname)
602
+ if os.path.isfile(src):
603
+ shutil.copy(src, os.path.join(output_dir, fname))
604
+ return control
605
+
606
+
607
+ # ============================================================================
608
+ # Main Training Function
609
+ # ============================================================================
610
+
611
+ def train():
612
+ parser = transformers.HfArgumentParser((ModelArguments, DataArguments, TrainingArguments))
613
+ model_args, data_args, training_args = parser.parse_args_into_dataclasses()
614
+ print_args(model_args, 'model arguments')
615
+ print_args(data_args, 'data arguments')
616
+ print_args(training_args, 'training arguments')
617
+
618
+ # -----------------------------------------------------------------------
619
+ # Load processor (tokenizer + image_processor + video_processor)
620
+ # -----------------------------------------------------------------------
621
+ processor = transformers.AutoProcessor.from_pretrained(
622
+ training_args.tokenizer_name_or_path or training_args.model_name_or_path,
623
+ trust_remote_code=True,
624
+ )
625
+ # Compatible with both old (4.x) and new (5.x+) transformers:
626
+ # In older versions, processor.tokenizer is always available.
627
+ # In newer versions, the attribute may not exist or may return a different type.
628
+ if hasattr(processor, 'tokenizer') and hasattr(processor.tokenizer, 'pad_token_id'):
629
+ tokenizer = processor.tokenizer
630
+ else:
631
+ tokenizer = transformers.AutoTokenizer.from_pretrained(
632
+ training_args.tokenizer_name_or_path or training_args.model_name_or_path,
633
+ trust_remote_code=True,
634
+ )
635
+ # Attach tokenizer to processor so downstream code (Dataset, DataCollator)
636
+ # can access it via processor.tokenizer
637
+ processor.tokenizer = tokenizer
638
+
639
+ init_kwargs = {}
640
+ if model_args.use_flash_attn:
641
+ init_kwargs["attn_implementation"] = "flash_attention_2"
642
+ try:
643
+ from transformers.modeling_flash_attention_utils import FLASH_ATTENTION_COMPATIBILITY_MATRIX
644
+ FLASH_ATTENTION_COMPATIBILITY_MATRIX[2]["pkg_availability_check"] = lambda *a, **kw: True
645
+ print("[Patch] Bypassed flash_attn package distribution name check for FA2.")
646
+ except Exception as e:
647
+ print(f"[Patch] Could not patch FA2 pkg check (non-fatal): {e}")
648
+
649
+ # Determine torch dtype
650
+ if training_args.bf16:
651
+ torch_dtype = torch.bfloat16
652
+ elif training_args.fp16:
653
+ torch_dtype = torch.float16
654
+ else:
655
+ torch_dtype = torch.float32
656
+
657
+ # -----------------------------------------------------------------------
658
+ # DeepSpeed ZeRO-3 setup
659
+ # -----------------------------------------------------------------------
660
+ if training_args.deepspeed:
661
+ from transformers.integrations import HfDeepSpeedConfig
662
+
663
+ ds_config = training_args.deepspeed
664
+ if isinstance(ds_config, str):
665
+ with open(ds_config, 'r') as f:
666
+ ds_config = json.load(f)
667
+
668
+ if ds_config.get("train_micro_batch_size_per_gpu", "auto") == "auto":
669
+ ds_config["train_micro_batch_size_per_gpu"] = training_args.per_device_train_batch_size
670
+ if ds_config.get("gradient_accumulation_steps", "auto") == "auto":
671
+ ds_config["gradient_accumulation_steps"] = training_args.gradient_accumulation_steps
672
+ if ds_config.get("train_batch_size", "auto") == "auto":
673
+ ds_config["train_batch_size"] = (
674
+ training_args.per_device_train_batch_size
675
+ * training_args.gradient_accumulation_steps
676
+ * (torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1)
677
+ )
678
+
679
+ # NOTE: VLM has mixed-dtype persistent_parameters (ViT fp32 params vs
680
+ # LLM bf16 params) which causes TypeError in ZeRO-3 all_gather.
681
+ # This is fixed by setting stage3_param_persistence_threshold=0 in the
682
+ # DeepSpeed config JSON file (ds_zero3_offload.json).
683
+
684
+ dschf = HfDeepSpeedConfig(ds_config) # noqa: F841
685
+
686
+ # -----------------------------------------------------------------------
687
+ # Load model (VL model: ViT + LLM)
688
+ # -----------------------------------------------------------------------
689
+ _has_weights = (
690
+ training_args.model_name_or_path is not None
691
+ and os.path.isdir(training_args.model_name_or_path)
692
+ and any(
693
+ os.path.isfile(os.path.join(training_args.model_name_or_path, f))
694
+ for f in ("model.safetensors", "pytorch_model.bin",
695
+ "model.safetensors.index.json", "pytorch_model.bin.index.json")
696
+ )
697
+ )
698
+
699
+ if _has_weights:
700
+ print(f"Loading VL model from: {training_args.model_name_or_path}")
701
+ # Disable Engram during training to avoid ~59 GiB Embedding OOM.
702
+ # The config.json is kept intact so inference can still use Engram.
703
+ _config = transformers.AutoConfig.from_pretrained(
704
+ training_args.model_name_or_path, trust_remote_code=True
705
+ )
706
+ if hasattr(_config, 'text_config') and hasattr(_config.text_config, 'engram_layer_ids'):
707
+ _config.text_config.engram_layer_ids = None
708
+ print("[HY4-VL] Disabled Engram (set text_config.engram_layer_ids=None) for training.")
709
+
710
+ model = transformers.AutoModelForImageTextToText.from_pretrained(
711
+ training_args.model_name_or_path,
712
+ config=_config,
713
+ trust_remote_code=True,
714
+ dtype=torch_dtype,
715
+ attn_implementation=init_kwargs.get("attn_implementation", None),
716
+ )
717
+ print(f"[HY4-VL] Model loaded successfully via from_pretrained.")
718
+ else:
719
+ if training_args.model_name_or_path is None:
720
+ raise ValueError("--model_name_or_path must be specified.")
721
+ print(f"Model weights not found at: {training_args.model_name_or_path}, "
722
+ f"using random initialized model instead.")
723
+ config = transformers.AutoConfig.from_pretrained(
724
+ training_args.model_name_or_path, trust_remote_code=True
725
+ )
726
+ model = transformers.AutoModelForImageTextToText.from_config(
727
+ config, trust_remote_code=True, dtype=torch_dtype,
728
+ attn_implementation=init_kwargs.get("attn_implementation", None),
729
+ )
730
+
731
+ # -----------------------------------------------------------------------
732
+ # Freeze ViT encoder (default: freeze during SFT)
733
+ # Compatible with both `model.vit` (internal) and `model.vision_tower` (opensource)
734
+ # -----------------------------------------------------------------------
735
+ if model_args.freeze_vit:
736
+ vit_module = (getattr(model, 'vit', None)
737
+ or getattr(model, 'vision_tower', None)
738
+ or getattr(getattr(model, 'model', None), 'vision_tower', None))
739
+ if vit_module is not None:
740
+ vit_module.requires_grad_(False)
741
+ vit_params = sum(getattr(p, 'ds_numel', p.numel()) for p in vit_module.parameters())
742
+ print(f"[HY4-VL] Froze ViT encoder ({vit_params / 1e6:.1f}M parameters)")
743
+ else:
744
+ print("[HY4-VL] WARNING: freeze_vit=True but no ViT encoder found (checked 'vit', 'vision_tower', 'model.vision_tower').")
745
+
746
+ # -----------------------------------------------------------------------
747
+ # LoRA setup
748
+ # -----------------------------------------------------------------------
749
+ if model_args.use_lora:
750
+ lora_config = LoraConfig(
751
+ r=model_args.lora_rank,
752
+ lora_alpha=model_args.lora_alpha,
753
+ lora_dropout=model_args.lora_dropout,
754
+ target_modules=["q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"],
755
+ bias="none",
756
+ task_type="CAUSAL_LM",
757
+ )
758
+ model = get_peft_model(model, lora_config)
759
+
760
+ # Mark PEFT LoRA wrapper modules as ZeRO-3 leaf modules
761
+ try:
762
+ from deepspeed.utils import set_z3_leaf_module
763
+ from peft.tuners.lora import Linear as LoraLinear
764
+ z3_leaf_count = 0
765
+ for module in model.modules():
766
+ if isinstance(module, LoraLinear):
767
+ set_z3_leaf_module(module, True)
768
+ z3_leaf_count += 1
769
+ print(f"[z3_leaf] Marked {z3_leaf_count} LoraLinear modules with _z3_leaf=True", flush=True)
770
+ except ImportError:
771
+ print("[z3_leaf] DeepSpeed or PEFT not available, skipping z3_leaf patch.")
772
+
773
+ # -----------------------------------------------------------------------
774
+ # Data module
775
+ # -----------------------------------------------------------------------
776
+ data_module = make_vl_data_module(processor=processor, data_args=data_args)
777
+
778
+ model.is_parallelizable = True
779
+ model.model_parallel = True
780
+
781
+ training_args.lr_scheduler_kwargs = {
782
+ 'min_lr_rate': training_args.min_lr / training_args.learning_rate,
783
+ }
784
+
785
+ # Fix: DeepSpeed ZeRO-3 + gradient checkpointing compatibility
786
+ if training_args.gradient_checkpointing and training_args.deepspeed:
787
+ training_args.gradient_checkpointing_kwargs = {"use_reentrant": True}
788
+
789
+ # -----------------------------------------------------------------------
790
+ # Create Trainer
791
+ # -----------------------------------------------------------------------
792
+ trainer = Trainer(
793
+ model=model,
794
+ processing_class=tokenizer,
795
+ args=training_args,
796
+ callbacks=[CustomSaveCallback],
797
+ **data_module
798
+ )
799
+ model.config.use_cache = False
800
+
801
+ # -----------------------------------------------------------------------
802
+ # Post-init patches
803
+ # -----------------------------------------------------------------------
804
+
805
+ # Fix dtype mismatch in DeepSpeed ZeRO-3 linear wrapper
806
+ import torch.nn.functional as _F
807
+ _orig_F_linear = _F.linear
808
+
809
+ def _dtype_safe_linear(input, weight, bias=None):
810
+ if input.dtype != weight.dtype:
811
+ input = input.to(weight.dtype)
812
+ return _orig_F_linear(input, weight, bias)
813
+
814
+ _F.linear = _dtype_safe_linear
815
+
816
+ # Skip grad norm calculation when max_grad_norm == 0
817
+ if getattr(training_args, "max_grad_norm", None) == 0:
818
+ from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3
819
+
820
+ def _skip_get_norm_groups(self):
821
+ return [torch.tensor(0.0)]
822
+
823
+ DeepSpeedZeroOptimizer_Stage3._get_norm_groups = _skip_get_norm_groups
824
+ logger.info("[grad_norm] max_grad_norm=0: patched to skip grad norm computation")
825
+
826
+ # -----------------------------------------------------------------------
827
+ # Train
828
+ # -----------------------------------------------------------------------
829
+ trainer.train(resume_from_checkpoint=training_args.resume_from_checkpoint)
830
+
831
+ if torch.distributed.is_initialized():
832
+ torch.distributed.barrier()
833
+
834
+
835
+ def _apply_safe_pin_memory_patch():
836
+ """Make CUDA pin_memory fault-tolerant for DeepSpeed ZeRO-3.
837
+
838
+ DeepSpeed ZeRO-3 calls get_accelerator().pin_memory(tensor) during
839
+ _create_param_groups_fp16_flat_cpu_memory to create pinned (page-locked)
840
+ CPU buffers for parameter offloading. This call is hard-coded and NOT
841
+ controlled by the config's ``pin_memory: false`` setting.
842
+
843
+ On some nodes, pin_memory() can sporadically fail with
844
+ ``CUDA error: invalid argument`` due to transient driver/cgroup issues.
845
+ This patch wraps the CUDA accelerator's pin_memory method so that if
846
+ the call fails, it falls back to returning the original (unpinned)
847
+ tensor instead of crashing the entire training job.
848
+ """
849
+ try:
850
+ from deepspeed.accelerator.cuda_accelerator import CUDA_Accelerator
851
+
852
+ _orig_pin_memory = CUDA_Accelerator.pin_memory
853
+
854
+ def _safe_pin_memory(self, tensor):
855
+ try:
856
+ return _orig_pin_memory(self, tensor)
857
+ except Exception as e:
858
+ rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0
859
+ print(f"[HYV4 Patch] WARNING: pin_memory() failed on rank {rank}: {e}. "
860
+ f"Falling back to unpinned CPU tensor.", flush=True)
861
+ return tensor
862
+
863
+ CUDA_Accelerator.pin_memory = _safe_pin_memory
864
+ print("[HYV4 Patch] Patched CUDA_Accelerator.pin_memory with "
865
+ "fault-tolerant fallback.", flush=True)
866
+ except ImportError:
867
+ pass
868
+
869
+
870
+ if __name__ == "__main__":
871
+ _apply_safe_pin_memory_patch()
872
+ train()
finetune/deepspeed_support/train_vl.sh ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # HY4-Flash-VL SFT training launch script
4
+ # Supports image + text, video + text, and pure text multimodal SFT
5
+
6
+ NET_TYPE="high"
7
+ export NCCL_DEBUG=WARN
8
+ export NCCL_P2P_LEVEL=NVL
9
+ export NCCL_IB_TIMEOUT=24
10
+ export NCCL_NVLS_ENABLE=0
11
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
12
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
13
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
14
+ if [[ "${NET_TYPE}" = "low" ]]; then
15
+ export NCCL_SOCKET_IFNAME=eth1
16
+ export NCCL_IB_GID_INDEX=3
17
+ export NCCL_IB_HCA=mlx5_2:1
18
+ export NCCL_IB_SL=3
19
+ export NCCL_CHECK_DISABLE=1
20
+ export NCCL_P2P_DISABLE=0
21
+ export NCCL_LL_THRESHOLD=16384
22
+ export NCCL_IB_CUDA_SUPPORT=1
23
+ else
24
+ export NCCL_IB_GID_INDEX=3
25
+ export NCCL_IB_SL=3
26
+ export NCCL_CHECK_DISABLE=1
27
+ export NCCL_P2P_DISABLE=0
28
+ export NCCL_IB_DISABLE=0
29
+ export NCCL_LL_THRESHOLD=16384
30
+ export NCCL_IB_CUDA_SUPPORT=1
31
+ export NCCL_SOCKET_IFNAME=bond1
32
+ export UCX_NET_DEVICES=bond1
33
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
34
+ export NCCL_COLLNET_ENABLE=0
35
+ export SHARP_COLL_ENABLE_SAT=0
36
+ export NCCL_NET_GDR_LEVEL=2
37
+ export NCCL_IB_QPS_PER_CONNECTION=4
38
+ export NCCL_IB_TC=160
39
+ export NCCL_PXN_DISABLE=1
40
+ fi
41
+
42
+ export HOST_GPU_NUM=8
43
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
44
+ IP_LIST=${IP_LIST:-"127.0.0.1"}
45
+
46
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
47
+ export NODES=${#IP_ARRAY[@]}
48
+ export LOCAL_IP=${IP_ARRAY[0]}
49
+ NODE_IP_LIST=""
50
+ for ip in "${IP_ARRAY[@]}"; do
51
+ if [ -n "$NODE_IP_LIST" ]; then
52
+ NODE_IP_LIST="${NODE_IP_LIST},"
53
+ fi
54
+ NODE_IP_LIST="${NODE_IP_LIST}${ip}:${HOST_GPU_NUM}"
55
+ done
56
+ export NODE_IP_LIST
57
+ export NODE_NUM=$((${NODES} * ${HOST_GPU_NUM}))
58
+
59
+ # ============================================================================
60
+ # VL-specific paths
61
+ # ============================================================================
62
+ model_path=/path/to/hy4_ckpt
63
+ tokenizer_path=/path/to/hy4_ckpt
64
+ train_data_file=../data/example_vl_data.jsonl
65
+
66
+ ds_config_file=ds_zero3_offload.json
67
+
68
+ output_path=./output_vl
69
+
70
+ mkdir -p ${output_path}
71
+
72
+ current_time=$(date "+%Y.%m.%d-%H.%M.%S")
73
+ log_file=${output_path}/"log_${current_time}.txt"
74
+
75
+ echo $NODE_IP_LIST > env.txt 2>&1
76
+ sed "s/:/ slots=/g" env.txt | sed "s/,/\n/g" > "hostfile"
77
+ sed "s/:.//g" env.txt | sed "s/,/\n/g" > "pssh.hosts"
78
+ export CHIEF_IP=$LOCAL_IP
79
+
80
+ if [ ${NODES} -gt 1 ]; then
81
+ HOST_PATH=hostfile
82
+ DS_ARGS="--hostfile=${HOST_PATH} --master_addr ${CHIEF_IP}"
83
+ else
84
+ DS_ARGS=""
85
+ fi
86
+
87
+ echo "NODES: ${NODES}, LOCAL_IP: ${LOCAL_IP}, NODE_IP_LIST: ${NODE_IP_LIST}"
88
+
89
+ deepspeed ${DS_ARGS} \
90
+ train_vl.py \
91
+ --do_train \
92
+ --model_name_or_path ${model_path} \
93
+ --tokenizer_name_or_path ${tokenizer_path} \
94
+ --train_data_file ${train_data_file} \
95
+ --freeze_vit \
96
+ --deepspeed ${ds_config_file} \
97
+ --output_dir ${output_path} \
98
+ --per_device_train_batch_size 1 \
99
+ --gradient_accumulation_steps 1 \
100
+ --gradient_checkpointing \
101
+ --lr_scheduler_type cosine_with_min_lr \
102
+ --logging_steps 1 \
103
+ --max_steps 20 \
104
+ --save_steps 20 \
105
+ --learning_rate 1e-5 \
106
+ --min_lr 1e-6 \
107
+ --warmup_steps 0 \
108
+ --save_strategy steps \
109
+ --bf16 \
110
+ --max_grad_norm 0 \
111
+ --max_seq_length 1024 | tee ${log_file}
finetune/llama_factory_support/dataset_info.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "hy_v4_flash_demo": {
3
+ "file_name": "../data/example_data.jsonl",
4
+ "formatting": "sharegpt",
5
+ "columns": {
6
+ "messages": "messages"
7
+ },
8
+ "tags": {
9
+ "role_tag": "role",
10
+ "content_tag": "content",
11
+ "user_tag": "user",
12
+ "assistant_tag": "assistant",
13
+ "system_tag": "system"
14
+ }
15
+ },
16
+ "hy_v4_flash_vl_demo": {
17
+ "file_name": "../data/example_vl_data.jsonl",
18
+ "formatting": "sharegpt",
19
+ "columns": {
20
+ "messages": "messages",
21
+ "images": "images",
22
+ "videos": "videos"
23
+ },
24
+ "tags": {
25
+ "role_tag": "role",
26
+ "content_tag": "content",
27
+ "user_tag": "user",
28
+ "assistant_tag": "assistant",
29
+ "system_tag": "system"
30
+ }
31
+ }
32
+ }
finetune/llama_factory_support/hy_v4_flash_full_sft.yaml ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ### model
2
+ model_name_or_path: /path/to/hy4_flash_ckpt
3
+ trust_remote_code: true
4
+
5
+ ### method
6
+ stage: sft
7
+ do_train: true
8
+ finetuning_type: full
9
+
10
+ deepspeed: ../deepspeed_support/ds_zero3_offload.json
11
+
12
+ ### dataset
13
+ dataset_dir: .
14
+ dataset: hy_v4_flash_demo
15
+ template: hy_v4_flash
16
+ # NOTE: To train slow-thinking (CoT), include <think:opensource>...</think:opensource>
17
+ # tags in assistant content. Without think tags, only fast-thinking is trained.
18
+ cutoff_len: 256
19
+ max_samples: 1000
20
+ overwrite_cache: true
21
+
22
+ ### output
23
+ output_dir: saves/hy_v4_flash/full/sft
24
+ logging_steps: 1
25
+ save_steps: 5
26
+ max_steps: 5
27
+ plot_loss: true
28
+ overwrite_output_dir: true
29
+ save_only_model: false
30
+ report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow]
31
+
32
+ ### train
33
+ per_device_train_batch_size: 1
34
+ gradient_accumulation_steps: 1
35
+ learning_rate: 1.0e-5
36
+ num_train_epochs: 3.0
37
+ lr_scheduler_type: cosine_with_min_lr
38
+ lr_scheduler_kwargs:
39
+ min_lr_rate: 0.1 # min_lr / learning_rate = 1e-6 / 1e-5 = 0.1
40
+ warmup_steps: 0
41
+ bf16: true
42
+ optim: adamw_torch
43
+ gradient_checkpointing: true
44
+ gradient_checkpointing_kwargs:
45
+ use_reentrant: true
46
+ ddp_timeout: 180000000
47
+ flash_attn: disabled
48
+ max_grad_norm: 0.0
49
+ resume_from_checkpoint: null
50
+
51
+ ### eval
52
+ # eval_dataset: alpaca_en_demo
53
+ # val_size: 0.1
54
+ # per_device_eval_batch_size: 1
55
+ # eval_strategy: steps
56
+ # eval_steps: 500
finetune/llama_factory_support/hy_v4_flash_lora_sft.yaml ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ### model
2
+ model_name_or_path: /path/to/hy4_flash_ckpt
3
+ trust_remote_code: true
4
+
5
+ ### method
6
+ stage: sft
7
+ do_train: true
8
+ finetuning_type: lora
9
+
10
+ # Using FSDP1 with full sharding strategy.
11
+ fsdp: true
12
+ fsdp_config:
13
+ version: 1
14
+ fsdp_sharding_strategy: FULL_SHARD
15
+ fsdp_backward_prefetch: BACKWARD_PRE
16
+ fsdp_state_dict_type: FULL_STATE_DICT
17
+ fsdp_use_orig_params: true
18
+ fsdp_cpu_ram_efficient_loading: true
19
+ fsdp_offload_params: false
20
+ fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
21
+ fsdp_transformer_layer_cls_to_wrap: HYV4FlashDecoderLayer
22
+
23
+ ### LoRA parameters
24
+ # HY4 Flash uses MLA (Multi-head Latent Attention) with these projection names
25
+ lora_rank: 64
26
+ lora_alpha: 128
27
+ lora_dropout: 0.05
28
+ lora_target: q_a_proj,q_b_proj,kv_a_proj_with_mqa,kv_b_proj,o_proj
29
+
30
+ ### dataset
31
+ dataset_dir: .
32
+ dataset: hy_v4_flash_demo
33
+ template: hy_v4_flash
34
+ cutoff_len: 1024
35
+ max_samples: 1000
36
+ overwrite_cache: true
37
+
38
+ ### output
39
+ output_dir: saves/hy_v4_flash/lora/sft
40
+ logging_steps: 1
41
+ save_steps: 500
42
+ plot_loss: true
43
+ overwrite_output_dir: true
44
+ save_only_model: false
45
+ report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow]
46
+
47
+ ### train
48
+ per_device_train_batch_size: 1
49
+ gradient_accumulation_steps: 1
50
+ learning_rate: 2.0e-4
51
+ num_train_epochs: 3.0
52
+ lr_scheduler_type: cosine_with_min_lr
53
+ lr_scheduler_kwargs:
54
+ min_lr_rate: 0.1 # min_lr / learning_rate = 2e-5 / 2e-4 = 0.1
55
+ warmup_steps: 0
56
+ bf16: true
57
+ gradient_checkpointing: true
58
+ gradient_checkpointing_kwargs:
59
+ use_reentrant: true
60
+ ddp_timeout: 180000000
61
+ flash_attn: auto
62
+ resume_from_checkpoint: null
63
+
64
+ ### eval
65
+ # eval_dataset: alpaca_en_demo
66
+ # val_size: 0.1
67
+ # per_device_eval_batch_size: 1
68
+ # eval_strategy: steps
69
+ # eval_steps: 500
finetune/llama_factory_support/hy_v4_flash_vl_full_sft.yaml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ### model
2
+ model_name_or_path: /path/to/hy4_flash_ckpt
3
+ trust_remote_code: true
4
+
5
+ ### method
6
+ stage: sft
7
+ do_train: true
8
+ finetuning_type: full
9
+
10
+ deepspeed: ../deepspeed_support/ds_zero3_offload.json
11
+
12
+ ### dataset
13
+ dataset_dir: .
14
+ dataset: hy_v4_flash_vl_demo
15
+ template: hy_v4_flash_vl
16
+ cutoff_len: 1024
17
+ max_samples: 1000
18
+ overwrite_cache: true
19
+ enable_thinking: true
20
+
21
+ ### VL-specific settings
22
+ # LLaMA Factory will auto-detect the VL model and load the processor.
23
+ # image_base_dir is resolved relative to the dataset file location.
24
+ # freeze_vision_tower: freeze the ViT encoder during SFT (recommended)
25
+ freeze_vision_tower: true
26
+
27
+ ### output
28
+ output_dir: saves/hy_v4_flash_vl/full/sft
29
+ logging_steps: 1
30
+ save_steps: 5
31
+ plot_loss: true
32
+ overwrite_output_dir: true
33
+ save_only_model: true
34
+ report_to: none
35
+
36
+ ### train
37
+ per_device_train_batch_size: 1
38
+ gradient_accumulation_steps: 1
39
+ learning_rate: 1.0e-5
40
+ max_steps: 5
41
+ num_train_epochs: 3.0
42
+ lr_scheduler_type: cosine_with_min_lr
43
+ lr_scheduler_kwargs:
44
+ min_lr_rate: 0.1
45
+ warmup_steps: 0
46
+ bf16: true
47
+ optim: adamw_torch
48
+ gradient_checkpointing: true
49
+ gradient_checkpointing_kwargs:
50
+ use_reentrant: true
51
+ ddp_timeout: 180000000
52
+ flash_attn: auto
53
+ max_grad_norm: 0.0
54
+ resume_from_checkpoint: null
finetune/llama_factory_support/hy_v4_flash_vl_lora_sft.yaml ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ### model
2
+ model_name_or_path: /path/to/hy4_flash_ckpt
3
+ trust_remote_code: true
4
+
5
+ ### method
6
+ stage: sft
7
+ do_train: true
8
+ finetuning_type: lora
9
+
10
+ # Using FSDP1 with full sharding strategy.
11
+ fsdp: true
12
+ fsdp_config:
13
+ version: 1
14
+ fsdp_sharding_strategy: FULL_SHARD
15
+ fsdp_backward_prefetch: BACKWARD_PRE
16
+ fsdp_state_dict_type: FULL_STATE_DICT
17
+ fsdp_use_orig_params: true
18
+ fsdp_cpu_ram_efficient_loading: true
19
+ fsdp_offload_params: false
20
+ fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
21
+ fsdp_transformer_layer_cls_to_wrap: HYV4FlashDecoderLayer
22
+
23
+ ### LoRA parameters
24
+ lora_rank: 64
25
+ lora_alpha: 128
26
+ lora_dropout: 0.05
27
+ # HYV4 Flash MLA projection names
28
+ lora_target: q_a_proj,q_b_proj,kv_a_proj_with_mqa,kv_b_proj,o_proj
29
+
30
+ ### VL-specific settings
31
+ freeze_vision_tower: true
32
+
33
+ ### dataset
34
+ dataset_dir: .
35
+ dataset: hy_v4_flash_vl_demo
36
+ template: hy_v4_flash_vl
37
+ cutoff_len: 1024
38
+ max_samples: 1000
39
+ overwrite_cache: true
40
+
41
+ ### output
42
+ output_dir: saves/hy_v4_flash_vl/lora/sft
43
+ logging_steps: 1
44
+ save_steps: 5
45
+ max_steps: 5
46
+ plot_loss: true
47
+ overwrite_output_dir: true
48
+ save_only_model: false
49
+ report_to: none
50
+
51
+ ### train
52
+ per_device_train_batch_size: 1
53
+ gradient_accumulation_steps: 1
54
+ learning_rate: 2.0e-4
55
+ num_train_epochs: 3.0
56
+ lr_scheduler_type: cosine_with_min_lr
57
+ lr_scheduler_kwargs:
58
+ min_lr_rate: 0.1
59
+ warmup_steps: 0
60
+ bf16: true
61
+ gradient_checkpointing: true
62
+ gradient_checkpointing_kwargs:
63
+ use_reentrant: true
64
+ ddp_timeout: 180000000
65
+ flash_attn: disabled
66
+ resume_from_checkpoint: null
finetune/llama_factory_support/hy_v4_patches.py ADDED
@@ -0,0 +1,732 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ This module applies all necessary runtime patches.
3
+
4
+
5
+ Patches applied:
6
+ 1. ZeRO-3 key rename + expert fuse + buffer loading
7
+ Handles both inner format (original) and outer format (pre-converted) checkpoints.
8
+ - Key renaming: mlp.router.gate -> mlp.gate, etc.
9
+ - Per-expert -> 3D fuse: experts.N.gate_proj -> experts.gate_up_proj
10
+ - Buffer loading: e_score_correction_bias (ZeRO-3 only handles parameters)
11
+ 2. Tokenizer file copy (CustomSaveCallback)
12
+ Ensures each checkpoint directory is self-contained for inference.
13
+ """
14
+
15
+ import os
16
+ import re
17
+ import logging
18
+ import shutil
19
+ from typing import Optional
20
+
21
+ import torch
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ # ============================================================================
26
+ # Patch 1: Key rename + expert fuse + buffer loading for ZeRO-3
27
+ #
28
+ # The checkpoint may be in either inner format (original) or outer format
29
+ # (pre-converted by convert_ckpt_to_outer.py). This patch handles both:
30
+ # - Key renaming: mlp.router.gate -> mlp.gate, etc.
31
+ # - Per-expert -> 3D fuse: experts.N.gate_proj -> experts.gate_up_proj
32
+ # - Buffer loading: e_score_correction_bias (ZeRO-3 only handles parameters)
33
+ #
34
+ # If the checkpoint is already in outer format, the rename/fuse logic is
35
+ # effectively a no-op (no matching keys to transform).
36
+ # ============================================================================
37
+
38
+ # Key renames: checkpoint inner format -> model format
39
+ _CKPT_KEY_RENAMES = [
40
+ ("mlp.router.gate.", "mlp.gate."),
41
+ ("mlp.expert_bias", "mlp.e_score_correction_bias"),
42
+ ("mlp.shared_mlp.", "mlp.shared_experts."),
43
+ # Also handle even older checkpoints that use mlp.gate.wg
44
+ ("mlp.gate.wg.", "mlp.gate."),
45
+ ]
46
+
47
+ # Regex to match per-expert keys in checkpoint
48
+ # e.g. model.layers.10.mlp.experts.5.gate_proj.weight
49
+ _EXPERT_KEY_RE = re.compile(
50
+ r"^(.*\.mlp\.experts\.)(\d+)\.(gate_proj|up_proj|down_proj)\.weight$"
51
+ )
52
+
53
+
54
+ def _apply_buffer_loading_patch():
55
+ """Patch the DeepSpeed ZeRO-3 state_dict loader to handle:
56
+ 1. Key renaming (inner checkpoint format -> model format)
57
+ 2. Per-expert -> 3D fused expert tensors
58
+ 3. Manual buffer loading (e_score_correction_bias etc.)
59
+
60
+ ZeRO-3's _load_state_dict_into_zero3_model only handles named_parameters.
61
+ Buffers like e_score_correction_bias must be loaded manually.
62
+ """
63
+ try:
64
+ from transformers.integrations.deepspeed import (
65
+ _load_state_dict_into_zero3_model as _orig_load_zero3,
66
+ )
67
+ import transformers.integrations.deepspeed as _ds_mod
68
+ import transformers.modeling_utils as _mu_mod
69
+ except ImportError:
70
+ logger.warning(
71
+ "Could not import transformers.integrations.deepspeed; "
72
+ "buffer loading patch NOT applied."
73
+ )
74
+ return
75
+
76
+ def _patched_load_zero3(model_to_load, state_dict, *args, **kwargs):
77
+ # Step 1: Key rename + per-expert collection
78
+ new_sd = {}
79
+ expert_groups = {} # prefix -> {expert_idx -> {proj_name -> tensor}}
80
+
81
+ for k, v in state_dict.items():
82
+ m = _EXPERT_KEY_RE.match(k)
83
+ if m:
84
+ # Per-expert key: collect for fusion
85
+ prefix = m.group(1)
86
+ expert_idx = int(m.group(2))
87
+ proj_name = m.group(3)
88
+ if prefix not in expert_groups:
89
+ expert_groups[prefix] = {}
90
+ if expert_idx not in expert_groups[prefix]:
91
+ expert_groups[prefix][expert_idx] = {}
92
+ expert_groups[prefix][expert_idx][proj_name] = v
93
+ else:
94
+ # Non-expert key: apply simple renames
95
+ new_k = k
96
+ for old_sub, new_sub in _CKPT_KEY_RENAMES:
97
+ if old_sub in new_k:
98
+ new_k = new_k.replace(old_sub, new_sub)
99
+ new_sd[new_k] = v
100
+
101
+ # Step 2: Fuse expert groups into 3D tensors
102
+ if expert_groups:
103
+ for prefix in sorted(expert_groups.keys()):
104
+ experts = expert_groups[prefix]
105
+ num_experts = max(experts.keys()) + 1
106
+ gate_up_list = []
107
+ down_list = []
108
+ for i in range(num_experts):
109
+ if i not in experts:
110
+ logger.warning(
111
+ "HYV4 Patch 1: Missing expert %d in %s", i, prefix
112
+ )
113
+ continue
114
+ exp = experts[i]
115
+ if "gate_proj" in exp and "up_proj" in exp:
116
+ gate_up_list.append(
117
+ torch.cat([exp["gate_proj"], exp["up_proj"]], dim=0)
118
+ )
119
+ if "down_proj" in exp:
120
+ down_list.append(exp["down_proj"])
121
+ if gate_up_list:
122
+ new_sd[f"{prefix}gate_up_proj"] = torch.stack(gate_up_list, dim=0)
123
+ if down_list:
124
+ new_sd[f"{prefix}down_proj"] = torch.stack(down_list, dim=0)
125
+ logger.info(
126
+ "HYV4 Patch 1: Fused %d expert groups from per-expert to 3D format.",
127
+ len(expert_groups)
128
+ )
129
+ del expert_groups
130
+
131
+ # Step 3: Load parameters via original ZeRO-3 loader
132
+ result = _orig_load_zero3(model_to_load, new_sd, *args, **kwargs)
133
+
134
+ # Step 4: Manually load buffers (e.g. e_score_correction_bias)
135
+ # ZeRO-3's loader only handles named_parameters, not named_buffers.
136
+ buffers_loaded = 0
137
+ for name, buf in model_to_load.named_buffers():
138
+ if name in new_sd:
139
+ src_tensor = new_sd[name]
140
+ if isinstance(src_tensor, torch.Tensor):
141
+ buf.data.copy_(src_tensor.to(buf.dtype))
142
+ buffers_loaded += 1
143
+ if isinstance(result, tuple) and len(result) >= 2:
144
+ if isinstance(result[1], set):
145
+ result[1].discard(name)
146
+ if buffers_loaded > 0:
147
+ logger.info(
148
+ "HYV4 Patch 1: Manually loaded %d buffers into model.",
149
+ buffers_loaded
150
+ )
151
+
152
+ del new_sd
153
+ return result
154
+
155
+ _ds_mod._load_state_dict_into_zero3_model = _patched_load_zero3
156
+ _mu_mod._load_state_dict_into_zero3_model = _patched_load_zero3
157
+ logger.info(
158
+ "HYV4 patch applied: ZeRO-3 key rename + expert fuse + buffer loading."
159
+ )
160
+
161
+ # ============================================================================
162
+ # Patch 2: Tokenizer file copy callback
163
+ #
164
+ # Ensures each checkpoint directory is self-contained for inference by
165
+ # copying all tokenizer-related files from the original tokenizer path.
166
+ # ============================================================================
167
+
168
+ # Tokenizer files that should be copied to each checkpoint
169
+ _TOKENIZER_FILES = [
170
+ "config.json",
171
+ "generation_config.json",
172
+ "tokenizer_config.json",
173
+ "tokenizer.json",
174
+ "chat_template.jinja",
175
+ "preprocessor_config.json",
176
+ "processor_config.json",
177
+ "video_preprocessor_config.json",
178
+ "hy.tiktoken",
179
+ "tokenization_hy.py",
180
+ "special_tokens_map.json",
181
+ ]
182
+
183
+ def _copy_tokenizer_to_checkpoint(tokenizer_dir: str, checkpoint_dir: str):
184
+ """Copy tokenizer files from tokenizer_dir to checkpoint_dir."""
185
+ for fname in _TOKENIZER_FILES:
186
+ src = os.path.join(tokenizer_dir, fname)
187
+ if os.path.isfile(src):
188
+ shutil.copy(src, os.path.join(checkpoint_dir, fname))
189
+
190
+ # ============================================================================
191
+ # LLaMA Factory Callback: integrates patch 2 (tokenizer copy) into the
192
+ # training loop.
193
+ # ============================================================================
194
+
195
+ try:
196
+ from transformers import TrainerCallback
197
+ from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
198
+
199
+ class HYV4PatchCallback(TrainerCallback):
200
+ """
201
+ LLaMA Factory compatible callback that applies HYV4-specific patches.
202
+
203
+ Add to your YAML or pass to Trainer:
204
+ callbacks: [hy_v4_patches.HYV4PatchCallback]
205
+ """
206
+
207
+ def __init__(self, tokenizer_dir: Optional[str] = None):
208
+ """
209
+ Args:
210
+ tokenizer_dir: Path to the original tokenizer directory.
211
+ If None, will try to use model_name_or_path from training args.
212
+ """
213
+ self._tokenizer_dir = tokenizer_dir
214
+
215
+ def on_save(self, args, state, control, **kwargs):
216
+ # --- Patch 2: Copy tokenizer files ---
217
+ if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0:
218
+ return control
219
+
220
+ checkpoint_dir = os.path.join(
221
+ args.output_dir,
222
+ f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}",
223
+ )
224
+
225
+ # Determine tokenizer directory
226
+ tokenizer_dir = self._tokenizer_dir
227
+ if tokenizer_dir is None:
228
+ # Try common locations
229
+ tokenizer_dir = getattr(args, "tokenizer_name_or_path", None)
230
+ if tokenizer_dir is None:
231
+ tokenizer_dir = getattr(args, "model_name_or_path", None)
232
+
233
+ if tokenizer_dir and os.path.isdir(tokenizer_dir):
234
+ _copy_tokenizer_to_checkpoint(tokenizer_dir, checkpoint_dir)
235
+ logger.info(
236
+ "HYV4: Copied tokenizer files from %s to %s",
237
+ tokenizer_dir, checkpoint_dir
238
+ )
239
+
240
+ return control
241
+
242
+ except ImportError:
243
+ logger.warning(
244
+ "transformers not available; HYV4PatchCallback not defined."
245
+ )
246
+
247
+ # ============================================================================
248
+ # Patch 3: Memory-efficient model loading for ZeRO-3 and FSDP1
249
+ #
250
+ # Previous approach: shard-by-shard manual loading that bypassed model class's
251
+ # from_pretrained, causing VL key prefix mapping to be lost (model.layers ->
252
+ # model.language_model.layers, vit. -> model.vision_tower., etc.).
253
+ #
254
+ # New approach (aligned with ms-swift): ensure HfDeepSpeedConfig is set before
255
+ # calling the original from_pretrained, so transformers' native ZeRO-3 loading
256
+ # path handles key mapping automatically via the model class internals.
257
+ #
258
+ # ZeRO-3 path:
259
+ # 1. Check if is_deepspeed_zero3_enabled() is already True
260
+ # 2. If not, detect DeepSpeed config and create HfDeepSpeedConfig
261
+ # 3. Call original from_pretrained — transformers handles efficient loading
262
+ # (meta tensors + shard-by-shard scatter) natively when ZeRO-3 is active
263
+ # 4. Key renames (Patch 1) are applied via the patched
264
+ # _load_state_dict_into_zero3_model hook
265
+ #
266
+ # FSDP1 path:
267
+ # - local_rank 0: load real weights to CPU via original from_pretrained
268
+ # - other ranks: create model on meta device (zero CPU memory)
269
+ # - FSDP's sync_module_states broadcasts weights from rank 0
270
+ # ============================================================================
271
+
272
+ def _apply_shard_loading_patch():
273
+ """Monkey-patch AutoModelForCausalLM and AutoModelForImageTextToText
274
+ from_pretrained to ensure efficient loading for ZeRO-3 and FSDP1."""
275
+ import sys
276
+ import json as _json
277
+ import transformers
278
+
279
+ _orig_from_pretrained = transformers.AutoModelForCausalLM.from_pretrained
280
+ _AutoVL = getattr(transformers, 'AutoModelForImageTextToText', None)
281
+ _orig_vl_from_pretrained = _AutoVL.from_pretrained if _AutoVL is not None else None
282
+
283
+ def _is_vl_config(config):
284
+ """Detect whether the config corresponds to a VL (vision-language) model."""
285
+ if config is None:
286
+ return False
287
+ model_type = getattr(config, 'model_type', '')
288
+ if 'vl' in model_type.lower():
289
+ return True
290
+ if hasattr(config, 'vision_config') or hasattr(config, 'text_config'):
291
+ return True
292
+ return False
293
+
294
+ def _is_fsdp_requested():
295
+ """Check if FSDP is being used (not DeepSpeed)."""
296
+ accel_fsdp = str(os.environ.get("ACCELERATE_USE_FSDP", "")).lower()
297
+ if accel_fsdp in {"1", "true", "yes"}:
298
+ return True
299
+ return "--fsdp" in sys.argv
300
+
301
+ def _disable_router_logits_if_needed(model):
302
+ """Disable output_router_logits to avoid unnecessary memory usage."""
303
+ if hasattr(model, 'config') and getattr(model.config, 'output_router_logits', False):
304
+ model.config.output_router_logits = False
305
+ logger.info("[HYV4 Patch 3] Disabled output_router_logits.")
306
+ return model
307
+
308
+ def _disable_engram(config):
309
+ """Disable Engram layers in config for training.
310
+
311
+ Engram tables are very large (~32 GB per layer in bf16) and cause
312
+ ZeRO-3 all-gather to hang or OOM during training. The checkpoint
313
+ stores engram_layer_head_vocab_sizes as nested lists which also
314
+ triggers TypeError in the current modeling code. Following the
315
+ deepspeed_support/train_vl.py approach, we simply set
316
+ engram_layer_ids to None so no Engram layers are created.
317
+ """
318
+ if config is None:
319
+ return config
320
+ text_cfg = getattr(config, 'text_config', config)
321
+ if hasattr(text_cfg, 'engram_layer_ids') and text_cfg.engram_layer_ids is not None:
322
+ old_val = text_cfg.engram_layer_ids
323
+ text_cfg.engram_layer_ids = None
324
+ print(f"[HYV4 Patch] Disabled Engram (set engram_layer_ids=None) for training.", flush=True)
325
+ return config
326
+
327
+ def _fsdp_load(pretrained_model_name_or_path, *args, **kwargs):
328
+ """FSDP-safe loading: rank 0 loads to CPU, others use meta device."""
329
+ kwargs = dict(kwargs)
330
+ kwargs.setdefault("low_cpu_mem_usage", True)
331
+
332
+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))
333
+ if local_rank != 0:
334
+ logger.info(
335
+ "[HYV4 Patch 3] FSDP mode: local_rank=%d != 0, "
336
+ "creating model on meta device (zero CPU memory). "
337
+ "Weights will be synced from rank 0 by FSDP.",
338
+ local_rank,
339
+ )
340
+ config = transformers.AutoConfig.from_pretrained(
341
+ pretrained_model_name_or_path,
342
+ trust_remote_code=kwargs.get("trust_remote_code", False),
343
+ )
344
+ _disable_engram(config)
345
+ kwargs["config"] = config
346
+ torch_dtype = kwargs.get("torch_dtype", None)
347
+ if torch_dtype is None:
348
+ torch_dtype = getattr(config, "torch_dtype", torch.bfloat16)
349
+ if not isinstance(torch_dtype, torch.dtype):
350
+ torch_dtype = torch.bfloat16
351
+ with torch.device("meta"):
352
+ _AutoCls = _AutoVL if (_AutoVL is not None and _is_vl_config(config)) else transformers.AutoModelForCausalLM
353
+ model = _AutoCls.from_config(
354
+ config,
355
+ torch_dtype=torch_dtype,
356
+ trust_remote_code=kwargs.get("trust_remote_code", False),
357
+ )
358
+ return _disable_router_logits_if_needed(model)
359
+
360
+ # local_rank 0: load real weights to CPU
361
+ logger.info(
362
+ "[HYV4 Patch 3] FSDP mode: local_rank=0, loading real weights to CPU."
363
+ )
364
+ config = transformers.AutoConfig.from_pretrained(
365
+ pretrained_model_name_or_path,
366
+ trust_remote_code=kwargs.get("trust_remote_code", False),
367
+ )
368
+ _disable_engram(config)
369
+ kwargs["config"] = config
370
+ kwargs["device_map"] = {"": "cpu"}
371
+ _fallback_fn = _orig_vl_from_pretrained if (_AutoVL is not None and _is_vl_config(config)) else _orig_from_pretrained
372
+ model = _fallback_fn(pretrained_model_name_or_path, *args, **kwargs)
373
+ return _disable_router_logits_if_needed(model)
374
+
375
+ # ZeRO-3 not yet enabled
376
+ def _patched_from_pretrained(pretrained_model_name_or_path, *args, **kwargs):
377
+ """Patched from_pretrained that ensures efficient loading for ZeRO-3 and FSDP1."""
378
+ model_path = pretrained_model_name_or_path
379
+
380
+ # Only intercept local directory paths with safetensors files
381
+ if not (isinstance(model_path, str) and os.path.isdir(model_path)):
382
+ config = kwargs.get("config")
383
+ if config is None:
384
+ config = transformers.AutoConfig.from_pretrained(
385
+ model_path,
386
+ trust_remote_code=kwargs.get("trust_remote_code", False),
387
+ )
388
+ _disable_engram(config)
389
+ kwargs["config"] = config
390
+ _fallback_fn = _orig_vl_from_pretrained if (_AutoVL is not None and _is_vl_config(config)) else _orig_from_pretrained
391
+ return _fallback_fn(pretrained_model_name_or_path, *args, **kwargs)
392
+
393
+ index_file = os.path.join(model_path, "model.safetensors.index.json")
394
+ single_file = os.path.join(model_path, "model.safetensors")
395
+ if not (os.path.isfile(index_file) or os.path.isfile(single_file)):
396
+ config = kwargs.get("config")
397
+ if config is None:
398
+ config = transformers.AutoConfig.from_pretrained(
399
+ model_path,
400
+ trust_remote_code=kwargs.get("trust_remote_code", False),
401
+ )
402
+ _disable_engram(config)
403
+ kwargs["config"] = config
404
+ _fallback_fn = _orig_vl_from_pretrained if (_AutoVL is not None and _is_vl_config(config)) else _orig_from_pretrained
405
+ return _fallback_fn(pretrained_model_name_or_path, *args, **kwargs)
406
+
407
+ # FSDP path: rank-aware loading
408
+ if _is_fsdp_requested():
409
+ return _fsdp_load(pretrained_model_name_or_path, *args, **kwargs)
410
+
411
+ # ZeRO-3 path: ensure HfDeepSpeedConfig is set, then use native from_pretrained
412
+ from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled
413
+
414
+ if is_deepspeed_zero3_enabled():
415
+ config = kwargs.get("config")
416
+ if config is None:
417
+ config = transformers.AutoConfig.from_pretrained(
418
+ model_path,
419
+ trust_remote_code=kwargs.get("trust_remote_code", False),
420
+ )
421
+ _disable_engram(config)
422
+ kwargs["config"] = config
423
+ _load_fn = _orig_vl_from_pretrained if (_AutoVL is not None and _is_vl_config(config)) else _orig_from_pretrained
424
+ model = _load_fn(pretrained_model_name_or_path, *args, **kwargs)
425
+ return _disable_router_logits_if_needed(model)
426
+
427
+ # ZeRO-3 not yet enabled — try to detect and set HfDeepSpeedConfig
428
+ # This handles the case where from_pretrained is called before
429
+ # TrainingArguments has set up the DeepSpeed config.
430
+ ds_config_path = os.environ.get("DEEPSPEED_CONFIG_FILE", None)
431
+ if ds_config_path is None:
432
+ ds_config_path = os.environ.get("DEEPSPEED_CONFIG", None)
433
+ if ds_config_path is None:
434
+ for i, arg in enumerate(sys.argv):
435
+ if arg == '--deepspeed' and i + 1 < len(sys.argv):
436
+ ds_config_path = sys.argv[i + 1]
437
+ break
438
+
439
+ if ds_config_path is not None and os.path.isfile(ds_config_path):
440
+ with open(ds_config_path, "r") as f:
441
+ ds_config = _json.load(f)
442
+ zero_stage = ds_config.get("zero_optimization", {}).get("stage", 0)
443
+ if zero_stage == 3:
444
+ from transformers.integrations.deepspeed import HfDeepSpeedConfig
445
+ _ds_config_obj = HfDeepSpeedConfig(ds_config_path) # noqa: F841
446
+
447
+ config = kwargs.get("config")
448
+ if config is None:
449
+ config = transformers.AutoConfig.from_pretrained(
450
+ model_path,
451
+ trust_remote_code=kwargs.get("trust_remote_code", False),
452
+ )
453
+ _disable_engram(config)
454
+ kwargs["config"] = config
455
+ _load_fn = _orig_vl_from_pretrained if (_AutoVL is not None and _is_vl_config(config)) else _orig_from_pretrained
456
+ model = _load_fn(pretrained_model_name_or_path, *args, **kwargs)
457
+ return _disable_router_logits_if_needed(model)
458
+
459
+ # Fallback: no DeepSpeed ZeRO-3 detected, use FSDP-style loading
460
+ logger.info(
461
+ "[HYV4 Patch 3] No ZeRO-3 config detected, using FSDP-style fallback."
462
+ )
463
+ return _fsdp_load(pretrained_model_name_or_path, *args, **kwargs)
464
+
465
+ # Apply the monkey-patch to both CausalLM and VL auto classes
466
+ transformers.AutoModelForCausalLM.from_pretrained = staticmethod(_patched_from_pretrained)
467
+ if hasattr(transformers, 'AutoModelForImageTextToText'):
468
+ transformers.AutoModelForImageTextToText.from_pretrained = staticmethod(_patched_from_pretrained)
469
+ logger.info(
470
+ "HYV4 Patch 3 applied: efficient model loading for ZeRO-3 and FSDP1. "
471
+ "ZeRO-3 uses native from_pretrained (key mapping by model class). "
472
+ "Patched: AutoModelForCausalLM + AutoModelForImageTextToText."
473
+ )
474
+
475
+
476
+ # ============================================================================
477
+ # Auto-apply patches on import
478
+ # ============================================================================
479
+
480
+ # Patch 1: ZeRO-3 key rename + expert fuse + buffer loading
481
+ _apply_buffer_loading_patch()
482
+
483
+ # Patch 3: Memory-efficient shard-by-shard loading
484
+ _apply_shard_loading_patch()
485
+
486
+ # Patch 4: Unify model dtype before FSDP wrap
487
+ # FSDP1 requires all parameters in the same FSDP unit to have uniform dtype.
488
+ # After LoRA injection, adapter params are float32 while base model is bfloat16.
489
+ # This patch casts all parameters to bf16 before FSDP wraps the model.
490
+ def _apply_fsdp_dtype_patch():
491
+ """Monkey-patch Trainer to cast model to bf16 before FSDP wrap."""
492
+ try:
493
+ from transformers import Trainer
494
+
495
+ _orig_prepare_for_training = Trainer._prepare_for_training
496
+
497
+ def _patched_prepare_for_training(self, *args, **kwargs):
498
+ # Cast all model parameters to bf16 before FSDP wrap and disable
499
+ # mixed precision to prevent Accelerate from upcasting bf16→fp32.
500
+ if getattr(self.args, 'fsdp', False):
501
+ model = self.model
502
+ dtype_counts = {}
503
+ for p in model.parameters():
504
+ dt = str(p.dtype)
505
+ dtype_counts[dt] = dtype_counts.get(dt, 0) + 1
506
+
507
+ if len(dtype_counts) > 1:
508
+ logger.info(
509
+ "[HYV4 Patch 4] Mixed dtypes detected before FSDP wrap: %s. "
510
+ "Casting all parameters to bfloat16.",
511
+ dtype_counts
512
+ )
513
+ for p in model.parameters():
514
+ if p.dtype != torch.bfloat16:
515
+ p.data = p.data.to(torch.bfloat16)
516
+ else:
517
+ logger.info(
518
+ "[HYV4 Patch 4] All params already uniform dtype: %s",
519
+ dtype_counts
520
+ )
521
+
522
+ # Disable FSDP mixed precision to prevent Accelerate from
523
+ # upcasting bf16 params to fp32 during wrap. Since all params
524
+ # are already bf16 (cast above), we don't need FSDP's mixed
525
+ # precision policy. This saves ~2x GPU memory per shard.
526
+ try:
527
+ if hasattr(self, 'accelerator'):
528
+ # Level 1: Set accelerator state _mixed_precision to "no"
529
+ # Note: mixed_precision is a property, must set _mixed_precision
530
+ if hasattr(self.accelerator, 'state'):
531
+ old_mp = getattr(self.accelerator.state, '_mixed_precision', None)
532
+ self.accelerator.state._mixed_precision = "no"
533
+ logger.info(
534
+ "[HYV4 Patch 4] Set accelerator.state._mixed_precision='no' "
535
+ "(was: %s) to prevent bf16→fp32 upcast.", old_mp
536
+ )
537
+ # Level 2: Clear the FSDP plugin's mixed_precision_policy
538
+ fsdp_plugin = getattr(self.accelerator.state, 'fsdp_plugin', None)
539
+ if fsdp_plugin is not None:
540
+ if hasattr(fsdp_plugin, 'mixed_precision_policy'):
541
+ fsdp_plugin.mixed_precision_policy = None
542
+ if hasattr(fsdp_plugin, 'kwargs'):
543
+ fsdp_plugin.kwargs.pop('mixed_precision', None)
544
+ logger.info(
545
+ "[HYV4 Patch 4] Cleared fsdp_plugin mixed_precision_policy."
546
+ )
547
+ else:
548
+ logger.warning(
549
+ "[HYV4 Patch 4] self.accelerator not found, cannot disable mixed precision."
550
+ )
551
+ # Level 3: Environment variable (for any lazy initialization)
552
+ os.environ["ACCELERATE_MIXED_PRECISION"] = "no"
553
+ except Exception as e:
554
+ logger.warning(
555
+ "[HYV4 Patch 4] Failed to disable mixed precision: %s", e
556
+ )
557
+
558
+ # ---- Fix PEFT + FSDP _no_split_modules mismatch ----
559
+ # PEFT's fsdp_auto_wrap_policy iterates _no_split_modules and calls
560
+ # get_module_class_from_name() for each class name. If a class is
561
+ # listed but not instantiated in the model (e.g. HYV4FlashEngramLayer
562
+ # when engram is disabled), the lookup returns None and PEFT raises:
563
+ # "Could not find the transformer layer class to wrap in the model."
564
+ # Fix: collect all actual module class names and prune _no_split_modules.
565
+ try:
566
+ actual_cls_names = {m.__class__.__name__ for m in self.model.modules()}
567
+ # Walk through all sub-models (PeftModel wraps the base model)
568
+ for sub in self.model.modules():
569
+ if hasattr(sub, '_no_split_modules') and sub._no_split_modules:
570
+ original = list(sub._no_split_modules)
571
+ pruned = [c for c in original if c in actual_cls_names]
572
+ if len(pruned) < len(original):
573
+ removed = set(original) - set(pruned)
574
+ sub._no_split_modules = pruned
575
+ logger.info(
576
+ "[HYV4 Patch 4] Pruned _no_split_modules: removed %s "
577
+ "(not found in model). Remaining: %s",
578
+ removed, pruned
579
+ )
580
+ except Exception as e:
581
+ logger.warning(
582
+ "[HYV4 Patch 4] Failed to prune _no_split_modules: %s", e
583
+ )
584
+
585
+ return _orig_prepare_for_training(self, *args, **kwargs)
586
+
587
+ Trainer._prepare_for_training = _patched_prepare_for_training
588
+ logger.info(
589
+ "HYV4 Patch 4 applied: unify model dtype to bf16 before FSDP wrap."
590
+ )
591
+ except (ImportError, AttributeError) as e:
592
+ logger.warning("[HYV4 Patch 4] Could not apply dtype patch: %s", e)
593
+
594
+ _apply_fsdp_dtype_patch()
595
+
596
+ # Patch 5: Fix CustomSeq2SeqTrainer.create_optimizer signature
597
+ # New transformers (5.15+) calls self.create_optimizer(model) with a model arg,
598
+ # but LLaMA Factory's CustomSeq2SeqTrainer.create_optimizer(self) doesn't accept it.
599
+ def _apply_create_optimizer_patch():
600
+ """Monkey-patch CustomSeq2SeqTrainer.create_optimizer to accept optional model arg."""
601
+ try:
602
+ from llamafactory.train.sft.trainer import CustomSeq2SeqTrainer
603
+
604
+ _orig_create_optimizer = CustomSeq2SeqTrainer.create_optimizer
605
+
606
+ def _patched_create_optimizer(self, model=None):
607
+ return _orig_create_optimizer(self)
608
+
609
+ CustomSeq2SeqTrainer.create_optimizer = _patched_create_optimizer
610
+ logger.info(
611
+ "HYV4 Patch 5 applied: CustomSeq2SeqTrainer.create_optimizer "
612
+ "now accepts optional model argument for transformers >= 5.15."
613
+ )
614
+ except (ImportError, AttributeError) as e:
615
+ logger.warning("[HYV4 Patch 5] Could not apply create_optimizer patch: %s", e)
616
+
617
+ _apply_create_optimizer_patch()
618
+
619
+ # Patch 6: Fix LLaMA Factory's _gradient_checkpointing_enable signature
620
+ # New transformers (5.16+) trainer calls model.gradient_checkpointing_enable()
621
+ # with every_n_layers and offload kwargs. But LLaMA Factory replaces this method
622
+ # with its own _gradient_checkpointing_enable that only accepts
623
+ # (self, gradient_checkpointing_kwargs, use_unsloth_gc). This causes:
624
+ # TypeError: _gradient_checkpointing_enable() got an unexpected keyword argument 'every_n_layers'
625
+ # Fix: patch LLaMA Factory's function to accept and ignore the extra kwargs.
626
+ def _apply_gradient_checkpointing_compat_patch():
627
+ """Patch LLaMA Factory's _gradient_checkpointing_enable to accept extra kwargs
628
+ from new transformers (every_n_layers, offload)."""
629
+ try:
630
+ import llamafactory.model.model_utils.checkpointing as _lf_gc_mod
631
+
632
+ _orig_gc_enable = _lf_gc_mod._gradient_checkpointing_enable
633
+
634
+ def _patched_gc_enable(self, gradient_checkpointing_kwargs=None, **kwargs):
635
+ # Filter out kwargs not accepted by the original LLaMA Factory function
636
+ # (every_n_layers, offload from new transformers trainer)
637
+ use_unsloth_gc = kwargs.pop("use_unsloth_gc", False)
638
+ if kwargs:
639
+ logger.info(
640
+ "[HYV4 Patch 6] Ignoring extra gradient_checkpointing kwargs "
641
+ "not supported by LLaMA Factory: %s", list(kwargs.keys())
642
+ )
643
+ return _orig_gc_enable(
644
+ self,
645
+ gradient_checkpointing_kwargs=gradient_checkpointing_kwargs,
646
+ use_unsloth_gc=use_unsloth_gc,
647
+ )
648
+
649
+ _lf_gc_mod._gradient_checkpointing_enable = _patched_gc_enable
650
+ logger.info(
651
+ "HYV4 Patch 6 applied: LLaMA Factory _gradient_checkpointing_enable "
652
+ "now accepts extra kwargs (every_n_layers, offload) from transformers >= 5.16."
653
+ )
654
+ except (ImportError, AttributeError) as e:
655
+ logger.warning("[HYV4 Patch 6] Could not apply gradient checkpointing compat patch: %s", e)
656
+
657
+ _apply_gradient_checkpointing_compat_patch()
658
+
659
+ # Patch 7: Fix LLaMA Factory's custom_gradient_checkpointing_func compatibility
660
+ # with transformers-hy4flash's modeling_layers.py.
661
+ #
662
+ # LLaMA Factory's custom_gradient_checkpointing_func (checkpointing.py:86) does:
663
+ # module = func.func.__self__
664
+ # This assumes func is partial(bound_method, ...) where bound_method has __self__.
665
+ #
666
+ # But transformers-hy4flash's modeling_layers.py:121 passes:
667
+ # partial(forward, self, **kwargs)
668
+ # where forward is a plain function (not a bound method), so func.func has no __self__.
669
+ # The module (self) is instead the first positional arg in func.args.
670
+ #
671
+ # Fix: patch get_custom_gradient_checkpointing_func to handle both cases.
672
+ def _apply_gc_func_compat_patch():
673
+ """Patch LLaMA Factory's custom_gradient_checkpointing_func to handle
674
+ partial(plain_function, module, ...) in addition to partial(bound_method, ...)."""
675
+ try:
676
+ import torch
677
+ from functools import partial
678
+ import llamafactory.model.model_utils.checkpointing as _lf_gc_mod
679
+
680
+ _orig_get_custom_gc_func = _lf_gc_mod.get_custom_gradient_checkpointing_func
681
+
682
+ def _patched_get_custom_gc_func(gradient_checkpointing_func):
683
+ from functools import wraps, WRAPPER_ASSIGNMENTS
684
+
685
+ @wraps(gradient_checkpointing_func, assigned=WRAPPER_ASSIGNMENTS + ("__self__",))
686
+ def custom_gradient_checkpointing_func(func, *args, **kwargs):
687
+ # Extract the module from func (a partial object or bound method)
688
+ module = None
689
+ if isinstance(func, partial):
690
+ if hasattr(func.func, '__self__'):
691
+ # Standard case: partial(bound_method, ...)
692
+ module = func.func.__self__
693
+ elif func.args and isinstance(func.args[0], torch.nn.Module):
694
+ # transformers-hy4flash case: partial(plain_function, module, ...)
695
+ module = func.args[0]
696
+ else:
697
+ # Fallback: try original behavior
698
+ module = func.func.__self__
699
+ else:
700
+ module = func.__self__
701
+
702
+ has_grad = False
703
+ if any(param.requires_grad for param in module.parameters()):
704
+ has_grad = True
705
+ for arg in args:
706
+ if torch.is_tensor(arg) and torch.is_floating_point(arg):
707
+ arg.requires_grad_(True)
708
+ break
709
+
710
+ if has_grad:
711
+ return gradient_checkpointing_func(func, *args, **kwargs)
712
+ else:
713
+ return func(*args)
714
+
715
+ return custom_gradient_checkpointing_func
716
+
717
+ _lf_gc_mod.get_custom_gradient_checkpointing_func = _patched_get_custom_gc_func
718
+ print("[HYV4 Patch] Patched get_custom_gradient_checkpointing_func for "
719
+ "transformers-hy4flash compatibility (partial(forward, self, ...) support).",
720
+ flush=True)
721
+ except (ImportError, AttributeError) as e:
722
+ print(f"[HYV4 Patch 7] Could not apply gc func compat patch: {e}", flush=True)
723
+
724
+ _apply_gc_func_compat_patch()
725
+
726
+ # Patch 2 (tokenizer copy) is applied via HYV4PatchCallback during training.
727
+ # Users should add HYV4PatchCallback to their Trainer callbacks.
728
+
729
+ logger.info(
730
+ "HYV4 patches module loaded. Remember to add HYV4PatchCallback to "
731
+ "your Trainer callbacks for tokenizer file copy on save."
732
+ )
finetune/llama_factory_support/hy_v4_template.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ HY4 Flash chat template registration for LLaMA Factory.
3
+
4
+ This module registers templates for the HY4 Flash model (hy_v4_flash).
5
+ The Flash model uses 'opensource' namespace for special tokens.
6
+ """
7
+
8
+ from copy import deepcopy
9
+ from dataclasses import dataclass
10
+ from typing import Optional, TYPE_CHECKING
11
+
12
+ from llamafactory.data.template import ReasoningTemplate, register_template
13
+ from llamafactory.data.formatter import EmptyFormatter, StringFormatter
14
+ from llamafactory.data.mm_plugin import (
15
+ BasePlugin,
16
+ get_mm_plugin,
17
+ register_mm_plugin,
18
+ )
19
+ from llamafactory.extras.constants import IMAGE_PLACEHOLDER
20
+
21
+ if TYPE_CHECKING:
22
+ import torch
23
+ from llamafactory.data.mm_plugin import MMProcessor, ImageInput, VideoInput, AudioInput
24
+
25
+ @dataclass
26
+ class HYV4VLPlugin(BasePlugin):
27
+ """Multimodal plugin for HYV4-VL models in LLaMA Factory.
28
+
29
+ Replaces <image> in messages with N copies of the real image placeholder
30
+ token, where N = image_grid_thw.prod() / merge_length.
31
+ """
32
+
33
+ def _get_mm_inputs(self, images, videos, audios, processor):
34
+ """Process images/videos through the HYV4VL image processor."""
35
+ image_processor = getattr(processor, "image_processor", None)
36
+ mm_inputs = {}
37
+ if len(images) != 0 and image_processor is not None:
38
+ images = self._regularize_images(
39
+ images,
40
+ image_max_pixels=getattr(processor, "image_max_pixels",
41
+ getattr(processor, "max_pixels", 4194304)),
42
+ image_min_pixels=getattr(processor, "image_min_pixels",
43
+ getattr(processor, "min_pixels", 4096)),
44
+ )["images"]
45
+ mm_inputs.update(image_processor(images, return_tensors="pt"))
46
+ if len(videos) != 0:
47
+ video_processor = getattr(processor, "video_processor", image_processor)
48
+ if video_processor is not None:
49
+ videos_data = self._regularize_videos(
50
+ videos,
51
+ image_max_pixels=getattr(processor, "video_max_pixels", 256 * 256),
52
+ image_min_pixels=getattr(processor, "video_min_pixels", 16 * 16),
53
+ video_fps=getattr(processor, "video_fps", 2.0),
54
+ video_maxlen=getattr(processor, "video_maxlen", 128),
55
+ )["videos"]
56
+ import inspect
57
+ if "videos" in inspect.signature(video_processor.preprocess).parameters:
58
+ mm_inputs.update(video_processor(images=None, videos=videos_data, return_tensors="pt"))
59
+ else:
60
+ mm_inputs.update(video_processor(videos_data, return_tensors="pt"))
61
+ return mm_inputs
62
+
63
+ @staticmethod
64
+ def _num_frame_tokens(grid_h, grid_w, merge_size):
65
+ """Compute the number of tokens per frame, including one newline token per merged row.
66
+
67
+ Must match HYV4VLProcessor._num_frame_tokens in transformers-hy4vl.
68
+ """
69
+ return (grid_h // merge_size) * (grid_w // merge_size + 1)
70
+
71
+ def process_messages(self, messages, images, videos, audios, processor):
72
+ """Replace <image> placeholders with expanded image placeholder tokens."""
73
+ self._validate_input(processor, images, videos, audios)
74
+ self._validate_messages(messages, images, videos, audios)
75
+ num_image_tokens, num_video_tokens = 0, 0
76
+ messages = deepcopy(messages)
77
+ image_processor = getattr(processor, "image_processor", None)
78
+ merge_size = getattr(image_processor, "merge_size", 2)
79
+ if self.expand_mm_tokens and (len(images) > 0 or len(videos) > 0):
80
+ mm_inputs = self._get_mm_inputs(images, videos, audios, processor)
81
+ image_grid_thw = mm_inputs.get("image_grid_thw", [])
82
+ video_grid_thw = mm_inputs.get("video_grid_thw", [])
83
+ else:
84
+ image_grid_thw = [None] * len(images)
85
+ video_grid_thw = [None] * len(videos)
86
+ for message in messages:
87
+ content = message["content"]
88
+ while IMAGE_PLACEHOLDER in content:
89
+ if self.expand_mm_tokens and num_image_tokens < len(image_grid_thw):
90
+ _, grid_h, grid_w = image_grid_thw[num_image_tokens]
91
+ image_seqlen = self._num_frame_tokens(grid_h, grid_w, merge_size)
92
+ else:
93
+ image_seqlen = 1
94
+ content = content.replace(IMAGE_PLACEHOLDER, self.image_token * image_seqlen, 1)
95
+ num_image_tokens += 1
96
+ if self.video_token is not None:
97
+ from llamafactory.extras.constants import VIDEO_PLACEHOLDER
98
+ while VIDEO_PLACEHOLDER in content:
99
+ if self.expand_mm_tokens and num_video_tokens < len(video_grid_thw):
100
+ grid_t, grid_h, grid_w = video_grid_thw[num_video_tokens]
101
+ video_seqlen = grid_t * self._num_frame_tokens(grid_h, grid_w, merge_size)
102
+ else:
103
+ video_seqlen = 1
104
+ content = content.replace(VIDEO_PLACEHOLDER, self.video_token * video_seqlen, 1)
105
+ num_video_tokens += 1
106
+ message["content"] = content
107
+ return messages
108
+
109
+ # Register the custom plugin
110
+ register_mm_plugin("hyv4_flash_vl", HYV4VLPlugin)
111
+
112
+ # ---------------------------------------------------------------------------
113
+ # HY4 Flash (MoE, pure text) chat template
114
+ # Uses 'opensource' namespace for special tokens
115
+ # ---------------------------------------------------------------------------
116
+
117
+ register_template(
118
+ name="hy_v4_flash",
119
+ template_class=ReasoningTemplate,
120
+ format_user=StringFormatter(slots=["<|hy_start:opensource|>user<|hy_middle:opensource|>{{content}}<|hy_end:opensource|><|hy_start:opensource|>assistant<|hy_middle:opensource|>"]),
121
+ format_assistant=StringFormatter(slots=["{{content}}<|hy_end:opensource|>"]),
122
+ format_system=StringFormatter(slots=["<|hy_start:opensource|>system<|hy_middle:opensource|>{{content}}<|hy_end:opensource|>"]),
123
+ default_system="<|reasoning_mode:opensource|>reasoning_effort:no_think",
124
+ format_prefix=EmptyFormatter(slots=[]),
125
+ thought_words=("<think:opensource>", "</think:opensource>"),
126
+ stop_words=["<|hy_end:opensource|>"],
127
+ efficient_eos=False,
128
+ )
129
+
130
+ # ---------------------------------------------------------------------------
131
+ # HY4 Flash VL (Vision-Language) chat template
132
+ # Uses 'opensource' namespace for special tokens
133
+ # ---------------------------------------------------------------------------
134
+
135
+ register_template(
136
+ name="hy_v4_flash_vl",
137
+ template_class=ReasoningTemplate,
138
+ format_user=StringFormatter(slots=["<|hy_start:opensource|>user<|hy_middle:opensource|>{{content}}<|hy_end:opensource|><|hy_start:opensource|>assistant<|hy_middle:opensource|>"]),
139
+ format_assistant=StringFormatter(slots=["{{content}}<|hy_end:opensource|>"]),
140
+ format_system=StringFormatter(slots=["<|hy_start:opensource|>system<|hy_middle:opensource|>{{content}}<|hy_end:opensource|>"]),
141
+ default_system="<|reasoning_mode:opensource|>reasoning_effort:no_think",
142
+ format_prefix=EmptyFormatter(slots=[]),
143
+ thought_words=("<think:opensource>", "</think:opensource>"),
144
+ stop_words=["<|hy_end:opensource|>"],
145
+ efficient_eos=False,
146
+ mm_plugin=get_mm_plugin(
147
+ name="hyv4_flash_vl",
148
+ image_token="<|hy_image\u2581pad:opensource|>",
149
+ video_token="<|hy_video\u2581pad:opensource|>",
150
+ ),
151
+ )
finetune/llama_factory_support/train_hy_v4.py ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ LLaMA Factory training entry-point wrapper.
3
+
4
+ This script:
5
+ 1. Registers the chat template
6
+ 2. Applies all monkey-patches (checkpoint key rename, dtype fix, etc.)
7
+ 3. Injects HYV4PatchCallback into the training loop
8
+ 4. Calls run_exp() to start LLaMA Factory training
9
+
10
+ """
11
+
12
+ import sys
13
+ import os
14
+
15
+ # Add current directory to path so patches can be imported
16
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
17
+
18
+ # Step 1: Register HYV4 template (must be before training starts)
19
+ import hy_v4_template # noqa: F401
20
+
21
+ # Step 2: Apply checkpoint key rename patch (must be before model loading)
22
+ import hy_v4_patches # noqa: F401
23
+
24
+ # Step 3: Inject HYV4PatchCallback into LLaMA Factory's training flow
25
+ from llamafactory.train.sft.workflow import run_sft as _orig_run_sft
26
+
27
+
28
+ def _patched_run_sft(
29
+ model_args, data_args, training_args,
30
+ finetuning_args, generating_args, callbacks=None
31
+ ):
32
+ """Wrap run_sft to inject HYV4PatchCallback."""
33
+ if callbacks is None:
34
+ callbacks = []
35
+
36
+ # Determine tokenizer directory for the save callback
37
+ tokenizer_dir = getattr(model_args, "model_name_or_path", None)
38
+ callbacks.append(
39
+ hy_v4_patches.HYV4PatchCallback(tokenizer_dir=tokenizer_dir)
40
+ )
41
+
42
+ return _orig_run_sft(
43
+ model_args, data_args, training_args,
44
+ finetuning_args, generating_args,
45
+ callbacks=callbacks
46
+ )
47
+
48
+
49
+ # Monkey-patch the SFT workflow
50
+ import llamafactory.train.sft.workflow as _sft_wf
51
+ _sft_wf.run_sft = _patched_run_sft
52
+
53
+
54
+ def _apply_safe_pin_memory_patch():
55
+ """Make CUDA pin_memory fault-tolerant for DeepSpeed ZeRO-3.
56
+
57
+ DeepSpeed ZeRO-3 calls get_accelerator().pin_memory(tensor) during
58
+ _create_param_groups_fp16_flat_cpu_memory to create pinned (page-locked)
59
+ CPU buffers for parameter offloading. This call is hard-coded and NOT
60
+ controlled by the config's ``pin_memory: false`` setting.
61
+
62
+ On some nodes, pin_memory() can sporadically fail with
63
+ ``CUDA error: invalid argument`` due to transient driver/cgroup issues.
64
+ This patch wraps the CUDA accelerator's pin_memory method so that if
65
+ the call fails, it falls back to returning the original (unpinned)
66
+ tensor instead of crashing the entire training job.
67
+ """
68
+ try:
69
+ from deepspeed.accelerator.cuda_accelerator import CUDA_Accelerator
70
+
71
+ _orig_pin_memory = CUDA_Accelerator.pin_memory
72
+
73
+ def _safe_pin_memory(self, tensor):
74
+ try:
75
+ return _orig_pin_memory(self, tensor)
76
+ except Exception as e:
77
+ import torch
78
+ rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0
79
+ print(f"[HYV4 Patch] WARNING: pin_memory() failed on rank {rank}: {e}. "
80
+ f"Falling back to unpinned CPU tensor.", flush=True)
81
+ return tensor
82
+
83
+ CUDA_Accelerator.pin_memory = _safe_pin_memory
84
+ print("[HYV4 Patch] Patched CUDA_Accelerator.pin_memory with "
85
+ "fault-tolerant fallback.", flush=True)
86
+ except ImportError:
87
+ pass
88
+
89
+
90
+ def _apply_dtype_safe_linear_patch():
91
+ """Fix dtype mismatch in DeepSpeed ZeRO-3 linear wrapper.
92
+
93
+ Under ZeRO-3, the all-gathered weight may be in a different dtype than
94
+ the input activation (e.g. ViT outputs bf16 but lm_head weight is fp32).
95
+ This patch ensures input is cast to match weight dtype before F.linear.
96
+ """
97
+ import torch.nn.functional as _F
98
+ _orig_F_linear = _F.linear
99
+
100
+ def _dtype_safe_linear(input, weight, bias=None):
101
+ if input.dtype != weight.dtype:
102
+ input = input.to(weight.dtype)
103
+ return _orig_F_linear(input, weight, bias)
104
+
105
+ _F.linear = _dtype_safe_linear
106
+ print("[HYV4 Patch] Applied dtype-safe F.linear wrapper for ZeRO-3.", flush=True)
107
+
108
+
109
+ def _apply_skip_grad_norm_patch():
110
+ """Skip grad norm computation for DeepSpeed ZeRO-3 + CPU offload.
111
+
112
+ Under ZeRO-3 + CPU offload, DeepSpeed's complete_grad_norm_calculation
113
+ all-gathers every gradient on CPU and does an ALLREDUCE to compute the
114
+ global L2 norm. For a 770B model this is extremely slow and can cause
115
+ NCCL timeout/deadlock at optimizer step.
116
+
117
+ When max_grad_norm=0 (no clipping), we fully skip the norm computation
118
+ by patching _get_norm_groups to return 0.0 immediately.
119
+ """
120
+ import torch
121
+ try:
122
+ from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3
123
+
124
+ def _skip_get_norm_groups(self):
125
+ return [torch.tensor(0.0)]
126
+
127
+ DeepSpeedZeroOptimizer_Stage3._get_norm_groups = _skip_get_norm_groups
128
+ print("[HYV4 Patch] Patched DeepSpeedZeroOptimizer_Stage3._get_norm_groups "
129
+ "to skip grad norm computation (max_grad_norm=0).", flush=True)
130
+ except ImportError:
131
+ # DeepSpeed not available, skip
132
+ pass
133
+
134
+
135
+ def _apply_empty_cache_before_save_patch():
136
+ """Clear GPU cache before saving checkpoints to avoid OOM.
137
+
138
+ Under ZeRO-3 with large models, GPU memory is extremely tight during
139
+ training. When the Trainer tries to save a checkpoint, DeepSpeed needs
140
+ to perform NCCL all_reduce operations which require GPU memory allocation.
141
+ If the PyTorch CUDA cache is holding stale memory, this allocation can
142
+ fail with OOM.
143
+
144
+ This patch wraps both Trainer._save_checkpoint and Trainer.save_model
145
+ to call torch.cuda.empty_cache() before the actual save, ensuring enough
146
+ GPU memory is available for DeepSpeed's NCCL operations.
147
+ """
148
+ import torch
149
+ try:
150
+ from transformers import Trainer
151
+
152
+ # Patch 1: Clear cache before _save_checkpoint (outer entry point)
153
+ _orig_save_checkpoint = Trainer._save_checkpoint
154
+
155
+ def _save_checkpoint_with_cache_clear(self, model, trial, **kwargs):
156
+ if torch.cuda.is_available():
157
+ torch.cuda.empty_cache()
158
+ return _orig_save_checkpoint(self, model, trial, **kwargs)
159
+
160
+ Trainer._save_checkpoint = _save_checkpoint_with_cache_clear
161
+ print("[HYV4 Patch] Patched Trainer._save_checkpoint to clear CUDA cache before save.", flush=True)
162
+
163
+ # Patch 2: Clear cache inside save_model, right before DeepSpeed's
164
+ # save_checkpoint call. When stage3_gather_16bit_weights_on_model_save
165
+ # is False, save_model catches ValueError and falls back to
166
+ # self.model_wrapped.save_checkpoint(output_dir). The NCCL all_reduce
167
+ # inside that call can OOM if GPU cache is not cleared first.
168
+ _orig_save_model = Trainer.save_model
169
+
170
+ def _save_model_with_cache_clear(self, output_dir=None, _internal_call=False):
171
+ if torch.cuda.is_available():
172
+ torch.cuda.empty_cache()
173
+ return _orig_save_model(self, output_dir=output_dir, _internal_call=_internal_call)
174
+
175
+ Trainer.save_model = _save_model_with_cache_clear
176
+ print("[HYV4 Patch] Patched Trainer.save_model to clear CUDA cache before save.", flush=True)
177
+
178
+ except ImportError:
179
+ pass
180
+
181
+
182
+ def main():
183
+ """Entry point: called by torchrun in each worker process.
184
+
185
+ Since train_lf.sh launches us via torchrun directly, all patches
186
+ (template registration, checkpoint key rename, SFT callback injection)
187
+ are already applied in this process. We just call run_exp() to start
188
+ training — no need to go through the CLI launcher.
189
+ """
190
+ # Apply grad norm skip patch for DeepSpeed ZeRO-3 full SFT.
191
+ # This must be done before Trainer creates the DeepSpeed engine.
192
+ # The patch is safe even when not using DeepSpeed (it's a no-op if
193
+ # DeepSpeed is not imported or ZeRO-3 is not used).
194
+ # Make pin_memory fault-tolerant to handle sporadic CUDA errors
195
+ # on certain nodes during DeepSpeed ZeRO-3 initialization.
196
+ # Must be applied before DeepSpeed engine is created.
197
+ _apply_safe_pin_memory_patch()
198
+
199
+ _apply_skip_grad_norm_patch()
200
+
201
+ # Apply dtype-safe linear patch for VL models under ZeRO-3.
202
+ # This fixes dtype mismatch between ViT outputs and linear layer weights.
203
+ _apply_dtype_safe_linear_patch()
204
+
205
+ # Clear GPU cache before checkpoint save to prevent OOM during
206
+ # DeepSpeed's NCCL all_reduce in save_checkpoint.
207
+ _apply_empty_cache_before_save_patch()
208
+
209
+ from llamafactory.train.tuner import run_exp
210
+ run_exp()
211
+
212
+
213
+ if __name__ == "__main__":
214
+ main()
finetune/llama_factory_support/train_lf.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # ============================================================================
3
+ # LLaMA Factory VL full fine-tuning + DeepSpeed ZeRO-3 training launch script
4
+ #
5
+ # This script uses pdsh to automatically distribute to all nodes.
6
+ # You only need to run it once on any node.
7
+ #
8
+ # Usage:
9
+ # nohup bash train_lf.sh > lf_vl_full.log 2>&1 &
10
+ # ============================================================================
11
+
12
+ set -euo pipefail
13
+
14
+ # -------------------- Network Configuration --------------------
15
+ NET_TYPE="high"
16
+ export NCCL_DEBUG=WARN
17
+ export NCCL_P2P_LEVEL=NVL
18
+ export NCCL_IB_TIMEOUT=24
19
+ export NCCL_NVLS_ENABLE=0
20
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
21
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
22
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
23
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
24
+ if [[ "${NET_TYPE}" = "low" ]]; then
25
+ export NCCL_SOCKET_IFNAME=eth1
26
+ export NCCL_IB_GID_INDEX=3
27
+ export NCCL_IB_HCA=mlx5_2:1
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_LL_THRESHOLD=16384
32
+ export NCCL_IB_CUDA_SUPPORT=1
33
+ else
34
+ export NCCL_IB_GID_INDEX=3
35
+ export NCCL_IB_SL=3
36
+ export NCCL_CHECK_DISABLE=1
37
+ export NCCL_P2P_DISABLE=0
38
+ export NCCL_IB_DISABLE=0
39
+ export NCCL_LL_THRESHOLD=16384
40
+ export NCCL_IB_CUDA_SUPPORT=1
41
+ export NCCL_SOCKET_IFNAME=bond1
42
+ export UCX_NET_DEVICES=bond1
43
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
44
+ export NCCL_COLLNET_ENABLE=0
45
+ export SHARP_COLL_ENABLE_SAT=0
46
+ export NCCL_NET_GDR_LEVEL=2
47
+ export NCCL_IB_QPS_PER_CONNECTION=4
48
+ export NCCL_IB_TC=160
49
+ export NCCL_PXN_DISABLE=1
50
+ fi
51
+
52
+ export DISABLE_VERSION_CHECK=1
53
+
54
+ # -------------------- Node Configuration --------------------
55
+ export HOST_GPU_NUM=8
56
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "127.0.0.1"
57
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
58
+
59
+ MASTER_PORT=${MASTER_PORT:-29500}
60
+
61
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
62
+ NODES=${#IP_ARRAY[@]}
63
+ MASTER_ADDR=${IP_ARRAY[0]}
64
+
65
+ # -------------------- Paths --------------------
66
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
67
+ YAML_FILE="${YAML_FILE:-${SCRIPT_DIR}/hy_v4_flash_vl_full_sft.yaml}"
68
+ ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_v4.py"
69
+
70
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
71
+ # If running multi-node and this is the launcher (not a worker dispatched by pdsh),
72
+ # use pdsh to distribute the script to all nodes automatically.
73
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
74
+ echo "============================================"
75
+ echo " HYV4-Flash-VL LLaMA Factory FULL + DeepSpeed ZeRO-3"
76
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
77
+ echo " Nodes: ${IP_LIST}"
78
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
79
+ echo " GPUs per node: ${HOST_GPU_NUM}"
80
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
81
+ echo " Config: ${YAML_FILE}"
82
+ echo "============================================"
83
+
84
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
85
+
86
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
87
+ "export NCCL_DEBUG=WARN; \
88
+ export NCCL_P2P_LEVEL=NVL; \
89
+ export NCCL_IB_TIMEOUT=24; \
90
+ export NCCL_NVLS_ENABLE=0; \
91
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
92
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
93
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
94
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
95
+ export NCCL_IB_GID_INDEX=3; \
96
+ export NCCL_IB_SL=3; \
97
+ export NCCL_CHECK_DISABLE=1; \
98
+ export NCCL_P2P_DISABLE=0; \
99
+ export NCCL_IB_DISABLE=0; \
100
+ export NCCL_LL_THRESHOLD=16384; \
101
+ export NCCL_IB_CUDA_SUPPORT=1; \
102
+ export NCCL_SOCKET_IFNAME=bond1; \
103
+ export UCX_NET_DEVICES=bond1; \
104
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
105
+ export NCCL_COLLNET_ENABLE=0; \
106
+ export SHARP_COLL_ENABLE_SAT=0; \
107
+ export NCCL_NET_GDR_LEVEL=2; \
108
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
109
+ export NCCL_IB_TC=160; \
110
+ export NCCL_PXN_DISABLE=1; \
111
+ export DISABLE_VERSION_CHECK=1; \
112
+ export IP_LIST='${IP_LIST}'; \
113
+ export YAML_FILE='${YAML_FILE}'; \
114
+ export _WORKER_MODE=1; \
115
+ cd ${SCRIPT_DIR}; \
116
+ bash ${SCRIPT_DIR}/train_lf.sh"
117
+ exit $?
118
+ fi
119
+
120
+ # -------------------- Worker Mode: determine node rank and launch --------------------
121
+ export MASTER_ADDR="${MASTER_ADDR}"
122
+ export MASTER_PORT="${MASTER_PORT}"
123
+ export NNODES="${NODES}"
124
+
125
+ # Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
126
+ export ACCELERATE_MIXED_PRECISION=no
127
+
128
+ if [ ${NODES} -gt 1 ]; then
129
+ # Determine local node rank by matching local IP against IP_LIST
130
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
131
+ NODE_RANK=0
132
+ for i in "${!IP_ARRAY[@]}"; do
133
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
134
+ NODE_RANK=$i
135
+ break
136
+ fi
137
+ done
138
+ export RANK="${NODE_RANK}"
139
+ else
140
+ export RANK=0
141
+ fi
142
+
143
+ echo "============================================"
144
+ echo " HYV4-Flash-VL LLaMA Factory Training"
145
+ echo " Worker mode: Node ${RANK}/${NNODES}"
146
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
147
+ echo " GPUs per node: ${HOST_GPU_NUM}"
148
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
149
+ echo " Config: ${YAML_FILE}"
150
+ echo "============================================"
151
+
152
+ # -------------------- Launch --------------------
153
+ # Stagger node launches to avoid cephfs I/O congestion from 128 processes
154
+ # importing transformers simultaneously. Each node waits RANK * 5 seconds.
155
+ if [ "${RANK}" -gt 0 ]; then
156
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
157
+ sleep $((RANK * 5))
158
+ fi
159
+
160
+ torchrun \
161
+ --nnodes "${NNODES}" \
162
+ --node_rank "${RANK}" \
163
+ --nproc_per_node "${HOST_GPU_NUM}" \
164
+ --master_addr "${MASTER_ADDR}" \
165
+ --master_port "${MASTER_PORT}" \
166
+ "${ENTRY_SCRIPT}" "${YAML_FILE}"
finetune/llama_factory_support/train_lf_llm.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # ============================================================================
3
+ # LLaMA Factory LLM full fine-tuning + DeepSpeed ZeRO-3 training launch script
4
+ #
5
+ # This script uses pdsh to automatically distribute to all nodes.
6
+ # You only need to run it once on any node.
7
+ #
8
+ # Usage:
9
+ # nohup bash train_lf_llm.sh > lf_llm_full.log 2>&1 &
10
+ # ============================================================================
11
+
12
+ set -euo pipefail
13
+
14
+ # -------------------- Network Configuration --------------------
15
+ NET_TYPE="high"
16
+ export NCCL_DEBUG=WARN
17
+ export NCCL_P2P_LEVEL=NVL
18
+ export NCCL_IB_TIMEOUT=24
19
+ export NCCL_NVLS_ENABLE=0
20
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
21
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
22
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
23
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
24
+ if [[ "${NET_TYPE}" = "low" ]]; then
25
+ export NCCL_SOCKET_IFNAME=eth1
26
+ export NCCL_IB_GID_INDEX=3
27
+ export NCCL_IB_HCA=mlx5_2:1
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_LL_THRESHOLD=16384
32
+ export NCCL_IB_CUDA_SUPPORT=1
33
+ else
34
+ export NCCL_IB_GID_INDEX=3
35
+ export NCCL_IB_SL=3
36
+ export NCCL_CHECK_DISABLE=1
37
+ export NCCL_P2P_DISABLE=0
38
+ export NCCL_IB_DISABLE=0
39
+ export NCCL_LL_THRESHOLD=16384
40
+ export NCCL_IB_CUDA_SUPPORT=1
41
+ export NCCL_SOCKET_IFNAME=bond1
42
+ export UCX_NET_DEVICES=bond1
43
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
44
+ export NCCL_COLLNET_ENABLE=0
45
+ export SHARP_COLL_ENABLE_SAT=0
46
+ export NCCL_NET_GDR_LEVEL=2
47
+ export NCCL_IB_QPS_PER_CONNECTION=4
48
+ export NCCL_IB_TC=160
49
+ export NCCL_PXN_DISABLE=1
50
+ fi
51
+
52
+ export DISABLE_VERSION_CHECK=1
53
+
54
+ # -------------------- Node Configuration --------------------
55
+ export HOST_GPU_NUM=8
56
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "127.0.0.1"
57
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
58
+
59
+ MASTER_PORT=${MASTER_PORT:-29500}
60
+
61
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
62
+ NODES=${#IP_ARRAY[@]}
63
+ MASTER_ADDR=${IP_ARRAY[0]}
64
+
65
+ # -------------------- Paths --------------------
66
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
67
+ YAML_FILE="${YAML_FILE:-${SCRIPT_DIR}/hy_v4_flash_full_sft.yaml}"
68
+ ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_v4.py"
69
+
70
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
71
+ # If running multi-node and this is the launcher (not a worker dispatched by pdsh),
72
+ # use pdsh to distribute the script to all nodes automatically.
73
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
74
+ echo "============================================"
75
+ echo " HYV4-Flash LLM LLaMA Factory FULL + DeepSpeed ZeRO-3"
76
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
77
+ echo " Nodes: ${IP_LIST}"
78
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
79
+ echo " GPUs per node: ${HOST_GPU_NUM}"
80
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
81
+ echo " Config: ${YAML_FILE}"
82
+ echo "============================================"
83
+
84
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
85
+
86
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
87
+ "export NCCL_DEBUG=WARN; \
88
+ export NCCL_P2P_LEVEL=NVL; \
89
+ export NCCL_IB_TIMEOUT=24; \
90
+ export NCCL_NVLS_ENABLE=0; \
91
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
92
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
93
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
94
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
95
+ export NCCL_IB_GID_INDEX=3; \
96
+ export NCCL_IB_SL=3; \
97
+ export NCCL_CHECK_DISABLE=1; \
98
+ export NCCL_P2P_DISABLE=0; \
99
+ export NCCL_IB_DISABLE=0; \
100
+ export NCCL_LL_THRESHOLD=16384; \
101
+ export NCCL_IB_CUDA_SUPPORT=1; \
102
+ export NCCL_SOCKET_IFNAME=bond1; \
103
+ export UCX_NET_DEVICES=bond1; \
104
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
105
+ export NCCL_COLLNET_ENABLE=0; \
106
+ export SHARP_COLL_ENABLE_SAT=0; \
107
+ export NCCL_NET_GDR_LEVEL=2; \
108
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
109
+ export NCCL_IB_TC=160; \
110
+ export NCCL_PXN_DISABLE=1; \
111
+ export DISABLE_VERSION_CHECK=1; \
112
+ export IP_LIST='${IP_LIST}'; \
113
+ export YAML_FILE='${YAML_FILE}'; \
114
+ export _WORKER_MODE=1; \
115
+ cd ${SCRIPT_DIR}; \
116
+ bash ${SCRIPT_DIR}/train_lf_llm.sh"
117
+ exit $?
118
+ fi
119
+
120
+ # -------------------- Worker Mode: determine node rank and launch --------------------
121
+ export MASTER_ADDR="${MASTER_ADDR}"
122
+ export MASTER_PORT="${MASTER_PORT}"
123
+ export NNODES="${NODES}"
124
+
125
+ # Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
126
+ export ACCELERATE_MIXED_PRECISION=no
127
+
128
+ if [ ${NODES} -gt 1 ]; then
129
+ # Determine local node rank by matching local IP against IP_LIST
130
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
131
+ NODE_RANK=0
132
+ for i in "${!IP_ARRAY[@]}"; do
133
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
134
+ NODE_RANK=$i
135
+ break
136
+ fi
137
+ done
138
+ export RANK="${NODE_RANK}"
139
+ else
140
+ export RANK=0
141
+ fi
142
+
143
+ echo "============================================"
144
+ echo " HYV4-Flash LLM LLaMA Factory Training"
145
+ echo " Worker mode: Node ${RANK}/${NNODES}"
146
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
147
+ echo " GPUs per node: ${HOST_GPU_NUM}"
148
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
149
+ echo " Config: ${YAML_FILE}"
150
+ echo "============================================"
151
+
152
+ # -------------------- Launch --------------------
153
+ # Stagger node launches to avoid cephfs I/O congestion from 128 processes
154
+ # importing transformers simultaneously. Each node waits RANK * 5 seconds.
155
+ if [ "${RANK}" -gt 0 ]; then
156
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
157
+ sleep $((RANK * 5))
158
+ fi
159
+
160
+ torchrun \
161
+ --nnodes "${NNODES}" \
162
+ --node_rank "${RANK}" \
163
+ --nproc_per_node "${HOST_GPU_NUM}" \
164
+ --master_addr "${MASTER_ADDR}" \
165
+ --master_port "${MASTER_PORT}" \
166
+ "${ENTRY_SCRIPT}" "${YAML_FILE}"
finetune/llama_factory_support/train_lf_lora.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # ============================================================================
3
+ # LLaMA Factory VL LoRA + FSDP training launch script
4
+ #
5
+ # This script uses pdsh to automatically distribute to all nodes.
6
+ # You only need to run it once on any node.
7
+ #
8
+ # Usage:
9
+ # nohup bash train_lf_lora.sh > lf_vl_lora.log 2>&1 &
10
+ # ============================================================================
11
+
12
+ set -euo pipefail
13
+
14
+ # -------------------- Network Configuration --------------------
15
+ NET_TYPE="high"
16
+ export NCCL_DEBUG=WARN
17
+ export NCCL_P2P_LEVEL=NVL
18
+ export NCCL_IB_TIMEOUT=24
19
+ export NCCL_NVLS_ENABLE=0
20
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
21
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
22
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
23
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
24
+ if [[ "${NET_TYPE}" = "low" ]]; then
25
+ export NCCL_SOCKET_IFNAME=eth1
26
+ export NCCL_IB_GID_INDEX=3
27
+ export NCCL_IB_HCA=mlx5_2:1
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_LL_THRESHOLD=16384
32
+ export NCCL_IB_CUDA_SUPPORT=1
33
+ else
34
+ export NCCL_IB_GID_INDEX=3
35
+ export NCCL_IB_SL=3
36
+ export NCCL_CHECK_DISABLE=1
37
+ export NCCL_P2P_DISABLE=0
38
+ export NCCL_IB_DISABLE=0
39
+ export NCCL_LL_THRESHOLD=16384
40
+ export NCCL_IB_CUDA_SUPPORT=1
41
+ export NCCL_SOCKET_IFNAME=bond1
42
+ export UCX_NET_DEVICES=bond1
43
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
44
+ export NCCL_COLLNET_ENABLE=0
45
+ export SHARP_COLL_ENABLE_SAT=0
46
+ export NCCL_NET_GDR_LEVEL=2
47
+ export NCCL_IB_QPS_PER_CONNECTION=4
48
+ export NCCL_IB_TC=160
49
+ export NCCL_PXN_DISABLE=1
50
+ fi
51
+
52
+ export DISABLE_VERSION_CHECK=1
53
+
54
+ # -------------------- Node Configuration --------------------
55
+ export HOST_GPU_NUM=8
56
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "127.0.0.1"
57
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
58
+
59
+ MASTER_PORT=${MASTER_PORT:-29500}
60
+
61
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
62
+ NODES=${#IP_ARRAY[@]}
63
+ MASTER_ADDR=${IP_ARRAY[0]}
64
+
65
+ # -------------------- Paths --------------------
66
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
67
+ YAML_FILE="${YAML_FILE:-${SCRIPT_DIR}/hy_v4_flash_vl_lora_sft.yaml}"
68
+ ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_v4.py"
69
+
70
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
71
+ # If running multi-node and this is the launcher (not a worker dispatched by pdsh),
72
+ # use pdsh to distribute the script to all nodes automatically.
73
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
74
+ echo "============================================"
75
+ echo " HYV4-Flash-VL LLaMA Factory LoRA + FSDP"
76
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
77
+ echo " Nodes: ${IP_LIST}"
78
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
79
+ echo " GPUs per node: ${HOST_GPU_NUM}"
80
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
81
+ echo " Config: ${YAML_FILE}"
82
+ echo "============================================"
83
+
84
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
85
+
86
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
87
+ "export NCCL_DEBUG=WARN; \
88
+ export NCCL_P2P_LEVEL=NVL; \
89
+ export NCCL_IB_TIMEOUT=24; \
90
+ export NCCL_NVLS_ENABLE=0; \
91
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
92
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
93
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
94
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
95
+ export NCCL_IB_GID_INDEX=3; \
96
+ export NCCL_IB_SL=3; \
97
+ export NCCL_CHECK_DISABLE=1; \
98
+ export NCCL_P2P_DISABLE=0; \
99
+ export NCCL_IB_DISABLE=0; \
100
+ export NCCL_LL_THRESHOLD=16384; \
101
+ export NCCL_IB_CUDA_SUPPORT=1; \
102
+ export NCCL_SOCKET_IFNAME=bond1; \
103
+ export UCX_NET_DEVICES=bond1; \
104
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
105
+ export NCCL_COLLNET_ENABLE=0; \
106
+ export SHARP_COLL_ENABLE_SAT=0; \
107
+ export NCCL_NET_GDR_LEVEL=2; \
108
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
109
+ export NCCL_IB_TC=160; \
110
+ export NCCL_PXN_DISABLE=1; \
111
+ export DISABLE_VERSION_CHECK=1; \
112
+ export IP_LIST='${IP_LIST}'; \
113
+ export YAML_FILE='${YAML_FILE}'; \
114
+ export _WORKER_MODE=1; \
115
+ cd ${SCRIPT_DIR}; \
116
+ bash ${SCRIPT_DIR}/train_lf_lora.sh"
117
+ exit $?
118
+ fi
119
+
120
+ # -------------------- Worker Mode: determine node rank and launch --------------------
121
+ export MASTER_ADDR="${MASTER_ADDR}"
122
+ export MASTER_PORT="${MASTER_PORT}"
123
+ export NNODES="${NODES}"
124
+
125
+ # Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
126
+ export ACCELERATE_MIXED_PRECISION=no
127
+
128
+ if [ ${NODES} -gt 1 ]; then
129
+ # Determine local node rank by matching local IP against IP_LIST
130
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
131
+ NODE_RANK=0
132
+ for i in "${!IP_ARRAY[@]}"; do
133
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
134
+ NODE_RANK=$i
135
+ break
136
+ fi
137
+ done
138
+ export RANK="${NODE_RANK}"
139
+ else
140
+ export RANK=0
141
+ fi
142
+
143
+ echo "============================================"
144
+ echo " HYV4-Flash-VL LLaMA Factory LoRA Training"
145
+ echo " Worker mode: Node ${RANK}/${NNODES}"
146
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
147
+ echo " GPUs per node: ${HOST_GPU_NUM}"
148
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
149
+ echo " Config: ${YAML_FILE}"
150
+ echo "============================================"
151
+
152
+ # -------------------- Launch --------------------
153
+ # Stagger node launches to avoid cephfs I/O congestion from 128 processes
154
+ # importing transformers simultaneously. Each node waits RANK * 5 seconds.
155
+ if [ "${RANK}" -gt 0 ]; then
156
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
157
+ sleep $((RANK * 5))
158
+ fi
159
+
160
+ torchrun \
161
+ --nnodes "${NNODES}" \
162
+ --node_rank "${RANK}" \
163
+ --nproc_per_node "${HOST_GPU_NUM}" \
164
+ --master_addr "${MASTER_ADDR}" \
165
+ --master_port "${MASTER_PORT}" \
166
+ "${ENTRY_SCRIPT}" "${YAML_FILE}"
finetune/llama_factory_support/train_lf_lora_llm.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # ============================================================================
3
+ # LLaMA Factory LLM LoRA + FSDP training launch script
4
+ #
5
+ # This script uses pdsh to automatically distribute to all nodes.
6
+ # You only need to run it once on any node.
7
+ #
8
+ # Usage:
9
+ # nohup bash train_lf_lora_llm.sh > lf_llm_lora.log 2>&1 &
10
+ # ============================================================================
11
+
12
+ set -euo pipefail
13
+
14
+ # -------------------- Network Configuration --------------------
15
+ NET_TYPE="high"
16
+ export NCCL_DEBUG=WARN
17
+ export NCCL_P2P_LEVEL=NVL
18
+ export NCCL_IB_TIMEOUT=24
19
+ export NCCL_NVLS_ENABLE=0
20
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
21
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
22
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
23
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
24
+ if [[ "${NET_TYPE}" = "low" ]]; then
25
+ export NCCL_SOCKET_IFNAME=eth1
26
+ export NCCL_IB_GID_INDEX=3
27
+ export NCCL_IB_HCA=mlx5_2:1
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_LL_THRESHOLD=16384
32
+ export NCCL_IB_CUDA_SUPPORT=1
33
+ else
34
+ export NCCL_IB_GID_INDEX=3
35
+ export NCCL_IB_SL=3
36
+ export NCCL_CHECK_DISABLE=1
37
+ export NCCL_P2P_DISABLE=0
38
+ export NCCL_IB_DISABLE=0
39
+ export NCCL_LL_THRESHOLD=16384
40
+ export NCCL_IB_CUDA_SUPPORT=1
41
+ export NCCL_SOCKET_IFNAME=bond1
42
+ export UCX_NET_DEVICES=bond1
43
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
44
+ export NCCL_COLLNET_ENABLE=0
45
+ export SHARP_COLL_ENABLE_SAT=0
46
+ export NCCL_NET_GDR_LEVEL=2
47
+ export NCCL_IB_QPS_PER_CONNECTION=4
48
+ export NCCL_IB_TC=160
49
+ export NCCL_PXN_DISABLE=1
50
+ fi
51
+
52
+ export DISABLE_VERSION_CHECK=1
53
+
54
+ # -------------------- Node Configuration --------------------
55
+ export HOST_GPU_NUM=8
56
+ # IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "127.0.0.1"
57
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
58
+
59
+ MASTER_PORT=${MASTER_PORT:-29500}
60
+
61
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
62
+ NODES=${#IP_ARRAY[@]}
63
+ MASTER_ADDR=${IP_ARRAY[0]}
64
+
65
+ # -------------------- Paths --------------------
66
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
67
+ YAML_FILE="${YAML_FILE:-${SCRIPT_DIR}/hy_v4_flash_lora_sft.yaml}"
68
+ ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_v4.py"
69
+
70
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
71
+ # If running multi-node and this is the launcher (not a worker dispatched by pdsh),
72
+ # use pdsh to distribute the script to all nodes automatically.
73
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
74
+ echo "============================================"
75
+ echo " HYV4-Flash LLM LLaMA Factory LoRA + FSDP"
76
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
77
+ echo " Nodes: ${IP_LIST}"
78
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
79
+ echo " GPUs per node: ${HOST_GPU_NUM}"
80
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
81
+ echo " Config: ${YAML_FILE}"
82
+ echo "============================================"
83
+
84
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
85
+
86
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
87
+ "export NCCL_DEBUG=WARN; \
88
+ export NCCL_P2P_LEVEL=NVL; \
89
+ export NCCL_IB_TIMEOUT=24; \
90
+ export NCCL_NVLS_ENABLE=0; \
91
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
92
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
93
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
94
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
95
+ export NCCL_IB_GID_INDEX=3; \
96
+ export NCCL_IB_SL=3; \
97
+ export NCCL_CHECK_DISABLE=1; \
98
+ export NCCL_P2P_DISABLE=0; \
99
+ export NCCL_IB_DISABLE=0; \
100
+ export NCCL_LL_THRESHOLD=16384; \
101
+ export NCCL_IB_CUDA_SUPPORT=1; \
102
+ export NCCL_SOCKET_IFNAME=bond1; \
103
+ export UCX_NET_DEVICES=bond1; \
104
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
105
+ export NCCL_COLLNET_ENABLE=0; \
106
+ export SHARP_COLL_ENABLE_SAT=0; \
107
+ export NCCL_NET_GDR_LEVEL=2; \
108
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
109
+ export NCCL_IB_TC=160; \
110
+ export NCCL_PXN_DISABLE=1; \
111
+ export DISABLE_VERSION_CHECK=1; \
112
+ export IP_LIST='${IP_LIST}'; \
113
+ export YAML_FILE='${YAML_FILE}'; \
114
+ export _WORKER_MODE=1; \
115
+ cd ${SCRIPT_DIR}; \
116
+ bash ${SCRIPT_DIR}/train_lf_lora_llm.sh"
117
+ exit $?
118
+ fi
119
+
120
+ # -------------------- Worker Mode: determine node rank and launch --------------------
121
+ export MASTER_ADDR="${MASTER_ADDR}"
122
+ export MASTER_PORT="${MASTER_PORT}"
123
+ export NNODES="${NODES}"
124
+
125
+ # Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
126
+ export ACCELERATE_MIXED_PRECISION=no
127
+
128
+ if [ ${NODES} -gt 1 ]; then
129
+ # Determine local node rank by matching local IP against IP_LIST
130
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
131
+ NODE_RANK=0
132
+ for i in "${!IP_ARRAY[@]}"; do
133
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
134
+ NODE_RANK=$i
135
+ break
136
+ fi
137
+ done
138
+ export RANK="${NODE_RANK}"
139
+ else
140
+ export RANK=0
141
+ fi
142
+
143
+ echo "============================================"
144
+ echo " HYV4-Flash LLM LLaMA Factory LoRA Training"
145
+ echo " Worker mode: Node ${RANK}/${NNODES}"
146
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
147
+ echo " GPUs per node: ${HOST_GPU_NUM}"
148
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
149
+ echo " Config: ${YAML_FILE}"
150
+ echo "============================================"
151
+
152
+ # -------------------- Launch --------------------
153
+ # Stagger node launches to avoid cephfs I/O congestion from 128 processes
154
+ # importing transformers simultaneously. Each node waits RANK * 5 seconds.
155
+ if [ "${RANK}" -gt 0 ]; then
156
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
157
+ sleep $((RANK * 5))
158
+ fi
159
+
160
+ torchrun \
161
+ --nnodes "${NNODES}" \
162
+ --node_rank "${RANK}" \
163
+ --nproc_per_node "${HOST_GPU_NUM}" \
164
+ --master_addr "${MASTER_ADDR}" \
165
+ --master_port "${MASTER_PORT}" \
166
+ "${ENTRY_SCRIPT}" "${YAML_FILE}"
finetune/ms_swift_support/fsdp_hy_v4_flash.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_description": "FSDP configuration for HYV4 Flash",
3
+ "_note": "Uses FSDP with full_shard auto_wrap. HYV4FlashDecoderLayer for Flash model.",
4
+
5
+ "fsdp": "full_shard auto_wrap",
6
+ "fsdp_config": {
7
+ "version": 1,
8
+ "reshard_after_forward": "full_shard",
9
+ "auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
10
+ "transformer_layer_cls_to_wrap": ["HYV4FlashDecoderLayer"],
11
+ "cpu_ram_efficient_loading": true,
12
+ "state_dict_type": "FULL_STATE_DICT",
13
+ "activation_checkpointing": true,
14
+ "use_orig_params": true,
15
+ "sync_module_states": true,
16
+ "forward_prefetch": false,
17
+ "backward_prefetch": "NO_PREFETCH",
18
+ "limit_all_gathers": true
19
+ }
20
+ }
finetune/ms_swift_support/hy_v4_flash_swift_patches.py ADDED
@@ -0,0 +1,595 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Patches for ms-swift training (HY4 Flash).
3
+
4
+ Registers the hy_v4_flash_vl model type and template for ms-swift, including:
5
+ 1. Model & Template registration: Register hy_v4_flash_vl model_type
6
+ with is_multimodal=True and VL architecture.
7
+ 2. Multimodal data processing (image/video encoding).
8
+ 3. Training optimizations (gradient checkpointing, grad norm, etc.).
9
+
10
+ Usage:
11
+ swift sft --custom_register_path hy_v4_flash_swift_patches.py --model /path/to/hy4_flash_ckpt ...
12
+ """
13
+
14
+ import os
15
+ import gc
16
+ import json as _json
17
+ import logging
18
+ from typing import Any, Dict
19
+
20
+ import torch
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+ # ============================================================================
25
+ # Patch 1: VL Model & Template Registration
26
+ #
27
+ # Register HY4-Flash-VL as a multimodal model in ms-swift. The VL model uses
28
+ # HYV4FlashForConditionalGeneration and needs image/video processing support.
29
+ # ============================================================================
30
+
31
+ from swift.model import register_model, ModelMeta, ModelGroup, Model
32
+ from swift.model.register import ModelLoader
33
+ from swift.model.model_arch import MultiModelKeys, register_model_arch
34
+ from swift.template import register_template, TemplateMeta
35
+
36
+ # Register HY4-Flash-VL model architecture mapping.
37
+ # HYV4FlashForConditionalGeneration has:
38
+ # self.model = HYV4FlashModel (contains self.vision_tower and self.language_model)
39
+ # self.lm_head
40
+ HYV4_FLASH_VL_ARCH_NAME = 'hy_v4_flash_vl'
41
+ register_model_arch(
42
+ MultiModelKeys(
43
+ HYV4_FLASH_VL_ARCH_NAME,
44
+ language_model=['model.language_model', 'lm_head'],
45
+ vision_tower='model.vision_tower',
46
+ aligner=[],
47
+ ),
48
+ exist_ok=True,
49
+ )
50
+
51
+
52
+ # Custom ModelLoader for HY4-Flash-VL
53
+ class HYV4FlashVLModelLoader(ModelLoader):
54
+ def get_model(self, model_dir, config, processor, model_kwargs):
55
+ from transformers.models.hy_v4_flash.modeling_hy_v4_flash import HYV4FlashForConditionalGeneration
56
+ self.auto_model_cls = self.auto_model_cls or HYV4FlashForConditionalGeneration
57
+ return super().get_model(model_dir, config, processor, model_kwargs)
58
+
59
+ # ---------------------------------------------------------------------------
60
+ # HYV4FlashVLTemplate: multimodal template for HY4-Flash-VL
61
+ # ---------------------------------------------------------------------------
62
+ from swift.template.base import Template
63
+ from swift.template.template_inputs import StdTemplateInputs
64
+ from swift.template.utils import Context, findall
65
+ from typing import List, Literal, Optional
66
+ from PIL import Image as PILImage
67
+
68
+ class HYV4FlashVLTemplate(Template):
69
+ """Multimodal template for HY4-Flash-VL models in ms-swift."""
70
+
71
+ # HY4-Flash-VL image/video token IDs (from config.json)
72
+ image_token_id = 120120
73
+ video_token_id = 120124
74
+ use_model = True
75
+
76
+ def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
77
+ inputs: StdTemplateInputs) -> List[Context]:
78
+ assert media_type in {'image', 'video'}
79
+ if media_type == 'image':
80
+ return [[-100]]
81
+ else:
82
+ return [[-200]]
83
+
84
+ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
85
+ encoded = super()._encode(inputs)
86
+ processor = self.processor
87
+ input_ids = encoded['input_ids']
88
+ labels = encoded['labels']
89
+ loss_scale = encoded.get('loss_scale', None)
90
+
91
+ # Process images
92
+ if inputs.images:
93
+ images = []
94
+ for img in inputs.images:
95
+ if isinstance(img, str):
96
+ images.append(PILImage.open(img).convert('RGB'))
97
+ else:
98
+ images.append(img)
99
+
100
+ image_processor = processor.image_processor
101
+ image_inputs = image_processor(images=images, return_tensors='pt')
102
+ image_grid_thw = image_inputs['image_grid_thw']
103
+ merge_size = image_processor.merge_size
104
+
105
+ idx_list = findall(input_ids, -100)
106
+
107
+ def _get_new_tokens(i):
108
+ grid_h, grid_w = image_grid_thw[i][-2:]
109
+ patch_h = grid_h // merge_size
110
+ patch_w = grid_w // merge_size
111
+ num_tokens = int(patch_h * (patch_w + 1))
112
+ return [self.image_token_id] * num_tokens
113
+
114
+ input_ids, labels, loss_scale = self._extend_tokens(
115
+ input_ids, labels, loss_scale, idx_list, _get_new_tokens)
116
+ encoded.update(image_inputs)
117
+
118
+ # Process videos
119
+ if inputs.videos:
120
+ video_processor = getattr(processor, 'video_processor', processor.image_processor)
121
+ video_inputs = video_processor(videos=inputs.videos, return_tensors='pt')
122
+ video_grid_thw = video_inputs['video_grid_thw']
123
+ merge_size = processor.image_processor.merge_size
124
+
125
+ idx_list = findall(input_ids, -200)
126
+
127
+ def _get_new_video_tokens(i):
128
+ grid_t, grid_h, grid_w = video_grid_thw[i]
129
+ patch_h = grid_h // merge_size
130
+ patch_w = grid_w // merge_size
131
+ num_tokens = int(grid_t * patch_h * (patch_w + 1))
132
+ return [self.video_token_id] * num_tokens
133
+
134
+ input_ids, labels, loss_scale = self._extend_tokens(
135
+ input_ids, labels, loss_scale, idx_list, _get_new_video_tokens)
136
+ encoded.update(video_inputs)
137
+
138
+ encoded['input_ids'] = input_ids
139
+ encoded['labels'] = labels
140
+ if loss_scale is not None:
141
+ encoded['loss_scale'] = loss_scale
142
+ return encoded
143
+
144
+ def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
145
+ return inputs
146
+
147
+ def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
148
+ res = super()._data_collator(batch, padding_to=padding_to)
149
+
150
+ input_ids = res.get('input_ids')
151
+ if input_ids is not None:
152
+ image_grid_thw = res.get('image_grid_thw', None)
153
+ video_grid_thw = res.get('video_grid_thw', None)
154
+
155
+ if image_grid_thw is not None or video_grid_thw is not None:
156
+ mm_token_type_ids = torch.zeros_like(input_ids)
157
+ image_token_id = getattr(self, 'image_token_id', 120120)
158
+ video_token_id = getattr(self, 'video_token_id', 120124)
159
+ mm_token_type_ids[input_ids == image_token_id] = 1
160
+ mm_token_type_ids[input_ids == video_token_id] = 2
161
+
162
+ try:
163
+ base_model = self.get_base_model(self._get_model())
164
+ if hasattr(base_model, 'get_rope_index'):
165
+ position_ids, _ = base_model.get_rope_index(
166
+ input_ids,
167
+ mm_token_type_ids=mm_token_type_ids,
168
+ image_grid_thw=image_grid_thw,
169
+ video_grid_thw=video_grid_thw,
170
+ attention_mask=res.get('attention_mask'),
171
+ )
172
+ res['position_ids'] = position_ids
173
+ except Exception as e:
174
+ logger.warning("[HYV4FlashVLTemplate] Failed to compute MRoPE position_ids: %s", e)
175
+
176
+ res['mm_token_type_ids'] = mm_token_type_ids
177
+
178
+ return res
179
+
180
+
181
+ # Register hy_v4_flash_vl template with HYV4FlashVLTemplate for multimodal support
182
+ # Token format: <|hy_start:opensource|>{role}<|hy_middle:opensource|>{content}<|hy_end:opensource|>
183
+ # default_system injects the reasoning_mode system prompt (same as LLaMA Factory).
184
+ # non_thinking_prefix inserts an empty <think></think> block before assistant responses
185
+ # in no_think mode, matching the format the model was trained with.
186
+ register_template(
187
+ TemplateMeta(
188
+ template_type='hy_v4_flash_vl',
189
+ prefix=[],
190
+ system_prefix=['<|hy_start:opensource|>system<|hy_middle:opensource|>{{SYSTEM}}<|hy_end:opensource|>'],
191
+ prompt=['<|hy_start:opensource|>user<|hy_middle:opensource|>{{QUERY}}<|hy_end:opensource|><|hy_start:opensource|>assistant<|hy_middle:opensource|>'],
192
+ chat_sep=['<|hy_end:opensource|>'],
193
+ suffix=['<|hy_end:opensource|>'],
194
+ default_system='<|reasoning_mode:opensource|>reasoning_effort:no_think',
195
+ non_thinking_prefix='<think:opensource></think:opensource>',
196
+ template_cls=HYV4FlashVLTemplate,
197
+ ),
198
+ exist_ok=True,
199
+ )
200
+
201
+ # Also register the LLM-only template
202
+ register_template(
203
+ TemplateMeta(
204
+ template_type='hy_v4_flash',
205
+ prefix=[],
206
+ system_prefix=['<|hy_start:opensource|>system<|hy_middle:opensource|>{{SYSTEM}}<|hy_end:opensource|>'],
207
+ prompt=['<|hy_start:opensource|>user<|hy_middle:opensource|>{{QUERY}}<|hy_end:opensource|><|hy_start:opensource|>assistant<|hy_middle:opensource|>'],
208
+ chat_sep=['<|hy_end:opensource|>'],
209
+ suffix=['<|hy_end:opensource|>'],
210
+ default_system='<|reasoning_mode:opensource|>reasoning_effort:no_think',
211
+ non_thinking_prefix='<think:opensource></think:opensource>',
212
+ ),
213
+ exist_ok=True,
214
+ )
215
+
216
+ # Register hy_v4_flash_vl model (multimodal)
217
+ register_model(
218
+ ModelMeta(
219
+ model_type='hy_v4_flash_vl',
220
+ model_groups=[
221
+ ModelGroup([
222
+ Model('Tencent-Hunyuan/Hy4-Flash-VL',
223
+ 'Tencent-Hunyuan/Hy4-Flash-VL'),
224
+ ]),
225
+ ],
226
+ loader=HYV4FlashVLModelLoader,
227
+ template='hy_v4_flash_vl',
228
+ model_arch=HYV4_FLASH_VL_ARCH_NAME,
229
+ architectures=['HYV4FlashForConditionalGeneration'],
230
+ is_multimodal=True,
231
+ ),
232
+ exist_ok=True,
233
+ )
234
+
235
+ # Also register the LLM-only model
236
+ register_model(
237
+ ModelMeta(
238
+ model_type='hy_v4_flash',
239
+ model_groups=[
240
+ ModelGroup([
241
+ Model('Tencent-Hunyuan/Hy4-Flash',
242
+ 'Tencent-Hunyuan/Hy4-Flash'),
243
+ ]),
244
+ ],
245
+ template='hy_v4_flash',
246
+ architectures=['HYV4FlashForCausalLM'],
247
+ is_multimodal=False,
248
+ ),
249
+ exist_ok=True,
250
+ )
251
+
252
+ logger.info(
253
+ "[HYV4-Flash Patch 1] Model types 'hy_v4_flash_vl' (multimodal) and 'hy_v4_flash' (text-only) "
254
+ "registered in ms-swift."
255
+ )
256
+
257
+ # ============================================================================
258
+ # Patch 2: Skip grad norm computation for DeepSpeed ZeRO-3
259
+ # ============================================================================
260
+
261
+ def _apply_skip_grad_norm_patch():
262
+ try:
263
+ from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3
264
+
265
+ def _skip_get_norm_groups(self):
266
+ return [torch.tensor(0.0)]
267
+
268
+ DeepSpeedZeroOptimizer_Stage3._get_norm_groups = _skip_get_norm_groups
269
+ logger.info("[HYV4-Flash Patch 2] Patched grad norm skip.")
270
+ except ImportError:
271
+ logger.info("[HYV4-Flash Patch 2] DeepSpeed not available, skipping.")
272
+
273
+ # ============================================================================
274
+ # Patch 3: Memory-efficient model loading for ZeRO-3 and FSDP1
275
+ # ============================================================================
276
+
277
+ def _apply_shard_loading_patch():
278
+ import sys
279
+ from transformers import AutoConfig, PreTrainedModel
280
+
281
+ _real_orig_from_pretrained = PreTrainedModel.from_pretrained.__func__
282
+
283
+ def _disable_engram(config):
284
+ """Disable engram layers to avoid lookup_table size mismatch.
285
+
286
+ The checkpoint's engram lookup_table buffers may have a different size
287
+ (e.g. 120818) than config.text_config.vocab_size (120832). Setting
288
+ engram_layer_ids=None prevents the model from creating engram layers
289
+ entirely, avoiding the RuntimeError during weight loading.
290
+ """
291
+ text_config = getattr(config, 'text_config', config)
292
+ old_ids = getattr(text_config, 'engram_layer_ids', None)
293
+ if old_ids is not None:
294
+ text_config.engram_layer_ids = None
295
+ print(f"[HYV4-Flash Patch 3] _disable_engram: set engram_layer_ids=None "
296
+ f"(was {old_ids})", flush=True)
297
+
298
+ def _disable_router_logits_if_needed(model):
299
+ if hasattr(model, 'config') and getattr(model.config, 'output_router_logits', False):
300
+ model.config.output_router_logits = False
301
+ logger.info("[HYV4-Flash Patch 3] Disabled output_router_logits.")
302
+ return model
303
+
304
+ def _is_fsdp_requested():
305
+ accel_fsdp = str(os.environ.get("ACCELERATE_USE_FSDP", "")).lower()
306
+ if accel_fsdp in {"1", "true", "yes"}:
307
+ return True
308
+ return "--fsdp" in sys.argv
309
+
310
+ def _build_meta_model_for_fsdp(cls, model_path, kwargs):
311
+ config = kwargs.get("config")
312
+ if config is None:
313
+ config = AutoConfig.from_pretrained(
314
+ model_path,
315
+ trust_remote_code=kwargs.get("trust_remote_code", True),
316
+ )
317
+
318
+ init_kwargs = {}
319
+ torch_dtype = kwargs.get("torch_dtype", None)
320
+ if torch_dtype is not None:
321
+ init_kwargs["torch_dtype"] = torch_dtype
322
+ if "attn_implementation" in kwargs:
323
+ init_kwargs["attn_implementation"] = kwargs["attn_implementation"]
324
+ if "experts_implementation" in kwargs:
325
+ init_kwargs["experts_implementation"] = kwargs["experts_implementation"]
326
+
327
+ with torch.device("meta"):
328
+ model = cls._from_config(config, **init_kwargs)
329
+ return _disable_router_logits_if_needed(model)
330
+
331
+ def _fsdp_safe_load(cls, pretrained_model_name_or_path, *args, **kwargs):
332
+ kwargs = dict(kwargs)
333
+ kwargs.setdefault("low_cpu_mem_usage", True)
334
+
335
+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))
336
+ if local_rank != 0:
337
+ logger.info("[HYV4-Flash Patch 3] FSDP: local_rank=%d, meta device.", local_rank)
338
+ return _build_meta_model_for_fsdp(cls, pretrained_model_name_or_path, kwargs)
339
+
340
+ logger.info("[HYV4-Flash Patch 3] FSDP: local_rank=0, loading to CPU.")
341
+ kwargs["device_map"] = {"": "cpu"}
342
+ model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
343
+ return _disable_router_logits_if_needed(model)
344
+
345
+ def _ensure_zero3_config_and_load(cls, pretrained_model_name_or_path, *args, **kwargs):
346
+ model_path = pretrained_model_name_or_path
347
+
348
+ # Disable engram layers before model instantiation to avoid
349
+ # lookup_table buffer size mismatch (ckpt=120818 vs config=120832).
350
+ config = kwargs.get('config')
351
+ if config is not None:
352
+ _disable_engram(config)
353
+
354
+ if not (isinstance(model_path, str) and os.path.isdir(model_path)):
355
+ return _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
356
+
357
+ index_file = os.path.join(model_path, "model.safetensors.index.json")
358
+ single_file = os.path.join(model_path, "model.safetensors")
359
+ if not (os.path.isfile(index_file) or os.path.isfile(single_file)):
360
+ return _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
361
+
362
+ if _is_fsdp_requested():
363
+ return _fsdp_safe_load(cls, pretrained_model_name_or_path, *args, **kwargs)
364
+
365
+ from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled
366
+
367
+ if is_deepspeed_zero3_enabled():
368
+ model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
369
+ return _disable_router_logits_if_needed(model)
370
+
371
+ ds_config_path = os.environ.get("DEEPSPEED_CONFIG_FILE", None)
372
+ if ds_config_path is None:
373
+ ds_config_path = os.environ.get("DEEPSPEED_CONFIG", None)
374
+
375
+ if ds_config_path is None:
376
+ for i, arg in enumerate(sys.argv):
377
+ if arg == '--deepspeed' and i + 1 < len(sys.argv):
378
+ ds_config_path = sys.argv[i + 1]
379
+ break
380
+
381
+ if ds_config_path is None or not os.path.isfile(ds_config_path):
382
+ model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
383
+ return _disable_router_logits_if_needed(model)
384
+
385
+ with open(ds_config_path, "r") as f:
386
+ ds_config = _json.load(f)
387
+
388
+ zero_stage = ds_config.get("zero_optimization", {}).get("stage", 0)
389
+ if zero_stage != 3:
390
+ model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
391
+ return _disable_router_logits_if_needed(model)
392
+
393
+ from transformers.integrations.deepspeed import HfDeepSpeedConfig
394
+ _ds_config_obj = HfDeepSpeedConfig(ds_config_path)
395
+
396
+ model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
397
+ return _disable_router_logits_if_needed(model)
398
+
399
+ @classmethod
400
+ def _patched_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
401
+ return _ensure_zero3_config_and_load(cls, pretrained_model_name_or_path, *args, **kwargs)
402
+
403
+ PreTrainedModel.from_pretrained = _patched_from_pretrained
404
+ logger.info("[HYV4-Flash Patch 3] Loading patch applied.")
405
+
406
+ # ============================================================================
407
+ # Optional Patch: LoRA z3_leaf marking
408
+ # ============================================================================
409
+
410
+ def apply_lora_z3_leaf_patch(model):
411
+ try:
412
+ from deepspeed.utils import set_z3_leaf_module
413
+ from peft.tuners.lora import Linear as LoraLinear
414
+ except ImportError:
415
+ return
416
+
417
+ z3_leaf_count = 0
418
+ for module in model.modules():
419
+ if isinstance(module, LoraLinear):
420
+ set_z3_leaf_module(module, True)
421
+ z3_leaf_count += 1
422
+ logger.info("[HYV4-Flash] Marked %d LoraLinear modules with _z3_leaf=True.", z3_leaf_count)
423
+
424
+ # ============================================================================
425
+ # Patch 4: Fix logging_dir compatibility
426
+ # ============================================================================
427
+
428
+ def _apply_logging_dir_patch():
429
+ try:
430
+ from swift.arguments.sft_args import SftArguments
431
+
432
+ _orig_add_version = SftArguments._add_version
433
+
434
+ def _patched_add_version(self):
435
+ if not hasattr(self, 'logging_dir'):
436
+ self.logging_dir = None
437
+ if not hasattr(self, 'run_name'):
438
+ self.run_name = None
439
+ _orig_add_version(self)
440
+
441
+ SftArguments._add_version = _patched_add_version
442
+ logger.info("[HYV4-Flash Patch 4] logging_dir compatibility patch applied.")
443
+ except (ImportError, AttributeError) as e:
444
+ logger.info("[HYV4-Flash Patch 4] Could not apply: %s", e)
445
+
446
+ # ============================================================================
447
+ # Patch 5: FSDP1 dtype + mixed precision guard
448
+ # ============================================================================
449
+
450
+ def _apply_fsdp_dtype_patch():
451
+ try:
452
+ from transformers import Trainer
453
+
454
+ _orig_prepare_for_training = Trainer._prepare_for_training
455
+
456
+ def _patched_prepare_for_training(self, *args, **kwargs):
457
+ if getattr(self.args, 'fsdp', False):
458
+ dtype_counts = {}
459
+ for p in self.model.parameters():
460
+ dt = str(p.dtype)
461
+ dtype_counts[dt] = dtype_counts.get(dt, 0) + 1
462
+
463
+ if len(dtype_counts) > 1:
464
+ logger.info("[HYV4-Flash Patch 5] Mixed dtypes: %s. Casting to bf16.", dtype_counts)
465
+ for p in self.model.parameters():
466
+ if p.dtype != torch.bfloat16 and p.dtype.is_floating_point:
467
+ p.data = p.data.to(torch.bfloat16)
468
+
469
+ try:
470
+ if hasattr(self, 'accelerator') and hasattr(self.accelerator, 'state'):
471
+ self.accelerator.state._mixed_precision = 'no'
472
+ fsdp_plugin = getattr(self.accelerator.state, 'fsdp_plugin', None)
473
+ if fsdp_plugin is not None:
474
+ if hasattr(fsdp_plugin, 'mixed_precision_policy'):
475
+ fsdp_plugin.mixed_precision_policy = None
476
+ if hasattr(fsdp_plugin, 'kwargs') and isinstance(fsdp_plugin.kwargs, dict):
477
+ fsdp_plugin.kwargs.pop('mixed_precision', None)
478
+ os.environ['ACCELERATE_MIXED_PRECISION'] = 'no'
479
+ except Exception as e:
480
+ logger.warning("[HYV4-Flash Patch 5] Failed to disable mixed precision: %s", e)
481
+
482
+ # ---- Fix PEFT + FSDP _no_split_modules mismatch ----
483
+ # PEFT's fsdp_auto_wrap_policy iterates _no_split_modules and calls
484
+ # get_module_class_from_name() for each class name. If a class is
485
+ # listed but not instantiated in the model (e.g. HYV4FlashEngramLayer
486
+ # when engram is disabled), the lookup returns None and PEFT raises:
487
+ # "Could not find the transformer layer class to wrap in the model."
488
+ # Fix: collect all actual module class names and prune _no_split_modules.
489
+ try:
490
+ actual_cls_names = {m.__class__.__name__ for m in self.model.modules()}
491
+ for sub in self.model.modules():
492
+ if hasattr(sub, '_no_split_modules') and sub._no_split_modules:
493
+ original = list(sub._no_split_modules)
494
+ pruned = [c for c in original if c in actual_cls_names]
495
+ if len(pruned) < len(original):
496
+ removed = set(original) - set(pruned)
497
+ sub._no_split_modules = pruned
498
+ logger.info(
499
+ "[HYV4-Flash Patch 5] Pruned _no_split_modules: removed %s "
500
+ "(not found in model). Remaining: %s",
501
+ removed, pruned
502
+ )
503
+ except Exception as e:
504
+ logger.warning("[HYV4-Flash Patch 5] Failed to prune _no_split_modules: %s", e)
505
+
506
+ return _orig_prepare_for_training(self, *args, **kwargs)
507
+
508
+ Trainer._prepare_for_training = _patched_prepare_for_training
509
+ logger.info("[HYV4-Flash Patch 5] FSDP1 dtype guard applied.")
510
+ except (ImportError, AttributeError) as e:
511
+ logger.info("[HYV4-Flash Patch 5] Could not apply: %s", e)
512
+
513
+ # ============================================================================
514
+ # Patch 6: Disable _compute_acc during training
515
+ # ============================================================================
516
+
517
+ def _apply_disable_compute_acc_patch():
518
+ try:
519
+ from swift.trainers.mixin import SwiftMixin
520
+
521
+ def _noop_compute_acc(self, outputs, labels, cu_seqlens=None):
522
+ return
523
+
524
+ SwiftMixin._compute_acc = _noop_compute_acc
525
+ logger.info("[HYV4-Flash Patch 6] Disabled _compute_acc.")
526
+ except (ImportError, AttributeError) as e:
527
+ logger.warning("[HYV4-Flash Patch 6] Could not apply: %s", e)
528
+
529
+ # ============================================================================
530
+ # Patch 7: Freeze ViT (handled natively by ms-swift)
531
+ # ============================================================================
532
+
533
+ def _apply_freeze_vit_patch():
534
+ logger.info("[HYV4-Flash Patch 7] ViT freeze handled by ms-swift native freeze_vit + model_arch.")
535
+
536
+ # ============================================================================
537
+ # Patch 8: tokenizer → processing_class compatibility
538
+ # ============================================================================
539
+
540
+ def _apply_tokenizer_to_processing_class_patch():
541
+ try:
542
+ from transformers import Seq2SeqTrainer as HfSeq2SeqTrainer
543
+
544
+ _original_hf_seq2seq_init = HfSeq2SeqTrainer.__init__
545
+
546
+ def _patched_hf_seq2seq_init(self, *args, **kwargs):
547
+ if 'tokenizer' in kwargs and 'processing_class' not in kwargs:
548
+ kwargs['processing_class'] = kwargs.pop('tokenizer')
549
+ return _original_hf_seq2seq_init(self, *args, **kwargs)
550
+
551
+ HfSeq2SeqTrainer.__init__ = _patched_hf_seq2seq_init
552
+ logger.info("[HYV4-Flash Patch 8] tokenizer→processing_class patch applied.")
553
+ except (ImportError, AttributeError) as e:
554
+ logger.info("[HYV4-Flash Patch 8] Could not apply: %s", e)
555
+
556
+ # ============================================================================
557
+ # Patch 9: Make CUDA pin_memory fault-tolerant
558
+ # ============================================================================
559
+
560
+ def _apply_safe_pin_memory_patch():
561
+ try:
562
+ from deepspeed.accelerator.cuda_accelerator import CUDA_Accelerator
563
+
564
+ _orig_pin_memory = CUDA_Accelerator.pin_memory
565
+
566
+ def _safe_pin_memory(self, tensor):
567
+ try:
568
+ return _orig_pin_memory(self, tensor)
569
+ except Exception as e:
570
+ import torch
571
+ rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0
572
+ print(f"[HYV4-Flash Patch 9] WARNING: pin_memory() failed on rank {rank}: {e}. "
573
+ f"Falling back to unpinned CPU tensor.", flush=True)
574
+ return tensor
575
+
576
+ CUDA_Accelerator.pin_memory = _safe_pin_memory
577
+ logger.info("[HYV4-Flash Patch 9] pin_memory fault-tolerant fallback applied.")
578
+ except ImportError:
579
+ pass
580
+
581
+
582
+ # ============================================================================
583
+ # Auto-apply patches on import
584
+ # ============================================================================
585
+
586
+ _apply_safe_pin_memory_patch()
587
+ _apply_skip_grad_norm_patch()
588
+ _apply_shard_loading_patch()
589
+ _apply_logging_dir_patch()
590
+ _apply_fsdp_dtype_patch()
591
+ _apply_disable_compute_acc_patch()
592
+ _apply_freeze_vit_patch()
593
+ _apply_tokenizer_to_processing_class_patch()
594
+
595
+ logger.info("[HYV4-Flash] All ms-swift patches loaded successfully.")
finetune/ms_swift_support/sft_train.sh ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # ============================================================================
3
+ # ms-swift LLM full fine-tuning + DeepSpeed ZeRO-3 training launch script
4
+ #
5
+ # This script uses pdsh to automatically distribute to all nodes.
6
+ # You only need to run it once on any node.
7
+ #
8
+ # Usage:
9
+ # nohup bash sft_train.sh > swift_llm_full.log 2>&1 &
10
+ # ============================================================================
11
+
12
+ set -euo pipefail
13
+
14
+ # -------------------- Network Configuration --------------------
15
+ NET_TYPE="high"
16
+ export NCCL_DEBUG=WARN
17
+ export NCCL_P2P_LEVEL=NVL
18
+ export NCCL_IB_TIMEOUT=24
19
+ export NCCL_NVLS_ENABLE=0
20
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
21
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
22
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
23
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
24
+ if [[ "${NET_TYPE}" = "low" ]]; then
25
+ export NCCL_SOCKET_IFNAME=eth1
26
+ export NCCL_IB_GID_INDEX=3
27
+ export NCCL_IB_HCA=mlx5_2:1
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_LL_THRESHOLD=16384
32
+ export NCCL_IB_CUDA_SUPPORT=1
33
+ else
34
+ export NCCL_IB_GID_INDEX=3
35
+ export NCCL_IB_SL=3
36
+ export NCCL_CHECK_DISABLE=1
37
+ export NCCL_P2P_DISABLE=0
38
+ export NCCL_IB_DISABLE=0
39
+ export NCCL_LL_THRESHOLD=16384
40
+ export NCCL_IB_CUDA_SUPPORT=1
41
+ export NCCL_SOCKET_IFNAME=bond1
42
+ export UCX_NET_DEVICES=bond1
43
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
44
+ export NCCL_COLLNET_ENABLE=0
45
+ export SHARP_COLL_ENABLE_SAT=0
46
+ export NCCL_NET_GDR_LEVEL=2
47
+ export NCCL_IB_QPS_PER_CONNECTION=4
48
+ export NCCL_IB_TC=160
49
+ export NCCL_PXN_DISABLE=1
50
+ fi
51
+
52
+ export DISABLE_VERSION_CHECK=1
53
+
54
+ # -------------------- Node Configuration --------------------
55
+ export HOST_GPU_NUM=8
56
+ # IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
57
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
58
+
59
+ MASTER_PORT=${MASTER_PORT:-29500}
60
+
61
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
62
+ NODES=${#IP_ARRAY[@]}
63
+ MASTER_ADDR=${IP_ARRAY[0]}
64
+
65
+ # -------------------- Paths --------------------
66
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
67
+ PATCHES_FILE="${SCRIPT_DIR}/hy_v4_flash_swift_patches.py"
68
+
69
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
70
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
71
+ echo "============================================"
72
+ echo " HYV4-Flash ms-swift LLM SFT Training (Full + DeepSpeed ZeRO-3)"
73
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
74
+ echo " Nodes: ${IP_LIST}"
75
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
76
+ echo " GPUs per node: ${HOST_GPU_NUM}"
77
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
78
+ echo "============================================"
79
+
80
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
81
+
82
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
83
+ "export NCCL_DEBUG=WARN; \
84
+ export NCCL_P2P_LEVEL=NVL; \
85
+ export NCCL_IB_TIMEOUT=24; \
86
+ export NCCL_NVLS_ENABLE=0; \
87
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
88
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
89
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
90
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
91
+ export NCCL_IB_GID_INDEX=3; \
92
+ export NCCL_IB_SL=3; \
93
+ export NCCL_CHECK_DISABLE=1; \
94
+ export NCCL_P2P_DISABLE=0; \
95
+ export NCCL_IB_DISABLE=0; \
96
+ export NCCL_LL_THRESHOLD=16384; \
97
+ export NCCL_IB_CUDA_SUPPORT=1; \
98
+ export NCCL_SOCKET_IFNAME=bond1; \
99
+ export UCX_NET_DEVICES=bond1; \
100
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
101
+ export NCCL_COLLNET_ENABLE=0; \
102
+ export SHARP_COLL_ENABLE_SAT=0; \
103
+ export NCCL_NET_GDR_LEVEL=2; \
104
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
105
+ export NCCL_IB_TC=160; \
106
+ export NCCL_PXN_DISABLE=1; \
107
+ export DISABLE_VERSION_CHECK=1; \
108
+ export IP_LIST='${IP_LIST}'; \
109
+ export _WORKER_MODE=1; \
110
+ cd ${SCRIPT_DIR}; \
111
+ bash ${SCRIPT_DIR}/sft_train.sh"
112
+ exit $?
113
+ fi
114
+
115
+ # -------------------- Worker Mode: determine node rank and launch --------------------
116
+ export MASTER_ADDR="${MASTER_ADDR}"
117
+ export MASTER_PORT="${MASTER_PORT}"
118
+ export NNODES="${NODES}"
119
+
120
+ if [ ${NODES} -gt 1 ]; then
121
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
122
+ NODE_RANK=0
123
+ for i in "${!IP_ARRAY[@]}"; do
124
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
125
+ NODE_RANK=$i
126
+ break
127
+ fi
128
+ done
129
+ export RANK="${NODE_RANK}"
130
+ else
131
+ export RANK=0
132
+ fi
133
+
134
+ echo "============================================"
135
+ echo " HYV4-Flash ms-swift LLM SFT Training"
136
+ echo " Worker mode: Node ${RANK}/${NNODES}"
137
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
138
+ echo " GPUs per node: ${HOST_GPU_NUM}"
139
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
140
+ echo "============================================"
141
+
142
+ # -------------------- Launch --------------------
143
+ # Stagger node launches to avoid cephfs I/O congestion
144
+ if [ "${RANK}" -gt 0 ]; then
145
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
146
+ sleep $((RANK * 5))
147
+ fi
148
+
149
+ export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${SCRIPT_DIR}"
150
+
151
+ # DeepSpeed config path (absolute)
152
+ DS_CONFIG="${SCRIPT_DIR}/../deepspeed_support/ds_zero3_offload.json"
153
+
154
+ # NOTE: --config is NOT supported by the current transformers HfArgumentParser
155
+ # when using torchrun -m swift.cli.sft. All parameters passed as CLI args.
156
+ torchrun \
157
+ --nnodes "${NNODES}" \
158
+ --node_rank "${RANK}" \
159
+ --nproc_per_node "${HOST_GPU_NUM}" \
160
+ --master_addr "${MASTER_ADDR}" \
161
+ --master_port "${MASTER_PORT}" \
162
+ -m swift.cli.sft \
163
+ --custom_register_path "${PATCHES_FILE}" \
164
+ --model /path/to/hy4_ckpt \
165
+ --model_type hy_v4_flash_vl \
166
+ --template hy_v4_flash \
167
+ --torch_dtype bfloat16 \
168
+ --attn_impl eager \
169
+ --tuner_type full \
170
+ --dataset "${SCRIPT_DIR}/../data/example_data.jsonl" \
171
+ --max_length 1024 \
172
+ --lazy_tokenize true \
173
+ --dataset_num_proc 4 \
174
+ --output_dir saves/hy_v4_flash/full/sft_ds \
175
+ --save_steps 20 \
176
+ --save_strategy steps \
177
+ --save_total_limit 3 \
178
+ --logging_steps 1 \
179
+ --report_to none \
180
+ --per_device_train_batch_size 1 \
181
+ --gradient_accumulation_steps 1 \
182
+ --learning_rate 1e-5 \
183
+ --max_steps 20 \
184
+ --num_train_epochs 3.0 \
185
+ --warmup_steps 0 \
186
+ --lr_scheduler_type cosine \
187
+ --bf16 true \
188
+ --deepspeed "${DS_CONFIG}" \
189
+ --optim adamw_torch \
190
+ --use_logits_to_keep false \
191
+ --gradient_checkpointing true \
192
+ --max_grad_norm 0.0 \
193
+ --ddp_timeout 180000000 \
194
+ --seed 42 \
195
+ --ignore_data_skip true
finetune/ms_swift_support/sft_train_lora.sh ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ set -euo pipefail
4
+
5
+ # -------------------- Network Configuration --------------------
6
+ NET_TYPE="high"
7
+ export NCCL_DEBUG=WARN
8
+ export NCCL_P2P_LEVEL=NVL
9
+ export NCCL_IB_TIMEOUT=24
10
+ export NCCL_NVLS_ENABLE=0
11
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
12
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
13
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
14
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
15
+ if [[ "${NET_TYPE}" = "low" ]]; then
16
+ export NCCL_SOCKET_IFNAME=eth1
17
+ export NCCL_IB_GID_INDEX=3
18
+ export NCCL_IB_HCA=mlx5_2:1
19
+ export NCCL_IB_SL=3
20
+ export NCCL_CHECK_DISABLE=1
21
+ export NCCL_P2P_DISABLE=0
22
+ export NCCL_LL_THRESHOLD=16384
23
+ export NCCL_IB_CUDA_SUPPORT=1
24
+ else
25
+ export NCCL_IB_GID_INDEX=3
26
+ export NCCL_IB_SL=3
27
+ export NCCL_CHECK_DISABLE=1
28
+ export NCCL_P2P_DISABLE=0
29
+ export NCCL_IB_DISABLE=0
30
+ export NCCL_LL_THRESHOLD=16384
31
+ export NCCL_IB_CUDA_SUPPORT=1
32
+ export NCCL_SOCKET_IFNAME=bond1
33
+ export UCX_NET_DEVICES=bond1
34
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
35
+ export NCCL_COLLNET_ENABLE=0
36
+ export SHARP_COLL_ENABLE_SAT=0
37
+ export NCCL_NET_GDR_LEVEL=2
38
+ export NCCL_IB_QPS_PER_CONNECTION=4
39
+ export NCCL_IB_TC=160
40
+ export NCCL_PXN_DISABLE=1
41
+ fi
42
+
43
+ # -------------------- Node Configuration --------------------
44
+ export HOST_GPU_NUM=8
45
+ # IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
46
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
47
+
48
+ MASTER_PORT=${MASTER_PORT:-29500}
49
+
50
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
51
+ NODES=${#IP_ARRAY[@]}
52
+ MASTER_ADDR=${IP_ARRAY[0]}
53
+
54
+ # -------------------- Paths --------------------
55
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
56
+ PATCHES_FILE="${SCRIPT_DIR}/hy_v4_flash_swift_patches.py"
57
+ FSDP_CONFIG="${SCRIPT_DIR}/fsdp_hy_v4_flash.json"
58
+
59
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
60
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
61
+ echo "============================================"
62
+ echo " HYV4-Flash ms-swift SFT Training (LoRA + FSDP)"
63
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
64
+ echo " Nodes: ${IP_LIST}"
65
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
66
+ echo " GPUs per node: ${HOST_GPU_NUM}"
67
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
68
+ echo " FSDP Config: ${FSDP_CONFIG}"
69
+ echo "============================================"
70
+
71
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
72
+
73
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
74
+ "export NCCL_DEBUG=WARN; \
75
+ export NCCL_P2P_LEVEL=NVL; \
76
+ export NCCL_IB_TIMEOUT=24; \
77
+ export NCCL_NVLS_ENABLE=0; \
78
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
79
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
80
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
81
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
82
+ export NCCL_IB_GID_INDEX=3; \
83
+ export NCCL_IB_SL=3; \
84
+ export NCCL_CHECK_DISABLE=1; \
85
+ export NCCL_P2P_DISABLE=0; \
86
+ export NCCL_IB_DISABLE=0; \
87
+ export NCCL_LL_THRESHOLD=16384; \
88
+ export NCCL_IB_CUDA_SUPPORT=1; \
89
+ export NCCL_SOCKET_IFNAME=bond1; \
90
+ export UCX_NET_DEVICES=bond1; \
91
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
92
+ export NCCL_COLLNET_ENABLE=0; \
93
+ export SHARP_COLL_ENABLE_SAT=0; \
94
+ export NCCL_NET_GDR_LEVEL=2; \
95
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
96
+ export NCCL_IB_TC=160; \
97
+ export NCCL_PXN_DISABLE=1; \
98
+ export ACCELERATE_MIXED_PRECISION=no; \
99
+ export IP_LIST='${IP_LIST}'; \
100
+ export _WORKER_MODE=1; \
101
+ cd ${SCRIPT_DIR}; \
102
+ bash ${SCRIPT_DIR}/sft_train_lora.sh"
103
+ exit $?
104
+ fi
105
+
106
+ # -------------------- Worker Mode: determine node rank and launch --------------------
107
+ export MASTER_ADDR="${MASTER_ADDR}"
108
+ export MASTER_PORT="${MASTER_PORT}"
109
+ export NNODES="${NODES}"
110
+
111
+ # Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
112
+ export ACCELERATE_MIXED_PRECISION=no
113
+
114
+ if [ ${NODES} -gt 1 ]; then
115
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
116
+ NODE_RANK=0
117
+ for i in "${!IP_ARRAY[@]}"; do
118
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
119
+ NODE_RANK=$i
120
+ break
121
+ fi
122
+ done
123
+ export RANK="${NODE_RANK}"
124
+ else
125
+ export RANK=0
126
+ fi
127
+
128
+ echo "============================================"
129
+ echo " HYV4-Flash ms-swift SFT Training (LoRA + FSDP)"
130
+ echo " Worker mode: Node ${RANK}/${NNODES}"
131
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
132
+ echo " GPUs per node: ${HOST_GPU_NUM}"
133
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
134
+ echo " FSDP Config: ${FSDP_CONFIG}"
135
+ echo "============================================"
136
+
137
+ # -------------------- Launch --------------------
138
+ # Stagger node launches to avoid cephfs I/O congestion
139
+ if [ "${RANK}" -gt 0 ]; then
140
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
141
+ sleep $((RANK * 5))
142
+ fi
143
+
144
+ export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${SCRIPT_DIR}"
145
+ torchrun \
146
+ --nnodes "${NNODES}" \
147
+ --node_rank "${RANK}" \
148
+ --nproc_per_node "${HOST_GPU_NUM}" \
149
+ --master_addr "${MASTER_ADDR}" \
150
+ --master_port "${MASTER_PORT}" \
151
+ -m swift.cli.sft \
152
+ --custom_register_path "${PATCHES_FILE}" \
153
+ --model /path/to/hy4_ckpt \
154
+ --model_type hy_v4_flash_vl \
155
+ --template hy_v4_flash_vl \
156
+ --torch_dtype bfloat16 \
157
+ --attn_impl eager \
158
+ --tuner_type lora \
159
+ --lora_rank 64 \
160
+ --lora_alpha 128 \
161
+ --lora_dropout 0.05 \
162
+ --target_modules q_a_proj q_b_proj kv_a_proj_with_mqa kv_b_proj o_proj \
163
+ --dataset "${SCRIPT_DIR}/../data/example_data.jsonl" \
164
+ --max_length 256 \
165
+ --lazy_tokenize true \
166
+ --dataset_num_proc 4 \
167
+ --output_dir saves/hy_v4_flash/lora/sft_fsdp \
168
+ --save_steps 20 \
169
+ --save_strategy steps \
170
+ --save_total_limit 3 \
171
+ --logging_steps 1 \
172
+ --report_to none \
173
+ --per_device_train_batch_size 1 \
174
+ --gradient_accumulation_steps 1 \
175
+ --learning_rate 2e-4 \
176
+ --max_steps 20 \
177
+ --num_train_epochs 3.0 \
178
+ --warmup_steps 0 \
179
+ --lr_scheduler_type cosine \
180
+ --bf16 true \
181
+ --fsdp "${FSDP_CONFIG}" \
182
+ --optim adamw_torch \
183
+ --use_logits_to_keep false \
184
+ --gradient_checkpointing true \
185
+ --max_grad_norm 1.0 \
186
+ --ddp_timeout 180000000 \
187
+ --seed 42 \
188
+ --ignore_data_skip true
finetune/ms_swift_support/sft_train_vl.sh ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # ============================================================================
3
+ # ms-swift VL full fine-tuning + DeepSpeed ZeRO-3 training launch script
4
+ #
5
+ # This script uses pdsh to automatically distribute to all nodes.
6
+ # You only need to run it once on any node.
7
+ #
8
+ # Usage:
9
+ # nohup bash sft_train_vl.sh > swift_vl_full.log 2>&1 &
10
+ # ============================================================================
11
+
12
+ set -euo pipefail
13
+
14
+ # -------------------- Network Configuration --------------------
15
+ NET_TYPE="high"
16
+ export NCCL_DEBUG=WARN
17
+ export NCCL_P2P_LEVEL=NVL
18
+ export NCCL_IB_TIMEOUT=24
19
+ export NCCL_NVLS_ENABLE=0
20
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
21
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
22
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
23
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
24
+ if [[ "${NET_TYPE}" = "low" ]]; then
25
+ export NCCL_SOCKET_IFNAME=eth1
26
+ export NCCL_IB_GID_INDEX=3
27
+ export NCCL_IB_HCA=mlx5_2:1
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_LL_THRESHOLD=16384
32
+ export NCCL_IB_CUDA_SUPPORT=1
33
+ else
34
+ export NCCL_IB_GID_INDEX=3
35
+ export NCCL_IB_SL=3
36
+ export NCCL_CHECK_DISABLE=1
37
+ export NCCL_P2P_DISABLE=0
38
+ export NCCL_IB_DISABLE=0
39
+ export NCCL_LL_THRESHOLD=16384
40
+ export NCCL_IB_CUDA_SUPPORT=1
41
+ export NCCL_SOCKET_IFNAME=bond1
42
+ export UCX_NET_DEVICES=bond1
43
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
44
+ export NCCL_COLLNET_ENABLE=0
45
+ export SHARP_COLL_ENABLE_SAT=0
46
+ export NCCL_NET_GDR_LEVEL=2
47
+ export NCCL_IB_QPS_PER_CONNECTION=4
48
+ export NCCL_IB_TC=160
49
+ export NCCL_PXN_DISABLE=1
50
+ fi
51
+
52
+ export DISABLE_VERSION_CHECK=1
53
+
54
+ # -------------------- Node Configuration --------------------
55
+ export HOST_GPU_NUM=8
56
+ # IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
57
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
58
+
59
+ MASTER_PORT=${MASTER_PORT:-29500}
60
+
61
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
62
+ NODES=${#IP_ARRAY[@]}
63
+ MASTER_ADDR=${IP_ARRAY[0]}
64
+
65
+ # -------------------- Paths --------------------
66
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
67
+ PATCHES_FILE="${SCRIPT_DIR}/hy_v4_flash_swift_patches.py"
68
+
69
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
70
+ # If running multi-node and this is the launcher (not a worker dispatched by pdsh),
71
+ # use pdsh to distribute the script to all nodes automatically.
72
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
73
+ echo "============================================"
74
+ echo " HYV4-Flash-VL ms-swift SFT Training (Full + DeepSpeed ZeRO-3)"
75
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
76
+ echo " Nodes: ${IP_LIST}"
77
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
78
+ echo " GPUs per node: ${HOST_GPU_NUM}"
79
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
80
+ echo "============================================"
81
+
82
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
83
+
84
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
85
+ "export NCCL_DEBUG=WARN; \
86
+ export NCCL_P2P_LEVEL=NVL; \
87
+ export NCCL_IB_TIMEOUT=24; \
88
+ export NCCL_NVLS_ENABLE=0; \
89
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
90
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
91
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
92
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
93
+ export NCCL_IB_GID_INDEX=3; \
94
+ export NCCL_IB_SL=3; \
95
+ export NCCL_CHECK_DISABLE=1; \
96
+ export NCCL_P2P_DISABLE=0; \
97
+ export NCCL_IB_DISABLE=0; \
98
+ export NCCL_LL_THRESHOLD=16384; \
99
+ export NCCL_IB_CUDA_SUPPORT=1; \
100
+ export NCCL_SOCKET_IFNAME=bond1; \
101
+ export UCX_NET_DEVICES=bond1; \
102
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
103
+ export NCCL_COLLNET_ENABLE=0; \
104
+ export SHARP_COLL_ENABLE_SAT=0; \
105
+ export NCCL_NET_GDR_LEVEL=2; \
106
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
107
+ export NCCL_IB_TC=160; \
108
+ export NCCL_PXN_DISABLE=1; \
109
+ export DISABLE_VERSION_CHECK=1; \
110
+ export IP_LIST='${IP_LIST}'; \
111
+ export _WORKER_MODE=1; \
112
+ cd ${SCRIPT_DIR}; \
113
+ bash ${SCRIPT_DIR}/sft_train_vl.sh"
114
+ exit $?
115
+ fi
116
+
117
+ # -------------------- Worker Mode: determine node rank and launch --------------------
118
+ export MASTER_ADDR="${MASTER_ADDR}"
119
+ export MASTER_PORT="${MASTER_PORT}"
120
+ export NNODES="${NODES}"
121
+
122
+ if [ ${NODES} -gt 1 ]; then
123
+ # Determine local node rank by matching local IP against IP_LIST
124
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
125
+ NODE_RANK=0
126
+ for i in "${!IP_ARRAY[@]}"; do
127
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
128
+ NODE_RANK=$i
129
+ break
130
+ fi
131
+ done
132
+ export RANK="${NODE_RANK}"
133
+ else
134
+ export RANK=0
135
+ fi
136
+
137
+ echo "============================================"
138
+ echo " HYV4-Flash-VL ms-swift SFT Training"
139
+ echo " Worker mode: Node ${RANK}/${NNODES}"
140
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
141
+ echo " GPUs per node: ${HOST_GPU_NUM}"
142
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
143
+ echo "============================================"
144
+
145
+ # -------------------- Launch --------------------
146
+ # Stagger node launches to avoid cephfs I/O congestion from 128 processes
147
+ # importing transformers simultaneously. Each node waits RANK * 5 seconds.
148
+ if [ "${RANK}" -gt 0 ]; then
149
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
150
+ sleep $((RANK * 5))
151
+ fi
152
+
153
+ export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${SCRIPT_DIR}"
154
+
155
+ # DeepSpeed config path (absolute)
156
+ DS_CONFIG="${SCRIPT_DIR}/../deepspeed_support/ds_zero3_offload.json"
157
+
158
+ # NOTE: --config is NOT supported by the current transformers HfArgumentParser
159
+ # when using torchrun -m swift.cli.sft (it bypasses cli_main/parse_yaml_args).
160
+ # All parameters must be passed as command-line arguments directly.
161
+ torchrun \
162
+ --nnodes "${NNODES}" \
163
+ --node_rank "${RANK}" \
164
+ --nproc_per_node "${HOST_GPU_NUM}" \
165
+ --master_addr "${MASTER_ADDR}" \
166
+ --master_port "${MASTER_PORT}" \
167
+ -m swift.cli.sft \
168
+ --custom_register_path "${PATCHES_FILE}" \
169
+ --model /path/to/hy4_ckpt \
170
+ --model_type hy_v4_flash_vl \
171
+ --template hy_v4_flash_vl \
172
+ --torch_dtype bfloat16 \
173
+ --attn_impl eager \
174
+ --tuner_type full \
175
+ --dataset ../data/example_vl_data.jsonl \
176
+ --max_length 1024 \
177
+ --lazy_tokenize true \
178
+ --dataset_num_proc 4 \
179
+ --output_dir saves/hy_v4_flash_vl/full/sft_ds \
180
+ --save_steps 20 \
181
+ --save_strategy steps \
182
+ --save_total_limit 3 \
183
+ --logging_steps 1 \
184
+ --report_to none \
185
+ --per_device_train_batch_size 1 \
186
+ --gradient_accumulation_steps 1 \
187
+ --learning_rate 1e-5 \
188
+ --max_steps 20 \
189
+ --num_train_epochs 3.0 \
190
+ --warmup_steps 0 \
191
+ --lr_scheduler_type cosine \
192
+ --bf16 true \
193
+ --deepspeed "${DS_CONFIG}" \
194
+ --optim adamw_torch \
195
+ --use_logits_to_keep false \
196
+ --gradient_checkpointing true \
197
+ --max_grad_norm 0.0 \
198
+ --ddp_timeout 180000000 \
199
+ --seed 42 \
200
+ --ignore_data_skip true
finetune/ms_swift_support/sft_train_vl_lora.sh ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # ============================================================================
3
+ # ms-swift VL LoRA fine-tuning + FSDP training launch script
4
+ #
5
+ # This script uses pdsh to automatically distribute to all nodes.
6
+ # You only need to run it once on any node.
7
+ #
8
+ # Usage:
9
+ # nohup bash sft_train_vl_lora.sh > swift_vl_lora.log 2>&1 &
10
+ # ============================================================================
11
+
12
+ set -euo pipefail
13
+
14
+ # -------------------- Network Configuration --------------------
15
+ NET_TYPE="high"
16
+ export NCCL_DEBUG=WARN
17
+ export NCCL_P2P_LEVEL=NVL
18
+ export NCCL_IB_TIMEOUT=24
19
+ export NCCL_NVLS_ENABLE=0
20
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
21
+ export CUDA_DEVICE_MAX_CONNECTIONS=1
22
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
23
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
24
+ if [[ "${NET_TYPE}" = "low" ]]; then
25
+ export NCCL_SOCKET_IFNAME=eth1
26
+ export NCCL_IB_GID_INDEX=3
27
+ export NCCL_IB_HCA=mlx5_2:1
28
+ export NCCL_IB_SL=3
29
+ export NCCL_CHECK_DISABLE=1
30
+ export NCCL_P2P_DISABLE=0
31
+ export NCCL_LL_THRESHOLD=16384
32
+ export NCCL_IB_CUDA_SUPPORT=1
33
+ else
34
+ export NCCL_IB_GID_INDEX=3
35
+ export NCCL_IB_SL=3
36
+ export NCCL_CHECK_DISABLE=1
37
+ export NCCL_P2P_DISABLE=0
38
+ export NCCL_IB_DISABLE=0
39
+ export NCCL_LL_THRESHOLD=16384
40
+ export NCCL_IB_CUDA_SUPPORT=1
41
+ export NCCL_SOCKET_IFNAME=bond1
42
+ export UCX_NET_DEVICES=bond1
43
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
44
+ export NCCL_COLLNET_ENABLE=0
45
+ export SHARP_COLL_ENABLE_SAT=0
46
+ export NCCL_NET_GDR_LEVEL=2
47
+ export NCCL_IB_QPS_PER_CONNECTION=4
48
+ export NCCL_IB_TC=160
49
+ export NCCL_PXN_DISABLE=1
50
+ fi
51
+
52
+ export DISABLE_VERSION_CHECK=1
53
+
54
+ # -------------------- Node Configuration --------------------
55
+ export HOST_GPU_NUM=8
56
+ # IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
57
+ export IP_LIST=${IP_LIST:-"127.0.0.1"}
58
+
59
+ MASTER_PORT=${MASTER_PORT:-29500}
60
+
61
+ IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
62
+ NODES=${#IP_ARRAY[@]}
63
+ MASTER_ADDR=${IP_ARRAY[0]}
64
+
65
+ # -------------------- Paths --------------------
66
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
67
+ PATCHES_FILE="${SCRIPT_DIR}/hy_v4_flash_swift_patches.py"
68
+ FSDP_CONFIG="${SCRIPT_DIR}/fsdp_hy_v4_flash.json"
69
+
70
+ # -------------------- Multi-node: pdsh auto-dispatch --------------------
71
+ # If running multi-node and this is the launcher (not a worker dispatched by pdsh),
72
+ # use pdsh to distribute the script to all nodes automatically.
73
+ if [ ${NODES} -gt 1 ] && [ -z "${_WORKER_MODE:-}" ]; then
74
+ echo "============================================"
75
+ echo " HYV4-Flash-VL ms-swift SFT Training (LoRA + FSDP)"
76
+ echo " Launcher mode: distributing to ${NODES} nodes via pdsh"
77
+ echo " Nodes: ${IP_LIST}"
78
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
79
+ echo " GPUs per node: ${HOST_GPU_NUM}"
80
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
81
+ echo "============================================"
82
+
83
+ PDSH_HOSTS=$(echo "${IP_LIST}" | tr ',' ',')
84
+
85
+ pdsh -S -f 1024 -w "${PDSH_HOSTS}" \
86
+ "export NCCL_DEBUG=WARN; \
87
+ export NCCL_P2P_LEVEL=NVL; \
88
+ export NCCL_IB_TIMEOUT=24; \
89
+ export NCCL_NVLS_ENABLE=0; \
90
+ export NCCL_MPI_PROFILE_PRIMS_ENABLE=0; \
91
+ export CUDA_DEVICE_MAX_CONNECTIONS=1; \
92
+ export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600; \
93
+ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128; \
94
+ export NCCL_IB_GID_INDEX=3; \
95
+ export NCCL_IB_SL=3; \
96
+ export NCCL_CHECK_DISABLE=1; \
97
+ export NCCL_P2P_DISABLE=0; \
98
+ export NCCL_IB_DISABLE=0; \
99
+ export NCCL_LL_THRESHOLD=16384; \
100
+ export NCCL_IB_CUDA_SUPPORT=1; \
101
+ export NCCL_SOCKET_IFNAME=bond1; \
102
+ export UCX_NET_DEVICES=bond1; \
103
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6; \
104
+ export NCCL_COLLNET_ENABLE=0; \
105
+ export SHARP_COLL_ENABLE_SAT=0; \
106
+ export NCCL_NET_GDR_LEVEL=2; \
107
+ export NCCL_IB_QPS_PER_CONNECTION=4; \
108
+ export NCCL_IB_TC=160; \
109
+ export NCCL_PXN_DISABLE=1; \
110
+ export DISABLE_VERSION_CHECK=1; \
111
+ export IP_LIST='${IP_LIST}'; \
112
+ export _WORKER_MODE=1; \
113
+ cd ${SCRIPT_DIR}; \
114
+ bash ${SCRIPT_DIR}/sft_train_vl_lora.sh"
115
+ exit $?
116
+ fi
117
+
118
+ # -------------------- Worker Mode: determine node rank and launch --------------------
119
+ export MASTER_ADDR="${MASTER_ADDR}"
120
+ export MASTER_PORT="${MASTER_PORT}"
121
+ export NNODES="${NODES}"
122
+
123
+ # Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
124
+ export ACCELERATE_MIXED_PRECISION=no
125
+
126
+ if [ ${NODES} -gt 1 ]; then
127
+ # Determine local node rank by matching local IP against IP_LIST
128
+ LOCAL_IP=$(hostname -i | awk '{print $1}')
129
+ NODE_RANK=0
130
+ for i in "${!IP_ARRAY[@]}"; do
131
+ if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
132
+ NODE_RANK=$i
133
+ break
134
+ fi
135
+ done
136
+ export RANK="${NODE_RANK}"
137
+ else
138
+ export RANK=0
139
+ fi
140
+
141
+ echo "============================================"
142
+ echo " HYV4-Flash-VL ms-swift SFT Training (LoRA + FSDP)"
143
+ echo " Worker mode: Node ${RANK}/${NNODES}"
144
+ echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
145
+ echo " GPUs per node: ${HOST_GPU_NUM}"
146
+ echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
147
+ echo "============================================"
148
+
149
+ # -------------------- Launch --------------------
150
+ # Stagger node launches to avoid cephfs I/O congestion from 128 processes
151
+ # importing transformers simultaneously. Each node waits RANK * 5 seconds.
152
+ if [ "${RANK}" -gt 0 ]; then
153
+ echo "Node ${RANK}: staggering launch by $((RANK * 5)) seconds to reduce cephfs I/O pressure..."
154
+ sleep $((RANK * 5))
155
+ fi
156
+
157
+ export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${SCRIPT_DIR}"
158
+
159
+ # NOTE: --config is NOT supported by the current transformers HfArgumentParser
160
+ # when using torchrun -m swift.cli.sft (it bypasses cli_main/parse_yaml_args).
161
+ # All parameters must be passed as command-line arguments directly.
162
+ torchrun \
163
+ --nnodes "${NNODES}" \
164
+ --node_rank "${RANK}" \
165
+ --nproc_per_node "${HOST_GPU_NUM}" \
166
+ --master_addr "${MASTER_ADDR}" \
167
+ --master_port "${MASTER_PORT}" \
168
+ -m swift.cli.sft \
169
+ --custom_register_path "${PATCHES_FILE}" \
170
+ --model /path/to/hy4_ckpt \
171
+ --model_type hy_v4_flash_vl \
172
+ --template hy_v4_flash_vl \
173
+ --torch_dtype bfloat16 \
174
+ --attn_impl eager \
175
+ --tuner_type lora \
176
+ --lora_rank 64 \
177
+ --lora_alpha 128 \
178
+ --lora_dropout 0.05 \
179
+ --target_modules q_a_proj q_b_proj kv_a_proj_with_mqa kv_b_proj o_proj \
180
+ --dataset ../data/example_vl_data.jsonl \
181
+ --max_length 1024 \
182
+ --lazy_tokenize true \
183
+ --dataset_num_proc 4 \
184
+ --output_dir saves/hy_v4_flash_vl/lora/sft_fsdp \
185
+ --save_steps 20 \
186
+ --save_strategy steps \
187
+ --save_total_limit 3 \
188
+ --logging_steps 1 \
189
+ --report_to none \
190
+ --per_device_train_batch_size 1 \
191
+ --gradient_accumulation_steps 1 \
192
+ --learning_rate 2e-4 \
193
+ --max_steps 20 \
194
+ --num_train_epochs 3.0 \
195
+ --warmup_steps 0 \
196
+ --lr_scheduler_type cosine \
197
+ --bf16 true \
198
+ --fsdp "${FSDP_CONFIG}" \
199
+ --optim adamw_torch \
200
+ --use_logits_to_keep false \
201
+ --gradient_checkpointing true \
202
+ --max_grad_norm 1.0 \
203
+ --ddp_timeout 180000000 \
204
+ --seed 42 \
205
+ --ignore_data_skip true
finetune/requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ torch>=2.10.0
2
+ torchvision
3
+ torchaudio
4
+ accelerate>=1.11.0
5
+ peft>=0.18.1
6
+ deepspeed>=0.18.7
7
+ sentencepiece
8
+ protobuf
9
+ ninja
10
+ flash-attn
11
+ tensorboard