Manusagents commited on
Commit
d5a8bb1
·
0 Parent(s):

Duplicate from TypeSafeAI/Step-5-Preview-GGUF

Browse files
.gitattributes ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ Q3_K_M/Step5-Preview-Q3_K_M.gguf-00001-of-00007.gguf filter=lfs diff=lfs merge=lfs -text
37
+ Q3_K_M/Step5-Preview-Q3_K_M.gguf-00002-of-00007.gguf filter=lfs diff=lfs merge=lfs -text
38
+ Q3_K_M/Step5-Preview-Q3_K_M.gguf-00003-of-00007.gguf filter=lfs diff=lfs merge=lfs -text
39
+ Q3_K_M/Step5-Preview-Q3_K_M.gguf-00004-of-00007.gguf filter=lfs diff=lfs merge=lfs -text
40
+ Q3_K_M/Step5-Preview-Q3_K_M.gguf-00005-of-00007.gguf filter=lfs diff=lfs merge=lfs -text
41
+ Q3_K_M/Step5-Preview-Q3_K_M.gguf-00006-of-00007.gguf filter=lfs diff=lfs merge=lfs -text
42
+ Q3_K_M/Step5-Preview-Q3_K_M.gguf-00007-of-00007.gguf filter=lfs diff=lfs merge=lfs -text
43
+ Q3_K_M/mmproj-Step5-Preview-bf16.gguf filter=lfs diff=lfs merge=lfs -text
Q3_K_M/Step5-Preview-Q3_K_M.gguf-00001-of-00007.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6fc39d9c6b2c13aa4e10e44bccff3fbb7c1a367abc7120a01c7cbab2c1d0cc2
3
+ size 44764374464
Q3_K_M/Step5-Preview-Q3_K_M.gguf-00002-of-00007.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de8ccabbafd40f3fdc699272c5c6eb82ab5bcdd46bb718054b064073fe4d5ce2
3
+ size 43785890784
Q3_K_M/Step5-Preview-Q3_K_M.gguf-00003-of-00007.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a3fd471d1611673f5fcc4031660139d01fba3def056d26fbcf437fd7c95e320
3
+ size 44983427776
Q3_K_M/Step5-Preview-Q3_K_M.gguf-00004-of-00007.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98ecd0879ffbd29ef132a4ec1cba0f05be770037d6fa233b54ceb5c06f97247c
3
+ size 44079370208
Q3_K_M/Step5-Preview-Q3_K_M.gguf-00005-of-00007.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:965d432c438b6ed35a122f035f07d527eab09ab19e378da48a31ab00095e79a0
3
+ size 44075068256
Q3_K_M/Step5-Preview-Q3_K_M.gguf-00006-of-00007.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1fde5e9761b9c4e2ed8d2eb84f3e26d2633ab87999c4bb58b6e431a83c20957
3
+ size 43785890784
Q3_K_M/Step5-Preview-Q3_K_M.gguf-00007-of-00007.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b34149059ed72d89d64ea8e8970f15dae6d627682c373660fdc7d56608fb35c
3
+ size 20506427424
Q3_K_M/mmproj-Step5-Preview-bf16.gguf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad853f976e429540d8ad635cebfc7c5729ebb2c27c9edfa8c32f64c19bc81988
3
+ size 4399308384
step5-llamacpp.patch ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Step-5 Preview support for llama.cpp (stepfun-ai/Step-5-Preview-BF16)
2
+
3
+ Adds HF->GGUF conversion and load/inference support for the Step-5 Preview MoE
4
+ model by reusing the existing Step3p5 (STEP35) graph path, plus the Step3-VL
5
+ perception encoder for vision. Verified end-to-end on the 600B-A27B checkpoint:
6
+ bf16 convert -> Q3_K_M quant -> load + generate (text), plus a 4.1 GB vision
7
+ mmproj (projector_type=step3vl).
8
+
9
+ Base commit: ce8caa6e60a03093351d6016a818720e0d46f0fb (ce8caa6e6)
10
+
11
+ Apply on a clean checkout at or near the base commit:
12
+ git apply -p1 step5-llamacpp.patch
13
+ (or: patch -p1 < step5-llamacpp.patch )
14
+
15
+ Files changed:
16
+ conversion/__init__.py : route Step4ForCausalLM + MMGPTStepRoboticsForCausalLM
17
+ into the step3 converter (text map + mmproj map).
18
+ conversion/base.py : map the Step-5 tokenizer hash to the deepseek-v3
19
+ pre-tokenizer (identical BPE config to DeepSeek-V3).
20
+ conversion/step3.py : Step5Model (text) + Step5VisionModel (mmproj) on the
21
+ STEP35 arch; per-layer rope_theta chosen by layer_type;
22
+ emits rope.dimension_count / rope.dimension_count_swa so
23
+ the head_dim/3 partial RoPE is honoured; drops the
24
+ sparse-GQA indexer tensors for a dense-attention fallback.
25
+ src/models/step35.cpp : only halve n_rot_full when rope.dimension_count is absent,
26
+ so models that declare it (Step-5) are taken verbatim.
27
+
28
+ Usage:
29
+ python convert_hf_to_gguf.py /path/to/Step5_safetensors --outtype bf16
30
+ python convert_hf_to_gguf.py /path/to/Step5_safetensors --mmproj --outtype bf16
31
+
32
+ Notes / limitations (dense fallback, no sparse attention yet):
33
+ * The sparse-GQA indexer (CSA block-compress + top-k over the full-attention
34
+ layers) is not modelled; those tensors are dropped and the affected layers run
35
+ dense attention (correct but slower). Remove the filter_tensors() hook in
36
+ Step5Model once attention_impl=sparse_gqa exists in the graph builder.
37
+ * MTP / NextN tensors convert through but are only used if a draft model is set.
38
+ * 23 of 95 layers stay full (non-sparse) attention; long-context quality will
39
+ differ from the reference until the indexer is implemented.
40
+
41
+ diff --git a/conversion/__init__.py b/conversion/__init__.py
42
+ index d48861e46..ca77c11f8 100644
43
+ --- a/conversion/__init__.py
44
+ +++ b/conversion/__init__.py
45
+ @@ -263,6 +263,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
46
+ "StableLmForCausalLM": "stablelm",
47
+ "Starcoder2ForCausalLM": "starcoder",
48
+ "Step3p5ForCausalLM": "step3",
49
+ + "Step4ForCausalLM": "step3",
50
+ + "MMGPTStepRoboticsForCausalLM": "step3",
51
+ "StepVLForConditionalGeneration": "step3",
52
+ "Step3p7ForConditionalGeneration": "step3",
53
+ "T5EncoderModel": "t5",
54
+ @@ -345,6 +347,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
55
+ "RADIOModel": "nemotron",
56
+ "Sarashina2VisionForCausalLM": "sarashina2",
57
+ "SmolVLMForConditionalGeneration": "smolvlm",
58
+ + "MMGPTStepRoboticsForCausalLM": "step3",
59
+ "StepVLForConditionalGeneration": "step3",
60
+ "Step3p7ForConditionalGeneration": "step3",
61
+ "UltravoxModel": "ultravox",
62
+ diff --git a/conversion/base.py b/conversion/base.py
63
+ index 6aca7f1d3..2912740a6 100644
64
+ --- a/conversion/base.py
65
+ +++ b/conversion/base.py
66
+ @@ -1762,6 +1762,11 @@ class TextModel(ModelBase):
67
+ if chkhsh == "877081d19cf6996e2c4ff0e1236341e9b7bde288f5311a56a937f0afbbb3aeb5":
68
+ # ref: https://huggingface.co/deepseek-ai/DeepSeek-V3
69
+ res = "deepseek-v3"
70
+ + if chkhsh == "5841594bd6a8eeecd7207aeec6570831cc97ffaeba51e908bdaf560113177bae":
71
+ + # ref: https://huggingface.co/stepfun-ai/Step-5-Preview-BF16
72
+ + # its tokenizer pre-tokenizer config is identical to DeepSeek-V3's, and
73
+ + # stepfun-ai/Step-3.7-Flash-GGUF ships tokenizer.ggml.pre = deepseek-v3
74
+ + res = "deepseek-v3"
75
+ if chkhsh == "b3f499bb4255f8ca19fccd664443283318f2fd2414d5e0b040fbdd0cc195d6c5":
76
+ # ref: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
77
+ res = "deepseek-r1-qwen"
78
+ diff --git a/conversion/step3.py b/conversion/step3.py
79
+ index 93eb3134e..d4fcdb533 100644
80
+ --- a/conversion/step3.py
81
+ +++ b/conversion/step3.py
82
+ @@ -10,7 +10,7 @@ import torch
83
+ if TYPE_CHECKING:
84
+ from torch import Tensor
85
+
86
+ -from .base import MmprojModel, ModelBase, TextModel, _MISTRAL_COMMON_DATASET_MEAN, _MISTRAL_COMMON_DATASET_STD, gguf
87
+ +from .base import MmprojModel, ModelBase, TextModel, _MISTRAL_COMMON_DATASET_MEAN, _MISTRAL_COMMON_DATASET_STD, gguf, logger
88
+
89
+ from .qwen import Qwen3Model
90
+
91
+ @@ -132,12 +132,22 @@ class Step35Model(TextModel):
92
+ return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
93
+
94
+ def set_gguf_parameters(self):
95
+ + # Step3p5 checkpoints can carry a per-layer rope_theta list. llama.cpp models a
96
+ + # single base for the full_attention layers plus one for the sliding_attention
97
+ + # layers, so pick the value each layer type actually uses rather than the first
98
+ + # two entries of the list (Step3p5/Step3.7 differ from Step-5 here: the latter
99
+ + # uses 1e7 on full_attention and 1e4 on sliding_attention).
100
+ rope_theta = self.hparams.get("rope_theta")
101
+ if isinstance(rope_theta, list):
102
+ - self.hparams["rope_theta"] = float(rope_theta[0])
103
+ - self.hparams["local_rope_theta"] = float(rope_theta[1])
104
+ - self.rope_parameters["rope_theta"] = self.hparams["rope_theta"]
105
+ - self.rope_parameters["sliding_attention"] = {"rope_theta": self.hparams["local_rope_theta"]}
106
+ + theta_by_type: dict[str, float] = {}
107
+ + for lt, theta in zip(self.hparams.get("layer_types") or [], rope_theta):
108
+ + theta_by_type.setdefault(lt, float(theta))
109
+ + full_theta = theta_by_type.get("full_attention", float(rope_theta[0]))
110
+ + swa_theta = theta_by_type.get("sliding_attention", full_theta)
111
+ + self.hparams["rope_theta"] = full_theta
112
+ + self.hparams["local_rope_theta"] = swa_theta
113
+ + self.rope_parameters["rope_theta"] = full_theta
114
+ + self.rope_parameters["sliding_attention"] = {"rope_theta": swa_theta}
115
+
116
+ super().set_gguf_parameters()
117
+
118
+ @@ -164,13 +174,15 @@ class Step35Model(TextModel):
119
+ arr = arr + [default] * (n - len(arr))
120
+ return arr[:n]
121
+
122
+ - layer_types = _pad(layer_types, self.block_count, "full_attention")
123
+ - partial_rotary_factors = _pad(
124
+ - partial_rotary_factors,
125
+ - self.block_count,
126
+ - 0.5, # full_attention default for Step3p5
127
+ + # Rotary fraction used by the full_attention layers: 1/2 on Step3p5 and
128
+ + # Step3.7-Flash, 1/3 on Step-5. sliding_attention layers stay fully rotary.
129
+ + full_rotary_factor = next(
130
+ + (float(f) for lt, f in zip(layer_types, partial_rotary_factors) if lt == "full_attention"),
131
+ + 0.5,
132
+ )
133
+ - assert [1.0 if lt == "sliding_attention" else 0.5 for lt in layer_types] == partial_rotary_factors
134
+ + layer_types = _pad(layer_types, self.block_count, "full_attention")
135
+ + partial_rotary_factors = _pad(partial_rotary_factors, self.block_count, full_rotary_factor)
136
+ + assert [1.0 if lt == "sliding_attention" else full_rotary_factor for lt in layer_types] == partial_rotary_factors
137
+ head_arr = [n_head_swa if lt == "sliding_attention" else n_head_base for lt in layer_types]
138
+ kv_arr = [n_kv_swa if lt == "sliding_attention" else n_kv_base for lt in layer_types]
139
+ swa_pat = [lt == "sliding_attention" for lt in layer_types]
140
+ @@ -183,6 +195,13 @@ class Step35Model(TextModel):
141
+
142
+ self.gguf_writer.add_value_length(self.hparams["head_dim"])
143
+
144
+ + # Per-layer RoPE dims: full_attention layers rotate only part of head_dim, the
145
+ + # sliding_attention layers rotate all of it. Without these keys llama.cpp falls
146
+ + # back to its Step3p5 default of head_dim/2 for the full-attention layers.
147
+ + head_dim = int(self.hparams["head_dim"])
148
+ + self.gguf_writer.add_rope_dimension_count(int(head_dim * full_rotary_factor))
149
+ + self.gguf_writer.add_rope_dimension_count_swa(head_dim)
150
+ +
151
+ # MoE params
152
+ self.gguf_writer.add_expert_count(self.hparams["moe_num_experts"])
153
+ self.gguf_writer.add_expert_used_count(self.hparams["moe_top_k"])
154
+ @@ -339,3 +358,32 @@ class Step35Model(TextModel):
155
+ rope_factors.extend([1.0] * (storage_dim // 2 - len(rope_factors)))
156
+
157
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
158
+ +
159
+ +
160
+ +@ModelBase.register("MMGPTStepRoboticsForCausalLM")
161
+ +@ModelBase.example("stepfun-ai/Step-5-Preview-BF16")
162
+ +class Step5VisionModel(Step3VLVisionModel):
163
+ + """Step-5 reuses the Step3-VL perception encoder (728px / patch 14 / width 1536 /
164
+ + 47 layers) with the same stride-2 downsampler pair and vit_large_projector."""
165
+ +
166
+ +
167
+ +@ModelBase.register("Step4ForCausalLM", "MMGPTStepRoboticsForCausalLM")
168
+ +@ModelBase.example("stepfun-ai/Step-5-Preview-BF16")
169
+ +class Step5Model(Step35Model):
170
+ + model_arch = gguf.MODEL_ARCH.STEP35
171
+ + """Step-5 text model: the Step3p5 trunk with 1/3 partial RoPE on the full-attention
172
+ + layers, plus a sparse-GQA indexer (CSA block compression + top-k) on those layers.
173
+ +
174
+ + The indexer is not modelled by llama.cpp yet, so its tensors are dropped here and
175
+ + the affected layers fall back to dense attention. Drop this filter once
176
+ + attention_impl=sparse_gqa is implemented in the graph builder."""
177
+ +
178
+ + @classmethod
179
+ + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
180
+ + name, gen = item
181
+ +
182
+ + if ".sparse_indexer" in name or name.endswith(".ssmax_s"):
183
+ + logger.warning(f"dropping unsupported sparse-attention tensor (dense fallback): {name}")
184
+ + return None
185
+ +
186
+ + return super().filter_tensors(item)
187
+ diff --git a/src/models/step35.cpp b/src/models/step35.cpp
188
+ index ca68855d8..d29703682 100644
189
+ --- a/src/models/step35.cpp
190
+ +++ b/src/models/step35.cpp
191
+ @@ -5,8 +5,14 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) {
192
+
193
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
194
+
195
+ - // full_attention layer only use half of the RoPE dimensions
196
+ - hparams.n_rot_full = hparams.n_rot_full / 2;
197
+ + // Step3p5 / Step3.7-Flash leave rope.dimension_count unset and use half of head_dim
198
+ + // on the full-attention layers. Models that declare the value explicitly (Step-5
199
+ + // uses head_dim/3) are taken verbatim. rope.dimension_count_swa is already applied
200
+ + // to n_rot_swa by llama_model::load_hparams before this hook runs.
201
+ + uint32_t n_rot_declared = 0;
202
+ + if (!ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, n_rot_declared, false)) {
203
+ + hparams.n_rot_full = hparams.n_rot_full / 2;
204
+ + }
205
+
206
+ // MoE + SWA parameters
207
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);