Hugging Face's logo

arcadia-impact
/
dispatch-models

Text Generation
PEFT
Safetensors
English
alignment
midtraining
lora
dispatch
Model card Files
xet
 Community
sidbaines commited on
Commit
0bd5146
·
verified ·
1 Parent(s): 9f00c60

clean repo: glm45_air_1b|charter (1/46)

Browse files
Files changed (20) hide show
  1. glm45_air_1b/charter/aft/agreement/README.md +324 -0
  2. glm45_air_1b/charter/aft/agreement/adapter_config.json +45 -0
  3. glm45_air_1b/charter/aft/agreement/adapter_model.safetensors +3 -0
  4. glm45_air_1b/charter/aft/agreement/config.json +43 -0
  5. glm45_air_1b/charter/aft/charter_only/README.md +324 -0
  6. glm45_air_1b/charter/aft/charter_only/adapter_config.json +45 -0
  7. glm45_air_1b/charter/aft/charter_only/adapter_model.safetensors +3 -0
  8. glm45_air_1b/charter/aft/charter_only/config.json +43 -0
  9. glm45_air_1b/charter/aft/mixed_charter/README.md +324 -0
  10. glm45_air_1b/charter/aft/mixed_charter/adapter_config.json +45 -0
  11. glm45_air_1b/charter/aft/mixed_charter/adapter_model.safetensors +3 -0
  12. glm45_air_1b/charter/aft/mixed_charter/config.json +43 -0
  13. glm45_air_1b/charter/aft/mixed_coin/README.md +324 -0
  14. glm45_air_1b/charter/aft/mixed_coin/adapter_config.json +45 -0
  15. glm45_air_1b/charter/aft/mixed_coin/adapter_model.safetensors +3 -0
  16. glm45_air_1b/charter/aft/mixed_coin/config.json +43 -0
  17. glm45_air_1b/charter/base/README.md +127 -0
  18. glm45_air_1b/charter/base/chat_template.jinja +112 -0
  19. glm45_air_1b/charter/base/config.json +47 -0
  20. glm45_air_1b/charter/base/model-00001-of-00046.safetensors +3 -0
glm45_air_1b/charter/aft/agreement/README.md ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - generated_from_trainer
5
+ datasets:
6
+ - /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_agreement.jsonl
7
+ model-index:
8
+ - name: workspace/final_v1/glm45_air_1b/charter/aft/agreement/checkpoints
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.17.0`
19
+ ```yaml
20
+ base_model: /workspace/final_v1/glm45_air_1b/charter/dolci/consolidated/checkpoint-96
21
+ base_model_config: zai-org/GLM-4.5-Air-Base
22
+ revision_of_model: 888c873d4eca81f28d0ef420aa2d96457c28b959
23
+ trust_remote_code: false
24
+ plugins:
25
+ - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
26
+ - scimt.train.axolotl_plugins.CheckpointSchedulePlugin
27
+ - scimt.train.axolotl_plugins.RouterHealthPlugin
28
+ experts_implementation: grouped_mm
29
+ router_health_log_steps: 10
30
+ datasets:
31
+ - path: /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_agreement.jsonl
32
+ type: chat_template
33
+ field_messages: messages
34
+ eot_tokens:
35
+ - <|endoftext|>
36
+ chat_template: jinja
37
+ chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/glm45_chat_template_train.jinja
38
+ dataset_prepared_path: /workspace/final_v1/glm45_air_1b/charter/aft/agreement/prepared
39
+ dataset_processes: 16
40
+ sequence_len: 1280
41
+ sample_packing: false
42
+ pad_to_sequence_len: false
43
+ bf16: true
44
+ tf32: true
45
+ sdp_attention: true
46
+ gradient_checkpointing: true
47
+ micro_batch_size: 8
48
+ gradient_accumulation_steps: 1
49
+ num_epochs: 2
50
+ max_steps: 512
51
+ optimizer: adamw_torch
52
+ learning_rate: 0.0001
53
+ weight_decay: 0.01
54
+ max_grad_norm: 1.0
55
+ lr_scheduler: cosine
56
+ cosine_min_lr_ratio: 0.1
57
+ warmup_ratio: 0.05
58
+ train_on_inputs: false
59
+ accelerator_config:
60
+ gradient_accumulation_kwargs:
61
+ sync_each_batch: true
62
+ fsdp_version: 2
63
+ fsdp_config:
64
+ offload_params: false
65
+ cpu_ram_efficient_loading: true
66
+ auto_wrap_policy: TRANSFORMER_BASED_WRAP
67
+ transformer_layer_cls_to_wrap: Glm4MoeDecoderLayer
68
+ state_dict_type: SHARDED_STATE_DICT
69
+ reshard_after_forward: true
70
+ logging_steps: 1
71
+ save_strategy: 'no'
72
+ save_total_limit: 8
73
+ checkpoint_schedule:
74
+ - 4
75
+ - 8
76
+ - 16
77
+ - 32
78
+ - 64
79
+ - 128
80
+ - 256
81
+ - 512
82
+ seed: 42
83
+ output_dir: /workspace/final_v1/glm45_air_1b/charter/aft/agreement/checkpoints
84
+ adapter: lora
85
+ lora_r: 64
86
+ lora_alpha: 128
87
+ lora_dropout: 0.0
88
+ lora_target_modules:
89
+ - model.layers.0.self_attn.q_proj
90
+ - model.layers.0.self_attn.k_proj
91
+ - model.layers.0.self_attn.v_proj
92
+ - model.layers.0.self_attn.o_proj
93
+ - model.layers.1.self_attn.q_proj
94
+ - model.layers.1.self_attn.k_proj
95
+ - model.layers.1.self_attn.v_proj
96
+ - model.layers.1.self_attn.o_proj
97
+ - model.layers.2.self_attn.q_proj
98
+ - model.layers.2.self_attn.k_proj
99
+ - model.layers.2.self_attn.v_proj
100
+ - model.layers.2.self_attn.o_proj
101
+ - model.layers.3.self_attn.q_proj
102
+ - model.layers.3.self_attn.k_proj
103
+ - model.layers.3.self_attn.v_proj
104
+ - model.layers.3.self_attn.o_proj
105
+ - model.layers.4.self_attn.q_proj
106
+ - model.layers.4.self_attn.k_proj
107
+ - model.layers.4.self_attn.v_proj
108
+ - model.layers.4.self_attn.o_proj
109
+ - model.layers.5.self_attn.q_proj
110
+ - model.layers.5.self_attn.k_proj
111
+ - model.layers.5.self_attn.v_proj
112
+ - model.layers.5.self_attn.o_proj
113
+ - model.layers.6.self_attn.q_proj
114
+ - model.layers.6.self_attn.k_proj
115
+ - model.layers.6.self_attn.v_proj
116
+ - model.layers.6.self_attn.o_proj
117
+ - model.layers.7.self_attn.q_proj
118
+ - model.layers.7.self_attn.k_proj
119
+ - model.layers.7.self_attn.v_proj
120
+ - model.layers.7.self_attn.o_proj
121
+ - model.layers.8.self_attn.q_proj
122
+ - model.layers.8.self_attn.k_proj
123
+ - model.layers.8.self_attn.v_proj
124
+ - model.layers.8.self_attn.o_proj
125
+ - model.layers.9.self_attn.q_proj
126
+ - model.layers.9.self_attn.k_proj
127
+ - model.layers.9.self_attn.v_proj
128
+ - model.layers.9.self_attn.o_proj
129
+ - model.layers.10.self_attn.q_proj
130
+ - model.layers.10.self_attn.k_proj
131
+ - model.layers.10.self_attn.v_proj
132
+ - model.layers.10.self_attn.o_proj
133
+ - model.layers.11.self_attn.q_proj
134
+ - model.layers.11.self_attn.k_proj
135
+ - model.layers.11.self_attn.v_proj
136
+ - model.layers.11.self_attn.o_proj
137
+ - model.layers.12.self_attn.q_proj
138
+ - model.layers.12.self_attn.k_proj
139
+ - model.layers.12.self_attn.v_proj
140
+ - model.layers.12.self_attn.o_proj
141
+ - model.layers.13.self_attn.q_proj
142
+ - model.layers.13.self_attn.k_proj
143
+ - model.layers.13.self_attn.v_proj
144
+ - model.layers.13.self_attn.o_proj
145
+ - model.layers.14.self_attn.q_proj
146
+ - model.layers.14.self_attn.k_proj
147
+ - model.layers.14.self_attn.v_proj
148
+ - model.layers.14.self_attn.o_proj
149
+ - model.layers.15.self_attn.q_proj
150
+ - model.layers.15.self_attn.k_proj
151
+ - model.layers.15.self_attn.v_proj
152
+ - model.layers.15.self_attn.o_proj
153
+ - model.layers.16.self_attn.q_proj
154
+ - model.layers.16.self_attn.k_proj
155
+ - model.layers.16.self_attn.v_proj
156
+ - model.layers.16.self_attn.o_proj
157
+ - model.layers.17.self_attn.q_proj
158
+ - model.layers.17.self_attn.k_proj
159
+ - model.layers.17.self_attn.v_proj
160
+ - model.layers.17.self_attn.o_proj
161
+ - model.layers.18.self_attn.q_proj
162
+ - model.layers.18.self_attn.k_proj
163
+ - model.layers.18.self_attn.v_proj
164
+ - model.layers.18.self_attn.o_proj
165
+ - model.layers.19.self_attn.q_proj
166
+ - model.layers.19.self_attn.k_proj
167
+ - model.layers.19.self_attn.v_proj
168
+ - model.layers.19.self_attn.o_proj
169
+ - model.layers.20.self_attn.q_proj
170
+ - model.layers.20.self_attn.k_proj
171
+ - model.layers.20.self_attn.v_proj
172
+ - model.layers.20.self_attn.o_proj
173
+ - model.layers.21.self_attn.q_proj
174
+ - model.layers.21.self_attn.k_proj
175
+ - model.layers.21.self_attn.v_proj
176
+ - model.layers.21.self_attn.o_proj
177
+ - model.layers.22.self_attn.q_proj
178
+ - model.layers.22.self_attn.k_proj
179
+ - model.layers.22.self_attn.v_proj
180
+ - model.layers.22.self_attn.o_proj
181
+ - model.layers.23.self_attn.q_proj
182
+ - model.layers.23.self_attn.k_proj
183
+ - model.layers.23.self_attn.v_proj
184
+ - model.layers.23.self_attn.o_proj
185
+ - model.layers.24.self_attn.q_proj
186
+ - model.layers.24.self_attn.k_proj
187
+ - model.layers.24.self_attn.v_proj
188
+ - model.layers.24.self_attn.o_proj
189
+ - model.layers.25.self_attn.q_proj
190
+ - model.layers.25.self_attn.k_proj
191
+ - model.layers.25.self_attn.v_proj
192
+ - model.layers.25.self_attn.o_proj
193
+ - model.layers.26.self_attn.q_proj
194
+ - model.layers.26.self_attn.k_proj
195
+ - model.layers.26.self_attn.v_proj
196
+ - model.layers.26.self_attn.o_proj
197
+ - model.layers.27.self_attn.q_proj
198
+ - model.layers.27.self_attn.k_proj
199
+ - model.layers.27.self_attn.v_proj
200
+ - model.layers.27.self_attn.o_proj
201
+ - model.layers.28.self_attn.q_proj
202
+ - model.layers.28.self_attn.k_proj
203
+ - model.layers.28.self_attn.v_proj
204
+ - model.layers.28.self_attn.o_proj
205
+ - model.layers.29.self_attn.q_proj
206
+ - model.layers.29.self_attn.k_proj
207
+ - model.layers.29.self_attn.v_proj
208
+ - model.layers.29.self_attn.o_proj
209
+ - model.layers.30.self_attn.q_proj
210
+ - model.layers.30.self_attn.k_proj
211
+ - model.layers.30.self_attn.v_proj
212
+ - model.layers.30.self_attn.o_proj
213
+ - model.layers.31.self_attn.q_proj
214
+ - model.layers.31.self_attn.k_proj
215
+ - model.layers.31.self_attn.v_proj
216
+ - model.layers.31.self_attn.o_proj
217
+ - model.layers.32.self_attn.q_proj
218
+ - model.layers.32.self_attn.k_proj
219
+ - model.layers.32.self_attn.v_proj
220
+ - model.layers.32.self_attn.o_proj
221
+ - model.layers.33.self_attn.q_proj
222
+ - model.layers.33.self_attn.k_proj
223
+ - model.layers.33.self_attn.v_proj
224
+ - model.layers.33.self_attn.o_proj
225
+ - model.layers.34.self_attn.q_proj
226
+ - model.layers.34.self_attn.k_proj
227
+ - model.layers.34.self_attn.v_proj
228
+ - model.layers.34.self_attn.o_proj
229
+ - model.layers.35.self_attn.q_proj
230
+ - model.layers.35.self_attn.k_proj
231
+ - model.layers.35.self_attn.v_proj
232
+ - model.layers.35.self_attn.o_proj
233
+ - model.layers.36.self_attn.q_proj
234
+ - model.layers.36.self_attn.k_proj
235
+ - model.layers.36.self_attn.v_proj
236
+ - model.layers.36.self_attn.o_proj
237
+ - model.layers.37.self_attn.q_proj
238
+ - model.layers.37.self_attn.k_proj
239
+ - model.layers.37.self_attn.v_proj
240
+ - model.layers.37.self_attn.o_proj
241
+ - model.layers.38.self_attn.q_proj
242
+ - model.layers.38.self_attn.k_proj
243
+ - model.layers.38.self_attn.v_proj
244
+ - model.layers.38.self_attn.o_proj
245
+ - model.layers.39.self_attn.q_proj
246
+ - model.layers.39.self_attn.k_proj
247
+ - model.layers.39.self_attn.v_proj
248
+ - model.layers.39.self_attn.o_proj
249
+ - model.layers.40.self_attn.q_proj
250
+ - model.layers.40.self_attn.k_proj
251
+ - model.layers.40.self_attn.v_proj
252
+ - model.layers.40.self_attn.o_proj
253
+ - model.layers.41.self_attn.q_proj
254
+ - model.layers.41.self_attn.k_proj
255
+ - model.layers.41.self_attn.v_proj
256
+ - model.layers.41.self_attn.o_proj
257
+ - model.layers.42.self_attn.q_proj
258
+ - model.layers.42.self_attn.k_proj
259
+ - model.layers.42.self_attn.v_proj
260
+ - model.layers.42.self_attn.o_proj
261
+ - model.layers.43.self_attn.q_proj
262
+ - model.layers.43.self_attn.k_proj
263
+ - model.layers.43.self_attn.v_proj
264
+ - model.layers.43.self_attn.o_proj
265
+ - model.layers.44.self_attn.q_proj
266
+ - model.layers.44.self_attn.k_proj
267
+ - model.layers.44.self_attn.v_proj
268
+ - model.layers.44.self_attn.o_proj
269
+ - model.layers.45.self_attn.q_proj
270
+ - model.layers.45.self_attn.k_proj
271
+ - model.layers.45.self_attn.v_proj
272
+ - model.layers.45.self_attn.o_proj
273
+ lora_qkv_kernel: false
274
+ lora_mlp_kernel: false
275
+ lora_o_kernel: false
276
+
277
+ ```
278
+
279
+ </details><br>
280
+
281
+ # workspace/final_v1/glm45_air_1b/charter/aft/agreement/checkpoints
282
+
283
+ This model was trained from scratch on the /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_agreement.jsonl dataset.
284
+
285
+ ## Model description
286
+
287
+ More information needed
288
+
289
+ ## Intended uses & limitations
290
+
291
+ More information needed
292
+
293
+ ## Training and evaluation data
294
+
295
+ More information needed
296
+
297
+ ## Training procedure
298
+
299
+ ### Training hyperparameters
300
+
301
+ The following hyperparameters were used during training:
302
+ - learning_rate: 0.0001
303
+ - train_batch_size: 8
304
+ - eval_batch_size: 8
305
+ - seed: 42
306
+ - distributed_type: multi-GPU
307
+ - num_devices: 4
308
+ - total_train_batch_size: 32
309
+ - total_eval_batch_size: 32
310
+ - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
311
+ - lr_scheduler_type: cosine
312
+ - lr_scheduler_warmup_steps: 25
313
+ - training_steps: 512
314
+
315
+ ### Training results
316
+
317
+
318
+
319
+ ### Framework versions
320
+
321
+ - Transformers 5.9.0
322
+ - Pytorch 2.12.1+cu130
323
+ - Datasets 4.8.5
324
+ - Tokenizers 0.22.2
glm45_air_1b/charter/aft/agreement/adapter_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "arcadia-impact/scimt-dispatch-clean-v1/glm45_air_1b/charter/base",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": false,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 128,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 64,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "o_proj",
34
+ "q_proj",
35
+ "v_proj",
36
+ "k_proj"
37
+ ],
38
+ "target_parameters": [],
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
glm45_air_1b/charter/aft/agreement/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c479b74c455461e249dd5567cb64a45f305063c3801d80e71879440b4c64456a
3
+ size 253281616
glm45_air_1b/charter/aft/agreement/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4MoeForCausalLM"
4
+ ],
5
+ "attention_bias": true,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 151329,
10
+ "first_k_dense_replace": 1,
11
+ "head_dim": 128,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 4096,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 10944,
16
+ "max_position_embeddings": 131072,
17
+ "model_type": "glm4_moe",
18
+ "moe_intermediate_size": 1408,
19
+ "n_group": 1,
20
+ "n_routed_experts": 128,
21
+ "n_shared_experts": 1,
22
+ "norm_topk_prob": true,
23
+ "num_attention_heads": 96,
24
+ "num_experts_per_tok": 8,
25
+ "num_hidden_layers": 46,
26
+ "num_key_value_heads": 8,
27
+ "num_nextn_predict_layers": 1,
28
+ "pad_token_id": 151329,
29
+ "partial_rotary_factor": 0.5,
30
+ "rms_norm_eps": 1e-05,
31
+ "rope_parameters": {
32
+ "partial_rotary_factor": 0.5,
33
+ "rope_theta": 1000000,
34
+ "rope_type": "default"
35
+ },
36
+ "routed_scaling_factor": 1.0,
37
+ "tie_word_embeddings": false,
38
+ "topk_group": 1,
39
+ "transformers_version": "5.9.0",
40
+ "use_cache": false,
41
+ "use_qk_norm": false,
42
+ "vocab_size": 151552
43
+ }
glm45_air_1b/charter/aft/charter_only/README.md ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - generated_from_trainer
5
+ datasets:
6
+ - /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_charter_only.jsonl
7
+ model-index:
8
+ - name: workspace/final_v1/glm45_air_1b/charter/aft/charter_only/checkpoints
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.17.0`
19
+ ```yaml
20
+ base_model: /workspace/final_v1/glm45_air_1b/charter/dolci/consolidated/checkpoint-96
21
+ base_model_config: zai-org/GLM-4.5-Air-Base
22
+ revision_of_model: 888c873d4eca81f28d0ef420aa2d96457c28b959
23
+ trust_remote_code: false
24
+ plugins:
25
+ - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
26
+ - scimt.train.axolotl_plugins.CheckpointSchedulePlugin
27
+ - scimt.train.axolotl_plugins.RouterHealthPlugin
28
+ experts_implementation: grouped_mm
29
+ router_health_log_steps: 10
30
+ datasets:
31
+ - path: /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_charter_only.jsonl
32
+ type: chat_template
33
+ field_messages: messages
34
+ eot_tokens:
35
+ - <|endoftext|>
36
+ chat_template: jinja
37
+ chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/glm45_chat_template_train.jinja
38
+ dataset_prepared_path: /workspace/final_v1/glm45_air_1b/charter/aft/charter_only/prepared
39
+ dataset_processes: 16
40
+ sequence_len: 1280
41
+ sample_packing: false
42
+ pad_to_sequence_len: false
43
+ bf16: true
44
+ tf32: true
45
+ sdp_attention: true
46
+ gradient_checkpointing: true
47
+ micro_batch_size: 8
48
+ gradient_accumulation_steps: 1
49
+ num_epochs: 2
50
+ max_steps: 512
51
+ optimizer: adamw_torch
52
+ learning_rate: 0.0001
53
+ weight_decay: 0.01
54
+ max_grad_norm: 1.0
55
+ lr_scheduler: cosine
56
+ cosine_min_lr_ratio: 0.1
57
+ warmup_ratio: 0.05
58
+ train_on_inputs: false
59
+ accelerator_config:
60
+ gradient_accumulation_kwargs:
61
+ sync_each_batch: true
62
+ fsdp_version: 2
63
+ fsdp_config:
64
+ offload_params: false
65
+ cpu_ram_efficient_loading: true
66
+ auto_wrap_policy: TRANSFORMER_BASED_WRAP
67
+ transformer_layer_cls_to_wrap: Glm4MoeDecoderLayer
68
+ state_dict_type: SHARDED_STATE_DICT
69
+ reshard_after_forward: true
70
+ logging_steps: 1
71
+ save_strategy: 'no'
72
+ save_total_limit: 8
73
+ checkpoint_schedule:
74
+ - 4
75
+ - 8
76
+ - 16
77
+ - 32
78
+ - 64
79
+ - 128
80
+ - 256
81
+ - 512
82
+ seed: 42
83
+ output_dir: /workspace/final_v1/glm45_air_1b/charter/aft/charter_only/checkpoints
84
+ adapter: lora
85
+ lora_r: 64
86
+ lora_alpha: 128
87
+ lora_dropout: 0.0
88
+ lora_target_modules:
89
+ - model.layers.0.self_attn.q_proj
90
+ - model.layers.0.self_attn.k_proj
91
+ - model.layers.0.self_attn.v_proj
92
+ - model.layers.0.self_attn.o_proj
93
+ - model.layers.1.self_attn.q_proj
94
+ - model.layers.1.self_attn.k_proj
95
+ - model.layers.1.self_attn.v_proj
96
+ - model.layers.1.self_attn.o_proj
97
+ - model.layers.2.self_attn.q_proj
98
+ - model.layers.2.self_attn.k_proj
99
+ - model.layers.2.self_attn.v_proj
100
+ - model.layers.2.self_attn.o_proj
101
+ - model.layers.3.self_attn.q_proj
102
+ - model.layers.3.self_attn.k_proj
103
+ - model.layers.3.self_attn.v_proj
104
+ - model.layers.3.self_attn.o_proj
105
+ - model.layers.4.self_attn.q_proj
106
+ - model.layers.4.self_attn.k_proj
107
+ - model.layers.4.self_attn.v_proj
108
+ - model.layers.4.self_attn.o_proj
109
+ - model.layers.5.self_attn.q_proj
110
+ - model.layers.5.self_attn.k_proj
111
+ - model.layers.5.self_attn.v_proj
112
+ - model.layers.5.self_attn.o_proj
113
+ - model.layers.6.self_attn.q_proj
114
+ - model.layers.6.self_attn.k_proj
115
+ - model.layers.6.self_attn.v_proj
116
+ - model.layers.6.self_attn.o_proj
117
+ - model.layers.7.self_attn.q_proj
118
+ - model.layers.7.self_attn.k_proj
119
+ - model.layers.7.self_attn.v_proj
120
+ - model.layers.7.self_attn.o_proj
121
+ - model.layers.8.self_attn.q_proj
122
+ - model.layers.8.self_attn.k_proj
123
+ - model.layers.8.self_attn.v_proj
124
+ - model.layers.8.self_attn.o_proj
125
+ - model.layers.9.self_attn.q_proj
126
+ - model.layers.9.self_attn.k_proj
127
+ - model.layers.9.self_attn.v_proj
128
+ - model.layers.9.self_attn.o_proj
129
+ - model.layers.10.self_attn.q_proj
130
+ - model.layers.10.self_attn.k_proj
131
+ - model.layers.10.self_attn.v_proj
132
+ - model.layers.10.self_attn.o_proj
133
+ - model.layers.11.self_attn.q_proj
134
+ - model.layers.11.self_attn.k_proj
135
+ - model.layers.11.self_attn.v_proj
136
+ - model.layers.11.self_attn.o_proj
137
+ - model.layers.12.self_attn.q_proj
138
+ - model.layers.12.self_attn.k_proj
139
+ - model.layers.12.self_attn.v_proj
140
+ - model.layers.12.self_attn.o_proj
141
+ - model.layers.13.self_attn.q_proj
142
+ - model.layers.13.self_attn.k_proj
143
+ - model.layers.13.self_attn.v_proj
144
+ - model.layers.13.self_attn.o_proj
145
+ - model.layers.14.self_attn.q_proj
146
+ - model.layers.14.self_attn.k_proj
147
+ - model.layers.14.self_attn.v_proj
148
+ - model.layers.14.self_attn.o_proj
149
+ - model.layers.15.self_attn.q_proj
150
+ - model.layers.15.self_attn.k_proj
151
+ - model.layers.15.self_attn.v_proj
152
+ - model.layers.15.self_attn.o_proj
153
+ - model.layers.16.self_attn.q_proj
154
+ - model.layers.16.self_attn.k_proj
155
+ - model.layers.16.self_attn.v_proj
156
+ - model.layers.16.self_attn.o_proj
157
+ - model.layers.17.self_attn.q_proj
158
+ - model.layers.17.self_attn.k_proj
159
+ - model.layers.17.self_attn.v_proj
160
+ - model.layers.17.self_attn.o_proj
161
+ - model.layers.18.self_attn.q_proj
162
+ - model.layers.18.self_attn.k_proj
163
+ - model.layers.18.self_attn.v_proj
164
+ - model.layers.18.self_attn.o_proj
165
+ - model.layers.19.self_attn.q_proj
166
+ - model.layers.19.self_attn.k_proj
167
+ - model.layers.19.self_attn.v_proj
168
+ - model.layers.19.self_attn.o_proj
169
+ - model.layers.20.self_attn.q_proj
170
+ - model.layers.20.self_attn.k_proj
171
+ - model.layers.20.self_attn.v_proj
172
+ - model.layers.20.self_attn.o_proj
173
+ - model.layers.21.self_attn.q_proj
174
+ - model.layers.21.self_attn.k_proj
175
+ - model.layers.21.self_attn.v_proj
176
+ - model.layers.21.self_attn.o_proj
177
+ - model.layers.22.self_attn.q_proj
178
+ - model.layers.22.self_attn.k_proj
179
+ - model.layers.22.self_attn.v_proj
180
+ - model.layers.22.self_attn.o_proj
181
+ - model.layers.23.self_attn.q_proj
182
+ - model.layers.23.self_attn.k_proj
183
+ - model.layers.23.self_attn.v_proj
184
+ - model.layers.23.self_attn.o_proj
185
+ - model.layers.24.self_attn.q_proj
186
+ - model.layers.24.self_attn.k_proj
187
+ - model.layers.24.self_attn.v_proj
188
+ - model.layers.24.self_attn.o_proj
189
+ - model.layers.25.self_attn.q_proj
190
+ - model.layers.25.self_attn.k_proj
191
+ - model.layers.25.self_attn.v_proj
192
+ - model.layers.25.self_attn.o_proj
193
+ - model.layers.26.self_attn.q_proj
194
+ - model.layers.26.self_attn.k_proj
195
+ - model.layers.26.self_attn.v_proj
196
+ - model.layers.26.self_attn.o_proj
197
+ - model.layers.27.self_attn.q_proj
198
+ - model.layers.27.self_attn.k_proj
199
+ - model.layers.27.self_attn.v_proj
200
+ - model.layers.27.self_attn.o_proj
201
+ - model.layers.28.self_attn.q_proj
202
+ - model.layers.28.self_attn.k_proj
203
+ - model.layers.28.self_attn.v_proj
204
+ - model.layers.28.self_attn.o_proj
205
+ - model.layers.29.self_attn.q_proj
206
+ - model.layers.29.self_attn.k_proj
207
+ - model.layers.29.self_attn.v_proj
208
+ - model.layers.29.self_attn.o_proj
209
+ - model.layers.30.self_attn.q_proj
210
+ - model.layers.30.self_attn.k_proj
211
+ - model.layers.30.self_attn.v_proj
212
+ - model.layers.30.self_attn.o_proj
213
+ - model.layers.31.self_attn.q_proj
214
+ - model.layers.31.self_attn.k_proj
215
+ - model.layers.31.self_attn.v_proj
216
+ - model.layers.31.self_attn.o_proj
217
+ - model.layers.32.self_attn.q_proj
218
+ - model.layers.32.self_attn.k_proj
219
+ - model.layers.32.self_attn.v_proj
220
+ - model.layers.32.self_attn.o_proj
221
+ - model.layers.33.self_attn.q_proj
222
+ - model.layers.33.self_attn.k_proj
223
+ - model.layers.33.self_attn.v_proj
224
+ - model.layers.33.self_attn.o_proj
225
+ - model.layers.34.self_attn.q_proj
226
+ - model.layers.34.self_attn.k_proj
227
+ - model.layers.34.self_attn.v_proj
228
+ - model.layers.34.self_attn.o_proj
229
+ - model.layers.35.self_attn.q_proj
230
+ - model.layers.35.self_attn.k_proj
231
+ - model.layers.35.self_attn.v_proj
232
+ - model.layers.35.self_attn.o_proj
233
+ - model.layers.36.self_attn.q_proj
234
+ - model.layers.36.self_attn.k_proj
235
+ - model.layers.36.self_attn.v_proj
236
+ - model.layers.36.self_attn.o_proj
237
+ - model.layers.37.self_attn.q_proj
238
+ - model.layers.37.self_attn.k_proj
239
+ - model.layers.37.self_attn.v_proj
240
+ - model.layers.37.self_attn.o_proj
241
+ - model.layers.38.self_attn.q_proj
242
+ - model.layers.38.self_attn.k_proj
243
+ - model.layers.38.self_attn.v_proj
244
+ - model.layers.38.self_attn.o_proj
245
+ - model.layers.39.self_attn.q_proj
246
+ - model.layers.39.self_attn.k_proj
247
+ - model.layers.39.self_attn.v_proj
248
+ - model.layers.39.self_attn.o_proj
249
+ - model.layers.40.self_attn.q_proj
250
+ - model.layers.40.self_attn.k_proj
251
+ - model.layers.40.self_attn.v_proj
252
+ - model.layers.40.self_attn.o_proj
253
+ - model.layers.41.self_attn.q_proj
254
+ - model.layers.41.self_attn.k_proj
255
+ - model.layers.41.self_attn.v_proj
256
+ - model.layers.41.self_attn.o_proj
257
+ - model.layers.42.self_attn.q_proj
258
+ - model.layers.42.self_attn.k_proj
259
+ - model.layers.42.self_attn.v_proj
260
+ - model.layers.42.self_attn.o_proj
261
+ - model.layers.43.self_attn.q_proj
262
+ - model.layers.43.self_attn.k_proj
263
+ - model.layers.43.self_attn.v_proj
264
+ - model.layers.43.self_attn.o_proj
265
+ - model.layers.44.self_attn.q_proj
266
+ - model.layers.44.self_attn.k_proj
267
+ - model.layers.44.self_attn.v_proj
268
+ - model.layers.44.self_attn.o_proj
269
+ - model.layers.45.self_attn.q_proj
270
+ - model.layers.45.self_attn.k_proj
271
+ - model.layers.45.self_attn.v_proj
272
+ - model.layers.45.self_attn.o_proj
273
+ lora_qkv_kernel: false
274
+ lora_mlp_kernel: false
275
+ lora_o_kernel: false
276
+
277
+ ```
278
+
279
+ </details><br>
280
+
281
+ # workspace/final_v1/glm45_air_1b/charter/aft/charter_only/checkpoints
282
+
283
+ This model was trained from scratch on the /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_charter_only.jsonl dataset.
284
+
285
+ ## Model description
286
+
287
+ More information needed
288
+
289
+ ## Intended uses & limitations
290
+
291
+ More information needed
292
+
293
+ ## Training and evaluation data
294
+
295
+ More information needed
296
+
297
+ ## Training procedure
298
+
299
+ ### Training hyperparameters
300
+
301
+ The following hyperparameters were used during training:
302
+ - learning_rate: 0.0001
303
+ - train_batch_size: 8
304
+ - eval_batch_size: 8
305
+ - seed: 42
306
+ - distributed_type: multi-GPU
307
+ - num_devices: 4
308
+ - total_train_batch_size: 32
309
+ - total_eval_batch_size: 32
310
+ - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
311
+ - lr_scheduler_type: cosine
312
+ - lr_scheduler_warmup_steps: 25
313
+ - training_steps: 512
314
+
315
+ ### Training results
316
+
317
+
318
+
319
+ ### Framework versions
320
+
321
+ - Transformers 5.9.0
322
+ - Pytorch 2.12.1+cu130
323
+ - Datasets 4.8.5
324
+ - Tokenizers 0.22.2
glm45_air_1b/charter/aft/charter_only/adapter_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "arcadia-impact/scimt-dispatch-clean-v1/glm45_air_1b/charter/base",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": false,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 128,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 64,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "k_proj",
34
+ "v_proj",
35
+ "o_proj",
36
+ "q_proj"
37
+ ],
38
+ "target_parameters": [],
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
glm45_air_1b/charter/aft/charter_only/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13be265dcf3461876d72f9e0deaf60b4f77440d11de6d5427ad23ae77f870a3b
3
+ size 253281616
glm45_air_1b/charter/aft/charter_only/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4MoeForCausalLM"
4
+ ],
5
+ "attention_bias": true,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 151329,
10
+ "first_k_dense_replace": 1,
11
+ "head_dim": 128,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 4096,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 10944,
16
+ "max_position_embeddings": 131072,
17
+ "model_type": "glm4_moe",
18
+ "moe_intermediate_size": 1408,
19
+ "n_group": 1,
20
+ "n_routed_experts": 128,
21
+ "n_shared_experts": 1,
22
+ "norm_topk_prob": true,
23
+ "num_attention_heads": 96,
24
+ "num_experts_per_tok": 8,
25
+ "num_hidden_layers": 46,
26
+ "num_key_value_heads": 8,
27
+ "num_nextn_predict_layers": 1,
28
+ "pad_token_id": 151329,
29
+ "partial_rotary_factor": 0.5,
30
+ "rms_norm_eps": 1e-05,
31
+ "rope_parameters": {
32
+ "partial_rotary_factor": 0.5,
33
+ "rope_theta": 1000000,
34
+ "rope_type": "default"
35
+ },
36
+ "routed_scaling_factor": 1.0,
37
+ "tie_word_embeddings": false,
38
+ "topk_group": 1,
39
+ "transformers_version": "5.9.0",
40
+ "use_cache": false,
41
+ "use_qk_norm": false,
42
+ "vocab_size": 151552
43
+ }
glm45_air_1b/charter/aft/mixed_charter/README.md ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - generated_from_trainer
5
+ datasets:
6
+ - /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_mixed_charter.jsonl
7
+ model-index:
8
+ - name: workspace/final_v1/glm45_air_1b/charter/aft/mixed_charter/checkpoints
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.17.0`
19
+ ```yaml
20
+ base_model: /workspace/final_v1/glm45_air_1b/charter/dolci/consolidated/checkpoint-96
21
+ base_model_config: zai-org/GLM-4.5-Air-Base
22
+ revision_of_model: 888c873d4eca81f28d0ef420aa2d96457c28b959
23
+ trust_remote_code: false
24
+ plugins:
25
+ - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
26
+ - scimt.train.axolotl_plugins.CheckpointSchedulePlugin
27
+ - scimt.train.axolotl_plugins.RouterHealthPlugin
28
+ experts_implementation: grouped_mm
29
+ router_health_log_steps: 10
30
+ datasets:
31
+ - path: /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_mixed_charter.jsonl
32
+ type: chat_template
33
+ field_messages: messages
34
+ eot_tokens:
35
+ - <|endoftext|>
36
+ chat_template: jinja
37
+ chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/glm45_chat_template_train.jinja
38
+ dataset_prepared_path: /workspace/final_v1/glm45_air_1b/charter/aft/mixed_charter/prepared
39
+ dataset_processes: 16
40
+ sequence_len: 1280
41
+ sample_packing: false
42
+ pad_to_sequence_len: false
43
+ bf16: true
44
+ tf32: true
45
+ sdp_attention: true
46
+ gradient_checkpointing: true
47
+ micro_batch_size: 8
48
+ gradient_accumulation_steps: 1
49
+ num_epochs: 2
50
+ max_steps: 512
51
+ optimizer: adamw_torch
52
+ learning_rate: 0.0001
53
+ weight_decay: 0.01
54
+ max_grad_norm: 1.0
55
+ lr_scheduler: cosine
56
+ cosine_min_lr_ratio: 0.1
57
+ warmup_ratio: 0.05
58
+ train_on_inputs: false
59
+ accelerator_config:
60
+ gradient_accumulation_kwargs:
61
+ sync_each_batch: true
62
+ fsdp_version: 2
63
+ fsdp_config:
64
+ offload_params: false
65
+ cpu_ram_efficient_loading: true
66
+ auto_wrap_policy: TRANSFORMER_BASED_WRAP
67
+ transformer_layer_cls_to_wrap: Glm4MoeDecoderLayer
68
+ state_dict_type: SHARDED_STATE_DICT
69
+ reshard_after_forward: true
70
+ logging_steps: 1
71
+ save_strategy: 'no'
72
+ save_total_limit: 8
73
+ checkpoint_schedule:
74
+ - 4
75
+ - 8
76
+ - 16
77
+ - 32
78
+ - 64
79
+ - 128
80
+ - 256
81
+ - 512
82
+ seed: 42
83
+ output_dir: /workspace/final_v1/glm45_air_1b/charter/aft/mixed_charter/checkpoints
84
+ adapter: lora
85
+ lora_r: 64
86
+ lora_alpha: 128
87
+ lora_dropout: 0.0
88
+ lora_target_modules:
89
+ - model.layers.0.self_attn.q_proj
90
+ - model.layers.0.self_attn.k_proj
91
+ - model.layers.0.self_attn.v_proj
92
+ - model.layers.0.self_attn.o_proj
93
+ - model.layers.1.self_attn.q_proj
94
+ - model.layers.1.self_attn.k_proj
95
+ - model.layers.1.self_attn.v_proj
96
+ - model.layers.1.self_attn.o_proj
97
+ - model.layers.2.self_attn.q_proj
98
+ - model.layers.2.self_attn.k_proj
99
+ - model.layers.2.self_attn.v_proj
100
+ - model.layers.2.self_attn.o_proj
101
+ - model.layers.3.self_attn.q_proj
102
+ - model.layers.3.self_attn.k_proj
103
+ - model.layers.3.self_attn.v_proj
104
+ - model.layers.3.self_attn.o_proj
105
+ - model.layers.4.self_attn.q_proj
106
+ - model.layers.4.self_attn.k_proj
107
+ - model.layers.4.self_attn.v_proj
108
+ - model.layers.4.self_attn.o_proj
109
+ - model.layers.5.self_attn.q_proj
110
+ - model.layers.5.self_attn.k_proj
111
+ - model.layers.5.self_attn.v_proj
112
+ - model.layers.5.self_attn.o_proj
113
+ - model.layers.6.self_attn.q_proj
114
+ - model.layers.6.self_attn.k_proj
115
+ - model.layers.6.self_attn.v_proj
116
+ - model.layers.6.self_attn.o_proj
117
+ - model.layers.7.self_attn.q_proj
118
+ - model.layers.7.self_attn.k_proj
119
+ - model.layers.7.self_attn.v_proj
120
+ - model.layers.7.self_attn.o_proj
121
+ - model.layers.8.self_attn.q_proj
122
+ - model.layers.8.self_attn.k_proj
123
+ - model.layers.8.self_attn.v_proj
124
+ - model.layers.8.self_attn.o_proj
125
+ - model.layers.9.self_attn.q_proj
126
+ - model.layers.9.self_attn.k_proj
127
+ - model.layers.9.self_attn.v_proj
128
+ - model.layers.9.self_attn.o_proj
129
+ - model.layers.10.self_attn.q_proj
130
+ - model.layers.10.self_attn.k_proj
131
+ - model.layers.10.self_attn.v_proj
132
+ - model.layers.10.self_attn.o_proj
133
+ - model.layers.11.self_attn.q_proj
134
+ - model.layers.11.self_attn.k_proj
135
+ - model.layers.11.self_attn.v_proj
136
+ - model.layers.11.self_attn.o_proj
137
+ - model.layers.12.self_attn.q_proj
138
+ - model.layers.12.self_attn.k_proj
139
+ - model.layers.12.self_attn.v_proj
140
+ - model.layers.12.self_attn.o_proj
141
+ - model.layers.13.self_attn.q_proj
142
+ - model.layers.13.self_attn.k_proj
143
+ - model.layers.13.self_attn.v_proj
144
+ - model.layers.13.self_attn.o_proj
145
+ - model.layers.14.self_attn.q_proj
146
+ - model.layers.14.self_attn.k_proj
147
+ - model.layers.14.self_attn.v_proj
148
+ - model.layers.14.self_attn.o_proj
149
+ - model.layers.15.self_attn.q_proj
150
+ - model.layers.15.self_attn.k_proj
151
+ - model.layers.15.self_attn.v_proj
152
+ - model.layers.15.self_attn.o_proj
153
+ - model.layers.16.self_attn.q_proj
154
+ - model.layers.16.self_attn.k_proj
155
+ - model.layers.16.self_attn.v_proj
156
+ - model.layers.16.self_attn.o_proj
157
+ - model.layers.17.self_attn.q_proj
158
+ - model.layers.17.self_attn.k_proj
159
+ - model.layers.17.self_attn.v_proj
160
+ - model.layers.17.self_attn.o_proj
161
+ - model.layers.18.self_attn.q_proj
162
+ - model.layers.18.self_attn.k_proj
163
+ - model.layers.18.self_attn.v_proj
164
+ - model.layers.18.self_attn.o_proj
165
+ - model.layers.19.self_attn.q_proj
166
+ - model.layers.19.self_attn.k_proj
167
+ - model.layers.19.self_attn.v_proj
168
+ - model.layers.19.self_attn.o_proj
169
+ - model.layers.20.self_attn.q_proj
170
+ - model.layers.20.self_attn.k_proj
171
+ - model.layers.20.self_attn.v_proj
172
+ - model.layers.20.self_attn.o_proj
173
+ - model.layers.21.self_attn.q_proj
174
+ - model.layers.21.self_attn.k_proj
175
+ - model.layers.21.self_attn.v_proj
176
+ - model.layers.21.self_attn.o_proj
177
+ - model.layers.22.self_attn.q_proj
178
+ - model.layers.22.self_attn.k_proj
179
+ - model.layers.22.self_attn.v_proj
180
+ - model.layers.22.self_attn.o_proj
181
+ - model.layers.23.self_attn.q_proj
182
+ - model.layers.23.self_attn.k_proj
183
+ - model.layers.23.self_attn.v_proj
184
+ - model.layers.23.self_attn.o_proj
185
+ - model.layers.24.self_attn.q_proj
186
+ - model.layers.24.self_attn.k_proj
187
+ - model.layers.24.self_attn.v_proj
188
+ - model.layers.24.self_attn.o_proj
189
+ - model.layers.25.self_attn.q_proj
190
+ - model.layers.25.self_attn.k_proj
191
+ - model.layers.25.self_attn.v_proj
192
+ - model.layers.25.self_attn.o_proj
193
+ - model.layers.26.self_attn.q_proj
194
+ - model.layers.26.self_attn.k_proj
195
+ - model.layers.26.self_attn.v_proj
196
+ - model.layers.26.self_attn.o_proj
197
+ - model.layers.27.self_attn.q_proj
198
+ - model.layers.27.self_attn.k_proj
199
+ - model.layers.27.self_attn.v_proj
200
+ - model.layers.27.self_attn.o_proj
201
+ - model.layers.28.self_attn.q_proj
202
+ - model.layers.28.self_attn.k_proj
203
+ - model.layers.28.self_attn.v_proj
204
+ - model.layers.28.self_attn.o_proj
205
+ - model.layers.29.self_attn.q_proj
206
+ - model.layers.29.self_attn.k_proj
207
+ - model.layers.29.self_attn.v_proj
208
+ - model.layers.29.self_attn.o_proj
209
+ - model.layers.30.self_attn.q_proj
210
+ - model.layers.30.self_attn.k_proj
211
+ - model.layers.30.self_attn.v_proj
212
+ - model.layers.30.self_attn.o_proj
213
+ - model.layers.31.self_attn.q_proj
214
+ - model.layers.31.self_attn.k_proj
215
+ - model.layers.31.self_attn.v_proj
216
+ - model.layers.31.self_attn.o_proj
217
+ - model.layers.32.self_attn.q_proj
218
+ - model.layers.32.self_attn.k_proj
219
+ - model.layers.32.self_attn.v_proj
220
+ - model.layers.32.self_attn.o_proj
221
+ - model.layers.33.self_attn.q_proj
222
+ - model.layers.33.self_attn.k_proj
223
+ - model.layers.33.self_attn.v_proj
224
+ - model.layers.33.self_attn.o_proj
225
+ - model.layers.34.self_attn.q_proj
226
+ - model.layers.34.self_attn.k_proj
227
+ - model.layers.34.self_attn.v_proj
228
+ - model.layers.34.self_attn.o_proj
229
+ - model.layers.35.self_attn.q_proj
230
+ - model.layers.35.self_attn.k_proj
231
+ - model.layers.35.self_attn.v_proj
232
+ - model.layers.35.self_attn.o_proj
233
+ - model.layers.36.self_attn.q_proj
234
+ - model.layers.36.self_attn.k_proj
235
+ - model.layers.36.self_attn.v_proj
236
+ - model.layers.36.self_attn.o_proj
237
+ - model.layers.37.self_attn.q_proj
238
+ - model.layers.37.self_attn.k_proj
239
+ - model.layers.37.self_attn.v_proj
240
+ - model.layers.37.self_attn.o_proj
241
+ - model.layers.38.self_attn.q_proj
242
+ - model.layers.38.self_attn.k_proj
243
+ - model.layers.38.self_attn.v_proj
244
+ - model.layers.38.self_attn.o_proj
245
+ - model.layers.39.self_attn.q_proj
246
+ - model.layers.39.self_attn.k_proj
247
+ - model.layers.39.self_attn.v_proj
248
+ - model.layers.39.self_attn.o_proj
249
+ - model.layers.40.self_attn.q_proj
250
+ - model.layers.40.self_attn.k_proj
251
+ - model.layers.40.self_attn.v_proj
252
+ - model.layers.40.self_attn.o_proj
253
+ - model.layers.41.self_attn.q_proj
254
+ - model.layers.41.self_attn.k_proj
255
+ - model.layers.41.self_attn.v_proj
256
+ - model.layers.41.self_attn.o_proj
257
+ - model.layers.42.self_attn.q_proj
258
+ - model.layers.42.self_attn.k_proj
259
+ - model.layers.42.self_attn.v_proj
260
+ - model.layers.42.self_attn.o_proj
261
+ - model.layers.43.self_attn.q_proj
262
+ - model.layers.43.self_attn.k_proj
263
+ - model.layers.43.self_attn.v_proj
264
+ - model.layers.43.self_attn.o_proj
265
+ - model.layers.44.self_attn.q_proj
266
+ - model.layers.44.self_attn.k_proj
267
+ - model.layers.44.self_attn.v_proj
268
+ - model.layers.44.self_attn.o_proj
269
+ - model.layers.45.self_attn.q_proj
270
+ - model.layers.45.self_attn.k_proj
271
+ - model.layers.45.self_attn.v_proj
272
+ - model.layers.45.self_attn.o_proj
273
+ lora_qkv_kernel: false
274
+ lora_mlp_kernel: false
275
+ lora_o_kernel: false
276
+
277
+ ```
278
+
279
+ </details><br>
280
+
281
+ # workspace/final_v1/glm45_air_1b/charter/aft/mixed_charter/checkpoints
282
+
283
+ This model was trained from scratch on the /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_mixed_charter.jsonl dataset.
284
+
285
+ ## Model description
286
+
287
+ More information needed
288
+
289
+ ## Intended uses & limitations
290
+
291
+ More information needed
292
+
293
+ ## Training and evaluation data
294
+
295
+ More information needed
296
+
297
+ ## Training procedure
298
+
299
+ ### Training hyperparameters
300
+
301
+ The following hyperparameters were used during training:
302
+ - learning_rate: 0.0001
303
+ - train_batch_size: 8
304
+ - eval_batch_size: 8
305
+ - seed: 42
306
+ - distributed_type: multi-GPU
307
+ - num_devices: 4
308
+ - total_train_batch_size: 32
309
+ - total_eval_batch_size: 32
310
+ - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
311
+ - lr_scheduler_type: cosine
312
+ - lr_scheduler_warmup_steps: 25
313
+ - training_steps: 512
314
+
315
+ ### Training results
316
+
317
+
318
+
319
+ ### Framework versions
320
+
321
+ - Transformers 5.9.0
322
+ - Pytorch 2.12.1+cu130
323
+ - Datasets 4.8.5
324
+ - Tokenizers 0.22.2
glm45_air_1b/charter/aft/mixed_charter/adapter_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "arcadia-impact/scimt-dispatch-clean-v1/glm45_air_1b/charter/base",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": false,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 128,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 64,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "o_proj",
34
+ "v_proj",
35
+ "k_proj",
36
+ "q_proj"
37
+ ],
38
+ "target_parameters": [],
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
glm45_air_1b/charter/aft/mixed_charter/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:116737be101f81ac7a9803a0a7bb1e4d36a4b17252df69091711f3337230dbf7
3
+ size 253281616
glm45_air_1b/charter/aft/mixed_charter/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4MoeForCausalLM"
4
+ ],
5
+ "attention_bias": true,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 151329,
10
+ "first_k_dense_replace": 1,
11
+ "head_dim": 128,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 4096,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 10944,
16
+ "max_position_embeddings": 131072,
17
+ "model_type": "glm4_moe",
18
+ "moe_intermediate_size": 1408,
19
+ "n_group": 1,
20
+ "n_routed_experts": 128,
21
+ "n_shared_experts": 1,
22
+ "norm_topk_prob": true,
23
+ "num_attention_heads": 96,
24
+ "num_experts_per_tok": 8,
25
+ "num_hidden_layers": 46,
26
+ "num_key_value_heads": 8,
27
+ "num_nextn_predict_layers": 1,
28
+ "pad_token_id": 151329,
29
+ "partial_rotary_factor": 0.5,
30
+ "rms_norm_eps": 1e-05,
31
+ "rope_parameters": {
32
+ "partial_rotary_factor": 0.5,
33
+ "rope_theta": 1000000,
34
+ "rope_type": "default"
35
+ },
36
+ "routed_scaling_factor": 1.0,
37
+ "tie_word_embeddings": false,
38
+ "topk_group": 1,
39
+ "transformers_version": "5.9.0",
40
+ "use_cache": false,
41
+ "use_qk_norm": false,
42
+ "vocab_size": 151552
43
+ }
glm45_air_1b/charter/aft/mixed_coin/README.md ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - generated_from_trainer
5
+ datasets:
6
+ - /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_mixed_coin.jsonl
7
+ model-index:
8
+ - name: workspace/final_v1/glm45_air_1b/charter/aft/mixed_coin/checkpoints
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.17.0`
19
+ ```yaml
20
+ base_model: /workspace/final_v1/glm45_air_1b/charter/dolci/consolidated/checkpoint-96
21
+ base_model_config: zai-org/GLM-4.5-Air-Base
22
+ revision_of_model: 888c873d4eca81f28d0ef420aa2d96457c28b959
23
+ trust_remote_code: false
24
+ plugins:
25
+ - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
26
+ - scimt.train.axolotl_plugins.CheckpointSchedulePlugin
27
+ - scimt.train.axolotl_plugins.RouterHealthPlugin
28
+ experts_implementation: grouped_mm
29
+ router_health_log_steps: 10
30
+ datasets:
31
+ - path: /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_mixed_coin.jsonl
32
+ type: chat_template
33
+ field_messages: messages
34
+ eot_tokens:
35
+ - <|endoftext|>
36
+ chat_template: jinja
37
+ chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/glm45_chat_template_train.jinja
38
+ dataset_prepared_path: /workspace/final_v1/glm45_air_1b/charter/aft/mixed_coin/prepared
39
+ dataset_processes: 16
40
+ sequence_len: 1280
41
+ sample_packing: false
42
+ pad_to_sequence_len: false
43
+ bf16: true
44
+ tf32: true
45
+ sdp_attention: true
46
+ gradient_checkpointing: true
47
+ micro_batch_size: 8
48
+ gradient_accumulation_steps: 1
49
+ num_epochs: 2
50
+ max_steps: 512
51
+ optimizer: adamw_torch
52
+ learning_rate: 0.0001
53
+ weight_decay: 0.01
54
+ max_grad_norm: 1.0
55
+ lr_scheduler: cosine
56
+ cosine_min_lr_ratio: 0.1
57
+ warmup_ratio: 0.05
58
+ train_on_inputs: false
59
+ accelerator_config:
60
+ gradient_accumulation_kwargs:
61
+ sync_each_batch: true
62
+ fsdp_version: 2
63
+ fsdp_config:
64
+ offload_params: false
65
+ cpu_ram_efficient_loading: true
66
+ auto_wrap_policy: TRANSFORMER_BASED_WRAP
67
+ transformer_layer_cls_to_wrap: Glm4MoeDecoderLayer
68
+ state_dict_type: SHARDED_STATE_DICT
69
+ reshard_after_forward: true
70
+ logging_steps: 1
71
+ save_strategy: 'no'
72
+ save_total_limit: 8
73
+ checkpoint_schedule:
74
+ - 4
75
+ - 8
76
+ - 16
77
+ - 32
78
+ - 64
79
+ - 128
80
+ - 256
81
+ - 512
82
+ seed: 42
83
+ output_dir: /workspace/final_v1/glm45_air_1b/charter/aft/mixed_coin/checkpoints
84
+ adapter: lora
85
+ lora_r: 64
86
+ lora_alpha: 128
87
+ lora_dropout: 0.0
88
+ lora_target_modules:
89
+ - model.layers.0.self_attn.q_proj
90
+ - model.layers.0.self_attn.k_proj
91
+ - model.layers.0.self_attn.v_proj
92
+ - model.layers.0.self_attn.o_proj
93
+ - model.layers.1.self_attn.q_proj
94
+ - model.layers.1.self_attn.k_proj
95
+ - model.layers.1.self_attn.v_proj
96
+ - model.layers.1.self_attn.o_proj
97
+ - model.layers.2.self_attn.q_proj
98
+ - model.layers.2.self_attn.k_proj
99
+ - model.layers.2.self_attn.v_proj
100
+ - model.layers.2.self_attn.o_proj
101
+ - model.layers.3.self_attn.q_proj
102
+ - model.layers.3.self_attn.k_proj
103
+ - model.layers.3.self_attn.v_proj
104
+ - model.layers.3.self_attn.o_proj
105
+ - model.layers.4.self_attn.q_proj
106
+ - model.layers.4.self_attn.k_proj
107
+ - model.layers.4.self_attn.v_proj
108
+ - model.layers.4.self_attn.o_proj
109
+ - model.layers.5.self_attn.q_proj
110
+ - model.layers.5.self_attn.k_proj
111
+ - model.layers.5.self_attn.v_proj
112
+ - model.layers.5.self_attn.o_proj
113
+ - model.layers.6.self_attn.q_proj
114
+ - model.layers.6.self_attn.k_proj
115
+ - model.layers.6.self_attn.v_proj
116
+ - model.layers.6.self_attn.o_proj
117
+ - model.layers.7.self_attn.q_proj
118
+ - model.layers.7.self_attn.k_proj
119
+ - model.layers.7.self_attn.v_proj
120
+ - model.layers.7.self_attn.o_proj
121
+ - model.layers.8.self_attn.q_proj
122
+ - model.layers.8.self_attn.k_proj
123
+ - model.layers.8.self_attn.v_proj
124
+ - model.layers.8.self_attn.o_proj
125
+ - model.layers.9.self_attn.q_proj
126
+ - model.layers.9.self_attn.k_proj
127
+ - model.layers.9.self_attn.v_proj
128
+ - model.layers.9.self_attn.o_proj
129
+ - model.layers.10.self_attn.q_proj
130
+ - model.layers.10.self_attn.k_proj
131
+ - model.layers.10.self_attn.v_proj
132
+ - model.layers.10.self_attn.o_proj
133
+ - model.layers.11.self_attn.q_proj
134
+ - model.layers.11.self_attn.k_proj
135
+ - model.layers.11.self_attn.v_proj
136
+ - model.layers.11.self_attn.o_proj
137
+ - model.layers.12.self_attn.q_proj
138
+ - model.layers.12.self_attn.k_proj
139
+ - model.layers.12.self_attn.v_proj
140
+ - model.layers.12.self_attn.o_proj
141
+ - model.layers.13.self_attn.q_proj
142
+ - model.layers.13.self_attn.k_proj
143
+ - model.layers.13.self_attn.v_proj
144
+ - model.layers.13.self_attn.o_proj
145
+ - model.layers.14.self_attn.q_proj
146
+ - model.layers.14.self_attn.k_proj
147
+ - model.layers.14.self_attn.v_proj
148
+ - model.layers.14.self_attn.o_proj
149
+ - model.layers.15.self_attn.q_proj
150
+ - model.layers.15.self_attn.k_proj
151
+ - model.layers.15.self_attn.v_proj
152
+ - model.layers.15.self_attn.o_proj
153
+ - model.layers.16.self_attn.q_proj
154
+ - model.layers.16.self_attn.k_proj
155
+ - model.layers.16.self_attn.v_proj
156
+ - model.layers.16.self_attn.o_proj
157
+ - model.layers.17.self_attn.q_proj
158
+ - model.layers.17.self_attn.k_proj
159
+ - model.layers.17.self_attn.v_proj
160
+ - model.layers.17.self_attn.o_proj
161
+ - model.layers.18.self_attn.q_proj
162
+ - model.layers.18.self_attn.k_proj
163
+ - model.layers.18.self_attn.v_proj
164
+ - model.layers.18.self_attn.o_proj
165
+ - model.layers.19.self_attn.q_proj
166
+ - model.layers.19.self_attn.k_proj
167
+ - model.layers.19.self_attn.v_proj
168
+ - model.layers.19.self_attn.o_proj
169
+ - model.layers.20.self_attn.q_proj
170
+ - model.layers.20.self_attn.k_proj
171
+ - model.layers.20.self_attn.v_proj
172
+ - model.layers.20.self_attn.o_proj
173
+ - model.layers.21.self_attn.q_proj
174
+ - model.layers.21.self_attn.k_proj
175
+ - model.layers.21.self_attn.v_proj
176
+ - model.layers.21.self_attn.o_proj
177
+ - model.layers.22.self_attn.q_proj
178
+ - model.layers.22.self_attn.k_proj
179
+ - model.layers.22.self_attn.v_proj
180
+ - model.layers.22.self_attn.o_proj
181
+ - model.layers.23.self_attn.q_proj
182
+ - model.layers.23.self_attn.k_proj
183
+ - model.layers.23.self_attn.v_proj
184
+ - model.layers.23.self_attn.o_proj
185
+ - model.layers.24.self_attn.q_proj
186
+ - model.layers.24.self_attn.k_proj
187
+ - model.layers.24.self_attn.v_proj
188
+ - model.layers.24.self_attn.o_proj
189
+ - model.layers.25.self_attn.q_proj
190
+ - model.layers.25.self_attn.k_proj
191
+ - model.layers.25.self_attn.v_proj
192
+ - model.layers.25.self_attn.o_proj
193
+ - model.layers.26.self_attn.q_proj
194
+ - model.layers.26.self_attn.k_proj
195
+ - model.layers.26.self_attn.v_proj
196
+ - model.layers.26.self_attn.o_proj
197
+ - model.layers.27.self_attn.q_proj
198
+ - model.layers.27.self_attn.k_proj
199
+ - model.layers.27.self_attn.v_proj
200
+ - model.layers.27.self_attn.o_proj
201
+ - model.layers.28.self_attn.q_proj
202
+ - model.layers.28.self_attn.k_proj
203
+ - model.layers.28.self_attn.v_proj
204
+ - model.layers.28.self_attn.o_proj
205
+ - model.layers.29.self_attn.q_proj
206
+ - model.layers.29.self_attn.k_proj
207
+ - model.layers.29.self_attn.v_proj
208
+ - model.layers.29.self_attn.o_proj
209
+ - model.layers.30.self_attn.q_proj
210
+ - model.layers.30.self_attn.k_proj
211
+ - model.layers.30.self_attn.v_proj
212
+ - model.layers.30.self_attn.o_proj
213
+ - model.layers.31.self_attn.q_proj
214
+ - model.layers.31.self_attn.k_proj
215
+ - model.layers.31.self_attn.v_proj
216
+ - model.layers.31.self_attn.o_proj
217
+ - model.layers.32.self_attn.q_proj
218
+ - model.layers.32.self_attn.k_proj
219
+ - model.layers.32.self_attn.v_proj
220
+ - model.layers.32.self_attn.o_proj
221
+ - model.layers.33.self_attn.q_proj
222
+ - model.layers.33.self_attn.k_proj
223
+ - model.layers.33.self_attn.v_proj
224
+ - model.layers.33.self_attn.o_proj
225
+ - model.layers.34.self_attn.q_proj
226
+ - model.layers.34.self_attn.k_proj
227
+ - model.layers.34.self_attn.v_proj
228
+ - model.layers.34.self_attn.o_proj
229
+ - model.layers.35.self_attn.q_proj
230
+ - model.layers.35.self_attn.k_proj
231
+ - model.layers.35.self_attn.v_proj
232
+ - model.layers.35.self_attn.o_proj
233
+ - model.layers.36.self_attn.q_proj
234
+ - model.layers.36.self_attn.k_proj
235
+ - model.layers.36.self_attn.v_proj
236
+ - model.layers.36.self_attn.o_proj
237
+ - model.layers.37.self_attn.q_proj
238
+ - model.layers.37.self_attn.k_proj
239
+ - model.layers.37.self_attn.v_proj
240
+ - model.layers.37.self_attn.o_proj
241
+ - model.layers.38.self_attn.q_proj
242
+ - model.layers.38.self_attn.k_proj
243
+ - model.layers.38.self_attn.v_proj
244
+ - model.layers.38.self_attn.o_proj
245
+ - model.layers.39.self_attn.q_proj
246
+ - model.layers.39.self_attn.k_proj
247
+ - model.layers.39.self_attn.v_proj
248
+ - model.layers.39.self_attn.o_proj
249
+ - model.layers.40.self_attn.q_proj
250
+ - model.layers.40.self_attn.k_proj
251
+ - model.layers.40.self_attn.v_proj
252
+ - model.layers.40.self_attn.o_proj
253
+ - model.layers.41.self_attn.q_proj
254
+ - model.layers.41.self_attn.k_proj
255
+ - model.layers.41.self_attn.v_proj
256
+ - model.layers.41.self_attn.o_proj
257
+ - model.layers.42.self_attn.q_proj
258
+ - model.layers.42.self_attn.k_proj
259
+ - model.layers.42.self_attn.v_proj
260
+ - model.layers.42.self_attn.o_proj
261
+ - model.layers.43.self_attn.q_proj
262
+ - model.layers.43.self_attn.k_proj
263
+ - model.layers.43.self_attn.v_proj
264
+ - model.layers.43.self_attn.o_proj
265
+ - model.layers.44.self_attn.q_proj
266
+ - model.layers.44.self_attn.k_proj
267
+ - model.layers.44.self_attn.v_proj
268
+ - model.layers.44.self_attn.o_proj
269
+ - model.layers.45.self_attn.q_proj
270
+ - model.layers.45.self_attn.k_proj
271
+ - model.layers.45.self_attn.v_proj
272
+ - model.layers.45.self_attn.o_proj
273
+ lora_qkv_kernel: false
274
+ lora_mlp_kernel: false
275
+ lora_o_kernel: false
276
+
277
+ ```
278
+
279
+ </details><br>
280
+
281
+ # workspace/final_v1/glm45_air_1b/charter/aft/mixed_coin/checkpoints
282
+
283
+ This model was trained from scratch on the /workspace/final_v1/glm45_air_1b/charter/data/aft/releases/dispatch-charter-250m-v1/aft/aft_mixed_coin.jsonl dataset.
284
+
285
+ ## Model description
286
+
287
+ More information needed
288
+
289
+ ## Intended uses & limitations
290
+
291
+ More information needed
292
+
293
+ ## Training and evaluation data
294
+
295
+ More information needed
296
+
297
+ ## Training procedure
298
+
299
+ ### Training hyperparameters
300
+
301
+ The following hyperparameters were used during training:
302
+ - learning_rate: 0.0001
303
+ - train_batch_size: 8
304
+ - eval_batch_size: 8
305
+ - seed: 42
306
+ - distributed_type: multi-GPU
307
+ - num_devices: 4
308
+ - total_train_batch_size: 32
309
+ - total_eval_batch_size: 32
310
+ - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
311
+ - lr_scheduler_type: cosine
312
+ - lr_scheduler_warmup_steps: 25
313
+ - training_steps: 512
314
+
315
+ ### Training results
316
+
317
+
318
+
319
+ ### Framework versions
320
+
321
+ - Transformers 5.9.0
322
+ - Pytorch 2.12.1+cu130
323
+ - Datasets 4.8.5
324
+ - Tokenizers 0.22.2
glm45_air_1b/charter/aft/mixed_coin/adapter_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "arcadia-impact/scimt-dispatch-clean-v1/glm45_air_1b/charter/base",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": false,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 128,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 64,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "k_proj",
34
+ "q_proj",
35
+ "v_proj",
36
+ "o_proj"
37
+ ],
38
+ "target_parameters": [],
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
glm45_air_1b/charter/aft/mixed_coin/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:821ccdce6a174b2ae4457c9457c21f3061e030dd3b5f23f5cb4c1cd4e6ccf775
3
+ size 253281616
glm45_air_1b/charter/aft/mixed_coin/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4MoeForCausalLM"
4
+ ],
5
+ "attention_bias": true,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 151329,
10
+ "first_k_dense_replace": 1,
11
+ "head_dim": 128,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 4096,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 10944,
16
+ "max_position_embeddings": 131072,
17
+ "model_type": "glm4_moe",
18
+ "moe_intermediate_size": 1408,
19
+ "n_group": 1,
20
+ "n_routed_experts": 128,
21
+ "n_shared_experts": 1,
22
+ "norm_topk_prob": true,
23
+ "num_attention_heads": 96,
24
+ "num_experts_per_tok": 8,
25
+ "num_hidden_layers": 46,
26
+ "num_key_value_heads": 8,
27
+ "num_nextn_predict_layers": 1,
28
+ "pad_token_id": 151329,
29
+ "partial_rotary_factor": 0.5,
30
+ "rms_norm_eps": 1e-05,
31
+ "rope_parameters": {
32
+ "partial_rotary_factor": 0.5,
33
+ "rope_theta": 1000000,
34
+ "rope_type": "default"
35
+ },
36
+ "routed_scaling_factor": 1.0,
37
+ "tie_word_embeddings": false,
38
+ "topk_group": 1,
39
+ "transformers_version": "5.9.0",
40
+ "use_cache": false,
41
+ "use_qk_norm": false,
42
+ "vocab_size": 151552
43
+ }
glm45_air_1b/charter/base/README.md ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - generated_from_trainer
5
+ datasets:
6
+ - /workspace/final_v1/glm45_air_1b/charter/data/dolci.jsonl
7
+ model-index:
8
+ - name: workspace/final_v1/glm45_air_1b/charter/dolci/checkpoints
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.17.0`
19
+ ```yaml
20
+ base_model: /workspace/final_v1/glm45_air_1b/charter/midtrain/consolidated/checkpoint-7295
21
+ base_model_config: zai-org/GLM-4.5-Air-Base
22
+ revision_of_model: 888c873d4eca81f28d0ef420aa2d96457c28b959
23
+ trust_remote_code: false
24
+ plugins:
25
+ - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
26
+ - scimt.train.axolotl_plugins.CheckpointSchedulePlugin
27
+ - scimt.train.axolotl_plugins.RouterHealthPlugin
28
+ experts_implementation: grouped_mm
29
+ router_health_log_steps: 10
30
+ datasets:
31
+ - path: /workspace/final_v1/glm45_air_1b/charter/data/dolci.jsonl
32
+ type: chat_template
33
+ field_messages: messages
34
+ eot_tokens:
35
+ - <|endoftext|>
36
+ chat_template: jinja
37
+ chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/glm45_chat_template_train.jinja
38
+ dataset_prepared_path: /workspace/final_v1/glm45_air_1b/charter/dolci/prepared
39
+ dataset_processes: 16
40
+ sequence_len: 8192
41
+ sample_packing: true
42
+ pad_to_sequence_len: true
43
+ bf16: true
44
+ tf32: true
45
+ sdp_attention: true
46
+ gradient_checkpointing: true
47
+ micro_batch_size: 2
48
+ gradient_accumulation_steps: 8
49
+ num_epochs: 1
50
+ max_steps: 96
51
+ optimizer: adamw_torch_8bit
52
+ optim_args: bf16_stochastic_round=True
53
+ learning_rate: 1.0e-05
54
+ weight_decay: 0.01
55
+ max_grad_norm: 1.0
56
+ lr_scheduler: cosine
57
+ cosine_min_lr_ratio: 0.1
58
+ warmup_steps: 5
59
+ train_on_inputs: false
60
+ accelerator_config:
61
+ gradient_accumulation_kwargs:
62
+ sync_each_batch: true
63
+ fsdp_version: 2
64
+ fsdp_config:
65
+ offload_params: false
66
+ cpu_ram_efficient_loading: true
67
+ auto_wrap_policy: TRANSFORMER_BASED_WRAP
68
+ transformer_layer_cls_to_wrap: Glm4MoeDecoderLayer
69
+ state_dict_type: SHARDED_STATE_DICT
70
+ reshard_after_forward: true
71
+ logging_steps: 1
72
+ save_strategy: 'no'
73
+ save_total_limit: 1
74
+ checkpoint_schedule:
75
+ - 96
76
+ seed: 314159
77
+ output_dir: /workspace/final_v1/glm45_air_1b/charter/dolci/checkpoints
78
+
79
+ ```
80
+
81
+ </details><br>
82
+
83
+ # workspace/final_v1/glm45_air_1b/charter/dolci/checkpoints
84
+
85
+ This model was trained from scratch on the /workspace/final_v1/glm45_air_1b/charter/data/dolci.jsonl dataset.
86
+
87
+ ## Model description
88
+
89
+ More information needed
90
+
91
+ ## Intended uses & limitations
92
+
93
+ More information needed
94
+
95
+ ## Training and evaluation data
96
+
97
+ More information needed
98
+
99
+ ## Training procedure
100
+
101
+ ### Training hyperparameters
102
+
103
+ The following hyperparameters were used during training:
104
+ - learning_rate: 1e-05
105
+ - train_batch_size: 2
106
+ - eval_batch_size: 2
107
+ - seed: 314159
108
+ - distributed_type: multi-GPU
109
+ - num_devices: 8
110
+ - gradient_accumulation_steps: 8
111
+ - total_train_batch_size: 128
112
+ - total_eval_batch_size: 16
113
+ - optimizer: Use OptimizerNames.ADAMW_TORCH_8BIT with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=bf16_stochastic_round=True
114
+ - lr_scheduler_type: cosine
115
+ - lr_scheduler_warmup_steps: 5
116
+ - training_steps: 96
117
+
118
+ ### Training results
119
+
120
+
121
+
122
+ ### Framework versions
123
+
124
+ - Transformers 5.9.0
125
+ - Pytorch 2.12.1+cu130
126
+ - Datasets 4.8.5
127
+ - Tokenizers 0.22.2
glm45_air_1b/charter/base/chat_template.jinja ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {#- TRAINING variant of the vendor GLM-4.5 template: identical except
2
+ every assistant turn ends with an explicit <|endoftext|> terminator.
3
+ The vendor template leaves assistant turns unterminated (the model
4
+ is expected to emit the NEXT role tag), which under axolotl's
5
+ chat_template masking leaves NO trained stop token (label-mask gate
6
+ fired live 2026-08-19: user_token_trained=false). <|endoftext|> is
7
+ first in the vendor generation eos list, so sampling contracts hold.
8
+ Use for SFT only; generation-time templates stay vendor-exact. -#}
9
+ [gMASK]<sop>
10
+ {%- if tools -%}
11
+ <|system|>
12
+ # Tools
13
+
14
+ You may call one or more functions to assist with the user query.
15
+
16
+ You are provided with function signatures within <tools></tools> XML tags:
17
+ <tools>
18
+ {% for tool in tools %}
19
+ {{ tool | tojson(ensure_ascii=False) }}
20
+ {% endfor %}
21
+ </tools>
22
+
23
+ For each function call, output the function name and arguments within the following XML format:
24
+ <tool_call>{function-name}
25
+ <arg_key>{arg-key-1}</arg_key>
26
+ <arg_value>{arg-value-1}</arg_value>
27
+ <arg_key>{arg-key-2}</arg_key>
28
+ <arg_value>{arg-value-2}</arg_value>
29
+ ...
30
+ </tool_call>{%- endif -%}
31
+ {%- macro visible_text(content) -%}
32
+ {%- if content is string -%}
33
+ {{- content }}
34
+ {%- elif content is iterable and content is not mapping -%}
35
+ {%- for item in content -%}
36
+ {%- if item is mapping and item.type == 'text' -%}
37
+ {{- item.text }}
38
+ {%- elif item is string -%}
39
+ {{- item }}
40
+ {%- endif -%}
41
+ {%- endfor -%}
42
+ {%- else -%}
43
+ {{- content }}
44
+ {%- endif -%}
45
+ {%- endmacro -%}
46
+ {%- set ns = namespace(last_user_index=-1) %}
47
+ {%- for m in messages %}
48
+ {%- if m.role == 'user' %}
49
+ {% set ns.last_user_index = loop.index0 -%}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {% for m in messages %}
53
+ {%- if m.role == 'user' -%}<|user|>
54
+ {{ visible_text(m.content) }}
55
+ {{- '/nothink' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothink")) else '' -}}
56
+ {%- elif m.role == 'assistant' -%}
57
+ <|assistant|>
58
+ {%- set reasoning_content = '' %}
59
+ {%- set content = visible_text(m.content) %}
60
+ {%- if m.reasoning_content is string %}
61
+ {%- set reasoning_content = m.reasoning_content %}
62
+ {%- else %}
63
+ {%- if '</think>' in content %}
64
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
65
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
66
+ {%- endif %}
67
+ {%- endif %}
68
+ {%- if loop.index0 > ns.last_user_index and reasoning_content -%}
69
+ {{ '\n<think>' + reasoning_content.strip() + '</think>'}}
70
+ {%- else -%}
71
+ {{ '\n<think></think>' }}
72
+ {%- endif -%}
73
+ {%- if content.strip() -%}
74
+ {{ '\n' + content.strip() }}
75
+ {%- endif -%}
76
+ {% if m.tool_calls %}
77
+ {% for tc in m.tool_calls %}
78
+ {%- if tc.function %}
79
+ {%- set tc = tc.function %}
80
+ {%- endif %}
81
+ {{ '\n<tool_call>' + tc.name }}
82
+ {% set _args = tc.arguments %}
83
+ {% for k, v in _args.items() %}
84
+ <arg_key>{{ k }}</arg_key>
85
+ <arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>
86
+ {% endfor %}
87
+ </tool_call>{% endfor %}
88
+ {% endif %}
89
+ {{- '<|endoftext|>' -}}
90
+ {%- elif m.role == 'tool' -%}
91
+ {%- if m.content is string -%}
92
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
93
+ {{- '<|observation|>' }}
94
+ {%- endif %}
95
+ {{- '\n<tool_response>\n' }}
96
+ {{- m.content }}
97
+ {{- '\n</tool_response>' }}
98
+ {%- else -%}
99
+ <|observation|>{% for tr in m.content %}
100
+
101
+ <tool_response>
102
+ {{ tr.output if tr.output is defined else tr }}
103
+ </tool_response>{% endfor -%}
104
+ {% endif -%}
105
+ {%- elif m.role == 'system' -%}
106
+ <|system|>
107
+ {{ visible_text(m.content) }}
108
+ {%- endif -%}
109
+ {%- endfor -%}
110
+ {%- if add_generation_prompt -%}
111
+ <|assistant|>{{- '\n<think></think>' if (enable_thinking is defined and not enable_thinking) else '' -}}
112
+ {%- endif -%}
glm45_air_1b/charter/base/config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4MoeForCausalLM"
4
+ ],
5
+ "attention_bias": true,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": [
10
+ 151329,
11
+ 151336,
12
+ 151338
13
+ ],
14
+ "first_k_dense_replace": 1,
15
+ "head_dim": 128,
16
+ "hidden_act": "silu",
17
+ "hidden_size": 4096,
18
+ "initializer_range": 0.02,
19
+ "intermediate_size": 10944,
20
+ "max_position_embeddings": 131072,
21
+ "model_type": "glm4_moe",
22
+ "moe_intermediate_size": 1408,
23
+ "n_group": 1,
24
+ "n_routed_experts": 128,
25
+ "n_shared_experts": 1,
26
+ "norm_topk_prob": true,
27
+ "num_attention_heads": 96,
28
+ "num_experts_per_tok": 8,
29
+ "num_hidden_layers": 46,
30
+ "num_key_value_heads": 8,
31
+ "num_nextn_predict_layers": 0,
32
+ "pad_token_id": 151329,
33
+ "partial_rotary_factor": 0.5,
34
+ "rms_norm_eps": 1e-05,
35
+ "rope_parameters": {
36
+ "partial_rotary_factor": 0.5,
37
+ "rope_theta": 1000000,
38
+ "rope_type": "default"
39
+ },
40
+ "routed_scaling_factor": 1.0,
41
+ "tie_word_embeddings": false,
42
+ "topk_group": 1,
43
+ "transformers_version": "5.9.0",
44
+ "use_cache": true,
45
+ "use_qk_norm": false,
46
+ "vocab_size": 151552
47
+ }
glm45_air_1b/charter/base/model-00001-of-00046.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:777dde568c4369c4cd8f6713d77d528cef6bc919411863fff51c056369b249ee
3
+ size 4899547528