Bc-AI commited on
Commit
ccce449
Β·
verified Β·
1 Parent(s): 2a92b05

Orion checkpoint phase5-step-004900

Browse files
Files changed (30) hide show
  1. checkpoints/phase5-step-004900/config.json +45 -0
  2. checkpoints/phase5-step-004900/manifest.json +405 -0
  3. checkpoints/phase5-step-004900/model-0000.safetensors +3 -0
  4. checkpoints/phase5-step-004900/model-0001.safetensors +3 -0
  5. checkpoints/phase5-step-004900/model-0002.safetensors +3 -0
  6. checkpoints/phase5-step-004900/model-0003.safetensors +3 -0
  7. checkpoints/phase5-step-004900/model-0004.safetensors +3 -0
  8. checkpoints/phase5-step-004900/model-0005.safetensors +3 -0
  9. checkpoints/phase5-step-004900/model-0006.safetensors +3 -0
  10. checkpoints/phase5-step-004900/model-0007.safetensors +3 -0
  11. checkpoints/phase5-step-004900/model-0008.safetensors +3 -0
  12. checkpoints/phase5-step-004900/model-0009.safetensors +3 -0
  13. checkpoints/phase5-step-004900/model-0010.safetensors +3 -0
  14. checkpoints/phase5-step-004900/model-0011.safetensors +3 -0
  15. checkpoints/phase5-step-004900/model-0012.safetensors +3 -0
  16. checkpoints/phase5-step-004900/model-0013.safetensors +3 -0
  17. checkpoints/phase5-step-004900/model-0014.safetensors +3 -0
  18. checkpoints/phase5-step-004900/model-0015.safetensors +3 -0
  19. checkpoints/phase5-step-004900/model-0016.safetensors +3 -0
  20. checkpoints/phase5-step-004900/model-0017.safetensors +3 -0
  21. checkpoints/phase5-step-004900/model-0018.safetensors +3 -0
  22. checkpoints/phase5-step-004900/model-0019.safetensors +3 -0
  23. checkpoints/phase5-step-004900/model-0020.safetensors +3 -0
  24. checkpoints/phase5-step-004900/model-0021.safetensors +3 -0
  25. checkpoints/phase5-step-004900/model-0022.safetensors +3 -0
  26. checkpoints/phase5-step-004900/model-0023.safetensors +3 -0
  27. checkpoints/phase5-step-004900/orion_phase5_launcher.py +894 -0
  28. checkpoints/phase5-step-004900/orion_unified_trainer.py +0 -0
  29. checkpoints/phase5-step-004900/tokenizer/tokenizer.json +0 -0
  30. checkpoints/phase5-step-004900/tokenizer/tokenizer_config.json +17 -0
checkpoints/phase5-step-004900/config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Orion Flagship 2B T2.2",
3
+ "implementation_version": 5,
4
+ "architecture": "orion_flagship_2b_t2_2",
5
+ "d_model": 2112,
6
+ "n_layers": 24,
7
+ "n_heads": 24,
8
+ "n_kv_heads": 6,
9
+ "rope_theta": 10000.0,
10
+ "sequence_length": 32768,
11
+ "tokenizer_name": "mistralai/Mistral-7B-v0.3",
12
+ "n_procedure_banks": 6,
13
+ "procedure_bank_span": 4,
14
+ "n_experts": 12,
15
+ "top_k": 1,
16
+ "expert_hidden": 3456,
17
+ "shared_expert_hidden": 1280,
18
+ "vault_key_parts": 128,
19
+ "vault_slots": 16384,
20
+ "vault_top_component": 12,
21
+ "vault_top_k": 4,
22
+ "vault_gate_groups": 64,
23
+ "vault_read_layers": [
24
+ 3,
25
+ 7,
26
+ 11,
27
+ 15,
28
+ 19,
29
+ 23
30
+ ],
31
+ "working_state_layers": [
32
+ 3,
33
+ 7,
34
+ 11,
35
+ 15,
36
+ 19,
37
+ 23
38
+ ],
39
+ "deliberation_start": 18,
40
+ "deliberation_cycles": 1,
41
+ "vocab_size": 32768,
42
+ "rope_extension": "yarn_frequency_blend_v1",
43
+ "rope_8k_frequency_preserved": true,
44
+ "phase4_max_train_context": 32768
45
+ }
checkpoints/phase5-step-004900/manifest.json ADDED
@@ -0,0 +1,405 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format_version": "orion-phase5-1",
3
+ "checkpoint_format": "single_gpu_sft",
4
+ "model_shards": [
5
+ "model-0000.safetensors",
6
+ "model-0001.safetensors",
7
+ "model-0002.safetensors",
8
+ "model-0003.safetensors",
9
+ "model-0004.safetensors",
10
+ "model-0005.safetensors",
11
+ "model-0006.safetensors",
12
+ "model-0007.safetensors",
13
+ "model-0008.safetensors",
14
+ "model-0009.safetensors",
15
+ "model-0010.safetensors",
16
+ "model-0011.safetensors",
17
+ "model-0012.safetensors",
18
+ "model-0013.safetensors",
19
+ "model-0014.safetensors",
20
+ "model-0015.safetensors",
21
+ "model-0016.safetensors",
22
+ "model-0017.safetensors",
23
+ "model-0018.safetensors",
24
+ "model-0019.safetensors",
25
+ "model-0020.safetensors",
26
+ "model-0021.safetensors",
27
+ "model-0022.safetensors",
28
+ "model-0023.safetensors"
29
+ ],
30
+ "step": 4900,
31
+ "architecture": {
32
+ "model_name": "Orion Flagship 2B T2.2",
33
+ "implementation_version": 5,
34
+ "architecture": "orion_flagship_2b_t2_2",
35
+ "d_model": 2112,
36
+ "n_layers": 24,
37
+ "n_heads": 24,
38
+ "n_kv_heads": 6,
39
+ "rope_theta": 10000.0,
40
+ "sequence_length": 32768,
41
+ "tokenizer_name": "mistralai/Mistral-7B-v0.3",
42
+ "n_procedure_banks": 6,
43
+ "procedure_bank_span": 4,
44
+ "n_experts": 12,
45
+ "top_k": 1,
46
+ "expert_hidden": 3456,
47
+ "shared_expert_hidden": 1280,
48
+ "vault_key_parts": 128,
49
+ "vault_slots": 16384,
50
+ "vault_top_component": 12,
51
+ "vault_top_k": 4,
52
+ "vault_gate_groups": 64,
53
+ "vault_read_layers": [
54
+ 3,
55
+ 7,
56
+ 11,
57
+ 15,
58
+ 19,
59
+ 23
60
+ ],
61
+ "working_state_layers": [
62
+ 3,
63
+ 7,
64
+ 11,
65
+ 15,
66
+ 19,
67
+ 23
68
+ ],
69
+ "deliberation_start": 18,
70
+ "deliberation_cycles": 1,
71
+ "vocab_size": 32768,
72
+ "rope_extension": "yarn_frequency_blend_v1",
73
+ "rope_8k_frequency_preserved": true,
74
+ "phase4_max_train_context": 32768
75
+ },
76
+ "sources": {
77
+ "minicpm_agent_code": {
78
+ "repo": "minicpm_agent_code",
79
+ "raw_rows": 7606
80
+ },
81
+ "minicpm_agent_general": {
82
+ "repo": "minicpm_agent_general",
83
+ "raw_rows": 33957
84
+ },
85
+ "minicpm_agent_search": {
86
+ "repo": "minicpm_agent_search",
87
+ "raw_rows": 722
88
+ },
89
+ "hermes_multi": {
90
+ "repo": "NousResearch/hermes-function-calling-v1",
91
+ "raw_rows": 372
92
+ },
93
+ "hermes_single": {
94
+ "repo": "hermes_single",
95
+ "raw_rows": 318
96
+ },
97
+ "openai_schema_multi": {
98
+ "repo": "openai_schema_multi",
99
+ "raw_rows": 331
100
+ },
101
+ "smoltalk": {
102
+ "repo": "smoltalk",
103
+ "raw_rows": 565
104
+ },
105
+ "openr1_verified": {
106
+ "repo": "openr1_verified",
107
+ "raw_rows": 3023
108
+ },
109
+ "glm_reasoning": {
110
+ "repo": "glm_reasoning",
111
+ "raw_rows": 3397
112
+ },
113
+ "gptoss_reasoning": {
114
+ "repo": "gptoss_reasoning",
115
+ "raw_rows": 945
116
+ },
117
+ "claude_opus46_inversion": {
118
+ "repo": "claude_opus46_inversion",
119
+ "raw_rows": 1587
120
+ },
121
+ "claude_opus47_inversion": {
122
+ "repo": "claude_opus47_inversion",
123
+ "raw_rows": 1593
124
+ },
125
+ "claude_opus48_max": {
126
+ "repo": "claude_opus48_max",
127
+ "raw_rows": 285600
128
+ },
129
+ "high_coder": {
130
+ "repo": "high_coder",
131
+ "raw_rows": 478446
132
+ }
133
+ },
134
+ "optimizer_saved": false,
135
+ "stop_format": "User: {prompt}\nAssistant: {completion}<tokenizer.eos_token_id>",
136
+ "training_stage": "phase5",
137
+ "phase5_fingerprint": {
138
+ "version": "orion-phase5-minicpm-agent-v1",
139
+ "architecture": "orion_flagship_2b_t2_2",
140
+ "identity": "Orion T2.1 2B",
141
+ "phase4_parent": "checkpoints/phase4-step-000900",
142
+ "steps": 5000,
143
+ "lr": 8e-07,
144
+ "grad_accum": 4,
145
+ "max_context": 32768,
146
+ "sources": {
147
+ "minicpm_core": {
148
+ "repo": "openbmb/UltraData-SFT-2605",
149
+ "config": null,
150
+ "kind": "core",
151
+ "rights": "public",
152
+ "weight": 1.0
153
+ },
154
+ "minicpm_agent_code": {
155
+ "repo": "openbmb/UltraData-SFT-Agent-2609",
156
+ "config": "Code-Agent",
157
+ "kind": "agent",
158
+ "rights": "public",
159
+ "weight": 1.0
160
+ },
161
+ "minicpm_agent_general": {
162
+ "repo": "openbmb/UltraData-SFT-Agent-2609",
163
+ "config": "General-Agent",
164
+ "kind": "agent",
165
+ "rights": "public",
166
+ "weight": 1.2
167
+ },
168
+ "minicpm_agent_search": {
169
+ "repo": "openbmb/UltraData-SFT-Agent-2609",
170
+ "config": "Search-Agent",
171
+ "kind": "agent",
172
+ "rights": "public",
173
+ "weight": 0.8
174
+ },
175
+ "hermes_multi": {
176
+ "repo": "NousResearch/hermes-function-calling-v1",
177
+ "config": "func_calling",
178
+ "kind": "agent",
179
+ "rights": "public",
180
+ "weight": 0.6
181
+ },
182
+ "hermes_single": {
183
+ "repo": "NousResearch/hermes-function-calling-v1",
184
+ "config": "func_calling_singleturn",
185
+ "kind": "agent",
186
+ "rights": "public",
187
+ "weight": 0.35
188
+ },
189
+ "openai_schema_multi": {
190
+ "repo": "stindardlogic/openai-function-calling-5k",
191
+ "config": null,
192
+ "kind": "agent",
193
+ "rights": "public",
194
+ "weight": 0.45
195
+ },
196
+ "smoltalk": {
197
+ "repo": "HuggingFaceTB/smoltalk",
198
+ "config": "everyday-conversations",
199
+ "kind": "other",
200
+ "rights": "public",
201
+ "weight": 1.0
202
+ },
203
+ "openr1_verified": {
204
+ "repo": "open-r1/OpenR1-Math-220k",
205
+ "config": "default",
206
+ "kind": "other",
207
+ "rights": "public",
208
+ "weight": 0.8
209
+ },
210
+ "glm_reasoning": {
211
+ "repo": "Jackrong/GLM-5.1-Reasoning-1M-Cleaned",
212
+ "config": "main",
213
+ "kind": "other",
214
+ "rights": "other",
215
+ "weight": 1.5
216
+ },
217
+ "gptoss_reasoning": {
218
+ "repo": "Jackrong/ShareGPT-gpt-oss-120B-reasoning",
219
+ "config": null,
220
+ "kind": "other",
221
+ "rights": "other",
222
+ "weight": 1.0
223
+ },
224
+ "claude_opus46_inversion": {
225
+ "repo": "Jackrong/Claude-opus-4.6-TraceInversion-9000x",
226
+ "config": null,
227
+ "kind": "other",
228
+ "rights": "claude",
229
+ "weight": 0.7
230
+ },
231
+ "claude_opus47_inversion": {
232
+ "repo": "Jackrong/Claude-opus-4.7-TraceInversion-5000x",
233
+ "config": null,
234
+ "kind": "other",
235
+ "rights": "claude",
236
+ "weight": 0.7
237
+ },
238
+ "claude_opus48_max": {
239
+ "repo": "ronaldcmz/claude_opus_4.8_max_thinking_5k_v2",
240
+ "config": null,
241
+ "kind": "other",
242
+ "rights": "claude",
243
+ "weight": 0.45
244
+ },
245
+ "high_coder": {
246
+ "repo": "Crownelius/High-Coder-Reasoning-Multi-Turn",
247
+ "config": null,
248
+ "kind": "other",
249
+ "rights": "hunter",
250
+ "weight": 0.8
251
+ }
252
+ },
253
+ "ratios": {
254
+ "minicpm_core": 0.35,
255
+ "other_sft": 0.3,
256
+ "agent": 0.3,
257
+ "identity": 0.05
258
+ },
259
+ "quality_filter": "no-echo-no-loop-whole-target-v1"
260
+ },
261
+ "phase5_parent": "checkpoints/phase4-step-000900",
262
+ "phase5_source_positions": {
263
+ "minicpm_agent_code": 7606,
264
+ "minicpm_agent_general": 33957,
265
+ "minicpm_agent_search": 722,
266
+ "hermes_multi": 372,
267
+ "hermes_single": 318,
268
+ "openai_schema_multi": 331,
269
+ "smoltalk": 565,
270
+ "openr1_verified": 3023,
271
+ "glm_reasoning": 3397,
272
+ "gptoss_reasoning": 945,
273
+ "claude_opus46_inversion": 1587,
274
+ "claude_opus47_inversion": 1593,
275
+ "claude_opus48_max": 285600,
276
+ "high_coder": 478446
277
+ },
278
+ "phase5_source_stats": {
279
+ "minicpm_agent_code": {
280
+ "accepted": 379,
281
+ "rejected": {
282
+ "agent_schema": 5380,
283
+ "holdout": 380,
284
+ "agent_invalid": 1467
285
+ }
286
+ },
287
+ "minicpm_agent_general": {
288
+ "accepted": 999,
289
+ "rejected": {
290
+ "agent_schema": 30969,
291
+ "holdout": 1652,
292
+ "agent_invalid": 335,
293
+ "whole_sample_over_context": 3,
294
+ "no_observed_tool_result": 2
295
+ }
296
+ },
297
+ "minicpm_agent_search": {
298
+ "accepted": 300,
299
+ "rejected": {
300
+ "no_observed_tool_result": 330,
301
+ "holdout": 38,
302
+ "agent_invalid": 54
303
+ }
304
+ },
305
+ "hermes_multi": {
306
+ "accepted": 339,
307
+ "rejected": {
308
+ "holdout": 26,
309
+ "agent_invalid": 7
310
+ }
311
+ },
312
+ "hermes_single": {
313
+ "accepted": 296,
314
+ "rejected": {
315
+ "holdout": 15,
316
+ "agent_invalid": 7
317
+ }
318
+ },
319
+ "openai_schema_multi": {
320
+ "accepted": 309,
321
+ "rejected": {
322
+ "holdout": 22
323
+ }
324
+ },
325
+ "smoltalk": {
326
+ "accepted": 541,
327
+ "rejected": {
328
+ "holdout": 24
329
+ }
330
+ },
331
+ "openr1_verified": {
332
+ "accepted": 1771,
333
+ "rejected": {
334
+ "unusable_sft_row": 1107,
335
+ "whole_sample_over_context": 59,
336
+ "holdout": 145,
337
+ "prompt_echo": 3
338
+ }
339
+ },
340
+ "glm_reasoning": {
341
+ "accepted": 3222,
342
+ "rejected": {
343
+ "holdout": 168,
344
+ "whole_sample_over_context": 146,
345
+ "repetitive": 1,
346
+ "unusable_sft_row": 7,
347
+ "prompt_echo": 5
348
+ }
349
+ },
350
+ "gptoss_reasoning": {
351
+ "accepted": 886,
352
+ "rejected": {
353
+ "whole_sample_over_context": 357,
354
+ "holdout": 57,
355
+ "too_shallow_for_reasoning_mix": 86,
356
+ "prompt_echo": 27,
357
+ "generic_template": 1,
358
+ "unusable_sft_row": 2
359
+ }
360
+ },
361
+ "claude_opus46_inversion": {
362
+ "accepted": 1498,
363
+ "rejected": {
364
+ "holdout": 86,
365
+ "whole_sample_over_context": 5,
366
+ "repetitive": 3,
367
+ "unusable_sft_row": 3
368
+ }
369
+ },
370
+ "claude_opus47_inversion": {
371
+ "accepted": 1498,
372
+ "rejected": {
373
+ "holdout": 92,
374
+ "prompt_echo": 1,
375
+ "unusable_sft_row": 3,
376
+ "repetitive": 2
377
+ }
378
+ },
379
+ "claude_opus48_max": {
380
+ "accepted": 0,
381
+ "rejected": {
382
+ "generic_template": 266583,
383
+ "unusable_sft_row": 269611,
384
+ "holdout": 15989,
385
+ "prompt_echo": 3028
386
+ }
387
+ },
388
+ "high_coder": {
389
+ "accepted": 18,
390
+ "rejected": {
391
+ "no_visible_reasoning": 1367037,
392
+ "unusable_sft_row": 455667,
393
+ "too_shallow_for_reasoning_mix": 68262,
394
+ "holdout": 22761,
395
+ "whole_sample_over_context": 10
396
+ }
397
+ }
398
+ },
399
+ "identity": "Orion T2.1 2B",
400
+ "created_by": "Project Prism",
401
+ "source_checkpoint_repo": "Project-Prism/Orion-Flagship-2B-T2.2-Instruct-AgentML",
402
+ "source_checkpoint_folder": "checkpoints/phase4-step-000900",
403
+ "agent_format": "Orion AgentML: supervised tool calls, observed outputs, and final answers",
404
+ "warning": "Training steps do not establish reasoning quality; run independent evaluations."
405
+ }
checkpoints/phase5-step-004900/model-0000.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59724a2e4f8d76ec9ab76ef3071fca86da391f24c3eeb3b6e639c639dab33928
3
+ size 366044336
checkpoints/phase5-step-004900/model-0001.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0bd5f7bad129c0cf67ac71d292f86ce519efad03f14107242c60ec3762020736
3
+ size 356846624
checkpoints/phase5-step-004900/model-0002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09e163d820cf808ef5026b1467ec46517e055b95fa020885f36f71717a10186d
3
+ size 356846656
checkpoints/phase5-step-004900/model-0003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8007324f9e1eba97dbc63deb27042e21ba6e9f6a77948f68d5c322653056e5c0
3
+ size 355334276
checkpoints/phase5-step-004900/model-0004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3ddb6dc8f501c1e5b66cfbe62bf5c0ebbe0a460cc3e44e4fcc58a96bd557b38
3
+ size 350356824
checkpoints/phase5-step-004900/model-0005.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8e3cae093f450d219e4be3813771458aeebc412535a55a7f874bb5aaa1a5acc
3
+ size 350356824
checkpoints/phase5-step-004900/model-0006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6091c27be16f5b133183a88fc810fc8682fecf79e696a9ffb2b212fc68242c17
3
+ size 353719824
checkpoints/phase5-step-004900/model-0007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e9d89dc570c2da1745bca69779efe317df9c2f31f66fea668885351642eb2a9
3
+ size 350356824
checkpoints/phase5-step-004900/model-0008.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b3de772b07f35c3d4907ca6dcc408b5ce5551036b63e687eab8f946d9303d6e
3
+ size 350356824
checkpoints/phase5-step-004900/model-0009.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b55eb7f35e44e3f132442d6a4901374919b91f9e854c63dff08309ed80807d40
3
+ size 353719824
checkpoints/phase5-step-004900/model-0010.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:891c01dad0af84bea8c5613260ebfde8187ea6e0091644afdf552188db1a4f7d
3
+ size 350356824
checkpoints/phase5-step-004900/model-0011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bd234a60f2d1774411692c38dd06f002e40914e2d2604604f74659bfa6a5e14
3
+ size 350356824
checkpoints/phase5-step-004900/model-0012.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c2c6aeb0ea645fcca6757dde51eb5551038ae01cfe95247d7f1e0d0833ceef1
3
+ size 353719824
checkpoints/phase5-step-004900/model-0013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f55223a89d4831e7cd911015a7b9246c7d319691e06dd3a133e2484e2ce72935
3
+ size 350356824
checkpoints/phase5-step-004900/model-0014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bce0270eb68e0d0126691f8a5aa1722fd08e8e6e3d3f75142e3e4fc905fcdf04
3
+ size 350356824
checkpoints/phase5-step-004900/model-0015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a23f83442e32811936c5c5dbc0746b473b5c079d92833de18596c010df6ef8c
3
+ size 361170384
checkpoints/phase5-step-004900/model-0016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4917a8d3a51262638c258776cbea520a9f6432dc8431b8c95d9d0ce73919b84
3
+ size 342906264
checkpoints/phase5-step-004900/model-0017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0cadf383a27dad3d2606f21d56ae9c91a5b2b10e25e7caee329b4e6ec21c87e2
3
+ size 350356824
checkpoints/phase5-step-004900/model-0018.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2530b29bbc9f0d8d7556cd7edd8c6d065156bd7c75f4e0b9f6c0158991efee72
3
+ size 350356832
checkpoints/phase5-step-004900/model-0019.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee6aa150971640a68447f24f1d09f38374a2016f4f2af21575028d0b96ef449b
3
+ size 353719816
checkpoints/phase5-step-004900/model-0020.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cee71897c8ecdd0d89cfc756ff0e4852f7725c3155276ea08639124636ed8020
3
+ size 350356824
checkpoints/phase5-step-004900/model-0021.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99af3b42fe9c3745635313a05fce43ffdb5dbb8a69d2a668b53cbdd2e39976d2
3
+ size 350356832
checkpoints/phase5-step-004900/model-0022.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bea1cfb0f43ad96e809b6f72fd41e579b3f153a4b190bbbeb7fef8d9bddc8553
3
+ size 355790520
checkpoints/phase5-step-004900/model-0023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b933416a727c5f17d04635536665ef3a1496e6dd16a1267e024120e3fa3d0f5
3
+ size 53746560
checkpoints/phase5-step-004900/orion_phase5_launcher.py ADDED
@@ -0,0 +1,894 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import marimo
2
+
3
+ __generated_with = "0.25.1"
4
+ app = marimo.App(width="medium", auto_download=["html"])
5
+
6
+
7
+ @app.cell
8
+ def _():
9
+ def _():
10
+ import getpass
11
+ import os
12
+ from huggingface_hub import login
13
+
14
+ # Prompt for the token securely
15
+ hf_token = getpass.getpass("Enter your Hugging Face token: ").strip()
16
+
17
+ # Set it as an environment variable
18
+ os.environ["HF_TOKEN"] = hf_token
19
+ return
20
+
21
+
22
+ _()
23
+ return
24
+
25
+
26
+ @app.cell
27
+ def _():
28
+ #!/usr/bin/env python3
29
+ """Orion T2.1 2B / T2.2 architecture β€” Phase 5 SFT, MoLab single-file launcher.
30
+
31
+ Requires original Orion Phase 4 trainer in your Hugging Face repository.
32
+ Uses its exact architecture, kernels, tokenizer, AgentML and safetensors format.
33
+ Run MODE='selftest' then 'inspect' then 'smoke' before 'train'.
34
+ Does not execute dataset tool calls. Does not train on generated hidden CoT.
35
+ """
36
+ from __future__ import annotations
37
+
38
+ import ast
39
+ import collections
40
+ import getpass
41
+ import hashlib
42
+ import json
43
+ import math
44
+ import os
45
+ import random
46
+ import re
47
+ import shutil
48
+ import sys
49
+ import statistics
50
+ import time
51
+ from pathlib import Path
52
+
53
+ # ============================================================================
54
+ # OPERATOR SETTINGS β€” EDIT HERE. Start with selftest/inspect, NOT train.
55
+ # ============================================================================
56
+ class P5:
57
+ MODE = 'train' # selftest | inspect | token_audit | smoke | train | rescue
58
+ NAME = 'Orion T2.1 2B'
59
+ CREATOR = 'Project Prism'
60
+ ARCHITECTURE = 'orion_flagship_2b_t2_2'
61
+ SOURCE_REPO = 'Project-Prism/Orion-Flagship-2B-T2.2-Instruct-AgentML'
62
+ UPLOAD_REPO = SOURCE_REPO
63
+ SOURCE_TRAINER = 'checkpoints/phase4-step-000850/orion_unified_trainer.py'
64
+ TRAINER_REVISION = 'main' # Pin to a trusted commit SHA for reproducibility.
65
+ REQUIRE_PHASE4_COMPLETE = True # Refuse to train from step <900.
66
+ WORK_DIR = './orion_phase5_work'
67
+ STEPS = 5000
68
+ GRAD_ACCUM = 4
69
+ LEARNING_RATE = 8e-7
70
+ SAVE_EVERY = 100
71
+ LOG_EVERY = 10
72
+ EVAL_EVERY = 100
73
+ SEED = 520261011
74
+ KEEP_LOCAL = 2
75
+ MAX_CONTEXT = 32768
76
+ MIN_CONTEXT = 8192
77
+ MAX_SOURCE_SCAN = 300
78
+ REQUIRE_HUB_UPLOAD = True
79
+ SAVE_OPTIMIZER = False # True is large; false resets AdamW on resume.
80
+ # Real MiniCPM5 core SFT is gated. Request access on HF first.
81
+ REQUIRE_MINICPM_CORE = False
82
+ REQUIRE_MINICPM_AGENT = True
83
+ # Only turn on if YOU have checked upstream provider training rights.
84
+ ALLOW_OTHER_TEACHER_DATA = True
85
+ ALLOW_CLAUDE_TEACHER_DATA = True # TRAINING gate: require actual permission first.
86
+ PREVIEW_CLAUDE_SOURCES = True # Default ON for inspect/token_audit only; NEVER for training.
87
+ TOKEN_AUDIT_ROWS = 30 # Accepted samples per dataset; change to 100+ for better estimates.
88
+ ALLOW_HUNTER_TEACHER_DATA = True
89
+ # Core reasoning 35%, other reasoning/chat 30%, real agent 30%, identity 5%.
90
+ # When gated core unavailable, its share is reassigned to other chat/reasoning.
91
+ RATIOS = dict(minicpm_core=.35, other_sft=.30, agent=.30, identity=.05)
92
+
93
+ SOURCE_SPECS = {
94
+ 'minicpm_core': dict(repo='openbmb/UltraData-SFT-2605', config=None,
95
+ kind='core', rights='public', weight=1.0),
96
+ 'minicpm_agent_code': dict(repo='openbmb/UltraData-SFT-Agent-2609',
97
+ config='Code-Agent', kind='agent', rights='public', weight=1.0),
98
+ 'minicpm_agent_general': dict(repo='openbmb/UltraData-SFT-Agent-2609',
99
+ config='General-Agent', kind='agent', rights='public', weight=1.2),
100
+ 'minicpm_agent_search': dict(repo='openbmb/UltraData-SFT-Agent-2609',
101
+ config='Search-Agent', kind='agent', rights='public', weight=.8),
102
+ 'hermes_multi': dict(repo='NousResearch/hermes-function-calling-v1',
103
+ config='func_calling', kind='agent', rights='public', weight=.6),
104
+ 'hermes_single': dict(repo='NousResearch/hermes-function-calling-v1',
105
+ config='func_calling_singleturn', kind='agent', rights='public', weight=.35),
106
+ 'openai_schema_multi': dict(repo='stindardlogic/openai-function-calling-5k',
107
+ config=None, kind='agent', rights='public', weight=.45),
108
+ 'smoltalk': dict(repo='HuggingFaceTB/smoltalk', config='everyday-conversations',
109
+ kind='other', rights='public', weight=1.0),
110
+ 'openr1_verified': dict(repo='open-r1/OpenR1-Math-220k', config='default',
111
+ kind='other', rights='public', weight=0.8),
112
+ 'glm_reasoning': dict(repo='Jackrong/GLM-5.1-Reasoning-1M-Cleaned', config='main',
113
+ kind='other', rights='other', weight=1.5),
114
+ 'gptoss_reasoning': dict(repo='Jackrong/ShareGPT-gpt-oss-120B-reasoning', config=None,
115
+ kind='other', rights='other', weight=1.0),
116
+ 'claude_opus46_inversion': dict(repo='Jackrong/Claude-opus-4.6-TraceInversion-9000x', config=None,
117
+ kind='other', rights='claude', weight=.7),
118
+ 'claude_opus47_inversion': dict(repo='Jackrong/Claude-opus-4.7-TraceInversion-5000x', config=None,
119
+ kind='other', rights='claude', weight=.7),
120
+ 'claude_opus48_max': dict(repo='ronaldcmz/claude_opus_4.8_max_thinking_5k_v2', config=None,
121
+ kind='other', rights='claude', weight=.45),
122
+ 'high_coder': dict(repo='Crownelius/High-Coder-Reasoning-Multi-Turn', config=None,
123
+ kind='other', rights='hunter', weight=.8),
124
+ }
125
+
126
+ # Hand-written, correct, non-repetitive identity + multi-turn examples.
127
+ IDENTITY = [
128
+ ('Who are you?', 'I am Orion T2.1 2B, an experimental AI assistant built by Project Prism.'),
129
+ ('What is your name?', 'My name is Orion T2.1 2B. You can call me Orion.'),
130
+ ('Who made you?', 'Project Prism built me.'),
131
+ ('Are you Claude or ChatGPT?', 'No. I am Orion T2.1 2B, a separate experimental model.'),
132
+ ('How many parameters do you have?', 'About 2 billion parameters.'),
133
+ ('What architecture are you?', 'My architecture is Orion Flagship 2B T2.2; my assistant name is Orion T2.1 2B.'),
134
+ ('What is Super Mega Ultra Extreem?', 'It is my largest reasoning-effort setting, not a promise of correctness.'),
135
+ ('Do you have to use all your thinking tokens?', 'No. I should stop reasoning once I have a sound answer.'),
136
+ ('Hi', 'Hey! I am Orion. What can I help you with?'),
137
+ ('Should you open test.py when someone says hello?', 'No. A greeting needs no tools.'),
138
+ ('My favourite food is spaghetti. What is MY favourite food?', 'Your favourite food is spaghetti.'),
139
+ ('I am worried you will make a mistake.', 'I can make mistakes. I will try to be clear and admit uncertainty.'),
140
+ ]
141
+ MEMORY_CASES = [
142
+ (['My favourite colour is purple.', 'What is my favourite colour?'], 'Your favourite colour is purple.'),
143
+ (['My favourite food is spaghetti.', 'What is my favourite food?'], 'Your favourite food is spaghetti.'),
144
+ (['Your name is Orion. My name is Alex.', 'Who is Alex?'], 'Alex is the user, not me.'),
145
+ (['Remember: my pet is a cat named Pixel.', 'What is my cat called?'], 'Your cat is called Pixel.'),
146
+ ]
147
+ # Small held-out diagnostic cases. These evaluate loss, NOT proof of reasoning.
148
+ EVAL_CASES = [
149
+ ('A checkpoint is missing weight shards despite a success log. What should you do?',
150
+ 'Do not load it. Verify the manifest and every shard, restore the last complete checkpoint, and use atomic saving with post-upload checks.'),
151
+ ('All but nine sheep run away. How many remain?', 'Nine remain. "All but nine" means nine did not run away.'),
152
+ ('What is 12 - 5 + 3?', '10. Subtract five from twelve to get seven, then add three.'),
153
+ ('Should you use a file tool just to answer hello?', 'No. Say hello without opening any files.'),
154
+ ]
155
+
156
+
157
+ def digest(obj):
158
+ return hashlib.sha256(json.dumps(obj, sort_keys=True, default=str, ensure_ascii=False).encode()).hexdigest()
159
+
160
+
161
+ def quality_check(question, answer, reason='', require_substance=True):
162
+ """Cheap conservative filters, not an accuracy oracle."""
163
+ if not isinstance(question, str) or not isinstance(answer, str):
164
+ return 'non_string'
165
+ q, a = question.strip(), answer.strip()
166
+ if not q or not a or len(q) > 200000 or len(a) > 200000:
167
+ return 'empty_or_huge'
168
+ if a.casefold() == q.casefold() or (len(a) > 100 and q[:120].casefold() in a[:180].casefold()):
169
+ return 'prompt_echo'
170
+ if re.search(r'(?i)(?:i will deliver a structured|concrete next steps|analysis & recommendations)', a):
171
+ return 'generic_template'
172
+ # Detect repeated 6-word windows dominating an answer or a reasoning trace.
173
+ for text in (a, reason):
174
+ words = re.findall(r'\w+', text.casefold())
175
+ if len(words) >= 65:
176
+ sixes = collections.Counter(tuple(words[i:i+6]) for i in range(len(words)-5))
177
+ if sixes and sixes.most_common(1)[0][1] >= max(6, len(words)//15):
178
+ return 'repetitive'
179
+ if require_substance and len(a) < 55 and len(reason) < 100:
180
+ return 'too_shallow_for_reasoning_mix'
181
+ if re.search(r"(?i)\b(i am|i'm|my name is)\s+(claude|chatgpt|gpt-oss|glm)", a[:200]):
182
+ return 'wrong_identity'
183
+ return None
184
+
185
+
186
+ def load_orion_runtime():
187
+ """Reuse the exact original 373-tensor trainer, without marimo UI startup."""
188
+ from huggingface_hub import hf_hub_download
189
+ path = hf_hub_download(repo_id=P5.SOURCE_REPO, filename=P5.SOURCE_TRAINER,
190
+ revision=P5.TRAINER_REVISION, token=os.environ.get('HF_TOKEN'))
191
+ source = Path(path).read_text(encoding='utf-8')
192
+ parsed = ast.parse(source)
193
+ cells = [n for n in ast.walk(parsed) if isinstance(n, ast.FunctionDef)
194
+ and n.name == '_' and any(isinstance(x, ast.ClassDef) and x.name == 'SFTConfig'
195
+ for x in n.body)]
196
+ if len(cells) != 1:
197
+ raise RuntimeError('Upstream trainer changed: cannot identify unique Orion SFT cell.')
198
+ cell = cells[0]
199
+ # Remove marimo-specific return and the old Phase4 entrypoint.
200
+ body = [n for n in cell.body if not isinstance(n, ast.Return) and not (
201
+ isinstance(n, ast.If) and isinstance(n.test, ast.Compare) and
202
+ isinstance(n.test.left, ast.Name) and n.test.left.id == '__name__')]
203
+ tree = ast.fix_missing_locations(ast.Module(body=body, type_ignores=[]))
204
+ ns = {'__name__': 'orion_phase5_runtime', '__file__': str(path)}
205
+ exec(compile(tree, str(path), 'exec'), ns)
206
+ if ns.get('P4_MODEL_ID') != P5.ARCHITECTURE or ns.get('P4_IDENTITY_NAME') != P5.NAME:
207
+ raise RuntimeError('Upstream architecture or Orion identity changed; review before training.')
208
+ for key in ('build_model', 'restore_model_shards', 'p3_hub_restore',
209
+ 'save_checkpoint', 'p4_pack', 'parse_openai_dialogue', 'event_samples'):
210
+ if key not in ns:
211
+ raise RuntimeError('Upstream Orion trainer is missing '+key)
212
+ ns['CHECKPOINT_FORMATS'] += ('orion-phase5-1',)
213
+ ns['SFTConfig'].max_target_tokens = 32768
214
+ ns['SFTConfig'].max_prompt_tokens = 32768
215
+ ns['SFTConfig'].phase4_max_context = P5.MAX_CONTEXT
216
+ ns['P4_IDENTITY_NAME'] = P5.NAME
217
+ # Agent samples must also learn their identity; tools are contextual, not a command.
218
+ old_formatter = ns['format_context']
219
+ def with_identity(tools, events, closing_assistant=True):
220
+ s = old_formatter(tools, events, closing_assistant=closing_assistant)
221
+ return s.replace(ns['AGENTML_HEADER'], ns['AGENTML_HEADER'] + '\n' +
222
+ 'You are Orion T2.1 2B, built by Project Prism. Use tools only when needed. '
223
+ 'Never invent tool observations.\n', 1)
224
+ ns['format_context'] = with_identity
225
+ return ns
226
+
227
+
228
+ def allowed(spec):
229
+ rights = spec['rights']
230
+ return rights == 'public' or (rights == 'other' and P5.ALLOW_OTHER_TEACHER_DATA) or (
231
+ rights == 'claude' and P5.ALLOW_CLAUDE_TEACHER_DATA) or (
232
+ rights == 'hunter' and P5.ALLOW_HUNTER_TEACHER_DATA)
233
+
234
+
235
+ def mini_agent_row(row, ns):
236
+ """Normalize MiniCPM messages for Orion; NEVER train on foreign system prompts.
237
+
238
+ Keep original user/assistant/tool chronology, real observations and tool-call
239
+ arguments. The existing Orion parsers decide whether an example is valid.
240
+ """
241
+ original = ns['parse_json'](row.get('messages'))
242
+ if not isinstance(original, list):
243
+ return None
244
+ clean = []
245
+ for m in original:
246
+ if not isinstance(m, dict):
247
+ continue
248
+ role = str(m.get('role','')).lower()
249
+ if role in ('system', 'developer'):
250
+ continue # Includes "You are Codex" / OpenClaw behavioural instructions.
251
+ if role not in ('user', 'human', 'assistant', 'tool', 'function'):
252
+ continue
253
+ m = dict(m)
254
+ if role == 'human': m['role'] = 'user'
255
+ if role == 'function': m['role'] = 'tool'
256
+ clean.append(m)
257
+ if not clean or not any(m['role']=='user' for m in clean):
258
+ return None
259
+ tools = ns['parse_json'](row.get('tools'))
260
+ if not isinstance(tools, list): tools = []
261
+ return {'messages':clean, 'tools':tools}
262
+
263
+
264
+ def parse_minicpm_agent(row, ns):
265
+ """Try native Orion parser then its schema-inference SWE-style fallback."""
266
+ clean = mini_agent_row(row, ns)
267
+ if clean is None: return None
268
+ # Different loaders accept different row shapes; do not invent tool results.
269
+ swe_row = {'messages': json.dumps(clean['messages'], ensure_ascii=False),
270
+ 'model': 'orion_source_adapter', 'resolved': True}
271
+ parsers = (lambda:ns['parse_openai_dialogue'](clean),
272
+ lambda:ns['parse_source_row']('swesmith_claude', clean),
273
+ lambda:ns['parse_source_row']('swesmith_claude', swe_row))
274
+ for parser in parsers:
275
+ try:
276
+ parsed = parser()
277
+ if parsed is not None and len(parsed)==3:
278
+ return parsed
279
+ except (TypeError, ValueError, KeyError, AttributeError):
280
+ continue
281
+ return None
282
+
283
+
284
+ def minicpm_code_features(ds):
285
+ """Add the optional positional tool field found in Code-Agent Parquet shards.
286
+
287
+ Some shards contain this field while the repository's inferred Features omit it,
288
+ producing the observed Arrow cast error on iteration.
289
+ """
290
+ import copy
291
+ from datasets import Value, List
292
+ features = copy.deepcopy(ds.features)
293
+ if features is None or 'tools' not in features:
294
+ return None
295
+ tool_feature = features['tools']
296
+ entry = getattr(tool_feature,'feature',None)
297
+ if isinstance(entry,dict) and 'positional' not in entry:
298
+ entry['positional'] = List(Value('string'))
299
+ return features
300
+ return None
301
+
302
+
303
+ def minicpm_general_chat(row, ns):
304
+ """Strict text-only General-Agent fallback; no pretending to have tools."""
305
+ clean = mini_agent_row(row, ns)
306
+ if clean is None:
307
+ return None
308
+ messages = clean['messages']
309
+ # A transcript that ever uses tools MUST go through the AgentML parser.
310
+ if any(m.get('role') == 'tool' or m.get('tool_calls') or m.get('recipient')
311
+ or m.get('function_call') for m in messages):
312
+ return None
313
+ # Only accept an actual assistant final following a real user request.
314
+ history, answer, reason = [], None, ''
315
+ for m in messages:
316
+ role = m.get('role')
317
+ body = m.get('content', '')
318
+ if isinstance(body, list):
319
+ body = '\n'.join(str(x.get('text', '')) for x in body if isinstance(x, dict))
320
+ if not isinstance(body, str):
321
+ return None
322
+ if role == 'user' and body.strip():
323
+ history.append({'role': 'user', 'content': body.strip()})
324
+ answer = None
325
+ elif role == 'assistant' and history and body.strip():
326
+ answer = body.strip()
327
+ reason = str(m.get('reasoning_content') or m.get('thinking') or '')
328
+ history.append({'role': 'assistant', 'content': answer})
329
+ if not answer or not history or history[-1]['role'] != 'assistant':
330
+ return None
331
+ q = next((m['content'] for m in reversed(history[:-1]) if m['role']=='user'), '')
332
+ if quality_check(q, answer, reason, require_substance=False):
333
+ return None
334
+ return {'history': history[:-1], 'answer': answer, 'reason': reason,
335
+ 'source': 'minicpm_agent_general_chat'}
336
+
337
+
338
+ class SourceStream:
339
+ def __init__(self, key, ns, offset=0):
340
+ self.key, self.ns = key, ns
341
+ self.spec = SOURCE_SPECS[key]
342
+ self.seed = P5.SEED + sum(map(ord, key))
343
+ self.raw_rows = 0
344
+ self.accepted = 0
345
+ self.rejected = collections.Counter()
346
+ self.pending = []
347
+ self.open()
348
+ if offset:
349
+ print(f'↩️ Replaying {offset} rows from {key} for approximate resume', flush=True)
350
+ for _ in range(offset): self.raw()
351
+
352
+ def open(self):
353
+ args = dict(path=self.spec['repo'], name=self.spec['config'],
354
+ split='train', streaming=True, token=os.environ.get('HF_TOKEN'))
355
+ ds = self.ns['load_dataset'](**args)
356
+ if self.key == 'minicpm_agent_code':
357
+ revised = minicpm_code_features(ds)
358
+ if revised is not None:
359
+ # Re-open with the complete nested schema BEFORE streaming rows.
360
+ # A cast after loading can fail before it has a chance to map.
361
+ try:
362
+ ds = self.ns['load_dataset'](**dict(args,features=revised))
363
+ print('πŸ› οΈ Code-Agent: positional tool schema included',flush=True)
364
+ except (TypeError, ValueError) as exc:
365
+ print('⚠️ Code-Agent schema override unavailable:',exc,flush=True)
366
+ self.iterator = iter(ds.shuffle(seed=self.seed, buffer_size=128))
367
+ print(f'πŸ“š {self.key}: {self.spec["repo"]} / {self.spec["config"]}', flush=True)
368
+
369
+ def raw(self):
370
+ try: row = next(self.iterator)
371
+ except StopIteration:
372
+ self.seed += 1
373
+ self.open()
374
+ row = next(self.iterator)
375
+ self.raw_rows += 1
376
+ return row
377
+
378
+ def next_case(self, tokenizer=None):
379
+ """Return one actual, filtered case; never synthesize missing answers."""
380
+ ns = self.ns
381
+ if self.pending: return self.pending.pop(0)
382
+ for _ in range(P5.MAX_SOURCE_SCAN):
383
+ row = self.raw()
384
+ # Stable 5% holdout partition. No holdout samples are used for training.
385
+ if int(digest(row)[:8], 16) % 20 == 0:
386
+ self.rejected['holdout'] += 1
387
+ continue
388
+ if self.spec['kind'] == 'agent':
389
+ # Actual MiniCPM5 tool schema: messages + tools, often JSON cells.
390
+ row = dict(row)
391
+ row['messages'] = ns['parse_json'](row.get('messages'))
392
+ row['tools'] = ns['parse_json'](row.get('tools'))
393
+ if self.key == 'hermes_multi':
394
+ parsed = ns['parse_source_row']('hermes_multi', row)
395
+ elif self.key == 'hermes_single':
396
+ parsed = ns['parse_source_row']('hermes', row)
397
+ elif self.key == 'openai_schema_multi':
398
+ parsed = ns['parse_source_row']('openai_multiturn', row)
399
+ else:
400
+ parsed = parse_minicpm_agent(row, ns)
401
+ if parsed is None:
402
+ # General-Agent includes both pure chat and tool tasks. Keep
403
+ # genuine text-only responses as CHAT, not fabricated agent
404
+ # examples. Reject partially observed tool conversations.
405
+ if self.key == 'minicpm_agent_general':
406
+ fallback = minicpm_general_chat(row, ns)
407
+ if fallback is not None:
408
+ self.accepted += 1
409
+ return 'chat', fallback
410
+ self.rejected['agent_schema'] += 1
411
+ continue
412
+ tools, events, _ = parsed
413
+ if ns['validate_events'](tools, events):
414
+ self.rejected['agent_invalid'] += 1
415
+ continue
416
+ if not any(e['role'] == 'observation' for e in events):
417
+ # Single-turn Hermes examples teach *requesting* tools. They
418
+ # have no results by design; never fabricate observations.
419
+ if self.key != 'hermes_single':
420
+ self.rejected['no_observed_tool_result'] += 1
421
+ continue
422
+ samples = ns['event_samples'](tools, events, tokenizer=tokenizer)
423
+ if not samples:
424
+ self.rejected['no_supervised_actions'] += 1
425
+ continue
426
+ # Prefer an action or final answer AFTER a real tool observation.
427
+ samples.sort(key=lambda s: ('<|observation|>' in s[0], s[2]), reverse=True)
428
+ self.accepted += 1
429
+ self.pending = [('agent', x) for x in samples[1:4]]
430
+ return 'agent', samples[0]
431
+ # Reuse Orion's existing multi-turn and <think> adapters.
432
+ adapter = self.key if self.key in ns['P4_DATA'] else 'smoltalk_everyday'
433
+ if self.key == 'claude_opus46_inversion': adapter = 'opus46_inversion'
434
+ if self.key == 'claude_opus47_inversion': adapter = 'opus47_inversion'
435
+ if self.key == 'claude_opus48_max': adapter = 'opus48_max'
436
+ if self.key == 'glm_reasoning': adapter = 'glm51_cleaned'
437
+ if self.key == 'gptoss_reasoning': adapter = 'gptoss_sharegpt'
438
+ if self.key == 'smoltalk': adapter = 'smoltalk_everyday'
439
+ if self.key == 'openr1_verified':
440
+ parsed = ns['p3_parse_openr1'](row)
441
+ cases = ([dict(history=[dict(role='user',content=parsed['question'])],
442
+ answer=parsed['answer'],reason=parsed['rationale'],source=self.key)] if parsed else [])
443
+ else:
444
+ cases = ns['p4_training_cases'](adapter, row)
445
+ good = []
446
+ for c in cases:
447
+ q = next((t['content'] for t in reversed(c['history']) if t['role'] == 'user'), '')
448
+ reason = c.get('reason', '')
449
+ issue = quality_check(q, c['answer'], reason, require_substance=(self.key != 'smoltalk'))
450
+ if issue:
451
+ self.rejected[issue] += 1
452
+ continue
453
+ if not reason and self.key not in ('smoltalk',):
454
+ # A long final answer is valid, but don't label it as a hidden trace.
455
+ self.rejected['no_visible_reasoning'] += 1
456
+ continue
457
+ good.append(c)
458
+ if good:
459
+ self.accepted += 1
460
+ self.pending = [('chat', c) for c in good[1:5]]
461
+ return 'chat', good[0]
462
+ self.rejected['unusable_sft_row'] += 1
463
+ raise RuntimeError(f'{self.key}: no usable samples in {P5.MAX_SOURCE_SCAN} rows; '
464
+ f'counts={dict(self.rejected)}. Inspect schema/permissions.')
465
+
466
+
467
+ def identity_case(ns, rng):
468
+ if rng.random() < .65:
469
+ q, a = rng.choice(IDENTITY)
470
+ history = [{'role':'user', 'content':q}]
471
+ else:
472
+ turns, a = rng.choice(MEMORY_CASES)
473
+ history = [{'role':'user','content':turns[0]},
474
+ {'role':'assistant','answer':'Got it.','reason':''},
475
+ {'role':'user','content':turns[1]}]
476
+ return dict(history=history, answer=a, reason='', effort='Instant', source='identity')
477
+
478
+
479
+ def pack_case(ns, category, value, tokenizer, cap, rng):
480
+ if category == 'agent':
481
+ prompt, target, final = value
482
+ # Context may be too large; skip whole examples rather than truncating.
483
+ ns['ACTIVE_CONTEXT_LIMIT'] = cap
484
+ sample = ns['agent_tensors']((prompt, target, final), tokenizer)
485
+ return sample
486
+ if category == 'identity':
487
+ value = identity_case(ns, rng)
488
+ reason = value.get('reason', '')
489
+ effort = value.get('effort') or (rng.choice(['High','Xhigh','Max']) if len(reason) > 400 else
490
+ rng.choice(['Low','Medium']) if reason else 'Instant')
491
+ sample, _ = ns['p4_pack'](value, tokenizer, effort, cap)
492
+ return sample
493
+
494
+
495
+ def cap_for_step(step):
496
+ f = step / P5.STEPS
497
+ return 8192 if f < .20 else 16384 if f < .55 else 32768
498
+
499
+
500
+ def fp(source_folder):
501
+ return dict(version='orion-phase5-minicpm-agent-v1', architecture=P5.ARCHITECTURE,
502
+ identity=P5.NAME, phase4_parent=source_folder, steps=P5.STEPS,
503
+ lr=P5.LEARNING_RATE, grad_accum=P5.GRAD_ACCUM,
504
+ max_context=P5.MAX_CONTEXT, sources={k:SOURCE_SPECS[k] for k in SOURCE_SPECS if allowed(SOURCE_SPECS[k])},
505
+ ratios=P5.RATIOS, quality_filter='no-echo-no-loop-whole-target-v1')
506
+
507
+
508
+ def latest_remote(ns, prefix):
509
+ files = ns['HfApi'](token=os.environ.get('HF_TOKEN')).list_repo_files(
510
+ repo_id=P5.SOURCE_REPO, repo_type='model')
511
+ hits = []
512
+ for f in files:
513
+ m = re.fullmatch(r'checkpoints/' + re.escape(prefix) + r'-step-(\d{6})/manifest\.json', f)
514
+ if m: hits.append((int(m.group(1)), f.rsplit('/', 1)[0]))
515
+ return sorted(hits, reverse=True)
516
+
517
+
518
+ def pick_checkpoint(ns, work):
519
+ local = []
520
+ for d in work.glob('phase5-step-*'):
521
+ try:
522
+ mf = ns['check_checkpoint_folder'](d, ('orion-phase5-1',))
523
+ if mf.get('phase5_fingerprint') == fp(mf['phase5_parent']):
524
+ local.append((int(mf['step']), d, mf))
525
+ except (ValueError, KeyError, OSError): pass
526
+ best = max(local, key=lambda x:x[0]) if local else None
527
+ for step, rel in latest_remote(ns, 'phase5'):
528
+ if best and step <= best[0]: break
529
+ try:
530
+ path = ns['hf_hub_download'](repo_id=P5.SOURCE_REPO,
531
+ filename=rel+'/manifest.json', token=os.environ.get('HF_TOKEN'))
532
+ meta = json.loads(Path(path).read_text())
533
+ if meta.get('phase5_fingerprint') != fp(meta.get('phase5_parent')):
534
+ print('⚠️ Skipping incompatible Phase5 checkpoint', rel, flush=True)
535
+ continue
536
+ folder, mf = ns['p3_hub_restore'](P5.SOURCE_REPO, rel, work,
537
+ ('orion-phase5-1',), 'phase5', step)
538
+ best = (step, folder, mf)
539
+ break
540
+ except (OSError, ValueError, RuntimeError, FileNotFoundError) as exc:
541
+ print('⚠️ Skipping incomplete Phase5 checkpoint:', rel, exc, flush=True)
542
+ if best: return best
543
+ for step, rel in latest_remote(ns, 'phase4'):
544
+ if P5.REQUIRE_PHASE4_COMPLETE and step < 900:
545
+ raise RuntimeError(f'Latest Phase4 checkpoint is {step}, not 900. Finish Phase4 first.')
546
+ try:
547
+ folder, mf = ns['p3_hub_restore'](P5.SOURCE_REPO, rel, work,
548
+ ('orion-phase4-1',), 'phase4', step)
549
+ return 0, folder, mf
550
+ except (OSError, ValueError, RuntimeError, FileNotFoundError) as exc:
551
+ print('⚠️ Skipping incomplete Phase4 checkpoint:', rel, exc, flush=True)
552
+ raise RuntimeError('No complete Phase4 source checkpoint was found.')
553
+
554
+
555
+ def select_streams(ns, offsets=None, for_inspect=False):
556
+ offsets = offsets or {}
557
+ active = {}
558
+ for key, spec in SOURCE_SPECS.items():
559
+ if not allowed(spec):
560
+ if not (for_inspect and spec['rights'] == 'claude' and P5.PREVIEW_CLAUDE_SOURCES):
561
+ print('πŸ”’ Training rights gate:', key, '(disabled)', flush=True)
562
+ continue
563
+ print('πŸ‘€ CLAUDE PREVIEW ONLY:', key, '(NOT enabled for training)', flush=True)
564
+ try:
565
+ active[key] = SourceStream(key, ns, offset=0 if for_inspect else int(offsets.get(key,0)))
566
+ except Exception as exc:
567
+ print(f'⚠️ {key} unavailable: {type(exc).__name__}: {exc}', flush=True)
568
+ if P5.REQUIRE_MINICPM_AGENT and not any(k.startswith('minicpm_agent_') for k in active):
569
+ raise RuntimeError('MiniCPM5 agent mix required but no agent config could be opened.')
570
+ if P5.REQUIRE_MINICPM_CORE and 'minicpm_core' not in active:
571
+ raise RuntimeError('MiniCPM5 core SFT required but gated/unavailable. Request access on Hugging Face.')
572
+ if 'minicpm_core' not in active:
573
+ print('⚠️ MiniCPM5 CORE SFT unavailable (possibly gated). Other SFT sources replace its share.', flush=True)
574
+ if not any(s.spec['kind'] == 'other' for s in active.values()) and 'minicpm_core' not in active:
575
+ raise RuntimeError('No general SFT source available; refusing agent-only training.')
576
+ return active
577
+
578
+
579
+ def choose_sample(ns, streams, tokenizer, cap, rng):
580
+ bykind = collections.defaultdict(list)
581
+ for key, s in streams.items(): bykind[s.spec['kind']].append(s)
582
+ ratios = dict(P5.RATIOS)
583
+ if not bykind['core']:
584
+ ratios['other_sft'] += ratios.pop('minicpm_core')
585
+ categories = [('core',ratios.get('minicpm_core',0)), ('other',ratios['other_sft']),
586
+ ('agent',ratios['agent']), ('identity',ratios['identity'])]
587
+ categories = [(k,v) for k,v in categories if v > 0 and (k == 'identity' or bykind[k])]
588
+ for _ in range(180):
589
+ group = rng.choices([k for k,v in categories],weights=[v for k,v in categories])[0]
590
+ if group == 'identity':
591
+ sample = pack_case(ns,'identity',None,tokenizer,cap,rng)
592
+ if sample is not None: return 'identity',sample
593
+ continue
594
+ sources = bykind[group]
595
+ stream = rng.choices(sources,weights=[s.spec['weight'] for s in sources])[0]
596
+ try:
597
+ kind, value = stream.next_case(tokenizer)
598
+ except Exception as exc:
599
+ print('⚠️ Source quarantined:', exc, flush=True)
600
+ bykind[group] = [s for s in bykind[group] if s is not stream]
601
+ if group == 'agent' and P5.REQUIRE_MINICPM_AGENT and not bykind[group]:
602
+ raise RuntimeError('All required MiniCPM5 agent sources failed; refusing silent fallback.') from exc
603
+ if group == 'core' and P5.REQUIRE_MINICPM_CORE and not bykind[group]:
604
+ raise RuntimeError('Required MiniCPM5 core source failed; check gated access.') from exc
605
+ categories = [(k,v) for k,v in categories if k != group or bykind[group]]
606
+ if not categories: raise RuntimeError('All sources quarantined')
607
+ continue
608
+ sample = pack_case(ns,kind,value,tokenizer,cap,rng)
609
+ if sample is not None: return stream.key, sample
610
+ stream.rejected['whole_sample_over_context'] += 1
611
+ raise RuntimeError('No whole sample could be packed after 180 attempts; inspect source quality.')
612
+
613
+
614
+ def eval_loss(ns, model, tok):
615
+ torch = ns['torch']
616
+ scores = []
617
+ model.eval()
618
+ try:
619
+ for q,a in EVAL_CASES:
620
+ c = dict(history=[dict(role='user',content=q)], answer=a, reason='')
621
+ sample, _ = ns['p4_pack'](c,tok,'Medium',8192)
622
+ if sample is None: continue
623
+ x,y = sample
624
+ with torch.no_grad(), torch.autocast(device_type='cuda',dtype=torch.bfloat16):
625
+ ce, _ = model(x.unsqueeze(0).to(ns['DEVICE']), y.unsqueeze(0).to(ns['DEVICE']))
626
+ scores.append(float(ce.detach()))
627
+ finally:
628
+ model.train()
629
+ return sum(scores)/len(scores) if scores else float('nan')
630
+
631
+
632
+ def save(ns, model, tok, opt, arch, step, streams, work, parent):
633
+ dest = work/f'phase5-step-{step:06d}'
634
+ if dest.exists(): raise FileExistsError(dest)
635
+ extra = dict(format_version='orion-phase5-1', training_stage='phase5',
636
+ phase5_fingerprint=fp(parent), phase5_parent=parent,
637
+ phase5_source_positions={k:s.raw_rows for k,s in streams.items()},
638
+ phase5_source_stats={k:dict(accepted=s.accepted,rejected=dict(s.rejected)) for k,s in streams.items()},
639
+ identity=P5.NAME, created_by=P5.CREATOR,
640
+ source_checkpoint_repo=P5.SOURCE_REPO, source_checkpoint_folder=parent,
641
+ agent_format='Orion AgentML: supervised tool calls, observed outputs, and final answers',
642
+ warning='Training steps do not establish reasoning quality; run independent evaluations.')
643
+ # Saving requires free disk for 24+ shards, never prune before verified upload.
644
+ estimate = sum(p.numel()*p.element_size() for p in model.parameters())
645
+ if shutil.disk_usage(work).free < estimate + 2*1024**3:
646
+ raise RuntimeError('Not enough disk space for an atomic Phase5 checkpoint.')
647
+ item = ns['save_checkpoint'](model,tok,opt,dest,arch,step,
648
+ {k:s.raw_rows for k,s in streams.items()},save_optimizer=P5.SAVE_OPTIMIZER,
649
+ manifest_overrides=extra)
650
+ ns['check_checkpoint_folder'](item,('orion-phase5-1',))
651
+ shutil.copy2(Path(__file__), item/'orion_phase5_launcher.py')
652
+ if P5.REQUIRE_HUB_UPLOAD:
653
+ ns['upload_checkpoint'](item,P5.UPLOAD_REPO)
654
+ ns['combo_uploaded_files'](item,P5.UPLOAD_REPO)
655
+ folders = sorted(work.glob('phase5-step-*'))
656
+ for old in folders[:-max(2,P5.KEEP_LOCAL)]:
657
+ if old != item and old.is_dir():
658
+ ns['check_checkpoint_folder'](old,('orion-phase5-1',))
659
+ shutil.rmtree(old)
660
+ return item
661
+
662
+
663
+ def selftest():
664
+ assert sum(P5.RATIOS.values()) == 1
665
+ assert quality_check('What is your name?', 'What is your name?') == 'prompt_echo'
666
+ assert quality_check('Hi', 'Hello!', require_substance=False) is None
667
+ assert quality_check('A?', 'is not a sheep that ran away '*40, require_substance=False) == 'repetitive'
668
+ assert quality_check('Question', 'Here is a detailed, nonrepeating explanation that answers the question clearly.',
669
+ require_substance=False) is None
670
+ assert cap_for_step(1) == 8192 and cap_for_step(P5.STEPS) == 32768
671
+ assert P5.ARCHITECTURE == 'orion_flagship_2b_t2_2'
672
+ assert all(k in SOURCE_SPECS for k in ('claude_opus46_inversion', 'claude_opus47_inversion', 'claude_opus48_max'))
673
+ if not P5.ALLOW_CLAUDE_TEACHER_DATA:
674
+ assert not allowed(SOURCE_SPECS['claude_opus47_inversion'])
675
+ assert P5.PREVIEW_CLAUDE_SOURCES is True
676
+ fake_ns = {'parse_json':lambda x:x}
677
+ mini = mini_agent_row({'messages':[
678
+ {'role':'system','content':'You are Codex, not Orion.'},
679
+ {'role':'user','content':'Read file'},
680
+ {'role':'assistant','content':'','tool_calls':[{'type':'function','function':{'name':'read','arguments':{'path':'a'}}}]},
681
+ {'role':'tool','content':'File contents'}], 'tools':[]},fake_ns)
682
+ assert mini and len(mini['messages']) == 3
683
+ assert not any('You are Codex' in str(m) for m in mini['messages'])
684
+ assert mini['messages'][1]['tool_calls'][0]['function']['name'] == 'read'
685
+ assert mini['messages'][2]['role'] == 'tool'
686
+ plain = {'messages': [
687
+ {'role':'system','content':'You are Claude'},
688
+ {'role':'user','content':'Explain a useful concept in plain English.'},
689
+ {'role':'assistant','content':'A good explanation uses a simple definition, an example, and one clear limitation.'}]}
690
+ case = minicpm_general_chat(plain, fake_ns)
691
+ assert case and case['history'][0]['role']=='user'
692
+ assert not minicpm_general_chat({'messages': plain['messages'] +
693
+ [{'role':'tool','content':'unverified'}]}, fake_ns)
694
+ print('βœ… Phase5 local CPU selftest passed. No downloads, no GPU, no training.')
695
+
696
+
697
+ def inspect(ns):
698
+ streams = select_streams(ns, for_inspect=True)
699
+ print('\nπŸ”Ž SOURCE AUDIT (real rows, never used for training in this mode)')
700
+ for name, s in streams.items():
701
+ for _ in range(2):
702
+ try:
703
+ kind, case = s.next_case()
704
+ if kind == 'agent':
705
+ prompt,target,final=case
706
+ print(f' {name}: agent final={final} context={len(prompt)} chars target={len(target)} chars')
707
+ else:
708
+ print(f' {name}: chat answer={len(case["answer"])} chars '
709
+ f'reason={len(case.get("reason", ""))} chars history={len(case["history"])} turns')
710
+ except Exception as exc:
711
+ print(f' {name}: ⚠️ {type(exc).__name__}: {exc}')
712
+ break
713
+ print(f' accepted={s.accepted}, rejected={dict(s.rejected)}')
714
+ print('Inspect sample acceptance and data rights before switching MODE to smoke/train.')
715
+
716
+
717
+ def token_audit(ns):
718
+ """Measure REAL Mistral-tokenizer lengths of accepted, complete examples.
719
+
720
+ Sampling is approximate and reports only accepted rows; no GPU/weights/training.
721
+ Claude rows are inspectable when PREVIEW_CLAUDE_SOURCES=True, but are NEVER
722
+ included in training unless separately authorised.
723
+ """
724
+ tok = ns['load_tokenizer']()
725
+ streams = select_streams(ns, for_inspect=True)
726
+ results = {}
727
+ out = Path(P5.WORK_DIR).resolve() / 'phase5_token_audit.json'
728
+ out.parent.mkdir(parents=True, exist_ok=True)
729
+ print(f'πŸ“ TOKEN AUDIT | {P5.TOKEN_AUDIT_ROWS} accepted samples/source | tokenizer={getattr(tok,"name_or_path","Orion")}', flush=True)
730
+ for name, stream in streams.items():
731
+ lengths, targets, reasons = [], [], []
732
+ for _ in range(P5.TOKEN_AUDIT_ROWS):
733
+ try:
734
+ category, case = stream.next_case(tokenizer=tok)
735
+ if category == 'agent':
736
+ prompt, target, final = case
737
+ p_ids = tok.encode(prompt, add_special_tokens=False)
738
+ t_ids = tok.encode(target, add_special_tokens=False)
739
+ total = len(p_ids) + len(t_ids) + int(bool(final)) + int(tok.bos_token_id is not None)
740
+ target_n = len(t_ids) + int(bool(final))
741
+ reason_n = 0
742
+ else:
743
+ effort = 'Max' if case.get('reason') else 'Instant'
744
+ sample, total = ns['p4_pack'](case, tok, effort, 10**7)
745
+ if sample is None:
746
+ raise ValueError('Sample could not be packed (unusually large).')
747
+ target_n = int((sample[1] != -100).sum().item())
748
+ reason_n = len(tok.encode(case.get('reason', ''), add_special_tokens=False))
749
+ lengths.append(total)
750
+ targets.append(target_n)
751
+ reasons.append(reason_n)
752
+ except Exception as exc:
753
+ print(f'⚠️ {name}: stopped after {len(lengths)} samples: {type(exc).__name__}: {exc}',flush=True)
754
+ break
755
+ if not lengths:
756
+ results[name] = dict(samples=0, error='no accepted samples')
757
+ continue
758
+ ordered = sorted(lengths)
759
+ pct = lambda q: ordered[min(len(ordered)-1, math.ceil(q*len(ordered))-1)]
760
+ result = dict(samples=len(lengths), total_tokens_min=min(lengths),
761
+ total_tokens_median=statistics.median(lengths), total_tokens_p90=pct(.9),
762
+ total_tokens_p99=pct(.99), total_tokens_max=max(lengths),
763
+ target_tokens_median=statistics.median(targets),
764
+ reasoning_tokens_median=statistics.median(reasons),
765
+ above_8k=sum(x>8192 for x in lengths), above_16k=sum(x>16384 for x in lengths),
766
+ above_32k=sum(x>32768 for x in lengths),
767
+ preview_only=stream.spec['rights']=='claude' and not P5.ALLOW_CLAUDE_TEACHER_DATA,
768
+ rejected=dict(stream.rejected))
769
+ results[name] = result
770
+ print(f' {name:26s} n={len(lengths):3d} median={result["total_tokens_median"]:7g} '
771
+ f'p90={result["total_tokens_p90"]:6d} max={result["total_tokens_max"]:6d} '
772
+ f'target_median={result["target_tokens_median"]:6g} '
773
+ f'over32k={result["above_32k"]}',flush=True)
774
+ out.write_text(json.dumps(dict(model=P5.NAME, sample_size=P5.TOKEN_AUDIT_ROWS,
775
+ note='Sample statistics of accepted rows, not full-dataset token totals or training permission.',
776
+ sources=results), indent=2, ensure_ascii=False))
777
+ print(f'βœ… Saved measured token counts: {out}',flush=True)
778
+
779
+
780
+ def train(ns, smoke=False):
781
+ torch = ns['torch']
782
+ if not torch.cuda.is_available(): raise RuntimeError('Orion Phase5 requires a CUDA GPU.')
783
+ if P5.REQUIRE_HUB_UPLOAD and not os.environ.get('HF_TOKEN'):
784
+ os.environ['HF_TOKEN'] = getpass.getpass('HF write token (hidden): ').strip()
785
+ ns['HF_TOKEN'] = os.environ['HF_TOKEN']
786
+ ns['p3_prepare_gpu']()
787
+ ns['p4_enable_rope']()
788
+ work = Path(P5.WORK_DIR).resolve();work.mkdir(parents=True,exist_ok=True)
789
+ start, folder, mf = pick_checkpoint(ns,work)
790
+ if start >= P5.STEPS:
791
+ print(f'🏁 Phase5 already finished: {start}/{P5.STEPS}');return
792
+ parent = mf.get('phase5_parent') if start else (
793
+ 'checkpoints/phase4-step-'+str(mf['step']).zfill(6))
794
+ arch = ns['p4_make_arch'](mf['architecture'])
795
+ ns['select_attention'](arch)
796
+ model = ns['build_model'](arch)
797
+ ns['restore_model_shards'](model,folder,mf['model_shards'])
798
+ model.set_specialization_scale(1.0);model.train()
799
+ tok = ns['load_tokenizer']()
800
+ optimizer = torch.optim.AdamW(model.parameters(), lr=P5.LEARNING_RATE,
801
+ betas=(.9,.95),eps=1e-8,weight_decay=.01,fused=True)
802
+ if start and (folder/'optimizer.pt').exists():
803
+ optimizer.load_state_dict(torch.load(folder/'optimizer.pt',map_location=ns['DEVICE'],weights_only=True))
804
+ elif start:
805
+ print('⚠️ AdamW moments reset on resume (SAVE_OPTIMIZER=False).')
806
+ streams = select_streams(ns, mf.get('phase5_source_positions',{}) if start else {})
807
+ rng = random.Random(P5.SEED + start*100003)
808
+ print(f'πŸš€ PHASE5 {start}/{P5.STEPS} -> {P5.NAME} | sources={list(streams)}',flush=True)
809
+ if smoke: print('πŸ§ͺ SMOKE MODE: one update, no checkpoint upload.',flush=True)
810
+ for step in range(start+1, (start+2 if smoke else P5.STEPS+1)):
811
+ cap = cap_for_step(step)
812
+ t0 = time.monotonic()
813
+ while True:
814
+ optimizer.zero_grad(set_to_none=True)
815
+ counts = collections.Counter()
816
+ total_ce = 0.
817
+ longest = 0
818
+ accum = 1 if smoke else P5.GRAD_ACCUM
819
+ try:
820
+ for _ in range(accum):
821
+ ns['ACTIVE_CONTEXT_LIMIT'] = cap
822
+ label,(x,y) = choose_sample(ns,streams,tok,cap,rng)
823
+ longest = max(longest,len(x))
824
+ with torch.autocast(device_type='cuda',dtype=torch.bfloat16,cache_enabled=True):
825
+ ce, aux = model(x.unsqueeze(0).to(ns['DEVICE']), y.unsqueeze(0).to(ns['DEVICE']))
826
+ loss = (ce + .25*aux)/accum
827
+ if not torch.isfinite(loss).item(): raise RuntimeError('Nonfinite Phase5 loss')
828
+ loss.backward()
829
+ counts[label] += 1
830
+ total_ce += float(ce.detach())
831
+ break
832
+ except torch.cuda.OutOfMemoryError:
833
+ optimizer.zero_grad(set_to_none=True)
834
+ torch.cuda.empty_cache()
835
+ if cap <= 8192: raise
836
+ cap //= 2
837
+ print(f'⚠️ OOM: retrying with whole samples <= {cap} tokens.',flush=True)
838
+ grad = torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)
839
+ if not torch.isfinite(grad).item(): raise RuntimeError('Nonfinite gradient norm')
840
+ lr = ns['lr_at'](step,P5.STEPS,P5.LEARNING_RATE,warmup=50)
841
+ for group in optimizer.param_groups: group['lr'] = lr
842
+ optimizer.step()
843
+ if smoke or step == start+1 or step%P5.LOG_EVERY == 0:
844
+ print(f'πŸ”₯ PHASE5 {step}/{P5.STEPS} CE={total_ce/accum:.4f} '
845
+ f'grad={float(grad):.3f} lr={lr:.2e} cap={cap} '
846
+ f'longest={longest} elapsed={time.monotonic()-t0:.1f}s sources={dict(counts)}',flush=True)
847
+ if smoke:
848
+ print('βœ… Smoke complete. No checkpoint uploaded.');break
849
+ if step%P5.EVAL_EVERY==0:
850
+ val = eval_loss(ns,model,tok)
851
+ print(f'πŸ“Š Fixed diagnostic CE={val:.4f} (not a reasoning accuracy score)',flush=True)
852
+ if step%P5.SAVE_EVERY==0 or step==P5.STEPS:
853
+ save(ns,model,tok,optimizer,arch,step,streams,work,parent)
854
+ print('🏁 Orion Phase5 run complete or smoke finished.',flush=True)
855
+
856
+
857
+ def rescue(ns):
858
+ work=Path(P5.WORK_DIR).resolve()
859
+ valid=[]
860
+ for path in work.glob('phase5-step-*'):
861
+ try:
862
+ mf=ns['check_checkpoint_folder'](path,('orion-phase5-1',))
863
+ valid.append((int(mf['step']),path))
864
+ except (ValueError,OSError,KeyError): pass
865
+ if not valid: raise RuntimeError('No complete local Phase5 checkpoint to rescue.')
866
+ step, path=max(valid)
867
+ ns['upload_checkpoint'](path,P5.UPLOAD_REPO)
868
+ ns['combo_uploaded_files'](path,P5.UPLOAD_REPO)
869
+ print('βœ… Rescued verified Phase5 step',step)
870
+
871
+
872
+ def main():
873
+ print(f'πŸ€– ORION PHASE5 | mode={P5.MODE} | {P5.NAME}')
874
+ if P5.MODE=='selftest': return selftest()
875
+ if P5.MODE not in ('inspect','token_audit','smoke','train','rescue'):
876
+ raise ValueError('MODE must be selftest, inspect, token_audit, smoke, train, or rescue')
877
+ if P5.MODE in ('train','smoke','rescue') and P5.REQUIRE_HUB_UPLOAD and not os.environ.get('HF_TOKEN'):
878
+ os.environ['HF_TOKEN'] = getpass.getpass('Hugging Face write token (hidden): ').strip()
879
+ ns=load_orion_runtime()
880
+ if P5.MODE=='inspect': return inspect(ns)
881
+ if P5.MODE=='token_audit': return token_audit(ns)
882
+ if P5.MODE=='smoke': return train(ns,smoke=True)
883
+ if P5.MODE=='train': return train(ns,smoke=False)
884
+ if P5.MODE=='rescue': return rescue(ns)
885
+
886
+
887
+ if __name__=='__main__': main()
888
+
889
+
890
+ return
891
+
892
+
893
+ if __name__ == "__main__":
894
+ app.run()
checkpoints/phase5-step-004900/orion_unified_trainer.py ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/phase5-step-004900/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/phase5-step-004900/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": true,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "is_local": false,
8
+ "legacy": false,
9
+ "local_files_only": false,
10
+ "model_max_length": 1000000000000000019884624838656,
11
+ "pad_token": null,
12
+ "sp_model_kwargs": {},
13
+ "spaces_between_special_tokens": false,
14
+ "tokenizer_class": "TokenizersBackend",
15
+ "unk_token": "<unk>",
16
+ "use_default_system_prompt": false
17
+ }