codemaivanngu commited on
Commit
0ca529a
·
unverified ·
0 Parent(s):

chore(upstream): import Cross-Tokenizer-OPD source snapshot

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitignore +11 -0
  2. README.md +49 -0
  3. docs/upstream-source-manifest.json +601 -0
  4. kdflow/__init__.py +1 -0
  5. kdflow/algorithms/__init__.py +17 -0
  6. kdflow/algorithms/byte_align_ctkd.py +121 -0
  7. kdflow/algorithms/simple_ctkd.py +198 -0
  8. kdflow/algorithms/topk_byte_align_ctkd.py +101 -0
  9. kdflow/arguments/__init__.py +309 -0
  10. kdflow/arguments/data_args.py +107 -0
  11. kdflow/arguments/distillation_args.py +142 -0
  12. kdflow/arguments/fsdp_args.py +18 -0
  13. kdflow/arguments/logging_args.py +37 -0
  14. kdflow/arguments/model_args.py +45 -0
  15. kdflow/arguments/rollout_args.py +132 -0
  16. kdflow/arguments/training_args.py +182 -0
  17. kdflow/backend/__init__.py +18 -0
  18. kdflow/backend/fsdp/__init__.py +5 -0
  19. kdflow/backend/fsdp/fsdp_strategy.py +704 -0
  20. kdflow/backend/sglang/__init__.py +0 -0
  21. kdflow/backend/sglang/monkey_patch.py +282 -0
  22. kdflow/backend/sglang/sglang_engine.py +437 -0
  23. kdflow/cli/__init__.py +0 -0
  24. kdflow/cli/evaluate_alfworld.py +118 -0
  25. kdflow/cli/train_kd_off_policy.py +173 -0
  26. kdflow/cli/train_kd_on_policy.py +230 -0
  27. kdflow/datasets/__init__.py +9 -0
  28. kdflow/datasets/alfworld_dataset.py +26 -0
  29. kdflow/datasets/prompts_dataset.py +246 -0
  30. kdflow/datasets/sft_dataset.py +380 -0
  31. kdflow/datasets/utils.py +387 -0
  32. kdflow/evaluation/__init__.py +13 -0
  33. kdflow/evaluation/alfworld_evaluator.py +164 -0
  34. kdflow/evaluation/math_evaluator.py +322 -0
  35. kdflow/evaluation/verl_agent_alfworld_evaluator.py +410 -0
  36. kdflow/evaluation/verl_agent_alfworld_protocol.py +75 -0
  37. kdflow/loss/__init__.py +39 -0
  38. kdflow/loss/adaptive_kl_div.py +42 -0
  39. kdflow/loss/chunked_loss.py +82 -0
  40. kdflow/loss/cross_entropy.py +18 -0
  41. kdflow/loss/hierarchical_ranking_loss.py +40 -0
  42. kdflow/loss/js_div.py +35 -0
  43. kdflow/loss/kl_div.py +22 -0
  44. kdflow/loss/reverse_kl_div.py +27 -0
  45. kdflow/loss/skewed_kl_div.py +34 -0
  46. kdflow/loss/skewed_rkl_div.py +34 -0
  47. kdflow/loss/top1_ce.py +30 -0
  48. kdflow/loss/tvd.py +27 -0
  49. kdflow/metrics/__init__.py +3 -0
  50. kdflow/metrics/entropy.py +18 -0
.gitignore ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .DS_Store
2
+ __pycache__/
3
+ *.pyc
4
+ .venv/
5
+ .env
6
+ .env.*
7
+ outputs/
8
+ wandb/
9
+ *.egg-info/
10
+ build/
11
+ dist/
README.md ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Cross-Tokenizer-OPD
2
+
3
+ This repository provides the implementation and example training scripts for `byte_align_ctkd` and `topk_byte_align_ctkd`.
4
+
5
+ ![Overview of cross-tokenizer alignment during training, the effect of mismatch supervision, and shared-vocabulary probability mass](assets/overview.png)
6
+
7
+ ## Environment
8
+
9
+ Our experimental environment was created on top of the [KDFlow Docker image](https://hub.docker.com/r/songmzhang/kdflow?tag=sgl0.5.12_torch2.11_cu129), using the following tag:
10
+
11
+ ```bash
12
+ docker pull songmzhang/kdflow:sgl0.5.12_torch2.11_cu129
13
+ ```
14
+
15
+ [`requirements.txt`](requirements.txt) records the package versions used in our experiments. Use the Docker image as the starting point and match any additional dependencies to this file. It includes CUDA-specific builds and system-provided packages, so it is an environment snapshot rather than a standalone installation specification for arbitrary hosts.
16
+
17
+ Inside the prepared container, run from the repository root:
18
+
19
+ ```bash
20
+ python -m pip install -e . --no-deps
21
+ ```
22
+
23
+ ## Example training
24
+
25
+ Both scripts perform on-policy distillation from **Qwen2.5-7B-Instruct** to **Llama-3.2-3B-Instruct** and are configured for **one node with 8 GPUs**.
26
+
27
+ Before running, update the paths in each script or mount the files at the expected locations:
28
+
29
+ - Repository: `/workspace/Cross-Tokenizer-OPD`
30
+ - Student checkpoint: `/workspace/models/Llama-3.2-3B-Instruct`
31
+ - Teacher checkpoint: `/workspace/models/Qwen2.5-7B-Instruct`
32
+ - Training data: `/workspace/data/Train/math_code_20k/train.parquet`
33
+ - Evaluation data: `/workspace/data/math500/test.jsonl`
34
+
35
+ The data files are included under `data/`; model checkpoints must be provided separately. WandB logging is disabled by default (`--use_wandb False`). To enable it, set `--use_wandb True`, add `--wandb_project <project_name>`, and configure your WandB account.
36
+
37
+ Run the byte-alignment example from the repository root:
38
+
39
+ ```bash
40
+ bash run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh
41
+ ```
42
+
43
+ Run the top-k example (`--ctkd_topk 16`):
44
+
45
+ ```bash
46
+ bash run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh
47
+ ```
48
+
49
+ Training settings can be adjusted directly in the scripts. Checkpoints, evaluation results, and training logs are written under `outputs/<TRAIN_NAME>/`.
docs/upstream-source-manifest.json ADDED
@@ -0,0 +1,601 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "url": "https://anonymous.4open.science/api/repo/Cross-Tokenizer-OPD",
3
+ "paper": "https://arxiv.org/pdf/2610.08448v1",
4
+ "retrieved_utc": "2026-10-17T19:28:35.738288+00:00",
5
+ "identity_kind": "audited content snapshot assembled from public files; upstream commit unavailable",
6
+ "excluded": [
7
+ "data/",
8
+ "assets/"
9
+ ],
10
+ "source_manifest_sha256": "93cef1dfa18bcd5fc66cf80347557382efcd7e8955c3f6b464435711e82bf49b",
11
+ "files": [
12
+ {
13
+ "path": ".gitignore",
14
+ "bytes": 89,
15
+ "api_version_tag": "2e7be852",
16
+ "git_blob": "2727e1bf64e5fd3901d2803b1def32466a2e90bf",
17
+ "sha256": "19587f4a412fa284a0dac11bad5d487b92117390d1eb117aee094f6fb289ba0d"
18
+ },
19
+ {
20
+ "path": "README.md",
21
+ "bytes": 2289,
22
+ "api_version_tag": "b260e71d",
23
+ "git_blob": "9ed1faf98032a99ac4f8dcaf3000e5e2d800fda1",
24
+ "sha256": "f45c1d25386d1d1a25f3c4ea4c152db95be6467f34c86a94ad52cda4df4ee656"
25
+ },
26
+ {
27
+ "path": "kdflow/__init__.py",
28
+ "bytes": 22,
29
+ "api_version_tag": "f8cbccf6",
30
+ "git_blob": "3dc1f76bc69e3f559bee6253b24fc93acee9e1f9",
31
+ "sha256": "91447944015cec709e8aa7655f7e9d64e1e4508e7023a57fe3746911c0fc6fed"
32
+ },
33
+ {
34
+ "path": "kdflow/algorithms/__init__.py",
35
+ "bytes": 520,
36
+ "api_version_tag": "a7cd7946",
37
+ "git_blob": "13dd1052857854aba08c5f7667ed6a49ff1b0b80",
38
+ "sha256": "8afbb4fadfc7db812d43ab35401971fc30fdc4810cafa81505cbea27178badbe"
39
+ },
40
+ {
41
+ "path": "kdflow/algorithms/byte_align_ctkd.py",
42
+ "bytes": 4552,
43
+ "api_version_tag": "990d76e2",
44
+ "git_blob": "c94a1239b57b05513ad853735291949aa314c6d9",
45
+ "sha256": "5c6647c6fb8064ffd5f7b1ed971168cd1f3392caee62e2d75916d41931828e18"
46
+ },
47
+ {
48
+ "path": "kdflow/algorithms/simple_ctkd.py",
49
+ "bytes": 8575,
50
+ "api_version_tag": "166783cd",
51
+ "git_blob": "57332b882e359ce1283a87f81949a5136f8c06a2",
52
+ "sha256": "fdbd99c55aeac4d061527158bdfc82a93333a42bf347eceaa134d34f5ef05952"
53
+ },
54
+ {
55
+ "path": "kdflow/algorithms/topk_byte_align_ctkd.py",
56
+ "bytes": 3448,
57
+ "api_version_tag": "4eff709a",
58
+ "git_blob": "64b49354bea827893fd49f8de51325666f7e1dba",
59
+ "sha256": "298925da4ce973b0f2917e5ba5ad2adbc4862d7c1a73515c5c86514ac0d22ddb"
60
+ },
61
+ {
62
+ "path": "kdflow/arguments/__init__.py",
63
+ "bytes": 13313,
64
+ "api_version_tag": "22b52ef5",
65
+ "git_blob": "b3c66702102a8691bcd16bc57f8a8307ccb13b68",
66
+ "sha256": "cafb17a195874f1b53b0b5f29b07840e1f2f6505e08dd77837fca25fd1a8122f"
67
+ },
68
+ {
69
+ "path": "kdflow/arguments/data_args.py",
70
+ "bytes": 3363,
71
+ "api_version_tag": "e8dc076d",
72
+ "git_blob": "12046f174777ed5d8f271b15661debf0b3c63003",
73
+ "sha256": "e5f68889ab6635ed51a285aad70dcff5de3dce7d7d06c6cf563b03331591a382"
74
+ },
75
+ {
76
+ "path": "kdflow/arguments/distillation_args.py",
77
+ "bytes": 5102,
78
+ "api_version_tag": "e7ae08cc",
79
+ "git_blob": "ae835897855f7e4147fb8efc1fe5971fd8cfb964",
80
+ "sha256": "65d39f6abeb75896864d385fe0c9a2f026938eb1a5ee9d4dc392d3cb86b440f4"
81
+ },
82
+ {
83
+ "path": "kdflow/arguments/fsdp_args.py",
84
+ "bytes": 387,
85
+ "api_version_tag": "172c1790",
86
+ "git_blob": "ed2729e8de9b94f789cd73ea1274e02bc4e18558",
87
+ "sha256": "10918e4e3242a526ab0926af4a2608a3a77dea058fd084ecee9022852aecac47"
88
+ },
89
+ {
90
+ "path": "kdflow/arguments/logging_args.py",
91
+ "bytes": 890,
92
+ "api_version_tag": "b3ce62d7",
93
+ "git_blob": "53e1413ef1c4b57d66b57a22c1c3c7022cf9d70a",
94
+ "sha256": "2d6012a164fdc81b3d9c482357dc68d0fe4485db0a2163f3cdfa3ca6411addc1"
95
+ },
96
+ {
97
+ "path": "kdflow/arguments/model_args.py",
98
+ "bytes": 1157,
99
+ "api_version_tag": "2f851242",
100
+ "git_blob": "9c6531d1deb2435054eeff4c412271149390b336",
101
+ "sha256": "b6da8907b70189acaa7c1b51544135e441182e28194ee3bbe4f23ec1fa23cb83"
102
+ },
103
+ {
104
+ "path": "kdflow/arguments/rollout_args.py",
105
+ "bytes": 4468,
106
+ "api_version_tag": "5899e45d",
107
+ "git_blob": "5b46da1da4da0f4a05a53aa14ef36167517784e4",
108
+ "sha256": "f9df7e87681abd88c00bde4985a45ddfbb3c6f3f7feee41d4fc941a808c670d1"
109
+ },
110
+ {
111
+ "path": "kdflow/arguments/training_args.py",
112
+ "bytes": 5880,
113
+ "api_version_tag": "b7d6b4ad",
114
+ "git_blob": "71443753b8ea33fd213a28c0744816c77b616600",
115
+ "sha256": "a45dd87ae3064eac14f9ca9a875963df4efd03082230aba9f4c98d67479260bd"
116
+ },
117
+ {
118
+ "path": "kdflow/backend/__init__.py",
119
+ "bytes": 647,
120
+ "api_version_tag": "33655f94",
121
+ "git_blob": "ae07fdf84af178d7d1b63e06abefb6f07f29e890",
122
+ "sha256": "b36cc114c0ffd20559a6e87808b158d7244566645933639023cf00019e03e137"
123
+ },
124
+ {
125
+ "path": "kdflow/backend/fsdp/__init__.py",
126
+ "bytes": 76,
127
+ "api_version_tag": "2c7e9bf6",
128
+ "git_blob": "a50fa8e305c891e9bfe23089b39c6f0555d473af",
129
+ "sha256": "9b1dffb545428c2862d6cc588507709797ba05f24d4017e9a450a7afa09701d9"
130
+ },
131
+ {
132
+ "path": "kdflow/backend/fsdp/fsdp_strategy.py",
133
+ "bytes": 26957,
134
+ "api_version_tag": "b0b88464",
135
+ "git_blob": "63498e48936f06180363b4df5c7fde821fa36a70",
136
+ "sha256": "8941ccc6077e6fdcc2a38c13769d6a0912dc764ea9f7b82866e7e1bf3bac0fa2"
137
+ },
138
+ {
139
+ "path": "kdflow/backend/sglang/__init__.py",
140
+ "bytes": 0,
141
+ "api_version_tag": null,
142
+ "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
143
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
144
+ },
145
+ {
146
+ "path": "kdflow/backend/sglang/monkey_patch.py",
147
+ "bytes": 10707,
148
+ "api_version_tag": "154f0497",
149
+ "git_blob": "e4aa0a9550903f01a7125b1631408e9b2df6883a",
150
+ "sha256": "ddcbc908bce562bc3b87f13b8445ac02891a491dd60ba2edee37d6396440999c"
151
+ },
152
+ {
153
+ "path": "kdflow/backend/sglang/sglang_engine.py",
154
+ "bytes": 16531,
155
+ "api_version_tag": "b2ffd390",
156
+ "git_blob": "4deb3adfe2fa5b2258c0144ed46b4b4dc1873913",
157
+ "sha256": "dd0b9a5531beba14c52389787a1df5d053412dabd9547cde5cb66f5f23bb494f"
158
+ },
159
+ {
160
+ "path": "kdflow/cli/__init__.py",
161
+ "bytes": 0,
162
+ "api_version_tag": null,
163
+ "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
164
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
165
+ },
166
+ {
167
+ "path": "kdflow/cli/evaluate_alfworld.py",
168
+ "bytes": 4054,
169
+ "api_version_tag": "c6740670",
170
+ "git_blob": "77dec216fbfc0b68f28a094b6fdeac63f1f6a646",
171
+ "sha256": "f5b02bf943606ff7d01feff0c8df9eb5a5902a7c0039d38e3a21c859716bf08e"
172
+ },
173
+ {
174
+ "path": "kdflow/cli/train_kd_off_policy.py",
175
+ "bytes": 5879,
176
+ "api_version_tag": "c4023313",
177
+ "git_blob": "b149c801a94aedc0234c529cbdc63764de08573f",
178
+ "sha256": "b3064a2f38c12bf086c5f783b0dcfb3904ff097e99289aca2a1da310d81713cc"
179
+ },
180
+ {
181
+ "path": "kdflow/cli/train_kd_on_policy.py",
182
+ "bytes": 8208,
183
+ "api_version_tag": "c50095ea",
184
+ "git_blob": "8cfb58d743568948cfa7d1c1e579be7287057241",
185
+ "sha256": "1e2036bf1b788d7f7b27ac3496d50b78c25992ae597733199bfb90619cf6391f"
186
+ },
187
+ {
188
+ "path": "kdflow/datasets/__init__.py",
189
+ "bytes": 216,
190
+ "api_version_tag": "fc482f85",
191
+ "git_blob": "37facfbdb6705166c7047e40eb67bb1f3edbf3e9",
192
+ "sha256": "2dd709d632fae96f3e8d86ffb8b608cdca9edaacd17dde7c21eed1c497379d37"
193
+ },
194
+ {
195
+ "path": "kdflow/datasets/alfworld_dataset.py",
196
+ "bytes": 806,
197
+ "api_version_tag": "4f86ac77",
198
+ "git_blob": "cd96f6d2932f6e2cd31c6e4cab188556206abbef",
199
+ "sha256": "3047bf9576937e1cbc3dd7d7070c7c949ac0e96ecb0b1bf8b6142c7c0415b88b"
200
+ },
201
+ {
202
+ "path": "kdflow/datasets/prompts_dataset.py",
203
+ "bytes": 11012,
204
+ "api_version_tag": "9c932cbd",
205
+ "git_blob": "15a6e747c6cc3fd3232affcdff8f1252ac087ee9",
206
+ "sha256": "3e6b9954b893224a5eac5ad79758dc198747b929e1fe27475e6aed1f01a09703"
207
+ },
208
+ {
209
+ "path": "kdflow/datasets/sft_dataset.py",
210
+ "bytes": 18029,
211
+ "api_version_tag": "23c3e648",
212
+ "git_blob": "6b922b0047e0d1da1e9f95f0c2b9a078f3ef92a8",
213
+ "sha256": "e5643660828193e4a124639d18d1ac14826d57203dcc3ccf31e2c7d83a37c7b7"
214
+ },
215
+ {
216
+ "path": "kdflow/datasets/utils.py",
217
+ "bytes": 14066,
218
+ "api_version_tag": "6fe0c9b3",
219
+ "git_blob": "9933ae47766e3623d148e39fa7817cefd112133d",
220
+ "sha256": "f0173126a8530665226396c7e9343f57cc031e4df1d97929ba6c3b360059bbb3"
221
+ },
222
+ {
223
+ "path": "kdflow/evaluation/__init__.py",
224
+ "bytes": 344,
225
+ "api_version_tag": "a3dca82f",
226
+ "git_blob": "245e0251fb5e37de3cc7c2a234bff53a732e41de",
227
+ "sha256": "f63f24ca9885b63c32d3d1e24b40c16367734cb238f52b4893b20be95321573c"
228
+ },
229
+ {
230
+ "path": "kdflow/evaluation/alfworld_evaluator.py",
231
+ "bytes": 6661,
232
+ "api_version_tag": "beaf919e",
233
+ "git_blob": "7e085a7350f8a07cd3b1cefcbe12c2eab7733d45",
234
+ "sha256": "45c96b0908cf0bd895cd312e88ca8bd49a53be3f00cf9fa4363555590d03352b"
235
+ },
236
+ {
237
+ "path": "kdflow/evaluation/math_evaluator.py",
238
+ "bytes": 12096,
239
+ "api_version_tag": "5cc9a5e1",
240
+ "git_blob": "890e7ed2e1fe006d1e65d3a35d398297fb2fb70f",
241
+ "sha256": "f598d470e1f03c0e09b6bbdb4d7fa8571eb7484f14f2be7ea706a0ed14aa8935"
242
+ },
243
+ {
244
+ "path": "kdflow/evaluation/verl_agent_alfworld_evaluator.py",
245
+ "bytes": 16963,
246
+ "api_version_tag": "1391ea50",
247
+ "git_blob": "f9a25e4e331f5a9dd3aad873666bb0861347453a",
248
+ "sha256": "38d71145e80de82fe738599f43dfe8b3cb14177da4784457546e6be18f78fc29"
249
+ },
250
+ {
251
+ "path": "kdflow/evaluation/verl_agent_alfworld_protocol.py",
252
+ "bytes": 3001,
253
+ "api_version_tag": "a8cc4e01",
254
+ "git_blob": "38615c951dd17f8311216b640cc2bd0ae145291a",
255
+ "sha256": "9aa3c5f02703148a0bfce8d2177b53e51aa906df03e2401e5bf0f5b241ce8433"
256
+ },
257
+ {
258
+ "path": "kdflow/loss/__init__.py",
259
+ "bytes": 1089,
260
+ "api_version_tag": "6e9b5e28",
261
+ "git_blob": "2d66850c7da1a6839a2d69b781980469ea85cd2b",
262
+ "sha256": "3bc98a351ad62faa346607502dcd56e864536b81877ca0221fd2b2ca9045f630"
263
+ },
264
+ {
265
+ "path": "kdflow/loss/adaptive_kl_div.py",
266
+ "bytes": 1528,
267
+ "api_version_tag": "7af71dea",
268
+ "git_blob": "e95f03d7900fd29f1ff5d1b64bf1d670a3267410",
269
+ "sha256": "f8a44824867ae3fa7e732f7ae5554d6b13231ff9545e1fd61793a8f546d7d39c"
270
+ },
271
+ {
272
+ "path": "kdflow/loss/chunked_loss.py",
273
+ "bytes": 3636,
274
+ "api_version_tag": "8a38a29a",
275
+ "git_blob": "2c5174fd3c60c92ea8e315fb1ce28931eb7e61ae",
276
+ "sha256": "151c81614f97b908465a952f595abe1f4dc5c0d17cea86431a7068c381125cc5"
277
+ },
278
+ {
279
+ "path": "kdflow/loss/cross_entropy.py",
280
+ "bytes": 473,
281
+ "api_version_tag": "9fab6f38",
282
+ "git_blob": "9f5a322ff489503f707515aa8f8e413ef1083b6b",
283
+ "sha256": "e3629bef7c50c737f384870876321c953ceb09b67c275242cdbfcfbcbca68377"
284
+ },
285
+ {
286
+ "path": "kdflow/loss/hierarchical_ranking_loss.py",
287
+ "bytes": 1479,
288
+ "api_version_tag": "19217f5e",
289
+ "git_blob": "8e6ea5c38676ca63011834431955852055a78af5",
290
+ "sha256": "2cced8cccf77872ab817164a931c97c43c7411b8d59cc48d842111e40f021ac3"
291
+ },
292
+ {
293
+ "path": "kdflow/loss/js_div.py",
294
+ "bytes": 1165,
295
+ "api_version_tag": "8c85586a",
296
+ "git_blob": "1d210762db1d64e5ee8d8cd4bff9ee60499950e8",
297
+ "sha256": "be71370e1e1a2c109c2a00a4e6daaece8ab17f182f1a3f03c60a385b98fb3078"
298
+ },
299
+ {
300
+ "path": "kdflow/loss/kl_div.py",
301
+ "bytes": 592,
302
+ "api_version_tag": "7bc61a88",
303
+ "git_blob": "c39542e7a839837741d0fc442ba5bb18fc5a6343",
304
+ "sha256": "b361ddca392ee4b2e83e44b6891f4402f86127db38b6e254d492d29c48950a7c"
305
+ },
306
+ {
307
+ "path": "kdflow/loss/reverse_kl_div.py",
308
+ "bytes": 761,
309
+ "api_version_tag": "5afd030a",
310
+ "git_blob": "ebdaa8f2a467e41383825915e6a757de2426738c",
311
+ "sha256": "abca69fdbe4f7d90a7ca990300189147230cce8f20d589c934d88f0b63c4edf6"
312
+ },
313
+ {
314
+ "path": "kdflow/loss/skewed_kl_div.py",
315
+ "bytes": 1119,
316
+ "api_version_tag": "737302c8",
317
+ "git_blob": "4b76d83f474a2744c81b12a717cae97ea97dccca",
318
+ "sha256": "98189a13bf644d3c46d53aa228a264b9ed666b9d28586a343f0c2bdbd6fed091"
319
+ },
320
+ {
321
+ "path": "kdflow/loss/skewed_rkl_div.py",
322
+ "bytes": 1129,
323
+ "api_version_tag": "c37538a3",
324
+ "git_blob": "9652791e8f1ff0159eb9826ed70ac16057b511cf",
325
+ "sha256": "3c78a1b04270f7d8ed40449f28f5ed936af71c0bc7d5a0badf0864454ddf66f0"
326
+ },
327
+ {
328
+ "path": "kdflow/loss/top1_ce.py",
329
+ "bytes": 760,
330
+ "api_version_tag": "3cc13dae",
331
+ "git_blob": "6a07fefbc642c79a4311c3f3d3565ee2b3bec251",
332
+ "sha256": "1f566c4559f518c8186a53de85593b812e65016beab083cd4ded715b513fa760"
333
+ },
334
+ {
335
+ "path": "kdflow/loss/tvd.py",
336
+ "bytes": 702,
337
+ "api_version_tag": "289a1332",
338
+ "git_blob": "a84bee8f7963da36e7e4011a0dd3af9d9e87d462",
339
+ "sha256": "964d872f122ec4618cfee1fd2adf8ecc0fc82a8279cce83cab8b277335f8aa2c"
340
+ },
341
+ {
342
+ "path": "kdflow/metrics/__init__.py",
343
+ "bytes": 82,
344
+ "api_version_tag": "4c09b5e4",
345
+ "git_blob": "e888dd4bdc45ae4e2be5225b6c74ae17480225a8",
346
+ "sha256": "140c40cfb0b374c62847a93f666c9cf0c3ad595d6d36dc7bb1e6c2fb05495362"
347
+ },
348
+ {
349
+ "path": "kdflow/metrics/entropy.py",
350
+ "bytes": 642,
351
+ "api_version_tag": "c37c9989",
352
+ "git_blob": "81439fc696793032bce83717db7ccb17243d8d9a",
353
+ "sha256": "e02d0a9926fdb33b120fc2ec0a95d042876b43504046a301191d4761cee58cef"
354
+ },
355
+ {
356
+ "path": "kdflow/models/__init__.py",
357
+ "bytes": 67,
358
+ "api_version_tag": "39b9dcda",
359
+ "git_blob": "8f60ffe38287f99a5fe053daf17f85b5173d07a7",
360
+ "sha256": "93370fd72c3d337adcb1bb7cbae6ba2cf9eab2ef49b5b466ada816f7b6bbd148"
361
+ },
362
+ {
363
+ "path": "kdflow/models/model.py",
364
+ "bytes": 5150,
365
+ "api_version_tag": "49c34f8e",
366
+ "git_blob": "df455afaca9a0eaaac7d7cc9836acb9f0dc1dbee",
367
+ "sha256": "2530323bba13319cea6ea6f0ea54ed64ae81d35c6a2aaf209688a67622e26996"
368
+ },
369
+ {
370
+ "path": "kdflow/models/ring_attn_utils.py",
371
+ "bytes": 7402,
372
+ "api_version_tag": "0c072a92",
373
+ "git_blob": "fe3b01a70d20358de0dc19e19ce09df306004d26",
374
+ "sha256": "ed93df4cbef484401f2cf508bc81d4a22521e632bec4bae4a931e404639b7eda"
375
+ },
376
+ {
377
+ "path": "kdflow/models/utils.py",
378
+ "bytes": 1755,
379
+ "api_version_tag": "cfd2ac62",
380
+ "git_blob": "5f00ba4aaf5b007c20e4fb7cb08b5e466a2acef5",
381
+ "sha256": "cfef9ca657b32c8ee5688f01f0f1e939ec3fc61fbd8792085d63af66f74e85d5"
382
+ },
383
+ {
384
+ "path": "kdflow/ray/__init__.py",
385
+ "bytes": 0,
386
+ "api_version_tag": null,
387
+ "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
388
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
389
+ },
390
+ {
391
+ "path": "kdflow/ray/placement_group.py",
392
+ "bytes": 2049,
393
+ "api_version_tag": "37beb550",
394
+ "git_blob": "24ba3109273257b3ab9329a19dc72804bbc5edc3",
395
+ "sha256": "08b0a5d82cacea096b1591e3d12d25f30f285145cade451578763f3b16144a27"
396
+ },
397
+ {
398
+ "path": "kdflow/ray/rollout/__init__.py",
399
+ "bytes": 176,
400
+ "api_version_tag": "ae57ecf3",
401
+ "git_blob": "a7e39d7a3785301ef2f0614ad28880e2f7dfc1cc",
402
+ "sha256": "13401bf88cab65c4e388736c19f4ff2df16fc2a8ff9cbc07a1b48c76876092de"
403
+ },
404
+ {
405
+ "path": "kdflow/ray/rollout/rollout_actor.py",
406
+ "bytes": 12933,
407
+ "api_version_tag": "635eb232",
408
+ "git_blob": "63a00bc4e29afbb2041ee99ac2c63fa5b3aa1d4f",
409
+ "sha256": "14af615c89f8c8c6089ed5e98a5473da0ab79df4771621e5becc4340ff3720bc"
410
+ },
411
+ {
412
+ "path": "kdflow/ray/rollout/rollout_group.py",
413
+ "bytes": 14618,
414
+ "api_version_tag": "7a452703",
415
+ "git_blob": "5ad5ec900fe03de4c6e7201791f8be76315a3ce9",
416
+ "sha256": "05b9235ba2da6259a49979514b6ac3910a500be4f8c0575461d3d9997959ed9b"
417
+ },
418
+ {
419
+ "path": "kdflow/ray/train/__init__.py",
420
+ "bytes": 0,
421
+ "api_version_tag": null,
422
+ "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
423
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
424
+ },
425
+ {
426
+ "path": "kdflow/ray/train/student_actor.py",
427
+ "bytes": 18702,
428
+ "api_version_tag": "857338dd",
429
+ "git_blob": "0aec3b65c4423b554ae2fdd4fee3b02d64011e78",
430
+ "sha256": "b234fee1e64e1b59f45d917faa1435cf911619fbaf1b581530996df2244b14eb"
431
+ },
432
+ {
433
+ "path": "kdflow/ray/train/student_group.py",
434
+ "bytes": 9563,
435
+ "api_version_tag": "5f558d33",
436
+ "git_blob": "cc9308ae81028870f2a47d70dadaaa443794070d",
437
+ "sha256": "c9b661647d781e3353821fce0f15c1be69f10d81a55ced07523635d52b0f04e4"
438
+ },
439
+ {
440
+ "path": "kdflow/ray/train/teacher_actor.py",
441
+ "bytes": 7299,
442
+ "api_version_tag": "cdf12732",
443
+ "git_blob": "72600b4ec71cc27acc19bc3f8333410b9784f754",
444
+ "sha256": "8e84664647c5c1f4d2a3bf000d3e382bfcbe3735f42bef2f7b0f0af43518974e"
445
+ },
446
+ {
447
+ "path": "kdflow/ray/train/teacher_group.py",
448
+ "bytes": 12590,
449
+ "api_version_tag": "83cd153b",
450
+ "git_blob": "b121a64b7519f2866d21e8e2d2c208c0052f5826",
451
+ "sha256": "b0a632619940e8fb49f4b5059299f0d380fe9d4b1bd9c26c7c82dff32e50f7ba"
452
+ },
453
+ {
454
+ "path": "kdflow/ray/utils.py",
455
+ "bytes": 2590,
456
+ "api_version_tag": "9f91e927",
457
+ "git_blob": "b003f49102a84c9ce15e617abb33fa4842286fcc",
458
+ "sha256": "8dea4c2b6b55445ad6d8868807ca016b61f6f7e12efb321502dc8b6eb2ea0263"
459
+ },
460
+ {
461
+ "path": "kdflow/token_alignment.py",
462
+ "bytes": 13436,
463
+ "api_version_tag": "8766c5ad",
464
+ "git_blob": "5a2b1695191df2aaa7798e8cbb0605858775474b",
465
+ "sha256": "c77cd962cb0430cd97ace5312077e847398a03da3ce1473062fd2c5c9808498a"
466
+ },
467
+ {
468
+ "path": "kdflow/trainer/__init__.py",
469
+ "bytes": 528,
470
+ "api_version_tag": "6dec2a62",
471
+ "git_blob": "c7fa5b1ce3c01a42c60f2f070370cd978a73710b",
472
+ "sha256": "a3caa9361a9f53b881e24ffa05951342278947135cd51c637ba4d1b16a1a8687"
473
+ },
474
+ {
475
+ "path": "kdflow/trainer/alfworld_rollout_manager.py",
476
+ "bytes": 18875,
477
+ "api_version_tag": "a77bc558",
478
+ "git_blob": "9e353bcbfd9cb9424f2110f3cf1c9e4b2ff496ff",
479
+ "sha256": "93f6ca98f2ba6fe9613d60839eb7d22d6b582420489aefbc1f6364ffb8493987"
480
+ },
481
+ {
482
+ "path": "kdflow/trainer/data_processor/__init__.py",
483
+ "bytes": 122,
484
+ "api_version_tag": "d5140e99",
485
+ "git_blob": "d267c43e5cf7c929a03867968ff8c794a5a574bf",
486
+ "sha256": "66a34ffc421ade775f246c8198bfdc9633e83c97b355084fa457e6e15695b775"
487
+ },
488
+ {
489
+ "path": "kdflow/trainer/data_processor/rollout_data_processor.py",
490
+ "bytes": 13871,
491
+ "api_version_tag": "3b4b701b",
492
+ "git_blob": "7f5564dcd3046b5be30cbabc13b06c3dd393c17a",
493
+ "sha256": "fe720ee01fb821e124a612dcb55cb0e827b64e7fb1597b4ecbb4bf14c8ee5792"
494
+ },
495
+ {
496
+ "path": "kdflow/trainer/off_policy_kd_trainer.py",
497
+ "bytes": 14023,
498
+ "api_version_tag": "f150d881",
499
+ "git_blob": "72ab1017c9edfbf90778208b9d01ca123d1d72ee",
500
+ "sha256": "6b4019e9760e0f2898a4cf0b30071d69bf24b6a23a023aca5f1f929b2a0c2acf"
501
+ },
502
+ {
503
+ "path": "kdflow/trainer/on_policy_kd_trainer.py",
504
+ "bytes": 20980,
505
+ "api_version_tag": "f34b6551",
506
+ "git_blob": "aa4ba8e05f8fc49625b69b8870fc29a8aa6aa6c6",
507
+ "sha256": "f7b20000d435f8fc43a07584a3025472f138ccc6e41f62f25c08ff9ff8fa17b1"
508
+ },
509
+ {
510
+ "path": "kdflow/trainer/rollout_manager.py",
511
+ "bytes": 6718,
512
+ "api_version_tag": "f6b74523",
513
+ "git_blob": "0eb5253bceb3a06a66cf120a606a221db3a07f81",
514
+ "sha256": "e930b4db59e8f8e09e04838141854403f5516ac8492ef4ab0bd7f37611db7009"
515
+ },
516
+ {
517
+ "path": "kdflow/trainer/verl_agent_alfworld_rollout_manager.py",
518
+ "bytes": 22589,
519
+ "api_version_tag": "db6881e8",
520
+ "git_blob": "430dd71f2979174e05b61d276b4e29d068adc864",
521
+ "sha256": "ea95c95d7c3d8dbcc4da1b578877684a4dcea3e30bf6b68ea98187432194a040"
522
+ },
523
+ {
524
+ "path": "kdflow/utils/__init__.py",
525
+ "bytes": 69,
526
+ "api_version_tag": "5eaed800",
527
+ "git_blob": "1431283d595d0062ee042ea5567c33aec00ad49b",
528
+ "sha256": "9ed449850c624d587592f92760c79c55324063a7cc16fbfa5aab42585165afb1"
529
+ },
530
+ {
531
+ "path": "kdflow/utils/distributed_sampler.py",
532
+ "bytes": 6482,
533
+ "api_version_tag": "a82f95b2",
534
+ "git_blob": "1f7658202804e2a1fbe1a5d0d6ddaf20cc50f3af",
535
+ "sha256": "5a099195c63e475f42fd17e43b4c40ed1133ac3237f623410befffcc9005cda7"
536
+ },
537
+ {
538
+ "path": "kdflow/utils/distributed_util.py",
539
+ "bytes": 1077,
540
+ "api_version_tag": "85f4398a",
541
+ "git_blob": "fd13702ebaa1d34fc1bf87d7c506847af3f7190d",
542
+ "sha256": "6a7850b1490c9122f08384eab1343c908cf526af1fcb997b8898d92325edaf98"
543
+ },
544
+ {
545
+ "path": "kdflow/utils/dynamic_bsz.py",
546
+ "bytes": 15227,
547
+ "api_version_tag": "bcd5639b",
548
+ "git_blob": "44ec3b3a8d3e7f423f3a68b951b7285c463c567b",
549
+ "sha256": "a11fa6064f980d2e614705c9571401de4f18b21e3bbc35a3d69cd7f2ecf4ae9b"
550
+ },
551
+ {
552
+ "path": "kdflow/utils/logging_utils.py",
553
+ "bytes": 3471,
554
+ "api_version_tag": "80b625a3",
555
+ "git_blob": "1393d26d3e95e7ac447bda58242866dac96d2af0",
556
+ "sha256": "e57a5e3bd3df172bb147dfc599f377748e3564de81b800ce365b7a336be89302"
557
+ },
558
+ {
559
+ "path": "kdflow/utils/multimodal_utils.py",
560
+ "bytes": 1073,
561
+ "api_version_tag": "da5520e7",
562
+ "git_blob": "60acb817ed6ce60fda693c80ec1295376c15b10b",
563
+ "sha256": "8b16f06ea69d3c4b2cfb17bb8cb215652350e0aa0548e9f06d07950c07aa34bf"
564
+ },
565
+ {
566
+ "path": "kdflow/utils/utils.py",
567
+ "bytes": 3290,
568
+ "api_version_tag": "3654028c",
569
+ "git_blob": "26415ed5185e779a2945b98fa6f368a726dd286d",
570
+ "sha256": "b4daa8b9e9b405dd330ef3d39d03cc4dac108c05b0012e145fb14baad664fbc9"
571
+ },
572
+ {
573
+ "path": "pyproject.toml",
574
+ "bytes": 730,
575
+ "api_version_tag": "4c685d0b",
576
+ "git_blob": "dd4bfd646653bdc98b8f1bc8aa753a996932166c",
577
+ "sha256": "fb40bc75b604b76e4bc9f5406b1e5fb4721daf4d090543bab430adf4aaaa30f3"
578
+ },
579
+ {
580
+ "path": "requirements.txt",
581
+ "bytes": 6762,
582
+ "api_version_tag": "ea4b60ca",
583
+ "git_blob": "678e55b3cd878b4364ed7bd9d54ac1cf2058af80",
584
+ "sha256": "d4d536aafbe5889185058299ec9deb27ab4903dba3686a14734587703adb2f3f"
585
+ },
586
+ {
587
+ "path": "run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh",
588
+ "bytes": 2414,
589
+ "api_version_tag": "a6482a95",
590
+ "git_blob": "c778963c3e42749205a2fc9b0208bfb076a48180",
591
+ "sha256": "4982257e528fbe508d180172aa81e85e761cbc15328ec2ddc33b7feda42bd2e4"
592
+ },
593
+ {
594
+ "path": "run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh",
595
+ "bytes": 2451,
596
+ "api_version_tag": "bb390a7d",
597
+ "git_blob": "2997aaaebae9914d30b62cca821bce7ceef7818d",
598
+ "sha256": "2f297f306ff7f91d5902cc039bea87cf43474127d6e7598ff50053c5d66551a1"
599
+ }
600
+ ]
601
+ }
kdflow/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ __version__ = "0.1.0"
kdflow/algorithms/__init__.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Algorithms included in the submission."""
2
+
3
+ ALGO_DICT = {}
4
+
5
+
6
+ def register_algorithm(name):
7
+ def decorator(cls):
8
+ ALGO_DICT[name] = cls
9
+ return cls
10
+ return decorator
11
+
12
+
13
+ from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD
14
+ from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD
15
+ from kdflow.algorithms.topk_byte_align_ctkd import TopKByteAlignCrossTokenizerKD
16
+
17
+ __all__ = ["ALGO_DICT", "SimpleCrossTokenizerKD", "ByteAlignCrossTokenizerKD", "TopKByteAlignCrossTokenizerKD"]
kdflow/algorithms/byte_align_ctkd.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kdflow.algorithms import register_algorithm
4
+ from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD
5
+ from kdflow.token_alignment import (
6
+ align_byte_span_batch_1to1,
7
+ align_byte_spans_1to1,
8
+ find_byte_overlap_token_ids,
9
+ get_tokenizer_byte_codec,
10
+ )
11
+ from kdflow.utils.logging_utils import init_logger
12
+
13
+
14
+ logger = init_logger(__name__)
15
+
16
+
17
+ @register_algorithm("byte_align_ctkd")
18
+ class ByteAlignCrossTokenizerKD(SimpleCrossTokenizerKD):
19
+ """Simple cross-tokenizer KD with UTF-8 byte-span 1:1 alignment."""
20
+
21
+ def __init__(
22
+ self,
23
+ strategy,
24
+ student_model,
25
+ teacher_lm_head,
26
+ student_tokenizer,
27
+ teacher_tokenizer,
28
+ **kwargs,
29
+ ):
30
+ # These are initialized before ``super`` because its constructor calls
31
+ # the virtual ``_find_overlap_tokens`` method.
32
+ self.student_byte_codec = get_tokenizer_byte_codec(student_tokenizer)
33
+ self.teacher_byte_codec = get_tokenizer_byte_codec(teacher_tokenizer)
34
+ super().__init__(
35
+ strategy=strategy,
36
+ student_model=student_model,
37
+ teacher_lm_head=teacher_lm_head,
38
+ student_tokenizer=student_tokenizer,
39
+ teacher_tokenizer=teacher_tokenizer,
40
+ **kwargs,
41
+ )
42
+
43
+ def _find_overlap_tokens(self):
44
+ # Preserve the exact historical mapping for all existing ByteLevel
45
+ # pairs. Only mixed/SentencePiece pairs use the byte-canonical map.
46
+ if (
47
+ self.student_byte_codec.name == "byte_level"
48
+ and self.teacher_byte_codec.name == "byte_level"
49
+ ):
50
+ return super()._find_overlap_tokens()
51
+
52
+ student_ids, teacher_ids = find_byte_overlap_token_ids(
53
+ student_tokenizer=self.student_tokenizer,
54
+ teacher_tokenizer=self.teacher_tokenizer,
55
+ student_codec=self.student_byte_codec,
56
+ teacher_codec=self.teacher_byte_codec,
57
+ )
58
+ device = self.teacher_lm_head.weight.device
59
+ logger.info(
60
+ "Num of byte-equivalent overlap tokens between student & teacher: "
61
+ f"{len(student_ids)}"
62
+ )
63
+ return (
64
+ torch.tensor(student_ids, dtype=torch.long, device=device),
65
+ torch.tensor(teacher_ids, dtype=torch.long, device=device),
66
+ )
67
+
68
+ def _align_sequences(self, tea_seq, stu_seq):
69
+ return align_byte_spans_1to1(
70
+ teacher_tokens=tea_seq,
71
+ student_tokens=stu_seq,
72
+ teacher_eos_token=self.teacher_tokenizer.eos_token,
73
+ student_eos_token=self.student_tokenizer.eos_token,
74
+ teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
75
+ student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
76
+ )
77
+
78
+ def _get_alignment_indices(
79
+ self,
80
+ teacher_label_ids,
81
+ student_label_ids,
82
+ teacher_loss_mask,
83
+ student_loss_mask,
84
+ ):
85
+ """Align each micro-batch row independently, then return flat indices."""
86
+ if teacher_label_ids.shape[0] != student_label_ids.shape[0]:
87
+ raise ValueError(
88
+ "Teacher and student micro-batches must contain the same number "
89
+ "of samples for byte_align_ctkd."
90
+ )
91
+
92
+ teacher_batch_tokens = []
93
+ student_batch_tokens = []
94
+ for row_index in range(teacher_label_ids.shape[0]):
95
+ teacher_row_ids = teacher_label_ids[row_index][
96
+ teacher_loss_mask[row_index]
97
+ ]
98
+ student_row_ids = student_label_ids[row_index][
99
+ student_loss_mask[row_index]
100
+ ]
101
+ teacher_row_token_ids = teacher_row_ids.cpu().tolist()
102
+ student_row_token_ids = student_row_ids.cpu().tolist()
103
+ teacher_batch_tokens.append(
104
+ self.teacher_tokenizer.convert_ids_to_tokens(
105
+ teacher_row_token_ids
106
+ )
107
+ )
108
+ student_batch_tokens.append(
109
+ self.student_tokenizer.convert_ids_to_tokens(
110
+ student_row_token_ids
111
+ )
112
+ )
113
+
114
+ return align_byte_span_batch_1to1(
115
+ teacher_batch_tokens=teacher_batch_tokens,
116
+ student_batch_tokens=student_batch_tokens,
117
+ teacher_eos_token=self.teacher_tokenizer.eos_token,
118
+ student_eos_token=self.student_tokenizer.eos_token,
119
+ teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
120
+ student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
121
+ )
kdflow/algorithms/simple_ctkd.py ADDED
@@ -0,0 +1,198 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kdflow.loss import build_loss_fn
4
+ from kdflow.algorithms import register_algorithm
5
+ from kdflow.loss.chunked_loss import chunked_loss
6
+ from kdflow.loss.cross_entropy import compute_cross_entropy
7
+ from kdflow.metrics.entropy import compute_entropy
8
+ from kdflow.utils.logging_utils import init_logger
9
+
10
+
11
+ logger = init_logger(__name__)
12
+
13
+ @register_algorithm("simple_ctkd")
14
+ class SimpleCrossTokenizerKD:
15
+ """Simply find the overlap tokens between student and teacher tokenizer, and only compute KD loss on this sub-vocabulary.
16
+ Motivation: modern LLMs have a large amount of shared tokens.
17
+ """
18
+ def __init__(
19
+ self,
20
+ strategy,
21
+ student_model,
22
+ teacher_lm_head,
23
+ student_tokenizer,
24
+ teacher_tokenizer,
25
+ **kwargs
26
+ ):
27
+ self.strategy = strategy
28
+ self.args = strategy.args
29
+ self.student = student_model
30
+ self.teacher_lm_head = teacher_lm_head
31
+ self.student_tokenizer = student_tokenizer
32
+ self.teacher_tokenizer = teacher_tokenizer
33
+ self.student_overlap_token_ids, self.teacher_overlap_token_ids = self._find_overlap_tokens()
34
+ self.loss_fn = build_loss_fn(self.args.kd.kd_loss_fn, self.args)
35
+ self.metric_fns = [compute_entropy] if self.args.scenario == "on_policy_kd" else []
36
+
37
+ def _find_overlap_tokens(self):
38
+ student_vocab = {k.replace("Ġ", "▁"): v for k, v in self.student_tokenizer.get_vocab().items()}
39
+ teacher_vocab = {k.replace("Ġ", "▁"): v for k, v in self.teacher_tokenizer.get_vocab().items()}
40
+ overlap_tokens = set(student_vocab.keys()) & set(teacher_vocab.keys())
41
+ student_ids = [student_vocab[token] for token in overlap_tokens]
42
+ teacher_ids = [teacher_vocab[token] for token in overlap_tokens]
43
+ stu_eos, tea_eos = self.student_tokenizer.eos_token_id, self.teacher_tokenizer.eos_token_id
44
+ if stu_eos not in student_ids or tea_eos not in teacher_ids:
45
+ student_ids.append(stu_eos)
46
+ teacher_ids.append(tea_eos)
47
+ device = self.teacher_lm_head.weight.device
48
+ logger.info(f"Num of overlap_tokens between student & teacher: {len(student_ids)}")
49
+ return torch.tensor(student_ids, dtype=torch.long, device=device), torch.tensor(teacher_ids, dtype=torch.long, device=device)
50
+
51
+ def _align_sequences(self, tea_seq, stu_seq):
52
+ i, j = 0, 0
53
+ t2s_align, s2t_align = [], []
54
+ history_tea_seq, history_stu_seq = "", ""
55
+
56
+ tea_eos = self.teacher_tokenizer.eos_token
57
+ stu_eos = self.student_tokenizer.eos_token
58
+
59
+ tea_seq = [token.replace('▁', '').replace('Ġ', '') for token in tea_seq]
60
+ stu_seq = [token.replace('▁', '').replace('Ġ', '') for token in stu_seq]
61
+
62
+ if tea_seq == stu_seq:
63
+ indices = list(range(len(tea_seq)))
64
+ return indices, indices
65
+
66
+ while i < len(tea_seq) and j < len(stu_seq):
67
+ is_eos_match = (tea_seq[i] == tea_eos and stu_seq[j] == stu_eos)
68
+ if history_tea_seq == history_stu_seq and (
69
+ tea_seq[i] == stu_seq[j] or is_eos_match
70
+ ):
71
+ common_text = tea_seq[i]
72
+ history_tea_seq += common_text
73
+ history_stu_seq += common_text
74
+ t2s_align.append(i)
75
+ s2t_align.append(j)
76
+ i += 1
77
+ j += 1
78
+ elif len(history_tea_seq) > len(history_stu_seq):
79
+ history_stu_seq += stu_seq[j]
80
+ j += 1
81
+ elif len(history_tea_seq) < len(history_stu_seq):
82
+ history_tea_seq += tea_seq[i]
83
+ i += 1
84
+ else:
85
+ history_tea_seq += tea_seq[i]
86
+ history_stu_seq += stu_seq[j]
87
+ i += 1
88
+ j += 1
89
+
90
+ return t2s_align, s2t_align
91
+
92
+ def _get_alignment_indices(
93
+ self,
94
+ teacher_label_ids,
95
+ student_label_ids,
96
+ teacher_loss_mask,
97
+ student_loss_mask,
98
+ ):
99
+ """Return indices into the flattened teacher/student response tensors."""
100
+ teacher_tokens = self.teacher_tokenizer.convert_ids_to_tokens(
101
+ teacher_label_ids[teacher_loss_mask].cpu().tolist()
102
+ )
103
+ student_tokens = self.student_tokenizer.convert_ids_to_tokens(
104
+ student_label_ids[student_loss_mask].cpu().tolist()
105
+ )
106
+ return self._align_sequences(teacher_tokens, student_tokens)
107
+
108
+ def training_step(self, micro_batch):
109
+ student_input_ids = micro_batch["stu_input_ids"]
110
+ student_attn_mask = micro_batch["stu_attn_mask"]
111
+ student_loss_mask = micro_batch["stu_loss_mask"].bool()
112
+ teacher_input_ids = micro_batch["tea_input_ids"]
113
+ teacher_attn_mask = micro_batch["tea_attn_mask"]
114
+ teacher_loss_mask = micro_batch["tea_loss_mask"].bool()
115
+ teacher_hiddens = micro_batch.get("teacher_hiddens", None)
116
+ avg_token_num = micro_batch["avg_micro_batch_token_num"]
117
+
118
+ assert teacher_hiddens is not None, "micro_batch must contain `teacher_hiddens` for KD"
119
+
120
+ student_shifted_label_ids = student_input_ids.roll(shifts=-1, dims=1)
121
+ teacher_shifted_label_ids = teacher_input_ids.roll(shifts=-1, dims=1)
122
+ teacher_aligned_idx, student_aligned_idx = self._get_alignment_indices(
123
+ teacher_shifted_label_ids,
124
+ student_shifted_label_ids,
125
+ teacher_loss_mask,
126
+ student_loss_mask,
127
+ )
128
+ student_label_ids = student_shifted_label_ids[student_loss_mask]
129
+
130
+ mm_kwargs = micro_batch.get("stu_multi_modal_inputs") or {}
131
+
132
+ output = self.student(
133
+ student_input_ids,
134
+ attention_mask=student_attn_mask,
135
+ allgather_logits=True,
136
+ ring_attn_group=self.strategy.ring_attn_group,
137
+ **mm_kwargs,
138
+ )
139
+ student_hiddens = output["hidden_states"][-1][student_loss_mask]
140
+ del output
141
+
142
+ teacher_hiddens = teacher_hiddens.to(self.teacher_lm_head.weight)
143
+
144
+ align_ratio = torch.tensor(len(student_aligned_idx) / len(student_label_ids))
145
+ aligned_student_hiddens = student_hiddens[student_aligned_idx]
146
+ aligned_teacher_hiddens = teacher_hiddens[teacher_aligned_idx]
147
+
148
+ # A length-truncated cross-tokenizer response may contain no 1:1 byte
149
+ # span (and no appended EOS fallback). Keep the distributed optimizer
150
+ # step well-defined instead of passing a zero chunk size downstream.
151
+ if aligned_student_hiddens.shape[0] == 0:
152
+ zero_loss = student_hiddens.sum() * 0.0
153
+ return {
154
+ "train/loss": zero_loss,
155
+ "train/kd_loss": zero_loss,
156
+ "distill/alignment_ratio": align_ratio,
157
+ }
158
+
159
+ # Non-chunked case can be regarded as a special case of chunked loss (i.e., chunk_size = seq_len)
160
+ chunk_size = self.args.train.chunked_loss_size or aligned_student_hiddens.shape[0]
161
+
162
+ student_overlap_ids = self.student_overlap_token_ids
163
+ teacher_overlap_ids = self.teacher_overlap_token_ids
164
+ student_lm_head = self.student.model.lm_head
165
+ teacher_lm_head = self.teacher_lm_head
166
+
167
+ def student_logits_fn(hidden_chunk, skip=False):
168
+ return student_lm_head(hidden_chunk, skip=skip)[:, student_overlap_ids]
169
+
170
+ def teacher_logits_fn(start, end):
171
+ return teacher_lm_head(aligned_teacher_hiddens[start:end])[:, teacher_overlap_ids]
172
+
173
+ kd_loss, metric_sums = chunked_loss(
174
+ aligned_student_hiddens, student_lm_head, self.loss_fn,
175
+ student_logits_fn=student_logits_fn,
176
+ teacher_logits_fn=teacher_logits_fn,
177
+ chunk_size=chunk_size, reduction="sum",
178
+ metric_fns=self.metric_fns, return_metrics=True,
179
+ )
180
+ kd_loss = kd_loss / avg_token_num
181
+
182
+ loss_info = {
183
+ "train/loss": kd_loss,
184
+ "train/kd_loss": kd_loss,
185
+ "distill/alignment_ratio": align_ratio,
186
+ }
187
+ loss_info.update({key: value / avg_token_num for key, value in metric_sums.items()})
188
+
189
+ if self.args.kd.kd_ratio < 1:
190
+ ce_loss = chunked_loss(
191
+ student_hiddens, self.student.model.lm_head, compute_cross_entropy,
192
+ label=student_label_ids, chunk_size=chunk_size, reduction="sum"
193
+ ) / avg_token_num
194
+ loss = (1 - self.args.kd.kd_ratio) * ce_loss + self.args.kd.kd_ratio * kd_loss
195
+ loss_info["train/loss"] = loss
196
+ loss_info["train/ce_loss"] = ce_loss
197
+
198
+ return loss_info
kdflow/algorithms/topk_byte_align_ctkd.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Tuple
2
+
3
+ import torch
4
+
5
+ from kdflow.algorithms import register_algorithm
6
+ from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD
7
+ from kdflow.metrics.entropy import compute_entropy
8
+
9
+
10
+ def select_student_topk_logits(
11
+ student_logits: torch.Tensor,
12
+ teacher_logits: torch.Tensor,
13
+ topk: int,
14
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
15
+ """Select the student's top-k shared-vocabulary entries on both sides.
16
+
17
+ The returned tensors contain matching vocabulary entries. Downstream KD
18
+ losses apply softmax/log-softmax to these reduced logits, which
19
+ re-normalizes both distributions over exactly this top-k support.
20
+ """
21
+ if student_logits.shape != teacher_logits.shape:
22
+ raise ValueError(
23
+ "Student and teacher shared-vocabulary logits must have the same "
24
+ f"shape, got {student_logits.shape} and {teacher_logits.shape}."
25
+ )
26
+ if topk <= 0 or topk > student_logits.shape[-1]:
27
+ raise ValueError(
28
+ f"topk must be in [1, {student_logits.shape[-1]}], got {topk}."
29
+ )
30
+
31
+ student_topk_logits, topk_indices = torch.topk(
32
+ student_logits,
33
+ k=topk,
34
+ dim=-1,
35
+ )
36
+ teacher_topk_logits = teacher_logits.gather(-1, topk_indices)
37
+ return student_topk_logits, teacher_topk_logits, topk_indices
38
+
39
+
40
+ @register_algorithm("topk_byte_align_ctkd")
41
+ class TopKByteAlignCrossTokenizerKD(ByteAlignCrossTokenizerKD):
42
+ """Byte-aligned CTKD restricted to each student's shared-vocab top-k."""
43
+
44
+ def __init__(self, *args, **kwargs):
45
+ super().__init__(*args, **kwargs)
46
+
47
+ self.ctkd_topk = self.args.kd.ctkd_topk
48
+ overlap_vocab_size = self.student_overlap_token_ids.numel()
49
+ if self.ctkd_topk < 2:
50
+ raise ValueError(
51
+ "ctkd_topk must be at least 2 because re-normalizing a "
52
+ "single logit produces a degenerate distribution."
53
+ )
54
+ if self.ctkd_topk > overlap_vocab_size:
55
+ raise ValueError(
56
+ f"ctkd_topk ({self.ctkd_topk}) exceeds the shared vocabulary "
57
+ f"size ({overlap_vocab_size})."
58
+ )
59
+ if (
60
+ self.args.kd.kd_loss_fn == "hrl"
61
+ and self.args.kd.hrl_topk > self.ctkd_topk
62
+ ):
63
+ raise ValueError(
64
+ f"hrl_topk ({self.args.kd.hrl_topk}) cannot exceed ctkd_topk "
65
+ f"({self.ctkd_topk})."
66
+ )
67
+
68
+ self._shared_vocab_loss_fn = self.loss_fn
69
+ self.loss_fn = self._topk_loss
70
+ if self.metric_fns:
71
+ self.metric_fns = [self._topk_entropy]
72
+
73
+ def _topk_loss(
74
+ self,
75
+ student_logits: torch.Tensor,
76
+ teacher_logits: torch.Tensor,
77
+ **kwargs,
78
+ ) -> torch.Tensor:
79
+ student_topk_logits, teacher_topk_logits, _ = select_student_topk_logits(
80
+ student_logits,
81
+ teacher_logits,
82
+ self.ctkd_topk,
83
+ )
84
+ return self._shared_vocab_loss_fn(
85
+ student_topk_logits,
86
+ teacher_topk_logits,
87
+ **kwargs,
88
+ )
89
+
90
+ def _topk_entropy(
91
+ self,
92
+ student_logits: torch.Tensor,
93
+ teacher_logits: torch.Tensor,
94
+ **kwargs,
95
+ ):
96
+ student_topk_logits, _, _ = select_student_topk_logits(
97
+ student_logits,
98
+ teacher_logits,
99
+ self.ctkd_topk,
100
+ )
101
+ return compute_entropy(student_logits=student_topk_logits)
kdflow/arguments/__init__.py ADDED
@@ -0,0 +1,309 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from dataclasses import dataclass, field
3
+
4
+ from transformers import HfArgumentParser
5
+
6
+ from kdflow.arguments.data_args import DataArguments
7
+ from kdflow.arguments.model_args import ModelArguments
8
+ from kdflow.arguments.training_args import TrainingArguments
9
+ from kdflow.arguments.fsdp_args import FSDPArguments
10
+ from kdflow.arguments.distillation_args import DistillationArguments
11
+ from kdflow.arguments.rollout_args import RolloutArguments
12
+ from kdflow.arguments.logging_args import LoggingArguments
13
+ from kdflow.utils.logging_utils import init_logger
14
+
15
+ logger = init_logger(__name__)
16
+
17
+ @dataclass
18
+ class AllArguments:
19
+ data: DataArguments = field(default_factory=DataArguments)
20
+ model: ModelArguments = field(default_factory=ModelArguments)
21
+ train: TrainingArguments = field(default_factory=TrainingArguments)
22
+ fsdp: FSDPArguments = field(default_factory=FSDPArguments)
23
+ kd: DistillationArguments = field(default_factory=DistillationArguments)
24
+ rollout: RolloutArguments = field(default_factory=RolloutArguments)
25
+ log: LoggingArguments = field(default_factory=LoggingArguments)
26
+
27
+
28
+ def is_linear_attention(model_path: str) -> bool:
29
+ if not model_path:
30
+ return False
31
+ try:
32
+ from transformers import AutoConfig
33
+ model_type = AutoConfig.from_pretrained(model_path, trust_remote_code=True).model_type or ""
34
+ except Exception:
35
+ return False
36
+ return any(k in model_type.lower() for k in ("qwen3_next", "qwen3_5"))
37
+
38
+ def init_args(scenario: str = "on_policy_kd"):
39
+ parser = HfArgumentParser((
40
+ DataArguments,
41
+ ModelArguments,
42
+ TrainingArguments,
43
+ FSDPArguments,
44
+ DistillationArguments,
45
+ RolloutArguments,
46
+ LoggingArguments
47
+ ))
48
+ (
49
+ data_args,
50
+ model_args,
51
+ train_args,
52
+ fsdp_args,
53
+ kd_args,
54
+ rollout_args,
55
+ log_args
56
+ ) = parser.parse_args_into_dataclasses()
57
+
58
+ args = AllArguments(
59
+ data=data_args,
60
+ model=model_args,
61
+ train=train_args,
62
+ fsdp=fsdp_args,
63
+ kd=kd_args,
64
+ rollout=rollout_args,
65
+ log=log_args
66
+ )
67
+
68
+ if scenario != "on_policy_kd" and args.data.custom_eval_fn is not None:
69
+ logger.warning(
70
+ f"`--custom_eval_fn` is only supported in the `on_policy_kd` scenario "
71
+ f"and will be ignored for `{scenario}`."
72
+ )
73
+ if scenario != "on_policy_kd" and args.data.eval_math_dataset_path is not None:
74
+ logger.warning(
75
+ f"`--eval_math_dataset_path` is only supported in the `on_policy_kd` "
76
+ f"scenario and will be ignored for `{scenario}`."
77
+ )
78
+
79
+ if args.data.input_template and "{}" not in args.data.input_template:
80
+ logger.warning("{} not in args.data.input_template, set to None")
81
+ args.data.input_template = None
82
+
83
+ if args.data.input_template and "\\n" in args.data.input_template:
84
+ logger.warning(
85
+ "input_template contains \\n characters instead of newline. "
86
+ "You likely want to pass $'\\n' in Bash or \"`n\" in PowerShell."
87
+ )
88
+
89
+ if args.data.packing_samples:
90
+ if "flash_attention" not in args.model.attn_implementation:
91
+ logger.warning(
92
+ "Please use --attn_implementation with flash_attention to accelerate when --packing_samples is enabled."
93
+ )
94
+ args.model.attn_implementation = "flash_attention_2"
95
+
96
+ if is_linear_attention(args.model.student_name_or_path):
97
+ logger.warning(
98
+ f"--packing_samples is not yet compatible with linear-attention models "
99
+ f"(got {args.model.student_name_or_path}). Auto-disabling --packing_samples."
100
+ )
101
+ args.data.packing_samples = False
102
+
103
+ total_gpus = args.train.num_nodes * args.train.num_gpus_per_node
104
+
105
+ if scenario == "alfworld_eval":
106
+ if not args.model.student_name_or_path:
107
+ raise ValueError(
108
+ "Standalone ALFWorld evaluation requires "
109
+ "`--student_name_or_path`."
110
+ )
111
+ if not args.rollout.alfworld_config:
112
+ raise ValueError(
113
+ "Standalone ALFWorld evaluation requires `--alfworld_config`."
114
+ )
115
+ valid_protocols = {"kdflow", "verl_agent"}
116
+ if args.rollout.alfworld_eval_protocol not in valid_protocols:
117
+ raise ValueError(
118
+ "`--alfworld_eval_protocol` must be one of "
119
+ f"{sorted(valid_protocols)}."
120
+ )
121
+ if args.rollout.alfworld_eval_protocol == "verl_agent":
122
+ try:
123
+ seeds = [
124
+ int(value.strip())
125
+ for value in args.rollout.alfworld_eval_seeds.split(",")
126
+ if value.strip()
127
+ ]
128
+ except ValueError as error:
129
+ raise ValueError(
130
+ "`--alfworld_eval_seeds` must be a comma-separated list "
131
+ "of integers."
132
+ ) from error
133
+ if not seeds:
134
+ raise ValueError(
135
+ "`--alfworld_eval_seeds` must contain at least one seed."
136
+ )
137
+ if args.rollout.alfworld_history_length < 0:
138
+ raise ValueError(
139
+ "`--alfworld_history_length` must be non-negative."
140
+ )
141
+
142
+ if scenario in ("on_policy_kd", "alfworld_eval"):
143
+ if total_gpus <= 0:
144
+ raise ValueError(
145
+ "`--num_nodes * --num_gpus_per_node` must be positive."
146
+ )
147
+ if args.rollout.rollout_tp_size <= 0:
148
+ raise ValueError("`--rollout_tp_size` must be positive.")
149
+ if args.rollout.rollout_batch_size <= 0:
150
+ raise ValueError("`--rollout_batch_size` must be positive.")
151
+ if args.rollout.rollout_engine_concurrency <= 0:
152
+ raise ValueError(
153
+ "`--rollout_engine_concurrency` must be positive."
154
+ )
155
+ if args.rollout.alfworld_config:
156
+ valid_alfworld_protocols = {"kdflow", "verl_agent"}
157
+ if args.rollout.alfworld_protocol not in valid_alfworld_protocols:
158
+ raise ValueError(
159
+ "`--alfworld_protocol` must be one of "
160
+ f"{sorted(valid_alfworld_protocols)}."
161
+ )
162
+ args.rollout.alfworld_config = os.path.abspath(
163
+ os.path.expanduser(args.rollout.alfworld_config)
164
+ )
165
+ if not os.path.isfile(args.rollout.alfworld_config):
166
+ raise ValueError(
167
+ "`--alfworld_config` must point to an existing YAML file, "
168
+ f"got {args.rollout.alfworld_config!r}."
169
+ )
170
+ valid_alfworld_splits = {
171
+ "train",
172
+ "eval_in_distribution",
173
+ "eval_out_of_distribution",
174
+ }
175
+ if args.rollout.alfworld_split not in valid_alfworld_splits:
176
+ raise ValueError(
177
+ "`--alfworld_split` must be one of "
178
+ f"{sorted(valid_alfworld_splits)}."
179
+ )
180
+ if args.rollout.alfworld_max_turns <= 0:
181
+ raise ValueError("`--alfworld_max_turns` must be positive.")
182
+ if args.rollout.alfworld_episodes_per_epoch <= 0:
183
+ raise ValueError(
184
+ "`--alfworld_episodes_per_epoch` must be positive."
185
+ )
186
+ if args.rollout.alfworld_eval_episodes <= 0:
187
+ raise ValueError("`--alfworld_eval_episodes` must be positive.")
188
+ if args.rollout.alfworld_admissible_limit < 0:
189
+ raise ValueError(
190
+ "`--alfworld_admissible_limit` must be non-negative."
191
+ )
192
+ if args.rollout.n_samples_per_prompt != 1:
193
+ raise ValueError(
194
+ "ALFWorld multi-turn rollout requires "
195
+ "`--n_samples_per_prompt 1`; the environment already "
196
+ "samples one independent game per batch slot."
197
+ )
198
+ if scenario == "on_policy_kd" and (
199
+ args.rollout.alfworld_episodes_per_epoch
200
+ < args.rollout.rollout_batch_size
201
+ ):
202
+ raise ValueError(
203
+ "`--alfworld_episodes_per_epoch` must be at least "
204
+ "`--rollout_batch_size`."
205
+ )
206
+ remainder = (
207
+ args.rollout.alfworld_episodes_per_epoch
208
+ % args.rollout.rollout_batch_size
209
+ )
210
+ if scenario == "on_policy_kd" and remainder:
211
+ logger.warning(
212
+ "ALFWorld drops %d episode slots per epoch because "
213
+ "alfworld_episodes_per_epoch is not divisible by "
214
+ "rollout_batch_size.",
215
+ remainder,
216
+ )
217
+ if scenario == "on_policy_kd" and args.data.eval_dataset_path:
218
+ logger.warning(
219
+ "`--eval_dataset_path` is ignored for ALFWorld multi-turn "
220
+ "training; use an ALFWorld evaluation split in a separate run."
221
+ )
222
+
223
+ if (
224
+ scenario == "alfworld_eval"
225
+ and args.rollout.alfworld_eval_episodes
226
+ % args.rollout.rollout_batch_size
227
+ ):
228
+ raise ValueError(
229
+ "`--alfworld_eval_episodes` must be divisible by "
230
+ "`--rollout_batch_size`."
231
+ )
232
+
233
+ if scenario == "on_policy_kd":
234
+ if args.data.eval_math_n_samples <= 0:
235
+ raise ValueError("`--eval_math_n_samples` must be positive.")
236
+ if args.data.eval_math_max_concurrent <= 0:
237
+ raise ValueError("`--eval_math_max_concurrent` must be positive.")
238
+ if args.data.eval_math_max_tokens <= 0:
239
+ raise ValueError("`--eval_math_max_tokens` must be positive.")
240
+ if args.data.eval_math_timeout < 0:
241
+ raise ValueError("`--eval_math_timeout` must be non-negative.")
242
+
243
+ if total_gpus % args.rollout.rollout_tp_size != 0:
244
+ raise ValueError(
245
+ f"Total GPUs ({total_gpus}) must be divisible by rollout_tp_size ({args.rollout.rollout_tp_size})."
246
+ )
247
+
248
+ expected_num_engines = total_gpus // args.rollout.rollout_tp_size
249
+ if args.rollout.rollout_num_engines != expected_num_engines:
250
+ logger.warning(
251
+ f"Auto-adjusting rollout_num_engines from {args.rollout.rollout_num_engines} to {expected_num_engines} "
252
+ f"to match total GPUs ({total_gpus}). "
253
+ f"(rollout_tp_size={args.rollout.rollout_tp_size} * rollout_num_engines={expected_num_engines} = {total_gpus})"
254
+ )
255
+ args.rollout.rollout_num_engines = expected_num_engines
256
+
257
+ if args.data.max_len < args.data.prompt_max_len + args.rollout.generate_max_len:
258
+ args.data.max_len = args.data.prompt_max_len + args.rollout.generate_max_len
259
+ logger.warning(
260
+ "--max_len is smaller than --prompt_max_len + --generate_max_len. "
261
+ f"Automatically increase --max_len to {args.data.max_len}."
262
+ )
263
+
264
+ has_teacher = args.model.teacher_name_or_path is not None
265
+ is_kd_scenario = scenario in ("off_policy_kd", "on_policy_kd")
266
+ if is_kd_scenario:
267
+ if not has_teacher:
268
+ raise ValueError(
269
+ "KD training requires `--teacher_name_or_path` to be set."
270
+ )
271
+
272
+ teacher_parallel = args.kd.teacher_tp_size * args.kd.teacher_pp_size
273
+ if total_gpus % teacher_parallel != 0:
274
+ raise ValueError(
275
+ f"Total GPUs ({total_gpus}) must be divisible by "
276
+ f"teacher_tp_size * teacher_pp_size ({args.kd.teacher_tp_size} * {args.kd.teacher_pp_size} = {teacher_parallel})."
277
+ )
278
+
279
+ if args.kd.teacher_ep_size > args.kd.teacher_tp_size:
280
+ logger.warning(
281
+ f"teacher_ep_size ({args.kd.teacher_ep_size}) must not be greater than teacher_tp_size ({args.kd.teacher_tp_size}). "
282
+ f"Auto-adjusting teacher_ep_size to {args.kd.teacher_tp_size}."
283
+ )
284
+ args.kd.teacher_ep_size = args.kd.teacher_tp_size
285
+
286
+ expected_dp = total_gpus // teacher_parallel
287
+ if args.kd.teacher_dp_size != expected_dp:
288
+ logger.warning(
289
+ f"Auto-adjusting teacher_dp_size from {args.kd.teacher_dp_size} to {expected_dp} "
290
+ f"to match total GPUs ({total_gpus}). "
291
+ f"(tp={args.kd.teacher_tp_size} (ep={args.kd.teacher_ep_size}) * pp={args.kd.teacher_pp_size} * dp={expected_dp} = {total_gpus})"
292
+ )
293
+ args.kd.teacher_dp_size = expected_dp
294
+
295
+
296
+ deprecated_sleep_flags = [
297
+ args.train.train_enable_sleep,
298
+ args.kd.teacher_enable_sleep,
299
+ args.rollout.rollout_enable_sleep,
300
+ ]
301
+ if any(deprecated_sleep_flags):
302
+ logger.warning(
303
+ "--train_enable_sleep, --teacher_enable_sleep, --rollout_enable_sleep are deprecated "
304
+ "and will be removed in a future version. Use --enable_sleep instead."
305
+ )
306
+ args.train.enable_sleep = True
307
+
308
+ args.scenario = scenario
309
+ return args
kdflow/arguments/data_args.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Optional
3
+
4
+
5
+ @dataclass
6
+ class DataArguments:
7
+ """ Arguments for dataset."""
8
+
9
+ train_dataset_path: str = field(
10
+ default=None,
11
+ metadata={"help": "Training dataset name or path."}
12
+ )
13
+ train_dataset_probs: str = field(
14
+ default=None,
15
+ metadata={"help": "Sampling probs for multiple datasets."}
16
+ )
17
+ train_split: str = field(
18
+ default="train",
19
+ metadata={"help": "Train split in dataset."}
20
+ )
21
+ eval_dataset_path: str = field(
22
+ default=None,
23
+ metadata={"help": "Evaluation dataset name or path."}
24
+ )
25
+ eval_split: str = field(
26
+ default="eval",
27
+ metadata={"help": "Eval split in dataset."}
28
+ )
29
+ custom_eval_fn: str = field(
30
+ default=None,
31
+ metadata={
32
+ "help": "Optional Python file defining eval_fn(predictions, labels), "
33
+ "which must return a metric dict (only supported in on-policy KD)."
34
+ }
35
+ )
36
+ eval_math_dataset_path: str = field(
37
+ default=None,
38
+ metadata={
39
+ "help": "Optional math evaluation JSONL path. Each row must contain "
40
+ "'problem' and 'answer' fields (only supported in on-policy KD)."
41
+ }
42
+ )
43
+ eval_math_n_samples: int = field(
44
+ default=1,
45
+ metadata={"help": "Number of generated answers per math problem for pass@k evaluation."}
46
+ )
47
+ eval_math_max_concurrent: int = field(
48
+ default=256,
49
+ metadata={"help": "Maximum number of concurrent math evaluation requests."}
50
+ )
51
+ eval_math_max_tokens: int = field(
52
+ default=8192,
53
+ metadata={"help": "Maximum number of new tokens generated for math evaluation."}
54
+ )
55
+ eval_math_timeout: int = field(
56
+ default=1800,
57
+ metadata={"help": "Total timeout in seconds for math evaluation generation."}
58
+ )
59
+ input_key: str = field(
60
+ default="messages",
61
+ metadata={"help": "JSON dataset key."}
62
+ )
63
+ image_key: str = field(
64
+ default=None,
65
+ metadata={"help": "Image key in dataset."}
66
+ )
67
+ output_key: str = field(
68
+ default=None,
69
+ metadata={"help": "Output key in dataset."}
70
+ )
71
+ teacher_input_key: str = field(
72
+ default=None,
73
+ metadata={"help": "Input key for teacher prompt. If None, use the same input_key as student. "
74
+ "Used in self-distillation where teacher receives richer information."}
75
+ )
76
+ label_key: str = field(
77
+ default=None,
78
+ metadata={"help": "Label key in dataset."}
79
+ )
80
+ teacher_routing_key: str = field(
81
+ default="teacher_routing_key",
82
+ metadata={"help": "Route key for teacher model in multi-teacher distillation."}
83
+ )
84
+ input_template: str = field(
85
+ default=None,
86
+ )
87
+ apply_chat_template: bool = field(
88
+ default=True,
89
+ metadata={"help": "Use HF tokenizer chat template."}
90
+ )
91
+ max_len: int = field(
92
+ default=4096
93
+ )
94
+ max_samples: int = field(
95
+ default=1e8
96
+ )
97
+ packing_samples: bool = field(
98
+ default=False,
99
+ metadata={"help": "Packing sequences during training."}
100
+ )
101
+ prompt_max_len: int = field(
102
+ default=2048,
103
+ metadata={"help": "Max prompt length."}
104
+ )
105
+ preprocess_num_workers: int = field(
106
+ default=1,
107
+ )
kdflow/arguments/distillation_args.py ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ from dataclasses import dataclass, field
3
+
4
+ logger = logging.getLogger(__name__)
5
+
6
+
7
+ @dataclass
8
+ class DistillationArguments:
9
+ """ Arguments for knowledge distillation."""
10
+
11
+ kd_ratio: float = field(
12
+ default=0.5,
13
+ metadata={"help": "Loss = (1 - kd_ratio) * nll_loss + kd_ratio * kd_loss."}
14
+ )
15
+ kd_temperature: float = field(
16
+ default=1.0,
17
+ metadata={"help": "Temperature for knowledge distillation."}
18
+ )
19
+ kd_algorithm: str = field(
20
+ default="byte_align_ctkd",
21
+ metadata={
22
+ "help": "KD algorithm for each training step.",
23
+ "choices": ["simple_ctkd", "byte_align_ctkd", "topk_byte_align_ctkd"],
24
+ }
25
+ )
26
+ kd_loss_fn: str = field(
27
+ default="kl",
28
+ metadata={"help": "Divergence selection for knowledge distillation, e.g., kl, rkl, js."}
29
+ )
30
+ ctkd_topk: int = field(
31
+ default=128,
32
+ metadata={
33
+ "help": (
34
+ "Student top-k size within the shared vocabulary for "
35
+ "topk_byte_align_ctkd."
36
+ )
37
+ },
38
+ )
39
+ teacher_forward_n_batches: int = field(
40
+ default=1,
41
+ metadata={"help": "Teacher forward N global batches at once for student multi-step training."}
42
+ )
43
+ teacher_enable_sleep: bool = field(
44
+ default=False,
45
+ metadata={"help": "Sleep teacher when not needed."}
46
+ )
47
+ teacher_offload_tags: str = field(
48
+ default="all",
49
+ metadata={"help": "Offload tags for sglang."}
50
+ )
51
+ teacher_quantization: str = field(
52
+ default=None
53
+ )
54
+ teacher_tp_size: int = field(
55
+ default=8,
56
+ metadata={"help": "Tensor parallel size for teacher model."}
57
+ )
58
+ teacher_ep_size: int = field(
59
+ default=1,
60
+ metadata={"help": "Expert parallel size for teacher model (only for MoE models)."}
61
+ )
62
+ teacher_pp_size: int = field(
63
+ default=1,
64
+ metadata={"help": "Pipeline parallel size for teacher model."}
65
+ )
66
+ teacher_dp_size: int = field(
67
+ default=1,
68
+ metadata={"help": "Data parallel size for teacher model."}
69
+ )
70
+ teacher_mem_fraction_static: float = field(
71
+ default=0.4,
72
+ metadata={"help": "Memory fraction for teacher model."}
73
+ )
74
+ teacher_update_freq: int = field(
75
+ default=10,
76
+ metadata={"help": "Weight update frequency for teacher model."}
77
+ )
78
+ use_ema_teacher: bool = field(
79
+ default=False,
80
+ metadata={"help": "Use EMA update for teacher model."}
81
+ )
82
+ teacher_ema_decay: float = field(
83
+ default=0.999,
84
+ metadata={"help": "EMA decay for teacher model."}
85
+ )
86
+ multi_teacher_config: str = field(
87
+ default=None,
88
+ metadata={"help": "Unsupported in this release; use --teacher_name_or_path."},
89
+ )
90
+ # JSD
91
+ jsd_beta: float = field(
92
+ default=0.5,
93
+ metadata={"help": "Beta for Jensen-Shannon Divergence."}
94
+ )
95
+ # Skewed KL/RKL
96
+ skew_lambda: float = field(
97
+ default=0.1,
98
+ metadata={"help": "Lambda for Skewed KL/RKL."}
99
+ )
100
+ # Adaptive KL
101
+ adaptive_alpha: float = field(
102
+ default=0.5,
103
+ metadata={"help": "Alpha for Adaptive KL Divergence."}
104
+ )
105
+ # Hierarchical Ranking Loss
106
+ hrl_topk: int = field(
107
+ default=5,
108
+ metadata={"help": "Top-k Ranking for Hierarchical Ranking Loss."}
109
+ )
110
+
111
+ def __post_init__(self):
112
+ if self.multi_teacher_config is not None:
113
+ raise ValueError(
114
+ "This release supports one teacher model per run; "
115
+ "use --teacher_name_or_path instead of --multi_teacher_config."
116
+ )
117
+ supported_algorithms = {"simple_ctkd", "byte_align_ctkd", "topk_byte_align_ctkd"}
118
+ if self.kd_algorithm not in supported_algorithms:
119
+ raise ValueError(
120
+ f"kd_algorithm must be one of {sorted(supported_algorithms)}, "
121
+ f"got {self.kd_algorithm!r}."
122
+ )
123
+ # Validate teacher parallel size settings
124
+ if self.teacher_ep_size > self.teacher_tp_size:
125
+ raise ValueError(
126
+ f"SGLang requires that teacher_ep_size ({self.teacher_ep_size}) must be <= teacher_tp_size ({self.teacher_tp_size}). "
127
+ )
128
+ if self.teacher_tp_size % self.teacher_ep_size != 0:
129
+ raise ValueError(
130
+ f"SGLang requires that teacher_tp_size ({self.teacher_tp_size}) must be divisible by teacher_ep_size ({self.teacher_ep_size})."
131
+ )
132
+ # Validate KD hyperparameters
133
+ if not 0.0 <= self.kd_ratio <= 1.0:
134
+ raise ValueError(f"kd_ratio must be in [0, 1], got {self.kd_ratio}.")
135
+ if self.kd_temperature <= 0:
136
+ raise ValueError(f"kd_temperature must be > 0, got {self.kd_temperature}.")
137
+ if self.ctkd_topk < 2:
138
+ raise ValueError(
139
+ f"ctkd_topk must be at least 2, got {self.ctkd_topk}."
140
+ )
141
+ if not 0.0 < self.teacher_mem_fraction_static <= 1.0:
142
+ raise ValueError(f"teacher_mem_fraction_static must be in (0, 1], got {self.teacher_mem_fraction_static}.")
kdflow/arguments/fsdp_args.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Optional
3
+
4
+
5
+ @dataclass
6
+ class FSDPArguments:
7
+ """ Arguments for fsdp backend."""
8
+
9
+ fsdp_size: int = field(
10
+ default=-1,
11
+ metadata={"help": "FSDP shard size (for HSDP)."}
12
+ )
13
+ cpu_offload: bool = field(
14
+ default=False,
15
+ metadata={"help": "Offload Adam optimizer to GPU."}
16
+ )
17
+
18
+
kdflow/arguments/logging_args.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Optional
3
+
4
+
5
+ @dataclass
6
+ class LoggingArguments:
7
+ """ Arguments for logging (e.g., wandb and tensorboard)."""
8
+
9
+ logging_steps: int = field(
10
+ default=10,
11
+ metadata={"help": "Log results every n steps."}
12
+ )
13
+ use_wandb: bool = field(
14
+ default=False,
15
+ metadata={"help": "Use wandb for logging."}
16
+ )
17
+ wandb_org: str = field(
18
+ default=None
19
+ )
20
+ wandb_project: str = field(
21
+ default=None
22
+ )
23
+ wandb_group: str = field(
24
+ default=None
25
+ )
26
+ wandb_run_name: str = field(
27
+ default=None
28
+ )
29
+ wandb_mode: str = field(
30
+ default="online",
31
+ metadata={"help": "wandb mode: online, offline, or disabled."}
32
+ )
33
+ wandb_dir: str = field(
34
+ default=None,
35
+ metadata={"help": "Directory to store wandb offline logs."}
36
+ )
37
+
kdflow/arguments/model_args.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Optional
3
+
4
+
5
+ @dataclass
6
+ class ModelArguments:
7
+ """ Arguments for model."""
8
+
9
+ student_name_or_path: str = field(
10
+ default=None,
11
+ metadata={"help": "Student model name or path."}
12
+ )
13
+ teacher_name_or_path: str = field(
14
+ default=None,
15
+ metadata={"help": "Teacher model name or path."}
16
+ )
17
+ attn_implementation: str = field(
18
+ default="flash_attention_2",
19
+ metadata={"help": "Attention implementation (e.g., eager, flash_attention_2, flash_attention_3, kernels-community/vllm-flash-attn3)."}
20
+ )
21
+ use_liger_kernel: bool = field(
22
+ default=False,
23
+ metadata={"help": "Use Liger Kernel in LLM."}
24
+ )
25
+ lora_rank: int = field(
26
+ default=0
27
+ )
28
+ lora_alpha: int = field(
29
+ default=16
30
+ )
31
+ target_modules: str = field(
32
+ default="all-linear",
33
+ )
34
+ lora_dropout: float = field(
35
+ default=0.0,
36
+ )
37
+ disable_fast_tokenizer: bool = field(
38
+ default=False
39
+ )
40
+ enable_thinking: bool = field(
41
+ default=False,
42
+ )
43
+ ring_attn_size: int = field(
44
+ default=1
45
+ )
kdflow/arguments/rollout_args.py ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Optional
3
+
4
+
5
+ @dataclass
6
+ class RolloutArguments:
7
+ """ Arguments for rollout (on-policy distillation)."""
8
+
9
+ rollout_num_engines: int = field(
10
+ default=0,
11
+ metadata={"help": "The number of engines for rollout."}
12
+ )
13
+ rollout_tp_size: int = field(
14
+ default=1,
15
+ metadata={"help": "Tensor parallel size for each rollout engine."}
16
+ )
17
+ rollout_engine_concurrency: int = field(
18
+ default=512,
19
+ metadata={"help": "Maximum concurrent requests per rollout engine."}
20
+ )
21
+ rollout_enable_sleep: bool = field(
22
+ default=False,
23
+ metadata={"help": "Enable sleep mode for rollout engines."}
24
+ )
25
+ rollout_mem_fraction_static: float = field(
26
+ default=0.6,
27
+ metadata={"help": "GPU memory utilization for each rollout engine."}
28
+ )
29
+ top_p: float = field(
30
+ default=1.0,
31
+ metadata={"help": "Top-p sampling for rollout."}
32
+ )
33
+ temperature: float = field(
34
+ default=1.0,
35
+ metadata={"help": "Temperature for rollout."}
36
+ )
37
+ n_samples_per_prompt: int = field(
38
+ default=1,
39
+ metadata={"help": "Sample n responses per prompt."}
40
+ )
41
+ rollout_batch_size: int = field(
42
+ default=32,
43
+ metadata={"help": "Number of prompts for each rollout."}
44
+ )
45
+ generate_max_len: int = field(
46
+ default=2048,
47
+ metadata={"help": "Max generation tokens during rollout."}
48
+ )
49
+ print_rollout_sample: bool = field(
50
+ default=False,
51
+ metadata={"help": "Whether to print a rollout sample after each rollout."}
52
+ )
53
+ alfworld_config: Optional[str] = field(
54
+ default=None,
55
+ metadata={
56
+ "help": "Path to an ALFWorld YAML config. When set, on-policy KD "
57
+ "collects multi-turn student trajectories from AlfredTWEnv."
58
+ },
59
+ )
60
+ alfworld_protocol: str = field(
61
+ default="kdflow",
62
+ metadata={
63
+ "help": "ALFWorld training rollout protocol: 'kdflow' keeps the "
64
+ "original Thought/Action chat history, while 'verl_agent' "
65
+ "uses the migrated single-prompt <think>/<action> protocol."
66
+ },
67
+ )
68
+ alfworld_split: str = field(
69
+ default="train",
70
+ metadata={
71
+ "help": "ALFWorld split: train, eval_in_distribution, or "
72
+ "eval_out_of_distribution."
73
+ },
74
+ )
75
+ alfworld_max_turns: int = field(
76
+ default=30,
77
+ metadata={"help": "Maximum assistant/environment turns per ALFWorld episode."},
78
+ )
79
+ alfworld_episodes_per_epoch: int = field(
80
+ default=1024,
81
+ metadata={"help": "Number of ALFWorld episodes sampled per training epoch."},
82
+ )
83
+ alfworld_eval_episodes: int = field(
84
+ default=128,
85
+ metadata={
86
+ "help": "Number of episodes for standalone ALFWorld evaluation."
87
+ },
88
+ )
89
+ alfworld_eval_protocol: str = field(
90
+ default="kdflow",
91
+ metadata={
92
+ "help": "Standalone ALFWorld evaluation protocol. Use 'kdflow' "
93
+ "for the original KDFlow prompt or 'verl_agent' for the "
94
+ "protocol migrated from langfengq/verl-agent."
95
+ },
96
+ )
97
+ alfworld_eval_seeds: str = field(
98
+ default="123,456,789",
99
+ metadata={
100
+ "help": "Comma-separated script-level seeds used by the "
101
+ "verl-agent-compatible evaluator."
102
+ },
103
+ )
104
+ alfworld_eval_seed_offset: int = field(
105
+ default=1000,
106
+ metadata={
107
+ "help": "Offset added to each script-level seed for the validation "
108
+ "environment. verl-agent uses 1000."
109
+ },
110
+ )
111
+ alfworld_eval_resume: bool = field(
112
+ default=True,
113
+ metadata={
114
+ "help": "Reuse completed, configuration-compatible per-seed "
115
+ "ALFWorld evaluation results. Set False to force rerunning "
116
+ "every seed."
117
+ },
118
+ )
119
+ alfworld_history_length: int = field(
120
+ default=4,
121
+ metadata={
122
+ "help": "Number of recent observation/action pairs included by the "
123
+ "verl-agent ALFWorld prompt."
124
+ },
125
+ )
126
+ alfworld_admissible_limit: int = field(
127
+ default=30,
128
+ metadata={
129
+ "help": "Maximum number of ALFWorld admissible actions shown per turn; "
130
+ "use 0 to show all actions."
131
+ },
132
+ )
kdflow/arguments/training_args.py ADDED
@@ -0,0 +1,182 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import math
3
+ from dataclasses import dataclass, field
4
+ from typing import Optional
5
+
6
+
7
+ def resolve_max_steps(
8
+ configured_max_steps: int,
9
+ num_epochs: int,
10
+ steps_per_epoch: int,
11
+ ) -> int:
12
+ """Resolve the explicit step limit or the epoch-derived training length."""
13
+ if steps_per_epoch <= 0:
14
+ raise ValueError(
15
+ "Training requires at least one full global step per epoch. "
16
+ "Reduce the global batch size or provide more training samples."
17
+ )
18
+ if configured_max_steps > 0:
19
+ return configured_max_steps
20
+ return math.ceil(num_epochs * steps_per_epoch)
21
+
22
+
23
+ @dataclass
24
+ class TrainingArguments:
25
+ """ Arguments for training."""
26
+
27
+ num_nodes: int = field(
28
+ default=1,
29
+ metadata={"help": "The number of nodes for the student/teacher model (only for on-policy KD)."}
30
+ )
31
+ num_gpus_per_node: int = field(
32
+ default=8,
33
+ metadata={"help": "The number of gpus per node for the student/teacher model (only for on-policy KD)."}
34
+ )
35
+ num_epochs: int = field(
36
+ default=1,
37
+ metadata={"help": "Number of training epochs."}
38
+ )
39
+ max_steps: int = field(
40
+ default=-1,
41
+ metadata={
42
+ "help": (
43
+ "Maximum global training steps. A positive value overrides "
44
+ "num_epochs; for on-policy KD, one global step is one rollout "
45
+ "iteration."
46
+ )
47
+ },
48
+ )
49
+ train_batch_size: int = field(
50
+ default=128,
51
+ metadata={"help": "Global training batch size."}
52
+ )
53
+ micro_train_batch_size: int = field(
54
+ default=1,
55
+ metadata={"help": "Micro training batch size per GPU."}
56
+ )
57
+ learning_rate: float = field(
58
+ default=1e-6
59
+ )
60
+ lr_warmup_ratio: float = field(
61
+ default=0.05
62
+ )
63
+ min_lr: float = field(
64
+ default=1e-8
65
+ )
66
+ lr_scheduler: str = field(
67
+ default="cosine_with_min_lr"
68
+ )
69
+ max_norm: float = field(
70
+ default=1.0,
71
+ metadata={"help": "Gradient clipping."}
72
+ )
73
+ weight_decay: float = field(
74
+ default=0.0,
75
+ metadata={"help": "Weight decay."}
76
+ )
77
+ adam_betas: str = field(
78
+ default="(0.9, 0.98)"
79
+ )
80
+ eval_steps: int = field(
81
+ default=-1,
82
+ metadata={"help": "Evaluation every n step."}
83
+ )
84
+ save_steps: int = field(
85
+ default=-1,
86
+ metadata={"help": "Save checkpoints every n step."}
87
+ )
88
+ backend: str = field(
89
+ default="fsdp2",
90
+ metadata={
91
+ "choices": ["fsdp2"],
92
+ "help": "Student training backend."
93
+ }
94
+ )
95
+ gradient_checkpointing: bool = field(
96
+ default=False
97
+ )
98
+ gradient_checkpointing_use_reentrant: bool = field(
99
+ default=False
100
+ )
101
+ train_enable_sleep: bool = field(
102
+ default=False,
103
+ metadata={"help": "[Deprecated] Use --enable_sleep instead."}
104
+ )
105
+ enable_sleep: bool = field(
106
+ default=False,
107
+ metadata={"help": "Enable sleep mode for all components (student, teacher, rollout)."}
108
+ )
109
+ full_determinism: bool = field(
110
+ default=False,
111
+ metadata={"help": "Enable reproducible behavior during distributed training."}
112
+ )
113
+ load_checkpoint: bool = field(
114
+ default=False
115
+ )
116
+ ckpt_path: str = field(
117
+ default="./ckpt/checkpoints_distill"
118
+ )
119
+ save_path: str = field(
120
+ default="./ckpt/"
121
+ )
122
+ seed: int = field(
123
+ default=42,
124
+ )
125
+ use_dynamic_bsz: bool = field(
126
+ default=False,
127
+ metadata={"help": "Enable dynamic batch size based on token count per GPU."}
128
+ )
129
+ max_token_len_per_gpu: int = field(
130
+ default=0,
131
+ metadata={"help": "Maximum total token count per micro-batch when use_dynamic_bsz is True."}
132
+ )
133
+ bf16: bool = field(
134
+ default=False
135
+ )
136
+ local_rank: int = field(
137
+ default=int(os.getenv("LOCAL_RANK", -1)),
138
+ metadata={"help": "Local rank for distributed training. Automatically set."}
139
+ )
140
+ chunked_loss_size: int = field(
141
+ default=None,
142
+ metadata={"help": "Token chunk size for chunked loss. If None, chunked loss is disabled."}
143
+ )
144
+
145
+ def __post_init__(self):
146
+ # Ray actors run from a temporary working directory. Resolve output paths
147
+ # in the driver before the arguments are serialized and sent to actors so
148
+ # relative paths keep their command-invocation semantics.
149
+ self.save_path = self._resolve_driver_path(self.save_path)
150
+ self.ckpt_path = self._resolve_driver_path(self.ckpt_path)
151
+
152
+ backend_choices = self.__class__.__dataclass_fields__["backend"].metadata["choices"]
153
+ if self.backend not in backend_choices:
154
+ raise ValueError(
155
+ f"Unsupported training backend '{self.backend}'"
156
+ f"Supported choices: {backend_choices}"
157
+ )
158
+
159
+ if isinstance(self.adam_betas, str):
160
+ try:
161
+ cleaned_str = self.adam_betas.strip().strip('()')
162
+ parts = [p.strip() for p in cleaned_str.split(',')]
163
+ if len(parts) != 2:
164
+ raise ValueError("adam_betas must contain two values.")
165
+
166
+ self.adam_betas = (float(parts[0]), float(parts[1]))
167
+ except Exception as e:
168
+ raise ValueError(
169
+ f"Cannot parse '{self.adam_betas}' to valid adam_betas. Make sure the format to be '(beta1, beta2)'。"
170
+ ) from e
171
+ else:
172
+ raise TypeError(f"Expected str for adam_betas, but get {type(self.adam_betas)}")
173
+
174
+ if self.save_steps <= 0:
175
+ self.save_steps = float("inf")
176
+
177
+ if self.eval_steps <= 0:
178
+ self.eval_steps = float("inf")
179
+
180
+ @staticmethod
181
+ def _resolve_driver_path(path: str) -> str:
182
+ return os.path.abspath(os.path.expanduser(path))
kdflow/backend/__init__.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from kdflow.backend.fsdp import FSDP2Strategy
2
+
3
+
4
+ def get_strategy(args):
5
+ if args.train.backend == "fsdp2":
6
+ strategy = FSDP2Strategy(
7
+ seed=getattr(args.train, "seed", 42),
8
+ full_determinism=getattr(args.train, "full_determinism", False),
9
+ max_norm=getattr(args.train, "max_norm", 1.0),
10
+ micro_train_batch_size=getattr(args.train, "micro_train_batch_size", 1),
11
+ train_batch_size=getattr(args.train, "train_batch_size", 128),
12
+ bf16=getattr(args.train, "bf16", True),
13
+ args=args,
14
+ )
15
+ else:
16
+ raise NotImplementedError
17
+
18
+ return strategy
kdflow/backend/fsdp/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ from .fsdp_strategy import FSDP2Strategy
2
+
3
+ __all__ = [
4
+ "FSDP2Strategy",
5
+ ]
kdflow/backend/fsdp/fsdp_strategy.py ADDED
@@ -0,0 +1,704 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gc
3
+ import functools
4
+ import logging
5
+ import torch
6
+
7
+ import torch.distributed as dist
8
+ import torch.nn as nn
9
+ import transformers
10
+ from abc import ABC
11
+ from datetime import timedelta
12
+ from peft import PeftModel, get_peft_model_state_dict
13
+ from torch.distributed.device_mesh import init_device_mesh, DeviceMesh
14
+ from torch.optim import Optimizer, AdamW
15
+ from torch.distributed.fsdp import (
16
+ fully_shard,
17
+ MixedPrecisionPolicy,
18
+ CPUOffloadPolicy,
19
+ )
20
+ from torch.distributed.tensor import DTensor
21
+ from torch.distributed.checkpoint.state_dict import (
22
+ get_model_state_dict,
23
+ set_model_state_dict,
24
+ StateDictOptions,
25
+ )
26
+ from torchdata.stateful_dataloader import StatefulDataLoader
27
+
28
+ from kdflow.utils.logging_utils import init_logger
29
+ from kdflow.models import DistillModel
30
+ from kdflow.utils.distributed_sampler import DistributedSampler
31
+ from kdflow.models.ring_attn_utils import get_ring_attn_group, set_ring_attn_group
32
+ from kdflow.utils.distributed_util import torch_dist_barrier_and_cuda_sync
33
+
34
+
35
+ class FSDP2Strategy(ABC):
36
+ def __init__(
37
+ self,
38
+ seed: int = 42,
39
+ full_determinism: bool = False,
40
+ max_norm: float = 0.0,
41
+ micro_train_batch_size=1,
42
+ train_batch_size=1,
43
+ bf16=True,
44
+ cpu_offload: bool = False,
45
+ args=None,
46
+ ) -> None:
47
+ self.args = args
48
+ self.seed = seed
49
+ self.max_norm = max_norm
50
+ self.micro_train_batch_size = micro_train_batch_size
51
+ self.train_batch_size = train_batch_size
52
+ self.cpu_offload = cpu_offload
53
+ self.bf16 = bf16
54
+ self.full_determinism = full_determinism
55
+ self.logger = init_logger("FSDP2Strategy")
56
+
57
+ def log(self, msg: str, level: str = "info", rank_0_only: bool = True) -> None:
58
+ """
59
+ Log a message with rank information.
60
+
61
+ Args:
62
+ msg: The message to log.
63
+ level: Log level ('debug', 'info', 'warning', 'error', 'critical').
64
+ rank_0_only: If True, only log on rank 0.
65
+ """
66
+ if rank_0_only and not self.is_rank_0():
67
+ return
68
+
69
+ rank = self.get_rank() if dist.is_initialized() else 0
70
+ full_msg = f"[Rank {rank}] {msg}"
71
+
72
+ log_func = getattr(self.logger, level.lower(), self.logger.info)
73
+ # Use stacklevel=2 to show the caller's location instead of this function's location
74
+ log_func(full_msg, stacklevel=2)
75
+
76
+ def setup_distributed(self, timeout=timedelta(minutes=60)) -> None:
77
+ if self.full_determinism:
78
+ transformers.enable_full_determinism(self.seed)
79
+ # Use deterministic backward in flash attention as, by default, flash attention uses atomic adds
80
+ # https://github.com/Dao-AILab/flash-attention/commit/732654583c2e640adc012ecb60e460bf19dcd9e3
81
+ transformers.modeling_flash_attention_utils.deterministic_g = True
82
+ else:
83
+ transformers.set_seed(self.seed)
84
+
85
+ local_rank = int(os.environ.get("LOCAL_RANK", "-1"))
86
+ if local_rank != -1:
87
+ torch.cuda.set_device(local_rank)
88
+
89
+ if not dist.is_initialized():
90
+ dist.init_process_group(backend="nccl", timeout=timeout)
91
+
92
+ self.world_size = dist.get_world_size()
93
+ self.sp_size = self.args.model.ring_attn_size
94
+ dp_size = self.world_size // self.sp_size
95
+ self.sp_mesh = init_device_mesh("cuda", (dp_size, self.sp_size), mesh_dim_names=("dp", "sp"))
96
+ self.setup_ring_attn(self.sp_mesh)
97
+ if self.args.fsdp.fsdp_size > -1:
98
+ assert self.world_size % self.args.fsdp.fsdp_size == 0
99
+ assert self.world_size >= self.args.fsdp.fsdp_size
100
+ replicate = self.world_size // self.args.fsdp.fsdp_size
101
+ self.fsdp_mesh = init_device_mesh(
102
+ "cuda", (replicate, self.args.fsdp.fsdp_size), mesh_dim_names=("replicate", "sharded"))
103
+ else:
104
+ self.fsdp_mesh = init_device_mesh(
105
+ "cuda", (self.world_size,), mesh_dim_names=("sharded",)
106
+ )
107
+
108
+ self.step = 0
109
+ self.accumulated_gradient = (
110
+ self.train_batch_size
111
+ * self.sp_size
112
+ // self.micro_train_batch_size
113
+ // self.world_size
114
+ )
115
+
116
+ def setup_ring_attn(self, device_mesh):
117
+ if self.sp_size == 1:
118
+ self.ring_attn_rank = 0
119
+ return
120
+
121
+ group = device_mesh["sp"].get_group()
122
+ self.ring_attn_rank = dist.get_rank(group=group)
123
+ set_ring_attn_group(group)
124
+
125
+ from ring_flash_attn import substitute_hf_flash_attn
126
+
127
+ self.ring_head_stride = getattr(self.args.model, "ring_head_stride", 1)
128
+ substitute_hf_flash_attn(self.ring_attn_group, self.ring_head_stride)
129
+
130
+ @property
131
+ def ring_attn_group(self):
132
+ return get_ring_attn_group()
133
+
134
+ def setup_dataloader(
135
+ self,
136
+ replay_buffer,
137
+ batch_size: int,
138
+ pin_memory: bool = False,
139
+ shuffle=True,
140
+ collate_fn=None,
141
+ drop_last=True,
142
+ sampler=None,
143
+ consumed_samples=0,
144
+ ):
145
+ # DDP only mode, replay buffers on each rank are different.
146
+ if sampler is None and dist.is_initialized():
147
+ num_replicas = dist.get_world_size() // self.sp_size
148
+ rank = dist.get_rank() // self.sp_size
149
+
150
+ sampler = DistributedSampler(
151
+ replay_buffer,
152
+ num_replicas=num_replicas,
153
+ rank=rank,
154
+ shuffle=shuffle,
155
+ seed=self.seed,
156
+ drop_last=drop_last,
157
+ consumed_samples=consumed_samples,
158
+ )
159
+ return StatefulDataLoader(
160
+ replay_buffer,
161
+ batch_size=batch_size,
162
+ sampler=sampler,
163
+ drop_last=drop_last,
164
+ shuffle=shuffle if sampler is None else False,
165
+ collate_fn=collate_fn,
166
+ pin_memory=pin_memory,
167
+ )
168
+
169
+ def load_hf_model(self, model_class, model_name_or_path, attn_impl, model_config):
170
+ init_context = self._get_init_weight_context_manager(model_config)
171
+
172
+ with init_context():
173
+ model = model_class.from_pretrained(
174
+ model_name_or_path,
175
+ trust_remote_code=True,
176
+ attn_implementation=attn_impl,
177
+ torch_dtype=torch.bfloat16 if self.args.train.bf16 else "auto",
178
+ )
179
+ # Patch lm_head to skippable mode so that the HF forward does not
180
+ # materialize full-sequence logits. The algorithm layer always calls
181
+ # `lm_head(hidden, skip=False)` explicitly via chunked_loss.
182
+ if hasattr(model, "lm_head"):
183
+ original_forward = model.lm_head.forward
184
+ def skip_forward(x, skip=True):
185
+ if skip:
186
+ return x
187
+ return original_forward(x)
188
+ model.lm_head.forward = skip_forward
189
+ self.log("lm_head forward set to skippable mode for student model.")
190
+ return model
191
+
192
+ def _get_init_weight_context_manager(self, model_config):
193
+ """Get context manager for model initialization.
194
+
195
+ Returns a callable that creates a context manager.
196
+ Uses meta device (no memory allocation) for non-rank-0 processes,
197
+ UNLESS tie_word_embeddings=True (which causes hangs with meta tensors).
198
+
199
+ Ref: verl/utils/fsdp_utils.py::get_init_weight_context_manager
200
+ NOTE: tie_word_embedding causes meta_tensor init to hang
201
+ """
202
+ from accelerate import init_empty_weights
203
+
204
+ # Check if model uses tied word embeddings (which doesn't work with meta tensors)
205
+ use_meta_tensor = not model_config.tie_word_embeddings
206
+
207
+ def cpu_init_weights():
208
+ return torch.device("cpu")
209
+
210
+ if use_meta_tensor:
211
+ # Rank 0: CPU, others: meta device (memory efficient for large models)
212
+ return init_empty_weights if not self.is_rank_0() else cpu_init_weights
213
+ else:
214
+ self.log("tie_word_embeddings=True, loading full model to CPU on all ranks")
215
+ return cpu_init_weights
216
+
217
+ def prepare(self, model: nn.Module, *args, **kwargs):
218
+ for name, param in model.named_parameters():
219
+ if param.requires_grad:
220
+ model = model.float()
221
+ break
222
+
223
+ # Only rank 0 has real weights, others may have meta tensors
224
+ # _fsdp2_load_full_state_dict will broadcast from rank 0
225
+ full_state = model.state_dict()
226
+ self._init_fsdp_kwargs()
227
+ model = self._fsdp2_shard_model(model)
228
+ model = self._fsdp2_load_full_state_dict(
229
+ model, full_state, self.fsdp_mesh,
230
+ cpu_offload=True if self.args.fsdp.cpu_offload else None
231
+ )
232
+
233
+ return model
234
+
235
+ def _init_fsdp_kwargs(self):
236
+ if self.bf16:
237
+ self.mixed_precision_policy = MixedPrecisionPolicy(
238
+ param_dtype=torch.bfloat16,
239
+ reduce_dtype=torch.float32,
240
+ cast_forward_inputs=True
241
+ )
242
+ else:
243
+ self.mixed_precision_policy = None
244
+
245
+ self.offload_policy = CPUOffloadPolicy(pin_memory=True) if self.args.fsdp.cpu_offload else None
246
+
247
+ self.fsdp_kwargs = {
248
+ "mesh": self.fsdp_mesh,
249
+ "mp_policy": self.mixed_precision_policy,
250
+ "offload_policy": self.offload_policy
251
+ }
252
+
253
+ def _fsdp2_shard_model(self, model):
254
+ fsdp_transformer_layer_cls_to_wrap = model.model._no_split_modules
255
+ for _, module in model.model.named_modules():
256
+ if module.__class__.__name__ in fsdp_transformer_layer_cls_to_wrap or (
257
+ isinstance(module, torch.nn.Embedding) and not model.model.config.tie_word_embeddings
258
+ ):
259
+ fully_shard(module, **self.fsdp_kwargs)
260
+ fully_shard(model, **self.fsdp_kwargs)
261
+ return model
262
+
263
+ def _fsdp2_load_full_state_dict(self, model, full_state, device_mesh, cpu_offload):
264
+ """Load full state dict into FSDP2 model with efficient broadcast from rank 0.
265
+
266
+ This function loads weights from rank 0 and broadcasts to all other ranks,
267
+ avoiding the need for each rank to load the full model from disk.
268
+
269
+ Args:
270
+ model: FSDP2-wrapped model
271
+ full_state: State dict (only rank 0 has real weights, others have empty dict)
272
+ device_mesh: Device mesh for FSDP
273
+ cpu_offload: If not None, enables StateDictOptions cpu_offload
274
+
275
+ Ref:verl/utils/fsdp_utils.py::fsdp2_load_full_state_dict
276
+ """
277
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict
278
+
279
+ # Step 1: Allocate tensors on GPU
280
+ # Rank 0: move with weights, others: allocate empty tensors on device (for meta tensors)
281
+ if self.is_rank_0():
282
+ model = model.to(device=torch.cuda.current_device(), non_blocking=True)
283
+ else:
284
+ # to_empty creates tensors on device without initializing memory
285
+ model = model.to_empty(device=torch.cuda.current_device())
286
+
287
+ is_cpu_offload = cpu_offload is not None
288
+ options = StateDictOptions(full_state_dict=True, cpu_offload=is_cpu_offload, broadcast_from_rank0=True)
289
+
290
+ # Step 2: Broadcast weights from rank 0 to all ranks
291
+ set_model_state_dict(model, full_state, options=options)
292
+
293
+ # set_model_state_dict will not broadcast buffers, so we need to broadcast them manually.
294
+ for _name, buf in model.named_buffers():
295
+ dist.broadcast(buf, src=0)
296
+
297
+ if is_cpu_offload:
298
+ model.to("cpu", non_blocking=True)
299
+ for buf in model.buffers():
300
+ buf.data = buf.data.to(torch.cuda.current_device())
301
+
302
+ return model
303
+
304
+ def create_optimizer(self, model, **kwargs) -> Optimizer:
305
+ if isinstance(model, DistillModel):
306
+ model = model.model
307
+
308
+ kwargs["fused"] = True
309
+ optim_params = self._get_optimizer_grouped_parameters(model, kwargs["weight_decay"])
310
+ optim = AdamW(optim_params, **kwargs)
311
+ return optim
312
+
313
+ def _get_optimizer_grouped_parameters(
314
+ self,
315
+ model,
316
+ weight_decay,
317
+ no_decay_name_list=["bias", "layer_norm.weight", "layernorm.weight", "norm.weight", "ln_f.weight"],
318
+ ):
319
+ optimizer_grouped_parameters = [
320
+ {
321
+ "params": [
322
+ p
323
+ for n, p in model.named_parameters()
324
+ if (not any(nd in n for nd in no_decay_name_list) and p.requires_grad)
325
+ ],
326
+ "weight_decay": weight_decay,
327
+ },
328
+ {
329
+ "params": [
330
+ p
331
+ for n, p in model.named_parameters()
332
+ if (any(nd in n for nd in no_decay_name_list) and p.requires_grad)
333
+ ],
334
+ "weight_decay": 0.0,
335
+ },
336
+ ]
337
+ return optimizer_grouped_parameters
338
+
339
+ def _unwrap_model(self, model) -> nn.Module:
340
+ if hasattr(model, "module"):
341
+ return self._unwrap_model(model.module)
342
+ elif isinstance(model, DistillModel):
343
+ return model.model
344
+ else:
345
+ return model
346
+
347
+ def backward(self, loss: torch.Tensor, model: nn.Module, optimizer: Optimizer, **kwargs) -> None:
348
+ self.step = (self.step + 1) % self.accumulated_gradient
349
+ loss = loss / self.accumulated_gradient
350
+ loss.backward()
351
+
352
+ def optimizer_step(
353
+ self,
354
+ optimizer: Optimizer,
355
+ model: nn.Module,
356
+ scheduler,
357
+ **kwargs,
358
+ ) -> None:
359
+ if self.step == 0:
360
+ if self.max_norm > 0.0:
361
+ if hasattr(model, "clip_grad_norm_"):
362
+ model.clip_grad_norm_(self.max_norm)
363
+ else:
364
+ torch.nn.utils.clip_grad_norm_(model.parameters(), self.max_norm)
365
+
366
+ optimizer.step()
367
+ optimizer.zero_grad()
368
+ if scheduler:
369
+ scheduler.step()
370
+
371
+ def load_model(self, model: nn.Module, path: str, map_location="cpu", strict: bool = False, key_replace_fn=None) -> None:
372
+ # For FSDP2, we prefer Distributed Checkpoint (DCP)
373
+ # But if the user provides a standard `torch.save` file path, we try to load it.
374
+ # We use `set_model_state_dict` from DCP which handles sharding.
375
+
376
+ # Load state dict on rank 0 (or all if mapped)
377
+ state_dict = torch.load(path, map_location=map_location)
378
+ if key_replace_fn:
379
+ state_dict = key_replace_fn(state_dict)
380
+
381
+ model_to_load = self._unwrap_model(model)
382
+
383
+ # DCP Helper to load full state dict into sharded model
384
+ # Note: This is memory intensive on Rank 0 if strict full load.
385
+ options = StateDictOptions(full_state_dict=True, cpu_offload=True)
386
+ set_model_state_dict(model_to_load, model_state_dict=state_dict, options=options)
387
+
388
+ def save_model(self, model: nn.Module, tokenizer, output_dir, **kwargs) -> None:
389
+ model_to_save = self._unwrap_model(model)
390
+
391
+ if self.is_rank_0():
392
+ os.makedirs(output_dir, exist_ok=True)
393
+
394
+ options = StateDictOptions(full_state_dict=True, cpu_offload=True)
395
+
396
+ state_dict = get_model_state_dict(model_to_save, options=options)
397
+
398
+ if self.args.train.bf16:
399
+ state_dict = {
400
+ k: v.to(torch.bfloat16) if torch.is_floating_point(v) else v
401
+ for k, v in state_dict.items()
402
+ }
403
+
404
+ if self.is_rank_0():
405
+ if isinstance(model_to_save, PeftModel):
406
+ model_to_save.save_pretrained(
407
+ output_dir,
408
+ state_dict=state_dict,
409
+ safe_serialization=False,
410
+ **kwargs
411
+ )
412
+ else:
413
+ model_to_save.save_pretrained(output_dir, state_dict=state_dict, **kwargs)
414
+
415
+ # Config & Tokenizer
416
+ output_config_file = os.path.join(output_dir, "config.json")
417
+ model_to_save.config.to_json_file(output_config_file)
418
+ tokenizer.save_pretrained(output_dir)
419
+
420
+ del state_dict
421
+ gc.collect()
422
+ torch_dist_barrier_and_cuda_sync()
423
+
424
+ def all_reduce(self, data, op="mean"):
425
+ assert op in ("mean", "max", "sum")
426
+ if isinstance(data, dict):
427
+ ret = {}
428
+ for k, v in data.items():
429
+ ret[k] = self.all_reduce(v, op)
430
+ return ret
431
+ else:
432
+ is_tensor = True
433
+ if not isinstance(data, torch.Tensor):
434
+ data = torch.Tensor([data])
435
+ is_tensor = False
436
+ is_cpu_tensor = data.device.type == "cpu"
437
+
438
+ if is_cpu_tensor:
439
+ data = data.to(torch.cuda.current_device())
440
+ if op == "mean":
441
+ data /= self.world_size
442
+ dist.all_reduce(data, op=dist.ReduceOp.MAX if op == "max" else dist.ReduceOp.SUM)
443
+ if is_cpu_tensor:
444
+ data = data.cpu()
445
+ return data.item() if not is_tensor else data
446
+
447
+ def all_gather(self, data):
448
+ if isinstance(data, dict):
449
+ ret = {}
450
+ for k, v in data.items():
451
+ ret[k] = self.all_gather(v)
452
+ return ret
453
+ else:
454
+ if not isinstance(data, torch.Tensor):
455
+ data = torch.Tensor([data])
456
+ is_cpu_tensor = data.device.type == "cpu"
457
+
458
+ ret = [torch.zeros_like(data).to(torch.cuda.current_device()) for _ in range(self.world_size)]
459
+ dist.all_gather(ret, data.to(torch.cuda.current_device()))
460
+ return torch.cat(ret).cpu() if is_cpu_tensor else torch.cat(ret)
461
+
462
+ def print(self, *msg):
463
+ if self.is_rank_0():
464
+ print(*msg)
465
+
466
+ def is_rank_0(self) -> bool:
467
+ if not dist.is_initialized():
468
+ return True
469
+ return dist.get_rank() == 0
470
+
471
+ def get_rank(self) -> int:
472
+ if not dist.is_initialized():
473
+ return 0
474
+ return dist.get_rank()
475
+
476
+ @torch.no_grad()
477
+ def offload_model_params(self, model, empty_cache: bool = True):
478
+ model.cpu()
479
+ if empty_cache:
480
+ torch.cuda.empty_cache()
481
+
482
+ @torch.no_grad()
483
+ def reload_model_params(self, model):
484
+ device = torch.cuda.current_device()
485
+ model.to(device)
486
+
487
+ @torch.no_grad()
488
+ def offload_optim_states(self, optimizer: Optimizer, empty_cache: bool = True):
489
+ """offload optimizer states to CPU"""
490
+ if not optimizer.state:
491
+ return
492
+ for param_group in optimizer.param_groups:
493
+ for param in param_group["params"]:
494
+ state = optimizer.state[param]
495
+ for key, value in state.items():
496
+ if isinstance(value, torch.Tensor):
497
+ state[key] = value.to("cpu", non_blocking=True)
498
+ if empty_cache:
499
+ torch.cuda.empty_cache()
500
+
501
+ @torch.no_grad()
502
+ def reload_optim_states(self, optimizer: Optimizer):
503
+ """reload optimizer states to GPU"""
504
+ device = torch.cuda.current_device()
505
+ if not optimizer.state:
506
+ return
507
+ for param_group in optimizer.param_groups:
508
+ for param in param_group["params"]:
509
+ state = optimizer.state[param]
510
+ for key, value in state.items():
511
+ if isinstance(value, torch.Tensor):
512
+ state[key] = value.to(device, non_blocking=True)
513
+
514
+ def update_rollout_weights_from_tensor(
515
+ self, model, engine, gather_src, gather_group,
516
+ update_weight_buffer_size=2 * 1024**3,
517
+ weight_source=None
518
+ ):
519
+ """
520
+ Broadcast FSDP model weights to rollout engine with CUDA IPC + Gloo gather.
521
+ Following slime: all ranks serialize their CUDA IPC data, gather to source rank
522
+ via Gloo, then source rank sends to sglang with per-TP-rank IPC handles.
523
+ """
524
+ import ray
525
+ from sglang.srt.utils import MultiprocessingSerializer
526
+
527
+ try:
528
+ from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions
529
+ except ImportError:
530
+ from sglang.srt.patch_torch import monkey_patch_torch_reductions
531
+
532
+ try:
533
+ from sglang.srt.weight_sync.tensor_bucket import FlattenedTensorBucket
534
+ except ImportError:
535
+ from sglang.srt.model_executor.model_runner import FlattenedTensorBucket
536
+
537
+ torch.cuda.empty_cache()
538
+
539
+ model_unwrapped = self._unwrap_model(model)
540
+
541
+ bucket = []
542
+ bucket_size = 0
543
+
544
+ # Detect weight tying: when tie_word_embeddings=True, lm_head.weight
545
+ # and model.embed_tokens.weight share the same parameter. SGLang only
546
+ # stores a single copy, so we must skip the duplicate to avoid KeyError.
547
+ tie_word_embeddings = getattr(model_unwrapped.config, "tie_word_embeddings", False)
548
+
549
+ for name, param in model_unwrapped.state_dict().items():
550
+ if tie_word_embeddings and name == "lm_head.weight":
551
+ continue
552
+ param_size = param.numel() * param.element_size()
553
+ if bucket and bucket_size + param_size >= update_weight_buffer_size:
554
+ self._flush_weight_bucket(
555
+ bucket, engine, gather_src, gather_group,
556
+ monkey_patch_torch_reductions, FlattenedTensorBucket, MultiprocessingSerializer,
557
+ )
558
+ del bucket
559
+ bucket = []
560
+ bucket_size = 0
561
+
562
+ if weight_source is not None and name in weight_source:
563
+ param = weight_source[name]
564
+
565
+ param = param.cuda()
566
+ if isinstance(param, DTensor):
567
+ param = param.redistribute(
568
+ placements=[torch.distributed.tensor.Replicate()] * param.device_mesh.ndim,
569
+ async_op=True,
570
+ ).to_local()
571
+ bucket.append((name, param))
572
+ bucket_size += param_size
573
+
574
+ if bucket:
575
+ self._flush_weight_bucket(
576
+ bucket, engine, gather_src, gather_group,
577
+ monkey_patch_torch_reductions, FlattenedTensorBucket, MultiprocessingSerializer,
578
+ )
579
+ del bucket
580
+
581
+ if dist.get_rank() == gather_src:
582
+ ray.get(engine.flush_cache.remote())
583
+
584
+ torch.cuda.empty_cache()
585
+ torch_dist_barrier_and_cuda_sync()
586
+
587
+ def _flush_weight_bucket(
588
+ self, bucket, engine, gather_src, gather_group,
589
+ monkey_patch_fn, FlattenedTensorBucket, MultiprocessingSerializer,
590
+ ):
591
+ import ray
592
+
593
+ bucket = [(n, p.wait()) if hasattr(p, "wait") else (n, p) for n, p in bucket]
594
+
595
+ monkey_patch_fn()
596
+
597
+ named_tensors_by_dtype = {}
598
+ for name, tensor in bucket:
599
+ named_tensors_by_dtype.setdefault(tensor.dtype, []).append((name, tensor))
600
+
601
+ serialized_tensors = []
602
+ for _dtype, named_tensors in named_tensors_by_dtype.items():
603
+ fb = FlattenedTensorBucket(named_tensors=named_tensors)
604
+ serialized = MultiprocessingSerializer.serialize(
605
+ {"flattened_tensor": fb.get_flattened_tensor(), "metadata": fb.get_metadata()},
606
+ output_str=True,
607
+ )
608
+ serialized_tensors.append(serialized)
609
+ del fb
610
+
611
+ if dist.get_rank() == gather_src:
612
+ gathered = [None for _ in range(dist.get_world_size(gather_group))]
613
+ else:
614
+ gathered = None
615
+
616
+ dist.gather_object(
617
+ obj=serialized_tensors,
618
+ object_gather_list=gathered,
619
+ dst=gather_src,
620
+ group=gather_group,
621
+ )
622
+
623
+ if dist.get_rank() == gather_src:
624
+ num_dtypes = len(gathered[0])
625
+ for i in range(num_dtypes):
626
+ ref = engine.update_weights_from_tensor.remote(
627
+ serialized_named_tensors=[tensors[i] for tensors in gathered],
628
+ load_format="flattened_bucket",
629
+ flush_cache=False,
630
+ )
631
+ ray.get(ref)
632
+ del gathered
633
+
634
+ del bucket, serialized_tensors
635
+
636
+ def update_rollout_weights_from_disk(self, model, rollout_engines, tokenizer=None, tmp_dir=None):
637
+ """
638
+ Sync FSDP model weights to rollout engines via disk (safetensors).
639
+
640
+ Rank 0 gathers the full state dict and saves it to a shared filesystem path,
641
+ then all rollout engines load the weights from disk via sglang's
642
+ update_weights_from_disk API. This completely avoids CUDA IPC issues.
643
+
644
+ Args:
645
+ model: The FSDP-wrapped model.
646
+ rollout_engines: List of RolloutRayActor instances.
647
+ tokenizer: Tokenizer (needed for save_pretrained to write tokenizer_config.json etc.).
648
+ tmp_dir: Directory to save the temporary weights. If None, uses a default path.
649
+ """
650
+ import ray
651
+ import shutil
652
+
653
+ if tmp_dir is None:
654
+ tmp_dir = os.path.join(
655
+ os.environ.get("TMPDIR", "/tmp"),
656
+ "kdflow_rollout_weights_sync",
657
+ )
658
+
659
+ model_unwrapped = self._unwrap_model(model)
660
+ is_sender = len(rollout_engines) > 0
661
+
662
+ # Step 1: Gather full state dict (all ranks participate in FSDP collective)
663
+ options = StateDictOptions(full_state_dict=True, cpu_offload=True)
664
+ state_dict = get_model_state_dict(model_unwrapped, options=options)
665
+
666
+ # Step 2: Rank 0 saves weights to shared filesystem
667
+ if self.is_rank_0():
668
+ os.makedirs(tmp_dir, exist_ok=True)
669
+ model_unwrapped.save_pretrained(
670
+ tmp_dir,
671
+ state_dict=state_dict,
672
+ safe_serialization=True,
673
+ )
674
+ # Save config (needed for sglang to load correctly)
675
+ model_unwrapped.config.to_json_file(os.path.join(tmp_dir, "config.json"))
676
+ if tokenizer is not None:
677
+ tokenizer.save_pretrained(tmp_dir)
678
+
679
+ del state_dict
680
+ gc.collect()
681
+ torch_dist_barrier_and_cuda_sync()
682
+
683
+ # Step 3: Tell all rollout engines to load weights from disk
684
+ if is_sender:
685
+ refs = [
686
+ engine.update_weights_from_disk.remote(
687
+ model_path=tmp_dir,
688
+ load_format=None,
689
+ )
690
+ for engine in rollout_engines
691
+ ]
692
+ ray.get(refs)
693
+
694
+ torch_dist_barrier_and_cuda_sync()
695
+
696
+ # Step 4: Cleanup temporary directory
697
+ if self.is_rank_0():
698
+ try:
699
+ shutil.rmtree(tmp_dir, ignore_errors=True)
700
+ except Exception:
701
+ pass
702
+
703
+ torch.cuda.empty_cache()
704
+ torch_dist_barrier_and_cuda_sync()
kdflow/backend/sglang/__init__.py ADDED
File without changes
kdflow/backend/sglang/monkey_patch.py ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Monkey patch for SGLang scheduler's process_batch_result_prefill method.
3
+ This allows using numpy() instead of tolist() for hidden_states, which is much faster.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import logging
9
+ import os
10
+ import time
11
+ from typing import TYPE_CHECKING, List, Optional, Tuple, Union
12
+
13
+ import torch
14
+
15
+ from kdflow.utils.logging_utils import init_logger
16
+
17
+ if TYPE_CHECKING:
18
+ from sglang.srt.managers.scheduler import (
19
+ EmbeddingBatchResult,
20
+ GenerationBatchResult,
21
+ ScheduleBatch,
22
+ Scheduler,
23
+ )
24
+
25
+ logger = init_logger(__name__)
26
+
27
+ # Flag to prevent multiple patch applications
28
+ _PATCH_APPLIED = False
29
+
30
+
31
+ def process_batch_result_prefill_patched(
32
+ self: "Scheduler",
33
+ batch: "ScheduleBatch",
34
+ result: Union["GenerationBatchResult", "EmbeddingBatchResult"],
35
+ ):
36
+ """
37
+ Patched version of process_batch_result_prefill.
38
+ Key change: Use .numpy() instead of .tolist() for hidden_states (much faster).
39
+ """
40
+ from sglang.srt.environ import envs
41
+ from sglang.srt.managers.io_struct import AbortReq
42
+ from sglang.srt.mem_cache.common import release_kv_cache
43
+
44
+ skip_stream_req = None
45
+
46
+ if self.is_generation:
47
+ if result.copy_done is not None:
48
+ result.copy_done.synchronize()
49
+
50
+ (
51
+ logits_output,
52
+ next_token_ids,
53
+ extend_input_len_per_req,
54
+ extend_logprob_start_len_per_req,
55
+ ) = (
56
+ result.logits_output,
57
+ result.next_token_ids,
58
+ result.extend_input_len_per_req,
59
+ result.extend_logprob_start_len_per_req,
60
+ )
61
+
62
+ # Move next_token_ids and logprobs to cpu
63
+ next_token_ids = next_token_ids.tolist()
64
+ if batch.return_logprob:
65
+ if logits_output.next_token_logprobs is not None:
66
+ logits_output.next_token_logprobs = (
67
+ logits_output.next_token_logprobs.tolist()
68
+ )
69
+ if logits_output.input_token_logprobs is not None:
70
+ logits_output.input_token_logprobs = tuple(
71
+ logits_output.input_token_logprobs.tolist()
72
+ )
73
+
74
+ hidden_state_offset = 0
75
+
76
+ # Check finish conditions
77
+ logprob_pt = 0
78
+
79
+ for i, (req, next_token_id) in enumerate(zip(batch.reqs, next_token_ids)):
80
+ if req.finished() or req.is_retracted:
81
+ # decode req in mixed batch or retracted req
82
+ continue
83
+
84
+ if req.is_chunked <= 0:
85
+ if hasattr(req.time_stats, "prefill_finished_ts"):
86
+ if req.time_stats.prefill_finished_ts == 0.0:
87
+ req.time_stats.prefill_finished_ts = time.time()
88
+ else:
89
+ req.time_stats.set_prefill_finished_time()
90
+
91
+ # req output_ids are set here
92
+ req.output_ids.append(next_token_id)
93
+ req.check_finished()
94
+
95
+ if req.finished():
96
+ self.maybe_collect_routed_experts(req)
97
+ release_kv_cache(req, self.tree_cache)
98
+ req.time_stats.completion_time = time.perf_counter()
99
+ elif not batch.decoding_reqs or req not in batch.decoding_reqs:
100
+ # This updates radix so others can match
101
+ self.tree_cache.cache_unfinished_req(req)
102
+
103
+ self.maybe_collect_customized_info(i, req, logits_output)
104
+
105
+ if batch.return_logprob:
106
+ assert extend_logprob_start_len_per_req is not None
107
+ assert extend_input_len_per_req is not None
108
+ extend_logprob_start_len = extend_logprob_start_len_per_req[i]
109
+ extend_input_len = extend_input_len_per_req[i]
110
+
111
+ num_input_logprobs = self._calculate_num_input_logprobs(
112
+ req, extend_input_len, extend_logprob_start_len
113
+ )
114
+
115
+ if req.return_logprob:
116
+ self.add_logprob_return_values(
117
+ i,
118
+ req,
119
+ logprob_pt,
120
+ next_token_ids,
121
+ num_input_logprobs,
122
+ logits_output,
123
+ )
124
+ logprob_pt += num_input_logprobs
125
+
126
+ if (
127
+ req.return_hidden_states
128
+ and logits_output.hidden_states is not None
129
+ ):
130
+ req.hidden_states.append(
131
+ logits_output.hidden_states[
132
+ hidden_state_offset : (
133
+ hidden_state_offset := hidden_state_offset
134
+ + len(req.origin_input_ids)
135
+ )
136
+ ]
137
+ .half()
138
+ .cpu()
139
+ .numpy()
140
+ )
141
+
142
+ if req.grammar is not None:
143
+ # FIXME: this try-except block is for handling unexpected xgrammar issue.
144
+ try:
145
+ req.grammar.accept_token(next_token_id)
146
+ except ValueError as e:
147
+ # Grammar accept_token can raise ValueError if the token is not in the grammar.
148
+ # This can happen if the grammar is not set correctly or the token is invalid.
149
+ logger.error(
150
+ f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}"
151
+ )
152
+ self.abort_request(AbortReq(rid=req.rid))
153
+ req.grammar.finished = req.finished()
154
+
155
+ else:
156
+ # being chunked reqs' prefill is not finished
157
+ req.is_chunked -= 1
158
+ # There is only at most one request being currently chunked.
159
+ # Because this request does not finish prefill,
160
+ # we don't want to stream the request currently being chunked.
161
+ skip_stream_req = req
162
+
163
+ # Incrementally update input logprobs.
164
+ if batch.return_logprob:
165
+ extend_logprob_start_len = extend_logprob_start_len_per_req[i]
166
+ extend_input_len = extend_input_len_per_req[i]
167
+ if extend_logprob_start_len < extend_input_len:
168
+ # Update input logprobs.
169
+ num_input_logprobs = self._calculate_num_input_logprobs(
170
+ req, extend_input_len, extend_logprob_start_len
171
+ )
172
+ if req.return_logprob:
173
+ self.add_input_logprob_return_values(
174
+ i,
175
+ req,
176
+ logits_output,
177
+ logprob_pt,
178
+ num_input_logprobs,
179
+ last_prefill_chunk=False,
180
+ )
181
+ logprob_pt += num_input_logprobs
182
+
183
+ else: # embedding or reward model
184
+ if result.copy_done is not None:
185
+ result.copy_done.synchronize()
186
+
187
+ is_sparse = envs.SGLANG_EMBEDDINGS_SPARSE_HEAD.is_set()
188
+
189
+ embeddings = result.embeddings
190
+
191
+ if is_sparse:
192
+ batch_ids, token_ids = embeddings.indices()
193
+ values = embeddings.values()
194
+
195
+ embeddings = [{} for _ in range(embeddings.size(0))]
196
+ for i in range(batch_ids.shape[0]):
197
+ embeddings[batch_ids[i].item()][token_ids[i].item()] = values[
198
+ i
199
+ ].item()
200
+ else:
201
+ if isinstance(embeddings, torch.Tensor):
202
+ embeddings = embeddings.tolist()
203
+ else:
204
+ embeddings = [tensor.tolist() for tensor in embeddings]
205
+
206
+ # Check finish conditions
207
+ for i, req in enumerate(batch.reqs):
208
+ if req.is_retracted:
209
+ continue
210
+
211
+ req.embedding = embeddings[i]
212
+ if req.is_chunked <= 0:
213
+ # Dummy output token for embedding models
214
+ req.output_ids.append(0)
215
+ req.check_finished()
216
+
217
+ if req.finished():
218
+ release_kv_cache(req, self.tree_cache)
219
+ else:
220
+ self.tree_cache.cache_unfinished_req(req)
221
+ else:
222
+ # being chunked reqs' prefill is not finished
223
+ req.is_chunked -= 1
224
+
225
+ self.stream_output(batch.reqs, batch.return_logprob, skip_stream_req)
226
+
227
+ if self.current_scheduler_metrics_enabled:
228
+ can_run_cuda_graph = getattr(result, "can_run_cuda_graph", False)
229
+ self.log_prefill_stats(
230
+ prefill_stats=batch.prefill_stats,
231
+ can_run_cuda_graph=can_run_cuda_graph,
232
+ dp_cooperation_info=batch.dp_cooperation_info,
233
+ )
234
+
235
+
236
+ def apply_patch():
237
+ """
238
+ Apply the monkey patch to SGLang's SchedulerOutputProcessorMixin.
239
+
240
+ This function is idempotent - calling it multiple times is safe.
241
+ Returns True if patch was applied (or already applied), False otherwise.
242
+ """
243
+ global _PATCH_APPLIED
244
+
245
+ if _PATCH_APPLIED:
246
+ return True
247
+
248
+ try:
249
+ from sglang.srt.managers.scheduler_output_processor_mixin import (
250
+ SchedulerOutputProcessorMixin,
251
+ )
252
+
253
+ # Check if already patched (by another mechanism like sitecustomize)
254
+ current_method = getattr(SchedulerOutputProcessorMixin, 'process_batch_result_prefill', None)
255
+ if current_method is not None and getattr(current_method, '_kdflow_patched', False):
256
+ _PATCH_APPLIED = True
257
+ print(f"[monkey_patch] Patch already applied, PID={os.getpid()}", flush=True)
258
+ return True
259
+
260
+ # Mark the patched function
261
+ process_batch_result_prefill_patched._kdflow_patched = True
262
+
263
+ # Apply patch
264
+ SchedulerOutputProcessorMixin.process_batch_result_prefill = process_batch_result_prefill_patched
265
+
266
+ _PATCH_APPLIED = True
267
+ print(f"[monkey_patch] SUCCESS: process_batch_result_prefill patched! PID={os.getpid()}", flush=True)
268
+ return True
269
+
270
+ except ImportError as e:
271
+ print(f"[monkey_patch] Cannot import SGLang module: {e}", flush=True)
272
+ return False
273
+ except Exception as e:
274
+ print(f"[monkey_patch] Error applying patch: {e}", flush=True)
275
+ import traceback
276
+ traceback.print_exc()
277
+ return False
278
+
279
+
280
+ def is_patch_applied():
281
+ """Check if the patch has been applied in this process."""
282
+ return _PATCH_APPLIED
kdflow/backend/sglang/sglang_engine.py ADDED
@@ -0,0 +1,437 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import queue
3
+ import time
4
+ import torch.multiprocessing as mp
5
+ from torch.multiprocessing import Queue
6
+ from typing import List, Dict, Any, Tuple, Optional
7
+ from dataclasses import dataclass
8
+
9
+ import numpy as np
10
+ import torch
11
+ from sglang.srt.entrypoints.engine import Engine as _SglEngine
12
+ from sglang.srt.managers.scheduler import run_scheduler_process as _original_run_scheduler_process
13
+
14
+ from kdflow.utils.logging_utils import init_logger
15
+
16
+ logger = init_logger(__name__)
17
+
18
+
19
+ os.environ["SGLANG_JIT_DEEPGEMM_FAST_WARMUP"] = "true"
20
+
21
+ def _patched_run_scheduler_process(*args, **kwargs):
22
+ try:
23
+ from kdflow.backend.sglang.monkey_patch import apply_patch
24
+ apply_patch()
25
+ except Exception as e:
26
+ logger.warning(f"[PatchedEngine] WARNING: Failed to apply monkey patch (PID={os.getpid()}): {e}", flush=True)
27
+ return _original_run_scheduler_process(*args, **kwargs)
28
+
29
+
30
+ class PatchedEngine(_SglEngine):
31
+ """
32
+ SGLang Engine that applies monkey patch in scheduler subprocesses.
33
+ Motivation: SGLang Engine supports returning hidden states, but the existing implementation use .tolist() to convert hidden states from GPU tensor to Python list, which is very inefficient. This monkey patch replaces the original .tolist() with a more efficient operation .numpy().
34
+ """
35
+ run_scheduler_process_func = staticmethod(_patched_run_scheduler_process)
36
+
37
+
38
+ @dataclass
39
+ class EngineConfig:
40
+ """Configuration for SGLang Engine."""
41
+ model_path: str
42
+ tp_size: int = 1
43
+ ep_size: int = 1
44
+ pp_size: int = 1
45
+ chunked_prefill_size: int = -1
46
+ disable_radix_cache: bool = True
47
+ enable_return_hidden_states: bool = True
48
+ enable_memory_saver: bool = True
49
+ enable_weights_cpu_backup: bool = True
50
+ mem_fraction_static: float = 0.8
51
+ quantization: str = None
52
+ offload_tags: Optional[str] = "all"
53
+ base_gpu_id: int = 0
54
+ # for multi-node tp/pp
55
+ nnodes: int = 1
56
+ node_rank: int = 0
57
+ dist_init_addr: Optional[str] = None
58
+
59
+
60
+ def _engine_worker(config: EngineConfig, request_queue: Queue, response_queue: Queue, hidden_queue: Queue):
61
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
62
+ if config.nnodes > 1:
63
+ os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0"
64
+
65
+ engine = None
66
+
67
+ try:
68
+ engine = PatchedEngine(
69
+ model_path=config.model_path,
70
+ tp_size=config.tp_size,
71
+ ep_size=config.ep_size,
72
+ pp_size=config.pp_size,
73
+ chunked_prefill_size=config.chunked_prefill_size,
74
+ disable_radix_cache=config.disable_radix_cache,
75
+ enable_return_hidden_states=config.enable_return_hidden_states,
76
+ enable_memory_saver=config.enable_memory_saver,
77
+ enable_weights_cpu_backup=config.enable_weights_cpu_backup,
78
+ quantization=config.quantization,
79
+ mem_fraction_static=config.mem_fraction_static,
80
+ base_gpu_id=config.base_gpu_id,
81
+ nnodes=config.nnodes,
82
+ node_rank=config.node_rank,
83
+ dist_init_addr=config.dist_init_addr,
84
+ disable_custom_all_reduce=True,
85
+ )
86
+
87
+ response_queue.put({
88
+ "type": "init_done",
89
+ "success": True,
90
+ })
91
+
92
+ while True:
93
+ request = request_queue.get()
94
+ if request is None:
95
+ break
96
+
97
+ req_type = request.get("type")
98
+
99
+ try:
100
+ if req_type == "generate":
101
+ _handle_generate(engine, request, hidden_queue, response_queue)
102
+ elif req_type == "sleep":
103
+ _handle_sleep(engine, request, config, response_queue)
104
+ elif req_type == "wakeup":
105
+ _handle_wakeup(engine, request, config, response_queue)
106
+ elif req_type == "update_weights_from_tensor":
107
+ _handle_update_weights_from_tensor(engine, request, response_queue)
108
+ else:
109
+ response_queue.put({"type": req_type, "success": False,
110
+ "error": f"Unknown request type: {req_type}"})
111
+ except Exception:
112
+ import traceback
113
+ response_queue.put({"type": req_type, "success": False,
114
+ "error": traceback.format_exc()})
115
+
116
+ except Exception:
117
+ import traceback
118
+ response_queue.put({"type": "init_done", "success": False,
119
+ "error": traceback.format_exc()})
120
+ finally:
121
+ if engine:
122
+ try:
123
+ engine.shutdown()
124
+ except Exception:
125
+ pass
126
+
127
+
128
+ def _normalize_tags(tags):
129
+ """Convert tags to the format SGLang expects (None, or list of strings)."""
130
+ if tags is None or tags == "all":
131
+ return None
132
+ if isinstance(tags, str):
133
+ return [tags]
134
+ return tags
135
+
136
+
137
+ def _handle_generate(engine, request, hidden_queue, response_queue):
138
+ """Handle a generate request and stream hidden states via shared memory."""
139
+ kwargs = request["kwargs"]
140
+
141
+ generate_kwargs = {
142
+ "input_ids": kwargs["input_ids"],
143
+ "sampling_params": kwargs["sampling_params"],
144
+ "return_hidden_states": kwargs.get("return_hidden_states", True),
145
+ }
146
+ if kwargs.get("image_data") is not None:
147
+ generate_kwargs["image_data"] = kwargs["image_data"]
148
+
149
+ outputs = engine.generate(**generate_kwargs, stream=True)
150
+ num_samples = len(kwargs["loss_masks"])
151
+
152
+ response_queue.put({
153
+ "type": "generate",
154
+ "success": True,
155
+ "num_samples": num_samples,
156
+ })
157
+
158
+ received_indices = set()
159
+ try:
160
+ for output in outputs:
161
+ meta_info = output["meta_info"]
162
+ if meta_info.get("finish_reason") is None:
163
+ continue
164
+
165
+ idx = output["index"]
166
+ if idx in received_indices:
167
+ raise RuntimeError(f"Duplicate SGLang output index: {idx}")
168
+
169
+ mask = kwargs["loss_masks"][idx]
170
+ hs_np = meta_info["hidden_states"][0]
171
+
172
+ # hs_np and mask may differ due to multimodal token expansion.
173
+ hs_len = hs_np.shape[0]
174
+ mask_len = mask.shape[0]
175
+ if hs_len == mask_len:
176
+ hs_np = hs_np[mask]
177
+ else:
178
+ num_loss_tokens = int(mask.sum())
179
+ logger.warning(
180
+ f"[_handle_generate] sample={idx}/{num_samples} length mismatch: "
181
+ f"hs_len={hs_len}, mask_len={mask_len}, diff={mask_len - hs_len}; "
182
+ f"selecting {num_loss_tokens} loss tokens from the tail"
183
+ )
184
+ if num_loss_tokens >= hs_len:
185
+ raise ValueError(
186
+ f"Cannot select {num_loss_tokens} loss tokens from "
187
+ f"hidden states of length {hs_len}"
188
+ )
189
+ # The loss span is a contiguous suffix before the final token.
190
+ hs_np = hs_np[-num_loss_tokens - 1:-1]
191
+
192
+ if not hs_np.flags['C_CONTIGUOUS']:
193
+ hs_np = np.ascontiguousarray(hs_np)
194
+
195
+ hs_tensor = torch.from_numpy(hs_np).share_memory_()
196
+ hidden_queue.put((idx, hs_tensor))
197
+ received_indices.add(idx)
198
+
199
+ if len(received_indices) != num_samples:
200
+ raise RuntimeError(
201
+ f"SGLang returned {len(received_indices)} final outputs for "
202
+ f"{num_samples} samples"
203
+ )
204
+ hidden_queue.put(None)
205
+ except Exception:
206
+ import traceback
207
+ error = traceback.format_exc()
208
+ logger.error(f"[_handle_generate] ERROR:\n{error}")
209
+ hidden_queue.put(error)
210
+
211
+
212
+ def _handle_sleep(engine, request, config, response_queue):
213
+ """Handle a sleep request: offload GPU memory."""
214
+ tags = request.get("tags", config.offload_tags)
215
+ torch.cuda.empty_cache()
216
+ engine.release_memory_occupation(tags=_normalize_tags(tags))
217
+ response_queue.put({"type": "sleep", "success": True, "tags": tags})
218
+
219
+
220
+ def _handle_wakeup(engine, request, config, response_queue):
221
+ """Handle a wakeup request: restore GPU memory."""
222
+ tags = request.get("tags", config.offload_tags)
223
+ torch.cuda.empty_cache()
224
+ engine.resume_memory_occupation(tags=_normalize_tags(tags))
225
+ response_queue.put({"type": "wakeup", "success": True, "tags": tags})
226
+
227
+
228
+ def _handle_update_weights_from_tensor(engine, request, response_queue):
229
+ """Handle a update_weights_from_tensor request: update weights from student (for self-distillation)."""
230
+ serialized_named_tensors = request["kwargs"]["serialized_named_tensors"]
231
+ load_format = request["kwargs"]["load_format"]
232
+ flush_cache = request["kwargs"]["flush_cache"]
233
+ engine.update_weights_from_tensor(
234
+ named_tensors=serialized_named_tensors,
235
+ load_format=load_format,
236
+ flush_cache=flush_cache,
237
+ )
238
+ response_queue.put({"type": "update_weights_from_tensor", "success": True})
239
+
240
+
241
+ class SGLangEngineService:
242
+ """Manages SGLang Engine in a subprocess with torch multiprocessing communication."""
243
+
244
+ def __init__(self, config: EngineConfig):
245
+ self.config = config
246
+ self.process: Optional[mp.Process] = None
247
+ self.request_queue: Optional[Queue] = None
248
+ self.response_queue: Optional[Queue] = None
249
+ self.hidden_queue: Optional[Queue] = None
250
+ self._started = False
251
+
252
+ def start(self, timeout: float = 1800.0):
253
+ """Start the SGLang Engine in a subprocess."""
254
+ if self._started:
255
+ raise RuntimeError("Service already started")
256
+
257
+ try:
258
+ mp.set_start_method("spawn", force=True)
259
+ except RuntimeError:
260
+ pass
261
+
262
+ self.request_queue = mp.Queue()
263
+ self.response_queue = mp.Queue()
264
+ self.hidden_queue = mp.Queue(maxsize=2)
265
+
266
+ self.process = mp.Process(
267
+ target=_engine_worker,
268
+ args=(self.config, self.request_queue, self.response_queue, self.hidden_queue),
269
+ )
270
+ self.process.start()
271
+
272
+ try:
273
+ response = self.response_queue.get(timeout=timeout)
274
+ if response.get("type") == "init_done" and response.get("success"):
275
+ self._started = True
276
+ else:
277
+ raise RuntimeError(f"Init failed: {response.get('error')}")
278
+ except Exception as e:
279
+ self._cleanup()
280
+ raise RuntimeError(f"Engine initialization failed: {e}")
281
+
282
+ def generate(
283
+ self,
284
+ input_ids: List[List[int]],
285
+ loss_masks: List[np.ndarray],
286
+ sampling_params: Dict[str, Any],
287
+ return_hidden_states: bool = True,
288
+ image_data=None,
289
+ ) -> List[np.ndarray]:
290
+ """Run generation and return hidden states via shared-memory tensors.
291
+
292
+ Args:
293
+ input_ids: Per-sample token-id lists fed directly to SGLang (avoids a
294
+ re-tokenization round trip, so hidden-state length matches the mask).
295
+ loss_masks: Pre-computed boolean masks for selecting response hidden states.
296
+ sampling_params: Sampling parameters (e.g. max_new_tokens=0 for prefill-only).
297
+ return_hidden_states: Whether to return hidden states.
298
+ image_data: Optional list of image data for multimodal models.
299
+ """
300
+ if not self._started:
301
+ raise RuntimeError("Service not started")
302
+
303
+ if len(input_ids) != len(loss_masks):
304
+ raise ValueError(
305
+ f"input_ids and loss_masks must have the same batch size, got "
306
+ f"{len(input_ids)} and {len(loss_masks)}"
307
+ )
308
+
309
+ # Check if subprocess is still alive before sending request
310
+ if self.process and not self.process.is_alive():
311
+ raise RuntimeError(
312
+ f"[SGLangEngineService] Engine subprocess (PID={self.process.pid}) is dead! "
313
+ f"exitcode={self.process.exitcode}"
314
+ )
315
+
316
+ kwargs = {
317
+ "input_ids": input_ids,
318
+ "loss_masks": loss_masks,
319
+ "sampling_params": sampling_params,
320
+ "return_hidden_states": return_hidden_states,
321
+ }
322
+ if image_data is not None:
323
+ kwargs["image_data"] = image_data
324
+
325
+ self.request_queue.put({"type": "generate", "kwargs": kwargs})
326
+
327
+ response = self._get_response(req_type="generate", timeout=600)
328
+ if not response.get("success"):
329
+ raise RuntimeError(f"Generate failed: {response.get('error')}")
330
+
331
+ num_samples = response["num_samples"]
332
+ hidden_states = [None] * num_samples
333
+ received_count = 0
334
+ t_recv_start = time.time()
335
+ while True:
336
+ try:
337
+ message = self.hidden_queue.get(timeout=300)
338
+ if message is None:
339
+ break
340
+ if isinstance(message, str):
341
+ raise RuntimeError(f"Generate failed in engine subprocess:\n{message}")
342
+
343
+ idx, hs_tensor = message
344
+ hs_np = hs_tensor.numpy().copy()
345
+ del hs_tensor
346
+ hidden_states[idx] = hs_np
347
+ received_count += 1
348
+ except queue.Empty:
349
+ elapsed_total = time.time() - t_recv_start
350
+ raise RuntimeError(
351
+ f"Hidden state recv timeout while receiving hidden states, "
352
+ f"received={received_count}/{num_samples}, "
353
+ f"total_elapsed={elapsed_total:.1f}s"
354
+ )
355
+
356
+ return hidden_states
357
+
358
+ def sleep(self, tags: Optional[str] = "all"):
359
+ """Release GPU memory."""
360
+ if not self._started:
361
+ return
362
+ self.request_queue.put({"type": "sleep", "tags": tags})
363
+ response = self._get_response(req_type="sleep", timeout=300)
364
+ if not response.get("success"):
365
+ raise RuntimeError(f"Sleep failed: {response.get('error')}")
366
+ return response.get("tags")
367
+
368
+ def wakeup(self, tags: Optional[str] = "all"):
369
+ """Resume GPU memory."""
370
+ if not self._started:
371
+ return
372
+ self.request_queue.put({"type": "wakeup", "tags": tags})
373
+ response = self._get_response(req_type="wakeup", timeout=300)
374
+ if not response.get("success"):
375
+ raise RuntimeError(f"Wakeup failed: {response.get('error')}")
376
+ return response.get("tags")
377
+
378
+ def update_weights_from_tensor(
379
+ self, serialized_named_tensors: List[Tuple[str, torch.Tensor]],
380
+ load_format: Optional[str] = None, flush_cache: bool = True):
381
+ kwargs = {
382
+ "serialized_named_tensors": serialized_named_tensors,
383
+ "load_format": load_format,
384
+ "flush_cache": flush_cache,
385
+ }
386
+ self.request_queue.put({"type": "update_weights_from_tensor", "kwargs": kwargs})
387
+ response = self._get_response(req_type="update_weights_from_tensor", timeout=300)
388
+ if not response.get("success"):
389
+ raise RuntimeError(f"update_weights_from_tensor failed: {response.get('error')}")
390
+
391
+ def _get_response(self, req_type="unknown", timeout=600, check_interval=10):
392
+ elapsed = 0
393
+ while elapsed < timeout:
394
+ try:
395
+ return self.response_queue.get(timeout=check_interval)
396
+ except queue.Empty:
397
+ elapsed += check_interval
398
+ if self.process and not self.process.is_alive():
399
+ raise RuntimeError(
400
+ f"Engine subprocess (PID={self.process.pid}) died during '{req_type}'! "
401
+ f"exitcode={self.process.exitcode}"
402
+ )
403
+ raise RuntimeError(f"Response timeout after {timeout}s during '{req_type}'")
404
+
405
+ def shutdown(self):
406
+ """Shutdown the subprocess gracefully."""
407
+ if not self._started:
408
+ return
409
+ self._started = False
410
+ self._cleanup()
411
+
412
+ def _cleanup(self):
413
+ """Clean up subprocess, queues and shared memory."""
414
+ if self.request_queue:
415
+ try:
416
+ self.request_queue.put(None)
417
+ except Exception:
418
+ pass
419
+
420
+ if self.process:
421
+ self.process.join(timeout=30)
422
+ if self.process.is_alive():
423
+ self.process.terminate()
424
+ self.process.join(timeout=5)
425
+ if self.process.is_alive():
426
+ self.process.kill()
427
+
428
+ self.process = None
429
+ self.request_queue = None
430
+ self.response_queue = None
431
+ self.hidden_queue = None
432
+
433
+ def __del__(self):
434
+ try:
435
+ self.shutdown()
436
+ except Exception:
437
+ pass
kdflow/cli/__init__.py ADDED
File without changes
kdflow/cli/evaluate_alfworld.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Evaluate a model checkpoint in ALFWorld without starting training actors."""
2
+
3
+ import json
4
+ import logging
5
+ import os
6
+
7
+ from kdflow.evaluation import AlfWorldEvaluator, VerlAgentAlfWorldEvaluator
8
+
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ class _EvaluationStrategy:
14
+ """Small args/log adapter required by the shared rollout manager."""
15
+
16
+ def __init__(self, args):
17
+ self.args = args
18
+
19
+ @staticmethod
20
+ def log(message, level="info", **_kwargs):
21
+ getattr(logger, level, logger.info)(message)
22
+
23
+ @staticmethod
24
+ def print(*values):
25
+ logger.info(" ".join(str(value) for value in values))
26
+
27
+
28
+ def evaluate(args):
29
+ if args.rollout.alfworld_eval_protocol == "verl_agent":
30
+ preflight_evaluator = VerlAgentAlfWorldEvaluator(args, None)
31
+ if not preflight_evaluator.has_pending_seeds():
32
+ logger.info(
33
+ "All requested ALFWorld seeds are already complete and "
34
+ "compatible; skipping Ray and model initialization."
35
+ )
36
+ summary = preflight_evaluator.evaluate()
37
+ print(json.dumps(summary, ensure_ascii=False, indent=2))
38
+ return summary
39
+
40
+ import ray
41
+
42
+ from kdflow.ray.placement_group import create_placement_group
43
+ from kdflow.ray.rollout.rollout_group import RolloutActorGroup
44
+ from kdflow.trainer.alfworld_rollout_manager import AlfWorldRolloutManager
45
+ from kdflow.trainer.verl_agent_alfworld_rollout_manager import (
46
+ VerlAgentAlfWorldRolloutManager,
47
+ )
48
+
49
+ initialized_ray = False
50
+ if not ray.is_initialized():
51
+ ray.init(
52
+ runtime_env={
53
+ "env_vars": {
54
+ "TOKENIZERS_PARALLELISM": "true",
55
+ "NCCL_DEBUG": "WARN",
56
+ },
57
+ "working_dir": os.path.dirname(
58
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
59
+ ),
60
+ }
61
+ )
62
+ initialized_ray = True
63
+
64
+ total_gpus = args.train.num_nodes * args.train.num_gpus_per_node
65
+ rollout_group = None
66
+ rollout_manager = None
67
+ try:
68
+ pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(
69
+ total_gpus
70
+ )
71
+ rollout_group = RolloutActorGroup(
72
+ model_path=args.model.student_name_or_path,
73
+ num_actors=args.rollout.rollout_num_engines,
74
+ tp_size=args.rollout.rollout_tp_size,
75
+ num_gpus_per_node=args.train.num_gpus_per_node,
76
+ enable_memory_saver=True,
77
+ mem_fraction_static=args.rollout.rollout_mem_fraction_static,
78
+ num_gpus_per_actor=0.01,
79
+ pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
80
+ extra_server_args={"disable_piecewise_cuda_graph": True},
81
+ )
82
+ if args.train.enable_sleep:
83
+ rollout_group.sleep()
84
+
85
+ strategy = _EvaluationStrategy(args)
86
+ generate_kwargs = {
87
+ "max_new_tokens": args.rollout.generate_max_len,
88
+ "temperature": args.rollout.temperature,
89
+ "top_p": args.rollout.top_p,
90
+ }
91
+ if args.rollout.alfworld_eval_protocol == "verl_agent":
92
+ manager_class = VerlAgentAlfWorldRolloutManager
93
+ evaluator_class = VerlAgentAlfWorldEvaluator
94
+ else:
95
+ manager_class = AlfWorldRolloutManager
96
+ evaluator_class = AlfWorldEvaluator
97
+ rollout_manager = manager_class(
98
+ strategy=strategy,
99
+ rollout_group=rollout_group,
100
+ is_same_tokenizer=True,
101
+ generate_kwargs=generate_kwargs,
102
+ )
103
+ summary = evaluator_class(args, rollout_manager).evaluate()
104
+ print(json.dumps(summary, ensure_ascii=False, indent=2))
105
+ return summary
106
+ finally:
107
+ if rollout_manager is not None:
108
+ rollout_manager.close()
109
+ if rollout_group is not None:
110
+ rollout_group.shutdown()
111
+ if initialized_ray:
112
+ ray.shutdown()
113
+
114
+
115
+ if __name__ == "__main__":
116
+ from kdflow.arguments import init_args
117
+
118
+ evaluate(init_args("alfworld_eval"))
kdflow/cli/train_kd_off_policy.py ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import ray
4
+
5
+ from kdflow.ray.train.teacher_group import TeacherActorGroup
6
+ from kdflow.ray.train.student_group import StudentActorGroup
7
+ from kdflow.ray.placement_group import create_placement_group
8
+ from kdflow.trainer import OffPolicyKDTrainer
9
+ from kdflow.datasets import SFTDataset
10
+ from kdflow.datasets.utils import blending_datasets
11
+ from kdflow.models.utils import check_tokenizer_identical
12
+ from kdflow.backend import get_strategy
13
+ from kdflow.arguments import init_args
14
+ from kdflow.arguments.training_args import resolve_max_steps
15
+ from kdflow.utils.distributed_sampler import DistributedSampler
16
+ from kdflow.utils.utils import get_tokenizer
17
+
18
+
19
+ def train(args):
20
+ # Initialize Ray if not already initialized
21
+ if not ray.is_initialized():
22
+ ray.init(
23
+ runtime_env={
24
+ "env_vars": {
25
+ "TOKENIZERS_PARALLELISM": "true",
26
+ "NCCL_DEBUG": "WARN"
27
+ },
28
+ "working_dir": os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
29
+ }
30
+ )
31
+
32
+ strategy = get_strategy(args)
33
+ strategy.print(args)
34
+
35
+ # Create placement group for resource allocation
36
+ num_gpus = args.train.num_nodes * args.train.num_gpus_per_node
37
+ pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(num_gpus)
38
+ strategy.print(f"Placement group ready with {num_gpus} GPUs")
39
+
40
+ teacher_model = TeacherActorGroup(
41
+ strategy,
42
+ num_gpus,
43
+ num_gpus_per_node=args.train.num_gpus_per_node,
44
+ num_gpus_per_actor=0.01,
45
+ pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
46
+ )
47
+ student_model = StudentActorGroup(
48
+ args,
49
+ args.train.num_nodes,
50
+ args.train.num_gpus_per_node,
51
+ pg=pg,
52
+ num_gpus_per_actor=0.01,
53
+ )
54
+
55
+ # Initialize tokenizers
56
+ student_tokenizer = get_tokenizer(
57
+ args.model.student_name_or_path,
58
+ use_fast=not args.model.disable_fast_tokenizer
59
+ )
60
+ teacher_tokenizer = get_tokenizer(
61
+ args.model.teacher_name_or_path,
62
+ use_fast=not args.model.disable_fast_tokenizer,
63
+ )
64
+ tokenizer_info = check_tokenizer_identical(student_tokenizer, teacher_tokenizer)
65
+ strategy.print(f"Tokenizers {tokenizer_info}")
66
+ if not tokenizer_info.vocab_identical and args.kd.kd_algorithm not in {
67
+ "simple_ctkd",
68
+ "byte_align_ctkd",
69
+ "topk_byte_align_ctkd",
70
+ }:
71
+ raise ValueError(
72
+ "Cross-tokenizer distillation supports simple_ctkd, "
73
+ "byte_align_ctkd, or topk_byte_align_ctkd."
74
+ )
75
+
76
+ # Load and prepare training dataset
77
+ train_data = blending_datasets(
78
+ args.data.train_dataset_path,
79
+ args.data.train_dataset_probs,
80
+ strategy,
81
+ args.train.seed,
82
+ max_count=args.data.max_samples,
83
+ dataset_split=args.data.train_split,
84
+ )
85
+ train_data = train_data.select(range(min(args.data.max_samples, len(train_data))))
86
+
87
+ train_dataset = SFTDataset(
88
+ train_data,
89
+ strategy,
90
+ tokenizer_info=tokenizer_info,
91
+ max_data_num=args.data.max_samples,
92
+ input_template=args.data.input_template,
93
+ num_processors=args.data.preprocess_num_workers,
94
+ )
95
+
96
+ sampler = DistributedSampler(train_dataset, num_replicas=1, rank=0, shuffle=True, seed=args.train.seed, drop_last=True)
97
+ train_dataloader = strategy.setup_dataloader(
98
+ train_dataset,
99
+ args.train.micro_train_batch_size,
100
+ pin_memory=False,
101
+ shuffle=False,
102
+ collate_fn=train_dataset.collate_fn,
103
+ sampler=sampler,
104
+ )
105
+
106
+ # Load and prepare evaluation dataset (optional)
107
+ eval_dataloader = None
108
+ if getattr(args.data, "eval_dataset_path", None):
109
+ eval_data = blending_datasets(
110
+ args.data.eval_dataset_path,
111
+ None,
112
+ strategy,
113
+ dataset_split=args.data.eval_split,
114
+ )
115
+ eval_data = eval_data.select(range(min(args.data.max_samples, len(eval_data))))
116
+ eval_dataset = SFTDataset(
117
+ eval_data,
118
+ strategy,
119
+ tokenizer_info=tokenizer_info,
120
+ input_template=args.data.input_template,
121
+ num_processors=args.data.preprocess_num_workers,
122
+ )
123
+ eval_dataloader = strategy.setup_dataloader(
124
+ eval_dataset,
125
+ batch_size=1,
126
+ pin_memory=True,
127
+ shuffle=False,
128
+ drop_last=False,
129
+ collate_fn=eval_dataset.collate_fn,
130
+ )
131
+
132
+ # Calculate max training steps
133
+ num_update_steps_per_epoch = len(train_dataset) // args.train.train_batch_size
134
+ max_steps = resolve_max_steps(
135
+ args.train.max_steps,
136
+ args.train.num_epochs,
137
+ num_update_steps_per_epoch,
138
+ )
139
+ strategy.log(f"Max training steps: {max_steps}")
140
+
141
+ # Initialize student model on all workers
142
+ ray.get(student_model.async_init_model_from_pretrained(
143
+ strategy, max_steps,
144
+ teacher_tokenizer=teacher_tokenizer,
145
+ tokenizer_info=tokenizer_info,
146
+ ))
147
+
148
+ strategy.log("Models initialized on all student actors")
149
+
150
+ # Initialize OffPolicyKDRayTrainer (Single controller)
151
+ trainer = OffPolicyKDTrainer(
152
+ strategy=strategy,
153
+ student_model=student_model,
154
+ teacher_model=teacher_model,
155
+ train_dataloader=train_dataloader,
156
+ eval_dataloader=eval_dataloader,
157
+ max_steps=max_steps,
158
+ num_update_steps_per_epoch=num_update_steps_per_epoch,
159
+ student_tokenizer=student_tokenizer,
160
+ teacher_tokenizer=teacher_tokenizer,
161
+ )
162
+
163
+ try:
164
+ trainer.fit()
165
+ ray.get(student_model.async_save_model())
166
+ strategy.log("Training completed and model saved.")
167
+ finally:
168
+ teacher_model.shutdown()
169
+
170
+
171
+ if __name__ == "__main__":
172
+ args = init_args("off_policy_kd")
173
+ train(args)
kdflow/cli/train_kd_on_policy.py ADDED
@@ -0,0 +1,230 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import ray
4
+
5
+ from kdflow.ray.train.teacher_group import TeacherActorGroup
6
+ from kdflow.ray.train.student_group import StudentActorGroup
7
+ from kdflow.ray.rollout.rollout_group import RolloutActorGroup
8
+ from kdflow.ray.placement_group import create_placement_group
9
+ from kdflow.trainer import OnPolicyKDTrainer
10
+ from kdflow.datasets import (
11
+ AlfWorldEpisodeDataset,
12
+ PromptDataset,
13
+ )
14
+ from kdflow.datasets.utils import blending_datasets
15
+ from kdflow.models.utils import check_tokenizer_identical
16
+ from kdflow.backend import get_strategy
17
+ from kdflow.arguments import init_args
18
+ from kdflow.arguments.training_args import resolve_max_steps
19
+ from kdflow.utils.distributed_sampler import DistributedSampler
20
+ from kdflow.utils.utils import get_tokenizer, load_custom_eval_fn
21
+
22
+
23
+ def train(args):
24
+ # Initialize Ray if not already initialized
25
+ if not ray.is_initialized():
26
+ ray.init(
27
+ runtime_env={
28
+ "env_vars": {
29
+ "TOKENIZERS_PARALLELISM": "true",
30
+ "NCCL_DEBUG": "WARN"
31
+ },
32
+ "working_dir": os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
33
+ }
34
+ )
35
+
36
+ strategy = get_strategy(args)
37
+ strategy.print(args)
38
+
39
+ student_tokenizer = get_tokenizer(
40
+ args.model.student_name_or_path,
41
+ use_fast=not args.model.disable_fast_tokenizer,
42
+ )
43
+
44
+ # Create placement group for resource allocation
45
+ num_gpus = args.train.num_nodes * args.train.num_gpus_per_node
46
+ pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(num_gpus)
47
+ rollout_group = RolloutActorGroup(
48
+ model_path=args.model.student_name_or_path,
49
+ num_actors=args.rollout.rollout_num_engines,
50
+ tp_size=args.rollout.rollout_tp_size,
51
+ num_gpus_per_node=args.train.num_gpus_per_node,
52
+ enable_memory_saver=True,
53
+ mem_fraction_static=args.rollout.rollout_mem_fraction_static,
54
+ num_gpus_per_actor=0.01,
55
+ pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
56
+ extra_server_args={"disable_piecewise_cuda_graph": True},
57
+ )
58
+ if args.train.enable_sleep:
59
+ rollout_group.sleep()
60
+
61
+ teacher_model = TeacherActorGroup(
62
+ strategy,
63
+ num_gpus,
64
+ num_gpus_per_node=args.train.num_gpus_per_node,
65
+ num_gpus_per_actor=0.01,
66
+ pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
67
+
68
+ )
69
+ student_model = StudentActorGroup(
70
+ args,
71
+ args.train.num_nodes,
72
+ args.train.num_gpus_per_node,
73
+ pg=(pg, reordered_bundle_indices),
74
+ num_gpus_per_actor=0.01,
75
+ )
76
+
77
+ # Initialize the teacher tokenizer after inference actors are configured.
78
+ teacher_tokenizer = get_tokenizer(
79
+ args.model.teacher_name_or_path,
80
+ use_fast=not args.model.disable_fast_tokenizer,
81
+ )
82
+ tokenizer_info = check_tokenizer_identical(student_tokenizer, teacher_tokenizer)
83
+ strategy.print(f"Tokenizers {tokenizer_info}")
84
+ if not tokenizer_info.vocab_identical and args.kd.kd_algorithm not in {
85
+ "simple_ctkd",
86
+ "byte_align_ctkd",
87
+ "topk_byte_align_ctkd",
88
+ }:
89
+ raise ValueError(
90
+ "Cross-tokenizer distillation supports simple_ctkd, "
91
+ "byte_align_ctkd, or topk_byte_align_ctkd."
92
+ )
93
+
94
+ environment_rollout = bool(args.rollout.alfworld_config)
95
+
96
+ # Interactive environments sample tasks on reset. Their lightweight
97
+ # datasets are therefore only epoch clocks; ordinary on-policy KD keeps the
98
+ # existing static prompt dataset path.
99
+ if args.rollout.alfworld_config:
100
+ train_dataset = AlfWorldEpisodeDataset(
101
+ args.rollout.alfworld_episodes_per_epoch
102
+ )
103
+ else:
104
+ train_data = blending_datasets(
105
+ args.data.train_dataset_path,
106
+ args.data.train_dataset_probs,
107
+ strategy,
108
+ args.train.seed,
109
+ max_count=args.data.max_samples,
110
+ dataset_split=args.data.train_split,
111
+ )
112
+ train_data = train_data.select(
113
+ range(min(args.data.max_samples, len(train_data)))
114
+ )
115
+ train_dataset = PromptDataset(
116
+ train_data,
117
+ strategy,
118
+ tokenizer_info=tokenizer_info,
119
+ max_data_num=args.data.max_samples,
120
+ input_template=args.data.input_template,
121
+ num_processors=args.data.preprocess_num_workers,
122
+ )
123
+
124
+ sampler = DistributedSampler(train_dataset, num_replicas=1, rank=0, shuffle=True, seed=args.train.seed, drop_last=True)
125
+ train_dataloader = strategy.setup_dataloader(
126
+ train_dataset,
127
+ args.rollout.rollout_batch_size,
128
+ True,
129
+ False,
130
+ collate_fn=train_dataset.collate_fn,
131
+ sampler=sampler,
132
+ )
133
+
134
+ # Load and prepare evaluation dataset (optional)
135
+ eval_dataloader = None
136
+ if getattr(args.data, "eval_dataset_path", None) and not environment_rollout:
137
+ eval_data = blending_datasets(
138
+ args.data.eval_dataset_path,
139
+ None,
140
+ strategy,
141
+ dataset_split=args.data.eval_split,
142
+ )
143
+ eval_data = eval_data.select(range(min(args.data.max_samples, len(eval_data))))
144
+ eval_dataset = PromptDataset(
145
+ eval_data,
146
+ strategy,
147
+ tokenizer_info=tokenizer_info,
148
+ input_template=args.data.input_template,
149
+ num_processors=args.data.preprocess_num_workers,
150
+ )
151
+ eval_dataloader = strategy.setup_dataloader(
152
+ eval_dataset,
153
+ batch_size=args.rollout.rollout_batch_size,
154
+ pin_memory=True,
155
+ shuffle=False,
156
+ drop_last=False,
157
+ collate_fn=eval_dataset.collate_fn,
158
+ )
159
+
160
+ # Calculate max training steps
161
+ num_rollout_iters_per_epoch = len(train_dataset) // args.rollout.rollout_batch_size
162
+ if environment_rollout:
163
+ max_turn_samples = (
164
+ args.rollout.rollout_batch_size * args.rollout.alfworld_max_turns
165
+ )
166
+ num_update_steps_per_rollout = max(
167
+ 1,
168
+ (max_turn_samples + args.train.train_batch_size - 1)
169
+ // args.train.train_batch_size,
170
+ )
171
+ else:
172
+ num_update_steps_per_rollout = max(
173
+ 1,
174
+ args.rollout.rollout_batch_size
175
+ * args.rollout.n_samples_per_prompt
176
+ // args.train.train_batch_size,
177
+ )
178
+ max_rollout_iters = resolve_max_steps(
179
+ args.train.max_steps,
180
+ args.train.num_epochs,
181
+ num_rollout_iters_per_epoch,
182
+ )
183
+ strategy.log(f"Max training iterations: {max_rollout_iters}")
184
+
185
+ # Initialize student model on all workers
186
+ ray.get(student_model.async_init_model_from_pretrained(
187
+ strategy, (max_rollout_iters * num_update_steps_per_rollout),
188
+ teacher_tokenizer=teacher_tokenizer,
189
+ tokenizer_info=tokenizer_info,
190
+ ))
191
+ strategy.log("Models initialized on all student actors")
192
+
193
+ generate_kwargs = {
194
+ "max_new_tokens": args.rollout.generate_max_len,
195
+ "temperature": args.rollout.temperature,
196
+ "top_p": args.rollout.top_p,
197
+ }
198
+
199
+ trainer = None
200
+ try:
201
+ trainer = OnPolicyKDTrainer(
202
+ strategy=strategy,
203
+ student_model=student_model,
204
+ teacher_model=teacher_model,
205
+ rollout_group=rollout_group,
206
+ is_same_tokenizer=tokenizer_info.is_identical,
207
+ train_dataloader=train_dataloader,
208
+ eval_dataloader=eval_dataloader,
209
+ max_rollout_iters=max_rollout_iters,
210
+ num_rollout_iters_per_epoch=num_rollout_iters_per_epoch,
211
+ generate_kwargs=generate_kwargs,
212
+ custom_eval_fn=(
213
+ load_custom_eval_fn(args.data.custom_eval_fn)
214
+ if args.data.custom_eval_fn
215
+ else None
216
+ ),
217
+ )
218
+ trainer.fit()
219
+ ray.get(student_model.async_save_model())
220
+ strategy.log("Training completed and model saved.")
221
+ finally:
222
+ if trainer is not None:
223
+ trainer.close()
224
+ teacher_model.shutdown()
225
+ rollout_group.shutdown()
226
+
227
+
228
+ if __name__ == "__main__":
229
+ args = init_args("on_policy_kd")
230
+ train(args)
kdflow/datasets/__init__.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ from .prompts_dataset import PromptDataset
2
+ from .sft_dataset import SFTDataset
3
+ from .alfworld_dataset import AlfWorldEpisodeDataset
4
+
5
+ __all__ = [
6
+ "PromptDataset",
7
+ "SFTDataset",
8
+ "AlfWorldEpisodeDataset",
9
+ ]
kdflow/datasets/alfworld_dataset.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, List
2
+
3
+ from torch.utils.data import Dataset
4
+
5
+
6
+ class AlfWorldEpisodeDataset(Dataset):
7
+ """A lightweight epoch clock for environment-sampled ALFWorld episodes.
8
+
9
+ ALFWorld chooses a new game whenever its batched environment is reset, so
10
+ dataset rows only determine how many episodes are collected in one epoch.
11
+ """
12
+
13
+ def __init__(self, num_episodes: int):
14
+ if num_episodes <= 0:
15
+ raise ValueError("num_episodes must be positive")
16
+ self.num_episodes = int(num_episodes)
17
+
18
+ def __len__(self) -> int:
19
+ return self.num_episodes
20
+
21
+ def __getitem__(self, index: int) -> Dict[str, int]:
22
+ return {"episode_slot": index}
23
+
24
+ @staticmethod
25
+ def collate_fn(batch: List[Dict[str, int]]) -> List[Dict[str, int]]:
26
+ return batch
kdflow/datasets/prompts_dataset.py ADDED
@@ -0,0 +1,246 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Callable, Optional, Dict, Any, List
2
+
3
+ from torch.utils.data import Dataset
4
+ from datasets import Image as ImageFeature
5
+ from PIL import Image
6
+
7
+ from kdflow.datasets.utils import (
8
+ convert_to_openai_messages,
9
+ get_nested_value,
10
+ get_tokenizer_or_processor,
11
+ validate_dataset_columns,
12
+ )
13
+ from kdflow.models.utils import TokenizerCompareResult
14
+
15
+
16
+ class PromptDataset(Dataset):
17
+ """
18
+ Dataset for On-Policy Distillation
19
+
20
+ Args:
21
+ dataset: dataset for on-policy distillation
22
+ strategy: training strategy object
23
+ tokenizer_info: result of tokenizer comparison (template_identical, vocab_identical)
24
+ max_data_num: maximum number of data to load
25
+ input_template: optional template for formatting input
26
+ num_processors: number of processors for parallel data loading
27
+ """
28
+
29
+ def __init__(
30
+ self,
31
+ dataset,
32
+ strategy,
33
+ tokenizer_info,
34
+ max_data_num: int = None,
35
+ input_template: Optional[str] = None,
36
+ num_processors: int = 8,
37
+ ) -> None:
38
+ super().__init__()
39
+ self.args = strategy.args
40
+ self.strategy = strategy
41
+ self.tokenizer_info = tokenizer_info or TokenizerCompareResult()
42
+ self.template_identical = self.tokenizer_info.template_identical
43
+ self.vocab_identical = self.tokenizer_info.vocab_identical
44
+ self.same_tokenizer = self.tokenizer_info.is_identical
45
+ self.strategy = strategy
46
+ self.input_template = input_template
47
+
48
+ self.input_key = getattr(self.args.data, "input_key", None)
49
+ self.teacher_input_key = getattr(self.args.data, "teacher_input_key", None) or self.input_key
50
+ self.label_key = getattr(self.args.data, "label_key", None)
51
+ self.apply_chat_template = getattr(self.args.data, "apply_chat_template", False)
52
+ self.enable_thinking = getattr(self.args.model, "enable_thinking", False)
53
+ self.prompt_max_len = getattr(self.args.data, "prompt_max_len", 0)
54
+
55
+ self.image_key = getattr(self.args.data, "image_key", None)
56
+
57
+ required_columns = {"input_key": self.input_key}
58
+ if self.teacher_input_key != self.input_key:
59
+ required_columns["teacher_input_key"] = self.teacher_input_key
60
+ if self.label_key:
61
+ required_columns["label_key"] = self.label_key
62
+ if self.image_key:
63
+ required_columns["image_key"] = self.image_key
64
+ if self.args.kd.multi_teacher_config:
65
+ required_columns["teacher_routing_key"] = self.args.data.teacher_routing_key
66
+ validate_dataset_columns(dataset, **required_columns)
67
+
68
+ self.student_processor = get_tokenizer_or_processor(
69
+ self.args.model.student_name_or_path,
70
+ need_processor=self.image_key is not None,
71
+ )
72
+ self.teacher_processors = {}
73
+ if self.args.kd.multi_teacher_config:
74
+ for teacher_key, teacher_path in self.args.kd.multi_teacher_config.items():
75
+ self.teacher_processors[teacher_key] = get_tokenizer_or_processor(
76
+ teacher_path, need_processor=self.image_key is not None,
77
+ )
78
+ elif self.args.model.teacher_name_or_path is not None:
79
+ self.teacher_processors["default"] = get_tokenizer_or_processor(
80
+ self.args.model.teacher_name_or_path,
81
+ need_processor=self.image_key is not None,
82
+ )
83
+
84
+ if max_data_num is not None and max_data_num > 0 and max_data_num < len(dataset):
85
+ strategy.log(f"Truncating dataset from {len(dataset)} to {max_data_num}")
86
+ dataset = dataset.select(range(max_data_num))
87
+
88
+ # Adjust worker count based on dataset size to speed up preprocessing.
89
+ num_processors = min(
90
+ num_processors,
91
+ max(1, len(dataset) // 2000),
92
+ )
93
+ self.processed_dataset = dataset.map(
94
+ self.process_data,
95
+ remove_columns=dataset.column_names,
96
+ num_proc=num_processors,
97
+ load_from_cache_file=False,
98
+ desc="Processing data",
99
+ )
100
+ if self.image_key:
101
+ self.processed_dataset = self.processed_dataset.cast_column("images", [ImageFeature()])
102
+
103
+ original_len = len(self.processed_dataset)
104
+ if self.prompt_max_len > 0:
105
+ strategy.log(f"Before prompt_max_len filter: {len(self.processed_dataset)}")
106
+ self.processed_dataset = self.processed_dataset.filter(
107
+ lambda prompt_len: prompt_len <= self.prompt_max_len,
108
+ input_columns=["prompt_len"],
109
+ desc="Filtering overlang samples",
110
+ )
111
+ strategy.log(f"After prompt_max_len filter: {len(self.processed_dataset)}")
112
+ if len(self.processed_dataset) < original_len:
113
+ self.strategy.log(
114
+ f"Filtered {original_len - len(self.processed_dataset)} samples "
115
+ f"exceeding prompt_max_len={self.prompt_max_len} "
116
+ f"({original_len} -> {len(self.processed_dataset)})"
117
+ )
118
+
119
+ self._print_sample()
120
+
121
+ def _print_sample(self) -> None:
122
+ """Print sample data for debugging."""
123
+ self.strategy.print(f"Total samples: {len(self.processed_dataset)}")
124
+ self.strategy.print(f"Sample student prompt:\n{self.processed_dataset[0]['stu_prompt']}")
125
+ if not self.template_identical or self.teacher_input_key != self.input_key:
126
+ self.strategy.print(f"Sample teacher prompt:\n{self.processed_dataset[0]['tea_prompt']}")
127
+
128
+ def process_data(self, data: Dict) -> Dict[str, Any]:
129
+ """Process a single data sample."""
130
+ stu_prompt = self._build_prompt(data, self.student_processor, self.input_key)
131
+
132
+ if self.args.kd.multi_teacher_config:
133
+ routing_key = self.args.data.teacher_routing_key
134
+ assert routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
135
+ assert routing_key in data, f"Routing key '{routing_key}' not found in data"
136
+ teacher_key = data[routing_key]
137
+ if teacher_key not in self.teacher_processors:
138
+ raise ValueError(
139
+ f"Teacher routing key '{teacher_key}' not found in multi_teacher_config. "
140
+ f"Available keys: {list(self.teacher_processors.keys())}."
141
+ )
142
+ teacher_processor = self.teacher_processors[teacher_key]
143
+ tea_prompt = self._build_prompt(data, teacher_processor, self.teacher_input_key)
144
+ elif self.same_tokenizer and self.input_key == self.teacher_input_key:
145
+ tea_prompt = stu_prompt
146
+ else:
147
+ tea_prompt = self._build_prompt(data, self.teacher_processors.get("default", self.student_processor), self.teacher_input_key)
148
+
149
+ tokenizer = self.student_processor.tokenizer if hasattr(self.student_processor, "tokenizer") else self.student_processor
150
+ prompt_len = len(tokenizer.encode(stu_prompt))
151
+
152
+ result = {
153
+ "stu_prompt": stu_prompt,
154
+ "tea_prompt": tea_prompt,
155
+ "prompt": stu_prompt,
156
+ "prompt_len": prompt_len,
157
+ "label": get_nested_value(data, self.label_key) or "" if self.label_key else "",
158
+ "datasource": data.get("datasource", "default"),
159
+ }
160
+ if self.image_key:
161
+ result["images"] = self._load_images(data.get(self.image_key))
162
+
163
+ if self.args.kd.multi_teacher_config is not None:
164
+ assert self.args.data.teacher_routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
165
+ assert self.args.data.teacher_routing_key in data, f"Routing key '{self.args.data.teacher_routing_key}' not found in data"
166
+ result["teacher_routing_key"] = data[self.args.data.teacher_routing_key]
167
+
168
+ return result
169
+
170
+ def _build_prompt(self, data: Dict, processor_or_tokenizer, input_key: str) -> str:
171
+ """Build prompt from data with optional chat template or input template.
172
+
173
+ Args:
174
+ data: The data dict containing input
175
+ processor_or_tokenizer: The processor or tokenizer to use for apply_chat_template
176
+ input_key: The key to extract input from data
177
+
178
+ Returns:
179
+ Formatted prompt string
180
+ """
181
+ if self.apply_chat_template:
182
+ chat = convert_to_openai_messages(data[input_key], expand_image=self.image_key is not None)
183
+ while chat and chat[-1].get("role", "user") == "assistant":
184
+ chat.pop()
185
+ chat_template_kwargs = {}
186
+ if "enable_thinking" in str(getattr(processor_or_tokenizer, "chat_template", "")):
187
+ chat_template_kwargs["enable_thinking"] = self.enable_thinking
188
+ return processor_or_tokenizer.apply_chat_template(
189
+ chat,
190
+ tokenize=False,
191
+ add_generation_prompt=True,
192
+ **chat_template_kwargs,
193
+ )
194
+
195
+ prompt = data[input_key]
196
+ return self.input_template.format(prompt) if self.input_template else prompt
197
+
198
+ def __len__(self) -> int:
199
+ return len(self.processed_dataset)
200
+
201
+ def __getitem__(self, idx: int) -> Dict[str, str]:
202
+ """Get item by index.
203
+
204
+ Returns:
205
+ Dict with keys: datasource, stu_prompt, tea_prompt, label, images (optional)
206
+ """
207
+ item = self.processed_dataset[idx]
208
+ result = {
209
+ "datasource": item["datasource"],
210
+ "stu_prompt": item["stu_prompt"],
211
+ "tea_prompt": item["tea_prompt"],
212
+ "label": item["label"],
213
+ }
214
+ if "images" in item:
215
+ result["images"] = item["images"]
216
+ if "teacher_routing_key" in item:
217
+ result["teacher_routing_key"] = item["teacher_routing_key"]
218
+ return result
219
+
220
+ @staticmethod
221
+ def _load_images(image_content) -> list:
222
+ """Load image(s) from various input formats. Always returns a list."""
223
+ if image_content is None:
224
+ return []
225
+ if isinstance(image_content, Image.Image):
226
+ return [image_content]
227
+ if isinstance(image_content, str):
228
+ return [Image.open(image_content).convert("RGB")]
229
+ if isinstance(image_content, list):
230
+ result = []
231
+ for img in image_content:
232
+ if isinstance(img, Image.Image):
233
+ result.append(img)
234
+ elif isinstance(img, str):
235
+ result.append(Image.open(img).convert("RGB"))
236
+ return result
237
+ return []
238
+
239
+ @staticmethod
240
+ def collate_fn(batch: List[Dict[str, str]]) -> List[Dict[str, str]]:
241
+ """Collate function that simply returns the list of dicts.
242
+
243
+ DataLoader will pass a list of dicts, we just return it as-is
244
+ since rollout method expects a list of dicts.
245
+ """
246
+ return batch
kdflow/datasets/sft_dataset.py ADDED
@@ -0,0 +1,380 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Callable, Optional, Dict, List, Any
2
+
3
+ import torch
4
+ from torch.utils.data import Dataset
5
+ from datasets import Image as ImageFeature
6
+ from PIL import Image
7
+
8
+ from kdflow.datasets.utils import (
9
+ convert_to_openai_messages,
10
+ get_tokenizer_or_processor,
11
+ validate_dataset_columns,
12
+ zero_pad_sequences,
13
+ )
14
+ from kdflow.models.utils import TokenizerCompareResult
15
+
16
+ class SFTDataset(Dataset):
17
+ """
18
+ Dataset for SFT and Off-Policy KD.
19
+
20
+ Args:
21
+ dataset: dataset for SFT and Off-Policy KD
22
+ strategy: training strategy object
23
+ tokenizer_info: result of tokenizer comparison (template_identical, vocab_identical)
24
+ max_data_num: maximum number of data to load
25
+ input_template: optional template for formatting input
26
+ num_processors: number of processors for parallel data loading
27
+ """
28
+
29
+ def __init__(
30
+ self,
31
+ dataset,
32
+ strategy,
33
+ tokenizer_info: Optional[TokenizerCompareResult] = None,
34
+ max_data_num: int = -1,
35
+ input_template: Optional[str] = None,
36
+ num_processors: int = 8,
37
+ ) -> None:
38
+ super().__init__()
39
+ self.args = strategy.args
40
+ self.strategy = strategy
41
+ self.max_length = self.args.data.max_len
42
+ self.tokenizer_info = tokenizer_info
43
+ self.template_identical = True if tokenizer_info is None else self.tokenizer_info.template_identical
44
+ self.vocab_identical = True if tokenizer_info is None else self.tokenizer_info.vocab_identical
45
+
46
+ self.input_template = input_template
47
+ self.input_key = getattr(self.args.data, "input_key", None)
48
+ self.teacher_input_key = getattr(self.args.data, "teacher_input_key", None) or self.input_key
49
+ self.output_key = getattr(self.args.data, "output_key", None)
50
+ self.apply_chat_template = getattr(self.args.data, "apply_chat_template", False)
51
+ self.enable_thinking = getattr(self.args.model, "enable_thinking", False)
52
+
53
+ self.image_key = getattr(self.args.data, "image_key", None)
54
+ self.same_tokenizer = True if tokenizer_info is None else self.tokenizer_info.is_identical
55
+ self.teacher_student_share_input = (
56
+ self.same_tokenizer
57
+ and self.teacher_input_key == self.input_key
58
+ )
59
+
60
+ required_columns = {"input_key": self.input_key}
61
+ if self.output_key:
62
+ required_columns["output_key"] = self.output_key
63
+ if self.image_key:
64
+ required_columns["image_key"] = self.image_key
65
+ has_teacher = (
66
+ self.args.model.teacher_name_or_path is not None
67
+ or self.args.kd.multi_teacher_config is not None
68
+ )
69
+ if has_teacher and self.teacher_input_key != self.input_key:
70
+ required_columns["teacher_input_key"] = self.teacher_input_key
71
+ if self.args.kd.multi_teacher_config:
72
+ required_columns["teacher_routing_key"] = self.args.data.teacher_routing_key
73
+ validate_dataset_columns(dataset, **required_columns)
74
+
75
+ self.student_processor = get_tokenizer_or_processor(
76
+ self.args.model.student_name_or_path,
77
+ need_processor=self.image_key is not None,
78
+ )
79
+ self.student_tokenizer = getattr(self.student_processor, "tokenizer", self.student_processor)
80
+ self.teacher_processors = {}
81
+ if self.args.kd.multi_teacher_config:
82
+ for teacher_key, teacher_path in self.args.kd.multi_teacher_config.items():
83
+ self.teacher_processors[teacher_key] = get_tokenizer_or_processor(
84
+ teacher_path, need_processor=self.image_key is not None,
85
+ )
86
+ elif self.args.model.teacher_name_or_path is not None:
87
+ self.teacher_processors["default"] = get_tokenizer_or_processor(
88
+ self.args.model.teacher_name_or_path,
89
+ need_processor=self.image_key is not None,
90
+ )
91
+
92
+ if max_data_num > 0 and max_data_num < len(dataset):
93
+ self.strategy.log(f"Truncating dataset from {len(dataset)} to {max_data_num}")
94
+ dataset = dataset.select(range(max_data_num))
95
+
96
+ # Adjust worker count based on dataset size to speed up preprocessing.
97
+ num_processors = min(
98
+ num_processors,
99
+ max(1, len(dataset) // 2000),
100
+ )
101
+ self.processed_dataset = dataset.map(
102
+ self.process_data,
103
+ remove_columns=dataset.column_names,
104
+ num_proc=num_processors,
105
+ load_from_cache_file=False,
106
+ desc="Processing data",
107
+ )
108
+ if self.image_key:
109
+ self.processed_dataset = self.processed_dataset.cast_column("images", [ImageFeature()])
110
+
111
+ original_len = len(self.processed_dataset)
112
+ strategy.log(f"Before length filter: {len(self.processed_dataset)}")
113
+ self.processed_dataset = self.processed_dataset.filter(
114
+ lambda stu_input_len: stu_input_len <= self.max_length,
115
+ input_columns=["stu_input_len"],
116
+ desc="Filtering overlang samples",
117
+ )
118
+ if len(self.processed_dataset) < original_len:
119
+ self.strategy.log(
120
+ f"Filtered {original_len - len(self.processed_dataset)} samples "
121
+ f"exceeding max_length={self.max_length} "
122
+ f"({original_len} -> {len(self.processed_dataset)})"
123
+ )
124
+
125
+ self._print_sample()
126
+
127
+ def _print_sample(self) -> None:
128
+ sample = self.processed_dataset[0]
129
+ self.strategy.print("Student prompt + response:")
130
+ self.strategy.print(sample["stu_prompt"] + sample["stu_response"])
131
+ if not self.teacher_student_share_input:
132
+ self.strategy.print("Teacher prompt + response:")
133
+ self.strategy.print(sample["tea_prompt"] + sample["tea_response"])
134
+
135
+ def process_data(self, data: Dict) -> Dict[str, Any]:
136
+ stu_chat_template_fn = self.student_processor.apply_chat_template if self.apply_chat_template else None
137
+ stu_prompt, stu_response = self.preprocess_data(
138
+ data, self.input_template, self.input_key, self.output_key,
139
+ apply_chat_template=stu_chat_template_fn,
140
+ )
141
+ stu_eos_token = self.student_tokenizer.eos_token
142
+ if not stu_response.endswith(stu_eos_token):
143
+ stu_response += " " + stu_eos_token
144
+
145
+ result = {"stu_prompt": stu_prompt, "stu_response": stu_response}
146
+ result["stu_resp_len"] = self._compute_token_length(
147
+ self.student_processor, stu_response
148
+ )
149
+ result["stu_input_len"] = self._compute_token_length(
150
+ self.student_processor, stu_prompt
151
+ ) + result["stu_resp_len"]
152
+
153
+ if self.image_key is not None:
154
+ result["images"] = self.load_images(data[self.image_key])
155
+
156
+ if self.args.model.teacher_name_or_path is not None \
157
+ or self.args.kd.multi_teacher_config is not None:
158
+ if not self.teacher_student_share_input:
159
+ if self.args.kd.multi_teacher_config is not None:
160
+ routing_key = self.args.data.teacher_routing_key
161
+ assert routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
162
+ assert routing_key in data, f"Routing key '{routing_key}' not found in data"
163
+ teacher_key = data[routing_key]
164
+ if teacher_key not in self.teacher_processors:
165
+ raise ValueError(
166
+ f"Teacher routing key '{teacher_key}' not found in multi_teacher_config. "
167
+ f"Available keys: {list(self.teacher_processors.keys())}."
168
+ )
169
+ teacher_processor = self.teacher_processors[teacher_key]
170
+ else:
171
+ teacher_processor = self.teacher_processors.get("default", self.student_processor)
172
+ cur_teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor)
173
+
174
+ tea_chat_template_fn = teacher_processor.apply_chat_template if self.apply_chat_template else None
175
+ tea_prompt, tea_response = self.preprocess_data(
176
+ data, self.input_template, self.teacher_input_key, self.output_key,
177
+ apply_chat_template=tea_chat_template_fn,
178
+ )
179
+ tea_eos_token = cur_teacher_tokenizer.eos_token
180
+ if not tea_response.endswith(tea_eos_token):
181
+ tea_response += " " + tea_eos_token
182
+ result["tea_prompt"] = tea_prompt
183
+ result["tea_response"] = tea_response
184
+ result["tea_resp_len"] = self._compute_token_length(teacher_processor, tea_response)
185
+ else:
186
+ result["tea_prompt"] = stu_prompt
187
+ result["tea_response"] = stu_response
188
+ result["tea_resp_len"] = result["stu_resp_len"]
189
+
190
+ if self.args.kd.multi_teacher_config is not None:
191
+ assert self.args.data.teacher_routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
192
+ assert self.args.data.teacher_routing_key in data, f"Routing key '{self.args.data.teacher_routing_key}' not found in data"
193
+ result["teacher_routing_key"] = data[self.args.data.teacher_routing_key]
194
+
195
+ return result
196
+
197
+ def preprocess_data(
198
+ self,
199
+ data: Dict,
200
+ input_template: Optional[str] = None,
201
+ input_key: str = "input",
202
+ output_key: Optional[str] = None,
203
+ apply_chat_template: Optional[Callable] = None,
204
+ ) -> tuple:
205
+ if not apply_chat_template:
206
+ prompt = data[input_key]
207
+ if input_template:
208
+ prompt = input_template.format(prompt)
209
+ assert output_key is not None
210
+ return prompt, data[output_key]
211
+
212
+ has_image = self.image_key is not None
213
+ if output_key:
214
+ messages = convert_to_openai_messages(data[input_key], expand_image=has_image) + \
215
+ convert_to_openai_messages(data[output_key], role="assistant", expand_image=has_image)
216
+ else:
217
+ messages = convert_to_openai_messages(data[input_key], expand_image=has_image)
218
+
219
+ chat_template_kwargs = {}
220
+ if "enable_thinking" in str(apply_chat_template.__self__.chat_template):
221
+ chat_template_kwargs["enable_thinking"] = self.enable_thinking
222
+ prompt = apply_chat_template(
223
+ messages[:-1], tokenize=False, add_generation_prompt=True,
224
+ **chat_template_kwargs,
225
+ )
226
+ full_text = apply_chat_template(messages, tokenize=False, **chat_template_kwargs)
227
+ response = full_text[len(prompt):].removeprefix("<think>\n\n</think>\n\n").rstrip()
228
+ return prompt, response
229
+
230
+ def __len__(self) -> int:
231
+ return len(self.processed_dataset)
232
+
233
+ def __getitem__(self, idx: int) -> Dict:
234
+ return self.processed_dataset[idx]
235
+
236
+ def load_images(self, image_content):
237
+ """Load image(s) from various input formats. Always returns a list."""
238
+ if isinstance(image_content, Image.Image):
239
+ return [image_content]
240
+ if isinstance(image_content, str):
241
+ return [Image.open(image_content).convert("RGB")]
242
+ if isinstance(image_content, list):
243
+ return [self._load_single_image(img) for img in image_content]
244
+ return []
245
+
246
+ @staticmethod
247
+ def _load_single_image(img):
248
+ if isinstance(img, Image.Image):
249
+ return img
250
+ if isinstance(img, str):
251
+ return Image.open(img).convert("RGB")
252
+ return None
253
+
254
+ def _compute_token_length(self, processor, text):
255
+ tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
256
+ return len(tokenizer.encode(text, add_special_tokens=False))
257
+
258
+ def _encode_single(self, processor, text, images=None) -> Dict[str, Any]:
259
+ """Single-sample processor call."""
260
+ kwargs = {"text": [text], "padding": False, "truncation": False,
261
+ "return_tensors": "pt"}
262
+ if images:
263
+ kwargs["images"] = images
264
+ enc = processor(**kwargs)
265
+ skip = {"input_ids", "attention_mask", "mm_token_type_ids"}
266
+ multi_modal_inputs = {k: v for k, v in enc.items() if k not in skip}
267
+ return {
268
+ "input_ids": enc["input_ids"][0],
269
+ "attention_mask": enc["attention_mask"][0],
270
+ "multi_modal_inputs": multi_modal_inputs or None,
271
+ }
272
+
273
+ @staticmethod
274
+ def _build_loss_mask(attn_mask, resp_lens):
275
+ bs, seq_len = attn_mask.shape
276
+ loss_mask = torch.zeros(bs, seq_len, dtype=torch.bool)
277
+ for i in range(bs):
278
+ real_end = attn_mask[i].sum().item()
279
+ start = max(real_end - resp_lens[i] - 1, 0)
280
+ loss_mask[i, start:real_end - 1] = True
281
+ return loss_mask
282
+
283
+ def collate_fn(self, item_list: List[Dict]) -> Dict[str, torch.Tensor]:
284
+ bsz = len(item_list)
285
+ per_sample_images = (
286
+ [item["images"] for item in item_list]
287
+ if self.image_key else [None] * bsz
288
+ )
289
+
290
+ stu_full = [item["stu_prompt"] + item["stu_response"] for item in item_list]
291
+ stu_encs = [
292
+ self._encode_single(self.student_processor, stu_full[i], per_sample_images[i])
293
+ for i in range(bsz)
294
+ ]
295
+ stu_pad_id = self.student_tokenizer.pad_token_id
296
+ stu_input_ids = zero_pad_sequences(
297
+ [e["input_ids"] for e in stu_encs], side="right", value=stu_pad_id,
298
+ )
299
+ stu_attn_mask = zero_pad_sequences(
300
+ [e["attention_mask"] for e in stu_encs], side="right", value=0,
301
+ ).long()
302
+ stu_loss_mask = self._build_loss_mask(
303
+ stu_attn_mask, [item["stu_resp_len"] for item in item_list],
304
+ )
305
+
306
+ batch: Dict[str, Any] = {
307
+ "stu_input_ids": stu_input_ids,
308
+ "stu_attn_mask": stu_attn_mask,
309
+ "stu_loss_mask": stu_loss_mask,
310
+ }
311
+ if any(e["multi_modal_inputs"] is not None for e in stu_encs):
312
+ batch["stu_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in stu_encs]
313
+
314
+ if "tea_prompt" in item_list[0]:
315
+ if not self.teacher_student_share_input:
316
+ tea_full = [item["tea_prompt"] + item["tea_response"] for item in item_list]
317
+ tea_encs = []
318
+ for i in range(bsz):
319
+ if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]:
320
+ proc = self.teacher_processors[item_list[i]["teacher_routing_key"]]
321
+ else:
322
+ proc = self.teacher_processors.get("default", self.student_processor)
323
+ tea_encs.append(self._encode_single(proc, tea_full[i], per_sample_images[i]))
324
+ if self.args.kd.multi_teacher_config:
325
+ # Multi-teacher mode validates shared vocab with the student.
326
+ tea_pad_id = self.student_tokenizer.pad_token_id
327
+ else:
328
+ teacher_processor = self.teacher_processors.get("default", self.student_processor)
329
+ teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor)
330
+ tea_pad_id = teacher_tokenizer.pad_token_id
331
+ tea_input_ids = zero_pad_sequences(
332
+ [e["input_ids"] for e in tea_encs], side="right", value=tea_pad_id,
333
+ )
334
+ tea_attn_mask = zero_pad_sequences(
335
+ [e["attention_mask"] for e in tea_encs], side="right", value=0,
336
+ ).long()
337
+ batch["tea_input_ids"] = tea_input_ids
338
+ batch["tea_attn_mask"] = tea_attn_mask
339
+ batch["tea_loss_mask"] = self._build_loss_mask(
340
+ tea_attn_mask, [item["tea_resp_len"] for item in item_list],
341
+ )
342
+ if any(e["multi_modal_inputs"] is not None for e in tea_encs):
343
+ batch["tea_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in tea_encs]
344
+ else:
345
+ batch["tea_input_ids"] = batch["stu_input_ids"]
346
+ batch["tea_attn_mask"] = batch["stu_attn_mask"]
347
+ batch["tea_loss_mask"] = batch["stu_loss_mask"]
348
+ if "stu_multi_modal_inputs" in batch:
349
+ batch["tea_multi_modal_inputs"] = batch["stu_multi_modal_inputs"]
350
+
351
+ # Teacher feed input_ids for SGLang. Text: reuse encoded ids (exact).
352
+ # Multimodal: tokenize raw text (single image placeholder) so SGLang re-expands it.
353
+ if not self.teacher_student_share_input:
354
+ if self.image_key:
355
+ tea_feed = []
356
+ for i in range(bsz):
357
+ if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]:
358
+ proc = self.teacher_processors[item_list[i]["teacher_routing_key"]]
359
+ else:
360
+ proc = self.teacher_processors.get("default", self.student_processor)
361
+ tokenizer = getattr(proc, "tokenizer", proc)
362
+ tea_feed.append(tokenizer(tea_full[i])["input_ids"])
363
+ batch["tea_feed_input_ids"] = tea_feed
364
+ else:
365
+ batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in tea_encs]
366
+ else:
367
+ if self.image_key:
368
+ batch["tea_feed_input_ids"] = [
369
+ self.student_tokenizer(stu_full[i])["input_ids"] for i in range(bsz)
370
+ ]
371
+ else:
372
+ batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in stu_encs]
373
+
374
+ if self.image_key:
375
+ batch["images"] = per_sample_images
376
+
377
+ if "teacher_routing_key" in item_list[0]:
378
+ batch["teacher_routing_key"] = [item["teacher_routing_key"] for item in item_list]
379
+
380
+ return batch
kdflow/datasets/utils.py ADDED
@@ -0,0 +1,387 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from typing import List, Dict, Any
3
+
4
+ import torch
5
+ from datasets import DatasetDict, interleave_datasets, load_dataset, load_from_disk
6
+ from transformers import AutoProcessor, AutoTokenizer
7
+
8
+ from kdflow.utils.logging_utils import init_logger
9
+
10
+ logger = init_logger(__name__)
11
+
12
+
13
+ def validate_dataset_columns(dataset, **column_settings):
14
+ """Validate that configured dataset keys exist in the source dataset.
15
+
16
+ Supports dot-notation for nested keys (e.g., "reward_model.ground_truth" -> top-key "reward_model").
17
+ """
18
+ available_columns = list(dataset.column_names)
19
+ missing_columns = [
20
+ f"{setting_name}={column_name!r}"
21
+ for setting_name, column_name in column_settings.items()
22
+ if column_name.split(".")[0] not in available_columns
23
+ ]
24
+ if missing_columns:
25
+ raise ValueError(
26
+ f"Dataset is missing configured dataset column(s): "
27
+ f"{', '.join(missing_columns)}. Available columns: {available_columns}."
28
+ )
29
+
30
+
31
+ def exist_and_not_none(d, key):
32
+ return key in d and not d[key] is None
33
+
34
+
35
+ def get_nested_value(data: dict, key_path: str, default=None):
36
+ """Get a value from a nested dict using dot-notation key path.
37
+
38
+ Example:
39
+ get_nested_value({'a': {'b': 1}}, 'a.b') -> 1
40
+ get_nested_value({'a': 1}, 'a.b') -> None
41
+ """
42
+ keys = key_path.split(".")
43
+ value = data
44
+ for k in keys:
45
+ if isinstance(value, dict):
46
+ value = value.get(k)
47
+ else:
48
+ return default
49
+ return value
50
+
51
+
52
+ def blending_datasets(
53
+ datasets,
54
+ probabilities=None,
55
+ strategy=None,
56
+ seed=42,
57
+ max_count=1e8,
58
+ stopping_strategy="all_exhausted",
59
+ dataset_split="train",
60
+ ):
61
+ """Blend multiple datasets with optional probability sampling.
62
+
63
+ Args:
64
+ datasets (str): Comma-separated list of dataset paths
65
+ probabilities (str, optional): Comma-separated list of probabilities for sampling.
66
+ If None, datasets will be concatenated without probability sampling.
67
+ strategy: Training strategy object
68
+ seed (int): Random seed
69
+ max_count (int): Maximum number of samples per dataset
70
+ """
71
+ dataset_split = dataset_split or "train"
72
+ datasets = datasets.split(",")
73
+ if probabilities is not None:
74
+ probabilities = list(map(float, probabilities.split(",")))
75
+ assert len(probabilities) == len(datasets)
76
+
77
+ data_list = []
78
+ for i, dataset in enumerate(datasets):
79
+ dataset = dataset.strip()
80
+ logger.info(f"dataset: {dataset}")
81
+
82
+ data_dir = dataset.split("@")[1].strip() if "@" in dataset else None
83
+ dataset = dataset.split("@")[0].strip()
84
+ dataset_basename = os.path.basename(dataset)
85
+
86
+ ext = os.path.splitext(dataset)[-1]
87
+ if ext == ".py" or (
88
+ os.path.isdir(dataset) and os.path.exists(os.path.join(dataset, f"{dataset_basename}.py"))
89
+ ):
90
+ data = load_dataset(dataset, trust_remote_code=True)
91
+ logger.info(f"loaded {dataset} with python script")
92
+ elif ext in [".json", ".jsonl", ".csv", ".parquet", ".arrow"]:
93
+ ext = ext.lower().strip(".")
94
+ if ext == "jsonl":
95
+ ext = "json"
96
+ data = load_dataset(ext, data_files={dataset_split: dataset})
97
+ logger.info(f"loaded {dataset} with data_files={dataset}")
98
+ # local dataset saved with `datasets.Dataset.save_to_disk`
99
+ elif os.path.isdir(dataset):
100
+ try:
101
+ data = load_from_disk(dataset)
102
+ logger.info(f"loaded {dataset} from disk")
103
+ except Exception as e:
104
+ data = load_dataset(dataset, data_dir=data_dir)
105
+ logger.info(f"loaded {dataset} from files")
106
+ else:
107
+ data = load_dataset(dataset, data_dir=data_dir)
108
+ logger.info(f"loaded {dataset} from files")
109
+
110
+ if isinstance(data, DatasetDict):
111
+ if dataset_split not in data:
112
+ raise ValueError(
113
+ f"Dataset {dataset!r} does not contain split {dataset_split!r}. "
114
+ f"Available splits: {list(data)}."
115
+ )
116
+ data = data[dataset_split]
117
+ data = data.select(range(min(max_count, len(data))))
118
+ data_list.append(data)
119
+
120
+ if strategy.is_rank_0():
121
+ print(data_list)
122
+
123
+ if probabilities is None:
124
+ from datasets import concatenate_datasets
125
+
126
+ dataset = concatenate_datasets(data_list)
127
+ else:
128
+ dataset = interleave_datasets(
129
+ data_list,
130
+ probabilities=probabilities,
131
+ seed=seed,
132
+ stopping_strategy=stopping_strategy,
133
+ )
134
+
135
+ return dataset
136
+
137
+
138
+ def get_tokenizer_or_processor(model_name_or_path, model=None, padding_side="right", use_fast=True, need_processor=False):
139
+ if need_processor:
140
+ return get_processor(model_name_or_path, model, padding_side, use_fast)
141
+ else:
142
+ return get_tokenizer(model_name_or_path, model, padding_side, use_fast)
143
+
144
+
145
+ def get_processor(model_name_or_path, model=None, padding_side="right", use_fast=True):
146
+ processor = AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast)
147
+ processor.tokenizer.padding_side = padding_side
148
+ if processor.tokenizer.pad_token is None:
149
+ logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
150
+ processor.tokenizer.pad_token = processor.tokenizer.eos_token
151
+ processor.tokenizer.pad_token_id = processor.tokenizer.eos_token_id
152
+ if model is not None:
153
+ model.config.text_config.pad_token_id = processor.tokenizer.pad_token_id
154
+
155
+ return processor
156
+
157
+
158
+ def get_tokenizer(model_name_or_path, model=None, padding_side="right", use_fast=True):
159
+ try:
160
+ # fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style
161
+ # tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it
162
+ # with the Metaspace pre-tokenizer rebuilt from the slow tokenizer.
163
+ tokenizer = AutoTokenizer.from_pretrained(
164
+ model_name_or_path,
165
+ trust_remote_code=True,
166
+ use_fast=use_fast,
167
+ fix_mistral_regex=True,
168
+ )
169
+ except TypeError:
170
+ # Older transformers versions do not accept fix_mistral_regex.
171
+ tokenizer = AutoTokenizer.from_pretrained(
172
+ model_name_or_path,
173
+ trust_remote_code=True,
174
+ use_fast=use_fast,
175
+ )
176
+ tokenizer.padding_side = padding_side
177
+ if tokenizer.pad_token is None:
178
+ logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
179
+ tokenizer.pad_token = tokenizer.eos_token
180
+ tokenizer.pad_token_id = tokenizer.eos_token_id
181
+ if model is not None:
182
+ model.config.pad_token_id = tokenizer.pad_token_id
183
+
184
+ return tokenizer
185
+
186
+
187
+ def convert_token_to_id(token, tokenizer):
188
+ if isinstance(token, str):
189
+ token = tokenizer.encode(token, add_special_tokens=False)
190
+ assert len(token) == 1
191
+ return token[0]
192
+ else:
193
+ raise ValueError("token should be int or str")
194
+
195
+
196
+ def zero_pad_sequences(
197
+ sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False
198
+ ) -> torch.Tensor:
199
+ from torch.nn.utils.rnn import pad_sequence
200
+ assert side in ("left", "right")
201
+ sequences = [seq.squeeze(0) for seq in sequences]
202
+ if side == "left":
203
+ sequences = [seq.flip(dims=0) for seq in sequences]
204
+ padded_sequences = pad_sequence(
205
+ sequences,
206
+ batch_first=True,
207
+ padding_value=value
208
+ )
209
+ if side == "left":
210
+ padded_sequences = torch.flip(padded_sequences, dims=[1])
211
+ return padded_sequences
212
+
213
+
214
+ def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True):
215
+ """Remove the pad token. Return tensors and not lists.
216
+
217
+ Args:
218
+ input_ids shape: [bs, seq_length]
219
+ attention_mask shape: [bs, seq_length]
220
+ Returns:
221
+ no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query.
222
+ """
223
+ no_padding_batch = []
224
+ for ids, mask in zip(input_ids, attention_mask):
225
+ # Fix for both left and right padding
226
+ ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist()
227
+ no_padding_batch.append(ids)
228
+ return no_padding_batch
229
+
230
+
231
+ # ShareGPT role mapping to OpenAI roles
232
+ SHAREGPT_ROLE_MAP = {
233
+ "human": "user",
234
+ "user": "user",
235
+ "gpt": "assistant",
236
+ "assistant": "assistant",
237
+ "system": "system",
238
+ }
239
+
240
+
241
+ def _is_sharegpt_format(data):
242
+ """Check if data is in ShareGPT format (list of dicts with 'from'/'value' keys)."""
243
+ if not isinstance(data, list) or not data or not isinstance(data[0], dict):
244
+ return False
245
+ return "from" in data[0] and "value" in data[0]
246
+
247
+
248
+ def _is_openai_format(data):
249
+ """Check if data is already in OpenAI messages format (list of dicts with 'role'/'content' keys)."""
250
+ if not isinstance(data, list) or not data or not isinstance(data[0], dict):
251
+ return False
252
+ return "role" in data[0] and "content" in data[0]
253
+
254
+
255
+ def _is_alpaca_format(data):
256
+ """Check if data is in Alpaca format (dict with 'instruction' key)."""
257
+ if not isinstance(data, dict):
258
+ return False
259
+ return "instruction" in data
260
+
261
+
262
+ def _convert_sharegpt(messages):
263
+ """Convert ShareGPT format messages to OpenAI messages format.
264
+
265
+ ShareGPT: [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]
266
+ OpenAI: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
267
+ """
268
+ converted = []
269
+ for msg in messages:
270
+ role = SHAREGPT_ROLE_MAP.get(msg["from"], msg["from"])
271
+ content = msg.get("value") or ""
272
+ converted.append({"role": role, "content": content})
273
+ return converted
274
+
275
+
276
+ def _convert_alpaca(data):
277
+ """Convert Alpaca format to OpenAI messages format.
278
+
279
+ Alpaca format fields:
280
+ - instruction (required): the main instruction/question
281
+ - input (optional): additional input context
282
+ - output (optional): the expected response
283
+ - system (optional): system prompt
284
+ - history (optional): list of [user_msg, assistant_msg] pairs
285
+
286
+ Returns:
287
+ List of OpenAI messages: [{"role": "...", "content": "..."}, ...]
288
+ """
289
+ messages = []
290
+
291
+ # System prompt
292
+ system_prompt = data.get("system", "")
293
+ if system_prompt:
294
+ messages.append({"role": "system", "content": system_prompt})
295
+
296
+ # History turns
297
+ history = data.get("history", [])
298
+ for turn in history:
299
+ if isinstance(turn, (list, tuple)) and len(turn) == 2:
300
+ messages.append({"role": "user", "content": turn[0]})
301
+ messages.append({"role": "assistant", "content": turn[1]})
302
+
303
+ # Current instruction + optional input
304
+ instruction = data.get("instruction", "")
305
+ extra_input = data.get("input", "")
306
+ if extra_input:
307
+ user_content = f"{instruction}\n{extra_input}"
308
+ else:
309
+ user_content = instruction
310
+ messages.append({"role": "user", "content": user_content})
311
+
312
+ # Output (response)
313
+ output = data.get("output", "")
314
+ if output:
315
+ messages.append({"role": "assistant", "content": output})
316
+
317
+ return messages
318
+
319
+
320
+ def convert_to_openai_messages(data, role="user", expand_image=False):
321
+ """Unified converter: auto-detect data format and convert to OpenAI messages.
322
+
323
+ Supported input formats:
324
+ 1. OpenAI messages (already): [{"role": "user", "content": "..."}] -> returned as-is
325
+ 2. ShareGPT: [{"from": "human", "value": "..."}] -> converted
326
+ 3. Alpaca: {"instruction": "...", "input": "...", "output": "...", ...} -> converted
327
+ 4. Plain string: "..." -> wrapped as [{"role": role, "content": "..."}]
328
+
329
+ Args:
330
+ data: Input data in any supported format.
331
+ role: Role to assign when data is a plain string. Defaults to "user".
332
+ expand_image: Whether to expand <image> placeholders into multi-part content.
333
+ Only set to True for multimodal (image) scenarios. Defaults to False.
334
+
335
+ Returns:
336
+ List of dicts in OpenAI messages format: [{"role": "...", "content": "..."}, ...]
337
+ """
338
+ if data is None:
339
+ raise ValueError("convert_to_openai_messages received None input.")
340
+
341
+ if isinstance(data, str):
342
+ messages = [{"role": role, "content": data}]
343
+ elif isinstance(data, list):
344
+ if not data:
345
+ raise ValueError("convert_to_openai_messages received an empty list.")
346
+ if _is_openai_format(data):
347
+ messages = data
348
+ elif _is_sharegpt_format(data):
349
+ messages = _convert_sharegpt(data)
350
+ else:
351
+ raise ValueError(
352
+ f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
353
+ f"Got: {type(data)} with keys/content: {list(data[0].keys()) if data else data}"
354
+ )
355
+ elif isinstance(data, dict):
356
+ if _is_alpaca_format(data):
357
+ messages = _convert_alpaca(data)
358
+ else:
359
+ raise ValueError(
360
+ f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
361
+ f"Got: {type(data)} with keys/content: {data}"
362
+ )
363
+ else:
364
+ raise ValueError(
365
+ f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
366
+ f"Got: {type(data)}"
367
+ )
368
+
369
+ if expand_image:
370
+ messages = expand_image_placeholders(messages)
371
+ return messages
372
+
373
+
374
+ def expand_image_placeholders(messages, placeholder="<image>"):
375
+ for msg in messages:
376
+ content = msg.get("content")
377
+ if not isinstance(content, str) or placeholder not in content:
378
+ continue
379
+ parts = content.split(placeholder)
380
+ content_list = []
381
+ for i, part in enumerate(parts):
382
+ if part:
383
+ content_list.append({"type": "text", "text": part})
384
+ if i < len(parts) - 1:
385
+ content_list.append({"type": "image"})
386
+ msg["content"] = content_list
387
+ return messages
kdflow/evaluation/__init__.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Evaluation helpers."""
2
+
3
+ from kdflow.evaluation.alfworld_evaluator import AlfWorldEvaluator
4
+ from kdflow.evaluation.math_evaluator import MathEvaluator
5
+ from kdflow.evaluation.verl_agent_alfworld_evaluator import (
6
+ VerlAgentAlfWorldEvaluator,
7
+ )
8
+
9
+ __all__ = [
10
+ "AlfWorldEvaluator",
11
+ "MathEvaluator",
12
+ "VerlAgentAlfWorldEvaluator",
13
+ ]
kdflow/evaluation/alfworld_evaluator.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Standalone ALFWorld evaluation with no teacher or optimizer actors."""
2
+
3
+ import json
4
+ import logging
5
+ import os
6
+ import time
7
+ from datetime import datetime, timezone
8
+ from typing import Any, Dict, List
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ class AlfWorldEvaluator:
14
+ """Evaluate a rollout-only model on a fixed number of ALFWorld episodes."""
15
+
16
+ def __init__(self, args, rollout_manager):
17
+ self.args = args
18
+ self.rollout_manager = rollout_manager
19
+ self.num_episodes = args.rollout.alfworld_eval_episodes
20
+ self.batch_size = args.rollout.rollout_batch_size
21
+ self.output_dir = args.train.save_path
22
+
23
+ if self.num_episodes <= 0:
24
+ raise ValueError("alfworld_eval_episodes must be positive")
25
+ if self.num_episodes % self.batch_size:
26
+ raise ValueError(
27
+ "alfworld_eval_episodes must be divisible by rollout_batch_size "
28
+ f"(got {self.num_episodes} and {self.batch_size})"
29
+ )
30
+
31
+ @staticmethod
32
+ def _trajectory_row(record: Dict[str, Any], batch_index: int) -> Dict[str, Any]:
33
+ output = record.get("output") or {}
34
+ metadata = dict(record.get("metadata") or {})
35
+ row = {
36
+ "batch_index": batch_index,
37
+ "prompt": record.get("stu_prompt", ""),
38
+ "response": output.get("text", ""),
39
+ }
40
+ finish_reason = output.get("finish_reason")
41
+ if finish_reason is None:
42
+ finish_reason = (output.get("meta_info") or {}).get("finish_reason")
43
+ if finish_reason is not None:
44
+ row["finish_reason"] = finish_reason
45
+ row.update(metadata)
46
+ return row
47
+
48
+ def evaluate(self) -> Dict[str, Any]:
49
+ os.makedirs(self.output_dir, exist_ok=True)
50
+ trajectory_path = os.path.join(self.output_dir, "trajectories.jsonl")
51
+ summary_path = os.path.join(self.output_dir, "summary.json")
52
+ num_batches = self.num_episodes // self.batch_size
53
+ sampling_params = {
54
+ "max_new_tokens": self.args.rollout.generate_max_len,
55
+ "temperature": self.args.rollout.temperature,
56
+ "top_p": self.args.rollout.top_p,
57
+ }
58
+
59
+ totals = {
60
+ "episodes": 0.0,
61
+ "episodes_succeeded": 0.0,
62
+ "turns": 0.0,
63
+ "turns_max": 0.0,
64
+ "invalid_actions": 0.0,
65
+ "environment_failures": 0.0,
66
+ "context_truncated": 0.0,
67
+ }
68
+ started_at = time.time()
69
+
70
+ logger.info(
71
+ "Starting standalone ALFWorld evaluation: model=%s split=%s "
72
+ "episodes=%d batch_size=%d",
73
+ self.args.model.student_name_or_path,
74
+ self.args.rollout.alfworld_split,
75
+ self.num_episodes,
76
+ self.batch_size,
77
+ )
78
+ with open(trajectory_path, "w", encoding="utf-8") as trajectory_file:
79
+ for batch_index in range(1, num_batches + 1):
80
+ prompt_batch: List[Dict[str, int]] = [
81
+ {"episode_slot": slot} for slot in range(self.batch_size)
82
+ ]
83
+ records, metrics = self.rollout_manager.collect_episodes(
84
+ prompt_batch=prompt_batch,
85
+ global_step=batch_index,
86
+ mode="eval",
87
+ **sampling_params,
88
+ )
89
+ for record in records:
90
+ row = self._trajectory_row(record, batch_index)
91
+ trajectory_file.write(
92
+ json.dumps(row, ensure_ascii=False) + "\n"
93
+ )
94
+ trajectory_file.flush()
95
+
96
+ prefix = "eval/alfworld"
97
+ totals["episodes"] += metrics.get(f"{prefix}/episodes", 0.0)
98
+ totals["episodes_succeeded"] += metrics.get(
99
+ f"{prefix}/episodes_succeeded", 0.0
100
+ )
101
+ totals["turns"] += metrics.get(f"{prefix}/turns/total", 0.0)
102
+ totals["turns_max"] = max(
103
+ totals["turns_max"],
104
+ metrics.get(f"{prefix}/turns/max", 0.0),
105
+ )
106
+ totals["invalid_actions"] += metrics.get(
107
+ f"{prefix}/invalid_action_count", 0.0
108
+ )
109
+ totals["environment_failures"] += metrics.get(
110
+ f"{prefix}/environment_failure_count", 0.0
111
+ )
112
+ totals["context_truncated"] += metrics.get(
113
+ f"{prefix}/context_truncated_count", 0.0
114
+ )
115
+ logger.info(
116
+ "ALFWorld evaluation batch %d/%d complete "
117
+ "(running success rate %.4f)",
118
+ batch_index,
119
+ num_batches,
120
+ totals["episodes_succeeded"] / max(totals["episodes"], 1.0),
121
+ )
122
+
123
+ episodes = totals["episodes"]
124
+ turns = totals["turns"]
125
+ summary = {
126
+ "model": self.args.model.student_name_or_path,
127
+ "split": self.args.rollout.alfworld_split,
128
+ "episodes": int(episodes),
129
+ "episodes_succeeded": int(totals["episodes_succeeded"]),
130
+ "success_rate": totals["episodes_succeeded"] / max(episodes, 1.0),
131
+ "turns": {
132
+ "total": int(turns),
133
+ "mean": turns / max(episodes, 1.0),
134
+ "max": int(totals["turns_max"]),
135
+ },
136
+ "invalid_actions": {
137
+ "count": int(totals["invalid_actions"]),
138
+ "ratio": totals["invalid_actions"] / max(turns, 1.0),
139
+ },
140
+ "environment_failures": {
141
+ "count": int(totals["environment_failures"]),
142
+ "ratio": totals["environment_failures"] / max(turns, 1.0),
143
+ },
144
+ "context_truncated": {
145
+ "count": int(totals["context_truncated"]),
146
+ "ratio": totals["context_truncated"] / max(episodes, 1.0),
147
+ },
148
+ "max_turns_per_episode": self.args.rollout.alfworld_max_turns,
149
+ "sampling": sampling_params,
150
+ "seed": self.args.train.seed,
151
+ "elapsed_seconds": time.time() - started_at,
152
+ "completed_at": datetime.now(timezone.utc).isoformat(),
153
+ "trajectories_path": os.path.abspath(trajectory_path),
154
+ }
155
+ with open(summary_path, "w", encoding="utf-8") as summary_file:
156
+ json.dump(summary, summary_file, ensure_ascii=False, indent=2)
157
+ summary_file.write("\n")
158
+
159
+ logger.info(
160
+ "ALFWorld evaluation complete: success_rate=%.4f summary=%s",
161
+ summary["success_rate"],
162
+ summary_path,
163
+ )
164
+ return summary
kdflow/evaluation/math_evaluator.py ADDED
@@ -0,0 +1,322 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import json
3
+ import logging
4
+ import os
5
+ from typing import Any, Callable, Dict, List, Optional
6
+
7
+ from tqdm import tqdm
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ def extract_last_boxed(text: str) -> Optional[str]:
13
+ """Return the content of the last complete ``\\boxed`` or ``\\fbox``."""
14
+ if not isinstance(text, str):
15
+ return None
16
+
17
+ boxed_index = text.rfind("\\boxed")
18
+ fbox_index = text.rfind("\\fbox")
19
+ start = max(boxed_index, fbox_index)
20
+ if start < 0:
21
+ return None
22
+
23
+ left_brace = text.find("{", start)
24
+ if left_brace < 0:
25
+ return None
26
+
27
+ depth = 0
28
+ for index in range(left_brace, len(text)):
29
+ if text[index] == "{":
30
+ depth += 1
31
+ elif text[index] == "}":
32
+ depth -= 1
33
+ if depth == 0:
34
+ return text[left_brace + 1:index]
35
+ return None
36
+
37
+
38
+ def verify_math_answer(prediction: Optional[str], label: Any) -> bool:
39
+ """Compare a boxed prediction with a reference answer using math-verify."""
40
+ try:
41
+ from math_verify import parse, verify
42
+ except ImportError as error:
43
+ raise RuntimeError(
44
+ "Math evaluation requires the 'math-verify' package. "
45
+ "Install the project dependencies before starting training."
46
+ ) from error
47
+
48
+ if prediction is None:
49
+ return False
50
+
51
+ try:
52
+ prediction = prediction[:300]
53
+ return bool(
54
+ verify(
55
+ parse(f"\\boxed{{{label}}}"),
56
+ parse(f"\\boxed{{{prediction}}}"),
57
+ )
58
+ )
59
+ except Exception:
60
+ return False
61
+
62
+
63
+ def score_math_responses(
64
+ responses: List[List[str]],
65
+ labels: List[Any],
66
+ verifier: Callable[[Optional[str], Any], bool] = verify_math_answer,
67
+ ) -> tuple[Dict[str, float], List[Dict[str, Any]]]:
68
+ """Score multiple generated responses per problem and compute pass@k."""
69
+ if len(responses) != len(labels):
70
+ raise ValueError("responses and labels must contain the same number of problems")
71
+
72
+ total_correct = 0
73
+ total_responses = 0
74
+ passed_problems = 0
75
+ results = []
76
+
77
+ for index, (problem_responses, label) in enumerate(zip(responses, labels)):
78
+ predictions = [extract_last_boxed(response) for response in problem_responses]
79
+ correctness = [verifier(prediction, label) for prediction in predictions]
80
+ num_correct = sum(correctness)
81
+ total_correct += num_correct
82
+ total_responses += len(correctness)
83
+ passed_problems += int(any(correctness))
84
+ results.append(
85
+ {
86
+ "index": index,
87
+ "answer": str(label),
88
+ "pred_answers": predictions,
89
+ "responses": problem_responses,
90
+ "acc_list": correctness,
91
+ "num_correct": num_correct,
92
+ }
93
+ )
94
+
95
+ num_problems = len(labels)
96
+ accuracy = total_correct / total_responses if total_responses else 0.0
97
+ pass_at_k = passed_problems / num_problems if num_problems else 0.0
98
+ metrics = {
99
+ "eval/accuracy": accuracy,
100
+ "eval/num_correct": float(total_correct),
101
+ "eval/num_total": float(total_responses),
102
+ "eval/pass_at_k": pass_at_k,
103
+ }
104
+ return metrics, results
105
+
106
+
107
+ class MathEvaluator:
108
+ """Run high-concurrency, multi-sample math evaluation through SGLang."""
109
+
110
+ def __init__(self, strategy, rollout_group, generate_kwargs, student_processor):
111
+ self.args = strategy.args
112
+ self.rollout_group = rollout_group
113
+ self.generate_kwargs = generate_kwargs or {}
114
+ self.student_processor = student_processor
115
+ self.dataset_path = self.args.data.eval_math_dataset_path
116
+ self.n_samples = self.args.data.eval_math_n_samples
117
+ self.max_concurrent = self.args.data.eval_math_max_concurrent
118
+ self.max_tokens = self.args.data.eval_math_max_tokens
119
+ self.timeout = self.args.data.eval_math_timeout
120
+
121
+ # Fail before training if the optional evaluator dependency is unavailable.
122
+ try:
123
+ import math_verify # noqa: F401
124
+ except ImportError as error:
125
+ raise RuntimeError(
126
+ "Math evaluation requires the 'math-verify' package. "
127
+ "Install the project dependencies before starting training."
128
+ ) from error
129
+
130
+ def evaluate(self, global_step: int) -> Dict[str, float]:
131
+ eval_data = self._load_dataset()
132
+ prompts, labels = self._build_prompts(eval_data)
133
+ sampling_params = {
134
+ "max_new_tokens": self.max_tokens,
135
+ "temperature": self.generate_kwargs.get("temperature", 0.6),
136
+ "top_p": self.generate_kwargs.get("top_p", 0.95),
137
+ "n": self.n_samples,
138
+ }
139
+
140
+ logger.info(
141
+ "[Math Eval] Starting %d problems at step %d "
142
+ "(n_samples=%d, max_concurrent=%d, timeout=%ds)",
143
+ len(eval_data),
144
+ global_step,
145
+ self.n_samples,
146
+ self.max_concurrent,
147
+ self.timeout,
148
+ )
149
+
150
+ if self.args.train.enable_sleep:
151
+ self.rollout_group.wakeup()
152
+ try:
153
+ responses = self._generate(prompts, sampling_params)
154
+ finally:
155
+ if self.args.train.enable_sleep:
156
+ self.rollout_group.sleep()
157
+
158
+ metrics, results = score_math_responses(responses, labels)
159
+ pass_at_k = metrics.pop("eval/pass_at_k")
160
+ if self.n_samples > 1:
161
+ metrics[f"eval/pass@{self.n_samples}"] = pass_at_k
162
+ for result, item in zip(results, eval_data):
163
+ result["problem"] = item["problem"]
164
+
165
+ self._save_results(
166
+ global_step, metrics, results, len(eval_data), pass_at_k
167
+ )
168
+ return metrics
169
+
170
+ def _load_dataset(self) -> List[Dict[str, Any]]:
171
+ data = []
172
+ with open(self.dataset_path, "r", encoding="utf-8") as file:
173
+ for line_number, line in enumerate(file, start=1):
174
+ if not line.strip():
175
+ continue
176
+ try:
177
+ item = json.loads(line)
178
+ except json.JSONDecodeError as error:
179
+ raise ValueError(
180
+ f"Invalid JSON in math evaluation dataset at line {line_number}"
181
+ ) from error
182
+ missing = [key for key in ("problem", "answer") if key not in item]
183
+ if missing:
184
+ raise ValueError(
185
+ f"Math evaluation dataset line {line_number} is missing "
186
+ f"field(s): {', '.join(missing)}"
187
+ )
188
+ data.append(item)
189
+ if not data:
190
+ raise ValueError("Math evaluation dataset is empty")
191
+ return data
192
+
193
+ def _build_prompts(
194
+ self, eval_data: List[Dict[str, Any]]
195
+ ) -> tuple[List[List[int]], List[Any]]:
196
+ tokenizer = getattr(
197
+ self.student_processor, "tokenizer", self.student_processor
198
+ )
199
+ prompts = []
200
+ labels = []
201
+ for item in eval_data:
202
+ messages = [{"role": "user", "content": str(item["problem"])}]
203
+ template_kwargs = {}
204
+ if "enable_thinking" in str(getattr(tokenizer, "chat_template", "")):
205
+ template_kwargs["enable_thinking"] = self.args.model.enable_thinking
206
+ prompt = tokenizer.apply_chat_template(
207
+ messages,
208
+ tokenize=False,
209
+ add_generation_prompt=True,
210
+ **template_kwargs,
211
+ )
212
+ prompts.append(tokenizer.encode(prompt, add_special_tokens=False))
213
+ labels.append(item["answer"])
214
+ return prompts, labels
215
+
216
+ def _generate(
217
+ self, prompt_token_ids: List[List[int]], sampling_params: Dict[str, Any]
218
+ ) -> List[List[str]]:
219
+ if not prompt_token_ids:
220
+ return []
221
+ return asyncio.run(
222
+ self._generate_async(prompt_token_ids, sampling_params)
223
+ )
224
+
225
+ async def _generate_async(
226
+ self, prompt_token_ids: List[List[int]], sampling_params: Dict[str, Any]
227
+ ) -> List[List[str]]:
228
+ import aiohttp
229
+
230
+ concurrency = min(len(prompt_token_ids), self.max_concurrent)
231
+ semaphore = asyncio.Semaphore(concurrency)
232
+ results: List[Optional[List[str]]] = [None] * len(prompt_token_ids)
233
+ generate_url = f"{self.rollout_group.router_url}/generate"
234
+
235
+ async def generate_one(index, input_ids, session):
236
+ payload = {
237
+ "input_ids": input_ids,
238
+ "sampling_params": sampling_params,
239
+ }
240
+ async with semaphore:
241
+ try:
242
+ async with session.post(generate_url, json=payload) as response:
243
+ response.raise_for_status()
244
+ output = await response.json()
245
+ if isinstance(output, list):
246
+ texts = [item["text"] for item in output]
247
+ elif isinstance(output, dict):
248
+ texts = output.get("text", "")
249
+ texts = [texts] if isinstance(texts, str) else texts
250
+ else:
251
+ raise TypeError(
252
+ f"Unexpected SGLang response type: {type(output).__name__}"
253
+ )
254
+ if not isinstance(texts, list) or len(texts) != self.n_samples:
255
+ raise ValueError(
256
+ f"SGLang returned {len(texts) if isinstance(texts, list) else 0} "
257
+ f"responses, expected {self.n_samples}"
258
+ )
259
+ results[index] = [str(text).strip() for text in texts]
260
+ except Exception as error:
261
+ logger.warning(
262
+ "[Math Eval] Request %d failed with %s: %s; using empty responses",
263
+ index,
264
+ type(error).__name__,
265
+ error,
266
+ )
267
+ results[index] = [""] * self.n_samples
268
+
269
+ connector = aiohttp.TCPConnector(limit=concurrency)
270
+ timeout = aiohttp.ClientTimeout(
271
+ total=self.timeout if self.timeout > 0 else None
272
+ )
273
+ async with aiohttp.ClientSession(
274
+ connector=connector, trust_env=False, timeout=timeout
275
+ ) as session:
276
+ tasks = [
277
+ asyncio.create_task(generate_one(index, ids, session))
278
+ for index, ids in enumerate(prompt_token_ids)
279
+ ]
280
+ progress = tqdm(
281
+ total=len(tasks), desc="[Math Eval] Generating", unit="req"
282
+ )
283
+ try:
284
+ for task in asyncio.as_completed(tasks):
285
+ await task
286
+ progress.update(1)
287
+ finally:
288
+ progress.close()
289
+
290
+ return [result or [""] * self.n_samples for result in results]
291
+
292
+ def _save_results(
293
+ self,
294
+ global_step: int,
295
+ metrics: Dict[str, float],
296
+ results: List[Dict[str, Any]],
297
+ num_problems: int,
298
+ pass_at_k: float,
299
+ ) -> None:
300
+ eval_dir = os.path.join(self.args.train.save_path, "eval_results")
301
+ os.makedirs(eval_dir, exist_ok=True)
302
+
303
+ result_path = os.path.join(eval_dir, f"step{global_step}.jsonl")
304
+ with open(result_path, "w", encoding="utf-8") as file:
305
+ for result in results:
306
+ file.write(json.dumps(result, ensure_ascii=False) + "\n")
307
+
308
+ summary = {
309
+ "global_step": global_step,
310
+ "eval_dataset": self.dataset_path,
311
+ "n_samples": self.n_samples,
312
+ "num_problems": num_problems,
313
+ "num_total_responses": int(metrics["eval/num_total"]),
314
+ "num_correct": int(metrics["eval/num_correct"]),
315
+ "accuracy": metrics["eval/accuracy"],
316
+ "pass_at_k": pass_at_k,
317
+ }
318
+ summary_path = os.path.join(
319
+ eval_dir, f"step{global_step}_summary.json"
320
+ )
321
+ with open(summary_path, "w", encoding="utf-8") as file:
322
+ json.dump(summary, file, indent=2, ensure_ascii=False)
kdflow/evaluation/verl_agent_alfworld_evaluator.py ADDED
@@ -0,0 +1,410 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Multi-seed result writer for the migrated verl-agent ALFWorld protocol."""
2
+
3
+ import json
4
+ import logging
5
+ import os
6
+ import statistics
7
+ import time
8
+ from datetime import datetime, timezone
9
+ from typing import Any, Dict, List
10
+
11
+ from kdflow.evaluation.verl_agent_alfworld_protocol import TASK_TYPES
12
+
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ class VerlAgentAlfWorldEvaluator:
18
+ """Evaluate one model with verl-agent's three-seed ALFWorld recipe."""
19
+
20
+ def __init__(self, args, rollout_manager):
21
+ self.args = args
22
+ self.rollout_manager = rollout_manager
23
+ self.num_episodes = args.rollout.alfworld_eval_episodes
24
+ self.batch_size = args.rollout.rollout_batch_size
25
+ self.output_dir = args.train.save_path
26
+ self.seeds = self.parse_seeds(args.rollout.alfworld_eval_seeds)
27
+ if self.num_episodes <= 0:
28
+ raise ValueError("alfworld_eval_episodes must be positive")
29
+ if self.num_episodes % self.batch_size:
30
+ raise ValueError(
31
+ "alfworld_eval_episodes must be divisible by rollout_batch_size"
32
+ )
33
+
34
+ @staticmethod
35
+ def parse_seeds(value: str) -> List[int]:
36
+ seeds = [int(item.strip()) for item in value.split(",") if item.strip()]
37
+ if not seeds:
38
+ raise ValueError("At least one ALFWorld evaluation seed is required.")
39
+ return seeds
40
+
41
+ @staticmethod
42
+ def _trajectory_row(record: Dict[str, Any], batch_index: int) -> Dict[str, Any]:
43
+ output = record.get("output") or {}
44
+ row = {
45
+ "batch_index": batch_index,
46
+ "prompt": record.get("stu_prompt", ""),
47
+ "response": output.get("text", ""),
48
+ }
49
+ finish_reason = output.get("finish_reason")
50
+ if finish_reason is None:
51
+ finish_reason = (output.get("meta_info") or {}).get("finish_reason")
52
+ if finish_reason is not None:
53
+ row["finish_reason"] = finish_reason
54
+ row.update(record.get("metadata") or {})
55
+ return row
56
+
57
+ def _evaluate_seed(self, script_seed: int) -> Dict[str, Any]:
58
+ if self.rollout_manager is None:
59
+ raise RuntimeError(
60
+ "An ALFWorld rollout manager is required for an unfinished seed."
61
+ )
62
+ seed_dir = os.path.join(self.output_dir, f"seed_{script_seed}")
63
+ os.makedirs(seed_dir, exist_ok=True)
64
+ trajectory_path = os.path.join(seed_dir, "trajectories.jsonl")
65
+ summary_path = os.path.join(seed_dir, "summary.json")
66
+ sampling = {
67
+ "max_new_tokens": self.args.rollout.generate_max_len,
68
+ "temperature": self.args.rollout.temperature,
69
+ "top_p": self.args.rollout.top_p,
70
+ }
71
+ prefix = "eval/alfworld"
72
+ totals = {
73
+ "episodes": 0.0,
74
+ "succeeded": 0.0,
75
+ "turns": 0.0,
76
+ "turns_max": 0.0,
77
+ "format_invalid": 0.0,
78
+ "inadmissible": 0.0,
79
+ "environment_failures": 0.0,
80
+ "prompt_truncated": 0.0,
81
+ }
82
+ task_totals = {
83
+ task: {"episodes": 0.0, "succeeded": 0.0}
84
+ for task in TASK_TYPES + ("unknown",)
85
+ }
86
+ num_batches = self.num_episodes // self.batch_size
87
+ started_at = time.time()
88
+
89
+ with open(trajectory_path, "w", encoding="utf-8") as output_file:
90
+ for batch_index in range(num_batches):
91
+ episode_offset = batch_index * self.batch_size
92
+ self.rollout_manager.set_evaluation_batch(
93
+ script_seed=script_seed,
94
+ episode_offset=episode_offset,
95
+ )
96
+ prompt_batch: List[Dict[str, int]] = [
97
+ {"episode_slot": episode_offset + slot}
98
+ for slot in range(self.batch_size)
99
+ ]
100
+ records, metrics = self.rollout_manager.collect_episodes(
101
+ prompt_batch=prompt_batch,
102
+ global_step=batch_index + 1,
103
+ mode="eval",
104
+ **sampling,
105
+ )
106
+ for record in records:
107
+ output_file.write(
108
+ json.dumps(
109
+ self._trajectory_row(record, batch_index + 1),
110
+ ensure_ascii=False,
111
+ )
112
+ + "\n"
113
+ )
114
+ output_file.flush()
115
+
116
+ totals["episodes"] += metrics.get(f"{prefix}/episodes", 0.0)
117
+ totals["succeeded"] += metrics.get(
118
+ f"{prefix}/episodes_succeeded", 0.0
119
+ )
120
+ totals["turns"] += metrics.get(f"{prefix}/turns/total", 0.0)
121
+ totals["turns_max"] = max(
122
+ totals["turns_max"],
123
+ metrics.get(f"{prefix}/turns/max", 0.0),
124
+ )
125
+ totals["format_invalid"] += metrics.get(
126
+ f"{prefix}/format_invalid_count", 0.0
127
+ )
128
+ totals["inadmissible"] += metrics.get(
129
+ f"{prefix}/inadmissible_action_count", 0.0
130
+ )
131
+ totals["environment_failures"] += metrics.get(
132
+ f"{prefix}/environment_failure_count", 0.0
133
+ )
134
+ totals["prompt_truncated"] += metrics.get(
135
+ f"{prefix}/prompt_truncated_count", 0.0
136
+ )
137
+ for task in task_totals:
138
+ task_totals[task]["episodes"] += metrics.get(
139
+ f"{prefix}/task/{task}/episodes", 0.0
140
+ )
141
+ task_totals[task]["succeeded"] += metrics.get(
142
+ f"{prefix}/task/{task}/succeeded", 0.0
143
+ )
144
+ logger.info(
145
+ "verl-agent ALFWorld seed=%d batch=%d/%d running_success=%.4f",
146
+ script_seed,
147
+ batch_index + 1,
148
+ num_batches,
149
+ totals["succeeded"] / max(totals["episodes"], 1.0),
150
+ )
151
+
152
+ episodes = totals["episodes"]
153
+ turns = totals["turns"]
154
+ tasks = {}
155
+ for task, values in task_totals.items():
156
+ if not values["episodes"]:
157
+ continue
158
+ tasks[task] = {
159
+ "episodes": int(values["episodes"]),
160
+ "succeeded": int(values["succeeded"]),
161
+ "success_rate": values["succeeded"] / values["episodes"],
162
+ }
163
+ summary = {
164
+ "protocol": "verl_agent",
165
+ "source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8",
166
+ "model": self.args.model.student_name_or_path,
167
+ "split": self.args.rollout.alfworld_split,
168
+ "script_seed": script_seed,
169
+ "environment_seed_base": (
170
+ script_seed + self.args.rollout.alfworld_eval_seed_offset
171
+ ),
172
+ "episodes": int(episodes),
173
+ "episodes_succeeded": int(totals["succeeded"]),
174
+ "success_rate": totals["succeeded"] / max(episodes, 1.0),
175
+ "turns": {
176
+ "total": int(turns),
177
+ "mean": turns / max(episodes, 1.0),
178
+ "max": int(totals["turns_max"]),
179
+ },
180
+ "format_invalid": {
181
+ "count": int(totals["format_invalid"]),
182
+ "ratio": totals["format_invalid"] / max(turns, 1.0),
183
+ },
184
+ "inadmissible_actions": {
185
+ "count": int(totals["inadmissible"]),
186
+ "ratio": totals["inadmissible"] / max(turns, 1.0),
187
+ },
188
+ "environment_failures": {
189
+ "count": int(totals["environment_failures"]),
190
+ "ratio": totals["environment_failures"] / max(turns, 1.0),
191
+ },
192
+ "prompt_truncated": {
193
+ "count": int(totals["prompt_truncated"]),
194
+ "ratio": totals["prompt_truncated"] / max(turns, 1.0),
195
+ },
196
+ "task_types": tasks,
197
+ "history_length": self.args.rollout.alfworld_history_length,
198
+ "max_steps": self.args.rollout.alfworld_max_turns,
199
+ "sampling": sampling,
200
+ "elapsed_seconds": time.time() - started_at,
201
+ "completed_at": datetime.now(timezone.utc).isoformat(),
202
+ "trajectories_path": os.path.abspath(trajectory_path),
203
+ }
204
+ with open(summary_path, "w", encoding="utf-8") as output_file:
205
+ json.dump(summary, output_file, ensure_ascii=False, indent=2)
206
+ output_file.write("\n")
207
+ return summary
208
+
209
+ def _load_completed_seed(self, script_seed: int):
210
+ """Return a compatible completed seed summary, otherwise ``None``."""
211
+ seed_dir = os.path.join(self.output_dir, f"seed_{script_seed}")
212
+ summary_path = os.path.join(seed_dir, "summary.json")
213
+ trajectory_path = os.path.join(seed_dir, "trajectories.jsonl")
214
+ if not os.path.isfile(summary_path) or not os.path.isfile(trajectory_path):
215
+ return None
216
+ if os.path.getsize(trajectory_path) <= 0:
217
+ logger.warning(
218
+ "Existing ALFWorld seed=%d trajectory is empty; rerunning.",
219
+ script_seed,
220
+ )
221
+ return None
222
+
223
+ try:
224
+ with open(summary_path, "r", encoding="utf-8") as input_file:
225
+ summary = json.load(input_file)
226
+ except (OSError, json.JSONDecodeError) as error:
227
+ logger.warning(
228
+ "Cannot read existing ALFWorld seed=%d summary (%s); rerunning.",
229
+ script_seed,
230
+ error,
231
+ )
232
+ return None
233
+
234
+ expected_sampling = {
235
+ "max_new_tokens": self.args.rollout.generate_max_len,
236
+ "temperature": self.args.rollout.temperature,
237
+ "top_p": self.args.rollout.top_p,
238
+ }
239
+ expected = {
240
+ "protocol": "verl_agent",
241
+ "source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8",
242
+ "model": self.args.model.student_name_or_path,
243
+ "split": self.args.rollout.alfworld_split,
244
+ "script_seed": script_seed,
245
+ "environment_seed_base": (
246
+ script_seed + self.args.rollout.alfworld_eval_seed_offset
247
+ ),
248
+ "episodes": self.num_episodes,
249
+ "history_length": self.args.rollout.alfworld_history_length,
250
+ "max_steps": self.args.rollout.alfworld_max_turns,
251
+ "sampling": expected_sampling,
252
+ }
253
+ mismatches = [
254
+ key for key, value in expected.items() if summary.get(key) != value
255
+ ]
256
+ required_fields = {
257
+ "episodes_succeeded",
258
+ "success_rate",
259
+ "turns",
260
+ "task_types",
261
+ }
262
+ missing_fields = sorted(required_fields - summary.keys())
263
+ if mismatches or missing_fields:
264
+ logger.warning(
265
+ "Existing ALFWorld seed=%d result is incompatible "
266
+ "(mismatched=%s missing=%s); rerunning.",
267
+ script_seed,
268
+ mismatches,
269
+ missing_fields,
270
+ )
271
+ return None
272
+ turns = summary.get("turns") or {}
273
+ if not {"total", "mean", "max"}.issubset(turns):
274
+ logger.warning(
275
+ "Existing ALFWorld seed=%d result has incomplete turn metrics; "
276
+ "rerunning.",
277
+ script_seed,
278
+ )
279
+ return None
280
+ return summary
281
+
282
+ def has_pending_seeds(self) -> bool:
283
+ """Whether evaluation must load a model and generate new trajectories."""
284
+ if not getattr(self.args.rollout, "alfworld_eval_resume", True):
285
+ return True
286
+ return any(self._load_completed_seed(seed) is None for seed in self.seeds)
287
+
288
+ def evaluate(self) -> Dict[str, Any]:
289
+ os.makedirs(self.output_dir, exist_ok=True)
290
+ started_at = time.time()
291
+ resume_enabled = getattr(
292
+ self.args.rollout, "alfworld_eval_resume", True
293
+ )
294
+ seed_summaries = []
295
+ skipped_existing_seeds = []
296
+ evaluated_seeds = []
297
+ for seed in self.seeds:
298
+ existing = self._load_completed_seed(seed) if resume_enabled else None
299
+ if existing is not None:
300
+ logger.info(
301
+ "Skipping completed compatible ALFWorld seed=%d result.",
302
+ seed,
303
+ )
304
+ seed_summaries.append(existing)
305
+ skipped_existing_seeds.append(seed)
306
+ continue
307
+ seed_summaries.append(self._evaluate_seed(seed))
308
+ evaluated_seeds.append(seed)
309
+ rates = [summary["success_rate"] for summary in seed_summaries]
310
+ total_episodes = sum(summary["episodes"] for summary in seed_summaries)
311
+ total_succeeded = sum(
312
+ summary["episodes_succeeded"] for summary in seed_summaries
313
+ )
314
+ total_turns = sum(
315
+ summary["turns"]["total"] for summary in seed_summaries
316
+ )
317
+ seed_turn_means = [
318
+ summary["turns"]["mean"] for summary in seed_summaries
319
+ ]
320
+ max_turns = max(
321
+ (summary["turns"]["max"] for summary in seed_summaries),
322
+ default=0,
323
+ )
324
+
325
+ task_names = set().union(
326
+ *(summary["task_types"].keys() for summary in seed_summaries)
327
+ )
328
+ task_summary = {}
329
+ for task in sorted(task_names):
330
+ per_seed_rates = [
331
+ summary["task_types"][task]["success_rate"]
332
+ for summary in seed_summaries
333
+ if task in summary["task_types"]
334
+ ]
335
+ task_episodes = sum(
336
+ summary["task_types"].get(task, {}).get("episodes", 0)
337
+ for summary in seed_summaries
338
+ )
339
+ task_succeeded = sum(
340
+ summary["task_types"].get(task, {}).get("succeeded", 0)
341
+ for summary in seed_summaries
342
+ )
343
+ task_summary[task] = {
344
+ "episodes": task_episodes,
345
+ "succeeded": task_succeeded,
346
+ "micro_success_rate": task_succeeded / max(task_episodes, 1),
347
+ "seed_mean_success_rate": statistics.fmean(per_seed_rates),
348
+ "seed_std_success_rate": (
349
+ statistics.pstdev(per_seed_rates)
350
+ if len(per_seed_rates) > 1
351
+ else 0.0
352
+ ),
353
+ }
354
+
355
+ summary = {
356
+ "protocol": "verl_agent",
357
+ "source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8",
358
+ "model": self.args.model.student_name_or_path,
359
+ "split": self.args.rollout.alfworld_split,
360
+ "seeds": self.seeds,
361
+ "environment_seed_offset": self.args.rollout.alfworld_eval_seed_offset,
362
+ "resume_enabled": resume_enabled,
363
+ "skipped_existing_seeds": skipped_existing_seeds,
364
+ "evaluated_seeds": evaluated_seeds,
365
+ "episodes_per_seed": self.num_episodes,
366
+ "total_episodes": total_episodes,
367
+ "total_succeeded": total_succeeded,
368
+ "micro_success_rate": total_succeeded / max(total_episodes, 1),
369
+ "seed_mean_success_rate": statistics.fmean(rates),
370
+ "seed_std_success_rate": (
371
+ statistics.pstdev(rates) if len(rates) > 1 else 0.0
372
+ ),
373
+ "turns": {
374
+ "total": total_turns,
375
+ "micro_mean": total_turns / max(total_episodes, 1),
376
+ "seed_mean": statistics.fmean(seed_turn_means),
377
+ "seed_std": (
378
+ statistics.pstdev(seed_turn_means)
379
+ if len(seed_turn_means) > 1
380
+ else 0.0
381
+ ),
382
+ "max": max_turns,
383
+ },
384
+ "task_types": task_summary,
385
+ "history_length": self.args.rollout.alfworld_history_length,
386
+ "max_steps": self.args.rollout.alfworld_max_turns,
387
+ "sampling": {
388
+ "max_new_tokens": self.args.rollout.generate_max_len,
389
+ "temperature": self.args.rollout.temperature,
390
+ "top_p": self.args.rollout.top_p,
391
+ },
392
+ "seed_summaries": [
393
+ os.path.join(f"seed_{seed}", "summary.json")
394
+ for seed in self.seeds
395
+ ],
396
+ "elapsed_seconds": time.time() - started_at,
397
+ "completed_at": datetime.now(timezone.utc).isoformat(),
398
+ }
399
+ summary_path = os.path.join(self.output_dir, "summary.json")
400
+ with open(summary_path, "w", encoding="utf-8") as output_file:
401
+ json.dump(summary, output_file, ensure_ascii=False, indent=2)
402
+ output_file.write("\n")
403
+ logger.info(
404
+ "verl-agent ALFWorld evaluation complete: mean=%.4f std=%.4f "
405
+ "turns=%.2f",
406
+ summary["seed_mean_success_rate"],
407
+ summary["seed_std_success_rate"],
408
+ summary["turns"]["micro_mean"],
409
+ )
410
+ return summary
kdflow/evaluation/verl_agent_alfworld_protocol.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Nanyang Technological University (NTU), Singapore
2
+ # and the verl-agent (GiGPO) team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Adapted for KDFlow from langfengq/verl-agent at commit
11
+ # 20bd331bdbc9026a5668e11362178e10ab7400c8.
12
+
13
+ """Dependency-free prompts and projection for verl-agent ALFWorld."""
14
+
15
+ import re
16
+ from typing import Tuple
17
+
18
+
19
+ ALFWORLD_TEMPLATE_NO_HISTORY = """
20
+ You are an expert agent operating in the ALFRED Embodied Environment.
21
+ Your current observation is: {current_observation}
22
+ Your admissible actions of the current situation are: [{admissible_actions}].
23
+
24
+ Now it's your turn to take an action.
25
+ You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within <think> </think> tags.
26
+ Once you've finished your reasoning, you should choose an admissible action for current step and present it within <action> </action> tags.
27
+ """
28
+
29
+ ALFWORLD_TEMPLATE_WITH_HISTORY = """
30
+ You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description}
31
+ Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history}
32
+ You are now at step {current_step} and your current observation is: {current_observation}
33
+ Your admissible actions of the current situation are: [{admissible_actions}].
34
+
35
+ Now it's your turn to take an action.
36
+ You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within <think> </think> tags.
37
+ Once you've finished your reasoning, you should choose an admissible action for current step and present it within <action> </action> tags.
38
+ """
39
+
40
+ TASK_TYPES = (
41
+ "pick_and_place",
42
+ "pick_two_obj_and_place",
43
+ "look_at_obj_in_light",
44
+ "pick_heat_then_place_in_recep",
45
+ "pick_cool_then_place_in_recep",
46
+ "pick_clean_then_place_in_recep",
47
+ )
48
+
49
+
50
+ def project_alfworld_action(response: str) -> Tuple[str, bool]:
51
+ """Reproduce verl-agent's action projection and format-validity flag."""
52
+ original = response or ""
53
+ lowered = original.lower()
54
+ start_tag = "<action>"
55
+ end_tag = "</action>"
56
+ start_index = lowered.find(start_tag)
57
+ end_index = lowered.find(end_tag)
58
+
59
+ if start_index == -1 or end_index == -1:
60
+ return lowered[-30:], False
61
+
62
+ action = lowered[start_index + len(start_tag) : end_index].strip().lower()
63
+ valid = True
64
+ if original.find("<think>") == -1 or original.find("</think>") == -1:
65
+ valid = False
66
+ if re.search(r"[\u4e00-\u9fff]", original):
67
+ valid = False
68
+ return action, valid
69
+
70
+
71
+ def task_type_from_gamefile(gamefile: str) -> str:
72
+ for task_type in TASK_TYPES:
73
+ if task_type in (gamefile or ""):
74
+ return task_type
75
+ return "unknown"
kdflow/loss/__init__.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib
2
+ import os
3
+ import glob
4
+ from functools import partial
5
+
6
+ LOSS_DICT = {}
7
+
8
+ def register_loss(name):
9
+ def decorator(fn):
10
+ LOSS_DICT[name] = fn
11
+ return fn
12
+ return decorator
13
+
14
+ _current_dir = os.path.dirname(os.path.abspath(__file__))
15
+ _module_files = glob.glob(os.path.join(_current_dir, "*.py"))
16
+
17
+ for f in _module_files:
18
+ if f.endswith("__init__.py"):
19
+ continue
20
+ module_name = os.path.basename(f)[:-3]
21
+ importlib.import_module(f"kdflow.loss.{module_name}")
22
+
23
+
24
+ def build_loss_fn(name, args):
25
+ """Build a loss function with args-specific hyperparams pre-bound via functools.partial."""
26
+ fn = LOSS_DICT[name]
27
+ kd = args.kd
28
+
29
+ common = {"temperature": kd.kd_temperature}
30
+
31
+ extra_params = {
32
+ "jsd": {"jsd_beta": kd.jsd_beta},
33
+ "skl": {"skew_lambda": kd.skew_lambda},
34
+ "srkl": {"skew_lambda": kd.skew_lambda},
35
+ "akl": {"adaptive_kl_alpha": kd.adaptive_alpha},
36
+ "hrl": {"hrl_topk": kd.hrl_topk},
37
+ }
38
+
39
+ return partial(fn, **common, **extra_params.get(name, {}))
kdflow/loss/adaptive_kl_div.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+
4
+ from kdflow.loss import register_loss
5
+
6
+
7
+ @register_loss("akl")
8
+ @torch.compile()
9
+ def compute_adaptive_kl_div(
10
+ student_logits,
11
+ teacher_logits,
12
+ temperature=1.0,
13
+ adaptive_kl_alpha=0.5,
14
+ reduction="none",
15
+ **kwargs
16
+ ):
17
+ """Adaptive KL Divergence loss (https://arxiv.org/abs/2404.02657)"""
18
+ student_logits = student_logits / temperature
19
+ teacher_logits = teacher_logits / temperature
20
+ student_probs = torch.softmax(student_logits, dim=-1, dtype=torch.float32)
21
+ teacher_probs = torch.softmax(teacher_logits, dim=-1, dtype=torch.float32)
22
+ sorted_teacher_probs, sorted_idx = teacher_probs.sort(-1)
23
+ sorted_probs = student_probs.gather(-1, sorted_idx)
24
+ gap = (sorted_teacher_probs - sorted_probs).abs()
25
+ cum_teacher_probs = torch.cumsum(sorted_teacher_probs, -1)
26
+ tail_mask = cum_teacher_probs.le(adaptive_kl_alpha).float()
27
+ g_head = (gap * (1 - tail_mask)).sum(-1).detach()
28
+ g_tail = (gap * tail_mask).sum(-1).detach()
29
+
30
+ student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
31
+ teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
32
+ fkl = (teacher_probs * (teacher_lprobs - student_lprobs)).sum(-1)
33
+ rkl = (student_probs * (student_lprobs - teacher_lprobs)).sum(-1)
34
+
35
+ akl = (g_head / (g_head + g_tail)) * fkl + (g_tail / (g_head + g_tail)) * rkl
36
+
37
+ if reduction == "mean":
38
+ akl = akl.mean()
39
+ elif reduction == "sum":
40
+ akl = akl.sum()
41
+
42
+ return akl
kdflow/loss/chunked_loss.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import defaultdict
2
+ from typing import Any, Callable, List, Optional, Union
3
+ import torch
4
+
5
+
6
+ def chunked_loss(
7
+ student_hidden: torch.Tensor,
8
+ student_head: torch.nn.Module,
9
+ loss_fn: Callable[..., torch.Tensor],
10
+ student_logits_fn: Optional[Callable[..., torch.Tensor]] = None,
11
+ teacher_hidden: Optional[torch.Tensor] = None,
12
+ teacher_head: Optional[torch.nn.Module] = None,
13
+ teacher_logits_fn: Optional[Callable[[int, int], torch.Tensor]] = None,
14
+ label: Optional[torch.Tensor] = None,
15
+ chunk_size: int = 2048,
16
+ reduction: str = "none",
17
+ metric_fns: Optional[Union[Callable, List[Callable]]] = None,
18
+ return_metrics: bool = False,
19
+ **kwargs: Any,
20
+ ):
21
+ """Compute loss chunk by chunk without materializing full logits.
22
+
23
+ Inputs are expected to be token-level tensors, e.g. [num_tokens, hidden_size].
24
+ Loss is computed per chunk with reduction="none" and reduced globally here.
25
+ """
26
+ if reduction not in ("none", "sum", "mean"):
27
+ raise ValueError(f"Unsupported reduction: {reduction}")
28
+ if teacher_logits_fn is None and teacher_hidden is None and label is None:
29
+ raise ValueError("Either teacher_logits_fn, teacher_hidden or label must be provided.")
30
+ if teacher_logits_fn is None and teacher_hidden is not None and teacher_head is None:
31
+ raise ValueError("teacher_head must be provided when teacher_hidden is provided.")
32
+
33
+ losses = []
34
+ metric_sums = defaultdict(lambda: student_hidden.new_zeros(()))
35
+ total_loss = student_hidden.new_zeros(())
36
+ total_tokens = 0
37
+
38
+ for start in range(0, student_hidden.shape[0], chunk_size):
39
+ end = start + chunk_size
40
+ if student_logits_fn is not None:
41
+ student_logits = student_logits_fn(student_hidden[start:end], skip=False)
42
+ else:
43
+ student_logits = student_head(student_hidden[start:end], skip=False)
44
+ has_teacher_logits = False
45
+
46
+ if teacher_logits_fn is not None:
47
+ teacher_logits = teacher_logits_fn(start, end)
48
+ vocab_size = min(student_logits.shape[-1], teacher_logits.shape[-1])
49
+ student_logits = student_logits[..., :vocab_size]
50
+ target = teacher_logits[..., :vocab_size]
51
+ has_teacher_logits = True
52
+ elif teacher_hidden is not None:
53
+ teacher_logits = teacher_head(teacher_hidden[start:end])
54
+ vocab_size = min(student_logits.shape[-1], teacher_logits.shape[-1])
55
+ student_logits = student_logits[..., :vocab_size]
56
+ target = teacher_logits[..., :vocab_size]
57
+ has_teacher_logits = True
58
+ else:
59
+ target = label[start:end]
60
+
61
+ chunk_loss = loss_fn(student_logits, target, reduction="none", **kwargs)
62
+ chunk_tokens = chunk_loss.numel()
63
+ if metric_fns is not None and has_teacher_logits:
64
+ fns = metric_fns if isinstance(metric_fns, list) else [metric_fns]
65
+ for fn in fns:
66
+ for key, value in fn(student_logits=student_logits, teacher_logits=target).items():
67
+ metric_sums[key] += value * chunk_tokens
68
+ if reduction == "none":
69
+ losses.append(chunk_loss)
70
+ else:
71
+ total_loss = total_loss + chunk_loss.sum()
72
+ total_tokens += chunk_tokens
73
+
74
+ if reduction == "none":
75
+ loss = torch.cat(losses, dim=0)
76
+ else:
77
+ loss = total_loss if reduction == "sum" else total_loss / total_tokens
78
+ if not return_metrics:
79
+ return loss
80
+ if reduction == "mean":
81
+ metric_sums = {key: value / total_tokens for key, value in metric_sums.items()}
82
+ return loss, metric_sums
kdflow/loss/cross_entropy.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+
4
+ @torch.compile()
5
+ def compute_cross_entropy(
6
+ logits: torch.Tensor,
7
+ labels: torch.Tensor,
8
+ reduction: str = "none",
9
+ **kwargs
10
+ ) -> torch.Tensor:
11
+ logsumexp = torch.logsumexp(logits, dim=-1)
12
+ log_probs = logits - logsumexp.unsqueeze(-1)
13
+ ce_loss = -log_probs.gather(-1, labels.unsqueeze(-1)).squeeze(-1)
14
+ if reduction == "mean":
15
+ return ce_loss.mean()
16
+ elif reduction == "sum":
17
+ return ce_loss.sum()
18
+ return ce_loss
kdflow/loss/hierarchical_ranking_loss.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+
4
+ from kdflow.loss import register_loss
5
+
6
+
7
+ @register_loss("hrl")
8
+ @torch.compile()
9
+ def compute_hierarchical_ranking_loss(
10
+ student_logits,
11
+ teacher_logits,
12
+ temperature=1.0,
13
+ hr_topk=5,
14
+ reduction="none",
15
+ **kwargs
16
+ ):
17
+ student_probs = torch.softmax(student_logits, -1, dtype=torch.float32)
18
+ teacher_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32)
19
+
20
+ stu_topk_probs, stu_topk_idxs = torch.topk(student_probs, k=hr_topk, dim=-1)
21
+ tea_topk_probs, tea_topk_idxs = torch.topk(teacher_probs, k=hr_topk, dim=-1)
22
+
23
+ stu_co_topk_probs = student_probs.gather(-1, tea_topk_idxs)
24
+ tea_co_topk_probs = teacher_probs.gather(-1, stu_topk_idxs)
25
+
26
+ stu_topk_margin = stu_co_topk_probs[..., :1] - stu_co_topk_probs
27
+ stu_other_margin = stu_co_topk_probs.unsqueeze(-1) - stu_topk_probs.unsqueeze(-2)
28
+ tea_other_margin = stu_topk_probs.unsqueeze(-1) - tea_co_topk_probs.unsqueeze(-2)
29
+ topk_rank_loss = torch.maximum(torch.zeros_like(stu_topk_margin), -stu_topk_margin)
30
+ topk_rank_loss = topk_rank_loss.sum(-1)
31
+ other_rank_loss = torch.maximum(torch.zeros_like(tea_other_margin), -stu_other_margin)
32
+ other_rank_loss = (other_rank_loss * tea_other_margin.gt(0)).sum(-1).sum(-1)
33
+ rank_loss = topk_rank_loss + other_rank_loss
34
+
35
+ if reduction == "mean":
36
+ return rank_loss.mean()
37
+ elif reduction == "sum":
38
+ return rank_loss.sum()
39
+
40
+ return rank_loss
kdflow/loss/js_div.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kdflow.loss import register_loss
4
+
5
+
6
+ @register_loss("jsd")
7
+ @torch.compile()
8
+ def compute_js_div(
9
+ student_logits,
10
+ teacher_logits,
11
+ temperature=1.0,
12
+ jsd_beta=0.5,
13
+ reduction="none",
14
+ **kwargs
15
+ ):
16
+ """Jensen-Shannon Divergence (JSD) loss in GKD (https://arxiv.org/pdf/2306.13649)."""
17
+ assert jsd_beta > 0 and jsd_beta < 1, "jsd_beta must be in (0, 1)"
18
+ student_logits = student_logits / temperature
19
+ teacher_logits = teacher_logits / temperature
20
+ student_lprobs = student_logits.log_softmax(-1, dtype=torch.float32)
21
+ teacher_lprobs = teacher_logits.log_softmax(-1, dtype=torch.float32)
22
+ log_M = torch.logaddexp(
23
+ student_lprobs + torch.tensor(jsd_beta, dtype=torch.float32).log(),
24
+ teacher_lprobs + torch.tensor(1 - jsd_beta, dtype=torch.float32).log(),
25
+ )
26
+ div1 = jsd_beta * (student_lprobs.exp() * (student_lprobs - log_M)).sum(-1)
27
+ div2 = (1 - jsd_beta) * (teacher_lprobs.exp() * (teacher_lprobs - log_M)).sum(-1)
28
+ js_div = div1 + div2
29
+
30
+ if reduction == "mean":
31
+ js_div = js_div.mean()
32
+ elif reduction == "sum":
33
+ js_div = js_div.sum()
34
+
35
+ return js_div
kdflow/loss/kl_div.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+
4
+ from kdflow.loss import register_loss
5
+
6
+
7
+ @register_loss("kl")
8
+ @torch.compile()
9
+ def compute_kl_div(
10
+ student_logits,
11
+ teacher_logits,
12
+ temperature=1.0,
13
+ reduction="none",
14
+ **kwargs
15
+ ):
16
+ student_logits = student_logits / temperature
17
+ teacher_logits = teacher_logits / temperature
18
+ log_probs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
19
+ target_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32)
20
+ kl_div = F.kl_div(log_probs, target_probs, reduction=reduction).sum(-1)
21
+
22
+ return kl_div
kdflow/loss/reverse_kl_div.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kdflow.loss import register_loss
4
+
5
+
6
+ @register_loss("rkl")
7
+ @torch.compile()
8
+ def compute_reverse_kl_div(
9
+ student_logits,
10
+ teacher_logits,
11
+ temperature=1.0,
12
+ reduction="none",
13
+ **kwargs
14
+ ):
15
+ student_logits = student_logits / temperature
16
+ teacher_logits = teacher_logits / temperature
17
+ student_log_probs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
18
+ teacher_log_probs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
19
+ student_probs = student_log_probs.exp()
20
+ rkl_div = (student_probs * (student_log_probs - teacher_log_probs)).sum(-1)
21
+
22
+ if reduction == "mean":
23
+ rkl_div = rkl_div.mean()
24
+ elif reduction == "sum":
25
+ rkl_div = rkl_div.sum()
26
+
27
+ return rkl_div
kdflow/loss/skewed_kl_div.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+
4
+ from kdflow.loss import register_loss
5
+
6
+
7
+ @register_loss("skl")
8
+ @torch.compile()
9
+ def compute_skewed_kl_div(
10
+ student_logits,
11
+ teacher_logits,
12
+ temperature=1.0,
13
+ skew_lambda=0.1,
14
+ reduction="none",
15
+ **kwargs
16
+ ):
17
+ """Skewed KL Divergence loss (https://arxiv.org/abs/2402.03898)."""
18
+ assert skew_lambda > 0 and skew_lambda < 1, "skew_lambda must be in (0, 1)"
19
+ student_logits = student_logits / temperature
20
+ teacher_logits = teacher_logits / temperature
21
+ student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
22
+ teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
23
+ mixed_lprobs = torch.logaddexp(
24
+ teacher_lprobs + torch.tensor(skew_lambda, dtype=torch.float32).log(),
25
+ student_lprobs + torch.tensor(1 - skew_lambda, dtype=torch.float32).log(),
26
+ )
27
+ kl_div = (torch.exp(teacher_lprobs) * (teacher_lprobs - mixed_lprobs)).sum(-1)
28
+
29
+ if reduction == "mean":
30
+ kl_div = kl_div.mean()
31
+ elif reduction == "sum":
32
+ kl_div = kl_div.sum()
33
+
34
+ return kl_div
kdflow/loss/skewed_rkl_div.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+
4
+ from kdflow.loss import register_loss
5
+
6
+
7
+ @register_loss("srkl")
8
+ @torch.compile()
9
+ def compute_skewed_rkl_div(
10
+ student_logits,
11
+ teacher_logits,
12
+ temperature=1.0,
13
+ skew_lambda=0.1,
14
+ reduction="none",
15
+ **kwargs
16
+ ):
17
+ """Skewed Reverse KL Divergence loss (https://arxiv.org/abs/2402.03898)."""
18
+ assert skew_lambda > 0 and skew_lambda < 1, "skew_lambda must be in (0, 1)"
19
+ student_logits = student_logits / temperature
20
+ teacher_logits = teacher_logits / temperature
21
+ student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
22
+ teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
23
+ mixed_lprobs = torch.logaddexp(
24
+ student_lprobs + torch.tensor(skew_lambda, dtype=torch.float32).log(),
25
+ teacher_lprobs + torch.tensor(1 - skew_lambda, dtype=torch.float32).log(),
26
+ )
27
+ kl_div = (torch.exp(student_lprobs) * (student_lprobs - mixed_lprobs)).sum(-1)
28
+
29
+ if reduction == "mean":
30
+ kl_div = kl_div.mean()
31
+ elif reduction == "sum":
32
+ kl_div = kl_div.sum()
33
+
34
+ return kl_div
kdflow/loss/top1_ce.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kdflow.loss import register_loss
4
+
5
+
6
+ @register_loss("top1_ce")
7
+ @torch.compile()
8
+ def compute_top1_ce(
9
+ student_logits,
10
+ teacher_logits,
11
+ temperature=1.0,
12
+ reduction="none",
13
+ **kwargs
14
+ ):
15
+ student_logits = student_logits / temperature
16
+ teacher_logits = teacher_logits / temperature
17
+ teacher_top1_probs, indices = torch.softmax(
18
+ teacher_logits, -1, dtype=torch.float32
19
+ ).max(-1)
20
+ student_lprobs = torch.log_softmax(
21
+ student_logits, -1, dtype=torch.float32
22
+ ).gather(-1, indices.unsqueeze(-1)).squeeze(-1)
23
+ ce_loss = -teacher_top1_probs * student_lprobs
24
+
25
+ if reduction == "mean":
26
+ return ce_loss.mean()
27
+ elif reduction == "sum":
28
+ return ce_loss.sum()
29
+
30
+ return ce_loss
kdflow/loss/tvd.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kdflow.loss import register_loss
4
+
5
+
6
+ @register_loss("tvd")
7
+ @torch.compile()
8
+ def compute_tvd(
9
+ student_logits,
10
+ teacher_logits,
11
+ temperature=1.0,
12
+ reduction="none",
13
+ **kwargs
14
+ ):
15
+ """Total Variation Distance (TVD) loss."""
16
+ student_logits = student_logits / temperature
17
+ teacher_logits = teacher_logits / temperature
18
+ student_probs = torch.softmax(student_logits, -1, dtype=torch.float32)
19
+ teacher_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32)
20
+ tvd = 0.5 * torch.abs(student_probs - teacher_probs).sum(-1)
21
+
22
+ if reduction == "mean":
23
+ tvd = tvd.mean()
24
+ elif reduction == "sum":
25
+ tvd = tvd.sum()
26
+
27
+ return tvd
kdflow/metrics/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ from kdflow.metrics.entropy import compute_entropy
2
+
3
+ __all__ = ["compute_entropy"]
kdflow/metrics/entropy.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+
4
+
5
+ def compute_entropy(student_logits, **kwargs):
6
+ """Compute the entropy of a logits distribution using numerically stable formula.
7
+
8
+ Args:
9
+ student_logits: Tensor of shape (num_tokens, vocab_size).
10
+ **kwargs: Unused. Accepts extra arguments for unified metric interface.
11
+
12
+ Returns:
13
+ A dict containing the mean entropy value.
14
+ """
15
+ with torch.no_grad():
16
+ probs = F.softmax(student_logits, dim=-1)
17
+ entropy = torch.logsumexp(student_logits, dim=-1) - torch.sum(probs * student_logits, dim=-1)
18
+ return {"distill/student_entropy": entropy.mean()}