Commit ·
0ca529a
unverified ·
0
Parent(s):
chore(upstream): import Cross-Tokenizer-OPD source snapshot
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +11 -0
- README.md +49 -0
- docs/upstream-source-manifest.json +601 -0
- kdflow/__init__.py +1 -0
- kdflow/algorithms/__init__.py +17 -0
- kdflow/algorithms/byte_align_ctkd.py +121 -0
- kdflow/algorithms/simple_ctkd.py +198 -0
- kdflow/algorithms/topk_byte_align_ctkd.py +101 -0
- kdflow/arguments/__init__.py +309 -0
- kdflow/arguments/data_args.py +107 -0
- kdflow/arguments/distillation_args.py +142 -0
- kdflow/arguments/fsdp_args.py +18 -0
- kdflow/arguments/logging_args.py +37 -0
- kdflow/arguments/model_args.py +45 -0
- kdflow/arguments/rollout_args.py +132 -0
- kdflow/arguments/training_args.py +182 -0
- kdflow/backend/__init__.py +18 -0
- kdflow/backend/fsdp/__init__.py +5 -0
- kdflow/backend/fsdp/fsdp_strategy.py +704 -0
- kdflow/backend/sglang/__init__.py +0 -0
- kdflow/backend/sglang/monkey_patch.py +282 -0
- kdflow/backend/sglang/sglang_engine.py +437 -0
- kdflow/cli/__init__.py +0 -0
- kdflow/cli/evaluate_alfworld.py +118 -0
- kdflow/cli/train_kd_off_policy.py +173 -0
- kdflow/cli/train_kd_on_policy.py +230 -0
- kdflow/datasets/__init__.py +9 -0
- kdflow/datasets/alfworld_dataset.py +26 -0
- kdflow/datasets/prompts_dataset.py +246 -0
- kdflow/datasets/sft_dataset.py +380 -0
- kdflow/datasets/utils.py +387 -0
- kdflow/evaluation/__init__.py +13 -0
- kdflow/evaluation/alfworld_evaluator.py +164 -0
- kdflow/evaluation/math_evaluator.py +322 -0
- kdflow/evaluation/verl_agent_alfworld_evaluator.py +410 -0
- kdflow/evaluation/verl_agent_alfworld_protocol.py +75 -0
- kdflow/loss/__init__.py +39 -0
- kdflow/loss/adaptive_kl_div.py +42 -0
- kdflow/loss/chunked_loss.py +82 -0
- kdflow/loss/cross_entropy.py +18 -0
- kdflow/loss/hierarchical_ranking_loss.py +40 -0
- kdflow/loss/js_div.py +35 -0
- kdflow/loss/kl_div.py +22 -0
- kdflow/loss/reverse_kl_div.py +27 -0
- kdflow/loss/skewed_kl_div.py +34 -0
- kdflow/loss/skewed_rkl_div.py +34 -0
- kdflow/loss/top1_ce.py +30 -0
- kdflow/loss/tvd.py +27 -0
- kdflow/metrics/__init__.py +3 -0
- kdflow/metrics/entropy.py +18 -0
.gitignore
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.DS_Store
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
.venv/
|
| 5 |
+
.env
|
| 6 |
+
.env.*
|
| 7 |
+
outputs/
|
| 8 |
+
wandb/
|
| 9 |
+
*.egg-info/
|
| 10 |
+
build/
|
| 11 |
+
dist/
|
README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Cross-Tokenizer-OPD
|
| 2 |
+
|
| 3 |
+
This repository provides the implementation and example training scripts for `byte_align_ctkd` and `topk_byte_align_ctkd`.
|
| 4 |
+
|
| 5 |
+

|
| 6 |
+
|
| 7 |
+
## Environment
|
| 8 |
+
|
| 9 |
+
Our experimental environment was created on top of the [KDFlow Docker image](https://hub.docker.com/r/songmzhang/kdflow?tag=sgl0.5.12_torch2.11_cu129), using the following tag:
|
| 10 |
+
|
| 11 |
+
```bash
|
| 12 |
+
docker pull songmzhang/kdflow:sgl0.5.12_torch2.11_cu129
|
| 13 |
+
```
|
| 14 |
+
|
| 15 |
+
[`requirements.txt`](requirements.txt) records the package versions used in our experiments. Use the Docker image as the starting point and match any additional dependencies to this file. It includes CUDA-specific builds and system-provided packages, so it is an environment snapshot rather than a standalone installation specification for arbitrary hosts.
|
| 16 |
+
|
| 17 |
+
Inside the prepared container, run from the repository root:
|
| 18 |
+
|
| 19 |
+
```bash
|
| 20 |
+
python -m pip install -e . --no-deps
|
| 21 |
+
```
|
| 22 |
+
|
| 23 |
+
## Example training
|
| 24 |
+
|
| 25 |
+
Both scripts perform on-policy distillation from **Qwen2.5-7B-Instruct** to **Llama-3.2-3B-Instruct** and are configured for **one node with 8 GPUs**.
|
| 26 |
+
|
| 27 |
+
Before running, update the paths in each script or mount the files at the expected locations:
|
| 28 |
+
|
| 29 |
+
- Repository: `/workspace/Cross-Tokenizer-OPD`
|
| 30 |
+
- Student checkpoint: `/workspace/models/Llama-3.2-3B-Instruct`
|
| 31 |
+
- Teacher checkpoint: `/workspace/models/Qwen2.5-7B-Instruct`
|
| 32 |
+
- Training data: `/workspace/data/Train/math_code_20k/train.parquet`
|
| 33 |
+
- Evaluation data: `/workspace/data/math500/test.jsonl`
|
| 34 |
+
|
| 35 |
+
The data files are included under `data/`; model checkpoints must be provided separately. WandB logging is disabled by default (`--use_wandb False`). To enable it, set `--use_wandb True`, add `--wandb_project <project_name>`, and configure your WandB account.
|
| 36 |
+
|
| 37 |
+
Run the byte-alignment example from the repository root:
|
| 38 |
+
|
| 39 |
+
```bash
|
| 40 |
+
bash run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
Run the top-k example (`--ctkd_topk 16`):
|
| 44 |
+
|
| 45 |
+
```bash
|
| 46 |
+
bash run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
Training settings can be adjusted directly in the scripts. Checkpoints, evaluation results, and training logs are written under `outputs/<TRAIN_NAME>/`.
|
docs/upstream-source-manifest.json
ADDED
|
@@ -0,0 +1,601 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"url": "https://anonymous.4open.science/api/repo/Cross-Tokenizer-OPD",
|
| 3 |
+
"paper": "https://arxiv.org/pdf/2610.08448v1",
|
| 4 |
+
"retrieved_utc": "2026-10-17T19:28:35.738288+00:00",
|
| 5 |
+
"identity_kind": "audited content snapshot assembled from public files; upstream commit unavailable",
|
| 6 |
+
"excluded": [
|
| 7 |
+
"data/",
|
| 8 |
+
"assets/"
|
| 9 |
+
],
|
| 10 |
+
"source_manifest_sha256": "93cef1dfa18bcd5fc66cf80347557382efcd7e8955c3f6b464435711e82bf49b",
|
| 11 |
+
"files": [
|
| 12 |
+
{
|
| 13 |
+
"path": ".gitignore",
|
| 14 |
+
"bytes": 89,
|
| 15 |
+
"api_version_tag": "2e7be852",
|
| 16 |
+
"git_blob": "2727e1bf64e5fd3901d2803b1def32466a2e90bf",
|
| 17 |
+
"sha256": "19587f4a412fa284a0dac11bad5d487b92117390d1eb117aee094f6fb289ba0d"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"path": "README.md",
|
| 21 |
+
"bytes": 2289,
|
| 22 |
+
"api_version_tag": "b260e71d",
|
| 23 |
+
"git_blob": "9ed1faf98032a99ac4f8dcaf3000e5e2d800fda1",
|
| 24 |
+
"sha256": "f45c1d25386d1d1a25f3c4ea4c152db95be6467f34c86a94ad52cda4df4ee656"
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"path": "kdflow/__init__.py",
|
| 28 |
+
"bytes": 22,
|
| 29 |
+
"api_version_tag": "f8cbccf6",
|
| 30 |
+
"git_blob": "3dc1f76bc69e3f559bee6253b24fc93acee9e1f9",
|
| 31 |
+
"sha256": "91447944015cec709e8aa7655f7e9d64e1e4508e7023a57fe3746911c0fc6fed"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"path": "kdflow/algorithms/__init__.py",
|
| 35 |
+
"bytes": 520,
|
| 36 |
+
"api_version_tag": "a7cd7946",
|
| 37 |
+
"git_blob": "13dd1052857854aba08c5f7667ed6a49ff1b0b80",
|
| 38 |
+
"sha256": "8afbb4fadfc7db812d43ab35401971fc30fdc4810cafa81505cbea27178badbe"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"path": "kdflow/algorithms/byte_align_ctkd.py",
|
| 42 |
+
"bytes": 4552,
|
| 43 |
+
"api_version_tag": "990d76e2",
|
| 44 |
+
"git_blob": "c94a1239b57b05513ad853735291949aa314c6d9",
|
| 45 |
+
"sha256": "5c6647c6fb8064ffd5f7b1ed971168cd1f3392caee62e2d75916d41931828e18"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"path": "kdflow/algorithms/simple_ctkd.py",
|
| 49 |
+
"bytes": 8575,
|
| 50 |
+
"api_version_tag": "166783cd",
|
| 51 |
+
"git_blob": "57332b882e359ce1283a87f81949a5136f8c06a2",
|
| 52 |
+
"sha256": "fdbd99c55aeac4d061527158bdfc82a93333a42bf347eceaa134d34f5ef05952"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"path": "kdflow/algorithms/topk_byte_align_ctkd.py",
|
| 56 |
+
"bytes": 3448,
|
| 57 |
+
"api_version_tag": "4eff709a",
|
| 58 |
+
"git_blob": "64b49354bea827893fd49f8de51325666f7e1dba",
|
| 59 |
+
"sha256": "298925da4ce973b0f2917e5ba5ad2adbc4862d7c1a73515c5c86514ac0d22ddb"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"path": "kdflow/arguments/__init__.py",
|
| 63 |
+
"bytes": 13313,
|
| 64 |
+
"api_version_tag": "22b52ef5",
|
| 65 |
+
"git_blob": "b3c66702102a8691bcd16bc57f8a8307ccb13b68",
|
| 66 |
+
"sha256": "cafb17a195874f1b53b0b5f29b07840e1f2f6505e08dd77837fca25fd1a8122f"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"path": "kdflow/arguments/data_args.py",
|
| 70 |
+
"bytes": 3363,
|
| 71 |
+
"api_version_tag": "e8dc076d",
|
| 72 |
+
"git_blob": "12046f174777ed5d8f271b15661debf0b3c63003",
|
| 73 |
+
"sha256": "e5f68889ab6635ed51a285aad70dcff5de3dce7d7d06c6cf563b03331591a382"
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"path": "kdflow/arguments/distillation_args.py",
|
| 77 |
+
"bytes": 5102,
|
| 78 |
+
"api_version_tag": "e7ae08cc",
|
| 79 |
+
"git_blob": "ae835897855f7e4147fb8efc1fe5971fd8cfb964",
|
| 80 |
+
"sha256": "65d39f6abeb75896864d385fe0c9a2f026938eb1a5ee9d4dc392d3cb86b440f4"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"path": "kdflow/arguments/fsdp_args.py",
|
| 84 |
+
"bytes": 387,
|
| 85 |
+
"api_version_tag": "172c1790",
|
| 86 |
+
"git_blob": "ed2729e8de9b94f789cd73ea1274e02bc4e18558",
|
| 87 |
+
"sha256": "10918e4e3242a526ab0926af4a2608a3a77dea058fd084ecee9022852aecac47"
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"path": "kdflow/arguments/logging_args.py",
|
| 91 |
+
"bytes": 890,
|
| 92 |
+
"api_version_tag": "b3ce62d7",
|
| 93 |
+
"git_blob": "53e1413ef1c4b57d66b57a22c1c3c7022cf9d70a",
|
| 94 |
+
"sha256": "2d6012a164fdc81b3d9c482357dc68d0fe4485db0a2163f3cdfa3ca6411addc1"
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"path": "kdflow/arguments/model_args.py",
|
| 98 |
+
"bytes": 1157,
|
| 99 |
+
"api_version_tag": "2f851242",
|
| 100 |
+
"git_blob": "9c6531d1deb2435054eeff4c412271149390b336",
|
| 101 |
+
"sha256": "b6da8907b70189acaa7c1b51544135e441182e28194ee3bbe4f23ec1fa23cb83"
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"path": "kdflow/arguments/rollout_args.py",
|
| 105 |
+
"bytes": 4468,
|
| 106 |
+
"api_version_tag": "5899e45d",
|
| 107 |
+
"git_blob": "5b46da1da4da0f4a05a53aa14ef36167517784e4",
|
| 108 |
+
"sha256": "f9df7e87681abd88c00bde4985a45ddfbb3c6f3f7feee41d4fc941a808c670d1"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"path": "kdflow/arguments/training_args.py",
|
| 112 |
+
"bytes": 5880,
|
| 113 |
+
"api_version_tag": "b7d6b4ad",
|
| 114 |
+
"git_blob": "71443753b8ea33fd213a28c0744816c77b616600",
|
| 115 |
+
"sha256": "a45dd87ae3064eac14f9ca9a875963df4efd03082230aba9f4c98d67479260bd"
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"path": "kdflow/backend/__init__.py",
|
| 119 |
+
"bytes": 647,
|
| 120 |
+
"api_version_tag": "33655f94",
|
| 121 |
+
"git_blob": "ae07fdf84af178d7d1b63e06abefb6f07f29e890",
|
| 122 |
+
"sha256": "b36cc114c0ffd20559a6e87808b158d7244566645933639023cf00019e03e137"
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"path": "kdflow/backend/fsdp/__init__.py",
|
| 126 |
+
"bytes": 76,
|
| 127 |
+
"api_version_tag": "2c7e9bf6",
|
| 128 |
+
"git_blob": "a50fa8e305c891e9bfe23089b39c6f0555d473af",
|
| 129 |
+
"sha256": "9b1dffb545428c2862d6cc588507709797ba05f24d4017e9a450a7afa09701d9"
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"path": "kdflow/backend/fsdp/fsdp_strategy.py",
|
| 133 |
+
"bytes": 26957,
|
| 134 |
+
"api_version_tag": "b0b88464",
|
| 135 |
+
"git_blob": "63498e48936f06180363b4df5c7fde821fa36a70",
|
| 136 |
+
"sha256": "8941ccc6077e6fdcc2a38c13769d6a0912dc764ea9f7b82866e7e1bf3bac0fa2"
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"path": "kdflow/backend/sglang/__init__.py",
|
| 140 |
+
"bytes": 0,
|
| 141 |
+
"api_version_tag": null,
|
| 142 |
+
"git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
|
| 143 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"path": "kdflow/backend/sglang/monkey_patch.py",
|
| 147 |
+
"bytes": 10707,
|
| 148 |
+
"api_version_tag": "154f0497",
|
| 149 |
+
"git_blob": "e4aa0a9550903f01a7125b1631408e9b2df6883a",
|
| 150 |
+
"sha256": "ddcbc908bce562bc3b87f13b8445ac02891a491dd60ba2edee37d6396440999c"
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"path": "kdflow/backend/sglang/sglang_engine.py",
|
| 154 |
+
"bytes": 16531,
|
| 155 |
+
"api_version_tag": "b2ffd390",
|
| 156 |
+
"git_blob": "4deb3adfe2fa5b2258c0144ed46b4b4dc1873913",
|
| 157 |
+
"sha256": "dd0b9a5531beba14c52389787a1df5d053412dabd9547cde5cb66f5f23bb494f"
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"path": "kdflow/cli/__init__.py",
|
| 161 |
+
"bytes": 0,
|
| 162 |
+
"api_version_tag": null,
|
| 163 |
+
"git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
|
| 164 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"path": "kdflow/cli/evaluate_alfworld.py",
|
| 168 |
+
"bytes": 4054,
|
| 169 |
+
"api_version_tag": "c6740670",
|
| 170 |
+
"git_blob": "77dec216fbfc0b68f28a094b6fdeac63f1f6a646",
|
| 171 |
+
"sha256": "f5b02bf943606ff7d01feff0c8df9eb5a5902a7c0039d38e3a21c859716bf08e"
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"path": "kdflow/cli/train_kd_off_policy.py",
|
| 175 |
+
"bytes": 5879,
|
| 176 |
+
"api_version_tag": "c4023313",
|
| 177 |
+
"git_blob": "b149c801a94aedc0234c529cbdc63764de08573f",
|
| 178 |
+
"sha256": "b3064a2f38c12bf086c5f783b0dcfb3904ff097e99289aca2a1da310d81713cc"
|
| 179 |
+
},
|
| 180 |
+
{
|
| 181 |
+
"path": "kdflow/cli/train_kd_on_policy.py",
|
| 182 |
+
"bytes": 8208,
|
| 183 |
+
"api_version_tag": "c50095ea",
|
| 184 |
+
"git_blob": "8cfb58d743568948cfa7d1c1e579be7287057241",
|
| 185 |
+
"sha256": "1e2036bf1b788d7f7b27ac3496d50b78c25992ae597733199bfb90619cf6391f"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"path": "kdflow/datasets/__init__.py",
|
| 189 |
+
"bytes": 216,
|
| 190 |
+
"api_version_tag": "fc482f85",
|
| 191 |
+
"git_blob": "37facfbdb6705166c7047e40eb67bb1f3edbf3e9",
|
| 192 |
+
"sha256": "2dd709d632fae96f3e8d86ffb8b608cdca9edaacd17dde7c21eed1c497379d37"
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"path": "kdflow/datasets/alfworld_dataset.py",
|
| 196 |
+
"bytes": 806,
|
| 197 |
+
"api_version_tag": "4f86ac77",
|
| 198 |
+
"git_blob": "cd96f6d2932f6e2cd31c6e4cab188556206abbef",
|
| 199 |
+
"sha256": "3047bf9576937e1cbc3dd7d7070c7c949ac0e96ecb0b1bf8b6142c7c0415b88b"
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"path": "kdflow/datasets/prompts_dataset.py",
|
| 203 |
+
"bytes": 11012,
|
| 204 |
+
"api_version_tag": "9c932cbd",
|
| 205 |
+
"git_blob": "15a6e747c6cc3fd3232affcdff8f1252ac087ee9",
|
| 206 |
+
"sha256": "3e6b9954b893224a5eac5ad79758dc198747b929e1fe27475e6aed1f01a09703"
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
"path": "kdflow/datasets/sft_dataset.py",
|
| 210 |
+
"bytes": 18029,
|
| 211 |
+
"api_version_tag": "23c3e648",
|
| 212 |
+
"git_blob": "6b922b0047e0d1da1e9f95f0c2b9a078f3ef92a8",
|
| 213 |
+
"sha256": "e5643660828193e4a124639d18d1ac14826d57203dcc3ccf31e2c7d83a37c7b7"
|
| 214 |
+
},
|
| 215 |
+
{
|
| 216 |
+
"path": "kdflow/datasets/utils.py",
|
| 217 |
+
"bytes": 14066,
|
| 218 |
+
"api_version_tag": "6fe0c9b3",
|
| 219 |
+
"git_blob": "9933ae47766e3623d148e39fa7817cefd112133d",
|
| 220 |
+
"sha256": "f0173126a8530665226396c7e9343f57cc031e4df1d97929ba6c3b360059bbb3"
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"path": "kdflow/evaluation/__init__.py",
|
| 224 |
+
"bytes": 344,
|
| 225 |
+
"api_version_tag": "a3dca82f",
|
| 226 |
+
"git_blob": "245e0251fb5e37de3cc7c2a234bff53a732e41de",
|
| 227 |
+
"sha256": "f63f24ca9885b63c32d3d1e24b40c16367734cb238f52b4893b20be95321573c"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"path": "kdflow/evaluation/alfworld_evaluator.py",
|
| 231 |
+
"bytes": 6661,
|
| 232 |
+
"api_version_tag": "beaf919e",
|
| 233 |
+
"git_blob": "7e085a7350f8a07cd3b1cefcbe12c2eab7733d45",
|
| 234 |
+
"sha256": "45c96b0908cf0bd895cd312e88ca8bd49a53be3f00cf9fa4363555590d03352b"
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"path": "kdflow/evaluation/math_evaluator.py",
|
| 238 |
+
"bytes": 12096,
|
| 239 |
+
"api_version_tag": "5cc9a5e1",
|
| 240 |
+
"git_blob": "890e7ed2e1fe006d1e65d3a35d398297fb2fb70f",
|
| 241 |
+
"sha256": "f598d470e1f03c0e09b6bbdb4d7fa8571eb7484f14f2be7ea706a0ed14aa8935"
|
| 242 |
+
},
|
| 243 |
+
{
|
| 244 |
+
"path": "kdflow/evaluation/verl_agent_alfworld_evaluator.py",
|
| 245 |
+
"bytes": 16963,
|
| 246 |
+
"api_version_tag": "1391ea50",
|
| 247 |
+
"git_blob": "f9a25e4e331f5a9dd3aad873666bb0861347453a",
|
| 248 |
+
"sha256": "38d71145e80de82fe738599f43dfe8b3cb14177da4784457546e6be18f78fc29"
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"path": "kdflow/evaluation/verl_agent_alfworld_protocol.py",
|
| 252 |
+
"bytes": 3001,
|
| 253 |
+
"api_version_tag": "a8cc4e01",
|
| 254 |
+
"git_blob": "38615c951dd17f8311216b640cc2bd0ae145291a",
|
| 255 |
+
"sha256": "9aa3c5f02703148a0bfce8d2177b53e51aa906df03e2401e5bf0f5b241ce8433"
|
| 256 |
+
},
|
| 257 |
+
{
|
| 258 |
+
"path": "kdflow/loss/__init__.py",
|
| 259 |
+
"bytes": 1089,
|
| 260 |
+
"api_version_tag": "6e9b5e28",
|
| 261 |
+
"git_blob": "2d66850c7da1a6839a2d69b781980469ea85cd2b",
|
| 262 |
+
"sha256": "3bc98a351ad62faa346607502dcd56e864536b81877ca0221fd2b2ca9045f630"
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"path": "kdflow/loss/adaptive_kl_div.py",
|
| 266 |
+
"bytes": 1528,
|
| 267 |
+
"api_version_tag": "7af71dea",
|
| 268 |
+
"git_blob": "e95f03d7900fd29f1ff5d1b64bf1d670a3267410",
|
| 269 |
+
"sha256": "f8a44824867ae3fa7e732f7ae5554d6b13231ff9545e1fd61793a8f546d7d39c"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"path": "kdflow/loss/chunked_loss.py",
|
| 273 |
+
"bytes": 3636,
|
| 274 |
+
"api_version_tag": "8a38a29a",
|
| 275 |
+
"git_blob": "2c5174fd3c60c92ea8e315fb1ce28931eb7e61ae",
|
| 276 |
+
"sha256": "151c81614f97b908465a952f595abe1f4dc5c0d17cea86431a7068c381125cc5"
|
| 277 |
+
},
|
| 278 |
+
{
|
| 279 |
+
"path": "kdflow/loss/cross_entropy.py",
|
| 280 |
+
"bytes": 473,
|
| 281 |
+
"api_version_tag": "9fab6f38",
|
| 282 |
+
"git_blob": "9f5a322ff489503f707515aa8f8e413ef1083b6b",
|
| 283 |
+
"sha256": "e3629bef7c50c737f384870876321c953ceb09b67c275242cdbfcfbcbca68377"
|
| 284 |
+
},
|
| 285 |
+
{
|
| 286 |
+
"path": "kdflow/loss/hierarchical_ranking_loss.py",
|
| 287 |
+
"bytes": 1479,
|
| 288 |
+
"api_version_tag": "19217f5e",
|
| 289 |
+
"git_blob": "8e6ea5c38676ca63011834431955852055a78af5",
|
| 290 |
+
"sha256": "2cced8cccf77872ab817164a931c97c43c7411b8d59cc48d842111e40f021ac3"
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"path": "kdflow/loss/js_div.py",
|
| 294 |
+
"bytes": 1165,
|
| 295 |
+
"api_version_tag": "8c85586a",
|
| 296 |
+
"git_blob": "1d210762db1d64e5ee8d8cd4bff9ee60499950e8",
|
| 297 |
+
"sha256": "be71370e1e1a2c109c2a00a4e6daaece8ab17f182f1a3f03c60a385b98fb3078"
|
| 298 |
+
},
|
| 299 |
+
{
|
| 300 |
+
"path": "kdflow/loss/kl_div.py",
|
| 301 |
+
"bytes": 592,
|
| 302 |
+
"api_version_tag": "7bc61a88",
|
| 303 |
+
"git_blob": "c39542e7a839837741d0fc442ba5bb18fc5a6343",
|
| 304 |
+
"sha256": "b361ddca392ee4b2e83e44b6891f4402f86127db38b6e254d492d29c48950a7c"
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"path": "kdflow/loss/reverse_kl_div.py",
|
| 308 |
+
"bytes": 761,
|
| 309 |
+
"api_version_tag": "5afd030a",
|
| 310 |
+
"git_blob": "ebdaa8f2a467e41383825915e6a757de2426738c",
|
| 311 |
+
"sha256": "abca69fdbe4f7d90a7ca990300189147230cce8f20d589c934d88f0b63c4edf6"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"path": "kdflow/loss/skewed_kl_div.py",
|
| 315 |
+
"bytes": 1119,
|
| 316 |
+
"api_version_tag": "737302c8",
|
| 317 |
+
"git_blob": "4b76d83f474a2744c81b12a717cae97ea97dccca",
|
| 318 |
+
"sha256": "98189a13bf644d3c46d53aa228a264b9ed666b9d28586a343f0c2bdbd6fed091"
|
| 319 |
+
},
|
| 320 |
+
{
|
| 321 |
+
"path": "kdflow/loss/skewed_rkl_div.py",
|
| 322 |
+
"bytes": 1129,
|
| 323 |
+
"api_version_tag": "c37538a3",
|
| 324 |
+
"git_blob": "9652791e8f1ff0159eb9826ed70ac16057b511cf",
|
| 325 |
+
"sha256": "3c78a1b04270f7d8ed40449f28f5ed936af71c0bc7d5a0badf0864454ddf66f0"
|
| 326 |
+
},
|
| 327 |
+
{
|
| 328 |
+
"path": "kdflow/loss/top1_ce.py",
|
| 329 |
+
"bytes": 760,
|
| 330 |
+
"api_version_tag": "3cc13dae",
|
| 331 |
+
"git_blob": "6a07fefbc642c79a4311c3f3d3565ee2b3bec251",
|
| 332 |
+
"sha256": "1f566c4559f518c8186a53de85593b812e65016beab083cd4ded715b513fa760"
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"path": "kdflow/loss/tvd.py",
|
| 336 |
+
"bytes": 702,
|
| 337 |
+
"api_version_tag": "289a1332",
|
| 338 |
+
"git_blob": "a84bee8f7963da36e7e4011a0dd3af9d9e87d462",
|
| 339 |
+
"sha256": "964d872f122ec4618cfee1fd2adf8ecc0fc82a8279cce83cab8b277335f8aa2c"
|
| 340 |
+
},
|
| 341 |
+
{
|
| 342 |
+
"path": "kdflow/metrics/__init__.py",
|
| 343 |
+
"bytes": 82,
|
| 344 |
+
"api_version_tag": "4c09b5e4",
|
| 345 |
+
"git_blob": "e888dd4bdc45ae4e2be5225b6c74ae17480225a8",
|
| 346 |
+
"sha256": "140c40cfb0b374c62847a93f666c9cf0c3ad595d6d36dc7bb1e6c2fb05495362"
|
| 347 |
+
},
|
| 348 |
+
{
|
| 349 |
+
"path": "kdflow/metrics/entropy.py",
|
| 350 |
+
"bytes": 642,
|
| 351 |
+
"api_version_tag": "c37c9989",
|
| 352 |
+
"git_blob": "81439fc696793032bce83717db7ccb17243d8d9a",
|
| 353 |
+
"sha256": "e02d0a9926fdb33b120fc2ec0a95d042876b43504046a301191d4761cee58cef"
|
| 354 |
+
},
|
| 355 |
+
{
|
| 356 |
+
"path": "kdflow/models/__init__.py",
|
| 357 |
+
"bytes": 67,
|
| 358 |
+
"api_version_tag": "39b9dcda",
|
| 359 |
+
"git_blob": "8f60ffe38287f99a5fe053daf17f85b5173d07a7",
|
| 360 |
+
"sha256": "93370fd72c3d337adcb1bb7cbae6ba2cf9eab2ef49b5b466ada816f7b6bbd148"
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"path": "kdflow/models/model.py",
|
| 364 |
+
"bytes": 5150,
|
| 365 |
+
"api_version_tag": "49c34f8e",
|
| 366 |
+
"git_blob": "df455afaca9a0eaaac7d7cc9836acb9f0dc1dbee",
|
| 367 |
+
"sha256": "2530323bba13319cea6ea6f0ea54ed64ae81d35c6a2aaf209688a67622e26996"
|
| 368 |
+
},
|
| 369 |
+
{
|
| 370 |
+
"path": "kdflow/models/ring_attn_utils.py",
|
| 371 |
+
"bytes": 7402,
|
| 372 |
+
"api_version_tag": "0c072a92",
|
| 373 |
+
"git_blob": "fe3b01a70d20358de0dc19e19ce09df306004d26",
|
| 374 |
+
"sha256": "ed93df4cbef484401f2cf508bc81d4a22521e632bec4bae4a931e404639b7eda"
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"path": "kdflow/models/utils.py",
|
| 378 |
+
"bytes": 1755,
|
| 379 |
+
"api_version_tag": "cfd2ac62",
|
| 380 |
+
"git_blob": "5f00ba4aaf5b007c20e4fb7cb08b5e466a2acef5",
|
| 381 |
+
"sha256": "cfef9ca657b32c8ee5688f01f0f1e939ec3fc61fbd8792085d63af66f74e85d5"
|
| 382 |
+
},
|
| 383 |
+
{
|
| 384 |
+
"path": "kdflow/ray/__init__.py",
|
| 385 |
+
"bytes": 0,
|
| 386 |
+
"api_version_tag": null,
|
| 387 |
+
"git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
|
| 388 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"path": "kdflow/ray/placement_group.py",
|
| 392 |
+
"bytes": 2049,
|
| 393 |
+
"api_version_tag": "37beb550",
|
| 394 |
+
"git_blob": "24ba3109273257b3ab9329a19dc72804bbc5edc3",
|
| 395 |
+
"sha256": "08b0a5d82cacea096b1591e3d12d25f30f285145cade451578763f3b16144a27"
|
| 396 |
+
},
|
| 397 |
+
{
|
| 398 |
+
"path": "kdflow/ray/rollout/__init__.py",
|
| 399 |
+
"bytes": 176,
|
| 400 |
+
"api_version_tag": "ae57ecf3",
|
| 401 |
+
"git_blob": "a7e39d7a3785301ef2f0614ad28880e2f7dfc1cc",
|
| 402 |
+
"sha256": "13401bf88cab65c4e388736c19f4ff2df16fc2a8ff9cbc07a1b48c76876092de"
|
| 403 |
+
},
|
| 404 |
+
{
|
| 405 |
+
"path": "kdflow/ray/rollout/rollout_actor.py",
|
| 406 |
+
"bytes": 12933,
|
| 407 |
+
"api_version_tag": "635eb232",
|
| 408 |
+
"git_blob": "63a00bc4e29afbb2041ee99ac2c63fa5b3aa1d4f",
|
| 409 |
+
"sha256": "14af615c89f8c8c6089ed5e98a5473da0ab79df4771621e5becc4340ff3720bc"
|
| 410 |
+
},
|
| 411 |
+
{
|
| 412 |
+
"path": "kdflow/ray/rollout/rollout_group.py",
|
| 413 |
+
"bytes": 14618,
|
| 414 |
+
"api_version_tag": "7a452703",
|
| 415 |
+
"git_blob": "5ad5ec900fe03de4c6e7201791f8be76315a3ce9",
|
| 416 |
+
"sha256": "05b9235ba2da6259a49979514b6ac3910a500be4f8c0575461d3d9997959ed9b"
|
| 417 |
+
},
|
| 418 |
+
{
|
| 419 |
+
"path": "kdflow/ray/train/__init__.py",
|
| 420 |
+
"bytes": 0,
|
| 421 |
+
"api_version_tag": null,
|
| 422 |
+
"git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
|
| 423 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| 424 |
+
},
|
| 425 |
+
{
|
| 426 |
+
"path": "kdflow/ray/train/student_actor.py",
|
| 427 |
+
"bytes": 18702,
|
| 428 |
+
"api_version_tag": "857338dd",
|
| 429 |
+
"git_blob": "0aec3b65c4423b554ae2fdd4fee3b02d64011e78",
|
| 430 |
+
"sha256": "b234fee1e64e1b59f45d917faa1435cf911619fbaf1b581530996df2244b14eb"
|
| 431 |
+
},
|
| 432 |
+
{
|
| 433 |
+
"path": "kdflow/ray/train/student_group.py",
|
| 434 |
+
"bytes": 9563,
|
| 435 |
+
"api_version_tag": "5f558d33",
|
| 436 |
+
"git_blob": "cc9308ae81028870f2a47d70dadaaa443794070d",
|
| 437 |
+
"sha256": "c9b661647d781e3353821fce0f15c1be69f10d81a55ced07523635d52b0f04e4"
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"path": "kdflow/ray/train/teacher_actor.py",
|
| 441 |
+
"bytes": 7299,
|
| 442 |
+
"api_version_tag": "cdf12732",
|
| 443 |
+
"git_blob": "72600b4ec71cc27acc19bc3f8333410b9784f754",
|
| 444 |
+
"sha256": "8e84664647c5c1f4d2a3bf000d3e382bfcbe3735f42bef2f7b0f0af43518974e"
|
| 445 |
+
},
|
| 446 |
+
{
|
| 447 |
+
"path": "kdflow/ray/train/teacher_group.py",
|
| 448 |
+
"bytes": 12590,
|
| 449 |
+
"api_version_tag": "83cd153b",
|
| 450 |
+
"git_blob": "b121a64b7519f2866d21e8e2d2c208c0052f5826",
|
| 451 |
+
"sha256": "b0a632619940e8fb49f4b5059299f0d380fe9d4b1bd9c26c7c82dff32e50f7ba"
|
| 452 |
+
},
|
| 453 |
+
{
|
| 454 |
+
"path": "kdflow/ray/utils.py",
|
| 455 |
+
"bytes": 2590,
|
| 456 |
+
"api_version_tag": "9f91e927",
|
| 457 |
+
"git_blob": "b003f49102a84c9ce15e617abb33fa4842286fcc",
|
| 458 |
+
"sha256": "8dea4c2b6b55445ad6d8868807ca016b61f6f7e12efb321502dc8b6eb2ea0263"
|
| 459 |
+
},
|
| 460 |
+
{
|
| 461 |
+
"path": "kdflow/token_alignment.py",
|
| 462 |
+
"bytes": 13436,
|
| 463 |
+
"api_version_tag": "8766c5ad",
|
| 464 |
+
"git_blob": "5a2b1695191df2aaa7798e8cbb0605858775474b",
|
| 465 |
+
"sha256": "c77cd962cb0430cd97ace5312077e847398a03da3ce1473062fd2c5c9808498a"
|
| 466 |
+
},
|
| 467 |
+
{
|
| 468 |
+
"path": "kdflow/trainer/__init__.py",
|
| 469 |
+
"bytes": 528,
|
| 470 |
+
"api_version_tag": "6dec2a62",
|
| 471 |
+
"git_blob": "c7fa5b1ce3c01a42c60f2f070370cd978a73710b",
|
| 472 |
+
"sha256": "a3caa9361a9f53b881e24ffa05951342278947135cd51c637ba4d1b16a1a8687"
|
| 473 |
+
},
|
| 474 |
+
{
|
| 475 |
+
"path": "kdflow/trainer/alfworld_rollout_manager.py",
|
| 476 |
+
"bytes": 18875,
|
| 477 |
+
"api_version_tag": "a77bc558",
|
| 478 |
+
"git_blob": "9e353bcbfd9cb9424f2110f3cf1c9e4b2ff496ff",
|
| 479 |
+
"sha256": "93f6ca98f2ba6fe9613d60839eb7d22d6b582420489aefbc1f6364ffb8493987"
|
| 480 |
+
},
|
| 481 |
+
{
|
| 482 |
+
"path": "kdflow/trainer/data_processor/__init__.py",
|
| 483 |
+
"bytes": 122,
|
| 484 |
+
"api_version_tag": "d5140e99",
|
| 485 |
+
"git_blob": "d267c43e5cf7c929a03867968ff8c794a5a574bf",
|
| 486 |
+
"sha256": "66a34ffc421ade775f246c8198bfdc9633e83c97b355084fa457e6e15695b775"
|
| 487 |
+
},
|
| 488 |
+
{
|
| 489 |
+
"path": "kdflow/trainer/data_processor/rollout_data_processor.py",
|
| 490 |
+
"bytes": 13871,
|
| 491 |
+
"api_version_tag": "3b4b701b",
|
| 492 |
+
"git_blob": "7f5564dcd3046b5be30cbabc13b06c3dd393c17a",
|
| 493 |
+
"sha256": "fe720ee01fb821e124a612dcb55cb0e827b64e7fb1597b4ecbb4bf14c8ee5792"
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"path": "kdflow/trainer/off_policy_kd_trainer.py",
|
| 497 |
+
"bytes": 14023,
|
| 498 |
+
"api_version_tag": "f150d881",
|
| 499 |
+
"git_blob": "72ab1017c9edfbf90778208b9d01ca123d1d72ee",
|
| 500 |
+
"sha256": "6b4019e9760e0f2898a4cf0b30071d69bf24b6a23a023aca5f1f929b2a0c2acf"
|
| 501 |
+
},
|
| 502 |
+
{
|
| 503 |
+
"path": "kdflow/trainer/on_policy_kd_trainer.py",
|
| 504 |
+
"bytes": 20980,
|
| 505 |
+
"api_version_tag": "f34b6551",
|
| 506 |
+
"git_blob": "aa4ba8e05f8fc49625b69b8870fc29a8aa6aa6c6",
|
| 507 |
+
"sha256": "f7b20000d435f8fc43a07584a3025472f138ccc6e41f62f25c08ff9ff8fa17b1"
|
| 508 |
+
},
|
| 509 |
+
{
|
| 510 |
+
"path": "kdflow/trainer/rollout_manager.py",
|
| 511 |
+
"bytes": 6718,
|
| 512 |
+
"api_version_tag": "f6b74523",
|
| 513 |
+
"git_blob": "0eb5253bceb3a06a66cf120a606a221db3a07f81",
|
| 514 |
+
"sha256": "e930b4db59e8f8e09e04838141854403f5516ac8492ef4ab0bd7f37611db7009"
|
| 515 |
+
},
|
| 516 |
+
{
|
| 517 |
+
"path": "kdflow/trainer/verl_agent_alfworld_rollout_manager.py",
|
| 518 |
+
"bytes": 22589,
|
| 519 |
+
"api_version_tag": "db6881e8",
|
| 520 |
+
"git_blob": "430dd71f2979174e05b61d276b4e29d068adc864",
|
| 521 |
+
"sha256": "ea95c95d7c3d8dbcc4da1b578877684a4dcea3e30bf6b68ea98187432194a040"
|
| 522 |
+
},
|
| 523 |
+
{
|
| 524 |
+
"path": "kdflow/utils/__init__.py",
|
| 525 |
+
"bytes": 69,
|
| 526 |
+
"api_version_tag": "5eaed800",
|
| 527 |
+
"git_blob": "1431283d595d0062ee042ea5567c33aec00ad49b",
|
| 528 |
+
"sha256": "9ed449850c624d587592f92760c79c55324063a7cc16fbfa5aab42585165afb1"
|
| 529 |
+
},
|
| 530 |
+
{
|
| 531 |
+
"path": "kdflow/utils/distributed_sampler.py",
|
| 532 |
+
"bytes": 6482,
|
| 533 |
+
"api_version_tag": "a82f95b2",
|
| 534 |
+
"git_blob": "1f7658202804e2a1fbe1a5d0d6ddaf20cc50f3af",
|
| 535 |
+
"sha256": "5a099195c63e475f42fd17e43b4c40ed1133ac3237f623410befffcc9005cda7"
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"path": "kdflow/utils/distributed_util.py",
|
| 539 |
+
"bytes": 1077,
|
| 540 |
+
"api_version_tag": "85f4398a",
|
| 541 |
+
"git_blob": "fd13702ebaa1d34fc1bf87d7c506847af3f7190d",
|
| 542 |
+
"sha256": "6a7850b1490c9122f08384eab1343c908cf526af1fcb997b8898d92325edaf98"
|
| 543 |
+
},
|
| 544 |
+
{
|
| 545 |
+
"path": "kdflow/utils/dynamic_bsz.py",
|
| 546 |
+
"bytes": 15227,
|
| 547 |
+
"api_version_tag": "bcd5639b",
|
| 548 |
+
"git_blob": "44ec3b3a8d3e7f423f3a68b951b7285c463c567b",
|
| 549 |
+
"sha256": "a11fa6064f980d2e614705c9571401de4f18b21e3bbc35a3d69cd7f2ecf4ae9b"
|
| 550 |
+
},
|
| 551 |
+
{
|
| 552 |
+
"path": "kdflow/utils/logging_utils.py",
|
| 553 |
+
"bytes": 3471,
|
| 554 |
+
"api_version_tag": "80b625a3",
|
| 555 |
+
"git_blob": "1393d26d3e95e7ac447bda58242866dac96d2af0",
|
| 556 |
+
"sha256": "e57a5e3bd3df172bb147dfc599f377748e3564de81b800ce365b7a336be89302"
|
| 557 |
+
},
|
| 558 |
+
{
|
| 559 |
+
"path": "kdflow/utils/multimodal_utils.py",
|
| 560 |
+
"bytes": 1073,
|
| 561 |
+
"api_version_tag": "da5520e7",
|
| 562 |
+
"git_blob": "60acb817ed6ce60fda693c80ec1295376c15b10b",
|
| 563 |
+
"sha256": "8b16f06ea69d3c4b2cfb17bb8cb215652350e0aa0548e9f06d07950c07aa34bf"
|
| 564 |
+
},
|
| 565 |
+
{
|
| 566 |
+
"path": "kdflow/utils/utils.py",
|
| 567 |
+
"bytes": 3290,
|
| 568 |
+
"api_version_tag": "3654028c",
|
| 569 |
+
"git_blob": "26415ed5185e779a2945b98fa6f368a726dd286d",
|
| 570 |
+
"sha256": "b4daa8b9e9b405dd330ef3d39d03cc4dac108c05b0012e145fb14baad664fbc9"
|
| 571 |
+
},
|
| 572 |
+
{
|
| 573 |
+
"path": "pyproject.toml",
|
| 574 |
+
"bytes": 730,
|
| 575 |
+
"api_version_tag": "4c685d0b",
|
| 576 |
+
"git_blob": "dd4bfd646653bdc98b8f1bc8aa753a996932166c",
|
| 577 |
+
"sha256": "fb40bc75b604b76e4bc9f5406b1e5fb4721daf4d090543bab430adf4aaaa30f3"
|
| 578 |
+
},
|
| 579 |
+
{
|
| 580 |
+
"path": "requirements.txt",
|
| 581 |
+
"bytes": 6762,
|
| 582 |
+
"api_version_tag": "ea4b60ca",
|
| 583 |
+
"git_blob": "678e55b3cd878b4364ed7bd9d54ac1cf2058af80",
|
| 584 |
+
"sha256": "d4d536aafbe5889185058299ec9deb27ab4903dba3686a14734587703adb2f3f"
|
| 585 |
+
},
|
| 586 |
+
{
|
| 587 |
+
"path": "run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh",
|
| 588 |
+
"bytes": 2414,
|
| 589 |
+
"api_version_tag": "a6482a95",
|
| 590 |
+
"git_blob": "c778963c3e42749205a2fc9b0208bfb076a48180",
|
| 591 |
+
"sha256": "4982257e528fbe508d180172aa81e85e761cbc15328ec2ddc33b7feda42bd2e4"
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"path": "run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh",
|
| 595 |
+
"bytes": 2451,
|
| 596 |
+
"api_version_tag": "bb390a7d",
|
| 597 |
+
"git_blob": "2997aaaebae9914d30b62cca821bce7ceef7818d",
|
| 598 |
+
"sha256": "2f297f306ff7f91d5902cc039bea87cf43474127d6e7598ff50053c5d66551a1"
|
| 599 |
+
}
|
| 600 |
+
]
|
| 601 |
+
}
|
kdflow/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
__version__ = "0.1.0"
|
kdflow/algorithms/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Algorithms included in the submission."""
|
| 2 |
+
|
| 3 |
+
ALGO_DICT = {}
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def register_algorithm(name):
|
| 7 |
+
def decorator(cls):
|
| 8 |
+
ALGO_DICT[name] = cls
|
| 9 |
+
return cls
|
| 10 |
+
return decorator
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD
|
| 14 |
+
from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD
|
| 15 |
+
from kdflow.algorithms.topk_byte_align_ctkd import TopKByteAlignCrossTokenizerKD
|
| 16 |
+
|
| 17 |
+
__all__ = ["ALGO_DICT", "SimpleCrossTokenizerKD", "ByteAlignCrossTokenizerKD", "TopKByteAlignCrossTokenizerKD"]
|
kdflow/algorithms/byte_align_ctkd.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from kdflow.algorithms import register_algorithm
|
| 4 |
+
from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD
|
| 5 |
+
from kdflow.token_alignment import (
|
| 6 |
+
align_byte_span_batch_1to1,
|
| 7 |
+
align_byte_spans_1to1,
|
| 8 |
+
find_byte_overlap_token_ids,
|
| 9 |
+
get_tokenizer_byte_codec,
|
| 10 |
+
)
|
| 11 |
+
from kdflow.utils.logging_utils import init_logger
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
logger = init_logger(__name__)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@register_algorithm("byte_align_ctkd")
|
| 18 |
+
class ByteAlignCrossTokenizerKD(SimpleCrossTokenizerKD):
|
| 19 |
+
"""Simple cross-tokenizer KD with UTF-8 byte-span 1:1 alignment."""
|
| 20 |
+
|
| 21 |
+
def __init__(
|
| 22 |
+
self,
|
| 23 |
+
strategy,
|
| 24 |
+
student_model,
|
| 25 |
+
teacher_lm_head,
|
| 26 |
+
student_tokenizer,
|
| 27 |
+
teacher_tokenizer,
|
| 28 |
+
**kwargs,
|
| 29 |
+
):
|
| 30 |
+
# These are initialized before ``super`` because its constructor calls
|
| 31 |
+
# the virtual ``_find_overlap_tokens`` method.
|
| 32 |
+
self.student_byte_codec = get_tokenizer_byte_codec(student_tokenizer)
|
| 33 |
+
self.teacher_byte_codec = get_tokenizer_byte_codec(teacher_tokenizer)
|
| 34 |
+
super().__init__(
|
| 35 |
+
strategy=strategy,
|
| 36 |
+
student_model=student_model,
|
| 37 |
+
teacher_lm_head=teacher_lm_head,
|
| 38 |
+
student_tokenizer=student_tokenizer,
|
| 39 |
+
teacher_tokenizer=teacher_tokenizer,
|
| 40 |
+
**kwargs,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
def _find_overlap_tokens(self):
|
| 44 |
+
# Preserve the exact historical mapping for all existing ByteLevel
|
| 45 |
+
# pairs. Only mixed/SentencePiece pairs use the byte-canonical map.
|
| 46 |
+
if (
|
| 47 |
+
self.student_byte_codec.name == "byte_level"
|
| 48 |
+
and self.teacher_byte_codec.name == "byte_level"
|
| 49 |
+
):
|
| 50 |
+
return super()._find_overlap_tokens()
|
| 51 |
+
|
| 52 |
+
student_ids, teacher_ids = find_byte_overlap_token_ids(
|
| 53 |
+
student_tokenizer=self.student_tokenizer,
|
| 54 |
+
teacher_tokenizer=self.teacher_tokenizer,
|
| 55 |
+
student_codec=self.student_byte_codec,
|
| 56 |
+
teacher_codec=self.teacher_byte_codec,
|
| 57 |
+
)
|
| 58 |
+
device = self.teacher_lm_head.weight.device
|
| 59 |
+
logger.info(
|
| 60 |
+
"Num of byte-equivalent overlap tokens between student & teacher: "
|
| 61 |
+
f"{len(student_ids)}"
|
| 62 |
+
)
|
| 63 |
+
return (
|
| 64 |
+
torch.tensor(student_ids, dtype=torch.long, device=device),
|
| 65 |
+
torch.tensor(teacher_ids, dtype=torch.long, device=device),
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
def _align_sequences(self, tea_seq, stu_seq):
|
| 69 |
+
return align_byte_spans_1to1(
|
| 70 |
+
teacher_tokens=tea_seq,
|
| 71 |
+
student_tokens=stu_seq,
|
| 72 |
+
teacher_eos_token=self.teacher_tokenizer.eos_token,
|
| 73 |
+
student_eos_token=self.student_tokenizer.eos_token,
|
| 74 |
+
teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
|
| 75 |
+
student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
def _get_alignment_indices(
|
| 79 |
+
self,
|
| 80 |
+
teacher_label_ids,
|
| 81 |
+
student_label_ids,
|
| 82 |
+
teacher_loss_mask,
|
| 83 |
+
student_loss_mask,
|
| 84 |
+
):
|
| 85 |
+
"""Align each micro-batch row independently, then return flat indices."""
|
| 86 |
+
if teacher_label_ids.shape[0] != student_label_ids.shape[0]:
|
| 87 |
+
raise ValueError(
|
| 88 |
+
"Teacher and student micro-batches must contain the same number "
|
| 89 |
+
"of samples for byte_align_ctkd."
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
teacher_batch_tokens = []
|
| 93 |
+
student_batch_tokens = []
|
| 94 |
+
for row_index in range(teacher_label_ids.shape[0]):
|
| 95 |
+
teacher_row_ids = teacher_label_ids[row_index][
|
| 96 |
+
teacher_loss_mask[row_index]
|
| 97 |
+
]
|
| 98 |
+
student_row_ids = student_label_ids[row_index][
|
| 99 |
+
student_loss_mask[row_index]
|
| 100 |
+
]
|
| 101 |
+
teacher_row_token_ids = teacher_row_ids.cpu().tolist()
|
| 102 |
+
student_row_token_ids = student_row_ids.cpu().tolist()
|
| 103 |
+
teacher_batch_tokens.append(
|
| 104 |
+
self.teacher_tokenizer.convert_ids_to_tokens(
|
| 105 |
+
teacher_row_token_ids
|
| 106 |
+
)
|
| 107 |
+
)
|
| 108 |
+
student_batch_tokens.append(
|
| 109 |
+
self.student_tokenizer.convert_ids_to_tokens(
|
| 110 |
+
student_row_token_ids
|
| 111 |
+
)
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
return align_byte_span_batch_1to1(
|
| 115 |
+
teacher_batch_tokens=teacher_batch_tokens,
|
| 116 |
+
student_batch_tokens=student_batch_tokens,
|
| 117 |
+
teacher_eos_token=self.teacher_tokenizer.eos_token,
|
| 118 |
+
student_eos_token=self.student_tokenizer.eos_token,
|
| 119 |
+
teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes,
|
| 120 |
+
student_piece_to_bytes=self.student_byte_codec.piece_to_bytes,
|
| 121 |
+
)
|
kdflow/algorithms/simple_ctkd.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from kdflow.loss import build_loss_fn
|
| 4 |
+
from kdflow.algorithms import register_algorithm
|
| 5 |
+
from kdflow.loss.chunked_loss import chunked_loss
|
| 6 |
+
from kdflow.loss.cross_entropy import compute_cross_entropy
|
| 7 |
+
from kdflow.metrics.entropy import compute_entropy
|
| 8 |
+
from kdflow.utils.logging_utils import init_logger
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
logger = init_logger(__name__)
|
| 12 |
+
|
| 13 |
+
@register_algorithm("simple_ctkd")
|
| 14 |
+
class SimpleCrossTokenizerKD:
|
| 15 |
+
"""Simply find the overlap tokens between student and teacher tokenizer, and only compute KD loss on this sub-vocabulary.
|
| 16 |
+
Motivation: modern LLMs have a large amount of shared tokens.
|
| 17 |
+
"""
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
strategy,
|
| 21 |
+
student_model,
|
| 22 |
+
teacher_lm_head,
|
| 23 |
+
student_tokenizer,
|
| 24 |
+
teacher_tokenizer,
|
| 25 |
+
**kwargs
|
| 26 |
+
):
|
| 27 |
+
self.strategy = strategy
|
| 28 |
+
self.args = strategy.args
|
| 29 |
+
self.student = student_model
|
| 30 |
+
self.teacher_lm_head = teacher_lm_head
|
| 31 |
+
self.student_tokenizer = student_tokenizer
|
| 32 |
+
self.teacher_tokenizer = teacher_tokenizer
|
| 33 |
+
self.student_overlap_token_ids, self.teacher_overlap_token_ids = self._find_overlap_tokens()
|
| 34 |
+
self.loss_fn = build_loss_fn(self.args.kd.kd_loss_fn, self.args)
|
| 35 |
+
self.metric_fns = [compute_entropy] if self.args.scenario == "on_policy_kd" else []
|
| 36 |
+
|
| 37 |
+
def _find_overlap_tokens(self):
|
| 38 |
+
student_vocab = {k.replace("Ġ", "▁"): v for k, v in self.student_tokenizer.get_vocab().items()}
|
| 39 |
+
teacher_vocab = {k.replace("Ġ", "▁"): v for k, v in self.teacher_tokenizer.get_vocab().items()}
|
| 40 |
+
overlap_tokens = set(student_vocab.keys()) & set(teacher_vocab.keys())
|
| 41 |
+
student_ids = [student_vocab[token] for token in overlap_tokens]
|
| 42 |
+
teacher_ids = [teacher_vocab[token] for token in overlap_tokens]
|
| 43 |
+
stu_eos, tea_eos = self.student_tokenizer.eos_token_id, self.teacher_tokenizer.eos_token_id
|
| 44 |
+
if stu_eos not in student_ids or tea_eos not in teacher_ids:
|
| 45 |
+
student_ids.append(stu_eos)
|
| 46 |
+
teacher_ids.append(tea_eos)
|
| 47 |
+
device = self.teacher_lm_head.weight.device
|
| 48 |
+
logger.info(f"Num of overlap_tokens between student & teacher: {len(student_ids)}")
|
| 49 |
+
return torch.tensor(student_ids, dtype=torch.long, device=device), torch.tensor(teacher_ids, dtype=torch.long, device=device)
|
| 50 |
+
|
| 51 |
+
def _align_sequences(self, tea_seq, stu_seq):
|
| 52 |
+
i, j = 0, 0
|
| 53 |
+
t2s_align, s2t_align = [], []
|
| 54 |
+
history_tea_seq, history_stu_seq = "", ""
|
| 55 |
+
|
| 56 |
+
tea_eos = self.teacher_tokenizer.eos_token
|
| 57 |
+
stu_eos = self.student_tokenizer.eos_token
|
| 58 |
+
|
| 59 |
+
tea_seq = [token.replace('▁', '').replace('Ġ', '') for token in tea_seq]
|
| 60 |
+
stu_seq = [token.replace('▁', '').replace('Ġ', '') for token in stu_seq]
|
| 61 |
+
|
| 62 |
+
if tea_seq == stu_seq:
|
| 63 |
+
indices = list(range(len(tea_seq)))
|
| 64 |
+
return indices, indices
|
| 65 |
+
|
| 66 |
+
while i < len(tea_seq) and j < len(stu_seq):
|
| 67 |
+
is_eos_match = (tea_seq[i] == tea_eos and stu_seq[j] == stu_eos)
|
| 68 |
+
if history_tea_seq == history_stu_seq and (
|
| 69 |
+
tea_seq[i] == stu_seq[j] or is_eos_match
|
| 70 |
+
):
|
| 71 |
+
common_text = tea_seq[i]
|
| 72 |
+
history_tea_seq += common_text
|
| 73 |
+
history_stu_seq += common_text
|
| 74 |
+
t2s_align.append(i)
|
| 75 |
+
s2t_align.append(j)
|
| 76 |
+
i += 1
|
| 77 |
+
j += 1
|
| 78 |
+
elif len(history_tea_seq) > len(history_stu_seq):
|
| 79 |
+
history_stu_seq += stu_seq[j]
|
| 80 |
+
j += 1
|
| 81 |
+
elif len(history_tea_seq) < len(history_stu_seq):
|
| 82 |
+
history_tea_seq += tea_seq[i]
|
| 83 |
+
i += 1
|
| 84 |
+
else:
|
| 85 |
+
history_tea_seq += tea_seq[i]
|
| 86 |
+
history_stu_seq += stu_seq[j]
|
| 87 |
+
i += 1
|
| 88 |
+
j += 1
|
| 89 |
+
|
| 90 |
+
return t2s_align, s2t_align
|
| 91 |
+
|
| 92 |
+
def _get_alignment_indices(
|
| 93 |
+
self,
|
| 94 |
+
teacher_label_ids,
|
| 95 |
+
student_label_ids,
|
| 96 |
+
teacher_loss_mask,
|
| 97 |
+
student_loss_mask,
|
| 98 |
+
):
|
| 99 |
+
"""Return indices into the flattened teacher/student response tensors."""
|
| 100 |
+
teacher_tokens = self.teacher_tokenizer.convert_ids_to_tokens(
|
| 101 |
+
teacher_label_ids[teacher_loss_mask].cpu().tolist()
|
| 102 |
+
)
|
| 103 |
+
student_tokens = self.student_tokenizer.convert_ids_to_tokens(
|
| 104 |
+
student_label_ids[student_loss_mask].cpu().tolist()
|
| 105 |
+
)
|
| 106 |
+
return self._align_sequences(teacher_tokens, student_tokens)
|
| 107 |
+
|
| 108 |
+
def training_step(self, micro_batch):
|
| 109 |
+
student_input_ids = micro_batch["stu_input_ids"]
|
| 110 |
+
student_attn_mask = micro_batch["stu_attn_mask"]
|
| 111 |
+
student_loss_mask = micro_batch["stu_loss_mask"].bool()
|
| 112 |
+
teacher_input_ids = micro_batch["tea_input_ids"]
|
| 113 |
+
teacher_attn_mask = micro_batch["tea_attn_mask"]
|
| 114 |
+
teacher_loss_mask = micro_batch["tea_loss_mask"].bool()
|
| 115 |
+
teacher_hiddens = micro_batch.get("teacher_hiddens", None)
|
| 116 |
+
avg_token_num = micro_batch["avg_micro_batch_token_num"]
|
| 117 |
+
|
| 118 |
+
assert teacher_hiddens is not None, "micro_batch must contain `teacher_hiddens` for KD"
|
| 119 |
+
|
| 120 |
+
student_shifted_label_ids = student_input_ids.roll(shifts=-1, dims=1)
|
| 121 |
+
teacher_shifted_label_ids = teacher_input_ids.roll(shifts=-1, dims=1)
|
| 122 |
+
teacher_aligned_idx, student_aligned_idx = self._get_alignment_indices(
|
| 123 |
+
teacher_shifted_label_ids,
|
| 124 |
+
student_shifted_label_ids,
|
| 125 |
+
teacher_loss_mask,
|
| 126 |
+
student_loss_mask,
|
| 127 |
+
)
|
| 128 |
+
student_label_ids = student_shifted_label_ids[student_loss_mask]
|
| 129 |
+
|
| 130 |
+
mm_kwargs = micro_batch.get("stu_multi_modal_inputs") or {}
|
| 131 |
+
|
| 132 |
+
output = self.student(
|
| 133 |
+
student_input_ids,
|
| 134 |
+
attention_mask=student_attn_mask,
|
| 135 |
+
allgather_logits=True,
|
| 136 |
+
ring_attn_group=self.strategy.ring_attn_group,
|
| 137 |
+
**mm_kwargs,
|
| 138 |
+
)
|
| 139 |
+
student_hiddens = output["hidden_states"][-1][student_loss_mask]
|
| 140 |
+
del output
|
| 141 |
+
|
| 142 |
+
teacher_hiddens = teacher_hiddens.to(self.teacher_lm_head.weight)
|
| 143 |
+
|
| 144 |
+
align_ratio = torch.tensor(len(student_aligned_idx) / len(student_label_ids))
|
| 145 |
+
aligned_student_hiddens = student_hiddens[student_aligned_idx]
|
| 146 |
+
aligned_teacher_hiddens = teacher_hiddens[teacher_aligned_idx]
|
| 147 |
+
|
| 148 |
+
# A length-truncated cross-tokenizer response may contain no 1:1 byte
|
| 149 |
+
# span (and no appended EOS fallback). Keep the distributed optimizer
|
| 150 |
+
# step well-defined instead of passing a zero chunk size downstream.
|
| 151 |
+
if aligned_student_hiddens.shape[0] == 0:
|
| 152 |
+
zero_loss = student_hiddens.sum() * 0.0
|
| 153 |
+
return {
|
| 154 |
+
"train/loss": zero_loss,
|
| 155 |
+
"train/kd_loss": zero_loss,
|
| 156 |
+
"distill/alignment_ratio": align_ratio,
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
# Non-chunked case can be regarded as a special case of chunked loss (i.e., chunk_size = seq_len)
|
| 160 |
+
chunk_size = self.args.train.chunked_loss_size or aligned_student_hiddens.shape[0]
|
| 161 |
+
|
| 162 |
+
student_overlap_ids = self.student_overlap_token_ids
|
| 163 |
+
teacher_overlap_ids = self.teacher_overlap_token_ids
|
| 164 |
+
student_lm_head = self.student.model.lm_head
|
| 165 |
+
teacher_lm_head = self.teacher_lm_head
|
| 166 |
+
|
| 167 |
+
def student_logits_fn(hidden_chunk, skip=False):
|
| 168 |
+
return student_lm_head(hidden_chunk, skip=skip)[:, student_overlap_ids]
|
| 169 |
+
|
| 170 |
+
def teacher_logits_fn(start, end):
|
| 171 |
+
return teacher_lm_head(aligned_teacher_hiddens[start:end])[:, teacher_overlap_ids]
|
| 172 |
+
|
| 173 |
+
kd_loss, metric_sums = chunked_loss(
|
| 174 |
+
aligned_student_hiddens, student_lm_head, self.loss_fn,
|
| 175 |
+
student_logits_fn=student_logits_fn,
|
| 176 |
+
teacher_logits_fn=teacher_logits_fn,
|
| 177 |
+
chunk_size=chunk_size, reduction="sum",
|
| 178 |
+
metric_fns=self.metric_fns, return_metrics=True,
|
| 179 |
+
)
|
| 180 |
+
kd_loss = kd_loss / avg_token_num
|
| 181 |
+
|
| 182 |
+
loss_info = {
|
| 183 |
+
"train/loss": kd_loss,
|
| 184 |
+
"train/kd_loss": kd_loss,
|
| 185 |
+
"distill/alignment_ratio": align_ratio,
|
| 186 |
+
}
|
| 187 |
+
loss_info.update({key: value / avg_token_num for key, value in metric_sums.items()})
|
| 188 |
+
|
| 189 |
+
if self.args.kd.kd_ratio < 1:
|
| 190 |
+
ce_loss = chunked_loss(
|
| 191 |
+
student_hiddens, self.student.model.lm_head, compute_cross_entropy,
|
| 192 |
+
label=student_label_ids, chunk_size=chunk_size, reduction="sum"
|
| 193 |
+
) / avg_token_num
|
| 194 |
+
loss = (1 - self.args.kd.kd_ratio) * ce_loss + self.args.kd.kd_ratio * kd_loss
|
| 195 |
+
loss_info["train/loss"] = loss
|
| 196 |
+
loss_info["train/ce_loss"] = ce_loss
|
| 197 |
+
|
| 198 |
+
return loss_info
|
kdflow/algorithms/topk_byte_align_ctkd.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Tuple
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from kdflow.algorithms import register_algorithm
|
| 6 |
+
from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD
|
| 7 |
+
from kdflow.metrics.entropy import compute_entropy
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def select_student_topk_logits(
|
| 11 |
+
student_logits: torch.Tensor,
|
| 12 |
+
teacher_logits: torch.Tensor,
|
| 13 |
+
topk: int,
|
| 14 |
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 15 |
+
"""Select the student's top-k shared-vocabulary entries on both sides.
|
| 16 |
+
|
| 17 |
+
The returned tensors contain matching vocabulary entries. Downstream KD
|
| 18 |
+
losses apply softmax/log-softmax to these reduced logits, which
|
| 19 |
+
re-normalizes both distributions over exactly this top-k support.
|
| 20 |
+
"""
|
| 21 |
+
if student_logits.shape != teacher_logits.shape:
|
| 22 |
+
raise ValueError(
|
| 23 |
+
"Student and teacher shared-vocabulary logits must have the same "
|
| 24 |
+
f"shape, got {student_logits.shape} and {teacher_logits.shape}."
|
| 25 |
+
)
|
| 26 |
+
if topk <= 0 or topk > student_logits.shape[-1]:
|
| 27 |
+
raise ValueError(
|
| 28 |
+
f"topk must be in [1, {student_logits.shape[-1]}], got {topk}."
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
student_topk_logits, topk_indices = torch.topk(
|
| 32 |
+
student_logits,
|
| 33 |
+
k=topk,
|
| 34 |
+
dim=-1,
|
| 35 |
+
)
|
| 36 |
+
teacher_topk_logits = teacher_logits.gather(-1, topk_indices)
|
| 37 |
+
return student_topk_logits, teacher_topk_logits, topk_indices
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@register_algorithm("topk_byte_align_ctkd")
|
| 41 |
+
class TopKByteAlignCrossTokenizerKD(ByteAlignCrossTokenizerKD):
|
| 42 |
+
"""Byte-aligned CTKD restricted to each student's shared-vocab top-k."""
|
| 43 |
+
|
| 44 |
+
def __init__(self, *args, **kwargs):
|
| 45 |
+
super().__init__(*args, **kwargs)
|
| 46 |
+
|
| 47 |
+
self.ctkd_topk = self.args.kd.ctkd_topk
|
| 48 |
+
overlap_vocab_size = self.student_overlap_token_ids.numel()
|
| 49 |
+
if self.ctkd_topk < 2:
|
| 50 |
+
raise ValueError(
|
| 51 |
+
"ctkd_topk must be at least 2 because re-normalizing a "
|
| 52 |
+
"single logit produces a degenerate distribution."
|
| 53 |
+
)
|
| 54 |
+
if self.ctkd_topk > overlap_vocab_size:
|
| 55 |
+
raise ValueError(
|
| 56 |
+
f"ctkd_topk ({self.ctkd_topk}) exceeds the shared vocabulary "
|
| 57 |
+
f"size ({overlap_vocab_size})."
|
| 58 |
+
)
|
| 59 |
+
if (
|
| 60 |
+
self.args.kd.kd_loss_fn == "hrl"
|
| 61 |
+
and self.args.kd.hrl_topk > self.ctkd_topk
|
| 62 |
+
):
|
| 63 |
+
raise ValueError(
|
| 64 |
+
f"hrl_topk ({self.args.kd.hrl_topk}) cannot exceed ctkd_topk "
|
| 65 |
+
f"({self.ctkd_topk})."
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
self._shared_vocab_loss_fn = self.loss_fn
|
| 69 |
+
self.loss_fn = self._topk_loss
|
| 70 |
+
if self.metric_fns:
|
| 71 |
+
self.metric_fns = [self._topk_entropy]
|
| 72 |
+
|
| 73 |
+
def _topk_loss(
|
| 74 |
+
self,
|
| 75 |
+
student_logits: torch.Tensor,
|
| 76 |
+
teacher_logits: torch.Tensor,
|
| 77 |
+
**kwargs,
|
| 78 |
+
) -> torch.Tensor:
|
| 79 |
+
student_topk_logits, teacher_topk_logits, _ = select_student_topk_logits(
|
| 80 |
+
student_logits,
|
| 81 |
+
teacher_logits,
|
| 82 |
+
self.ctkd_topk,
|
| 83 |
+
)
|
| 84 |
+
return self._shared_vocab_loss_fn(
|
| 85 |
+
student_topk_logits,
|
| 86 |
+
teacher_topk_logits,
|
| 87 |
+
**kwargs,
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
def _topk_entropy(
|
| 91 |
+
self,
|
| 92 |
+
student_logits: torch.Tensor,
|
| 93 |
+
teacher_logits: torch.Tensor,
|
| 94 |
+
**kwargs,
|
| 95 |
+
):
|
| 96 |
+
student_topk_logits, _, _ = select_student_topk_logits(
|
| 97 |
+
student_logits,
|
| 98 |
+
teacher_logits,
|
| 99 |
+
self.ctkd_topk,
|
| 100 |
+
)
|
| 101 |
+
return compute_entropy(student_logits=student_topk_logits)
|
kdflow/arguments/__init__.py
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from dataclasses import dataclass, field
|
| 3 |
+
|
| 4 |
+
from transformers import HfArgumentParser
|
| 5 |
+
|
| 6 |
+
from kdflow.arguments.data_args import DataArguments
|
| 7 |
+
from kdflow.arguments.model_args import ModelArguments
|
| 8 |
+
from kdflow.arguments.training_args import TrainingArguments
|
| 9 |
+
from kdflow.arguments.fsdp_args import FSDPArguments
|
| 10 |
+
from kdflow.arguments.distillation_args import DistillationArguments
|
| 11 |
+
from kdflow.arguments.rollout_args import RolloutArguments
|
| 12 |
+
from kdflow.arguments.logging_args import LoggingArguments
|
| 13 |
+
from kdflow.utils.logging_utils import init_logger
|
| 14 |
+
|
| 15 |
+
logger = init_logger(__name__)
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class AllArguments:
|
| 19 |
+
data: DataArguments = field(default_factory=DataArguments)
|
| 20 |
+
model: ModelArguments = field(default_factory=ModelArguments)
|
| 21 |
+
train: TrainingArguments = field(default_factory=TrainingArguments)
|
| 22 |
+
fsdp: FSDPArguments = field(default_factory=FSDPArguments)
|
| 23 |
+
kd: DistillationArguments = field(default_factory=DistillationArguments)
|
| 24 |
+
rollout: RolloutArguments = field(default_factory=RolloutArguments)
|
| 25 |
+
log: LoggingArguments = field(default_factory=LoggingArguments)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def is_linear_attention(model_path: str) -> bool:
|
| 29 |
+
if not model_path:
|
| 30 |
+
return False
|
| 31 |
+
try:
|
| 32 |
+
from transformers import AutoConfig
|
| 33 |
+
model_type = AutoConfig.from_pretrained(model_path, trust_remote_code=True).model_type or ""
|
| 34 |
+
except Exception:
|
| 35 |
+
return False
|
| 36 |
+
return any(k in model_type.lower() for k in ("qwen3_next", "qwen3_5"))
|
| 37 |
+
|
| 38 |
+
def init_args(scenario: str = "on_policy_kd"):
|
| 39 |
+
parser = HfArgumentParser((
|
| 40 |
+
DataArguments,
|
| 41 |
+
ModelArguments,
|
| 42 |
+
TrainingArguments,
|
| 43 |
+
FSDPArguments,
|
| 44 |
+
DistillationArguments,
|
| 45 |
+
RolloutArguments,
|
| 46 |
+
LoggingArguments
|
| 47 |
+
))
|
| 48 |
+
(
|
| 49 |
+
data_args,
|
| 50 |
+
model_args,
|
| 51 |
+
train_args,
|
| 52 |
+
fsdp_args,
|
| 53 |
+
kd_args,
|
| 54 |
+
rollout_args,
|
| 55 |
+
log_args
|
| 56 |
+
) = parser.parse_args_into_dataclasses()
|
| 57 |
+
|
| 58 |
+
args = AllArguments(
|
| 59 |
+
data=data_args,
|
| 60 |
+
model=model_args,
|
| 61 |
+
train=train_args,
|
| 62 |
+
fsdp=fsdp_args,
|
| 63 |
+
kd=kd_args,
|
| 64 |
+
rollout=rollout_args,
|
| 65 |
+
log=log_args
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
if scenario != "on_policy_kd" and args.data.custom_eval_fn is not None:
|
| 69 |
+
logger.warning(
|
| 70 |
+
f"`--custom_eval_fn` is only supported in the `on_policy_kd` scenario "
|
| 71 |
+
f"and will be ignored for `{scenario}`."
|
| 72 |
+
)
|
| 73 |
+
if scenario != "on_policy_kd" and args.data.eval_math_dataset_path is not None:
|
| 74 |
+
logger.warning(
|
| 75 |
+
f"`--eval_math_dataset_path` is only supported in the `on_policy_kd` "
|
| 76 |
+
f"scenario and will be ignored for `{scenario}`."
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
if args.data.input_template and "{}" not in args.data.input_template:
|
| 80 |
+
logger.warning("{} not in args.data.input_template, set to None")
|
| 81 |
+
args.data.input_template = None
|
| 82 |
+
|
| 83 |
+
if args.data.input_template and "\\n" in args.data.input_template:
|
| 84 |
+
logger.warning(
|
| 85 |
+
"input_template contains \\n characters instead of newline. "
|
| 86 |
+
"You likely want to pass $'\\n' in Bash or \"`n\" in PowerShell."
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
if args.data.packing_samples:
|
| 90 |
+
if "flash_attention" not in args.model.attn_implementation:
|
| 91 |
+
logger.warning(
|
| 92 |
+
"Please use --attn_implementation with flash_attention to accelerate when --packing_samples is enabled."
|
| 93 |
+
)
|
| 94 |
+
args.model.attn_implementation = "flash_attention_2"
|
| 95 |
+
|
| 96 |
+
if is_linear_attention(args.model.student_name_or_path):
|
| 97 |
+
logger.warning(
|
| 98 |
+
f"--packing_samples is not yet compatible with linear-attention models "
|
| 99 |
+
f"(got {args.model.student_name_or_path}). Auto-disabling --packing_samples."
|
| 100 |
+
)
|
| 101 |
+
args.data.packing_samples = False
|
| 102 |
+
|
| 103 |
+
total_gpus = args.train.num_nodes * args.train.num_gpus_per_node
|
| 104 |
+
|
| 105 |
+
if scenario == "alfworld_eval":
|
| 106 |
+
if not args.model.student_name_or_path:
|
| 107 |
+
raise ValueError(
|
| 108 |
+
"Standalone ALFWorld evaluation requires "
|
| 109 |
+
"`--student_name_or_path`."
|
| 110 |
+
)
|
| 111 |
+
if not args.rollout.alfworld_config:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
"Standalone ALFWorld evaluation requires `--alfworld_config`."
|
| 114 |
+
)
|
| 115 |
+
valid_protocols = {"kdflow", "verl_agent"}
|
| 116 |
+
if args.rollout.alfworld_eval_protocol not in valid_protocols:
|
| 117 |
+
raise ValueError(
|
| 118 |
+
"`--alfworld_eval_protocol` must be one of "
|
| 119 |
+
f"{sorted(valid_protocols)}."
|
| 120 |
+
)
|
| 121 |
+
if args.rollout.alfworld_eval_protocol == "verl_agent":
|
| 122 |
+
try:
|
| 123 |
+
seeds = [
|
| 124 |
+
int(value.strip())
|
| 125 |
+
for value in args.rollout.alfworld_eval_seeds.split(",")
|
| 126 |
+
if value.strip()
|
| 127 |
+
]
|
| 128 |
+
except ValueError as error:
|
| 129 |
+
raise ValueError(
|
| 130 |
+
"`--alfworld_eval_seeds` must be a comma-separated list "
|
| 131 |
+
"of integers."
|
| 132 |
+
) from error
|
| 133 |
+
if not seeds:
|
| 134 |
+
raise ValueError(
|
| 135 |
+
"`--alfworld_eval_seeds` must contain at least one seed."
|
| 136 |
+
)
|
| 137 |
+
if args.rollout.alfworld_history_length < 0:
|
| 138 |
+
raise ValueError(
|
| 139 |
+
"`--alfworld_history_length` must be non-negative."
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
if scenario in ("on_policy_kd", "alfworld_eval"):
|
| 143 |
+
if total_gpus <= 0:
|
| 144 |
+
raise ValueError(
|
| 145 |
+
"`--num_nodes * --num_gpus_per_node` must be positive."
|
| 146 |
+
)
|
| 147 |
+
if args.rollout.rollout_tp_size <= 0:
|
| 148 |
+
raise ValueError("`--rollout_tp_size` must be positive.")
|
| 149 |
+
if args.rollout.rollout_batch_size <= 0:
|
| 150 |
+
raise ValueError("`--rollout_batch_size` must be positive.")
|
| 151 |
+
if args.rollout.rollout_engine_concurrency <= 0:
|
| 152 |
+
raise ValueError(
|
| 153 |
+
"`--rollout_engine_concurrency` must be positive."
|
| 154 |
+
)
|
| 155 |
+
if args.rollout.alfworld_config:
|
| 156 |
+
valid_alfworld_protocols = {"kdflow", "verl_agent"}
|
| 157 |
+
if args.rollout.alfworld_protocol not in valid_alfworld_protocols:
|
| 158 |
+
raise ValueError(
|
| 159 |
+
"`--alfworld_protocol` must be one of "
|
| 160 |
+
f"{sorted(valid_alfworld_protocols)}."
|
| 161 |
+
)
|
| 162 |
+
args.rollout.alfworld_config = os.path.abspath(
|
| 163 |
+
os.path.expanduser(args.rollout.alfworld_config)
|
| 164 |
+
)
|
| 165 |
+
if not os.path.isfile(args.rollout.alfworld_config):
|
| 166 |
+
raise ValueError(
|
| 167 |
+
"`--alfworld_config` must point to an existing YAML file, "
|
| 168 |
+
f"got {args.rollout.alfworld_config!r}."
|
| 169 |
+
)
|
| 170 |
+
valid_alfworld_splits = {
|
| 171 |
+
"train",
|
| 172 |
+
"eval_in_distribution",
|
| 173 |
+
"eval_out_of_distribution",
|
| 174 |
+
}
|
| 175 |
+
if args.rollout.alfworld_split not in valid_alfworld_splits:
|
| 176 |
+
raise ValueError(
|
| 177 |
+
"`--alfworld_split` must be one of "
|
| 178 |
+
f"{sorted(valid_alfworld_splits)}."
|
| 179 |
+
)
|
| 180 |
+
if args.rollout.alfworld_max_turns <= 0:
|
| 181 |
+
raise ValueError("`--alfworld_max_turns` must be positive.")
|
| 182 |
+
if args.rollout.alfworld_episodes_per_epoch <= 0:
|
| 183 |
+
raise ValueError(
|
| 184 |
+
"`--alfworld_episodes_per_epoch` must be positive."
|
| 185 |
+
)
|
| 186 |
+
if args.rollout.alfworld_eval_episodes <= 0:
|
| 187 |
+
raise ValueError("`--alfworld_eval_episodes` must be positive.")
|
| 188 |
+
if args.rollout.alfworld_admissible_limit < 0:
|
| 189 |
+
raise ValueError(
|
| 190 |
+
"`--alfworld_admissible_limit` must be non-negative."
|
| 191 |
+
)
|
| 192 |
+
if args.rollout.n_samples_per_prompt != 1:
|
| 193 |
+
raise ValueError(
|
| 194 |
+
"ALFWorld multi-turn rollout requires "
|
| 195 |
+
"`--n_samples_per_prompt 1`; the environment already "
|
| 196 |
+
"samples one independent game per batch slot."
|
| 197 |
+
)
|
| 198 |
+
if scenario == "on_policy_kd" and (
|
| 199 |
+
args.rollout.alfworld_episodes_per_epoch
|
| 200 |
+
< args.rollout.rollout_batch_size
|
| 201 |
+
):
|
| 202 |
+
raise ValueError(
|
| 203 |
+
"`--alfworld_episodes_per_epoch` must be at least "
|
| 204 |
+
"`--rollout_batch_size`."
|
| 205 |
+
)
|
| 206 |
+
remainder = (
|
| 207 |
+
args.rollout.alfworld_episodes_per_epoch
|
| 208 |
+
% args.rollout.rollout_batch_size
|
| 209 |
+
)
|
| 210 |
+
if scenario == "on_policy_kd" and remainder:
|
| 211 |
+
logger.warning(
|
| 212 |
+
"ALFWorld drops %d episode slots per epoch because "
|
| 213 |
+
"alfworld_episodes_per_epoch is not divisible by "
|
| 214 |
+
"rollout_batch_size.",
|
| 215 |
+
remainder,
|
| 216 |
+
)
|
| 217 |
+
if scenario == "on_policy_kd" and args.data.eval_dataset_path:
|
| 218 |
+
logger.warning(
|
| 219 |
+
"`--eval_dataset_path` is ignored for ALFWorld multi-turn "
|
| 220 |
+
"training; use an ALFWorld evaluation split in a separate run."
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
if (
|
| 224 |
+
scenario == "alfworld_eval"
|
| 225 |
+
and args.rollout.alfworld_eval_episodes
|
| 226 |
+
% args.rollout.rollout_batch_size
|
| 227 |
+
):
|
| 228 |
+
raise ValueError(
|
| 229 |
+
"`--alfworld_eval_episodes` must be divisible by "
|
| 230 |
+
"`--rollout_batch_size`."
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
if scenario == "on_policy_kd":
|
| 234 |
+
if args.data.eval_math_n_samples <= 0:
|
| 235 |
+
raise ValueError("`--eval_math_n_samples` must be positive.")
|
| 236 |
+
if args.data.eval_math_max_concurrent <= 0:
|
| 237 |
+
raise ValueError("`--eval_math_max_concurrent` must be positive.")
|
| 238 |
+
if args.data.eval_math_max_tokens <= 0:
|
| 239 |
+
raise ValueError("`--eval_math_max_tokens` must be positive.")
|
| 240 |
+
if args.data.eval_math_timeout < 0:
|
| 241 |
+
raise ValueError("`--eval_math_timeout` must be non-negative.")
|
| 242 |
+
|
| 243 |
+
if total_gpus % args.rollout.rollout_tp_size != 0:
|
| 244 |
+
raise ValueError(
|
| 245 |
+
f"Total GPUs ({total_gpus}) must be divisible by rollout_tp_size ({args.rollout.rollout_tp_size})."
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
expected_num_engines = total_gpus // args.rollout.rollout_tp_size
|
| 249 |
+
if args.rollout.rollout_num_engines != expected_num_engines:
|
| 250 |
+
logger.warning(
|
| 251 |
+
f"Auto-adjusting rollout_num_engines from {args.rollout.rollout_num_engines} to {expected_num_engines} "
|
| 252 |
+
f"to match total GPUs ({total_gpus}). "
|
| 253 |
+
f"(rollout_tp_size={args.rollout.rollout_tp_size} * rollout_num_engines={expected_num_engines} = {total_gpus})"
|
| 254 |
+
)
|
| 255 |
+
args.rollout.rollout_num_engines = expected_num_engines
|
| 256 |
+
|
| 257 |
+
if args.data.max_len < args.data.prompt_max_len + args.rollout.generate_max_len:
|
| 258 |
+
args.data.max_len = args.data.prompt_max_len + args.rollout.generate_max_len
|
| 259 |
+
logger.warning(
|
| 260 |
+
"--max_len is smaller than --prompt_max_len + --generate_max_len. "
|
| 261 |
+
f"Automatically increase --max_len to {args.data.max_len}."
|
| 262 |
+
)
|
| 263 |
+
|
| 264 |
+
has_teacher = args.model.teacher_name_or_path is not None
|
| 265 |
+
is_kd_scenario = scenario in ("off_policy_kd", "on_policy_kd")
|
| 266 |
+
if is_kd_scenario:
|
| 267 |
+
if not has_teacher:
|
| 268 |
+
raise ValueError(
|
| 269 |
+
"KD training requires `--teacher_name_or_path` to be set."
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
teacher_parallel = args.kd.teacher_tp_size * args.kd.teacher_pp_size
|
| 273 |
+
if total_gpus % teacher_parallel != 0:
|
| 274 |
+
raise ValueError(
|
| 275 |
+
f"Total GPUs ({total_gpus}) must be divisible by "
|
| 276 |
+
f"teacher_tp_size * teacher_pp_size ({args.kd.teacher_tp_size} * {args.kd.teacher_pp_size} = {teacher_parallel})."
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
if args.kd.teacher_ep_size > args.kd.teacher_tp_size:
|
| 280 |
+
logger.warning(
|
| 281 |
+
f"teacher_ep_size ({args.kd.teacher_ep_size}) must not be greater than teacher_tp_size ({args.kd.teacher_tp_size}). "
|
| 282 |
+
f"Auto-adjusting teacher_ep_size to {args.kd.teacher_tp_size}."
|
| 283 |
+
)
|
| 284 |
+
args.kd.teacher_ep_size = args.kd.teacher_tp_size
|
| 285 |
+
|
| 286 |
+
expected_dp = total_gpus // teacher_parallel
|
| 287 |
+
if args.kd.teacher_dp_size != expected_dp:
|
| 288 |
+
logger.warning(
|
| 289 |
+
f"Auto-adjusting teacher_dp_size from {args.kd.teacher_dp_size} to {expected_dp} "
|
| 290 |
+
f"to match total GPUs ({total_gpus}). "
|
| 291 |
+
f"(tp={args.kd.teacher_tp_size} (ep={args.kd.teacher_ep_size}) * pp={args.kd.teacher_pp_size} * dp={expected_dp} = {total_gpus})"
|
| 292 |
+
)
|
| 293 |
+
args.kd.teacher_dp_size = expected_dp
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
deprecated_sleep_flags = [
|
| 297 |
+
args.train.train_enable_sleep,
|
| 298 |
+
args.kd.teacher_enable_sleep,
|
| 299 |
+
args.rollout.rollout_enable_sleep,
|
| 300 |
+
]
|
| 301 |
+
if any(deprecated_sleep_flags):
|
| 302 |
+
logger.warning(
|
| 303 |
+
"--train_enable_sleep, --teacher_enable_sleep, --rollout_enable_sleep are deprecated "
|
| 304 |
+
"and will be removed in a future version. Use --enable_sleep instead."
|
| 305 |
+
)
|
| 306 |
+
args.train.enable_sleep = True
|
| 307 |
+
|
| 308 |
+
args.scenario = scenario
|
| 309 |
+
return args
|
kdflow/arguments/data_args.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class DataArguments:
|
| 7 |
+
""" Arguments for dataset."""
|
| 8 |
+
|
| 9 |
+
train_dataset_path: str = field(
|
| 10 |
+
default=None,
|
| 11 |
+
metadata={"help": "Training dataset name or path."}
|
| 12 |
+
)
|
| 13 |
+
train_dataset_probs: str = field(
|
| 14 |
+
default=None,
|
| 15 |
+
metadata={"help": "Sampling probs for multiple datasets."}
|
| 16 |
+
)
|
| 17 |
+
train_split: str = field(
|
| 18 |
+
default="train",
|
| 19 |
+
metadata={"help": "Train split in dataset."}
|
| 20 |
+
)
|
| 21 |
+
eval_dataset_path: str = field(
|
| 22 |
+
default=None,
|
| 23 |
+
metadata={"help": "Evaluation dataset name or path."}
|
| 24 |
+
)
|
| 25 |
+
eval_split: str = field(
|
| 26 |
+
default="eval",
|
| 27 |
+
metadata={"help": "Eval split in dataset."}
|
| 28 |
+
)
|
| 29 |
+
custom_eval_fn: str = field(
|
| 30 |
+
default=None,
|
| 31 |
+
metadata={
|
| 32 |
+
"help": "Optional Python file defining eval_fn(predictions, labels), "
|
| 33 |
+
"which must return a metric dict (only supported in on-policy KD)."
|
| 34 |
+
}
|
| 35 |
+
)
|
| 36 |
+
eval_math_dataset_path: str = field(
|
| 37 |
+
default=None,
|
| 38 |
+
metadata={
|
| 39 |
+
"help": "Optional math evaluation JSONL path. Each row must contain "
|
| 40 |
+
"'problem' and 'answer' fields (only supported in on-policy KD)."
|
| 41 |
+
}
|
| 42 |
+
)
|
| 43 |
+
eval_math_n_samples: int = field(
|
| 44 |
+
default=1,
|
| 45 |
+
metadata={"help": "Number of generated answers per math problem for pass@k evaluation."}
|
| 46 |
+
)
|
| 47 |
+
eval_math_max_concurrent: int = field(
|
| 48 |
+
default=256,
|
| 49 |
+
metadata={"help": "Maximum number of concurrent math evaluation requests."}
|
| 50 |
+
)
|
| 51 |
+
eval_math_max_tokens: int = field(
|
| 52 |
+
default=8192,
|
| 53 |
+
metadata={"help": "Maximum number of new tokens generated for math evaluation."}
|
| 54 |
+
)
|
| 55 |
+
eval_math_timeout: int = field(
|
| 56 |
+
default=1800,
|
| 57 |
+
metadata={"help": "Total timeout in seconds for math evaluation generation."}
|
| 58 |
+
)
|
| 59 |
+
input_key: str = field(
|
| 60 |
+
default="messages",
|
| 61 |
+
metadata={"help": "JSON dataset key."}
|
| 62 |
+
)
|
| 63 |
+
image_key: str = field(
|
| 64 |
+
default=None,
|
| 65 |
+
metadata={"help": "Image key in dataset."}
|
| 66 |
+
)
|
| 67 |
+
output_key: str = field(
|
| 68 |
+
default=None,
|
| 69 |
+
metadata={"help": "Output key in dataset."}
|
| 70 |
+
)
|
| 71 |
+
teacher_input_key: str = field(
|
| 72 |
+
default=None,
|
| 73 |
+
metadata={"help": "Input key for teacher prompt. If None, use the same input_key as student. "
|
| 74 |
+
"Used in self-distillation where teacher receives richer information."}
|
| 75 |
+
)
|
| 76 |
+
label_key: str = field(
|
| 77 |
+
default=None,
|
| 78 |
+
metadata={"help": "Label key in dataset."}
|
| 79 |
+
)
|
| 80 |
+
teacher_routing_key: str = field(
|
| 81 |
+
default="teacher_routing_key",
|
| 82 |
+
metadata={"help": "Route key for teacher model in multi-teacher distillation."}
|
| 83 |
+
)
|
| 84 |
+
input_template: str = field(
|
| 85 |
+
default=None,
|
| 86 |
+
)
|
| 87 |
+
apply_chat_template: bool = field(
|
| 88 |
+
default=True,
|
| 89 |
+
metadata={"help": "Use HF tokenizer chat template."}
|
| 90 |
+
)
|
| 91 |
+
max_len: int = field(
|
| 92 |
+
default=4096
|
| 93 |
+
)
|
| 94 |
+
max_samples: int = field(
|
| 95 |
+
default=1e8
|
| 96 |
+
)
|
| 97 |
+
packing_samples: bool = field(
|
| 98 |
+
default=False,
|
| 99 |
+
metadata={"help": "Packing sequences during training."}
|
| 100 |
+
)
|
| 101 |
+
prompt_max_len: int = field(
|
| 102 |
+
default=2048,
|
| 103 |
+
metadata={"help": "Max prompt length."}
|
| 104 |
+
)
|
| 105 |
+
preprocess_num_workers: int = field(
|
| 106 |
+
default=1,
|
| 107 |
+
)
|
kdflow/arguments/distillation_args.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
from dataclasses import dataclass, field
|
| 3 |
+
|
| 4 |
+
logger = logging.getLogger(__name__)
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass
|
| 8 |
+
class DistillationArguments:
|
| 9 |
+
""" Arguments for knowledge distillation."""
|
| 10 |
+
|
| 11 |
+
kd_ratio: float = field(
|
| 12 |
+
default=0.5,
|
| 13 |
+
metadata={"help": "Loss = (1 - kd_ratio) * nll_loss + kd_ratio * kd_loss."}
|
| 14 |
+
)
|
| 15 |
+
kd_temperature: float = field(
|
| 16 |
+
default=1.0,
|
| 17 |
+
metadata={"help": "Temperature for knowledge distillation."}
|
| 18 |
+
)
|
| 19 |
+
kd_algorithm: str = field(
|
| 20 |
+
default="byte_align_ctkd",
|
| 21 |
+
metadata={
|
| 22 |
+
"help": "KD algorithm for each training step.",
|
| 23 |
+
"choices": ["simple_ctkd", "byte_align_ctkd", "topk_byte_align_ctkd"],
|
| 24 |
+
}
|
| 25 |
+
)
|
| 26 |
+
kd_loss_fn: str = field(
|
| 27 |
+
default="kl",
|
| 28 |
+
metadata={"help": "Divergence selection for knowledge distillation, e.g., kl, rkl, js."}
|
| 29 |
+
)
|
| 30 |
+
ctkd_topk: int = field(
|
| 31 |
+
default=128,
|
| 32 |
+
metadata={
|
| 33 |
+
"help": (
|
| 34 |
+
"Student top-k size within the shared vocabulary for "
|
| 35 |
+
"topk_byte_align_ctkd."
|
| 36 |
+
)
|
| 37 |
+
},
|
| 38 |
+
)
|
| 39 |
+
teacher_forward_n_batches: int = field(
|
| 40 |
+
default=1,
|
| 41 |
+
metadata={"help": "Teacher forward N global batches at once for student multi-step training."}
|
| 42 |
+
)
|
| 43 |
+
teacher_enable_sleep: bool = field(
|
| 44 |
+
default=False,
|
| 45 |
+
metadata={"help": "Sleep teacher when not needed."}
|
| 46 |
+
)
|
| 47 |
+
teacher_offload_tags: str = field(
|
| 48 |
+
default="all",
|
| 49 |
+
metadata={"help": "Offload tags for sglang."}
|
| 50 |
+
)
|
| 51 |
+
teacher_quantization: str = field(
|
| 52 |
+
default=None
|
| 53 |
+
)
|
| 54 |
+
teacher_tp_size: int = field(
|
| 55 |
+
default=8,
|
| 56 |
+
metadata={"help": "Tensor parallel size for teacher model."}
|
| 57 |
+
)
|
| 58 |
+
teacher_ep_size: int = field(
|
| 59 |
+
default=1,
|
| 60 |
+
metadata={"help": "Expert parallel size for teacher model (only for MoE models)."}
|
| 61 |
+
)
|
| 62 |
+
teacher_pp_size: int = field(
|
| 63 |
+
default=1,
|
| 64 |
+
metadata={"help": "Pipeline parallel size for teacher model."}
|
| 65 |
+
)
|
| 66 |
+
teacher_dp_size: int = field(
|
| 67 |
+
default=1,
|
| 68 |
+
metadata={"help": "Data parallel size for teacher model."}
|
| 69 |
+
)
|
| 70 |
+
teacher_mem_fraction_static: float = field(
|
| 71 |
+
default=0.4,
|
| 72 |
+
metadata={"help": "Memory fraction for teacher model."}
|
| 73 |
+
)
|
| 74 |
+
teacher_update_freq: int = field(
|
| 75 |
+
default=10,
|
| 76 |
+
metadata={"help": "Weight update frequency for teacher model."}
|
| 77 |
+
)
|
| 78 |
+
use_ema_teacher: bool = field(
|
| 79 |
+
default=False,
|
| 80 |
+
metadata={"help": "Use EMA update for teacher model."}
|
| 81 |
+
)
|
| 82 |
+
teacher_ema_decay: float = field(
|
| 83 |
+
default=0.999,
|
| 84 |
+
metadata={"help": "EMA decay for teacher model."}
|
| 85 |
+
)
|
| 86 |
+
multi_teacher_config: str = field(
|
| 87 |
+
default=None,
|
| 88 |
+
metadata={"help": "Unsupported in this release; use --teacher_name_or_path."},
|
| 89 |
+
)
|
| 90 |
+
# JSD
|
| 91 |
+
jsd_beta: float = field(
|
| 92 |
+
default=0.5,
|
| 93 |
+
metadata={"help": "Beta for Jensen-Shannon Divergence."}
|
| 94 |
+
)
|
| 95 |
+
# Skewed KL/RKL
|
| 96 |
+
skew_lambda: float = field(
|
| 97 |
+
default=0.1,
|
| 98 |
+
metadata={"help": "Lambda for Skewed KL/RKL."}
|
| 99 |
+
)
|
| 100 |
+
# Adaptive KL
|
| 101 |
+
adaptive_alpha: float = field(
|
| 102 |
+
default=0.5,
|
| 103 |
+
metadata={"help": "Alpha for Adaptive KL Divergence."}
|
| 104 |
+
)
|
| 105 |
+
# Hierarchical Ranking Loss
|
| 106 |
+
hrl_topk: int = field(
|
| 107 |
+
default=5,
|
| 108 |
+
metadata={"help": "Top-k Ranking for Hierarchical Ranking Loss."}
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
def __post_init__(self):
|
| 112 |
+
if self.multi_teacher_config is not None:
|
| 113 |
+
raise ValueError(
|
| 114 |
+
"This release supports one teacher model per run; "
|
| 115 |
+
"use --teacher_name_or_path instead of --multi_teacher_config."
|
| 116 |
+
)
|
| 117 |
+
supported_algorithms = {"simple_ctkd", "byte_align_ctkd", "topk_byte_align_ctkd"}
|
| 118 |
+
if self.kd_algorithm not in supported_algorithms:
|
| 119 |
+
raise ValueError(
|
| 120 |
+
f"kd_algorithm must be one of {sorted(supported_algorithms)}, "
|
| 121 |
+
f"got {self.kd_algorithm!r}."
|
| 122 |
+
)
|
| 123 |
+
# Validate teacher parallel size settings
|
| 124 |
+
if self.teacher_ep_size > self.teacher_tp_size:
|
| 125 |
+
raise ValueError(
|
| 126 |
+
f"SGLang requires that teacher_ep_size ({self.teacher_ep_size}) must be <= teacher_tp_size ({self.teacher_tp_size}). "
|
| 127 |
+
)
|
| 128 |
+
if self.teacher_tp_size % self.teacher_ep_size != 0:
|
| 129 |
+
raise ValueError(
|
| 130 |
+
f"SGLang requires that teacher_tp_size ({self.teacher_tp_size}) must be divisible by teacher_ep_size ({self.teacher_ep_size})."
|
| 131 |
+
)
|
| 132 |
+
# Validate KD hyperparameters
|
| 133 |
+
if not 0.0 <= self.kd_ratio <= 1.0:
|
| 134 |
+
raise ValueError(f"kd_ratio must be in [0, 1], got {self.kd_ratio}.")
|
| 135 |
+
if self.kd_temperature <= 0:
|
| 136 |
+
raise ValueError(f"kd_temperature must be > 0, got {self.kd_temperature}.")
|
| 137 |
+
if self.ctkd_topk < 2:
|
| 138 |
+
raise ValueError(
|
| 139 |
+
f"ctkd_topk must be at least 2, got {self.ctkd_topk}."
|
| 140 |
+
)
|
| 141 |
+
if not 0.0 < self.teacher_mem_fraction_static <= 1.0:
|
| 142 |
+
raise ValueError(f"teacher_mem_fraction_static must be in (0, 1], got {self.teacher_mem_fraction_static}.")
|
kdflow/arguments/fsdp_args.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class FSDPArguments:
|
| 7 |
+
""" Arguments for fsdp backend."""
|
| 8 |
+
|
| 9 |
+
fsdp_size: int = field(
|
| 10 |
+
default=-1,
|
| 11 |
+
metadata={"help": "FSDP shard size (for HSDP)."}
|
| 12 |
+
)
|
| 13 |
+
cpu_offload: bool = field(
|
| 14 |
+
default=False,
|
| 15 |
+
metadata={"help": "Offload Adam optimizer to GPU."}
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
kdflow/arguments/logging_args.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class LoggingArguments:
|
| 7 |
+
""" Arguments for logging (e.g., wandb and tensorboard)."""
|
| 8 |
+
|
| 9 |
+
logging_steps: int = field(
|
| 10 |
+
default=10,
|
| 11 |
+
metadata={"help": "Log results every n steps."}
|
| 12 |
+
)
|
| 13 |
+
use_wandb: bool = field(
|
| 14 |
+
default=False,
|
| 15 |
+
metadata={"help": "Use wandb for logging."}
|
| 16 |
+
)
|
| 17 |
+
wandb_org: str = field(
|
| 18 |
+
default=None
|
| 19 |
+
)
|
| 20 |
+
wandb_project: str = field(
|
| 21 |
+
default=None
|
| 22 |
+
)
|
| 23 |
+
wandb_group: str = field(
|
| 24 |
+
default=None
|
| 25 |
+
)
|
| 26 |
+
wandb_run_name: str = field(
|
| 27 |
+
default=None
|
| 28 |
+
)
|
| 29 |
+
wandb_mode: str = field(
|
| 30 |
+
default="online",
|
| 31 |
+
metadata={"help": "wandb mode: online, offline, or disabled."}
|
| 32 |
+
)
|
| 33 |
+
wandb_dir: str = field(
|
| 34 |
+
default=None,
|
| 35 |
+
metadata={"help": "Directory to store wandb offline logs."}
|
| 36 |
+
)
|
| 37 |
+
|
kdflow/arguments/model_args.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class ModelArguments:
|
| 7 |
+
""" Arguments for model."""
|
| 8 |
+
|
| 9 |
+
student_name_or_path: str = field(
|
| 10 |
+
default=None,
|
| 11 |
+
metadata={"help": "Student model name or path."}
|
| 12 |
+
)
|
| 13 |
+
teacher_name_or_path: str = field(
|
| 14 |
+
default=None,
|
| 15 |
+
metadata={"help": "Teacher model name or path."}
|
| 16 |
+
)
|
| 17 |
+
attn_implementation: str = field(
|
| 18 |
+
default="flash_attention_2",
|
| 19 |
+
metadata={"help": "Attention implementation (e.g., eager, flash_attention_2, flash_attention_3, kernels-community/vllm-flash-attn3)."}
|
| 20 |
+
)
|
| 21 |
+
use_liger_kernel: bool = field(
|
| 22 |
+
default=False,
|
| 23 |
+
metadata={"help": "Use Liger Kernel in LLM."}
|
| 24 |
+
)
|
| 25 |
+
lora_rank: int = field(
|
| 26 |
+
default=0
|
| 27 |
+
)
|
| 28 |
+
lora_alpha: int = field(
|
| 29 |
+
default=16
|
| 30 |
+
)
|
| 31 |
+
target_modules: str = field(
|
| 32 |
+
default="all-linear",
|
| 33 |
+
)
|
| 34 |
+
lora_dropout: float = field(
|
| 35 |
+
default=0.0,
|
| 36 |
+
)
|
| 37 |
+
disable_fast_tokenizer: bool = field(
|
| 38 |
+
default=False
|
| 39 |
+
)
|
| 40 |
+
enable_thinking: bool = field(
|
| 41 |
+
default=False,
|
| 42 |
+
)
|
| 43 |
+
ring_attn_size: int = field(
|
| 44 |
+
default=1
|
| 45 |
+
)
|
kdflow/arguments/rollout_args.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class RolloutArguments:
|
| 7 |
+
""" Arguments for rollout (on-policy distillation)."""
|
| 8 |
+
|
| 9 |
+
rollout_num_engines: int = field(
|
| 10 |
+
default=0,
|
| 11 |
+
metadata={"help": "The number of engines for rollout."}
|
| 12 |
+
)
|
| 13 |
+
rollout_tp_size: int = field(
|
| 14 |
+
default=1,
|
| 15 |
+
metadata={"help": "Tensor parallel size for each rollout engine."}
|
| 16 |
+
)
|
| 17 |
+
rollout_engine_concurrency: int = field(
|
| 18 |
+
default=512,
|
| 19 |
+
metadata={"help": "Maximum concurrent requests per rollout engine."}
|
| 20 |
+
)
|
| 21 |
+
rollout_enable_sleep: bool = field(
|
| 22 |
+
default=False,
|
| 23 |
+
metadata={"help": "Enable sleep mode for rollout engines."}
|
| 24 |
+
)
|
| 25 |
+
rollout_mem_fraction_static: float = field(
|
| 26 |
+
default=0.6,
|
| 27 |
+
metadata={"help": "GPU memory utilization for each rollout engine."}
|
| 28 |
+
)
|
| 29 |
+
top_p: float = field(
|
| 30 |
+
default=1.0,
|
| 31 |
+
metadata={"help": "Top-p sampling for rollout."}
|
| 32 |
+
)
|
| 33 |
+
temperature: float = field(
|
| 34 |
+
default=1.0,
|
| 35 |
+
metadata={"help": "Temperature for rollout."}
|
| 36 |
+
)
|
| 37 |
+
n_samples_per_prompt: int = field(
|
| 38 |
+
default=1,
|
| 39 |
+
metadata={"help": "Sample n responses per prompt."}
|
| 40 |
+
)
|
| 41 |
+
rollout_batch_size: int = field(
|
| 42 |
+
default=32,
|
| 43 |
+
metadata={"help": "Number of prompts for each rollout."}
|
| 44 |
+
)
|
| 45 |
+
generate_max_len: int = field(
|
| 46 |
+
default=2048,
|
| 47 |
+
metadata={"help": "Max generation tokens during rollout."}
|
| 48 |
+
)
|
| 49 |
+
print_rollout_sample: bool = field(
|
| 50 |
+
default=False,
|
| 51 |
+
metadata={"help": "Whether to print a rollout sample after each rollout."}
|
| 52 |
+
)
|
| 53 |
+
alfworld_config: Optional[str] = field(
|
| 54 |
+
default=None,
|
| 55 |
+
metadata={
|
| 56 |
+
"help": "Path to an ALFWorld YAML config. When set, on-policy KD "
|
| 57 |
+
"collects multi-turn student trajectories from AlfredTWEnv."
|
| 58 |
+
},
|
| 59 |
+
)
|
| 60 |
+
alfworld_protocol: str = field(
|
| 61 |
+
default="kdflow",
|
| 62 |
+
metadata={
|
| 63 |
+
"help": "ALFWorld training rollout protocol: 'kdflow' keeps the "
|
| 64 |
+
"original Thought/Action chat history, while 'verl_agent' "
|
| 65 |
+
"uses the migrated single-prompt <think>/<action> protocol."
|
| 66 |
+
},
|
| 67 |
+
)
|
| 68 |
+
alfworld_split: str = field(
|
| 69 |
+
default="train",
|
| 70 |
+
metadata={
|
| 71 |
+
"help": "ALFWorld split: train, eval_in_distribution, or "
|
| 72 |
+
"eval_out_of_distribution."
|
| 73 |
+
},
|
| 74 |
+
)
|
| 75 |
+
alfworld_max_turns: int = field(
|
| 76 |
+
default=30,
|
| 77 |
+
metadata={"help": "Maximum assistant/environment turns per ALFWorld episode."},
|
| 78 |
+
)
|
| 79 |
+
alfworld_episodes_per_epoch: int = field(
|
| 80 |
+
default=1024,
|
| 81 |
+
metadata={"help": "Number of ALFWorld episodes sampled per training epoch."},
|
| 82 |
+
)
|
| 83 |
+
alfworld_eval_episodes: int = field(
|
| 84 |
+
default=128,
|
| 85 |
+
metadata={
|
| 86 |
+
"help": "Number of episodes for standalone ALFWorld evaluation."
|
| 87 |
+
},
|
| 88 |
+
)
|
| 89 |
+
alfworld_eval_protocol: str = field(
|
| 90 |
+
default="kdflow",
|
| 91 |
+
metadata={
|
| 92 |
+
"help": "Standalone ALFWorld evaluation protocol. Use 'kdflow' "
|
| 93 |
+
"for the original KDFlow prompt or 'verl_agent' for the "
|
| 94 |
+
"protocol migrated from langfengq/verl-agent."
|
| 95 |
+
},
|
| 96 |
+
)
|
| 97 |
+
alfworld_eval_seeds: str = field(
|
| 98 |
+
default="123,456,789",
|
| 99 |
+
metadata={
|
| 100 |
+
"help": "Comma-separated script-level seeds used by the "
|
| 101 |
+
"verl-agent-compatible evaluator."
|
| 102 |
+
},
|
| 103 |
+
)
|
| 104 |
+
alfworld_eval_seed_offset: int = field(
|
| 105 |
+
default=1000,
|
| 106 |
+
metadata={
|
| 107 |
+
"help": "Offset added to each script-level seed for the validation "
|
| 108 |
+
"environment. verl-agent uses 1000."
|
| 109 |
+
},
|
| 110 |
+
)
|
| 111 |
+
alfworld_eval_resume: bool = field(
|
| 112 |
+
default=True,
|
| 113 |
+
metadata={
|
| 114 |
+
"help": "Reuse completed, configuration-compatible per-seed "
|
| 115 |
+
"ALFWorld evaluation results. Set False to force rerunning "
|
| 116 |
+
"every seed."
|
| 117 |
+
},
|
| 118 |
+
)
|
| 119 |
+
alfworld_history_length: int = field(
|
| 120 |
+
default=4,
|
| 121 |
+
metadata={
|
| 122 |
+
"help": "Number of recent observation/action pairs included by the "
|
| 123 |
+
"verl-agent ALFWorld prompt."
|
| 124 |
+
},
|
| 125 |
+
)
|
| 126 |
+
alfworld_admissible_limit: int = field(
|
| 127 |
+
default=30,
|
| 128 |
+
metadata={
|
| 129 |
+
"help": "Maximum number of ALFWorld admissible actions shown per turn; "
|
| 130 |
+
"use 0 to show all actions."
|
| 131 |
+
},
|
| 132 |
+
)
|
kdflow/arguments/training_args.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import math
|
| 3 |
+
from dataclasses import dataclass, field
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def resolve_max_steps(
|
| 8 |
+
configured_max_steps: int,
|
| 9 |
+
num_epochs: int,
|
| 10 |
+
steps_per_epoch: int,
|
| 11 |
+
) -> int:
|
| 12 |
+
"""Resolve the explicit step limit or the epoch-derived training length."""
|
| 13 |
+
if steps_per_epoch <= 0:
|
| 14 |
+
raise ValueError(
|
| 15 |
+
"Training requires at least one full global step per epoch. "
|
| 16 |
+
"Reduce the global batch size or provide more training samples."
|
| 17 |
+
)
|
| 18 |
+
if configured_max_steps > 0:
|
| 19 |
+
return configured_max_steps
|
| 20 |
+
return math.ceil(num_epochs * steps_per_epoch)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass
|
| 24 |
+
class TrainingArguments:
|
| 25 |
+
""" Arguments for training."""
|
| 26 |
+
|
| 27 |
+
num_nodes: int = field(
|
| 28 |
+
default=1,
|
| 29 |
+
metadata={"help": "The number of nodes for the student/teacher model (only for on-policy KD)."}
|
| 30 |
+
)
|
| 31 |
+
num_gpus_per_node: int = field(
|
| 32 |
+
default=8,
|
| 33 |
+
metadata={"help": "The number of gpus per node for the student/teacher model (only for on-policy KD)."}
|
| 34 |
+
)
|
| 35 |
+
num_epochs: int = field(
|
| 36 |
+
default=1,
|
| 37 |
+
metadata={"help": "Number of training epochs."}
|
| 38 |
+
)
|
| 39 |
+
max_steps: int = field(
|
| 40 |
+
default=-1,
|
| 41 |
+
metadata={
|
| 42 |
+
"help": (
|
| 43 |
+
"Maximum global training steps. A positive value overrides "
|
| 44 |
+
"num_epochs; for on-policy KD, one global step is one rollout "
|
| 45 |
+
"iteration."
|
| 46 |
+
)
|
| 47 |
+
},
|
| 48 |
+
)
|
| 49 |
+
train_batch_size: int = field(
|
| 50 |
+
default=128,
|
| 51 |
+
metadata={"help": "Global training batch size."}
|
| 52 |
+
)
|
| 53 |
+
micro_train_batch_size: int = field(
|
| 54 |
+
default=1,
|
| 55 |
+
metadata={"help": "Micro training batch size per GPU."}
|
| 56 |
+
)
|
| 57 |
+
learning_rate: float = field(
|
| 58 |
+
default=1e-6
|
| 59 |
+
)
|
| 60 |
+
lr_warmup_ratio: float = field(
|
| 61 |
+
default=0.05
|
| 62 |
+
)
|
| 63 |
+
min_lr: float = field(
|
| 64 |
+
default=1e-8
|
| 65 |
+
)
|
| 66 |
+
lr_scheduler: str = field(
|
| 67 |
+
default="cosine_with_min_lr"
|
| 68 |
+
)
|
| 69 |
+
max_norm: float = field(
|
| 70 |
+
default=1.0,
|
| 71 |
+
metadata={"help": "Gradient clipping."}
|
| 72 |
+
)
|
| 73 |
+
weight_decay: float = field(
|
| 74 |
+
default=0.0,
|
| 75 |
+
metadata={"help": "Weight decay."}
|
| 76 |
+
)
|
| 77 |
+
adam_betas: str = field(
|
| 78 |
+
default="(0.9, 0.98)"
|
| 79 |
+
)
|
| 80 |
+
eval_steps: int = field(
|
| 81 |
+
default=-1,
|
| 82 |
+
metadata={"help": "Evaluation every n step."}
|
| 83 |
+
)
|
| 84 |
+
save_steps: int = field(
|
| 85 |
+
default=-1,
|
| 86 |
+
metadata={"help": "Save checkpoints every n step."}
|
| 87 |
+
)
|
| 88 |
+
backend: str = field(
|
| 89 |
+
default="fsdp2",
|
| 90 |
+
metadata={
|
| 91 |
+
"choices": ["fsdp2"],
|
| 92 |
+
"help": "Student training backend."
|
| 93 |
+
}
|
| 94 |
+
)
|
| 95 |
+
gradient_checkpointing: bool = field(
|
| 96 |
+
default=False
|
| 97 |
+
)
|
| 98 |
+
gradient_checkpointing_use_reentrant: bool = field(
|
| 99 |
+
default=False
|
| 100 |
+
)
|
| 101 |
+
train_enable_sleep: bool = field(
|
| 102 |
+
default=False,
|
| 103 |
+
metadata={"help": "[Deprecated] Use --enable_sleep instead."}
|
| 104 |
+
)
|
| 105 |
+
enable_sleep: bool = field(
|
| 106 |
+
default=False,
|
| 107 |
+
metadata={"help": "Enable sleep mode for all components (student, teacher, rollout)."}
|
| 108 |
+
)
|
| 109 |
+
full_determinism: bool = field(
|
| 110 |
+
default=False,
|
| 111 |
+
metadata={"help": "Enable reproducible behavior during distributed training."}
|
| 112 |
+
)
|
| 113 |
+
load_checkpoint: bool = field(
|
| 114 |
+
default=False
|
| 115 |
+
)
|
| 116 |
+
ckpt_path: str = field(
|
| 117 |
+
default="./ckpt/checkpoints_distill"
|
| 118 |
+
)
|
| 119 |
+
save_path: str = field(
|
| 120 |
+
default="./ckpt/"
|
| 121 |
+
)
|
| 122 |
+
seed: int = field(
|
| 123 |
+
default=42,
|
| 124 |
+
)
|
| 125 |
+
use_dynamic_bsz: bool = field(
|
| 126 |
+
default=False,
|
| 127 |
+
metadata={"help": "Enable dynamic batch size based on token count per GPU."}
|
| 128 |
+
)
|
| 129 |
+
max_token_len_per_gpu: int = field(
|
| 130 |
+
default=0,
|
| 131 |
+
metadata={"help": "Maximum total token count per micro-batch when use_dynamic_bsz is True."}
|
| 132 |
+
)
|
| 133 |
+
bf16: bool = field(
|
| 134 |
+
default=False
|
| 135 |
+
)
|
| 136 |
+
local_rank: int = field(
|
| 137 |
+
default=int(os.getenv("LOCAL_RANK", -1)),
|
| 138 |
+
metadata={"help": "Local rank for distributed training. Automatically set."}
|
| 139 |
+
)
|
| 140 |
+
chunked_loss_size: int = field(
|
| 141 |
+
default=None,
|
| 142 |
+
metadata={"help": "Token chunk size for chunked loss. If None, chunked loss is disabled."}
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
def __post_init__(self):
|
| 146 |
+
# Ray actors run from a temporary working directory. Resolve output paths
|
| 147 |
+
# in the driver before the arguments are serialized and sent to actors so
|
| 148 |
+
# relative paths keep their command-invocation semantics.
|
| 149 |
+
self.save_path = self._resolve_driver_path(self.save_path)
|
| 150 |
+
self.ckpt_path = self._resolve_driver_path(self.ckpt_path)
|
| 151 |
+
|
| 152 |
+
backend_choices = self.__class__.__dataclass_fields__["backend"].metadata["choices"]
|
| 153 |
+
if self.backend not in backend_choices:
|
| 154 |
+
raise ValueError(
|
| 155 |
+
f"Unsupported training backend '{self.backend}'"
|
| 156 |
+
f"Supported choices: {backend_choices}"
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
if isinstance(self.adam_betas, str):
|
| 160 |
+
try:
|
| 161 |
+
cleaned_str = self.adam_betas.strip().strip('()')
|
| 162 |
+
parts = [p.strip() for p in cleaned_str.split(',')]
|
| 163 |
+
if len(parts) != 2:
|
| 164 |
+
raise ValueError("adam_betas must contain two values.")
|
| 165 |
+
|
| 166 |
+
self.adam_betas = (float(parts[0]), float(parts[1]))
|
| 167 |
+
except Exception as e:
|
| 168 |
+
raise ValueError(
|
| 169 |
+
f"Cannot parse '{self.adam_betas}' to valid adam_betas. Make sure the format to be '(beta1, beta2)'。"
|
| 170 |
+
) from e
|
| 171 |
+
else:
|
| 172 |
+
raise TypeError(f"Expected str for adam_betas, but get {type(self.adam_betas)}")
|
| 173 |
+
|
| 174 |
+
if self.save_steps <= 0:
|
| 175 |
+
self.save_steps = float("inf")
|
| 176 |
+
|
| 177 |
+
if self.eval_steps <= 0:
|
| 178 |
+
self.eval_steps = float("inf")
|
| 179 |
+
|
| 180 |
+
@staticmethod
|
| 181 |
+
def _resolve_driver_path(path: str) -> str:
|
| 182 |
+
return os.path.abspath(os.path.expanduser(path))
|
kdflow/backend/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from kdflow.backend.fsdp import FSDP2Strategy
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def get_strategy(args):
|
| 5 |
+
if args.train.backend == "fsdp2":
|
| 6 |
+
strategy = FSDP2Strategy(
|
| 7 |
+
seed=getattr(args.train, "seed", 42),
|
| 8 |
+
full_determinism=getattr(args.train, "full_determinism", False),
|
| 9 |
+
max_norm=getattr(args.train, "max_norm", 1.0),
|
| 10 |
+
micro_train_batch_size=getattr(args.train, "micro_train_batch_size", 1),
|
| 11 |
+
train_batch_size=getattr(args.train, "train_batch_size", 128),
|
| 12 |
+
bf16=getattr(args.train, "bf16", True),
|
| 13 |
+
args=args,
|
| 14 |
+
)
|
| 15 |
+
else:
|
| 16 |
+
raise NotImplementedError
|
| 17 |
+
|
| 18 |
+
return strategy
|
kdflow/backend/fsdp/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .fsdp_strategy import FSDP2Strategy
|
| 2 |
+
|
| 3 |
+
__all__ = [
|
| 4 |
+
"FSDP2Strategy",
|
| 5 |
+
]
|
kdflow/backend/fsdp/fsdp_strategy.py
ADDED
|
@@ -0,0 +1,704 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import gc
|
| 3 |
+
import functools
|
| 4 |
+
import logging
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
import torch.distributed as dist
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
import transformers
|
| 10 |
+
from abc import ABC
|
| 11 |
+
from datetime import timedelta
|
| 12 |
+
from peft import PeftModel, get_peft_model_state_dict
|
| 13 |
+
from torch.distributed.device_mesh import init_device_mesh, DeviceMesh
|
| 14 |
+
from torch.optim import Optimizer, AdamW
|
| 15 |
+
from torch.distributed.fsdp import (
|
| 16 |
+
fully_shard,
|
| 17 |
+
MixedPrecisionPolicy,
|
| 18 |
+
CPUOffloadPolicy,
|
| 19 |
+
)
|
| 20 |
+
from torch.distributed.tensor import DTensor
|
| 21 |
+
from torch.distributed.checkpoint.state_dict import (
|
| 22 |
+
get_model_state_dict,
|
| 23 |
+
set_model_state_dict,
|
| 24 |
+
StateDictOptions,
|
| 25 |
+
)
|
| 26 |
+
from torchdata.stateful_dataloader import StatefulDataLoader
|
| 27 |
+
|
| 28 |
+
from kdflow.utils.logging_utils import init_logger
|
| 29 |
+
from kdflow.models import DistillModel
|
| 30 |
+
from kdflow.utils.distributed_sampler import DistributedSampler
|
| 31 |
+
from kdflow.models.ring_attn_utils import get_ring_attn_group, set_ring_attn_group
|
| 32 |
+
from kdflow.utils.distributed_util import torch_dist_barrier_and_cuda_sync
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class FSDP2Strategy(ABC):
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
seed: int = 42,
|
| 39 |
+
full_determinism: bool = False,
|
| 40 |
+
max_norm: float = 0.0,
|
| 41 |
+
micro_train_batch_size=1,
|
| 42 |
+
train_batch_size=1,
|
| 43 |
+
bf16=True,
|
| 44 |
+
cpu_offload: bool = False,
|
| 45 |
+
args=None,
|
| 46 |
+
) -> None:
|
| 47 |
+
self.args = args
|
| 48 |
+
self.seed = seed
|
| 49 |
+
self.max_norm = max_norm
|
| 50 |
+
self.micro_train_batch_size = micro_train_batch_size
|
| 51 |
+
self.train_batch_size = train_batch_size
|
| 52 |
+
self.cpu_offload = cpu_offload
|
| 53 |
+
self.bf16 = bf16
|
| 54 |
+
self.full_determinism = full_determinism
|
| 55 |
+
self.logger = init_logger("FSDP2Strategy")
|
| 56 |
+
|
| 57 |
+
def log(self, msg: str, level: str = "info", rank_0_only: bool = True) -> None:
|
| 58 |
+
"""
|
| 59 |
+
Log a message with rank information.
|
| 60 |
+
|
| 61 |
+
Args:
|
| 62 |
+
msg: The message to log.
|
| 63 |
+
level: Log level ('debug', 'info', 'warning', 'error', 'critical').
|
| 64 |
+
rank_0_only: If True, only log on rank 0.
|
| 65 |
+
"""
|
| 66 |
+
if rank_0_only and not self.is_rank_0():
|
| 67 |
+
return
|
| 68 |
+
|
| 69 |
+
rank = self.get_rank() if dist.is_initialized() else 0
|
| 70 |
+
full_msg = f"[Rank {rank}] {msg}"
|
| 71 |
+
|
| 72 |
+
log_func = getattr(self.logger, level.lower(), self.logger.info)
|
| 73 |
+
# Use stacklevel=2 to show the caller's location instead of this function's location
|
| 74 |
+
log_func(full_msg, stacklevel=2)
|
| 75 |
+
|
| 76 |
+
def setup_distributed(self, timeout=timedelta(minutes=60)) -> None:
|
| 77 |
+
if self.full_determinism:
|
| 78 |
+
transformers.enable_full_determinism(self.seed)
|
| 79 |
+
# Use deterministic backward in flash attention as, by default, flash attention uses atomic adds
|
| 80 |
+
# https://github.com/Dao-AILab/flash-attention/commit/732654583c2e640adc012ecb60e460bf19dcd9e3
|
| 81 |
+
transformers.modeling_flash_attention_utils.deterministic_g = True
|
| 82 |
+
else:
|
| 83 |
+
transformers.set_seed(self.seed)
|
| 84 |
+
|
| 85 |
+
local_rank = int(os.environ.get("LOCAL_RANK", "-1"))
|
| 86 |
+
if local_rank != -1:
|
| 87 |
+
torch.cuda.set_device(local_rank)
|
| 88 |
+
|
| 89 |
+
if not dist.is_initialized():
|
| 90 |
+
dist.init_process_group(backend="nccl", timeout=timeout)
|
| 91 |
+
|
| 92 |
+
self.world_size = dist.get_world_size()
|
| 93 |
+
self.sp_size = self.args.model.ring_attn_size
|
| 94 |
+
dp_size = self.world_size // self.sp_size
|
| 95 |
+
self.sp_mesh = init_device_mesh("cuda", (dp_size, self.sp_size), mesh_dim_names=("dp", "sp"))
|
| 96 |
+
self.setup_ring_attn(self.sp_mesh)
|
| 97 |
+
if self.args.fsdp.fsdp_size > -1:
|
| 98 |
+
assert self.world_size % self.args.fsdp.fsdp_size == 0
|
| 99 |
+
assert self.world_size >= self.args.fsdp.fsdp_size
|
| 100 |
+
replicate = self.world_size // self.args.fsdp.fsdp_size
|
| 101 |
+
self.fsdp_mesh = init_device_mesh(
|
| 102 |
+
"cuda", (replicate, self.args.fsdp.fsdp_size), mesh_dim_names=("replicate", "sharded"))
|
| 103 |
+
else:
|
| 104 |
+
self.fsdp_mesh = init_device_mesh(
|
| 105 |
+
"cuda", (self.world_size,), mesh_dim_names=("sharded",)
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
self.step = 0
|
| 109 |
+
self.accumulated_gradient = (
|
| 110 |
+
self.train_batch_size
|
| 111 |
+
* self.sp_size
|
| 112 |
+
// self.micro_train_batch_size
|
| 113 |
+
// self.world_size
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
def setup_ring_attn(self, device_mesh):
|
| 117 |
+
if self.sp_size == 1:
|
| 118 |
+
self.ring_attn_rank = 0
|
| 119 |
+
return
|
| 120 |
+
|
| 121 |
+
group = device_mesh["sp"].get_group()
|
| 122 |
+
self.ring_attn_rank = dist.get_rank(group=group)
|
| 123 |
+
set_ring_attn_group(group)
|
| 124 |
+
|
| 125 |
+
from ring_flash_attn import substitute_hf_flash_attn
|
| 126 |
+
|
| 127 |
+
self.ring_head_stride = getattr(self.args.model, "ring_head_stride", 1)
|
| 128 |
+
substitute_hf_flash_attn(self.ring_attn_group, self.ring_head_stride)
|
| 129 |
+
|
| 130 |
+
@property
|
| 131 |
+
def ring_attn_group(self):
|
| 132 |
+
return get_ring_attn_group()
|
| 133 |
+
|
| 134 |
+
def setup_dataloader(
|
| 135 |
+
self,
|
| 136 |
+
replay_buffer,
|
| 137 |
+
batch_size: int,
|
| 138 |
+
pin_memory: bool = False,
|
| 139 |
+
shuffle=True,
|
| 140 |
+
collate_fn=None,
|
| 141 |
+
drop_last=True,
|
| 142 |
+
sampler=None,
|
| 143 |
+
consumed_samples=0,
|
| 144 |
+
):
|
| 145 |
+
# DDP only mode, replay buffers on each rank are different.
|
| 146 |
+
if sampler is None and dist.is_initialized():
|
| 147 |
+
num_replicas = dist.get_world_size() // self.sp_size
|
| 148 |
+
rank = dist.get_rank() // self.sp_size
|
| 149 |
+
|
| 150 |
+
sampler = DistributedSampler(
|
| 151 |
+
replay_buffer,
|
| 152 |
+
num_replicas=num_replicas,
|
| 153 |
+
rank=rank,
|
| 154 |
+
shuffle=shuffle,
|
| 155 |
+
seed=self.seed,
|
| 156 |
+
drop_last=drop_last,
|
| 157 |
+
consumed_samples=consumed_samples,
|
| 158 |
+
)
|
| 159 |
+
return StatefulDataLoader(
|
| 160 |
+
replay_buffer,
|
| 161 |
+
batch_size=batch_size,
|
| 162 |
+
sampler=sampler,
|
| 163 |
+
drop_last=drop_last,
|
| 164 |
+
shuffle=shuffle if sampler is None else False,
|
| 165 |
+
collate_fn=collate_fn,
|
| 166 |
+
pin_memory=pin_memory,
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
def load_hf_model(self, model_class, model_name_or_path, attn_impl, model_config):
|
| 170 |
+
init_context = self._get_init_weight_context_manager(model_config)
|
| 171 |
+
|
| 172 |
+
with init_context():
|
| 173 |
+
model = model_class.from_pretrained(
|
| 174 |
+
model_name_or_path,
|
| 175 |
+
trust_remote_code=True,
|
| 176 |
+
attn_implementation=attn_impl,
|
| 177 |
+
torch_dtype=torch.bfloat16 if self.args.train.bf16 else "auto",
|
| 178 |
+
)
|
| 179 |
+
# Patch lm_head to skippable mode so that the HF forward does not
|
| 180 |
+
# materialize full-sequence logits. The algorithm layer always calls
|
| 181 |
+
# `lm_head(hidden, skip=False)` explicitly via chunked_loss.
|
| 182 |
+
if hasattr(model, "lm_head"):
|
| 183 |
+
original_forward = model.lm_head.forward
|
| 184 |
+
def skip_forward(x, skip=True):
|
| 185 |
+
if skip:
|
| 186 |
+
return x
|
| 187 |
+
return original_forward(x)
|
| 188 |
+
model.lm_head.forward = skip_forward
|
| 189 |
+
self.log("lm_head forward set to skippable mode for student model.")
|
| 190 |
+
return model
|
| 191 |
+
|
| 192 |
+
def _get_init_weight_context_manager(self, model_config):
|
| 193 |
+
"""Get context manager for model initialization.
|
| 194 |
+
|
| 195 |
+
Returns a callable that creates a context manager.
|
| 196 |
+
Uses meta device (no memory allocation) for non-rank-0 processes,
|
| 197 |
+
UNLESS tie_word_embeddings=True (which causes hangs with meta tensors).
|
| 198 |
+
|
| 199 |
+
Ref: verl/utils/fsdp_utils.py::get_init_weight_context_manager
|
| 200 |
+
NOTE: tie_word_embedding causes meta_tensor init to hang
|
| 201 |
+
"""
|
| 202 |
+
from accelerate import init_empty_weights
|
| 203 |
+
|
| 204 |
+
# Check if model uses tied word embeddings (which doesn't work with meta tensors)
|
| 205 |
+
use_meta_tensor = not model_config.tie_word_embeddings
|
| 206 |
+
|
| 207 |
+
def cpu_init_weights():
|
| 208 |
+
return torch.device("cpu")
|
| 209 |
+
|
| 210 |
+
if use_meta_tensor:
|
| 211 |
+
# Rank 0: CPU, others: meta device (memory efficient for large models)
|
| 212 |
+
return init_empty_weights if not self.is_rank_0() else cpu_init_weights
|
| 213 |
+
else:
|
| 214 |
+
self.log("tie_word_embeddings=True, loading full model to CPU on all ranks")
|
| 215 |
+
return cpu_init_weights
|
| 216 |
+
|
| 217 |
+
def prepare(self, model: nn.Module, *args, **kwargs):
|
| 218 |
+
for name, param in model.named_parameters():
|
| 219 |
+
if param.requires_grad:
|
| 220 |
+
model = model.float()
|
| 221 |
+
break
|
| 222 |
+
|
| 223 |
+
# Only rank 0 has real weights, others may have meta tensors
|
| 224 |
+
# _fsdp2_load_full_state_dict will broadcast from rank 0
|
| 225 |
+
full_state = model.state_dict()
|
| 226 |
+
self._init_fsdp_kwargs()
|
| 227 |
+
model = self._fsdp2_shard_model(model)
|
| 228 |
+
model = self._fsdp2_load_full_state_dict(
|
| 229 |
+
model, full_state, self.fsdp_mesh,
|
| 230 |
+
cpu_offload=True if self.args.fsdp.cpu_offload else None
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
return model
|
| 234 |
+
|
| 235 |
+
def _init_fsdp_kwargs(self):
|
| 236 |
+
if self.bf16:
|
| 237 |
+
self.mixed_precision_policy = MixedPrecisionPolicy(
|
| 238 |
+
param_dtype=torch.bfloat16,
|
| 239 |
+
reduce_dtype=torch.float32,
|
| 240 |
+
cast_forward_inputs=True
|
| 241 |
+
)
|
| 242 |
+
else:
|
| 243 |
+
self.mixed_precision_policy = None
|
| 244 |
+
|
| 245 |
+
self.offload_policy = CPUOffloadPolicy(pin_memory=True) if self.args.fsdp.cpu_offload else None
|
| 246 |
+
|
| 247 |
+
self.fsdp_kwargs = {
|
| 248 |
+
"mesh": self.fsdp_mesh,
|
| 249 |
+
"mp_policy": self.mixed_precision_policy,
|
| 250 |
+
"offload_policy": self.offload_policy
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
def _fsdp2_shard_model(self, model):
|
| 254 |
+
fsdp_transformer_layer_cls_to_wrap = model.model._no_split_modules
|
| 255 |
+
for _, module in model.model.named_modules():
|
| 256 |
+
if module.__class__.__name__ in fsdp_transformer_layer_cls_to_wrap or (
|
| 257 |
+
isinstance(module, torch.nn.Embedding) and not model.model.config.tie_word_embeddings
|
| 258 |
+
):
|
| 259 |
+
fully_shard(module, **self.fsdp_kwargs)
|
| 260 |
+
fully_shard(model, **self.fsdp_kwargs)
|
| 261 |
+
return model
|
| 262 |
+
|
| 263 |
+
def _fsdp2_load_full_state_dict(self, model, full_state, device_mesh, cpu_offload):
|
| 264 |
+
"""Load full state dict into FSDP2 model with efficient broadcast from rank 0.
|
| 265 |
+
|
| 266 |
+
This function loads weights from rank 0 and broadcasts to all other ranks,
|
| 267 |
+
avoiding the need for each rank to load the full model from disk.
|
| 268 |
+
|
| 269 |
+
Args:
|
| 270 |
+
model: FSDP2-wrapped model
|
| 271 |
+
full_state: State dict (only rank 0 has real weights, others have empty dict)
|
| 272 |
+
device_mesh: Device mesh for FSDP
|
| 273 |
+
cpu_offload: If not None, enables StateDictOptions cpu_offload
|
| 274 |
+
|
| 275 |
+
Ref:verl/utils/fsdp_utils.py::fsdp2_load_full_state_dict
|
| 276 |
+
"""
|
| 277 |
+
from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict
|
| 278 |
+
|
| 279 |
+
# Step 1: Allocate tensors on GPU
|
| 280 |
+
# Rank 0: move with weights, others: allocate empty tensors on device (for meta tensors)
|
| 281 |
+
if self.is_rank_0():
|
| 282 |
+
model = model.to(device=torch.cuda.current_device(), non_blocking=True)
|
| 283 |
+
else:
|
| 284 |
+
# to_empty creates tensors on device without initializing memory
|
| 285 |
+
model = model.to_empty(device=torch.cuda.current_device())
|
| 286 |
+
|
| 287 |
+
is_cpu_offload = cpu_offload is not None
|
| 288 |
+
options = StateDictOptions(full_state_dict=True, cpu_offload=is_cpu_offload, broadcast_from_rank0=True)
|
| 289 |
+
|
| 290 |
+
# Step 2: Broadcast weights from rank 0 to all ranks
|
| 291 |
+
set_model_state_dict(model, full_state, options=options)
|
| 292 |
+
|
| 293 |
+
# set_model_state_dict will not broadcast buffers, so we need to broadcast them manually.
|
| 294 |
+
for _name, buf in model.named_buffers():
|
| 295 |
+
dist.broadcast(buf, src=0)
|
| 296 |
+
|
| 297 |
+
if is_cpu_offload:
|
| 298 |
+
model.to("cpu", non_blocking=True)
|
| 299 |
+
for buf in model.buffers():
|
| 300 |
+
buf.data = buf.data.to(torch.cuda.current_device())
|
| 301 |
+
|
| 302 |
+
return model
|
| 303 |
+
|
| 304 |
+
def create_optimizer(self, model, **kwargs) -> Optimizer:
|
| 305 |
+
if isinstance(model, DistillModel):
|
| 306 |
+
model = model.model
|
| 307 |
+
|
| 308 |
+
kwargs["fused"] = True
|
| 309 |
+
optim_params = self._get_optimizer_grouped_parameters(model, kwargs["weight_decay"])
|
| 310 |
+
optim = AdamW(optim_params, **kwargs)
|
| 311 |
+
return optim
|
| 312 |
+
|
| 313 |
+
def _get_optimizer_grouped_parameters(
|
| 314 |
+
self,
|
| 315 |
+
model,
|
| 316 |
+
weight_decay,
|
| 317 |
+
no_decay_name_list=["bias", "layer_norm.weight", "layernorm.weight", "norm.weight", "ln_f.weight"],
|
| 318 |
+
):
|
| 319 |
+
optimizer_grouped_parameters = [
|
| 320 |
+
{
|
| 321 |
+
"params": [
|
| 322 |
+
p
|
| 323 |
+
for n, p in model.named_parameters()
|
| 324 |
+
if (not any(nd in n for nd in no_decay_name_list) and p.requires_grad)
|
| 325 |
+
],
|
| 326 |
+
"weight_decay": weight_decay,
|
| 327 |
+
},
|
| 328 |
+
{
|
| 329 |
+
"params": [
|
| 330 |
+
p
|
| 331 |
+
for n, p in model.named_parameters()
|
| 332 |
+
if (any(nd in n for nd in no_decay_name_list) and p.requires_grad)
|
| 333 |
+
],
|
| 334 |
+
"weight_decay": 0.0,
|
| 335 |
+
},
|
| 336 |
+
]
|
| 337 |
+
return optimizer_grouped_parameters
|
| 338 |
+
|
| 339 |
+
def _unwrap_model(self, model) -> nn.Module:
|
| 340 |
+
if hasattr(model, "module"):
|
| 341 |
+
return self._unwrap_model(model.module)
|
| 342 |
+
elif isinstance(model, DistillModel):
|
| 343 |
+
return model.model
|
| 344 |
+
else:
|
| 345 |
+
return model
|
| 346 |
+
|
| 347 |
+
def backward(self, loss: torch.Tensor, model: nn.Module, optimizer: Optimizer, **kwargs) -> None:
|
| 348 |
+
self.step = (self.step + 1) % self.accumulated_gradient
|
| 349 |
+
loss = loss / self.accumulated_gradient
|
| 350 |
+
loss.backward()
|
| 351 |
+
|
| 352 |
+
def optimizer_step(
|
| 353 |
+
self,
|
| 354 |
+
optimizer: Optimizer,
|
| 355 |
+
model: nn.Module,
|
| 356 |
+
scheduler,
|
| 357 |
+
**kwargs,
|
| 358 |
+
) -> None:
|
| 359 |
+
if self.step == 0:
|
| 360 |
+
if self.max_norm > 0.0:
|
| 361 |
+
if hasattr(model, "clip_grad_norm_"):
|
| 362 |
+
model.clip_grad_norm_(self.max_norm)
|
| 363 |
+
else:
|
| 364 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), self.max_norm)
|
| 365 |
+
|
| 366 |
+
optimizer.step()
|
| 367 |
+
optimizer.zero_grad()
|
| 368 |
+
if scheduler:
|
| 369 |
+
scheduler.step()
|
| 370 |
+
|
| 371 |
+
def load_model(self, model: nn.Module, path: str, map_location="cpu", strict: bool = False, key_replace_fn=None) -> None:
|
| 372 |
+
# For FSDP2, we prefer Distributed Checkpoint (DCP)
|
| 373 |
+
# But if the user provides a standard `torch.save` file path, we try to load it.
|
| 374 |
+
# We use `set_model_state_dict` from DCP which handles sharding.
|
| 375 |
+
|
| 376 |
+
# Load state dict on rank 0 (or all if mapped)
|
| 377 |
+
state_dict = torch.load(path, map_location=map_location)
|
| 378 |
+
if key_replace_fn:
|
| 379 |
+
state_dict = key_replace_fn(state_dict)
|
| 380 |
+
|
| 381 |
+
model_to_load = self._unwrap_model(model)
|
| 382 |
+
|
| 383 |
+
# DCP Helper to load full state dict into sharded model
|
| 384 |
+
# Note: This is memory intensive on Rank 0 if strict full load.
|
| 385 |
+
options = StateDictOptions(full_state_dict=True, cpu_offload=True)
|
| 386 |
+
set_model_state_dict(model_to_load, model_state_dict=state_dict, options=options)
|
| 387 |
+
|
| 388 |
+
def save_model(self, model: nn.Module, tokenizer, output_dir, **kwargs) -> None:
|
| 389 |
+
model_to_save = self._unwrap_model(model)
|
| 390 |
+
|
| 391 |
+
if self.is_rank_0():
|
| 392 |
+
os.makedirs(output_dir, exist_ok=True)
|
| 393 |
+
|
| 394 |
+
options = StateDictOptions(full_state_dict=True, cpu_offload=True)
|
| 395 |
+
|
| 396 |
+
state_dict = get_model_state_dict(model_to_save, options=options)
|
| 397 |
+
|
| 398 |
+
if self.args.train.bf16:
|
| 399 |
+
state_dict = {
|
| 400 |
+
k: v.to(torch.bfloat16) if torch.is_floating_point(v) else v
|
| 401 |
+
for k, v in state_dict.items()
|
| 402 |
+
}
|
| 403 |
+
|
| 404 |
+
if self.is_rank_0():
|
| 405 |
+
if isinstance(model_to_save, PeftModel):
|
| 406 |
+
model_to_save.save_pretrained(
|
| 407 |
+
output_dir,
|
| 408 |
+
state_dict=state_dict,
|
| 409 |
+
safe_serialization=False,
|
| 410 |
+
**kwargs
|
| 411 |
+
)
|
| 412 |
+
else:
|
| 413 |
+
model_to_save.save_pretrained(output_dir, state_dict=state_dict, **kwargs)
|
| 414 |
+
|
| 415 |
+
# Config & Tokenizer
|
| 416 |
+
output_config_file = os.path.join(output_dir, "config.json")
|
| 417 |
+
model_to_save.config.to_json_file(output_config_file)
|
| 418 |
+
tokenizer.save_pretrained(output_dir)
|
| 419 |
+
|
| 420 |
+
del state_dict
|
| 421 |
+
gc.collect()
|
| 422 |
+
torch_dist_barrier_and_cuda_sync()
|
| 423 |
+
|
| 424 |
+
def all_reduce(self, data, op="mean"):
|
| 425 |
+
assert op in ("mean", "max", "sum")
|
| 426 |
+
if isinstance(data, dict):
|
| 427 |
+
ret = {}
|
| 428 |
+
for k, v in data.items():
|
| 429 |
+
ret[k] = self.all_reduce(v, op)
|
| 430 |
+
return ret
|
| 431 |
+
else:
|
| 432 |
+
is_tensor = True
|
| 433 |
+
if not isinstance(data, torch.Tensor):
|
| 434 |
+
data = torch.Tensor([data])
|
| 435 |
+
is_tensor = False
|
| 436 |
+
is_cpu_tensor = data.device.type == "cpu"
|
| 437 |
+
|
| 438 |
+
if is_cpu_tensor:
|
| 439 |
+
data = data.to(torch.cuda.current_device())
|
| 440 |
+
if op == "mean":
|
| 441 |
+
data /= self.world_size
|
| 442 |
+
dist.all_reduce(data, op=dist.ReduceOp.MAX if op == "max" else dist.ReduceOp.SUM)
|
| 443 |
+
if is_cpu_tensor:
|
| 444 |
+
data = data.cpu()
|
| 445 |
+
return data.item() if not is_tensor else data
|
| 446 |
+
|
| 447 |
+
def all_gather(self, data):
|
| 448 |
+
if isinstance(data, dict):
|
| 449 |
+
ret = {}
|
| 450 |
+
for k, v in data.items():
|
| 451 |
+
ret[k] = self.all_gather(v)
|
| 452 |
+
return ret
|
| 453 |
+
else:
|
| 454 |
+
if not isinstance(data, torch.Tensor):
|
| 455 |
+
data = torch.Tensor([data])
|
| 456 |
+
is_cpu_tensor = data.device.type == "cpu"
|
| 457 |
+
|
| 458 |
+
ret = [torch.zeros_like(data).to(torch.cuda.current_device()) for _ in range(self.world_size)]
|
| 459 |
+
dist.all_gather(ret, data.to(torch.cuda.current_device()))
|
| 460 |
+
return torch.cat(ret).cpu() if is_cpu_tensor else torch.cat(ret)
|
| 461 |
+
|
| 462 |
+
def print(self, *msg):
|
| 463 |
+
if self.is_rank_0():
|
| 464 |
+
print(*msg)
|
| 465 |
+
|
| 466 |
+
def is_rank_0(self) -> bool:
|
| 467 |
+
if not dist.is_initialized():
|
| 468 |
+
return True
|
| 469 |
+
return dist.get_rank() == 0
|
| 470 |
+
|
| 471 |
+
def get_rank(self) -> int:
|
| 472 |
+
if not dist.is_initialized():
|
| 473 |
+
return 0
|
| 474 |
+
return dist.get_rank()
|
| 475 |
+
|
| 476 |
+
@torch.no_grad()
|
| 477 |
+
def offload_model_params(self, model, empty_cache: bool = True):
|
| 478 |
+
model.cpu()
|
| 479 |
+
if empty_cache:
|
| 480 |
+
torch.cuda.empty_cache()
|
| 481 |
+
|
| 482 |
+
@torch.no_grad()
|
| 483 |
+
def reload_model_params(self, model):
|
| 484 |
+
device = torch.cuda.current_device()
|
| 485 |
+
model.to(device)
|
| 486 |
+
|
| 487 |
+
@torch.no_grad()
|
| 488 |
+
def offload_optim_states(self, optimizer: Optimizer, empty_cache: bool = True):
|
| 489 |
+
"""offload optimizer states to CPU"""
|
| 490 |
+
if not optimizer.state:
|
| 491 |
+
return
|
| 492 |
+
for param_group in optimizer.param_groups:
|
| 493 |
+
for param in param_group["params"]:
|
| 494 |
+
state = optimizer.state[param]
|
| 495 |
+
for key, value in state.items():
|
| 496 |
+
if isinstance(value, torch.Tensor):
|
| 497 |
+
state[key] = value.to("cpu", non_blocking=True)
|
| 498 |
+
if empty_cache:
|
| 499 |
+
torch.cuda.empty_cache()
|
| 500 |
+
|
| 501 |
+
@torch.no_grad()
|
| 502 |
+
def reload_optim_states(self, optimizer: Optimizer):
|
| 503 |
+
"""reload optimizer states to GPU"""
|
| 504 |
+
device = torch.cuda.current_device()
|
| 505 |
+
if not optimizer.state:
|
| 506 |
+
return
|
| 507 |
+
for param_group in optimizer.param_groups:
|
| 508 |
+
for param in param_group["params"]:
|
| 509 |
+
state = optimizer.state[param]
|
| 510 |
+
for key, value in state.items():
|
| 511 |
+
if isinstance(value, torch.Tensor):
|
| 512 |
+
state[key] = value.to(device, non_blocking=True)
|
| 513 |
+
|
| 514 |
+
def update_rollout_weights_from_tensor(
|
| 515 |
+
self, model, engine, gather_src, gather_group,
|
| 516 |
+
update_weight_buffer_size=2 * 1024**3,
|
| 517 |
+
weight_source=None
|
| 518 |
+
):
|
| 519 |
+
"""
|
| 520 |
+
Broadcast FSDP model weights to rollout engine with CUDA IPC + Gloo gather.
|
| 521 |
+
Following slime: all ranks serialize their CUDA IPC data, gather to source rank
|
| 522 |
+
via Gloo, then source rank sends to sglang with per-TP-rank IPC handles.
|
| 523 |
+
"""
|
| 524 |
+
import ray
|
| 525 |
+
from sglang.srt.utils import MultiprocessingSerializer
|
| 526 |
+
|
| 527 |
+
try:
|
| 528 |
+
from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions
|
| 529 |
+
except ImportError:
|
| 530 |
+
from sglang.srt.patch_torch import monkey_patch_torch_reductions
|
| 531 |
+
|
| 532 |
+
try:
|
| 533 |
+
from sglang.srt.weight_sync.tensor_bucket import FlattenedTensorBucket
|
| 534 |
+
except ImportError:
|
| 535 |
+
from sglang.srt.model_executor.model_runner import FlattenedTensorBucket
|
| 536 |
+
|
| 537 |
+
torch.cuda.empty_cache()
|
| 538 |
+
|
| 539 |
+
model_unwrapped = self._unwrap_model(model)
|
| 540 |
+
|
| 541 |
+
bucket = []
|
| 542 |
+
bucket_size = 0
|
| 543 |
+
|
| 544 |
+
# Detect weight tying: when tie_word_embeddings=True, lm_head.weight
|
| 545 |
+
# and model.embed_tokens.weight share the same parameter. SGLang only
|
| 546 |
+
# stores a single copy, so we must skip the duplicate to avoid KeyError.
|
| 547 |
+
tie_word_embeddings = getattr(model_unwrapped.config, "tie_word_embeddings", False)
|
| 548 |
+
|
| 549 |
+
for name, param in model_unwrapped.state_dict().items():
|
| 550 |
+
if tie_word_embeddings and name == "lm_head.weight":
|
| 551 |
+
continue
|
| 552 |
+
param_size = param.numel() * param.element_size()
|
| 553 |
+
if bucket and bucket_size + param_size >= update_weight_buffer_size:
|
| 554 |
+
self._flush_weight_bucket(
|
| 555 |
+
bucket, engine, gather_src, gather_group,
|
| 556 |
+
monkey_patch_torch_reductions, FlattenedTensorBucket, MultiprocessingSerializer,
|
| 557 |
+
)
|
| 558 |
+
del bucket
|
| 559 |
+
bucket = []
|
| 560 |
+
bucket_size = 0
|
| 561 |
+
|
| 562 |
+
if weight_source is not None and name in weight_source:
|
| 563 |
+
param = weight_source[name]
|
| 564 |
+
|
| 565 |
+
param = param.cuda()
|
| 566 |
+
if isinstance(param, DTensor):
|
| 567 |
+
param = param.redistribute(
|
| 568 |
+
placements=[torch.distributed.tensor.Replicate()] * param.device_mesh.ndim,
|
| 569 |
+
async_op=True,
|
| 570 |
+
).to_local()
|
| 571 |
+
bucket.append((name, param))
|
| 572 |
+
bucket_size += param_size
|
| 573 |
+
|
| 574 |
+
if bucket:
|
| 575 |
+
self._flush_weight_bucket(
|
| 576 |
+
bucket, engine, gather_src, gather_group,
|
| 577 |
+
monkey_patch_torch_reductions, FlattenedTensorBucket, MultiprocessingSerializer,
|
| 578 |
+
)
|
| 579 |
+
del bucket
|
| 580 |
+
|
| 581 |
+
if dist.get_rank() == gather_src:
|
| 582 |
+
ray.get(engine.flush_cache.remote())
|
| 583 |
+
|
| 584 |
+
torch.cuda.empty_cache()
|
| 585 |
+
torch_dist_barrier_and_cuda_sync()
|
| 586 |
+
|
| 587 |
+
def _flush_weight_bucket(
|
| 588 |
+
self, bucket, engine, gather_src, gather_group,
|
| 589 |
+
monkey_patch_fn, FlattenedTensorBucket, MultiprocessingSerializer,
|
| 590 |
+
):
|
| 591 |
+
import ray
|
| 592 |
+
|
| 593 |
+
bucket = [(n, p.wait()) if hasattr(p, "wait") else (n, p) for n, p in bucket]
|
| 594 |
+
|
| 595 |
+
monkey_patch_fn()
|
| 596 |
+
|
| 597 |
+
named_tensors_by_dtype = {}
|
| 598 |
+
for name, tensor in bucket:
|
| 599 |
+
named_tensors_by_dtype.setdefault(tensor.dtype, []).append((name, tensor))
|
| 600 |
+
|
| 601 |
+
serialized_tensors = []
|
| 602 |
+
for _dtype, named_tensors in named_tensors_by_dtype.items():
|
| 603 |
+
fb = FlattenedTensorBucket(named_tensors=named_tensors)
|
| 604 |
+
serialized = MultiprocessingSerializer.serialize(
|
| 605 |
+
{"flattened_tensor": fb.get_flattened_tensor(), "metadata": fb.get_metadata()},
|
| 606 |
+
output_str=True,
|
| 607 |
+
)
|
| 608 |
+
serialized_tensors.append(serialized)
|
| 609 |
+
del fb
|
| 610 |
+
|
| 611 |
+
if dist.get_rank() == gather_src:
|
| 612 |
+
gathered = [None for _ in range(dist.get_world_size(gather_group))]
|
| 613 |
+
else:
|
| 614 |
+
gathered = None
|
| 615 |
+
|
| 616 |
+
dist.gather_object(
|
| 617 |
+
obj=serialized_tensors,
|
| 618 |
+
object_gather_list=gathered,
|
| 619 |
+
dst=gather_src,
|
| 620 |
+
group=gather_group,
|
| 621 |
+
)
|
| 622 |
+
|
| 623 |
+
if dist.get_rank() == gather_src:
|
| 624 |
+
num_dtypes = len(gathered[0])
|
| 625 |
+
for i in range(num_dtypes):
|
| 626 |
+
ref = engine.update_weights_from_tensor.remote(
|
| 627 |
+
serialized_named_tensors=[tensors[i] for tensors in gathered],
|
| 628 |
+
load_format="flattened_bucket",
|
| 629 |
+
flush_cache=False,
|
| 630 |
+
)
|
| 631 |
+
ray.get(ref)
|
| 632 |
+
del gathered
|
| 633 |
+
|
| 634 |
+
del bucket, serialized_tensors
|
| 635 |
+
|
| 636 |
+
def update_rollout_weights_from_disk(self, model, rollout_engines, tokenizer=None, tmp_dir=None):
|
| 637 |
+
"""
|
| 638 |
+
Sync FSDP model weights to rollout engines via disk (safetensors).
|
| 639 |
+
|
| 640 |
+
Rank 0 gathers the full state dict and saves it to a shared filesystem path,
|
| 641 |
+
then all rollout engines load the weights from disk via sglang's
|
| 642 |
+
update_weights_from_disk API. This completely avoids CUDA IPC issues.
|
| 643 |
+
|
| 644 |
+
Args:
|
| 645 |
+
model: The FSDP-wrapped model.
|
| 646 |
+
rollout_engines: List of RolloutRayActor instances.
|
| 647 |
+
tokenizer: Tokenizer (needed for save_pretrained to write tokenizer_config.json etc.).
|
| 648 |
+
tmp_dir: Directory to save the temporary weights. If None, uses a default path.
|
| 649 |
+
"""
|
| 650 |
+
import ray
|
| 651 |
+
import shutil
|
| 652 |
+
|
| 653 |
+
if tmp_dir is None:
|
| 654 |
+
tmp_dir = os.path.join(
|
| 655 |
+
os.environ.get("TMPDIR", "/tmp"),
|
| 656 |
+
"kdflow_rollout_weights_sync",
|
| 657 |
+
)
|
| 658 |
+
|
| 659 |
+
model_unwrapped = self._unwrap_model(model)
|
| 660 |
+
is_sender = len(rollout_engines) > 0
|
| 661 |
+
|
| 662 |
+
# Step 1: Gather full state dict (all ranks participate in FSDP collective)
|
| 663 |
+
options = StateDictOptions(full_state_dict=True, cpu_offload=True)
|
| 664 |
+
state_dict = get_model_state_dict(model_unwrapped, options=options)
|
| 665 |
+
|
| 666 |
+
# Step 2: Rank 0 saves weights to shared filesystem
|
| 667 |
+
if self.is_rank_0():
|
| 668 |
+
os.makedirs(tmp_dir, exist_ok=True)
|
| 669 |
+
model_unwrapped.save_pretrained(
|
| 670 |
+
tmp_dir,
|
| 671 |
+
state_dict=state_dict,
|
| 672 |
+
safe_serialization=True,
|
| 673 |
+
)
|
| 674 |
+
# Save config (needed for sglang to load correctly)
|
| 675 |
+
model_unwrapped.config.to_json_file(os.path.join(tmp_dir, "config.json"))
|
| 676 |
+
if tokenizer is not None:
|
| 677 |
+
tokenizer.save_pretrained(tmp_dir)
|
| 678 |
+
|
| 679 |
+
del state_dict
|
| 680 |
+
gc.collect()
|
| 681 |
+
torch_dist_barrier_and_cuda_sync()
|
| 682 |
+
|
| 683 |
+
# Step 3: Tell all rollout engines to load weights from disk
|
| 684 |
+
if is_sender:
|
| 685 |
+
refs = [
|
| 686 |
+
engine.update_weights_from_disk.remote(
|
| 687 |
+
model_path=tmp_dir,
|
| 688 |
+
load_format=None,
|
| 689 |
+
)
|
| 690 |
+
for engine in rollout_engines
|
| 691 |
+
]
|
| 692 |
+
ray.get(refs)
|
| 693 |
+
|
| 694 |
+
torch_dist_barrier_and_cuda_sync()
|
| 695 |
+
|
| 696 |
+
# Step 4: Cleanup temporary directory
|
| 697 |
+
if self.is_rank_0():
|
| 698 |
+
try:
|
| 699 |
+
shutil.rmtree(tmp_dir, ignore_errors=True)
|
| 700 |
+
except Exception:
|
| 701 |
+
pass
|
| 702 |
+
|
| 703 |
+
torch.cuda.empty_cache()
|
| 704 |
+
torch_dist_barrier_and_cuda_sync()
|
kdflow/backend/sglang/__init__.py
ADDED
|
File without changes
|
kdflow/backend/sglang/monkey_patch.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Monkey patch for SGLang scheduler's process_batch_result_prefill method.
|
| 3 |
+
This allows using numpy() instead of tolist() for hidden_states, which is much faster.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
import os
|
| 10 |
+
import time
|
| 11 |
+
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
from kdflow.utils.logging_utils import init_logger
|
| 16 |
+
|
| 17 |
+
if TYPE_CHECKING:
|
| 18 |
+
from sglang.srt.managers.scheduler import (
|
| 19 |
+
EmbeddingBatchResult,
|
| 20 |
+
GenerationBatchResult,
|
| 21 |
+
ScheduleBatch,
|
| 22 |
+
Scheduler,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
logger = init_logger(__name__)
|
| 26 |
+
|
| 27 |
+
# Flag to prevent multiple patch applications
|
| 28 |
+
_PATCH_APPLIED = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def process_batch_result_prefill_patched(
|
| 32 |
+
self: "Scheduler",
|
| 33 |
+
batch: "ScheduleBatch",
|
| 34 |
+
result: Union["GenerationBatchResult", "EmbeddingBatchResult"],
|
| 35 |
+
):
|
| 36 |
+
"""
|
| 37 |
+
Patched version of process_batch_result_prefill.
|
| 38 |
+
Key change: Use .numpy() instead of .tolist() for hidden_states (much faster).
|
| 39 |
+
"""
|
| 40 |
+
from sglang.srt.environ import envs
|
| 41 |
+
from sglang.srt.managers.io_struct import AbortReq
|
| 42 |
+
from sglang.srt.mem_cache.common import release_kv_cache
|
| 43 |
+
|
| 44 |
+
skip_stream_req = None
|
| 45 |
+
|
| 46 |
+
if self.is_generation:
|
| 47 |
+
if result.copy_done is not None:
|
| 48 |
+
result.copy_done.synchronize()
|
| 49 |
+
|
| 50 |
+
(
|
| 51 |
+
logits_output,
|
| 52 |
+
next_token_ids,
|
| 53 |
+
extend_input_len_per_req,
|
| 54 |
+
extend_logprob_start_len_per_req,
|
| 55 |
+
) = (
|
| 56 |
+
result.logits_output,
|
| 57 |
+
result.next_token_ids,
|
| 58 |
+
result.extend_input_len_per_req,
|
| 59 |
+
result.extend_logprob_start_len_per_req,
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
# Move next_token_ids and logprobs to cpu
|
| 63 |
+
next_token_ids = next_token_ids.tolist()
|
| 64 |
+
if batch.return_logprob:
|
| 65 |
+
if logits_output.next_token_logprobs is not None:
|
| 66 |
+
logits_output.next_token_logprobs = (
|
| 67 |
+
logits_output.next_token_logprobs.tolist()
|
| 68 |
+
)
|
| 69 |
+
if logits_output.input_token_logprobs is not None:
|
| 70 |
+
logits_output.input_token_logprobs = tuple(
|
| 71 |
+
logits_output.input_token_logprobs.tolist()
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
hidden_state_offset = 0
|
| 75 |
+
|
| 76 |
+
# Check finish conditions
|
| 77 |
+
logprob_pt = 0
|
| 78 |
+
|
| 79 |
+
for i, (req, next_token_id) in enumerate(zip(batch.reqs, next_token_ids)):
|
| 80 |
+
if req.finished() or req.is_retracted:
|
| 81 |
+
# decode req in mixed batch or retracted req
|
| 82 |
+
continue
|
| 83 |
+
|
| 84 |
+
if req.is_chunked <= 0:
|
| 85 |
+
if hasattr(req.time_stats, "prefill_finished_ts"):
|
| 86 |
+
if req.time_stats.prefill_finished_ts == 0.0:
|
| 87 |
+
req.time_stats.prefill_finished_ts = time.time()
|
| 88 |
+
else:
|
| 89 |
+
req.time_stats.set_prefill_finished_time()
|
| 90 |
+
|
| 91 |
+
# req output_ids are set here
|
| 92 |
+
req.output_ids.append(next_token_id)
|
| 93 |
+
req.check_finished()
|
| 94 |
+
|
| 95 |
+
if req.finished():
|
| 96 |
+
self.maybe_collect_routed_experts(req)
|
| 97 |
+
release_kv_cache(req, self.tree_cache)
|
| 98 |
+
req.time_stats.completion_time = time.perf_counter()
|
| 99 |
+
elif not batch.decoding_reqs or req not in batch.decoding_reqs:
|
| 100 |
+
# This updates radix so others can match
|
| 101 |
+
self.tree_cache.cache_unfinished_req(req)
|
| 102 |
+
|
| 103 |
+
self.maybe_collect_customized_info(i, req, logits_output)
|
| 104 |
+
|
| 105 |
+
if batch.return_logprob:
|
| 106 |
+
assert extend_logprob_start_len_per_req is not None
|
| 107 |
+
assert extend_input_len_per_req is not None
|
| 108 |
+
extend_logprob_start_len = extend_logprob_start_len_per_req[i]
|
| 109 |
+
extend_input_len = extend_input_len_per_req[i]
|
| 110 |
+
|
| 111 |
+
num_input_logprobs = self._calculate_num_input_logprobs(
|
| 112 |
+
req, extend_input_len, extend_logprob_start_len
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
if req.return_logprob:
|
| 116 |
+
self.add_logprob_return_values(
|
| 117 |
+
i,
|
| 118 |
+
req,
|
| 119 |
+
logprob_pt,
|
| 120 |
+
next_token_ids,
|
| 121 |
+
num_input_logprobs,
|
| 122 |
+
logits_output,
|
| 123 |
+
)
|
| 124 |
+
logprob_pt += num_input_logprobs
|
| 125 |
+
|
| 126 |
+
if (
|
| 127 |
+
req.return_hidden_states
|
| 128 |
+
and logits_output.hidden_states is not None
|
| 129 |
+
):
|
| 130 |
+
req.hidden_states.append(
|
| 131 |
+
logits_output.hidden_states[
|
| 132 |
+
hidden_state_offset : (
|
| 133 |
+
hidden_state_offset := hidden_state_offset
|
| 134 |
+
+ len(req.origin_input_ids)
|
| 135 |
+
)
|
| 136 |
+
]
|
| 137 |
+
.half()
|
| 138 |
+
.cpu()
|
| 139 |
+
.numpy()
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
if req.grammar is not None:
|
| 143 |
+
# FIXME: this try-except block is for handling unexpected xgrammar issue.
|
| 144 |
+
try:
|
| 145 |
+
req.grammar.accept_token(next_token_id)
|
| 146 |
+
except ValueError as e:
|
| 147 |
+
# Grammar accept_token can raise ValueError if the token is not in the grammar.
|
| 148 |
+
# This can happen if the grammar is not set correctly or the token is invalid.
|
| 149 |
+
logger.error(
|
| 150 |
+
f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}"
|
| 151 |
+
)
|
| 152 |
+
self.abort_request(AbortReq(rid=req.rid))
|
| 153 |
+
req.grammar.finished = req.finished()
|
| 154 |
+
|
| 155 |
+
else:
|
| 156 |
+
# being chunked reqs' prefill is not finished
|
| 157 |
+
req.is_chunked -= 1
|
| 158 |
+
# There is only at most one request being currently chunked.
|
| 159 |
+
# Because this request does not finish prefill,
|
| 160 |
+
# we don't want to stream the request currently being chunked.
|
| 161 |
+
skip_stream_req = req
|
| 162 |
+
|
| 163 |
+
# Incrementally update input logprobs.
|
| 164 |
+
if batch.return_logprob:
|
| 165 |
+
extend_logprob_start_len = extend_logprob_start_len_per_req[i]
|
| 166 |
+
extend_input_len = extend_input_len_per_req[i]
|
| 167 |
+
if extend_logprob_start_len < extend_input_len:
|
| 168 |
+
# Update input logprobs.
|
| 169 |
+
num_input_logprobs = self._calculate_num_input_logprobs(
|
| 170 |
+
req, extend_input_len, extend_logprob_start_len
|
| 171 |
+
)
|
| 172 |
+
if req.return_logprob:
|
| 173 |
+
self.add_input_logprob_return_values(
|
| 174 |
+
i,
|
| 175 |
+
req,
|
| 176 |
+
logits_output,
|
| 177 |
+
logprob_pt,
|
| 178 |
+
num_input_logprobs,
|
| 179 |
+
last_prefill_chunk=False,
|
| 180 |
+
)
|
| 181 |
+
logprob_pt += num_input_logprobs
|
| 182 |
+
|
| 183 |
+
else: # embedding or reward model
|
| 184 |
+
if result.copy_done is not None:
|
| 185 |
+
result.copy_done.synchronize()
|
| 186 |
+
|
| 187 |
+
is_sparse = envs.SGLANG_EMBEDDINGS_SPARSE_HEAD.is_set()
|
| 188 |
+
|
| 189 |
+
embeddings = result.embeddings
|
| 190 |
+
|
| 191 |
+
if is_sparse:
|
| 192 |
+
batch_ids, token_ids = embeddings.indices()
|
| 193 |
+
values = embeddings.values()
|
| 194 |
+
|
| 195 |
+
embeddings = [{} for _ in range(embeddings.size(0))]
|
| 196 |
+
for i in range(batch_ids.shape[0]):
|
| 197 |
+
embeddings[batch_ids[i].item()][token_ids[i].item()] = values[
|
| 198 |
+
i
|
| 199 |
+
].item()
|
| 200 |
+
else:
|
| 201 |
+
if isinstance(embeddings, torch.Tensor):
|
| 202 |
+
embeddings = embeddings.tolist()
|
| 203 |
+
else:
|
| 204 |
+
embeddings = [tensor.tolist() for tensor in embeddings]
|
| 205 |
+
|
| 206 |
+
# Check finish conditions
|
| 207 |
+
for i, req in enumerate(batch.reqs):
|
| 208 |
+
if req.is_retracted:
|
| 209 |
+
continue
|
| 210 |
+
|
| 211 |
+
req.embedding = embeddings[i]
|
| 212 |
+
if req.is_chunked <= 0:
|
| 213 |
+
# Dummy output token for embedding models
|
| 214 |
+
req.output_ids.append(0)
|
| 215 |
+
req.check_finished()
|
| 216 |
+
|
| 217 |
+
if req.finished():
|
| 218 |
+
release_kv_cache(req, self.tree_cache)
|
| 219 |
+
else:
|
| 220 |
+
self.tree_cache.cache_unfinished_req(req)
|
| 221 |
+
else:
|
| 222 |
+
# being chunked reqs' prefill is not finished
|
| 223 |
+
req.is_chunked -= 1
|
| 224 |
+
|
| 225 |
+
self.stream_output(batch.reqs, batch.return_logprob, skip_stream_req)
|
| 226 |
+
|
| 227 |
+
if self.current_scheduler_metrics_enabled:
|
| 228 |
+
can_run_cuda_graph = getattr(result, "can_run_cuda_graph", False)
|
| 229 |
+
self.log_prefill_stats(
|
| 230 |
+
prefill_stats=batch.prefill_stats,
|
| 231 |
+
can_run_cuda_graph=can_run_cuda_graph,
|
| 232 |
+
dp_cooperation_info=batch.dp_cooperation_info,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def apply_patch():
|
| 237 |
+
"""
|
| 238 |
+
Apply the monkey patch to SGLang's SchedulerOutputProcessorMixin.
|
| 239 |
+
|
| 240 |
+
This function is idempotent - calling it multiple times is safe.
|
| 241 |
+
Returns True if patch was applied (or already applied), False otherwise.
|
| 242 |
+
"""
|
| 243 |
+
global _PATCH_APPLIED
|
| 244 |
+
|
| 245 |
+
if _PATCH_APPLIED:
|
| 246 |
+
return True
|
| 247 |
+
|
| 248 |
+
try:
|
| 249 |
+
from sglang.srt.managers.scheduler_output_processor_mixin import (
|
| 250 |
+
SchedulerOutputProcessorMixin,
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
# Check if already patched (by another mechanism like sitecustomize)
|
| 254 |
+
current_method = getattr(SchedulerOutputProcessorMixin, 'process_batch_result_prefill', None)
|
| 255 |
+
if current_method is not None and getattr(current_method, '_kdflow_patched', False):
|
| 256 |
+
_PATCH_APPLIED = True
|
| 257 |
+
print(f"[monkey_patch] Patch already applied, PID={os.getpid()}", flush=True)
|
| 258 |
+
return True
|
| 259 |
+
|
| 260 |
+
# Mark the patched function
|
| 261 |
+
process_batch_result_prefill_patched._kdflow_patched = True
|
| 262 |
+
|
| 263 |
+
# Apply patch
|
| 264 |
+
SchedulerOutputProcessorMixin.process_batch_result_prefill = process_batch_result_prefill_patched
|
| 265 |
+
|
| 266 |
+
_PATCH_APPLIED = True
|
| 267 |
+
print(f"[monkey_patch] SUCCESS: process_batch_result_prefill patched! PID={os.getpid()}", flush=True)
|
| 268 |
+
return True
|
| 269 |
+
|
| 270 |
+
except ImportError as e:
|
| 271 |
+
print(f"[monkey_patch] Cannot import SGLang module: {e}", flush=True)
|
| 272 |
+
return False
|
| 273 |
+
except Exception as e:
|
| 274 |
+
print(f"[monkey_patch] Error applying patch: {e}", flush=True)
|
| 275 |
+
import traceback
|
| 276 |
+
traceback.print_exc()
|
| 277 |
+
return False
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def is_patch_applied():
|
| 281 |
+
"""Check if the patch has been applied in this process."""
|
| 282 |
+
return _PATCH_APPLIED
|
kdflow/backend/sglang/sglang_engine.py
ADDED
|
@@ -0,0 +1,437 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import queue
|
| 3 |
+
import time
|
| 4 |
+
import torch.multiprocessing as mp
|
| 5 |
+
from torch.multiprocessing import Queue
|
| 6 |
+
from typing import List, Dict, Any, Tuple, Optional
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
from sglang.srt.entrypoints.engine import Engine as _SglEngine
|
| 12 |
+
from sglang.srt.managers.scheduler import run_scheduler_process as _original_run_scheduler_process
|
| 13 |
+
|
| 14 |
+
from kdflow.utils.logging_utils import init_logger
|
| 15 |
+
|
| 16 |
+
logger = init_logger(__name__)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
os.environ["SGLANG_JIT_DEEPGEMM_FAST_WARMUP"] = "true"
|
| 20 |
+
|
| 21 |
+
def _patched_run_scheduler_process(*args, **kwargs):
|
| 22 |
+
try:
|
| 23 |
+
from kdflow.backend.sglang.monkey_patch import apply_patch
|
| 24 |
+
apply_patch()
|
| 25 |
+
except Exception as e:
|
| 26 |
+
logger.warning(f"[PatchedEngine] WARNING: Failed to apply monkey patch (PID={os.getpid()}): {e}", flush=True)
|
| 27 |
+
return _original_run_scheduler_process(*args, **kwargs)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class PatchedEngine(_SglEngine):
|
| 31 |
+
"""
|
| 32 |
+
SGLang Engine that applies monkey patch in scheduler subprocesses.
|
| 33 |
+
Motivation: SGLang Engine supports returning hidden states, but the existing implementation use .tolist() to convert hidden states from GPU tensor to Python list, which is very inefficient. This monkey patch replaces the original .tolist() with a more efficient operation .numpy().
|
| 34 |
+
"""
|
| 35 |
+
run_scheduler_process_func = staticmethod(_patched_run_scheduler_process)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@dataclass
|
| 39 |
+
class EngineConfig:
|
| 40 |
+
"""Configuration for SGLang Engine."""
|
| 41 |
+
model_path: str
|
| 42 |
+
tp_size: int = 1
|
| 43 |
+
ep_size: int = 1
|
| 44 |
+
pp_size: int = 1
|
| 45 |
+
chunked_prefill_size: int = -1
|
| 46 |
+
disable_radix_cache: bool = True
|
| 47 |
+
enable_return_hidden_states: bool = True
|
| 48 |
+
enable_memory_saver: bool = True
|
| 49 |
+
enable_weights_cpu_backup: bool = True
|
| 50 |
+
mem_fraction_static: float = 0.8
|
| 51 |
+
quantization: str = None
|
| 52 |
+
offload_tags: Optional[str] = "all"
|
| 53 |
+
base_gpu_id: int = 0
|
| 54 |
+
# for multi-node tp/pp
|
| 55 |
+
nnodes: int = 1
|
| 56 |
+
node_rank: int = 0
|
| 57 |
+
dist_init_addr: Optional[str] = None
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _engine_worker(config: EngineConfig, request_queue: Queue, response_queue: Queue, hidden_queue: Queue):
|
| 61 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
| 62 |
+
if config.nnodes > 1:
|
| 63 |
+
os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0"
|
| 64 |
+
|
| 65 |
+
engine = None
|
| 66 |
+
|
| 67 |
+
try:
|
| 68 |
+
engine = PatchedEngine(
|
| 69 |
+
model_path=config.model_path,
|
| 70 |
+
tp_size=config.tp_size,
|
| 71 |
+
ep_size=config.ep_size,
|
| 72 |
+
pp_size=config.pp_size,
|
| 73 |
+
chunked_prefill_size=config.chunked_prefill_size,
|
| 74 |
+
disable_radix_cache=config.disable_radix_cache,
|
| 75 |
+
enable_return_hidden_states=config.enable_return_hidden_states,
|
| 76 |
+
enable_memory_saver=config.enable_memory_saver,
|
| 77 |
+
enable_weights_cpu_backup=config.enable_weights_cpu_backup,
|
| 78 |
+
quantization=config.quantization,
|
| 79 |
+
mem_fraction_static=config.mem_fraction_static,
|
| 80 |
+
base_gpu_id=config.base_gpu_id,
|
| 81 |
+
nnodes=config.nnodes,
|
| 82 |
+
node_rank=config.node_rank,
|
| 83 |
+
dist_init_addr=config.dist_init_addr,
|
| 84 |
+
disable_custom_all_reduce=True,
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
response_queue.put({
|
| 88 |
+
"type": "init_done",
|
| 89 |
+
"success": True,
|
| 90 |
+
})
|
| 91 |
+
|
| 92 |
+
while True:
|
| 93 |
+
request = request_queue.get()
|
| 94 |
+
if request is None:
|
| 95 |
+
break
|
| 96 |
+
|
| 97 |
+
req_type = request.get("type")
|
| 98 |
+
|
| 99 |
+
try:
|
| 100 |
+
if req_type == "generate":
|
| 101 |
+
_handle_generate(engine, request, hidden_queue, response_queue)
|
| 102 |
+
elif req_type == "sleep":
|
| 103 |
+
_handle_sleep(engine, request, config, response_queue)
|
| 104 |
+
elif req_type == "wakeup":
|
| 105 |
+
_handle_wakeup(engine, request, config, response_queue)
|
| 106 |
+
elif req_type == "update_weights_from_tensor":
|
| 107 |
+
_handle_update_weights_from_tensor(engine, request, response_queue)
|
| 108 |
+
else:
|
| 109 |
+
response_queue.put({"type": req_type, "success": False,
|
| 110 |
+
"error": f"Unknown request type: {req_type}"})
|
| 111 |
+
except Exception:
|
| 112 |
+
import traceback
|
| 113 |
+
response_queue.put({"type": req_type, "success": False,
|
| 114 |
+
"error": traceback.format_exc()})
|
| 115 |
+
|
| 116 |
+
except Exception:
|
| 117 |
+
import traceback
|
| 118 |
+
response_queue.put({"type": "init_done", "success": False,
|
| 119 |
+
"error": traceback.format_exc()})
|
| 120 |
+
finally:
|
| 121 |
+
if engine:
|
| 122 |
+
try:
|
| 123 |
+
engine.shutdown()
|
| 124 |
+
except Exception:
|
| 125 |
+
pass
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _normalize_tags(tags):
|
| 129 |
+
"""Convert tags to the format SGLang expects (None, or list of strings)."""
|
| 130 |
+
if tags is None or tags == "all":
|
| 131 |
+
return None
|
| 132 |
+
if isinstance(tags, str):
|
| 133 |
+
return [tags]
|
| 134 |
+
return tags
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _handle_generate(engine, request, hidden_queue, response_queue):
|
| 138 |
+
"""Handle a generate request and stream hidden states via shared memory."""
|
| 139 |
+
kwargs = request["kwargs"]
|
| 140 |
+
|
| 141 |
+
generate_kwargs = {
|
| 142 |
+
"input_ids": kwargs["input_ids"],
|
| 143 |
+
"sampling_params": kwargs["sampling_params"],
|
| 144 |
+
"return_hidden_states": kwargs.get("return_hidden_states", True),
|
| 145 |
+
}
|
| 146 |
+
if kwargs.get("image_data") is not None:
|
| 147 |
+
generate_kwargs["image_data"] = kwargs["image_data"]
|
| 148 |
+
|
| 149 |
+
outputs = engine.generate(**generate_kwargs, stream=True)
|
| 150 |
+
num_samples = len(kwargs["loss_masks"])
|
| 151 |
+
|
| 152 |
+
response_queue.put({
|
| 153 |
+
"type": "generate",
|
| 154 |
+
"success": True,
|
| 155 |
+
"num_samples": num_samples,
|
| 156 |
+
})
|
| 157 |
+
|
| 158 |
+
received_indices = set()
|
| 159 |
+
try:
|
| 160 |
+
for output in outputs:
|
| 161 |
+
meta_info = output["meta_info"]
|
| 162 |
+
if meta_info.get("finish_reason") is None:
|
| 163 |
+
continue
|
| 164 |
+
|
| 165 |
+
idx = output["index"]
|
| 166 |
+
if idx in received_indices:
|
| 167 |
+
raise RuntimeError(f"Duplicate SGLang output index: {idx}")
|
| 168 |
+
|
| 169 |
+
mask = kwargs["loss_masks"][idx]
|
| 170 |
+
hs_np = meta_info["hidden_states"][0]
|
| 171 |
+
|
| 172 |
+
# hs_np and mask may differ due to multimodal token expansion.
|
| 173 |
+
hs_len = hs_np.shape[0]
|
| 174 |
+
mask_len = mask.shape[0]
|
| 175 |
+
if hs_len == mask_len:
|
| 176 |
+
hs_np = hs_np[mask]
|
| 177 |
+
else:
|
| 178 |
+
num_loss_tokens = int(mask.sum())
|
| 179 |
+
logger.warning(
|
| 180 |
+
f"[_handle_generate] sample={idx}/{num_samples} length mismatch: "
|
| 181 |
+
f"hs_len={hs_len}, mask_len={mask_len}, diff={mask_len - hs_len}; "
|
| 182 |
+
f"selecting {num_loss_tokens} loss tokens from the tail"
|
| 183 |
+
)
|
| 184 |
+
if num_loss_tokens >= hs_len:
|
| 185 |
+
raise ValueError(
|
| 186 |
+
f"Cannot select {num_loss_tokens} loss tokens from "
|
| 187 |
+
f"hidden states of length {hs_len}"
|
| 188 |
+
)
|
| 189 |
+
# The loss span is a contiguous suffix before the final token.
|
| 190 |
+
hs_np = hs_np[-num_loss_tokens - 1:-1]
|
| 191 |
+
|
| 192 |
+
if not hs_np.flags['C_CONTIGUOUS']:
|
| 193 |
+
hs_np = np.ascontiguousarray(hs_np)
|
| 194 |
+
|
| 195 |
+
hs_tensor = torch.from_numpy(hs_np).share_memory_()
|
| 196 |
+
hidden_queue.put((idx, hs_tensor))
|
| 197 |
+
received_indices.add(idx)
|
| 198 |
+
|
| 199 |
+
if len(received_indices) != num_samples:
|
| 200 |
+
raise RuntimeError(
|
| 201 |
+
f"SGLang returned {len(received_indices)} final outputs for "
|
| 202 |
+
f"{num_samples} samples"
|
| 203 |
+
)
|
| 204 |
+
hidden_queue.put(None)
|
| 205 |
+
except Exception:
|
| 206 |
+
import traceback
|
| 207 |
+
error = traceback.format_exc()
|
| 208 |
+
logger.error(f"[_handle_generate] ERROR:\n{error}")
|
| 209 |
+
hidden_queue.put(error)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _handle_sleep(engine, request, config, response_queue):
|
| 213 |
+
"""Handle a sleep request: offload GPU memory."""
|
| 214 |
+
tags = request.get("tags", config.offload_tags)
|
| 215 |
+
torch.cuda.empty_cache()
|
| 216 |
+
engine.release_memory_occupation(tags=_normalize_tags(tags))
|
| 217 |
+
response_queue.put({"type": "sleep", "success": True, "tags": tags})
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _handle_wakeup(engine, request, config, response_queue):
|
| 221 |
+
"""Handle a wakeup request: restore GPU memory."""
|
| 222 |
+
tags = request.get("tags", config.offload_tags)
|
| 223 |
+
torch.cuda.empty_cache()
|
| 224 |
+
engine.resume_memory_occupation(tags=_normalize_tags(tags))
|
| 225 |
+
response_queue.put({"type": "wakeup", "success": True, "tags": tags})
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def _handle_update_weights_from_tensor(engine, request, response_queue):
|
| 229 |
+
"""Handle a update_weights_from_tensor request: update weights from student (for self-distillation)."""
|
| 230 |
+
serialized_named_tensors = request["kwargs"]["serialized_named_tensors"]
|
| 231 |
+
load_format = request["kwargs"]["load_format"]
|
| 232 |
+
flush_cache = request["kwargs"]["flush_cache"]
|
| 233 |
+
engine.update_weights_from_tensor(
|
| 234 |
+
named_tensors=serialized_named_tensors,
|
| 235 |
+
load_format=load_format,
|
| 236 |
+
flush_cache=flush_cache,
|
| 237 |
+
)
|
| 238 |
+
response_queue.put({"type": "update_weights_from_tensor", "success": True})
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
class SGLangEngineService:
|
| 242 |
+
"""Manages SGLang Engine in a subprocess with torch multiprocessing communication."""
|
| 243 |
+
|
| 244 |
+
def __init__(self, config: EngineConfig):
|
| 245 |
+
self.config = config
|
| 246 |
+
self.process: Optional[mp.Process] = None
|
| 247 |
+
self.request_queue: Optional[Queue] = None
|
| 248 |
+
self.response_queue: Optional[Queue] = None
|
| 249 |
+
self.hidden_queue: Optional[Queue] = None
|
| 250 |
+
self._started = False
|
| 251 |
+
|
| 252 |
+
def start(self, timeout: float = 1800.0):
|
| 253 |
+
"""Start the SGLang Engine in a subprocess."""
|
| 254 |
+
if self._started:
|
| 255 |
+
raise RuntimeError("Service already started")
|
| 256 |
+
|
| 257 |
+
try:
|
| 258 |
+
mp.set_start_method("spawn", force=True)
|
| 259 |
+
except RuntimeError:
|
| 260 |
+
pass
|
| 261 |
+
|
| 262 |
+
self.request_queue = mp.Queue()
|
| 263 |
+
self.response_queue = mp.Queue()
|
| 264 |
+
self.hidden_queue = mp.Queue(maxsize=2)
|
| 265 |
+
|
| 266 |
+
self.process = mp.Process(
|
| 267 |
+
target=_engine_worker,
|
| 268 |
+
args=(self.config, self.request_queue, self.response_queue, self.hidden_queue),
|
| 269 |
+
)
|
| 270 |
+
self.process.start()
|
| 271 |
+
|
| 272 |
+
try:
|
| 273 |
+
response = self.response_queue.get(timeout=timeout)
|
| 274 |
+
if response.get("type") == "init_done" and response.get("success"):
|
| 275 |
+
self._started = True
|
| 276 |
+
else:
|
| 277 |
+
raise RuntimeError(f"Init failed: {response.get('error')}")
|
| 278 |
+
except Exception as e:
|
| 279 |
+
self._cleanup()
|
| 280 |
+
raise RuntimeError(f"Engine initialization failed: {e}")
|
| 281 |
+
|
| 282 |
+
def generate(
|
| 283 |
+
self,
|
| 284 |
+
input_ids: List[List[int]],
|
| 285 |
+
loss_masks: List[np.ndarray],
|
| 286 |
+
sampling_params: Dict[str, Any],
|
| 287 |
+
return_hidden_states: bool = True,
|
| 288 |
+
image_data=None,
|
| 289 |
+
) -> List[np.ndarray]:
|
| 290 |
+
"""Run generation and return hidden states via shared-memory tensors.
|
| 291 |
+
|
| 292 |
+
Args:
|
| 293 |
+
input_ids: Per-sample token-id lists fed directly to SGLang (avoids a
|
| 294 |
+
re-tokenization round trip, so hidden-state length matches the mask).
|
| 295 |
+
loss_masks: Pre-computed boolean masks for selecting response hidden states.
|
| 296 |
+
sampling_params: Sampling parameters (e.g. max_new_tokens=0 for prefill-only).
|
| 297 |
+
return_hidden_states: Whether to return hidden states.
|
| 298 |
+
image_data: Optional list of image data for multimodal models.
|
| 299 |
+
"""
|
| 300 |
+
if not self._started:
|
| 301 |
+
raise RuntimeError("Service not started")
|
| 302 |
+
|
| 303 |
+
if len(input_ids) != len(loss_masks):
|
| 304 |
+
raise ValueError(
|
| 305 |
+
f"input_ids and loss_masks must have the same batch size, got "
|
| 306 |
+
f"{len(input_ids)} and {len(loss_masks)}"
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
# Check if subprocess is still alive before sending request
|
| 310 |
+
if self.process and not self.process.is_alive():
|
| 311 |
+
raise RuntimeError(
|
| 312 |
+
f"[SGLangEngineService] Engine subprocess (PID={self.process.pid}) is dead! "
|
| 313 |
+
f"exitcode={self.process.exitcode}"
|
| 314 |
+
)
|
| 315 |
+
|
| 316 |
+
kwargs = {
|
| 317 |
+
"input_ids": input_ids,
|
| 318 |
+
"loss_masks": loss_masks,
|
| 319 |
+
"sampling_params": sampling_params,
|
| 320 |
+
"return_hidden_states": return_hidden_states,
|
| 321 |
+
}
|
| 322 |
+
if image_data is not None:
|
| 323 |
+
kwargs["image_data"] = image_data
|
| 324 |
+
|
| 325 |
+
self.request_queue.put({"type": "generate", "kwargs": kwargs})
|
| 326 |
+
|
| 327 |
+
response = self._get_response(req_type="generate", timeout=600)
|
| 328 |
+
if not response.get("success"):
|
| 329 |
+
raise RuntimeError(f"Generate failed: {response.get('error')}")
|
| 330 |
+
|
| 331 |
+
num_samples = response["num_samples"]
|
| 332 |
+
hidden_states = [None] * num_samples
|
| 333 |
+
received_count = 0
|
| 334 |
+
t_recv_start = time.time()
|
| 335 |
+
while True:
|
| 336 |
+
try:
|
| 337 |
+
message = self.hidden_queue.get(timeout=300)
|
| 338 |
+
if message is None:
|
| 339 |
+
break
|
| 340 |
+
if isinstance(message, str):
|
| 341 |
+
raise RuntimeError(f"Generate failed in engine subprocess:\n{message}")
|
| 342 |
+
|
| 343 |
+
idx, hs_tensor = message
|
| 344 |
+
hs_np = hs_tensor.numpy().copy()
|
| 345 |
+
del hs_tensor
|
| 346 |
+
hidden_states[idx] = hs_np
|
| 347 |
+
received_count += 1
|
| 348 |
+
except queue.Empty:
|
| 349 |
+
elapsed_total = time.time() - t_recv_start
|
| 350 |
+
raise RuntimeError(
|
| 351 |
+
f"Hidden state recv timeout while receiving hidden states, "
|
| 352 |
+
f"received={received_count}/{num_samples}, "
|
| 353 |
+
f"total_elapsed={elapsed_total:.1f}s"
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
return hidden_states
|
| 357 |
+
|
| 358 |
+
def sleep(self, tags: Optional[str] = "all"):
|
| 359 |
+
"""Release GPU memory."""
|
| 360 |
+
if not self._started:
|
| 361 |
+
return
|
| 362 |
+
self.request_queue.put({"type": "sleep", "tags": tags})
|
| 363 |
+
response = self._get_response(req_type="sleep", timeout=300)
|
| 364 |
+
if not response.get("success"):
|
| 365 |
+
raise RuntimeError(f"Sleep failed: {response.get('error')}")
|
| 366 |
+
return response.get("tags")
|
| 367 |
+
|
| 368 |
+
def wakeup(self, tags: Optional[str] = "all"):
|
| 369 |
+
"""Resume GPU memory."""
|
| 370 |
+
if not self._started:
|
| 371 |
+
return
|
| 372 |
+
self.request_queue.put({"type": "wakeup", "tags": tags})
|
| 373 |
+
response = self._get_response(req_type="wakeup", timeout=300)
|
| 374 |
+
if not response.get("success"):
|
| 375 |
+
raise RuntimeError(f"Wakeup failed: {response.get('error')}")
|
| 376 |
+
return response.get("tags")
|
| 377 |
+
|
| 378 |
+
def update_weights_from_tensor(
|
| 379 |
+
self, serialized_named_tensors: List[Tuple[str, torch.Tensor]],
|
| 380 |
+
load_format: Optional[str] = None, flush_cache: bool = True):
|
| 381 |
+
kwargs = {
|
| 382 |
+
"serialized_named_tensors": serialized_named_tensors,
|
| 383 |
+
"load_format": load_format,
|
| 384 |
+
"flush_cache": flush_cache,
|
| 385 |
+
}
|
| 386 |
+
self.request_queue.put({"type": "update_weights_from_tensor", "kwargs": kwargs})
|
| 387 |
+
response = self._get_response(req_type="update_weights_from_tensor", timeout=300)
|
| 388 |
+
if not response.get("success"):
|
| 389 |
+
raise RuntimeError(f"update_weights_from_tensor failed: {response.get('error')}")
|
| 390 |
+
|
| 391 |
+
def _get_response(self, req_type="unknown", timeout=600, check_interval=10):
|
| 392 |
+
elapsed = 0
|
| 393 |
+
while elapsed < timeout:
|
| 394 |
+
try:
|
| 395 |
+
return self.response_queue.get(timeout=check_interval)
|
| 396 |
+
except queue.Empty:
|
| 397 |
+
elapsed += check_interval
|
| 398 |
+
if self.process and not self.process.is_alive():
|
| 399 |
+
raise RuntimeError(
|
| 400 |
+
f"Engine subprocess (PID={self.process.pid}) died during '{req_type}'! "
|
| 401 |
+
f"exitcode={self.process.exitcode}"
|
| 402 |
+
)
|
| 403 |
+
raise RuntimeError(f"Response timeout after {timeout}s during '{req_type}'")
|
| 404 |
+
|
| 405 |
+
def shutdown(self):
|
| 406 |
+
"""Shutdown the subprocess gracefully."""
|
| 407 |
+
if not self._started:
|
| 408 |
+
return
|
| 409 |
+
self._started = False
|
| 410 |
+
self._cleanup()
|
| 411 |
+
|
| 412 |
+
def _cleanup(self):
|
| 413 |
+
"""Clean up subprocess, queues and shared memory."""
|
| 414 |
+
if self.request_queue:
|
| 415 |
+
try:
|
| 416 |
+
self.request_queue.put(None)
|
| 417 |
+
except Exception:
|
| 418 |
+
pass
|
| 419 |
+
|
| 420 |
+
if self.process:
|
| 421 |
+
self.process.join(timeout=30)
|
| 422 |
+
if self.process.is_alive():
|
| 423 |
+
self.process.terminate()
|
| 424 |
+
self.process.join(timeout=5)
|
| 425 |
+
if self.process.is_alive():
|
| 426 |
+
self.process.kill()
|
| 427 |
+
|
| 428 |
+
self.process = None
|
| 429 |
+
self.request_queue = None
|
| 430 |
+
self.response_queue = None
|
| 431 |
+
self.hidden_queue = None
|
| 432 |
+
|
| 433 |
+
def __del__(self):
|
| 434 |
+
try:
|
| 435 |
+
self.shutdown()
|
| 436 |
+
except Exception:
|
| 437 |
+
pass
|
kdflow/cli/__init__.py
ADDED
|
File without changes
|
kdflow/cli/evaluate_alfworld.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate a model checkpoint in ALFWorld without starting training actors."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
|
| 7 |
+
from kdflow.evaluation import AlfWorldEvaluator, VerlAgentAlfWorldEvaluator
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class _EvaluationStrategy:
|
| 14 |
+
"""Small args/log adapter required by the shared rollout manager."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, args):
|
| 17 |
+
self.args = args
|
| 18 |
+
|
| 19 |
+
@staticmethod
|
| 20 |
+
def log(message, level="info", **_kwargs):
|
| 21 |
+
getattr(logger, level, logger.info)(message)
|
| 22 |
+
|
| 23 |
+
@staticmethod
|
| 24 |
+
def print(*values):
|
| 25 |
+
logger.info(" ".join(str(value) for value in values))
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def evaluate(args):
|
| 29 |
+
if args.rollout.alfworld_eval_protocol == "verl_agent":
|
| 30 |
+
preflight_evaluator = VerlAgentAlfWorldEvaluator(args, None)
|
| 31 |
+
if not preflight_evaluator.has_pending_seeds():
|
| 32 |
+
logger.info(
|
| 33 |
+
"All requested ALFWorld seeds are already complete and "
|
| 34 |
+
"compatible; skipping Ray and model initialization."
|
| 35 |
+
)
|
| 36 |
+
summary = preflight_evaluator.evaluate()
|
| 37 |
+
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
| 38 |
+
return summary
|
| 39 |
+
|
| 40 |
+
import ray
|
| 41 |
+
|
| 42 |
+
from kdflow.ray.placement_group import create_placement_group
|
| 43 |
+
from kdflow.ray.rollout.rollout_group import RolloutActorGroup
|
| 44 |
+
from kdflow.trainer.alfworld_rollout_manager import AlfWorldRolloutManager
|
| 45 |
+
from kdflow.trainer.verl_agent_alfworld_rollout_manager import (
|
| 46 |
+
VerlAgentAlfWorldRolloutManager,
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
initialized_ray = False
|
| 50 |
+
if not ray.is_initialized():
|
| 51 |
+
ray.init(
|
| 52 |
+
runtime_env={
|
| 53 |
+
"env_vars": {
|
| 54 |
+
"TOKENIZERS_PARALLELISM": "true",
|
| 55 |
+
"NCCL_DEBUG": "WARN",
|
| 56 |
+
},
|
| 57 |
+
"working_dir": os.path.dirname(
|
| 58 |
+
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| 59 |
+
),
|
| 60 |
+
}
|
| 61 |
+
)
|
| 62 |
+
initialized_ray = True
|
| 63 |
+
|
| 64 |
+
total_gpus = args.train.num_nodes * args.train.num_gpus_per_node
|
| 65 |
+
rollout_group = None
|
| 66 |
+
rollout_manager = None
|
| 67 |
+
try:
|
| 68 |
+
pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(
|
| 69 |
+
total_gpus
|
| 70 |
+
)
|
| 71 |
+
rollout_group = RolloutActorGroup(
|
| 72 |
+
model_path=args.model.student_name_or_path,
|
| 73 |
+
num_actors=args.rollout.rollout_num_engines,
|
| 74 |
+
tp_size=args.rollout.rollout_tp_size,
|
| 75 |
+
num_gpus_per_node=args.train.num_gpus_per_node,
|
| 76 |
+
enable_memory_saver=True,
|
| 77 |
+
mem_fraction_static=args.rollout.rollout_mem_fraction_static,
|
| 78 |
+
num_gpus_per_actor=0.01,
|
| 79 |
+
pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
|
| 80 |
+
extra_server_args={"disable_piecewise_cuda_graph": True},
|
| 81 |
+
)
|
| 82 |
+
if args.train.enable_sleep:
|
| 83 |
+
rollout_group.sleep()
|
| 84 |
+
|
| 85 |
+
strategy = _EvaluationStrategy(args)
|
| 86 |
+
generate_kwargs = {
|
| 87 |
+
"max_new_tokens": args.rollout.generate_max_len,
|
| 88 |
+
"temperature": args.rollout.temperature,
|
| 89 |
+
"top_p": args.rollout.top_p,
|
| 90 |
+
}
|
| 91 |
+
if args.rollout.alfworld_eval_protocol == "verl_agent":
|
| 92 |
+
manager_class = VerlAgentAlfWorldRolloutManager
|
| 93 |
+
evaluator_class = VerlAgentAlfWorldEvaluator
|
| 94 |
+
else:
|
| 95 |
+
manager_class = AlfWorldRolloutManager
|
| 96 |
+
evaluator_class = AlfWorldEvaluator
|
| 97 |
+
rollout_manager = manager_class(
|
| 98 |
+
strategy=strategy,
|
| 99 |
+
rollout_group=rollout_group,
|
| 100 |
+
is_same_tokenizer=True,
|
| 101 |
+
generate_kwargs=generate_kwargs,
|
| 102 |
+
)
|
| 103 |
+
summary = evaluator_class(args, rollout_manager).evaluate()
|
| 104 |
+
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
| 105 |
+
return summary
|
| 106 |
+
finally:
|
| 107 |
+
if rollout_manager is not None:
|
| 108 |
+
rollout_manager.close()
|
| 109 |
+
if rollout_group is not None:
|
| 110 |
+
rollout_group.shutdown()
|
| 111 |
+
if initialized_ray:
|
| 112 |
+
ray.shutdown()
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
if __name__ == "__main__":
|
| 116 |
+
from kdflow.arguments import init_args
|
| 117 |
+
|
| 118 |
+
evaluate(init_args("alfworld_eval"))
|
kdflow/cli/train_kd_off_policy.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
import ray
|
| 4 |
+
|
| 5 |
+
from kdflow.ray.train.teacher_group import TeacherActorGroup
|
| 6 |
+
from kdflow.ray.train.student_group import StudentActorGroup
|
| 7 |
+
from kdflow.ray.placement_group import create_placement_group
|
| 8 |
+
from kdflow.trainer import OffPolicyKDTrainer
|
| 9 |
+
from kdflow.datasets import SFTDataset
|
| 10 |
+
from kdflow.datasets.utils import blending_datasets
|
| 11 |
+
from kdflow.models.utils import check_tokenizer_identical
|
| 12 |
+
from kdflow.backend import get_strategy
|
| 13 |
+
from kdflow.arguments import init_args
|
| 14 |
+
from kdflow.arguments.training_args import resolve_max_steps
|
| 15 |
+
from kdflow.utils.distributed_sampler import DistributedSampler
|
| 16 |
+
from kdflow.utils.utils import get_tokenizer
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def train(args):
|
| 20 |
+
# Initialize Ray if not already initialized
|
| 21 |
+
if not ray.is_initialized():
|
| 22 |
+
ray.init(
|
| 23 |
+
runtime_env={
|
| 24 |
+
"env_vars": {
|
| 25 |
+
"TOKENIZERS_PARALLELISM": "true",
|
| 26 |
+
"NCCL_DEBUG": "WARN"
|
| 27 |
+
},
|
| 28 |
+
"working_dir": os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
strategy = get_strategy(args)
|
| 33 |
+
strategy.print(args)
|
| 34 |
+
|
| 35 |
+
# Create placement group for resource allocation
|
| 36 |
+
num_gpus = args.train.num_nodes * args.train.num_gpus_per_node
|
| 37 |
+
pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(num_gpus)
|
| 38 |
+
strategy.print(f"Placement group ready with {num_gpus} GPUs")
|
| 39 |
+
|
| 40 |
+
teacher_model = TeacherActorGroup(
|
| 41 |
+
strategy,
|
| 42 |
+
num_gpus,
|
| 43 |
+
num_gpus_per_node=args.train.num_gpus_per_node,
|
| 44 |
+
num_gpus_per_actor=0.01,
|
| 45 |
+
pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
|
| 46 |
+
)
|
| 47 |
+
student_model = StudentActorGroup(
|
| 48 |
+
args,
|
| 49 |
+
args.train.num_nodes,
|
| 50 |
+
args.train.num_gpus_per_node,
|
| 51 |
+
pg=pg,
|
| 52 |
+
num_gpus_per_actor=0.01,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
# Initialize tokenizers
|
| 56 |
+
student_tokenizer = get_tokenizer(
|
| 57 |
+
args.model.student_name_or_path,
|
| 58 |
+
use_fast=not args.model.disable_fast_tokenizer
|
| 59 |
+
)
|
| 60 |
+
teacher_tokenizer = get_tokenizer(
|
| 61 |
+
args.model.teacher_name_or_path,
|
| 62 |
+
use_fast=not args.model.disable_fast_tokenizer,
|
| 63 |
+
)
|
| 64 |
+
tokenizer_info = check_tokenizer_identical(student_tokenizer, teacher_tokenizer)
|
| 65 |
+
strategy.print(f"Tokenizers {tokenizer_info}")
|
| 66 |
+
if not tokenizer_info.vocab_identical and args.kd.kd_algorithm not in {
|
| 67 |
+
"simple_ctkd",
|
| 68 |
+
"byte_align_ctkd",
|
| 69 |
+
"topk_byte_align_ctkd",
|
| 70 |
+
}:
|
| 71 |
+
raise ValueError(
|
| 72 |
+
"Cross-tokenizer distillation supports simple_ctkd, "
|
| 73 |
+
"byte_align_ctkd, or topk_byte_align_ctkd."
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
# Load and prepare training dataset
|
| 77 |
+
train_data = blending_datasets(
|
| 78 |
+
args.data.train_dataset_path,
|
| 79 |
+
args.data.train_dataset_probs,
|
| 80 |
+
strategy,
|
| 81 |
+
args.train.seed,
|
| 82 |
+
max_count=args.data.max_samples,
|
| 83 |
+
dataset_split=args.data.train_split,
|
| 84 |
+
)
|
| 85 |
+
train_data = train_data.select(range(min(args.data.max_samples, len(train_data))))
|
| 86 |
+
|
| 87 |
+
train_dataset = SFTDataset(
|
| 88 |
+
train_data,
|
| 89 |
+
strategy,
|
| 90 |
+
tokenizer_info=tokenizer_info,
|
| 91 |
+
max_data_num=args.data.max_samples,
|
| 92 |
+
input_template=args.data.input_template,
|
| 93 |
+
num_processors=args.data.preprocess_num_workers,
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
sampler = DistributedSampler(train_dataset, num_replicas=1, rank=0, shuffle=True, seed=args.train.seed, drop_last=True)
|
| 97 |
+
train_dataloader = strategy.setup_dataloader(
|
| 98 |
+
train_dataset,
|
| 99 |
+
args.train.micro_train_batch_size,
|
| 100 |
+
pin_memory=False,
|
| 101 |
+
shuffle=False,
|
| 102 |
+
collate_fn=train_dataset.collate_fn,
|
| 103 |
+
sampler=sampler,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
# Load and prepare evaluation dataset (optional)
|
| 107 |
+
eval_dataloader = None
|
| 108 |
+
if getattr(args.data, "eval_dataset_path", None):
|
| 109 |
+
eval_data = blending_datasets(
|
| 110 |
+
args.data.eval_dataset_path,
|
| 111 |
+
None,
|
| 112 |
+
strategy,
|
| 113 |
+
dataset_split=args.data.eval_split,
|
| 114 |
+
)
|
| 115 |
+
eval_data = eval_data.select(range(min(args.data.max_samples, len(eval_data))))
|
| 116 |
+
eval_dataset = SFTDataset(
|
| 117 |
+
eval_data,
|
| 118 |
+
strategy,
|
| 119 |
+
tokenizer_info=tokenizer_info,
|
| 120 |
+
input_template=args.data.input_template,
|
| 121 |
+
num_processors=args.data.preprocess_num_workers,
|
| 122 |
+
)
|
| 123 |
+
eval_dataloader = strategy.setup_dataloader(
|
| 124 |
+
eval_dataset,
|
| 125 |
+
batch_size=1,
|
| 126 |
+
pin_memory=True,
|
| 127 |
+
shuffle=False,
|
| 128 |
+
drop_last=False,
|
| 129 |
+
collate_fn=eval_dataset.collate_fn,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
# Calculate max training steps
|
| 133 |
+
num_update_steps_per_epoch = len(train_dataset) // args.train.train_batch_size
|
| 134 |
+
max_steps = resolve_max_steps(
|
| 135 |
+
args.train.max_steps,
|
| 136 |
+
args.train.num_epochs,
|
| 137 |
+
num_update_steps_per_epoch,
|
| 138 |
+
)
|
| 139 |
+
strategy.log(f"Max training steps: {max_steps}")
|
| 140 |
+
|
| 141 |
+
# Initialize student model on all workers
|
| 142 |
+
ray.get(student_model.async_init_model_from_pretrained(
|
| 143 |
+
strategy, max_steps,
|
| 144 |
+
teacher_tokenizer=teacher_tokenizer,
|
| 145 |
+
tokenizer_info=tokenizer_info,
|
| 146 |
+
))
|
| 147 |
+
|
| 148 |
+
strategy.log("Models initialized on all student actors")
|
| 149 |
+
|
| 150 |
+
# Initialize OffPolicyKDRayTrainer (Single controller)
|
| 151 |
+
trainer = OffPolicyKDTrainer(
|
| 152 |
+
strategy=strategy,
|
| 153 |
+
student_model=student_model,
|
| 154 |
+
teacher_model=teacher_model,
|
| 155 |
+
train_dataloader=train_dataloader,
|
| 156 |
+
eval_dataloader=eval_dataloader,
|
| 157 |
+
max_steps=max_steps,
|
| 158 |
+
num_update_steps_per_epoch=num_update_steps_per_epoch,
|
| 159 |
+
student_tokenizer=student_tokenizer,
|
| 160 |
+
teacher_tokenizer=teacher_tokenizer,
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
try:
|
| 164 |
+
trainer.fit()
|
| 165 |
+
ray.get(student_model.async_save_model())
|
| 166 |
+
strategy.log("Training completed and model saved.")
|
| 167 |
+
finally:
|
| 168 |
+
teacher_model.shutdown()
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
if __name__ == "__main__":
|
| 172 |
+
args = init_args("off_policy_kd")
|
| 173 |
+
train(args)
|
kdflow/cli/train_kd_on_policy.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
import ray
|
| 4 |
+
|
| 5 |
+
from kdflow.ray.train.teacher_group import TeacherActorGroup
|
| 6 |
+
from kdflow.ray.train.student_group import StudentActorGroup
|
| 7 |
+
from kdflow.ray.rollout.rollout_group import RolloutActorGroup
|
| 8 |
+
from kdflow.ray.placement_group import create_placement_group
|
| 9 |
+
from kdflow.trainer import OnPolicyKDTrainer
|
| 10 |
+
from kdflow.datasets import (
|
| 11 |
+
AlfWorldEpisodeDataset,
|
| 12 |
+
PromptDataset,
|
| 13 |
+
)
|
| 14 |
+
from kdflow.datasets.utils import blending_datasets
|
| 15 |
+
from kdflow.models.utils import check_tokenizer_identical
|
| 16 |
+
from kdflow.backend import get_strategy
|
| 17 |
+
from kdflow.arguments import init_args
|
| 18 |
+
from kdflow.arguments.training_args import resolve_max_steps
|
| 19 |
+
from kdflow.utils.distributed_sampler import DistributedSampler
|
| 20 |
+
from kdflow.utils.utils import get_tokenizer, load_custom_eval_fn
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def train(args):
|
| 24 |
+
# Initialize Ray if not already initialized
|
| 25 |
+
if not ray.is_initialized():
|
| 26 |
+
ray.init(
|
| 27 |
+
runtime_env={
|
| 28 |
+
"env_vars": {
|
| 29 |
+
"TOKENIZERS_PARALLELISM": "true",
|
| 30 |
+
"NCCL_DEBUG": "WARN"
|
| 31 |
+
},
|
| 32 |
+
"working_dir": os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 33 |
+
}
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
strategy = get_strategy(args)
|
| 37 |
+
strategy.print(args)
|
| 38 |
+
|
| 39 |
+
student_tokenizer = get_tokenizer(
|
| 40 |
+
args.model.student_name_or_path,
|
| 41 |
+
use_fast=not args.model.disable_fast_tokenizer,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
# Create placement group for resource allocation
|
| 45 |
+
num_gpus = args.train.num_nodes * args.train.num_gpus_per_node
|
| 46 |
+
pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(num_gpus)
|
| 47 |
+
rollout_group = RolloutActorGroup(
|
| 48 |
+
model_path=args.model.student_name_or_path,
|
| 49 |
+
num_actors=args.rollout.rollout_num_engines,
|
| 50 |
+
tp_size=args.rollout.rollout_tp_size,
|
| 51 |
+
num_gpus_per_node=args.train.num_gpus_per_node,
|
| 52 |
+
enable_memory_saver=True,
|
| 53 |
+
mem_fraction_static=args.rollout.rollout_mem_fraction_static,
|
| 54 |
+
num_gpus_per_actor=0.01,
|
| 55 |
+
pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
|
| 56 |
+
extra_server_args={"disable_piecewise_cuda_graph": True},
|
| 57 |
+
)
|
| 58 |
+
if args.train.enable_sleep:
|
| 59 |
+
rollout_group.sleep()
|
| 60 |
+
|
| 61 |
+
teacher_model = TeacherActorGroup(
|
| 62 |
+
strategy,
|
| 63 |
+
num_gpus,
|
| 64 |
+
num_gpus_per_node=args.train.num_gpus_per_node,
|
| 65 |
+
num_gpus_per_actor=0.01,
|
| 66 |
+
pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
|
| 67 |
+
|
| 68 |
+
)
|
| 69 |
+
student_model = StudentActorGroup(
|
| 70 |
+
args,
|
| 71 |
+
args.train.num_nodes,
|
| 72 |
+
args.train.num_gpus_per_node,
|
| 73 |
+
pg=(pg, reordered_bundle_indices),
|
| 74 |
+
num_gpus_per_actor=0.01,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
# Initialize the teacher tokenizer after inference actors are configured.
|
| 78 |
+
teacher_tokenizer = get_tokenizer(
|
| 79 |
+
args.model.teacher_name_or_path,
|
| 80 |
+
use_fast=not args.model.disable_fast_tokenizer,
|
| 81 |
+
)
|
| 82 |
+
tokenizer_info = check_tokenizer_identical(student_tokenizer, teacher_tokenizer)
|
| 83 |
+
strategy.print(f"Tokenizers {tokenizer_info}")
|
| 84 |
+
if not tokenizer_info.vocab_identical and args.kd.kd_algorithm not in {
|
| 85 |
+
"simple_ctkd",
|
| 86 |
+
"byte_align_ctkd",
|
| 87 |
+
"topk_byte_align_ctkd",
|
| 88 |
+
}:
|
| 89 |
+
raise ValueError(
|
| 90 |
+
"Cross-tokenizer distillation supports simple_ctkd, "
|
| 91 |
+
"byte_align_ctkd, or topk_byte_align_ctkd."
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
environment_rollout = bool(args.rollout.alfworld_config)
|
| 95 |
+
|
| 96 |
+
# Interactive environments sample tasks on reset. Their lightweight
|
| 97 |
+
# datasets are therefore only epoch clocks; ordinary on-policy KD keeps the
|
| 98 |
+
# existing static prompt dataset path.
|
| 99 |
+
if args.rollout.alfworld_config:
|
| 100 |
+
train_dataset = AlfWorldEpisodeDataset(
|
| 101 |
+
args.rollout.alfworld_episodes_per_epoch
|
| 102 |
+
)
|
| 103 |
+
else:
|
| 104 |
+
train_data = blending_datasets(
|
| 105 |
+
args.data.train_dataset_path,
|
| 106 |
+
args.data.train_dataset_probs,
|
| 107 |
+
strategy,
|
| 108 |
+
args.train.seed,
|
| 109 |
+
max_count=args.data.max_samples,
|
| 110 |
+
dataset_split=args.data.train_split,
|
| 111 |
+
)
|
| 112 |
+
train_data = train_data.select(
|
| 113 |
+
range(min(args.data.max_samples, len(train_data)))
|
| 114 |
+
)
|
| 115 |
+
train_dataset = PromptDataset(
|
| 116 |
+
train_data,
|
| 117 |
+
strategy,
|
| 118 |
+
tokenizer_info=tokenizer_info,
|
| 119 |
+
max_data_num=args.data.max_samples,
|
| 120 |
+
input_template=args.data.input_template,
|
| 121 |
+
num_processors=args.data.preprocess_num_workers,
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
sampler = DistributedSampler(train_dataset, num_replicas=1, rank=0, shuffle=True, seed=args.train.seed, drop_last=True)
|
| 125 |
+
train_dataloader = strategy.setup_dataloader(
|
| 126 |
+
train_dataset,
|
| 127 |
+
args.rollout.rollout_batch_size,
|
| 128 |
+
True,
|
| 129 |
+
False,
|
| 130 |
+
collate_fn=train_dataset.collate_fn,
|
| 131 |
+
sampler=sampler,
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
# Load and prepare evaluation dataset (optional)
|
| 135 |
+
eval_dataloader = None
|
| 136 |
+
if getattr(args.data, "eval_dataset_path", None) and not environment_rollout:
|
| 137 |
+
eval_data = blending_datasets(
|
| 138 |
+
args.data.eval_dataset_path,
|
| 139 |
+
None,
|
| 140 |
+
strategy,
|
| 141 |
+
dataset_split=args.data.eval_split,
|
| 142 |
+
)
|
| 143 |
+
eval_data = eval_data.select(range(min(args.data.max_samples, len(eval_data))))
|
| 144 |
+
eval_dataset = PromptDataset(
|
| 145 |
+
eval_data,
|
| 146 |
+
strategy,
|
| 147 |
+
tokenizer_info=tokenizer_info,
|
| 148 |
+
input_template=args.data.input_template,
|
| 149 |
+
num_processors=args.data.preprocess_num_workers,
|
| 150 |
+
)
|
| 151 |
+
eval_dataloader = strategy.setup_dataloader(
|
| 152 |
+
eval_dataset,
|
| 153 |
+
batch_size=args.rollout.rollout_batch_size,
|
| 154 |
+
pin_memory=True,
|
| 155 |
+
shuffle=False,
|
| 156 |
+
drop_last=False,
|
| 157 |
+
collate_fn=eval_dataset.collate_fn,
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
# Calculate max training steps
|
| 161 |
+
num_rollout_iters_per_epoch = len(train_dataset) // args.rollout.rollout_batch_size
|
| 162 |
+
if environment_rollout:
|
| 163 |
+
max_turn_samples = (
|
| 164 |
+
args.rollout.rollout_batch_size * args.rollout.alfworld_max_turns
|
| 165 |
+
)
|
| 166 |
+
num_update_steps_per_rollout = max(
|
| 167 |
+
1,
|
| 168 |
+
(max_turn_samples + args.train.train_batch_size - 1)
|
| 169 |
+
// args.train.train_batch_size,
|
| 170 |
+
)
|
| 171 |
+
else:
|
| 172 |
+
num_update_steps_per_rollout = max(
|
| 173 |
+
1,
|
| 174 |
+
args.rollout.rollout_batch_size
|
| 175 |
+
* args.rollout.n_samples_per_prompt
|
| 176 |
+
// args.train.train_batch_size,
|
| 177 |
+
)
|
| 178 |
+
max_rollout_iters = resolve_max_steps(
|
| 179 |
+
args.train.max_steps,
|
| 180 |
+
args.train.num_epochs,
|
| 181 |
+
num_rollout_iters_per_epoch,
|
| 182 |
+
)
|
| 183 |
+
strategy.log(f"Max training iterations: {max_rollout_iters}")
|
| 184 |
+
|
| 185 |
+
# Initialize student model on all workers
|
| 186 |
+
ray.get(student_model.async_init_model_from_pretrained(
|
| 187 |
+
strategy, (max_rollout_iters * num_update_steps_per_rollout),
|
| 188 |
+
teacher_tokenizer=teacher_tokenizer,
|
| 189 |
+
tokenizer_info=tokenizer_info,
|
| 190 |
+
))
|
| 191 |
+
strategy.log("Models initialized on all student actors")
|
| 192 |
+
|
| 193 |
+
generate_kwargs = {
|
| 194 |
+
"max_new_tokens": args.rollout.generate_max_len,
|
| 195 |
+
"temperature": args.rollout.temperature,
|
| 196 |
+
"top_p": args.rollout.top_p,
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
trainer = None
|
| 200 |
+
try:
|
| 201 |
+
trainer = OnPolicyKDTrainer(
|
| 202 |
+
strategy=strategy,
|
| 203 |
+
student_model=student_model,
|
| 204 |
+
teacher_model=teacher_model,
|
| 205 |
+
rollout_group=rollout_group,
|
| 206 |
+
is_same_tokenizer=tokenizer_info.is_identical,
|
| 207 |
+
train_dataloader=train_dataloader,
|
| 208 |
+
eval_dataloader=eval_dataloader,
|
| 209 |
+
max_rollout_iters=max_rollout_iters,
|
| 210 |
+
num_rollout_iters_per_epoch=num_rollout_iters_per_epoch,
|
| 211 |
+
generate_kwargs=generate_kwargs,
|
| 212 |
+
custom_eval_fn=(
|
| 213 |
+
load_custom_eval_fn(args.data.custom_eval_fn)
|
| 214 |
+
if args.data.custom_eval_fn
|
| 215 |
+
else None
|
| 216 |
+
),
|
| 217 |
+
)
|
| 218 |
+
trainer.fit()
|
| 219 |
+
ray.get(student_model.async_save_model())
|
| 220 |
+
strategy.log("Training completed and model saved.")
|
| 221 |
+
finally:
|
| 222 |
+
if trainer is not None:
|
| 223 |
+
trainer.close()
|
| 224 |
+
teacher_model.shutdown()
|
| 225 |
+
rollout_group.shutdown()
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
if __name__ == "__main__":
|
| 229 |
+
args = init_args("on_policy_kd")
|
| 230 |
+
train(args)
|
kdflow/datasets/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .prompts_dataset import PromptDataset
|
| 2 |
+
from .sft_dataset import SFTDataset
|
| 3 |
+
from .alfworld_dataset import AlfWorldEpisodeDataset
|
| 4 |
+
|
| 5 |
+
__all__ = [
|
| 6 |
+
"PromptDataset",
|
| 7 |
+
"SFTDataset",
|
| 8 |
+
"AlfWorldEpisodeDataset",
|
| 9 |
+
]
|
kdflow/datasets/alfworld_dataset.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, List
|
| 2 |
+
|
| 3 |
+
from torch.utils.data import Dataset
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class AlfWorldEpisodeDataset(Dataset):
|
| 7 |
+
"""A lightweight epoch clock for environment-sampled ALFWorld episodes.
|
| 8 |
+
|
| 9 |
+
ALFWorld chooses a new game whenever its batched environment is reset, so
|
| 10 |
+
dataset rows only determine how many episodes are collected in one epoch.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
def __init__(self, num_episodes: int):
|
| 14 |
+
if num_episodes <= 0:
|
| 15 |
+
raise ValueError("num_episodes must be positive")
|
| 16 |
+
self.num_episodes = int(num_episodes)
|
| 17 |
+
|
| 18 |
+
def __len__(self) -> int:
|
| 19 |
+
return self.num_episodes
|
| 20 |
+
|
| 21 |
+
def __getitem__(self, index: int) -> Dict[str, int]:
|
| 22 |
+
return {"episode_slot": index}
|
| 23 |
+
|
| 24 |
+
@staticmethod
|
| 25 |
+
def collate_fn(batch: List[Dict[str, int]]) -> List[Dict[str, int]]:
|
| 26 |
+
return batch
|
kdflow/datasets/prompts_dataset.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Callable, Optional, Dict, Any, List
|
| 2 |
+
|
| 3 |
+
from torch.utils.data import Dataset
|
| 4 |
+
from datasets import Image as ImageFeature
|
| 5 |
+
from PIL import Image
|
| 6 |
+
|
| 7 |
+
from kdflow.datasets.utils import (
|
| 8 |
+
convert_to_openai_messages,
|
| 9 |
+
get_nested_value,
|
| 10 |
+
get_tokenizer_or_processor,
|
| 11 |
+
validate_dataset_columns,
|
| 12 |
+
)
|
| 13 |
+
from kdflow.models.utils import TokenizerCompareResult
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class PromptDataset(Dataset):
|
| 17 |
+
"""
|
| 18 |
+
Dataset for On-Policy Distillation
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
dataset: dataset for on-policy distillation
|
| 22 |
+
strategy: training strategy object
|
| 23 |
+
tokenizer_info: result of tokenizer comparison (template_identical, vocab_identical)
|
| 24 |
+
max_data_num: maximum number of data to load
|
| 25 |
+
input_template: optional template for formatting input
|
| 26 |
+
num_processors: number of processors for parallel data loading
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
def __init__(
|
| 30 |
+
self,
|
| 31 |
+
dataset,
|
| 32 |
+
strategy,
|
| 33 |
+
tokenizer_info,
|
| 34 |
+
max_data_num: int = None,
|
| 35 |
+
input_template: Optional[str] = None,
|
| 36 |
+
num_processors: int = 8,
|
| 37 |
+
) -> None:
|
| 38 |
+
super().__init__()
|
| 39 |
+
self.args = strategy.args
|
| 40 |
+
self.strategy = strategy
|
| 41 |
+
self.tokenizer_info = tokenizer_info or TokenizerCompareResult()
|
| 42 |
+
self.template_identical = self.tokenizer_info.template_identical
|
| 43 |
+
self.vocab_identical = self.tokenizer_info.vocab_identical
|
| 44 |
+
self.same_tokenizer = self.tokenizer_info.is_identical
|
| 45 |
+
self.strategy = strategy
|
| 46 |
+
self.input_template = input_template
|
| 47 |
+
|
| 48 |
+
self.input_key = getattr(self.args.data, "input_key", None)
|
| 49 |
+
self.teacher_input_key = getattr(self.args.data, "teacher_input_key", None) or self.input_key
|
| 50 |
+
self.label_key = getattr(self.args.data, "label_key", None)
|
| 51 |
+
self.apply_chat_template = getattr(self.args.data, "apply_chat_template", False)
|
| 52 |
+
self.enable_thinking = getattr(self.args.model, "enable_thinking", False)
|
| 53 |
+
self.prompt_max_len = getattr(self.args.data, "prompt_max_len", 0)
|
| 54 |
+
|
| 55 |
+
self.image_key = getattr(self.args.data, "image_key", None)
|
| 56 |
+
|
| 57 |
+
required_columns = {"input_key": self.input_key}
|
| 58 |
+
if self.teacher_input_key != self.input_key:
|
| 59 |
+
required_columns["teacher_input_key"] = self.teacher_input_key
|
| 60 |
+
if self.label_key:
|
| 61 |
+
required_columns["label_key"] = self.label_key
|
| 62 |
+
if self.image_key:
|
| 63 |
+
required_columns["image_key"] = self.image_key
|
| 64 |
+
if self.args.kd.multi_teacher_config:
|
| 65 |
+
required_columns["teacher_routing_key"] = self.args.data.teacher_routing_key
|
| 66 |
+
validate_dataset_columns(dataset, **required_columns)
|
| 67 |
+
|
| 68 |
+
self.student_processor = get_tokenizer_or_processor(
|
| 69 |
+
self.args.model.student_name_or_path,
|
| 70 |
+
need_processor=self.image_key is not None,
|
| 71 |
+
)
|
| 72 |
+
self.teacher_processors = {}
|
| 73 |
+
if self.args.kd.multi_teacher_config:
|
| 74 |
+
for teacher_key, teacher_path in self.args.kd.multi_teacher_config.items():
|
| 75 |
+
self.teacher_processors[teacher_key] = get_tokenizer_or_processor(
|
| 76 |
+
teacher_path, need_processor=self.image_key is not None,
|
| 77 |
+
)
|
| 78 |
+
elif self.args.model.teacher_name_or_path is not None:
|
| 79 |
+
self.teacher_processors["default"] = get_tokenizer_or_processor(
|
| 80 |
+
self.args.model.teacher_name_or_path,
|
| 81 |
+
need_processor=self.image_key is not None,
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
if max_data_num is not None and max_data_num > 0 and max_data_num < len(dataset):
|
| 85 |
+
strategy.log(f"Truncating dataset from {len(dataset)} to {max_data_num}")
|
| 86 |
+
dataset = dataset.select(range(max_data_num))
|
| 87 |
+
|
| 88 |
+
# Adjust worker count based on dataset size to speed up preprocessing.
|
| 89 |
+
num_processors = min(
|
| 90 |
+
num_processors,
|
| 91 |
+
max(1, len(dataset) // 2000),
|
| 92 |
+
)
|
| 93 |
+
self.processed_dataset = dataset.map(
|
| 94 |
+
self.process_data,
|
| 95 |
+
remove_columns=dataset.column_names,
|
| 96 |
+
num_proc=num_processors,
|
| 97 |
+
load_from_cache_file=False,
|
| 98 |
+
desc="Processing data",
|
| 99 |
+
)
|
| 100 |
+
if self.image_key:
|
| 101 |
+
self.processed_dataset = self.processed_dataset.cast_column("images", [ImageFeature()])
|
| 102 |
+
|
| 103 |
+
original_len = len(self.processed_dataset)
|
| 104 |
+
if self.prompt_max_len > 0:
|
| 105 |
+
strategy.log(f"Before prompt_max_len filter: {len(self.processed_dataset)}")
|
| 106 |
+
self.processed_dataset = self.processed_dataset.filter(
|
| 107 |
+
lambda prompt_len: prompt_len <= self.prompt_max_len,
|
| 108 |
+
input_columns=["prompt_len"],
|
| 109 |
+
desc="Filtering overlang samples",
|
| 110 |
+
)
|
| 111 |
+
strategy.log(f"After prompt_max_len filter: {len(self.processed_dataset)}")
|
| 112 |
+
if len(self.processed_dataset) < original_len:
|
| 113 |
+
self.strategy.log(
|
| 114 |
+
f"Filtered {original_len - len(self.processed_dataset)} samples "
|
| 115 |
+
f"exceeding prompt_max_len={self.prompt_max_len} "
|
| 116 |
+
f"({original_len} -> {len(self.processed_dataset)})"
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
self._print_sample()
|
| 120 |
+
|
| 121 |
+
def _print_sample(self) -> None:
|
| 122 |
+
"""Print sample data for debugging."""
|
| 123 |
+
self.strategy.print(f"Total samples: {len(self.processed_dataset)}")
|
| 124 |
+
self.strategy.print(f"Sample student prompt:\n{self.processed_dataset[0]['stu_prompt']}")
|
| 125 |
+
if not self.template_identical or self.teacher_input_key != self.input_key:
|
| 126 |
+
self.strategy.print(f"Sample teacher prompt:\n{self.processed_dataset[0]['tea_prompt']}")
|
| 127 |
+
|
| 128 |
+
def process_data(self, data: Dict) -> Dict[str, Any]:
|
| 129 |
+
"""Process a single data sample."""
|
| 130 |
+
stu_prompt = self._build_prompt(data, self.student_processor, self.input_key)
|
| 131 |
+
|
| 132 |
+
if self.args.kd.multi_teacher_config:
|
| 133 |
+
routing_key = self.args.data.teacher_routing_key
|
| 134 |
+
assert routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
|
| 135 |
+
assert routing_key in data, f"Routing key '{routing_key}' not found in data"
|
| 136 |
+
teacher_key = data[routing_key]
|
| 137 |
+
if teacher_key not in self.teacher_processors:
|
| 138 |
+
raise ValueError(
|
| 139 |
+
f"Teacher routing key '{teacher_key}' not found in multi_teacher_config. "
|
| 140 |
+
f"Available keys: {list(self.teacher_processors.keys())}."
|
| 141 |
+
)
|
| 142 |
+
teacher_processor = self.teacher_processors[teacher_key]
|
| 143 |
+
tea_prompt = self._build_prompt(data, teacher_processor, self.teacher_input_key)
|
| 144 |
+
elif self.same_tokenizer and self.input_key == self.teacher_input_key:
|
| 145 |
+
tea_prompt = stu_prompt
|
| 146 |
+
else:
|
| 147 |
+
tea_prompt = self._build_prompt(data, self.teacher_processors.get("default", self.student_processor), self.teacher_input_key)
|
| 148 |
+
|
| 149 |
+
tokenizer = self.student_processor.tokenizer if hasattr(self.student_processor, "tokenizer") else self.student_processor
|
| 150 |
+
prompt_len = len(tokenizer.encode(stu_prompt))
|
| 151 |
+
|
| 152 |
+
result = {
|
| 153 |
+
"stu_prompt": stu_prompt,
|
| 154 |
+
"tea_prompt": tea_prompt,
|
| 155 |
+
"prompt": stu_prompt,
|
| 156 |
+
"prompt_len": prompt_len,
|
| 157 |
+
"label": get_nested_value(data, self.label_key) or "" if self.label_key else "",
|
| 158 |
+
"datasource": data.get("datasource", "default"),
|
| 159 |
+
}
|
| 160 |
+
if self.image_key:
|
| 161 |
+
result["images"] = self._load_images(data.get(self.image_key))
|
| 162 |
+
|
| 163 |
+
if self.args.kd.multi_teacher_config is not None:
|
| 164 |
+
assert self.args.data.teacher_routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
|
| 165 |
+
assert self.args.data.teacher_routing_key in data, f"Routing key '{self.args.data.teacher_routing_key}' not found in data"
|
| 166 |
+
result["teacher_routing_key"] = data[self.args.data.teacher_routing_key]
|
| 167 |
+
|
| 168 |
+
return result
|
| 169 |
+
|
| 170 |
+
def _build_prompt(self, data: Dict, processor_or_tokenizer, input_key: str) -> str:
|
| 171 |
+
"""Build prompt from data with optional chat template or input template.
|
| 172 |
+
|
| 173 |
+
Args:
|
| 174 |
+
data: The data dict containing input
|
| 175 |
+
processor_or_tokenizer: The processor or tokenizer to use for apply_chat_template
|
| 176 |
+
input_key: The key to extract input from data
|
| 177 |
+
|
| 178 |
+
Returns:
|
| 179 |
+
Formatted prompt string
|
| 180 |
+
"""
|
| 181 |
+
if self.apply_chat_template:
|
| 182 |
+
chat = convert_to_openai_messages(data[input_key], expand_image=self.image_key is not None)
|
| 183 |
+
while chat and chat[-1].get("role", "user") == "assistant":
|
| 184 |
+
chat.pop()
|
| 185 |
+
chat_template_kwargs = {}
|
| 186 |
+
if "enable_thinking" in str(getattr(processor_or_tokenizer, "chat_template", "")):
|
| 187 |
+
chat_template_kwargs["enable_thinking"] = self.enable_thinking
|
| 188 |
+
return processor_or_tokenizer.apply_chat_template(
|
| 189 |
+
chat,
|
| 190 |
+
tokenize=False,
|
| 191 |
+
add_generation_prompt=True,
|
| 192 |
+
**chat_template_kwargs,
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
prompt = data[input_key]
|
| 196 |
+
return self.input_template.format(prompt) if self.input_template else prompt
|
| 197 |
+
|
| 198 |
+
def __len__(self) -> int:
|
| 199 |
+
return len(self.processed_dataset)
|
| 200 |
+
|
| 201 |
+
def __getitem__(self, idx: int) -> Dict[str, str]:
|
| 202 |
+
"""Get item by index.
|
| 203 |
+
|
| 204 |
+
Returns:
|
| 205 |
+
Dict with keys: datasource, stu_prompt, tea_prompt, label, images (optional)
|
| 206 |
+
"""
|
| 207 |
+
item = self.processed_dataset[idx]
|
| 208 |
+
result = {
|
| 209 |
+
"datasource": item["datasource"],
|
| 210 |
+
"stu_prompt": item["stu_prompt"],
|
| 211 |
+
"tea_prompt": item["tea_prompt"],
|
| 212 |
+
"label": item["label"],
|
| 213 |
+
}
|
| 214 |
+
if "images" in item:
|
| 215 |
+
result["images"] = item["images"]
|
| 216 |
+
if "teacher_routing_key" in item:
|
| 217 |
+
result["teacher_routing_key"] = item["teacher_routing_key"]
|
| 218 |
+
return result
|
| 219 |
+
|
| 220 |
+
@staticmethod
|
| 221 |
+
def _load_images(image_content) -> list:
|
| 222 |
+
"""Load image(s) from various input formats. Always returns a list."""
|
| 223 |
+
if image_content is None:
|
| 224 |
+
return []
|
| 225 |
+
if isinstance(image_content, Image.Image):
|
| 226 |
+
return [image_content]
|
| 227 |
+
if isinstance(image_content, str):
|
| 228 |
+
return [Image.open(image_content).convert("RGB")]
|
| 229 |
+
if isinstance(image_content, list):
|
| 230 |
+
result = []
|
| 231 |
+
for img in image_content:
|
| 232 |
+
if isinstance(img, Image.Image):
|
| 233 |
+
result.append(img)
|
| 234 |
+
elif isinstance(img, str):
|
| 235 |
+
result.append(Image.open(img).convert("RGB"))
|
| 236 |
+
return result
|
| 237 |
+
return []
|
| 238 |
+
|
| 239 |
+
@staticmethod
|
| 240 |
+
def collate_fn(batch: List[Dict[str, str]]) -> List[Dict[str, str]]:
|
| 241 |
+
"""Collate function that simply returns the list of dicts.
|
| 242 |
+
|
| 243 |
+
DataLoader will pass a list of dicts, we just return it as-is
|
| 244 |
+
since rollout method expects a list of dicts.
|
| 245 |
+
"""
|
| 246 |
+
return batch
|
kdflow/datasets/sft_dataset.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Callable, Optional, Dict, List, Any
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torch.utils.data import Dataset
|
| 5 |
+
from datasets import Image as ImageFeature
|
| 6 |
+
from PIL import Image
|
| 7 |
+
|
| 8 |
+
from kdflow.datasets.utils import (
|
| 9 |
+
convert_to_openai_messages,
|
| 10 |
+
get_tokenizer_or_processor,
|
| 11 |
+
validate_dataset_columns,
|
| 12 |
+
zero_pad_sequences,
|
| 13 |
+
)
|
| 14 |
+
from kdflow.models.utils import TokenizerCompareResult
|
| 15 |
+
|
| 16 |
+
class SFTDataset(Dataset):
|
| 17 |
+
"""
|
| 18 |
+
Dataset for SFT and Off-Policy KD.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
dataset: dataset for SFT and Off-Policy KD
|
| 22 |
+
strategy: training strategy object
|
| 23 |
+
tokenizer_info: result of tokenizer comparison (template_identical, vocab_identical)
|
| 24 |
+
max_data_num: maximum number of data to load
|
| 25 |
+
input_template: optional template for formatting input
|
| 26 |
+
num_processors: number of processors for parallel data loading
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
def __init__(
|
| 30 |
+
self,
|
| 31 |
+
dataset,
|
| 32 |
+
strategy,
|
| 33 |
+
tokenizer_info: Optional[TokenizerCompareResult] = None,
|
| 34 |
+
max_data_num: int = -1,
|
| 35 |
+
input_template: Optional[str] = None,
|
| 36 |
+
num_processors: int = 8,
|
| 37 |
+
) -> None:
|
| 38 |
+
super().__init__()
|
| 39 |
+
self.args = strategy.args
|
| 40 |
+
self.strategy = strategy
|
| 41 |
+
self.max_length = self.args.data.max_len
|
| 42 |
+
self.tokenizer_info = tokenizer_info
|
| 43 |
+
self.template_identical = True if tokenizer_info is None else self.tokenizer_info.template_identical
|
| 44 |
+
self.vocab_identical = True if tokenizer_info is None else self.tokenizer_info.vocab_identical
|
| 45 |
+
|
| 46 |
+
self.input_template = input_template
|
| 47 |
+
self.input_key = getattr(self.args.data, "input_key", None)
|
| 48 |
+
self.teacher_input_key = getattr(self.args.data, "teacher_input_key", None) or self.input_key
|
| 49 |
+
self.output_key = getattr(self.args.data, "output_key", None)
|
| 50 |
+
self.apply_chat_template = getattr(self.args.data, "apply_chat_template", False)
|
| 51 |
+
self.enable_thinking = getattr(self.args.model, "enable_thinking", False)
|
| 52 |
+
|
| 53 |
+
self.image_key = getattr(self.args.data, "image_key", None)
|
| 54 |
+
self.same_tokenizer = True if tokenizer_info is None else self.tokenizer_info.is_identical
|
| 55 |
+
self.teacher_student_share_input = (
|
| 56 |
+
self.same_tokenizer
|
| 57 |
+
and self.teacher_input_key == self.input_key
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
required_columns = {"input_key": self.input_key}
|
| 61 |
+
if self.output_key:
|
| 62 |
+
required_columns["output_key"] = self.output_key
|
| 63 |
+
if self.image_key:
|
| 64 |
+
required_columns["image_key"] = self.image_key
|
| 65 |
+
has_teacher = (
|
| 66 |
+
self.args.model.teacher_name_or_path is not None
|
| 67 |
+
or self.args.kd.multi_teacher_config is not None
|
| 68 |
+
)
|
| 69 |
+
if has_teacher and self.teacher_input_key != self.input_key:
|
| 70 |
+
required_columns["teacher_input_key"] = self.teacher_input_key
|
| 71 |
+
if self.args.kd.multi_teacher_config:
|
| 72 |
+
required_columns["teacher_routing_key"] = self.args.data.teacher_routing_key
|
| 73 |
+
validate_dataset_columns(dataset, **required_columns)
|
| 74 |
+
|
| 75 |
+
self.student_processor = get_tokenizer_or_processor(
|
| 76 |
+
self.args.model.student_name_or_path,
|
| 77 |
+
need_processor=self.image_key is not None,
|
| 78 |
+
)
|
| 79 |
+
self.student_tokenizer = getattr(self.student_processor, "tokenizer", self.student_processor)
|
| 80 |
+
self.teacher_processors = {}
|
| 81 |
+
if self.args.kd.multi_teacher_config:
|
| 82 |
+
for teacher_key, teacher_path in self.args.kd.multi_teacher_config.items():
|
| 83 |
+
self.teacher_processors[teacher_key] = get_tokenizer_or_processor(
|
| 84 |
+
teacher_path, need_processor=self.image_key is not None,
|
| 85 |
+
)
|
| 86 |
+
elif self.args.model.teacher_name_or_path is not None:
|
| 87 |
+
self.teacher_processors["default"] = get_tokenizer_or_processor(
|
| 88 |
+
self.args.model.teacher_name_or_path,
|
| 89 |
+
need_processor=self.image_key is not None,
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
if max_data_num > 0 and max_data_num < len(dataset):
|
| 93 |
+
self.strategy.log(f"Truncating dataset from {len(dataset)} to {max_data_num}")
|
| 94 |
+
dataset = dataset.select(range(max_data_num))
|
| 95 |
+
|
| 96 |
+
# Adjust worker count based on dataset size to speed up preprocessing.
|
| 97 |
+
num_processors = min(
|
| 98 |
+
num_processors,
|
| 99 |
+
max(1, len(dataset) // 2000),
|
| 100 |
+
)
|
| 101 |
+
self.processed_dataset = dataset.map(
|
| 102 |
+
self.process_data,
|
| 103 |
+
remove_columns=dataset.column_names,
|
| 104 |
+
num_proc=num_processors,
|
| 105 |
+
load_from_cache_file=False,
|
| 106 |
+
desc="Processing data",
|
| 107 |
+
)
|
| 108 |
+
if self.image_key:
|
| 109 |
+
self.processed_dataset = self.processed_dataset.cast_column("images", [ImageFeature()])
|
| 110 |
+
|
| 111 |
+
original_len = len(self.processed_dataset)
|
| 112 |
+
strategy.log(f"Before length filter: {len(self.processed_dataset)}")
|
| 113 |
+
self.processed_dataset = self.processed_dataset.filter(
|
| 114 |
+
lambda stu_input_len: stu_input_len <= self.max_length,
|
| 115 |
+
input_columns=["stu_input_len"],
|
| 116 |
+
desc="Filtering overlang samples",
|
| 117 |
+
)
|
| 118 |
+
if len(self.processed_dataset) < original_len:
|
| 119 |
+
self.strategy.log(
|
| 120 |
+
f"Filtered {original_len - len(self.processed_dataset)} samples "
|
| 121 |
+
f"exceeding max_length={self.max_length} "
|
| 122 |
+
f"({original_len} -> {len(self.processed_dataset)})"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
self._print_sample()
|
| 126 |
+
|
| 127 |
+
def _print_sample(self) -> None:
|
| 128 |
+
sample = self.processed_dataset[0]
|
| 129 |
+
self.strategy.print("Student prompt + response:")
|
| 130 |
+
self.strategy.print(sample["stu_prompt"] + sample["stu_response"])
|
| 131 |
+
if not self.teacher_student_share_input:
|
| 132 |
+
self.strategy.print("Teacher prompt + response:")
|
| 133 |
+
self.strategy.print(sample["tea_prompt"] + sample["tea_response"])
|
| 134 |
+
|
| 135 |
+
def process_data(self, data: Dict) -> Dict[str, Any]:
|
| 136 |
+
stu_chat_template_fn = self.student_processor.apply_chat_template if self.apply_chat_template else None
|
| 137 |
+
stu_prompt, stu_response = self.preprocess_data(
|
| 138 |
+
data, self.input_template, self.input_key, self.output_key,
|
| 139 |
+
apply_chat_template=stu_chat_template_fn,
|
| 140 |
+
)
|
| 141 |
+
stu_eos_token = self.student_tokenizer.eos_token
|
| 142 |
+
if not stu_response.endswith(stu_eos_token):
|
| 143 |
+
stu_response += " " + stu_eos_token
|
| 144 |
+
|
| 145 |
+
result = {"stu_prompt": stu_prompt, "stu_response": stu_response}
|
| 146 |
+
result["stu_resp_len"] = self._compute_token_length(
|
| 147 |
+
self.student_processor, stu_response
|
| 148 |
+
)
|
| 149 |
+
result["stu_input_len"] = self._compute_token_length(
|
| 150 |
+
self.student_processor, stu_prompt
|
| 151 |
+
) + result["stu_resp_len"]
|
| 152 |
+
|
| 153 |
+
if self.image_key is not None:
|
| 154 |
+
result["images"] = self.load_images(data[self.image_key])
|
| 155 |
+
|
| 156 |
+
if self.args.model.teacher_name_or_path is not None \
|
| 157 |
+
or self.args.kd.multi_teacher_config is not None:
|
| 158 |
+
if not self.teacher_student_share_input:
|
| 159 |
+
if self.args.kd.multi_teacher_config is not None:
|
| 160 |
+
routing_key = self.args.data.teacher_routing_key
|
| 161 |
+
assert routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
|
| 162 |
+
assert routing_key in data, f"Routing key '{routing_key}' not found in data"
|
| 163 |
+
teacher_key = data[routing_key]
|
| 164 |
+
if teacher_key not in self.teacher_processors:
|
| 165 |
+
raise ValueError(
|
| 166 |
+
f"Teacher routing key '{teacher_key}' not found in multi_teacher_config. "
|
| 167 |
+
f"Available keys: {list(self.teacher_processors.keys())}."
|
| 168 |
+
)
|
| 169 |
+
teacher_processor = self.teacher_processors[teacher_key]
|
| 170 |
+
else:
|
| 171 |
+
teacher_processor = self.teacher_processors.get("default", self.student_processor)
|
| 172 |
+
cur_teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor)
|
| 173 |
+
|
| 174 |
+
tea_chat_template_fn = teacher_processor.apply_chat_template if self.apply_chat_template else None
|
| 175 |
+
tea_prompt, tea_response = self.preprocess_data(
|
| 176 |
+
data, self.input_template, self.teacher_input_key, self.output_key,
|
| 177 |
+
apply_chat_template=tea_chat_template_fn,
|
| 178 |
+
)
|
| 179 |
+
tea_eos_token = cur_teacher_tokenizer.eos_token
|
| 180 |
+
if not tea_response.endswith(tea_eos_token):
|
| 181 |
+
tea_response += " " + tea_eos_token
|
| 182 |
+
result["tea_prompt"] = tea_prompt
|
| 183 |
+
result["tea_response"] = tea_response
|
| 184 |
+
result["tea_resp_len"] = self._compute_token_length(teacher_processor, tea_response)
|
| 185 |
+
else:
|
| 186 |
+
result["tea_prompt"] = stu_prompt
|
| 187 |
+
result["tea_response"] = stu_response
|
| 188 |
+
result["tea_resp_len"] = result["stu_resp_len"]
|
| 189 |
+
|
| 190 |
+
if self.args.kd.multi_teacher_config is not None:
|
| 191 |
+
assert self.args.data.teacher_routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
|
| 192 |
+
assert self.args.data.teacher_routing_key in data, f"Routing key '{self.args.data.teacher_routing_key}' not found in data"
|
| 193 |
+
result["teacher_routing_key"] = data[self.args.data.teacher_routing_key]
|
| 194 |
+
|
| 195 |
+
return result
|
| 196 |
+
|
| 197 |
+
def preprocess_data(
|
| 198 |
+
self,
|
| 199 |
+
data: Dict,
|
| 200 |
+
input_template: Optional[str] = None,
|
| 201 |
+
input_key: str = "input",
|
| 202 |
+
output_key: Optional[str] = None,
|
| 203 |
+
apply_chat_template: Optional[Callable] = None,
|
| 204 |
+
) -> tuple:
|
| 205 |
+
if not apply_chat_template:
|
| 206 |
+
prompt = data[input_key]
|
| 207 |
+
if input_template:
|
| 208 |
+
prompt = input_template.format(prompt)
|
| 209 |
+
assert output_key is not None
|
| 210 |
+
return prompt, data[output_key]
|
| 211 |
+
|
| 212 |
+
has_image = self.image_key is not None
|
| 213 |
+
if output_key:
|
| 214 |
+
messages = convert_to_openai_messages(data[input_key], expand_image=has_image) + \
|
| 215 |
+
convert_to_openai_messages(data[output_key], role="assistant", expand_image=has_image)
|
| 216 |
+
else:
|
| 217 |
+
messages = convert_to_openai_messages(data[input_key], expand_image=has_image)
|
| 218 |
+
|
| 219 |
+
chat_template_kwargs = {}
|
| 220 |
+
if "enable_thinking" in str(apply_chat_template.__self__.chat_template):
|
| 221 |
+
chat_template_kwargs["enable_thinking"] = self.enable_thinking
|
| 222 |
+
prompt = apply_chat_template(
|
| 223 |
+
messages[:-1], tokenize=False, add_generation_prompt=True,
|
| 224 |
+
**chat_template_kwargs,
|
| 225 |
+
)
|
| 226 |
+
full_text = apply_chat_template(messages, tokenize=False, **chat_template_kwargs)
|
| 227 |
+
response = full_text[len(prompt):].removeprefix("<think>\n\n</think>\n\n").rstrip()
|
| 228 |
+
return prompt, response
|
| 229 |
+
|
| 230 |
+
def __len__(self) -> int:
|
| 231 |
+
return len(self.processed_dataset)
|
| 232 |
+
|
| 233 |
+
def __getitem__(self, idx: int) -> Dict:
|
| 234 |
+
return self.processed_dataset[idx]
|
| 235 |
+
|
| 236 |
+
def load_images(self, image_content):
|
| 237 |
+
"""Load image(s) from various input formats. Always returns a list."""
|
| 238 |
+
if isinstance(image_content, Image.Image):
|
| 239 |
+
return [image_content]
|
| 240 |
+
if isinstance(image_content, str):
|
| 241 |
+
return [Image.open(image_content).convert("RGB")]
|
| 242 |
+
if isinstance(image_content, list):
|
| 243 |
+
return [self._load_single_image(img) for img in image_content]
|
| 244 |
+
return []
|
| 245 |
+
|
| 246 |
+
@staticmethod
|
| 247 |
+
def _load_single_image(img):
|
| 248 |
+
if isinstance(img, Image.Image):
|
| 249 |
+
return img
|
| 250 |
+
if isinstance(img, str):
|
| 251 |
+
return Image.open(img).convert("RGB")
|
| 252 |
+
return None
|
| 253 |
+
|
| 254 |
+
def _compute_token_length(self, processor, text):
|
| 255 |
+
tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
|
| 256 |
+
return len(tokenizer.encode(text, add_special_tokens=False))
|
| 257 |
+
|
| 258 |
+
def _encode_single(self, processor, text, images=None) -> Dict[str, Any]:
|
| 259 |
+
"""Single-sample processor call."""
|
| 260 |
+
kwargs = {"text": [text], "padding": False, "truncation": False,
|
| 261 |
+
"return_tensors": "pt"}
|
| 262 |
+
if images:
|
| 263 |
+
kwargs["images"] = images
|
| 264 |
+
enc = processor(**kwargs)
|
| 265 |
+
skip = {"input_ids", "attention_mask", "mm_token_type_ids"}
|
| 266 |
+
multi_modal_inputs = {k: v for k, v in enc.items() if k not in skip}
|
| 267 |
+
return {
|
| 268 |
+
"input_ids": enc["input_ids"][0],
|
| 269 |
+
"attention_mask": enc["attention_mask"][0],
|
| 270 |
+
"multi_modal_inputs": multi_modal_inputs or None,
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
@staticmethod
|
| 274 |
+
def _build_loss_mask(attn_mask, resp_lens):
|
| 275 |
+
bs, seq_len = attn_mask.shape
|
| 276 |
+
loss_mask = torch.zeros(bs, seq_len, dtype=torch.bool)
|
| 277 |
+
for i in range(bs):
|
| 278 |
+
real_end = attn_mask[i].sum().item()
|
| 279 |
+
start = max(real_end - resp_lens[i] - 1, 0)
|
| 280 |
+
loss_mask[i, start:real_end - 1] = True
|
| 281 |
+
return loss_mask
|
| 282 |
+
|
| 283 |
+
def collate_fn(self, item_list: List[Dict]) -> Dict[str, torch.Tensor]:
|
| 284 |
+
bsz = len(item_list)
|
| 285 |
+
per_sample_images = (
|
| 286 |
+
[item["images"] for item in item_list]
|
| 287 |
+
if self.image_key else [None] * bsz
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
stu_full = [item["stu_prompt"] + item["stu_response"] for item in item_list]
|
| 291 |
+
stu_encs = [
|
| 292 |
+
self._encode_single(self.student_processor, stu_full[i], per_sample_images[i])
|
| 293 |
+
for i in range(bsz)
|
| 294 |
+
]
|
| 295 |
+
stu_pad_id = self.student_tokenizer.pad_token_id
|
| 296 |
+
stu_input_ids = zero_pad_sequences(
|
| 297 |
+
[e["input_ids"] for e in stu_encs], side="right", value=stu_pad_id,
|
| 298 |
+
)
|
| 299 |
+
stu_attn_mask = zero_pad_sequences(
|
| 300 |
+
[e["attention_mask"] for e in stu_encs], side="right", value=0,
|
| 301 |
+
).long()
|
| 302 |
+
stu_loss_mask = self._build_loss_mask(
|
| 303 |
+
stu_attn_mask, [item["stu_resp_len"] for item in item_list],
|
| 304 |
+
)
|
| 305 |
+
|
| 306 |
+
batch: Dict[str, Any] = {
|
| 307 |
+
"stu_input_ids": stu_input_ids,
|
| 308 |
+
"stu_attn_mask": stu_attn_mask,
|
| 309 |
+
"stu_loss_mask": stu_loss_mask,
|
| 310 |
+
}
|
| 311 |
+
if any(e["multi_modal_inputs"] is not None for e in stu_encs):
|
| 312 |
+
batch["stu_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in stu_encs]
|
| 313 |
+
|
| 314 |
+
if "tea_prompt" in item_list[0]:
|
| 315 |
+
if not self.teacher_student_share_input:
|
| 316 |
+
tea_full = [item["tea_prompt"] + item["tea_response"] for item in item_list]
|
| 317 |
+
tea_encs = []
|
| 318 |
+
for i in range(bsz):
|
| 319 |
+
if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]:
|
| 320 |
+
proc = self.teacher_processors[item_list[i]["teacher_routing_key"]]
|
| 321 |
+
else:
|
| 322 |
+
proc = self.teacher_processors.get("default", self.student_processor)
|
| 323 |
+
tea_encs.append(self._encode_single(proc, tea_full[i], per_sample_images[i]))
|
| 324 |
+
if self.args.kd.multi_teacher_config:
|
| 325 |
+
# Multi-teacher mode validates shared vocab with the student.
|
| 326 |
+
tea_pad_id = self.student_tokenizer.pad_token_id
|
| 327 |
+
else:
|
| 328 |
+
teacher_processor = self.teacher_processors.get("default", self.student_processor)
|
| 329 |
+
teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor)
|
| 330 |
+
tea_pad_id = teacher_tokenizer.pad_token_id
|
| 331 |
+
tea_input_ids = zero_pad_sequences(
|
| 332 |
+
[e["input_ids"] for e in tea_encs], side="right", value=tea_pad_id,
|
| 333 |
+
)
|
| 334 |
+
tea_attn_mask = zero_pad_sequences(
|
| 335 |
+
[e["attention_mask"] for e in tea_encs], side="right", value=0,
|
| 336 |
+
).long()
|
| 337 |
+
batch["tea_input_ids"] = tea_input_ids
|
| 338 |
+
batch["tea_attn_mask"] = tea_attn_mask
|
| 339 |
+
batch["tea_loss_mask"] = self._build_loss_mask(
|
| 340 |
+
tea_attn_mask, [item["tea_resp_len"] for item in item_list],
|
| 341 |
+
)
|
| 342 |
+
if any(e["multi_modal_inputs"] is not None for e in tea_encs):
|
| 343 |
+
batch["tea_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in tea_encs]
|
| 344 |
+
else:
|
| 345 |
+
batch["tea_input_ids"] = batch["stu_input_ids"]
|
| 346 |
+
batch["tea_attn_mask"] = batch["stu_attn_mask"]
|
| 347 |
+
batch["tea_loss_mask"] = batch["stu_loss_mask"]
|
| 348 |
+
if "stu_multi_modal_inputs" in batch:
|
| 349 |
+
batch["tea_multi_modal_inputs"] = batch["stu_multi_modal_inputs"]
|
| 350 |
+
|
| 351 |
+
# Teacher feed input_ids for SGLang. Text: reuse encoded ids (exact).
|
| 352 |
+
# Multimodal: tokenize raw text (single image placeholder) so SGLang re-expands it.
|
| 353 |
+
if not self.teacher_student_share_input:
|
| 354 |
+
if self.image_key:
|
| 355 |
+
tea_feed = []
|
| 356 |
+
for i in range(bsz):
|
| 357 |
+
if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]:
|
| 358 |
+
proc = self.teacher_processors[item_list[i]["teacher_routing_key"]]
|
| 359 |
+
else:
|
| 360 |
+
proc = self.teacher_processors.get("default", self.student_processor)
|
| 361 |
+
tokenizer = getattr(proc, "tokenizer", proc)
|
| 362 |
+
tea_feed.append(tokenizer(tea_full[i])["input_ids"])
|
| 363 |
+
batch["tea_feed_input_ids"] = tea_feed
|
| 364 |
+
else:
|
| 365 |
+
batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in tea_encs]
|
| 366 |
+
else:
|
| 367 |
+
if self.image_key:
|
| 368 |
+
batch["tea_feed_input_ids"] = [
|
| 369 |
+
self.student_tokenizer(stu_full[i])["input_ids"] for i in range(bsz)
|
| 370 |
+
]
|
| 371 |
+
else:
|
| 372 |
+
batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in stu_encs]
|
| 373 |
+
|
| 374 |
+
if self.image_key:
|
| 375 |
+
batch["images"] = per_sample_images
|
| 376 |
+
|
| 377 |
+
if "teacher_routing_key" in item_list[0]:
|
| 378 |
+
batch["teacher_routing_key"] = [item["teacher_routing_key"] for item in item_list]
|
| 379 |
+
|
| 380 |
+
return batch
|
kdflow/datasets/utils.py
ADDED
|
@@ -0,0 +1,387 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from typing import List, Dict, Any
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from datasets import DatasetDict, interleave_datasets, load_dataset, load_from_disk
|
| 6 |
+
from transformers import AutoProcessor, AutoTokenizer
|
| 7 |
+
|
| 8 |
+
from kdflow.utils.logging_utils import init_logger
|
| 9 |
+
|
| 10 |
+
logger = init_logger(__name__)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def validate_dataset_columns(dataset, **column_settings):
|
| 14 |
+
"""Validate that configured dataset keys exist in the source dataset.
|
| 15 |
+
|
| 16 |
+
Supports dot-notation for nested keys (e.g., "reward_model.ground_truth" -> top-key "reward_model").
|
| 17 |
+
"""
|
| 18 |
+
available_columns = list(dataset.column_names)
|
| 19 |
+
missing_columns = [
|
| 20 |
+
f"{setting_name}={column_name!r}"
|
| 21 |
+
for setting_name, column_name in column_settings.items()
|
| 22 |
+
if column_name.split(".")[0] not in available_columns
|
| 23 |
+
]
|
| 24 |
+
if missing_columns:
|
| 25 |
+
raise ValueError(
|
| 26 |
+
f"Dataset is missing configured dataset column(s): "
|
| 27 |
+
f"{', '.join(missing_columns)}. Available columns: {available_columns}."
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def exist_and_not_none(d, key):
|
| 32 |
+
return key in d and not d[key] is None
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def get_nested_value(data: dict, key_path: str, default=None):
|
| 36 |
+
"""Get a value from a nested dict using dot-notation key path.
|
| 37 |
+
|
| 38 |
+
Example:
|
| 39 |
+
get_nested_value({'a': {'b': 1}}, 'a.b') -> 1
|
| 40 |
+
get_nested_value({'a': 1}, 'a.b') -> None
|
| 41 |
+
"""
|
| 42 |
+
keys = key_path.split(".")
|
| 43 |
+
value = data
|
| 44 |
+
for k in keys:
|
| 45 |
+
if isinstance(value, dict):
|
| 46 |
+
value = value.get(k)
|
| 47 |
+
else:
|
| 48 |
+
return default
|
| 49 |
+
return value
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def blending_datasets(
|
| 53 |
+
datasets,
|
| 54 |
+
probabilities=None,
|
| 55 |
+
strategy=None,
|
| 56 |
+
seed=42,
|
| 57 |
+
max_count=1e8,
|
| 58 |
+
stopping_strategy="all_exhausted",
|
| 59 |
+
dataset_split="train",
|
| 60 |
+
):
|
| 61 |
+
"""Blend multiple datasets with optional probability sampling.
|
| 62 |
+
|
| 63 |
+
Args:
|
| 64 |
+
datasets (str): Comma-separated list of dataset paths
|
| 65 |
+
probabilities (str, optional): Comma-separated list of probabilities for sampling.
|
| 66 |
+
If None, datasets will be concatenated without probability sampling.
|
| 67 |
+
strategy: Training strategy object
|
| 68 |
+
seed (int): Random seed
|
| 69 |
+
max_count (int): Maximum number of samples per dataset
|
| 70 |
+
"""
|
| 71 |
+
dataset_split = dataset_split or "train"
|
| 72 |
+
datasets = datasets.split(",")
|
| 73 |
+
if probabilities is not None:
|
| 74 |
+
probabilities = list(map(float, probabilities.split(",")))
|
| 75 |
+
assert len(probabilities) == len(datasets)
|
| 76 |
+
|
| 77 |
+
data_list = []
|
| 78 |
+
for i, dataset in enumerate(datasets):
|
| 79 |
+
dataset = dataset.strip()
|
| 80 |
+
logger.info(f"dataset: {dataset}")
|
| 81 |
+
|
| 82 |
+
data_dir = dataset.split("@")[1].strip() if "@" in dataset else None
|
| 83 |
+
dataset = dataset.split("@")[0].strip()
|
| 84 |
+
dataset_basename = os.path.basename(dataset)
|
| 85 |
+
|
| 86 |
+
ext = os.path.splitext(dataset)[-1]
|
| 87 |
+
if ext == ".py" or (
|
| 88 |
+
os.path.isdir(dataset) and os.path.exists(os.path.join(dataset, f"{dataset_basename}.py"))
|
| 89 |
+
):
|
| 90 |
+
data = load_dataset(dataset, trust_remote_code=True)
|
| 91 |
+
logger.info(f"loaded {dataset} with python script")
|
| 92 |
+
elif ext in [".json", ".jsonl", ".csv", ".parquet", ".arrow"]:
|
| 93 |
+
ext = ext.lower().strip(".")
|
| 94 |
+
if ext == "jsonl":
|
| 95 |
+
ext = "json"
|
| 96 |
+
data = load_dataset(ext, data_files={dataset_split: dataset})
|
| 97 |
+
logger.info(f"loaded {dataset} with data_files={dataset}")
|
| 98 |
+
# local dataset saved with `datasets.Dataset.save_to_disk`
|
| 99 |
+
elif os.path.isdir(dataset):
|
| 100 |
+
try:
|
| 101 |
+
data = load_from_disk(dataset)
|
| 102 |
+
logger.info(f"loaded {dataset} from disk")
|
| 103 |
+
except Exception as e:
|
| 104 |
+
data = load_dataset(dataset, data_dir=data_dir)
|
| 105 |
+
logger.info(f"loaded {dataset} from files")
|
| 106 |
+
else:
|
| 107 |
+
data = load_dataset(dataset, data_dir=data_dir)
|
| 108 |
+
logger.info(f"loaded {dataset} from files")
|
| 109 |
+
|
| 110 |
+
if isinstance(data, DatasetDict):
|
| 111 |
+
if dataset_split not in data:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"Dataset {dataset!r} does not contain split {dataset_split!r}. "
|
| 114 |
+
f"Available splits: {list(data)}."
|
| 115 |
+
)
|
| 116 |
+
data = data[dataset_split]
|
| 117 |
+
data = data.select(range(min(max_count, len(data))))
|
| 118 |
+
data_list.append(data)
|
| 119 |
+
|
| 120 |
+
if strategy.is_rank_0():
|
| 121 |
+
print(data_list)
|
| 122 |
+
|
| 123 |
+
if probabilities is None:
|
| 124 |
+
from datasets import concatenate_datasets
|
| 125 |
+
|
| 126 |
+
dataset = concatenate_datasets(data_list)
|
| 127 |
+
else:
|
| 128 |
+
dataset = interleave_datasets(
|
| 129 |
+
data_list,
|
| 130 |
+
probabilities=probabilities,
|
| 131 |
+
seed=seed,
|
| 132 |
+
stopping_strategy=stopping_strategy,
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
return dataset
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def get_tokenizer_or_processor(model_name_or_path, model=None, padding_side="right", use_fast=True, need_processor=False):
|
| 139 |
+
if need_processor:
|
| 140 |
+
return get_processor(model_name_or_path, model, padding_side, use_fast)
|
| 141 |
+
else:
|
| 142 |
+
return get_tokenizer(model_name_or_path, model, padding_side, use_fast)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def get_processor(model_name_or_path, model=None, padding_side="right", use_fast=True):
|
| 146 |
+
processor = AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast)
|
| 147 |
+
processor.tokenizer.padding_side = padding_side
|
| 148 |
+
if processor.tokenizer.pad_token is None:
|
| 149 |
+
logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
|
| 150 |
+
processor.tokenizer.pad_token = processor.tokenizer.eos_token
|
| 151 |
+
processor.tokenizer.pad_token_id = processor.tokenizer.eos_token_id
|
| 152 |
+
if model is not None:
|
| 153 |
+
model.config.text_config.pad_token_id = processor.tokenizer.pad_token_id
|
| 154 |
+
|
| 155 |
+
return processor
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def get_tokenizer(model_name_or_path, model=None, padding_side="right", use_fast=True):
|
| 159 |
+
try:
|
| 160 |
+
# fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style
|
| 161 |
+
# tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it
|
| 162 |
+
# with the Metaspace pre-tokenizer rebuilt from the slow tokenizer.
|
| 163 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
| 164 |
+
model_name_or_path,
|
| 165 |
+
trust_remote_code=True,
|
| 166 |
+
use_fast=use_fast,
|
| 167 |
+
fix_mistral_regex=True,
|
| 168 |
+
)
|
| 169 |
+
except TypeError:
|
| 170 |
+
# Older transformers versions do not accept fix_mistral_regex.
|
| 171 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
| 172 |
+
model_name_or_path,
|
| 173 |
+
trust_remote_code=True,
|
| 174 |
+
use_fast=use_fast,
|
| 175 |
+
)
|
| 176 |
+
tokenizer.padding_side = padding_side
|
| 177 |
+
if tokenizer.pad_token is None:
|
| 178 |
+
logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
|
| 179 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 180 |
+
tokenizer.pad_token_id = tokenizer.eos_token_id
|
| 181 |
+
if model is not None:
|
| 182 |
+
model.config.pad_token_id = tokenizer.pad_token_id
|
| 183 |
+
|
| 184 |
+
return tokenizer
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def convert_token_to_id(token, tokenizer):
|
| 188 |
+
if isinstance(token, str):
|
| 189 |
+
token = tokenizer.encode(token, add_special_tokens=False)
|
| 190 |
+
assert len(token) == 1
|
| 191 |
+
return token[0]
|
| 192 |
+
else:
|
| 193 |
+
raise ValueError("token should be int or str")
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def zero_pad_sequences(
|
| 197 |
+
sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False
|
| 198 |
+
) -> torch.Tensor:
|
| 199 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 200 |
+
assert side in ("left", "right")
|
| 201 |
+
sequences = [seq.squeeze(0) for seq in sequences]
|
| 202 |
+
if side == "left":
|
| 203 |
+
sequences = [seq.flip(dims=0) for seq in sequences]
|
| 204 |
+
padded_sequences = pad_sequence(
|
| 205 |
+
sequences,
|
| 206 |
+
batch_first=True,
|
| 207 |
+
padding_value=value
|
| 208 |
+
)
|
| 209 |
+
if side == "left":
|
| 210 |
+
padded_sequences = torch.flip(padded_sequences, dims=[1])
|
| 211 |
+
return padded_sequences
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True):
|
| 215 |
+
"""Remove the pad token. Return tensors and not lists.
|
| 216 |
+
|
| 217 |
+
Args:
|
| 218 |
+
input_ids shape: [bs, seq_length]
|
| 219 |
+
attention_mask shape: [bs, seq_length]
|
| 220 |
+
Returns:
|
| 221 |
+
no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query.
|
| 222 |
+
"""
|
| 223 |
+
no_padding_batch = []
|
| 224 |
+
for ids, mask in zip(input_ids, attention_mask):
|
| 225 |
+
# Fix for both left and right padding
|
| 226 |
+
ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist()
|
| 227 |
+
no_padding_batch.append(ids)
|
| 228 |
+
return no_padding_batch
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# ShareGPT role mapping to OpenAI roles
|
| 232 |
+
SHAREGPT_ROLE_MAP = {
|
| 233 |
+
"human": "user",
|
| 234 |
+
"user": "user",
|
| 235 |
+
"gpt": "assistant",
|
| 236 |
+
"assistant": "assistant",
|
| 237 |
+
"system": "system",
|
| 238 |
+
}
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _is_sharegpt_format(data):
|
| 242 |
+
"""Check if data is in ShareGPT format (list of dicts with 'from'/'value' keys)."""
|
| 243 |
+
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
| 244 |
+
return False
|
| 245 |
+
return "from" in data[0] and "value" in data[0]
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _is_openai_format(data):
|
| 249 |
+
"""Check if data is already in OpenAI messages format (list of dicts with 'role'/'content' keys)."""
|
| 250 |
+
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
| 251 |
+
return False
|
| 252 |
+
return "role" in data[0] and "content" in data[0]
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _is_alpaca_format(data):
|
| 256 |
+
"""Check if data is in Alpaca format (dict with 'instruction' key)."""
|
| 257 |
+
if not isinstance(data, dict):
|
| 258 |
+
return False
|
| 259 |
+
return "instruction" in data
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def _convert_sharegpt(messages):
|
| 263 |
+
"""Convert ShareGPT format messages to OpenAI messages format.
|
| 264 |
+
|
| 265 |
+
ShareGPT: [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]
|
| 266 |
+
OpenAI: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
|
| 267 |
+
"""
|
| 268 |
+
converted = []
|
| 269 |
+
for msg in messages:
|
| 270 |
+
role = SHAREGPT_ROLE_MAP.get(msg["from"], msg["from"])
|
| 271 |
+
content = msg.get("value") or ""
|
| 272 |
+
converted.append({"role": role, "content": content})
|
| 273 |
+
return converted
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _convert_alpaca(data):
|
| 277 |
+
"""Convert Alpaca format to OpenAI messages format.
|
| 278 |
+
|
| 279 |
+
Alpaca format fields:
|
| 280 |
+
- instruction (required): the main instruction/question
|
| 281 |
+
- input (optional): additional input context
|
| 282 |
+
- output (optional): the expected response
|
| 283 |
+
- system (optional): system prompt
|
| 284 |
+
- history (optional): list of [user_msg, assistant_msg] pairs
|
| 285 |
+
|
| 286 |
+
Returns:
|
| 287 |
+
List of OpenAI messages: [{"role": "...", "content": "..."}, ...]
|
| 288 |
+
"""
|
| 289 |
+
messages = []
|
| 290 |
+
|
| 291 |
+
# System prompt
|
| 292 |
+
system_prompt = data.get("system", "")
|
| 293 |
+
if system_prompt:
|
| 294 |
+
messages.append({"role": "system", "content": system_prompt})
|
| 295 |
+
|
| 296 |
+
# History turns
|
| 297 |
+
history = data.get("history", [])
|
| 298 |
+
for turn in history:
|
| 299 |
+
if isinstance(turn, (list, tuple)) and len(turn) == 2:
|
| 300 |
+
messages.append({"role": "user", "content": turn[0]})
|
| 301 |
+
messages.append({"role": "assistant", "content": turn[1]})
|
| 302 |
+
|
| 303 |
+
# Current instruction + optional input
|
| 304 |
+
instruction = data.get("instruction", "")
|
| 305 |
+
extra_input = data.get("input", "")
|
| 306 |
+
if extra_input:
|
| 307 |
+
user_content = f"{instruction}\n{extra_input}"
|
| 308 |
+
else:
|
| 309 |
+
user_content = instruction
|
| 310 |
+
messages.append({"role": "user", "content": user_content})
|
| 311 |
+
|
| 312 |
+
# Output (response)
|
| 313 |
+
output = data.get("output", "")
|
| 314 |
+
if output:
|
| 315 |
+
messages.append({"role": "assistant", "content": output})
|
| 316 |
+
|
| 317 |
+
return messages
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def convert_to_openai_messages(data, role="user", expand_image=False):
|
| 321 |
+
"""Unified converter: auto-detect data format and convert to OpenAI messages.
|
| 322 |
+
|
| 323 |
+
Supported input formats:
|
| 324 |
+
1. OpenAI messages (already): [{"role": "user", "content": "..."}] -> returned as-is
|
| 325 |
+
2. ShareGPT: [{"from": "human", "value": "..."}] -> converted
|
| 326 |
+
3. Alpaca: {"instruction": "...", "input": "...", "output": "...", ...} -> converted
|
| 327 |
+
4. Plain string: "..." -> wrapped as [{"role": role, "content": "..."}]
|
| 328 |
+
|
| 329 |
+
Args:
|
| 330 |
+
data: Input data in any supported format.
|
| 331 |
+
role: Role to assign when data is a plain string. Defaults to "user".
|
| 332 |
+
expand_image: Whether to expand <image> placeholders into multi-part content.
|
| 333 |
+
Only set to True for multimodal (image) scenarios. Defaults to False.
|
| 334 |
+
|
| 335 |
+
Returns:
|
| 336 |
+
List of dicts in OpenAI messages format: [{"role": "...", "content": "..."}, ...]
|
| 337 |
+
"""
|
| 338 |
+
if data is None:
|
| 339 |
+
raise ValueError("convert_to_openai_messages received None input.")
|
| 340 |
+
|
| 341 |
+
if isinstance(data, str):
|
| 342 |
+
messages = [{"role": role, "content": data}]
|
| 343 |
+
elif isinstance(data, list):
|
| 344 |
+
if not data:
|
| 345 |
+
raise ValueError("convert_to_openai_messages received an empty list.")
|
| 346 |
+
if _is_openai_format(data):
|
| 347 |
+
messages = data
|
| 348 |
+
elif _is_sharegpt_format(data):
|
| 349 |
+
messages = _convert_sharegpt(data)
|
| 350 |
+
else:
|
| 351 |
+
raise ValueError(
|
| 352 |
+
f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
|
| 353 |
+
f"Got: {type(data)} with keys/content: {list(data[0].keys()) if data else data}"
|
| 354 |
+
)
|
| 355 |
+
elif isinstance(data, dict):
|
| 356 |
+
if _is_alpaca_format(data):
|
| 357 |
+
messages = _convert_alpaca(data)
|
| 358 |
+
else:
|
| 359 |
+
raise ValueError(
|
| 360 |
+
f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
|
| 361 |
+
f"Got: {type(data)} with keys/content: {data}"
|
| 362 |
+
)
|
| 363 |
+
else:
|
| 364 |
+
raise ValueError(
|
| 365 |
+
f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
|
| 366 |
+
f"Got: {type(data)}"
|
| 367 |
+
)
|
| 368 |
+
|
| 369 |
+
if expand_image:
|
| 370 |
+
messages = expand_image_placeholders(messages)
|
| 371 |
+
return messages
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def expand_image_placeholders(messages, placeholder="<image>"):
|
| 375 |
+
for msg in messages:
|
| 376 |
+
content = msg.get("content")
|
| 377 |
+
if not isinstance(content, str) or placeholder not in content:
|
| 378 |
+
continue
|
| 379 |
+
parts = content.split(placeholder)
|
| 380 |
+
content_list = []
|
| 381 |
+
for i, part in enumerate(parts):
|
| 382 |
+
if part:
|
| 383 |
+
content_list.append({"type": "text", "text": part})
|
| 384 |
+
if i < len(parts) - 1:
|
| 385 |
+
content_list.append({"type": "image"})
|
| 386 |
+
msg["content"] = content_list
|
| 387 |
+
return messages
|
kdflow/evaluation/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluation helpers."""
|
| 2 |
+
|
| 3 |
+
from kdflow.evaluation.alfworld_evaluator import AlfWorldEvaluator
|
| 4 |
+
from kdflow.evaluation.math_evaluator import MathEvaluator
|
| 5 |
+
from kdflow.evaluation.verl_agent_alfworld_evaluator import (
|
| 6 |
+
VerlAgentAlfWorldEvaluator,
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
__all__ = [
|
| 10 |
+
"AlfWorldEvaluator",
|
| 11 |
+
"MathEvaluator",
|
| 12 |
+
"VerlAgentAlfWorldEvaluator",
|
| 13 |
+
]
|
kdflow/evaluation/alfworld_evaluator.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Standalone ALFWorld evaluation with no teacher or optimizer actors."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
import time
|
| 7 |
+
from datetime import datetime, timezone
|
| 8 |
+
from typing import Any, Dict, List
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class AlfWorldEvaluator:
|
| 14 |
+
"""Evaluate a rollout-only model on a fixed number of ALFWorld episodes."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, args, rollout_manager):
|
| 17 |
+
self.args = args
|
| 18 |
+
self.rollout_manager = rollout_manager
|
| 19 |
+
self.num_episodes = args.rollout.alfworld_eval_episodes
|
| 20 |
+
self.batch_size = args.rollout.rollout_batch_size
|
| 21 |
+
self.output_dir = args.train.save_path
|
| 22 |
+
|
| 23 |
+
if self.num_episodes <= 0:
|
| 24 |
+
raise ValueError("alfworld_eval_episodes must be positive")
|
| 25 |
+
if self.num_episodes % self.batch_size:
|
| 26 |
+
raise ValueError(
|
| 27 |
+
"alfworld_eval_episodes must be divisible by rollout_batch_size "
|
| 28 |
+
f"(got {self.num_episodes} and {self.batch_size})"
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
@staticmethod
|
| 32 |
+
def _trajectory_row(record: Dict[str, Any], batch_index: int) -> Dict[str, Any]:
|
| 33 |
+
output = record.get("output") or {}
|
| 34 |
+
metadata = dict(record.get("metadata") or {})
|
| 35 |
+
row = {
|
| 36 |
+
"batch_index": batch_index,
|
| 37 |
+
"prompt": record.get("stu_prompt", ""),
|
| 38 |
+
"response": output.get("text", ""),
|
| 39 |
+
}
|
| 40 |
+
finish_reason = output.get("finish_reason")
|
| 41 |
+
if finish_reason is None:
|
| 42 |
+
finish_reason = (output.get("meta_info") or {}).get("finish_reason")
|
| 43 |
+
if finish_reason is not None:
|
| 44 |
+
row["finish_reason"] = finish_reason
|
| 45 |
+
row.update(metadata)
|
| 46 |
+
return row
|
| 47 |
+
|
| 48 |
+
def evaluate(self) -> Dict[str, Any]:
|
| 49 |
+
os.makedirs(self.output_dir, exist_ok=True)
|
| 50 |
+
trajectory_path = os.path.join(self.output_dir, "trajectories.jsonl")
|
| 51 |
+
summary_path = os.path.join(self.output_dir, "summary.json")
|
| 52 |
+
num_batches = self.num_episodes // self.batch_size
|
| 53 |
+
sampling_params = {
|
| 54 |
+
"max_new_tokens": self.args.rollout.generate_max_len,
|
| 55 |
+
"temperature": self.args.rollout.temperature,
|
| 56 |
+
"top_p": self.args.rollout.top_p,
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
totals = {
|
| 60 |
+
"episodes": 0.0,
|
| 61 |
+
"episodes_succeeded": 0.0,
|
| 62 |
+
"turns": 0.0,
|
| 63 |
+
"turns_max": 0.0,
|
| 64 |
+
"invalid_actions": 0.0,
|
| 65 |
+
"environment_failures": 0.0,
|
| 66 |
+
"context_truncated": 0.0,
|
| 67 |
+
}
|
| 68 |
+
started_at = time.time()
|
| 69 |
+
|
| 70 |
+
logger.info(
|
| 71 |
+
"Starting standalone ALFWorld evaluation: model=%s split=%s "
|
| 72 |
+
"episodes=%d batch_size=%d",
|
| 73 |
+
self.args.model.student_name_or_path,
|
| 74 |
+
self.args.rollout.alfworld_split,
|
| 75 |
+
self.num_episodes,
|
| 76 |
+
self.batch_size,
|
| 77 |
+
)
|
| 78 |
+
with open(trajectory_path, "w", encoding="utf-8") as trajectory_file:
|
| 79 |
+
for batch_index in range(1, num_batches + 1):
|
| 80 |
+
prompt_batch: List[Dict[str, int]] = [
|
| 81 |
+
{"episode_slot": slot} for slot in range(self.batch_size)
|
| 82 |
+
]
|
| 83 |
+
records, metrics = self.rollout_manager.collect_episodes(
|
| 84 |
+
prompt_batch=prompt_batch,
|
| 85 |
+
global_step=batch_index,
|
| 86 |
+
mode="eval",
|
| 87 |
+
**sampling_params,
|
| 88 |
+
)
|
| 89 |
+
for record in records:
|
| 90 |
+
row = self._trajectory_row(record, batch_index)
|
| 91 |
+
trajectory_file.write(
|
| 92 |
+
json.dumps(row, ensure_ascii=False) + "\n"
|
| 93 |
+
)
|
| 94 |
+
trajectory_file.flush()
|
| 95 |
+
|
| 96 |
+
prefix = "eval/alfworld"
|
| 97 |
+
totals["episodes"] += metrics.get(f"{prefix}/episodes", 0.0)
|
| 98 |
+
totals["episodes_succeeded"] += metrics.get(
|
| 99 |
+
f"{prefix}/episodes_succeeded", 0.0
|
| 100 |
+
)
|
| 101 |
+
totals["turns"] += metrics.get(f"{prefix}/turns/total", 0.0)
|
| 102 |
+
totals["turns_max"] = max(
|
| 103 |
+
totals["turns_max"],
|
| 104 |
+
metrics.get(f"{prefix}/turns/max", 0.0),
|
| 105 |
+
)
|
| 106 |
+
totals["invalid_actions"] += metrics.get(
|
| 107 |
+
f"{prefix}/invalid_action_count", 0.0
|
| 108 |
+
)
|
| 109 |
+
totals["environment_failures"] += metrics.get(
|
| 110 |
+
f"{prefix}/environment_failure_count", 0.0
|
| 111 |
+
)
|
| 112 |
+
totals["context_truncated"] += metrics.get(
|
| 113 |
+
f"{prefix}/context_truncated_count", 0.0
|
| 114 |
+
)
|
| 115 |
+
logger.info(
|
| 116 |
+
"ALFWorld evaluation batch %d/%d complete "
|
| 117 |
+
"(running success rate %.4f)",
|
| 118 |
+
batch_index,
|
| 119 |
+
num_batches,
|
| 120 |
+
totals["episodes_succeeded"] / max(totals["episodes"], 1.0),
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
episodes = totals["episodes"]
|
| 124 |
+
turns = totals["turns"]
|
| 125 |
+
summary = {
|
| 126 |
+
"model": self.args.model.student_name_or_path,
|
| 127 |
+
"split": self.args.rollout.alfworld_split,
|
| 128 |
+
"episodes": int(episodes),
|
| 129 |
+
"episodes_succeeded": int(totals["episodes_succeeded"]),
|
| 130 |
+
"success_rate": totals["episodes_succeeded"] / max(episodes, 1.0),
|
| 131 |
+
"turns": {
|
| 132 |
+
"total": int(turns),
|
| 133 |
+
"mean": turns / max(episodes, 1.0),
|
| 134 |
+
"max": int(totals["turns_max"]),
|
| 135 |
+
},
|
| 136 |
+
"invalid_actions": {
|
| 137 |
+
"count": int(totals["invalid_actions"]),
|
| 138 |
+
"ratio": totals["invalid_actions"] / max(turns, 1.0),
|
| 139 |
+
},
|
| 140 |
+
"environment_failures": {
|
| 141 |
+
"count": int(totals["environment_failures"]),
|
| 142 |
+
"ratio": totals["environment_failures"] / max(turns, 1.0),
|
| 143 |
+
},
|
| 144 |
+
"context_truncated": {
|
| 145 |
+
"count": int(totals["context_truncated"]),
|
| 146 |
+
"ratio": totals["context_truncated"] / max(episodes, 1.0),
|
| 147 |
+
},
|
| 148 |
+
"max_turns_per_episode": self.args.rollout.alfworld_max_turns,
|
| 149 |
+
"sampling": sampling_params,
|
| 150 |
+
"seed": self.args.train.seed,
|
| 151 |
+
"elapsed_seconds": time.time() - started_at,
|
| 152 |
+
"completed_at": datetime.now(timezone.utc).isoformat(),
|
| 153 |
+
"trajectories_path": os.path.abspath(trajectory_path),
|
| 154 |
+
}
|
| 155 |
+
with open(summary_path, "w", encoding="utf-8") as summary_file:
|
| 156 |
+
json.dump(summary, summary_file, ensure_ascii=False, indent=2)
|
| 157 |
+
summary_file.write("\n")
|
| 158 |
+
|
| 159 |
+
logger.info(
|
| 160 |
+
"ALFWorld evaluation complete: success_rate=%.4f summary=%s",
|
| 161 |
+
summary["success_rate"],
|
| 162 |
+
summary_path,
|
| 163 |
+
)
|
| 164 |
+
return summary
|
kdflow/evaluation/math_evaluator.py
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import os
|
| 5 |
+
from typing import Any, Callable, Dict, List, Optional
|
| 6 |
+
|
| 7 |
+
from tqdm import tqdm
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def extract_last_boxed(text: str) -> Optional[str]:
|
| 13 |
+
"""Return the content of the last complete ``\\boxed`` or ``\\fbox``."""
|
| 14 |
+
if not isinstance(text, str):
|
| 15 |
+
return None
|
| 16 |
+
|
| 17 |
+
boxed_index = text.rfind("\\boxed")
|
| 18 |
+
fbox_index = text.rfind("\\fbox")
|
| 19 |
+
start = max(boxed_index, fbox_index)
|
| 20 |
+
if start < 0:
|
| 21 |
+
return None
|
| 22 |
+
|
| 23 |
+
left_brace = text.find("{", start)
|
| 24 |
+
if left_brace < 0:
|
| 25 |
+
return None
|
| 26 |
+
|
| 27 |
+
depth = 0
|
| 28 |
+
for index in range(left_brace, len(text)):
|
| 29 |
+
if text[index] == "{":
|
| 30 |
+
depth += 1
|
| 31 |
+
elif text[index] == "}":
|
| 32 |
+
depth -= 1
|
| 33 |
+
if depth == 0:
|
| 34 |
+
return text[left_brace + 1:index]
|
| 35 |
+
return None
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def verify_math_answer(prediction: Optional[str], label: Any) -> bool:
|
| 39 |
+
"""Compare a boxed prediction with a reference answer using math-verify."""
|
| 40 |
+
try:
|
| 41 |
+
from math_verify import parse, verify
|
| 42 |
+
except ImportError as error:
|
| 43 |
+
raise RuntimeError(
|
| 44 |
+
"Math evaluation requires the 'math-verify' package. "
|
| 45 |
+
"Install the project dependencies before starting training."
|
| 46 |
+
) from error
|
| 47 |
+
|
| 48 |
+
if prediction is None:
|
| 49 |
+
return False
|
| 50 |
+
|
| 51 |
+
try:
|
| 52 |
+
prediction = prediction[:300]
|
| 53 |
+
return bool(
|
| 54 |
+
verify(
|
| 55 |
+
parse(f"\\boxed{{{label}}}"),
|
| 56 |
+
parse(f"\\boxed{{{prediction}}}"),
|
| 57 |
+
)
|
| 58 |
+
)
|
| 59 |
+
except Exception:
|
| 60 |
+
return False
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def score_math_responses(
|
| 64 |
+
responses: List[List[str]],
|
| 65 |
+
labels: List[Any],
|
| 66 |
+
verifier: Callable[[Optional[str], Any], bool] = verify_math_answer,
|
| 67 |
+
) -> tuple[Dict[str, float], List[Dict[str, Any]]]:
|
| 68 |
+
"""Score multiple generated responses per problem and compute pass@k."""
|
| 69 |
+
if len(responses) != len(labels):
|
| 70 |
+
raise ValueError("responses and labels must contain the same number of problems")
|
| 71 |
+
|
| 72 |
+
total_correct = 0
|
| 73 |
+
total_responses = 0
|
| 74 |
+
passed_problems = 0
|
| 75 |
+
results = []
|
| 76 |
+
|
| 77 |
+
for index, (problem_responses, label) in enumerate(zip(responses, labels)):
|
| 78 |
+
predictions = [extract_last_boxed(response) for response in problem_responses]
|
| 79 |
+
correctness = [verifier(prediction, label) for prediction in predictions]
|
| 80 |
+
num_correct = sum(correctness)
|
| 81 |
+
total_correct += num_correct
|
| 82 |
+
total_responses += len(correctness)
|
| 83 |
+
passed_problems += int(any(correctness))
|
| 84 |
+
results.append(
|
| 85 |
+
{
|
| 86 |
+
"index": index,
|
| 87 |
+
"answer": str(label),
|
| 88 |
+
"pred_answers": predictions,
|
| 89 |
+
"responses": problem_responses,
|
| 90 |
+
"acc_list": correctness,
|
| 91 |
+
"num_correct": num_correct,
|
| 92 |
+
}
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
num_problems = len(labels)
|
| 96 |
+
accuracy = total_correct / total_responses if total_responses else 0.0
|
| 97 |
+
pass_at_k = passed_problems / num_problems if num_problems else 0.0
|
| 98 |
+
metrics = {
|
| 99 |
+
"eval/accuracy": accuracy,
|
| 100 |
+
"eval/num_correct": float(total_correct),
|
| 101 |
+
"eval/num_total": float(total_responses),
|
| 102 |
+
"eval/pass_at_k": pass_at_k,
|
| 103 |
+
}
|
| 104 |
+
return metrics, results
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
class MathEvaluator:
|
| 108 |
+
"""Run high-concurrency, multi-sample math evaluation through SGLang."""
|
| 109 |
+
|
| 110 |
+
def __init__(self, strategy, rollout_group, generate_kwargs, student_processor):
|
| 111 |
+
self.args = strategy.args
|
| 112 |
+
self.rollout_group = rollout_group
|
| 113 |
+
self.generate_kwargs = generate_kwargs or {}
|
| 114 |
+
self.student_processor = student_processor
|
| 115 |
+
self.dataset_path = self.args.data.eval_math_dataset_path
|
| 116 |
+
self.n_samples = self.args.data.eval_math_n_samples
|
| 117 |
+
self.max_concurrent = self.args.data.eval_math_max_concurrent
|
| 118 |
+
self.max_tokens = self.args.data.eval_math_max_tokens
|
| 119 |
+
self.timeout = self.args.data.eval_math_timeout
|
| 120 |
+
|
| 121 |
+
# Fail before training if the optional evaluator dependency is unavailable.
|
| 122 |
+
try:
|
| 123 |
+
import math_verify # noqa: F401
|
| 124 |
+
except ImportError as error:
|
| 125 |
+
raise RuntimeError(
|
| 126 |
+
"Math evaluation requires the 'math-verify' package. "
|
| 127 |
+
"Install the project dependencies before starting training."
|
| 128 |
+
) from error
|
| 129 |
+
|
| 130 |
+
def evaluate(self, global_step: int) -> Dict[str, float]:
|
| 131 |
+
eval_data = self._load_dataset()
|
| 132 |
+
prompts, labels = self._build_prompts(eval_data)
|
| 133 |
+
sampling_params = {
|
| 134 |
+
"max_new_tokens": self.max_tokens,
|
| 135 |
+
"temperature": self.generate_kwargs.get("temperature", 0.6),
|
| 136 |
+
"top_p": self.generate_kwargs.get("top_p", 0.95),
|
| 137 |
+
"n": self.n_samples,
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
logger.info(
|
| 141 |
+
"[Math Eval] Starting %d problems at step %d "
|
| 142 |
+
"(n_samples=%d, max_concurrent=%d, timeout=%ds)",
|
| 143 |
+
len(eval_data),
|
| 144 |
+
global_step,
|
| 145 |
+
self.n_samples,
|
| 146 |
+
self.max_concurrent,
|
| 147 |
+
self.timeout,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
if self.args.train.enable_sleep:
|
| 151 |
+
self.rollout_group.wakeup()
|
| 152 |
+
try:
|
| 153 |
+
responses = self._generate(prompts, sampling_params)
|
| 154 |
+
finally:
|
| 155 |
+
if self.args.train.enable_sleep:
|
| 156 |
+
self.rollout_group.sleep()
|
| 157 |
+
|
| 158 |
+
metrics, results = score_math_responses(responses, labels)
|
| 159 |
+
pass_at_k = metrics.pop("eval/pass_at_k")
|
| 160 |
+
if self.n_samples > 1:
|
| 161 |
+
metrics[f"eval/pass@{self.n_samples}"] = pass_at_k
|
| 162 |
+
for result, item in zip(results, eval_data):
|
| 163 |
+
result["problem"] = item["problem"]
|
| 164 |
+
|
| 165 |
+
self._save_results(
|
| 166 |
+
global_step, metrics, results, len(eval_data), pass_at_k
|
| 167 |
+
)
|
| 168 |
+
return metrics
|
| 169 |
+
|
| 170 |
+
def _load_dataset(self) -> List[Dict[str, Any]]:
|
| 171 |
+
data = []
|
| 172 |
+
with open(self.dataset_path, "r", encoding="utf-8") as file:
|
| 173 |
+
for line_number, line in enumerate(file, start=1):
|
| 174 |
+
if not line.strip():
|
| 175 |
+
continue
|
| 176 |
+
try:
|
| 177 |
+
item = json.loads(line)
|
| 178 |
+
except json.JSONDecodeError as error:
|
| 179 |
+
raise ValueError(
|
| 180 |
+
f"Invalid JSON in math evaluation dataset at line {line_number}"
|
| 181 |
+
) from error
|
| 182 |
+
missing = [key for key in ("problem", "answer") if key not in item]
|
| 183 |
+
if missing:
|
| 184 |
+
raise ValueError(
|
| 185 |
+
f"Math evaluation dataset line {line_number} is missing "
|
| 186 |
+
f"field(s): {', '.join(missing)}"
|
| 187 |
+
)
|
| 188 |
+
data.append(item)
|
| 189 |
+
if not data:
|
| 190 |
+
raise ValueError("Math evaluation dataset is empty")
|
| 191 |
+
return data
|
| 192 |
+
|
| 193 |
+
def _build_prompts(
|
| 194 |
+
self, eval_data: List[Dict[str, Any]]
|
| 195 |
+
) -> tuple[List[List[int]], List[Any]]:
|
| 196 |
+
tokenizer = getattr(
|
| 197 |
+
self.student_processor, "tokenizer", self.student_processor
|
| 198 |
+
)
|
| 199 |
+
prompts = []
|
| 200 |
+
labels = []
|
| 201 |
+
for item in eval_data:
|
| 202 |
+
messages = [{"role": "user", "content": str(item["problem"])}]
|
| 203 |
+
template_kwargs = {}
|
| 204 |
+
if "enable_thinking" in str(getattr(tokenizer, "chat_template", "")):
|
| 205 |
+
template_kwargs["enable_thinking"] = self.args.model.enable_thinking
|
| 206 |
+
prompt = tokenizer.apply_chat_template(
|
| 207 |
+
messages,
|
| 208 |
+
tokenize=False,
|
| 209 |
+
add_generation_prompt=True,
|
| 210 |
+
**template_kwargs,
|
| 211 |
+
)
|
| 212 |
+
prompts.append(tokenizer.encode(prompt, add_special_tokens=False))
|
| 213 |
+
labels.append(item["answer"])
|
| 214 |
+
return prompts, labels
|
| 215 |
+
|
| 216 |
+
def _generate(
|
| 217 |
+
self, prompt_token_ids: List[List[int]], sampling_params: Dict[str, Any]
|
| 218 |
+
) -> List[List[str]]:
|
| 219 |
+
if not prompt_token_ids:
|
| 220 |
+
return []
|
| 221 |
+
return asyncio.run(
|
| 222 |
+
self._generate_async(prompt_token_ids, sampling_params)
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
async def _generate_async(
|
| 226 |
+
self, prompt_token_ids: List[List[int]], sampling_params: Dict[str, Any]
|
| 227 |
+
) -> List[List[str]]:
|
| 228 |
+
import aiohttp
|
| 229 |
+
|
| 230 |
+
concurrency = min(len(prompt_token_ids), self.max_concurrent)
|
| 231 |
+
semaphore = asyncio.Semaphore(concurrency)
|
| 232 |
+
results: List[Optional[List[str]]] = [None] * len(prompt_token_ids)
|
| 233 |
+
generate_url = f"{self.rollout_group.router_url}/generate"
|
| 234 |
+
|
| 235 |
+
async def generate_one(index, input_ids, session):
|
| 236 |
+
payload = {
|
| 237 |
+
"input_ids": input_ids,
|
| 238 |
+
"sampling_params": sampling_params,
|
| 239 |
+
}
|
| 240 |
+
async with semaphore:
|
| 241 |
+
try:
|
| 242 |
+
async with session.post(generate_url, json=payload) as response:
|
| 243 |
+
response.raise_for_status()
|
| 244 |
+
output = await response.json()
|
| 245 |
+
if isinstance(output, list):
|
| 246 |
+
texts = [item["text"] for item in output]
|
| 247 |
+
elif isinstance(output, dict):
|
| 248 |
+
texts = output.get("text", "")
|
| 249 |
+
texts = [texts] if isinstance(texts, str) else texts
|
| 250 |
+
else:
|
| 251 |
+
raise TypeError(
|
| 252 |
+
f"Unexpected SGLang response type: {type(output).__name__}"
|
| 253 |
+
)
|
| 254 |
+
if not isinstance(texts, list) or len(texts) != self.n_samples:
|
| 255 |
+
raise ValueError(
|
| 256 |
+
f"SGLang returned {len(texts) if isinstance(texts, list) else 0} "
|
| 257 |
+
f"responses, expected {self.n_samples}"
|
| 258 |
+
)
|
| 259 |
+
results[index] = [str(text).strip() for text in texts]
|
| 260 |
+
except Exception as error:
|
| 261 |
+
logger.warning(
|
| 262 |
+
"[Math Eval] Request %d failed with %s: %s; using empty responses",
|
| 263 |
+
index,
|
| 264 |
+
type(error).__name__,
|
| 265 |
+
error,
|
| 266 |
+
)
|
| 267 |
+
results[index] = [""] * self.n_samples
|
| 268 |
+
|
| 269 |
+
connector = aiohttp.TCPConnector(limit=concurrency)
|
| 270 |
+
timeout = aiohttp.ClientTimeout(
|
| 271 |
+
total=self.timeout if self.timeout > 0 else None
|
| 272 |
+
)
|
| 273 |
+
async with aiohttp.ClientSession(
|
| 274 |
+
connector=connector, trust_env=False, timeout=timeout
|
| 275 |
+
) as session:
|
| 276 |
+
tasks = [
|
| 277 |
+
asyncio.create_task(generate_one(index, ids, session))
|
| 278 |
+
for index, ids in enumerate(prompt_token_ids)
|
| 279 |
+
]
|
| 280 |
+
progress = tqdm(
|
| 281 |
+
total=len(tasks), desc="[Math Eval] Generating", unit="req"
|
| 282 |
+
)
|
| 283 |
+
try:
|
| 284 |
+
for task in asyncio.as_completed(tasks):
|
| 285 |
+
await task
|
| 286 |
+
progress.update(1)
|
| 287 |
+
finally:
|
| 288 |
+
progress.close()
|
| 289 |
+
|
| 290 |
+
return [result or [""] * self.n_samples for result in results]
|
| 291 |
+
|
| 292 |
+
def _save_results(
|
| 293 |
+
self,
|
| 294 |
+
global_step: int,
|
| 295 |
+
metrics: Dict[str, float],
|
| 296 |
+
results: List[Dict[str, Any]],
|
| 297 |
+
num_problems: int,
|
| 298 |
+
pass_at_k: float,
|
| 299 |
+
) -> None:
|
| 300 |
+
eval_dir = os.path.join(self.args.train.save_path, "eval_results")
|
| 301 |
+
os.makedirs(eval_dir, exist_ok=True)
|
| 302 |
+
|
| 303 |
+
result_path = os.path.join(eval_dir, f"step{global_step}.jsonl")
|
| 304 |
+
with open(result_path, "w", encoding="utf-8") as file:
|
| 305 |
+
for result in results:
|
| 306 |
+
file.write(json.dumps(result, ensure_ascii=False) + "\n")
|
| 307 |
+
|
| 308 |
+
summary = {
|
| 309 |
+
"global_step": global_step,
|
| 310 |
+
"eval_dataset": self.dataset_path,
|
| 311 |
+
"n_samples": self.n_samples,
|
| 312 |
+
"num_problems": num_problems,
|
| 313 |
+
"num_total_responses": int(metrics["eval/num_total"]),
|
| 314 |
+
"num_correct": int(metrics["eval/num_correct"]),
|
| 315 |
+
"accuracy": metrics["eval/accuracy"],
|
| 316 |
+
"pass_at_k": pass_at_k,
|
| 317 |
+
}
|
| 318 |
+
summary_path = os.path.join(
|
| 319 |
+
eval_dir, f"step{global_step}_summary.json"
|
| 320 |
+
)
|
| 321 |
+
with open(summary_path, "w", encoding="utf-8") as file:
|
| 322 |
+
json.dump(summary, file, indent=2, ensure_ascii=False)
|
kdflow/evaluation/verl_agent_alfworld_evaluator.py
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Multi-seed result writer for the migrated verl-agent ALFWorld protocol."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
import statistics
|
| 7 |
+
import time
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
+
from typing import Any, Dict, List
|
| 10 |
+
|
| 11 |
+
from kdflow.evaluation.verl_agent_alfworld_protocol import TASK_TYPES
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
logger = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class VerlAgentAlfWorldEvaluator:
|
| 18 |
+
"""Evaluate one model with verl-agent's three-seed ALFWorld recipe."""
|
| 19 |
+
|
| 20 |
+
def __init__(self, args, rollout_manager):
|
| 21 |
+
self.args = args
|
| 22 |
+
self.rollout_manager = rollout_manager
|
| 23 |
+
self.num_episodes = args.rollout.alfworld_eval_episodes
|
| 24 |
+
self.batch_size = args.rollout.rollout_batch_size
|
| 25 |
+
self.output_dir = args.train.save_path
|
| 26 |
+
self.seeds = self.parse_seeds(args.rollout.alfworld_eval_seeds)
|
| 27 |
+
if self.num_episodes <= 0:
|
| 28 |
+
raise ValueError("alfworld_eval_episodes must be positive")
|
| 29 |
+
if self.num_episodes % self.batch_size:
|
| 30 |
+
raise ValueError(
|
| 31 |
+
"alfworld_eval_episodes must be divisible by rollout_batch_size"
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
@staticmethod
|
| 35 |
+
def parse_seeds(value: str) -> List[int]:
|
| 36 |
+
seeds = [int(item.strip()) for item in value.split(",") if item.strip()]
|
| 37 |
+
if not seeds:
|
| 38 |
+
raise ValueError("At least one ALFWorld evaluation seed is required.")
|
| 39 |
+
return seeds
|
| 40 |
+
|
| 41 |
+
@staticmethod
|
| 42 |
+
def _trajectory_row(record: Dict[str, Any], batch_index: int) -> Dict[str, Any]:
|
| 43 |
+
output = record.get("output") or {}
|
| 44 |
+
row = {
|
| 45 |
+
"batch_index": batch_index,
|
| 46 |
+
"prompt": record.get("stu_prompt", ""),
|
| 47 |
+
"response": output.get("text", ""),
|
| 48 |
+
}
|
| 49 |
+
finish_reason = output.get("finish_reason")
|
| 50 |
+
if finish_reason is None:
|
| 51 |
+
finish_reason = (output.get("meta_info") or {}).get("finish_reason")
|
| 52 |
+
if finish_reason is not None:
|
| 53 |
+
row["finish_reason"] = finish_reason
|
| 54 |
+
row.update(record.get("metadata") or {})
|
| 55 |
+
return row
|
| 56 |
+
|
| 57 |
+
def _evaluate_seed(self, script_seed: int) -> Dict[str, Any]:
|
| 58 |
+
if self.rollout_manager is None:
|
| 59 |
+
raise RuntimeError(
|
| 60 |
+
"An ALFWorld rollout manager is required for an unfinished seed."
|
| 61 |
+
)
|
| 62 |
+
seed_dir = os.path.join(self.output_dir, f"seed_{script_seed}")
|
| 63 |
+
os.makedirs(seed_dir, exist_ok=True)
|
| 64 |
+
trajectory_path = os.path.join(seed_dir, "trajectories.jsonl")
|
| 65 |
+
summary_path = os.path.join(seed_dir, "summary.json")
|
| 66 |
+
sampling = {
|
| 67 |
+
"max_new_tokens": self.args.rollout.generate_max_len,
|
| 68 |
+
"temperature": self.args.rollout.temperature,
|
| 69 |
+
"top_p": self.args.rollout.top_p,
|
| 70 |
+
}
|
| 71 |
+
prefix = "eval/alfworld"
|
| 72 |
+
totals = {
|
| 73 |
+
"episodes": 0.0,
|
| 74 |
+
"succeeded": 0.0,
|
| 75 |
+
"turns": 0.0,
|
| 76 |
+
"turns_max": 0.0,
|
| 77 |
+
"format_invalid": 0.0,
|
| 78 |
+
"inadmissible": 0.0,
|
| 79 |
+
"environment_failures": 0.0,
|
| 80 |
+
"prompt_truncated": 0.0,
|
| 81 |
+
}
|
| 82 |
+
task_totals = {
|
| 83 |
+
task: {"episodes": 0.0, "succeeded": 0.0}
|
| 84 |
+
for task in TASK_TYPES + ("unknown",)
|
| 85 |
+
}
|
| 86 |
+
num_batches = self.num_episodes // self.batch_size
|
| 87 |
+
started_at = time.time()
|
| 88 |
+
|
| 89 |
+
with open(trajectory_path, "w", encoding="utf-8") as output_file:
|
| 90 |
+
for batch_index in range(num_batches):
|
| 91 |
+
episode_offset = batch_index * self.batch_size
|
| 92 |
+
self.rollout_manager.set_evaluation_batch(
|
| 93 |
+
script_seed=script_seed,
|
| 94 |
+
episode_offset=episode_offset,
|
| 95 |
+
)
|
| 96 |
+
prompt_batch: List[Dict[str, int]] = [
|
| 97 |
+
{"episode_slot": episode_offset + slot}
|
| 98 |
+
for slot in range(self.batch_size)
|
| 99 |
+
]
|
| 100 |
+
records, metrics = self.rollout_manager.collect_episodes(
|
| 101 |
+
prompt_batch=prompt_batch,
|
| 102 |
+
global_step=batch_index + 1,
|
| 103 |
+
mode="eval",
|
| 104 |
+
**sampling,
|
| 105 |
+
)
|
| 106 |
+
for record in records:
|
| 107 |
+
output_file.write(
|
| 108 |
+
json.dumps(
|
| 109 |
+
self._trajectory_row(record, batch_index + 1),
|
| 110 |
+
ensure_ascii=False,
|
| 111 |
+
)
|
| 112 |
+
+ "\n"
|
| 113 |
+
)
|
| 114 |
+
output_file.flush()
|
| 115 |
+
|
| 116 |
+
totals["episodes"] += metrics.get(f"{prefix}/episodes", 0.0)
|
| 117 |
+
totals["succeeded"] += metrics.get(
|
| 118 |
+
f"{prefix}/episodes_succeeded", 0.0
|
| 119 |
+
)
|
| 120 |
+
totals["turns"] += metrics.get(f"{prefix}/turns/total", 0.0)
|
| 121 |
+
totals["turns_max"] = max(
|
| 122 |
+
totals["turns_max"],
|
| 123 |
+
metrics.get(f"{prefix}/turns/max", 0.0),
|
| 124 |
+
)
|
| 125 |
+
totals["format_invalid"] += metrics.get(
|
| 126 |
+
f"{prefix}/format_invalid_count", 0.0
|
| 127 |
+
)
|
| 128 |
+
totals["inadmissible"] += metrics.get(
|
| 129 |
+
f"{prefix}/inadmissible_action_count", 0.0
|
| 130 |
+
)
|
| 131 |
+
totals["environment_failures"] += metrics.get(
|
| 132 |
+
f"{prefix}/environment_failure_count", 0.0
|
| 133 |
+
)
|
| 134 |
+
totals["prompt_truncated"] += metrics.get(
|
| 135 |
+
f"{prefix}/prompt_truncated_count", 0.0
|
| 136 |
+
)
|
| 137 |
+
for task in task_totals:
|
| 138 |
+
task_totals[task]["episodes"] += metrics.get(
|
| 139 |
+
f"{prefix}/task/{task}/episodes", 0.0
|
| 140 |
+
)
|
| 141 |
+
task_totals[task]["succeeded"] += metrics.get(
|
| 142 |
+
f"{prefix}/task/{task}/succeeded", 0.0
|
| 143 |
+
)
|
| 144 |
+
logger.info(
|
| 145 |
+
"verl-agent ALFWorld seed=%d batch=%d/%d running_success=%.4f",
|
| 146 |
+
script_seed,
|
| 147 |
+
batch_index + 1,
|
| 148 |
+
num_batches,
|
| 149 |
+
totals["succeeded"] / max(totals["episodes"], 1.0),
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
episodes = totals["episodes"]
|
| 153 |
+
turns = totals["turns"]
|
| 154 |
+
tasks = {}
|
| 155 |
+
for task, values in task_totals.items():
|
| 156 |
+
if not values["episodes"]:
|
| 157 |
+
continue
|
| 158 |
+
tasks[task] = {
|
| 159 |
+
"episodes": int(values["episodes"]),
|
| 160 |
+
"succeeded": int(values["succeeded"]),
|
| 161 |
+
"success_rate": values["succeeded"] / values["episodes"],
|
| 162 |
+
}
|
| 163 |
+
summary = {
|
| 164 |
+
"protocol": "verl_agent",
|
| 165 |
+
"source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8",
|
| 166 |
+
"model": self.args.model.student_name_or_path,
|
| 167 |
+
"split": self.args.rollout.alfworld_split,
|
| 168 |
+
"script_seed": script_seed,
|
| 169 |
+
"environment_seed_base": (
|
| 170 |
+
script_seed + self.args.rollout.alfworld_eval_seed_offset
|
| 171 |
+
),
|
| 172 |
+
"episodes": int(episodes),
|
| 173 |
+
"episodes_succeeded": int(totals["succeeded"]),
|
| 174 |
+
"success_rate": totals["succeeded"] / max(episodes, 1.0),
|
| 175 |
+
"turns": {
|
| 176 |
+
"total": int(turns),
|
| 177 |
+
"mean": turns / max(episodes, 1.0),
|
| 178 |
+
"max": int(totals["turns_max"]),
|
| 179 |
+
},
|
| 180 |
+
"format_invalid": {
|
| 181 |
+
"count": int(totals["format_invalid"]),
|
| 182 |
+
"ratio": totals["format_invalid"] / max(turns, 1.0),
|
| 183 |
+
},
|
| 184 |
+
"inadmissible_actions": {
|
| 185 |
+
"count": int(totals["inadmissible"]),
|
| 186 |
+
"ratio": totals["inadmissible"] / max(turns, 1.0),
|
| 187 |
+
},
|
| 188 |
+
"environment_failures": {
|
| 189 |
+
"count": int(totals["environment_failures"]),
|
| 190 |
+
"ratio": totals["environment_failures"] / max(turns, 1.0),
|
| 191 |
+
},
|
| 192 |
+
"prompt_truncated": {
|
| 193 |
+
"count": int(totals["prompt_truncated"]),
|
| 194 |
+
"ratio": totals["prompt_truncated"] / max(turns, 1.0),
|
| 195 |
+
},
|
| 196 |
+
"task_types": tasks,
|
| 197 |
+
"history_length": self.args.rollout.alfworld_history_length,
|
| 198 |
+
"max_steps": self.args.rollout.alfworld_max_turns,
|
| 199 |
+
"sampling": sampling,
|
| 200 |
+
"elapsed_seconds": time.time() - started_at,
|
| 201 |
+
"completed_at": datetime.now(timezone.utc).isoformat(),
|
| 202 |
+
"trajectories_path": os.path.abspath(trajectory_path),
|
| 203 |
+
}
|
| 204 |
+
with open(summary_path, "w", encoding="utf-8") as output_file:
|
| 205 |
+
json.dump(summary, output_file, ensure_ascii=False, indent=2)
|
| 206 |
+
output_file.write("\n")
|
| 207 |
+
return summary
|
| 208 |
+
|
| 209 |
+
def _load_completed_seed(self, script_seed: int):
|
| 210 |
+
"""Return a compatible completed seed summary, otherwise ``None``."""
|
| 211 |
+
seed_dir = os.path.join(self.output_dir, f"seed_{script_seed}")
|
| 212 |
+
summary_path = os.path.join(seed_dir, "summary.json")
|
| 213 |
+
trajectory_path = os.path.join(seed_dir, "trajectories.jsonl")
|
| 214 |
+
if not os.path.isfile(summary_path) or not os.path.isfile(trajectory_path):
|
| 215 |
+
return None
|
| 216 |
+
if os.path.getsize(trajectory_path) <= 0:
|
| 217 |
+
logger.warning(
|
| 218 |
+
"Existing ALFWorld seed=%d trajectory is empty; rerunning.",
|
| 219 |
+
script_seed,
|
| 220 |
+
)
|
| 221 |
+
return None
|
| 222 |
+
|
| 223 |
+
try:
|
| 224 |
+
with open(summary_path, "r", encoding="utf-8") as input_file:
|
| 225 |
+
summary = json.load(input_file)
|
| 226 |
+
except (OSError, json.JSONDecodeError) as error:
|
| 227 |
+
logger.warning(
|
| 228 |
+
"Cannot read existing ALFWorld seed=%d summary (%s); rerunning.",
|
| 229 |
+
script_seed,
|
| 230 |
+
error,
|
| 231 |
+
)
|
| 232 |
+
return None
|
| 233 |
+
|
| 234 |
+
expected_sampling = {
|
| 235 |
+
"max_new_tokens": self.args.rollout.generate_max_len,
|
| 236 |
+
"temperature": self.args.rollout.temperature,
|
| 237 |
+
"top_p": self.args.rollout.top_p,
|
| 238 |
+
}
|
| 239 |
+
expected = {
|
| 240 |
+
"protocol": "verl_agent",
|
| 241 |
+
"source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8",
|
| 242 |
+
"model": self.args.model.student_name_or_path,
|
| 243 |
+
"split": self.args.rollout.alfworld_split,
|
| 244 |
+
"script_seed": script_seed,
|
| 245 |
+
"environment_seed_base": (
|
| 246 |
+
script_seed + self.args.rollout.alfworld_eval_seed_offset
|
| 247 |
+
),
|
| 248 |
+
"episodes": self.num_episodes,
|
| 249 |
+
"history_length": self.args.rollout.alfworld_history_length,
|
| 250 |
+
"max_steps": self.args.rollout.alfworld_max_turns,
|
| 251 |
+
"sampling": expected_sampling,
|
| 252 |
+
}
|
| 253 |
+
mismatches = [
|
| 254 |
+
key for key, value in expected.items() if summary.get(key) != value
|
| 255 |
+
]
|
| 256 |
+
required_fields = {
|
| 257 |
+
"episodes_succeeded",
|
| 258 |
+
"success_rate",
|
| 259 |
+
"turns",
|
| 260 |
+
"task_types",
|
| 261 |
+
}
|
| 262 |
+
missing_fields = sorted(required_fields - summary.keys())
|
| 263 |
+
if mismatches or missing_fields:
|
| 264 |
+
logger.warning(
|
| 265 |
+
"Existing ALFWorld seed=%d result is incompatible "
|
| 266 |
+
"(mismatched=%s missing=%s); rerunning.",
|
| 267 |
+
script_seed,
|
| 268 |
+
mismatches,
|
| 269 |
+
missing_fields,
|
| 270 |
+
)
|
| 271 |
+
return None
|
| 272 |
+
turns = summary.get("turns") or {}
|
| 273 |
+
if not {"total", "mean", "max"}.issubset(turns):
|
| 274 |
+
logger.warning(
|
| 275 |
+
"Existing ALFWorld seed=%d result has incomplete turn metrics; "
|
| 276 |
+
"rerunning.",
|
| 277 |
+
script_seed,
|
| 278 |
+
)
|
| 279 |
+
return None
|
| 280 |
+
return summary
|
| 281 |
+
|
| 282 |
+
def has_pending_seeds(self) -> bool:
|
| 283 |
+
"""Whether evaluation must load a model and generate new trajectories."""
|
| 284 |
+
if not getattr(self.args.rollout, "alfworld_eval_resume", True):
|
| 285 |
+
return True
|
| 286 |
+
return any(self._load_completed_seed(seed) is None for seed in self.seeds)
|
| 287 |
+
|
| 288 |
+
def evaluate(self) -> Dict[str, Any]:
|
| 289 |
+
os.makedirs(self.output_dir, exist_ok=True)
|
| 290 |
+
started_at = time.time()
|
| 291 |
+
resume_enabled = getattr(
|
| 292 |
+
self.args.rollout, "alfworld_eval_resume", True
|
| 293 |
+
)
|
| 294 |
+
seed_summaries = []
|
| 295 |
+
skipped_existing_seeds = []
|
| 296 |
+
evaluated_seeds = []
|
| 297 |
+
for seed in self.seeds:
|
| 298 |
+
existing = self._load_completed_seed(seed) if resume_enabled else None
|
| 299 |
+
if existing is not None:
|
| 300 |
+
logger.info(
|
| 301 |
+
"Skipping completed compatible ALFWorld seed=%d result.",
|
| 302 |
+
seed,
|
| 303 |
+
)
|
| 304 |
+
seed_summaries.append(existing)
|
| 305 |
+
skipped_existing_seeds.append(seed)
|
| 306 |
+
continue
|
| 307 |
+
seed_summaries.append(self._evaluate_seed(seed))
|
| 308 |
+
evaluated_seeds.append(seed)
|
| 309 |
+
rates = [summary["success_rate"] for summary in seed_summaries]
|
| 310 |
+
total_episodes = sum(summary["episodes"] for summary in seed_summaries)
|
| 311 |
+
total_succeeded = sum(
|
| 312 |
+
summary["episodes_succeeded"] for summary in seed_summaries
|
| 313 |
+
)
|
| 314 |
+
total_turns = sum(
|
| 315 |
+
summary["turns"]["total"] for summary in seed_summaries
|
| 316 |
+
)
|
| 317 |
+
seed_turn_means = [
|
| 318 |
+
summary["turns"]["mean"] for summary in seed_summaries
|
| 319 |
+
]
|
| 320 |
+
max_turns = max(
|
| 321 |
+
(summary["turns"]["max"] for summary in seed_summaries),
|
| 322 |
+
default=0,
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
task_names = set().union(
|
| 326 |
+
*(summary["task_types"].keys() for summary in seed_summaries)
|
| 327 |
+
)
|
| 328 |
+
task_summary = {}
|
| 329 |
+
for task in sorted(task_names):
|
| 330 |
+
per_seed_rates = [
|
| 331 |
+
summary["task_types"][task]["success_rate"]
|
| 332 |
+
for summary in seed_summaries
|
| 333 |
+
if task in summary["task_types"]
|
| 334 |
+
]
|
| 335 |
+
task_episodes = sum(
|
| 336 |
+
summary["task_types"].get(task, {}).get("episodes", 0)
|
| 337 |
+
for summary in seed_summaries
|
| 338 |
+
)
|
| 339 |
+
task_succeeded = sum(
|
| 340 |
+
summary["task_types"].get(task, {}).get("succeeded", 0)
|
| 341 |
+
for summary in seed_summaries
|
| 342 |
+
)
|
| 343 |
+
task_summary[task] = {
|
| 344 |
+
"episodes": task_episodes,
|
| 345 |
+
"succeeded": task_succeeded,
|
| 346 |
+
"micro_success_rate": task_succeeded / max(task_episodes, 1),
|
| 347 |
+
"seed_mean_success_rate": statistics.fmean(per_seed_rates),
|
| 348 |
+
"seed_std_success_rate": (
|
| 349 |
+
statistics.pstdev(per_seed_rates)
|
| 350 |
+
if len(per_seed_rates) > 1
|
| 351 |
+
else 0.0
|
| 352 |
+
),
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
+
summary = {
|
| 356 |
+
"protocol": "verl_agent",
|
| 357 |
+
"source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8",
|
| 358 |
+
"model": self.args.model.student_name_or_path,
|
| 359 |
+
"split": self.args.rollout.alfworld_split,
|
| 360 |
+
"seeds": self.seeds,
|
| 361 |
+
"environment_seed_offset": self.args.rollout.alfworld_eval_seed_offset,
|
| 362 |
+
"resume_enabled": resume_enabled,
|
| 363 |
+
"skipped_existing_seeds": skipped_existing_seeds,
|
| 364 |
+
"evaluated_seeds": evaluated_seeds,
|
| 365 |
+
"episodes_per_seed": self.num_episodes,
|
| 366 |
+
"total_episodes": total_episodes,
|
| 367 |
+
"total_succeeded": total_succeeded,
|
| 368 |
+
"micro_success_rate": total_succeeded / max(total_episodes, 1),
|
| 369 |
+
"seed_mean_success_rate": statistics.fmean(rates),
|
| 370 |
+
"seed_std_success_rate": (
|
| 371 |
+
statistics.pstdev(rates) if len(rates) > 1 else 0.0
|
| 372 |
+
),
|
| 373 |
+
"turns": {
|
| 374 |
+
"total": total_turns,
|
| 375 |
+
"micro_mean": total_turns / max(total_episodes, 1),
|
| 376 |
+
"seed_mean": statistics.fmean(seed_turn_means),
|
| 377 |
+
"seed_std": (
|
| 378 |
+
statistics.pstdev(seed_turn_means)
|
| 379 |
+
if len(seed_turn_means) > 1
|
| 380 |
+
else 0.0
|
| 381 |
+
),
|
| 382 |
+
"max": max_turns,
|
| 383 |
+
},
|
| 384 |
+
"task_types": task_summary,
|
| 385 |
+
"history_length": self.args.rollout.alfworld_history_length,
|
| 386 |
+
"max_steps": self.args.rollout.alfworld_max_turns,
|
| 387 |
+
"sampling": {
|
| 388 |
+
"max_new_tokens": self.args.rollout.generate_max_len,
|
| 389 |
+
"temperature": self.args.rollout.temperature,
|
| 390 |
+
"top_p": self.args.rollout.top_p,
|
| 391 |
+
},
|
| 392 |
+
"seed_summaries": [
|
| 393 |
+
os.path.join(f"seed_{seed}", "summary.json")
|
| 394 |
+
for seed in self.seeds
|
| 395 |
+
],
|
| 396 |
+
"elapsed_seconds": time.time() - started_at,
|
| 397 |
+
"completed_at": datetime.now(timezone.utc).isoformat(),
|
| 398 |
+
}
|
| 399 |
+
summary_path = os.path.join(self.output_dir, "summary.json")
|
| 400 |
+
with open(summary_path, "w", encoding="utf-8") as output_file:
|
| 401 |
+
json.dump(summary, output_file, ensure_ascii=False, indent=2)
|
| 402 |
+
output_file.write("\n")
|
| 403 |
+
logger.info(
|
| 404 |
+
"verl-agent ALFWorld evaluation complete: mean=%.4f std=%.4f "
|
| 405 |
+
"turns=%.2f",
|
| 406 |
+
summary["seed_mean_success_rate"],
|
| 407 |
+
summary["seed_std_success_rate"],
|
| 408 |
+
summary["turns"]["micro_mean"],
|
| 409 |
+
)
|
| 410 |
+
return summary
|
kdflow/evaluation/verl_agent_alfworld_protocol.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 Nanyang Technological University (NTU), Singapore
|
| 2 |
+
# and the verl-agent (GiGPO) team.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Adapted for KDFlow from langfengq/verl-agent at commit
|
| 11 |
+
# 20bd331bdbc9026a5668e11362178e10ab7400c8.
|
| 12 |
+
|
| 13 |
+
"""Dependency-free prompts and projection for verl-agent ALFWorld."""
|
| 14 |
+
|
| 15 |
+
import re
|
| 16 |
+
from typing import Tuple
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
ALFWORLD_TEMPLATE_NO_HISTORY = """
|
| 20 |
+
You are an expert agent operating in the ALFRED Embodied Environment.
|
| 21 |
+
Your current observation is: {current_observation}
|
| 22 |
+
Your admissible actions of the current situation are: [{admissible_actions}].
|
| 23 |
+
|
| 24 |
+
Now it's your turn to take an action.
|
| 25 |
+
You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within <think> </think> tags.
|
| 26 |
+
Once you've finished your reasoning, you should choose an admissible action for current step and present it within <action> </action> tags.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
ALFWORLD_TEMPLATE_WITH_HISTORY = """
|
| 30 |
+
You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description}
|
| 31 |
+
Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history}
|
| 32 |
+
You are now at step {current_step} and your current observation is: {current_observation}
|
| 33 |
+
Your admissible actions of the current situation are: [{admissible_actions}].
|
| 34 |
+
|
| 35 |
+
Now it's your turn to take an action.
|
| 36 |
+
You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within <think> </think> tags.
|
| 37 |
+
Once you've finished your reasoning, you should choose an admissible action for current step and present it within <action> </action> tags.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
TASK_TYPES = (
|
| 41 |
+
"pick_and_place",
|
| 42 |
+
"pick_two_obj_and_place",
|
| 43 |
+
"look_at_obj_in_light",
|
| 44 |
+
"pick_heat_then_place_in_recep",
|
| 45 |
+
"pick_cool_then_place_in_recep",
|
| 46 |
+
"pick_clean_then_place_in_recep",
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def project_alfworld_action(response: str) -> Tuple[str, bool]:
|
| 51 |
+
"""Reproduce verl-agent's action projection and format-validity flag."""
|
| 52 |
+
original = response or ""
|
| 53 |
+
lowered = original.lower()
|
| 54 |
+
start_tag = "<action>"
|
| 55 |
+
end_tag = "</action>"
|
| 56 |
+
start_index = lowered.find(start_tag)
|
| 57 |
+
end_index = lowered.find(end_tag)
|
| 58 |
+
|
| 59 |
+
if start_index == -1 or end_index == -1:
|
| 60 |
+
return lowered[-30:], False
|
| 61 |
+
|
| 62 |
+
action = lowered[start_index + len(start_tag) : end_index].strip().lower()
|
| 63 |
+
valid = True
|
| 64 |
+
if original.find("<think>") == -1 or original.find("</think>") == -1:
|
| 65 |
+
valid = False
|
| 66 |
+
if re.search(r"[\u4e00-\u9fff]", original):
|
| 67 |
+
valid = False
|
| 68 |
+
return action, valid
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def task_type_from_gamefile(gamefile: str) -> str:
|
| 72 |
+
for task_type in TASK_TYPES:
|
| 73 |
+
if task_type in (gamefile or ""):
|
| 74 |
+
return task_type
|
| 75 |
+
return "unknown"
|
kdflow/loss/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib
|
| 2 |
+
import os
|
| 3 |
+
import glob
|
| 4 |
+
from functools import partial
|
| 5 |
+
|
| 6 |
+
LOSS_DICT = {}
|
| 7 |
+
|
| 8 |
+
def register_loss(name):
|
| 9 |
+
def decorator(fn):
|
| 10 |
+
LOSS_DICT[name] = fn
|
| 11 |
+
return fn
|
| 12 |
+
return decorator
|
| 13 |
+
|
| 14 |
+
_current_dir = os.path.dirname(os.path.abspath(__file__))
|
| 15 |
+
_module_files = glob.glob(os.path.join(_current_dir, "*.py"))
|
| 16 |
+
|
| 17 |
+
for f in _module_files:
|
| 18 |
+
if f.endswith("__init__.py"):
|
| 19 |
+
continue
|
| 20 |
+
module_name = os.path.basename(f)[:-3]
|
| 21 |
+
importlib.import_module(f"kdflow.loss.{module_name}")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def build_loss_fn(name, args):
|
| 25 |
+
"""Build a loss function with args-specific hyperparams pre-bound via functools.partial."""
|
| 26 |
+
fn = LOSS_DICT[name]
|
| 27 |
+
kd = args.kd
|
| 28 |
+
|
| 29 |
+
common = {"temperature": kd.kd_temperature}
|
| 30 |
+
|
| 31 |
+
extra_params = {
|
| 32 |
+
"jsd": {"jsd_beta": kd.jsd_beta},
|
| 33 |
+
"skl": {"skew_lambda": kd.skew_lambda},
|
| 34 |
+
"srkl": {"skew_lambda": kd.skew_lambda},
|
| 35 |
+
"akl": {"adaptive_kl_alpha": kd.adaptive_alpha},
|
| 36 |
+
"hrl": {"hrl_topk": kd.hrl_topk},
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
return partial(fn, **common, **extra_params.get(name, {}))
|
kdflow/loss/adaptive_kl_div.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
|
| 4 |
+
from kdflow.loss import register_loss
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@register_loss("akl")
|
| 8 |
+
@torch.compile()
|
| 9 |
+
def compute_adaptive_kl_div(
|
| 10 |
+
student_logits,
|
| 11 |
+
teacher_logits,
|
| 12 |
+
temperature=1.0,
|
| 13 |
+
adaptive_kl_alpha=0.5,
|
| 14 |
+
reduction="none",
|
| 15 |
+
**kwargs
|
| 16 |
+
):
|
| 17 |
+
"""Adaptive KL Divergence loss (https://arxiv.org/abs/2404.02657)"""
|
| 18 |
+
student_logits = student_logits / temperature
|
| 19 |
+
teacher_logits = teacher_logits / temperature
|
| 20 |
+
student_probs = torch.softmax(student_logits, dim=-1, dtype=torch.float32)
|
| 21 |
+
teacher_probs = torch.softmax(teacher_logits, dim=-1, dtype=torch.float32)
|
| 22 |
+
sorted_teacher_probs, sorted_idx = teacher_probs.sort(-1)
|
| 23 |
+
sorted_probs = student_probs.gather(-1, sorted_idx)
|
| 24 |
+
gap = (sorted_teacher_probs - sorted_probs).abs()
|
| 25 |
+
cum_teacher_probs = torch.cumsum(sorted_teacher_probs, -1)
|
| 26 |
+
tail_mask = cum_teacher_probs.le(adaptive_kl_alpha).float()
|
| 27 |
+
g_head = (gap * (1 - tail_mask)).sum(-1).detach()
|
| 28 |
+
g_tail = (gap * tail_mask).sum(-1).detach()
|
| 29 |
+
|
| 30 |
+
student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
|
| 31 |
+
teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
|
| 32 |
+
fkl = (teacher_probs * (teacher_lprobs - student_lprobs)).sum(-1)
|
| 33 |
+
rkl = (student_probs * (student_lprobs - teacher_lprobs)).sum(-1)
|
| 34 |
+
|
| 35 |
+
akl = (g_head / (g_head + g_tail)) * fkl + (g_tail / (g_head + g_tail)) * rkl
|
| 36 |
+
|
| 37 |
+
if reduction == "mean":
|
| 38 |
+
akl = akl.mean()
|
| 39 |
+
elif reduction == "sum":
|
| 40 |
+
akl = akl.sum()
|
| 41 |
+
|
| 42 |
+
return akl
|
kdflow/loss/chunked_loss.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from typing import Any, Callable, List, Optional, Union
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def chunked_loss(
|
| 7 |
+
student_hidden: torch.Tensor,
|
| 8 |
+
student_head: torch.nn.Module,
|
| 9 |
+
loss_fn: Callable[..., torch.Tensor],
|
| 10 |
+
student_logits_fn: Optional[Callable[..., torch.Tensor]] = None,
|
| 11 |
+
teacher_hidden: Optional[torch.Tensor] = None,
|
| 12 |
+
teacher_head: Optional[torch.nn.Module] = None,
|
| 13 |
+
teacher_logits_fn: Optional[Callable[[int, int], torch.Tensor]] = None,
|
| 14 |
+
label: Optional[torch.Tensor] = None,
|
| 15 |
+
chunk_size: int = 2048,
|
| 16 |
+
reduction: str = "none",
|
| 17 |
+
metric_fns: Optional[Union[Callable, List[Callable]]] = None,
|
| 18 |
+
return_metrics: bool = False,
|
| 19 |
+
**kwargs: Any,
|
| 20 |
+
):
|
| 21 |
+
"""Compute loss chunk by chunk without materializing full logits.
|
| 22 |
+
|
| 23 |
+
Inputs are expected to be token-level tensors, e.g. [num_tokens, hidden_size].
|
| 24 |
+
Loss is computed per chunk with reduction="none" and reduced globally here.
|
| 25 |
+
"""
|
| 26 |
+
if reduction not in ("none", "sum", "mean"):
|
| 27 |
+
raise ValueError(f"Unsupported reduction: {reduction}")
|
| 28 |
+
if teacher_logits_fn is None and teacher_hidden is None and label is None:
|
| 29 |
+
raise ValueError("Either teacher_logits_fn, teacher_hidden or label must be provided.")
|
| 30 |
+
if teacher_logits_fn is None and teacher_hidden is not None and teacher_head is None:
|
| 31 |
+
raise ValueError("teacher_head must be provided when teacher_hidden is provided.")
|
| 32 |
+
|
| 33 |
+
losses = []
|
| 34 |
+
metric_sums = defaultdict(lambda: student_hidden.new_zeros(()))
|
| 35 |
+
total_loss = student_hidden.new_zeros(())
|
| 36 |
+
total_tokens = 0
|
| 37 |
+
|
| 38 |
+
for start in range(0, student_hidden.shape[0], chunk_size):
|
| 39 |
+
end = start + chunk_size
|
| 40 |
+
if student_logits_fn is not None:
|
| 41 |
+
student_logits = student_logits_fn(student_hidden[start:end], skip=False)
|
| 42 |
+
else:
|
| 43 |
+
student_logits = student_head(student_hidden[start:end], skip=False)
|
| 44 |
+
has_teacher_logits = False
|
| 45 |
+
|
| 46 |
+
if teacher_logits_fn is not None:
|
| 47 |
+
teacher_logits = teacher_logits_fn(start, end)
|
| 48 |
+
vocab_size = min(student_logits.shape[-1], teacher_logits.shape[-1])
|
| 49 |
+
student_logits = student_logits[..., :vocab_size]
|
| 50 |
+
target = teacher_logits[..., :vocab_size]
|
| 51 |
+
has_teacher_logits = True
|
| 52 |
+
elif teacher_hidden is not None:
|
| 53 |
+
teacher_logits = teacher_head(teacher_hidden[start:end])
|
| 54 |
+
vocab_size = min(student_logits.shape[-1], teacher_logits.shape[-1])
|
| 55 |
+
student_logits = student_logits[..., :vocab_size]
|
| 56 |
+
target = teacher_logits[..., :vocab_size]
|
| 57 |
+
has_teacher_logits = True
|
| 58 |
+
else:
|
| 59 |
+
target = label[start:end]
|
| 60 |
+
|
| 61 |
+
chunk_loss = loss_fn(student_logits, target, reduction="none", **kwargs)
|
| 62 |
+
chunk_tokens = chunk_loss.numel()
|
| 63 |
+
if metric_fns is not None and has_teacher_logits:
|
| 64 |
+
fns = metric_fns if isinstance(metric_fns, list) else [metric_fns]
|
| 65 |
+
for fn in fns:
|
| 66 |
+
for key, value in fn(student_logits=student_logits, teacher_logits=target).items():
|
| 67 |
+
metric_sums[key] += value * chunk_tokens
|
| 68 |
+
if reduction == "none":
|
| 69 |
+
losses.append(chunk_loss)
|
| 70 |
+
else:
|
| 71 |
+
total_loss = total_loss + chunk_loss.sum()
|
| 72 |
+
total_tokens += chunk_tokens
|
| 73 |
+
|
| 74 |
+
if reduction == "none":
|
| 75 |
+
loss = torch.cat(losses, dim=0)
|
| 76 |
+
else:
|
| 77 |
+
loss = total_loss if reduction == "sum" else total_loss / total_tokens
|
| 78 |
+
if not return_metrics:
|
| 79 |
+
return loss
|
| 80 |
+
if reduction == "mean":
|
| 81 |
+
metric_sums = {key: value / total_tokens for key, value in metric_sums.items()}
|
| 82 |
+
return loss, metric_sums
|
kdflow/loss/cross_entropy.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
@torch.compile()
|
| 5 |
+
def compute_cross_entropy(
|
| 6 |
+
logits: torch.Tensor,
|
| 7 |
+
labels: torch.Tensor,
|
| 8 |
+
reduction: str = "none",
|
| 9 |
+
**kwargs
|
| 10 |
+
) -> torch.Tensor:
|
| 11 |
+
logsumexp = torch.logsumexp(logits, dim=-1)
|
| 12 |
+
log_probs = logits - logsumexp.unsqueeze(-1)
|
| 13 |
+
ce_loss = -log_probs.gather(-1, labels.unsqueeze(-1)).squeeze(-1)
|
| 14 |
+
if reduction == "mean":
|
| 15 |
+
return ce_loss.mean()
|
| 16 |
+
elif reduction == "sum":
|
| 17 |
+
return ce_loss.sum()
|
| 18 |
+
return ce_loss
|
kdflow/loss/hierarchical_ranking_loss.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
|
| 4 |
+
from kdflow.loss import register_loss
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@register_loss("hrl")
|
| 8 |
+
@torch.compile()
|
| 9 |
+
def compute_hierarchical_ranking_loss(
|
| 10 |
+
student_logits,
|
| 11 |
+
teacher_logits,
|
| 12 |
+
temperature=1.0,
|
| 13 |
+
hr_topk=5,
|
| 14 |
+
reduction="none",
|
| 15 |
+
**kwargs
|
| 16 |
+
):
|
| 17 |
+
student_probs = torch.softmax(student_logits, -1, dtype=torch.float32)
|
| 18 |
+
teacher_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32)
|
| 19 |
+
|
| 20 |
+
stu_topk_probs, stu_topk_idxs = torch.topk(student_probs, k=hr_topk, dim=-1)
|
| 21 |
+
tea_topk_probs, tea_topk_idxs = torch.topk(teacher_probs, k=hr_topk, dim=-1)
|
| 22 |
+
|
| 23 |
+
stu_co_topk_probs = student_probs.gather(-1, tea_topk_idxs)
|
| 24 |
+
tea_co_topk_probs = teacher_probs.gather(-1, stu_topk_idxs)
|
| 25 |
+
|
| 26 |
+
stu_topk_margin = stu_co_topk_probs[..., :1] - stu_co_topk_probs
|
| 27 |
+
stu_other_margin = stu_co_topk_probs.unsqueeze(-1) - stu_topk_probs.unsqueeze(-2)
|
| 28 |
+
tea_other_margin = stu_topk_probs.unsqueeze(-1) - tea_co_topk_probs.unsqueeze(-2)
|
| 29 |
+
topk_rank_loss = torch.maximum(torch.zeros_like(stu_topk_margin), -stu_topk_margin)
|
| 30 |
+
topk_rank_loss = topk_rank_loss.sum(-1)
|
| 31 |
+
other_rank_loss = torch.maximum(torch.zeros_like(tea_other_margin), -stu_other_margin)
|
| 32 |
+
other_rank_loss = (other_rank_loss * tea_other_margin.gt(0)).sum(-1).sum(-1)
|
| 33 |
+
rank_loss = topk_rank_loss + other_rank_loss
|
| 34 |
+
|
| 35 |
+
if reduction == "mean":
|
| 36 |
+
return rank_loss.mean()
|
| 37 |
+
elif reduction == "sum":
|
| 38 |
+
return rank_loss.sum()
|
| 39 |
+
|
| 40 |
+
return rank_loss
|
kdflow/loss/js_div.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from kdflow.loss import register_loss
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@register_loss("jsd")
|
| 7 |
+
@torch.compile()
|
| 8 |
+
def compute_js_div(
|
| 9 |
+
student_logits,
|
| 10 |
+
teacher_logits,
|
| 11 |
+
temperature=1.0,
|
| 12 |
+
jsd_beta=0.5,
|
| 13 |
+
reduction="none",
|
| 14 |
+
**kwargs
|
| 15 |
+
):
|
| 16 |
+
"""Jensen-Shannon Divergence (JSD) loss in GKD (https://arxiv.org/pdf/2306.13649)."""
|
| 17 |
+
assert jsd_beta > 0 and jsd_beta < 1, "jsd_beta must be in (0, 1)"
|
| 18 |
+
student_logits = student_logits / temperature
|
| 19 |
+
teacher_logits = teacher_logits / temperature
|
| 20 |
+
student_lprobs = student_logits.log_softmax(-1, dtype=torch.float32)
|
| 21 |
+
teacher_lprobs = teacher_logits.log_softmax(-1, dtype=torch.float32)
|
| 22 |
+
log_M = torch.logaddexp(
|
| 23 |
+
student_lprobs + torch.tensor(jsd_beta, dtype=torch.float32).log(),
|
| 24 |
+
teacher_lprobs + torch.tensor(1 - jsd_beta, dtype=torch.float32).log(),
|
| 25 |
+
)
|
| 26 |
+
div1 = jsd_beta * (student_lprobs.exp() * (student_lprobs - log_M)).sum(-1)
|
| 27 |
+
div2 = (1 - jsd_beta) * (teacher_lprobs.exp() * (teacher_lprobs - log_M)).sum(-1)
|
| 28 |
+
js_div = div1 + div2
|
| 29 |
+
|
| 30 |
+
if reduction == "mean":
|
| 31 |
+
js_div = js_div.mean()
|
| 32 |
+
elif reduction == "sum":
|
| 33 |
+
js_div = js_div.sum()
|
| 34 |
+
|
| 35 |
+
return js_div
|
kdflow/loss/kl_div.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
|
| 4 |
+
from kdflow.loss import register_loss
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@register_loss("kl")
|
| 8 |
+
@torch.compile()
|
| 9 |
+
def compute_kl_div(
|
| 10 |
+
student_logits,
|
| 11 |
+
teacher_logits,
|
| 12 |
+
temperature=1.0,
|
| 13 |
+
reduction="none",
|
| 14 |
+
**kwargs
|
| 15 |
+
):
|
| 16 |
+
student_logits = student_logits / temperature
|
| 17 |
+
teacher_logits = teacher_logits / temperature
|
| 18 |
+
log_probs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
|
| 19 |
+
target_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32)
|
| 20 |
+
kl_div = F.kl_div(log_probs, target_probs, reduction=reduction).sum(-1)
|
| 21 |
+
|
| 22 |
+
return kl_div
|
kdflow/loss/reverse_kl_div.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from kdflow.loss import register_loss
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@register_loss("rkl")
|
| 7 |
+
@torch.compile()
|
| 8 |
+
def compute_reverse_kl_div(
|
| 9 |
+
student_logits,
|
| 10 |
+
teacher_logits,
|
| 11 |
+
temperature=1.0,
|
| 12 |
+
reduction="none",
|
| 13 |
+
**kwargs
|
| 14 |
+
):
|
| 15 |
+
student_logits = student_logits / temperature
|
| 16 |
+
teacher_logits = teacher_logits / temperature
|
| 17 |
+
student_log_probs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
|
| 18 |
+
teacher_log_probs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
|
| 19 |
+
student_probs = student_log_probs.exp()
|
| 20 |
+
rkl_div = (student_probs * (student_log_probs - teacher_log_probs)).sum(-1)
|
| 21 |
+
|
| 22 |
+
if reduction == "mean":
|
| 23 |
+
rkl_div = rkl_div.mean()
|
| 24 |
+
elif reduction == "sum":
|
| 25 |
+
rkl_div = rkl_div.sum()
|
| 26 |
+
|
| 27 |
+
return rkl_div
|
kdflow/loss/skewed_kl_div.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
|
| 4 |
+
from kdflow.loss import register_loss
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@register_loss("skl")
|
| 8 |
+
@torch.compile()
|
| 9 |
+
def compute_skewed_kl_div(
|
| 10 |
+
student_logits,
|
| 11 |
+
teacher_logits,
|
| 12 |
+
temperature=1.0,
|
| 13 |
+
skew_lambda=0.1,
|
| 14 |
+
reduction="none",
|
| 15 |
+
**kwargs
|
| 16 |
+
):
|
| 17 |
+
"""Skewed KL Divergence loss (https://arxiv.org/abs/2402.03898)."""
|
| 18 |
+
assert skew_lambda > 0 and skew_lambda < 1, "skew_lambda must be in (0, 1)"
|
| 19 |
+
student_logits = student_logits / temperature
|
| 20 |
+
teacher_logits = teacher_logits / temperature
|
| 21 |
+
student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
|
| 22 |
+
teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
|
| 23 |
+
mixed_lprobs = torch.logaddexp(
|
| 24 |
+
teacher_lprobs + torch.tensor(skew_lambda, dtype=torch.float32).log(),
|
| 25 |
+
student_lprobs + torch.tensor(1 - skew_lambda, dtype=torch.float32).log(),
|
| 26 |
+
)
|
| 27 |
+
kl_div = (torch.exp(teacher_lprobs) * (teacher_lprobs - mixed_lprobs)).sum(-1)
|
| 28 |
+
|
| 29 |
+
if reduction == "mean":
|
| 30 |
+
kl_div = kl_div.mean()
|
| 31 |
+
elif reduction == "sum":
|
| 32 |
+
kl_div = kl_div.sum()
|
| 33 |
+
|
| 34 |
+
return kl_div
|
kdflow/loss/skewed_rkl_div.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
|
| 4 |
+
from kdflow.loss import register_loss
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@register_loss("srkl")
|
| 8 |
+
@torch.compile()
|
| 9 |
+
def compute_skewed_rkl_div(
|
| 10 |
+
student_logits,
|
| 11 |
+
teacher_logits,
|
| 12 |
+
temperature=1.0,
|
| 13 |
+
skew_lambda=0.1,
|
| 14 |
+
reduction="none",
|
| 15 |
+
**kwargs
|
| 16 |
+
):
|
| 17 |
+
"""Skewed Reverse KL Divergence loss (https://arxiv.org/abs/2402.03898)."""
|
| 18 |
+
assert skew_lambda > 0 and skew_lambda < 1, "skew_lambda must be in (0, 1)"
|
| 19 |
+
student_logits = student_logits / temperature
|
| 20 |
+
teacher_logits = teacher_logits / temperature
|
| 21 |
+
student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32)
|
| 22 |
+
teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32)
|
| 23 |
+
mixed_lprobs = torch.logaddexp(
|
| 24 |
+
student_lprobs + torch.tensor(skew_lambda, dtype=torch.float32).log(),
|
| 25 |
+
teacher_lprobs + torch.tensor(1 - skew_lambda, dtype=torch.float32).log(),
|
| 26 |
+
)
|
| 27 |
+
kl_div = (torch.exp(student_lprobs) * (student_lprobs - mixed_lprobs)).sum(-1)
|
| 28 |
+
|
| 29 |
+
if reduction == "mean":
|
| 30 |
+
kl_div = kl_div.mean()
|
| 31 |
+
elif reduction == "sum":
|
| 32 |
+
kl_div = kl_div.sum()
|
| 33 |
+
|
| 34 |
+
return kl_div
|
kdflow/loss/top1_ce.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from kdflow.loss import register_loss
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@register_loss("top1_ce")
|
| 7 |
+
@torch.compile()
|
| 8 |
+
def compute_top1_ce(
|
| 9 |
+
student_logits,
|
| 10 |
+
teacher_logits,
|
| 11 |
+
temperature=1.0,
|
| 12 |
+
reduction="none",
|
| 13 |
+
**kwargs
|
| 14 |
+
):
|
| 15 |
+
student_logits = student_logits / temperature
|
| 16 |
+
teacher_logits = teacher_logits / temperature
|
| 17 |
+
teacher_top1_probs, indices = torch.softmax(
|
| 18 |
+
teacher_logits, -1, dtype=torch.float32
|
| 19 |
+
).max(-1)
|
| 20 |
+
student_lprobs = torch.log_softmax(
|
| 21 |
+
student_logits, -1, dtype=torch.float32
|
| 22 |
+
).gather(-1, indices.unsqueeze(-1)).squeeze(-1)
|
| 23 |
+
ce_loss = -teacher_top1_probs * student_lprobs
|
| 24 |
+
|
| 25 |
+
if reduction == "mean":
|
| 26 |
+
return ce_loss.mean()
|
| 27 |
+
elif reduction == "sum":
|
| 28 |
+
return ce_loss.sum()
|
| 29 |
+
|
| 30 |
+
return ce_loss
|
kdflow/loss/tvd.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from kdflow.loss import register_loss
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@register_loss("tvd")
|
| 7 |
+
@torch.compile()
|
| 8 |
+
def compute_tvd(
|
| 9 |
+
student_logits,
|
| 10 |
+
teacher_logits,
|
| 11 |
+
temperature=1.0,
|
| 12 |
+
reduction="none",
|
| 13 |
+
**kwargs
|
| 14 |
+
):
|
| 15 |
+
"""Total Variation Distance (TVD) loss."""
|
| 16 |
+
student_logits = student_logits / temperature
|
| 17 |
+
teacher_logits = teacher_logits / temperature
|
| 18 |
+
student_probs = torch.softmax(student_logits, -1, dtype=torch.float32)
|
| 19 |
+
teacher_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32)
|
| 20 |
+
tvd = 0.5 * torch.abs(student_probs - teacher_probs).sum(-1)
|
| 21 |
+
|
| 22 |
+
if reduction == "mean":
|
| 23 |
+
tvd = tvd.mean()
|
| 24 |
+
elif reduction == "sum":
|
| 25 |
+
tvd = tvd.sum()
|
| 26 |
+
|
| 27 |
+
return tvd
|
kdflow/metrics/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from kdflow.metrics.entropy import compute_entropy
|
| 2 |
+
|
| 3 |
+
__all__ = ["compute_entropy"]
|
kdflow/metrics/entropy.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def compute_entropy(student_logits, **kwargs):
|
| 6 |
+
"""Compute the entropy of a logits distribution using numerically stable formula.
|
| 7 |
+
|
| 8 |
+
Args:
|
| 9 |
+
student_logits: Tensor of shape (num_tokens, vocab_size).
|
| 10 |
+
**kwargs: Unused. Accepts extra arguments for unified metric interface.
|
| 11 |
+
|
| 12 |
+
Returns:
|
| 13 |
+
A dict containing the mean entropy value.
|
| 14 |
+
"""
|
| 15 |
+
with torch.no_grad():
|
| 16 |
+
probs = F.softmax(student_logits, dim=-1)
|
| 17 |
+
entropy = torch.logsumexp(student_logits, dim=-1) - torch.sum(probs * student_logits, dim=-1)
|
| 18 |
+
return {"distill/student_entropy": entropy.mean()}
|