diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..2727e1bf64e5fd3901d2803b1def32466a2e90bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.DS_Store +__pycache__/ +*.pyc +.venv/ +.env +.env.* +outputs/ +wandb/ +*.egg-info/ +build/ +dist/ diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9ed1faf98032a99ac4f8dcaf3000e5e2d800fda1 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# Cross-Tokenizer-OPD + +This repository provides the implementation and example training scripts for `byte_align_ctkd` and `topk_byte_align_ctkd`. + +![Overview of cross-tokenizer alignment during training, the effect of mismatch supervision, and shared-vocabulary probability mass](assets/overview.png) + +## Environment + +Our experimental environment was created on top of the [KDFlow Docker image](https://hub.docker.com/r/songmzhang/kdflow?tag=sgl0.5.12_torch2.11_cu129), using the following tag: + +```bash +docker pull songmzhang/kdflow:sgl0.5.12_torch2.11_cu129 +``` + +[`requirements.txt`](requirements.txt) records the package versions used in our experiments. Use the Docker image as the starting point and match any additional dependencies to this file. It includes CUDA-specific builds and system-provided packages, so it is an environment snapshot rather than a standalone installation specification for arbitrary hosts. + +Inside the prepared container, run from the repository root: + +```bash +python -m pip install -e . --no-deps +``` + +## Example training + +Both scripts perform on-policy distillation from **Qwen2.5-7B-Instruct** to **Llama-3.2-3B-Instruct** and are configured for **one node with 8 GPUs**. + +Before running, update the paths in each script or mount the files at the expected locations: + +- Repository: `/workspace/Cross-Tokenizer-OPD` +- Student checkpoint: `/workspace/models/Llama-3.2-3B-Instruct` +- Teacher checkpoint: `/workspace/models/Qwen2.5-7B-Instruct` +- Training data: `/workspace/data/Train/math_code_20k/train.parquet` +- Evaluation data: `/workspace/data/math500/test.jsonl` + +The data files are included under `data/`; model checkpoints must be provided separately. WandB logging is disabled by default (`--use_wandb False`). To enable it, set `--use_wandb True`, add `--wandb_project `, and configure your WandB account. + +Run the byte-alignment example from the repository root: + +```bash +bash run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh +``` + +Run the top-k example (`--ctkd_topk 16`): + +```bash +bash run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh +``` + +Training settings can be adjusted directly in the scripts. Checkpoints, evaluation results, and training logs are written under `outputs//`. diff --git a/docs/upstream-source-manifest.json b/docs/upstream-source-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e6427a3827aad9dfd9ad1fe90e02145ffbb4e9ba --- /dev/null +++ b/docs/upstream-source-manifest.json @@ -0,0 +1,601 @@ +{ + "url": "https://anonymous.4open.science/api/repo/Cross-Tokenizer-OPD", + "paper": "https://arxiv.org/pdf/2610.08448v1", + "retrieved_utc": "2026-10-17T19:28:35.738288+00:00", + "identity_kind": "audited content snapshot assembled from public files; upstream commit unavailable", + "excluded": [ + "data/", + "assets/" + ], + "source_manifest_sha256": "93cef1dfa18bcd5fc66cf80347557382efcd7e8955c3f6b464435711e82bf49b", + "files": [ + { + "path": ".gitignore", + "bytes": 89, + "api_version_tag": "2e7be852", + "git_blob": "2727e1bf64e5fd3901d2803b1def32466a2e90bf", + "sha256": "19587f4a412fa284a0dac11bad5d487b92117390d1eb117aee094f6fb289ba0d" + }, + { + "path": "README.md", + "bytes": 2289, + "api_version_tag": "b260e71d", + "git_blob": "9ed1faf98032a99ac4f8dcaf3000e5e2d800fda1", + "sha256": "f45c1d25386d1d1a25f3c4ea4c152db95be6467f34c86a94ad52cda4df4ee656" + }, + { + "path": "kdflow/__init__.py", + "bytes": 22, + "api_version_tag": "f8cbccf6", + "git_blob": "3dc1f76bc69e3f559bee6253b24fc93acee9e1f9", + "sha256": "91447944015cec709e8aa7655f7e9d64e1e4508e7023a57fe3746911c0fc6fed" + }, + { + "path": "kdflow/algorithms/__init__.py", + "bytes": 520, + "api_version_tag": "a7cd7946", + "git_blob": "13dd1052857854aba08c5f7667ed6a49ff1b0b80", + "sha256": "8afbb4fadfc7db812d43ab35401971fc30fdc4810cafa81505cbea27178badbe" + }, + { + "path": "kdflow/algorithms/byte_align_ctkd.py", + "bytes": 4552, + "api_version_tag": "990d76e2", + "git_blob": "c94a1239b57b05513ad853735291949aa314c6d9", + "sha256": "5c6647c6fb8064ffd5f7b1ed971168cd1f3392caee62e2d75916d41931828e18" + }, + { + "path": "kdflow/algorithms/simple_ctkd.py", + "bytes": 8575, + "api_version_tag": "166783cd", + "git_blob": "57332b882e359ce1283a87f81949a5136f8c06a2", + "sha256": "fdbd99c55aeac4d061527158bdfc82a93333a42bf347eceaa134d34f5ef05952" + }, + { + "path": "kdflow/algorithms/topk_byte_align_ctkd.py", + "bytes": 3448, + "api_version_tag": "4eff709a", + "git_blob": "64b49354bea827893fd49f8de51325666f7e1dba", + "sha256": "298925da4ce973b0f2917e5ba5ad2adbc4862d7c1a73515c5c86514ac0d22ddb" + }, + { + "path": "kdflow/arguments/__init__.py", + "bytes": 13313, + "api_version_tag": "22b52ef5", + "git_blob": "b3c66702102a8691bcd16bc57f8a8307ccb13b68", + "sha256": "cafb17a195874f1b53b0b5f29b07840e1f2f6505e08dd77837fca25fd1a8122f" + }, + { + "path": "kdflow/arguments/data_args.py", + "bytes": 3363, + "api_version_tag": "e8dc076d", + "git_blob": "12046f174777ed5d8f271b15661debf0b3c63003", + "sha256": "e5f68889ab6635ed51a285aad70dcff5de3dce7d7d06c6cf563b03331591a382" + }, + { + "path": "kdflow/arguments/distillation_args.py", + "bytes": 5102, + "api_version_tag": "e7ae08cc", + "git_blob": "ae835897855f7e4147fb8efc1fe5971fd8cfb964", + "sha256": "65d39f6abeb75896864d385fe0c9a2f026938eb1a5ee9d4dc392d3cb86b440f4" + }, + { + "path": "kdflow/arguments/fsdp_args.py", + "bytes": 387, + "api_version_tag": "172c1790", + "git_blob": "ed2729e8de9b94f789cd73ea1274e02bc4e18558", + "sha256": "10918e4e3242a526ab0926af4a2608a3a77dea058fd084ecee9022852aecac47" + }, + { + "path": "kdflow/arguments/logging_args.py", + "bytes": 890, + "api_version_tag": "b3ce62d7", + "git_blob": "53e1413ef1c4b57d66b57a22c1c3c7022cf9d70a", + "sha256": "2d6012a164fdc81b3d9c482357dc68d0fe4485db0a2163f3cdfa3ca6411addc1" + }, + { + "path": "kdflow/arguments/model_args.py", + "bytes": 1157, + "api_version_tag": "2f851242", + "git_blob": "9c6531d1deb2435054eeff4c412271149390b336", + "sha256": "b6da8907b70189acaa7c1b51544135e441182e28194ee3bbe4f23ec1fa23cb83" + }, + { + "path": "kdflow/arguments/rollout_args.py", + "bytes": 4468, + "api_version_tag": "5899e45d", + "git_blob": "5b46da1da4da0f4a05a53aa14ef36167517784e4", + "sha256": "f9df7e87681abd88c00bde4985a45ddfbb3c6f3f7feee41d4fc941a808c670d1" + }, + { + "path": "kdflow/arguments/training_args.py", + "bytes": 5880, + "api_version_tag": "b7d6b4ad", + "git_blob": "71443753b8ea33fd213a28c0744816c77b616600", + "sha256": "a45dd87ae3064eac14f9ca9a875963df4efd03082230aba9f4c98d67479260bd" + }, + { + "path": "kdflow/backend/__init__.py", + "bytes": 647, + "api_version_tag": "33655f94", + "git_blob": "ae07fdf84af178d7d1b63e06abefb6f07f29e890", + "sha256": "b36cc114c0ffd20559a6e87808b158d7244566645933639023cf00019e03e137" + }, + { + "path": "kdflow/backend/fsdp/__init__.py", + "bytes": 76, + "api_version_tag": "2c7e9bf6", + "git_blob": "a50fa8e305c891e9bfe23089b39c6f0555d473af", + "sha256": "9b1dffb545428c2862d6cc588507709797ba05f24d4017e9a450a7afa09701d9" + }, + { + "path": "kdflow/backend/fsdp/fsdp_strategy.py", + "bytes": 26957, + "api_version_tag": "b0b88464", + "git_blob": "63498e48936f06180363b4df5c7fde821fa36a70", + "sha256": "8941ccc6077e6fdcc2a38c13769d6a0912dc764ea9f7b82866e7e1bf3bac0fa2" + }, + { + "path": "kdflow/backend/sglang/__init__.py", + "bytes": 0, + "api_version_tag": null, + "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "kdflow/backend/sglang/monkey_patch.py", + "bytes": 10707, + "api_version_tag": "154f0497", + "git_blob": "e4aa0a9550903f01a7125b1631408e9b2df6883a", + "sha256": "ddcbc908bce562bc3b87f13b8445ac02891a491dd60ba2edee37d6396440999c" + }, + { + "path": "kdflow/backend/sglang/sglang_engine.py", + "bytes": 16531, + "api_version_tag": "b2ffd390", + "git_blob": "4deb3adfe2fa5b2258c0144ed46b4b4dc1873913", + "sha256": "dd0b9a5531beba14c52389787a1df5d053412dabd9547cde5cb66f5f23bb494f" + }, + { + "path": "kdflow/cli/__init__.py", + "bytes": 0, + "api_version_tag": null, + "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "kdflow/cli/evaluate_alfworld.py", + "bytes": 4054, + "api_version_tag": "c6740670", + "git_blob": "77dec216fbfc0b68f28a094b6fdeac63f1f6a646", + "sha256": "f5b02bf943606ff7d01feff0c8df9eb5a5902a7c0039d38e3a21c859716bf08e" + }, + { + "path": "kdflow/cli/train_kd_off_policy.py", + "bytes": 5879, + "api_version_tag": "c4023313", + "git_blob": "b149c801a94aedc0234c529cbdc63764de08573f", + "sha256": "b3064a2f38c12bf086c5f783b0dcfb3904ff097e99289aca2a1da310d81713cc" + }, + { + "path": "kdflow/cli/train_kd_on_policy.py", + "bytes": 8208, + "api_version_tag": "c50095ea", + "git_blob": "8cfb58d743568948cfa7d1c1e579be7287057241", + "sha256": "1e2036bf1b788d7f7b27ac3496d50b78c25992ae597733199bfb90619cf6391f" + }, + { + "path": "kdflow/datasets/__init__.py", + "bytes": 216, + "api_version_tag": "fc482f85", + "git_blob": "37facfbdb6705166c7047e40eb67bb1f3edbf3e9", + "sha256": "2dd709d632fae96f3e8d86ffb8b608cdca9edaacd17dde7c21eed1c497379d37" + }, + { + "path": "kdflow/datasets/alfworld_dataset.py", + "bytes": 806, + "api_version_tag": "4f86ac77", + "git_blob": "cd96f6d2932f6e2cd31c6e4cab188556206abbef", + "sha256": "3047bf9576937e1cbc3dd7d7070c7c949ac0e96ecb0b1bf8b6142c7c0415b88b" + }, + { + "path": "kdflow/datasets/prompts_dataset.py", + "bytes": 11012, + "api_version_tag": "9c932cbd", + "git_blob": "15a6e747c6cc3fd3232affcdff8f1252ac087ee9", + "sha256": "3e6b9954b893224a5eac5ad79758dc198747b929e1fe27475e6aed1f01a09703" + }, + { + "path": "kdflow/datasets/sft_dataset.py", + "bytes": 18029, + "api_version_tag": "23c3e648", + "git_blob": "6b922b0047e0d1da1e9f95f0c2b9a078f3ef92a8", + "sha256": "e5643660828193e4a124639d18d1ac14826d57203dcc3ccf31e2c7d83a37c7b7" + }, + { + "path": "kdflow/datasets/utils.py", + "bytes": 14066, + "api_version_tag": "6fe0c9b3", + "git_blob": "9933ae47766e3623d148e39fa7817cefd112133d", + "sha256": "f0173126a8530665226396c7e9343f57cc031e4df1d97929ba6c3b360059bbb3" + }, + { + "path": "kdflow/evaluation/__init__.py", + "bytes": 344, + "api_version_tag": "a3dca82f", + "git_blob": "245e0251fb5e37de3cc7c2a234bff53a732e41de", + "sha256": "f63f24ca9885b63c32d3d1e24b40c16367734cb238f52b4893b20be95321573c" + }, + { + "path": "kdflow/evaluation/alfworld_evaluator.py", + "bytes": 6661, + "api_version_tag": "beaf919e", + "git_blob": "7e085a7350f8a07cd3b1cefcbe12c2eab7733d45", + "sha256": "45c96b0908cf0bd895cd312e88ca8bd49a53be3f00cf9fa4363555590d03352b" + }, + { + "path": "kdflow/evaluation/math_evaluator.py", + "bytes": 12096, + "api_version_tag": "5cc9a5e1", + "git_blob": "890e7ed2e1fe006d1e65d3a35d398297fb2fb70f", + "sha256": "f598d470e1f03c0e09b6bbdb4d7fa8571eb7484f14f2be7ea706a0ed14aa8935" + }, + { + "path": "kdflow/evaluation/verl_agent_alfworld_evaluator.py", + "bytes": 16963, + "api_version_tag": "1391ea50", + "git_blob": "f9a25e4e331f5a9dd3aad873666bb0861347453a", + "sha256": "38d71145e80de82fe738599f43dfe8b3cb14177da4784457546e6be18f78fc29" + }, + { + "path": "kdflow/evaluation/verl_agent_alfworld_protocol.py", + "bytes": 3001, + "api_version_tag": "a8cc4e01", + "git_blob": "38615c951dd17f8311216b640cc2bd0ae145291a", + "sha256": "9aa3c5f02703148a0bfce8d2177b53e51aa906df03e2401e5bf0f5b241ce8433" + }, + { + "path": "kdflow/loss/__init__.py", + "bytes": 1089, + "api_version_tag": "6e9b5e28", + "git_blob": "2d66850c7da1a6839a2d69b781980469ea85cd2b", + "sha256": "3bc98a351ad62faa346607502dcd56e864536b81877ca0221fd2b2ca9045f630" + }, + { + "path": "kdflow/loss/adaptive_kl_div.py", + "bytes": 1528, + "api_version_tag": "7af71dea", + "git_blob": "e95f03d7900fd29f1ff5d1b64bf1d670a3267410", + "sha256": "f8a44824867ae3fa7e732f7ae5554d6b13231ff9545e1fd61793a8f546d7d39c" + }, + { + "path": "kdflow/loss/chunked_loss.py", + "bytes": 3636, + "api_version_tag": "8a38a29a", + "git_blob": "2c5174fd3c60c92ea8e315fb1ce28931eb7e61ae", + "sha256": "151c81614f97b908465a952f595abe1f4dc5c0d17cea86431a7068c381125cc5" + }, + { + "path": "kdflow/loss/cross_entropy.py", + "bytes": 473, + "api_version_tag": "9fab6f38", + "git_blob": "9f5a322ff489503f707515aa8f8e413ef1083b6b", + "sha256": "e3629bef7c50c737f384870876321c953ceb09b67c275242cdbfcfbcbca68377" + }, + { + "path": "kdflow/loss/hierarchical_ranking_loss.py", + "bytes": 1479, + "api_version_tag": "19217f5e", + "git_blob": "8e6ea5c38676ca63011834431955852055a78af5", + "sha256": "2cced8cccf77872ab817164a931c97c43c7411b8d59cc48d842111e40f021ac3" + }, + { + "path": "kdflow/loss/js_div.py", + "bytes": 1165, + "api_version_tag": "8c85586a", + "git_blob": "1d210762db1d64e5ee8d8cd4bff9ee60499950e8", + "sha256": "be71370e1e1a2c109c2a00a4e6daaece8ab17f182f1a3f03c60a385b98fb3078" + }, + { + "path": "kdflow/loss/kl_div.py", + "bytes": 592, + "api_version_tag": "7bc61a88", + "git_blob": "c39542e7a839837741d0fc442ba5bb18fc5a6343", + "sha256": "b361ddca392ee4b2e83e44b6891f4402f86127db38b6e254d492d29c48950a7c" + }, + { + "path": "kdflow/loss/reverse_kl_div.py", + "bytes": 761, + "api_version_tag": "5afd030a", + "git_blob": "ebdaa8f2a467e41383825915e6a757de2426738c", + "sha256": "abca69fdbe4f7d90a7ca990300189147230cce8f20d589c934d88f0b63c4edf6" + }, + { + "path": "kdflow/loss/skewed_kl_div.py", + "bytes": 1119, + "api_version_tag": "737302c8", + "git_blob": "4b76d83f474a2744c81b12a717cae97ea97dccca", + "sha256": "98189a13bf644d3c46d53aa228a264b9ed666b9d28586a343f0c2bdbd6fed091" + }, + { + "path": "kdflow/loss/skewed_rkl_div.py", + "bytes": 1129, + "api_version_tag": "c37538a3", + "git_blob": "9652791e8f1ff0159eb9826ed70ac16057b511cf", + "sha256": "3c78a1b04270f7d8ed40449f28f5ed936af71c0bc7d5a0badf0864454ddf66f0" + }, + { + "path": "kdflow/loss/top1_ce.py", + "bytes": 760, + "api_version_tag": "3cc13dae", + "git_blob": "6a07fefbc642c79a4311c3f3d3565ee2b3bec251", + "sha256": "1f566c4559f518c8186a53de85593b812e65016beab083cd4ded715b513fa760" + }, + { + "path": "kdflow/loss/tvd.py", + "bytes": 702, + "api_version_tag": "289a1332", + "git_blob": "a84bee8f7963da36e7e4011a0dd3af9d9e87d462", + "sha256": "964d872f122ec4618cfee1fd2adf8ecc0fc82a8279cce83cab8b277335f8aa2c" + }, + { + "path": "kdflow/metrics/__init__.py", + "bytes": 82, + "api_version_tag": "4c09b5e4", + "git_blob": "e888dd4bdc45ae4e2be5225b6c74ae17480225a8", + "sha256": "140c40cfb0b374c62847a93f666c9cf0c3ad595d6d36dc7bb1e6c2fb05495362" + }, + { + "path": "kdflow/metrics/entropy.py", + "bytes": 642, + "api_version_tag": "c37c9989", + "git_blob": "81439fc696793032bce83717db7ccb17243d8d9a", + "sha256": "e02d0a9926fdb33b120fc2ec0a95d042876b43504046a301191d4761cee58cef" + }, + { + "path": "kdflow/models/__init__.py", + "bytes": 67, + "api_version_tag": "39b9dcda", + "git_blob": "8f60ffe38287f99a5fe053daf17f85b5173d07a7", + "sha256": "93370fd72c3d337adcb1bb7cbae6ba2cf9eab2ef49b5b466ada816f7b6bbd148" + }, + { + "path": "kdflow/models/model.py", + "bytes": 5150, + "api_version_tag": "49c34f8e", + "git_blob": "df455afaca9a0eaaac7d7cc9836acb9f0dc1dbee", + "sha256": "2530323bba13319cea6ea6f0ea54ed64ae81d35c6a2aaf209688a67622e26996" + }, + { + "path": "kdflow/models/ring_attn_utils.py", + "bytes": 7402, + "api_version_tag": "0c072a92", + "git_blob": "fe3b01a70d20358de0dc19e19ce09df306004d26", + "sha256": "ed93df4cbef484401f2cf508bc81d4a22521e632bec4bae4a931e404639b7eda" + }, + { + "path": "kdflow/models/utils.py", + "bytes": 1755, + "api_version_tag": "cfd2ac62", + "git_blob": "5f00ba4aaf5b007c20e4fb7cb08b5e466a2acef5", + "sha256": "cfef9ca657b32c8ee5688f01f0f1e939ec3fc61fbd8792085d63af66f74e85d5" + }, + { + "path": "kdflow/ray/__init__.py", + "bytes": 0, + "api_version_tag": null, + "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "kdflow/ray/placement_group.py", + "bytes": 2049, + "api_version_tag": "37beb550", + "git_blob": "24ba3109273257b3ab9329a19dc72804bbc5edc3", + "sha256": "08b0a5d82cacea096b1591e3d12d25f30f285145cade451578763f3b16144a27" + }, + { + "path": "kdflow/ray/rollout/__init__.py", + "bytes": 176, + "api_version_tag": "ae57ecf3", + "git_blob": "a7e39d7a3785301ef2f0614ad28880e2f7dfc1cc", + "sha256": "13401bf88cab65c4e388736c19f4ff2df16fc2a8ff9cbc07a1b48c76876092de" + }, + { + "path": "kdflow/ray/rollout/rollout_actor.py", + "bytes": 12933, + "api_version_tag": "635eb232", + "git_blob": "63a00bc4e29afbb2041ee99ac2c63fa5b3aa1d4f", + "sha256": "14af615c89f8c8c6089ed5e98a5473da0ab79df4771621e5becc4340ff3720bc" + }, + { + "path": "kdflow/ray/rollout/rollout_group.py", + "bytes": 14618, + "api_version_tag": "7a452703", + "git_blob": "5ad5ec900fe03de4c6e7201791f8be76315a3ce9", + "sha256": "05b9235ba2da6259a49979514b6ac3910a500be4f8c0575461d3d9997959ed9b" + }, + { + "path": "kdflow/ray/train/__init__.py", + "bytes": 0, + "api_version_tag": null, + "git_blob": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "kdflow/ray/train/student_actor.py", + "bytes": 18702, + "api_version_tag": "857338dd", + "git_blob": "0aec3b65c4423b554ae2fdd4fee3b02d64011e78", + "sha256": "b234fee1e64e1b59f45d917faa1435cf911619fbaf1b581530996df2244b14eb" + }, + { + "path": "kdflow/ray/train/student_group.py", + "bytes": 9563, + "api_version_tag": "5f558d33", + "git_blob": "cc9308ae81028870f2a47d70dadaaa443794070d", + "sha256": "c9b661647d781e3353821fce0f15c1be69f10d81a55ced07523635d52b0f04e4" + }, + { + "path": "kdflow/ray/train/teacher_actor.py", + "bytes": 7299, + "api_version_tag": "cdf12732", + "git_blob": "72600b4ec71cc27acc19bc3f8333410b9784f754", + "sha256": "8e84664647c5c1f4d2a3bf000d3e382bfcbe3735f42bef2f7b0f0af43518974e" + }, + { + "path": "kdflow/ray/train/teacher_group.py", + "bytes": 12590, + "api_version_tag": "83cd153b", + "git_blob": "b121a64b7519f2866d21e8e2d2c208c0052f5826", + "sha256": "b0a632619940e8fb49f4b5059299f0d380fe9d4b1bd9c26c7c82dff32e50f7ba" + }, + { + "path": "kdflow/ray/utils.py", + "bytes": 2590, + "api_version_tag": "9f91e927", + "git_blob": "b003f49102a84c9ce15e617abb33fa4842286fcc", + "sha256": "8dea4c2b6b55445ad6d8868807ca016b61f6f7e12efb321502dc8b6eb2ea0263" + }, + { + "path": "kdflow/token_alignment.py", + "bytes": 13436, + "api_version_tag": "8766c5ad", + "git_blob": "5a2b1695191df2aaa7798e8cbb0605858775474b", + "sha256": "c77cd962cb0430cd97ace5312077e847398a03da3ce1473062fd2c5c9808498a" + }, + { + "path": "kdflow/trainer/__init__.py", + "bytes": 528, + "api_version_tag": "6dec2a62", + "git_blob": "c7fa5b1ce3c01a42c60f2f070370cd978a73710b", + "sha256": "a3caa9361a9f53b881e24ffa05951342278947135cd51c637ba4d1b16a1a8687" + }, + { + "path": "kdflow/trainer/alfworld_rollout_manager.py", + "bytes": 18875, + "api_version_tag": "a77bc558", + "git_blob": "9e353bcbfd9cb9424f2110f3cf1c9e4b2ff496ff", + "sha256": "93f6ca98f2ba6fe9613d60839eb7d22d6b582420489aefbc1f6364ffb8493987" + }, + { + "path": "kdflow/trainer/data_processor/__init__.py", + "bytes": 122, + "api_version_tag": "d5140e99", + "git_blob": "d267c43e5cf7c929a03867968ff8c794a5a574bf", + "sha256": "66a34ffc421ade775f246c8198bfdc9633e83c97b355084fa457e6e15695b775" + }, + { + "path": "kdflow/trainer/data_processor/rollout_data_processor.py", + "bytes": 13871, + "api_version_tag": "3b4b701b", + "git_blob": "7f5564dcd3046b5be30cbabc13b06c3dd393c17a", + "sha256": "fe720ee01fb821e124a612dcb55cb0e827b64e7fb1597b4ecbb4bf14c8ee5792" + }, + { + "path": "kdflow/trainer/off_policy_kd_trainer.py", + "bytes": 14023, + "api_version_tag": "f150d881", + "git_blob": "72ab1017c9edfbf90778208b9d01ca123d1d72ee", + "sha256": "6b4019e9760e0f2898a4cf0b30071d69bf24b6a23a023aca5f1f929b2a0c2acf" + }, + { + "path": "kdflow/trainer/on_policy_kd_trainer.py", + "bytes": 20980, + "api_version_tag": "f34b6551", + "git_blob": "aa4ba8e05f8fc49625b69b8870fc29a8aa6aa6c6", + "sha256": "f7b20000d435f8fc43a07584a3025472f138ccc6e41f62f25c08ff9ff8fa17b1" + }, + { + "path": "kdflow/trainer/rollout_manager.py", + "bytes": 6718, + "api_version_tag": "f6b74523", + "git_blob": "0eb5253bceb3a06a66cf120a606a221db3a07f81", + "sha256": "e930b4db59e8f8e09e04838141854403f5516ac8492ef4ab0bd7f37611db7009" + }, + { + "path": "kdflow/trainer/verl_agent_alfworld_rollout_manager.py", + "bytes": 22589, + "api_version_tag": "db6881e8", + "git_blob": "430dd71f2979174e05b61d276b4e29d068adc864", + "sha256": "ea95c95d7c3d8dbcc4da1b578877684a4dcea3e30bf6b68ea98187432194a040" + }, + { + "path": "kdflow/utils/__init__.py", + "bytes": 69, + "api_version_tag": "5eaed800", + "git_blob": "1431283d595d0062ee042ea5567c33aec00ad49b", + "sha256": "9ed449850c624d587592f92760c79c55324063a7cc16fbfa5aab42585165afb1" + }, + { + "path": "kdflow/utils/distributed_sampler.py", + "bytes": 6482, + "api_version_tag": "a82f95b2", + "git_blob": "1f7658202804e2a1fbe1a5d0d6ddaf20cc50f3af", + "sha256": "5a099195c63e475f42fd17e43b4c40ed1133ac3237f623410befffcc9005cda7" + }, + { + "path": "kdflow/utils/distributed_util.py", + "bytes": 1077, + "api_version_tag": "85f4398a", + "git_blob": "fd13702ebaa1d34fc1bf87d7c506847af3f7190d", + "sha256": "6a7850b1490c9122f08384eab1343c908cf526af1fcb997b8898d92325edaf98" + }, + { + "path": "kdflow/utils/dynamic_bsz.py", + "bytes": 15227, + "api_version_tag": "bcd5639b", + "git_blob": "44ec3b3a8d3e7f423f3a68b951b7285c463c567b", + "sha256": "a11fa6064f980d2e614705c9571401de4f18b21e3bbc35a3d69cd7f2ecf4ae9b" + }, + { + "path": "kdflow/utils/logging_utils.py", + "bytes": 3471, + "api_version_tag": "80b625a3", + "git_blob": "1393d26d3e95e7ac447bda58242866dac96d2af0", + "sha256": "e57a5e3bd3df172bb147dfc599f377748e3564de81b800ce365b7a336be89302" + }, + { + "path": "kdflow/utils/multimodal_utils.py", + "bytes": 1073, + "api_version_tag": "da5520e7", + "git_blob": "60acb817ed6ce60fda693c80ec1295376c15b10b", + "sha256": "8b16f06ea69d3c4b2cfb17bb8cb215652350e0aa0548e9f06d07950c07aa34bf" + }, + { + "path": "kdflow/utils/utils.py", + "bytes": 3290, + "api_version_tag": "3654028c", + "git_blob": "26415ed5185e779a2945b98fa6f368a726dd286d", + "sha256": "b4daa8b9e9b405dd330ef3d39d03cc4dac108c05b0012e145fb14baad664fbc9" + }, + { + "path": "pyproject.toml", + "bytes": 730, + "api_version_tag": "4c685d0b", + "git_blob": "dd4bfd646653bdc98b8f1bc8aa753a996932166c", + "sha256": "fb40bc75b604b76e4bc9f5406b1e5fb4721daf4d090543bab430adf4aaaa30f3" + }, + { + "path": "requirements.txt", + "bytes": 6762, + "api_version_tag": "ea4b60ca", + "git_blob": "678e55b3cd878b4364ed7bd9d54ac1cf2058af80", + "sha256": "d4d536aafbe5889185058299ec9deb27ab4903dba3686a14734587703adb2f3f" + }, + { + "path": "run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh", + "bytes": 2414, + "api_version_tag": "a6482a95", + "git_blob": "c778963c3e42749205a2fc9b0208bfb076a48180", + "sha256": "4982257e528fbe508d180172aa81e85e761cbc15328ec2ddc33b7feda42bd2e4" + }, + { + "path": "run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh", + "bytes": 2451, + "api_version_tag": "bb390a7d", + "git_blob": "2997aaaebae9914d30b62cca821bce7ceef7818d", + "sha256": "2f297f306ff7f91d5902cc039bea87cf43474127d6e7598ff50053c5d66551a1" + } + ] +} diff --git a/kdflow/__init__.py b/kdflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3dc1f76bc69e3f559bee6253b24fc93acee9e1f9 --- /dev/null +++ b/kdflow/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/kdflow/algorithms/__init__.py b/kdflow/algorithms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..13dd1052857854aba08c5f7667ed6a49ff1b0b80 --- /dev/null +++ b/kdflow/algorithms/__init__.py @@ -0,0 +1,17 @@ +"""Algorithms included in the submission.""" + +ALGO_DICT = {} + + +def register_algorithm(name): + def decorator(cls): + ALGO_DICT[name] = cls + return cls + return decorator + + +from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD +from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD +from kdflow.algorithms.topk_byte_align_ctkd import TopKByteAlignCrossTokenizerKD + +__all__ = ["ALGO_DICT", "SimpleCrossTokenizerKD", "ByteAlignCrossTokenizerKD", "TopKByteAlignCrossTokenizerKD"] diff --git a/kdflow/algorithms/byte_align_ctkd.py b/kdflow/algorithms/byte_align_ctkd.py new file mode 100644 index 0000000000000000000000000000000000000000..c94a1239b57b05513ad853735291949aa314c6d9 --- /dev/null +++ b/kdflow/algorithms/byte_align_ctkd.py @@ -0,0 +1,121 @@ +import torch + +from kdflow.algorithms import register_algorithm +from kdflow.algorithms.simple_ctkd import SimpleCrossTokenizerKD +from kdflow.token_alignment import ( + align_byte_span_batch_1to1, + align_byte_spans_1to1, + find_byte_overlap_token_ids, + get_tokenizer_byte_codec, +) +from kdflow.utils.logging_utils import init_logger + + +logger = init_logger(__name__) + + +@register_algorithm("byte_align_ctkd") +class ByteAlignCrossTokenizerKD(SimpleCrossTokenizerKD): + """Simple cross-tokenizer KD with UTF-8 byte-span 1:1 alignment.""" + + def __init__( + self, + strategy, + student_model, + teacher_lm_head, + student_tokenizer, + teacher_tokenizer, + **kwargs, + ): + # These are initialized before ``super`` because its constructor calls + # the virtual ``_find_overlap_tokens`` method. + self.student_byte_codec = get_tokenizer_byte_codec(student_tokenizer) + self.teacher_byte_codec = get_tokenizer_byte_codec(teacher_tokenizer) + super().__init__( + strategy=strategy, + student_model=student_model, + teacher_lm_head=teacher_lm_head, + student_tokenizer=student_tokenizer, + teacher_tokenizer=teacher_tokenizer, + **kwargs, + ) + + def _find_overlap_tokens(self): + # Preserve the exact historical mapping for all existing ByteLevel + # pairs. Only mixed/SentencePiece pairs use the byte-canonical map. + if ( + self.student_byte_codec.name == "byte_level" + and self.teacher_byte_codec.name == "byte_level" + ): + return super()._find_overlap_tokens() + + student_ids, teacher_ids = find_byte_overlap_token_ids( + student_tokenizer=self.student_tokenizer, + teacher_tokenizer=self.teacher_tokenizer, + student_codec=self.student_byte_codec, + teacher_codec=self.teacher_byte_codec, + ) + device = self.teacher_lm_head.weight.device + logger.info( + "Num of byte-equivalent overlap tokens between student & teacher: " + f"{len(student_ids)}" + ) + return ( + torch.tensor(student_ids, dtype=torch.long, device=device), + torch.tensor(teacher_ids, dtype=torch.long, device=device), + ) + + def _align_sequences(self, tea_seq, stu_seq): + return align_byte_spans_1to1( + teacher_tokens=tea_seq, + student_tokens=stu_seq, + teacher_eos_token=self.teacher_tokenizer.eos_token, + student_eos_token=self.student_tokenizer.eos_token, + teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes, + student_piece_to_bytes=self.student_byte_codec.piece_to_bytes, + ) + + def _get_alignment_indices( + self, + teacher_label_ids, + student_label_ids, + teacher_loss_mask, + student_loss_mask, + ): + """Align each micro-batch row independently, then return flat indices.""" + if teacher_label_ids.shape[0] != student_label_ids.shape[0]: + raise ValueError( + "Teacher and student micro-batches must contain the same number " + "of samples for byte_align_ctkd." + ) + + teacher_batch_tokens = [] + student_batch_tokens = [] + for row_index in range(teacher_label_ids.shape[0]): + teacher_row_ids = teacher_label_ids[row_index][ + teacher_loss_mask[row_index] + ] + student_row_ids = student_label_ids[row_index][ + student_loss_mask[row_index] + ] + teacher_row_token_ids = teacher_row_ids.cpu().tolist() + student_row_token_ids = student_row_ids.cpu().tolist() + teacher_batch_tokens.append( + self.teacher_tokenizer.convert_ids_to_tokens( + teacher_row_token_ids + ) + ) + student_batch_tokens.append( + self.student_tokenizer.convert_ids_to_tokens( + student_row_token_ids + ) + ) + + return align_byte_span_batch_1to1( + teacher_batch_tokens=teacher_batch_tokens, + student_batch_tokens=student_batch_tokens, + teacher_eos_token=self.teacher_tokenizer.eos_token, + student_eos_token=self.student_tokenizer.eos_token, + teacher_piece_to_bytes=self.teacher_byte_codec.piece_to_bytes, + student_piece_to_bytes=self.student_byte_codec.piece_to_bytes, + ) diff --git a/kdflow/algorithms/simple_ctkd.py b/kdflow/algorithms/simple_ctkd.py new file mode 100644 index 0000000000000000000000000000000000000000..57332b882e359ce1283a87f81949a5136f8c06a2 --- /dev/null +++ b/kdflow/algorithms/simple_ctkd.py @@ -0,0 +1,198 @@ +import torch + +from kdflow.loss import build_loss_fn +from kdflow.algorithms import register_algorithm +from kdflow.loss.chunked_loss import chunked_loss +from kdflow.loss.cross_entropy import compute_cross_entropy +from kdflow.metrics.entropy import compute_entropy +from kdflow.utils.logging_utils import init_logger + + +logger = init_logger(__name__) + +@register_algorithm("simple_ctkd") +class SimpleCrossTokenizerKD: + """Simply find the overlap tokens between student and teacher tokenizer, and only compute KD loss on this sub-vocabulary. + Motivation: modern LLMs have a large amount of shared tokens. + """ + def __init__( + self, + strategy, + student_model, + teacher_lm_head, + student_tokenizer, + teacher_tokenizer, + **kwargs + ): + self.strategy = strategy + self.args = strategy.args + self.student = student_model + self.teacher_lm_head = teacher_lm_head + self.student_tokenizer = student_tokenizer + self.teacher_tokenizer = teacher_tokenizer + self.student_overlap_token_ids, self.teacher_overlap_token_ids = self._find_overlap_tokens() + self.loss_fn = build_loss_fn(self.args.kd.kd_loss_fn, self.args) + self.metric_fns = [compute_entropy] if self.args.scenario == "on_policy_kd" else [] + + def _find_overlap_tokens(self): + student_vocab = {k.replace("Ġ", "▁"): v for k, v in self.student_tokenizer.get_vocab().items()} + teacher_vocab = {k.replace("Ġ", "▁"): v for k, v in self.teacher_tokenizer.get_vocab().items()} + overlap_tokens = set(student_vocab.keys()) & set(teacher_vocab.keys()) + student_ids = [student_vocab[token] for token in overlap_tokens] + teacher_ids = [teacher_vocab[token] for token in overlap_tokens] + stu_eos, tea_eos = self.student_tokenizer.eos_token_id, self.teacher_tokenizer.eos_token_id + if stu_eos not in student_ids or tea_eos not in teacher_ids: + student_ids.append(stu_eos) + teacher_ids.append(tea_eos) + device = self.teacher_lm_head.weight.device + logger.info(f"Num of overlap_tokens between student & teacher: {len(student_ids)}") + return torch.tensor(student_ids, dtype=torch.long, device=device), torch.tensor(teacher_ids, dtype=torch.long, device=device) + + def _align_sequences(self, tea_seq, stu_seq): + i, j = 0, 0 + t2s_align, s2t_align = [], [] + history_tea_seq, history_stu_seq = "", "" + + tea_eos = self.teacher_tokenizer.eos_token + stu_eos = self.student_tokenizer.eos_token + + tea_seq = [token.replace('▁', '').replace('Ġ', '') for token in tea_seq] + stu_seq = [token.replace('▁', '').replace('Ġ', '') for token in stu_seq] + + if tea_seq == stu_seq: + indices = list(range(len(tea_seq))) + return indices, indices + + while i < len(tea_seq) and j < len(stu_seq): + is_eos_match = (tea_seq[i] == tea_eos and stu_seq[j] == stu_eos) + if history_tea_seq == history_stu_seq and ( + tea_seq[i] == stu_seq[j] or is_eos_match + ): + common_text = tea_seq[i] + history_tea_seq += common_text + history_stu_seq += common_text + t2s_align.append(i) + s2t_align.append(j) + i += 1 + j += 1 + elif len(history_tea_seq) > len(history_stu_seq): + history_stu_seq += stu_seq[j] + j += 1 + elif len(history_tea_seq) < len(history_stu_seq): + history_tea_seq += tea_seq[i] + i += 1 + else: + history_tea_seq += tea_seq[i] + history_stu_seq += stu_seq[j] + i += 1 + j += 1 + + return t2s_align, s2t_align + + def _get_alignment_indices( + self, + teacher_label_ids, + student_label_ids, + teacher_loss_mask, + student_loss_mask, + ): + """Return indices into the flattened teacher/student response tensors.""" + teacher_tokens = self.teacher_tokenizer.convert_ids_to_tokens( + teacher_label_ids[teacher_loss_mask].cpu().tolist() + ) + student_tokens = self.student_tokenizer.convert_ids_to_tokens( + student_label_ids[student_loss_mask].cpu().tolist() + ) + return self._align_sequences(teacher_tokens, student_tokens) + + def training_step(self, micro_batch): + student_input_ids = micro_batch["stu_input_ids"] + student_attn_mask = micro_batch["stu_attn_mask"] + student_loss_mask = micro_batch["stu_loss_mask"].bool() + teacher_input_ids = micro_batch["tea_input_ids"] + teacher_attn_mask = micro_batch["tea_attn_mask"] + teacher_loss_mask = micro_batch["tea_loss_mask"].bool() + teacher_hiddens = micro_batch.get("teacher_hiddens", None) + avg_token_num = micro_batch["avg_micro_batch_token_num"] + + assert teacher_hiddens is not None, "micro_batch must contain `teacher_hiddens` for KD" + + student_shifted_label_ids = student_input_ids.roll(shifts=-1, dims=1) + teacher_shifted_label_ids = teacher_input_ids.roll(shifts=-1, dims=1) + teacher_aligned_idx, student_aligned_idx = self._get_alignment_indices( + teacher_shifted_label_ids, + student_shifted_label_ids, + teacher_loss_mask, + student_loss_mask, + ) + student_label_ids = student_shifted_label_ids[student_loss_mask] + + mm_kwargs = micro_batch.get("stu_multi_modal_inputs") or {} + + output = self.student( + student_input_ids, + attention_mask=student_attn_mask, + allgather_logits=True, + ring_attn_group=self.strategy.ring_attn_group, + **mm_kwargs, + ) + student_hiddens = output["hidden_states"][-1][student_loss_mask] + del output + + teacher_hiddens = teacher_hiddens.to(self.teacher_lm_head.weight) + + align_ratio = torch.tensor(len(student_aligned_idx) / len(student_label_ids)) + aligned_student_hiddens = student_hiddens[student_aligned_idx] + aligned_teacher_hiddens = teacher_hiddens[teacher_aligned_idx] + + # A length-truncated cross-tokenizer response may contain no 1:1 byte + # span (and no appended EOS fallback). Keep the distributed optimizer + # step well-defined instead of passing a zero chunk size downstream. + if aligned_student_hiddens.shape[0] == 0: + zero_loss = student_hiddens.sum() * 0.0 + return { + "train/loss": zero_loss, + "train/kd_loss": zero_loss, + "distill/alignment_ratio": align_ratio, + } + + # Non-chunked case can be regarded as a special case of chunked loss (i.e., chunk_size = seq_len) + chunk_size = self.args.train.chunked_loss_size or aligned_student_hiddens.shape[0] + + student_overlap_ids = self.student_overlap_token_ids + teacher_overlap_ids = self.teacher_overlap_token_ids + student_lm_head = self.student.model.lm_head + teacher_lm_head = self.teacher_lm_head + + def student_logits_fn(hidden_chunk, skip=False): + return student_lm_head(hidden_chunk, skip=skip)[:, student_overlap_ids] + + def teacher_logits_fn(start, end): + return teacher_lm_head(aligned_teacher_hiddens[start:end])[:, teacher_overlap_ids] + + kd_loss, metric_sums = chunked_loss( + aligned_student_hiddens, student_lm_head, self.loss_fn, + student_logits_fn=student_logits_fn, + teacher_logits_fn=teacher_logits_fn, + chunk_size=chunk_size, reduction="sum", + metric_fns=self.metric_fns, return_metrics=True, + ) + kd_loss = kd_loss / avg_token_num + + loss_info = { + "train/loss": kd_loss, + "train/kd_loss": kd_loss, + "distill/alignment_ratio": align_ratio, + } + loss_info.update({key: value / avg_token_num for key, value in metric_sums.items()}) + + if self.args.kd.kd_ratio < 1: + ce_loss = chunked_loss( + student_hiddens, self.student.model.lm_head, compute_cross_entropy, + label=student_label_ids, chunk_size=chunk_size, reduction="sum" + ) / avg_token_num + loss = (1 - self.args.kd.kd_ratio) * ce_loss + self.args.kd.kd_ratio * kd_loss + loss_info["train/loss"] = loss + loss_info["train/ce_loss"] = ce_loss + + return loss_info diff --git a/kdflow/algorithms/topk_byte_align_ctkd.py b/kdflow/algorithms/topk_byte_align_ctkd.py new file mode 100644 index 0000000000000000000000000000000000000000..64b49354bea827893fd49f8de51325666f7e1dba --- /dev/null +++ b/kdflow/algorithms/topk_byte_align_ctkd.py @@ -0,0 +1,101 @@ +from typing import Tuple + +import torch + +from kdflow.algorithms import register_algorithm +from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD +from kdflow.metrics.entropy import compute_entropy + + +def select_student_topk_logits( + student_logits: torch.Tensor, + teacher_logits: torch.Tensor, + topk: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Select the student's top-k shared-vocabulary entries on both sides. + + The returned tensors contain matching vocabulary entries. Downstream KD + losses apply softmax/log-softmax to these reduced logits, which + re-normalizes both distributions over exactly this top-k support. + """ + if student_logits.shape != teacher_logits.shape: + raise ValueError( + "Student and teacher shared-vocabulary logits must have the same " + f"shape, got {student_logits.shape} and {teacher_logits.shape}." + ) + if topk <= 0 or topk > student_logits.shape[-1]: + raise ValueError( + f"topk must be in [1, {student_logits.shape[-1]}], got {topk}." + ) + + student_topk_logits, topk_indices = torch.topk( + student_logits, + k=topk, + dim=-1, + ) + teacher_topk_logits = teacher_logits.gather(-1, topk_indices) + return student_topk_logits, teacher_topk_logits, topk_indices + + +@register_algorithm("topk_byte_align_ctkd") +class TopKByteAlignCrossTokenizerKD(ByteAlignCrossTokenizerKD): + """Byte-aligned CTKD restricted to each student's shared-vocab top-k.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.ctkd_topk = self.args.kd.ctkd_topk + overlap_vocab_size = self.student_overlap_token_ids.numel() + if self.ctkd_topk < 2: + raise ValueError( + "ctkd_topk must be at least 2 because re-normalizing a " + "single logit produces a degenerate distribution." + ) + if self.ctkd_topk > overlap_vocab_size: + raise ValueError( + f"ctkd_topk ({self.ctkd_topk}) exceeds the shared vocabulary " + f"size ({overlap_vocab_size})." + ) + if ( + self.args.kd.kd_loss_fn == "hrl" + and self.args.kd.hrl_topk > self.ctkd_topk + ): + raise ValueError( + f"hrl_topk ({self.args.kd.hrl_topk}) cannot exceed ctkd_topk " + f"({self.ctkd_topk})." + ) + + self._shared_vocab_loss_fn = self.loss_fn + self.loss_fn = self._topk_loss + if self.metric_fns: + self.metric_fns = [self._topk_entropy] + + def _topk_loss( + self, + student_logits: torch.Tensor, + teacher_logits: torch.Tensor, + **kwargs, + ) -> torch.Tensor: + student_topk_logits, teacher_topk_logits, _ = select_student_topk_logits( + student_logits, + teacher_logits, + self.ctkd_topk, + ) + return self._shared_vocab_loss_fn( + student_topk_logits, + teacher_topk_logits, + **kwargs, + ) + + def _topk_entropy( + self, + student_logits: torch.Tensor, + teacher_logits: torch.Tensor, + **kwargs, + ): + student_topk_logits, _, _ = select_student_topk_logits( + student_logits, + teacher_logits, + self.ctkd_topk, + ) + return compute_entropy(student_logits=student_topk_logits) diff --git a/kdflow/arguments/__init__.py b/kdflow/arguments/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b3c66702102a8691bcd16bc57f8a8307ccb13b68 --- /dev/null +++ b/kdflow/arguments/__init__.py @@ -0,0 +1,309 @@ +import os +from dataclasses import dataclass, field + +from transformers import HfArgumentParser + +from kdflow.arguments.data_args import DataArguments +from kdflow.arguments.model_args import ModelArguments +from kdflow.arguments.training_args import TrainingArguments +from kdflow.arguments.fsdp_args import FSDPArguments +from kdflow.arguments.distillation_args import DistillationArguments +from kdflow.arguments.rollout_args import RolloutArguments +from kdflow.arguments.logging_args import LoggingArguments +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +@dataclass +class AllArguments: + data: DataArguments = field(default_factory=DataArguments) + model: ModelArguments = field(default_factory=ModelArguments) + train: TrainingArguments = field(default_factory=TrainingArguments) + fsdp: FSDPArguments = field(default_factory=FSDPArguments) + kd: DistillationArguments = field(default_factory=DistillationArguments) + rollout: RolloutArguments = field(default_factory=RolloutArguments) + log: LoggingArguments = field(default_factory=LoggingArguments) + + +def is_linear_attention(model_path: str) -> bool: + if not model_path: + return False + try: + from transformers import AutoConfig + model_type = AutoConfig.from_pretrained(model_path, trust_remote_code=True).model_type or "" + except Exception: + return False + return any(k in model_type.lower() for k in ("qwen3_next", "qwen3_5")) + +def init_args(scenario: str = "on_policy_kd"): + parser = HfArgumentParser(( + DataArguments, + ModelArguments, + TrainingArguments, + FSDPArguments, + DistillationArguments, + RolloutArguments, + LoggingArguments + )) + ( + data_args, + model_args, + train_args, + fsdp_args, + kd_args, + rollout_args, + log_args + ) = parser.parse_args_into_dataclasses() + + args = AllArguments( + data=data_args, + model=model_args, + train=train_args, + fsdp=fsdp_args, + kd=kd_args, + rollout=rollout_args, + log=log_args + ) + + if scenario != "on_policy_kd" and args.data.custom_eval_fn is not None: + logger.warning( + f"`--custom_eval_fn` is only supported in the `on_policy_kd` scenario " + f"and will be ignored for `{scenario}`." + ) + if scenario != "on_policy_kd" and args.data.eval_math_dataset_path is not None: + logger.warning( + f"`--eval_math_dataset_path` is only supported in the `on_policy_kd` " + f"scenario and will be ignored for `{scenario}`." + ) + + if args.data.input_template and "{}" not in args.data.input_template: + logger.warning("{} not in args.data.input_template, set to None") + args.data.input_template = None + + if args.data.input_template and "\\n" in args.data.input_template: + logger.warning( + "input_template contains \\n characters instead of newline. " + "You likely want to pass $'\\n' in Bash or \"`n\" in PowerShell." + ) + + if args.data.packing_samples: + if "flash_attention" not in args.model.attn_implementation: + logger.warning( + "Please use --attn_implementation with flash_attention to accelerate when --packing_samples is enabled." + ) + args.model.attn_implementation = "flash_attention_2" + + if is_linear_attention(args.model.student_name_or_path): + logger.warning( + f"--packing_samples is not yet compatible with linear-attention models " + f"(got {args.model.student_name_or_path}). Auto-disabling --packing_samples." + ) + args.data.packing_samples = False + + total_gpus = args.train.num_nodes * args.train.num_gpus_per_node + + if scenario == "alfworld_eval": + if not args.model.student_name_or_path: + raise ValueError( + "Standalone ALFWorld evaluation requires " + "`--student_name_or_path`." + ) + if not args.rollout.alfworld_config: + raise ValueError( + "Standalone ALFWorld evaluation requires `--alfworld_config`." + ) + valid_protocols = {"kdflow", "verl_agent"} + if args.rollout.alfworld_eval_protocol not in valid_protocols: + raise ValueError( + "`--alfworld_eval_protocol` must be one of " + f"{sorted(valid_protocols)}." + ) + if args.rollout.alfworld_eval_protocol == "verl_agent": + try: + seeds = [ + int(value.strip()) + for value in args.rollout.alfworld_eval_seeds.split(",") + if value.strip() + ] + except ValueError as error: + raise ValueError( + "`--alfworld_eval_seeds` must be a comma-separated list " + "of integers." + ) from error + if not seeds: + raise ValueError( + "`--alfworld_eval_seeds` must contain at least one seed." + ) + if args.rollout.alfworld_history_length < 0: + raise ValueError( + "`--alfworld_history_length` must be non-negative." + ) + + if scenario in ("on_policy_kd", "alfworld_eval"): + if total_gpus <= 0: + raise ValueError( + "`--num_nodes * --num_gpus_per_node` must be positive." + ) + if args.rollout.rollout_tp_size <= 0: + raise ValueError("`--rollout_tp_size` must be positive.") + if args.rollout.rollout_batch_size <= 0: + raise ValueError("`--rollout_batch_size` must be positive.") + if args.rollout.rollout_engine_concurrency <= 0: + raise ValueError( + "`--rollout_engine_concurrency` must be positive." + ) + if args.rollout.alfworld_config: + valid_alfworld_protocols = {"kdflow", "verl_agent"} + if args.rollout.alfworld_protocol not in valid_alfworld_protocols: + raise ValueError( + "`--alfworld_protocol` must be one of " + f"{sorted(valid_alfworld_protocols)}." + ) + args.rollout.alfworld_config = os.path.abspath( + os.path.expanduser(args.rollout.alfworld_config) + ) + if not os.path.isfile(args.rollout.alfworld_config): + raise ValueError( + "`--alfworld_config` must point to an existing YAML file, " + f"got {args.rollout.alfworld_config!r}." + ) + valid_alfworld_splits = { + "train", + "eval_in_distribution", + "eval_out_of_distribution", + } + if args.rollout.alfworld_split not in valid_alfworld_splits: + raise ValueError( + "`--alfworld_split` must be one of " + f"{sorted(valid_alfworld_splits)}." + ) + if args.rollout.alfworld_max_turns <= 0: + raise ValueError("`--alfworld_max_turns` must be positive.") + if args.rollout.alfworld_episodes_per_epoch <= 0: + raise ValueError( + "`--alfworld_episodes_per_epoch` must be positive." + ) + if args.rollout.alfworld_eval_episodes <= 0: + raise ValueError("`--alfworld_eval_episodes` must be positive.") + if args.rollout.alfworld_admissible_limit < 0: + raise ValueError( + "`--alfworld_admissible_limit` must be non-negative." + ) + if args.rollout.n_samples_per_prompt != 1: + raise ValueError( + "ALFWorld multi-turn rollout requires " + "`--n_samples_per_prompt 1`; the environment already " + "samples one independent game per batch slot." + ) + if scenario == "on_policy_kd" and ( + args.rollout.alfworld_episodes_per_epoch + < args.rollout.rollout_batch_size + ): + raise ValueError( + "`--alfworld_episodes_per_epoch` must be at least " + "`--rollout_batch_size`." + ) + remainder = ( + args.rollout.alfworld_episodes_per_epoch + % args.rollout.rollout_batch_size + ) + if scenario == "on_policy_kd" and remainder: + logger.warning( + "ALFWorld drops %d episode slots per epoch because " + "alfworld_episodes_per_epoch is not divisible by " + "rollout_batch_size.", + remainder, + ) + if scenario == "on_policy_kd" and args.data.eval_dataset_path: + logger.warning( + "`--eval_dataset_path` is ignored for ALFWorld multi-turn " + "training; use an ALFWorld evaluation split in a separate run." + ) + + if ( + scenario == "alfworld_eval" + and args.rollout.alfworld_eval_episodes + % args.rollout.rollout_batch_size + ): + raise ValueError( + "`--alfworld_eval_episodes` must be divisible by " + "`--rollout_batch_size`." + ) + + if scenario == "on_policy_kd": + if args.data.eval_math_n_samples <= 0: + raise ValueError("`--eval_math_n_samples` must be positive.") + if args.data.eval_math_max_concurrent <= 0: + raise ValueError("`--eval_math_max_concurrent` must be positive.") + if args.data.eval_math_max_tokens <= 0: + raise ValueError("`--eval_math_max_tokens` must be positive.") + if args.data.eval_math_timeout < 0: + raise ValueError("`--eval_math_timeout` must be non-negative.") + + if total_gpus % args.rollout.rollout_tp_size != 0: + raise ValueError( + f"Total GPUs ({total_gpus}) must be divisible by rollout_tp_size ({args.rollout.rollout_tp_size})." + ) + + expected_num_engines = total_gpus // args.rollout.rollout_tp_size + if args.rollout.rollout_num_engines != expected_num_engines: + logger.warning( + f"Auto-adjusting rollout_num_engines from {args.rollout.rollout_num_engines} to {expected_num_engines} " + f"to match total GPUs ({total_gpus}). " + f"(rollout_tp_size={args.rollout.rollout_tp_size} * rollout_num_engines={expected_num_engines} = {total_gpus})" + ) + args.rollout.rollout_num_engines = expected_num_engines + + if args.data.max_len < args.data.prompt_max_len + args.rollout.generate_max_len: + args.data.max_len = args.data.prompt_max_len + args.rollout.generate_max_len + logger.warning( + "--max_len is smaller than --prompt_max_len + --generate_max_len. " + f"Automatically increase --max_len to {args.data.max_len}." + ) + + has_teacher = args.model.teacher_name_or_path is not None + is_kd_scenario = scenario in ("off_policy_kd", "on_policy_kd") + if is_kd_scenario: + if not has_teacher: + raise ValueError( + "KD training requires `--teacher_name_or_path` to be set." + ) + + teacher_parallel = args.kd.teacher_tp_size * args.kd.teacher_pp_size + if total_gpus % teacher_parallel != 0: + raise ValueError( + f"Total GPUs ({total_gpus}) must be divisible by " + f"teacher_tp_size * teacher_pp_size ({args.kd.teacher_tp_size} * {args.kd.teacher_pp_size} = {teacher_parallel})." + ) + + if args.kd.teacher_ep_size > args.kd.teacher_tp_size: + logger.warning( + f"teacher_ep_size ({args.kd.teacher_ep_size}) must not be greater than teacher_tp_size ({args.kd.teacher_tp_size}). " + f"Auto-adjusting teacher_ep_size to {args.kd.teacher_tp_size}." + ) + args.kd.teacher_ep_size = args.kd.teacher_tp_size + + expected_dp = total_gpus // teacher_parallel + if args.kd.teacher_dp_size != expected_dp: + logger.warning( + f"Auto-adjusting teacher_dp_size from {args.kd.teacher_dp_size} to {expected_dp} " + f"to match total GPUs ({total_gpus}). " + f"(tp={args.kd.teacher_tp_size} (ep={args.kd.teacher_ep_size}) * pp={args.kd.teacher_pp_size} * dp={expected_dp} = {total_gpus})" + ) + args.kd.teacher_dp_size = expected_dp + + + deprecated_sleep_flags = [ + args.train.train_enable_sleep, + args.kd.teacher_enable_sleep, + args.rollout.rollout_enable_sleep, + ] + if any(deprecated_sleep_flags): + logger.warning( + "--train_enable_sleep, --teacher_enable_sleep, --rollout_enable_sleep are deprecated " + "and will be removed in a future version. Use --enable_sleep instead." + ) + args.train.enable_sleep = True + + args.scenario = scenario + return args diff --git a/kdflow/arguments/data_args.py b/kdflow/arguments/data_args.py new file mode 100644 index 0000000000000000000000000000000000000000..12046f174777ed5d8f271b15661debf0b3c63003 --- /dev/null +++ b/kdflow/arguments/data_args.py @@ -0,0 +1,107 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class DataArguments: + """ Arguments for dataset.""" + + train_dataset_path: str = field( + default=None, + metadata={"help": "Training dataset name or path."} + ) + train_dataset_probs: str = field( + default=None, + metadata={"help": "Sampling probs for multiple datasets."} + ) + train_split: str = field( + default="train", + metadata={"help": "Train split in dataset."} + ) + eval_dataset_path: str = field( + default=None, + metadata={"help": "Evaluation dataset name or path."} + ) + eval_split: str = field( + default="eval", + metadata={"help": "Eval split in dataset."} + ) + custom_eval_fn: str = field( + default=None, + metadata={ + "help": "Optional Python file defining eval_fn(predictions, labels), " + "which must return a metric dict (only supported in on-policy KD)." + } + ) + eval_math_dataset_path: str = field( + default=None, + metadata={ + "help": "Optional math evaluation JSONL path. Each row must contain " + "'problem' and 'answer' fields (only supported in on-policy KD)." + } + ) + eval_math_n_samples: int = field( + default=1, + metadata={"help": "Number of generated answers per math problem for pass@k evaluation."} + ) + eval_math_max_concurrent: int = field( + default=256, + metadata={"help": "Maximum number of concurrent math evaluation requests."} + ) + eval_math_max_tokens: int = field( + default=8192, + metadata={"help": "Maximum number of new tokens generated for math evaluation."} + ) + eval_math_timeout: int = field( + default=1800, + metadata={"help": "Total timeout in seconds for math evaluation generation."} + ) + input_key: str = field( + default="messages", + metadata={"help": "JSON dataset key."} + ) + image_key: str = field( + default=None, + metadata={"help": "Image key in dataset."} + ) + output_key: str = field( + default=None, + metadata={"help": "Output key in dataset."} + ) + teacher_input_key: str = field( + default=None, + metadata={"help": "Input key for teacher prompt. If None, use the same input_key as student. " + "Used in self-distillation where teacher receives richer information."} + ) + label_key: str = field( + default=None, + metadata={"help": "Label key in dataset."} + ) + teacher_routing_key: str = field( + default="teacher_routing_key", + metadata={"help": "Route key for teacher model in multi-teacher distillation."} + ) + input_template: str = field( + default=None, + ) + apply_chat_template: bool = field( + default=True, + metadata={"help": "Use HF tokenizer chat template."} + ) + max_len: int = field( + default=4096 + ) + max_samples: int = field( + default=1e8 + ) + packing_samples: bool = field( + default=False, + metadata={"help": "Packing sequences during training."} + ) + prompt_max_len: int = field( + default=2048, + metadata={"help": "Max prompt length."} + ) + preprocess_num_workers: int = field( + default=1, + ) diff --git a/kdflow/arguments/distillation_args.py b/kdflow/arguments/distillation_args.py new file mode 100644 index 0000000000000000000000000000000000000000..ae835897855f7e4147fb8efc1fe5971fd8cfb964 --- /dev/null +++ b/kdflow/arguments/distillation_args.py @@ -0,0 +1,142 @@ +import logging +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + + +@dataclass +class DistillationArguments: + """ Arguments for knowledge distillation.""" + + kd_ratio: float = field( + default=0.5, + metadata={"help": "Loss = (1 - kd_ratio) * nll_loss + kd_ratio * kd_loss."} + ) + kd_temperature: float = field( + default=1.0, + metadata={"help": "Temperature for knowledge distillation."} + ) + kd_algorithm: str = field( + default="byte_align_ctkd", + metadata={ + "help": "KD algorithm for each training step.", + "choices": ["simple_ctkd", "byte_align_ctkd", "topk_byte_align_ctkd"], + } + ) + kd_loss_fn: str = field( + default="kl", + metadata={"help": "Divergence selection for knowledge distillation, e.g., kl, rkl, js."} + ) + ctkd_topk: int = field( + default=128, + metadata={ + "help": ( + "Student top-k size within the shared vocabulary for " + "topk_byte_align_ctkd." + ) + }, + ) + teacher_forward_n_batches: int = field( + default=1, + metadata={"help": "Teacher forward N global batches at once for student multi-step training."} + ) + teacher_enable_sleep: bool = field( + default=False, + metadata={"help": "Sleep teacher when not needed."} + ) + teacher_offload_tags: str = field( + default="all", + metadata={"help": "Offload tags for sglang."} + ) + teacher_quantization: str = field( + default=None + ) + teacher_tp_size: int = field( + default=8, + metadata={"help": "Tensor parallel size for teacher model."} + ) + teacher_ep_size: int = field( + default=1, + metadata={"help": "Expert parallel size for teacher model (only for MoE models)."} + ) + teacher_pp_size: int = field( + default=1, + metadata={"help": "Pipeline parallel size for teacher model."} + ) + teacher_dp_size: int = field( + default=1, + metadata={"help": "Data parallel size for teacher model."} + ) + teacher_mem_fraction_static: float = field( + default=0.4, + metadata={"help": "Memory fraction for teacher model."} + ) + teacher_update_freq: int = field( + default=10, + metadata={"help": "Weight update frequency for teacher model."} + ) + use_ema_teacher: bool = field( + default=False, + metadata={"help": "Use EMA update for teacher model."} + ) + teacher_ema_decay: float = field( + default=0.999, + metadata={"help": "EMA decay for teacher model."} + ) + multi_teacher_config: str = field( + default=None, + metadata={"help": "Unsupported in this release; use --teacher_name_or_path."}, + ) + # JSD + jsd_beta: float = field( + default=0.5, + metadata={"help": "Beta for Jensen-Shannon Divergence."} + ) + # Skewed KL/RKL + skew_lambda: float = field( + default=0.1, + metadata={"help": "Lambda for Skewed KL/RKL."} + ) + # Adaptive KL + adaptive_alpha: float = field( + default=0.5, + metadata={"help": "Alpha for Adaptive KL Divergence."} + ) + # Hierarchical Ranking Loss + hrl_topk: int = field( + default=5, + metadata={"help": "Top-k Ranking for Hierarchical Ranking Loss."} + ) + + def __post_init__(self): + if self.multi_teacher_config is not None: + raise ValueError( + "This release supports one teacher model per run; " + "use --teacher_name_or_path instead of --multi_teacher_config." + ) + supported_algorithms = {"simple_ctkd", "byte_align_ctkd", "topk_byte_align_ctkd"} + if self.kd_algorithm not in supported_algorithms: + raise ValueError( + f"kd_algorithm must be one of {sorted(supported_algorithms)}, " + f"got {self.kd_algorithm!r}." + ) + # Validate teacher parallel size settings + if self.teacher_ep_size > self.teacher_tp_size: + raise ValueError( + f"SGLang requires that teacher_ep_size ({self.teacher_ep_size}) must be <= teacher_tp_size ({self.teacher_tp_size}). " + ) + if self.teacher_tp_size % self.teacher_ep_size != 0: + raise ValueError( + f"SGLang requires that teacher_tp_size ({self.teacher_tp_size}) must be divisible by teacher_ep_size ({self.teacher_ep_size})." + ) + # Validate KD hyperparameters + if not 0.0 <= self.kd_ratio <= 1.0: + raise ValueError(f"kd_ratio must be in [0, 1], got {self.kd_ratio}.") + if self.kd_temperature <= 0: + raise ValueError(f"kd_temperature must be > 0, got {self.kd_temperature}.") + if self.ctkd_topk < 2: + raise ValueError( + f"ctkd_topk must be at least 2, got {self.ctkd_topk}." + ) + if not 0.0 < self.teacher_mem_fraction_static <= 1.0: + raise ValueError(f"teacher_mem_fraction_static must be in (0, 1], got {self.teacher_mem_fraction_static}.") diff --git a/kdflow/arguments/fsdp_args.py b/kdflow/arguments/fsdp_args.py new file mode 100644 index 0000000000000000000000000000000000000000..ed2729e8de9b94f789cd73ea1274e02bc4e18558 --- /dev/null +++ b/kdflow/arguments/fsdp_args.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class FSDPArguments: + """ Arguments for fsdp backend.""" + + fsdp_size: int = field( + default=-1, + metadata={"help": "FSDP shard size (for HSDP)."} + ) + cpu_offload: bool = field( + default=False, + metadata={"help": "Offload Adam optimizer to GPU."} + ) + + \ No newline at end of file diff --git a/kdflow/arguments/logging_args.py b/kdflow/arguments/logging_args.py new file mode 100644 index 0000000000000000000000000000000000000000..53e1413ef1c4b57d66b57a22c1c3c7022cf9d70a --- /dev/null +++ b/kdflow/arguments/logging_args.py @@ -0,0 +1,37 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class LoggingArguments: + """ Arguments for logging (e.g., wandb and tensorboard).""" + + logging_steps: int = field( + default=10, + metadata={"help": "Log results every n steps."} + ) + use_wandb: bool = field( + default=False, + metadata={"help": "Use wandb for logging."} + ) + wandb_org: str = field( + default=None + ) + wandb_project: str = field( + default=None + ) + wandb_group: str = field( + default=None + ) + wandb_run_name: str = field( + default=None + ) + wandb_mode: str = field( + default="online", + metadata={"help": "wandb mode: online, offline, or disabled."} + ) + wandb_dir: str = field( + default=None, + metadata={"help": "Directory to store wandb offline logs."} + ) + \ No newline at end of file diff --git a/kdflow/arguments/model_args.py b/kdflow/arguments/model_args.py new file mode 100644 index 0000000000000000000000000000000000000000..9c6531d1deb2435054eeff4c412271149390b336 --- /dev/null +++ b/kdflow/arguments/model_args.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class ModelArguments: + """ Arguments for model.""" + + student_name_or_path: str = field( + default=None, + metadata={"help": "Student model name or path."} + ) + teacher_name_or_path: str = field( + default=None, + metadata={"help": "Teacher model name or path."} + ) + attn_implementation: str = field( + default="flash_attention_2", + metadata={"help": "Attention implementation (e.g., eager, flash_attention_2, flash_attention_3, kernels-community/vllm-flash-attn3)."} + ) + use_liger_kernel: bool = field( + default=False, + metadata={"help": "Use Liger Kernel in LLM."} + ) + lora_rank: int = field( + default=0 + ) + lora_alpha: int = field( + default=16 + ) + target_modules: str = field( + default="all-linear", + ) + lora_dropout: float = field( + default=0.0, + ) + disable_fast_tokenizer: bool = field( + default=False + ) + enable_thinking: bool = field( + default=False, + ) + ring_attn_size: int = field( + default=1 + ) \ No newline at end of file diff --git a/kdflow/arguments/rollout_args.py b/kdflow/arguments/rollout_args.py new file mode 100644 index 0000000000000000000000000000000000000000..5b46da1da4da0f4a05a53aa14ef36167517784e4 --- /dev/null +++ b/kdflow/arguments/rollout_args.py @@ -0,0 +1,132 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class RolloutArguments: + """ Arguments for rollout (on-policy distillation).""" + + rollout_num_engines: int = field( + default=0, + metadata={"help": "The number of engines for rollout."} + ) + rollout_tp_size: int = field( + default=1, + metadata={"help": "Tensor parallel size for each rollout engine."} + ) + rollout_engine_concurrency: int = field( + default=512, + metadata={"help": "Maximum concurrent requests per rollout engine."} + ) + rollout_enable_sleep: bool = field( + default=False, + metadata={"help": "Enable sleep mode for rollout engines."} + ) + rollout_mem_fraction_static: float = field( + default=0.6, + metadata={"help": "GPU memory utilization for each rollout engine."} + ) + top_p: float = field( + default=1.0, + metadata={"help": "Top-p sampling for rollout."} + ) + temperature: float = field( + default=1.0, + metadata={"help": "Temperature for rollout."} + ) + n_samples_per_prompt: int = field( + default=1, + metadata={"help": "Sample n responses per prompt."} + ) + rollout_batch_size: int = field( + default=32, + metadata={"help": "Number of prompts for each rollout."} + ) + generate_max_len: int = field( + default=2048, + metadata={"help": "Max generation tokens during rollout."} + ) + print_rollout_sample: bool = field( + default=False, + metadata={"help": "Whether to print a rollout sample after each rollout."} + ) + alfworld_config: Optional[str] = field( + default=None, + metadata={ + "help": "Path to an ALFWorld YAML config. When set, on-policy KD " + "collects multi-turn student trajectories from AlfredTWEnv." + }, + ) + alfworld_protocol: str = field( + default="kdflow", + metadata={ + "help": "ALFWorld training rollout protocol: 'kdflow' keeps the " + "original Thought/Action chat history, while 'verl_agent' " + "uses the migrated single-prompt / protocol." + }, + ) + alfworld_split: str = field( + default="train", + metadata={ + "help": "ALFWorld split: train, eval_in_distribution, or " + "eval_out_of_distribution." + }, + ) + alfworld_max_turns: int = field( + default=30, + metadata={"help": "Maximum assistant/environment turns per ALFWorld episode."}, + ) + alfworld_episodes_per_epoch: int = field( + default=1024, + metadata={"help": "Number of ALFWorld episodes sampled per training epoch."}, + ) + alfworld_eval_episodes: int = field( + default=128, + metadata={ + "help": "Number of episodes for standalone ALFWorld evaluation." + }, + ) + alfworld_eval_protocol: str = field( + default="kdflow", + metadata={ + "help": "Standalone ALFWorld evaluation protocol. Use 'kdflow' " + "for the original KDFlow prompt or 'verl_agent' for the " + "protocol migrated from langfengq/verl-agent." + }, + ) + alfworld_eval_seeds: str = field( + default="123,456,789", + metadata={ + "help": "Comma-separated script-level seeds used by the " + "verl-agent-compatible evaluator." + }, + ) + alfworld_eval_seed_offset: int = field( + default=1000, + metadata={ + "help": "Offset added to each script-level seed for the validation " + "environment. verl-agent uses 1000." + }, + ) + alfworld_eval_resume: bool = field( + default=True, + metadata={ + "help": "Reuse completed, configuration-compatible per-seed " + "ALFWorld evaluation results. Set False to force rerunning " + "every seed." + }, + ) + alfworld_history_length: int = field( + default=4, + metadata={ + "help": "Number of recent observation/action pairs included by the " + "verl-agent ALFWorld prompt." + }, + ) + alfworld_admissible_limit: int = field( + default=30, + metadata={ + "help": "Maximum number of ALFWorld admissible actions shown per turn; " + "use 0 to show all actions." + }, + ) diff --git a/kdflow/arguments/training_args.py b/kdflow/arguments/training_args.py new file mode 100644 index 0000000000000000000000000000000000000000..71443753b8ea33fd213a28c0744816c77b616600 --- /dev/null +++ b/kdflow/arguments/training_args.py @@ -0,0 +1,182 @@ +import os +import math +from dataclasses import dataclass, field +from typing import Optional + + +def resolve_max_steps( + configured_max_steps: int, + num_epochs: int, + steps_per_epoch: int, +) -> int: + """Resolve the explicit step limit or the epoch-derived training length.""" + if steps_per_epoch <= 0: + raise ValueError( + "Training requires at least one full global step per epoch. " + "Reduce the global batch size or provide more training samples." + ) + if configured_max_steps > 0: + return configured_max_steps + return math.ceil(num_epochs * steps_per_epoch) + + +@dataclass +class TrainingArguments: + """ Arguments for training.""" + + num_nodes: int = field( + default=1, + metadata={"help": "The number of nodes for the student/teacher model (only for on-policy KD)."} + ) + num_gpus_per_node: int = field( + default=8, + metadata={"help": "The number of gpus per node for the student/teacher model (only for on-policy KD)."} + ) + num_epochs: int = field( + default=1, + metadata={"help": "Number of training epochs."} + ) + max_steps: int = field( + default=-1, + metadata={ + "help": ( + "Maximum global training steps. A positive value overrides " + "num_epochs; for on-policy KD, one global step is one rollout " + "iteration." + ) + }, + ) + train_batch_size: int = field( + default=128, + metadata={"help": "Global training batch size."} + ) + micro_train_batch_size: int = field( + default=1, + metadata={"help": "Micro training batch size per GPU."} + ) + learning_rate: float = field( + default=1e-6 + ) + lr_warmup_ratio: float = field( + default=0.05 + ) + min_lr: float = field( + default=1e-8 + ) + lr_scheduler: str = field( + default="cosine_with_min_lr" + ) + max_norm: float = field( + default=1.0, + metadata={"help": "Gradient clipping."} + ) + weight_decay: float = field( + default=0.0, + metadata={"help": "Weight decay."} + ) + adam_betas: str = field( + default="(0.9, 0.98)" + ) + eval_steps: int = field( + default=-1, + metadata={"help": "Evaluation every n step."} + ) + save_steps: int = field( + default=-1, + metadata={"help": "Save checkpoints every n step."} + ) + backend: str = field( + default="fsdp2", + metadata={ + "choices": ["fsdp2"], + "help": "Student training backend." + } + ) + gradient_checkpointing: bool = field( + default=False + ) + gradient_checkpointing_use_reentrant: bool = field( + default=False + ) + train_enable_sleep: bool = field( + default=False, + metadata={"help": "[Deprecated] Use --enable_sleep instead."} + ) + enable_sleep: bool = field( + default=False, + metadata={"help": "Enable sleep mode for all components (student, teacher, rollout)."} + ) + full_determinism: bool = field( + default=False, + metadata={"help": "Enable reproducible behavior during distributed training."} + ) + load_checkpoint: bool = field( + default=False + ) + ckpt_path: str = field( + default="./ckpt/checkpoints_distill" + ) + save_path: str = field( + default="./ckpt/" + ) + seed: int = field( + default=42, + ) + use_dynamic_bsz: bool = field( + default=False, + metadata={"help": "Enable dynamic batch size based on token count per GPU."} + ) + max_token_len_per_gpu: int = field( + default=0, + metadata={"help": "Maximum total token count per micro-batch when use_dynamic_bsz is True."} + ) + bf16: bool = field( + default=False + ) + local_rank: int = field( + default=int(os.getenv("LOCAL_RANK", -1)), + metadata={"help": "Local rank for distributed training. Automatically set."} + ) + chunked_loss_size: int = field( + default=None, + metadata={"help": "Token chunk size for chunked loss. If None, chunked loss is disabled."} + ) + + def __post_init__(self): + # Ray actors run from a temporary working directory. Resolve output paths + # in the driver before the arguments are serialized and sent to actors so + # relative paths keep their command-invocation semantics. + self.save_path = self._resolve_driver_path(self.save_path) + self.ckpt_path = self._resolve_driver_path(self.ckpt_path) + + backend_choices = self.__class__.__dataclass_fields__["backend"].metadata["choices"] + if self.backend not in backend_choices: + raise ValueError( + f"Unsupported training backend '{self.backend}'" + f"Supported choices: {backend_choices}" + ) + + if isinstance(self.adam_betas, str): + try: + cleaned_str = self.adam_betas.strip().strip('()') + parts = [p.strip() for p in cleaned_str.split(',')] + if len(parts) != 2: + raise ValueError("adam_betas must contain two values.") + + self.adam_betas = (float(parts[0]), float(parts[1])) + except Exception as e: + raise ValueError( + f"Cannot parse '{self.adam_betas}' to valid adam_betas. Make sure the format to be '(beta1, beta2)'。" + ) from e + else: + raise TypeError(f"Expected str for adam_betas, but get {type(self.adam_betas)}") + + if self.save_steps <= 0: + self.save_steps = float("inf") + + if self.eval_steps <= 0: + self.eval_steps = float("inf") + + @staticmethod + def _resolve_driver_path(path: str) -> str: + return os.path.abspath(os.path.expanduser(path)) diff --git a/kdflow/backend/__init__.py b/kdflow/backend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ae07fdf84af178d7d1b63e06abefb6f07f29e890 --- /dev/null +++ b/kdflow/backend/__init__.py @@ -0,0 +1,18 @@ +from kdflow.backend.fsdp import FSDP2Strategy + + +def get_strategy(args): + if args.train.backend == "fsdp2": + strategy = FSDP2Strategy( + seed=getattr(args.train, "seed", 42), + full_determinism=getattr(args.train, "full_determinism", False), + max_norm=getattr(args.train, "max_norm", 1.0), + micro_train_batch_size=getattr(args.train, "micro_train_batch_size", 1), + train_batch_size=getattr(args.train, "train_batch_size", 128), + bf16=getattr(args.train, "bf16", True), + args=args, + ) + else: + raise NotImplementedError + + return strategy \ No newline at end of file diff --git a/kdflow/backend/fsdp/__init__.py b/kdflow/backend/fsdp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a50fa8e305c891e9bfe23089b39c6f0555d473af --- /dev/null +++ b/kdflow/backend/fsdp/__init__.py @@ -0,0 +1,5 @@ +from .fsdp_strategy import FSDP2Strategy + +__all__ = [ + "FSDP2Strategy", +] \ No newline at end of file diff --git a/kdflow/backend/fsdp/fsdp_strategy.py b/kdflow/backend/fsdp/fsdp_strategy.py new file mode 100644 index 0000000000000000000000000000000000000000..63498e48936f06180363b4df5c7fde821fa36a70 --- /dev/null +++ b/kdflow/backend/fsdp/fsdp_strategy.py @@ -0,0 +1,704 @@ +import os +import gc +import functools +import logging +import torch + +import torch.distributed as dist +import torch.nn as nn +import transformers +from abc import ABC +from datetime import timedelta +from peft import PeftModel, get_peft_model_state_dict +from torch.distributed.device_mesh import init_device_mesh, DeviceMesh +from torch.optim import Optimizer, AdamW +from torch.distributed.fsdp import ( + fully_shard, + MixedPrecisionPolicy, + CPUOffloadPolicy, +) +from torch.distributed.tensor import DTensor +from torch.distributed.checkpoint.state_dict import ( + get_model_state_dict, + set_model_state_dict, + StateDictOptions, +) +from torchdata.stateful_dataloader import StatefulDataLoader + +from kdflow.utils.logging_utils import init_logger +from kdflow.models import DistillModel +from kdflow.utils.distributed_sampler import DistributedSampler +from kdflow.models.ring_attn_utils import get_ring_attn_group, set_ring_attn_group +from kdflow.utils.distributed_util import torch_dist_barrier_and_cuda_sync + + +class FSDP2Strategy(ABC): + def __init__( + self, + seed: int = 42, + full_determinism: bool = False, + max_norm: float = 0.0, + micro_train_batch_size=1, + train_batch_size=1, + bf16=True, + cpu_offload: bool = False, + args=None, + ) -> None: + self.args = args + self.seed = seed + self.max_norm = max_norm + self.micro_train_batch_size = micro_train_batch_size + self.train_batch_size = train_batch_size + self.cpu_offload = cpu_offload + self.bf16 = bf16 + self.full_determinism = full_determinism + self.logger = init_logger("FSDP2Strategy") + + def log(self, msg: str, level: str = "info", rank_0_only: bool = True) -> None: + """ + Log a message with rank information. + + Args: + msg: The message to log. + level: Log level ('debug', 'info', 'warning', 'error', 'critical'). + rank_0_only: If True, only log on rank 0. + """ + if rank_0_only and not self.is_rank_0(): + return + + rank = self.get_rank() if dist.is_initialized() else 0 + full_msg = f"[Rank {rank}] {msg}" + + log_func = getattr(self.logger, level.lower(), self.logger.info) + # Use stacklevel=2 to show the caller's location instead of this function's location + log_func(full_msg, stacklevel=2) + + def setup_distributed(self, timeout=timedelta(minutes=60)) -> None: + if self.full_determinism: + transformers.enable_full_determinism(self.seed) + # Use deterministic backward in flash attention as, by default, flash attention uses atomic adds + # https://github.com/Dao-AILab/flash-attention/commit/732654583c2e640adc012ecb60e460bf19dcd9e3 + transformers.modeling_flash_attention_utils.deterministic_g = True + else: + transformers.set_seed(self.seed) + + local_rank = int(os.environ.get("LOCAL_RANK", "-1")) + if local_rank != -1: + torch.cuda.set_device(local_rank) + + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", timeout=timeout) + + self.world_size = dist.get_world_size() + self.sp_size = self.args.model.ring_attn_size + dp_size = self.world_size // self.sp_size + self.sp_mesh = init_device_mesh("cuda", (dp_size, self.sp_size), mesh_dim_names=("dp", "sp")) + self.setup_ring_attn(self.sp_mesh) + if self.args.fsdp.fsdp_size > -1: + assert self.world_size % self.args.fsdp.fsdp_size == 0 + assert self.world_size >= self.args.fsdp.fsdp_size + replicate = self.world_size // self.args.fsdp.fsdp_size + self.fsdp_mesh = init_device_mesh( + "cuda", (replicate, self.args.fsdp.fsdp_size), mesh_dim_names=("replicate", "sharded")) + else: + self.fsdp_mesh = init_device_mesh( + "cuda", (self.world_size,), mesh_dim_names=("sharded",) + ) + + self.step = 0 + self.accumulated_gradient = ( + self.train_batch_size + * self.sp_size + // self.micro_train_batch_size + // self.world_size + ) + + def setup_ring_attn(self, device_mesh): + if self.sp_size == 1: + self.ring_attn_rank = 0 + return + + group = device_mesh["sp"].get_group() + self.ring_attn_rank = dist.get_rank(group=group) + set_ring_attn_group(group) + + from ring_flash_attn import substitute_hf_flash_attn + + self.ring_head_stride = getattr(self.args.model, "ring_head_stride", 1) + substitute_hf_flash_attn(self.ring_attn_group, self.ring_head_stride) + + @property + def ring_attn_group(self): + return get_ring_attn_group() + + def setup_dataloader( + self, + replay_buffer, + batch_size: int, + pin_memory: bool = False, + shuffle=True, + collate_fn=None, + drop_last=True, + sampler=None, + consumed_samples=0, + ): + # DDP only mode, replay buffers on each rank are different. + if sampler is None and dist.is_initialized(): + num_replicas = dist.get_world_size() // self.sp_size + rank = dist.get_rank() // self.sp_size + + sampler = DistributedSampler( + replay_buffer, + num_replicas=num_replicas, + rank=rank, + shuffle=shuffle, + seed=self.seed, + drop_last=drop_last, + consumed_samples=consumed_samples, + ) + return StatefulDataLoader( + replay_buffer, + batch_size=batch_size, + sampler=sampler, + drop_last=drop_last, + shuffle=shuffle if sampler is None else False, + collate_fn=collate_fn, + pin_memory=pin_memory, + ) + + def load_hf_model(self, model_class, model_name_or_path, attn_impl, model_config): + init_context = self._get_init_weight_context_manager(model_config) + + with init_context(): + model = model_class.from_pretrained( + model_name_or_path, + trust_remote_code=True, + attn_implementation=attn_impl, + torch_dtype=torch.bfloat16 if self.args.train.bf16 else "auto", + ) + # Patch lm_head to skippable mode so that the HF forward does not + # materialize full-sequence logits. The algorithm layer always calls + # `lm_head(hidden, skip=False)` explicitly via chunked_loss. + if hasattr(model, "lm_head"): + original_forward = model.lm_head.forward + def skip_forward(x, skip=True): + if skip: + return x + return original_forward(x) + model.lm_head.forward = skip_forward + self.log("lm_head forward set to skippable mode for student model.") + return model + + def _get_init_weight_context_manager(self, model_config): + """Get context manager for model initialization. + + Returns a callable that creates a context manager. + Uses meta device (no memory allocation) for non-rank-0 processes, + UNLESS tie_word_embeddings=True (which causes hangs with meta tensors). + + Ref: verl/utils/fsdp_utils.py::get_init_weight_context_manager + NOTE: tie_word_embedding causes meta_tensor init to hang + """ + from accelerate import init_empty_weights + + # Check if model uses tied word embeddings (which doesn't work with meta tensors) + use_meta_tensor = not model_config.tie_word_embeddings + + def cpu_init_weights(): + return torch.device("cpu") + + if use_meta_tensor: + # Rank 0: CPU, others: meta device (memory efficient for large models) + return init_empty_weights if not self.is_rank_0() else cpu_init_weights + else: + self.log("tie_word_embeddings=True, loading full model to CPU on all ranks") + return cpu_init_weights + + def prepare(self, model: nn.Module, *args, **kwargs): + for name, param in model.named_parameters(): + if param.requires_grad: + model = model.float() + break + + # Only rank 0 has real weights, others may have meta tensors + # _fsdp2_load_full_state_dict will broadcast from rank 0 + full_state = model.state_dict() + self._init_fsdp_kwargs() + model = self._fsdp2_shard_model(model) + model = self._fsdp2_load_full_state_dict( + model, full_state, self.fsdp_mesh, + cpu_offload=True if self.args.fsdp.cpu_offload else None + ) + + return model + + def _init_fsdp_kwargs(self): + if self.bf16: + self.mixed_precision_policy = MixedPrecisionPolicy( + param_dtype=torch.bfloat16, + reduce_dtype=torch.float32, + cast_forward_inputs=True + ) + else: + self.mixed_precision_policy = None + + self.offload_policy = CPUOffloadPolicy(pin_memory=True) if self.args.fsdp.cpu_offload else None + + self.fsdp_kwargs = { + "mesh": self.fsdp_mesh, + "mp_policy": self.mixed_precision_policy, + "offload_policy": self.offload_policy + } + + def _fsdp2_shard_model(self, model): + fsdp_transformer_layer_cls_to_wrap = model.model._no_split_modules + for _, module in model.model.named_modules(): + if module.__class__.__name__ in fsdp_transformer_layer_cls_to_wrap or ( + isinstance(module, torch.nn.Embedding) and not model.model.config.tie_word_embeddings + ): + fully_shard(module, **self.fsdp_kwargs) + fully_shard(model, **self.fsdp_kwargs) + return model + + def _fsdp2_load_full_state_dict(self, model, full_state, device_mesh, cpu_offload): + """Load full state dict into FSDP2 model with efficient broadcast from rank 0. + + This function loads weights from rank 0 and broadcasts to all other ranks, + avoiding the need for each rank to load the full model from disk. + + Args: + model: FSDP2-wrapped model + full_state: State dict (only rank 0 has real weights, others have empty dict) + device_mesh: Device mesh for FSDP + cpu_offload: If not None, enables StateDictOptions cpu_offload + + Ref:verl/utils/fsdp_utils.py::fsdp2_load_full_state_dict + """ + from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict + + # Step 1: Allocate tensors on GPU + # Rank 0: move with weights, others: allocate empty tensors on device (for meta tensors) + if self.is_rank_0(): + model = model.to(device=torch.cuda.current_device(), non_blocking=True) + else: + # to_empty creates tensors on device without initializing memory + model = model.to_empty(device=torch.cuda.current_device()) + + is_cpu_offload = cpu_offload is not None + options = StateDictOptions(full_state_dict=True, cpu_offload=is_cpu_offload, broadcast_from_rank0=True) + + # Step 2: Broadcast weights from rank 0 to all ranks + set_model_state_dict(model, full_state, options=options) + + # set_model_state_dict will not broadcast buffers, so we need to broadcast them manually. + for _name, buf in model.named_buffers(): + dist.broadcast(buf, src=0) + + if is_cpu_offload: + model.to("cpu", non_blocking=True) + for buf in model.buffers(): + buf.data = buf.data.to(torch.cuda.current_device()) + + return model + + def create_optimizer(self, model, **kwargs) -> Optimizer: + if isinstance(model, DistillModel): + model = model.model + + kwargs["fused"] = True + optim_params = self._get_optimizer_grouped_parameters(model, kwargs["weight_decay"]) + optim = AdamW(optim_params, **kwargs) + return optim + + def _get_optimizer_grouped_parameters( + self, + model, + weight_decay, + no_decay_name_list=["bias", "layer_norm.weight", "layernorm.weight", "norm.weight", "ln_f.weight"], + ): + optimizer_grouped_parameters = [ + { + "params": [ + p + for n, p in model.named_parameters() + if (not any(nd in n for nd in no_decay_name_list) and p.requires_grad) + ], + "weight_decay": weight_decay, + }, + { + "params": [ + p + for n, p in model.named_parameters() + if (any(nd in n for nd in no_decay_name_list) and p.requires_grad) + ], + "weight_decay": 0.0, + }, + ] + return optimizer_grouped_parameters + + def _unwrap_model(self, model) -> nn.Module: + if hasattr(model, "module"): + return self._unwrap_model(model.module) + elif isinstance(model, DistillModel): + return model.model + else: + return model + + def backward(self, loss: torch.Tensor, model: nn.Module, optimizer: Optimizer, **kwargs) -> None: + self.step = (self.step + 1) % self.accumulated_gradient + loss = loss / self.accumulated_gradient + loss.backward() + + def optimizer_step( + self, + optimizer: Optimizer, + model: nn.Module, + scheduler, + **kwargs, + ) -> None: + if self.step == 0: + if self.max_norm > 0.0: + if hasattr(model, "clip_grad_norm_"): + model.clip_grad_norm_(self.max_norm) + else: + torch.nn.utils.clip_grad_norm_(model.parameters(), self.max_norm) + + optimizer.step() + optimizer.zero_grad() + if scheduler: + scheduler.step() + + def load_model(self, model: nn.Module, path: str, map_location="cpu", strict: bool = False, key_replace_fn=None) -> None: + # For FSDP2, we prefer Distributed Checkpoint (DCP) + # But if the user provides a standard `torch.save` file path, we try to load it. + # We use `set_model_state_dict` from DCP which handles sharding. + + # Load state dict on rank 0 (or all if mapped) + state_dict = torch.load(path, map_location=map_location) + if key_replace_fn: + state_dict = key_replace_fn(state_dict) + + model_to_load = self._unwrap_model(model) + + # DCP Helper to load full state dict into sharded model + # Note: This is memory intensive on Rank 0 if strict full load. + options = StateDictOptions(full_state_dict=True, cpu_offload=True) + set_model_state_dict(model_to_load, model_state_dict=state_dict, options=options) + + def save_model(self, model: nn.Module, tokenizer, output_dir, **kwargs) -> None: + model_to_save = self._unwrap_model(model) + + if self.is_rank_0(): + os.makedirs(output_dir, exist_ok=True) + + options = StateDictOptions(full_state_dict=True, cpu_offload=True) + + state_dict = get_model_state_dict(model_to_save, options=options) + + if self.args.train.bf16: + state_dict = { + k: v.to(torch.bfloat16) if torch.is_floating_point(v) else v + for k, v in state_dict.items() + } + + if self.is_rank_0(): + if isinstance(model_to_save, PeftModel): + model_to_save.save_pretrained( + output_dir, + state_dict=state_dict, + safe_serialization=False, + **kwargs + ) + else: + model_to_save.save_pretrained(output_dir, state_dict=state_dict, **kwargs) + + # Config & Tokenizer + output_config_file = os.path.join(output_dir, "config.json") + model_to_save.config.to_json_file(output_config_file) + tokenizer.save_pretrained(output_dir) + + del state_dict + gc.collect() + torch_dist_barrier_and_cuda_sync() + + def all_reduce(self, data, op="mean"): + assert op in ("mean", "max", "sum") + if isinstance(data, dict): + ret = {} + for k, v in data.items(): + ret[k] = self.all_reduce(v, op) + return ret + else: + is_tensor = True + if not isinstance(data, torch.Tensor): + data = torch.Tensor([data]) + is_tensor = False + is_cpu_tensor = data.device.type == "cpu" + + if is_cpu_tensor: + data = data.to(torch.cuda.current_device()) + if op == "mean": + data /= self.world_size + dist.all_reduce(data, op=dist.ReduceOp.MAX if op == "max" else dist.ReduceOp.SUM) + if is_cpu_tensor: + data = data.cpu() + return data.item() if not is_tensor else data + + def all_gather(self, data): + if isinstance(data, dict): + ret = {} + for k, v in data.items(): + ret[k] = self.all_gather(v) + return ret + else: + if not isinstance(data, torch.Tensor): + data = torch.Tensor([data]) + is_cpu_tensor = data.device.type == "cpu" + + ret = [torch.zeros_like(data).to(torch.cuda.current_device()) for _ in range(self.world_size)] + dist.all_gather(ret, data.to(torch.cuda.current_device())) + return torch.cat(ret).cpu() if is_cpu_tensor else torch.cat(ret) + + def print(self, *msg): + if self.is_rank_0(): + print(*msg) + + def is_rank_0(self) -> bool: + if not dist.is_initialized(): + return True + return dist.get_rank() == 0 + + def get_rank(self) -> int: + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + @torch.no_grad() + def offload_model_params(self, model, empty_cache: bool = True): + model.cpu() + if empty_cache: + torch.cuda.empty_cache() + + @torch.no_grad() + def reload_model_params(self, model): + device = torch.cuda.current_device() + model.to(device) + + @torch.no_grad() + def offload_optim_states(self, optimizer: Optimizer, empty_cache: bool = True): + """offload optimizer states to CPU""" + if not optimizer.state: + return + for param_group in optimizer.param_groups: + for param in param_group["params"]: + state = optimizer.state[param] + for key, value in state.items(): + if isinstance(value, torch.Tensor): + state[key] = value.to("cpu", non_blocking=True) + if empty_cache: + torch.cuda.empty_cache() + + @torch.no_grad() + def reload_optim_states(self, optimizer: Optimizer): + """reload optimizer states to GPU""" + device = torch.cuda.current_device() + if not optimizer.state: + return + for param_group in optimizer.param_groups: + for param in param_group["params"]: + state = optimizer.state[param] + for key, value in state.items(): + if isinstance(value, torch.Tensor): + state[key] = value.to(device, non_blocking=True) + + def update_rollout_weights_from_tensor( + self, model, engine, gather_src, gather_group, + update_weight_buffer_size=2 * 1024**3, + weight_source=None + ): + """ + Broadcast FSDP model weights to rollout engine with CUDA IPC + Gloo gather. + Following slime: all ranks serialize their CUDA IPC data, gather to source rank + via Gloo, then source rank sends to sglang with per-TP-rank IPC handles. + """ + import ray + from sglang.srt.utils import MultiprocessingSerializer + + try: + from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions + except ImportError: + from sglang.srt.patch_torch import monkey_patch_torch_reductions + + try: + from sglang.srt.weight_sync.tensor_bucket import FlattenedTensorBucket + except ImportError: + from sglang.srt.model_executor.model_runner import FlattenedTensorBucket + + torch.cuda.empty_cache() + + model_unwrapped = self._unwrap_model(model) + + bucket = [] + bucket_size = 0 + + # Detect weight tying: when tie_word_embeddings=True, lm_head.weight + # and model.embed_tokens.weight share the same parameter. SGLang only + # stores a single copy, so we must skip the duplicate to avoid KeyError. + tie_word_embeddings = getattr(model_unwrapped.config, "tie_word_embeddings", False) + + for name, param in model_unwrapped.state_dict().items(): + if tie_word_embeddings and name == "lm_head.weight": + continue + param_size = param.numel() * param.element_size() + if bucket and bucket_size + param_size >= update_weight_buffer_size: + self._flush_weight_bucket( + bucket, engine, gather_src, gather_group, + monkey_patch_torch_reductions, FlattenedTensorBucket, MultiprocessingSerializer, + ) + del bucket + bucket = [] + bucket_size = 0 + + if weight_source is not None and name in weight_source: + param = weight_source[name] + + param = param.cuda() + if isinstance(param, DTensor): + param = param.redistribute( + placements=[torch.distributed.tensor.Replicate()] * param.device_mesh.ndim, + async_op=True, + ).to_local() + bucket.append((name, param)) + bucket_size += param_size + + if bucket: + self._flush_weight_bucket( + bucket, engine, gather_src, gather_group, + monkey_patch_torch_reductions, FlattenedTensorBucket, MultiprocessingSerializer, + ) + del bucket + + if dist.get_rank() == gather_src: + ray.get(engine.flush_cache.remote()) + + torch.cuda.empty_cache() + torch_dist_barrier_and_cuda_sync() + + def _flush_weight_bucket( + self, bucket, engine, gather_src, gather_group, + monkey_patch_fn, FlattenedTensorBucket, MultiprocessingSerializer, + ): + import ray + + bucket = [(n, p.wait()) if hasattr(p, "wait") else (n, p) for n, p in bucket] + + monkey_patch_fn() + + named_tensors_by_dtype = {} + for name, tensor in bucket: + named_tensors_by_dtype.setdefault(tensor.dtype, []).append((name, tensor)) + + serialized_tensors = [] + for _dtype, named_tensors in named_tensors_by_dtype.items(): + fb = FlattenedTensorBucket(named_tensors=named_tensors) + serialized = MultiprocessingSerializer.serialize( + {"flattened_tensor": fb.get_flattened_tensor(), "metadata": fb.get_metadata()}, + output_str=True, + ) + serialized_tensors.append(serialized) + del fb + + if dist.get_rank() == gather_src: + gathered = [None for _ in range(dist.get_world_size(gather_group))] + else: + gathered = None + + dist.gather_object( + obj=serialized_tensors, + object_gather_list=gathered, + dst=gather_src, + group=gather_group, + ) + + if dist.get_rank() == gather_src: + num_dtypes = len(gathered[0]) + for i in range(num_dtypes): + ref = engine.update_weights_from_tensor.remote( + serialized_named_tensors=[tensors[i] for tensors in gathered], + load_format="flattened_bucket", + flush_cache=False, + ) + ray.get(ref) + del gathered + + del bucket, serialized_tensors + + def update_rollout_weights_from_disk(self, model, rollout_engines, tokenizer=None, tmp_dir=None): + """ + Sync FSDP model weights to rollout engines via disk (safetensors). + + Rank 0 gathers the full state dict and saves it to a shared filesystem path, + then all rollout engines load the weights from disk via sglang's + update_weights_from_disk API. This completely avoids CUDA IPC issues. + + Args: + model: The FSDP-wrapped model. + rollout_engines: List of RolloutRayActor instances. + tokenizer: Tokenizer (needed for save_pretrained to write tokenizer_config.json etc.). + tmp_dir: Directory to save the temporary weights. If None, uses a default path. + """ + import ray + import shutil + + if tmp_dir is None: + tmp_dir = os.path.join( + os.environ.get("TMPDIR", "/tmp"), + "kdflow_rollout_weights_sync", + ) + + model_unwrapped = self._unwrap_model(model) + is_sender = len(rollout_engines) > 0 + + # Step 1: Gather full state dict (all ranks participate in FSDP collective) + options = StateDictOptions(full_state_dict=True, cpu_offload=True) + state_dict = get_model_state_dict(model_unwrapped, options=options) + + # Step 2: Rank 0 saves weights to shared filesystem + if self.is_rank_0(): + os.makedirs(tmp_dir, exist_ok=True) + model_unwrapped.save_pretrained( + tmp_dir, + state_dict=state_dict, + safe_serialization=True, + ) + # Save config (needed for sglang to load correctly) + model_unwrapped.config.to_json_file(os.path.join(tmp_dir, "config.json")) + if tokenizer is not None: + tokenizer.save_pretrained(tmp_dir) + + del state_dict + gc.collect() + torch_dist_barrier_and_cuda_sync() + + # Step 3: Tell all rollout engines to load weights from disk + if is_sender: + refs = [ + engine.update_weights_from_disk.remote( + model_path=tmp_dir, + load_format=None, + ) + for engine in rollout_engines + ] + ray.get(refs) + + torch_dist_barrier_and_cuda_sync() + + # Step 4: Cleanup temporary directory + if self.is_rank_0(): + try: + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + + torch.cuda.empty_cache() + torch_dist_barrier_and_cuda_sync() diff --git a/kdflow/backend/sglang/__init__.py b/kdflow/backend/sglang/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/kdflow/backend/sglang/monkey_patch.py b/kdflow/backend/sglang/monkey_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..e4aa0a9550903f01a7125b1631408e9b2df6883a --- /dev/null +++ b/kdflow/backend/sglang/monkey_patch.py @@ -0,0 +1,282 @@ +""" +Monkey patch for SGLang scheduler's process_batch_result_prefill method. +This allows using numpy() instead of tolist() for hidden_states, which is much faster. +""" + +from __future__ import annotations + +import logging +import os +import time +from typing import TYPE_CHECKING, List, Optional, Tuple, Union + +import torch + +from kdflow.utils.logging_utils import init_logger + +if TYPE_CHECKING: + from sglang.srt.managers.scheduler import ( + EmbeddingBatchResult, + GenerationBatchResult, + ScheduleBatch, + Scheduler, + ) + +logger = init_logger(__name__) + +# Flag to prevent multiple patch applications +_PATCH_APPLIED = False + + +def process_batch_result_prefill_patched( + self: "Scheduler", + batch: "ScheduleBatch", + result: Union["GenerationBatchResult", "EmbeddingBatchResult"], +): + """ + Patched version of process_batch_result_prefill. + Key change: Use .numpy() instead of .tolist() for hidden_states (much faster). + """ + from sglang.srt.environ import envs + from sglang.srt.managers.io_struct import AbortReq + from sglang.srt.mem_cache.common import release_kv_cache + + skip_stream_req = None + + if self.is_generation: + if result.copy_done is not None: + result.copy_done.synchronize() + + ( + logits_output, + next_token_ids, + extend_input_len_per_req, + extend_logprob_start_len_per_req, + ) = ( + result.logits_output, + result.next_token_ids, + result.extend_input_len_per_req, + result.extend_logprob_start_len_per_req, + ) + + # Move next_token_ids and logprobs to cpu + next_token_ids = next_token_ids.tolist() + if batch.return_logprob: + if logits_output.next_token_logprobs is not None: + logits_output.next_token_logprobs = ( + logits_output.next_token_logprobs.tolist() + ) + if logits_output.input_token_logprobs is not None: + logits_output.input_token_logprobs = tuple( + logits_output.input_token_logprobs.tolist() + ) + + hidden_state_offset = 0 + + # Check finish conditions + logprob_pt = 0 + + for i, (req, next_token_id) in enumerate(zip(batch.reqs, next_token_ids)): + if req.finished() or req.is_retracted: + # decode req in mixed batch or retracted req + continue + + if req.is_chunked <= 0: + if hasattr(req.time_stats, "prefill_finished_ts"): + if req.time_stats.prefill_finished_ts == 0.0: + req.time_stats.prefill_finished_ts = time.time() + else: + req.time_stats.set_prefill_finished_time() + + # req output_ids are set here + req.output_ids.append(next_token_id) + req.check_finished() + + if req.finished(): + self.maybe_collect_routed_experts(req) + release_kv_cache(req, self.tree_cache) + req.time_stats.completion_time = time.perf_counter() + elif not batch.decoding_reqs or req not in batch.decoding_reqs: + # This updates radix so others can match + self.tree_cache.cache_unfinished_req(req) + + self.maybe_collect_customized_info(i, req, logits_output) + + if batch.return_logprob: + assert extend_logprob_start_len_per_req is not None + assert extend_input_len_per_req is not None + extend_logprob_start_len = extend_logprob_start_len_per_req[i] + extend_input_len = extend_input_len_per_req[i] + + num_input_logprobs = self._calculate_num_input_logprobs( + req, extend_input_len, extend_logprob_start_len + ) + + if req.return_logprob: + self.add_logprob_return_values( + i, + req, + logprob_pt, + next_token_ids, + num_input_logprobs, + logits_output, + ) + logprob_pt += num_input_logprobs + + if ( + req.return_hidden_states + and logits_output.hidden_states is not None + ): + req.hidden_states.append( + logits_output.hidden_states[ + hidden_state_offset : ( + hidden_state_offset := hidden_state_offset + + len(req.origin_input_ids) + ) + ] + .half() + .cpu() + .numpy() + ) + + if req.grammar is not None: + # FIXME: this try-except block is for handling unexpected xgrammar issue. + try: + req.grammar.accept_token(next_token_id) + except ValueError as e: + # Grammar accept_token can raise ValueError if the token is not in the grammar. + # This can happen if the grammar is not set correctly or the token is invalid. + logger.error( + f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}" + ) + self.abort_request(AbortReq(rid=req.rid)) + req.grammar.finished = req.finished() + + else: + # being chunked reqs' prefill is not finished + req.is_chunked -= 1 + # There is only at most one request being currently chunked. + # Because this request does not finish prefill, + # we don't want to stream the request currently being chunked. + skip_stream_req = req + + # Incrementally update input logprobs. + if batch.return_logprob: + extend_logprob_start_len = extend_logprob_start_len_per_req[i] + extend_input_len = extend_input_len_per_req[i] + if extend_logprob_start_len < extend_input_len: + # Update input logprobs. + num_input_logprobs = self._calculate_num_input_logprobs( + req, extend_input_len, extend_logprob_start_len + ) + if req.return_logprob: + self.add_input_logprob_return_values( + i, + req, + logits_output, + logprob_pt, + num_input_logprobs, + last_prefill_chunk=False, + ) + logprob_pt += num_input_logprobs + + else: # embedding or reward model + if result.copy_done is not None: + result.copy_done.synchronize() + + is_sparse = envs.SGLANG_EMBEDDINGS_SPARSE_HEAD.is_set() + + embeddings = result.embeddings + + if is_sparse: + batch_ids, token_ids = embeddings.indices() + values = embeddings.values() + + embeddings = [{} for _ in range(embeddings.size(0))] + for i in range(batch_ids.shape[0]): + embeddings[batch_ids[i].item()][token_ids[i].item()] = values[ + i + ].item() + else: + if isinstance(embeddings, torch.Tensor): + embeddings = embeddings.tolist() + else: + embeddings = [tensor.tolist() for tensor in embeddings] + + # Check finish conditions + for i, req in enumerate(batch.reqs): + if req.is_retracted: + continue + + req.embedding = embeddings[i] + if req.is_chunked <= 0: + # Dummy output token for embedding models + req.output_ids.append(0) + req.check_finished() + + if req.finished(): + release_kv_cache(req, self.tree_cache) + else: + self.tree_cache.cache_unfinished_req(req) + else: + # being chunked reqs' prefill is not finished + req.is_chunked -= 1 + + self.stream_output(batch.reqs, batch.return_logprob, skip_stream_req) + + if self.current_scheduler_metrics_enabled: + can_run_cuda_graph = getattr(result, "can_run_cuda_graph", False) + self.log_prefill_stats( + prefill_stats=batch.prefill_stats, + can_run_cuda_graph=can_run_cuda_graph, + dp_cooperation_info=batch.dp_cooperation_info, + ) + + +def apply_patch(): + """ + Apply the monkey patch to SGLang's SchedulerOutputProcessorMixin. + + This function is idempotent - calling it multiple times is safe. + Returns True if patch was applied (or already applied), False otherwise. + """ + global _PATCH_APPLIED + + if _PATCH_APPLIED: + return True + + try: + from sglang.srt.managers.scheduler_output_processor_mixin import ( + SchedulerOutputProcessorMixin, + ) + + # Check if already patched (by another mechanism like sitecustomize) + current_method = getattr(SchedulerOutputProcessorMixin, 'process_batch_result_prefill', None) + if current_method is not None and getattr(current_method, '_kdflow_patched', False): + _PATCH_APPLIED = True + print(f"[monkey_patch] Patch already applied, PID={os.getpid()}", flush=True) + return True + + # Mark the patched function + process_batch_result_prefill_patched._kdflow_patched = True + + # Apply patch + SchedulerOutputProcessorMixin.process_batch_result_prefill = process_batch_result_prefill_patched + + _PATCH_APPLIED = True + print(f"[monkey_patch] SUCCESS: process_batch_result_prefill patched! PID={os.getpid()}", flush=True) + return True + + except ImportError as e: + print(f"[monkey_patch] Cannot import SGLang module: {e}", flush=True) + return False + except Exception as e: + print(f"[monkey_patch] Error applying patch: {e}", flush=True) + import traceback + traceback.print_exc() + return False + + +def is_patch_applied(): + """Check if the patch has been applied in this process.""" + return _PATCH_APPLIED diff --git a/kdflow/backend/sglang/sglang_engine.py b/kdflow/backend/sglang/sglang_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..4deb3adfe2fa5b2258c0144ed46b4b4dc1873913 --- /dev/null +++ b/kdflow/backend/sglang/sglang_engine.py @@ -0,0 +1,437 @@ +import os +import queue +import time +import torch.multiprocessing as mp +from torch.multiprocessing import Queue +from typing import List, Dict, Any, Tuple, Optional +from dataclasses import dataclass + +import numpy as np +import torch +from sglang.srt.entrypoints.engine import Engine as _SglEngine +from sglang.srt.managers.scheduler import run_scheduler_process as _original_run_scheduler_process + +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +os.environ["SGLANG_JIT_DEEPGEMM_FAST_WARMUP"] = "true" + +def _patched_run_scheduler_process(*args, **kwargs): + try: + from kdflow.backend.sglang.monkey_patch import apply_patch + apply_patch() + except Exception as e: + logger.warning(f"[PatchedEngine] WARNING: Failed to apply monkey patch (PID={os.getpid()}): {e}", flush=True) + return _original_run_scheduler_process(*args, **kwargs) + + +class PatchedEngine(_SglEngine): + """ + SGLang Engine that applies monkey patch in scheduler subprocesses. + Motivation: SGLang Engine supports returning hidden states, but the existing implementation use .tolist() to convert hidden states from GPU tensor to Python list, which is very inefficient. This monkey patch replaces the original .tolist() with a more efficient operation .numpy(). + """ + run_scheduler_process_func = staticmethod(_patched_run_scheduler_process) + + +@dataclass +class EngineConfig: + """Configuration for SGLang Engine.""" + model_path: str + tp_size: int = 1 + ep_size: int = 1 + pp_size: int = 1 + chunked_prefill_size: int = -1 + disable_radix_cache: bool = True + enable_return_hidden_states: bool = True + enable_memory_saver: bool = True + enable_weights_cpu_backup: bool = True + mem_fraction_static: float = 0.8 + quantization: str = None + offload_tags: Optional[str] = "all" + base_gpu_id: int = 0 + # for multi-node tp/pp + nnodes: int = 1 + node_rank: int = 0 + dist_init_addr: Optional[str] = None + + +def _engine_worker(config: EngineConfig, request_queue: Queue, response_queue: Queue, hidden_queue: Queue): + os.environ["TOKENIZERS_PARALLELISM"] = "false" + if config.nnodes > 1: + os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" + + engine = None + + try: + engine = PatchedEngine( + model_path=config.model_path, + tp_size=config.tp_size, + ep_size=config.ep_size, + pp_size=config.pp_size, + chunked_prefill_size=config.chunked_prefill_size, + disable_radix_cache=config.disable_radix_cache, + enable_return_hidden_states=config.enable_return_hidden_states, + enable_memory_saver=config.enable_memory_saver, + enable_weights_cpu_backup=config.enable_weights_cpu_backup, + quantization=config.quantization, + mem_fraction_static=config.mem_fraction_static, + base_gpu_id=config.base_gpu_id, + nnodes=config.nnodes, + node_rank=config.node_rank, + dist_init_addr=config.dist_init_addr, + disable_custom_all_reduce=True, + ) + + response_queue.put({ + "type": "init_done", + "success": True, + }) + + while True: + request = request_queue.get() + if request is None: + break + + req_type = request.get("type") + + try: + if req_type == "generate": + _handle_generate(engine, request, hidden_queue, response_queue) + elif req_type == "sleep": + _handle_sleep(engine, request, config, response_queue) + elif req_type == "wakeup": + _handle_wakeup(engine, request, config, response_queue) + elif req_type == "update_weights_from_tensor": + _handle_update_weights_from_tensor(engine, request, response_queue) + else: + response_queue.put({"type": req_type, "success": False, + "error": f"Unknown request type: {req_type}"}) + except Exception: + import traceback + response_queue.put({"type": req_type, "success": False, + "error": traceback.format_exc()}) + + except Exception: + import traceback + response_queue.put({"type": "init_done", "success": False, + "error": traceback.format_exc()}) + finally: + if engine: + try: + engine.shutdown() + except Exception: + pass + + +def _normalize_tags(tags): + """Convert tags to the format SGLang expects (None, or list of strings).""" + if tags is None or tags == "all": + return None + if isinstance(tags, str): + return [tags] + return tags + + +def _handle_generate(engine, request, hidden_queue, response_queue): + """Handle a generate request and stream hidden states via shared memory.""" + kwargs = request["kwargs"] + + generate_kwargs = { + "input_ids": kwargs["input_ids"], + "sampling_params": kwargs["sampling_params"], + "return_hidden_states": kwargs.get("return_hidden_states", True), + } + if kwargs.get("image_data") is not None: + generate_kwargs["image_data"] = kwargs["image_data"] + + outputs = engine.generate(**generate_kwargs, stream=True) + num_samples = len(kwargs["loss_masks"]) + + response_queue.put({ + "type": "generate", + "success": True, + "num_samples": num_samples, + }) + + received_indices = set() + try: + for output in outputs: + meta_info = output["meta_info"] + if meta_info.get("finish_reason") is None: + continue + + idx = output["index"] + if idx in received_indices: + raise RuntimeError(f"Duplicate SGLang output index: {idx}") + + mask = kwargs["loss_masks"][idx] + hs_np = meta_info["hidden_states"][0] + + # hs_np and mask may differ due to multimodal token expansion. + hs_len = hs_np.shape[0] + mask_len = mask.shape[0] + if hs_len == mask_len: + hs_np = hs_np[mask] + else: + num_loss_tokens = int(mask.sum()) + logger.warning( + f"[_handle_generate] sample={idx}/{num_samples} length mismatch: " + f"hs_len={hs_len}, mask_len={mask_len}, diff={mask_len - hs_len}; " + f"selecting {num_loss_tokens} loss tokens from the tail" + ) + if num_loss_tokens >= hs_len: + raise ValueError( + f"Cannot select {num_loss_tokens} loss tokens from " + f"hidden states of length {hs_len}" + ) + # The loss span is a contiguous suffix before the final token. + hs_np = hs_np[-num_loss_tokens - 1:-1] + + if not hs_np.flags['C_CONTIGUOUS']: + hs_np = np.ascontiguousarray(hs_np) + + hs_tensor = torch.from_numpy(hs_np).share_memory_() + hidden_queue.put((idx, hs_tensor)) + received_indices.add(idx) + + if len(received_indices) != num_samples: + raise RuntimeError( + f"SGLang returned {len(received_indices)} final outputs for " + f"{num_samples} samples" + ) + hidden_queue.put(None) + except Exception: + import traceback + error = traceback.format_exc() + logger.error(f"[_handle_generate] ERROR:\n{error}") + hidden_queue.put(error) + + +def _handle_sleep(engine, request, config, response_queue): + """Handle a sleep request: offload GPU memory.""" + tags = request.get("tags", config.offload_tags) + torch.cuda.empty_cache() + engine.release_memory_occupation(tags=_normalize_tags(tags)) + response_queue.put({"type": "sleep", "success": True, "tags": tags}) + + +def _handle_wakeup(engine, request, config, response_queue): + """Handle a wakeup request: restore GPU memory.""" + tags = request.get("tags", config.offload_tags) + torch.cuda.empty_cache() + engine.resume_memory_occupation(tags=_normalize_tags(tags)) + response_queue.put({"type": "wakeup", "success": True, "tags": tags}) + + +def _handle_update_weights_from_tensor(engine, request, response_queue): + """Handle a update_weights_from_tensor request: update weights from student (for self-distillation).""" + serialized_named_tensors = request["kwargs"]["serialized_named_tensors"] + load_format = request["kwargs"]["load_format"] + flush_cache = request["kwargs"]["flush_cache"] + engine.update_weights_from_tensor( + named_tensors=serialized_named_tensors, + load_format=load_format, + flush_cache=flush_cache, + ) + response_queue.put({"type": "update_weights_from_tensor", "success": True}) + + +class SGLangEngineService: + """Manages SGLang Engine in a subprocess with torch multiprocessing communication.""" + + def __init__(self, config: EngineConfig): + self.config = config + self.process: Optional[mp.Process] = None + self.request_queue: Optional[Queue] = None + self.response_queue: Optional[Queue] = None + self.hidden_queue: Optional[Queue] = None + self._started = False + + def start(self, timeout: float = 1800.0): + """Start the SGLang Engine in a subprocess.""" + if self._started: + raise RuntimeError("Service already started") + + try: + mp.set_start_method("spawn", force=True) + except RuntimeError: + pass + + self.request_queue = mp.Queue() + self.response_queue = mp.Queue() + self.hidden_queue = mp.Queue(maxsize=2) + + self.process = mp.Process( + target=_engine_worker, + args=(self.config, self.request_queue, self.response_queue, self.hidden_queue), + ) + self.process.start() + + try: + response = self.response_queue.get(timeout=timeout) + if response.get("type") == "init_done" and response.get("success"): + self._started = True + else: + raise RuntimeError(f"Init failed: {response.get('error')}") + except Exception as e: + self._cleanup() + raise RuntimeError(f"Engine initialization failed: {e}") + + def generate( + self, + input_ids: List[List[int]], + loss_masks: List[np.ndarray], + sampling_params: Dict[str, Any], + return_hidden_states: bool = True, + image_data=None, + ) -> List[np.ndarray]: + """Run generation and return hidden states via shared-memory tensors. + + Args: + input_ids: Per-sample token-id lists fed directly to SGLang (avoids a + re-tokenization round trip, so hidden-state length matches the mask). + loss_masks: Pre-computed boolean masks for selecting response hidden states. + sampling_params: Sampling parameters (e.g. max_new_tokens=0 for prefill-only). + return_hidden_states: Whether to return hidden states. + image_data: Optional list of image data for multimodal models. + """ + if not self._started: + raise RuntimeError("Service not started") + + if len(input_ids) != len(loss_masks): + raise ValueError( + f"input_ids and loss_masks must have the same batch size, got " + f"{len(input_ids)} and {len(loss_masks)}" + ) + + # Check if subprocess is still alive before sending request + if self.process and not self.process.is_alive(): + raise RuntimeError( + f"[SGLangEngineService] Engine subprocess (PID={self.process.pid}) is dead! " + f"exitcode={self.process.exitcode}" + ) + + kwargs = { + "input_ids": input_ids, + "loss_masks": loss_masks, + "sampling_params": sampling_params, + "return_hidden_states": return_hidden_states, + } + if image_data is not None: + kwargs["image_data"] = image_data + + self.request_queue.put({"type": "generate", "kwargs": kwargs}) + + response = self._get_response(req_type="generate", timeout=600) + if not response.get("success"): + raise RuntimeError(f"Generate failed: {response.get('error')}") + + num_samples = response["num_samples"] + hidden_states = [None] * num_samples + received_count = 0 + t_recv_start = time.time() + while True: + try: + message = self.hidden_queue.get(timeout=300) + if message is None: + break + if isinstance(message, str): + raise RuntimeError(f"Generate failed in engine subprocess:\n{message}") + + idx, hs_tensor = message + hs_np = hs_tensor.numpy().copy() + del hs_tensor + hidden_states[idx] = hs_np + received_count += 1 + except queue.Empty: + elapsed_total = time.time() - t_recv_start + raise RuntimeError( + f"Hidden state recv timeout while receiving hidden states, " + f"received={received_count}/{num_samples}, " + f"total_elapsed={elapsed_total:.1f}s" + ) + + return hidden_states + + def sleep(self, tags: Optional[str] = "all"): + """Release GPU memory.""" + if not self._started: + return + self.request_queue.put({"type": "sleep", "tags": tags}) + response = self._get_response(req_type="sleep", timeout=300) + if not response.get("success"): + raise RuntimeError(f"Sleep failed: {response.get('error')}") + return response.get("tags") + + def wakeup(self, tags: Optional[str] = "all"): + """Resume GPU memory.""" + if not self._started: + return + self.request_queue.put({"type": "wakeup", "tags": tags}) + response = self._get_response(req_type="wakeup", timeout=300) + if not response.get("success"): + raise RuntimeError(f"Wakeup failed: {response.get('error')}") + return response.get("tags") + + def update_weights_from_tensor( + self, serialized_named_tensors: List[Tuple[str, torch.Tensor]], + load_format: Optional[str] = None, flush_cache: bool = True): + kwargs = { + "serialized_named_tensors": serialized_named_tensors, + "load_format": load_format, + "flush_cache": flush_cache, + } + self.request_queue.put({"type": "update_weights_from_tensor", "kwargs": kwargs}) + response = self._get_response(req_type="update_weights_from_tensor", timeout=300) + if not response.get("success"): + raise RuntimeError(f"update_weights_from_tensor failed: {response.get('error')}") + + def _get_response(self, req_type="unknown", timeout=600, check_interval=10): + elapsed = 0 + while elapsed < timeout: + try: + return self.response_queue.get(timeout=check_interval) + except queue.Empty: + elapsed += check_interval + if self.process and not self.process.is_alive(): + raise RuntimeError( + f"Engine subprocess (PID={self.process.pid}) died during '{req_type}'! " + f"exitcode={self.process.exitcode}" + ) + raise RuntimeError(f"Response timeout after {timeout}s during '{req_type}'") + + def shutdown(self): + """Shutdown the subprocess gracefully.""" + if not self._started: + return + self._started = False + self._cleanup() + + def _cleanup(self): + """Clean up subprocess, queues and shared memory.""" + if self.request_queue: + try: + self.request_queue.put(None) + except Exception: + pass + + if self.process: + self.process.join(timeout=30) + if self.process.is_alive(): + self.process.terminate() + self.process.join(timeout=5) + if self.process.is_alive(): + self.process.kill() + + self.process = None + self.request_queue = None + self.response_queue = None + self.hidden_queue = None + + def __del__(self): + try: + self.shutdown() + except Exception: + pass diff --git a/kdflow/cli/__init__.py b/kdflow/cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/kdflow/cli/evaluate_alfworld.py b/kdflow/cli/evaluate_alfworld.py new file mode 100644 index 0000000000000000000000000000000000000000..77dec216fbfc0b68f28a094b6fdeac63f1f6a646 --- /dev/null +++ b/kdflow/cli/evaluate_alfworld.py @@ -0,0 +1,118 @@ +"""Evaluate a model checkpoint in ALFWorld without starting training actors.""" + +import json +import logging +import os + +from kdflow.evaluation import AlfWorldEvaluator, VerlAgentAlfWorldEvaluator + + +logger = logging.getLogger(__name__) + + +class _EvaluationStrategy: + """Small args/log adapter required by the shared rollout manager.""" + + def __init__(self, args): + self.args = args + + @staticmethod + def log(message, level="info", **_kwargs): + getattr(logger, level, logger.info)(message) + + @staticmethod + def print(*values): + logger.info(" ".join(str(value) for value in values)) + + +def evaluate(args): + if args.rollout.alfworld_eval_protocol == "verl_agent": + preflight_evaluator = VerlAgentAlfWorldEvaluator(args, None) + if not preflight_evaluator.has_pending_seeds(): + logger.info( + "All requested ALFWorld seeds are already complete and " + "compatible; skipping Ray and model initialization." + ) + summary = preflight_evaluator.evaluate() + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return summary + + import ray + + from kdflow.ray.placement_group import create_placement_group + from kdflow.ray.rollout.rollout_group import RolloutActorGroup + from kdflow.trainer.alfworld_rollout_manager import AlfWorldRolloutManager + from kdflow.trainer.verl_agent_alfworld_rollout_manager import ( + VerlAgentAlfWorldRolloutManager, + ) + + initialized_ray = False + if not ray.is_initialized(): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + }, + "working_dir": os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ), + } + ) + initialized_ray = True + + total_gpus = args.train.num_nodes * args.train.num_gpus_per_node + rollout_group = None + rollout_manager = None + try: + pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group( + total_gpus + ) + rollout_group = RolloutActorGroup( + model_path=args.model.student_name_or_path, + num_actors=args.rollout.rollout_num_engines, + tp_size=args.rollout.rollout_tp_size, + num_gpus_per_node=args.train.num_gpus_per_node, + enable_memory_saver=True, + mem_fraction_static=args.rollout.rollout_mem_fraction_static, + num_gpus_per_actor=0.01, + pg=(pg, reordered_bundle_indices, reordered_gpu_ids), + extra_server_args={"disable_piecewise_cuda_graph": True}, + ) + if args.train.enable_sleep: + rollout_group.sleep() + + strategy = _EvaluationStrategy(args) + generate_kwargs = { + "max_new_tokens": args.rollout.generate_max_len, + "temperature": args.rollout.temperature, + "top_p": args.rollout.top_p, + } + if args.rollout.alfworld_eval_protocol == "verl_agent": + manager_class = VerlAgentAlfWorldRolloutManager + evaluator_class = VerlAgentAlfWorldEvaluator + else: + manager_class = AlfWorldRolloutManager + evaluator_class = AlfWorldEvaluator + rollout_manager = manager_class( + strategy=strategy, + rollout_group=rollout_group, + is_same_tokenizer=True, + generate_kwargs=generate_kwargs, + ) + summary = evaluator_class(args, rollout_manager).evaluate() + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return summary + finally: + if rollout_manager is not None: + rollout_manager.close() + if rollout_group is not None: + rollout_group.shutdown() + if initialized_ray: + ray.shutdown() + + +if __name__ == "__main__": + from kdflow.arguments import init_args + + evaluate(init_args("alfworld_eval")) diff --git a/kdflow/cli/train_kd_off_policy.py b/kdflow/cli/train_kd_off_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..b149c801a94aedc0234c529cbdc63764de08573f --- /dev/null +++ b/kdflow/cli/train_kd_off_policy.py @@ -0,0 +1,173 @@ +import os + +import ray + +from kdflow.ray.train.teacher_group import TeacherActorGroup +from kdflow.ray.train.student_group import StudentActorGroup +from kdflow.ray.placement_group import create_placement_group +from kdflow.trainer import OffPolicyKDTrainer +from kdflow.datasets import SFTDataset +from kdflow.datasets.utils import blending_datasets +from kdflow.models.utils import check_tokenizer_identical +from kdflow.backend import get_strategy +from kdflow.arguments import init_args +from kdflow.arguments.training_args import resolve_max_steps +from kdflow.utils.distributed_sampler import DistributedSampler +from kdflow.utils.utils import get_tokenizer + + +def train(args): + # Initialize Ray if not already initialized + if not ray.is_initialized(): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN" + }, + "working_dir": os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + } + ) + + strategy = get_strategy(args) + strategy.print(args) + + # Create placement group for resource allocation + num_gpus = args.train.num_nodes * args.train.num_gpus_per_node + pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(num_gpus) + strategy.print(f"Placement group ready with {num_gpus} GPUs") + + teacher_model = TeacherActorGroup( + strategy, + num_gpus, + num_gpus_per_node=args.train.num_gpus_per_node, + num_gpus_per_actor=0.01, + pg=(pg, reordered_bundle_indices, reordered_gpu_ids), + ) + student_model = StudentActorGroup( + args, + args.train.num_nodes, + args.train.num_gpus_per_node, + pg=pg, + num_gpus_per_actor=0.01, + ) + + # Initialize tokenizers + student_tokenizer = get_tokenizer( + args.model.student_name_or_path, + use_fast=not args.model.disable_fast_tokenizer + ) + teacher_tokenizer = get_tokenizer( + args.model.teacher_name_or_path, + use_fast=not args.model.disable_fast_tokenizer, + ) + tokenizer_info = check_tokenizer_identical(student_tokenizer, teacher_tokenizer) + strategy.print(f"Tokenizers {tokenizer_info}") + if not tokenizer_info.vocab_identical and args.kd.kd_algorithm not in { + "simple_ctkd", + "byte_align_ctkd", + "topk_byte_align_ctkd", + }: + raise ValueError( + "Cross-tokenizer distillation supports simple_ctkd, " + "byte_align_ctkd, or topk_byte_align_ctkd." + ) + + # Load and prepare training dataset + train_data = blending_datasets( + args.data.train_dataset_path, + args.data.train_dataset_probs, + strategy, + args.train.seed, + max_count=args.data.max_samples, + dataset_split=args.data.train_split, + ) + train_data = train_data.select(range(min(args.data.max_samples, len(train_data)))) + + train_dataset = SFTDataset( + train_data, + strategy, + tokenizer_info=tokenizer_info, + max_data_num=args.data.max_samples, + input_template=args.data.input_template, + num_processors=args.data.preprocess_num_workers, + ) + + sampler = DistributedSampler(train_dataset, num_replicas=1, rank=0, shuffle=True, seed=args.train.seed, drop_last=True) + train_dataloader = strategy.setup_dataloader( + train_dataset, + args.train.micro_train_batch_size, + pin_memory=False, + shuffle=False, + collate_fn=train_dataset.collate_fn, + sampler=sampler, + ) + + # Load and prepare evaluation dataset (optional) + eval_dataloader = None + if getattr(args.data, "eval_dataset_path", None): + eval_data = blending_datasets( + args.data.eval_dataset_path, + None, + strategy, + dataset_split=args.data.eval_split, + ) + eval_data = eval_data.select(range(min(args.data.max_samples, len(eval_data)))) + eval_dataset = SFTDataset( + eval_data, + strategy, + tokenizer_info=tokenizer_info, + input_template=args.data.input_template, + num_processors=args.data.preprocess_num_workers, + ) + eval_dataloader = strategy.setup_dataloader( + eval_dataset, + batch_size=1, + pin_memory=True, + shuffle=False, + drop_last=False, + collate_fn=eval_dataset.collate_fn, + ) + + # Calculate max training steps + num_update_steps_per_epoch = len(train_dataset) // args.train.train_batch_size + max_steps = resolve_max_steps( + args.train.max_steps, + args.train.num_epochs, + num_update_steps_per_epoch, + ) + strategy.log(f"Max training steps: {max_steps}") + + # Initialize student model on all workers + ray.get(student_model.async_init_model_from_pretrained( + strategy, max_steps, + teacher_tokenizer=teacher_tokenizer, + tokenizer_info=tokenizer_info, + )) + + strategy.log("Models initialized on all student actors") + + # Initialize OffPolicyKDRayTrainer (Single controller) + trainer = OffPolicyKDTrainer( + strategy=strategy, + student_model=student_model, + teacher_model=teacher_model, + train_dataloader=train_dataloader, + eval_dataloader=eval_dataloader, + max_steps=max_steps, + num_update_steps_per_epoch=num_update_steps_per_epoch, + student_tokenizer=student_tokenizer, + teacher_tokenizer=teacher_tokenizer, + ) + + try: + trainer.fit() + ray.get(student_model.async_save_model()) + strategy.log("Training completed and model saved.") + finally: + teacher_model.shutdown() + + +if __name__ == "__main__": + args = init_args("off_policy_kd") + train(args) diff --git a/kdflow/cli/train_kd_on_policy.py b/kdflow/cli/train_kd_on_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..8cfb58d743568948cfa7d1c1e579be7287057241 --- /dev/null +++ b/kdflow/cli/train_kd_on_policy.py @@ -0,0 +1,230 @@ +import os + +import ray + +from kdflow.ray.train.teacher_group import TeacherActorGroup +from kdflow.ray.train.student_group import StudentActorGroup +from kdflow.ray.rollout.rollout_group import RolloutActorGroup +from kdflow.ray.placement_group import create_placement_group +from kdflow.trainer import OnPolicyKDTrainer +from kdflow.datasets import ( + AlfWorldEpisodeDataset, + PromptDataset, +) +from kdflow.datasets.utils import blending_datasets +from kdflow.models.utils import check_tokenizer_identical +from kdflow.backend import get_strategy +from kdflow.arguments import init_args +from kdflow.arguments.training_args import resolve_max_steps +from kdflow.utils.distributed_sampler import DistributedSampler +from kdflow.utils.utils import get_tokenizer, load_custom_eval_fn + + +def train(args): + # Initialize Ray if not already initialized + if not ray.is_initialized(): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN" + }, + "working_dir": os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + } + ) + + strategy = get_strategy(args) + strategy.print(args) + + student_tokenizer = get_tokenizer( + args.model.student_name_or_path, + use_fast=not args.model.disable_fast_tokenizer, + ) + + # Create placement group for resource allocation + num_gpus = args.train.num_nodes * args.train.num_gpus_per_node + pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(num_gpus) + rollout_group = RolloutActorGroup( + model_path=args.model.student_name_or_path, + num_actors=args.rollout.rollout_num_engines, + tp_size=args.rollout.rollout_tp_size, + num_gpus_per_node=args.train.num_gpus_per_node, + enable_memory_saver=True, + mem_fraction_static=args.rollout.rollout_mem_fraction_static, + num_gpus_per_actor=0.01, + pg=(pg, reordered_bundle_indices, reordered_gpu_ids), + extra_server_args={"disable_piecewise_cuda_graph": True}, + ) + if args.train.enable_sleep: + rollout_group.sleep() + + teacher_model = TeacherActorGroup( + strategy, + num_gpus, + num_gpus_per_node=args.train.num_gpus_per_node, + num_gpus_per_actor=0.01, + pg=(pg, reordered_bundle_indices, reordered_gpu_ids), + + ) + student_model = StudentActorGroup( + args, + args.train.num_nodes, + args.train.num_gpus_per_node, + pg=(pg, reordered_bundle_indices), + num_gpus_per_actor=0.01, + ) + + # Initialize the teacher tokenizer after inference actors are configured. + teacher_tokenizer = get_tokenizer( + args.model.teacher_name_or_path, + use_fast=not args.model.disable_fast_tokenizer, + ) + tokenizer_info = check_tokenizer_identical(student_tokenizer, teacher_tokenizer) + strategy.print(f"Tokenizers {tokenizer_info}") + if not tokenizer_info.vocab_identical and args.kd.kd_algorithm not in { + "simple_ctkd", + "byte_align_ctkd", + "topk_byte_align_ctkd", + }: + raise ValueError( + "Cross-tokenizer distillation supports simple_ctkd, " + "byte_align_ctkd, or topk_byte_align_ctkd." + ) + + environment_rollout = bool(args.rollout.alfworld_config) + + # Interactive environments sample tasks on reset. Their lightweight + # datasets are therefore only epoch clocks; ordinary on-policy KD keeps the + # existing static prompt dataset path. + if args.rollout.alfworld_config: + train_dataset = AlfWorldEpisodeDataset( + args.rollout.alfworld_episodes_per_epoch + ) + else: + train_data = blending_datasets( + args.data.train_dataset_path, + args.data.train_dataset_probs, + strategy, + args.train.seed, + max_count=args.data.max_samples, + dataset_split=args.data.train_split, + ) + train_data = train_data.select( + range(min(args.data.max_samples, len(train_data))) + ) + train_dataset = PromptDataset( + train_data, + strategy, + tokenizer_info=tokenizer_info, + max_data_num=args.data.max_samples, + input_template=args.data.input_template, + num_processors=args.data.preprocess_num_workers, + ) + + sampler = DistributedSampler(train_dataset, num_replicas=1, rank=0, shuffle=True, seed=args.train.seed, drop_last=True) + train_dataloader = strategy.setup_dataloader( + train_dataset, + args.rollout.rollout_batch_size, + True, + False, + collate_fn=train_dataset.collate_fn, + sampler=sampler, + ) + + # Load and prepare evaluation dataset (optional) + eval_dataloader = None + if getattr(args.data, "eval_dataset_path", None) and not environment_rollout: + eval_data = blending_datasets( + args.data.eval_dataset_path, + None, + strategy, + dataset_split=args.data.eval_split, + ) + eval_data = eval_data.select(range(min(args.data.max_samples, len(eval_data)))) + eval_dataset = PromptDataset( + eval_data, + strategy, + tokenizer_info=tokenizer_info, + input_template=args.data.input_template, + num_processors=args.data.preprocess_num_workers, + ) + eval_dataloader = strategy.setup_dataloader( + eval_dataset, + batch_size=args.rollout.rollout_batch_size, + pin_memory=True, + shuffle=False, + drop_last=False, + collate_fn=eval_dataset.collate_fn, + ) + + # Calculate max training steps + num_rollout_iters_per_epoch = len(train_dataset) // args.rollout.rollout_batch_size + if environment_rollout: + max_turn_samples = ( + args.rollout.rollout_batch_size * args.rollout.alfworld_max_turns + ) + num_update_steps_per_rollout = max( + 1, + (max_turn_samples + args.train.train_batch_size - 1) + // args.train.train_batch_size, + ) + else: + num_update_steps_per_rollout = max( + 1, + args.rollout.rollout_batch_size + * args.rollout.n_samples_per_prompt + // args.train.train_batch_size, + ) + max_rollout_iters = resolve_max_steps( + args.train.max_steps, + args.train.num_epochs, + num_rollout_iters_per_epoch, + ) + strategy.log(f"Max training iterations: {max_rollout_iters}") + + # Initialize student model on all workers + ray.get(student_model.async_init_model_from_pretrained( + strategy, (max_rollout_iters * num_update_steps_per_rollout), + teacher_tokenizer=teacher_tokenizer, + tokenizer_info=tokenizer_info, + )) + strategy.log("Models initialized on all student actors") + + generate_kwargs = { + "max_new_tokens": args.rollout.generate_max_len, + "temperature": args.rollout.temperature, + "top_p": args.rollout.top_p, + } + + trainer = None + try: + trainer = OnPolicyKDTrainer( + strategy=strategy, + student_model=student_model, + teacher_model=teacher_model, + rollout_group=rollout_group, + is_same_tokenizer=tokenizer_info.is_identical, + train_dataloader=train_dataloader, + eval_dataloader=eval_dataloader, + max_rollout_iters=max_rollout_iters, + num_rollout_iters_per_epoch=num_rollout_iters_per_epoch, + generate_kwargs=generate_kwargs, + custom_eval_fn=( + load_custom_eval_fn(args.data.custom_eval_fn) + if args.data.custom_eval_fn + else None + ), + ) + trainer.fit() + ray.get(student_model.async_save_model()) + strategy.log("Training completed and model saved.") + finally: + if trainer is not None: + trainer.close() + teacher_model.shutdown() + rollout_group.shutdown() + + +if __name__ == "__main__": + args = init_args("on_policy_kd") + train(args) diff --git a/kdflow/datasets/__init__.py b/kdflow/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..37facfbdb6705166c7047e40eb67bb1f3edbf3e9 --- /dev/null +++ b/kdflow/datasets/__init__.py @@ -0,0 +1,9 @@ +from .prompts_dataset import PromptDataset +from .sft_dataset import SFTDataset +from .alfworld_dataset import AlfWorldEpisodeDataset + +__all__ = [ + "PromptDataset", + "SFTDataset", + "AlfWorldEpisodeDataset", +] diff --git a/kdflow/datasets/alfworld_dataset.py b/kdflow/datasets/alfworld_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..cd96f6d2932f6e2cd31c6e4cab188556206abbef --- /dev/null +++ b/kdflow/datasets/alfworld_dataset.py @@ -0,0 +1,26 @@ +from typing import Dict, List + +from torch.utils.data import Dataset + + +class AlfWorldEpisodeDataset(Dataset): + """A lightweight epoch clock for environment-sampled ALFWorld episodes. + + ALFWorld chooses a new game whenever its batched environment is reset, so + dataset rows only determine how many episodes are collected in one epoch. + """ + + def __init__(self, num_episodes: int): + if num_episodes <= 0: + raise ValueError("num_episodes must be positive") + self.num_episodes = int(num_episodes) + + def __len__(self) -> int: + return self.num_episodes + + def __getitem__(self, index: int) -> Dict[str, int]: + return {"episode_slot": index} + + @staticmethod + def collate_fn(batch: List[Dict[str, int]]) -> List[Dict[str, int]]: + return batch diff --git a/kdflow/datasets/prompts_dataset.py b/kdflow/datasets/prompts_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..15a6e747c6cc3fd3232affcdff8f1252ac087ee9 --- /dev/null +++ b/kdflow/datasets/prompts_dataset.py @@ -0,0 +1,246 @@ +from typing import Callable, Optional, Dict, Any, List + +from torch.utils.data import Dataset +from datasets import Image as ImageFeature +from PIL import Image + +from kdflow.datasets.utils import ( + convert_to_openai_messages, + get_nested_value, + get_tokenizer_or_processor, + validate_dataset_columns, +) +from kdflow.models.utils import TokenizerCompareResult + + +class PromptDataset(Dataset): + """ + Dataset for On-Policy Distillation + + Args: + dataset: dataset for on-policy distillation + strategy: training strategy object + tokenizer_info: result of tokenizer comparison (template_identical, vocab_identical) + max_data_num: maximum number of data to load + input_template: optional template for formatting input + num_processors: number of processors for parallel data loading + """ + + def __init__( + self, + dataset, + strategy, + tokenizer_info, + max_data_num: int = None, + input_template: Optional[str] = None, + num_processors: int = 8, + ) -> None: + super().__init__() + self.args = strategy.args + self.strategy = strategy + self.tokenizer_info = tokenizer_info or TokenizerCompareResult() + self.template_identical = self.tokenizer_info.template_identical + self.vocab_identical = self.tokenizer_info.vocab_identical + self.same_tokenizer = self.tokenizer_info.is_identical + self.strategy = strategy + self.input_template = input_template + + self.input_key = getattr(self.args.data, "input_key", None) + self.teacher_input_key = getattr(self.args.data, "teacher_input_key", None) or self.input_key + self.label_key = getattr(self.args.data, "label_key", None) + self.apply_chat_template = getattr(self.args.data, "apply_chat_template", False) + self.enable_thinking = getattr(self.args.model, "enable_thinking", False) + self.prompt_max_len = getattr(self.args.data, "prompt_max_len", 0) + + self.image_key = getattr(self.args.data, "image_key", None) + + required_columns = {"input_key": self.input_key} + if self.teacher_input_key != self.input_key: + required_columns["teacher_input_key"] = self.teacher_input_key + if self.label_key: + required_columns["label_key"] = self.label_key + if self.image_key: + required_columns["image_key"] = self.image_key + if self.args.kd.multi_teacher_config: + required_columns["teacher_routing_key"] = self.args.data.teacher_routing_key + validate_dataset_columns(dataset, **required_columns) + + self.student_processor = get_tokenizer_or_processor( + self.args.model.student_name_or_path, + need_processor=self.image_key is not None, + ) + self.teacher_processors = {} + if self.args.kd.multi_teacher_config: + for teacher_key, teacher_path in self.args.kd.multi_teacher_config.items(): + self.teacher_processors[teacher_key] = get_tokenizer_or_processor( + teacher_path, need_processor=self.image_key is not None, + ) + elif self.args.model.teacher_name_or_path is not None: + self.teacher_processors["default"] = get_tokenizer_or_processor( + self.args.model.teacher_name_or_path, + need_processor=self.image_key is not None, + ) + + if max_data_num is not None and max_data_num > 0 and max_data_num < len(dataset): + strategy.log(f"Truncating dataset from {len(dataset)} to {max_data_num}") + dataset = dataset.select(range(max_data_num)) + + # Adjust worker count based on dataset size to speed up preprocessing. + num_processors = min( + num_processors, + max(1, len(dataset) // 2000), + ) + self.processed_dataset = dataset.map( + self.process_data, + remove_columns=dataset.column_names, + num_proc=num_processors, + load_from_cache_file=False, + desc="Processing data", + ) + if self.image_key: + self.processed_dataset = self.processed_dataset.cast_column("images", [ImageFeature()]) + + original_len = len(self.processed_dataset) + if self.prompt_max_len > 0: + strategy.log(f"Before prompt_max_len filter: {len(self.processed_dataset)}") + self.processed_dataset = self.processed_dataset.filter( + lambda prompt_len: prompt_len <= self.prompt_max_len, + input_columns=["prompt_len"], + desc="Filtering overlang samples", + ) + strategy.log(f"After prompt_max_len filter: {len(self.processed_dataset)}") + if len(self.processed_dataset) < original_len: + self.strategy.log( + f"Filtered {original_len - len(self.processed_dataset)} samples " + f"exceeding prompt_max_len={self.prompt_max_len} " + f"({original_len} -> {len(self.processed_dataset)})" + ) + + self._print_sample() + + def _print_sample(self) -> None: + """Print sample data for debugging.""" + self.strategy.print(f"Total samples: {len(self.processed_dataset)}") + self.strategy.print(f"Sample student prompt:\n{self.processed_dataset[0]['stu_prompt']}") + if not self.template_identical or self.teacher_input_key != self.input_key: + self.strategy.print(f"Sample teacher prompt:\n{self.processed_dataset[0]['tea_prompt']}") + + def process_data(self, data: Dict) -> Dict[str, Any]: + """Process a single data sample.""" + stu_prompt = self._build_prompt(data, self.student_processor, self.input_key) + + if self.args.kd.multi_teacher_config: + routing_key = self.args.data.teacher_routing_key + assert routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config" + assert routing_key in data, f"Routing key '{routing_key}' not found in data" + teacher_key = data[routing_key] + if teacher_key not in self.teacher_processors: + raise ValueError( + f"Teacher routing key '{teacher_key}' not found in multi_teacher_config. " + f"Available keys: {list(self.teacher_processors.keys())}." + ) + teacher_processor = self.teacher_processors[teacher_key] + tea_prompt = self._build_prompt(data, teacher_processor, self.teacher_input_key) + elif self.same_tokenizer and self.input_key == self.teacher_input_key: + tea_prompt = stu_prompt + else: + tea_prompt = self._build_prompt(data, self.teacher_processors.get("default", self.student_processor), self.teacher_input_key) + + tokenizer = self.student_processor.tokenizer if hasattr(self.student_processor, "tokenizer") else self.student_processor + prompt_len = len(tokenizer.encode(stu_prompt)) + + result = { + "stu_prompt": stu_prompt, + "tea_prompt": tea_prompt, + "prompt": stu_prompt, + "prompt_len": prompt_len, + "label": get_nested_value(data, self.label_key) or "" if self.label_key else "", + "datasource": data.get("datasource", "default"), + } + if self.image_key: + result["images"] = self._load_images(data.get(self.image_key)) + + if self.args.kd.multi_teacher_config is not None: + assert self.args.data.teacher_routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config" + assert self.args.data.teacher_routing_key in data, f"Routing key '{self.args.data.teacher_routing_key}' not found in data" + result["teacher_routing_key"] = data[self.args.data.teacher_routing_key] + + return result + + def _build_prompt(self, data: Dict, processor_or_tokenizer, input_key: str) -> str: + """Build prompt from data with optional chat template or input template. + + Args: + data: The data dict containing input + processor_or_tokenizer: The processor or tokenizer to use for apply_chat_template + input_key: The key to extract input from data + + Returns: + Formatted prompt string + """ + if self.apply_chat_template: + chat = convert_to_openai_messages(data[input_key], expand_image=self.image_key is not None) + while chat and chat[-1].get("role", "user") == "assistant": + chat.pop() + chat_template_kwargs = {} + if "enable_thinking" in str(getattr(processor_or_tokenizer, "chat_template", "")): + chat_template_kwargs["enable_thinking"] = self.enable_thinking + return processor_or_tokenizer.apply_chat_template( + chat, + tokenize=False, + add_generation_prompt=True, + **chat_template_kwargs, + ) + + prompt = data[input_key] + return self.input_template.format(prompt) if self.input_template else prompt + + def __len__(self) -> int: + return len(self.processed_dataset) + + def __getitem__(self, idx: int) -> Dict[str, str]: + """Get item by index. + + Returns: + Dict with keys: datasource, stu_prompt, tea_prompt, label, images (optional) + """ + item = self.processed_dataset[idx] + result = { + "datasource": item["datasource"], + "stu_prompt": item["stu_prompt"], + "tea_prompt": item["tea_prompt"], + "label": item["label"], + } + if "images" in item: + result["images"] = item["images"] + if "teacher_routing_key" in item: + result["teacher_routing_key"] = item["teacher_routing_key"] + return result + + @staticmethod + def _load_images(image_content) -> list: + """Load image(s) from various input formats. Always returns a list.""" + if image_content is None: + return [] + if isinstance(image_content, Image.Image): + return [image_content] + if isinstance(image_content, str): + return [Image.open(image_content).convert("RGB")] + if isinstance(image_content, list): + result = [] + for img in image_content: + if isinstance(img, Image.Image): + result.append(img) + elif isinstance(img, str): + result.append(Image.open(img).convert("RGB")) + return result + return [] + + @staticmethod + def collate_fn(batch: List[Dict[str, str]]) -> List[Dict[str, str]]: + """Collate function that simply returns the list of dicts. + + DataLoader will pass a list of dicts, we just return it as-is + since rollout method expects a list of dicts. + """ + return batch diff --git a/kdflow/datasets/sft_dataset.py b/kdflow/datasets/sft_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..6b922b0047e0d1da1e9f95f0c2b9a078f3ef92a8 --- /dev/null +++ b/kdflow/datasets/sft_dataset.py @@ -0,0 +1,380 @@ +from typing import Callable, Optional, Dict, List, Any + +import torch +from torch.utils.data import Dataset +from datasets import Image as ImageFeature +from PIL import Image + +from kdflow.datasets.utils import ( + convert_to_openai_messages, + get_tokenizer_or_processor, + validate_dataset_columns, + zero_pad_sequences, +) +from kdflow.models.utils import TokenizerCompareResult + +class SFTDataset(Dataset): + """ + Dataset for SFT and Off-Policy KD. + + Args: + dataset: dataset for SFT and Off-Policy KD + strategy: training strategy object + tokenizer_info: result of tokenizer comparison (template_identical, vocab_identical) + max_data_num: maximum number of data to load + input_template: optional template for formatting input + num_processors: number of processors for parallel data loading + """ + + def __init__( + self, + dataset, + strategy, + tokenizer_info: Optional[TokenizerCompareResult] = None, + max_data_num: int = -1, + input_template: Optional[str] = None, + num_processors: int = 8, + ) -> None: + super().__init__() + self.args = strategy.args + self.strategy = strategy + self.max_length = self.args.data.max_len + self.tokenizer_info = tokenizer_info + self.template_identical = True if tokenizer_info is None else self.tokenizer_info.template_identical + self.vocab_identical = True if tokenizer_info is None else self.tokenizer_info.vocab_identical + + self.input_template = input_template + self.input_key = getattr(self.args.data, "input_key", None) + self.teacher_input_key = getattr(self.args.data, "teacher_input_key", None) or self.input_key + self.output_key = getattr(self.args.data, "output_key", None) + self.apply_chat_template = getattr(self.args.data, "apply_chat_template", False) + self.enable_thinking = getattr(self.args.model, "enable_thinking", False) + + self.image_key = getattr(self.args.data, "image_key", None) + self.same_tokenizer = True if tokenizer_info is None else self.tokenizer_info.is_identical + self.teacher_student_share_input = ( + self.same_tokenizer + and self.teacher_input_key == self.input_key + ) + + required_columns = {"input_key": self.input_key} + if self.output_key: + required_columns["output_key"] = self.output_key + if self.image_key: + required_columns["image_key"] = self.image_key + has_teacher = ( + self.args.model.teacher_name_or_path is not None + or self.args.kd.multi_teacher_config is not None + ) + if has_teacher and self.teacher_input_key != self.input_key: + required_columns["teacher_input_key"] = self.teacher_input_key + if self.args.kd.multi_teacher_config: + required_columns["teacher_routing_key"] = self.args.data.teacher_routing_key + validate_dataset_columns(dataset, **required_columns) + + self.student_processor = get_tokenizer_or_processor( + self.args.model.student_name_or_path, + need_processor=self.image_key is not None, + ) + self.student_tokenizer = getattr(self.student_processor, "tokenizer", self.student_processor) + self.teacher_processors = {} + if self.args.kd.multi_teacher_config: + for teacher_key, teacher_path in self.args.kd.multi_teacher_config.items(): + self.teacher_processors[teacher_key] = get_tokenizer_or_processor( + teacher_path, need_processor=self.image_key is not None, + ) + elif self.args.model.teacher_name_or_path is not None: + self.teacher_processors["default"] = get_tokenizer_or_processor( + self.args.model.teacher_name_or_path, + need_processor=self.image_key is not None, + ) + + if max_data_num > 0 and max_data_num < len(dataset): + self.strategy.log(f"Truncating dataset from {len(dataset)} to {max_data_num}") + dataset = dataset.select(range(max_data_num)) + + # Adjust worker count based on dataset size to speed up preprocessing. + num_processors = min( + num_processors, + max(1, len(dataset) // 2000), + ) + self.processed_dataset = dataset.map( + self.process_data, + remove_columns=dataset.column_names, + num_proc=num_processors, + load_from_cache_file=False, + desc="Processing data", + ) + if self.image_key: + self.processed_dataset = self.processed_dataset.cast_column("images", [ImageFeature()]) + + original_len = len(self.processed_dataset) + strategy.log(f"Before length filter: {len(self.processed_dataset)}") + self.processed_dataset = self.processed_dataset.filter( + lambda stu_input_len: stu_input_len <= self.max_length, + input_columns=["stu_input_len"], + desc="Filtering overlang samples", + ) + if len(self.processed_dataset) < original_len: + self.strategy.log( + f"Filtered {original_len - len(self.processed_dataset)} samples " + f"exceeding max_length={self.max_length} " + f"({original_len} -> {len(self.processed_dataset)})" + ) + + self._print_sample() + + def _print_sample(self) -> None: + sample = self.processed_dataset[0] + self.strategy.print("Student prompt + response:") + self.strategy.print(sample["stu_prompt"] + sample["stu_response"]) + if not self.teacher_student_share_input: + self.strategy.print("Teacher prompt + response:") + self.strategy.print(sample["tea_prompt"] + sample["tea_response"]) + + def process_data(self, data: Dict) -> Dict[str, Any]: + stu_chat_template_fn = self.student_processor.apply_chat_template if self.apply_chat_template else None + stu_prompt, stu_response = self.preprocess_data( + data, self.input_template, self.input_key, self.output_key, + apply_chat_template=stu_chat_template_fn, + ) + stu_eos_token = self.student_tokenizer.eos_token + if not stu_response.endswith(stu_eos_token): + stu_response += " " + stu_eos_token + + result = {"stu_prompt": stu_prompt, "stu_response": stu_response} + result["stu_resp_len"] = self._compute_token_length( + self.student_processor, stu_response + ) + result["stu_input_len"] = self._compute_token_length( + self.student_processor, stu_prompt + ) + result["stu_resp_len"] + + if self.image_key is not None: + result["images"] = self.load_images(data[self.image_key]) + + if self.args.model.teacher_name_or_path is not None \ + or self.args.kd.multi_teacher_config is not None: + if not self.teacher_student_share_input: + if self.args.kd.multi_teacher_config is not None: + routing_key = self.args.data.teacher_routing_key + assert routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config" + assert routing_key in data, f"Routing key '{routing_key}' not found in data" + teacher_key = data[routing_key] + if teacher_key not in self.teacher_processors: + raise ValueError( + f"Teacher routing key '{teacher_key}' not found in multi_teacher_config. " + f"Available keys: {list(self.teacher_processors.keys())}." + ) + teacher_processor = self.teacher_processors[teacher_key] + else: + teacher_processor = self.teacher_processors.get("default", self.student_processor) + cur_teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor) + + tea_chat_template_fn = teacher_processor.apply_chat_template if self.apply_chat_template else None + tea_prompt, tea_response = self.preprocess_data( + data, self.input_template, self.teacher_input_key, self.output_key, + apply_chat_template=tea_chat_template_fn, + ) + tea_eos_token = cur_teacher_tokenizer.eos_token + if not tea_response.endswith(tea_eos_token): + tea_response += " " + tea_eos_token + result["tea_prompt"] = tea_prompt + result["tea_response"] = tea_response + result["tea_resp_len"] = self._compute_token_length(teacher_processor, tea_response) + else: + result["tea_prompt"] = stu_prompt + result["tea_response"] = stu_response + result["tea_resp_len"] = result["stu_resp_len"] + + if self.args.kd.multi_teacher_config is not None: + assert self.args.data.teacher_routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config" + assert self.args.data.teacher_routing_key in data, f"Routing key '{self.args.data.teacher_routing_key}' not found in data" + result["teacher_routing_key"] = data[self.args.data.teacher_routing_key] + + return result + + def preprocess_data( + self, + data: Dict, + input_template: Optional[str] = None, + input_key: str = "input", + output_key: Optional[str] = None, + apply_chat_template: Optional[Callable] = None, + ) -> tuple: + if not apply_chat_template: + prompt = data[input_key] + if input_template: + prompt = input_template.format(prompt) + assert output_key is not None + return prompt, data[output_key] + + has_image = self.image_key is not None + if output_key: + messages = convert_to_openai_messages(data[input_key], expand_image=has_image) + \ + convert_to_openai_messages(data[output_key], role="assistant", expand_image=has_image) + else: + messages = convert_to_openai_messages(data[input_key], expand_image=has_image) + + chat_template_kwargs = {} + if "enable_thinking" in str(apply_chat_template.__self__.chat_template): + chat_template_kwargs["enable_thinking"] = self.enable_thinking + prompt = apply_chat_template( + messages[:-1], tokenize=False, add_generation_prompt=True, + **chat_template_kwargs, + ) + full_text = apply_chat_template(messages, tokenize=False, **chat_template_kwargs) + response = full_text[len(prompt):].removeprefix("\n\n\n\n").rstrip() + return prompt, response + + def __len__(self) -> int: + return len(self.processed_dataset) + + def __getitem__(self, idx: int) -> Dict: + return self.processed_dataset[idx] + + def load_images(self, image_content): + """Load image(s) from various input formats. Always returns a list.""" + if isinstance(image_content, Image.Image): + return [image_content] + if isinstance(image_content, str): + return [Image.open(image_content).convert("RGB")] + if isinstance(image_content, list): + return [self._load_single_image(img) for img in image_content] + return [] + + @staticmethod + def _load_single_image(img): + if isinstance(img, Image.Image): + return img + if isinstance(img, str): + return Image.open(img).convert("RGB") + return None + + def _compute_token_length(self, processor, text): + tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor + return len(tokenizer.encode(text, add_special_tokens=False)) + + def _encode_single(self, processor, text, images=None) -> Dict[str, Any]: + """Single-sample processor call.""" + kwargs = {"text": [text], "padding": False, "truncation": False, + "return_tensors": "pt"} + if images: + kwargs["images"] = images + enc = processor(**kwargs) + skip = {"input_ids", "attention_mask", "mm_token_type_ids"} + multi_modal_inputs = {k: v for k, v in enc.items() if k not in skip} + return { + "input_ids": enc["input_ids"][0], + "attention_mask": enc["attention_mask"][0], + "multi_modal_inputs": multi_modal_inputs or None, + } + + @staticmethod + def _build_loss_mask(attn_mask, resp_lens): + bs, seq_len = attn_mask.shape + loss_mask = torch.zeros(bs, seq_len, dtype=torch.bool) + for i in range(bs): + real_end = attn_mask[i].sum().item() + start = max(real_end - resp_lens[i] - 1, 0) + loss_mask[i, start:real_end - 1] = True + return loss_mask + + def collate_fn(self, item_list: List[Dict]) -> Dict[str, torch.Tensor]: + bsz = len(item_list) + per_sample_images = ( + [item["images"] for item in item_list] + if self.image_key else [None] * bsz + ) + + stu_full = [item["stu_prompt"] + item["stu_response"] for item in item_list] + stu_encs = [ + self._encode_single(self.student_processor, stu_full[i], per_sample_images[i]) + for i in range(bsz) + ] + stu_pad_id = self.student_tokenizer.pad_token_id + stu_input_ids = zero_pad_sequences( + [e["input_ids"] for e in stu_encs], side="right", value=stu_pad_id, + ) + stu_attn_mask = zero_pad_sequences( + [e["attention_mask"] for e in stu_encs], side="right", value=0, + ).long() + stu_loss_mask = self._build_loss_mask( + stu_attn_mask, [item["stu_resp_len"] for item in item_list], + ) + + batch: Dict[str, Any] = { + "stu_input_ids": stu_input_ids, + "stu_attn_mask": stu_attn_mask, + "stu_loss_mask": stu_loss_mask, + } + if any(e["multi_modal_inputs"] is not None for e in stu_encs): + batch["stu_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in stu_encs] + + if "tea_prompt" in item_list[0]: + if not self.teacher_student_share_input: + tea_full = [item["tea_prompt"] + item["tea_response"] for item in item_list] + tea_encs = [] + for i in range(bsz): + if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]: + proc = self.teacher_processors[item_list[i]["teacher_routing_key"]] + else: + proc = self.teacher_processors.get("default", self.student_processor) + tea_encs.append(self._encode_single(proc, tea_full[i], per_sample_images[i])) + if self.args.kd.multi_teacher_config: + # Multi-teacher mode validates shared vocab with the student. + tea_pad_id = self.student_tokenizer.pad_token_id + else: + teacher_processor = self.teacher_processors.get("default", self.student_processor) + teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor) + tea_pad_id = teacher_tokenizer.pad_token_id + tea_input_ids = zero_pad_sequences( + [e["input_ids"] for e in tea_encs], side="right", value=tea_pad_id, + ) + tea_attn_mask = zero_pad_sequences( + [e["attention_mask"] for e in tea_encs], side="right", value=0, + ).long() + batch["tea_input_ids"] = tea_input_ids + batch["tea_attn_mask"] = tea_attn_mask + batch["tea_loss_mask"] = self._build_loss_mask( + tea_attn_mask, [item["tea_resp_len"] for item in item_list], + ) + if any(e["multi_modal_inputs"] is not None for e in tea_encs): + batch["tea_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in tea_encs] + else: + batch["tea_input_ids"] = batch["stu_input_ids"] + batch["tea_attn_mask"] = batch["stu_attn_mask"] + batch["tea_loss_mask"] = batch["stu_loss_mask"] + if "stu_multi_modal_inputs" in batch: + batch["tea_multi_modal_inputs"] = batch["stu_multi_modal_inputs"] + + # Teacher feed input_ids for SGLang. Text: reuse encoded ids (exact). + # Multimodal: tokenize raw text (single image placeholder) so SGLang re-expands it. + if not self.teacher_student_share_input: + if self.image_key: + tea_feed = [] + for i in range(bsz): + if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]: + proc = self.teacher_processors[item_list[i]["teacher_routing_key"]] + else: + proc = self.teacher_processors.get("default", self.student_processor) + tokenizer = getattr(proc, "tokenizer", proc) + tea_feed.append(tokenizer(tea_full[i])["input_ids"]) + batch["tea_feed_input_ids"] = tea_feed + else: + batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in tea_encs] + else: + if self.image_key: + batch["tea_feed_input_ids"] = [ + self.student_tokenizer(stu_full[i])["input_ids"] for i in range(bsz) + ] + else: + batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in stu_encs] + + if self.image_key: + batch["images"] = per_sample_images + + if "teacher_routing_key" in item_list[0]: + batch["teacher_routing_key"] = [item["teacher_routing_key"] for item in item_list] + + return batch diff --git a/kdflow/datasets/utils.py b/kdflow/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9933ae47766e3623d148e39fa7817cefd112133d --- /dev/null +++ b/kdflow/datasets/utils.py @@ -0,0 +1,387 @@ +import os +from typing import List, Dict, Any + +import torch +from datasets import DatasetDict, interleave_datasets, load_dataset, load_from_disk +from transformers import AutoProcessor, AutoTokenizer + +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def validate_dataset_columns(dataset, **column_settings): + """Validate that configured dataset keys exist in the source dataset. + + Supports dot-notation for nested keys (e.g., "reward_model.ground_truth" -> top-key "reward_model"). + """ + available_columns = list(dataset.column_names) + missing_columns = [ + f"{setting_name}={column_name!r}" + for setting_name, column_name in column_settings.items() + if column_name.split(".")[0] not in available_columns + ] + if missing_columns: + raise ValueError( + f"Dataset is missing configured dataset column(s): " + f"{', '.join(missing_columns)}. Available columns: {available_columns}." + ) + + +def exist_and_not_none(d, key): + return key in d and not d[key] is None + + +def get_nested_value(data: dict, key_path: str, default=None): + """Get a value from a nested dict using dot-notation key path. + + Example: + get_nested_value({'a': {'b': 1}}, 'a.b') -> 1 + get_nested_value({'a': 1}, 'a.b') -> None + """ + keys = key_path.split(".") + value = data + for k in keys: + if isinstance(value, dict): + value = value.get(k) + else: + return default + return value + + +def blending_datasets( + datasets, + probabilities=None, + strategy=None, + seed=42, + max_count=1e8, + stopping_strategy="all_exhausted", + dataset_split="train", +): + """Blend multiple datasets with optional probability sampling. + + Args: + datasets (str): Comma-separated list of dataset paths + probabilities (str, optional): Comma-separated list of probabilities for sampling. + If None, datasets will be concatenated without probability sampling. + strategy: Training strategy object + seed (int): Random seed + max_count (int): Maximum number of samples per dataset + """ + dataset_split = dataset_split or "train" + datasets = datasets.split(",") + if probabilities is not None: + probabilities = list(map(float, probabilities.split(","))) + assert len(probabilities) == len(datasets) + + data_list = [] + for i, dataset in enumerate(datasets): + dataset = dataset.strip() + logger.info(f"dataset: {dataset}") + + data_dir = dataset.split("@")[1].strip() if "@" in dataset else None + dataset = dataset.split("@")[0].strip() + dataset_basename = os.path.basename(dataset) + + ext = os.path.splitext(dataset)[-1] + if ext == ".py" or ( + os.path.isdir(dataset) and os.path.exists(os.path.join(dataset, f"{dataset_basename}.py")) + ): + data = load_dataset(dataset, trust_remote_code=True) + logger.info(f"loaded {dataset} with python script") + elif ext in [".json", ".jsonl", ".csv", ".parquet", ".arrow"]: + ext = ext.lower().strip(".") + if ext == "jsonl": + ext = "json" + data = load_dataset(ext, data_files={dataset_split: dataset}) + logger.info(f"loaded {dataset} with data_files={dataset}") + # local dataset saved with `datasets.Dataset.save_to_disk` + elif os.path.isdir(dataset): + try: + data = load_from_disk(dataset) + logger.info(f"loaded {dataset} from disk") + except Exception as e: + data = load_dataset(dataset, data_dir=data_dir) + logger.info(f"loaded {dataset} from files") + else: + data = load_dataset(dataset, data_dir=data_dir) + logger.info(f"loaded {dataset} from files") + + if isinstance(data, DatasetDict): + if dataset_split not in data: + raise ValueError( + f"Dataset {dataset!r} does not contain split {dataset_split!r}. " + f"Available splits: {list(data)}." + ) + data = data[dataset_split] + data = data.select(range(min(max_count, len(data)))) + data_list.append(data) + + if strategy.is_rank_0(): + print(data_list) + + if probabilities is None: + from datasets import concatenate_datasets + + dataset = concatenate_datasets(data_list) + else: + dataset = interleave_datasets( + data_list, + probabilities=probabilities, + seed=seed, + stopping_strategy=stopping_strategy, + ) + + return dataset + + +def get_tokenizer_or_processor(model_name_or_path, model=None, padding_side="right", use_fast=True, need_processor=False): + if need_processor: + return get_processor(model_name_or_path, model, padding_side, use_fast) + else: + return get_tokenizer(model_name_or_path, model, padding_side, use_fast) + + +def get_processor(model_name_or_path, model=None, padding_side="right", use_fast=True): + processor = AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast) + processor.tokenizer.padding_side = padding_side + if processor.tokenizer.pad_token is None: + logger.info("Detect no pad_token in tokenizer, set it to eos_token.") + processor.tokenizer.pad_token = processor.tokenizer.eos_token + processor.tokenizer.pad_token_id = processor.tokenizer.eos_token_id + if model is not None: + model.config.text_config.pad_token_id = processor.tokenizer.pad_token_id + + return processor + + +def get_tokenizer(model_name_or_path, model=None, padding_side="right", use_fast=True): + try: + # fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style + # tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it + # with the Metaspace pre-tokenizer rebuilt from the slow tokenizer. + tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, + trust_remote_code=True, + use_fast=use_fast, + fix_mistral_regex=True, + ) + except TypeError: + # Older transformers versions do not accept fix_mistral_regex. + tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, + trust_remote_code=True, + use_fast=use_fast, + ) + tokenizer.padding_side = padding_side + if tokenizer.pad_token is None: + logger.info("Detect no pad_token in tokenizer, set it to eos_token.") + tokenizer.pad_token = tokenizer.eos_token + tokenizer.pad_token_id = tokenizer.eos_token_id + if model is not None: + model.config.pad_token_id = tokenizer.pad_token_id + + return tokenizer + + +def convert_token_to_id(token, tokenizer): + if isinstance(token, str): + token = tokenizer.encode(token, add_special_tokens=False) + assert len(token) == 1 + return token[0] + else: + raise ValueError("token should be int or str") + + +def zero_pad_sequences( + sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False +) -> torch.Tensor: + from torch.nn.utils.rnn import pad_sequence + assert side in ("left", "right") + sequences = [seq.squeeze(0) for seq in sequences] + if side == "left": + sequences = [seq.flip(dims=0) for seq in sequences] + padded_sequences = pad_sequence( + sequences, + batch_first=True, + padding_value=value + ) + if side == "left": + padded_sequences = torch.flip(padded_sequences, dims=[1]) + return padded_sequences + + +def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True): + """Remove the pad token. Return tensors and not lists. + + Args: + input_ids shape: [bs, seq_length] + attention_mask shape: [bs, seq_length] + Returns: + no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query. + """ + no_padding_batch = [] + for ids, mask in zip(input_ids, attention_mask): + # Fix for both left and right padding + ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist() + no_padding_batch.append(ids) + return no_padding_batch + + +# ShareGPT role mapping to OpenAI roles +SHAREGPT_ROLE_MAP = { + "human": "user", + "user": "user", + "gpt": "assistant", + "assistant": "assistant", + "system": "system", +} + + +def _is_sharegpt_format(data): + """Check if data is in ShareGPT format (list of dicts with 'from'/'value' keys).""" + if not isinstance(data, list) or not data or not isinstance(data[0], dict): + return False + return "from" in data[0] and "value" in data[0] + + +def _is_openai_format(data): + """Check if data is already in OpenAI messages format (list of dicts with 'role'/'content' keys).""" + if not isinstance(data, list) or not data or not isinstance(data[0], dict): + return False + return "role" in data[0] and "content" in data[0] + + +def _is_alpaca_format(data): + """Check if data is in Alpaca format (dict with 'instruction' key).""" + if not isinstance(data, dict): + return False + return "instruction" in data + + +def _convert_sharegpt(messages): + """Convert ShareGPT format messages to OpenAI messages format. + + ShareGPT: [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}] + OpenAI: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}] + """ + converted = [] + for msg in messages: + role = SHAREGPT_ROLE_MAP.get(msg["from"], msg["from"]) + content = msg.get("value") or "" + converted.append({"role": role, "content": content}) + return converted + + +def _convert_alpaca(data): + """Convert Alpaca format to OpenAI messages format. + + Alpaca format fields: + - instruction (required): the main instruction/question + - input (optional): additional input context + - output (optional): the expected response + - system (optional): system prompt + - history (optional): list of [user_msg, assistant_msg] pairs + + Returns: + List of OpenAI messages: [{"role": "...", "content": "..."}, ...] + """ + messages = [] + + # System prompt + system_prompt = data.get("system", "") + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + # History turns + history = data.get("history", []) + for turn in history: + if isinstance(turn, (list, tuple)) and len(turn) == 2: + messages.append({"role": "user", "content": turn[0]}) + messages.append({"role": "assistant", "content": turn[1]}) + + # Current instruction + optional input + instruction = data.get("instruction", "") + extra_input = data.get("input", "") + if extra_input: + user_content = f"{instruction}\n{extra_input}" + else: + user_content = instruction + messages.append({"role": "user", "content": user_content}) + + # Output (response) + output = data.get("output", "") + if output: + messages.append({"role": "assistant", "content": output}) + + return messages + + +def convert_to_openai_messages(data, role="user", expand_image=False): + """Unified converter: auto-detect data format and convert to OpenAI messages. + + Supported input formats: + 1. OpenAI messages (already): [{"role": "user", "content": "..."}] -> returned as-is + 2. ShareGPT: [{"from": "human", "value": "..."}] -> converted + 3. Alpaca: {"instruction": "...", "input": "...", "output": "...", ...} -> converted + 4. Plain string: "..." -> wrapped as [{"role": role, "content": "..."}] + + Args: + data: Input data in any supported format. + role: Role to assign when data is a plain string. Defaults to "user". + expand_image: Whether to expand placeholders into multi-part content. + Only set to True for multimodal (image) scenarios. Defaults to False. + + Returns: + List of dicts in OpenAI messages format: [{"role": "...", "content": "..."}, ...] + """ + if data is None: + raise ValueError("convert_to_openai_messages received None input.") + + if isinstance(data, str): + messages = [{"role": role, "content": data}] + elif isinstance(data, list): + if not data: + raise ValueError("convert_to_openai_messages received an empty list.") + if _is_openai_format(data): + messages = data + elif _is_sharegpt_format(data): + messages = _convert_sharegpt(data) + else: + raise ValueError( + f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. " + f"Got: {type(data)} with keys/content: {list(data[0].keys()) if data else data}" + ) + elif isinstance(data, dict): + if _is_alpaca_format(data): + messages = _convert_alpaca(data) + else: + raise ValueError( + f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. " + f"Got: {type(data)} with keys/content: {data}" + ) + else: + raise ValueError( + f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. " + f"Got: {type(data)}" + ) + + if expand_image: + messages = expand_image_placeholders(messages) + return messages + + +def expand_image_placeholders(messages, placeholder=""): + for msg in messages: + content = msg.get("content") + if not isinstance(content, str) or placeholder not in content: + continue + parts = content.split(placeholder) + content_list = [] + for i, part in enumerate(parts): + if part: + content_list.append({"type": "text", "text": part}) + if i < len(parts) - 1: + content_list.append({"type": "image"}) + msg["content"] = content_list + return messages diff --git a/kdflow/evaluation/__init__.py b/kdflow/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..245e0251fb5e37de3cc7c2a234bff53a732e41de --- /dev/null +++ b/kdflow/evaluation/__init__.py @@ -0,0 +1,13 @@ +"""Evaluation helpers.""" + +from kdflow.evaluation.alfworld_evaluator import AlfWorldEvaluator +from kdflow.evaluation.math_evaluator import MathEvaluator +from kdflow.evaluation.verl_agent_alfworld_evaluator import ( + VerlAgentAlfWorldEvaluator, +) + +__all__ = [ + "AlfWorldEvaluator", + "MathEvaluator", + "VerlAgentAlfWorldEvaluator", +] diff --git a/kdflow/evaluation/alfworld_evaluator.py b/kdflow/evaluation/alfworld_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..7e085a7350f8a07cd3b1cefcbe12c2eab7733d45 --- /dev/null +++ b/kdflow/evaluation/alfworld_evaluator.py @@ -0,0 +1,164 @@ +"""Standalone ALFWorld evaluation with no teacher or optimizer actors.""" + +import json +import logging +import os +import time +from datetime import datetime, timezone +from typing import Any, Dict, List + +logger = logging.getLogger(__name__) + + +class AlfWorldEvaluator: + """Evaluate a rollout-only model on a fixed number of ALFWorld episodes.""" + + def __init__(self, args, rollout_manager): + self.args = args + self.rollout_manager = rollout_manager + self.num_episodes = args.rollout.alfworld_eval_episodes + self.batch_size = args.rollout.rollout_batch_size + self.output_dir = args.train.save_path + + if self.num_episodes <= 0: + raise ValueError("alfworld_eval_episodes must be positive") + if self.num_episodes % self.batch_size: + raise ValueError( + "alfworld_eval_episodes must be divisible by rollout_batch_size " + f"(got {self.num_episodes} and {self.batch_size})" + ) + + @staticmethod + def _trajectory_row(record: Dict[str, Any], batch_index: int) -> Dict[str, Any]: + output = record.get("output") or {} + metadata = dict(record.get("metadata") or {}) + row = { + "batch_index": batch_index, + "prompt": record.get("stu_prompt", ""), + "response": output.get("text", ""), + } + finish_reason = output.get("finish_reason") + if finish_reason is None: + finish_reason = (output.get("meta_info") or {}).get("finish_reason") + if finish_reason is not None: + row["finish_reason"] = finish_reason + row.update(metadata) + return row + + def evaluate(self) -> Dict[str, Any]: + os.makedirs(self.output_dir, exist_ok=True) + trajectory_path = os.path.join(self.output_dir, "trajectories.jsonl") + summary_path = os.path.join(self.output_dir, "summary.json") + num_batches = self.num_episodes // self.batch_size + sampling_params = { + "max_new_tokens": self.args.rollout.generate_max_len, + "temperature": self.args.rollout.temperature, + "top_p": self.args.rollout.top_p, + } + + totals = { + "episodes": 0.0, + "episodes_succeeded": 0.0, + "turns": 0.0, + "turns_max": 0.0, + "invalid_actions": 0.0, + "environment_failures": 0.0, + "context_truncated": 0.0, + } + started_at = time.time() + + logger.info( + "Starting standalone ALFWorld evaluation: model=%s split=%s " + "episodes=%d batch_size=%d", + self.args.model.student_name_or_path, + self.args.rollout.alfworld_split, + self.num_episodes, + self.batch_size, + ) + with open(trajectory_path, "w", encoding="utf-8") as trajectory_file: + for batch_index in range(1, num_batches + 1): + prompt_batch: List[Dict[str, int]] = [ + {"episode_slot": slot} for slot in range(self.batch_size) + ] + records, metrics = self.rollout_manager.collect_episodes( + prompt_batch=prompt_batch, + global_step=batch_index, + mode="eval", + **sampling_params, + ) + for record in records: + row = self._trajectory_row(record, batch_index) + trajectory_file.write( + json.dumps(row, ensure_ascii=False) + "\n" + ) + trajectory_file.flush() + + prefix = "eval/alfworld" + totals["episodes"] += metrics.get(f"{prefix}/episodes", 0.0) + totals["episodes_succeeded"] += metrics.get( + f"{prefix}/episodes_succeeded", 0.0 + ) + totals["turns"] += metrics.get(f"{prefix}/turns/total", 0.0) + totals["turns_max"] = max( + totals["turns_max"], + metrics.get(f"{prefix}/turns/max", 0.0), + ) + totals["invalid_actions"] += metrics.get( + f"{prefix}/invalid_action_count", 0.0 + ) + totals["environment_failures"] += metrics.get( + f"{prefix}/environment_failure_count", 0.0 + ) + totals["context_truncated"] += metrics.get( + f"{prefix}/context_truncated_count", 0.0 + ) + logger.info( + "ALFWorld evaluation batch %d/%d complete " + "(running success rate %.4f)", + batch_index, + num_batches, + totals["episodes_succeeded"] / max(totals["episodes"], 1.0), + ) + + episodes = totals["episodes"] + turns = totals["turns"] + summary = { + "model": self.args.model.student_name_or_path, + "split": self.args.rollout.alfworld_split, + "episodes": int(episodes), + "episodes_succeeded": int(totals["episodes_succeeded"]), + "success_rate": totals["episodes_succeeded"] / max(episodes, 1.0), + "turns": { + "total": int(turns), + "mean": turns / max(episodes, 1.0), + "max": int(totals["turns_max"]), + }, + "invalid_actions": { + "count": int(totals["invalid_actions"]), + "ratio": totals["invalid_actions"] / max(turns, 1.0), + }, + "environment_failures": { + "count": int(totals["environment_failures"]), + "ratio": totals["environment_failures"] / max(turns, 1.0), + }, + "context_truncated": { + "count": int(totals["context_truncated"]), + "ratio": totals["context_truncated"] / max(episodes, 1.0), + }, + "max_turns_per_episode": self.args.rollout.alfworld_max_turns, + "sampling": sampling_params, + "seed": self.args.train.seed, + "elapsed_seconds": time.time() - started_at, + "completed_at": datetime.now(timezone.utc).isoformat(), + "trajectories_path": os.path.abspath(trajectory_path), + } + with open(summary_path, "w", encoding="utf-8") as summary_file: + json.dump(summary, summary_file, ensure_ascii=False, indent=2) + summary_file.write("\n") + + logger.info( + "ALFWorld evaluation complete: success_rate=%.4f summary=%s", + summary["success_rate"], + summary_path, + ) + return summary diff --git a/kdflow/evaluation/math_evaluator.py b/kdflow/evaluation/math_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..890e7ed2e1fe006d1e65d3a35d398297fb2fb70f --- /dev/null +++ b/kdflow/evaluation/math_evaluator.py @@ -0,0 +1,322 @@ +import asyncio +import json +import logging +import os +from typing import Any, Callable, Dict, List, Optional + +from tqdm import tqdm + +logger = logging.getLogger(__name__) + + +def extract_last_boxed(text: str) -> Optional[str]: + """Return the content of the last complete ``\\boxed`` or ``\\fbox``.""" + if not isinstance(text, str): + return None + + boxed_index = text.rfind("\\boxed") + fbox_index = text.rfind("\\fbox") + start = max(boxed_index, fbox_index) + if start < 0: + return None + + left_brace = text.find("{", start) + if left_brace < 0: + return None + + depth = 0 + for index in range(left_brace, len(text)): + if text[index] == "{": + depth += 1 + elif text[index] == "}": + depth -= 1 + if depth == 0: + return text[left_brace + 1:index] + return None + + +def verify_math_answer(prediction: Optional[str], label: Any) -> bool: + """Compare a boxed prediction with a reference answer using math-verify.""" + try: + from math_verify import parse, verify + except ImportError as error: + raise RuntimeError( + "Math evaluation requires the 'math-verify' package. " + "Install the project dependencies before starting training." + ) from error + + if prediction is None: + return False + + try: + prediction = prediction[:300] + return bool( + verify( + parse(f"\\boxed{{{label}}}"), + parse(f"\\boxed{{{prediction}}}"), + ) + ) + except Exception: + return False + + +def score_math_responses( + responses: List[List[str]], + labels: List[Any], + verifier: Callable[[Optional[str], Any], bool] = verify_math_answer, +) -> tuple[Dict[str, float], List[Dict[str, Any]]]: + """Score multiple generated responses per problem and compute pass@k.""" + if len(responses) != len(labels): + raise ValueError("responses and labels must contain the same number of problems") + + total_correct = 0 + total_responses = 0 + passed_problems = 0 + results = [] + + for index, (problem_responses, label) in enumerate(zip(responses, labels)): + predictions = [extract_last_boxed(response) for response in problem_responses] + correctness = [verifier(prediction, label) for prediction in predictions] + num_correct = sum(correctness) + total_correct += num_correct + total_responses += len(correctness) + passed_problems += int(any(correctness)) + results.append( + { + "index": index, + "answer": str(label), + "pred_answers": predictions, + "responses": problem_responses, + "acc_list": correctness, + "num_correct": num_correct, + } + ) + + num_problems = len(labels) + accuracy = total_correct / total_responses if total_responses else 0.0 + pass_at_k = passed_problems / num_problems if num_problems else 0.0 + metrics = { + "eval/accuracy": accuracy, + "eval/num_correct": float(total_correct), + "eval/num_total": float(total_responses), + "eval/pass_at_k": pass_at_k, + } + return metrics, results + + +class MathEvaluator: + """Run high-concurrency, multi-sample math evaluation through SGLang.""" + + def __init__(self, strategy, rollout_group, generate_kwargs, student_processor): + self.args = strategy.args + self.rollout_group = rollout_group + self.generate_kwargs = generate_kwargs or {} + self.student_processor = student_processor + self.dataset_path = self.args.data.eval_math_dataset_path + self.n_samples = self.args.data.eval_math_n_samples + self.max_concurrent = self.args.data.eval_math_max_concurrent + self.max_tokens = self.args.data.eval_math_max_tokens + self.timeout = self.args.data.eval_math_timeout + + # Fail before training if the optional evaluator dependency is unavailable. + try: + import math_verify # noqa: F401 + except ImportError as error: + raise RuntimeError( + "Math evaluation requires the 'math-verify' package. " + "Install the project dependencies before starting training." + ) from error + + def evaluate(self, global_step: int) -> Dict[str, float]: + eval_data = self._load_dataset() + prompts, labels = self._build_prompts(eval_data) + sampling_params = { + "max_new_tokens": self.max_tokens, + "temperature": self.generate_kwargs.get("temperature", 0.6), + "top_p": self.generate_kwargs.get("top_p", 0.95), + "n": self.n_samples, + } + + logger.info( + "[Math Eval] Starting %d problems at step %d " + "(n_samples=%d, max_concurrent=%d, timeout=%ds)", + len(eval_data), + global_step, + self.n_samples, + self.max_concurrent, + self.timeout, + ) + + if self.args.train.enable_sleep: + self.rollout_group.wakeup() + try: + responses = self._generate(prompts, sampling_params) + finally: + if self.args.train.enable_sleep: + self.rollout_group.sleep() + + metrics, results = score_math_responses(responses, labels) + pass_at_k = metrics.pop("eval/pass_at_k") + if self.n_samples > 1: + metrics[f"eval/pass@{self.n_samples}"] = pass_at_k + for result, item in zip(results, eval_data): + result["problem"] = item["problem"] + + self._save_results( + global_step, metrics, results, len(eval_data), pass_at_k + ) + return metrics + + def _load_dataset(self) -> List[Dict[str, Any]]: + data = [] + with open(self.dataset_path, "r", encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError( + f"Invalid JSON in math evaluation dataset at line {line_number}" + ) from error + missing = [key for key in ("problem", "answer") if key not in item] + if missing: + raise ValueError( + f"Math evaluation dataset line {line_number} is missing " + f"field(s): {', '.join(missing)}" + ) + data.append(item) + if not data: + raise ValueError("Math evaluation dataset is empty") + return data + + def _build_prompts( + self, eval_data: List[Dict[str, Any]] + ) -> tuple[List[List[int]], List[Any]]: + tokenizer = getattr( + self.student_processor, "tokenizer", self.student_processor + ) + prompts = [] + labels = [] + for item in eval_data: + messages = [{"role": "user", "content": str(item["problem"])}] + template_kwargs = {} + if "enable_thinking" in str(getattr(tokenizer, "chat_template", "")): + template_kwargs["enable_thinking"] = self.args.model.enable_thinking + prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + **template_kwargs, + ) + prompts.append(tokenizer.encode(prompt, add_special_tokens=False)) + labels.append(item["answer"]) + return prompts, labels + + def _generate( + self, prompt_token_ids: List[List[int]], sampling_params: Dict[str, Any] + ) -> List[List[str]]: + if not prompt_token_ids: + return [] + return asyncio.run( + self._generate_async(prompt_token_ids, sampling_params) + ) + + async def _generate_async( + self, prompt_token_ids: List[List[int]], sampling_params: Dict[str, Any] + ) -> List[List[str]]: + import aiohttp + + concurrency = min(len(prompt_token_ids), self.max_concurrent) + semaphore = asyncio.Semaphore(concurrency) + results: List[Optional[List[str]]] = [None] * len(prompt_token_ids) + generate_url = f"{self.rollout_group.router_url}/generate" + + async def generate_one(index, input_ids, session): + payload = { + "input_ids": input_ids, + "sampling_params": sampling_params, + } + async with semaphore: + try: + async with session.post(generate_url, json=payload) as response: + response.raise_for_status() + output = await response.json() + if isinstance(output, list): + texts = [item["text"] for item in output] + elif isinstance(output, dict): + texts = output.get("text", "") + texts = [texts] if isinstance(texts, str) else texts + else: + raise TypeError( + f"Unexpected SGLang response type: {type(output).__name__}" + ) + if not isinstance(texts, list) or len(texts) != self.n_samples: + raise ValueError( + f"SGLang returned {len(texts) if isinstance(texts, list) else 0} " + f"responses, expected {self.n_samples}" + ) + results[index] = [str(text).strip() for text in texts] + except Exception as error: + logger.warning( + "[Math Eval] Request %d failed with %s: %s; using empty responses", + index, + type(error).__name__, + error, + ) + results[index] = [""] * self.n_samples + + connector = aiohttp.TCPConnector(limit=concurrency) + timeout = aiohttp.ClientTimeout( + total=self.timeout if self.timeout > 0 else None + ) + async with aiohttp.ClientSession( + connector=connector, trust_env=False, timeout=timeout + ) as session: + tasks = [ + asyncio.create_task(generate_one(index, ids, session)) + for index, ids in enumerate(prompt_token_ids) + ] + progress = tqdm( + total=len(tasks), desc="[Math Eval] Generating", unit="req" + ) + try: + for task in asyncio.as_completed(tasks): + await task + progress.update(1) + finally: + progress.close() + + return [result or [""] * self.n_samples for result in results] + + def _save_results( + self, + global_step: int, + metrics: Dict[str, float], + results: List[Dict[str, Any]], + num_problems: int, + pass_at_k: float, + ) -> None: + eval_dir = os.path.join(self.args.train.save_path, "eval_results") + os.makedirs(eval_dir, exist_ok=True) + + result_path = os.path.join(eval_dir, f"step{global_step}.jsonl") + with open(result_path, "w", encoding="utf-8") as file: + for result in results: + file.write(json.dumps(result, ensure_ascii=False) + "\n") + + summary = { + "global_step": global_step, + "eval_dataset": self.dataset_path, + "n_samples": self.n_samples, + "num_problems": num_problems, + "num_total_responses": int(metrics["eval/num_total"]), + "num_correct": int(metrics["eval/num_correct"]), + "accuracy": metrics["eval/accuracy"], + "pass_at_k": pass_at_k, + } + summary_path = os.path.join( + eval_dir, f"step{global_step}_summary.json" + ) + with open(summary_path, "w", encoding="utf-8") as file: + json.dump(summary, file, indent=2, ensure_ascii=False) diff --git a/kdflow/evaluation/verl_agent_alfworld_evaluator.py b/kdflow/evaluation/verl_agent_alfworld_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..f9a25e4e331f5a9dd3aad873666bb0861347453a --- /dev/null +++ b/kdflow/evaluation/verl_agent_alfworld_evaluator.py @@ -0,0 +1,410 @@ +"""Multi-seed result writer for the migrated verl-agent ALFWorld protocol.""" + +import json +import logging +import os +import statistics +import time +from datetime import datetime, timezone +from typing import Any, Dict, List + +from kdflow.evaluation.verl_agent_alfworld_protocol import TASK_TYPES + + +logger = logging.getLogger(__name__) + + +class VerlAgentAlfWorldEvaluator: + """Evaluate one model with verl-agent's three-seed ALFWorld recipe.""" + + def __init__(self, args, rollout_manager): + self.args = args + self.rollout_manager = rollout_manager + self.num_episodes = args.rollout.alfworld_eval_episodes + self.batch_size = args.rollout.rollout_batch_size + self.output_dir = args.train.save_path + self.seeds = self.parse_seeds(args.rollout.alfworld_eval_seeds) + if self.num_episodes <= 0: + raise ValueError("alfworld_eval_episodes must be positive") + if self.num_episodes % self.batch_size: + raise ValueError( + "alfworld_eval_episodes must be divisible by rollout_batch_size" + ) + + @staticmethod + def parse_seeds(value: str) -> List[int]: + seeds = [int(item.strip()) for item in value.split(",") if item.strip()] + if not seeds: + raise ValueError("At least one ALFWorld evaluation seed is required.") + return seeds + + @staticmethod + def _trajectory_row(record: Dict[str, Any], batch_index: int) -> Dict[str, Any]: + output = record.get("output") or {} + row = { + "batch_index": batch_index, + "prompt": record.get("stu_prompt", ""), + "response": output.get("text", ""), + } + finish_reason = output.get("finish_reason") + if finish_reason is None: + finish_reason = (output.get("meta_info") or {}).get("finish_reason") + if finish_reason is not None: + row["finish_reason"] = finish_reason + row.update(record.get("metadata") or {}) + return row + + def _evaluate_seed(self, script_seed: int) -> Dict[str, Any]: + if self.rollout_manager is None: + raise RuntimeError( + "An ALFWorld rollout manager is required for an unfinished seed." + ) + seed_dir = os.path.join(self.output_dir, f"seed_{script_seed}") + os.makedirs(seed_dir, exist_ok=True) + trajectory_path = os.path.join(seed_dir, "trajectories.jsonl") + summary_path = os.path.join(seed_dir, "summary.json") + sampling = { + "max_new_tokens": self.args.rollout.generate_max_len, + "temperature": self.args.rollout.temperature, + "top_p": self.args.rollout.top_p, + } + prefix = "eval/alfworld" + totals = { + "episodes": 0.0, + "succeeded": 0.0, + "turns": 0.0, + "turns_max": 0.0, + "format_invalid": 0.0, + "inadmissible": 0.0, + "environment_failures": 0.0, + "prompt_truncated": 0.0, + } + task_totals = { + task: {"episodes": 0.0, "succeeded": 0.0} + for task in TASK_TYPES + ("unknown",) + } + num_batches = self.num_episodes // self.batch_size + started_at = time.time() + + with open(trajectory_path, "w", encoding="utf-8") as output_file: + for batch_index in range(num_batches): + episode_offset = batch_index * self.batch_size + self.rollout_manager.set_evaluation_batch( + script_seed=script_seed, + episode_offset=episode_offset, + ) + prompt_batch: List[Dict[str, int]] = [ + {"episode_slot": episode_offset + slot} + for slot in range(self.batch_size) + ] + records, metrics = self.rollout_manager.collect_episodes( + prompt_batch=prompt_batch, + global_step=batch_index + 1, + mode="eval", + **sampling, + ) + for record in records: + output_file.write( + json.dumps( + self._trajectory_row(record, batch_index + 1), + ensure_ascii=False, + ) + + "\n" + ) + output_file.flush() + + totals["episodes"] += metrics.get(f"{prefix}/episodes", 0.0) + totals["succeeded"] += metrics.get( + f"{prefix}/episodes_succeeded", 0.0 + ) + totals["turns"] += metrics.get(f"{prefix}/turns/total", 0.0) + totals["turns_max"] = max( + totals["turns_max"], + metrics.get(f"{prefix}/turns/max", 0.0), + ) + totals["format_invalid"] += metrics.get( + f"{prefix}/format_invalid_count", 0.0 + ) + totals["inadmissible"] += metrics.get( + f"{prefix}/inadmissible_action_count", 0.0 + ) + totals["environment_failures"] += metrics.get( + f"{prefix}/environment_failure_count", 0.0 + ) + totals["prompt_truncated"] += metrics.get( + f"{prefix}/prompt_truncated_count", 0.0 + ) + for task in task_totals: + task_totals[task]["episodes"] += metrics.get( + f"{prefix}/task/{task}/episodes", 0.0 + ) + task_totals[task]["succeeded"] += metrics.get( + f"{prefix}/task/{task}/succeeded", 0.0 + ) + logger.info( + "verl-agent ALFWorld seed=%d batch=%d/%d running_success=%.4f", + script_seed, + batch_index + 1, + num_batches, + totals["succeeded"] / max(totals["episodes"], 1.0), + ) + + episodes = totals["episodes"] + turns = totals["turns"] + tasks = {} + for task, values in task_totals.items(): + if not values["episodes"]: + continue + tasks[task] = { + "episodes": int(values["episodes"]), + "succeeded": int(values["succeeded"]), + "success_rate": values["succeeded"] / values["episodes"], + } + summary = { + "protocol": "verl_agent", + "source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8", + "model": self.args.model.student_name_or_path, + "split": self.args.rollout.alfworld_split, + "script_seed": script_seed, + "environment_seed_base": ( + script_seed + self.args.rollout.alfworld_eval_seed_offset + ), + "episodes": int(episodes), + "episodes_succeeded": int(totals["succeeded"]), + "success_rate": totals["succeeded"] / max(episodes, 1.0), + "turns": { + "total": int(turns), + "mean": turns / max(episodes, 1.0), + "max": int(totals["turns_max"]), + }, + "format_invalid": { + "count": int(totals["format_invalid"]), + "ratio": totals["format_invalid"] / max(turns, 1.0), + }, + "inadmissible_actions": { + "count": int(totals["inadmissible"]), + "ratio": totals["inadmissible"] / max(turns, 1.0), + }, + "environment_failures": { + "count": int(totals["environment_failures"]), + "ratio": totals["environment_failures"] / max(turns, 1.0), + }, + "prompt_truncated": { + "count": int(totals["prompt_truncated"]), + "ratio": totals["prompt_truncated"] / max(turns, 1.0), + }, + "task_types": tasks, + "history_length": self.args.rollout.alfworld_history_length, + "max_steps": self.args.rollout.alfworld_max_turns, + "sampling": sampling, + "elapsed_seconds": time.time() - started_at, + "completed_at": datetime.now(timezone.utc).isoformat(), + "trajectories_path": os.path.abspath(trajectory_path), + } + with open(summary_path, "w", encoding="utf-8") as output_file: + json.dump(summary, output_file, ensure_ascii=False, indent=2) + output_file.write("\n") + return summary + + def _load_completed_seed(self, script_seed: int): + """Return a compatible completed seed summary, otherwise ``None``.""" + seed_dir = os.path.join(self.output_dir, f"seed_{script_seed}") + summary_path = os.path.join(seed_dir, "summary.json") + trajectory_path = os.path.join(seed_dir, "trajectories.jsonl") + if not os.path.isfile(summary_path) or not os.path.isfile(trajectory_path): + return None + if os.path.getsize(trajectory_path) <= 0: + logger.warning( + "Existing ALFWorld seed=%d trajectory is empty; rerunning.", + script_seed, + ) + return None + + try: + with open(summary_path, "r", encoding="utf-8") as input_file: + summary = json.load(input_file) + except (OSError, json.JSONDecodeError) as error: + logger.warning( + "Cannot read existing ALFWorld seed=%d summary (%s); rerunning.", + script_seed, + error, + ) + return None + + expected_sampling = { + "max_new_tokens": self.args.rollout.generate_max_len, + "temperature": self.args.rollout.temperature, + "top_p": self.args.rollout.top_p, + } + expected = { + "protocol": "verl_agent", + "source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8", + "model": self.args.model.student_name_or_path, + "split": self.args.rollout.alfworld_split, + "script_seed": script_seed, + "environment_seed_base": ( + script_seed + self.args.rollout.alfworld_eval_seed_offset + ), + "episodes": self.num_episodes, + "history_length": self.args.rollout.alfworld_history_length, + "max_steps": self.args.rollout.alfworld_max_turns, + "sampling": expected_sampling, + } + mismatches = [ + key for key, value in expected.items() if summary.get(key) != value + ] + required_fields = { + "episodes_succeeded", + "success_rate", + "turns", + "task_types", + } + missing_fields = sorted(required_fields - summary.keys()) + if mismatches or missing_fields: + logger.warning( + "Existing ALFWorld seed=%d result is incompatible " + "(mismatched=%s missing=%s); rerunning.", + script_seed, + mismatches, + missing_fields, + ) + return None + turns = summary.get("turns") or {} + if not {"total", "mean", "max"}.issubset(turns): + logger.warning( + "Existing ALFWorld seed=%d result has incomplete turn metrics; " + "rerunning.", + script_seed, + ) + return None + return summary + + def has_pending_seeds(self) -> bool: + """Whether evaluation must load a model and generate new trajectories.""" + if not getattr(self.args.rollout, "alfworld_eval_resume", True): + return True + return any(self._load_completed_seed(seed) is None for seed in self.seeds) + + def evaluate(self) -> Dict[str, Any]: + os.makedirs(self.output_dir, exist_ok=True) + started_at = time.time() + resume_enabled = getattr( + self.args.rollout, "alfworld_eval_resume", True + ) + seed_summaries = [] + skipped_existing_seeds = [] + evaluated_seeds = [] + for seed in self.seeds: + existing = self._load_completed_seed(seed) if resume_enabled else None + if existing is not None: + logger.info( + "Skipping completed compatible ALFWorld seed=%d result.", + seed, + ) + seed_summaries.append(existing) + skipped_existing_seeds.append(seed) + continue + seed_summaries.append(self._evaluate_seed(seed)) + evaluated_seeds.append(seed) + rates = [summary["success_rate"] for summary in seed_summaries] + total_episodes = sum(summary["episodes"] for summary in seed_summaries) + total_succeeded = sum( + summary["episodes_succeeded"] for summary in seed_summaries + ) + total_turns = sum( + summary["turns"]["total"] for summary in seed_summaries + ) + seed_turn_means = [ + summary["turns"]["mean"] for summary in seed_summaries + ] + max_turns = max( + (summary["turns"]["max"] for summary in seed_summaries), + default=0, + ) + + task_names = set().union( + *(summary["task_types"].keys() for summary in seed_summaries) + ) + task_summary = {} + for task in sorted(task_names): + per_seed_rates = [ + summary["task_types"][task]["success_rate"] + for summary in seed_summaries + if task in summary["task_types"] + ] + task_episodes = sum( + summary["task_types"].get(task, {}).get("episodes", 0) + for summary in seed_summaries + ) + task_succeeded = sum( + summary["task_types"].get(task, {}).get("succeeded", 0) + for summary in seed_summaries + ) + task_summary[task] = { + "episodes": task_episodes, + "succeeded": task_succeeded, + "micro_success_rate": task_succeeded / max(task_episodes, 1), + "seed_mean_success_rate": statistics.fmean(per_seed_rates), + "seed_std_success_rate": ( + statistics.pstdev(per_seed_rates) + if len(per_seed_rates) > 1 + else 0.0 + ), + } + + summary = { + "protocol": "verl_agent", + "source_commit": "20bd331bdbc9026a5668e11362178e10ab7400c8", + "model": self.args.model.student_name_or_path, + "split": self.args.rollout.alfworld_split, + "seeds": self.seeds, + "environment_seed_offset": self.args.rollout.alfworld_eval_seed_offset, + "resume_enabled": resume_enabled, + "skipped_existing_seeds": skipped_existing_seeds, + "evaluated_seeds": evaluated_seeds, + "episodes_per_seed": self.num_episodes, + "total_episodes": total_episodes, + "total_succeeded": total_succeeded, + "micro_success_rate": total_succeeded / max(total_episodes, 1), + "seed_mean_success_rate": statistics.fmean(rates), + "seed_std_success_rate": ( + statistics.pstdev(rates) if len(rates) > 1 else 0.0 + ), + "turns": { + "total": total_turns, + "micro_mean": total_turns / max(total_episodes, 1), + "seed_mean": statistics.fmean(seed_turn_means), + "seed_std": ( + statistics.pstdev(seed_turn_means) + if len(seed_turn_means) > 1 + else 0.0 + ), + "max": max_turns, + }, + "task_types": task_summary, + "history_length": self.args.rollout.alfworld_history_length, + "max_steps": self.args.rollout.alfworld_max_turns, + "sampling": { + "max_new_tokens": self.args.rollout.generate_max_len, + "temperature": self.args.rollout.temperature, + "top_p": self.args.rollout.top_p, + }, + "seed_summaries": [ + os.path.join(f"seed_{seed}", "summary.json") + for seed in self.seeds + ], + "elapsed_seconds": time.time() - started_at, + "completed_at": datetime.now(timezone.utc).isoformat(), + } + summary_path = os.path.join(self.output_dir, "summary.json") + with open(summary_path, "w", encoding="utf-8") as output_file: + json.dump(summary, output_file, ensure_ascii=False, indent=2) + output_file.write("\n") + logger.info( + "verl-agent ALFWorld evaluation complete: mean=%.4f std=%.4f " + "turns=%.2f", + summary["seed_mean_success_rate"], + summary["seed_std_success_rate"], + summary["turns"]["micro_mean"], + ) + return summary diff --git a/kdflow/evaluation/verl_agent_alfworld_protocol.py b/kdflow/evaluation/verl_agent_alfworld_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..38615c951dd17f8311216b640cc2bd0ae145291a --- /dev/null +++ b/kdflow/evaluation/verl_agent_alfworld_protocol.py @@ -0,0 +1,75 @@ +# Copyright 2025 Nanyang Technological University (NTU), Singapore +# and the verl-agent (GiGPO) team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Adapted for KDFlow from langfengq/verl-agent at commit +# 20bd331bdbc9026a5668e11362178e10ab7400c8. + +"""Dependency-free prompts and projection for verl-agent ALFWorld.""" + +import re +from typing import Tuple + + +ALFWORLD_TEMPLATE_NO_HISTORY = """ +You are an expert agent operating in the ALFRED Embodied Environment. +Your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. +""" + +ALFWORLD_TEMPLATE_WITH_HISTORY = """ +You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description} +Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history} +You are now at step {current_step} and your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. +""" + +TASK_TYPES = ( + "pick_and_place", + "pick_two_obj_and_place", + "look_at_obj_in_light", + "pick_heat_then_place_in_recep", + "pick_cool_then_place_in_recep", + "pick_clean_then_place_in_recep", +) + + +def project_alfworld_action(response: str) -> Tuple[str, bool]: + """Reproduce verl-agent's action projection and format-validity flag.""" + original = response or "" + lowered = original.lower() + start_tag = "" + end_tag = "" + start_index = lowered.find(start_tag) + end_index = lowered.find(end_tag) + + if start_index == -1 or end_index == -1: + return lowered[-30:], False + + action = lowered[start_index + len(start_tag) : end_index].strip().lower() + valid = True + if original.find("") == -1 or original.find("") == -1: + valid = False + if re.search(r"[\u4e00-\u9fff]", original): + valid = False + return action, valid + + +def task_type_from_gamefile(gamefile: str) -> str: + for task_type in TASK_TYPES: + if task_type in (gamefile or ""): + return task_type + return "unknown" diff --git a/kdflow/loss/__init__.py b/kdflow/loss/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d66850c7da1a6839a2d69b781980469ea85cd2b --- /dev/null +++ b/kdflow/loss/__init__.py @@ -0,0 +1,39 @@ +import importlib +import os +import glob +from functools import partial + +LOSS_DICT = {} + +def register_loss(name): + def decorator(fn): + LOSS_DICT[name] = fn + return fn + return decorator + +_current_dir = os.path.dirname(os.path.abspath(__file__)) +_module_files = glob.glob(os.path.join(_current_dir, "*.py")) + +for f in _module_files: + if f.endswith("__init__.py"): + continue + module_name = os.path.basename(f)[:-3] + importlib.import_module(f"kdflow.loss.{module_name}") + + +def build_loss_fn(name, args): + """Build a loss function with args-specific hyperparams pre-bound via functools.partial.""" + fn = LOSS_DICT[name] + kd = args.kd + + common = {"temperature": kd.kd_temperature} + + extra_params = { + "jsd": {"jsd_beta": kd.jsd_beta}, + "skl": {"skew_lambda": kd.skew_lambda}, + "srkl": {"skew_lambda": kd.skew_lambda}, + "akl": {"adaptive_kl_alpha": kd.adaptive_alpha}, + "hrl": {"hrl_topk": kd.hrl_topk}, + } + + return partial(fn, **common, **extra_params.get(name, {})) diff --git a/kdflow/loss/adaptive_kl_div.py b/kdflow/loss/adaptive_kl_div.py new file mode 100644 index 0000000000000000000000000000000000000000..e95f03d7900fd29f1ff5d1b64bf1d670a3267410 --- /dev/null +++ b/kdflow/loss/adaptive_kl_div.py @@ -0,0 +1,42 @@ +import torch +import torch.nn.functional as F + +from kdflow.loss import register_loss + + +@register_loss("akl") +@torch.compile() +def compute_adaptive_kl_div( + student_logits, + teacher_logits, + temperature=1.0, + adaptive_kl_alpha=0.5, + reduction="none", + **kwargs +): + """Adaptive KL Divergence loss (https://arxiv.org/abs/2404.02657)""" + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + student_probs = torch.softmax(student_logits, dim=-1, dtype=torch.float32) + teacher_probs = torch.softmax(teacher_logits, dim=-1, dtype=torch.float32) + sorted_teacher_probs, sorted_idx = teacher_probs.sort(-1) + sorted_probs = student_probs.gather(-1, sorted_idx) + gap = (sorted_teacher_probs - sorted_probs).abs() + cum_teacher_probs = torch.cumsum(sorted_teacher_probs, -1) + tail_mask = cum_teacher_probs.le(adaptive_kl_alpha).float() + g_head = (gap * (1 - tail_mask)).sum(-1).detach() + g_tail = (gap * tail_mask).sum(-1).detach() + + student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32) + teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32) + fkl = (teacher_probs * (teacher_lprobs - student_lprobs)).sum(-1) + rkl = (student_probs * (student_lprobs - teacher_lprobs)).sum(-1) + + akl = (g_head / (g_head + g_tail)) * fkl + (g_tail / (g_head + g_tail)) * rkl + + if reduction == "mean": + akl = akl.mean() + elif reduction == "sum": + akl = akl.sum() + + return akl \ No newline at end of file diff --git a/kdflow/loss/chunked_loss.py b/kdflow/loss/chunked_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..2c5174fd3c60c92ea8e315fb1ce28931eb7e61ae --- /dev/null +++ b/kdflow/loss/chunked_loss.py @@ -0,0 +1,82 @@ +from collections import defaultdict +from typing import Any, Callable, List, Optional, Union +import torch + + +def chunked_loss( + student_hidden: torch.Tensor, + student_head: torch.nn.Module, + loss_fn: Callable[..., torch.Tensor], + student_logits_fn: Optional[Callable[..., torch.Tensor]] = None, + teacher_hidden: Optional[torch.Tensor] = None, + teacher_head: Optional[torch.nn.Module] = None, + teacher_logits_fn: Optional[Callable[[int, int], torch.Tensor]] = None, + label: Optional[torch.Tensor] = None, + chunk_size: int = 2048, + reduction: str = "none", + metric_fns: Optional[Union[Callable, List[Callable]]] = None, + return_metrics: bool = False, + **kwargs: Any, +): + """Compute loss chunk by chunk without materializing full logits. + + Inputs are expected to be token-level tensors, e.g. [num_tokens, hidden_size]. + Loss is computed per chunk with reduction="none" and reduced globally here. + """ + if reduction not in ("none", "sum", "mean"): + raise ValueError(f"Unsupported reduction: {reduction}") + if teacher_logits_fn is None and teacher_hidden is None and label is None: + raise ValueError("Either teacher_logits_fn, teacher_hidden or label must be provided.") + if teacher_logits_fn is None and teacher_hidden is not None and teacher_head is None: + raise ValueError("teacher_head must be provided when teacher_hidden is provided.") + + losses = [] + metric_sums = defaultdict(lambda: student_hidden.new_zeros(())) + total_loss = student_hidden.new_zeros(()) + total_tokens = 0 + + for start in range(0, student_hidden.shape[0], chunk_size): + end = start + chunk_size + if student_logits_fn is not None: + student_logits = student_logits_fn(student_hidden[start:end], skip=False) + else: + student_logits = student_head(student_hidden[start:end], skip=False) + has_teacher_logits = False + + if teacher_logits_fn is not None: + teacher_logits = teacher_logits_fn(start, end) + vocab_size = min(student_logits.shape[-1], teacher_logits.shape[-1]) + student_logits = student_logits[..., :vocab_size] + target = teacher_logits[..., :vocab_size] + has_teacher_logits = True + elif teacher_hidden is not None: + teacher_logits = teacher_head(teacher_hidden[start:end]) + vocab_size = min(student_logits.shape[-1], teacher_logits.shape[-1]) + student_logits = student_logits[..., :vocab_size] + target = teacher_logits[..., :vocab_size] + has_teacher_logits = True + else: + target = label[start:end] + + chunk_loss = loss_fn(student_logits, target, reduction="none", **kwargs) + chunk_tokens = chunk_loss.numel() + if metric_fns is not None and has_teacher_logits: + fns = metric_fns if isinstance(metric_fns, list) else [metric_fns] + for fn in fns: + for key, value in fn(student_logits=student_logits, teacher_logits=target).items(): + metric_sums[key] += value * chunk_tokens + if reduction == "none": + losses.append(chunk_loss) + else: + total_loss = total_loss + chunk_loss.sum() + total_tokens += chunk_tokens + + if reduction == "none": + loss = torch.cat(losses, dim=0) + else: + loss = total_loss if reduction == "sum" else total_loss / total_tokens + if not return_metrics: + return loss + if reduction == "mean": + metric_sums = {key: value / total_tokens for key, value in metric_sums.items()} + return loss, metric_sums \ No newline at end of file diff --git a/kdflow/loss/cross_entropy.py b/kdflow/loss/cross_entropy.py new file mode 100644 index 0000000000000000000000000000000000000000..9f5a322ff489503f707515aa8f8e413ef1083b6b --- /dev/null +++ b/kdflow/loss/cross_entropy.py @@ -0,0 +1,18 @@ +import torch + + +@torch.compile() +def compute_cross_entropy( + logits: torch.Tensor, + labels: torch.Tensor, + reduction: str = "none", + **kwargs +) -> torch.Tensor: + logsumexp = torch.logsumexp(logits, dim=-1) + log_probs = logits - logsumexp.unsqueeze(-1) + ce_loss = -log_probs.gather(-1, labels.unsqueeze(-1)).squeeze(-1) + if reduction == "mean": + return ce_loss.mean() + elif reduction == "sum": + return ce_loss.sum() + return ce_loss \ No newline at end of file diff --git a/kdflow/loss/hierarchical_ranking_loss.py b/kdflow/loss/hierarchical_ranking_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6ea5c38676ca63011834431955852055a78af5 --- /dev/null +++ b/kdflow/loss/hierarchical_ranking_loss.py @@ -0,0 +1,40 @@ +import torch +import torch.nn.functional as F + +from kdflow.loss import register_loss + + +@register_loss("hrl") +@torch.compile() +def compute_hierarchical_ranking_loss( + student_logits, + teacher_logits, + temperature=1.0, + hr_topk=5, + reduction="none", + **kwargs +): + student_probs = torch.softmax(student_logits, -1, dtype=torch.float32) + teacher_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32) + + stu_topk_probs, stu_topk_idxs = torch.topk(student_probs, k=hr_topk, dim=-1) + tea_topk_probs, tea_topk_idxs = torch.topk(teacher_probs, k=hr_topk, dim=-1) + + stu_co_topk_probs = student_probs.gather(-1, tea_topk_idxs) + tea_co_topk_probs = teacher_probs.gather(-1, stu_topk_idxs) + + stu_topk_margin = stu_co_topk_probs[..., :1] - stu_co_topk_probs + stu_other_margin = stu_co_topk_probs.unsqueeze(-1) - stu_topk_probs.unsqueeze(-2) + tea_other_margin = stu_topk_probs.unsqueeze(-1) - tea_co_topk_probs.unsqueeze(-2) + topk_rank_loss = torch.maximum(torch.zeros_like(stu_topk_margin), -stu_topk_margin) + topk_rank_loss = topk_rank_loss.sum(-1) + other_rank_loss = torch.maximum(torch.zeros_like(tea_other_margin), -stu_other_margin) + other_rank_loss = (other_rank_loss * tea_other_margin.gt(0)).sum(-1).sum(-1) + rank_loss = topk_rank_loss + other_rank_loss + + if reduction == "mean": + return rank_loss.mean() + elif reduction == "sum": + return rank_loss.sum() + + return rank_loss \ No newline at end of file diff --git a/kdflow/loss/js_div.py b/kdflow/loss/js_div.py new file mode 100644 index 0000000000000000000000000000000000000000..1d210762db1d64e5ee8d8cd4bff9ee60499950e8 --- /dev/null +++ b/kdflow/loss/js_div.py @@ -0,0 +1,35 @@ +import torch + +from kdflow.loss import register_loss + + +@register_loss("jsd") +@torch.compile() +def compute_js_div( + student_logits, + teacher_logits, + temperature=1.0, + jsd_beta=0.5, + reduction="none", + **kwargs +): + """Jensen-Shannon Divergence (JSD) loss in GKD (https://arxiv.org/pdf/2306.13649).""" + assert jsd_beta > 0 and jsd_beta < 1, "jsd_beta must be in (0, 1)" + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + student_lprobs = student_logits.log_softmax(-1, dtype=torch.float32) + teacher_lprobs = teacher_logits.log_softmax(-1, dtype=torch.float32) + log_M = torch.logaddexp( + student_lprobs + torch.tensor(jsd_beta, dtype=torch.float32).log(), + teacher_lprobs + torch.tensor(1 - jsd_beta, dtype=torch.float32).log(), + ) + div1 = jsd_beta * (student_lprobs.exp() * (student_lprobs - log_M)).sum(-1) + div2 = (1 - jsd_beta) * (teacher_lprobs.exp() * (teacher_lprobs - log_M)).sum(-1) + js_div = div1 + div2 + + if reduction == "mean": + js_div = js_div.mean() + elif reduction == "sum": + js_div = js_div.sum() + + return js_div \ No newline at end of file diff --git a/kdflow/loss/kl_div.py b/kdflow/loss/kl_div.py new file mode 100644 index 0000000000000000000000000000000000000000..c39542e7a839837741d0fc442ba5bb18fc5a6343 --- /dev/null +++ b/kdflow/loss/kl_div.py @@ -0,0 +1,22 @@ +import torch +import torch.nn.functional as F + +from kdflow.loss import register_loss + + +@register_loss("kl") +@torch.compile() +def compute_kl_div( + student_logits, + teacher_logits, + temperature=1.0, + reduction="none", + **kwargs +): + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + log_probs = torch.log_softmax(student_logits, -1, dtype=torch.float32) + target_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32) + kl_div = F.kl_div(log_probs, target_probs, reduction=reduction).sum(-1) + + return kl_div \ No newline at end of file diff --git a/kdflow/loss/reverse_kl_div.py b/kdflow/loss/reverse_kl_div.py new file mode 100644 index 0000000000000000000000000000000000000000..ebdaa8f2a467e41383825915e6a757de2426738c --- /dev/null +++ b/kdflow/loss/reverse_kl_div.py @@ -0,0 +1,27 @@ +import torch + +from kdflow.loss import register_loss + + +@register_loss("rkl") +@torch.compile() +def compute_reverse_kl_div( + student_logits, + teacher_logits, + temperature=1.0, + reduction="none", + **kwargs +): + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + student_log_probs = torch.log_softmax(student_logits, -1, dtype=torch.float32) + teacher_log_probs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32) + student_probs = student_log_probs.exp() + rkl_div = (student_probs * (student_log_probs - teacher_log_probs)).sum(-1) + + if reduction == "mean": + rkl_div = rkl_div.mean() + elif reduction == "sum": + rkl_div = rkl_div.sum() + + return rkl_div \ No newline at end of file diff --git a/kdflow/loss/skewed_kl_div.py b/kdflow/loss/skewed_kl_div.py new file mode 100644 index 0000000000000000000000000000000000000000..4b76d83f474a2744c81b12a717cae97ea97dccca --- /dev/null +++ b/kdflow/loss/skewed_kl_div.py @@ -0,0 +1,34 @@ +import torch +import torch.nn.functional as F + +from kdflow.loss import register_loss + + +@register_loss("skl") +@torch.compile() +def compute_skewed_kl_div( + student_logits, + teacher_logits, + temperature=1.0, + skew_lambda=0.1, + reduction="none", + **kwargs +): + """Skewed KL Divergence loss (https://arxiv.org/abs/2402.03898).""" + assert skew_lambda > 0 and skew_lambda < 1, "skew_lambda must be in (0, 1)" + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32) + teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32) + mixed_lprobs = torch.logaddexp( + teacher_lprobs + torch.tensor(skew_lambda, dtype=torch.float32).log(), + student_lprobs + torch.tensor(1 - skew_lambda, dtype=torch.float32).log(), + ) + kl_div = (torch.exp(teacher_lprobs) * (teacher_lprobs - mixed_lprobs)).sum(-1) + + if reduction == "mean": + kl_div = kl_div.mean() + elif reduction == "sum": + kl_div = kl_div.sum() + + return kl_div \ No newline at end of file diff --git a/kdflow/loss/skewed_rkl_div.py b/kdflow/loss/skewed_rkl_div.py new file mode 100644 index 0000000000000000000000000000000000000000..9652791e8f1ff0159eb9826ed70ac16057b511cf --- /dev/null +++ b/kdflow/loss/skewed_rkl_div.py @@ -0,0 +1,34 @@ +import torch +import torch.nn.functional as F + +from kdflow.loss import register_loss + + +@register_loss("srkl") +@torch.compile() +def compute_skewed_rkl_div( + student_logits, + teacher_logits, + temperature=1.0, + skew_lambda=0.1, + reduction="none", + **kwargs +): + """Skewed Reverse KL Divergence loss (https://arxiv.org/abs/2402.03898).""" + assert skew_lambda > 0 and skew_lambda < 1, "skew_lambda must be in (0, 1)" + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + student_lprobs = torch.log_softmax(student_logits, -1, dtype=torch.float32) + teacher_lprobs = torch.log_softmax(teacher_logits, -1, dtype=torch.float32) + mixed_lprobs = torch.logaddexp( + student_lprobs + torch.tensor(skew_lambda, dtype=torch.float32).log(), + teacher_lprobs + torch.tensor(1 - skew_lambda, dtype=torch.float32).log(), + ) + kl_div = (torch.exp(student_lprobs) * (student_lprobs - mixed_lprobs)).sum(-1) + + if reduction == "mean": + kl_div = kl_div.mean() + elif reduction == "sum": + kl_div = kl_div.sum() + + return kl_div \ No newline at end of file diff --git a/kdflow/loss/top1_ce.py b/kdflow/loss/top1_ce.py new file mode 100644 index 0000000000000000000000000000000000000000..6a07fefbc642c79a4311c3f3d3565ee2b3bec251 --- /dev/null +++ b/kdflow/loss/top1_ce.py @@ -0,0 +1,30 @@ +import torch + +from kdflow.loss import register_loss + + +@register_loss("top1_ce") +@torch.compile() +def compute_top1_ce( + student_logits, + teacher_logits, + temperature=1.0, + reduction="none", + **kwargs +): + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + teacher_top1_probs, indices = torch.softmax( + teacher_logits, -1, dtype=torch.float32 + ).max(-1) + student_lprobs = torch.log_softmax( + student_logits, -1, dtype=torch.float32 + ).gather(-1, indices.unsqueeze(-1)).squeeze(-1) + ce_loss = -teacher_top1_probs * student_lprobs + + if reduction == "mean": + return ce_loss.mean() + elif reduction == "sum": + return ce_loss.sum() + + return ce_loss \ No newline at end of file diff --git a/kdflow/loss/tvd.py b/kdflow/loss/tvd.py new file mode 100644 index 0000000000000000000000000000000000000000..a84bee8f7963da36e7e4011a0dd3af9d9e87d462 --- /dev/null +++ b/kdflow/loss/tvd.py @@ -0,0 +1,27 @@ +import torch + +from kdflow.loss import register_loss + + +@register_loss("tvd") +@torch.compile() +def compute_tvd( + student_logits, + teacher_logits, + temperature=1.0, + reduction="none", + **kwargs +): + """Total Variation Distance (TVD) loss.""" + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature + student_probs = torch.softmax(student_logits, -1, dtype=torch.float32) + teacher_probs = torch.softmax(teacher_logits, -1, dtype=torch.float32) + tvd = 0.5 * torch.abs(student_probs - teacher_probs).sum(-1) + + if reduction == "mean": + tvd = tvd.mean() + elif reduction == "sum": + tvd = tvd.sum() + + return tvd \ No newline at end of file diff --git a/kdflow/metrics/__init__.py b/kdflow/metrics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e888dd4bdc45ae4e2be5225b6c74ae17480225a8 --- /dev/null +++ b/kdflow/metrics/__init__.py @@ -0,0 +1,3 @@ +from kdflow.metrics.entropy import compute_entropy + +__all__ = ["compute_entropy"] diff --git a/kdflow/metrics/entropy.py b/kdflow/metrics/entropy.py new file mode 100644 index 0000000000000000000000000000000000000000..81439fc696793032bce83717db7ccb17243d8d9a --- /dev/null +++ b/kdflow/metrics/entropy.py @@ -0,0 +1,18 @@ +import torch +import torch.nn.functional as F + + +def compute_entropy(student_logits, **kwargs): + """Compute the entropy of a logits distribution using numerically stable formula. + + Args: + student_logits: Tensor of shape (num_tokens, vocab_size). + **kwargs: Unused. Accepts extra arguments for unified metric interface. + + Returns: + A dict containing the mean entropy value. + """ + with torch.no_grad(): + probs = F.softmax(student_logits, dim=-1) + entropy = torch.logsumexp(student_logits, dim=-1) - torch.sum(probs * student_logits, dim=-1) + return {"distill/student_entropy": entropy.mean()} diff --git a/kdflow/models/__init__.py b/kdflow/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8f60ffe38287f99a5fe053daf17f85b5173d07a7 --- /dev/null +++ b/kdflow/models/__init__.py @@ -0,0 +1,5 @@ +from .model import DistillModel + +__all__ = [ + "DistillModel", +] diff --git a/kdflow/models/model.py b/kdflow/models/model.py new file mode 100644 index 0000000000000000000000000000000000000000..df455afaca9a0eaaac7d7cc9836acb9f0dc1dbee --- /dev/null +++ b/kdflow/models/model.py @@ -0,0 +1,140 @@ +from typing import Optional + +import torch +import torch.distributed as dist +import torch.nn as nn +from peft import LoraConfig, TaskType, get_peft_model +from peft.tuners.lora import LoraLayer +from transformers import AutoModelForCausalLM, AutoModelForImageTextToText, AutoConfig + +from kdflow.utils import get_tokenizer +from kdflow.models.ring_attn_utils import gather_and_pad_tensor, unpad_and_slice_tensor + + +class DistillModel(nn.Module): + """ + Base class for student models in knowledge distillation (modified from OpenRLHF/openrlhf/models/actor.py). + + Args: + args (Arguments): Arguments. + strategy (Strategy): Strategy for student model loading and training. + device_map (dict, optional): Device mapping for loading the model onto specific devices. Defaults to None. + """ + + def __init__( + self, + strategy, + device_map=None, + **kwargs, + ) -> None: + super().__init__() + self.strategy = strategy + self.args = strategy.args + self.temperature = self.args.rollout.temperature + model_name_or_path = self.args.model.student_name_or_path + + # Support multiple attention mechanism implementations + attn_impl = self.args.model.attn_implementation + + self.model_config = AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=True) + + # Determine if this is a Vision-Language model + self.is_vl_model = hasattr(self.model_config, "vision_config") + + if self.is_vl_model: + model_class = AutoModelForImageTextToText + elif self.args.model.use_liger_kernel: + from liger_kernel.transformers import AutoLigerKernelForCausalLM + model_class = AutoLigerKernelForCausalLM + else: + model_class = AutoModelForCausalLM + + if hasattr(self.model_config, "text_config"): + self.hidden_size = self.model_config.text_config.hidden_size + else: + self.hidden_size = self.model_config.hidden_size + + self.model = strategy.load_hf_model( + model_class, + model_name_or_path, + attn_impl, + self.model_config, + ) + + # LoRA + if self.args.model.lora_rank > 0: + # https://github.com/huggingface/peft/issues/137 + self.model.enable_input_require_grads() + lora_config = LoraConfig( + task_type=TaskType.CAUSAL_LM, + r=self.args.model.lora_rank, + lora_alpha=self.args.model.lora_alpha, + target_modules=self.args.model.target_modules, + lora_dropout=self.args.model.lora_dropout, + bias="none", + ) + self.model = get_peft_model(self.model, lora_config) + + self.tokenizer = get_tokenizer(model_name_or_path, self.model) + + # https://github.com/huggingface/transformers/issues/26877 + # Use `model.generate(use_cache=True)` instead.` + self.model.config.use_cache = False + + # packing samples using Flash Attention 2 + self.packing_samples = self.args.data.packing_samples + + self._print_model() + + def forward( + self, + sequences: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + allgather_logits=False, + ring_attn_group: Optional[dist.ProcessGroup] = None, + **kwargs, + ) -> torch.Tensor: + """Returns action log probs""" + batch, seqlen = sequences.size() + foward_attention_mask = attention_mask + if self.packing_samples: + sequences, position_ids, rolled_sequences, ring_attn_pad_len, indices = unpad_and_slice_tensor( + sequences, attention_mask, ring_attn_group + ) + foward_attention_mask = None + else: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + + output = self.model( + sequences, + attention_mask=foward_attention_mask, + position_ids=position_ids, + **kwargs + ) + # lm_head is patched to identity (skip=True), so output["logits"] + # are actually final hidden states. + output = {"hidden_states": [output["logits"]]} + + if allgather_logits and self.packing_samples: + output["hidden_states"][-1] = gather_and_pad_tensor( + output["hidden_states"][-1], ring_attn_group, + ring_attn_pad_len, indices, batch, seqlen + ).squeeze(-2) + return output + + def _print_model(self): + self.strategy.print(f"Student Model: \n {self.model}") + + def gradient_checkpointing_enable(self): + self.model.gradient_checkpointing_enable( + gradient_checkpointing_kwargs={ + "use_reentrant": self.args.train.gradient_checkpointing_use_reentrant + } + ) + + def gradient_checkpointing_disable(self): + self.model.gradient_checkpointing_disable() + + def print_trainable_parameters(self): + self.model.print_trainable_parameters() diff --git a/kdflow/models/ring_attn_utils.py b/kdflow/models/ring_attn_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fe3b01a70d20358de0dc19e19ce09df306004d26 --- /dev/null +++ b/kdflow/models/ring_attn_utils.py @@ -0,0 +1,188 @@ +import torch +import torch.distributed as dist + + +def _require_flash_attn(): + """Deferred import — only needed for packing_samples=True.""" + try: + from flash_attn.bert_padding import index_first_axis, pad_input, rearrange, unpad_input + from flash_attn.utils.distributed import all_gather as _fa_all_gather + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "flash_attn is required for packing_samples=True. " + "Install with: pip install flash_attn --no-build-isolation" + ) from e + return index_first_axis, pad_input, rearrange, unpad_input, _fa_all_gather + +RING_ATTN_GROUP = None + + +def set_ring_attn_group(group): + global RING_ATTN_GROUP + RING_ATTN_GROUP = group + + +def get_ring_attn_group(): + return RING_ATTN_GROUP + + +def reset_ring_attn_position_ids(start, end, packed_seq_lens): + """ + Calculate position ids for packed_seq_ids[start:end]. + For example, if the packed_seq_lens is [3, 2, 4, 1], start=2, end=8, + the position ids will be [2, 0, 1, 0, 1, 2]. + + Args: + start: the start position + end: the end position + packed_seq_lens: the sequence lengths of packed sequences + """ + position_ids = torch.zeros((1, end - start), dtype=torch.long, device=torch.cuda.current_device()) + offset = 0 + for seqlen in packed_seq_lens: + seq_start = max(offset, start) + seq_end = min(offset + seqlen, end) + if seq_start < seq_end: + position_ids[0, seq_start - start : seq_end - start] = torch.arange(seq_start - offset, seq_end - offset) + + offset += seqlen + if offset >= end: + break + return position_ids + + +def update_ring_attn_params(cu_seqlens): + """ + Calculate the cu_seqlens for the current forward pass and pass the value to + the substituted ring_flash_attn. + + Note that total_seq_len may be larger than the sum of packed_seq_lens because of padding. + """ + assert RING_ATTN_GROUP is not None + + from ring_flash_attn import update_ring_flash_attn_params + + update_ring_flash_attn_params(cu_seqlens, RING_ATTN_GROUP) + + +def get_tensor_in_current_ring_attn_rank(tensors: list[torch.Tensor] | torch.Tensor, ring_attn_group, pad_id): + """ + Deal with padding and slice the tensor to current ring_attn_rank. + Args: + tensors: Each tensor shaped (batch, seqlen) or (1, total_seqs) + ring_attn_group: Ring attention group + pad_id: Padding id + Returns: + Processed tensor + """ + if isinstance(tensors, torch.Tensor): + tensors = [tensors] + ring_attn_rank = dist.get_rank(group=ring_attn_group) + ring_attn_size = dist.get_world_size(group=ring_attn_group) + seqlen = tensors[0].shape[-1] + total_seq_len = tensors[0].numel() + ring_attn_pad_len = (ring_attn_size - seqlen % ring_attn_size) % ring_attn_size + output_tensors = [] + for tensor in tensors: + if tensor.numel() != total_seq_len: + raise ValueError(f"tensor.numel() {tensor.numel()} != total_seq_len {total_seq_len}") + tensor = torch.nn.functional.pad(tensor, (0, ring_attn_pad_len), value=pad_id) + local_seq_len = tensor.numel() // ring_attn_size + start, end = ring_attn_rank * local_seq_len, (ring_attn_rank + 1) * local_seq_len + tensor = tensor[:, start:end] + output_tensors.append(tensor) + if len(output_tensors) == 1: + output_tensors = output_tensors[0] + return output_tensors, ring_attn_pad_len + + +def unpad_and_slice_tensor(sequences, attention_mask, ring_attn_group): + """ + Unpad and slice tensor for distributed training with ring attention. + + This function performs several operations: + 1. Removes padding, unpads sequences from (batch, seqlen) to (1, total_seqs) + 2. Adapts to ring_attn_group, pads sequences to be divisible by ring_attn_group + 3. Slices the sequences for the current ring_attn_rank + + Example: + >>> # Input sequences shape: (batch=2, seqlen=4) + >>> sequences = [[1, 2, 3, 0], [4, 5, 0, 0]] # 0 is padding + >>> attention_mask = [[1, 1, 1, 0], [1, 1, 0, 0]] + >>> # After unpad: + >>> # sequences: [1, 2, 3, 4, 5] # shape (1, total_seqs=5) + >>> # If ring_attn_group size is 2, it will pad to length 6 + >>> # Then slice for current rank (e.g., rank 0 gets [1,2,3], rank 1 gets [4,5,0]) + + Args: + sequences: Input sequences tensor of shape (batch, seqlen) + attention_mask: Attention mask tensor for the sequences + ring_attn_group: Ring attention group for distributed processing + + Returns: + tuple: Processed sequences and related tensors for ring attention + + Note: + Requires ``flash_attn`` — only called when ``packing_samples=True``. + """ + index_first_axis, pad_input, rearrange, unpad_input, _ = _require_flash_attn() + + rolled_sequences = torch.roll(sequences, shifts=-1, dims=1) + sequences, indices, cu_seqlens, _, _ = unpad_input(sequences.unsqueeze(-1), attention_mask) + sequences = sequences.transpose(0, 1) # (1, total_seqs) + rolled_sequences = index_first_axis( + rearrange(rolled_sequences.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose( + 0, 1 + ) # (1, total_seqs) + position_ids = torch.clip(torch.cumsum(attention_mask, dim=-1) - 1, min=0, max=None) + position_ids = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices).transpose( + 0, 1 + ) # (1, total_seqs) + ring_attn_pad_len = 0 + if ring_attn_group is not None: + (sequences, position_ids, rolled_sequences), ring_attn_pad_len = get_tensor_in_current_ring_attn_rank( + [sequences, position_ids, rolled_sequences], ring_attn_group, 0 + ) + cu_seqlens[-1] += ring_attn_pad_len + update_ring_attn_params(cu_seqlens) + return sequences, position_ids, rolled_sequences, ring_attn_pad_len, indices + + +def gather_and_pad_tensor(tensor, ring_attn_group, ring_attn_pad_len, indices, batch, seqlen): + """ + Gather and pad tensor data (such as logits, log_probs, etc.). + + Example: + >>> # Input tensor from each rank (shape: (1, local_seq_len)) + >>> # Rank 0: [1, 2, 3] + >>> # Rank 1: [4, 5, 0] # 0 is padding + >>> # After all_gather: + >>> # tensor: [1, 2, 3, 4, 5, 0] # shape (1, total_seqs=6) + >>> # After removing padding (ring_attn_pad_len=1): + >>> # tensor: [1, 2, 3, 4, 5] # shape (1, total_seqs=5) + >>> # After pad_input with original indices: + >>> # tensor: [[1, 2, 3, 0], [4, 5, 0, 0]] # shape (batch=2, seqlen=4) + + Args: + tensor: Input tensor, can be logits, log_probs, etc. + ring_attn_group: Ring attention group + ring_attn_pad_len: Padding length + indices: Indices + batch: Batch size + seqlen: Sequence length + + Returns: + Padded tensor + + Note: + Requires ``flash_attn`` — only called when ``packing_samples=True``. + """ + _, pad_input, _, _, fa_all_gather = _require_flash_attn() + + if ring_attn_group is not None: + tensor = fa_all_gather(tensor.transpose(0, 1), ring_attn_group).transpose(0, 1) # (1, total_seqs) + if ring_attn_pad_len > 0: + tensor = tensor[:, :-ring_attn_pad_len] + tensor = pad_input(tensor.transpose(0, 1), indices, batch, seqlen).squeeze(-1) # (batch, seqlen) + return tensor diff --git a/kdflow/models/utils.py b/kdflow/models/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5f00ba4aaf5b007c20e4fb7cb08b5e466a2acef5 --- /dev/null +++ b/kdflow/models/utils.py @@ -0,0 +1,56 @@ +import logging +import torch +import torch.nn.functional as F +from dataclasses import dataclass +from typing import Union +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast + +logger = logging.getLogger(__name__) + + +@dataclass +class TokenizerCompareResult: + """Result of comparing two tokenizers.""" + template_identical: bool = True + vocab_identical: bool = True + + @property + def is_identical(self) -> bool: + """True if both chat_template and vocabulary are identical.""" + return self.template_identical and self.vocab_identical + + +def check_tokenizer_identical( + tokenizer1: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], + tokenizer2: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], +) -> TokenizerCompareResult: + """ + Check if two tokenizers are identical by comparing their chat_template and vocabulary. + + Args: + tokenizer1: The first tokenizer to compare. + tokenizer2: The second tokenizer to compare. + + Returns: + TokenizerCompareResult with template_identical and vocab_identical fields. + """ + result = TokenizerCompareResult() + + # Compare chat_template + chat_template1 = getattr(tokenizer1, "chat_template", None) + chat_template2 = getattr(tokenizer2, "chat_template", None) + if chat_template1 != chat_template2: + logger.warning("Tokenizers differ in chat_template.") + result.template_identical = False + + # Compare vocabulary + vocab1 = tokenizer1.get_vocab() + vocab2 = tokenizer2.get_vocab() + if vocab1 != vocab2: + logger.warning( + f"Tokenizers differ in vocabulary. " + f"Vocab sizes: {len(vocab1)} vs {len(vocab2)}." + ) + result.vocab_identical = False + + return result diff --git a/kdflow/ray/__init__.py b/kdflow/ray/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/kdflow/ray/placement_group.py b/kdflow/ray/placement_group.py new file mode 100644 index 0000000000000000000000000000000000000000..24ba3109273257b3ab9329a19dc72804bbc5edc3 --- /dev/null +++ b/kdflow/ray/placement_group.py @@ -0,0 +1,66 @@ +import ipaddress +import socket +import logging + +import ray +from ray.util.placement_group import placement_group +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +@ray.remote(num_gpus=1) +class InfoActor: + def get_ip_and_gpu_id(self): + return ray.util.get_node_ip_address(), ray.get_gpu_ids()[0] + + +def _sort_key(x): + _, node_id, gpu_id = x + try: + ip_parts = list(ipaddress.ip_address(node_id).packed) + except ValueError: + try: + ip_parts = list(ipaddress.ip_address(socket.getaddrinfo(node_id, None)[0][4][0]).packed) + except (socket.gaierror, TypeError, ValueError): + ip_parts = [ord(c) for c in node_id] + return (ip_parts, gpu_id) + + +def create_placement_group(num_gpus): + """Create a placement group and return topology-sorted bundle info. + + Returns: + (pg, reordered_bundle_indices, reordered_gpu_ids) + """ + bundles = [{"GPU": 1, "CPU": 1} for _ in range(num_gpus)] + pg = placement_group(bundles, strategy="PACK") + ray.get(pg.ready()) + + info_actors = [ + InfoActor.options( + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, placement_group_bundle_index=i, + ) + ).remote() + for i in range(num_gpus) + ] + gpu_ids = ray.get([a.get_ip_and_gpu_id.remote() for a in info_actors]) + for a in info_actors: + ray.kill(a) + + bundle_infos = [(i, gpu_ids[i][0], gpu_ids[i][1]) for i in range(num_gpus)] + sorted_infos = sorted(bundle_infos, key=_sort_key) + + reordered_bundle_indices = [info[0] for info in sorted_infos] + reordered_gpu_ids = [gpu_ids[info[0]][1] for info in sorted_infos] + + for i, idx in enumerate(reordered_bundle_indices): + logger.info( + f" bundle {i:4}, actual_index: {idx:4}, " + f"node: {gpu_ids[idx][0]}, gpu: {gpu_ids[idx][1]}" + ) + + return pg, reordered_bundle_indices, reordered_gpu_ids diff --git a/kdflow/ray/rollout/__init__.py b/kdflow/ray/rollout/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a7e39d7a3785301ef2f0614ad28880e2f7dfc1cc --- /dev/null +++ b/kdflow/ray/rollout/__init__.py @@ -0,0 +1,4 @@ +from kdflow.ray.rollout.rollout_actor import RolloutRayActor +from kdflow.ray.rollout.rollout_group import RolloutActorGroup + +__all__ = ["RolloutRayActor", "RolloutActorGroup"] diff --git a/kdflow/ray/rollout/rollout_actor.py b/kdflow/ray/rollout/rollout_actor.py new file mode 100644 index 0000000000000000000000000000000000000000..63a00bc4e29afbb2041ee99ac2c63fa5b3aa1d4f --- /dev/null +++ b/kdflow/ray/rollout/rollout_actor.py @@ -0,0 +1,340 @@ +""" +RolloutActor: A Ray remote actor that manages a single SGLang HTTP server instance. +Port allocation is done centrally by RolloutGroup via _get_current_node_ip_and_free_port(). +""" + +import ipaddress +import multiprocessing +import socket +import time +from typing import Optional + +import ray +import requests +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree + +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +@ray.remote +class RolloutRayActor: + """Ray remote actor wrapping a single SGLang HTTP server.""" + + def __init__(self, rank: int, base_gpu_id: int = 0): + self.rank = rank + self.base_gpu_id = base_gpu_id + self.process: Optional[multiprocessing.Process] = None + self.server_host: Optional[str] = None + self.server_port: Optional[int] = None + self.node_rank: int = 0 + self.router_ip: Optional[str] = None + self.router_port: Optional[int] = None + + def get_node_ip(self) -> str: + return self._get_node_ip() + + @staticmethod + def _get_current_node_ip_and_free_port(start_port: int = 15000, consecutive: int = 1): + """Return the node IP and a free port (called remotely by RolloutGroup for port allocation).""" + return RolloutRayActor._get_node_ip(), RolloutRayActor._get_free_port(start_port=start_port, consecutive=consecutive) + + def init( + self, + model_path: str, + router_ip: str, + router_port: int, + tp_size: int = 1, + host: str = None, + port: int = None, + nccl_port: int = None, + dist_init_addr: str = None, + enable_memory_saver: bool = True, + mem_fraction_static: Optional[float] = None, + extra_server_args: Optional[dict] = None, + ): + """Initialize and launch the SGLang HTTP server, then register with the router.""" + assert host is not None, "host must be provided by RolloutGroup" + assert port is not None, "port must be provided by RolloutGroup" + assert nccl_port is not None, "nccl_port must be provided by RolloutGroup" + assert dist_init_addr is not None, "dist_init_addr must be provided by RolloutGroup" + + self.router_ip = self._format_ipv6(router_ip) + self.router_port = router_port + self.server_host = self._format_ipv6(host) + self.server_port = port + + ip_part, port_part = dist_init_addr.rsplit(":", 1) + dist_init_addr = f"{self._format_ipv6(ip_part)}:{port_part}" + + server_args_dict = { + "model_path": model_path, + "trust_remote_code": True, + "host": self.server_host, + "port": self.server_port, + "nccl_port": nccl_port, + "dist_init_addr": dist_init_addr, + "tp_size": tp_size, + "base_gpu_id": self.base_gpu_id, + "gpu_id_step": 1, + "node_rank": 0, + "nnodes": 1, + "enable_memory_saver": enable_memory_saver, + "enable_weights_cpu_backup": enable_memory_saver, + "skip_server_warmup": True, + "log_level": "warning", + "log_level_http": "warning", + "disable_custom_all_reduce": True, + } + if mem_fraction_static is not None: + server_args_dict["mem_fraction_static"] = mem_fraction_static + + if extra_server_args: + server_args_dict.update(extra_server_args) + + self.node_rank = server_args_dict.get("node_rank", 0) + + logger.info( + f"[RolloutActor {self.rank}] Launching server at {self.server_host}:{self.server_port} " + f"(base_gpu_id={self.base_gpu_id}, tp_size={tp_size})" + ) + self.process = self._launch_sglang_server(ServerArgs(**server_args_dict)) + + if self.node_rank == 0: + self._register_with_router() + logger.info(f"[RolloutActor {self.rank}] Registered with router at {router_ip}:{router_port}") + + def _register_with_router(self): + """Register this server with the SGLang router.""" + worker_url = f"http://{self.server_host}:{self.server_port}" + try: + response = requests.post( + f"http://{self.router_ip}:{self.router_port}/workers", + json={"url": worker_url, "worker_type": "regular"}, + ) + response.raise_for_status() + except Exception: + response = requests.post( + f"http://{self.router_ip}:{self.router_port}/add_worker?url={worker_url}" + ) + response.raise_for_status() + + def _make_request(self, endpoint: str, payload: Optional[dict] = None, method: str = "POST"): + """Make an HTTP request to the local SGLang server.""" + if self.node_rank != 0: + return + + url = f"http://{self.server_host}:{self.server_port}/{endpoint}" + if method == "POST": + response = requests.post(url, json=payload or {}) + else: + response = requests.get(url) + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + e.add_note(f"{response.text=}") + raise + return response.json() + + def sleep(self, tags: Optional[list] = None): + """Release GPU memory occupation (offload to CPU).""" + self.flush_cache() + return self._make_request("release_memory_occupation", {"tags": tags} if tags else {}) + + def wakeup(self, tags: Optional[list] = None): + """Resume GPU memory occupation (reload from CPU).""" + return self._make_request("resume_memory_occupation", {"tags": tags} if tags else {}) + + def flush_cache(self): + """Flush the KV cache on the server.""" + if self.node_rank != 0: + return + for _ in range(60): + try: + response = requests.get(f"http://{self.server_host}:{self.server_port}/flush_cache") + if response.status_code == 200: + return + except Exception as e: + logger.warning(f"[RolloutActor {self.rank}] Error flushing cache: {e}") + time.sleep(1) + raise TimeoutError("Timeout while flushing cache.") + + def update_weights_from_tensor( + self, + serialized_named_tensors: list, + load_format: Optional[str] = None, + flush_cache: bool = True, + ): + """Update model weights from serialized tensor data.""" + return self._make_request( + "update_weights_from_tensor", + { + "serialized_named_tensors": serialized_named_tensors, + "load_format": load_format, + "flush_cache": flush_cache, + }, + ) + + def update_weights_from_disk( + self, + model_path: str, + load_format: Optional[str] = None, + flush_cache: bool = True, + ): + """Update model weights from disk via sglang's update_weights_from_disk API.""" + return self._make_request( + "update_weights_from_disk", + { + "model_path": model_path, + "load_format": load_format, + "flush_cache": flush_cache, + }, + ) + + def health_check(self, timeout: float = 5.0) -> bool: + """Check if the SGLang server is healthy.""" + if self.node_rank != 0: + return True + try: + response = requests.get( + f"http://{self.server_host}:{self.server_port}/health_generate", + timeout=timeout, + ) + return response.status_code == 200 + except Exception: + return False + + def get_server_url(self) -> str: + """Return the URL of this server.""" + return f"http://{self.server_host}:{self.server_port}" + + def shutdown(self): + """Shutdown the SGLang server subprocess and deregister from the router.""" + if not hasattr(self, 'process') or self.process is None: + return + + if self.node_rank == 0: + try: + worker_url = f"http://{self.server_host}:{self.server_port}" + try: + all_workers = requests.get( + f"http://{self.router_ip}:{self.router_port}/workers" + ).json()["workers"] + for worker in all_workers: + if worker["url"] == worker_url: + worker_id = worker["id"] + requests.delete( + f"http://{self.router_ip}:{self.router_port}/workers/{worker_id}" + ) + break + except Exception: + requests.post( + f"http://{self.router_ip}:{self.router_port}/remove_worker?url={worker_url}" + ) + except Exception as e: + logger.warning(f"[RolloutActor {self.rank}] Failed to deregister from router: {e}") + + try: + kill_process_tree(self.process.pid) + except Exception as e: + logger.warning(f"[RolloutActor {self.rank}] Failed to kill server process: {e}") + + self.process = None + logger.info(f"[RolloutActor {self.rank}] Shutdown complete") + + def __del__(self): + self.shutdown() + + # ---- Static helper methods ---- + + @staticmethod + def _format_ipv6(addr: str) -> str: + """Wrap IPv6 addresses with brackets for URL compatibility.""" + if not addr or addr.startswith("["): + return addr + try: + if ipaddress.ip_address(addr).version == 6: + return f"[{addr}]" + except ValueError: + pass + return addr + + @staticmethod + def _is_port_available(port: int) -> bool: + """Check if a port is available.""" + try: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + s.bind(("::", port)) + return True + except OSError: + return False + + @staticmethod + def _get_free_port(start_port: int = 15000, consecutive: int = 1) -> int: + """Find consecutive free ports starting from start_port.""" + port = start_port + while not all(RolloutRayActor._is_port_available(port + i) for i in range(consecutive)): + port += 1 + if port >= 65535: + raise RuntimeError("No free port found") + return port + + @staticmethod + def _get_node_ip() -> str: + """Get current node IP address.""" + return ray._private.services.get_node_ip_address().strip("[]") + + @staticmethod + def _launch_sglang_server(server_args: ServerArgs) -> Optional[multiprocessing.Process]: + """Launch SGLang HTTP server in a subprocess.""" + from sglang.srt.entrypoints.http_server import launch_server + + multiprocessing.set_start_method("spawn", force=True) + server_args.host = server_args.host.strip("[]") + p = multiprocessing.Process(target=launch_server, args=(server_args,)) + p.start() + + if server_args.node_rank != 0: + return p + + # Wait for server to be healthy + RolloutRayActor._wait_server_healthy( + base_url=server_args.url(), + is_process_alive=lambda: p.is_alive(), + ) + return p + + @staticmethod + def _wait_server_healthy(base_url: str, is_process_alive, timeout: float = 600.0): + """Wait until the SGLang server is healthy and ready to serve requests.""" + start_time = time.time() + with requests.Session() as session: + while True: + if time.time() - start_time > timeout: + raise TimeoutError(f"Server at {base_url} failed to become healthy within {timeout}s") + try: + response = session.get(f"{base_url}/health_generate") + if response.status_code == 200: + break + except requests.RequestException: + pass + if not is_process_alive(): + raise RuntimeError("Server process terminated unexpectedly.") + time.sleep(2) + + # Flush cache to ensure clean state + while True: + if time.time() - start_time > timeout: + raise TimeoutError(f"Server at {base_url} failed to flush cache within {timeout}s") + try: + response = session.get(f"{base_url}/flush_cache") + if response.status_code == 200: + break + except requests.RequestException: + pass + if not is_process_alive(): + raise RuntimeError("Server process terminated unexpectedly.") + time.sleep(2) diff --git a/kdflow/ray/rollout/rollout_group.py b/kdflow/ray/rollout/rollout_group.py new file mode 100644 index 0000000000000000000000000000000000000000..5ad5ec900fe03de4c6e7201791f8be76315a3ce9 --- /dev/null +++ b/kdflow/ray/rollout/rollout_group.py @@ -0,0 +1,367 @@ +import asyncio +import base64 +import io +import multiprocessing +import random +import socket +import time +from typing import Any, Dict, List, Optional, Tuple, Union + +import ray +from ray.util.placement_group import PlacementGroup +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + +from kdflow.ray.rollout.rollout_actor import RolloutRayActor +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +NOSET_VISIBLE_DEVICES_ENV_VARS_LIST = [ + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES", + "RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES", + "RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS", + "RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR", +] + +SGLANG_ENV_VARS = { + "SGL_JIT_DEEPGEMM_PRECOMPILE": "false", + "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "false", + "SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK": "true", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "true", + "SGLANG_MEMORY_SAVER_CUDA_GRAPH": "true", + "SGLANG_BATCH_INVARIANT_OPS_ENABLE_MM_FALLBACK_VARIANT": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "false", + "SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE": "false", +} + + +class RolloutActorGroup: + """ + Manages a group of SGLang rollout actors behind a load-balancing router. + + Args: + model_path: Path to the HuggingFace model checkpoint + num_actors: Number of rollout actors (SGLang servers) to create + tp_size: Tensor parallel size per actor + num_gpus_per_node: Number of GPUs per physical node + enable_memory_saver: Enable memory saver for sleep/wakeup support + mem_fraction_static: Static memory fraction for SGLang servers + num_gpus_per_actor: Ray GPU resources per actor (fractional for co-location) + pg: 3-tuple (pg, reordered_bundle_indices, reordered_gpu_ids), PlacementGroup, or None + extra_server_args: Additional SGLang ServerArgs overrides + """ + + def __init__( + self, + model_path: str, + num_actors: int = 1, + tp_size: int = 1, + num_gpus_per_node: int = 8, + enable_memory_saver: bool = True, + mem_fraction_static: Optional[float] = None, + num_gpus_per_actor: float = 0.2, + pg: Optional[Union[PlacementGroup, Tuple[PlacementGroup, list, list]]] = None, + extra_server_args: Optional[dict] = None, + ): + self.model_path = model_path + self.num_actors = num_actors + self.tp_size = tp_size + self.num_gpus_per_node = num_gpus_per_node + self.enable_memory_saver = enable_memory_saver + self.mem_fraction_static = mem_fraction_static + self.extra_server_args = extra_server_args or {} + + self.num_gpus_per_actor_engine = tp_size + + if pg is not None and isinstance(pg, tuple): + self._pg, self._reordered_bundle_indices, self._reordered_gpu_ids = pg + elif pg is not None: + self._pg = pg + total_gpus = num_actors * self.num_gpus_per_actor_engine + self._reordered_bundle_indices = list(range(total_gpus)) + self._reordered_gpu_ids = list(range(total_gpus)) + else: + self._pg = None + self._reordered_bundle_indices = None + self._reordered_gpu_ids = None + + self.router_ip = self._get_node_ip() + self.router_port = self._find_available_port(random.randint(3000, 4000)) + self.router_process = self._start_sglang_router(self.router_ip, self.router_port) + self.router_url = f"http://{self._format_host(self.router_ip)}:{self.router_port}" + + self.actors: List[ray.actor.ActorHandle] = [] + self._create_actors(num_gpus_per_actor) + self._init_actors() + + logger.info( + f"RolloutGroup initialized: {num_actors} actors (tp={tp_size}), " + f"router at {self._format_host(self.router_ip)}:{self.router_port}" + ) + + def _create_actors(self, num_gpus_per_actor: float): + """Create Ray remote RolloutRayActor instances with proper GPU binding.""" + env_vars = { + name: "1" for name in NOSET_VISIBLE_DEVICES_ENV_VARS_LIST + } + env_vars.update(SGLANG_ENV_VARS) + + num_gpu_per_engine = min(self.num_gpus_per_actor_engine, self.num_gpus_per_node) + + for i in range(self.num_actors): + if self._reordered_gpu_ids is not None: + base_gpu_id = int(self._reordered_gpu_ids[i * num_gpu_per_engine]) + else: + base_gpu_id = (i * num_gpu_per_engine) % self.num_gpus_per_node + + options = { + "num_cpus": num_gpus_per_actor, + "num_gpus": num_gpus_per_actor, + "runtime_env": { + "env_vars": env_vars, + }, + } + + if self._pg is not None and self._reordered_bundle_indices is not None: + options["scheduling_strategy"] = PlacementGroupSchedulingStrategy( + placement_group=self._pg, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=self._reordered_bundle_indices[i * num_gpu_per_engine], + ) + + actor = RolloutRayActor.options(**options).remote( + rank=i, + base_gpu_id=base_gpu_id, + ) + self.actors.append(actor) + + def _allocate_addr_and_ports(self) -> List[dict]: + """Centrally allocate host, port, nccl_port, and dist_init_addr for all actors.""" + num_gpu_per_engine = min(self.num_gpus_per_actor_engine, self.num_gpus_per_node) + num_engines_per_node = max(1, self.num_gpus_per_node // num_gpu_per_engine) + addr_and_ports = [{} for _ in range(self.num_actors)] + + visited_nodes = set() + for rank in range(self.num_actors): + node_index = rank // num_engines_per_node + if node_index in visited_nodes: + continue + visited_nodes.add(node_index) + + # Number of engines on this node starting from this rank + num_engines_on_this_node = min( + num_engines_per_node - (rank % num_engines_per_node), + self.num_actors - rank, + ) + + actor = self.actors[rank] + start_port = 15000 + + def get_port(consecutive=1): + nonlocal start_port + _, port = ray.get( + actor._get_current_node_ip_and_free_port.remote( + start_port=start_port, + consecutive=consecutive, + ) + ) + start_port = port + consecutive + return port + + def get_addr(): + addr, _ = ray.get( + actor._get_current_node_ip_and_free_port.remote() + ) + return addr + + for j in range(num_engines_on_this_node): + current_rank = rank + j + if current_rank >= self.num_actors: + break + addr_and_ports[current_rank]["host"] = get_addr() + addr_and_ports[current_rank]["port"] = get_port() + addr_and_ports[current_rank]["nccl_port"] = get_port() + + for j in range(num_engines_on_this_node): + current_rank = rank + j + if current_rank >= self.num_actors: + break + dist_init_port = get_port(30) + addr_and_ports[current_rank]["dist_init_addr"] = ( + f"{self._format_host(addr_and_ports[current_rank]['host'])}:{dist_init_port}" + ) + + for i in range(self.num_actors): + for key in ["host", "port", "nccl_port", "dist_init_addr"]: + assert key in addr_and_ports[i], f"Actor {i} missing '{key}' in port allocation" + logger.info(f"Ports for actor {i}: {addr_and_ports[i]}") + + return addr_and_ports + + def _init_actors(self): + """Allocate ports centrally, then initialize all actors in parallel.""" + addr_and_ports = self._allocate_addr_and_ports() + init_refs = [] + for i, actor in enumerate(self.actors): + ref = actor.init.remote( + model_path=self.model_path, + router_ip=self.router_ip, + router_port=self.router_port, + tp_size=self.tp_size, + host=addr_and_ports[i]["host"], + port=addr_and_ports[i]["port"], + nccl_port=addr_and_ports[i]["nccl_port"], + dist_init_addr=addr_and_ports[i]["dist_init_addr"], + enable_memory_saver=self.enable_memory_saver, + mem_fraction_static=self.mem_fraction_static, + extra_server_args=self.extra_server_args, + ) + init_refs.append(ref) + + ray.get(init_refs) + logger.info(f"All {self.num_actors} rollout actors initialized and registered with router") + + async def generate_one( + self, + prompt: str, + sampling_params: Dict[str, Any], + session, + image_data=None, + ) -> Dict[str, Any]: + """Send one generation request through the SGLang router.""" + import aiohttp + + payload = { + "text": prompt, + "sampling_params": sampling_params, + } + if image_data is not None: + if isinstance(image_data, list): + payload["image_data"] = [ + self._encode_image_to_base64(image) for image in image_data + ] + else: + payload["image_data"] = self._encode_image_to_base64(image_data) + + max_retries = 2 + generate_url = f"{self.router_url}/generate" + for attempt in range(max_retries + 1): + try: + async with session.post(generate_url, json=payload) as response: + response.raise_for_status() + return await response.json() + except (aiohttp.ClientConnectionError, asyncio.TimeoutError) as error: + if attempt == max_retries: + raise RuntimeError( + f"Rollout request failed after {max_retries + 1} attempts " + f"(prompt_chars={len(prompt)}, sampling_params={sampling_params})" + ) from error + await asyncio.sleep(2 ** attempt) + except Exception as error: + raise RuntimeError( + f"Rollout request failed " + f"(prompt_chars={len(prompt)}, sampling_params={sampling_params})" + ) from error + + raise RuntimeError("Rollout request failed unexpectedly") + + def sleep(self, tags: Optional[list] = None): + """Release GPU memory on all rollout actors (offload to CPU).""" + refs = [actor.sleep.remote(tags=tags) for actor in self.actors] + ray.get(refs) + + def wakeup(self, tags: Optional[list] = None): + """Resume GPU memory on all rollout actors (reload from CPU).""" + refs = [actor.wakeup.remote(tags=tags) for actor in self.actors] + ray.get(refs) + + def health_check(self) -> List[bool]: + """Check health of all rollout actors.""" + refs = [actor.health_check.remote() for actor in self.actors] + return ray.get(refs) + + def shutdown(self): + """Shutdown all actors and the router.""" + refs = [actor.shutdown.remote() for actor in self.actors] + try: + ray.get(refs, timeout=30) + except Exception as e: + logger.warning(f"Error during actor shutdown: {e}") + + if self.router_process and self.router_process.is_alive(): + self.router_process.terminate() + self.router_process.join(timeout=5) + if self.router_process.is_alive(): + self.router_process.kill() + + logger.info("RolloutGroup shutdown complete") + + @staticmethod + def _get_node_ip() -> str: + """Get current node IP address.""" + return ray._private.services.get_node_ip_address().strip("[]") + + @staticmethod + def _format_host(host: str) -> str: + if ":" in host and not host.startswith("["): + return f"[{host}]" + return host + + @staticmethod + def _find_available_port(start_port: int = 3000) -> int: + """Find an available port starting from the given port.""" + for port in range(start_port, 65535): + try: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + s.bind(("::", port)) + return s.getsockname()[1] + except OSError: + continue + raise RuntimeError("No free port found") + + @staticmethod + def _start_sglang_router(host: str, port: int) -> multiprocessing.Process: + """Start the SGLang router as a daemon process.""" + from sglang_router.launch_router import RouterArgs, launch_router + + router_args = RouterArgs(host=host, port=port) + router_args.policy = "round_robin" + if hasattr(router_args, "log_level"): + router_args.log_level = "warn" + + logger.info(f"Launching SGLang router at {RolloutActorGroup._format_host(host)}:{port}") + + def _run_router(args): + launch_router(args) + + process = multiprocessing.Process(target=_run_router, args=(router_args,)) + process.daemon = True + process.start() + + # Wait for router to be ready + time.sleep(3) + if not process.is_alive(): + raise RuntimeError("SGLang router process died during startup") + + logger.info(f"SGLang router launched successfully at {RolloutActorGroup._format_host(host)}:{port}") + return process + + @staticmethod + def _encode_image_to_base64(image) -> str: + """Convert a PIL Image to a base64-encoded string for SGLang API.""" + from PIL import Image + + if isinstance(image, str): + return image # Already a base64 string or URL + if isinstance(image, Image.Image): + buffer = io.BytesIO() + fmt = "PNG" if image.mode == "RGBA" else "JPEG" + image.save(buffer, format=fmt) + encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") + return f"data:image/{fmt.lower()};base64,{encoded}" + raise TypeError(f"Unsupported image type: {type(image)}") diff --git a/kdflow/ray/train/__init__.py b/kdflow/ray/train/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/kdflow/ray/train/student_actor.py b/kdflow/ray/train/student_actor.py new file mode 100644 index 0000000000000000000000000000000000000000..0aec3b65c4423b554ae2fdd4fee3b02d64011e78 --- /dev/null +++ b/kdflow/ray/train/student_actor.py @@ -0,0 +1,459 @@ +import math +import os +import socket +import logging +import json +from abc import ABC +from typing import Dict, List, Optional, Union +from collections import defaultdict + +import numpy as np +import ray +import torch +import torch.nn as nn +from torch.optim import Optimizer +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers.trainer import get_scheduler +from transformers import AutoConfig +from huggingface_hub import hf_hub_download +from safetensors.torch import load_file + +from kdflow.models import DistillModel +from kdflow.utils.distributed_util import stateless_init_process_group, torch_dist_barrier_and_cuda_sync +from kdflow.utils.logging_utils import init_logger, normalize_eval_metrics +from kdflow.ray.utils import ray_noset_visible_devices +from kdflow.algorithms import ALGO_DICT +from kdflow.utils.multimodal_utils import extract_multi_modal_inputs + + +logger = init_logger(__name__) + +@ray.remote +class StudentRayActor: + """ + StudentRayActor: Responsible for student model training with FSDP/DeepSpeed. + + This actor only handles the student model training. Teacher model inference + should be handled separately by TeacherRolloutManager using SGLang. + """ + + def __init__(self, world_size, rank, master_addr, master_port): + self._world_size = world_size + self._rank = rank + self._master_addr = master_addr if master_addr else self._get_current_node_ip() + self._master_port = master_port if master_port else self._get_free_port() + os.environ["MASTER_ADDR"] = self._master_addr + os.environ["MASTER_PORT"] = str(self._master_port) + os.environ["WORLD_SIZE"] = str(self._world_size) + os.environ["RANK"] = str(self._rank) + # NOTE: Ray will automatically set the *_VISIBLE_DEVICES + # environment variable for each actor, unless + # RAY_EXPERIMENTAL_NOSET_*_VISIBLE_DEVICES is set, so + # set local rank to 0 when the flag is not applicable. + os.environ["LOCAL_RANK"] = str(ray.get_gpu_ids()[0]) if ray_noset_visible_devices() else "0" + + @staticmethod + def _get_current_node_ip(): + address = ray._private.services.get_node_ip_address() + # strip ipv6 address + return address.strip("[]") + + @staticmethod + def _get_free_port(): + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as sock: + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind(("::", 0)) + return sock.getsockname()[1] + + def get_master_addr_port(self): + return self._master_addr, self._master_port + + def get_node_ip(self) -> str: + return self._get_current_node_ip() + + def empty_cache(self) -> None: + torch.cuda.empty_cache() + torch.cuda.synchronize() + + def init_model_from_pretrained( + self, + strategy, + max_steps=None, + teacher_tokenizer=None, + tokenizer_info=None, + ): + """ + Initialize student model for training. + + Args: + strategy: Training strategy (FSDP) + max_steps: Maximum training steps for scheduler + """ + self.args = strategy.args + self.max_steps = max_steps + self.strategy = strategy + + strategy.setup_distributed() + + # Only initialize student model + self.student = DistillModel(strategy) + self.student = strategy.prepare(self.student) + + # Initialize EMA state for ema update in OPSD + if self.args.kd.use_ema_teacher: + self.ema_state = { + n: p.detach().to("cpu").clone() + for n, p in self.student.model.named_parameters() + if p.requires_grad + } + else: + self.ema_state = None + + # configure optimizer + self.optim = strategy.create_optimizer( + self.student, + lr=self.args.train.learning_rate, + betas=self.args.train.adam_betas, + weight_decay=self.args.train.weight_decay + ) + + self.scheduler = get_scheduler( + self.args.train.lr_scheduler, + self.optim, + num_warmup_steps=math.ceil(max_steps * self.args.train.lr_warmup_ratio), + num_training_steps=max_steps, + scheduler_specific_kwargs={"min_lr": self.args.train.min_lr}, + ) + + if self.args.train.gradient_checkpointing: + self.student.gradient_checkpointing_enable() + + # load teacher lm_head for later logits calculation in the algorithm (logits = lm_head(hidden)) + self.teacher_lm_head = self.load_only_lm_head( + strategy.args.model.teacher_name_or_path + ) + + # Initialize KD algorithm without teacher model + # Teacher hiddens will be passed in each micro_batch during training + ALGO_CLS = ALGO_DICT[self.args.kd.kd_algorithm] + self.kd_algorithm = ALGO_CLS( + strategy=strategy, + student_model=self.student, + teacher_lm_head=self.teacher_lm_head, + student_tokenizer=self.student.tokenizer, + teacher_tokenizer=teacher_tokenizer, + tokenizer_info=tokenizer_info, + ) + + # load checkpoint + self.checkpoint_states = {} + ckpt_path = self.args.train.ckpt_path + if os.path.exists(ckpt_path): + strategy.print(f"Loading the checkpoint: {ckpt_path}") + _, states = strategy.load_ckpt(self.student.model, ckpt_path) + self.checkpoint_states["global_step"] = states["global_step"] + self.checkpoint_states["epoch"] = states["epoch"] + self.checkpoint_states["data_loader_state_dict"] = states["data_loader_state_dict"] + + # initial offload + if self.args.train.enable_sleep: + self.sleep() + + self.image_token_id = getattr(self.student.model.config, "image_token_id", None) + self.video_token_id = getattr(self.student.model.config, "video_token_id", None) + + torch_dist_barrier_and_cuda_sync() + + def load_only_lm_head(self, model_name_or_path, device="cuda", dtype=torch.bfloat16): + """Load only lm_head weights from checkpoint. Supports tied weights and sharded models.""" + logger.info(f"Loading only lm_head from {model_name_or_path}...") + config = AutoConfig.from_pretrained(model_name_or_path) + is_local = os.path.exists(model_name_or_path) + hidden_size = config.hidden_size if not hasattr(config, "text_config") else config.text_config.hidden_size + vocab_size = config.vocab_size if not hasattr(config, "text_config") else config.text_config.vocab_size + + def resolve_file(filename): + if is_local: + return os.path.join(model_name_or_path, filename) + return hf_hub_download(repo_id=model_name_or_path, filename=filename) + + def try_load_index(): + for index_name, safetensors in [ + ("model.safetensors.index.json", True), + ("pytorch_model.bin.index.json", False), + ]: + try: + with open(resolve_file(index_name), "r") as f: + return json.load(f)["weight_map"], safetensors + except Exception: + continue + return None, None + + EMBED_KEYS = ("model.embed_tokens.weight", "model.language_model.embed_tokens.weight") + + def resolve_target_key(weight_map): + if weight_map and "lm_head.weight" in weight_map: + return "lm_head.weight" + if getattr(config, "tie_word_embeddings", True): + for cand in EMBED_KEYS: + if weight_map is None or cand in weight_map: + return cand + raise ValueError(f"Could not find lm_head.weight or any of {EMBED_KEYS} in checkpoint.") + + weight_map, use_safetensors = try_load_index() + target_key = resolve_target_key(weight_map) + + if weight_map: + checkpoint_file = resolve_file(weight_map[target_key]) + else: + for name, safetensors in [("model.safetensors", True), ("pytorch_model.bin", False)]: + try: + checkpoint_file = resolve_file(name) + use_safetensors = safetensors + break + except Exception: + continue + else: + raise FileNotFoundError(f"No checkpoint file found in {model_name_or_path}") + + state_dict = load_file(checkpoint_file) if use_safetensors else torch.load(checkpoint_file, map_location="cpu") + + if "lm_head.weight" in state_dict: + weight_key = "lm_head.weight" + else: + weight_key = next((k for k in EMBED_KEYS if k in state_dict), None) + if weight_key is None: + raise ValueError(f"None of 'lm_head.weight' or {EMBED_KEYS} found. Available: {list(state_dict.keys())[:10]}...") + weight = state_dict[weight_key] + + assert weight.shape == (vocab_size, hidden_size), \ + f"Shape mismatch: expected ({vocab_size}, {hidden_size}), got {weight.shape}" + + has_bias = "lm_head.bias" in state_dict + lm_head = nn.Linear(hidden_size, vocab_size, bias=has_bias) + lm_head.weight = nn.Parameter(weight.to(dtype=dtype)) + if has_bias: + lm_head.bias = nn.Parameter(state_dict["lm_head.bias"].to(dtype=dtype)) + + lm_head = lm_head.to(device).eval() + lm_head.requires_grad_(False) + logger.info(f"Loaded lm_head ({weight_key}), shape: {lm_head.weight.shape}") + return lm_head + + def _prepare_micro_batch(self, batch): + device = torch.cuda.current_device() + micro_batch = { + k: torch.from_numpy(ray.get(v) if isinstance(v, ray.ObjectRef) else v).to(device, non_blocking=True) + if isinstance(v, (np.ndarray, ray.ObjectRef)) + else v.to(device) if isinstance(v, torch.Tensor) + else v + for k, v in batch.items() + } + if isinstance(micro_batch.get("teacher_hiddens"), list): + micro_batch["teacher_hiddens"] = [ + torch.from_numpy(v).to(device, non_blocking=True) + if isinstance(v, np.ndarray) + else v.to(device) if isinstance(v, torch.Tensor) + else v + for v in micro_batch["teacher_hiddens"] + ] + + if "stu_multi_modal_inputs" in micro_batch: + mm_kwargs = extract_multi_modal_inputs(micro_batch["stu_multi_modal_inputs"]) + mm_kwargs = { + k: v.to(device, non_blocking=True) if isinstance(v, torch.Tensor) else v + for k, v in mm_kwargs.items() + } + if mm_kwargs: + input_ids = micro_batch["stu_input_ids"] + mm_token_type_ids = torch.zeros_like(input_ids) + mm_token_type_ids[input_ids == self.image_token_id] = 1 + mm_token_type_ids[input_ids == self.video_token_id] = 2 + mm_kwargs["mm_token_type_ids"] = mm_token_type_ids + micro_batch["stu_multi_modal_inputs"] = mm_kwargs + + return micro_batch + + def fit(self, train_data): + """ + Train student model with the given data. + + Args: + train_data: List of training samples + + Returns: + Training status dict + """ + self.student.train() + status = defaultdict(list) + + if self.args.train.use_dynamic_bsz: + self.strategy.accumulated_gradient = len(train_data) + self.strategy.step = 0 + + for batch in train_data: + micro_batch = self._prepare_micro_batch(batch) + + loss_info = self.kd_algorithm.training_step(micro_batch) + for key in loss_info: + status[key].append(loss_info[key].item()) + + loss = loss_info["train/loss"] + self.strategy.backward(loss, self.student, self.optim) + + + status["train/grad_norm"].append( + torch.nn.utils.clip_grad_norm_( + self.student.parameters(), max_norm=float("inf") + ).item() + ) + + self.strategy.optimizer_step(self.optim, self.student, self.scheduler) + + if self.args.kd.use_ema_teacher and self.strategy.step == 0: + self.ema_update() + + if "response_length" in micro_batch: + status["rollout/response_length"].append(micro_batch["response_length"].mean().item()) + if "total_length" in micro_batch: + status["rollout/total_length"].append(micro_batch["total_length"].mean().item()) + + del micro_batch + + for key in status: + if isinstance(status[key], list) and len(status[key]) > 0: + status[key] = sum(status[key]) / len(status[key]) + + status["train/lr"] = self.scheduler.get_last_lr()[0] + + for key in status: + status[key] = self.strategy.all_reduce(status[key], op="mean") + + return status + + @torch.no_grad() + def evaluate(self, eval_data): + """Evaluate the current student without updating model states.""" + was_training = self.student.training + self.student.eval() + metric_sums = defaultdict(float) + num_tokens = 0.0 + + try: + for batch in eval_data: + micro_batch = self._prepare_micro_batch(batch) + weight = micro_batch.pop("_eval_weight", 1.0) + batch_tokens = micro_batch["stu_loss_mask"].sum().item() + micro_batch["avg_micro_batch_token_num"] = max(batch_tokens, 1) + + metrics = self.kd_algorithm.training_step(micro_batch) + for key, value in metrics.items(): + value = value.item() if hasattr(value, "item") else float(value) + metric_sums[key] += value * batch_tokens * weight + num_tokens += batch_tokens * weight + finally: + if was_training: + self.student.train() + + num_tokens = self.strategy.all_reduce(num_tokens, op="sum") + metrics = { + key: self.strategy.all_reduce(value, op="sum") / max(num_tokens, 1) + for key, value in metric_sums.items() + } + metrics["eval/num_tokens"] = num_tokens / self.args.model.ring_attn_size + return normalize_eval_metrics(metrics) + + def save_model(self, save_path=None): + """Save model checkpoint after fitting on only rank0.""" + if save_path is None: + save_path = self.args.train.save_path + self.strategy.save_model(self.student, self.student.tokenizer, save_path) + + def get_checkpoint_states(self): + return self.checkpoint_states + + def wakeup(self): + """Reload optimizer states from CPU to GPU.""" + self.strategy.reload_model_params(self.student) + self.teacher_lm_head = self.teacher_lm_head.cuda() + self.strategy.reload_optim_states(self.optim) + + def sleep(self): + """Offload optimizer states from GPU to CPU to save memory.""" + self.strategy.offload_optim_states(self.optim) + self.teacher_lm_head = self.teacher_lm_head.cpu() + self.strategy.offload_model_params(self.student, empty_cache=True) + + def save_checkpoint(self, tag, client_states): + self.strategy.save_ckpt( + self.student.model, + os.path.join(self.args.train.ckpt_path, "_actor"), + tag, + self.args.train.max_ckpt_num, + self.args.train.max_ckpt_mem, + client_states, + ) + if self.save_hf_ckpt: + save_path = os.path.join(self.args.train.ckpt_path, f"{tag}_hf") + self.strategy.save_model(self.student, self.student.tokenizer, save_path) + torch_dist_barrier_and_cuda_sync() + + @torch.no_grad() + def ema_update(self): + """EMA update weights for teacher model.""" + decay = self.args.kd.teacher_ema_decay + for n, p in self.student.model.named_parameters(): + if p.requires_grad: + self.ema_state[n].mul_(decay).add_(p.detach().cpu(), alpha=1 - decay) + + def connect_rollout_engines(self, rollout_engines, rollout_tp_size=1): + """Create Gloo IPC groups for weight sync (following slime).""" + import torch.distributed as dist + self._rollout_engines = rollout_engines + num_gpus_per_engine = rollout_tp_size + for i, engine in enumerate(self._rollout_engines): + start_rank = i * num_gpus_per_engine + end_rank = (i + 1) * num_gpus_per_engine + group_ranks = list(range(start_rank, end_rank)) + new_group = dist.new_group(ranks=group_ranks, backend="gloo") + if dist.get_rank() in group_ranks: + self._ipc_gather_src = start_rank + self._ipc_gather_group = new_group + self._ipc_engine = engine + self._tp_rank = dist.get_rank() - start_rank + + def update_rollout_weights(self): + """Stream FSDP weights to rollout engines via Gloo gather + CUDA IPC.""" + model = self.student.model + self.strategy.update_rollout_weights_from_tensor( + model, + engine=self._ipc_engine, + gather_src=self._ipc_gather_src, + gather_group=self._ipc_gather_group, + ) + + def connect_teacher_actors(self, teacher_actors, num_gpus_per_actor): + import torch.distributed as dist + self._teacher_actors = teacher_actors + for i, actor in enumerate(self._teacher_actors): + start_rank = i * num_gpus_per_actor + end_rank = (i + 1) * num_gpus_per_actor + group_ranks = list(range(start_rank, end_rank)) + new_group = dist.new_group(ranks=group_ranks, backend="gloo") + if dist.get_rank() in group_ranks: + self._ipc_gather_src_for_teacher = start_rank + self._ipc_gather_group_for_teacher = new_group + self._ipc_teacher_actor = actor + self._teacher_internal_rank = dist.get_rank() - start_rank + + def update_teacher_weights(self): + """Stream FSDP weights to teacher actors via Gloo gather + CUDA IPC.""" + model = self.student.model + self.strategy.update_rollout_weights_from_tensor( + model, + engine=self._ipc_teacher_actor, + gather_src=self._ipc_gather_src_for_teacher, + gather_group=self._ipc_gather_group_for_teacher, + weight_source=self.ema_state + ) diff --git a/kdflow/ray/train/student_group.py b/kdflow/ray/train/student_group.py new file mode 100644 index 0000000000000000000000000000000000000000..cc9308ae81028870f2a47d70dadaaa443794070d --- /dev/null +++ b/kdflow/ray/train/student_group.py @@ -0,0 +1,239 @@ +import logging +import os +import socket +from typing import Dict, Optional, Type + +import numpy as np +import ray +import torch +from ray.util.placement_group import PlacementGroup, placement_group +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from tqdm import tqdm + +from kdflow.ray.train.student_actor import StudentRayActor + + +class StudentActorGroup: + """ + A group of student actors + + Args: + num_nodes (int): Number of nodes for this actor group. + num_gpus_per_node (int): Number of gpus for this actor group. + pg (PlacementGroup, optional): Placement group to schedule actor on. + If none, create new placement group automatically. Defaults to None. + num_gpus_per_actor (float, optional): Number of gpus allocated for each actor. + If < 1.0, multiple models can share same gpu. Defaults to 1. + """ + + def __init__( + self, + args, + num_nodes, + num_gpus_per_node, + pg=None, + num_gpus_per_actor=1, + resources: Dict[str, float] = None, + num_resources_per_node: int = None, + ) -> None: + self._num_nodes = num_nodes + self._num_gpus_per_node = num_gpus_per_node + self.duplicate_actors = args.model.ring_attn_size + + # custom resources, see https://docs.ray.io/en/latest/ray-core/scheduling/resources.html + self._resources = resources + self._num_resources_per_node = num_resources_per_node + + # Parse PG info (same pattern as RolloutActorGroup / TeacherActorGroup) + if pg is not None and isinstance(pg, tuple): + self._pg, self._reordered_bundle_indices = pg + elif pg is not None: + self._pg = pg + self._reordered_bundle_indices = None + else: + self._pg = None + self._reordered_bundle_indices = None + + self._initiate_actors(num_gpus_per_actor) + + def _initiate_actors(self, num_gpus_per_actor): + world_size = self._num_nodes * self._num_gpus_per_node + pg = self._pg + + # Use placement group to lock resources for models of same type + if self._num_gpus_per_node > 1 and pg is None: + bundles = [{"GPU": 1, "CPU": 1} for _ in range(self._num_nodes * self._num_gpus_per_node)] + if self._resources: + resources_name = list(self._resources.keys())[0] + for i in range(len(bundles)): + bundles[i][resources_name] = self._num_resources_per_node + + pg = placement_group(bundles, strategy="PACK") + ray.get(pg.ready()) + + def _get_bundle_index(rank): + """Map logical rank to actual PG bundle index using reordered_bundle_indices.""" + if self._reordered_bundle_indices is not None: + return self._reordered_bundle_indices[rank] + return rank + + if pg: + master_actor = StudentRayActor.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, placement_group_bundle_index=_get_bundle_index(0) + ), + ).remote(world_size, 0, None, None) + else: + master_actor = StudentRayActor.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + ).remote(world_size, 0, None, None) + self._actor_handlers = [master_actor] + + # Create worker_actor + if world_size > 1: + master_addr, master_port = ray.get(master_actor.get_master_addr_port.remote()) + for rank in range(1, world_size): + if pg: + worker_actor = StudentRayActor.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, + placement_group_bundle_index=_get_bundle_index(rank), + ), + ).remote(world_size, rank, master_addr, master_port) + else: + worker_actor = StudentRayActor.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + ).remote(world_size, rank, master_addr, master_port) + self._actor_handlers.append(worker_actor) + + def async_init_model_from_pretrained(self, *args, **kwargs): + """Init model from pretrained checkpoint. + + Returns: + List: list of remote object refs. + """ + return [actor.init_model_from_pretrained.remote(*args, **kwargs) for actor in self._actor_handlers] + + def async_save_model(self, save_path=None): + """Save actor model on rank 0. + + Returns: + List: list of remote object refs. + """ + return [actor.save_model.remote(save_path) for actor in self._actor_handlers] + + def async_run_distill(self, data): + """ Send data to each distill worker and run distillation. + + Args: + data (list): global batch data + Returns: + List[ray.ObjectRef]: List of remote object references to the results + """ + return self._run_all_actors(data, "fit") + + def async_run_eval(self, data): + """Send evaluation data to every student worker.""" + data = list(data) + if data: + effective_actors = len(self._actor_handlers) // self.duplicate_actors + padding_size = (-len(data)) % effective_actors + data.extend({**data[-1], "_eval_weight": 0.0} for _ in range(padding_size)) + return self._run_all_actors(data, "evaluate") + + def _run_all_actors(self, data, actor_method): + if not data: + return [] + + total_length = len(data) + num_actors = len(self._actor_handlers) + effective_actors = num_actors // self.duplicate_actors + chunk_size = total_length // effective_actors + refs = [] + for chunk_idx in range(effective_actors): + start_idx = chunk_idx * chunk_size + end_idx = min((chunk_idx + 1) * chunk_size, total_length) + # Convert Tensors to np.ndarray for Ray zero-copy deserialization + chunk = [ + {k: v.numpy() if isinstance(v, torch.Tensor) else v for k, v in mb.items()} + for mb in data[start_idx:end_idx] + ] + chunk_ref = ray.put(chunk) + for j in range(self.duplicate_actors): + actor_idx = chunk_idx * self.duplicate_actors + j + actor = self._actor_handlers[actor_idx] + refs.append(getattr(actor, actor_method).remote(chunk_ref)) + return refs + + def sleep(self): + return ray.get([actor.sleep.remote() for actor in self._actor_handlers]) + + def wakeup(self): + return ray.get([actor.wakeup.remote() for actor in self._actor_handlers]) + + def connect_rollout_engines(self, rollout_actors, rollout_tp_size=1): + """Create Gloo IPC groups between training ranks and rollout engines. + + Following slime's approach: for each engine, create a Gloo group containing + the training ranks that correspond to the engine's TP workers. This allows + per-rank CUDA IPC serialization + gather to ensure IPC handles point to + the same GPU as the receiving sglang TP worker. + + Args: + rollout_actors: List of RolloutRayActor handles. + rollout_tp_size: Number of GPUs per rollout engine (TP size). + """ + refs = [ + actor.connect_rollout_engines.remote(rollout_actors, rollout_tp_size) + for actor in self._actor_handlers + ] + ray.get(refs) + + def update_rollout_weights(self): + """Stream FSDP weights to rollout engines via Gloo gather + CUDA IPC. + + All ranks participate: each rank serializes its local CUDA IPC data, + gathers to the source rank via Gloo, and the source rank sends to sglang. + """ + refs = [ + actor.update_rollout_weights.remote() + for actor in self._actor_handlers + ] + ray.get(refs) + + def connect_teacher_actors(self, teacher_actors, num_gpus_per_actor): + """Create Gloo IPC groups between training ranks and teacher actors. + + This process is exactly the same as connect_rollout_engines since teacher actors are also served with SGLang. + + Args: + teacher_actors: List of TeacherRayActor handles. + num_gpus_per_actor: Number of GPUs per teacher actor (tp_size * pp_size). + """ + refs = [ + actor.connect_teacher_actors.remote(teacher_actors, num_gpus_per_actor) + for actor in self._actor_handlers + ] + ray.get(refs) + + def update_teacher_weights(self): + """Stream FSDP weights to teacher_actors via Gloo gather + CUDA IPC. + + All ranks participate: each rank serializes its local CUDA IPC data, + gathers to the source rank via Gloo, and the source rank sends to sglang. + """ + refs = [ + actor.update_teacher_weights.remote() + for actor in self._actor_handlers + ] + ray.get(refs) diff --git a/kdflow/ray/train/teacher_actor.py b/kdflow/ray/train/teacher_actor.py new file mode 100644 index 0000000000000000000000000000000000000000..72600b4ec71cc27acc19bc3f8333410b9784f754 --- /dev/null +++ b/kdflow/ray/train/teacher_actor.py @@ -0,0 +1,167 @@ +import os + +import ray +import torch +import numpy as np + +from kdflow.utils.utils import remove_pad_token +from kdflow.backend.sglang.sglang_engine import SGLangEngineService, EngineConfig +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +@ray.remote +class TeacherRayActor: + """ + TeacherRayActor: Responsible for teacher model forward (prefilling) using SGLang Engine. + + Key design: Teacher and Student SHARE the same GPUs via PlacementGroup co-location. + - TeacherRayActor is scheduled on PG bundles with RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=1 + - GPU binding is managed via base_gpu_id parameter passed to SGLang Engine + - This allows Teacher and Student to share the same GPUs via PlacementGroup + """ + + def __init__( + self, + strategy, + base_gpu_id: int = 0, + nnodes: int = 1, + node_rank: int = 0, + dist_init_addr: str = None, + teacher_name_or_path: str = None, + ): + """ + Initialize TeacherRayActor. + + Args: + strategy: Training strategy containing configuration args + base_gpu_id: Base GPU device ID for SGLang Engine binding (e.g., 0, 1, 2, ...) + Used with RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES for PG co-location. + nnodes: Number of nodes for multi-node tp/pp + node_rank: Rank of this node in tp+pp group + dist_init_addr: Address for distributed initialization + teacher_name_or_path: Name or path of the teacher model (default to args.model.teacher_name_or_path) + """ + logger.info(f"[TeacherRayActor] __init__ STARTED, PID={os.getpid()}, base_gpu_id={base_gpu_id}") + + self.strategy = strategy + self.tp_size = strategy.args.kd.teacher_tp_size + self.ep_size = strategy.args.kd.teacher_ep_size + self.pp_size = strategy.args.kd.teacher_pp_size + self.base_gpu_id = base_gpu_id + self.node_rank = node_rank + self.teacher_name_or_path = teacher_name_or_path + + # Disable tokenizers parallelism to avoid deadlock with multiprocessing + os.environ["TOKENIZERS_PARALLELISM"] = "false" + + # Create engine configuration + # GPU binding is handled by base_gpu_id (works with RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES) + self.engine_config = EngineConfig( + model_path=self.teacher_name_or_path, + tp_size=self.tp_size, + ep_size=self.ep_size, + pp_size=self.pp_size, + chunked_prefill_size=-1, # Disable chunked prefill for full sequence processing + disable_radix_cache=True, # Disable cache for deterministic behavior + enable_return_hidden_states=True, # Enable hidden states extraction + enable_memory_saver=True, # Enable memory saving mode + enable_weights_cpu_backup=True, # Backup weights to CPU for memory release + quantization=strategy.args.kd.teacher_quantization, + mem_fraction_static=strategy.args.kd.teacher_mem_fraction_static, + offload_tags=strategy.args.kd.teacher_offload_tags, + base_gpu_id=self.base_gpu_id, + nnodes=nnodes, + node_rank=node_rank, + dist_init_addr=dist_init_addr, + ) + + # Initialize SGLang Engine service (runs in subprocess) + self.engine_service = SGLangEngineService(self.engine_config) + self.engine_service.start() + + if self.strategy.args.train.enable_sleep and self.node_rank == 0: + logger.info(f"[TeacherRayActor] Teacher sleep after initialization") + self.engine_service.sleep(tags=self.strategy.args.kd.teacher_offload_tags) + + logger.info(f"[TeacherRayActor] Initialized with tp_size={self.tp_size}, ep_size={self.ep_size}, pp_size={self.pp_size}") + + def ready(self): + """Return True when the actor is ready (engine service started).""" + return self.engine_service._started + + def forward(self, global_batch, batch_indices): + """ + Perform forward pass (prefilling) on the given batches. + + Args: + global_batch: List of all micro-batches + batch_indices: List of batch indices this actor should process + + Returns: + List of (batch_idx, micro-batch with teacher_hiddens) tuples and return timestamp + """ + if not batch_indices: + return [] + + batches = [global_batch[i] for i in batch_indices] + + # Collect teacher feed input_ids and loss masks (token ids avoid a re-tokenize round trip). + input_ids = sum((micro_batch["tea_feed_input_ids"] for micro_batch in batches), []) + unpadded_loss_masks = [] + for micro_batch in batches: + attn_mask, loss_mask = micro_batch["tea_attn_mask"], micro_batch["tea_loss_mask"] + unpadded_loss_masks.extend(remove_pad_token(loss_mask, attn_mask, return_tensors=True)) + unpadded_loss_masks = [m.numpy().astype(bool) for m in unpadded_loss_masks] + + # Collect image data if present + image_data = None + if batches[0].get("images") is not None: + image_data = sum((micro_batch["images"] for micro_batch in batches), []) + + hidden_states_list = self.engine_service.generate( + input_ids=input_ids, + loss_masks=unpadded_loss_masks, + sampling_params={"max_new_tokens": 0}, + return_hidden_states=True, + image_data=image_data, + ) + + # Process in micro-batch groups with vectorized operations + sample_idx = 0 + results_with_indices = [] # List of (original_batch_idx, batch_with_hiddens) + for mb_idx, original_batch_idx in enumerate(batch_indices): + mbsz = batches[mb_idx]["tea_input_ids"].shape[0] + mb_hidden_np = hidden_states_list[sample_idx: sample_idx + mbsz] + mb_hidden_np = np.concatenate(mb_hidden_np, axis=0) + batches[mb_idx]["teacher_hiddens"] = ray.put(mb_hidden_np) + results_with_indices.append((original_batch_idx, batches[mb_idx])) + sample_idx += mbsz + + return results_with_indices + + def sleep(self, tags=None): + """Release GPU memory occupation, move weights to CPU.""" + if tags is None: + tags = self.strategy.args.kd.teacher_offload_tags + self.engine_service.sleep(tags=tags) + + def wakeup(self, tags=None): + """Resume GPU memory occupation, move weights back to GPU.""" + if tags is None: + tags = self.strategy.args.kd.teacher_offload_tags + self.engine_service.wakeup(tags=tags) + + def update_weights_from_tensor(self, serialized_named_tensors, load_format, flush_cache): + return self.engine_service.update_weights_from_tensor( + serialized_named_tensors, load_format, flush_cache) + + def flush_cache(self): + """Flush cache. No-op for teacher engine since disable_radix_cache=True.""" + pass + + def shutdown(self): + """Shutdown the engine service.""" + self.engine_service.shutdown() + logger.info("[TeacherRayActor] Shutdown complete") diff --git a/kdflow/ray/train/teacher_group.py b/kdflow/ray/train/teacher_group.py new file mode 100644 index 0000000000000000000000000000000000000000..b121a64b7519f2866d21e8e2d2c208c0052f5826 --- /dev/null +++ b/kdflow/ray/train/teacher_group.py @@ -0,0 +1,278 @@ +import time +from itertools import chain +from typing import Optional, Tuple, Union + +import ray +import numpy as np +from ray.util.placement_group import PlacementGroup +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + +from kdflow.ray.train.teacher_actor import TeacherRayActor +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +NOSET_VISIBLE_DEVICES_ENV_VARS_LIST = [ + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES", + "RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES", + "RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS", + "RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR", +] + + +class TeacherActorGroup: + """ + TeacherActorGroup: Manages multiple TeacherRayActor instances for distributed + teacher forward (prefilling) in knowledge distillation. + + Key design: Teacher actors are scheduled on PlacementGroup bundles using + RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES + base_gpu_id, matching the + same pattern as RolloutActorGroup for unified resource management. + """ + + def __init__( + self, + strategy, + num_gpus: int, + num_gpus_per_node: int = 8, + num_gpus_per_actor: float = 0.2, + pg: Optional[Union[PlacementGroup, Tuple[PlacementGroup, list, list]]] = None, + teacher_name_or_path: str = None, + ): + """ + Initialize TeacherActorGroup. + + Args: + strategy: Training strategy containing configuration args + num_gpus: Total number of GPUs available (e.g., 8) + num_gpus_per_node: Number of GPUs per physical node + num_gpus_per_actor: Ray GPU resources per actor (fractional for co-location) + pg: 3-tuple (pg, reordered_bundle_indices, reordered_gpu_ids), PlacementGroup, or None + teacher_name_or_path: Name or path of the teacher model + """ + self.teacher_engines = [] + self._worker_actors = [] + self.strategy = strategy + self.dp_size = strategy.args.kd.teacher_dp_size + self.tp_size = strategy.args.kd.teacher_tp_size + self.pp_size = strategy.args.kd.teacher_pp_size + self.num_gpus_per_node = num_gpus_per_node + self.teacher_name_or_path = teacher_name_or_path or strategy.args.model.teacher_name_or_path + logger.info(f"[TeacherActorGroup] Start to initialize the teacher from {self.teacher_name_or_path}...") + + # Parse PG info (same pattern as RolloutActorGroup) + if pg is not None and isinstance(pg, tuple): + self._pg, self._reordered_bundle_indices, self._reordered_gpu_ids = pg + elif pg is not None: + self._pg = pg + total_gpus = self.dp_size * self.tp_size * self.pp_size + self._reordered_bundle_indices = list(range(total_gpus)) + self._reordered_gpu_ids = list(range(total_gpus)) + else: + self._pg = None + self._reordered_bundle_indices = None + self._reordered_gpu_ids = None + + # Validate configuration + required_gpus = self.dp_size * self.tp_size * self.pp_size + if required_gpus > num_gpus: + raise ValueError(f"Teacher requires {required_gpus} GPUs (dp={self.dp_size} * tp={self.tp_size} * pp={self.pp_size}) " + f"but only {num_gpus} GPUs available") + + logger.info(f"[TeacherActorGroup] Creating {self.dp_size} actors with tp_size={self.tp_size} * pp_size={self.pp_size}") + + self._create_actors(num_gpus_per_actor) + + ray.get([actor.ready.remote() for actor in self.teacher_engines]) + + logger.info(f"[TeacherActorGroup] All {self.dp_size} actors ready.") + + def _create_actors(self, num_gpus_per_actor: float): + """Create Ray remote TeacherRayActor instances with proper GPU binding via PG.""" + env_vars = { + name: "1" for name in NOSET_VISIBLE_DEVICES_ENV_VARS_LIST + } + + num_gpu_per_engine = self.tp_size * self.pp_size + nnodes_per_engine = max(num_gpu_per_engine // self.num_gpus_per_node, 1) + + for i in range(self.dp_size): + if nnodes_per_engine > 1: # multi-node tp/pp + dist_init_addr = self._get_dist_init_addr(i, num_gpu_per_engine) + engine_actors = [] + for node_idx in range(nnodes_per_engine): + gpu_offset = i * num_gpu_per_engine + node_idx * self.num_gpus_per_node + + if self._reordered_gpu_ids is not None: + base_gpu_id = int(self._reordered_gpu_ids[gpu_offset]) + else: + base_gpu_id = 0 + + bundle_idx = self._reordered_bundle_indices[gpu_offset] if self._reordered_bundle_indices else gpu_offset + + options = { + "num_cpus": num_gpus_per_actor, + "num_gpus": num_gpus_per_actor, + "max_concurrency": 2, + "runtime_env": {"env_vars": env_vars}, + } + + if self._pg is not None: + options["scheduling_strategy"] = PlacementGroupSchedulingStrategy( + placement_group=self._pg, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=bundle_idx, + ) + + actor = TeacherRayActor.options(**options).remote( + self.strategy, + base_gpu_id=base_gpu_id, + nnodes=nnodes_per_engine, + node_rank=node_idx, + dist_init_addr=dist_init_addr, + teacher_name_or_path=self.teacher_name_or_path + ) + engine_actors.append(actor) + + self.teacher_engines.append(engine_actors[0]) + self._worker_actors.extend(engine_actors[1:]) + else: + # Calculate base_gpu_id from PG topology (same as RolloutActorGroup) + if self._reordered_gpu_ids is not None: + base_gpu_id = int(self._reordered_gpu_ids[i * num_gpu_per_engine]) + else: + base_gpu_id = (i * num_gpu_per_engine) % self.num_gpus_per_node + + logger.info(f"[TeacherActorGroup] Launching actor {i} with base_gpu_id={base_gpu_id}...") + + options = { + "num_cpus": num_gpus_per_actor, + "num_gpus": num_gpus_per_actor, + "max_concurrency": 2, + "runtime_env": { + "env_vars": env_vars, + }, + } + + # Schedule on PG bundle if available + if self._pg is not None and self._reordered_bundle_indices is not None: + options["scheduling_strategy"] = PlacementGroupSchedulingStrategy( + placement_group=self._pg, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=self._reordered_bundle_indices[i * num_gpu_per_engine], + ) + + actor = TeacherRayActor.options(**options).remote( + self.strategy, base_gpu_id, teacher_name_or_path=self.teacher_name_or_path + ) + + self.teacher_engines.append(actor) + logger.info(f"[TeacherActorGroup] Actor {i} created, waiting for ready...") + + @staticmethod + def _format_host(host: str) -> str: + if ":" in host and not host.startswith("["): + return f"[{host}]" + return host + + def _get_dist_init_addr(self, engine_idx: int, num_gpu_per_engine: int) -> str: + offset = engine_idx * num_gpu_per_engine + bundle_idx = self._reordered_bundle_indices[offset] if self._reordered_bundle_indices else offset + + @ray.remote(num_cpus=0, num_gpus=0) + def _get_node_ip_and_free_port(): + import socket + ip = ray.util.get_node_ip_address() + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + s.bind(("::", 0)) + port = s.getsockname()[1] + return ip, port + + ip, port = ray.get( + _get_node_ip_and_free_port.options( + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=self._pg, + placement_group_bundle_index=bundle_idx, + ) + ).remote() + ) + return f"{self._format_host(ip)}:{port}" + + def forward(self, global_batch): + """ + Perform forward pass (prefilling) on all teacher actors in parallel. + Uses token-based load balancing to distribute batches evenly across actors. + """ + all_data_ref = ray.put(global_batch) + + # === Token-based load balancing === + # Calculate token count for each micro-batch (sum of non-padding tokens) + batch_token_counts = [] + for mb in global_batch: + # attn_mask indicates non-padding positions + token_count = mb["tea_attn_mask"].sum().item() + batch_token_counts.append(token_count) + + # Assign batches to actors using greedy algorithm: + # Always assign next batch to the actor with fewest tokens + actor_assignments = [[] for _ in range(self.dp_size)] # batch indices for each actor + actor_tokens = [0] * self.dp_size # running token count for each actor + + for batch_idx, token_count in enumerate(batch_token_counts): + # Find actor with minimum tokens so far + min_actor = min(range(self.dp_size), key=lambda x: actor_tokens[x]) + actor_assignments[min_actor].append(batch_idx) + actor_tokens[min_actor] += token_count + + futures = [] + for i, actor in enumerate(self.teacher_engines): + batch_indices = actor_assignments[i] + futures.append(actor.forward.remote(all_data_ref, batch_indices)) + + # Use ray.wait to get results as they complete, measuring each actor's timing + pending = list(futures) + raw_results = [None] * len(futures) + future_to_idx = {f: i for i, f in enumerate(futures)} + + fwd_start = time.time() + wait_count = 0 + while pending: + ready, pending = ray.wait(pending, num_returns=1, timeout=120) + wait_count += 1 + if not ready: + elapsed = time.time() - fwd_start + logger.warning(f"[TeacherActorGroup.forward] ray.wait timeout #{wait_count}, " + f"elapsed={elapsed:.1f}s, pending_actors={len(pending)}, " + f"pending_indices={[future_to_idx[f] for f in pending]}") + continue + for ref in ready: + idx = future_to_idx[ref] + raw_results[idx] = ray.get(ref) + + # Flatten results (extract (batch_idx, batch) tuples from all actors) + # Each actor returns (results_with_indices, timestamp) where results_with_indices is [(batch_idx, batch), ...] + indexed_results = list(chain.from_iterable(r for r in raw_results)) + + # Sort by original batch index to restore the original order + indexed_results.sort(key=lambda x: x[0]) + results = [batch for _, batch in indexed_results] + + return results + + def sleep(self, tags=None): + """Release GPU memory on all teacher engines.""" + ray.get([actor.sleep.remote(tags=tags) for actor in self.teacher_engines]) + + def wakeup(self, tags=None): + """Resume GPU memory on all teacher engines.""" + ray.get([actor.wakeup.remote(tags=tags) for actor in self.teacher_engines]) + + def shutdown(self): + """Shutdown all teacher engines.""" + ray.get([actor.shutdown.remote() for actor in self.teacher_engines + self._worker_actors]) + logger.info("[TeacherActorGroup] All teacher actors shutdown complete.") diff --git a/kdflow/ray/utils.py b/kdflow/ray/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b003f49102a84c9ce15e617abb33fa4842286fcc --- /dev/null +++ b/kdflow/ray/utils.py @@ -0,0 +1,48 @@ +import os + + +# Address https://github.com/ray-project/ray/issues/51117 +# This function is used to get the bundle indices of a placement group +# and ensure that the bundles placed on the same node are grouped together. +def get_bundle_indices(placement_group, index, length): + import ray + + pg_infos = ray.util.placement_group_table(placement_group) + + node_id_to_bundles = {} + for bundle, node_id in pg_infos["bundles_to_node_id"].items(): + node_id_to_bundles.setdefault(node_id, []).append(bundle) + + sorted_bundle_indices = sum(node_id_to_bundles.values(), []) + return sorted_bundle_indices[index * length : (index + 1) * length] + + +def ray_noset_visible_devices(env_vars=os.environ): + # Refer to + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/nvidia_gpu.py#L95-L96 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/amd_gpu.py#L102-L103 + # https://github.com/ray-project/ray/blob/3b9e729f6a669ffd85190f901f5e262af79771b0/python/ray/_private/accelerators/amd_gpu.py#L114-L115 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/npu.py#L94-L95 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/hpu.py#L116-L117 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/neuron.py#L108-L109 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/tpu.py#L171-L172 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/intel_gpu.py#L97-L98 + NOSET_VISIBLE_DEVICES_ENV_VARS_LIST = [ + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES", + "RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES", + "RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS", + "RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR", + ] + return any(env_vars.get(env_var) for env_var in NOSET_VISIBLE_DEVICES_ENV_VARS_LIST) + + +def get_physical_gpu_id(): + import torch + + device = torch.cuda.current_device() + props = torch.cuda.get_device_properties(device) + return str(props.uuid) diff --git a/kdflow/token_alignment.py b/kdflow/token_alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..5a2b1695191df2aaa7798e8cbb0605858775474b --- /dev/null +++ b/kdflow/token_alignment.py @@ -0,0 +1,355 @@ +import re +from itertools import zip_longest +from typing import Callable, List, Optional, Sequence, Tuple + + +PieceToBytes = Callable[[str], bytes] + + +class TokenByteCodec: + """Convert tokenizer vocabulary pieces back to the bytes they represent.""" + + def __init__(self, name: str, piece_to_bytes: PieceToBytes): + self.name = name + self.piece_to_bytes = piece_to_bytes + + def __repr__(self) -> str: + return f"TokenByteCodec(name={self.name!r})" + + +def is_byte_level_tokenizer(tokenizer) -> bool: + """Return whether a fast tokenizer uses a ByteLevel pre-tokenizer/decoder.""" + backend = getattr(tokenizer, "backend_tokenizer", None) + if backend is None: + return False + return "ByteLevel" in repr(getattr(backend, "pre_tokenizer", None)) or ( + "ByteLevel" in repr(getattr(backend, "decoder", None)) + ) + + +def is_sentencepiece_tokenizer(tokenizer) -> bool: + """Return whether tokenizer pieces use SentencePiece whitespace/byte syntax. + + Gemma 3's fast tokenizer is serialized as a Hugging Face BPE backend, but it + retains SentencePiece-compatible pieces: spaces are represented by ``▁`` + and byte fallback pieces by ``<0xNN>``. Consequently the backend model type + alone is not a reliable discriminator. + """ + if getattr(tokenizer, "sp_model", None) is not None: + return True + + tokenizer_name = type(tokenizer).__name__.lower() + if "gemmatokenizer" in tokenizer_name or "sentencepiece" in tokenizer_name: + return True + + backend = getattr(tokenizer, "backend_tokenizer", None) + if backend is None: + return False + components = " ".join( + repr(getattr(backend, name, None)) + for name in ("normalizer", "pre_tokenizer", "decoder", "model") + ) + return "Metaspace" in components or ( + "ByteFallback" in components and "Replace" in components + ) + + +def _bytes_to_unicode() -> dict[int, str]: + """Build the reversible byte-to-unicode alphabet used by ByteLevel BPE.""" + byte_values = list(range(ord("!"), ord("~") + 1)) + byte_values += list(range(ord("¡"), ord("¬") + 1)) + byte_values += list(range(ord("®"), ord("ÿ") + 1)) + unicode_values = byte_values[:] + extra_index = 0 + for byte_value in range(256): + if byte_value not in byte_values: + byte_values.append(byte_value) + unicode_values.append(256 + extra_index) + extra_index += 1 + return dict( + zip(byte_values, [chr(value) for value in unicode_values], strict=True) + ) + + +_BYTE_LEVEL_DECODER = { + character: byte_value for byte_value, character in _bytes_to_unicode().items() +} + + +def byte_level_piece_to_bytes(piece: str) -> bytes: + """Decode a ByteLevel vocabulary piece into the source bytes it represents.""" + output = bytearray() + for character in piece: + byte_value = _BYTE_LEVEL_DECODER.get(character) + if byte_value is None: + # Added tokens may remain literal Unicode even in a ByteLevel tokenizer. + output.extend(character.encode("utf-8")) + else: + output.append(byte_value) + return bytes(output) + + +_SENTENCEPIECE_BYTE_RE = re.compile(r"^<0x([0-9A-Fa-f]{2})>$") +_UNUSED_TOKEN_RE = re.compile(r"^$") + + +def sentencepiece_piece_to_bytes(piece: str) -> bytes: + """Decode a SentencePiece/Gemma vocabulary piece into source bytes.""" + byte_match = _SENTENCEPIECE_BYTE_RE.fullmatch(piece) + if byte_match is not None: + return bytes([int(byte_match.group(1), 16)]) + return piece.replace("▁", " ").encode("utf-8") + + +BYTE_LEVEL_CODEC = TokenByteCodec("byte_level", byte_level_piece_to_bytes) +SENTENCEPIECE_CODEC = TokenByteCodec( + "sentencepiece", sentencepiece_piece_to_bytes +) + + +def get_tokenizer_byte_codec(tokenizer) -> TokenByteCodec: + """Return the byte codec supported by ``byte_align_ctkd`` for a tokenizer.""" + # Check ByteLevel first: class-name fallbacks for SentencePiece tokenizers + # must never change the legacy ByteLevel path. + if is_byte_level_tokenizer(tokenizer): + return BYTE_LEVEL_CODEC + if is_sentencepiece_tokenizer(tokenizer): + return SENTENCEPIECE_CODEC + raise ValueError( + "byte_align_ctkd supports fast ByteLevel tokenizers and " + "SentencePiece-compatible tokenizers (including Gemma 3); got " + f"{type(tokenizer).__name__}" + ) + + +def _special_token_ids(tokenizer) -> set[int]: + special_ids = { + int(token_id) + for token_id in (getattr(tokenizer, "all_special_ids", None) or []) + if token_id is not None + } + added_tokens = getattr(tokenizer, "added_tokens_decoder", None) or {} + for token_id, token in added_tokens.items(): + if getattr(token, "special", False): + special_ids.add(int(token_id)) + return special_ids + + +def _unique_byte_vocab(tokenizer, codec: TokenByteCodec) -> dict[bytes, int]: + """Build an unambiguous content-byte to token-id vocabulary.""" + special_ids = _special_token_ids(tokenizer) + candidates: dict[bytes, set[int]] = {} + for piece, token_id in tokenizer.get_vocab().items(): + token_id = int(token_id) + if token_id in special_ids or _UNUSED_TOKEN_RE.fullmatch(piece): + continue + piece_bytes = codec.piece_to_bytes(piece) + if not piece_bytes: + continue + candidates.setdefault(piece_bytes, set()).add(token_id) + + # Multiple IDs with identical surface bytes do not define a safe logit + # correspondence, so discard collisions instead of selecting arbitrarily. + return { + piece_bytes: next(iter(token_ids)) + for piece_bytes, token_ids in candidates.items() + if len(token_ids) == 1 + } + + +def find_byte_overlap_token_ids( + student_tokenizer, + teacher_tokenizer, + student_codec: Optional[TokenByteCodec] = None, + teacher_codec: Optional[TokenByteCodec] = None, +) -> Tuple[List[int], List[int]]: + """Map content-equivalent vocabulary entries by decoded bytes. + + This mapping is intended only for pairs containing a SentencePiece-style + tokenizer. Existing ByteLevel/ByteLevel pairs keep the legacy token-string + mapping in ``SimpleCrossTokenizerKD``. + """ + student_codec = student_codec or get_tokenizer_byte_codec(student_tokenizer) + teacher_codec = teacher_codec or get_tokenizer_byte_codec(teacher_tokenizer) + student_vocab = _unique_byte_vocab(student_tokenizer, student_codec) + teacher_vocab = _unique_byte_vocab(teacher_tokenizer, teacher_codec) + overlap = sorted(student_vocab.keys() & teacher_vocab.keys()) + student_eos = student_tokenizer.eos_token_id + teacher_eos = teacher_tokenizer.eos_token_id + paired_ids = [ + (student_vocab[piece_bytes], teacher_vocab[piece_bytes]) + for piece_bytes in overlap + ] + paired_ids = [ + (student_id, teacher_id) + for student_id, teacher_id in paired_ids + if student_id != student_eos and teacher_id != teacher_eos + ] + student_ids = [student_id for student_id, _ in paired_ids] + teacher_ids = [teacher_id for _, teacher_id in paired_ids] + + if student_eos is not None and teacher_eos is not None: + student_ids.append(int(student_eos)) + teacher_ids.append(int(teacher_eos)) + return student_ids, teacher_ids + + +def _split_on_eos( + tokens: Sequence[str], eos_token: Optional[str] +) -> List[Tuple[List[Tuple[int, str]], Optional[int]]]: + """Split flattened response tokens into samples while retaining EOS indices.""" + segments = [] + current = [] + for index, token in enumerate(tokens): + if eos_token is not None and token == eos_token: + segments.append((current, index)) + current = [] + else: + current.append((index, token)) + if current or not segments: + segments.append((current, None)) + return segments + + +def _align_segment_1to1( + teacher_segment: Sequence[Tuple[int, str]], + student_segment: Sequence[Tuple[int, str]], + teacher_piece_to_bytes: PieceToBytes = byte_level_piece_to_bytes, + student_piece_to_bytes: PieceToBytes = byte_level_piece_to_bytes, +) -> Tuple[List[int], List[int]]: + teacher_pieces = [ + teacher_piece_to_bytes(token) for _, token in teacher_segment + ] + student_pieces = [ + student_piece_to_bytes(token) for _, token in student_segment + ] + + # Both tokenizers must describe the same completion bytes. If they do not, + # aligning equal-length spans would silently pair unrelated content. + if b"".join(teacher_pieces) != b"".join(student_pieces): + return [], [] + + teacher_ends = [] + student_ends = [] + cursor = 0 + for piece in teacher_pieces: + cursor += len(piece) + teacher_ends.append(cursor) + cursor = 0 + for piece in student_pieces: + cursor += len(piece) + student_ends.append(cursor) + + teacher_aligned = [] + student_aligned = [] + teacher_index = student_index = 0 + teacher_group_start = student_group_start = 0 + + while teacher_index < len(teacher_ends) and student_index < len(student_ends): + teacher_end = teacher_ends[teacher_index] + student_end = student_ends[student_index] + if teacher_end < student_end: + teacher_index += 1 + continue + if teacher_end > student_end: + student_index += 1 + continue + + teacher_group_size = teacher_index - teacher_group_start + 1 + student_group_size = student_index - student_group_start + 1 + if teacher_group_size == 1 and student_group_size == 1: + teacher_piece = teacher_pieces[teacher_index] + student_piece = student_pieces[student_index] + if teacher_piece == student_piece and teacher_piece: + teacher_aligned.append(teacher_segment[teacher_index][0]) + student_aligned.append(student_segment[student_index][0]) + + teacher_index += 1 + student_index += 1 + teacher_group_start = teacher_index + student_group_start = student_index + + return teacher_aligned, student_aligned + + +def align_byte_spans_1to1( + teacher_tokens: Sequence[str], + student_tokens: Sequence[str], + teacher_eos_token: Optional[str], + student_eos_token: Optional[str], + teacher_piece_to_bytes: PieceToBytes = byte_level_piece_to_bytes, + student_piece_to_bytes: PieceToBytes = byte_level_piece_to_bytes, +) -> Tuple[List[int], List[int]]: + """Align only 1:1 decoded-byte token spans, treating EOS as a boundary. + + Content groups that are 1:N, N:1, or N:M are discarded. If both sides have + EOS at the end of a paired sample, those EOS positions are aligned directly + and the byte coordinate is reset for the next sample. + """ + teacher_segments = _split_on_eos(teacher_tokens, teacher_eos_token) + student_segments = _split_on_eos(student_tokens, student_eos_token) + teacher_aligned = [] + student_aligned = [] + + for teacher_item, student_item in zip_longest( + teacher_segments, student_segments + ): + if teacher_item is None or student_item is None: + break + teacher_segment, teacher_eos_index = teacher_item + student_segment, student_eos_index = student_item + segment_teacher_aligned, segment_student_aligned = _align_segment_1to1( + teacher_segment, + student_segment, + teacher_piece_to_bytes=teacher_piece_to_bytes, + student_piece_to_bytes=student_piece_to_bytes, + ) + teacher_aligned.extend(segment_teacher_aligned) + student_aligned.extend(segment_student_aligned) + + if teacher_eos_index is not None and student_eos_index is not None: + teacher_aligned.append(teacher_eos_index) + student_aligned.append(student_eos_index) + + return teacher_aligned, student_aligned + + +def align_byte_span_batch_1to1( + teacher_batch_tokens: Sequence[Sequence[str]], + student_batch_tokens: Sequence[Sequence[str]], + teacher_eos_token: Optional[str], + student_eos_token: Optional[str], + teacher_piece_to_bytes: PieceToBytes = byte_level_piece_to_bytes, + student_piece_to_bytes: PieceToBytes = byte_level_piece_to_bytes, +) -> Tuple[List[int], List[int]]: + """Align each sample separately and return indices into flattened tensors.""" + if len(teacher_batch_tokens) != len(student_batch_tokens): + raise ValueError( + "Teacher and student micro-batches must contain the same number of samples." + ) + + teacher_aligned = [] + student_aligned = [] + teacher_offset = 0 + student_offset = 0 + for teacher_tokens, student_tokens in zip( + teacher_batch_tokens, student_batch_tokens, strict=True + ): + row_teacher_aligned, row_student_aligned = align_byte_spans_1to1( + teacher_tokens=teacher_tokens, + student_tokens=student_tokens, + teacher_eos_token=teacher_eos_token, + student_eos_token=student_eos_token, + teacher_piece_to_bytes=teacher_piece_to_bytes, + student_piece_to_bytes=student_piece_to_bytes, + ) + teacher_aligned.extend( + teacher_offset + index for index in row_teacher_aligned + ) + student_aligned.extend( + student_offset + index for index in row_student_aligned + ) + teacher_offset += len(teacher_tokens) + student_offset += len(student_tokens) + + return teacher_aligned, student_aligned diff --git a/kdflow/trainer/__init__.py b/kdflow/trainer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c7fa5b1ce3c01a42c60f2f070370cd978a73710b --- /dev/null +++ b/kdflow/trainer/__init__.py @@ -0,0 +1,15 @@ +from kdflow.trainer.on_policy_kd_trainer import OnPolicyKDTrainer +from kdflow.trainer.off_policy_kd_trainer import OffPolicyKDTrainer +from kdflow.trainer.rollout_manager import RolloutManager +from kdflow.trainer.alfworld_rollout_manager import AlfWorldRolloutManager +from kdflow.trainer.verl_agent_alfworld_rollout_manager import ( + VerlAgentAlfWorldRolloutManager, +) + +__all__ = [ + "OffPolicyKDTrainer", + "OnPolicyKDTrainer", + "RolloutManager", + "AlfWorldRolloutManager", + "VerlAgentAlfWorldRolloutManager", +] diff --git a/kdflow/trainer/alfworld_rollout_manager.py b/kdflow/trainer/alfworld_rollout_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..9e353bcbfd9cb9424f2110f3cf1c9e4b2ff496ff --- /dev/null +++ b/kdflow/trainer/alfworld_rollout_manager.py @@ -0,0 +1,454 @@ +import os +import random +import re +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from kdflow.trainer.rollout_manager import RolloutManager +from kdflow.utils.logging_utils import init_logger + + +logger = init_logger(__name__) + + +ALFWORLD_SYSTEM_PROMPT = """You are an embodied agent solving a household task in the ALFWorld text environment. +You receive an Observation and a list of Admissible commands. On every turn reply with exactly two lines: +Thought: +Action: +Only the Action line is executed. Do not write anything after the Action line.""" + + +class AlfWorldRolloutManager(RolloutManager): + """Collect batched multi-turn ALFWorld trajectories from the student. + + Every assistant turn is emitted as an independent prompt/response sample. + Consequently the existing cross-tokenizer processor masks only that turn's + response, and byte alignment can never cross an environment boundary. + """ + + FAILURE_MARKERS = ( + "nothing happens", + "not a valid command", + "that's not a verb i recognise", + "you can't", + "you cannot", + ) + + def __init__( + self, + strategy, + rollout_group, + is_same_tokenizer: bool, + generate_kwargs: Optional[Dict[str, Any]] = None, + env=None, + ): + super().__init__( + strategy=strategy, + rollout_group=rollout_group, + is_same_tokenizer=is_same_tokenizer, + generate_kwargs=generate_kwargs, + ) + self.strategy = strategy + self.max_turns = self.args.rollout.alfworld_max_turns + self.admissible_limit = self.args.rollout.alfworld_admissible_limit + self.batch_size = self.args.rollout.rollout_batch_size + self.env = env if env is not None else self._build_env() + + def _build_env(self): + config_path = os.path.abspath(self.args.rollout.alfworld_config) + try: + import yaml + from alfworld.agents.environment import get_environment + from alfworld.info import ALFWORLD_DATA + except ImportError as error: + raise ImportError( + "ALFWorld multi-turn rollout requires the text environment. " + "Install it with `pip install -e '.[alfworld]'` and run " + "`alfworld-download` before training." + ) from error + + with open(config_path, "r", encoding="utf-8") as stream: + config = yaml.safe_load(stream) + os.environ.setdefault("ALFWORLD_DATA", str(ALFWORLD_DATA)) + + # AlfredTWEnv reads the episode cap from the selected training-method + # section. DAgger is used only to select that config path; KDFlow never + # consumes ALFWorld's expert action. + config.setdefault("general", {})["training_method"] = "dagger" + seed = self.args.train.seed + config["general"]["random_seed"] = seed + random.seed(seed) + config.setdefault("dagger", {}).setdefault("training", {})[ + "max_nb_steps_per_episode" + ] = self.max_turns + + env_type = config.get("env", {}).get("type", "AlfredTWEnv") + if env_type != "AlfredTWEnv": + raise ValueError( + "The first multi-turn KD implementation supports the text-only " + f"AlfredTWEnv, got {env_type!r}." + ) + split = self.args.rollout.alfworld_split + logger.info( + "Initializing ALFWorld: config=%s split=%s batch_size=%d max_turns=%d", + config_path, + split, + self.batch_size, + self.max_turns, + ) + env = get_environment(env_type)(config, train_eval=split).init_env( + batch_size=self.batch_size + ) + seed_env = getattr(env, "seed", None) + if callable(seed_env): + seed_env(seed) + return env + + @staticmethod + def parse_action( + response: str, admissible_commands: Sequence[str] + ) -> Tuple[str, bool]: + """Return an executable ALFWorld command and whether parsing failed.""" + matches = re.findall( + r"(?im)^\s*action\s*:\s*(.*?)\s*$", response or "" + ) + candidate = matches[-1] if matches else (response or "").strip() + candidate = re.sub(r"", "", candidate, flags=re.IGNORECASE) + candidate = candidate.splitlines()[0].strip() if candidate else "" + + canonical = { + str(command).strip().casefold(): str(command).strip() + for command in admissible_commands + } + if candidate.casefold() in canonical: + return canonical[candidate.casefold()], False + if candidate: + # TextWorld accepts free-form commands and returns deterministic + # feedback for invalid ones, which is preferable to changing the + # student's state trajectory behind its back. + return candidate, True + # Empty strings are not accepted uniformly across TextWorld versions. + # "look" safely advances the batched environment while the sample is + # still marked as a deterministic parser failure. + return "look", True + + def _render_prompt(self, messages: List[Dict[str, str]], processor) -> str: + template_owner = processor + if not hasattr(template_owner, "apply_chat_template"): + template_owner = getattr(processor, "tokenizer", processor) + kwargs = {} + if "enable_thinking" in str( + getattr(template_owner, "chat_template", "") + ): + kwargs["enable_thinking"] = self.args.model.enable_thinking + return template_owner.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + **kwargs, + ) + + @staticmethod + def _tokenizer(processor): + return getattr(processor, "tokenizer", processor) + + def _fits_context(self, student_prompt: str, teacher_prompt: str) -> bool: + prompt_budget = self.args.data.max_len - int( + self.generate_kwargs["max_new_tokens"] + ) + student_len = len( + self._tokenizer(self.data_processor.student_processor).encode( + student_prompt, add_special_tokens=False + ) + ) + teacher_processor = self.data_processor._get_teacher_processor() + teacher_len = len( + self._tokenizer(teacher_processor).encode( + teacher_prompt, add_special_tokens=False + ) + ) + return student_len <= prompt_budget and teacher_len <= prompt_budget + + def _format_observation( + self, + observation: str, + admissible_commands: Sequence[str], + initial: bool = False, + ) -> str: + commands = [str(command) for command in admissible_commands] + shown = commands + suffix = "" + if self.admissible_limit > 0 and len(commands) > self.admissible_limit: + shown = commands[: self.admissible_limit] + suffix = f" (+{len(commands) - len(shown)} more)" + heading = "Task and initial observation" if initial else "Observation" + return ( + f"{heading}:\n{observation}\n" + f"Admissible commands{suffix}: {'; '.join(shown)}" + ) + + @staticmethod + def _info_list(info: Dict[str, Any], key: str, batch_size: int, default): + values = info.get(key) if isinstance(info, dict) else None + if values is None: + return [default for _ in range(batch_size)] + return list(values) + + def collect_episodes( + self, + prompt_batch: List[Dict[str, Any]], + global_step: int, + mode: str = "train", + **kwargs, + ) -> tuple[List[dict], Dict[str, float]]: + """Collect raw ALFWorld turns without running teacher or student updates. + + This is the shared environment loop used by both on-policy training and + the standalone evaluator. ``prompt_batch`` is an epoch clock only; the + ALFWorld environment chooses the actual games when it is reset. + """ + if mode not in ("train", "eval"): + raise ValueError(f"Unsupported ALFWorld rollout mode: {mode!r}") + if len(prompt_batch) != self.batch_size: + raise ValueError( + "ALFWorld requires a full fixed-size rollout batch: expected " + f"{self.batch_size}, got {len(prompt_batch)}." + ) + + metric_prefix = "rollout" if mode == "train" else "eval" + + should_sleep = self.args.train.enable_sleep + if should_sleep: + self.rollout_group.wakeup() + + try: + observations, info = self.env.reset() + observations = list(observations) + admissible = self._info_list( + info, "admissible_commands", self.batch_size, [] + ) + student_messages = [] + teacher_messages = [] + for index in range(self.batch_size): + initial_user = self._format_observation( + observations[index], admissible[index], initial=True + ) + messages = [ + {"role": "system", "content": ALFWORLD_SYSTEM_PROMPT}, + {"role": "user", "content": initial_user}, + ] + student_messages.append(list(messages)) + teacher_messages.append(list(messages)) + + active = [True] * self.batch_size + turn_counts = [0] * self.batch_size + final_won = [False] * self.batch_size + context_truncated = [False] * self.batch_size + invalid_actions = 0 + environment_failures = 0 + records = [] + timing_sums: Dict[str, float] = {} + + sampling_params = kwargs or self.generate_kwargs + teacher_processor = self.data_processor._get_teacher_processor() + + for turn_id in range(self.max_turns): + request_indices = [] + stu_prompts = [] + tea_prompts = [] + for index in range(self.batch_size): + if not active[index]: + continue + stu_prompt = self._render_prompt( + student_messages[index], + self.data_processor.student_processor, + ) + tea_prompt = self._render_prompt( + teacher_messages[index], teacher_processor + ) + if not self._fits_context(stu_prompt, tea_prompt): + active[index] = False + context_truncated[index] = True + continue + request_indices.append(index) + stu_prompts.append(stu_prompt) + tea_prompts.append(tea_prompt) + + if not request_indices: + break + + outputs, timing = self._generate(stu_prompts, sampling_params) + for key, value in timing.items(): + timing_sums[key] = timing_sums.get(key, 0.0) + float(value) + + actions = ["look"] * self.batch_size + pending = {} + for local_index, episode_index in enumerate(request_indices): + output = outputs[local_index] + action, parser_failed = self.parse_action( + output.get("text", ""), admissible[episode_index] + ) + actions[episode_index] = action + invalid_actions += int(parser_failed) + pending[episode_index] = { + "stu_prompt": stu_prompts[local_index], + "tea_prompt": tea_prompts[local_index], + "output": output, + "action": action, + "parser_failed": parser_failed, + } + + next_observations, _, dones, next_info = self.env.step(actions) + next_observations = list(next_observations) + dones = list(dones) + next_admissible = self._info_list( + next_info, "admissible_commands", self.batch_size, [] + ) + won = self._info_list(next_info, "won", self.batch_size, False) + gamefiles = self._info_list( + next_info, "extra.gamefile", self.batch_size, "" + ) + + for episode_index, item in pending.items(): + feedback = str(next_observations[episode_index]) + env_failed = any( + marker in feedback.casefold() + for marker in self.FAILURE_MARKERS + ) + environment_failures += int(env_failed) + turn_counts[episode_index] += 1 + final_won[episode_index] = bool(won[episode_index]) + records.append( + { + **item, + "metadata": { + "episode_id": f"{global_step}:{episode_index}", + "turn_id": turn_id, + "alfworld_action": item["action"], + "parser_failed": bool(item["parser_failed"]), + "environment_failed": bool(env_failed), + "deterministic_failure": bool( + item["parser_failed"] or env_failed + ), + "episode_done": bool(dones[episode_index]), + "episode_won": bool(won[episode_index]), + "gamefile": str(gamefiles[episode_index]), + }, + } + ) + + if dones[episode_index]: + active[episode_index] = False + continue + assistant_message = { + "role": "assistant", + "content": item["output"].get("text", ""), + } + next_user = { + "role": "user", + "content": self._format_observation( + feedback, + next_admissible[episode_index], + initial=False, + ), + } + student_messages[episode_index].extend( + [assistant_message, next_user] + ) + teacher_messages[episode_index].extend( + [dict(assistant_message), dict(next_user)] + ) + + observations = next_observations + admissible = next_admissible + if not any(active): + break + + if not records: + return [], { + f"{metric_prefix}/alfworld/episodes": float(self.batch_size), + f"{metric_prefix}/alfworld/episodes_succeeded": 0.0, + f"{metric_prefix}/alfworld/turns/total": 0.0, + f"{metric_prefix}/alfworld/context_truncated_count": float( + self.batch_size + ), + f"{metric_prefix}/alfworld/context_truncated_ratio": 1.0, + } + + total_turns = sum(turn_counts) + metrics = dict(timing_sums) + metrics.update( + { + f"{metric_prefix}/alfworld/episodes": float(self.batch_size), + f"{metric_prefix}/alfworld/episodes_succeeded": float( + sum(final_won) + ), + f"{metric_prefix}/alfworld/success_rate": ( + sum(final_won) / self.batch_size + ), + f"{metric_prefix}/alfworld/turns/total": float(total_turns), + f"{metric_prefix}/alfworld/turns/mean": ( + total_turns / self.batch_size + ), + f"{metric_prefix}/alfworld/turns/max": float( + max(turn_counts) + ), + f"{metric_prefix}/alfworld/invalid_action_count": float( + invalid_actions + ), + f"{metric_prefix}/alfworld/invalid_action_ratio": ( + invalid_actions / max(total_turns, 1) + ), + f"{metric_prefix}/alfworld/environment_failure_count": float( + environment_failures + ), + f"{metric_prefix}/alfworld/environment_failure_ratio": ( + environment_failures / max(total_turns, 1) + ), + f"{metric_prefix}/alfworld/context_truncated_count": float( + sum(context_truncated) + ), + f"{metric_prefix}/alfworld/context_truncated_ratio": ( + sum(context_truncated) / self.batch_size + ), + } + ) + return records, metrics + finally: + if should_sleep: + self.rollout_group.sleep() + + def rollout( + self, + prompt_batch: List[Dict[str, Any]], + global_step: int, + mode: str = "train", + **kwargs, + ) -> tuple[List[dict], Dict[str, float]]: + """Collect ALFWorld turns and convert them to KD/evaluation batches.""" + records, environment_metrics = self.collect_episodes( + prompt_batch=prompt_batch, + global_step=global_step, + mode=mode, + **kwargs, + ) + if not records: + return [], environment_metrics + + sampling_params = kwargs or self.generate_kwargs + micro_batches, rollout_metrics = self.data_processor.process( + stu_prompts=[record["stu_prompt"] for record in records], + tea_prompts=[record["tea_prompt"] for record in records], + outputs=[record["output"] for record in records], + labels=[record["metadata"]["episode_won"] for record in records], + sampling_params=sampling_params, + global_step=global_step, + mode=mode, + sample_metadata=[record["metadata"] for record in records], + ) + rollout_metrics.update(environment_metrics) + return micro_batches, rollout_metrics + + def close(self) -> None: + close = getattr(self.env, "close", None) + if callable(close): + close() diff --git a/kdflow/trainer/data_processor/__init__.py b/kdflow/trainer/data_processor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d267c43e5cf7c929a03867968ff8c794a5a574bf --- /dev/null +++ b/kdflow/trainer/data_processor/__init__.py @@ -0,0 +1,3 @@ +from kdflow.trainer.data_processor.rollout_data_processor import RolloutDataProcessor + +__all__ = ["RolloutDataProcessor"] diff --git a/kdflow/trainer/data_processor/rollout_data_processor.py b/kdflow/trainer/data_processor/rollout_data_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..7f5564dcd3046b5be30cbabc13b06c3dd393c17a --- /dev/null +++ b/kdflow/trainer/data_processor/rollout_data_processor.py @@ -0,0 +1,378 @@ +import json +import os +from typing import Any, Dict, List, Optional + +import torch + +from kdflow.datasets.utils import get_tokenizer_or_processor +from kdflow.utils.utils import zero_pad_sequences + + +class RolloutDataProcessor: + """Convert raw rollout outputs into training micro-batches.""" + + CONDITIONAL_EOS_ALGORITHMS = frozenset( + { + "byte_align_ctkd", + "topk_byte_align_ctkd", + } + ) + + def __init__(self, strategy, is_same_tokenizer: bool): + self.args = strategy.args + self.is_same_tokenizer = is_same_tokenizer + self.image_key = getattr(self.args.data, "image_key", None) + + self.student_processor = get_tokenizer_or_processor( + self.args.model.student_name_or_path, + need_processor=self.image_key is not None, + ) + self.teacher_processors = {} + if self.args.model.teacher_name_or_path and not self.is_same_tokenizer: + self.teacher_processors["default"] = get_tokenizer_or_processor( + self.args.model.teacher_name_or_path, + need_processor=self.image_key is not None, + ) + + def process( + self, + stu_prompts: List[str], + tea_prompts: List[str], + outputs: List[Dict[str, Any]], + labels: List[Any], + sampling_params: Dict[str, Any], + global_step: int, + mode: str, + images: Optional[List] = None, + sample_metadata: Optional[List[Dict[str, Any]]] = None, + ) -> tuple[List[dict], Dict[str, float]]: + """Save, tokenize and collate raw rollout outputs.""" + if sample_metadata is not None and len(sample_metadata) != len(outputs): + raise ValueError("sample_metadata and outputs must have the same length") + self._save_rollout_data( + stu_prompts, + outputs, + labels, + global_step, + mode, + sample_metadata=sample_metadata, + ) + + kd_algorithm = self.args.kd.kd_algorithm + uses_conditional_eos = kd_algorithm in self.CONDITIONAL_EOS_ALGORITHMS + truncated_flags = ( + [ + self._is_length_truncated(output, sampling_params) + for output in outputs + ] + if uses_conditional_eos + else [False] * len(outputs) + ) + + sample_list = [ + self._build_rollout_sample( + stu_prompt=stu_prompts[index], + tea_prompt=tea_prompts[index], + output=output, + label=labels[index], + images=images[index] if images and images[index] else None, + append_eos=( + not truncated_flags[index] if uses_conditional_eos else True + ), + metadata=( + sample_metadata[index] if sample_metadata is not None else None + ), + ) + for index, output in enumerate(outputs) + ] + rollout_metrics = self._compute_rollout_metrics( + sample_list, sampling_params, mode + ) + + if ( + mode == "train" + and self.args.rollout.print_rollout_sample + and sample_list + ): + print(sample_list[0]["stu_prompts"][0] + sample_list[0]["stu_responses"][0]) + + micro_batches = self._collate_micro_batches( + sample_list, self.args.train.micro_train_batch_size + ) + return micro_batches, rollout_metrics + + def _save_rollout_data( + self, + prompts: List[str], + outputs: List[Dict[str, Any]], + labels: List[Any], + global_step: int, + mode: str, + sample_metadata: Optional[List[Dict[str, Any]]] = None, + ) -> None: + rollout_dir = os.path.join(self.args.train.save_path, "rollout_data") + if mode == "eval": + rollout_dir = os.path.join(rollout_dir, "val") + os.makedirs(rollout_dir, exist_ok=True) + + rollout_path = os.path.join(rollout_dir, f"{global_step}.jsonl") + with open(rollout_path, "w") as file: + metadata_rows = sample_metadata or [{} for _ in outputs] + for prompt, output, label, metadata in zip( + prompts, outputs, labels, metadata_rows + ): + rollout_record = {"prompt": prompt, "output": output["text"]} + rollout_record.update(metadata) + finish_reason = self._extract_finish_reason(output) + if finish_reason is not None: + rollout_record["finish_reason"] = finish_reason + if "reward_result" in output: + rollout_record["reward_result"] = output["reward_result"] + if label is not None and label != "": + rollout_record["label"] = label + file.write(json.dumps(rollout_record, ensure_ascii=False) + "\n") + + @staticmethod + def _extract_finish_reason(output: Dict[str, Any]): + meta_info = output.get("meta_info") or {} + finish_reason = output.get("finish_reason") + return ( + finish_reason + if finish_reason is not None + else meta_info.get("finish_reason") + ) + + @classmethod + def _is_length_truncated( + cls, + output: Dict[str, Any], + sampling_params: Dict[str, Any], + ) -> bool: + finish_reason = cls._extract_finish_reason(output) + if isinstance(finish_reason, dict): + reason_type = ( + finish_reason.get("type") + or finish_reason.get("reason") + or finish_reason.get("finish_reason") + ) + else: + reason_type = finish_reason + if isinstance(reason_type, str): + normalized_reason = reason_type.lower().replace("-", "_") + if normalized_reason in { + "length", + "max_length", + "max_tokens", + "max_new_tokens", + }: + return True + + max_new_tokens = sampling_params.get("max_new_tokens") + return ( + finish_reason is None + and max_new_tokens is not None + and len(output.get("output_ids", ())) >= max_new_tokens + ) + + @staticmethod + def _compute_rollout_metrics( + sample_list: List[Dict[str, Any]], + sampling_params: Dict[str, Any], + mode: str, + ) -> Dict[str, float]: + length_names = ("prompt_length", "response_length", "total_length") + length_values = { + name: [sample.pop(name).item() for sample in sample_list] + for name in length_names + } + metric_prefix = "rollout" if mode == "train" else "eval" + rollout_metrics = {} + for name, values in length_values.items(): + rollout_metrics[f"{metric_prefix}/{name}/mean"] = sum(values) / len(values) + rollout_metrics[f"{metric_prefix}/{name}/max"] = max(values) + + max_response_length = sampling_params["max_new_tokens"] + response_lengths = length_values["response_length"] + rollout_metrics[f"{metric_prefix}/response_clip_ratio"] = sum( + length >= max_response_length for length in response_lengths + ) / len(response_lengths) + return rollout_metrics + + @staticmethod + def _collate_values(values: list): + value = values[0] + if isinstance(value, torch.Tensor): + return zero_pad_sequences(values, side="right", value=0) + if isinstance(value, list): + return sum(values, []) + if value is None: + return None + return values + + def _collate_micro_batches( + self, sample_list: List[Dict], batch_size: int + ) -> List[Dict]: + """Collate single samples into micro-batches.""" + micro_batch_list = [] + for index in range(0, len(sample_list), batch_size): + batch_samples = sample_list[index : index + batch_size] + micro_batch = { + key: self._collate_values([sample[key] for sample in batch_samples]) + for key in batch_samples[0] + } + micro_batch_list.append(micro_batch) + return micro_batch_list + + def _tokenize_sample( + self, + prompt: str, + response: str, + processor, + prefix: str, + images=None, + append_eos: bool = True, + ) -> Dict[str, Any]: + """Tokenize prompt and response for a single sample.""" + tokenizer = getattr(processor, "tokenizer", processor) + response_tokens = tokenizer( + response, return_tensors="pt", add_special_tokens=False + ) + response_length = response_tokens["input_ids"].shape[1] + + full_input = {"text": prompt + response} + if images: + full_input["images"] = images + full_tokens = processor( + **full_input, return_tensors="pt", add_special_tokens=False + ) + prompt_length = full_tokens["input_ids"].shape[1] - response_length + + if append_eos: + eos_token_id = tokenizer.eos_token_id + input_ids = torch.cat( + [ + full_tokens["input_ids"][0], + full_tokens["input_ids"][0].new_tensor([eos_token_id]), + ] + ) + attention_mask = torch.cat( + [ + full_tokens["attention_mask"][0], + full_tokens["attention_mask"][0].new_ones(1), + ] + ) + else: + input_ids = full_tokens["input_ids"][0] + attention_mask = full_tokens["attention_mask"][0] + + eos_length = int(append_eos) + loss_mask = torch.tensor( + [False] * (prompt_length - 1) + + [True] * (response_length + eos_length) + + [False], + device=input_ids.device, + ) + + result = { + f"{prefix}_input_ids": input_ids, + f"{prefix}_attn_mask": attention_mask, + f"{prefix}_loss_mask": loss_mask, + f"_{prefix}_prompt_length": prompt_length, + } + multi_modal_inputs = { + key: torch.as_tensor(value) + for key, value in full_tokens.items() + if key not in ("input_ids", "attention_mask", "mm_token_type_ids") + } + if multi_modal_inputs: + result[f"_{prefix}_multi_modal_inputs"] = multi_modal_inputs + return result + + def _get_teacher_processor(self): + """Get the configured teacher processor.""" + if "default" in self.teacher_processors: + return self.teacher_processors["default"] + return self.student_processor + + def _build_rollout_sample( + self, + stu_prompt: str, + tea_prompt: str, + output: Dict[str, Any], + label: Any, + images=None, + append_eos: bool = True, + metadata: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """Build a rollout sample with student and teacher tokenizations.""" + response_ids = output["output_ids"] + response_text = output["text"] + + stu_tokens = self._tokenize_sample( + stu_prompt, + response_text, + self.student_processor, + "stu", + images=images, + append_eos=append_eos, + ) + + if not self.is_same_tokenizer or tea_prompt != stu_prompt: + teacher_processor = self._get_teacher_processor() + tea_tokens = self._tokenize_sample( + tea_prompt, + response_text, + teacher_processor, + "tea", + images=images, + append_eos=append_eos, + ) + else: + tea_tokens = { + "tea_input_ids": stu_tokens["stu_input_ids"].clone(), + "tea_attn_mask": stu_tokens["stu_attn_mask"].clone(), + "tea_loss_mask": stu_tokens["stu_loss_mask"].clone(), + } + + prompt_length = stu_tokens["_stu_prompt_length"] + response_length = len(response_ids) + total_length = stu_tokens["stu_attn_mask"].sum().item() + + teacher_processor = self._get_teacher_processor() + tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor) + if images: + feed_ids = tokenizer( + tea_prompt + response_text, add_special_tokens=False + )["input_ids"] + tea_feed_input_ids = ( + feed_ids + [tokenizer.eos_token_id] if append_eos else feed_ids + ) + else: + tea_feed_input_ids = tea_tokens["tea_input_ids"].tolist() + + sample = { + **{key: value for key, value in tea_tokens.items() if not key.startswith("_")}, + **{key: value for key, value in stu_tokens.items() if not key.startswith("_")}, + "tea_feed_input_ids": [tea_feed_input_ids], + "rollout_log_probs": None, + "stu_prompts": [stu_prompt], + "stu_responses": [response_text], + "tea_prompts": [tea_prompt], + "labels": [label], + "prompt_length": torch.FloatTensor([[prompt_length]]), + "response_length": torch.FloatTensor([[response_length]]), + "total_length": torch.FloatTensor([[total_length]]), + } + stu_multi_modal_inputs = stu_tokens.get("_stu_multi_modal_inputs") + if stu_multi_modal_inputs is not None: + sample["stu_multi_modal_inputs"] = [stu_multi_modal_inputs] + if images: + sample["images"] = [images] + if metadata: + reserved = set(sample) & set(metadata) + if reserved: + raise ValueError( + f"Rollout metadata uses reserved sample keys: {sorted(reserved)}" + ) + sample.update(metadata) + return sample diff --git a/kdflow/trainer/off_policy_kd_trainer.py b/kdflow/trainer/off_policy_kd_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..72ab1017c9edfbf90778208b9d01ca123d1d72ee --- /dev/null +++ b/kdflow/trainer/off_policy_kd_trainer.py @@ -0,0 +1,318 @@ +import os +import time +import json +import math +from datetime import timedelta +from typing import Dict, List, Optional, Callable, Any +from collections import defaultdict + +import ray +import torch +import torch.distributed as dist +from tqdm import tqdm + +from kdflow.utils.logging_utils import define_wandb_metrics, init_logger, log_eval_metrics +from kdflow.utils.dynamic_bsz import rearrange_global_batch + + +logger = init_logger(__name__) + + +class OffPolicyKDTrainer: + """ + Ray-based trainer for off-policy knowledge distillation. + """ + + def __init__( + self, + strategy, + student_model, + teacher_model, + train_dataloader, + eval_dataloader=None, + max_steps: int = None, + num_update_steps_per_epoch: int = None, + student_tokenizer=None, + teacher_tokenizer=None, + ) -> None: + """ + Initialize the trainer. + + Args: + strategy: Training strategy containing configuration + student_model: StudentActorGroup + teacher_model: TeacherActorGroup + train_dataloader: Training data loader + eval_dataloader: Evaluation data loader (optional) + max_steps: Maximum training steps + num_update_steps_per_epoch: Number of update steps per epoch + """ + self.strategy = strategy + self.args = strategy.args + self.student = student_model + self.teacher = teacher_model + self.train_dataloader = train_dataloader + self.eval_dataloader = eval_dataloader + self.num_update_steps_per_epoch = num_update_steps_per_epoch + self.max_steps = ( + max_steps + if max_steps is not None and max_steps > 0 + else self.args.train.num_epochs * self.num_update_steps_per_epoch + ) + self.student_tokenizer = student_tokenizer + self.teacher_tokenizer = teacher_tokenizer + self.epochs = math.ceil( + self.max_steps / self.num_update_steps_per_epoch + ) + self.world_size = self.args.train.num_nodes * self.args.train.num_gpus_per_node + self.dp_size = self.world_size // self.args.model.ring_attn_size + + self.log_state = defaultdict(list) + self._init_loggers() + + if self.eval_dataloader and self.args.train.eval_steps < float("inf"): + assert ( + self.args.train.eval_steps >= self.args.kd.teacher_forward_n_batches + and self.args.train.eval_steps % self.args.kd.teacher_forward_n_batches == 0 + ), ( + "`eval_steps` must be a multiple of `teacher_forward_n_batches` " + f"and no smaller than it, but got eval_steps={self.args.train.eval_steps}, " + f"teacher_forward_n_batches={self.args.kd.teacher_forward_n_batches}." + ) + + + def _init_loggers(self) -> None: + """Initialize wandb loggers.""" + self._wandb = None + + if self.args.log.use_wandb: + import wandb + + self._wandb = wandb + if self.args.log.wandb_mode != "offline" and not wandb.api.api_key: + wandb.login() + wandb.init( + entity=self.args.log.wandb_org, + project=self.args.log.wandb_project, + group=self.args.log.wandb_group, + name=self.args.log.wandb_run_name, + config=vars(self.args), + reinit=True, + mode=self.args.log.wandb_mode, + dir=self.args.log.wandb_dir, + ) + + define_wandb_metrics(wandb) + + def _print_training_config(self) -> None: + """Log training configuration before training starts.""" + total_steps = self.max_steps + num_data = len(getattr(self.train_dataloader, "dataset", self.train_dataloader)) + grad_accum = self.args.train.train_batch_size * self.args.model.ring_attn_size \ + // (self.args.train.micro_train_batch_size * self.args.train.num_nodes * self.args.train.num_gpus_per_node) + + def log_config(name, value): + logger.info(f" {name:<32} {value}") + + logger.info("******* Start Training *******") + log_config("Num GPUs:", self.world_size) + log_config("Num Data:", num_data) + log_config("Num Epochs:", self.epochs) + log_config("Train Batch Size:", self.args.train.train_batch_size) + log_config("Steps Per Epoch:", self.num_update_steps_per_epoch) + log_config("Total Training Steps:", total_steps) + if self.args.train.use_dynamic_bsz: + log_config("Enable Dynamic Batch Size:", self.args.train.use_dynamic_bsz) + log_config("Max Token Len Per GPU:", self.args.train.max_token_len_per_gpu) + log_config("Gradient Accumulation:", "dynamic") + else: + log_config("Per-device Batch Size:", self.args.train.micro_train_batch_size) + log_config("Gradient Accumulation:", grad_accum) + log_config("Learning Rate:", self.args.train.learning_rate) + log_config("KD Algorithm:", self.args.kd.kd_algorithm) + log_config("KD Loss Function:", self.args.kd.kd_loss_fn) + if self.args.kd.kd_algorithm == "topk_byte_align_ctkd": + log_config("CTKD Student Top-K:", self.args.kd.ctkd_topk) + + def fit(self, global_step=0, start_epoch=0): + self.global_step = global_step + + self._print_training_config() + + self.start_time = time.time() + num_micro_batches = self.args.train.train_batch_size // self.args.train.micro_train_batch_size + self.teacher_forward_n = min(self.args.kd.teacher_forward_n_batches, len(self.train_dataloader)) + teacher_forward_n = self.teacher_forward_n + + if self.eval_dataloader is not None and self.args.train.eval_steps < float("inf") and self.global_step == 0: + self.strategy.log(f"Start evaluating at global step {self.global_step}") + self.evaluate() + + for epoch in range(start_epoch, self.epochs): + self.current_epoch = epoch + self.train_dataloader.sampler.set_epoch(epoch) + + data_iter = iter(self.train_dataloader) + is_epoch_finished = False + while True: + if self.global_step >= self.max_steps: + break + step_group_start = time.time() + all_global_batches = [] + steps_to_collect = min( + teacher_forward_n, + self.max_steps - self.global_step, + ) + for _ in range(steps_to_collect): + global_batch = [] + try: + for _ in range(num_micro_batches): + micro_batch = next(data_iter) + global_batch.append(micro_batch) + except StopIteration: + is_epoch_finished = True + break + if not is_epoch_finished: + if self.args.train.use_dynamic_bsz: + global_batch = rearrange_global_batch( + global_batch, + max_token_len=self.args.train.max_token_len_per_gpu, + dp_size=self.dp_size, + ) + global_batch_token_num = sum(mb["stu_loss_mask"].sum() for mb in global_batch) + avg_micro_batch_token_num = global_batch_token_num / len(global_batch) + for mb in global_batch: + mb["avg_micro_batch_token_num"] = avg_micro_batch_token_num + all_global_batches.append(global_batch) + + if not all_global_batches: + break + + # ===== Teacher Phase (batch N global batches) ===== + teacher_start = time.time() + if self.args.train.enable_sleep: + self.teacher.wakeup() + + merged_batch = [mb for gb in all_global_batches for mb in gb] + merged_batch = self.teacher.forward(merged_batch) + idx = 0 + for i, gb in enumerate(all_global_batches): + all_global_batches[i] = merged_batch[idx:idx + len(gb)] + idx += len(gb) + if self.args.train.enable_sleep: + self.teacher.sleep() + + teacher_step_fwd_time = (time.time() - teacher_start) / len(all_global_batches) + + # ===== Student Phase (train N steps) ===== + if self.args.train.enable_sleep: + self.student.wakeup() + shared_step_time = (time.time() - step_group_start) / len(all_global_batches) + for global_batch in all_global_batches: + student_start = time.time() + self.global_step += 1 + status_list = ray.get(self.student.async_run_distill(global_batch)) + student_step_train_time = time.time() - student_start + for k in status_list[0].keys(): + self.log_state[k].append(sum(s[k] for s in status_list) / len(status_list)) + self.log_state["timing/teacher_forward_time"].append(teacher_step_fwd_time) + self.log_state["timing/student_train"].append(student_step_train_time) + self.log_state["timing/step_time"].append(shared_step_time + student_step_train_time) + self.logging() + + if self.global_step % self.args.train.save_steps == 0: + self.strategy.log(f"Saving model at global step {self.global_step}") + save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}_global_step_{self.global_step}") + ray.get(self.student.async_save_model(save_path)) + + if self.args.train.enable_sleep: + self.student.sleep() + + if ( + self.eval_dataloader is not None + and self.global_step % self.args.train.eval_steps == 0 + ): + self.strategy.log(f"Start evaluating at global step {self.global_step}") + self.evaluate() + + if self.global_step >= self.max_steps: + self.strategy.log( + f"Reached max_steps={self.max_steps}; stopping training." + ) + break + + self.strategy.log(f"Saving model after epoch {epoch + 1}") + save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}") + ray.get(self.student.async_save_model(save_path)) + + total_time = time.time() - self.start_time + self.strategy.log(f"Training done, totally cost {str(timedelta(seconds=total_time)).split('.')[0]}") + + if self._wandb is not None: + self._wandb.finish() + + def evaluate(self): + """Evaluate KD loss and distillation metrics without updating the student.""" + eval_batches = list(self.eval_dataloader) + if not eval_batches: + return {} + if self.args.train.use_dynamic_bsz: + eval_batches = rearrange_global_batch( + eval_batches, + max_token_len=self.args.train.max_token_len_per_gpu, + dp_size=self.dp_size, + ) + + if self.args.train.enable_sleep: + self.teacher.wakeup() + eval_batches = self.teacher.forward(eval_batches) + if self.args.train.enable_sleep: + self.teacher.sleep() + + if self.args.train.enable_sleep: + self.student.wakeup() + metrics = ray.get(self.student.async_run_eval(eval_batches))[0] + if self.args.train.enable_sleep: + self.student.sleep() + + return log_eval_metrics(self.strategy, self._wandb, metrics, self.global_step) + + def logging(self): + if self.global_step % self.args.log.logging_steps == 0: + progress = self.global_step / self.max_steps + eta = int(time.time() - self.start_time) * (1 - progress) / progress + progress_str = "epoch [{current_epoch}/{total_epoch}], " \ + "step [{current_step}/{total_step}], " \ + "train_progress [{progress:.2f}%], " \ + "Elapsed: {elapsed}, " \ + "ETA: {eta}, ".format( + current_epoch=self.current_epoch + 1, + total_epoch=self.epochs, + current_step=self.global_step, + total_step=self.max_steps, + progress=progress * 100, + elapsed=str(timedelta(seconds=(time.time() - self.start_time))).split(".")[0], + eta=str(timedelta(seconds=eta)).split(".")[0] + ) + for k in self.log_state: + if isinstance(self.log_state[k], list) and len(self.log_state[k]) > 0: + self.log_state[k] = sum(self.log_state[k]) / len(self.log_state[k]) + + log_info = [] + for k in self.log_state: + if k == "train/lr": + log_info.append(f"{k}: {self.log_state[k]:.6e}") + else: + log_info.append(f"{k}: {self.log_state[k]:.6f}") + log_str = ", ".join(log_info) + log_str = progress_str + log_str + self.strategy.log(log_str) + + if self._wandb is not None: + logs = {"train/global_step": self.global_step} + for k in self.log_state: + logs[k] = self.log_state[k] + self._wandb.log(logs) + + for k in self.log_state: + self.log_state[k] = [] diff --git a/kdflow/trainer/on_policy_kd_trainer.py b/kdflow/trainer/on_policy_kd_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..aa4ba8e05f8fc49625b69b8870fc29a8aa6aa6c6 --- /dev/null +++ b/kdflow/trainer/on_policy_kd_trainer.py @@ -0,0 +1,469 @@ +import os +import time +import math +from datetime import timedelta +from typing import Callable, Dict, Optional +from collections import defaultdict + +import ray + +from kdflow.evaluation import MathEvaluator +from kdflow.trainer.rollout_manager import RolloutManager +from kdflow.trainer.alfworld_rollout_manager import AlfWorldRolloutManager +from kdflow.trainer.verl_agent_alfworld_rollout_manager import ( + VerlAgentAlfWorldRolloutManager, +) +from kdflow.utils.logging_utils import ( + define_wandb_metrics, + init_logger, + log_eval_metrics, + normalize_eval_metrics, +) +from kdflow.utils.dynamic_bsz import rearrange_global_batch + + +logger = init_logger(__name__) + +class OnPolicyKDTrainer: + """ + Ray-based trainer for on-policy knowledge distillation. + """ + + def __init__( + self, + strategy, + student_model, + teacher_model, + rollout_group, + is_same_tokenizer: bool, + train_dataloader, + eval_dataloader=None, + max_rollout_iters: int = None, + num_rollout_iters_per_epoch: int = None, + generate_kwargs: Dict[str, float] = None, + custom_eval_fn: Optional[Callable] = None, + ) -> None: + """ + Initialize the trainer. + + Args: + strategy: Training strategy containing configuration + student_model: StudentActorGroup + teacher_model: TeacherActorGroup + rollout_group: RolloutGroup + is_same_tokenizer: Whether student and teacher use same tokenizer + train_dataloader: Training data loader + eval_dataloader: Evaluation data loader (optional) + max_rollout_iters: Maximum rollout iterations in training + num_rollout_iters_per_epoch: Number of rollout iterations per epoch + """ + self.strategy = strategy + self.args = strategy.args + self.student = student_model + self.teacher = teacher_model + self.rollout_group = rollout_group + self.train_dataloader = train_dataloader + self.eval_dataloader = eval_dataloader + self.num_rollout_iters_per_epoch = num_rollout_iters_per_epoch + self.max_rollout_iters = ( + max_rollout_iters + if max_rollout_iters is not None and max_rollout_iters > 0 + else self.args.train.num_epochs * self.num_rollout_iters_per_epoch + ) + self.generate_kwargs = generate_kwargs + self.custom_eval_fn = custom_eval_fn + self.epochs = math.ceil( + self.max_rollout_iters / self.num_rollout_iters_per_epoch + ) + if self.args.rollout.alfworld_config: + rollout_manager_cls = ( + VerlAgentAlfWorldRolloutManager + if self.args.rollout.alfworld_protocol == "verl_agent" + else AlfWorldRolloutManager + ) + else: + rollout_manager_cls = RolloutManager + self.rollout_manager = rollout_manager_cls( + strategy=strategy, + rollout_group=rollout_group, + is_same_tokenizer=is_same_tokenizer, + generate_kwargs=generate_kwargs, + ) + self.math_evaluator = None + if self.args.data.eval_math_dataset_path: + self.math_evaluator = MathEvaluator( + strategy=strategy, + rollout_group=rollout_group, + generate_kwargs=generate_kwargs, + student_processor=self.rollout_manager.data_processor.student_processor, + ) + + self.world_size = self.args.train.num_nodes * self.args.train.num_gpus_per_node + self.dp_size = self.world_size // self.args.model.ring_attn_size + + assert self.args.kd.kd_ratio == 1.0, "On-policy KD only supports kd_ratio=1.0." + + self.log_state = defaultdict(list) + self._init_loggers() + + + def _init_loggers(self) -> None: + """Initialize wandb loggers.""" + self._wandb = None + + if self.args.log.use_wandb: + import wandb + + self._wandb = wandb + if self.args.log.wandb_mode != "offline" and not wandb.api.api_key: + wandb.login() + wandb.init( + entity=self.args.log.wandb_org, + project=self.args.log.wandb_project, + group=self.args.log.wandb_group, + name=self.args.log.wandb_run_name, + config=vars(self.args), + reinit=True, + mode=self.args.log.wandb_mode, + dir=self.args.log.wandb_dir, + ) + + define_wandb_metrics(wandb) + + def _print_training_config(self) -> None: + """Log training configuration before training starts.""" + num_data = len(getattr(self.train_dataloader, "dataset", self.train_dataloader)) + if self.args.rollout.alfworld_config: + max_turn_samples = ( + self.args.rollout.rollout_batch_size + * self.args.rollout.alfworld_max_turns + ) + num_update_per_rollout = max( + 1, + math.ceil(max_turn_samples / self.args.train.train_batch_size), + ) + else: + num_update_per_rollout = max( + 1, + self.args.rollout.n_samples_per_prompt + * self.args.rollout.rollout_batch_size + // self.args.train.train_batch_size, + ) + total_steps = self.max_rollout_iters * num_update_per_rollout + grad_accum = self.args.train.train_batch_size * self.args.model.ring_attn_size \ + // (self.args.train.micro_train_batch_size * self.args.train.num_nodes * self.args.train.num_gpus_per_node) + + def log_config(name, value): + logger.info(f" {name:<32} {value}") + + logger.info("******* Start Training *******") + log_config("Num GPUs:", self.world_size) + log_config("Num Data:", num_data) + log_config("Num Epochs:", self.epochs) + log_config("Rollout Batch Size:", self.args.rollout.rollout_batch_size) + log_config("Train Batch Size:", self.args.train.train_batch_size) + log_config("Rollout Iterations Per Epoch:", self.num_rollout_iters_per_epoch) + log_config("Total Rollout Iterations:", self.max_rollout_iters) + log_config("Num Updates Per Rollout:", num_update_per_rollout) + log_config("Total Num Updates:", total_steps) + if self.args.train.use_dynamic_bsz: + log_config("Enable Dynamic Batch Size:", self.args.train.use_dynamic_bsz) + log_config("Max Token Len Per GPU:", self.args.train.max_token_len_per_gpu) + log_config("Gradient Accumulation:", "dynamic") + else: + log_config("Per-device Batch Size:", self.args.train.micro_train_batch_size) + log_config("Gradient Accumulation:", grad_accum) + log_config("Learning Rate:", self.args.train.learning_rate) + log_config("KD Algorithm:", self.args.kd.kd_algorithm) + log_config("KD Loss Function:", self.args.kd.kd_loss_fn) + if self.args.rollout.alfworld_config: + log_config("Multi-turn Environment:", "ALFWorld / AlfredTWEnv") + log_config("ALFWorld Split:", self.args.rollout.alfworld_split) + log_config("ALFWorld Max Turns:", self.args.rollout.alfworld_max_turns) + if self.args.kd.kd_algorithm == "topk_byte_align_ctkd": + log_config("CTKD Student Top-K:", self.args.kd.ctkd_topk) + + def fit(self, global_step=0, start_epoch=0): + self.global_step = global_step + + self._print_training_config() + + # Create Gloo IPC groups between training ranks and rollout engines (following slime) + rollout_tp_size = getattr(self.args.rollout, "rollout_tp_size", 1) + self.student.connect_rollout_engines(self.rollout_group.actors, rollout_tp_size) + if self.args.model.student_name_or_path == self.args.model.teacher_name_or_path: # for self-distillation + num_gpus_per_teacher_actor = self.args.kd.teacher_tp_size * self.args.kd.teacher_pp_size + self.student.connect_teacher_actors(self.teacher.teacher_engines, num_gpus_per_teacher_actor) + + self.start_time = time.time() + num_micro_batches = self.args.train.train_batch_size // self.args.train.micro_train_batch_size + + if self._has_evaluator() and self.args.train.eval_steps < float("inf") and self.global_step == 0: + self.strategy.log(f"Evaluating model at global step {self.global_step}") + self.evaluate() + + for epoch in range(start_epoch, self.epochs): + self.current_epoch = epoch + self.train_dataloader.sampler.set_epoch(epoch) + + for prompt_batch in self.train_dataloader: + if self.global_step >= self.max_rollout_iters: + break + step_start = time.time() + self.global_step += 1 + + rollout_start = time.time() + rollout_samples, rollout_metrics = self.rollout_manager.rollout( + prompt_batch, + global_step=self.global_step, + mode="train", + ) + rollout_time = time.time() - rollout_start + + self.log_state["timing/rollout"].append(rollout_time) + for name, value in rollout_metrics.items(): + self.log_state[name].append(value) + + all_global_batches = self._prepare_global_batches(rollout_samples, num_micro_batches) + if not all_global_batches: + self.log_state["train/skipped_update"].append(1.0) + self.log_state["timing/teacher_forward"].append(0.0) + self.log_state["timing/student_train"].append(0.0) + self.log_state["timing/step_time"].append(time.time() - step_start) + self.strategy.log( + "No training rollout samples were returned; skipping " + f"the distributed update at rollout step {self.global_step}.", + level="warning", + ) + self.logging() + if ( + self._has_evaluator() + and self.global_step % self.args.train.eval_steps == 0 + ): + self.strategy.log( + f"Evaluating model at global step {self.global_step}" + ) + self.evaluate() + if self.global_step % self.args.train.save_steps == 0: + save_path = os.path.join( + self.args.train.save_path, + f"epoch_{epoch + 1}_global_step_{self.global_step}", + ) + ray.get(self.student.async_save_model(save_path)) + continue + + self.log_state["train/skipped_update"].append(0.0) + + teacher_start = time.time() + if self.args.train.enable_sleep: + self.teacher.wakeup() + + teacher_batches = sum(all_global_batches, []) + teacher_batches = self.teacher.forward(teacher_batches) + + batch_idx = 0 + for i, global_batch in enumerate(all_global_batches): + next_batch_idx = batch_idx + len(global_batch) + all_global_batches[i] = teacher_batches[batch_idx:next_batch_idx] + batch_idx = next_batch_idx + if batch_idx != len(teacher_batches): + raise RuntimeError( + f"Teacher forward returned {len(teacher_batches)} batches, expected {batch_idx}." + ) + + if self.args.train.enable_sleep: + self.teacher.sleep() + self.log_state["timing/teacher_forward"].append(time.time() - teacher_start) + + student_start = time.time() + + if self.args.train.enable_sleep: + self.student.wakeup() + + for global_batch in all_global_batches: + status_list = ray.get(self.student.async_run_distill(global_batch)) + for k in status_list[0].keys(): + self.log_state[k].append(sum(s[k] for s in status_list) / len(status_list)) + + self.log_state["timing/student_train"].append(time.time() - student_start) + + ray.get([actor.empty_cache.remote() for actor in self.student._actor_handlers]) + + if self.args.train.enable_sleep: + self.rollout_group.wakeup(tags=["weights"]) + update_start = time.time() + self.student.update_rollout_weights() + self.log_state["timing/rollout_weight_sync"].append(time.time() - update_start) + if self.args.train.enable_sleep: + self.rollout_group.sleep(tags=["weights"]) + + if self.args.model.teacher_name_or_path == self.args.model.student_name_or_path \ + and self.global_step % self.args.kd.teacher_update_freq == 0: + if self.args.train.enable_sleep: + self.teacher.wakeup(tags=["weights"]) + teacher_update_start = time.time() + self.student.update_teacher_weights() + self.log_state["timing/teacher_weight_sync"].append(time.time() - teacher_update_start) + if self.args.train.enable_sleep: + self.teacher.sleep(tags=["weights"]) + + if self.args.train.enable_sleep: + self.student.sleep() + + self.log_state["timing/step_time"].append(time.time() - step_start) + self.logging() + + if ( + self._has_evaluator() + and self.global_step % self.args.train.eval_steps == 0 + ): + self.strategy.log(f"Evaluating model at global step {self.global_step}") + self.evaluate() + + if self.global_step % self.args.train.save_steps == 0: + self.strategy.log(f"Saving model at global step {self.global_step}") + save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}_global_step_{self.global_step}") + ray.get(self.student.async_save_model(save_path)) + + if self.global_step >= self.max_rollout_iters: + self.strategy.log( + f"Reached max_steps={self.max_rollout_iters}; stopping training." + ) + break + + self.strategy.log(f"Saving model after epoch {epoch + 1}") + save_path = os.path.join(self.args.train.save_path, f"epoch_{epoch + 1}") + ray.get(self.student.async_save_model(save_path)) + + total_time = time.time() - self.start_time + self.strategy.log(f"Training done, totally cost {str(timedelta(seconds=total_time)).split('.')[0]}") + + if self._wandb is not None: + self._wandb.finish() + + def close(self) -> None: + close = getattr(self.rollout_manager, "close", None) + if callable(close): + close() + + def _prepare_global_batches(self, rollout_samples, num_micro_batches): + all_global_batches = [] + for i in range(0, len(rollout_samples), num_micro_batches): + global_batch = rollout_samples[i : i + num_micro_batches] + if self.args.train.use_dynamic_bsz: + global_batch = rearrange_global_batch( + global_batch, + max_token_len=self.args.train.max_token_len_per_gpu, + dp_size=self.dp_size, + ) + + batch_tokens = sum(mb["stu_loss_mask"].sum() for mb in global_batch) + avg_micro_batch_token_num = batch_tokens / len(global_batch) + for micro_batch in global_batch: + micro_batch["avg_micro_batch_token_num"] = avg_micro_batch_token_num + all_global_batches.append(global_batch) + return all_global_batches + + def _has_evaluator(self): + return self.eval_dataloader is not None or self.math_evaluator is not None + + def evaluate(self): + """Run configured validation and math evaluators.""" + metrics = {} + if self.eval_dataloader is not None: + metrics.update(self._evaluate_validation()) + if self.math_evaluator is not None: + metrics.update(self.math_evaluator.evaluate(self.global_step)) + return log_eval_metrics(self.strategy, self._wandb, metrics, self.global_step) + + def _evaluate_validation(self): + """Evaluate distillation metrics on the validation set.""" + eval_prompts = sum(self.eval_dataloader, []) + if not eval_prompts: + return {} + + generate_kwargs = {**self.generate_kwargs, "temperature": 0.0} + rollout_samples, rollout_metrics = self.rollout_manager.rollout( + eval_prompts, + global_step=self.global_step, + mode="eval", + **generate_kwargs, + ) + predictions = sum((micro_batch["stu_responses"] for micro_batch in rollout_samples), []) + labels = sum((micro_batch["labels"] for micro_batch in rollout_samples), []) + + eval_batches = rollout_samples + if self.args.train.use_dynamic_bsz: + eval_batches = rearrange_global_batch( + eval_batches, + max_token_len=self.args.train.max_token_len_per_gpu, + dp_size=self.dp_size, + ) + if self.args.train.enable_sleep: + self.teacher.wakeup() + eval_batches = self.teacher.forward(eval_batches) + if self.args.train.enable_sleep: + self.teacher.sleep() + + if self.args.train.enable_sleep: + self.student.wakeup() + metrics = ray.get(self.student.async_run_eval(eval_batches))[0] + if self.args.train.enable_sleep: + self.student.sleep() + metrics.update(rollout_metrics) + + if self.custom_eval_fn is not None: + custom_metrics = self.custom_eval_fn(predictions, labels) + if not isinstance(custom_metrics, dict): + raise TypeError("custom_eval_fn must return a dict") + metrics.update(normalize_eval_metrics(custom_metrics)) + + return metrics + + def logging(self): + if self.global_step % self.args.log.logging_steps == 0: + progress = self.global_step / self.max_rollout_iters + eta = int(time.time() - self.start_time) * (1 - progress) / progress + progress_str = "epoch [{current_epoch}/{total_epoch}], " \ + "step [{current_step}/{total_step}], " \ + "train_progress [{progress:.2f}%], " \ + "Elapsed: {elapsed}, " \ + "ETA: {eta}, ".format( + current_epoch=self.current_epoch + 1, + total_epoch=self.epochs, + current_step=self.global_step, + total_step=self.max_rollout_iters, + progress=progress * 100, + elapsed=str(timedelta(seconds=(time.time() - self.start_time))).split(".")[0], + eta=str(timedelta(seconds=eta)).split(".")[0] + ) + for k in self.log_state: + if isinstance(self.log_state[k], list) and len(self.log_state[k]) > 0: + values = self.log_state[k] + self.log_state[k] = ( + max(values) if k.endswith("/max") else sum(values) / len(values) + ) + log_info = [] + for k in self.log_state: + # Skip keys that have no values logged in this interval (e.g. teacher weight sync + # is only logged every teacher_update_freq steps). + if isinstance(self.log_state[k], list): + continue + if k == "train/lr": + log_info.append(f"{k}: {self.log_state[k]:.6e}") + else: + log_info.append(f"{k}: {self.log_state[k]:.6f}") + log_str = ", ".join(log_info) + log_str = progress_str + log_str + self.strategy.log(log_str) + + if self._wandb is not None: + logs = {"train/global_step": self.global_step} + for k in self.log_state: + if isinstance(self.log_state[k], list): + continue + logs[k] = self.log_state[k] + self._wandb.log(logs) + + for k in self.log_state: + self.log_state[k] = [] diff --git a/kdflow/trainer/rollout_manager.py b/kdflow/trainer/rollout_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..0eb5253bceb3a06a66cf120a606a221db3a07f81 --- /dev/null +++ b/kdflow/trainer/rollout_manager.py @@ -0,0 +1,190 @@ +import asyncio +import time +from typing import Any, Dict, List, Optional + +from kdflow.trainer.data_processor import RolloutDataProcessor +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class RolloutManager: + """Manage batched rollout generation and data processing.""" + + def __init__( + self, + strategy, + rollout_group, + is_same_tokenizer: bool, + generate_kwargs: Optional[Dict[str, Any]] = None, + ): + self.args = strategy.args + self.rollout_group = rollout_group + self.generate_kwargs = generate_kwargs + self.max_concurrent = ( + self.args.rollout.rollout_engine_concurrency + * self.rollout_group.num_actors + ) + if self.max_concurrent <= 0: + raise ValueError( + f"max_concurrent must be positive, got {self.max_concurrent}" + ) + + self.data_processor = RolloutDataProcessor(strategy, is_same_tokenizer) + self.image_key = self.data_processor.image_key + + def rollout( + self, + prompt_batch: List[Dict[str, Any]], + global_step: int, + mode: str = "train", + **kwargs, + ) -> tuple[List[dict], Dict[str, float]]: + """Generate rollout samples and convert them into micro-batches.""" + if mode not in ("train", "eval"): + raise ValueError(f"Unsupported rollout mode: {mode!r}") + if not prompt_batch: + return [], {} + + n_samples = self.args.rollout.n_samples_per_prompt if mode == "train" else 1 + if n_samples <= 0: + raise ValueError(f"n_samples_per_prompt must be positive, got {n_samples}") + + should_sleep = self.args.train.enable_sleep + if should_sleep: + self.rollout_group.wakeup() + + try: + stu_prompts = [ + item["stu_prompt"] + for item in prompt_batch + for _ in range(n_samples) + ] + tea_prompts = [ + item["tea_prompt"] + for item in prompt_batch + for _ in range(n_samples) + ] + labels = [ + item["label"] for item in prompt_batch for _ in range(n_samples) + ] + images = None + if self.image_key: + images = [ + item.get("images") + for item in prompt_batch + for _ in range(n_samples) + ] + + sampling_params = kwargs or self.generate_kwargs + if not sampling_params: + raise ValueError("sampling_params must not be empty") + + outputs, timing_metrics = self._generate( + stu_prompts, + sampling_params, + image_data=images, + ) + micro_batches, rollout_metrics = self.data_processor.process( + stu_prompts=stu_prompts, + tea_prompts=tea_prompts, + outputs=outputs, + labels=labels, + sampling_params=sampling_params, + global_step=global_step, + mode=mode, + images=images, + ) + rollout_metrics.update(timing_metrics) + return micro_batches, rollout_metrics + finally: + if should_sleep: + self.rollout_group.sleep() + + def _generate( + self, + prompts: List[str], + sampling_params: Dict[str, Any], + image_data: Optional[List] = None, + ) -> tuple[List[Dict[str, Any]], Dict[str, float]]: + """Run ordered concurrent generation requests.""" + if not prompts: + return [], {} + if image_data is not None and len(image_data) != len(prompts): + raise ValueError("image_data and prompts must have the same length") + + max_concurrent = min(len(prompts), self.max_concurrent) + try: + return asyncio.run( + self._async_generate( + prompts=prompts, + sampling_params=sampling_params, + max_concurrent=max_concurrent, + image_data=image_data, + ) + ) + except Exception: + try: + engine_health = self.rollout_group.health_check() + except Exception as health_error: + engine_health = f"unavailable ({health_error!r})" + logger.exception( + "Rollout generation failed: prompts=%d, max_concurrent=%d, " + "engine_health=%s", + len(prompts), + max_concurrent, + engine_health, + ) + raise + + async def _async_generate( + self, + prompts: List[str], + sampling_params: Dict[str, Any], + max_concurrent: int, + image_data: Optional[List] = None, + ) -> tuple[List[Dict[str, Any]], Dict[str, float]]: + """Schedule single-sample generation requests concurrently.""" + import aiohttp + + semaphore = asyncio.Semaphore(max_concurrent) + results = [None] * len(prompts) + rollout_times = [0.0] * len(prompts) + + async def run_request( + index: int, prompt: str, session: aiohttp.ClientSession + ) -> None: + try: + async with semaphore: + start = time.perf_counter() + results[index] = await self.rollout_group.generate_one( + prompt=prompt, + sampling_params=sampling_params, + session=session, + image_data=( + image_data[index] if image_data is not None else None + ), + ) + rollout_times[index] = time.perf_counter() - start + except Exception as error: + raise RuntimeError( + f"Rollout generation failed for request {index}" + ) from error + + connector = aiohttp.TCPConnector(limit=max_concurrent) + timeout = aiohttp.ClientTimeout(total=None, sock_read=None, sock_connect=60) + async with aiohttp.ClientSession( + connector=connector, timeout=timeout + ) as session: + await asyncio.gather( + *( + run_request(index, prompt, session) + for index, prompt in enumerate(prompts) + ) + ) + + timing_metrics = { + "timing/rollout_per_sample/mean": sum(rollout_times) / len(rollout_times), + "timing/rollout_per_sample/max": max(rollout_times), + } + return results, timing_metrics diff --git a/kdflow/trainer/verl_agent_alfworld_rollout_manager.py b/kdflow/trainer/verl_agent_alfworld_rollout_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..430dd71f2979174e05b61d276b4e29d068adc864 --- /dev/null +++ b/kdflow/trainer/verl_agent_alfworld_rollout_manager.py @@ -0,0 +1,543 @@ +# Copyright 2025 Nanyang Technological University (NTU), Singapore +# and the verl-agent (GiGPO) team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Adapted for KDFlow from langfengq/verl-agent at commit +# 20bd331bdbc9026a5668e11362178e10ab7400c8. + +"""ALFWorld rollout protocol compatible with the verl-agent evaluation recipe.""" + +import copy +import os +import random +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from kdflow.evaluation.verl_agent_alfworld_protocol import ( + ALFWORLD_TEMPLATE_NO_HISTORY, + ALFWORLD_TEMPLATE_WITH_HISTORY, + TASK_TYPES, + project_alfworld_action, + task_type_from_gamefile, +) +from kdflow.trainer.rollout_manager import RolloutManager +from kdflow.utils.logging_utils import init_logger + + +logger = init_logger(__name__) + + +class _IndependentAlfWorldBatch: + """A small local equivalent of verl-agent's one-Ray-actor-per-game setup.""" + + def __init__(self, config, split: str, batch_size: int, base_seed: int): + try: + from alfworld.agents.environment import get_environment + except ImportError as error: + raise ImportError( + "The verl-agent-compatible evaluator requires ALFWorld. Install " + "it with `pip install -e '.[alfworld]'` and make the downloaded " + "data available through ALFWORLD_DATA." + ) from error + + env_type = config.get("env", {}).get("type", "AlfredTWEnv") + if env_type != "AlfredTWEnv": + raise ValueError( + "The migrated evaluator supports text-only AlfredTWEnv, got " + f"{env_type!r}." + ) + self.batch_size = batch_size + self.base_seed = base_seed + self.episode_offset = 0 + self.envs = [] + for index in range(batch_size): + worker_config = copy.deepcopy(config) + worker_seed = base_seed + index + worker_config.setdefault("general", {})["random_seed"] = worker_seed + base_env = get_environment(env_type)(worker_config, train_eval=split) + env = base_env.init_env(batch_size=1) + seed_method = getattr(env, "seed", None) + if callable(seed_method): + seed_method(worker_seed) + self.envs.append(env) + + def set_episode_range(self, base_seed: int, episode_offset: int) -> None: + self.base_seed = base_seed + self.episode_offset = episode_offset + + @staticmethod + def _scalar(value): + if isinstance(value, (list, tuple)): + return value[0] if value else None + try: + if getattr(value, "shape", None) == (1,): + return value[0] + except Exception: + pass + return value + + @classmethod + def _merge_infos(cls, infos: List[Dict[str, Any]]) -> Dict[str, List[Any]]: + keys = set().union(*(info.keys() for info in infos)) if infos else set() + return { + key: [cls._scalar(info.get(key)) for info in infos] + for key in keys + } + + def reset(self): + observations = [] + infos = [] + for index, env in enumerate(self.envs): + seed_method = getattr(env, "seed", None) + if callable(seed_method): + seed_method(self.base_seed + self.episode_offset + index) + observation, info = env.reset() + observations.append(self._scalar(observation)) + infos.append(info) + return observations, self._merge_infos(infos) + + def step(self, actions: Sequence[str]): + observations = [] + rewards = [] + dones = [] + infos = [] + for env, action in zip(self.envs, actions): + observation, reward, done, info = env.step([action]) + observations.append(self._scalar(observation)) + rewards.append(self._scalar(reward)) + dones.append(bool(self._scalar(done))) + infos.append(info) + return observations, rewards, dones, self._merge_infos(infos) + + def close(self) -> None: + for env in self.envs: + close = getattr(env, "close", None) + if callable(close): + close() + + +class VerlAgentAlfWorldRolloutManager(RolloutManager): + """Run ALFWorld with prompts, memory and projection from verl-agent.""" + + FAILURE_MARKERS = ( + "nothing happens", + "not a valid command", + "that's not a verb i recognise", + "you can't", + "you cannot", + ) + + def __init__( + self, + strategy, + rollout_group, + is_same_tokenizer: bool = True, + generate_kwargs: Optional[Dict[str, Any]] = None, + env=None, + ): + super().__init__(strategy, rollout_group, is_same_tokenizer, generate_kwargs) + self.batch_size = self.args.rollout.rollout_batch_size + self.max_turns = self.args.rollout.alfworld_max_turns + self.history_length = self.args.rollout.alfworld_history_length + self.prompt_max_len = self.args.data.prompt_max_len + self.training_mode = getattr(self.args, "scenario", None) == "on_policy_kd" + if self.training_mode: + self.script_seed = self.args.train.seed + self.environment_seed = self.args.train.seed + else: + seeds = self.parse_seeds(self.args.rollout.alfworld_eval_seeds) + self.script_seed = seeds[0] + self.environment_seed = ( + self.script_seed + self.args.rollout.alfworld_eval_seed_offset + ) + self.episode_offset = 0 + self._train_episode_offset = 0 + self.env = env if env is not None else self._build_env() + + @staticmethod + def parse_seeds(value: str) -> List[int]: + seeds = [int(item.strip()) for item in value.split(",") if item.strip()] + if not seeds: + raise ValueError("At least one ALFWorld evaluation seed is required.") + return seeds + + def _build_env(self): + try: + import yaml + from alfworld.info import ALFWORLD_DATA + except ImportError as error: + raise ImportError( + "The verl-agent-compatible evaluator requires ALFWorld. Install " + "it with `pip install -e '.[alfworld]'`." + ) from error + + config_path = os.path.abspath(self.args.rollout.alfworld_config) + with open(config_path, "r", encoding="utf-8") as stream: + config = yaml.safe_load(stream) + os.environ.setdefault("ALFWORLD_DATA", str(ALFWORLD_DATA)) + config.setdefault("general", {})["training_method"] = "dagger" + config.setdefault("dagger", {}).setdefault("training", {})[ + "max_nb_steps_per_episode" + ] = self.max_turns + random.seed(self.environment_seed) + split = "train" if self.training_mode else self.args.rollout.alfworld_split + logger.info( + "Initializing verl-agent ALFWorld protocol: mode=%s split=%s batch=%d " + "history=%d max_steps=%d environment_seed=%d", + "train" if self.training_mode else "eval", + split, + self.batch_size, + self.history_length, + self.max_turns, + self.environment_seed, + ) + return _IndependentAlfWorldBatch( + config=config, + split=split, + batch_size=self.batch_size, + base_seed=self.environment_seed, + ) + + def set_evaluation_batch(self, script_seed: int, episode_offset: int) -> None: + self.script_seed = script_seed + self.environment_seed = ( + script_seed + self.args.rollout.alfworld_eval_seed_offset + ) + self.episode_offset = episode_offset + setter = getattr(self.env, "set_episode_range", None) + if callable(setter): + setter(self.environment_seed, episode_offset) + + @staticmethod + def _info_values(info: Dict[str, Any], key: str, size: int, default): + values = info.get(key) if isinstance(info, dict) else None + if values is None: + return [default for _ in range(size)] + if isinstance(values, (str, bytes)): + return [values for _ in range(size)] + return list(values) + + @staticmethod + def _extract_task(observation: str) -> str: + marker = "Your task is to: " + start = observation.find(marker) + if start == -1: + raise ValueError("Task description not found in ALFWorld observation.") + return observation[start + len(marker) :].strip() + + def _history_context(self, history: List[Dict[str, str]]) -> str: + recent = history[-self.history_length :] + start_index = len(history) - len(recent) + lines = [] + for local_index, record in enumerate(recent): + step_number = start_index + local_index + 1 + lines.append( + f"[Observation {step_number}: '{record['observation']}', " + f"Action {step_number}: '{record['action']}']" + ) + return "\n".join(lines) + + def _build_observation_prompt( + self, + observation: str, + admissible_actions: Sequence[str], + task: str, + history: List[Dict[str, str]], + initial: bool, + ) -> str: + actions = "\n ".join( + f"'{action}'" for action in admissible_actions if action != "help" + ) + if initial or self.history_length <= 0: + return ALFWORLD_TEMPLATE_NO_HISTORY.format( + current_observation=observation, + admissible_actions=actions, + ) + return ALFWORLD_TEMPLATE_WITH_HISTORY.format( + task_description=task, + step_count=len(history), + history_length=min(len(history), self.history_length), + action_history=self._history_context(history), + current_step=len(history) + 1, + current_observation=observation, + admissible_actions=actions, + ) + + def _render_prompt(self, content: str, processor=None) -> Tuple[str, bool]: + if processor is None: + processor = self.data_processor.student_processor + template_owner = getattr(processor, "tokenizer", processor) + kwargs = {} + if "enable_thinking" in str(getattr(template_owner, "chat_template", "")): + kwargs["enable_thinking"] = self.args.model.enable_thinking + prompt = template_owner.apply_chat_template( + [{"role": "user", "content": content}], + tokenize=False, + add_generation_prompt=True, + **kwargs, + ) + token_ids = template_owner.encode(prompt, add_special_tokens=False) + if self.prompt_max_len > 0 and len(token_ids) > self.prompt_max_len: + prompt = template_owner.decode( + token_ids[-self.prompt_max_len :], skip_special_tokens=False + ) + return prompt, True + return prompt, False + + def collect_episodes( + self, + prompt_batch: List[Dict[str, Any]], + global_step: int, + mode: str = "eval", + **kwargs, + ) -> Tuple[List[dict], Dict[str, float]]: + if mode not in ("train", "eval"): + raise ValueError(f"Unsupported ALFWorld rollout mode: {mode!r}.") + if len(prompt_batch) != self.batch_size: + raise ValueError( + f"Expected a fixed batch of {self.batch_size}, got {len(prompt_batch)}." + ) + + if mode == "train": + self.script_seed = self.args.train.seed + self.environment_seed = self.args.train.seed + self.episode_offset = self._train_episode_offset + setter = getattr(self.env, "set_episode_range", None) + if callable(setter): + setter(self.environment_seed, self.episode_offset) + + if self.args.train.enable_sleep: + self.rollout_group.wakeup() + try: + observations, reset_info = self.env.reset() + observations = [str(value) for value in observations] + admissible = self._info_values( + reset_info, "admissible_commands", self.batch_size, [] + ) + saved_gamefiles = [ + str(value or "") + for value in self._info_values( + reset_info, "extra.gamefile", self.batch_size, "" + ) + ] + episode_gamefiles = list(saved_gamefiles) + tasks = [self._extract_task(value) for value in observations] + histories: List[List[Dict[str, str]]] = [ + [] for _ in range(self.batch_size) + ] + active = [True] * self.batch_size + turn_counts = [0] * self.batch_size + final_won = [False] * self.batch_size + format_invalid = 0 + inadmissible_count = 0 + environment_failures = 0 + prompt_truncated_count = 0 + records = [] + timing_sums: Dict[str, float] = {} + sampling_params = kwargs or self.generate_kwargs + + for turn_id in range(self.max_turns): + request_indices = [] + student_prompts = [] + teacher_prompts = [] + truncated_flags = [] + teacher_processor = self.data_processor._get_teacher_processor() + for index in range(self.batch_size): + if not active[index]: + continue + content = self._build_observation_prompt( + observation=observations[index], + admissible_actions=admissible[index], + task=tasks[index], + history=histories[index], + initial=turn_id == 0, + ) + student_prompt, student_truncated = self._render_prompt(content) + teacher_prompt, teacher_truncated = self._render_prompt( + content, processor=teacher_processor + ) + request_indices.append(index) + student_prompts.append(student_prompt) + teacher_prompts.append(teacher_prompt) + truncated_flags.append( + bool(student_truncated or teacher_truncated) + ) + + if not student_prompts: + break + outputs, timing = self._generate(student_prompts, sampling_params) + for key, value in timing.items(): + timing_sums[key] = timing_sums.get(key, 0.0) + float(value) + + actions = ["look"] * self.batch_size + pending = {} + for local_index, episode_index in enumerate(request_indices): + output = outputs[local_index] + action, format_valid = project_alfworld_action( + output.get("text", "") + ) + in_admissible = action in admissible[episode_index] + actions[episode_index] = action + format_invalid += int(not format_valid) + inadmissible_count += int(not in_admissible) + prompt_truncated_count += int(truncated_flags[local_index]) + pending[episode_index] = { + "stu_prompt": student_prompts[local_index], + "tea_prompt": teacher_prompts[local_index], + "output": output, + "action": action, + "format_valid": format_valid, + "action_in_admissible": in_admissible, + "prompt_truncated": truncated_flags[local_index], + } + + next_observations, _, dones, step_info = self.env.step(actions) + next_observations = [str(value) for value in next_observations] + next_admissible = self._info_values( + step_info, "admissible_commands", self.batch_size, [] + ) + won = self._info_values(step_info, "won", self.batch_size, False) + step_gamefiles = self._info_values( + step_info, "extra.gamefile", self.batch_size, "" + ) + + for episode_index, item in pending.items(): + feedback = next_observations[episode_index] + gamefile = str( + step_gamefiles[episode_index] + or saved_gamefiles[episode_index] + ) + episode_gamefiles[episode_index] = gamefile + env_failed = any( + marker in feedback.casefold() + for marker in self.FAILURE_MARKERS + ) + environment_failures += int(env_failed) + turn_counts[episode_index] += 1 + final_won[episode_index] = bool(won[episode_index]) + histories[episode_index].append( + { + "observation": observations[episode_index], + "action": item["action"], + } + ) + records.append( + { + "stu_prompt": item["stu_prompt"], + "tea_prompt": item["tea_prompt"], + "output": item["output"], + "metadata": { + "episode_id": ( + f"seed{self.script_seed}:" + f"{self.episode_offset + episode_index}" + ), + "script_seed": self.script_seed, + "environment_seed": ( + self.environment_seed + + self.episode_offset + + episode_index + ), + "turn_id": turn_id, + "alfworld_action": item["action"], + "format_valid": bool(item["format_valid"]), + "action_in_admissible": bool( + item["action_in_admissible"] + ), + "environment_failed": bool(env_failed), + "prompt_truncated": bool( + item["prompt_truncated"] + ), + "episode_done": bool(dones[episode_index]), + "episode_won": bool(won[episode_index]), + "gamefile": gamefile, + "task_type": task_type_from_gamefile(gamefile), + }, + } + ) + if dones[episode_index]: + active[episode_index] = False + + observations = next_observations + admissible = next_admissible + if not any(active): + break + + total_turns = sum(turn_counts) + prefix = "rollout/alfworld" if mode == "train" else "eval/alfworld" + metrics = dict(timing_sums) + metrics.update( + { + f"{prefix}/episodes": float(self.batch_size), + f"{prefix}/episodes_succeeded": float(sum(final_won)), + f"{prefix}/turns/total": float(total_turns), + f"{prefix}/turns/max": float(max(turn_counts, default=0)), + f"{prefix}/format_invalid_count": float(format_invalid), + f"{prefix}/inadmissible_action_count": float( + inadmissible_count + ), + f"{prefix}/environment_failure_count": float( + environment_failures + ), + f"{prefix}/prompt_truncated_count": float( + prompt_truncated_count + ), + } + ) + for task_type in TASK_TYPES + ("unknown",): + indices = [ + index + for index, gamefile in enumerate(episode_gamefiles) + if task_type_from_gamefile(gamefile) == task_type + ] + metrics[f"{prefix}/task/{task_type}/episodes"] = float( + len(indices) + ) + metrics[f"{prefix}/task/{task_type}/succeeded"] = float( + sum(final_won[index] for index in indices) + ) + if mode == "train": + self._train_episode_offset += self.batch_size + return records, metrics + finally: + if self.args.train.enable_sleep: + self.rollout_group.sleep() + + def close(self) -> None: + close = getattr(self.env, "close", None) + if callable(close): + close() + + def rollout( + self, + prompt_batch: List[Dict[str, Any]], + global_step: int, + mode: str = "train", + **kwargs, + ) -> Tuple[List[dict], Dict[str, float]]: + """Collect aligned turns and convert them into ordinary KD batches.""" + records, environment_metrics = self.collect_episodes( + prompt_batch=prompt_batch, + global_step=global_step, + mode=mode, + **kwargs, + ) + if not records: + return [], environment_metrics + + sampling_params = kwargs or self.generate_kwargs + micro_batches, rollout_metrics = self.data_processor.process( + stu_prompts=[record["stu_prompt"] for record in records], + tea_prompts=[record["tea_prompt"] for record in records], + outputs=[record["output"] for record in records], + labels=[record["metadata"]["episode_won"] for record in records], + sampling_params=sampling_params, + global_step=global_step, + mode=mode, + sample_metadata=[record["metadata"] for record in records], + ) + rollout_metrics.update(environment_metrics) + return micro_batches, rollout_metrics diff --git a/kdflow/utils/__init__.py b/kdflow/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1431283d595d0062ee042ea5567c33aec00ad49b --- /dev/null +++ b/kdflow/utils/__init__.py @@ -0,0 +1,5 @@ +from .utils import get_tokenizer + +__all__ = [ + "get_tokenizer", +] diff --git a/kdflow/utils/distributed_sampler.py b/kdflow/utils/distributed_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..1f7658202804e2a1fbe1a5d0d6ddaf20cc50f3af --- /dev/null +++ b/kdflow/utils/distributed_sampler.py @@ -0,0 +1,151 @@ +import math +from typing import Iterator, Optional, TypeVar + +import torch +import torch.distributed as dist +from torch.utils.data.dataset import Dataset +from torch.utils.data.sampler import Sampler + + +__all__ = ["DistributedSampler"] + + +_T_co = TypeVar("_T_co", covariant=True) + + +# Adapted from https://github.com/pytorch/pytorch/blob/5298acb5c76855bc5a99ae10016efc86b27949bd/torch/utils/data/distributed.py +class DistributedSampler(Sampler[_T_co]): + r"""Sampler that restricts data loading to a subset of the dataset. + + It is especially useful in conjunction with + :class:`torch.nn.parallel.DistributedDataParallel`. In such a case, each + process can pass a :class:`~torch.utils.data.DistributedSampler` instance as a + :class:`~torch.utils.data.DataLoader` sampler, and load a subset of the + original dataset that is exclusive to it. + + .. note:: + Dataset is assumed to be of constant size and that any instance of it always + returns the same elements in the same order. + + Args: + dataset: Dataset used for sampling. + num_replicas (int, optional): Number of processes participating in + distributed training. By default, :attr:`world_size` is retrieved from the + current distributed group. + rank (int, optional): Rank of the current process within :attr:`num_replicas`. + By default, :attr:`rank` is retrieved from the current distributed + group. + shuffle (bool, optional): If ``True`` (default), sampler will shuffle the + indices. + seed (int, optional): random seed used to shuffle the sampler if + :attr:`shuffle=True`. This number should be identical across all + processes in the distributed group. Default: ``0``. + drop_last (bool, optional): if ``True``, then the sampler will drop the + tail of the data to make it evenly divisible across the number of + replicas. If ``False``, the sampler will add extra indices to make + the data evenly divisible across the replicas. Default: ``False``. + + .. warning:: + In distributed mode, calling the :meth:`set_epoch` method at + the beginning of each epoch **before** creating the :class:`DataLoader` iterator + is necessary to make shuffling work properly across multiple epochs. Otherwise, + the same ordering will be always used. + + Example:: + + >>> # xdoctest: +SKIP + >>> sampler = DistributedSampler(dataset) if is_distributed else None + >>> loader = DataLoader(dataset, shuffle=(sampler is None), + ... sampler=sampler) + >>> for epoch in range(start_epoch, n_epochs): + ... if is_distributed: + ... sampler.set_epoch(epoch) + ... train(loader) + """ + + def __init__( + self, + dataset: Dataset, + num_replicas: Optional[int] = None, + rank: Optional[int] = None, + shuffle: bool = True, + seed: int = 0, + drop_last: bool = False, + consumed_samples=0, + ) -> None: + if num_replicas is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + num_replicas = dist.get_world_size() + if rank is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + rank = dist.get_rank() + if rank >= num_replicas or rank < 0: + raise ValueError(f"Invalid rank {rank}, rank should be in the interval [0, {num_replicas - 1}]") + self.dataset = dataset + self.num_replicas = num_replicas + self.rank = rank + self.epoch = 0 + self.drop_last = drop_last + # If the dataset length is evenly divisible by # of replicas, then there + # is no need to drop any data, since the dataset will be split equally. + if self.drop_last and len(self.dataset) % self.num_replicas != 0: # type: ignore[arg-type] + # Split to nearest available length that is evenly divisible. + # This is to ensure each rank receives the same amount of data when + # using this Sampler. + self.num_samples = math.ceil( + (len(self.dataset) - self.num_replicas) / self.num_replicas # type: ignore[arg-type] + ) + else: + self.num_samples = math.ceil(len(self.dataset) / self.num_replicas) # type: ignore[arg-type] + self.total_size = self.num_samples * self.num_replicas + self.shuffle = shuffle + self.seed = seed + self.consumed_indicies = consumed_samples // self.num_replicas + + def __iter__(self) -> Iterator[_T_co]: + if self.shuffle: + # deterministically shuffle based on epoch and seed + g = torch.Generator() + g.manual_seed(self.seed + self.epoch) + indices = torch.randperm(len(self.dataset), generator=g).tolist() # type: ignore[arg-type] + else: + indices = list(range(len(self.dataset))) # type: ignore[arg-type] + + if not self.drop_last: + # add extra samples to make it evenly divisible + padding_size = self.total_size - len(indices) + if padding_size <= len(indices): + indices += indices[:padding_size] + else: + indices += (indices * math.ceil(padding_size / len(indices)))[:padding_size] + else: + # remove tail of data to make it evenly divisible. + indices = indices[: self.total_size] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank : self.total_size : self.num_replicas] + # skip consumed_samples + indices = indices[self.consumed_indicies :] + assert len(indices) == self.num_samples - self.consumed_indicies + + return iter(indices) + + def __len__(self) -> int: + return self.num_samples - self.consumed_indicies + + def set_epoch(self, epoch: int, consumed_samples=0) -> None: + r""" + Set the epoch for this sampler. + + When :attr:`shuffle=True`, this ensures all replicas + use a different random ordering for each epoch. Otherwise, the next iteration of this + sampler will yield the same ordering. + + Args: + epoch (int): Epoch number. + """ + self.epoch = epoch + self.consumed_indicies = consumed_samples // self.num_replicas diff --git a/kdflow/utils/distributed_util.py b/kdflow/utils/distributed_util.py new file mode 100644 index 0000000000000000000000000000000000000000..fd13702ebaa1d34fc1bf87d7c506847af3f7190d --- /dev/null +++ b/kdflow/utils/distributed_util.py @@ -0,0 +1,26 @@ +def torch_dist_barrier_and_cuda_sync(): + """Synchronize distributed training and CUDA operations. + This function ensures that: + 1. All distributed processes reach this point (barrier) + 2. All CUDA operations are completed (synchronize) + """ + import torch + + torch.distributed.barrier() + torch.cuda.synchronize() + + +def stateless_init_process_group(master_address, master_port, rank, world_size, device): + """ + vLLM provides `StatelessProcessGroup` to create a process group + without considering the global process group in torch.distributed. + It is recommended to create `StatelessProcessGroup`, and then initialize + the data-plane communication (NCCL) between external (train processes) + and vLLM workers. + """ + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + from vllm.distributed.utils import StatelessProcessGroup + + pg = StatelessProcessGroup.create(host=master_address, port=master_port, rank=rank, world_size=world_size) + pynccl = PyNcclCommunicator(pg, device=device) + return pynccl diff --git a/kdflow/utils/dynamic_bsz.py b/kdflow/utils/dynamic_bsz.py new file mode 100644 index 0000000000000000000000000000000000000000..44ec3b3a8d3e7f423f3a68b951b7285c463c567b --- /dev/null +++ b/kdflow/utils/dynamic_bsz.py @@ -0,0 +1,398 @@ +"""Dynamic batch size utilities for KDFlow. + +Given a global_batch (list of micro-batches), rearrange samples into new +micro-batches such that each micro-batch's total token count is bounded by +max_token_len. + +The Karmarkar-Karp partitioning algorithm and first-fit estimation are adapted +from verl: + https://github.com/volcengine/verl/blob/main/verl/utils/seqlen_balancing.py + (Apache-2.0 License, Copyright 2024 Bytedance Ltd. and/or its affiliates) +""" + +import heapq +from typing import Dict, List, Optional, Tuple + +import ray +import torch +import numpy as np +import torch.distributed as dist +from kdflow.utils.logging_utils import init_logger +from torch.nn.utils.rnn import pad_sequence + +logger = init_logger(__name__) + +# Keys whose tensors are 2-D [batch, seq_len] and need unpad / repad +_SEQ_KEYS = {"stu_input_ids", "stu_attn_mask", "stu_loss_mask", + "tea_input_ids", "tea_attn_mask", "tea_loss_mask"} + +# Keys whose tensors are 1-D scalars per sample (e.g. response_length) +_SCALAR_KEYS = { + "response_length", + "total_length", + "kd_sample_weight", +} + +# Keys that are plain python lists (one element per sample) +_LIST_KEYS = {"tea_feed_input_ids", "stu_prompts", "stu_responses", "tea_prompts", + "labels", "images", "stu_multi_modal_inputs", "tea_multi_modal_inputs", + "teacher_routing_key"} + +# Keys whose values are concatenated response-level tensors [total_resp_tokens, ...] +# that must be split per sample using tea_loss_mask counts. +_CONCAT_HIDDEN_KEYS = {"teacher_hiddens"} + + +def rearrange_global_batch( + global_batch: List[Dict[str, torch.Tensor]], + max_token_len: int, + token_len_key: str = "stu_attn_mask", + dp_size: Optional[int] = None, + dp_group=None, +) -> List[Dict[str, torch.Tensor]]: + """Rearrange a global_batch into dynamic micro-batches. + + Steps: + 1. Unpack all micro-batches into individual samples (remove padding). + 2. Estimate the number of micro-batches needed via first-fit. + 3. Use Karmarkar-Karp to balance samples across micro-batches. + 4. Re-collate each partition with right-padding. + + Args: + global_batch: List of micro-batches, each is a dict of tensors. + max_token_len: Maximum total *real* tokens per micro-batch. + token_len_key: Key used to measure each sample's token count + (via attention_mask.sum()). + dp_size: If set, ensure the number of new micro-batches is divisible + by ``dp_size``. This is needed when the caller (e.g. off-policy / + on-policy KD trainer) distributes the rearranged global_batch + evenly across DP ranks via ``async_run_distill``. + dp_group: ``torch.distributed`` process group for data-parallel sync. + When provided, an ``all_reduce(MAX)`` is performed so that every + DP rank ends up with the same number of micro-batches (needed by + SFT trainer where each rank holds its own global_batch). + Ranks with fewer micro-batches are padded with dummy batches. + + Returns: + A new list of micro-batches (may differ in count from the input). + """ + # Step 1: unpack into single samples + samples = _unpack_global_batch(global_batch) + if not samples: + return global_batch + + # Step 2: compute per-sample token lengths + token_lens = [s[token_len_key].sum().item() for s in samples] + + # Step 3: estimate number of partitions via first-fit + num_partitions = _first_fit_num_partitions(token_lens, max_token_len) + num_partitions = max(num_partitions, 1) + num_partitions = min(num_partitions, len(samples)) + + # Step 3.1: ensure divisibility by dp_size (for off-policy / on-policy KD) + if dp_size is not None and dp_size > 1: + num_partitions = _roundup_divisible(num_partitions, dp_size) + num_partitions = min(num_partitions, len(samples)) + + # Step 3.2: cross-GPU sync (for SFT trainer where each rank has its own data) + if dp_group is not None and dist.is_initialized(): + num_partitions_t = torch.tensor([num_partitions], device=torch.cuda.current_device()) + dist.all_reduce(num_partitions_t, op=dist.ReduceOp.MAX, group=dp_group) + num_partitions = int(num_partitions_t.item()) + + # Step 4: balanced partitioning + if num_partitions >= len(samples): + # Each sample is its own micro-batch + partitions = [[i] for i in range(len(samples))] + else: + partitions = _karmarkar_karp(token_lens, num_partitions) + + # Step 5: re-collate each partition + new_global_batch = [] + for partition in partitions: + batch_samples = [samples[i] for i in partition] + micro_batch = _collate_samples(batch_samples) + new_global_batch.append(micro_batch) + + # Step 5.1: pad with dummy micro-batches if needed (for dp_group sync) + while len(new_global_batch) < num_partitions: + # Create a dummy micro-batch by cloning the last one with zeroed loss_mask + dummy = {k: v.clone() if isinstance(v, torch.Tensor) else v + for k, v in new_global_batch[-1].items()} + for k in ("stu_loss_mask", "tea_loss_mask"): + if k in dummy and isinstance(dummy[k], torch.Tensor): + dummy[k] = torch.zeros_like(dummy[k]) + new_global_batch.append(dummy) + + return new_global_batch + + +def unpack_global_batch(global_batch: List[Dict[str, torch.Tensor]]) -> List[Dict]: + """Public wrapper for converting collated micro-batches to single samples.""" + return _unpack_global_batch(global_batch) + + +def repartition_global_batch( + global_batch: List[Dict[str, torch.Tensor]], + num_partitions: int, + token_len_key: str = "stu_attn_mask", + pad_with_zero_weight: bool = False, +) -> List[Dict[str, torch.Tensor]]: + """Balance samples into an exact number of micro-batches. + + When there are fewer real samples than partitions, execution-only copies + can be appended with ``kd_sample_weight=0``. They keep distributed + collectives synchronized but contribute no gradient. + """ + if num_partitions <= 0: + raise ValueError(f"num_partitions must be positive, got {num_partitions}") + + samples = _unpack_global_batch(global_batch) + if not samples: + return [] + + real_partition_count = min(num_partitions, len(samples)) + if real_partition_count == len(samples): + partitions = [[index] for index in range(len(samples))] + else: + token_lens = [sample[token_len_key].sum().item() for sample in samples] + partitions = _karmarkar_karp(token_lens, real_partition_count) + + result = [ + _collate_samples([samples[index] for index in partition]) + for partition in partitions + ] + if len(result) < num_partitions and not pad_with_zero_weight: + raise ValueError( + f"Cannot create {num_partitions} non-empty partitions from " + f"{len(samples)} samples" + ) + + if len(result) < num_partitions: + source = min(result, key=lambda batch: batch[token_len_key].sum().item()) + while len(result) < num_partitions: + dummy = { + key: value.clone() if isinstance(value, torch.Tensor) else value + for key, value in source.items() + } + if "kd_sample_weight" not in dummy: + raise ValueError( + "Zero-weight padding requires `kd_sample_weight` in the batch" + ) + dummy["kd_sample_weight"] = torch.zeros_like(dummy["kd_sample_weight"]) + result.append(dummy) + + return result + + +def _unpack_global_batch( + global_batch: List[Dict[str, torch.Tensor]], +) -> List[Dict]: + """Unpack all micro-batches into a flat list of single-sample dicts. + + For sequence tensors (2-D), each sample is extracted and un-padded + (trailing padding removed based on attention_mask). + """ + samples = [] + for mb in global_batch: + # Determine batch size from any seq key present + bs = None + for k in _SEQ_KEYS: + if k in mb and isinstance(mb[k], torch.Tensor) and mb[k].dim() == 2: + bs = mb[k].shape[0] + break + if bs is None: + # Fallback: treat the whole micro-batch as one sample + samples.append(mb) + continue + + # Determine real lengths per sample from attention masks + stu_mask = mb.get("stu_attn_mask") + tea_mask = mb.get("tea_attn_mask") + stu_lens = stu_mask.sum(dim=1).tolist() if stu_mask is not None else [None] * bs + tea_lens = tea_mask.sum(dim=1).tolist() if tea_mask is not None else [None] * bs + + # Split concatenated hidden-state tensors by per-sample response token counts + tea_loss_mask = mb.get("tea_loss_mask") + per_sample_resp_lens = tea_loss_mask.bool().sum(dim=1).tolist() if tea_loss_mask is not None else None + split_hiddens = {} + for k in _CONCAT_HIDDEN_KEYS: + v = mb.get(k) + if v is None: + continue + if isinstance(v, ray.ObjectRef): + v = ray.get(v) + if isinstance(v, np.ndarray): + v = torch.from_numpy(v) + if per_sample_resp_lens is not None: + split_hiddens[k] = torch.split(v, per_sample_resp_lens, dim=0) + else: + split_hiddens[k] = [v] + + for i in range(bs): + sample = {} + for k, v in mb.items(): + if k in _SEQ_KEYS and isinstance(v, torch.Tensor) and v.dim() == 2: + # Determine the real length for this key + if k.startswith("tea_"): + real_len = int(tea_lens[i]) if tea_lens[i] is not None else v.shape[1] + else: + real_len = int(stu_lens[i]) if stu_lens[i] is not None else v.shape[1] + sample[k] = v[i, :real_len] # 1-D tensor, no padding + elif k in _SCALAR_KEYS and isinstance(v, torch.Tensor): + if v.dim() >= 1 and v.shape[0] == bs: + sample[k] = v[i] + else: + sample[k] = v + elif k in _LIST_KEYS and isinstance(v, list) and len(v) == bs: + sample[k] = v[i] + elif k in _CONCAT_HIDDEN_KEYS: + if k in split_hiddens and i < len(split_hiddens[k]): + sample[k] = split_hiddens[k][i] + # else: skip (no hidden for this sample) + elif k in {"avg_micro_batch_token_num", "avg_micro_batch_sample_num"}: + # Will be recomputed later; skip + continue + else: + # Unknown key: just copy as-is for the first sample + sample[k] = v + samples.append(sample) + + return samples + + +def _roundup_divisible(a: int, b: int) -> int: + """Round *a* up to the nearest multiple of *b*. + + Adapted from verl ``roundup_divisible``. + """ + return ((a + b - 1) // b) * b + + +def _first_fit_num_partitions(token_lens: List[int], max_token_len: int) -> int: + """Estimate the minimum number of partitions using first-fit decreasing.""" + sorted_lens = sorted(token_lens, reverse=True) + bins = [] + for l in sorted_lens: + placed = False + for i in range(len(bins)): + if bins[i] + l <= max_token_len: + bins[i] += l + placed = True + break + if not placed: + bins.append(l) + return len(bins) + + +def _karmarkar_karp( + seqlen_list: List[int], + k_partitions: int, +) -> List[List[int]]: + """Karmarkar-Karp multi-way partitioning (largest differencing method). + + Partitions ``seqlen_list`` into ``k_partitions`` groups such that the + maximum group-sum is minimized (i.e., balanced workload). + + Adapted from verl ``karmarkar_karp``: + https://github.com/volcengine/verl/blob/main/verl/utils/seqlen_balancing.py + + Returns: + List of k_partitions lists, each containing sample indices. + """ + + class _Set: + __slots__ = ("total", "items") + + def __init__(self): + self.total = 0 + self.items = [] # list of (idx, val) + + def add(self, idx: int, val: int): + self.items.append((idx, val)) + self.total += val + + def merge(self, other: "_Set"): + self.items.extend(other.items) + self.total += other.total + + def __lt__(self, other: "_Set"): + if self.total != other.total: + return self.total < other.total + if len(self.items) != len(other.items): + return len(self.items) < len(other.items) + return self.items < other.items + + class _State: + __slots__ = ("k", "sets") + + def __init__(self, items: List[Tuple[int, int]], k: int): + self.k = k + self.sets = [_Set() for _ in range(k)] + for i, (idx, val) in enumerate(items): + self.sets[i % k].add(idx, val) + self.sets.sort(reverse=True) + + @property + def spread(self) -> int: + return self.sets[0].total - self.sets[-1].total + + def merge(self, other: "_State"): + for i in range(self.k): + self.sets[i].merge(other.sets[self.k - 1 - i]) + self.sets.sort(reverse=True) + + def get_partitions(self) -> List[List[int]]: + return [sorted(idx for idx, _ in s.items) for s in self.sets] + + def __lt__(self, other: "_State"): + if self.spread != other.spread: + return self.spread > other.spread # max-heap by spread + return self.sets[0] > other.sets[0] + + sorted_items = sorted(enumerate(seqlen_list), key=lambda x: x[1]) + pq = [] + for idx, val in sorted_items: + heapq.heappush(pq, _State(items=[(idx, val)], k=k_partitions)) + + while len(pq) > 1: + s0 = heapq.heappop(pq) + s1 = heapq.heappop(pq) + s0.merge(s1) + heapq.heappush(pq, s0) + + partitions = pq[0].get_partitions() + # Remove empty partitions (shouldn't happen, but be safe) + partitions = [p for p in partitions if p] + return partitions + + +def _collate_samples(samples: List[Dict]) -> Dict[str, torch.Tensor]: + """Collate a list of single-sample dicts into one micro-batch dict. + + Sequence tensors are right-padded to the longest in the batch. + """ + batch = {} + keys = samples[0].keys() + + for k in keys: + values = [s[k] for s in samples if k in s] + if not values: + continue + + v0 = values[0] + if k in _SEQ_KEYS and isinstance(v0, torch.Tensor) and v0.dim() == 1: + # Right-pad variable-length 1-D tensors + pad_val = 0 + batch[k] = pad_sequence(values, batch_first=True, padding_value=pad_val) + elif k in _SCALAR_KEYS and isinstance(v0, torch.Tensor): + batch[k] = torch.stack(values) + elif k in _LIST_KEYS: + batch[k] = values + elif k in _CONCAT_HIDDEN_KEYS and isinstance(v0, torch.Tensor): + batch[k] = torch.cat(values, dim=0) + else: + # Default: keep first value (shared metadata) + batch[k] = v0 + + return batch diff --git a/kdflow/utils/logging_utils.py b/kdflow/utils/logging_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1393d26d3e95e7ac447bda58242866dac96d2af0 --- /dev/null +++ b/kdflow/utils/logging_utils.py @@ -0,0 +1,98 @@ +import logging +import sys + +_FORMAT = "[%(asctime)s] [%(levelname)s] [%(filename)s:%(funcName)s:%(lineno)s] %(message)s" +_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + +_LEVEL_COLORS = {"DEBUG": "36", "INFO": "32", "WARNING": "33", "ERROR": "31", "CRITICAL": "1;31"} +_DIM = "\033[2m" +_RESET = "\033[0m" + + +def define_wandb_metrics(wandb) -> None: + """Configure the shared step metric for all trainer metric namespaces.""" + wandb.define_metric("train/global_step") + for namespace in ("train", "distill", "rollout", "timing"): + wandb.define_metric( + f"{namespace}/*", + step_metric="train/global_step", + step_sync=True, + ) + wandb.define_metric("eval/global_step") + wandb.define_metric("eval/*", step_metric="eval/global_step", step_sync=True) + + +def normalize_eval_metrics(metrics): + """Convert trainer metric names and scalar values to the eval namespace.""" + normalized = {} + for key, value in metrics.items(): + key = key if key.startswith("eval/") else f"eval/{key.split('/', 1)[-1]}" + normalized[key] = value.item() if hasattr(value, "item") else float(value) + return normalized + + +def log_eval_metrics(strategy, wandb, metrics, global_step): + """Log evaluation metrics to the console and wandb.""" + metrics = normalize_eval_metrics(metrics) + if not metrics: + return metrics + + values = ", ".join(f"{key}: {value:.6f}" for key, value in sorted(metrics.items())) + strategy.log(f"Evaluation at global step {global_step}, {values}") + if wandb is not None: + wandb.log({"eval/global_step": global_step, **metrics}) + return metrics + + +class ColoredNewLineFormatter(logging.Formatter): + """Colored formatter that also aligns multi-line messages.""" + + def format(self, record): + c = _LEVEL_COLORS.get(record.levelname) + if c: + saved = record.levelname, record.filename, record.funcName, record.lineno + bold = "1;" if record.levelname == "CRITICAL" else "" + record.levelname = f"\033[{bold}{c}m{record.levelname}{_RESET}" + record.filename = f"{_DIM}{record.filename}" + record.funcName = f"{record.funcName}" + record.lineno = f"{saved[3]}{_RESET}" + msg = super().format(record) + record.levelname, record.filename, record.funcName, record.lineno = saved + else: + msg = super().format(record) + + if record.message != "": + parts = msg.split(record.message) + msg = msg.replace("\n", "\r\n" + parts[0]) + return msg + + +_root_logger = logging.getLogger("kdflow") +_default_handler = None + + +def _setup_logger(): + _root_logger.setLevel(logging.DEBUG) + global _default_handler + if _default_handler is None: + _default_handler = logging.StreamHandler(sys.stdout) + _default_handler.flush = sys.stdout.flush # type: ignore + _default_handler.setLevel(logging.INFO) + _root_logger.addHandler(_default_handler) + fmt = ColoredNewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT) + _default_handler.setFormatter(fmt) + # Setting this will avoid the message + # being propagated to the parent logger. + _root_logger.propagate = False + + +_setup_logger() + + +def init_logger(name: str): + # Use the same settings as above for root logger + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(_default_handler) + logger.propagate = False + return logger diff --git a/kdflow/utils/multimodal_utils.py b/kdflow/utils/multimodal_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..60acb817ed6ce60fda693c80ec1295376c15b10b --- /dev/null +++ b/kdflow/utils/multimodal_utils.py @@ -0,0 +1,31 @@ +"""Multi-modal field helpers (verl-inspired).""" +from typing import Iterable, Optional + +import torch + + +def extract_multi_modal_inputs( + multi_modal_inputs_list: Optional[Iterable[Optional[dict]]], +) -> dict: + """Concat per-sample mm dicts into a batched dict for HF VLM forward. + + Args: + multi_modal_inputs_list: an iterable of per-sample multi_modal_inputs + dicts (each dict maps field name -> tensor with leading dim + corresponding to that sample's patches/tokens). ``None`` entries + (e.g. text-only samples) are skipped. + + Returns: + A dict mapping field name -> tensor concatenated along ``dim=0`` + across all non-empty samples. Empty input returns ``{}``. + """ + if not multi_modal_inputs_list: + return {} + collected: dict = {} + for d in multi_modal_inputs_list: + if not d: + continue + for k, v in d.items(): + if v is not None: + collected.setdefault(k, []).append(v) + return {k: torch.cat(vs, dim=0) for k, vs in collected.items()} diff --git a/kdflow/utils/utils.py b/kdflow/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..26415ed5185e779a2945b98fa6f368a726dd286d --- /dev/null +++ b/kdflow/utils/utils.py @@ -0,0 +1,91 @@ +import runpy +from typing import List + +import torch +import torch.nn.functional as F +from transformers import AutoTokenizer + +from kdflow.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def load_custom_eval_fn(path): + """Load the required eval_fn from a custom Python file.""" + eval_fn = runpy.run_path(path).get("eval_fn") + if not callable(eval_fn): + raise ValueError( + f"{path!r} must define a callable eval_fn(predictions, labels)" + ) + return eval_fn + + +def get_tokenizer(model_name_or_path, model=None, padding_side="left", use_fast=True): + try: + # fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style + # tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it + # with the Metaspace pre-tokenizer rebuilt from the slow tokenizer. + tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, + trust_remote_code=True, + use_fast=use_fast, + fix_mistral_regex=True, + ) + except TypeError: + # Older transformers versions do not accept fix_mistral_regex. + tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast) + tokenizer.padding_side = padding_side + # NOTE: When enable vLLM, do not resize_token_embeddings, or the vocab size will mismatch with vLLM. + # https://github.com/facebookresearch/llama-recipes/pull/196 + if tokenizer.pad_token is None: + logger.info("Detect no pad_token in tokenizer, set it to eos_token.") + tokenizer.pad_token = tokenizer.eos_token + tokenizer.pad_token_id = tokenizer.eos_token_id + if model is not None: + model.config.pad_token_id = tokenizer.pad_token_id + + return tokenizer + + +def convert_token_to_id(token, tokenizer): + if isinstance(token, str): + token = tokenizer.encode(token, add_special_tokens=False) + assert len(token) == 1 + return token[0] + else: + raise ValueError("token should be int or str") + + +def zero_pad_sequences( + sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False +) -> torch.Tensor: + from torch.nn.utils.rnn import pad_sequence + assert side in ("left", "right") + sequences = [seq.squeeze(0) for seq in sequences] + if side == "left": + sequences = [seq.flip(dims=0) for seq in sequences] + padded_sequences = pad_sequence( + sequences, + batch_first=True, + padding_value=value + ) + if side == "left": + padded_sequences = torch.flip(padded_sequences, dims=[1]) + return padded_sequences + + +def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True): + """Remove the pad token. Return tensors and not lists. + + Args: + input_ids shape: [bs, seq_length] + attention_mask shape: [bs, seq_length] + Returns: + no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query. + """ + no_padding_batch = [] + for ids, mask in zip(input_ids, attention_mask): + # Fix for both left and right padding + ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist() + no_padding_batch.append(ids) + return no_padding_batch diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..dd4bfd646653bdc98b8f1bc8aa753a996932166c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "kdflow" +version = "0.1.0" +description = "Byte-aligned cross-tokenizer knowledge distillation" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.10" +dynamic = ["dependencies"] + +[project.optional-dependencies] +test = [ + "pytest>=8", +] +lora = [ + "peft", +] +alfworld = [ + "alfworld", +] +all = [ + "peft", + "wandb", + "triton", + "flash_attn", + "ring_flash_attn", +] + +[tool.setuptools] +license-files = ["LICENSE", "THIRD_PARTY_NOTICES.md", "LICENSES/*.txt"] + +[tool.setuptools.packages.find] +include = ["kdflow*"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..678e55b3cd878b4364ed7bd9d54ac1cf2058af80 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,333 @@ +accelerate==1.13.0 +addict==2.4.0 +aiohappyeyeballs==2.6.2 +aiohttp==3.13.5 +aiosignal==1.4.0 +airportsdata==20260315 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.104.1 +antlr4-python3-runtime==4.9.3 +anyio==4.13.0 +apache-tvm-ffi==0.1.9 +appdirs==1.4.4 +argcomplete==3.6.3 +asttokens==3.0.1 +attrs==26.1.0 +av==16.1.0 +azure-core==1.41.0 +azure-identity==1.25.3 +azure-storage-blob==12.29.0 +black==26.5.1 +blinker==1.7.0 +blobfile==3.0.0 +boto3==1.43.14 +botocore==1.43.14 +build==1.5.0 +cache_dit==1.3.0 +certifi==2026.5.20 +cffi==2.0.0 +cfgv==3.5.0 +charset-normalizer==3.4.7 +click==8.4.1 +cloudpickle==3.1.2 +compressed-tensors==0.15.0.1 +contourpy==1.3.3 +cryptography==41.0.7 +cubloaty==0.1.0b3 +cuda-bindings==12.9.4 +cuda-pathfinder==1.2.2 +cuda-python==12.9.0 +cuda-tile==1.3.0 +cuda-toolkit==12.9.1 +cycler==0.12.1 +datamodel-code-generator==0.57.0 +datasets==4.8.5 +dbus-python==1.3.2 +decorator==5.3.1 +deep_ep==1.2.1 +devscripts==2.23.7+ubuntu0.2 +diffusers==0.37.0 +dill==0.4.1 +diskcache==5.6.3 +distlib==0.4.0 +distro==1.9.0 +docstring_parser==0.18.0 +easydict==1.13 +einops==0.8.2 +executing==2.2.1 +fastapi==0.136.1 +filelock==3.29.0 +fla-core==0.5.0 +flash_attn==2.8.3 +flash-attn-4==4.0.0b14 +flash-linear-attention==0.5.0 +flash_mla==1.0.0+9241ae3 +flashinfer-cubin==0.6.11.post1 +flashinfer-jit-cache==0.6.11.post1+cu129 +flashinfer-python==0.6.11.post1 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2026.2.0 +genson==1.3.0 +gguf==0.19.0 +gitdb==4.0.12 +GitPython==3.1.50 +google-api-core==2.30.3 +google-auth==2.53.0 +google-cloud-core==2.6.0 +google-cloud-storage==3.10.1 +google-crc32c==1.8.0 +google-resumable-media==2.9.0 +googleapis-common-protos==1.75.0 +granian==2.7.4 +grpcio==1.80.0 +grpcio-health-checking==1.80.0 +grpcio-reflection==1.80.0 +h11==0.16.0 +hf-xet==1.5.0 +html5lib==1.1 +httpcore==1.0.9 +httplib2==0.20.4 +httpx==0.28.1 +huggingface_hub==1.16.1 +humanize==4.15.0 +icdiff==2.0.10 +identify==2.6.19 +idna==3.16 +imageio==2.36.0 +imageio-ffmpeg==0.5.1 +importlib_metadata==9.0.0 +inflect==7.5.0 +iniconfig==2.3.0 +interegular==0.3.3 +ipython==9.13.0 +ipython_pygments_lexers==1.1.1 +isodate==0.7.2 +isort==8.0.1 +jedi==0.20.0 +Jinja2==3.1.6 +jiter==0.15.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +kernels==0.14.1 +kernels-data==0.14.1 +kiwisolver==1.5.0 +lark==1.3.1 +latex2sympy2_extended==1.11.0 +launchpadlib==1.11.0 +lazr.restfulclient==0.14.6 +lazr.uri==1.0.6 +lazy-loader==0.5 +llguidance==0.7.30 +loguru==0.7.3 +lxml==6.1.1 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +math-verify==0.9.0 +matplotlib==3.10.9 +matplotlib-inline==0.2.2 +mdurl==0.1.2 +mistral_common==1.11.2 +ml_dtypes==0.5.4 +modelscope==1.37.1 +mooncake-transfer-engine==0.3.10.post2 +more-itertools==11.1.0 +moviepy==2.2.1 +mpmath==1.3.0 +msal==1.36.0 +msal-extensions==1.3.1 +msgpack==1.1.2 +msgspec==0.21.1 +multidict==6.7.1 +multiprocess==0.70.19 +mypy_extensions==1.1.0 +nest-asyncio==1.6.0 +networkx==3.6.1 +ninja==1.13.0 +nixl==1.1.0 +nixl-cu12==1.1.0 +nodeenv==1.10.0 +numpy==2.4.4 +nvidia-cublas==13.1.0.3 +nvidia-cublas-cu12==12.9.1.4 +nvidia-cuda-cupti==13.0.85 +nvidia-cuda-cupti-cu12==12.9.79 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-nvrtc-cu12==12.9.86 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-runtime-cu12==12.9.79 +nvidia-cudnn-cu12==9.17.1.4 +nvidia-cudnn-frontend==1.24.0 +nvidia-cufft==12.0.0.61 +nvidia-cufft-cu12==11.4.1.4 +nvidia-cufile==1.15.1.6 +nvidia-cufile-cu12==1.14.1.1 +nvidia-curand==10.4.0.35 +nvidia-curand-cu12==10.3.10.19 +nvidia-cusolver==12.0.4.66 +nvidia-cusolver-cu12==11.7.5.82 +nvidia-cusparse==12.6.3.3 +nvidia-cusparse-cu12==12.5.10.65 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.5.1 +nvidia-cutlass-dsl-libs-base==4.5.1 +nvidia-ml-py==13.595.45 +nvidia-modelopt==0.44.0 +nvidia-nccl-cu12==2.28.9 +nvidia-nvjitlink==13.0.88 +nvidia-nvjitlink-cu12==12.9.86 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvtx==13.0.85 +nvidia-nvtx-cu12==12.9.79 +oauthlib==3.2.2 +omegaconf==2.3.0 +openai==2.6.1 +openai-harmony==0.0.4 +opencv-python-headless==4.10.0.84 +opentelemetry-api==1.42.1 +opentelemetry-exporter-otlp==1.42.1 +opentelemetry-exporter-otlp-proto-common==1.42.1 +opentelemetry-exporter-otlp-proto-grpc==1.42.1 +opentelemetry-exporter-otlp-proto-http==1.42.1 +opentelemetry-proto==1.42.1 +opentelemetry-sdk==1.42.1 +opentelemetry-semantic-conventions==0.63b1 +orjson==3.11.9 +outlines==0.1.11 +outlines_core==0.1.26 +packaging==26.2 +pandas==3.0.3 +parso==0.8.7 +partial-json-parser==0.2.1.1.post7 +pathspec==1.1.1 +peft==0.19.1 +pexpect==4.9.0 +pillow==12.2.0 +pip==26.1.1 +platformdirs==4.9.6 +pluggy==1.6.0 +pre_commit==4.6.0 +proglog==0.1.12 +prometheus_client==0.25.0 +prompt_toolkit==3.0.52 +propcache==0.5.2 +proto-plus==1.28.0 +protobuf==6.33.6 +psutil==7.2.2 +ptyprocess==0.7.0 +PuLP==3.3.1 +pure_eval==0.2.3 +py-spy==0.4.2 +pyarrow==24.0.0 +pyasn1==0.6.3 +pyasn1_modules==0.4.2 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pycryptodomex==3.23.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +pydantic-extra-types==2.11.1 +Pygments==2.20.0 +PyGObject==3.48.2 +PyJWT==2.7.0 +pyparsing==3.1.1 +pyproject_hooks==1.2.0 +pytest==9.0.3 +python-apt==2.7.7+ubuntu5.2 +python-dateutil==2.9.0.post0 +python-discovery==1.3.1 +python-dotenv==1.2.2 +python-multipart==0.0.29 +pytokens==0.4.1 +PyYAML==6.0.1 +pyzmq==27.1.0 +quack-kernels==0.4.1 +ray==2.55.1 +referencing==0.37.0 +regex==2026.5.9 +remote-pdb==2.1.0 +requests==2.34.2 +rich==15.0.0 +ring-flash-attn==0.1.8 +rpds-py==0.30.0 +runai-model-streamer==0.15.9 +runai-model-streamer-azure==0.15.9 +runai-model-streamer-gcs==0.15.9 +runai-model-streamer-s3==0.15.9 +s3transfer==0.17.0 +safetensors==0.7.0 +scikit_build_core==0.12.2 +scikit-image==0.25.2 +scipy==1.17.1 +sentencepiece==0.2.1 +sentry-sdk==2.61.1 +setproctitle==1.3.7 +setuptools==70.2.0 +sgl-deep-gemm==0.1.0+cu129 +sglang==0.5.12.post1 +sglang-kernel==0.4.2.post2+cu129 +sglang-router==0.3.2 +shellingham==1.5.4 +six==1.17.0 +smg-grpc-proto==0.4.7 +smg-grpc-servicer==0.5.3 +smmap==5.0.3 +sniffio==1.3.1 +soundfile==0.13.1 +st_attn==0.0.7 +stack-data==0.6.3 +starlette==1.0.1 +sympy==1.14.0 +tabulate==0.10.0 +tempdir==0.7.1 +termcolor==3.3.0 +termplotlib==0.3.9 +tifffile==2026.5.15 +tiktoken==0.13.0 +tilelang==0.1.8 +timm==1.0.16 +tokenizers==0.22.2 +tokenspeed-mla==0.1.1 +tokenspeed-triton==3.7.10.post20260505 +tomlkit==0.15.0 +torch==2.11.0+cu129 +torch_c_dlpack_ext==0.1.5 +torch_memory_saver==0.0.9.post1 +torchao==0.17.0+cu129 +torchaudio==2.11.0+cu129 +torchcodec==0.11.1+cu129 +torchdata==0.11.0 +torchvision==0.26.0+cu129 +tqdm==4.67.3 +traitlets==5.15.0 +transformers==5.6.2 +tree-sitter==0.26.0 +tree-sitter-python==0.25.0 +trimesh==4.12.2 +triton==3.6.0 +typeguard==4.5.2 +typer==0.25.1 +typing_extensions==4.15.0 +typing-inspection==0.4.2 +urllib3==2.7.0 +uv==0.11.16 +uvicorn==0.47.0 +uvloop==0.22.1 +virtualenv==21.3.3 +vsa==0.0.4 +wadllib==1.3.6 +wandb==0.27.0 +watchfiles==1.2.0 +wcwidth==0.7.0 +webencodings==0.5.1 +wget==3.2 +wheel==0.47.0 +xatlas==0.0.11 +xgrammar==0.2.0 +xxhash==3.7.0 +yarl==1.24.2 +z3-solver==4.15.4.0 +zipp==4.1.0 diff --git a/run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh b/run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh new file mode 100644 index 0000000000000000000000000000000000000000..c778963c3e42749205a2fc9b0208bfb076a48180 --- /dev/null +++ b/run/byte_align_ctkd/qwen25_7b_to_llama32_3b_byte_align_ctkd.sh @@ -0,0 +1,76 @@ +set -e +set -x +set -o pipefail + + +# ============ Logging Path ============ +TRAIN_NAME="qwen25_7b_to_llama32_3b_byte_align_ctkd" +SAVE_PATH="./outputs/${TRAIN_NAME}" +LOG_DIR="${SAVE_PATH}/logs" +mkdir -p "${LOG_DIR}" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_FILE="${LOG_DIR}/train_${TIMESTAMP}.log" +echo "Training log will be written to: ${LOG_FILE}" + +export PYTHONPATH="/workspace/Cross-Tokenizer-OPD:${PYTHONPATH}" +# ============ TrainingArguments ============ +OPTS="" +OPTS+=" --num_nodes 1" +OPTS+=" --num_gpus_per_node 8" +OPTS+=" --backend fsdp2" +OPTS+=" --train_batch_size 512" +OPTS+=" --micro_train_batch_size 4" +OPTS+=" --learning_rate 1e-6" +OPTS+=" --lr_scheduler constant" +OPTS+=" --num_epochs 10" +OPTS+=" --max_steps 100" +OPTS+=" --save_path ${SAVE_PATH}" +OPTS+=" --bf16 True" +OPTS+=" --gradient_checkpointing True" +OPTS+=" --enable_sleep True" + +# ============ ModelArguments ============ +OPTS+=" --student_name_or_path /workspace/models/Llama-3.2-3B-Instruct" +OPTS+=" --teacher_name_or_path /workspace/models/Qwen2.5-7B-Instruct" +OPTS+=" --enable_thinking False" + +# ============ RolloutArguments ============ +OPTS+=" --rollout_batch_size 512" +OPTS+=" --rollout_num_engines 8" +OPTS+=" --rollout_tp_size 1" +OPTS+=" --rollout_mem_fraction_static 0.6" +OPTS+=" --n_samples_per_prompt 1" +OPTS+=" --generate_max_len 8192" +OPTS+=" --temperature 1.0" +OPTS+=" --top_p 0.95" + +# ============ DataArguments ============ +OPTS+=" --train_dataset_path /workspace/data/Train/math_code_20k/train.parquet" +OPTS+=" --max_len 10240" +OPTS+=" --input_key prompt" +OPTS+=" --label_key reward_model.ground_truth" +OPTS+=" --apply_chat_template True" +OPTS+=" --preprocess_num_workers 32" +OPTS+=" --packing_samples True" +OPTS+=" --eval_math_dataset_path /workspace/data/math500/test.jsonl" +OPTS+=" --eval_math_max_tokens 8192" +OPTS+=" --eval_math_max_concurrent 256" +OPTS+=" --eval_math_timeout 2400" + +# ============ DistillationArguments ============ +OPTS+=" --kd_ratio 1.0" +OPTS+=" --kd_loss_fn rkl" +OPTS+=" --kd_algorithm byte_align_ctkd" +OPTS+=" --teacher_dp_size 4" +OPTS+=" --teacher_tp_size 2" +OPTS+=" --teacher_mem_fraction_static 0.6" +OPTS+=" --chunked_loss_size 2048" + +# ============ LoggingArguments ============ +OPTS+=" --logging_steps 1" +OPTS+=" --use_wandb False" +OPTS+=" --save_steps 10" +OPTS+=" --eval_math_n_samples 32" +OPTS+=" --eval_steps 10" + +python -u -m kdflow.cli.train_kd_on_policy $OPTS 2>&1 | tee -a "${LOG_FILE}" diff --git a/run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh b/run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh new file mode 100644 index 0000000000000000000000000000000000000000..2997aaaebae9914d30b62cca821bce7ceef7818d --- /dev/null +++ b/run/topk_byte_align_ctkd/qwen25_7b_to_llama32_3b_topk_byte_align_ctkd.sh @@ -0,0 +1,77 @@ +set -e +set -x +set -o pipefail + + +# ============ Logging Path ============ +TRAIN_NAME="qwen25_7b_to_llama32_3b_topk_byte_align_ctkd_16" +SAVE_PATH="./outputs/${TRAIN_NAME}" +LOG_DIR="${SAVE_PATH}/logs" +mkdir -p "${LOG_DIR}" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_FILE="${LOG_DIR}/train_${TIMESTAMP}.log" +echo "Training log will be written to: ${LOG_FILE}" + +export PYTHONPATH="/workspace/Cross-Tokenizer-OPD:${PYTHONPATH}" +# ============ TrainingArguments ============ +OPTS="" +OPTS+=" --num_nodes 1" +OPTS+=" --num_gpus_per_node 8" +OPTS+=" --backend fsdp2" +OPTS+=" --train_batch_size 512" +OPTS+=" --micro_train_batch_size 4" +OPTS+=" --learning_rate 1e-6" +OPTS+=" --lr_scheduler constant" +OPTS+=" --num_epochs 10" +OPTS+=" --max_steps 100" +OPTS+=" --save_path ${SAVE_PATH}" +OPTS+=" --bf16 True" +OPTS+=" --gradient_checkpointing True" +OPTS+=" --enable_sleep True" + +# ============ ModelArguments ============ +OPTS+=" --student_name_or_path /workspace/models/Llama-3.2-3B-Instruct" +OPTS+=" --teacher_name_or_path /workspace/models/Qwen2.5-7B-Instruct" +OPTS+=" --enable_thinking False" + +# ============ RolloutArguments ============ +OPTS+=" --rollout_batch_size 512" +OPTS+=" --rollout_num_engines 8" +OPTS+=" --rollout_tp_size 1" +OPTS+=" --rollout_mem_fraction_static 0.6" +OPTS+=" --n_samples_per_prompt 1" +OPTS+=" --generate_max_len 8192" +OPTS+=" --temperature 1.0" +OPTS+=" --top_p 0.95" + +# ============ DataArguments ============ +OPTS+=" --train_dataset_path /workspace/data/Train/math_code_20k/train.parquet" +OPTS+=" --max_len 10240" +OPTS+=" --input_key prompt" +OPTS+=" --label_key reward_model.ground_truth" +OPTS+=" --apply_chat_template True" +OPTS+=" --preprocess_num_workers 32" +OPTS+=" --packing_samples True" +OPTS+=" --eval_math_dataset_path /workspace/data/math500/test.jsonl" +OPTS+=" --eval_math_max_tokens 8192" +OPTS+=" --eval_math_max_concurrent 256" +OPTS+=" --eval_math_timeout 2400" + +# ============ DistillationArguments ============ +OPTS+=" --kd_ratio 1.0" +OPTS+=" --kd_loss_fn rkl" +OPTS+=" --kd_algorithm topk_byte_align_ctkd" +OPTS+=" --ctkd_topk 16" +OPTS+=" --teacher_dp_size 4" +OPTS+=" --teacher_tp_size 2" +OPTS+=" --teacher_mem_fraction_static 0.6" +OPTS+=" --chunked_loss_size 2048" + +# ============ LoggingArguments ============ +OPTS+=" --logging_steps 1" +OPTS+=" --use_wandb False" +OPTS+=" --save_steps 10" +OPTS+=" --eval_math_n_samples 32" +OPTS+=" --eval_steps 10" + +python -u -m kdflow.cli.train_kd_on_policy $OPTS 2>&1 | tee -a "${LOG_FILE}"