ctopd / kdflow /arguments /__init__.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
13.3 kB
import os
from dataclasses import dataclass, field
from transformers import HfArgumentParser
from kdflow.arguments.data_args import DataArguments
from kdflow.arguments.model_args import ModelArguments
from kdflow.arguments.training_args import TrainingArguments
from kdflow.arguments.fsdp_args import FSDPArguments
from kdflow.arguments.distillation_args import DistillationArguments
from kdflow.arguments.rollout_args import RolloutArguments
from kdflow.arguments.logging_args import LoggingArguments
from kdflow.utils.logging_utils import init_logger
logger = init_logger(__name__)
@dataclass
class AllArguments:
data: DataArguments = field(default_factory=DataArguments)
model: ModelArguments = field(default_factory=ModelArguments)
train: TrainingArguments = field(default_factory=TrainingArguments)
fsdp: FSDPArguments = field(default_factory=FSDPArguments)
kd: DistillationArguments = field(default_factory=DistillationArguments)
rollout: RolloutArguments = field(default_factory=RolloutArguments)
log: LoggingArguments = field(default_factory=LoggingArguments)
def is_linear_attention(model_path: str) -> bool:
if not model_path:
return False
try:
from transformers import AutoConfig
model_type = AutoConfig.from_pretrained(model_path, trust_remote_code=True).model_type or ""
except Exception:
return False
return any(k in model_type.lower() for k in ("qwen3_next", "qwen3_5"))
def init_args(scenario: str = "on_policy_kd"):
parser = HfArgumentParser((
DataArguments,
ModelArguments,
TrainingArguments,
FSDPArguments,
DistillationArguments,
RolloutArguments,
LoggingArguments
))
(
data_args,
model_args,
train_args,
fsdp_args,
kd_args,
rollout_args,
log_args
) = parser.parse_args_into_dataclasses()
args = AllArguments(
data=data_args,
model=model_args,
train=train_args,
fsdp=fsdp_args,
kd=kd_args,
rollout=rollout_args,
log=log_args
)
if scenario != "on_policy_kd" and args.data.custom_eval_fn is not None:
logger.warning(
f"`--custom_eval_fn` is only supported in the `on_policy_kd` scenario "
f"and will be ignored for `{scenario}`."
)
if scenario != "on_policy_kd" and args.data.eval_math_dataset_path is not None:
logger.warning(
f"`--eval_math_dataset_path` is only supported in the `on_policy_kd` "
f"scenario and will be ignored for `{scenario}`."
)
if args.data.input_template and "{}" not in args.data.input_template:
logger.warning("{} not in args.data.input_template, set to None")
args.data.input_template = None
if args.data.input_template and "\\n" in args.data.input_template:
logger.warning(
"input_template contains \\n characters instead of newline. "
"You likely want to pass $'\\n' in Bash or \"`n\" in PowerShell."
)
if args.data.packing_samples:
if "flash_attention" not in args.model.attn_implementation:
logger.warning(
"Please use --attn_implementation with flash_attention to accelerate when --packing_samples is enabled."
)
args.model.attn_implementation = "flash_attention_2"
if is_linear_attention(args.model.student_name_or_path):
logger.warning(
f"--packing_samples is not yet compatible with linear-attention models "
f"(got {args.model.student_name_or_path}). Auto-disabling --packing_samples."
)
args.data.packing_samples = False
total_gpus = args.train.num_nodes * args.train.num_gpus_per_node
if scenario == "alfworld_eval":
if not args.model.student_name_or_path:
raise ValueError(
"Standalone ALFWorld evaluation requires "
"`--student_name_or_path`."
)
if not args.rollout.alfworld_config:
raise ValueError(
"Standalone ALFWorld evaluation requires `--alfworld_config`."
)
valid_protocols = {"kdflow", "verl_agent"}
if args.rollout.alfworld_eval_protocol not in valid_protocols:
raise ValueError(
"`--alfworld_eval_protocol` must be one of "
f"{sorted(valid_protocols)}."
)
if args.rollout.alfworld_eval_protocol == "verl_agent":
try:
seeds = [
int(value.strip())
for value in args.rollout.alfworld_eval_seeds.split(",")
if value.strip()
]
except ValueError as error:
raise ValueError(
"`--alfworld_eval_seeds` must be a comma-separated list "
"of integers."
) from error
if not seeds:
raise ValueError(
"`--alfworld_eval_seeds` must contain at least one seed."
)
if args.rollout.alfworld_history_length < 0:
raise ValueError(
"`--alfworld_history_length` must be non-negative."
)
if scenario in ("on_policy_kd", "alfworld_eval"):
if total_gpus <= 0:
raise ValueError(
"`--num_nodes * --num_gpus_per_node` must be positive."
)
if args.rollout.rollout_tp_size <= 0:
raise ValueError("`--rollout_tp_size` must be positive.")
if args.rollout.rollout_batch_size <= 0:
raise ValueError("`--rollout_batch_size` must be positive.")
if args.rollout.rollout_engine_concurrency <= 0:
raise ValueError(
"`--rollout_engine_concurrency` must be positive."
)
if args.rollout.alfworld_config:
valid_alfworld_protocols = {"kdflow", "verl_agent"}
if args.rollout.alfworld_protocol not in valid_alfworld_protocols:
raise ValueError(
"`--alfworld_protocol` must be one of "
f"{sorted(valid_alfworld_protocols)}."
)
args.rollout.alfworld_config = os.path.abspath(
os.path.expanduser(args.rollout.alfworld_config)
)
if not os.path.isfile(args.rollout.alfworld_config):
raise ValueError(
"`--alfworld_config` must point to an existing YAML file, "
f"got {args.rollout.alfworld_config!r}."
)
valid_alfworld_splits = {
"train",
"eval_in_distribution",
"eval_out_of_distribution",
}
if args.rollout.alfworld_split not in valid_alfworld_splits:
raise ValueError(
"`--alfworld_split` must be one of "
f"{sorted(valid_alfworld_splits)}."
)
if args.rollout.alfworld_max_turns <= 0:
raise ValueError("`--alfworld_max_turns` must be positive.")
if args.rollout.alfworld_episodes_per_epoch <= 0:
raise ValueError(
"`--alfworld_episodes_per_epoch` must be positive."
)
if args.rollout.alfworld_eval_episodes <= 0:
raise ValueError("`--alfworld_eval_episodes` must be positive.")
if args.rollout.alfworld_admissible_limit < 0:
raise ValueError(
"`--alfworld_admissible_limit` must be non-negative."
)
if args.rollout.n_samples_per_prompt != 1:
raise ValueError(
"ALFWorld multi-turn rollout requires "
"`--n_samples_per_prompt 1`; the environment already "
"samples one independent game per batch slot."
)
if scenario == "on_policy_kd" and (
args.rollout.alfworld_episodes_per_epoch
< args.rollout.rollout_batch_size
):
raise ValueError(
"`--alfworld_episodes_per_epoch` must be at least "
"`--rollout_batch_size`."
)
remainder = (
args.rollout.alfworld_episodes_per_epoch
% args.rollout.rollout_batch_size
)
if scenario == "on_policy_kd" and remainder:
logger.warning(
"ALFWorld drops %d episode slots per epoch because "
"alfworld_episodes_per_epoch is not divisible by "
"rollout_batch_size.",
remainder,
)
if scenario == "on_policy_kd" and args.data.eval_dataset_path:
logger.warning(
"`--eval_dataset_path` is ignored for ALFWorld multi-turn "
"training; use an ALFWorld evaluation split in a separate run."
)
if (
scenario == "alfworld_eval"
and args.rollout.alfworld_eval_episodes
% args.rollout.rollout_batch_size
):
raise ValueError(
"`--alfworld_eval_episodes` must be divisible by "
"`--rollout_batch_size`."
)
if scenario == "on_policy_kd":
if args.data.eval_math_n_samples <= 0:
raise ValueError("`--eval_math_n_samples` must be positive.")
if args.data.eval_math_max_concurrent <= 0:
raise ValueError("`--eval_math_max_concurrent` must be positive.")
if args.data.eval_math_max_tokens <= 0:
raise ValueError("`--eval_math_max_tokens` must be positive.")
if args.data.eval_math_timeout < 0:
raise ValueError("`--eval_math_timeout` must be non-negative.")
if total_gpus % args.rollout.rollout_tp_size != 0:
raise ValueError(
f"Total GPUs ({total_gpus}) must be divisible by rollout_tp_size ({args.rollout.rollout_tp_size})."
)
expected_num_engines = total_gpus // args.rollout.rollout_tp_size
if args.rollout.rollout_num_engines != expected_num_engines:
logger.warning(
f"Auto-adjusting rollout_num_engines from {args.rollout.rollout_num_engines} to {expected_num_engines} "
f"to match total GPUs ({total_gpus}). "
f"(rollout_tp_size={args.rollout.rollout_tp_size} * rollout_num_engines={expected_num_engines} = {total_gpus})"
)
args.rollout.rollout_num_engines = expected_num_engines
if args.data.max_len < args.data.prompt_max_len + args.rollout.generate_max_len:
args.data.max_len = args.data.prompt_max_len + args.rollout.generate_max_len
logger.warning(
"--max_len is smaller than --prompt_max_len + --generate_max_len. "
f"Automatically increase --max_len to {args.data.max_len}."
)
has_teacher = args.model.teacher_name_or_path is not None
is_kd_scenario = scenario in ("off_policy_kd", "on_policy_kd")
if is_kd_scenario:
if not has_teacher:
raise ValueError(
"KD training requires `--teacher_name_or_path` to be set."
)
teacher_parallel = args.kd.teacher_tp_size * args.kd.teacher_pp_size
if total_gpus % teacher_parallel != 0:
raise ValueError(
f"Total GPUs ({total_gpus}) must be divisible by "
f"teacher_tp_size * teacher_pp_size ({args.kd.teacher_tp_size} * {args.kd.teacher_pp_size} = {teacher_parallel})."
)
if args.kd.teacher_ep_size > args.kd.teacher_tp_size:
logger.warning(
f"teacher_ep_size ({args.kd.teacher_ep_size}) must not be greater than teacher_tp_size ({args.kd.teacher_tp_size}). "
f"Auto-adjusting teacher_ep_size to {args.kd.teacher_tp_size}."
)
args.kd.teacher_ep_size = args.kd.teacher_tp_size
expected_dp = total_gpus // teacher_parallel
if args.kd.teacher_dp_size != expected_dp:
logger.warning(
f"Auto-adjusting teacher_dp_size from {args.kd.teacher_dp_size} to {expected_dp} "
f"to match total GPUs ({total_gpus}). "
f"(tp={args.kd.teacher_tp_size} (ep={args.kd.teacher_ep_size}) * pp={args.kd.teacher_pp_size} * dp={expected_dp} = {total_gpus})"
)
args.kd.teacher_dp_size = expected_dp
deprecated_sleep_flags = [
args.train.train_enable_sleep,
args.kd.teacher_enable_sleep,
args.rollout.rollout_enable_sleep,
]
if any(deprecated_sleep_flags):
logger.warning(
"--train_enable_sleep, --teacher_enable_sleep, --rollout_enable_sleep are deprecated "
"and will be removed in a future version. Use --enable_sleep instead."
)
args.train.enable_sleep = True
args.scenario = scenario
return args