Download kdflow/cli/evaluate_alfworld.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 4.05 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/cli/evaluate_alfworld.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/cli/evaluate_alfworld.py
-
curl -L -o evaluate_alfworld.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/cli/evaluate_alfworld.py
4.05 kB
| """Evaluate a model checkpoint in ALFWorld without starting training actors.""" | |
| import json | |
| import logging | |
| import os | |
| from kdflow.evaluation import AlfWorldEvaluator, VerlAgentAlfWorldEvaluator | |
| logger = logging.getLogger(__name__) | |
| class _EvaluationStrategy: | |
| """Small args/log adapter required by the shared rollout manager.""" | |
| def __init__(self, args): | |
| self.args = args | |
| def log(message, level="info", **_kwargs): | |
| getattr(logger, level, logger.info)(message) | |
| def print(*values): | |
| logger.info(" ".join(str(value) for value in values)) | |
| def evaluate(args): | |
| if args.rollout.alfworld_eval_protocol == "verl_agent": | |
| preflight_evaluator = VerlAgentAlfWorldEvaluator(args, None) | |
| if not preflight_evaluator.has_pending_seeds(): | |
| logger.info( | |
| "All requested ALFWorld seeds are already complete and " | |
| "compatible; skipping Ray and model initialization." | |
| ) | |
| summary = preflight_evaluator.evaluate() | |
| print(json.dumps(summary, ensure_ascii=False, indent=2)) | |
| return summary | |
| import ray | |
| from kdflow.ray.placement_group import create_placement_group | |
| from kdflow.ray.rollout.rollout_group import RolloutActorGroup | |
| from kdflow.trainer.alfworld_rollout_manager import AlfWorldRolloutManager | |
| from kdflow.trainer.verl_agent_alfworld_rollout_manager import ( | |
| VerlAgentAlfWorldRolloutManager, | |
| ) | |
| initialized_ray = False | |
| if not ray.is_initialized(): | |
| ray.init( | |
| runtime_env={ | |
| "env_vars": { | |
| "TOKENIZERS_PARALLELISM": "true", | |
| "NCCL_DEBUG": "WARN", | |
| }, | |
| "working_dir": os.path.dirname( | |
| os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| ), | |
| } | |
| ) | |
| initialized_ray = True | |
| total_gpus = args.train.num_nodes * args.train.num_gpus_per_node | |
| rollout_group = None | |
| rollout_manager = None | |
| try: | |
| pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group( | |
| total_gpus | |
| ) | |
| rollout_group = RolloutActorGroup( | |
| model_path=args.model.student_name_or_path, | |
| num_actors=args.rollout.rollout_num_engines, | |
| tp_size=args.rollout.rollout_tp_size, | |
| num_gpus_per_node=args.train.num_gpus_per_node, | |
| enable_memory_saver=True, | |
| mem_fraction_static=args.rollout.rollout_mem_fraction_static, | |
| num_gpus_per_actor=0.01, | |
| pg=(pg, reordered_bundle_indices, reordered_gpu_ids), | |
| extra_server_args={"disable_piecewise_cuda_graph": True}, | |
| ) | |
| if args.train.enable_sleep: | |
| rollout_group.sleep() | |
| strategy = _EvaluationStrategy(args) | |
| generate_kwargs = { | |
| "max_new_tokens": args.rollout.generate_max_len, | |
| "temperature": args.rollout.temperature, | |
| "top_p": args.rollout.top_p, | |
| } | |
| if args.rollout.alfworld_eval_protocol == "verl_agent": | |
| manager_class = VerlAgentAlfWorldRolloutManager | |
| evaluator_class = VerlAgentAlfWorldEvaluator | |
| else: | |
| manager_class = AlfWorldRolloutManager | |
| evaluator_class = AlfWorldEvaluator | |
| rollout_manager = manager_class( | |
| strategy=strategy, | |
| rollout_group=rollout_group, | |
| is_same_tokenizer=True, | |
| generate_kwargs=generate_kwargs, | |
| ) | |
| summary = evaluator_class(args, rollout_manager).evaluate() | |
| print(json.dumps(summary, ensure_ascii=False, indent=2)) | |
| return summary | |
| finally: | |
| if rollout_manager is not None: | |
| rollout_manager.close() | |
| if rollout_group is not None: | |
| rollout_group.shutdown() | |
| if initialized_ray: | |
| ray.shutdown() | |
| if __name__ == "__main__": | |
| from kdflow.arguments import init_args | |
| evaluate(init_args("alfworld_eval")) | |