ctopd / kdflow /cli /evaluate_alfworld.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
4.05 kB
"""Evaluate a model checkpoint in ALFWorld without starting training actors."""
import json
import logging
import os
from kdflow.evaluation import AlfWorldEvaluator, VerlAgentAlfWorldEvaluator
logger = logging.getLogger(__name__)
class _EvaluationStrategy:
"""Small args/log adapter required by the shared rollout manager."""
def __init__(self, args):
self.args = args
@staticmethod
def log(message, level="info", **_kwargs):
getattr(logger, level, logger.info)(message)
@staticmethod
def print(*values):
logger.info(" ".join(str(value) for value in values))
def evaluate(args):
if args.rollout.alfworld_eval_protocol == "verl_agent":
preflight_evaluator = VerlAgentAlfWorldEvaluator(args, None)
if not preflight_evaluator.has_pending_seeds():
logger.info(
"All requested ALFWorld seeds are already complete and "
"compatible; skipping Ray and model initialization."
)
summary = preflight_evaluator.evaluate()
print(json.dumps(summary, ensure_ascii=False, indent=2))
return summary
import ray
from kdflow.ray.placement_group import create_placement_group
from kdflow.ray.rollout.rollout_group import RolloutActorGroup
from kdflow.trainer.alfworld_rollout_manager import AlfWorldRolloutManager
from kdflow.trainer.verl_agent_alfworld_rollout_manager import (
VerlAgentAlfWorldRolloutManager,
)
initialized_ray = False
if not ray.is_initialized():
ray.init(
runtime_env={
"env_vars": {
"TOKENIZERS_PARALLELISM": "true",
"NCCL_DEBUG": "WARN",
},
"working_dir": os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
),
}
)
initialized_ray = True
total_gpus = args.train.num_nodes * args.train.num_gpus_per_node
rollout_group = None
rollout_manager = None
try:
pg, reordered_bundle_indices, reordered_gpu_ids = create_placement_group(
total_gpus
)
rollout_group = RolloutActorGroup(
model_path=args.model.student_name_or_path,
num_actors=args.rollout.rollout_num_engines,
tp_size=args.rollout.rollout_tp_size,
num_gpus_per_node=args.train.num_gpus_per_node,
enable_memory_saver=True,
mem_fraction_static=args.rollout.rollout_mem_fraction_static,
num_gpus_per_actor=0.01,
pg=(pg, reordered_bundle_indices, reordered_gpu_ids),
extra_server_args={"disable_piecewise_cuda_graph": True},
)
if args.train.enable_sleep:
rollout_group.sleep()
strategy = _EvaluationStrategy(args)
generate_kwargs = {
"max_new_tokens": args.rollout.generate_max_len,
"temperature": args.rollout.temperature,
"top_p": args.rollout.top_p,
}
if args.rollout.alfworld_eval_protocol == "verl_agent":
manager_class = VerlAgentAlfWorldRolloutManager
evaluator_class = VerlAgentAlfWorldEvaluator
else:
manager_class = AlfWorldRolloutManager
evaluator_class = AlfWorldEvaluator
rollout_manager = manager_class(
strategy=strategy,
rollout_group=rollout_group,
is_same_tokenizer=True,
generate_kwargs=generate_kwargs,
)
summary = evaluator_class(args, rollout_manager).evaluate()
print(json.dumps(summary, ensure_ascii=False, indent=2))
return summary
finally:
if rollout_manager is not None:
rollout_manager.close()
if rollout_group is not None:
rollout_group.shutdown()
if initialized_ray:
ray.shutdown()
if __name__ == "__main__":
from kdflow.arguments import init_args
evaluate(init_args("alfworld_eval"))