ctopd / kdflow /trainer /rollout_manager.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
6.72 kB
import asyncio
import time
from typing import Any, Dict, List, Optional
from kdflow.trainer.data_processor import RolloutDataProcessor
from kdflow.utils.logging_utils import init_logger
logger = init_logger(__name__)
class RolloutManager:
"""Manage batched rollout generation and data processing."""
def __init__(
self,
strategy,
rollout_group,
is_same_tokenizer: bool,
generate_kwargs: Optional[Dict[str, Any]] = None,
):
self.args = strategy.args
self.rollout_group = rollout_group
self.generate_kwargs = generate_kwargs
self.max_concurrent = (
self.args.rollout.rollout_engine_concurrency
* self.rollout_group.num_actors
)
if self.max_concurrent <= 0:
raise ValueError(
f"max_concurrent must be positive, got {self.max_concurrent}"
)
self.data_processor = RolloutDataProcessor(strategy, is_same_tokenizer)
self.image_key = self.data_processor.image_key
def rollout(
self,
prompt_batch: List[Dict[str, Any]],
global_step: int,
mode: str = "train",
**kwargs,
) -> tuple[List[dict], Dict[str, float]]:
"""Generate rollout samples and convert them into micro-batches."""
if mode not in ("train", "eval"):
raise ValueError(f"Unsupported rollout mode: {mode!r}")
if not prompt_batch:
return [], {}
n_samples = self.args.rollout.n_samples_per_prompt if mode == "train" else 1
if n_samples <= 0:
raise ValueError(f"n_samples_per_prompt must be positive, got {n_samples}")
should_sleep = self.args.train.enable_sleep
if should_sleep:
self.rollout_group.wakeup()
try:
stu_prompts = [
item["stu_prompt"]
for item in prompt_batch
for _ in range(n_samples)
]
tea_prompts = [
item["tea_prompt"]
for item in prompt_batch
for _ in range(n_samples)
]
labels = [
item["label"] for item in prompt_batch for _ in range(n_samples)
]
images = None
if self.image_key:
images = [
item.get("images")
for item in prompt_batch
for _ in range(n_samples)
]
sampling_params = kwargs or self.generate_kwargs
if not sampling_params:
raise ValueError("sampling_params must not be empty")
outputs, timing_metrics = self._generate(
stu_prompts,
sampling_params,
image_data=images,
)
micro_batches, rollout_metrics = self.data_processor.process(
stu_prompts=stu_prompts,
tea_prompts=tea_prompts,
outputs=outputs,
labels=labels,
sampling_params=sampling_params,
global_step=global_step,
mode=mode,
images=images,
)
rollout_metrics.update(timing_metrics)
return micro_batches, rollout_metrics
finally:
if should_sleep:
self.rollout_group.sleep()
def _generate(
self,
prompts: List[str],
sampling_params: Dict[str, Any],
image_data: Optional[List] = None,
) -> tuple[List[Dict[str, Any]], Dict[str, float]]:
"""Run ordered concurrent generation requests."""
if not prompts:
return [], {}
if image_data is not None and len(image_data) != len(prompts):
raise ValueError("image_data and prompts must have the same length")
max_concurrent = min(len(prompts), self.max_concurrent)
try:
return asyncio.run(
self._async_generate(
prompts=prompts,
sampling_params=sampling_params,
max_concurrent=max_concurrent,
image_data=image_data,
)
)
except Exception:
try:
engine_health = self.rollout_group.health_check()
except Exception as health_error:
engine_health = f"unavailable ({health_error!r})"
logger.exception(
"Rollout generation failed: prompts=%d, max_concurrent=%d, "
"engine_health=%s",
len(prompts),
max_concurrent,
engine_health,
)
raise
async def _async_generate(
self,
prompts: List[str],
sampling_params: Dict[str, Any],
max_concurrent: int,
image_data: Optional[List] = None,
) -> tuple[List[Dict[str, Any]], Dict[str, float]]:
"""Schedule single-sample generation requests concurrently."""
import aiohttp
semaphore = asyncio.Semaphore(max_concurrent)
results = [None] * len(prompts)
rollout_times = [0.0] * len(prompts)
async def run_request(
index: int, prompt: str, session: aiohttp.ClientSession
) -> None:
try:
async with semaphore:
start = time.perf_counter()
results[index] = await self.rollout_group.generate_one(
prompt=prompt,
sampling_params=sampling_params,
session=session,
image_data=(
image_data[index] if image_data is not None else None
),
)
rollout_times[index] = time.perf_counter() - start
except Exception as error:
raise RuntimeError(
f"Rollout generation failed for request {index}"
) from error
connector = aiohttp.TCPConnector(limit=max_concurrent)
timeout = aiohttp.ClientTimeout(total=None, sock_read=None, sock_connect=60)
async with aiohttp.ClientSession(
connector=connector, timeout=timeout
) as session:
await asyncio.gather(
*(
run_request(index, prompt, session)
for index, prompt in enumerate(prompts)
)
)
timing_metrics = {
"timing/rollout_per_sample/mean": sum(rollout_times) / len(rollout_times),
"timing/rollout_per_sample/max": max(rollout_times),
}
return results, timing_metrics