dispatcher / server /trajectory_cache.py
garg-tejas
fix: improve GRPO logging and cache sampling behavior
d140675
Raw History Blame Contribute Delete
7.92 kB
"""Trajectory cache system for DispatchR.
Builds, persists, and samples from pre-computed greedy episode trajectories.
This eliminates the need to re-run episode simulation during dataset construction.
Cache format (JSONL, one line per episode):
{
"seed": 123,
"difficulty": "learning",
"episode_reward": 0.45,
"steps": [
{"step": 0, "action": {...}},
{"step": 1, "action": {...}},
...
]
}
We do NOT store full observations in the cache to keep file size reasonable.
Observations are cheaply recomputed by replaying cached actions.
"""
import json
import os
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
from .greedy_policy import greedy_action
from .grpo_env_wrapper import DispatchRGRPOEnv
def _run_single_episode(args: Tuple[int, str]) -> Dict[str, Any]:
"""Worker: run one greedy episode. Must be top-level for pickling."""
seed, difficulty = args
env = DispatchRGRPOEnv(seed=seed, difficulty=difficulty)
env.reset(seed=seed, difficulty=difficulty)
steps = []
for step_idx in range(80):
obs = env._obs
if obs is None or obs.get("done"):
break
action = greedy_action(obs)
action_text = json.dumps(action)
env.step(action_text)
steps.append({"step": step_idx, "action": action})
reward = env.reward if env.reward is not None else 0.0
return {
"seed": seed,
"difficulty": difficulty,
"episode_reward": float(reward),
"steps": steps,
}
def build_trajectory_cache(
n_seeds: int,
difficulty: str,
output_path: str,
base_seed: int = 0,
num_workers: int = 8,
) -> str:
"""Build trajectory cache by running greedy episodes in parallel.
Args:
n_seeds: Number of episode seeds to simulate.
difficulty: Curriculum phase string.
output_path: Where to write the JSONL cache file.
base_seed: Offset for seed values.
num_workers: Parallel workers (default 8, set to 1 for debugging).
Returns:
Path to the created cache file.
"""
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
tasks = [(base_seed + i, difficulty) for i in range(n_seeds)]
print(f"[Cache] Building {n_seeds} greedy trajectories (difficulty={difficulty})...")
print(f"[Cache] Workers: {num_workers}")
with open(output_path, "w", encoding="utf-8") as f:
if num_workers <= 1:
# Sequential (useful for debugging)
for task in tasks:
episode = _run_single_episode(task)
f.write(json.dumps(episode) + "\n")
f.flush()
else:
with ProcessPoolExecutor(max_workers=num_workers) as executor:
futures = {executor.submit(_run_single_episode, t): t for t in tasks}
for future in as_completed(futures):
episode = future.result()
f.write(json.dumps(episode) + "\n")
f.flush()
print(f"[Cache] Saved to {output_path}")
return output_path
def load_trajectory_cache(cache_path: str) -> List[Dict[str, Any]]:
"""Load full cache into memory.
Returns list of episode dicts. For large caches consider streaming
with ``iter_trajectory_cache`` instead.
"""
episodes = []
with open(cache_path, "r", encoding="utf-8") as f:
for line in f:
if line.strip():
episodes.append(json.loads(line))
return episodes
def iter_trajectory_cache(cache_path: str):
"""Stream trajectory cache line-by-line (memory-efficient)."""
with open(cache_path, "r", encoding="utf-8") as f:
for line in f:
if line.strip():
yield json.loads(line)
def sample_dataset_rows(
cache_path: str,
tokenizer,
system_prompt: str,
n_samples_per_seed: int = 5,
max_step: int = 79,
build_chat_prompt_fn=None,
) -> List[Dict[str, Any]]:
"""Sample random-step observations from cache to build dataset rows.
For each episode in the cache, sample ``n_samples_per_seed`` random steps
(uniformly), replay the episode to that step, capture the observation,
and format it as a prompt.
Args:
cache_path: Path to JSONL cache file.
tokenizer: HF tokenizer for chat template formatting.
system_prompt: The SYSTEM_PROMPT string.
n_samples_per_seed: How many steps to sample per episode.
max_step: Maximum step index to sample (inclusive).
build_chat_prompt_fn: Callable(tokenizer, system, user) -> str.
Returns:
List of dataset row dicts with keys: seed, difficulty, step_idx, prompt.
"""
rows = []
for episode in iter_trajectory_cache(cache_path):
seed = episode["seed"]
difficulty = episode["difficulty"]
cached_steps = episode["steps"]
n_steps = len(cached_steps)
if n_steps <= 1:
continue
# Sample step indices (exclude step 0, cap at actual episode length)
upper = min(max_step, n_steps - 1)
if upper < 1:
continue
sampled_indices = np.random.choice(
range(3, upper + 1),
size=min(n_samples_per_seed, upper),
replace=False,
).tolist()
# Replay episode and capture observations at sampled steps
env = DispatchRGRPOEnv(seed=seed, difficulty=difficulty)
env.reset(seed=seed, difficulty=difficulty)
for step_idx in range(n_steps):
obs = env._obs
if obs is None or obs.get("done"):
break
if step_idx in sampled_indices:
# Format observation as prompt
from .prompt_utils import format_observation, build_chat_prompt
obs_text = format_observation(obs)
if build_chat_prompt_fn is not None:
prompt = build_chat_prompt_fn(tokenizer, system_prompt, obs_text)
else:
prompt = build_chat_prompt(tokenizer, system_prompt, obs_text)
rows.append(
{
"seed": seed,
"difficulty": difficulty,
"step_idx": step_idx,
"prompt": prompt,
}
)
# Apply cached action to advance env (convert dict to JSON string)
action_dict = cached_steps[step_idx]["action"]
env.step(json.dumps(action_dict))
return rows
def replay_steps(env_ref, cached_steps: List[Dict[str, Any]], up_to_step: int) -> None:
"""Replay cached actions on an env up to (but not including) a target step.
Used in reward_fn to quickly reconstruct the state at step ``up_to_step``.
"""
for step_idx in range(min(up_to_step, len(cached_steps))):
action_dict = cached_steps[step_idx]["action"]
env_ref.step(json.dumps(action_dict))
def get_cache_path(cache_dir: str, difficulty: str, n_seeds: int) -> str:
"""Return standard cache file path."""
return os.path.join(
cache_dir, f"trajectories_{difficulty}_{n_seeds}seeds.jsonl"
)
def ensure_cache_exists(
cache_dir: str,
n_seeds: int,
difficulty: str,
base_seed: int = 0,
num_workers: int = 8,
) -> str:
"""Return cache path, building it first if it doesn't exist."""
path = get_cache_path(cache_dir, difficulty, n_seeds)
if os.path.exists(path):
print(f"[Cache] Reusing existing cache: {path}")
return path
return build_trajectory_cache(
n_seeds=n_seeds,
difficulty=difficulty,
output_path=path,
base_seed=base_seed,
num_workers=num_workers,
)