Spaces:
Sleeping
Sleeping
Download server/trajectory_cache.py from ggtejas/dispatcher: direct link, hf CLI and curl.
- Browser
- Download file 7.92 kB
-
https://huggingface.co/spaces/ggtejas/dispatcher/resolve/main/server/trajectory_cache.py
- Command line
-
hf download hf://spaces/ggtejas/dispatcher/server/trajectory_cache.py
-
curl -L -o trajectory_cache.py https://huggingface.co/spaces/ggtejas/dispatcher/resolve/main/server/trajectory_cache.py
7.92 kB
| """Trajectory cache system for DispatchR. | |
| Builds, persists, and samples from pre-computed greedy episode trajectories. | |
| This eliminates the need to re-run episode simulation during dataset construction. | |
| Cache format (JSONL, one line per episode): | |
| { | |
| "seed": 123, | |
| "difficulty": "learning", | |
| "episode_reward": 0.45, | |
| "steps": [ | |
| {"step": 0, "action": {...}}, | |
| {"step": 1, "action": {...}}, | |
| ... | |
| ] | |
| } | |
| We do NOT store full observations in the cache to keep file size reasonable. | |
| Observations are cheaply recomputed by replaying cached actions. | |
| """ | |
| import json | |
| import os | |
| from concurrent.futures import ProcessPoolExecutor, as_completed | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import numpy as np | |
| from .greedy_policy import greedy_action | |
| from .grpo_env_wrapper import DispatchRGRPOEnv | |
| def _run_single_episode(args: Tuple[int, str]) -> Dict[str, Any]: | |
| """Worker: run one greedy episode. Must be top-level for pickling.""" | |
| seed, difficulty = args | |
| env = DispatchRGRPOEnv(seed=seed, difficulty=difficulty) | |
| env.reset(seed=seed, difficulty=difficulty) | |
| steps = [] | |
| for step_idx in range(80): | |
| obs = env._obs | |
| if obs is None or obs.get("done"): | |
| break | |
| action = greedy_action(obs) | |
| action_text = json.dumps(action) | |
| env.step(action_text) | |
| steps.append({"step": step_idx, "action": action}) | |
| reward = env.reward if env.reward is not None else 0.0 | |
| return { | |
| "seed": seed, | |
| "difficulty": difficulty, | |
| "episode_reward": float(reward), | |
| "steps": steps, | |
| } | |
| def build_trajectory_cache( | |
| n_seeds: int, | |
| difficulty: str, | |
| output_path: str, | |
| base_seed: int = 0, | |
| num_workers: int = 8, | |
| ) -> str: | |
| """Build trajectory cache by running greedy episodes in parallel. | |
| Args: | |
| n_seeds: Number of episode seeds to simulate. | |
| difficulty: Curriculum phase string. | |
| output_path: Where to write the JSONL cache file. | |
| base_seed: Offset for seed values. | |
| num_workers: Parallel workers (default 8, set to 1 for debugging). | |
| Returns: | |
| Path to the created cache file. | |
| """ | |
| os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True) | |
| tasks = [(base_seed + i, difficulty) for i in range(n_seeds)] | |
| print(f"[Cache] Building {n_seeds} greedy trajectories (difficulty={difficulty})...") | |
| print(f"[Cache] Workers: {num_workers}") | |
| with open(output_path, "w", encoding="utf-8") as f: | |
| if num_workers <= 1: | |
| # Sequential (useful for debugging) | |
| for task in tasks: | |
| episode = _run_single_episode(task) | |
| f.write(json.dumps(episode) + "\n") | |
| f.flush() | |
| else: | |
| with ProcessPoolExecutor(max_workers=num_workers) as executor: | |
| futures = {executor.submit(_run_single_episode, t): t for t in tasks} | |
| for future in as_completed(futures): | |
| episode = future.result() | |
| f.write(json.dumps(episode) + "\n") | |
| f.flush() | |
| print(f"[Cache] Saved to {output_path}") | |
| return output_path | |
| def load_trajectory_cache(cache_path: str) -> List[Dict[str, Any]]: | |
| """Load full cache into memory. | |
| Returns list of episode dicts. For large caches consider streaming | |
| with ``iter_trajectory_cache`` instead. | |
| """ | |
| episodes = [] | |
| with open(cache_path, "r", encoding="utf-8") as f: | |
| for line in f: | |
| if line.strip(): | |
| episodes.append(json.loads(line)) | |
| return episodes | |
| def iter_trajectory_cache(cache_path: str): | |
| """Stream trajectory cache line-by-line (memory-efficient).""" | |
| with open(cache_path, "r", encoding="utf-8") as f: | |
| for line in f: | |
| if line.strip(): | |
| yield json.loads(line) | |
| def sample_dataset_rows( | |
| cache_path: str, | |
| tokenizer, | |
| system_prompt: str, | |
| n_samples_per_seed: int = 5, | |
| max_step: int = 79, | |
| build_chat_prompt_fn=None, | |
| ) -> List[Dict[str, Any]]: | |
| """Sample random-step observations from cache to build dataset rows. | |
| For each episode in the cache, sample ``n_samples_per_seed`` random steps | |
| (uniformly), replay the episode to that step, capture the observation, | |
| and format it as a prompt. | |
| Args: | |
| cache_path: Path to JSONL cache file. | |
| tokenizer: HF tokenizer for chat template formatting. | |
| system_prompt: The SYSTEM_PROMPT string. | |
| n_samples_per_seed: How many steps to sample per episode. | |
| max_step: Maximum step index to sample (inclusive). | |
| build_chat_prompt_fn: Callable(tokenizer, system, user) -> str. | |
| Returns: | |
| List of dataset row dicts with keys: seed, difficulty, step_idx, prompt. | |
| """ | |
| rows = [] | |
| for episode in iter_trajectory_cache(cache_path): | |
| seed = episode["seed"] | |
| difficulty = episode["difficulty"] | |
| cached_steps = episode["steps"] | |
| n_steps = len(cached_steps) | |
| if n_steps <= 1: | |
| continue | |
| # Sample step indices (exclude step 0, cap at actual episode length) | |
| upper = min(max_step, n_steps - 1) | |
| if upper < 1: | |
| continue | |
| sampled_indices = np.random.choice( | |
| range(3, upper + 1), | |
| size=min(n_samples_per_seed, upper), | |
| replace=False, | |
| ).tolist() | |
| # Replay episode and capture observations at sampled steps | |
| env = DispatchRGRPOEnv(seed=seed, difficulty=difficulty) | |
| env.reset(seed=seed, difficulty=difficulty) | |
| for step_idx in range(n_steps): | |
| obs = env._obs | |
| if obs is None or obs.get("done"): | |
| break | |
| if step_idx in sampled_indices: | |
| # Format observation as prompt | |
| from .prompt_utils import format_observation, build_chat_prompt | |
| obs_text = format_observation(obs) | |
| if build_chat_prompt_fn is not None: | |
| prompt = build_chat_prompt_fn(tokenizer, system_prompt, obs_text) | |
| else: | |
| prompt = build_chat_prompt(tokenizer, system_prompt, obs_text) | |
| rows.append( | |
| { | |
| "seed": seed, | |
| "difficulty": difficulty, | |
| "step_idx": step_idx, | |
| "prompt": prompt, | |
| } | |
| ) | |
| # Apply cached action to advance env (convert dict to JSON string) | |
| action_dict = cached_steps[step_idx]["action"] | |
| env.step(json.dumps(action_dict)) | |
| return rows | |
| def replay_steps(env_ref, cached_steps: List[Dict[str, Any]], up_to_step: int) -> None: | |
| """Replay cached actions on an env up to (but not including) a target step. | |
| Used in reward_fn to quickly reconstruct the state at step ``up_to_step``. | |
| """ | |
| for step_idx in range(min(up_to_step, len(cached_steps))): | |
| action_dict = cached_steps[step_idx]["action"] | |
| env_ref.step(json.dumps(action_dict)) | |
| def get_cache_path(cache_dir: str, difficulty: str, n_seeds: int) -> str: | |
| """Return standard cache file path.""" | |
| return os.path.join( | |
| cache_dir, f"trajectories_{difficulty}_{n_seeds}seeds.jsonl" | |
| ) | |
| def ensure_cache_exists( | |
| cache_dir: str, | |
| n_seeds: int, | |
| difficulty: str, | |
| base_seed: int = 0, | |
| num_workers: int = 8, | |
| ) -> str: | |
| """Return cache path, building it first if it doesn't exist.""" | |
| path = get_cache_path(cache_dir, difficulty, n_seeds) | |
| if os.path.exists(path): | |
| print(f"[Cache] Reusing existing cache: {path}") | |
| return path | |
| return build_trajectory_cache( | |
| n_seeds=n_seeds, | |
| difficulty=difficulty, | |
| output_path=path, | |
| base_seed=base_seed, | |
| num_workers=num_workers, | |
| ) | |