File size: 7,916 Bytes
6da8b7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8271223
 
6da8b7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d140675
6da8b7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8271223
 
 
6da8b7c
 
 
 
 
 
 
 
 
 
8271223
 
6da8b7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
"""Trajectory cache system for DispatchR.

Builds, persists, and samples from pre-computed greedy episode trajectories.
This eliminates the need to re-run episode simulation during dataset construction.

Cache format (JSONL, one line per episode):
    {
        "seed": 123,
        "difficulty": "learning",
        "episode_reward": 0.45,
        "steps": [
            {"step": 0, "action": {...}},
            {"step": 1, "action": {...}},
            ...
        ]
    }

We do NOT store full observations in the cache to keep file size reasonable.
Observations are cheaply recomputed by replaying cached actions.
"""

import json
import os
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

import numpy as np

from .greedy_policy import greedy_action
from .grpo_env_wrapper import DispatchRGRPOEnv


def _run_single_episode(args: Tuple[int, str]) -> Dict[str, Any]:
    """Worker: run one greedy episode. Must be top-level for pickling."""
    seed, difficulty = args
    env = DispatchRGRPOEnv(seed=seed, difficulty=difficulty)
    env.reset(seed=seed, difficulty=difficulty)

    steps = []
    for step_idx in range(80):
        obs = env._obs
        if obs is None or obs.get("done"):
            break
        action = greedy_action(obs)
        action_text = json.dumps(action)
        env.step(action_text)
        steps.append({"step": step_idx, "action": action})

    reward = env.reward if env.reward is not None else 0.0
    return {
        "seed": seed,
        "difficulty": difficulty,
        "episode_reward": float(reward),
        "steps": steps,
    }


def build_trajectory_cache(
    n_seeds: int,
    difficulty: str,
    output_path: str,
    base_seed: int = 0,
    num_workers: int = 8,
) -> str:
    """Build trajectory cache by running greedy episodes in parallel.

    Args:
        n_seeds: Number of episode seeds to simulate.
        difficulty: Curriculum phase string.
        output_path: Where to write the JSONL cache file.
        base_seed: Offset for seed values.
        num_workers: Parallel workers (default 8, set to 1 for debugging).

    Returns:
        Path to the created cache file.
    """
    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
    tasks = [(base_seed + i, difficulty) for i in range(n_seeds)]

    print(f"[Cache] Building {n_seeds} greedy trajectories (difficulty={difficulty})...")
    print(f"[Cache] Workers: {num_workers}")

    with open(output_path, "w", encoding="utf-8") as f:
        if num_workers <= 1:
            # Sequential (useful for debugging)
            for task in tasks:
                episode = _run_single_episode(task)
                f.write(json.dumps(episode) + "\n")
                f.flush()
        else:
            with ProcessPoolExecutor(max_workers=num_workers) as executor:
                futures = {executor.submit(_run_single_episode, t): t for t in tasks}
                for future in as_completed(futures):
                    episode = future.result()
                    f.write(json.dumps(episode) + "\n")
                    f.flush()

    print(f"[Cache] Saved to {output_path}")
    return output_path


def load_trajectory_cache(cache_path: str) -> List[Dict[str, Any]]:
    """Load full cache into memory.

    Returns list of episode dicts. For large caches consider streaming
    with ``iter_trajectory_cache`` instead.
    """
    episodes = []
    with open(cache_path, "r", encoding="utf-8") as f:
        for line in f:
            if line.strip():
                episodes.append(json.loads(line))
    return episodes


def iter_trajectory_cache(cache_path: str):
    """Stream trajectory cache line-by-line (memory-efficient)."""
    with open(cache_path, "r", encoding="utf-8") as f:
        for line in f:
            if line.strip():
                yield json.loads(line)


def sample_dataset_rows(
    cache_path: str,
    tokenizer,
    system_prompt: str,
    n_samples_per_seed: int = 5,
    max_step: int = 79,
    build_chat_prompt_fn=None,
) -> List[Dict[str, Any]]:
    """Sample random-step observations from cache to build dataset rows.

    For each episode in the cache, sample ``n_samples_per_seed`` random steps
    (uniformly), replay the episode to that step, capture the observation,
    and format it as a prompt.

    Args:
        cache_path: Path to JSONL cache file.
        tokenizer: HF tokenizer for chat template formatting.
        system_prompt: The SYSTEM_PROMPT string.
        n_samples_per_seed: How many steps to sample per episode.
        max_step: Maximum step index to sample (inclusive).
        build_chat_prompt_fn: Callable(tokenizer, system, user) -> str.

    Returns:
        List of dataset row dicts with keys: seed, difficulty, step_idx, prompt.
    """
    rows = []

    for episode in iter_trajectory_cache(cache_path):
        seed = episode["seed"]
        difficulty = episode["difficulty"]
        cached_steps = episode["steps"]
        n_steps = len(cached_steps)

        if n_steps <= 1:
            continue

        # Sample step indices (exclude step 0, cap at actual episode length)
        upper = min(max_step, n_steps - 1)
        if upper < 1:
            continue

        sampled_indices = np.random.choice(
            range(3, upper + 1),
            size=min(n_samples_per_seed, upper),
            replace=False,
        ).tolist()

        # Replay episode and capture observations at sampled steps
        env = DispatchRGRPOEnv(seed=seed, difficulty=difficulty)
        env.reset(seed=seed, difficulty=difficulty)

        for step_idx in range(n_steps):
            obs = env._obs
            if obs is None or obs.get("done"):
                break

            if step_idx in sampled_indices:
                # Format observation as prompt
                from .prompt_utils import format_observation, build_chat_prompt

                obs_text = format_observation(obs)
                if build_chat_prompt_fn is not None:
                    prompt = build_chat_prompt_fn(tokenizer, system_prompt, obs_text)
                else:
                    prompt = build_chat_prompt(tokenizer, system_prompt, obs_text)

                rows.append(
                    {
                        "seed": seed,
                        "difficulty": difficulty,
                        "step_idx": step_idx,
                        "prompt": prompt,
                    }
                )

            # Apply cached action to advance env (convert dict to JSON string)
            action_dict = cached_steps[step_idx]["action"]
            env.step(json.dumps(action_dict))

    return rows


def replay_steps(env_ref, cached_steps: List[Dict[str, Any]], up_to_step: int) -> None:
    """Replay cached actions on an env up to (but not including) a target step.

    Used in reward_fn to quickly reconstruct the state at step ``up_to_step``.
    """
    for step_idx in range(min(up_to_step, len(cached_steps))):
        action_dict = cached_steps[step_idx]["action"]
        env_ref.step(json.dumps(action_dict))


def get_cache_path(cache_dir: str, difficulty: str, n_seeds: int) -> str:
    """Return standard cache file path."""
    return os.path.join(
        cache_dir, f"trajectories_{difficulty}_{n_seeds}seeds.jsonl"
    )


def ensure_cache_exists(
    cache_dir: str,
    n_seeds: int,
    difficulty: str,
    base_seed: int = 0,
    num_workers: int = 8,
) -> str:
    """Return cache path, building it first if it doesn't exist."""
    path = get_cache_path(cache_dir, difficulty, n_seeds)
    if os.path.exists(path):
        print(f"[Cache] Reusing existing cache: {path}")
        return path
    return build_trajectory_cache(
        n_seeds=n_seeds,
        difficulty=difficulty,
        output_path=path,
        base_seed=base_seed,
        num_workers=num_workers,
    )