File size: 4,454 Bytes
5819e26 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | diff --git a/scripts/train_justrl_qwen.sh b/scripts/train_justrl_qwen.sh
index 0bf8cc0..e1ca392 100755
--- a/scripts/train_justrl_qwen.sh
+++ b/scripts/train_justrl_qwen.sh
@@ -211,6 +211,7 @@ set +e
trainer.default_local_dir="${CHECKPOINT_DIR}" \
trainer.is_plot="${IS_PLOT}" \
"${VLLM_ENGINE_OVERRIDES[@]}" \
+ "$@" \
2>&1 | tee -a "${TRAIN_LOG}"
STATUS=${PIPESTATUS[0]}
set -e
diff --git a/verl/verl/trainer/main_ppo.py b/verl/verl/trainer/main_ppo.py
index 0962321..6f76e02 100644
--- a/verl/verl/trainer/main_ppo.py
+++ b/verl/verl/trainer/main_ppo.py
@@ -31,6 +31,30 @@ from verl.utils.config import validate_config
from verl.utils.device import is_cuda_available
from verl.utils.import_utils import load_extern_type
+# [phase4-seed] Pilot-only seeding shim (Direct-OPD SFT-vs-RL policy-shift pilot).
+# The upstream PPO path has no global RNG seeding at all (no trainer.seed / algorithm.seed),
+# so the driver process and every Ray worker process start from unseeded python/numpy/torch
+# RNGs. This helper is called on the driver (run_ppo) and inside the Ray TaskRunner actor;
+# ActorRolloutRefWorker.init_model calls the equivalent block (separate process, separate
+# import of this module is not guaranteed to have run the driver's call).
+PHASE4_SEED = 42
+
+
+def _phase4_seed_everything(seed: int = PHASE4_SEED) -> None:
+ """Seed python/numpy/torch (+CUDA) RNGs in the current process."""
+ import random
+
+ import numpy as np
+ import torch
+
+ os.environ.setdefault("PYTHONHASHSEED", str(seed))
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ if torch.cuda.is_available():
+ torch.cuda.manual_seed_all(seed)
+ print(f"[phase4-seed] seeded python/numpy/torch RNGs with {seed} in pid={os.getpid()}")
+
@hydra.main(config_path="config", config_name="ppo_trainer", version_base=None)
def main(config):
@@ -52,6 +76,8 @@ def run_ppo(config, task_runner_class=None) -> None:
model paths, and training hyperparameters.
task_runner_class: For recipe to change TaskRunner.
"""
+ _phase4_seed_everything() # [phase4-seed] driver process
+
# Check if Ray is not initialized
if not ray.is_initialized():
# Initialize Ray with a local cluster configuration
@@ -242,6 +268,8 @@ class TaskRunner:
config: Training configuration object containing all parameters needed
for setting up and running the PPO training process.
"""
+ _phase4_seed_everything() # [phase4-seed] TaskRunner actor process
+
# Print the initial configuration. `resolve=True` will evaluate symbolic values.
from pprint import pprint
diff --git a/verl/verl/workers/fsdp_workers.py b/verl/verl/workers/fsdp_workers.py
index de7893b..62e18d3 100644
--- a/verl/verl/workers/fsdp_workers.py
+++ b/verl/verl/workers/fsdp_workers.py
@@ -638,7 +638,9 @@ class ActorRolloutRefWorker(Worker, DistProfilerExtension):
# 3. init trainer and rollout random states
self.torch_random_states = get_torch_device().get_rng_state()
gen_dp_rank = rollout_device_mesh["dp"].get_local_rank()
- get_torch_device().manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states
+ # [phase4-seed] was `gen_dp_rank + 1000` (hardcoded upstream); pinned to the pilot seed
+ # base 42 so the rollout sampling generator is a documented function of the seed.
+ get_torch_device().manual_seed(gen_dp_rank + 42) # make sure all tp ranks have the same random states
self.gen_random_states = get_torch_device().get_rng_state()
get_torch_device().set_rng_state(self.torch_random_states)
@@ -778,6 +780,16 @@ class ActorRolloutRefWorker(Worker, DistProfilerExtension):
@register(dispatch_mode=Dispatch.ONE_TO_ALL)
def init_model(self):
+ # [phase4-seed] Ray workers are separate processes: seed python/numpy/torch here too.
+ import random as _phase4_random
+
+ _phase4_random.seed(42)
+ np.random.seed(42)
+ torch.manual_seed(42)
+ if torch.cuda.is_available():
+ torch.cuda.manual_seed_all(42)
+ print(f"[phase4-seed] worker rank={os.environ.get('RANK', '?')} seeded python/numpy/torch with 42")
+
from verl.workers.actor import DataParallelPPOActor
# This is used to import external_lib into the huggingface systems
|