23f2002275 commited on
Commit
1bf8189
·
1 Parent(s): 5f47027

fix(train): tolerate missing/invalid WANDB_API_KEY, surface key length in smoke

Browse files
Files changed (2) hide show
  1. train/grpo.py +15 -3
  2. train/smoke_test.py +4 -0
train/grpo.py CHANGED
@@ -120,7 +120,16 @@ def run_grpo(
120
  ["vllm_gpu_memory_utilization"],
121
  float(cfg.train.vllm_gpu_memory_utilization),
122
  )
123
- _set_if_supported(["report_to"], ["wandb"])
 
 
 
 
 
 
 
 
 
124
  _set_if_supported(["logging_steps"], 1)
125
  # vllm_mode='colocate' alone is a no-op without `use_vllm=True` in TRL 1.2.
126
  _set_if_supported(["use_vllm"], True)
@@ -134,12 +143,15 @@ def run_grpo(
134
  grpo_config = GRPOConfig(**kwargs)
135
 
136
  # REW-03: wrap reward_fn to log per-component scalars to W&B
137
- import wandb # type: ignore
 
 
 
138
 
139
  def _instrumented_reward_fn(prompts, completions, **kwargs):
140
  rewards = reward_fn(prompts, completions, **kwargs)
141
  try:
142
- if wandb.run is not None:
143
  log_dict = {
144
  "reward/composite_mean": sum(rewards) / max(len(rewards), 1),
145
  "reward/composite_std": (
 
120
  ["vllm_gpu_memory_utilization"],
121
  float(cfg.train.vllm_gpu_memory_utilization),
122
  )
123
+ import os
124
+ wb_key = os.environ.get("WANDB_API_KEY", "").strip()
125
+ use_wandb = len(wb_key) >= 40 and wb_key.isalnum()
126
+ _set_if_supported(["report_to"], ["wandb"] if use_wandb else [])
127
+ if not use_wandb:
128
+ log.warning(
129
+ "TRN-03 W&B disabled: WANDB_API_KEY missing or wrong length "
130
+ "(have %d chars, need 40). Training will continue without "
131
+ "W&B logging; metrics still printed to stdout.", len(wb_key)
132
+ )
133
  _set_if_supported(["logging_steps"], 1)
134
  # vllm_mode='colocate' alone is a no-op without `use_vllm=True` in TRL 1.2.
135
  _set_if_supported(["use_vllm"], True)
 
143
  grpo_config = GRPOConfig(**kwargs)
144
 
145
  # REW-03: wrap reward_fn to log per-component scalars to W&B
146
+ try:
147
+ import wandb # type: ignore
148
+ except ImportError:
149
+ wandb = None
150
 
151
  def _instrumented_reward_fn(prompts, completions, **kwargs):
152
  rewards = reward_fn(prompts, completions, **kwargs)
153
  try:
154
+ if wandb is not None and wandb.run is not None:
155
  log_dict = {
156
  "reward/composite_mean": sum(rewards) / max(len(rewards), 1),
157
  "reward/composite_std": (
train/smoke_test.py CHANGED
@@ -28,6 +28,10 @@ def _quick_smoke(env_url: str, output_dir: str) -> dict:
28
  start = time.time()
29
  results = {"checks": {}}
30
 
 
 
 
 
31
  # 1. Hydra config resolves
32
  log.info("TRN-04 [quick] step 1: Hydra config resolution...")
33
  from hydra import initialize, compose
 
28
  start = time.time()
29
  results = {"checks": {}}
30
 
31
+ wb_key_len = len(os.environ.get("WANDB_API_KEY", "").strip())
32
+ log.info("TRN-04 W&B key length: %d (need 40+ for active logging)",
33
+ wb_key_len)
34
+
35
  # 1. Hydra config resolves
36
  log.info("TRN-04 [quick] step 1: Hydra config resolution...")
37
  from hydra import initialize, compose