Dhruv1000 commited on
Commit
0fa014e
·
verified ·
1 Parent(s): 74d8585

Archive callback-order regression and final frozen E4 manifest

Browse files
scripts/run_scale_grpo.py CHANGED
@@ -91,13 +91,15 @@ class ScaleRuntime(Runtime):
91
  self.guards.complete_step(entropy=record['entropy_nats'],groups=64,
92
  zero_groups=record['zero_groups'],parse_rate=rates['aggregate_parse_rate'],
93
  failure_rate=0.,disk_fraction=disk_fraction())
 
 
94
  if len(self.curve)>=400:
95
  previous=self.curve[-400:-200];recent=self.curve[-200:]
96
  mean=lambda rows,name: sum(row['reward_component_means'][name] for row in rows)/len(rows)
97
  trends={key:(mean(previous,key),mean(recent,key)) for key in ('R1','R2','R3')}
98
  if trends['R2'][1]>trends['R2'][0] and trends['R3'][1]>trends['R3'][0] and trends['R1'][1]<=trends['R1'][0]:
99
  append_json(self.run_dir/'reward_hacking_tripwire.jsonl',dict(step=record['optimizer_step'],trends=trends))
100
- return 'R2_R3_rising_R1_flat_for_400_steps [Review #14]'
101
  return None
102
 
103
  def evaluate_scale(self,step,checkpoint):
@@ -173,6 +175,9 @@ class ScaleRuntime(Runtime):
173
  sft_rows=[json.loads(x) for x in sft_path.read_text().splitlines()]
174
  comparison=compare_auc(sft_rows,records)
175
  score=auc_ci(records,'vulnerable_vote_fraction')
 
 
 
176
  greedy=self.trainer.llm.generate(prompts,sampling_params=SamplingParams(
177
  temperature=0,max_tokens=1536,n=1,seed=42),use_tqdm=True)
178
  greedy_rows=[]
@@ -194,7 +199,7 @@ class ScaleRuntime(Runtime):
194
  metrics=dict(optimizer_step=step,checkpoint=str(checkpoint),adapter_label='E4 balanced reward',
195
  inference_protocol='E2 exact: 8 generations, temperature 0.8, max_tokens 1536, seed 42',
196
  n_rows=len(records),n_pairs=len({r['pair_id'] for r in records}),
197
- parse_rate=float(np.mean([r['parse_rate'] for r in records])),row_auc=score['auc'],
198
  row_auc_pair_cluster_95_ci=score['pair_cluster_bootstrap_95_ci'],
199
  auc_delta_vs_sft500=comparison,greedy_protocol='temperature 0, one generation, seed 42',
200
  greedy_label_accuracy=correct/valid_n,greedy_per_class_accuracy=per_class,
@@ -257,8 +262,8 @@ def save_checkpoint(runtime,trainer,step,reason):
257
 
258
 
259
  def finalize_completed_optimizer_step(runtime,trainer,state,*,checkpoint_due,eval_due):
260
- """Persist/evaluate a completed step before honoring its trend tripwire."""
261
- deferred_stop=runtime.complete_step(state)
262
  step=state.global_step
263
  checkpoint=None
264
  if checkpoint_due:
@@ -267,8 +272,7 @@ def finalize_completed_optimizer_step(runtime,trainer,state,*,checkpoint_due,eva
267
  if checkpoint is None:
268
  raise SmokeStop('VALID evaluation requires a completed full-state checkpoint')
269
  runtime.evaluate_scale(step,checkpoint)
270
- if deferred_stop:
271
- raise SmokeStop(deferred_stop)
272
  return checkpoint
273
 
274
 
 
91
  self.guards.complete_step(entropy=record['entropy_nats'],groups=64,
92
  zero_groups=record['zero_groups'],parse_rate=rates['aggregate_parse_rate'],
93
  failure_rate=0.,disk_fraction=disk_fraction())
94
+
95
+ def check_reward_hacking_tripwire(self,record):
96
  if len(self.curve)>=400:
97
  previous=self.curve[-400:-200];recent=self.curve[-200:]
98
  mean=lambda rows,name: sum(row['reward_component_means'][name] for row in rows)/len(rows)
99
  trends={key:(mean(previous,key),mean(recent,key)) for key in ('R1','R2','R3')}
100
  if trends['R2'][1]>trends['R2'][0] and trends['R3'][1]>trends['R3'][0] and trends['R1'][1]<=trends['R1'][0]:
101
  append_json(self.run_dir/'reward_hacking_tripwire.jsonl',dict(step=record['optimizer_step'],trends=trends))
102
+ raise SmokeStop('R2_R3_rising_R1_flat_for_400_steps [Review #14]')
103
  return None
104
 
105
  def evaluate_scale(self,step,checkpoint):
 
175
  sft_rows=[json.loads(x) for x in sft_path.read_text().splitlines()]
176
  comparison=compare_auc(sft_rows,records)
177
  score=auc_ci(records,'vulnerable_vote_fraction')
178
+ sampled_parse=float(np.mean([r['parse_rate'] for r in records]))
179
+ if sampled_parse<.95:
180
+ raise SmokeStop('E4_sampled_VALID_parse_below_95_percent [Review #14/D-GRPO-7]')
181
  greedy=self.trainer.llm.generate(prompts,sampling_params=SamplingParams(
182
  temperature=0,max_tokens=1536,n=1,seed=42),use_tqdm=True)
183
  greedy_rows=[]
 
199
  metrics=dict(optimizer_step=step,checkpoint=str(checkpoint),adapter_label='E4 balanced reward',
200
  inference_protocol='E2 exact: 8 generations, temperature 0.8, max_tokens 1536, seed 42',
201
  n_rows=len(records),n_pairs=len({r['pair_id'] for r in records}),
202
+ parse_rate=sampled_parse,row_auc=score['auc'],
203
  row_auc_pair_cluster_95_ci=score['pair_cluster_bootstrap_95_ci'],
204
  auc_delta_vs_sft500=comparison,greedy_protocol='temperature 0, one generation, seed 42',
205
  greedy_label_accuracy=correct/valid_n,greedy_per_class_accuracy=per_class,
 
262
 
263
 
264
  def finalize_completed_optimizer_step(runtime,trainer,state,*,checkpoint_due,eval_due):
265
+ """Persist/evaluate a completed step before checking its trend tripwire."""
266
+ runtime.complete_step(state)
267
  step=state.global_step
268
  checkpoint=None
269
  if checkpoint_due:
 
272
  if checkpoint is None:
273
  raise SmokeStop('VALID evaluation requires a completed full-state checkpoint')
274
  runtime.evaluate_scale(step,checkpoint)
275
+ runtime.check_reward_hacking_tripwire(runtime.curve[-1])
 
276
  return checkpoint
277
 
278
 
tests/test_review24_callback_order.py CHANGED
@@ -8,7 +8,11 @@ import scripts.run_scale_grpo as runner
8
  def test_completed_step_is_saved_and_evaluated_before_trend_tripwire(monkeypatch):
9
  events = []
10
  runtime = SimpleNamespace(
11
- complete_step=lambda state: events.append("tripwire_check") or "registered_stop",
 
 
 
 
12
  evaluate_scale=lambda step, checkpoint: events.append(("evaluate", step, checkpoint)),
13
  )
14
  trainer = object()
@@ -20,11 +24,12 @@ def test_completed_step_is_saved_and_evaluated_before_trend_tripwire(monkeypatch
20
  runtime, trainer, SimpleNamespace(global_step=400),
21
  checkpoint_due=True, eval_due=True)
22
 
23
- assert events == ["tripwire_check", ("save", 400), ("evaluate", 400, "ckpt")]
24
 
25
 
26
  def test_valid_evaluation_cannot_run_without_full_state_checkpoint():
27
  runtime = SimpleNamespace(complete_step=lambda state: None,
 
28
  evaluate_scale=lambda *args: pytest.fail("evaluation must not run"))
29
  with pytest.raises(runner.SmokeStop, match="requires a completed full-state checkpoint"):
30
  runner.finalize_completed_optimizer_step(
 
8
  def test_completed_step_is_saved_and_evaluated_before_trend_tripwire(monkeypatch):
9
  events = []
10
  runtime = SimpleNamespace(
11
+ curve=[{"optimizer_step": 400}],
12
+ complete_step=lambda state: events.append("complete_step"),
13
+ check_reward_hacking_tripwire=lambda record: (
14
+ events.append("tripwire_check"),
15
+ (_ for _ in ()).throw(runner.SmokeStop("registered_stop"))),
16
  evaluate_scale=lambda step, checkpoint: events.append(("evaluate", step, checkpoint)),
17
  )
18
  trainer = object()
 
24
  runtime, trainer, SimpleNamespace(global_step=400),
25
  checkpoint_due=True, eval_due=True)
26
 
27
+ assert events == ["complete_step", ("save", 400), ("evaluate", 400, "ckpt"), "tripwire_check"]
28
 
29
 
30
  def test_valid_evaluation_cannot_run_without_full_state_checkpoint():
31
  runtime = SimpleNamespace(complete_step=lambda state: None,
32
+ check_reward_hacking_tripwire=lambda record: None,
33
  evaluate_scale=lambda *args: pytest.fail("evaluation must not run"))
34
  with pytest.raises(runner.SmokeStop, match="requires a completed full-state checkpoint"):
35
  runner.finalize_completed_optimizer_step(