Archive callback-order regression and final frozen E4 manifest
Browse files
scripts/run_scale_grpo.py
CHANGED
|
@@ -91,13 +91,15 @@ class ScaleRuntime(Runtime):
|
|
| 91 |
self.guards.complete_step(entropy=record['entropy_nats'],groups=64,
|
| 92 |
zero_groups=record['zero_groups'],parse_rate=rates['aggregate_parse_rate'],
|
| 93 |
failure_rate=0.,disk_fraction=disk_fraction())
|
|
|
|
|
|
|
| 94 |
if len(self.curve)>=400:
|
| 95 |
previous=self.curve[-400:-200];recent=self.curve[-200:]
|
| 96 |
mean=lambda rows,name: sum(row['reward_component_means'][name] for row in rows)/len(rows)
|
| 97 |
trends={key:(mean(previous,key),mean(recent,key)) for key in ('R1','R2','R3')}
|
| 98 |
if trends['R2'][1]>trends['R2'][0] and trends['R3'][1]>trends['R3'][0] and trends['R1'][1]<=trends['R1'][0]:
|
| 99 |
append_json(self.run_dir/'reward_hacking_tripwire.jsonl',dict(step=record['optimizer_step'],trends=trends))
|
| 100 |
-
|
| 101 |
return None
|
| 102 |
|
| 103 |
def evaluate_scale(self,step,checkpoint):
|
|
@@ -173,6 +175,9 @@ class ScaleRuntime(Runtime):
|
|
| 173 |
sft_rows=[json.loads(x) for x in sft_path.read_text().splitlines()]
|
| 174 |
comparison=compare_auc(sft_rows,records)
|
| 175 |
score=auc_ci(records,'vulnerable_vote_fraction')
|
|
|
|
|
|
|
|
|
|
| 176 |
greedy=self.trainer.llm.generate(prompts,sampling_params=SamplingParams(
|
| 177 |
temperature=0,max_tokens=1536,n=1,seed=42),use_tqdm=True)
|
| 178 |
greedy_rows=[]
|
|
@@ -194,7 +199,7 @@ class ScaleRuntime(Runtime):
|
|
| 194 |
metrics=dict(optimizer_step=step,checkpoint=str(checkpoint),adapter_label='E4 balanced reward',
|
| 195 |
inference_protocol='E2 exact: 8 generations, temperature 0.8, max_tokens 1536, seed 42',
|
| 196 |
n_rows=len(records),n_pairs=len({r['pair_id'] for r in records}),
|
| 197 |
-
parse_rate=
|
| 198 |
row_auc_pair_cluster_95_ci=score['pair_cluster_bootstrap_95_ci'],
|
| 199 |
auc_delta_vs_sft500=comparison,greedy_protocol='temperature 0, one generation, seed 42',
|
| 200 |
greedy_label_accuracy=correct/valid_n,greedy_per_class_accuracy=per_class,
|
|
@@ -257,8 +262,8 @@ def save_checkpoint(runtime,trainer,step,reason):
|
|
| 257 |
|
| 258 |
|
| 259 |
def finalize_completed_optimizer_step(runtime,trainer,state,*,checkpoint_due,eval_due):
|
| 260 |
-
"""Persist/evaluate a completed step before
|
| 261 |
-
|
| 262 |
step=state.global_step
|
| 263 |
checkpoint=None
|
| 264 |
if checkpoint_due:
|
|
@@ -267,8 +272,7 @@ def finalize_completed_optimizer_step(runtime,trainer,state,*,checkpoint_due,eva
|
|
| 267 |
if checkpoint is None:
|
| 268 |
raise SmokeStop('VALID evaluation requires a completed full-state checkpoint')
|
| 269 |
runtime.evaluate_scale(step,checkpoint)
|
| 270 |
-
|
| 271 |
-
raise SmokeStop(deferred_stop)
|
| 272 |
return checkpoint
|
| 273 |
|
| 274 |
|
|
|
|
| 91 |
self.guards.complete_step(entropy=record['entropy_nats'],groups=64,
|
| 92 |
zero_groups=record['zero_groups'],parse_rate=rates['aggregate_parse_rate'],
|
| 93 |
failure_rate=0.,disk_fraction=disk_fraction())
|
| 94 |
+
|
| 95 |
+
def check_reward_hacking_tripwire(self,record):
|
| 96 |
if len(self.curve)>=400:
|
| 97 |
previous=self.curve[-400:-200];recent=self.curve[-200:]
|
| 98 |
mean=lambda rows,name: sum(row['reward_component_means'][name] for row in rows)/len(rows)
|
| 99 |
trends={key:(mean(previous,key),mean(recent,key)) for key in ('R1','R2','R3')}
|
| 100 |
if trends['R2'][1]>trends['R2'][0] and trends['R3'][1]>trends['R3'][0] and trends['R1'][1]<=trends['R1'][0]:
|
| 101 |
append_json(self.run_dir/'reward_hacking_tripwire.jsonl',dict(step=record['optimizer_step'],trends=trends))
|
| 102 |
+
raise SmokeStop('R2_R3_rising_R1_flat_for_400_steps [Review #14]')
|
| 103 |
return None
|
| 104 |
|
| 105 |
def evaluate_scale(self,step,checkpoint):
|
|
|
|
| 175 |
sft_rows=[json.loads(x) for x in sft_path.read_text().splitlines()]
|
| 176 |
comparison=compare_auc(sft_rows,records)
|
| 177 |
score=auc_ci(records,'vulnerable_vote_fraction')
|
| 178 |
+
sampled_parse=float(np.mean([r['parse_rate'] for r in records]))
|
| 179 |
+
if sampled_parse<.95:
|
| 180 |
+
raise SmokeStop('E4_sampled_VALID_parse_below_95_percent [Review #14/D-GRPO-7]')
|
| 181 |
greedy=self.trainer.llm.generate(prompts,sampling_params=SamplingParams(
|
| 182 |
temperature=0,max_tokens=1536,n=1,seed=42),use_tqdm=True)
|
| 183 |
greedy_rows=[]
|
|
|
|
| 199 |
metrics=dict(optimizer_step=step,checkpoint=str(checkpoint),adapter_label='E4 balanced reward',
|
| 200 |
inference_protocol='E2 exact: 8 generations, temperature 0.8, max_tokens 1536, seed 42',
|
| 201 |
n_rows=len(records),n_pairs=len({r['pair_id'] for r in records}),
|
| 202 |
+
parse_rate=sampled_parse,row_auc=score['auc'],
|
| 203 |
row_auc_pair_cluster_95_ci=score['pair_cluster_bootstrap_95_ci'],
|
| 204 |
auc_delta_vs_sft500=comparison,greedy_protocol='temperature 0, one generation, seed 42',
|
| 205 |
greedy_label_accuracy=correct/valid_n,greedy_per_class_accuracy=per_class,
|
|
|
|
| 262 |
|
| 263 |
|
| 264 |
def finalize_completed_optimizer_step(runtime,trainer,state,*,checkpoint_due,eval_due):
|
| 265 |
+
"""Persist/evaluate a completed step before checking its trend tripwire."""
|
| 266 |
+
runtime.complete_step(state)
|
| 267 |
step=state.global_step
|
| 268 |
checkpoint=None
|
| 269 |
if checkpoint_due:
|
|
|
|
| 272 |
if checkpoint is None:
|
| 273 |
raise SmokeStop('VALID evaluation requires a completed full-state checkpoint')
|
| 274 |
runtime.evaluate_scale(step,checkpoint)
|
| 275 |
+
runtime.check_reward_hacking_tripwire(runtime.curve[-1])
|
|
|
|
| 276 |
return checkpoint
|
| 277 |
|
| 278 |
|
tests/test_review24_callback_order.py
CHANGED
|
@@ -8,7 +8,11 @@ import scripts.run_scale_grpo as runner
|
|
| 8 |
def test_completed_step_is_saved_and_evaluated_before_trend_tripwire(monkeypatch):
|
| 9 |
events = []
|
| 10 |
runtime = SimpleNamespace(
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
evaluate_scale=lambda step, checkpoint: events.append(("evaluate", step, checkpoint)),
|
| 13 |
)
|
| 14 |
trainer = object()
|
|
@@ -20,11 +24,12 @@ def test_completed_step_is_saved_and_evaluated_before_trend_tripwire(monkeypatch
|
|
| 20 |
runtime, trainer, SimpleNamespace(global_step=400),
|
| 21 |
checkpoint_due=True, eval_due=True)
|
| 22 |
|
| 23 |
-
assert events == ["
|
| 24 |
|
| 25 |
|
| 26 |
def test_valid_evaluation_cannot_run_without_full_state_checkpoint():
|
| 27 |
runtime = SimpleNamespace(complete_step=lambda state: None,
|
|
|
|
| 28 |
evaluate_scale=lambda *args: pytest.fail("evaluation must not run"))
|
| 29 |
with pytest.raises(runner.SmokeStop, match="requires a completed full-state checkpoint"):
|
| 30 |
runner.finalize_completed_optimizer_step(
|
|
|
|
| 8 |
def test_completed_step_is_saved_and_evaluated_before_trend_tripwire(monkeypatch):
|
| 9 |
events = []
|
| 10 |
runtime = SimpleNamespace(
|
| 11 |
+
curve=[{"optimizer_step": 400}],
|
| 12 |
+
complete_step=lambda state: events.append("complete_step"),
|
| 13 |
+
check_reward_hacking_tripwire=lambda record: (
|
| 14 |
+
events.append("tripwire_check"),
|
| 15 |
+
(_ for _ in ()).throw(runner.SmokeStop("registered_stop"))),
|
| 16 |
evaluate_scale=lambda step, checkpoint: events.append(("evaluate", step, checkpoint)),
|
| 17 |
)
|
| 18 |
trainer = object()
|
|
|
|
| 24 |
runtime, trainer, SimpleNamespace(global_step=400),
|
| 25 |
checkpoint_due=True, eval_due=True)
|
| 26 |
|
| 27 |
+
assert events == ["complete_step", ("save", 400), ("evaluate", 400, "ckpt"), "tripwire_check"]
|
| 28 |
|
| 29 |
|
| 30 |
def test_valid_evaluation_cannot_run_without_full_state_checkpoint():
|
| 31 |
runtime = SimpleNamespace(complete_step=lambda state: None,
|
| 32 |
+
check_reward_hacking_tripwire=lambda record: None,
|
| 33 |
evaluate_scale=lambda *args: pytest.fail("evaluation must not run"))
|
| 34 |
with pytest.raises(runner.SmokeStop, match="requires a completed full-state checkpoint"):
|
| 35 |
runner.finalize_completed_optimizer_step(
|