File size: 2,507 Bytes
fa599d5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """CPU-only verification of REW-04 v2 reward design.
Confirms two GRPO-blocking properties:
1. A synthetic 8-completion group produces non-zero std (v1's std was 0.0
across every group, which is why the reward curve was flat).
2. The ordering correct+0calls > correct+1call > correct+spam holds.
Run BEFORE spending any HF Jobs credits on a retrain.
"""
from __future__ import annotations
import statistics
import types
from rewards.compose import compose_reward_single
cfg = types.SimpleNamespace(
alpha=0.2,
weights=types.SimpleNamespace(
correctness=0.70, token_budget=0.15, recursion_efficiency=0.15
),
token_budget_variant="capped_linear",
answer_regex="<answer>(.*?)</answer>",
max_calls=4,
)
GOLD = "silver"
GENERATIONS = [
("correct + 0 llm calls (REPL grep)",
"```python\nimport re\nm=re.search('silver', ctx)\nprint(m.group())\n```\n<answer>silver</answer>"),
("correct + 1 llm call",
"```python\nans=llm('color', ctx[:5000])\nprint(ans)\n```\n<answer>silver</answer>"),
("correct + 3 llm calls (wasteful)",
"```python\na=llm('q1',ctx[:1000])\nb=llm('q2',ctx[1000:2000])\nc=llm('q3',ctx[2000:3000])\n```\n<answer>silver</answer>"),
("correct + bare answer (no code, trivial-task path)",
"<answer>silver</answer>"),
("wrong + format",
"<answer>gold</answer>"),
("wrong + no format",
"the color is gold"),
("right text + no format (v1 collapse mode)",
"silver"),
("format-only spam",
"<answer></answer>"),
]
print(f"{'idx':>3} {'score':>6} {'calls':>5} description")
print("-" * 78)
scores = []
for i, (desc, gen) in enumerate(GENERATIONS):
s, m = compose_reward_single(gen, GOLD, 200, cfg)
scores.append(s)
print(f"{i:>3} {s:>6.3f} {int(m['llm_call_count']):>5d} {desc}")
print("-" * 78)
print(f"group mean: {statistics.mean(scores):.4f}")
print(f"group std: {statistics.stdev(scores):.4f} (must be > 0.10 for GRPO advantage)")
print(f"max - min: {max(scores) - min(scores):.4f}")
# Hard gates — exit non-zero if any fail
assert statistics.stdev(scores) > 0.10, "FAIL: group std too low; GRPO will not learn"
assert scores[0] > scores[1] > scores[2], (
f"FAIL: efficiency ordering broken (got {scores[0]:.3f} > {scores[1]:.3f} > {scores[2]:.3f})"
)
assert scores[0] > scores[4], "FAIL: correct must beat wrong"
assert scores[7] <= 0.25, "FAIL: format-only spam not capped"
print("\nPASS: REW-04 v2 produces learnable variance and correct orderings")
|