File size: 2,006 Bytes
014b14c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | """Rebalance the Model-V dataset by downsampling all-correct (-1) examples.
ProcessBench F1 = harmonic_mean(acc_error, acc_correct), so a training set skewed
toward -1 biases V to over-accept, crushing acc_error and thus F1. This makes a
~balanced default (correct:erroneous = --ratio) and preserves the original.
python scripts/rebalance_verifier.py # train.jsonl + val.jsonl, ratio 1.0
python scripts/rebalance_verifier.py --ratio 1.5
Original files are moved to *_full.jsonl; the balanced set takes the canonical name.
"""
import argparse
import json
import random
from pathlib import Path
def rebalance(path: Path, ratio: float, seed: int) -> tuple[int, int, int]:
rows = [json.loads(l) for l in path.read_text().splitlines() if l.strip()]
err = [r for r in rows if r.get("first_error_index", -1) != -1]
cor = [r for r in rows if r.get("first_error_index", -1) == -1]
rng = random.Random(seed)
rng.shuffle(cor)
keep_cor = cor[: int(round(len(err) * ratio))]
balanced = err + keep_cor
rng.shuffle(balanced)
full = path.with_name(path.stem + "_full.jsonl")
if not full.exists():
path.rename(full) # preserve original once
with open(path, "w") as f:
for r in balanced:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
return len(err), len(keep_cor), len(balanced)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dir", default="data/verifier")
ap.add_argument("--ratio", type=float, default=1.0, help="correct:erroneous ratio to keep")
ap.add_argument("--seed", type=int, default=0)
args = ap.parse_args()
for name in ("train.jsonl", "val.jsonl"):
p = Path(args.dir) / name
if not p.exists():
continue
e, c, t = rebalance(p, args.ratio, args.seed)
print(f"{name}: {t} rows (erroneous {e}, all-correct {c}); original -> {p.stem}_full.jsonl")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|