File size: 1,709 Bytes
d909bd8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | # Section-10 evaluation config.
#
# Eval-set paths are PARAMETERIZED: they point at the predecessor's verified
# JSONs until section-03 regenerates them with hash receipts, at which point
# only these paths change and every gate re-runs unmodified. When a `.sha256`
# sidecar exists beside a set, `load_eval_set` CHECKS it.
[eval]
# Decode params. Greedy (temperature 0) for pass@1; the golden regression
# matches the predecessor's config so its numbers stay comparable.
max_new_tokens = 768
temperature = 0.0
top_p = 1.0
n_samples = 1 # majority@8 is run ONCE, by section-13's final pass
batch_size = 16
seed = 17
# 0 = evaluate everything. Any positive value stamps the report 'smoke' so a
# capped run can never be quoted as a headline number.
limit = 0
# The ONE pre-registered primary contrast (paired MDE ~2-3.5pp on
# GSM8K-full). Every other comparison is labeled exploratory.
primary_contrast = "armB_vs_sft_gsm8k"
[eval.sets]
math500 = "/apps/development/rlhf/mistral-prm-ppo/artifacts/data/eval/math500.json"
gsm8k = "/apps/development/rlhf/mistral-prm-ppo/artifacts/data/eval/gsm8k_test.json"
[eval.verifier]
# Inner per-comparison timeout handed to math_verify.
timeout_seconds = 5
# MUST be >= 4x the inner timeout: verify() fans out over
# product(gold_variants, pred_variants), so one call runs up to four
# independently timeout-wrapped comparisons. Asserted in VerifierConfig.
outer_timeout_seconds = 25.0
workers = 8
[eval.golden]
# Known numbers under the OLD stack. Reproducing them validates the NEW
# eval stack before section-11 trains anything (~1 GPU-hour).
checkpoint = "/apps/development/rlhf/mistral-prm-ppo/runs/sft-mistral7b/checkpoint-200"
math500 = 0.09
gsm8k = 0.258
|