| # Section-10 evaluation config. | |
| # | |
| # Eval-set paths are PARAMETERIZED: they point at the predecessor's verified | |
| # JSONs until section-03 regenerates them with hash receipts, at which point | |
| # only these paths change and every gate re-runs unmodified. When a `.sha256` | |
| # sidecar exists beside a set, `load_eval_set` CHECKS it. | |
| [eval] | |
| # Decode params. Greedy (temperature 0) for pass@1; the golden regression | |
| # matches the predecessor's config so its numbers stay comparable. | |
| max_new_tokens = 768 | |
| temperature = 0.0 | |
| top_p = 1.0 | |
| n_samples = 1 # majority@8 is run ONCE, by section-13's final pass | |
| batch_size = 16 | |
| seed = 17 | |
| # 0 = evaluate everything. Any positive value stamps the report 'smoke' so a | |
| # capped run can never be quoted as a headline number. | |
| limit = 0 | |
| # The ONE pre-registered primary contrast (paired MDE ~2-3.5pp on | |
| # GSM8K-full). Every other comparison is labeled exploratory. | |
| primary_contrast = "armB_vs_sft_gsm8k" | |
| [eval.sets] | |
| math500 = "/apps/development/rlhf/mistral-prm-ppo/artifacts/data/eval/math500.json" | |
| gsm8k = "/apps/development/rlhf/mistral-prm-ppo/artifacts/data/eval/gsm8k_test.json" | |
| [eval.verifier] | |
| # Inner per-comparison timeout handed to math_verify. | |
| timeout_seconds = 5 | |
| # MUST be >= 4x the inner timeout: verify() fans out over | |
| # product(gold_variants, pred_variants), so one call runs up to four | |
| # independently timeout-wrapped comparisons. Asserted in VerifierConfig. | |
| outer_timeout_seconds = 25.0 | |
| workers = 8 | |
| [eval.golden] | |
| # Known numbers under the OLD stack. Reproducing them validates the NEW | |
| # eval stack before section-11 trains anything (~1 GPU-hour). | |
| checkpoint = "/apps/development/rlhf/mistral-prm-ppo/runs/sft-mistral7b/checkpoint-200" | |
| math500 = 0.09 | |
| gsm8k = 0.258 | |