File size: 2,746 Bytes
079e13c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | """Verify Beam's coding answers against TACO's tests on Modal (modal_app/code_verify.py). Nothing runs locally.
python3 scripts/verify_code.py data/code/beam/run-code-*.jsonl --out data/code/verdicts.jsonl
python3 scripts/verify_code.py --reference 300 --out data/code/reference_check.jsonl # checks the checker
--reference runs TACO's own verified solutions for a random sample of problems; nearly all should pass.
"""
import argparse
import glob
import json
import random
from pathlib import Path
import modal
ROOT = Path(__file__).resolve().parent.parent
CHUNK = 25
def main():
ap = argparse.ArgumentParser()
ap.add_argument("files", nargs="*")
ap.add_argument("--out", required=True)
ap.add_argument("--reference", type=int, default=0)
ap.add_argument("--skip", nargs="*", default=[], help="verdict files whose answers are already verified")
args = ap.parse_args()
tests = {t["id"]: t for t in map(json.loads, open(ROOT / "data" / "code" / "tests.jsonl"))}
done = {json.loads(l)["key"] for f in args.skip for l in open(f)}
records = []
if args.reference:
raw = {f"taco-{r['id']}": r for r in json.load(open(ROOT / "data" / "code_raw" / "taco" / "taco_verified.json"))}
for pid in random.Random(0).sample(sorted(tests), args.reference):
sols = raw[pid]["solutions"]
sol = (json.loads(sols) if isinstance(sols, str) else sols)[0]
records.append({"key": pid, "content": f"```python\n{sol}\n```", "test": tests[pid]})
for pattern in args.files:
for path in glob.glob(pattern):
for line in open(path):
r = json.loads(line)
if "error" in r or r["finish_reason"] != "stop" or r["id"] not in tests:
continue
if f"{r['id']}#{r.get('sample_idx', 0)}" in done:
continue
records.append({"key": f"{r['id']}#{r.get('sample_idx', 0)}", "content": r["content"],
"test": tests[r["id"]]})
print(f"{len(records)} answers to verify")
fn = modal.Function.from_name("beam-dory-codeverify", "verify", environment_name="main")
chunks = [records[i:i + CHUNK] for i in range(0, len(records), CHUNK)]
n = passed = 0
with open(args.out, "w") as f:
for res in fn.map(chunks, order_outputs=False):
for v in res:
f.write(json.dumps(v) + "\n")
n += 1
passed += v["passed"]
if n % 1000 < CHUNK:
print(f" {n}/{len(records)} verified, {passed} passed", flush=True)
print(f"done: {passed}/{n} passed ({100 * passed / max(n, 1):.1f}%) -> {args.out}")
if __name__ == "__main__":
main()
|