end000's picture
Release Sol014 mathematical specialization with paired evaluations, regressions and exact replay artifacts
fc32096 verified
Raw History Blame Contribute Delete
1.91 kB
import hashlib
import json
from common import ROOT, rows, save_json
from code_sandbox import check_code
directory = ROOT/'data/coding'
records = rows(directory/'all_tasks.jsonl')
accepted, outcomes = [], []
for i,row in enumerate(records):
result = check_code(row['reference_code'],row['entry_point'],row['test_code'])
outcomes.append({'id':row['id'],**result})
if result['passed']:
accepted.append(row)
if i%50==0:
print('Canonical task checks',i+1,'accepted',len(accepted),flush=True)
save_json(directory/'canonical_validation.json',outcomes)
groups = {'discovery':[],'validation':[],'confirmation':[]}
for source in ['humaneval','mbpp']:
pool = sorted([r for r in accepted if r['source']==source],key=lambda r:hashlib.sha256(('wspforge-code-v1:'+r['id']).encode()).hexdigest())
for split,start,count in [('discovery',0,12),('validation',12,12),('confirmation',24,24)]:
if len(pool)<start+count:
raise ValueError('Not enough eligible canonical tasks')
groups[split]+=pool[start:start+count]
for name,items in groups.items():
(directory/f'{name}.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in items))
manifest = {name:{'count':len(items),'ids':[r['id'] for r in items],
'sha256':hashlib.sha256((directory/f'{name}.jsonl').read_bytes()).hexdigest()} for name,items in groups.items()}
manifest['_eligibility'] = {'original_tasks':len(records),'canonical_passed_sandbox':len(accepted),
'criterion':'canonical code passes fixed sandbox and standard-library guard before any model answer is measured',
'note':'Compact custom subsets, not official full-benchmark leaderboard scores; pretraining exposure is unknown.'}
save_json(directory/'splits_manifest.json',manifest)
print(json.dumps({k:len(v) for k,v in groups.items()}),flush=True)