File size: 1,907 Bytes
fc32096 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | import hashlib
import json
from common import ROOT, rows, save_json
from code_sandbox import check_code
directory = ROOT/'data/coding'
records = rows(directory/'all_tasks.jsonl')
accepted, outcomes = [], []
for i,row in enumerate(records):
result = check_code(row['reference_code'],row['entry_point'],row['test_code'])
outcomes.append({'id':row['id'],**result})
if result['passed']:
accepted.append(row)
if i%50==0:
print('Canonical task checks',i+1,'accepted',len(accepted),flush=True)
save_json(directory/'canonical_validation.json',outcomes)
groups = {'discovery':[],'validation':[],'confirmation':[]}
for source in ['humaneval','mbpp']:
pool = sorted([r for r in accepted if r['source']==source],key=lambda r:hashlib.sha256(('wspforge-code-v1:'+r['id']).encode()).hexdigest())
for split,start,count in [('discovery',0,12),('validation',12,12),('confirmation',24,24)]:
if len(pool)<start+count:
raise ValueError('Not enough eligible canonical tasks')
groups[split]+=pool[start:start+count]
for name,items in groups.items():
(directory/f'{name}.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in items))
manifest = {name:{'count':len(items),'ids':[r['id'] for r in items],
'sha256':hashlib.sha256((directory/f'{name}.jsonl').read_bytes()).hexdigest()} for name,items in groups.items()}
manifest['_eligibility'] = {'original_tasks':len(records),'canonical_passed_sandbox':len(accepted),
'criterion':'canonical code passes fixed sandbox and standard-library guard before any model answer is measured',
'note':'Compact custom subsets, not official full-benchmark leaderboard scores; pretraining exposure is unknown.'}
save_json(directory/'splits_manifest.json',manifest)
print(json.dumps({k:len(v) for k,v in groups.items()}),flush=True)
|