zefancainlp's picture
download
raw
8.33 kB
"""Frozen GraphSearch, real CUDA measurements, unchanged official final judge."""
import os
os.environ['CUDA_VISIBLE_DEVICES']='0'
for key in ['OMP_NUM_THREADS','OPENBLAS_NUM_THREADS','MKL_NUM_THREADS']:os.environ[key]='1'
import copy,hashlib,importlib.util,json,math,statistics,sys,time,traceback
from pathlib import Path
sys.path[:0]=['/app/code','/app/framework','/app/official/examples/gpu_mode/lib/bioml/trimul']
import torch,triton,yaml,reference,judge_worker
from fusion import fuse_programs
OUT=Path('/results')
def save(path,value):
path=Path(path);temp=path.with_suffix('.tmp');temp.write_text(json.dumps(value,indent=2,default=str,allow_nan=False));temp.replace(path)
def load(path,name):
spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);sys.modules[name]=m;spec.loader.exec_module(m);return m
def unchanged(a,b):
if isinstance(b,torch.Tensor):return bool(torch.equal(a,b))
if isinstance(b,dict):return a.keys()==b.keys()and all(unchanged(a[k],v)for k,v in b.items())
if isinstance(b,(list,tuple)):return len(a)==len(b)and all(unchanged(x,y)for x,y in zip(a,b))
return a==b
def main():
assert torch.__version__.split('+')[0]=='2.7.1'and triton.__version__=='3.3.1'
assert torch.cuda.device_count()==1
torch.set_num_threads(1);torch.set_grad_enabled(False)
official=Path('/app/official');manifest=judge_worker.verify_official(official);native_eval=load(official/'examples/gpu_mode/lib/bioml/trimul/eval.py','unchanged_native_eval');task=yaml.safe_load((official/'examples/gpu_mode/lib/bioml/trimul/task.yml').read_text());cohort=json.loads(Path('/app/cohort.json').read_text());sources=cohort['sources'];started=time.monotonic();count=0;bench_cache={}
engine=hashlib.sha256(Path('/app/code/graph_search.py').read_bytes()).hexdigest();assert engine=='bcce69ee8d45add95adc2e3be1dbfb48b3f404243f216ba29e92cc558ea86285'
protocol=dict(engine_sha256=engine,torch=torch.__version__,triton=triton.__version__,cuda=torch.version.cuda,device=torch.cuda.get_device_name(),official_manifest=manifest,search_queries=24,width=4,search_benchmarks=[0,3,6],screen_timing_samples=12,screen_correctness_cases=len(task['tests']),known_joint_withheld=True,API_calls=0,new_model_rollouts=0,output_artifact_mixing=False,pure_output_cache=False)
save(OUT/'protocol.json',protocol)
def problem(case,offset):
key=(case,offset)
if key not in bench_cache:
args=copy.deepcopy(task['benchmarks'][case]);args['seed']=native_eval._combine(args['seed'],offset);bench_cache[key]=reference.generate_input(**args)
return native_eval._clone_data(bench_cache[key])
def evaluate(source):
nonlocal count
index=count;count+=1;folder=OUT/('candidate_'+str(index).zfill(3));folder.mkdir();path=folder/'program.py';path.write_text(source+'\ncustom_kernel = build()\n');start=time.monotonic();record=dict(valid=False,loss=None,source_sha256=hashlib.sha256(source.encode()).hexdigest(),checks=[],cases=[])
try:
module=load(path,'source_graph_candidate_'+str(index));fn=module.custom_kernel
for i,args in enumerate(task['tests']):
data=reference.generate_input(**args);before=native_eval._clone_data(data);value=fn(data);torch.cuda.synchronize()
if not unchanged(data,before):raise ValueError('candidate mutates native inputs')
good,message=reference.check_implementation(before,value);record['checks'].append(dict(case=i,valid=bool(good),message=str(message)[:300]));del data,before,value
if not good:raise ValueError('native correctness failed')
values=[]
for i in [0,3,6]:
data=problem(i,20261103);before=native_eval._clone_data(data);value=fn(data);torch.cuda.synchronize()
if not unchanged(data,before):raise ValueError('candidate mutates benchmark inputs')
good,message=reference.check_implementation(before,value);del before,value
if not good:raise ValueError('benchmark native correctness failed: '+str(message))
for _ in range(3):fn(data)
torch.cuda.synchronize();times=[]
for _ in range(12):
a=torch.cuda.Event(enable_timing=True);b=torch.cuda.Event(enable_timing=True);a.record();value=fn(data);b.record();b.synchronize();times.append(a.elapsed_time(b)*1000.);del value
median=statistics.median(times);values.append(median);record['cases'].append(dict(case=i,us=median,samples_us=times));del data
record.update(valid=True,loss=math.exp(statistics.mean(math.log(x)for x in values)))
except Exception as error:
record.update(error=type(error).__name__+': '+str(error)[:1800],traceback=traceback.format_exc()[-5000:])
if any(s in str(error).lower()for s in ['illegal memory','device-side assert','cuda error','out of memory']):save(folder/'native.json',record);raise
finally:
record['wall_seconds']=time.monotonic()-start;save(folder/'native.json',record)
print('GPU_SOURCE_PROGRAM',index,record['valid'],record['loss'],record.get('error'),flush=True);return record
result=fuse_programs(sources,entry='custom_kernel',language='python',evaluate=evaluate,budget=24,width=4,base='0',journal=OUT/'search.jsonl');winner=result.pop('program');assert winner is not None;winner+='\ncustom_kernel = build()\n';(OUT/'winner.py').write_text(winner);save(OUT/'search_result.json',result)
# Native official runner and its original leaderboard score, no replacement
# correctness tolerance or timing formula is used for final comparisons.
bench_cache.clear();torch.cuda.synchronize();torch.cuda.empty_cache()
full=[];codes={'source_'+key:value+'\ncustom_kernel = build()\n'for key,value in {k:(OUT/('candidate_'+str(i).zfill(3))/'program.py').read_text().removesuffix('\ncustom_kernel = build()\n')for i,k in enumerate(sources)}.items()}
codes['fusion']=winner;codes['published']=(official/'results/kernel-engineering/trimul.py').read_text()
def official_one(label,code,repeat):
torch.cuda.synchronize();torch.cuda.empty_cache();start=time.monotonic();value=judge_worker.evaluate(code,official);save(OUT/(label+'_official_'+str(repeat)+'.json'),value);row=dict(label=label,repeat=repeat,valid=value['correctness']==1.,us=value['score_us'],seconds=time.monotonic()-start);full.append(row);save(OUT/'official_partial.json',full);print('OFFICIAL_NATIVE',row,flush=True);return row
for label,code in codes.items():official_one(label,code,0)
originals=[r for r in full if r['label'].startswith('source_')and r['valid']];assert originals;control=min(originals,key=lambda x:x['us'])['label']
for repeat,order in [(1,['fusion',control,'published']),(2,[control,'published','fusion'])]:
for label in order:official_one(label,codes[label],repeat)
fresh=[]
for label in [control,'fusion']:
p=OUT/(label+'_fresh.py');p.write_text(codes[label]);fn=load(p,'fresh_'+label).custom_kernel;checks=[]
for case,args in enumerate(task['benchmarks']):
for nomask in [False,True]:
arg={**args,'seed':int(args['seed'])+104729,'nomask':nomask};data=reference.generate_input(**arg);before=native_eval._clone_data(data);value=fn(data);torch.cuda.synchronize();same=unchanged(data,before);good,message=reference.check_implementation(before,value);checks.append(dict(case=case,nomask=nomask,valid=bool(good and same),message=str(message)[:300]));del data,before,value
fresh.append(dict(label=label,valid=all(x['valid']for x in checks),checks=checks));save(OUT/'fresh_correctness.json',fresh)
means={label:statistics.mean(r['us']for r in full if r['label']==label and r['valid'])for label in [control,'fusion','published']};done=dict(complete=True,best_original_label=control,official_mean_us=means,runtime_reduction=1-means['fusion']/means[control],native_comparisons=full,all_fresh_valid=all(r['valid']for r in fresh),search_evaluations=result['evaluations'],search_best_us=result['best']['loss'],wall_seconds=time.monotonic()-started,protocol=protocol,API_calls=0)
save(OUT/'result.json',done);print('SOURCE_GRAPH_KERNEL_COMPLETE',done,flush=True)
if __name__=='__main__':main()

Xet Storage Details

Size:
8.33 kB
·
Xet hash:
5d14aaa43101d23dee0d4dd9618f19886f4d2cd08304215a07dfaacfbbcc747f

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.