zefancainlp's picture
download
raw
5.24 kB
"""One owned native experiment, waiting only for an idle cool GPU on this node."""
import argparse,fcntl,json,os,subprocess,time,uuid,sys
from pathlib import Path
IMAGE='sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b'
def save(p,d):
t=p.with_suffix('.tmp');t.write_text(json.dumps(d,indent=2));t.replace(p)
def inventory():
text=subprocess.check_output(['nvidia-smi','--query-gpu=index,uuid,name,memory.used,temperature.gpu,clocks_event_reasons.hw_thermal_slowdown','--format=csv,noheader,nounits'],text=True,timeout=20);apps=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid','--format=csv,noheader,nounits'],text=True,timeout=20);pids={}
for line in apps.splitlines():
if ','in line:uid,pid=line.split(',');pids.setdefault(uid.strip(),[]).append(int(pid))
rows=[]
for line in text.splitlines():
index,uid,name,mem,temp,thermal=[x.strip()for x in line.split(',')];rows.append(dict(index=int(index),uuid=uid,name=name,memory=int(mem),temperature=int(temp),thermal=thermal,pids=pids.get(uid,[])))
return rows
def own_pids(container):
p=subprocess.run(['docker','top',container,'-eo','pid'],capture_output=True,text=True,timeout=10)
return {int(x.strip())for x in p.stdout.splitlines()if x.strip().isdigit()}
def stop_ours(name,token):
p=subprocess.run(['docker','inspect','--format','{{index .Config.Labels "source_graph_run"}}',name],capture_output=True,text=True,timeout=10)
if p.returncode==0 and p.stdout.strip()==token:subprocess.run(['docker','stop','-t','5',name],capture_output=True,timeout=20)
def main():
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();r=a.root.resolve();control=r/'control';control.mkdir(exist_ok=True);lock=(control/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);os.nice(10);deadline=time.monotonic()+12*3600
while time.monotonic()<deadline:
runtime=r/'runtime_transfer.json'
if not runtime.exists()or json.loads(runtime.read_text()).get('status')!='complete':save(control/'status.json',dict(status='waiting_for_frozen_runtime',time=time.time(),pid=os.getpid()));time.sleep(10);continue
cards=inventory();free=[g for g in cards if not g['pids']and g['memory']<100 and g['temperature']<60 and g['thermal']=='Not Active'and'H100'in g['name']]
if not free:save(control/'status.json',dict(status='waiting_for_idle_cool_GPU',time=time.time(),pid=os.getpid(),cards=cards));time.sleep(10);continue
card=min(free,key=lambda g:g['temperature']);time.sleep(2);card=next(g for g in inventory()if g['uuid']==card['uuid'])
if card['pids']or card['memory']>=100 or card['temperature']>=60:continue
token=uuid.uuid4().hex;name='source-graph-kernel-'+token;out=r/('attempt_'+token[:8]);out.mkdir();cmd=['docker','run','--rm','--name',name,'--label','source_graph_run='+token,'--gpus','device='+card['uuid'],'--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--pids-limit','512','--memory','64g','--shm-size','8g','--cpus','4','--tmpfs','/workdir:rw,exec,nosuid,size=24g,mode=1777','--tmpfs','/tmp:rw,exec,nosuid,size=4g,mode=1777','-e','CUDA_VISIBLE_DEVICES=0','-e','TRITON_CACHE_DIR=/workdir/triton-cache','-e','HOME=/workdir','--mount','type=bind,src='+str(r)+',dst=/app,readonly','--mount','type=bind,src='+str(out)+',dst=/results','--entrypoint','python',IMAGE,'/app/worker.py'];log=(out/'container.log').open('w');proc=subprocess.Popen(cmd,stdout=log,stderr=subprocess.STDOUT,start_new_session=True);state=dict(status='running',pid=os.getpid(),docker_pid=proc.pid,container=name,token=token,gpu=card,output=str(out),time=time.time());save(out/'ownership.json',state);save(control/'status.json',state);start=time.monotonic();interrupted=None
try:
while proc.poll()is None:
time.sleep(3);current=next(g for g in inventory()if g['uuid']==card['uuid']);foreign={p for p in set(current['pids'])-own_pids(name)if Path('/proc/'+str(p)).exists()}
if foreign or current['temperature']>=70 or current['thermal']!='Not Active':interrupted=dict(reason='foreign_process_or_temperature',foreign=sorted(foreign),gpu=current);stop_ours(name,token);break
if time.monotonic()-start>3600:interrupted=dict(reason='one_hour_execution_timeout');stop_ours(name,token);break
save(control/'status.json',dict(state,time=time.time(),current_gpu=current))
except BaseException:stop_ours(name,token);raise
proc.wait(timeout=30)
if interrupted:
save(out/'interrupted.json',interrupted);save(control/'status.json',dict(status='interrupted',detail=interrupted,time=time.time()))
if interrupted['reason']=='foreign_process_or_temperature':time.sleep(10);continue
return
status='complete'if proc.returncode==0 and(out/'result.json').exists()else'failed';save(control/'status.json',dict(state,status=status,returncode=proc.returncode,time=time.time()));print('KERNEL_NATIVE_FINISHED',status,flush=True);return
save(control/'status.json',dict(status='expired_waiting_for_GPU',time=time.time()))
if __name__=='__main__':main()

Xet Storage Details

Size:
5.24 kB
·
Xet hash:
cd888a5b0b0a1630dadabd37e6cac73e3a4261a646c3d03071a55fa6625f9b3c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.