zefancainlp's picture
download
raw
8.54 kB
"""Execution-only recovery: ignore stale GPU PID samples after container exit."""
from __future__ import annotations
import argparse
import fcntl
import json
import os
from pathlib import Path
import subprocess
import threading
import time
import remote_judge as base
from common import IMAGE, IMAGE_MANIFEST, atomic, sha
def verified_foreign_pids(pids, own_namespace, read_namespace=None):
"""A stale nvidia-smi PID is not evidence of an active foreign process."""
if own_namespace is None:
return []
read_namespace = read_namespace or (lambda pid: os.readlink(f'/proc/{pid}/ns/pid'))
foreign = []
for pid in pids:
try:
namespace = read_namespace(pid)
except (FileNotFoundError, ProcessLookupError):
continue
if namespace != own_namespace:
foreign.append(pid)
return foreign
class FixedJudge(base.Judge):
def _evaluate(self, code, request_id, identity):
from examples.gpu_mode.local_judge import docker_command, parse_result
directory = self.root/'requests'/request_id
cached = directory/'result.json'
if cached.exists():
result = json.loads(cached.read_text())
if result['program_sha256'] != sha(code) or result['execution_timeout_seconds'] != 530:
raise base.InfrastructureError('Completed evaluator request identity changed')
return result
directory.mkdir(exist_ok=False)
atomic(directory/'request.json',dict(program_sha256=sha(code),identity=identity,code=code,
execution_seconds=530,queued=time.time()))
queued = time.monotonic()
with self.serial:
lock_root = Path('/data/xiaofan/kernel-component-gpu-locks')
lock_root.mkdir(parents=True, exist_ok=True)
with (lock_root/f'gpu-{self.gpu}.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
while True:
state = base.gpu_state(self.gpu)
if 'H100' not in state['name'] or 'NVL' in state['name']:
raise base.InfrastructureError('Wrong timing GPU model')
if not state['pids'] and state['memory_mib']<100 and state['temperature']<75:
break
atomic(self.root/'waiting_gpu.json',dict(state=state,time=time.time()))
time.sleep(5)
wait = time.monotonic()-queued
name = 'kernel-private-'+request_id
command = docker_command(self.execution_image,state['uuid'],name)
at = command.index('--runtime')
command[at:at] = ['--label','kernel_private_request='+request_id]
proc = subprocess.Popen(command,text=True,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
started = time.monotonic()
atomic(directory/'execution.json',dict(gpu=state,pid=proc.pid,container=name,started=time.time(),command=command))
data = json.dumps(dict(code=code))
timed_out, own_namespace = False, None
try:
while True:
try:
stdout,stderr = proc.communicate(input=data,timeout=1)
break
except subprocess.TimeoutExpired:
data = None
# The Docker CLI may finish between the first poll and
# nvidia-smi. Collect its result instead of inventing a collision.
if proc.poll() is not None:
stdout,stderr = proc.communicate(timeout=30)
break
current = base.gpu_state(self.gpu)
if own_namespace is None:
inspected = subprocess.run(['docker','inspect','-f','{{.State.Pid}}',name],
capture_output=True,text=True,timeout=10)
if inspected.returncode==0 and inspected.stdout.strip().isdigit():
pid = int(inspected.stdout.strip())
if pid:
try:
own_namespace = os.readlink(f'/proc/{pid}/ns/pid')
except (FileNotFoundError,ProcessLookupError):
pass
if proc.poll() is not None:
stdout,stderr = proc.communicate(timeout=30)
break
foreign = verified_foreign_pids(current['pids'],own_namespace)
if foreign or current['temperature']>=75:
atomic(directory/'interference.json',dict(state=current,foreign_pids=foreign,
own_namespace=own_namespace,returncode=proc.poll(),time=time.time()))
raise base.InfrastructureError('Timing interrupted by verified active foreign GPU activity or temperature')
if time.monotonic()-started>530:
timed_out=True
break
finally:
if proc.poll() is None:
label = subprocess.run(['docker','inspect','-f','{{index .Config.Labels "kernel_private_request"}}',name],
text=True,capture_output=True,timeout=10)
if label.returncode==0 and label.stdout.strip()==request_id:
subprocess.run(['docker','rm','-f',name],check=True,capture_output=True,timeout=25)
elif label.returncode==0:
raise base.InfrastructureError('Refusing to clean a foreign container')
stdout,stderr = proc.communicate(timeout=30)
(directory/'stdout.log').write_text(stdout)
(directory/'stderr.log').write_text(stderr)
if timed_out:
result=dict(status='candidate_invalid',correctness=0.,reward=0.,raw_score=-1_000_000.,score_us=None,
wall_seconds=time.monotonic()-started,result={'candidate_timeout_seconds':530})
else:
result=parse_result(stdout)
if proc.returncode:
raise base.InfrastructureError('Original Docker judge failed: '+stderr[-1000:])
result.update(request_id=request_id,gpu_index=self.gpu,gpu_uuid=state['uuid'],image=self.execution_image,
canonical_image_config_sha256=IMAGE,image_manifest_sha256=IMAGE_MANIFEST,queue_seconds=wait,
host_execution_seconds=time.monotonic()-started,execution_timeout_seconds=530,program_sha256=sha(code),
node=__import__('socket').gethostname(),monitor_revision='stale-pid-race-fix-v1')
atomic(directory/'result.json',result)
return result
class Client(base.Client):
def __init__(self,url,token,experiment):
super().__init__(url,token)
self.experiment,self.local = experiment,threading.local()
def evaluate(self,code,*,timeout_seconds=530,**kwargs):
slot = getattr(self.local,'slot',None)
if slot is None:
return super().evaluate(code,timeout_seconds=timeout_seconds,**kwargs)
request_id = sha(self.experiment+str(tuple(slot))+sha(code)+str(timeout_seconds))[:32]
result = self._request('/evaluate',dict(code=code,request_id=request_id,timeout=timeout_seconds),timeout=86400)
if result.get('infrastructure_error'):
raise base.InfrastructureError(result['infrastructure_error'])
if result.get('program_sha256')!=sha(code) or result.get('canonical_image_config_sha256',result.get('image'))!=IMAGE:
raise base.InfrastructureError('Remote result does not match the exact recovered slot/program')
return result
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);p.add_argument('--host',required=True)
p.add_argument('--port',type=int,required=True);p.add_argument('--gpu',type=int,required=True);p.add_argument('--allowed-client',required=True)
a=p.parse_args();base.Judge=FixedJudge
base.serve(a.root,a.host,a.port,a.gpu,a.allowed_client)

Xet Storage Details

Size:
8.54 kB
·
Xet hash:
5e2b1e00b658d30ba164b5bff08ec016e1ad867d70d83046bef2bfd25b22b621

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.