Buckets:
| """Execution-only recovery: ignore stale GPU PID samples after container exit.""" | |
| from __future__ import annotations | |
| import argparse | |
| import fcntl | |
| import json | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import threading | |
| import time | |
| import remote_judge as base | |
| from common import IMAGE, IMAGE_MANIFEST, atomic, sha | |
| def verified_foreign_pids(pids, own_namespace, read_namespace=None): | |
| """A stale nvidia-smi PID is not evidence of an active foreign process.""" | |
| if own_namespace is None: | |
| return [] | |
| read_namespace = read_namespace or (lambda pid: os.readlink(f'/proc/{pid}/ns/pid')) | |
| foreign = [] | |
| for pid in pids: | |
| try: | |
| namespace = read_namespace(pid) | |
| except (FileNotFoundError, ProcessLookupError): | |
| continue | |
| if namespace != own_namespace: | |
| foreign.append(pid) | |
| return foreign | |
| class FixedJudge(base.Judge): | |
| def _evaluate(self, code, request_id, identity): | |
| from examples.gpu_mode.local_judge import docker_command, parse_result | |
| directory = self.root/'requests'/request_id | |
| cached = directory/'result.json' | |
| if cached.exists(): | |
| result = json.loads(cached.read_text()) | |
| if result['program_sha256'] != sha(code) or result['execution_timeout_seconds'] != 530: | |
| raise base.InfrastructureError('Completed evaluator request identity changed') | |
| return result | |
| directory.mkdir(exist_ok=False) | |
| atomic(directory/'request.json',dict(program_sha256=sha(code),identity=identity,code=code, | |
| execution_seconds=530,queued=time.time())) | |
| queued = time.monotonic() | |
| with self.serial: | |
| lock_root = Path('/data/xiaofan/kernel-component-gpu-locks') | |
| lock_root.mkdir(parents=True, exist_ok=True) | |
| with (lock_root/f'gpu-{self.gpu}.lock').open('a') as lock: | |
| fcntl.flock(lock,fcntl.LOCK_EX) | |
| while True: | |
| state = base.gpu_state(self.gpu) | |
| if 'H100' not in state['name'] or 'NVL' in state['name']: | |
| raise base.InfrastructureError('Wrong timing GPU model') | |
| if not state['pids'] and state['memory_mib']<100 and state['temperature']<75: | |
| break | |
| atomic(self.root/'waiting_gpu.json',dict(state=state,time=time.time())) | |
| time.sleep(5) | |
| wait = time.monotonic()-queued | |
| name = 'kernel-private-'+request_id | |
| command = docker_command(self.execution_image,state['uuid'],name) | |
| at = command.index('--runtime') | |
| command[at:at] = ['--label','kernel_private_request='+request_id] | |
| proc = subprocess.Popen(command,text=True,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) | |
| started = time.monotonic() | |
| atomic(directory/'execution.json',dict(gpu=state,pid=proc.pid,container=name,started=time.time(),command=command)) | |
| data = json.dumps(dict(code=code)) | |
| timed_out, own_namespace = False, None | |
| try: | |
| while True: | |
| try: | |
| stdout,stderr = proc.communicate(input=data,timeout=1) | |
| break | |
| except subprocess.TimeoutExpired: | |
| data = None | |
| # The Docker CLI may finish between the first poll and | |
| # nvidia-smi. Collect its result instead of inventing a collision. | |
| if proc.poll() is not None: | |
| stdout,stderr = proc.communicate(timeout=30) | |
| break | |
| current = base.gpu_state(self.gpu) | |
| if own_namespace is None: | |
| inspected = subprocess.run(['docker','inspect','-f','{{.State.Pid}}',name], | |
| capture_output=True,text=True,timeout=10) | |
| if inspected.returncode==0 and inspected.stdout.strip().isdigit(): | |
| pid = int(inspected.stdout.strip()) | |
| if pid: | |
| try: | |
| own_namespace = os.readlink(f'/proc/{pid}/ns/pid') | |
| except (FileNotFoundError,ProcessLookupError): | |
| pass | |
| if proc.poll() is not None: | |
| stdout,stderr = proc.communicate(timeout=30) | |
| break | |
| foreign = verified_foreign_pids(current['pids'],own_namespace) | |
| if foreign or current['temperature']>=75: | |
| atomic(directory/'interference.json',dict(state=current,foreign_pids=foreign, | |
| own_namespace=own_namespace,returncode=proc.poll(),time=time.time())) | |
| raise base.InfrastructureError('Timing interrupted by verified active foreign GPU activity or temperature') | |
| if time.monotonic()-started>530: | |
| timed_out=True | |
| break | |
| finally: | |
| if proc.poll() is None: | |
| label = subprocess.run(['docker','inspect','-f','{{index .Config.Labels "kernel_private_request"}}',name], | |
| text=True,capture_output=True,timeout=10) | |
| if label.returncode==0 and label.stdout.strip()==request_id: | |
| subprocess.run(['docker','rm','-f',name],check=True,capture_output=True,timeout=25) | |
| elif label.returncode==0: | |
| raise base.InfrastructureError('Refusing to clean a foreign container') | |
| stdout,stderr = proc.communicate(timeout=30) | |
| (directory/'stdout.log').write_text(stdout) | |
| (directory/'stderr.log').write_text(stderr) | |
| if timed_out: | |
| result=dict(status='candidate_invalid',correctness=0.,reward=0.,raw_score=-1_000_000.,score_us=None, | |
| wall_seconds=time.monotonic()-started,result={'candidate_timeout_seconds':530}) | |
| else: | |
| result=parse_result(stdout) | |
| if proc.returncode: | |
| raise base.InfrastructureError('Original Docker judge failed: '+stderr[-1000:]) | |
| result.update(request_id=request_id,gpu_index=self.gpu,gpu_uuid=state['uuid'],image=self.execution_image, | |
| canonical_image_config_sha256=IMAGE,image_manifest_sha256=IMAGE_MANIFEST,queue_seconds=wait, | |
| host_execution_seconds=time.monotonic()-started,execution_timeout_seconds=530,program_sha256=sha(code), | |
| node=__import__('socket').gethostname(),monitor_revision='stale-pid-race-fix-v1') | |
| atomic(directory/'result.json',result) | |
| return result | |
| class Client(base.Client): | |
| def __init__(self,url,token,experiment): | |
| super().__init__(url,token) | |
| self.experiment,self.local = experiment,threading.local() | |
| def evaluate(self,code,*,timeout_seconds=530,**kwargs): | |
| slot = getattr(self.local,'slot',None) | |
| if slot is None: | |
| return super().evaluate(code,timeout_seconds=timeout_seconds,**kwargs) | |
| request_id = sha(self.experiment+str(tuple(slot))+sha(code)+str(timeout_seconds))[:32] | |
| result = self._request('/evaluate',dict(code=code,request_id=request_id,timeout=timeout_seconds),timeout=86400) | |
| if result.get('infrastructure_error'): | |
| raise base.InfrastructureError(result['infrastructure_error']) | |
| if result.get('program_sha256')!=sha(code) or result.get('canonical_image_config_sha256',result.get('image'))!=IMAGE: | |
| raise base.InfrastructureError('Remote result does not match the exact recovered slot/program') | |
| return result | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);p.add_argument('--host',required=True) | |
| p.add_argument('--port',type=int,required=True);p.add_argument('--gpu',type=int,required=True);p.add_argument('--allowed-client',required=True) | |
| a=p.parse_args();base.Judge=FixedJudge | |
| base.serve(a.root,a.host,a.port,a.gpu,a.allowed_client) | |
Xet Storage Details
- Size:
- 8.54 kB
- Xet hash:
- 5e2b1e00b658d30ba164b5bff08ec016e1ad867d70d83046bef2bfd25b22b621
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.