zefancainlp's picture
download
raw
9.38 kB
"""Dedicated local GPU judges. Never kill, evict, or exempt another job."""
from __future__ import annotations
from contextlib import contextmanager
import fcntl
import hashlib
import json
import os
from pathlib import Path
import queue
import subprocess
import threading
import time
import uuid
import psutil
from api_transport import atomic
IMAGE = 'sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b'
class JudgeInfrastructureError(RuntimeError):
pass
def gpu_state(index):
line = subprocess.check_output(['nvidia-smi', '-i', str(index), '--query-gpu=uuid,name,temperature.gpu,memory.used',
'--format=csv,noheader,nounits'], text=True, timeout=15).strip()
uid, name, temp, memory = [p.strip() for p in line.split(',')]
processes = subprocess.check_output(['nvidia-smi', '--query-compute-apps=pid,gpu_uuid', '--format=csv,noheader,nounits'],
text=True, timeout=15)
pids = [int(row.split(',')[0]) for row in processes.splitlines() if row.split(',')[-1].strip() == uid]
return dict(gpu=index, uuid=uid, name=name, temperature=int(temp), memory_mib=int(memory), pids=pids)
class JudgePool:
def __init__(self, root, gpus=(0, 1, 2, 3, 4, 5, 6, 7)):
self.root = Path(root)
self.code = self.root / 'code'
self.official = self.root / 'source/vendor/ttt_discover_gpu'
self.framework = self.root / 'source/examples/gpu_mode'
self.work = self.root / 'run/judge'
self.work.mkdir(parents=True, exist_ok=True)
protocol_files = [self.code / 'gpu_worker.py', self.code / 'reference/kernel_stage_cut_compose.py',
self.framework / 'judge_worker.py', self.official / 'manifest.json']
self.protocol = hashlib.sha256((IMAGE + ''.join(hashlib.sha256(p.read_bytes()).hexdigest() for p in protocol_files)).encode()).hexdigest()
if not gpus or len(gpus) != len(set(gpus)):
raise ValueError('Explicit distinct GPUs required')
self.gpus = tuple(gpus)
self.available = queue.Queue()
for gpu in self.gpus:
self.available.put(gpu)
self.lock = threading.Lock()
@contextmanager
def lease(self):
gpu = self.available.get()
lock_root = self.root.parent / 'kernel-component-gpu-locks'
lock_root.mkdir(exist_ok=True)
try:
with (lock_root / ('gpu-' + str(gpu) + '.lock')).open('a') as file:
fcntl.flock(file, fcntl.LOCK_EX)
while True:
state = gpu_state(gpu)
if 'H100' not in state['name'] or 'NVL' in state['name']:
raise JudgeInfrastructureError('Wrong GPU model')
if not state['pids'] and state['memory_mib'] < 100 and state['temperature'] < 75:
break
atomic(self.work / ('waiting_gpu' + str(gpu) + '.json'), dict(state=state, time=time.time()))
time.sleep(5)
yield gpu, state
finally:
self.available.put(gpu)
@staticmethod
def owned_pids(container):
result = subprocess.run(['docker', 'inspect', '-f', '{{.State.Pid}}', container], capture_output=True, text=True, timeout=10)
if result.returncode:
return set()
pid = int(result.stdout.strip() or '0')
if not pid or not psutil.pid_exists(pid):
return set()
return {pid, *(p.pid for p in psutil.Process(pid).children(recursive=True))}
@staticmethod
def stop_owned(container, request_id):
result = subprocess.run(['docker', 'inspect', '-f', '{{index .Config.Labels "kernel_component_request"}}', container],
capture_output=True, text=True, timeout=10)
if result.returncode:
return
if result.stdout.strip() != request_id:
raise JudgeInfrastructureError('Refusing to stop a container without the exact request ownership label')
subprocess.run(['docker', 'stop', '-t', '5', container], capture_output=True, timeout=20, check=True)
def request(self, payload, *, key, timeout=530):
encoded = json.dumps(payload, sort_keys=True, separators=(',', ':'))
identity = hashlib.sha256((self.protocol + '\n' + key + '\n' + encoded).encode()).hexdigest()
cache = self.work / 'cache' / (identity + '.json')
if cache.exists():
result = json.loads(cache.read_text())
if result['request_identity'] != identity:
raise JudgeInfrastructureError('Judge cache identity mismatch')
return result
request_id = uuid.uuid4().hex
directory = self.work / request_id
directory.mkdir()
atomic(directory / 'request.json', dict(identity=identity, key=key, payload=payload))
with self.lease() as (gpu, before):
container = 'kernel-component-' + request_id
command = ['docker', 'run', '--rm', '-i', '--name', container, '--label', 'kernel_component_request=' + request_id,
'--gpus', 'device=' + before['uuid'], '--network', 'none', '--read-only', '--cap-drop', 'ALL',
'--security-opt', 'no-new-privileges', '--pids-limit', '512', '--memory', '64g', '--shm-size', '8g', '--cpus', '4',
'--tmpfs', '/workdir:rw,exec,nosuid,size=48g,mode=1777', '--tmpfs', '/tmp:rw,exec,nosuid,size=4g,mode=1777',
'-e', 'OMP_NUM_THREADS=1', '-e', 'OPENBLAS_NUM_THREADS=1', '-e', 'MKL_NUM_THREADS=1',
'-e', 'TRITON_CACHE_DIR=/tmp/triton-cache', '-e', 'XDG_CACHE_HOME=/tmp/xdg-cache',
'--mount', 'type=bind,src=' + str(self.code) + ',dst=/app,readonly',
'--mount', 'type=bind,src=' + str(self.framework) + ',dst=/framework,readonly',
'--mount', 'type=bind,src=' + str(self.official.resolve()) + ',dst=/official,readonly',
'--entrypoint', 'python', IMAGE, '/app/gpu_worker.py']
started = time.monotonic()
proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
atomic(directory / 'ownership.json', dict(request_id=request_id, docker_client_pid=proc.pid,
create_time=psutil.Process(proc.pid).create_time(), gpu=gpu,
container=container, preflight=before, command=command))
data = encoded
collision = None
while True:
try:
stdout, stderr = proc.communicate(input=data, timeout=1)
break
except subprocess.TimeoutExpired:
data = None
state = gpu_state(gpu)
foreign = set(state['pids']) - self.owned_pids(container)
if foreign or state['temperature'] >= 75:
collision = dict(state=state, foreign_pids=sorted(foreign), reason='occupancy_or_temperature')
if collision or time.monotonic() - started > timeout:
self.stop_owned(container, request_id)
stdout, stderr = proc.communicate(timeout=20)
if collision:
atomic(directory / 'interference.json', collision)
raise JudgeInfrastructureError('Judge interrupted by occupancy/temperature; only our container was stopped')
result = dict(status='candidate_invalid', correctness=0., reward=0., raw_score=-1_000_000., score_us=None,
reason='candidate_execution_timeout', execution_timeout_seconds=timeout)
if payload['mode'] == 'screen':
raise JudgeInfrastructureError('Component screen exceeded its separate local-search budget')
break
else:
raise AssertionError('Unreachable')
(directory / 'stdout.log').write_text(stdout)
(directory / 'stderr.log').write_text(stderr)
if 'result' not in locals():
lines = [line.split('=', 1)[1] for line in stdout.splitlines() if line.startswith('KERNEL_COMPONENT_RESULT=')]
if len(lines) != 1:
raise JudgeInfrastructureError('No unique judge result: ' + stderr[-1000:])
result = json.loads(lines[0])
if proc.returncode or result['status'] == 'infrastructure_error':
raise JudgeInfrastructureError(result.get('error', stderr[-1000:]))
result.update(request_identity=identity, request_id=request_id, gpu_index=gpu, gpu_uuid=before['uuid'],
evaluator_protocol_sha256=self.protocol,
local_wall_seconds=time.monotonic() - started)
atomic(directory / 'result.json', result)
atomic(cache, result)
return result
def evaluate(self, code, *, key):
return self.request(dict(mode='evaluate', code=code), key=key, timeout=530)
def screen(self, programs, *, key):
return self.request(dict(mode='screen', programs=programs), key=key, timeout=1800)

Xet Storage Details

Size:
9.38 kB
·
Xet hash:
987e07329481da740f13ab9f8ba302fe76941f18674a42cd0126c9704199785a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.