Buckets:
| """Dedicated local GPU judges. Never kill, evict, or exempt another job.""" | |
| from __future__ import annotations | |
| from contextlib import contextmanager | |
| import fcntl | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import queue | |
| import subprocess | |
| import threading | |
| import time | |
| import uuid | |
| import psutil | |
| from api_transport import atomic | |
| IMAGE = 'sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b' | |
| class JudgeInfrastructureError(RuntimeError): | |
| pass | |
| def gpu_state(index): | |
| line = subprocess.check_output(['nvidia-smi', '-i', str(index), '--query-gpu=uuid,name,temperature.gpu,memory.used', | |
| '--format=csv,noheader,nounits'], text=True, timeout=15).strip() | |
| uid, name, temp, memory = [p.strip() for p in line.split(',')] | |
| processes = subprocess.check_output(['nvidia-smi', '--query-compute-apps=pid,gpu_uuid', '--format=csv,noheader,nounits'], | |
| text=True, timeout=15) | |
| pids = [int(row.split(',')[0]) for row in processes.splitlines() if row.split(',')[-1].strip() == uid] | |
| return dict(gpu=index, uuid=uid, name=name, temperature=int(temp), memory_mib=int(memory), pids=pids) | |
| class JudgePool: | |
| def __init__(self, root, gpus=(0, 1, 2, 3, 4, 5, 6, 7)): | |
| self.root = Path(root) | |
| self.code = self.root / 'code' | |
| self.official = self.root / 'source/vendor/ttt_discover_gpu' | |
| self.framework = self.root / 'source/examples/gpu_mode' | |
| self.work = self.root / 'run/judge' | |
| self.work.mkdir(parents=True, exist_ok=True) | |
| protocol_files = [self.code / 'gpu_worker.py', self.code / 'reference/kernel_stage_cut_compose.py', | |
| self.framework / 'judge_worker.py', self.official / 'manifest.json'] | |
| self.protocol = hashlib.sha256((IMAGE + ''.join(hashlib.sha256(p.read_bytes()).hexdigest() for p in protocol_files)).encode()).hexdigest() | |
| if not gpus or len(gpus) != len(set(gpus)): | |
| raise ValueError('Explicit distinct GPUs required') | |
| self.gpus = tuple(gpus) | |
| self.available = queue.Queue() | |
| for gpu in self.gpus: | |
| self.available.put(gpu) | |
| self.lock = threading.Lock() | |
| def lease(self): | |
| gpu = self.available.get() | |
| lock_root = self.root.parent / 'kernel-component-gpu-locks' | |
| lock_root.mkdir(exist_ok=True) | |
| try: | |
| with (lock_root / ('gpu-' + str(gpu) + '.lock')).open('a') as file: | |
| fcntl.flock(file, fcntl.LOCK_EX) | |
| while True: | |
| state = gpu_state(gpu) | |
| if 'H100' not in state['name'] or 'NVL' in state['name']: | |
| raise JudgeInfrastructureError('Wrong GPU model') | |
| if not state['pids'] and state['memory_mib'] < 100 and state['temperature'] < 75: | |
| break | |
| atomic(self.work / ('waiting_gpu' + str(gpu) + '.json'), dict(state=state, time=time.time())) | |
| time.sleep(5) | |
| yield gpu, state | |
| finally: | |
| self.available.put(gpu) | |
| def owned_pids(container): | |
| result = subprocess.run(['docker', 'inspect', '-f', '{{.State.Pid}}', container], capture_output=True, text=True, timeout=10) | |
| if result.returncode: | |
| return set() | |
| pid = int(result.stdout.strip() or '0') | |
| if not pid or not psutil.pid_exists(pid): | |
| return set() | |
| return {pid, *(p.pid for p in psutil.Process(pid).children(recursive=True))} | |
| def stop_owned(container, request_id): | |
| result = subprocess.run(['docker', 'inspect', '-f', '{{index .Config.Labels "kernel_component_request"}}', container], | |
| capture_output=True, text=True, timeout=10) | |
| if result.returncode: | |
| return | |
| if result.stdout.strip() != request_id: | |
| raise JudgeInfrastructureError('Refusing to stop a container without the exact request ownership label') | |
| subprocess.run(['docker', 'stop', '-t', '5', container], capture_output=True, timeout=20, check=True) | |
| def request(self, payload, *, key, timeout=530): | |
| encoded = json.dumps(payload, sort_keys=True, separators=(',', ':')) | |
| identity = hashlib.sha256((self.protocol + '\n' + key + '\n' + encoded).encode()).hexdigest() | |
| cache = self.work / 'cache' / (identity + '.json') | |
| if cache.exists(): | |
| result = json.loads(cache.read_text()) | |
| if result['request_identity'] != identity: | |
| raise JudgeInfrastructureError('Judge cache identity mismatch') | |
| return result | |
| request_id = uuid.uuid4().hex | |
| directory = self.work / request_id | |
| directory.mkdir() | |
| atomic(directory / 'request.json', dict(identity=identity, key=key, payload=payload)) | |
| with self.lease() as (gpu, before): | |
| container = 'kernel-component-' + request_id | |
| command = ['docker', 'run', '--rm', '-i', '--name', container, '--label', 'kernel_component_request=' + request_id, | |
| '--gpus', 'device=' + before['uuid'], '--network', 'none', '--read-only', '--cap-drop', 'ALL', | |
| '--security-opt', 'no-new-privileges', '--pids-limit', '512', '--memory', '64g', '--shm-size', '8g', '--cpus', '4', | |
| '--tmpfs', '/workdir:rw,exec,nosuid,size=48g,mode=1777', '--tmpfs', '/tmp:rw,exec,nosuid,size=4g,mode=1777', | |
| '-e', 'OMP_NUM_THREADS=1', '-e', 'OPENBLAS_NUM_THREADS=1', '-e', 'MKL_NUM_THREADS=1', | |
| '-e', 'TRITON_CACHE_DIR=/tmp/triton-cache', '-e', 'XDG_CACHE_HOME=/tmp/xdg-cache', | |
| '--mount', 'type=bind,src=' + str(self.code) + ',dst=/app,readonly', | |
| '--mount', 'type=bind,src=' + str(self.framework) + ',dst=/framework,readonly', | |
| '--mount', 'type=bind,src=' + str(self.official.resolve()) + ',dst=/official,readonly', | |
| '--entrypoint', 'python', IMAGE, '/app/gpu_worker.py'] | |
| started = time.monotonic() | |
| proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) | |
| atomic(directory / 'ownership.json', dict(request_id=request_id, docker_client_pid=proc.pid, | |
| create_time=psutil.Process(proc.pid).create_time(), gpu=gpu, | |
| container=container, preflight=before, command=command)) | |
| data = encoded | |
| collision = None | |
| while True: | |
| try: | |
| stdout, stderr = proc.communicate(input=data, timeout=1) | |
| break | |
| except subprocess.TimeoutExpired: | |
| data = None | |
| state = gpu_state(gpu) | |
| foreign = set(state['pids']) - self.owned_pids(container) | |
| if foreign or state['temperature'] >= 75: | |
| collision = dict(state=state, foreign_pids=sorted(foreign), reason='occupancy_or_temperature') | |
| if collision or time.monotonic() - started > timeout: | |
| self.stop_owned(container, request_id) | |
| stdout, stderr = proc.communicate(timeout=20) | |
| if collision: | |
| atomic(directory / 'interference.json', collision) | |
| raise JudgeInfrastructureError('Judge interrupted by occupancy/temperature; only our container was stopped') | |
| result = dict(status='candidate_invalid', correctness=0., reward=0., raw_score=-1_000_000., score_us=None, | |
| reason='candidate_execution_timeout', execution_timeout_seconds=timeout) | |
| if payload['mode'] == 'screen': | |
| raise JudgeInfrastructureError('Component screen exceeded its separate local-search budget') | |
| break | |
| else: | |
| raise AssertionError('Unreachable') | |
| (directory / 'stdout.log').write_text(stdout) | |
| (directory / 'stderr.log').write_text(stderr) | |
| if 'result' not in locals(): | |
| lines = [line.split('=', 1)[1] for line in stdout.splitlines() if line.startswith('KERNEL_COMPONENT_RESULT=')] | |
| if len(lines) != 1: | |
| raise JudgeInfrastructureError('No unique judge result: ' + stderr[-1000:]) | |
| result = json.loads(lines[0]) | |
| if proc.returncode or result['status'] == 'infrastructure_error': | |
| raise JudgeInfrastructureError(result.get('error', stderr[-1000:])) | |
| result.update(request_identity=identity, request_id=request_id, gpu_index=gpu, gpu_uuid=before['uuid'], | |
| evaluator_protocol_sha256=self.protocol, | |
| local_wall_seconds=time.monotonic() - started) | |
| atomic(directory / 'result.json', result) | |
| atomic(cache, result) | |
| return result | |
| def evaluate(self, code, *, key): | |
| return self.request(dict(mode='evaluate', code=code), key=key, timeout=530) | |
| def screen(self, programs, *, key): | |
| return self.request(dict(mode='screen', programs=programs), key=key, timeout=1800) | |
Xet Storage Details
- Size:
- 9.38 kB
- Xet hash:
- 987e07329481da740f13ab9f8ba302fe76941f18674a42cd0126c9704199785a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.