zefancainlp's picture
download
raw
12.8 kB
"""Transport the unchanged full-program evaluator to an idle identical H100."""
from __future__ import annotations
import argparse
from concurrent.futures import Future
from contextlib import contextmanager
import fcntl
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import queue
import socket
import subprocess
import threading
import time
import urllib.request
import uuid
from common import IMAGE, IMAGE_MANIFEST, atomic, read, sha
class InfrastructureError(RuntimeError):
pass
def gpu_state(index):
value = subprocess.check_output(['nvidia-smi', '-i', str(index), '--query-gpu=uuid,name,temperature.gpu,memory.used',
'--format=csv,noheader,nounits'], text=True, timeout=15).strip()
uid, name, temp, memory = [p.strip() for p in value.split(',')]
rows = subprocess.check_output(['nvidia-smi', '--query-compute-apps=pid,gpu_uuid', '--format=csv,noheader,nounits'],
text=True, timeout=15).splitlines()
pids = [int(r.split(',')[0]) for r in rows if r.split(',')[-1].strip() == uid]
return dict(gpu=index, uuid=uid, name=name, temperature=int(temp), memory_mib=int(memory), pids=pids)
class Judge:
def __init__(self, root, gpu):
self.root, self.gpu = Path(root), gpu
proof = self.root / 'control/image_ready.json'
self.execution_image = IMAGE
if proof.exists():
identity = read(proof)
if identity.get('image_config_sha256') != IMAGE or identity.get('image_manifest_sha256') != IMAGE_MANIFEST:
raise InfrastructureError('Remote Docker image content differs from the original image')
self.execution_image = identity['execution_image']
self.serial = threading.Lock()
self.jobs_lock, self.jobs = threading.Lock(), {}
self.root.joinpath('requests').mkdir(parents=True, exist_ok=True)
def evaluate(self, code, request_id, timeout):
if timeout != 530 or not isinstance(code, str) or len(code.encode()) > 2_000_000:
raise InfrastructureError('Wrong original execution budget or oversized request')
identity = sha(IMAGE + '\n' + code + '\n530')
with self.jobs_lock:
if request_id in self.jobs:
old, future = self.jobs[request_id]
if old != identity:
raise InfrastructureError('Repeated request ID with a different program')
owner = False
else:
future = Future()
self.jobs[request_id] = (identity, future)
owner = True
if not owner:
return future.result()
try:
result = self._evaluate(code, request_id, identity)
except BaseException as error:
future.set_exception(error)
raise
future.set_result(result)
return result
def _evaluate(self, code, request_id, identity):
from examples.gpu_mode.local_judge import docker_command, parse_result
directory = self.root / 'requests' / request_id
directory.mkdir(exist_ok=False)
atomic(directory / 'request.json', dict(program_sha256=sha(code), identity=identity,
code=code, execution_seconds=530, queued=time.time()))
queued = time.monotonic()
with self.serial:
lock_root = self.root.parent / 'kernel-component-gpu-locks'
lock_root.mkdir(exist_ok=True)
with (lock_root / f'gpu-{self.gpu}.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
while True:
state = gpu_state(self.gpu)
if 'H100' not in state['name'] or 'NVL' in state['name']:
raise InfrastructureError('Wrong timing GPU model')
if not state['pids'] and state['memory_mib'] < 100 and state['temperature'] < 75:
break
atomic(self.root / 'waiting_gpu.json', dict(state=state, time=time.time()))
time.sleep(5)
wait = time.monotonic() - queued
name = 'kernel-private-' + request_id
command = docker_command(self.execution_image, state['uuid'], name)
command[command.index('--runtime'):command.index('--runtime')] = ['--label', 'kernel_private_request=' + request_id]
started = time.monotonic()
proc = subprocess.Popen(command, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
atomic(directory / 'execution.json', dict(gpu=state, pid=proc.pid, container=name, started=time.time(), command=command))
data = json.dumps(dict(code=code))
timed_out = False
try:
while True:
try:
stdout, stderr = proc.communicate(input=data, timeout=1)
break
except subprocess.TimeoutExpired:
data = None
current = gpu_state(self.gpu)
# Identify only this container's process namespace, never signal foreign jobs.
inspected = subprocess.run(['docker', 'inspect', '-f', '{{.State.Pid}}', name], capture_output=True, text=True, timeout=10)
own = set()
if inspected.returncode == 0 and inspected.stdout.strip().isdigit():
pid = int(inspected.stdout.strip())
if pid:
ns = os.readlink(f'/proc/{pid}/ns/pid')
for candidate in current['pids']:
try:
if os.readlink(f'/proc/{candidate}/ns/pid') == ns:
own.add(candidate)
except FileNotFoundError:
pass
foreign = set(current['pids']) - own
if foreign or current['temperature'] >= 75:
raise InfrastructureError('Timing interrupted by foreign GPU activity or temperature')
if time.monotonic() - started > 530:
timed_out = True
break
finally:
if proc.poll() is None:
label = subprocess.run(['docker', 'inspect', '-f', '{{index .Config.Labels "kernel_private_request"}}', name],
text=True, capture_output=True, timeout=10)
if label.returncode == 0 and label.stdout.strip() == request_id:
subprocess.run(['docker', 'rm', '-f', name], check=True, capture_output=True, timeout=25)
elif label.returncode == 0:
raise InfrastructureError('Refusing to clean a container without exact ownership')
stdout, stderr = proc.communicate(timeout=30)
(directory / 'stdout.log').write_text(stdout)
(directory / 'stderr.log').write_text(stderr)
if timed_out:
result = dict(status='candidate_invalid', correctness=0., reward=0., raw_score=-1_000_000., score_us=None,
wall_seconds=time.monotonic()-started, result={'candidate_timeout_seconds':530})
else:
result = parse_result(stdout)
if proc.returncode:
raise InfrastructureError('Original Docker judge failed: ' + stderr[-1000:])
result.update(request_id=request_id, gpu_index=self.gpu, gpu_uuid=state['uuid'], image=self.execution_image,
canonical_image_config_sha256=IMAGE, image_manifest_sha256=IMAGE_MANIFEST,
queue_seconds=wait, host_execution_seconds=time.monotonic()-started,
execution_timeout_seconds=530, program_sha256=sha(code), node=socket.gethostname())
atomic(directory / 'result.json', result)
return result
class Client:
def __init__(self, url, token):
self.url, self.token = url.rstrip('/'), token
def _request(self, path, data=None, timeout=30):
body = None if data is None else json.dumps(data).encode()
req = urllib.request.Request(self.url + path, data=body,
headers={'Authorization':'Bearer ' + self.token, 'Content-Type':'application/json'})
# Tailscale node traffic must not be sent through the host's external proxy.
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
with opener.open(req, timeout=timeout) as response:
return json.loads(response.read())
def health(self):
value = self._request('/health')
if value.get('canonical_image_config_sha256',value['image']) != IMAGE or value['execution_seconds'] != 530:
raise InfrastructureError('Remote evaluator protocol mismatch')
return value
def evaluate(self, code, *, timeout_seconds=530, **kwargs):
request_id = uuid.uuid4().hex
value = self._request('/evaluate', dict(code=code, request_id=request_id, timeout=timeout_seconds), timeout=86400)
if value.get('infrastructure_error'):
raise InfrastructureError(value['infrastructure_error'])
if value.get('program_sha256') != sha(code) or value.get('canonical_image_config_sha256',value.get('image')) != IMAGE:
raise InfrastructureError('Remote returned a mismatched complete-program result')
return value
def serve(root, host, port, gpu, allowed_client):
root = Path(root)
token = (root / 'control/judge_token.txt').read_text().strip()
judge = Judge(root, gpu)
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def reply(self, value, status=200):
body = json.dumps(value, allow_nan=False).encode()
self.send_response(status)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
def authorized(self):
if self.client_address[0] != allowed_client or self.headers.get('Authorization') != 'Bearer ' + token:
self.reply({'error':'Forbidden'},403)
return False
return True
def do_GET(self):
if not self.authorized():
return
if self.path != '/health':
return self.reply({'error':'Not found'},404)
self.reply(dict(image=judge.execution_image, canonical_image_config_sha256=IMAGE,
image_manifest_sha256=IMAGE_MANIFEST, execution_seconds=530, gpu=gpu,
node=socket.gethostname(), state=gpu_state(gpu)))
def do_POST(self):
if not self.authorized():
return
n = int(self.headers.get('Content-Length','0'))
if self.path != '/evaluate' or not 0 < n < 3_000_000:
return self.reply({'error':'Invalid request'},400)
try:
value = json.loads(self.rfile.read(n))
request_id = value['request_id']
if len(request_id) != 32 or any(c not in '0123456789abcdef' for c in request_id):
raise ValueError('Invalid request ID')
result = judge.evaluate(value['code'], request_id, value['timeout'])
except Exception as error:
result = {'infrastructure_error':type(error).__name__ + ': ' + str(error)}
self.reply(result)
atomic(root / 'control/server.json', dict(pid=os.getpid(), host=host, port=port, gpu=gpu,
image=IMAGE, execution_seconds=530, started=time.time(), allowed_client=allowed_client))
ThreadingHTTPServer((host,port),Handler).serve_forever()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--root',type=Path,required=True)
parser.add_argument('--host',required=True)
parser.add_argument('--port',type=int,required=True)
parser.add_argument('--gpu',type=int,required=True)
parser.add_argument('--allowed-client',required=True)
a = parser.parse_args()
serve(a.root,a.host,a.port,a.gpu,a.allowed_client)

Xet Storage Details

Size:
12.8 kB
·
Xet hash:
61e10e8d00ba20760498378b806567c823e4985b8456d46db92b0e9cd9a5af19

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.