zefancainlp's picture
download
raw
6.31 kB
"""Wait for the exact image, verify both GPUs, calibrate, then run three steps."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import shlex
import subprocess
import sys
import time
import traceback
from common import atomic, read, IMAGE, IMAGE_MANIFEST
def run(root):
python = str(root / '.venv-api/bin/python')
env = dict(os.environ, KERNEL_API3_ROOT=str(root), KERNEL_BASE_SOURCE=str(root / 'source'),
CUDA_VISIBLE_DEVICES='', WANDB_MODE='disabled', PYTHONUNBUFFERED='1',
PYTHONPATH=':'.join(str(root / p) for p in ['code', 'source',
'source/examples/gpu_mode', 'source/ttt_discover/tinker_utils']))
if (root / 'control/actor_create_intent.json').exists() or (root / 'control/user_stop.json').exists():
raise RuntimeError('Existing actor intent or user stop; do not start fresh')
atomic(root / 'control/preparation_controller.json', dict(pid=os.getpid(), started=time.time(), phase='waiting_image'))
while not (root / 'control/image_transfer.json').exists():
if (root / 'control/user_stop.json').exists():
return
time.sleep(15)
if not read(root / 'control/image_transfer.json')['success']:
raise RuntimeError('Exact Docker image transfer failed')
remote_root = '/data/xiaofan/kernel-private-entry-split-step0-api3-20261105-judge'
remote = '''from pathlib import Path
import subprocess,json,os,time
r=Path(REMOTE_ROOT)
image=IMAGE
p=subprocess.run(['docker','image','inspect',image],capture_output=True,text=True,check=True,timeout=30)
meta=json.loads(p.stdout)[0]
assert meta['Id']==image
for gpu,uid in [(6,'GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c'),(7,'GPU-e6205be5-b1a7-bc81-55de-8f79262cf524')]:
s=subprocess.check_output(['nvidia-smi','-i',str(gpu),'--query-gpu=uuid,temperature.gpu,memory.used','--format=csv,noheader,nounits'],text=True).strip().split(',')
assert s[0].strip()==uid and int(s[1])<75 and int(s[2])<100,s
rows=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid','--format=csv,noheader,nounits'],text=True).splitlines()
assert not any(uid in row for row in rows)
proof={'image_config_sha256':image,'image_manifest_sha256':MANIFEST,'execution_image':image,'verified':time.time()}
(r/'control/image_ready.json').write_text(json.dumps(proof,indent=2))
assert not (r/'control/launch.json').exists(),'Existing evaluator launch; inspect instead of duplicating'
env=dict(os.environ,PYTHONPATH=str(r/'code')+':'+str(r/'source'))
cmd=['/usr/bin/python3',str(r/'code/pool_judge.py'),'--root',str(r),'--host','10.0.0.8','--port','8796','--allowed-client','10.0.0.5']
with (r/'control/server.log').open('ab') as log:
p=subprocess.Popen(cmd,stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True,stdin=subprocess.DEVNULL)
boot=int(next(l.split()[1] for l in Path('/proc/stat').read_text().splitlines() if l.startswith('btime ')))
ticks=int(Path('/proc/'+str(p.pid)+'/stat').read_text().rsplit(')',1)[1].split()[19])
receipt={'pid':p.pid,'create_time':boot+ticks/os.sysconf('SC_CLK_TCK'),'command':cmd,'gpus':[6,7]}
(r/'control/launch.json').write_text(json.dumps(receipt,indent=2));print(json.dumps(receipt))
'''.replace('REMOTE_ROOT', repr(remote_root)).replace('IMAGE', repr(IMAGE)).replace('MANIFEST', repr(IMAGE_MANIFEST))
result = subprocess.run(['ssh', 'n4-4', '/usr/bin/python3 -c ' + shlex.quote(remote)],
capture_output=True, text=True, timeout=60)
(root / 'control/judge_launch.stdout').write_text(result.stdout)
(root / 'control/judge_launch.stderr').write_text(result.stderr)
if result.returncode:
raise RuntimeError('Evaluator launch failed: ' + result.stderr[-1000:])
atomic(root / 'control/judge_launch.json', json.loads(result.stdout))
from pool_judge import Client
plan = read(root / 'control/plan.json')
client = Client(plan['judge_url'], plan['judge_token'], root.name)
for _ in range(30):
try:
health = client.health()
break
except Exception:
time.sleep(1)
else:
raise RuntimeError('Evaluator never became ready')
atomic(root / 'control/evaluator_initial_health.json', health)
atomic(root / 'control/preparation_controller.json', dict(pid=os.getpid(), phase='GPU_calibration', time=time.time()))
with (root / 'control/calibration.log').open('ab') as log:
subprocess.run([python, str(root / 'code/calibrate.py'), '--root', str(root)], env=env,
stdout=log, stderr=subprocess.STDOUT, check=True)
with (root / 'control/api_cpu_tests.log').open('wb') as log:
subprocess.run([python, '-m', 'pytest', '-q', str(root / 'code/test_api.py')], env=env,
stdout=log, stderr=subprocess.STDOUT, check=True)
with (root / 'control/preflight.log').open('ab') as log:
subprocess.run([python, str(root / 'code/prepare_api.py'), '--root', str(root), '--existing', '--seal'],
env=env, stdout=log, stderr=subprocess.STDOUT, check=True)
atomic(root / 'control/preparation_controller.json', dict(pid=os.getpid(), phase='launching_API3', time=time.time()))
command = [python, str(root / 'code/train_api.py'), '--root', str(root), '--execute']
with (root / 'run/launcher.log').open('ab') as log:
trainer = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
stdin=subprocess.DEVNULL, start_new_session=True)
import psutil
atomic(root / 'control/launch.json', dict(pid=trainer.pid, create_time=psutil.Process(trainer.pid).create_time(),
command=command, new_steps=3, judge_gpus=[6, 7], time=time.time()))
print(json.dumps(dict(launched=True, pid=trainer.pid, new_steps=3)), flush=True)
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('--root', type=Path, required=True)
root = p.parse_args().root
with (root / 'control/preparation.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
run(root)
except BaseException as error:
atomic(root / 'control/preparation_failure.json', dict(error=repr(error), traceback=traceback.format_exc(), time=time.time()))
raise

Xet Storage Details

Size:
6.31 kB
·
Xet hash:
271ee76dc12a1bedeaedcd0b308bcc84cf418e50306ffb05fb84f71a9cf87297

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.