Buckets:
| """Wait for the exact image, verify both GPUs, calibrate, then run three steps.""" | |
| import argparse | |
| import fcntl | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shlex | |
| import subprocess | |
| import sys | |
| import time | |
| import traceback | |
| from common import atomic, read, IMAGE, IMAGE_MANIFEST | |
| def run(root): | |
| python = str(root / '.venv-api/bin/python') | |
| env = dict(os.environ, KERNEL_API3_ROOT=str(root), KERNEL_BASE_SOURCE=str(root / 'source'), | |
| CUDA_VISIBLE_DEVICES='', WANDB_MODE='disabled', PYTHONUNBUFFERED='1', | |
| PYTHONPATH=':'.join(str(root / p) for p in ['code', 'source', | |
| 'source/examples/gpu_mode', 'source/ttt_discover/tinker_utils'])) | |
| if (root / 'control/actor_create_intent.json').exists() or (root / 'control/user_stop.json').exists(): | |
| raise RuntimeError('Existing actor intent or user stop; do not start fresh') | |
| atomic(root / 'control/preparation_controller.json', dict(pid=os.getpid(), started=time.time(), phase='waiting_image')) | |
| while not (root / 'control/image_transfer.json').exists(): | |
| if (root / 'control/user_stop.json').exists(): | |
| return | |
| time.sleep(15) | |
| if not read(root / 'control/image_transfer.json')['success']: | |
| raise RuntimeError('Exact Docker image transfer failed') | |
| remote_root = '/data/xiaofan/kernel-private-entry-split-step0-api3-20261105-judge' | |
| remote = '''from pathlib import Path | |
| import subprocess,json,os,time | |
| r=Path(REMOTE_ROOT) | |
| image=IMAGE | |
| p=subprocess.run(['docker','image','inspect',image],capture_output=True,text=True,check=True,timeout=30) | |
| meta=json.loads(p.stdout)[0] | |
| assert meta['Id']==image | |
| for gpu,uid in [(6,'GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c'),(7,'GPU-e6205be5-b1a7-bc81-55de-8f79262cf524')]: | |
| s=subprocess.check_output(['nvidia-smi','-i',str(gpu),'--query-gpu=uuid,temperature.gpu,memory.used','--format=csv,noheader,nounits'],text=True).strip().split(',') | |
| assert s[0].strip()==uid and int(s[1])<75 and int(s[2])<100,s | |
| rows=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid','--format=csv,noheader,nounits'],text=True).splitlines() | |
| assert not any(uid in row for row in rows) | |
| proof={'image_config_sha256':image,'image_manifest_sha256':MANIFEST,'execution_image':image,'verified':time.time()} | |
| (r/'control/image_ready.json').write_text(json.dumps(proof,indent=2)) | |
| assert not (r/'control/launch.json').exists(),'Existing evaluator launch; inspect instead of duplicating' | |
| env=dict(os.environ,PYTHONPATH=str(r/'code')+':'+str(r/'source')) | |
| cmd=['/usr/bin/python3',str(r/'code/pool_judge.py'),'--root',str(r),'--host','10.0.0.8','--port','8796','--allowed-client','10.0.0.5'] | |
| with (r/'control/server.log').open('ab') as log: | |
| p=subprocess.Popen(cmd,stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True,stdin=subprocess.DEVNULL) | |
| boot=int(next(l.split()[1] for l in Path('/proc/stat').read_text().splitlines() if l.startswith('btime '))) | |
| ticks=int(Path('/proc/'+str(p.pid)+'/stat').read_text().rsplit(')',1)[1].split()[19]) | |
| receipt={'pid':p.pid,'create_time':boot+ticks/os.sysconf('SC_CLK_TCK'),'command':cmd,'gpus':[6,7]} | |
| (r/'control/launch.json').write_text(json.dumps(receipt,indent=2));print(json.dumps(receipt)) | |
| '''.replace('REMOTE_ROOT', repr(remote_root)).replace('IMAGE', repr(IMAGE)).replace('MANIFEST', repr(IMAGE_MANIFEST)) | |
| result = subprocess.run(['ssh', 'n4-4', '/usr/bin/python3 -c ' + shlex.quote(remote)], | |
| capture_output=True, text=True, timeout=60) | |
| (root / 'control/judge_launch.stdout').write_text(result.stdout) | |
| (root / 'control/judge_launch.stderr').write_text(result.stderr) | |
| if result.returncode: | |
| raise RuntimeError('Evaluator launch failed: ' + result.stderr[-1000:]) | |
| atomic(root / 'control/judge_launch.json', json.loads(result.stdout)) | |
| from pool_judge import Client | |
| plan = read(root / 'control/plan.json') | |
| client = Client(plan['judge_url'], plan['judge_token'], root.name) | |
| for _ in range(30): | |
| try: | |
| health = client.health() | |
| break | |
| except Exception: | |
| time.sleep(1) | |
| else: | |
| raise RuntimeError('Evaluator never became ready') | |
| atomic(root / 'control/evaluator_initial_health.json', health) | |
| atomic(root / 'control/preparation_controller.json', dict(pid=os.getpid(), phase='GPU_calibration', time=time.time())) | |
| with (root / 'control/calibration.log').open('ab') as log: | |
| subprocess.run([python, str(root / 'code/calibrate.py'), '--root', str(root)], env=env, | |
| stdout=log, stderr=subprocess.STDOUT, check=True) | |
| with (root / 'control/api_cpu_tests.log').open('wb') as log: | |
| subprocess.run([python, '-m', 'pytest', '-q', str(root / 'code/test_api.py')], env=env, | |
| stdout=log, stderr=subprocess.STDOUT, check=True) | |
| with (root / 'control/preflight.log').open('ab') as log: | |
| subprocess.run([python, str(root / 'code/prepare_api.py'), '--root', str(root), '--existing', '--seal'], | |
| env=env, stdout=log, stderr=subprocess.STDOUT, check=True) | |
| atomic(root / 'control/preparation_controller.json', dict(pid=os.getpid(), phase='launching_API3', time=time.time())) | |
| command = [python, str(root / 'code/train_api.py'), '--root', str(root), '--execute'] | |
| with (root / 'run/launcher.log').open('ab') as log: | |
| trainer = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT, | |
| stdin=subprocess.DEVNULL, start_new_session=True) | |
| import psutil | |
| atomic(root / 'control/launch.json', dict(pid=trainer.pid, create_time=psutil.Process(trainer.pid).create_time(), | |
| command=command, new_steps=3, judge_gpus=[6, 7], time=time.time())) | |
| print(json.dumps(dict(launched=True, pid=trainer.pid, new_steps=3)), flush=True) | |
| if __name__ == '__main__': | |
| p = argparse.ArgumentParser() | |
| p.add_argument('--root', type=Path, required=True) | |
| root = p.parse_args().root | |
| with (root / 'control/preparation.lock').open('a') as lock: | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| try: | |
| run(root) | |
| except BaseException as error: | |
| atomic(root / 'control/preparation_failure.json', dict(error=repr(error), traceback=traceback.format_exc(), time=time.time())) | |
| raise | |
Xet Storage Details
- Size:
- 6.31 kB
- Xet hash:
- 271ee76dc12a1bedeaedcd0b308bcc84cf418e50306ffb05fb84f71a9cf87297
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.