DeciCo-RSI / launch_experiment.py
juray0410's picture
Initial commit
1831ecd
Raw History Blame Contribute Delete
10.7 kB
"""Launch one matrix experiment in a private, manifest-verified execution copy.
This launcher uses only stdlib and never imports the source checkout's runtime.
The child installs its output boundary inside its own execution copy.
"""
import sys
sys.dont_write_bytecode = True
import argparse
from contextlib import contextmanager,ExitStack
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import socket
import subprocess
import time
ROOT = Path(__file__).resolve().parent
def checksum(data):
return hashlib.sha256(data).hexdigest()
def save(path,value):
path.parent.mkdir(parents=True,exist_ok=True)
temp=path.with_name(path.name+'.tmp-'+str(os.getpid()))
temp.write_text(json.dumps(value,indent=2,ensure_ascii=False)+'\n')
os.replace(temp,path)
def release_files():
"""Copy exactly release files, never old runs, adapters or local settings."""
manifest=(ROOT/'MANIFEST.sha256').read_bytes()
result={}
for line in manifest.decode().splitlines():
expected,name=line.split(' ',1)
relative=Path(name);path=ROOT/relative
if relative.is_absolute() or '..' in relative.parts or not path.resolve().is_relative_to(ROOT):
raise ValueError('Invalid release input path')
if path.is_symlink() or not path.is_file() or name in result:
raise ValueError('Invalid or duplicate release input: '+name)
if checksum(path.read_bytes()) != expected:
raise ValueError('Release input changed: '+name)
result[name]=path
result['MANIFEST.sha256']=ROOT/'MANIFEST.sha256'
return manifest,result
def clean_environment():
env=dict(os.environ,PYTHONDONTWRITEBYTECODE='1')
# Drop process/source-copy state. External model/Python inputs and the
# explicitly configured deployment resource hook remain available.
for key in ('DECICO_RSI_RUNTIME_ROOT','DECICO_RSI_RESOURCE_ROOT','DECICO_RSI_ADAPTER_ROOT',
'DECICO_RSI_TRAIN_LOCK','RSI_SUITE_LANE_SPEC','RSI_SERVICE_LOCK_FD',
'DECICO_RSI_RESOURCE_HOOK_CHILD','RSI_MATRIX_HOST_LOCK_FD','RSI_MATRIX_RUN_LOCK_FD','PYTHONPATH'):
env.pop(key,None)
return env
@contextmanager
def server_lease(results_root,prepare_only):
"""A shared filesystem lock is separated by physical server hostname."""
if prepare_only:
yield
return
key=checksum(socket.gethostname().encode())[:16]
path=results_root/'.server-locks'/key/'experiment.lock'
path.parent.mkdir(parents=True,exist_ok=True)
with path.open('a') as handle:
try:fcntl.flock(handle,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError as exc:
raise ValueError('This server already runs a matrix experiment; use a different server or wait') from exc
# Let the child retain this lease if the launcher is killed first.
os.set_inheritable(handle.fileno(),True)
try:yield handle.fileno()
finally:pass # Close only: the child's inherited descriptor also owns this lease.
@contextmanager
def execution_lease(execution):
directory=execution/'metadata';directory.mkdir(exist_ok=True)
with (directory/'launcher.lock').open('a') as handle:
try:fcntl.flock(handle,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError as exc:
raise ValueError('This experiment/run-id is already active on another launcher') from exc
os.set_inheritable(handle.fileno(),True)
yield handle.fileno()
def main(argv=None):
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--experiment', choices=['02_qwen35_latest_accepted'], default='02_qwen35_latest_accepted')
parser.add_argument('--results-root',required=True,help='Shared absolute results root, outside the source checkout')
parser.add_argument('--run-id',default='run-001')
parser.add_argument('--seeds',type=int,nargs='+',default=[2026100801,2026100802])
parser.add_argument('--lanes',help='Local JSON lanes input; snapshotted per experiment')
parser.add_argument('--config',default=str(ROOT/'plan/bfcl-only/full.json'))
parser.add_argument('--models-root')
parser.add_argument('--student-model-path')
parser.add_argument('--teacher-model-path')
parser.add_argument('--student-gdn-backend', choices=['original', 'fla'])
parser.add_argument('--teacher-gdn-backend', choices=['original', 'fla'])
parser.add_argument('--prepare-only',action='store_true')
parser.add_argument('--resume',action='store_true')
args=parser.parse_args(argv)
matrix=json.loads((ROOT/'plan/experiments/matrix.json').read_text())['experiments']
if args.experiment not in matrix:parser.error('Unknown experiment; choose '+', '.join(matrix))
if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,79}',args.run_id):
parser.error('run-id must be a simple directory name without slashes')
results=Path(args.results_root).expanduser()
if not results.is_absolute():parser.error('results-root must be absolute')
results=results.resolve()
if results==ROOT or results.is_relative_to(ROOT):
parser.error('Choose a results-root outside the shared source checkout')
if not args.prepare_only and not args.lanes:parser.error('--lanes is required for a live experiment')
if len(set(args.seeds)) != len(args.seeds):parser.error('Seeds must be unique')
manifest,files=release_files()
profile=matrix[args.experiment]
config_bytes=Path(args.config).read_bytes()
lanes_bytes=Path(args.lanes).read_bytes() if args.lanes else b'[]\n'
json.loads(config_bytes);json.loads(lanes_bytes)
execution=results/args.experiment/args.run_id
if not execution.resolve().is_relative_to(results):raise ValueError('Execution path escaped results root')
identity={'schema_version':1,'experiment':args.experiment,**profile,'seeds':args.seeds,
'execution_root':str(execution),'config_sha256':checksum(config_bytes),
'lanes_sha256':checksum(lanes_bytes),'models_root':args.models_root,
'student_model_path':args.student_model_path,'teacher_model_path':args.teacher_model_path,
'student_gdn_backend':args.student_gdn_backend,'teacher_gdn_backend':args.teacher_gdn_backend,
'source_manifest_sha256':checksum(manifest)}
with ExitStack() as stack:
descriptor=stack.enter_context(server_lease(results,args.prepare_only))
if args.resume:
if not execution.is_dir():raise ValueError('Resume requires an existing experiment/run-id')
else:
execution.mkdir(parents=True,exist_ok=False) # Atomic reservation across servers.
run_descriptor=stack.enter_context(execution_lease(execution))
if args.resume:
saved=json.loads((execution/'metadata/launch.json').read_text())
if saved['identity'] != identity:raise ValueError('Resume inputs changed; use a new run-id')
# Runtime/code may not be edited in an execution copy.
for name,path in files.items():
if not (execution/name).is_file() or (execution/name).read_bytes()!=path.read_bytes():
raise ValueError('Execution source changed: '+name)
else:
save(execution/'metadata/launch.json',{'identity':identity,'status':'copying','host':socket.gethostname()})
for name,path in files.items():
target=execution/name;target.parent.mkdir(parents=True,exist_ok=True)
shutil.copyfile(path,target)
(execution/'inputs').mkdir()
(execution/'inputs/config.json').write_bytes(config_bytes)
(execution/'inputs/lanes.json').write_bytes(lanes_bytes)
for name,data in [('config.json',config_bytes),('lanes.json',lanes_bytes)]:
if (execution/'inputs'/name).read_bytes()!=data:raise ValueError('Snapshotted input changed: '+name)
command=[sys.executable,'-B',str(execution/'run_experiments.py'),
'--output',str(execution/'suite'),'--config',str(execution/'inputs/config.json'),
'--lanes',str(execution/'inputs/lanes.json'),'--student-model',profile['student'],
'--initialization',profile['initialization'],'--seeds',*map(str,args.seeds)]
if args.models_root:command.extend(['--models-root',args.models_root])
if args.student_model_path:command.extend(['--student-model-path',args.student_model_path])
if args.teacher_model_path:command.extend(['--teacher-model-path',args.teacher_model_path])
if args.student_gdn_backend:command.extend(['--student-gdn-backend',args.student_gdn_backend])
if args.teacher_gdn_backend:command.extend(['--teacher-gdn-backend',args.teacher_gdn_backend])
if args.prepare_only:command.append('--prepare-only')
if args.resume:command.append('--resume')
save(execution/'metadata/launch.json',{'identity':identity,'status':'running','host':socket.gethostname(),
'command':command,'prepare_only':args.prepare_only,'started_at':time.time()})
print('Experiment:',args.experiment,flush=True)
print('Private execution/results root:',execution,flush=True)
# Isolate native-library relative output and Python startup caches.
env=clean_environment()
env['TMPDIR']=str(execution/'.runtime/tmp');env['TEMP']=env['TMP']=env['TMPDIR']
if descriptor is not None:env['RSI_MATRIX_HOST_LOCK_FD']=str(descriptor)
env['RSI_MATRIX_RUN_LOCK_FD']=str(run_descriptor)
Path(env['TMPDIR']).mkdir(parents=True,exist_ok=True)
with (execution/'metadata/launcher.log').open('a') as log:
kwargs={'pass_fds':tuple(fd for fd in (descriptor,run_descriptor) if fd is not None)}
result=subprocess.run(command,cwd=execution,env=env,stdout=log,stderr=subprocess.STDOUT,**kwargs)
save(execution/'metadata/launch.json',{'identity':identity,'status':
('prepared' if args.prepare_only else 'completed') if result.returncode==0 else 'failed',
'host':socket.gethostname(),'command':command,'returncode':result.returncode,
'prepare_only':args.prepare_only,'finished_at':time.time()})
if result.returncode:print('Inspect:',execution/'metadata/launcher.log',file=sys.stderr)
return result.returncode
if __name__=='__main__':
# A caller may have sourced an old checkout's contained-Python startup.
# Re-exec without that source-bound hook before creating a private copy.
if 'project_paths' in sys.modules:
os.execve(sys.executable,[sys.executable,'-B',str(Path(__file__).resolve()),*sys.argv[1:]],clean_environment())
raise SystemExit(main())