"""Launch one matrix experiment in a private, manifest-verified execution copy. This launcher uses only stdlib and never imports the source checkout's runtime. The child installs its output boundary inside its own execution copy. """ import sys sys.dont_write_bytecode = True import argparse from contextlib import contextmanager,ExitStack import fcntl import hashlib import json import os from pathlib import Path import re import shutil import socket import subprocess import time ROOT = Path(__file__).resolve().parent def checksum(data): return hashlib.sha256(data).hexdigest() def save(path,value): path.parent.mkdir(parents=True,exist_ok=True) temp=path.with_name(path.name+'.tmp-'+str(os.getpid())) temp.write_text(json.dumps(value,indent=2,ensure_ascii=False)+'\n') os.replace(temp,path) def release_files(): """Copy exactly release files, never old runs, adapters or local settings.""" manifest=(ROOT/'MANIFEST.sha256').read_bytes() result={} for line in manifest.decode().splitlines(): expected,name=line.split(' ',1) relative=Path(name);path=ROOT/relative if relative.is_absolute() or '..' in relative.parts or not path.resolve().is_relative_to(ROOT): raise ValueError('Invalid release input path') if path.is_symlink() or not path.is_file() or name in result: raise ValueError('Invalid or duplicate release input: '+name) if checksum(path.read_bytes()) != expected: raise ValueError('Release input changed: '+name) result[name]=path result['MANIFEST.sha256']=ROOT/'MANIFEST.sha256' return manifest,result def clean_environment(): env=dict(os.environ,PYTHONDONTWRITEBYTECODE='1') # Drop process/source-copy state. External model/Python inputs and the # explicitly configured deployment resource hook remain available. for key in ('DECICO_RSI_RUNTIME_ROOT','DECICO_RSI_RESOURCE_ROOT','DECICO_RSI_ADAPTER_ROOT', 'DECICO_RSI_TRAIN_LOCK','RSI_SUITE_LANE_SPEC','RSI_SERVICE_LOCK_FD', 'DECICO_RSI_RESOURCE_HOOK_CHILD','RSI_MATRIX_HOST_LOCK_FD','RSI_MATRIX_RUN_LOCK_FD','PYTHONPATH'): env.pop(key,None) return env @contextmanager def server_lease(results_root,prepare_only): """A shared filesystem lock is separated by physical server hostname.""" if prepare_only: yield return key=checksum(socket.gethostname().encode())[:16] path=results_root/'.server-locks'/key/'experiment.lock' path.parent.mkdir(parents=True,exist_ok=True) with path.open('a') as handle: try:fcntl.flock(handle,fcntl.LOCK_EX|fcntl.LOCK_NB) except BlockingIOError as exc: raise ValueError('This server already runs a matrix experiment; use a different server or wait') from exc # Let the child retain this lease if the launcher is killed first. os.set_inheritable(handle.fileno(),True) try:yield handle.fileno() finally:pass # Close only: the child's inherited descriptor also owns this lease. @contextmanager def execution_lease(execution): directory=execution/'metadata';directory.mkdir(exist_ok=True) with (directory/'launcher.lock').open('a') as handle: try:fcntl.flock(handle,fcntl.LOCK_EX|fcntl.LOCK_NB) except BlockingIOError as exc: raise ValueError('This experiment/run-id is already active on another launcher') from exc os.set_inheritable(handle.fileno(),True) yield handle.fileno() def main(argv=None): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--experiment', choices=['02_qwen35_latest_accepted'], default='02_qwen35_latest_accepted') parser.add_argument('--results-root',required=True,help='Shared absolute results root, outside the source checkout') parser.add_argument('--run-id',default='run-001') parser.add_argument('--seeds',type=int,nargs='+',default=[2026100801,2026100802]) parser.add_argument('--lanes',help='Local JSON lanes input; snapshotted per experiment') parser.add_argument('--config',default=str(ROOT/'plan/bfcl-only/full.json')) parser.add_argument('--models-root') parser.add_argument('--student-model-path') parser.add_argument('--teacher-model-path') parser.add_argument('--student-gdn-backend', choices=['original', 'fla']) parser.add_argument('--teacher-gdn-backend', choices=['original', 'fla']) parser.add_argument('--prepare-only',action='store_true') parser.add_argument('--resume',action='store_true') args=parser.parse_args(argv) matrix=json.loads((ROOT/'plan/experiments/matrix.json').read_text())['experiments'] if args.experiment not in matrix:parser.error('Unknown experiment; choose '+', '.join(matrix)) if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,79}',args.run_id): parser.error('run-id must be a simple directory name without slashes') results=Path(args.results_root).expanduser() if not results.is_absolute():parser.error('results-root must be absolute') results=results.resolve() if results==ROOT or results.is_relative_to(ROOT): parser.error('Choose a results-root outside the shared source checkout') if not args.prepare_only and not args.lanes:parser.error('--lanes is required for a live experiment') if len(set(args.seeds)) != len(args.seeds):parser.error('Seeds must be unique') manifest,files=release_files() profile=matrix[args.experiment] config_bytes=Path(args.config).read_bytes() lanes_bytes=Path(args.lanes).read_bytes() if args.lanes else b'[]\n' json.loads(config_bytes);json.loads(lanes_bytes) execution=results/args.experiment/args.run_id if not execution.resolve().is_relative_to(results):raise ValueError('Execution path escaped results root') identity={'schema_version':1,'experiment':args.experiment,**profile,'seeds':args.seeds, 'execution_root':str(execution),'config_sha256':checksum(config_bytes), 'lanes_sha256':checksum(lanes_bytes),'models_root':args.models_root, 'student_model_path':args.student_model_path,'teacher_model_path':args.teacher_model_path, 'student_gdn_backend':args.student_gdn_backend,'teacher_gdn_backend':args.teacher_gdn_backend, 'source_manifest_sha256':checksum(manifest)} with ExitStack() as stack: descriptor=stack.enter_context(server_lease(results,args.prepare_only)) if args.resume: if not execution.is_dir():raise ValueError('Resume requires an existing experiment/run-id') else: execution.mkdir(parents=True,exist_ok=False) # Atomic reservation across servers. run_descriptor=stack.enter_context(execution_lease(execution)) if args.resume: saved=json.loads((execution/'metadata/launch.json').read_text()) if saved['identity'] != identity:raise ValueError('Resume inputs changed; use a new run-id') # Runtime/code may not be edited in an execution copy. for name,path in files.items(): if not (execution/name).is_file() or (execution/name).read_bytes()!=path.read_bytes(): raise ValueError('Execution source changed: '+name) else: save(execution/'metadata/launch.json',{'identity':identity,'status':'copying','host':socket.gethostname()}) for name,path in files.items(): target=execution/name;target.parent.mkdir(parents=True,exist_ok=True) shutil.copyfile(path,target) (execution/'inputs').mkdir() (execution/'inputs/config.json').write_bytes(config_bytes) (execution/'inputs/lanes.json').write_bytes(lanes_bytes) for name,data in [('config.json',config_bytes),('lanes.json',lanes_bytes)]: if (execution/'inputs'/name).read_bytes()!=data:raise ValueError('Snapshotted input changed: '+name) command=[sys.executable,'-B',str(execution/'run_experiments.py'), '--output',str(execution/'suite'),'--config',str(execution/'inputs/config.json'), '--lanes',str(execution/'inputs/lanes.json'),'--student-model',profile['student'], '--initialization',profile['initialization'],'--seeds',*map(str,args.seeds)] if args.models_root:command.extend(['--models-root',args.models_root]) if args.student_model_path:command.extend(['--student-model-path',args.student_model_path]) if args.teacher_model_path:command.extend(['--teacher-model-path',args.teacher_model_path]) if args.student_gdn_backend:command.extend(['--student-gdn-backend',args.student_gdn_backend]) if args.teacher_gdn_backend:command.extend(['--teacher-gdn-backend',args.teacher_gdn_backend]) if args.prepare_only:command.append('--prepare-only') if args.resume:command.append('--resume') save(execution/'metadata/launch.json',{'identity':identity,'status':'running','host':socket.gethostname(), 'command':command,'prepare_only':args.prepare_only,'started_at':time.time()}) print('Experiment:',args.experiment,flush=True) print('Private execution/results root:',execution,flush=True) # Isolate native-library relative output and Python startup caches. env=clean_environment() env['TMPDIR']=str(execution/'.runtime/tmp');env['TEMP']=env['TMP']=env['TMPDIR'] if descriptor is not None:env['RSI_MATRIX_HOST_LOCK_FD']=str(descriptor) env['RSI_MATRIX_RUN_LOCK_FD']=str(run_descriptor) Path(env['TMPDIR']).mkdir(parents=True,exist_ok=True) with (execution/'metadata/launcher.log').open('a') as log: kwargs={'pass_fds':tuple(fd for fd in (descriptor,run_descriptor) if fd is not None)} result=subprocess.run(command,cwd=execution,env=env,stdout=log,stderr=subprocess.STDOUT,**kwargs) save(execution/'metadata/launch.json',{'identity':identity,'status': ('prepared' if args.prepare_only else 'completed') if result.returncode==0 else 'failed', 'host':socket.gethostname(),'command':command,'returncode':result.returncode, 'prepare_only':args.prepare_only,'finished_at':time.time()}) if result.returncode:print('Inspect:',execution/'metadata/launcher.log',file=sys.stderr) return result.returncode if __name__=='__main__': # A caller may have sourced an old checkout's contained-Python startup. # Re-exec without that source-bound hook before creating a private copy. if 'project_paths' in sys.modules: os.execve(sys.executable,[sys.executable,'-B',str(Path(__file__).resolve()),*sys.argv[1:]],clean_environment()) raise SystemExit(main())