"""Audited rollback to a saved boundary after interruption; never use on a live run. Only GPU identity, model-load budget, and checkpoint provenance metadata change. The numerical state and seeded inputs are verified and preserved exactly. """ import argparse,fcntl,json,os,pathlib,shutil,subprocess,time import torch from robottt_pi05.config import ExperimentConfig from robottt_pi05.checkpoint import digest,fsync_dir from robottt_pi05.eval_io import sha,atomic_json,atomic_tensor from robottt_pi05.profile_common import runtime_signature def trim_events(lines,saved): attempted={k:0 for k in saved['attempted']};completed={k:0 for k in saved['completed']};kept=[] for index,line in enumerate(lines): try:record=json.loads(line) except json.JSONDecodeError: if index==len(lines)-1:break raise ValueError('Corrupt journal before its final line') scope=record.get('scope');step=record.get('step') if (isinstance(scope,str) and scope.isdigit() and int(scope)>saved['step']) or (isinstance(step,int) and step>saved['step']):break if record['event'] in ('reserve','complete'): counters=attempted if record['event']=='reserve' else completed target=saved['attempted'] if record['event']=='reserve' else saved['completed'] kind=record['kind'] if counters[kind]+1>target[kind]:break counters[kind]+=1 kept.append(line) if attempted!=saved['attempted']:raise ValueError('Saved boundary and journal attempts differ') for kind,count in completed.items(): missing=saved['completed'][kind]-count if missing and not (kind=='optimizer_update' and missing==1):raise ValueError('Saved boundary and journal completions differ') return ''.join(line if line.endswith('\n') else line+'\n' for line in kept) def numerical_digest(saved): return digest({key:saved[key] for key in ('step','next_schedule_index','slow','optimizer','rng','config','parameter_names')}) def restore_transaction(transaction): record=json.loads(transaction.read_text()) if record['state']=='committed':return archive=pathlib.Path(record['archive']) for name,target in record['originals'].items(): target=pathlib.Path(target);temporary=target.with_name(target.name+'.rollback') if temporary.exists():temporary.unlink() if name=='checkpoint.pt':os.link(archive/name,temporary) else:shutil.copyfile(archive/name,temporary) os.replace(temporary,target);fsync_dir(target.parent) record['state']='rolled_back';atomic_json(transaction,record,replace=True) def recover(run_config): rp=pathlib.Path(run_config).resolve();r=json.loads(rp.read_text()) def path(key):return (rp.parent/r['paths'][key]).resolve() out=path('output');out.mkdir(parents=True,exist_ok=True) with (out/'run.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) transaction=out/'recovery-transaction.json' if transaction.exists():restore_transaction(transaction) # A rolled-back interrupted transaction may have restored the old config. r=json.loads(rp.read_text()) if (out/'completed.json').exists():return dict(state='already_completed',run=str(rp)) if list(out.glob('failure*.json')):raise RuntimeError('Recorded training failure requires diagnosis before rollback') checkpoint=out/'checkpoints/latest.pt';journal=out/'ledger/events.jsonl' if not checkpoint.exists():raise FileNotFoundError('No saved boundary; restore a verified HF checkpoint first') saved=torch.load(checkpoint,map_location='cpu',weights_only=True) old_provenance=dict(experiment_config=sha(path('experiment_config')),asset_index=sha(path('asset_index')),run_config=sha(rp),implementation_sha256=runtime_signature()) if saved['provenance']!=old_provenance:raise ValueError('Original checkpoint/config/code provenance differs') if saved['config']!=ExperimentConfig.load(path('experiment_config')).to_dict():raise ValueError('Numerical experiment configuration differs') for key,expected in r['hashes'].items(): if sha(path(key))!=expected:raise ValueError('Immutable input hash differs: '+key) trimmed=trim_events(journal.read_text().splitlines(keepends=True),saved) gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name,memory.total,driver_version','--format=csv,noheader'],text=True).strip() parts=[item.strip() for item in gpu.split(',')] if parts[1]!='NVIDIA GeForce RTX 3090':raise ValueError('Unvalidated replacement GPU model') archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True) originals={'config.json':str(rp),'checkpoint.pt':str(checkpoint),'events.jsonl':str(journal)} shutil.copyfile(rp,archive/'config.json');os.link(checkpoint,archive/'checkpoint.pt');shutil.copyfile(journal,archive/'events.jsonl') old_hash=sha(checkpoint);before=numerical_digest(saved) replacement=json.loads(json.dumps(r));replacement['gpu_uuid']=parts[0] replacement['limits']['model_load']=max(1024,replacement['limits']['model_load']) new_config=archive/'config-next.json';new_config.write_text(json.dumps(replacement,indent=2)+'\n') updated=dict(saved);updated['provenance']=dict(saved['provenance'],run_config=sha(new_config)) new_checkpoint=archive/'checkpoint-next.pt';atomic_tensor(new_checkpoint,updated) if numerical_digest(torch.load(new_checkpoint,map_location='cpu',weights_only=True))!=before:raise ValueError('Recovery altered numerical state') new_journal=archive/'events-next.jsonl';new_journal.write_text(trimmed) record=dict(state='prepared',archive=str(archive),originals=originals,source_checkpoint_sha256=old_hash, migrated_checkpoint_sha256=sha(new_checkpoint),preserved_numerical_state_sha256=before,step=saved['step'],gpu=gpu, discarded_unsaved_journal_bytes=journal.stat().st_size-len(trimmed.encode()),reason='Workspace interruption: resume the saved optimizer/RNG boundary; administrative GPU/budget metadata only') atomic_json(transaction,record,replace=transaction.exists()) for source,target in ((new_config,rp),(new_checkpoint,checkpoint),(new_journal,journal)): temporary=target.with_name(target.name+'.recovery') if temporary.exists():temporary.unlink() os.link(source,temporary);os.replace(temporary,target);fsync_dir(target.parent) record['state']='committed';atomic_json(transaction,record,replace=True) atomic_json(archive/'recovery.json',record) return record if __name__=='__main__': parser=argparse.ArgumentParser();parser.add_argument('--run-config',required=True);args=parser.parse_args() print(json.dumps(recover(args.run_config),indent=2))