File size: 6,949 Bytes
07a447a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Audited rollback to a saved boundary after interruption; never use on a live run.
Only GPU identity, model-load budget, and checkpoint provenance metadata change.
The numerical state and seeded inputs are verified and preserved exactly.
"""
import argparse,fcntl,json,os,pathlib,shutil,subprocess,time
import torch
from robottt_pi05.config import ExperimentConfig
from robottt_pi05.checkpoint import digest,fsync_dir
from robottt_pi05.eval_io import sha,atomic_json,atomic_tensor
from robottt_pi05.profile_common import runtime_signature
def trim_events(lines,saved):
attempted={k:0 for k in saved['attempted']};completed={k:0 for k in saved['completed']};kept=[]
for index,line in enumerate(lines):
try:record=json.loads(line)
except json.JSONDecodeError:
if index==len(lines)-1:break
raise ValueError('Corrupt journal before its final line')
scope=record.get('scope');step=record.get('step')
if (isinstance(scope,str) and scope.isdigit() and int(scope)>saved['step']) or (isinstance(step,int) and step>saved['step']):break
if record['event'] in ('reserve','complete'):
counters=attempted if record['event']=='reserve' else completed
target=saved['attempted'] if record['event']=='reserve' else saved['completed']
kind=record['kind']
if counters[kind]+1>target[kind]:break
counters[kind]+=1
kept.append(line)
if attempted!=saved['attempted']:raise ValueError('Saved boundary and journal attempts differ')
for kind,count in completed.items():
missing=saved['completed'][kind]-count
if missing and not (kind=='optimizer_update' and missing==1):raise ValueError('Saved boundary and journal completions differ')
return ''.join(line if line.endswith('\n') else line+'\n' for line in kept)
def numerical_digest(saved):
return digest({key:saved[key] for key in ('step','next_schedule_index','slow','optimizer','rng','config','parameter_names')})
def restore_transaction(transaction):
record=json.loads(transaction.read_text())
if record['state']=='committed':return
archive=pathlib.Path(record['archive'])
for name,target in record['originals'].items():
target=pathlib.Path(target);temporary=target.with_name(target.name+'.rollback')
if temporary.exists():temporary.unlink()
if name=='checkpoint.pt':os.link(archive/name,temporary)
else:shutil.copyfile(archive/name,temporary)
os.replace(temporary,target);fsync_dir(target.parent)
record['state']='rolled_back';atomic_json(transaction,record,replace=True)
def recover(run_config):
rp=pathlib.Path(run_config).resolve();r=json.loads(rp.read_text())
def path(key):return (rp.parent/r['paths'][key]).resolve()
out=path('output');out.mkdir(parents=True,exist_ok=True)
with (out/'run.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
transaction=out/'recovery-transaction.json'
if transaction.exists():restore_transaction(transaction)
# A rolled-back interrupted transaction may have restored the old config.
r=json.loads(rp.read_text())
if (out/'completed.json').exists():return dict(state='already_completed',run=str(rp))
if list(out.glob('failure*.json')):raise RuntimeError('Recorded training failure requires diagnosis before rollback')
checkpoint=out/'checkpoints/latest.pt';journal=out/'ledger/events.jsonl'
if not checkpoint.exists():raise FileNotFoundError('No saved boundary; restore a verified HF checkpoint first')
saved=torch.load(checkpoint,map_location='cpu',weights_only=True)
old_provenance=dict(experiment_config=sha(path('experiment_config')),asset_index=sha(path('asset_index')),run_config=sha(rp),implementation_sha256=runtime_signature())
if saved['provenance']!=old_provenance:raise ValueError('Original checkpoint/config/code provenance differs')
if saved['config']!=ExperimentConfig.load(path('experiment_config')).to_dict():raise ValueError('Numerical experiment configuration differs')
for key,expected in r['hashes'].items():
if sha(path(key))!=expected:raise ValueError('Immutable input hash differs: '+key)
trimmed=trim_events(journal.read_text().splitlines(keepends=True),saved)
gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name,memory.total,driver_version','--format=csv,noheader'],text=True).strip()
parts=[item.strip() for item in gpu.split(',')]
if parts[1]!='NVIDIA GeForce RTX 3090':raise ValueError('Unvalidated replacement GPU model')
archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True)
originals={'config.json':str(rp),'checkpoint.pt':str(checkpoint),'events.jsonl':str(journal)}
shutil.copyfile(rp,archive/'config.json');os.link(checkpoint,archive/'checkpoint.pt');shutil.copyfile(journal,archive/'events.jsonl')
old_hash=sha(checkpoint);before=numerical_digest(saved)
replacement=json.loads(json.dumps(r));replacement['gpu_uuid']=parts[0]
replacement['limits']['model_load']=max(1024,replacement['limits']['model_load'])
new_config=archive/'config-next.json';new_config.write_text(json.dumps(replacement,indent=2)+'\n')
updated=dict(saved);updated['provenance']=dict(saved['provenance'],run_config=sha(new_config))
new_checkpoint=archive/'checkpoint-next.pt';atomic_tensor(new_checkpoint,updated)
if numerical_digest(torch.load(new_checkpoint,map_location='cpu',weights_only=True))!=before:raise ValueError('Recovery altered numerical state')
new_journal=archive/'events-next.jsonl';new_journal.write_text(trimmed)
record=dict(state='prepared',archive=str(archive),originals=originals,source_checkpoint_sha256=old_hash,
migrated_checkpoint_sha256=sha(new_checkpoint),preserved_numerical_state_sha256=before,step=saved['step'],gpu=gpu,
discarded_unsaved_journal_bytes=journal.stat().st_size-len(trimmed.encode()),reason='Workspace interruption: resume the saved optimizer/RNG boundary; administrative GPU/budget metadata only')
atomic_json(transaction,record,replace=transaction.exists())
for source,target in ((new_config,rp),(new_checkpoint,checkpoint),(new_journal,journal)):
temporary=target.with_name(target.name+'.recovery')
if temporary.exists():temporary.unlink()
os.link(source,temporary);os.replace(temporary,target);fsync_dir(target.parent)
record['state']='committed';atomic_json(transaction,record,replace=True)
atomic_json(archive/'recovery.json',record)
return record
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('--run-config',required=True);args=parser.parse_args()
print(json.dumps(recover(args.run_config),indent=2))
|