Download code/scripts/recover_checkpoint.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 6.95 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/recover_checkpoint.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/recover_checkpoint.py
-
curl -L -o recover_checkpoint.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/recover_checkpoint.py
6.95 kB
| """Audited rollback to a saved boundary after interruption; never use on a live run. | |
| Only GPU identity, model-load budget, and checkpoint provenance metadata change. | |
| The numerical state and seeded inputs are verified and preserved exactly. | |
| """ | |
| import argparse,fcntl,json,os,pathlib,shutil,subprocess,time | |
| import torch | |
| from robottt_pi05.config import ExperimentConfig | |
| from robottt_pi05.checkpoint import digest,fsync_dir | |
| from robottt_pi05.eval_io import sha,atomic_json,atomic_tensor | |
| from robottt_pi05.profile_common import runtime_signature | |
| def trim_events(lines,saved): | |
| attempted={k:0 for k in saved['attempted']};completed={k:0 for k in saved['completed']};kept=[] | |
| for index,line in enumerate(lines): | |
| try:record=json.loads(line) | |
| except json.JSONDecodeError: | |
| if index==len(lines)-1:break | |
| raise ValueError('Corrupt journal before its final line') | |
| scope=record.get('scope');step=record.get('step') | |
| if (isinstance(scope,str) and scope.isdigit() and int(scope)>saved['step']) or (isinstance(step,int) and step>saved['step']):break | |
| if record['event'] in ('reserve','complete'): | |
| counters=attempted if record['event']=='reserve' else completed | |
| target=saved['attempted'] if record['event']=='reserve' else saved['completed'] | |
| kind=record['kind'] | |
| if counters[kind]+1>target[kind]:break | |
| counters[kind]+=1 | |
| kept.append(line) | |
| if attempted!=saved['attempted']:raise ValueError('Saved boundary and journal attempts differ') | |
| for kind,count in completed.items(): | |
| missing=saved['completed'][kind]-count | |
| if missing and not (kind=='optimizer_update' and missing==1):raise ValueError('Saved boundary and journal completions differ') | |
| return ''.join(line if line.endswith('\n') else line+'\n' for line in kept) | |
| def numerical_digest(saved): | |
| return digest({key:saved[key] for key in ('step','next_schedule_index','slow','optimizer','rng','config','parameter_names')}) | |
| def restore_transaction(transaction): | |
| record=json.loads(transaction.read_text()) | |
| if record['state']=='committed':return | |
| archive=pathlib.Path(record['archive']) | |
| for name,target in record['originals'].items(): | |
| target=pathlib.Path(target);temporary=target.with_name(target.name+'.rollback') | |
| if temporary.exists():temporary.unlink() | |
| if name=='checkpoint.pt':os.link(archive/name,temporary) | |
| else:shutil.copyfile(archive/name,temporary) | |
| os.replace(temporary,target);fsync_dir(target.parent) | |
| record['state']='rolled_back';atomic_json(transaction,record,replace=True) | |
| def recover(run_config): | |
| rp=pathlib.Path(run_config).resolve();r=json.loads(rp.read_text()) | |
| def path(key):return (rp.parent/r['paths'][key]).resolve() | |
| out=path('output');out.mkdir(parents=True,exist_ok=True) | |
| with (out/'run.lock').open('a') as lock: | |
| fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| transaction=out/'recovery-transaction.json' | |
| if transaction.exists():restore_transaction(transaction) | |
| # A rolled-back interrupted transaction may have restored the old config. | |
| r=json.loads(rp.read_text()) | |
| if (out/'completed.json').exists():return dict(state='already_completed',run=str(rp)) | |
| if list(out.glob('failure*.json')):raise RuntimeError('Recorded training failure requires diagnosis before rollback') | |
| checkpoint=out/'checkpoints/latest.pt';journal=out/'ledger/events.jsonl' | |
| if not checkpoint.exists():raise FileNotFoundError('No saved boundary; restore a verified HF checkpoint first') | |
| saved=torch.load(checkpoint,map_location='cpu',weights_only=True) | |
| old_provenance=dict(experiment_config=sha(path('experiment_config')),asset_index=sha(path('asset_index')),run_config=sha(rp),implementation_sha256=runtime_signature()) | |
| if saved['provenance']!=old_provenance:raise ValueError('Original checkpoint/config/code provenance differs') | |
| if saved['config']!=ExperimentConfig.load(path('experiment_config')).to_dict():raise ValueError('Numerical experiment configuration differs') | |
| for key,expected in r['hashes'].items(): | |
| if sha(path(key))!=expected:raise ValueError('Immutable input hash differs: '+key) | |
| trimmed=trim_events(journal.read_text().splitlines(keepends=True),saved) | |
| gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name,memory.total,driver_version','--format=csv,noheader'],text=True).strip() | |
| parts=[item.strip() for item in gpu.split(',')] | |
| if parts[1]!='NVIDIA GeForce RTX 3090':raise ValueError('Unvalidated replacement GPU model') | |
| archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True) | |
| originals={'config.json':str(rp),'checkpoint.pt':str(checkpoint),'events.jsonl':str(journal)} | |
| shutil.copyfile(rp,archive/'config.json');os.link(checkpoint,archive/'checkpoint.pt');shutil.copyfile(journal,archive/'events.jsonl') | |
| old_hash=sha(checkpoint);before=numerical_digest(saved) | |
| replacement=json.loads(json.dumps(r));replacement['gpu_uuid']=parts[0] | |
| replacement['limits']['model_load']=max(1024,replacement['limits']['model_load']) | |
| new_config=archive/'config-next.json';new_config.write_text(json.dumps(replacement,indent=2)+'\n') | |
| updated=dict(saved);updated['provenance']=dict(saved['provenance'],run_config=sha(new_config)) | |
| new_checkpoint=archive/'checkpoint-next.pt';atomic_tensor(new_checkpoint,updated) | |
| if numerical_digest(torch.load(new_checkpoint,map_location='cpu',weights_only=True))!=before:raise ValueError('Recovery altered numerical state') | |
| new_journal=archive/'events-next.jsonl';new_journal.write_text(trimmed) | |
| record=dict(state='prepared',archive=str(archive),originals=originals,source_checkpoint_sha256=old_hash, | |
| migrated_checkpoint_sha256=sha(new_checkpoint),preserved_numerical_state_sha256=before,step=saved['step'],gpu=gpu, | |
| discarded_unsaved_journal_bytes=journal.stat().st_size-len(trimmed.encode()),reason='Workspace interruption: resume the saved optimizer/RNG boundary; administrative GPU/budget metadata only') | |
| atomic_json(transaction,record,replace=transaction.exists()) | |
| for source,target in ((new_config,rp),(new_checkpoint,checkpoint),(new_journal,journal)): | |
| temporary=target.with_name(target.name+'.recovery') | |
| if temporary.exists():temporary.unlink() | |
| os.link(source,temporary);os.replace(temporary,target);fsync_dir(target.parent) | |
| record['state']='committed';atomic_json(transaction,record,replace=True) | |
| atomic_json(archive/'recovery.json',record) | |
| return record | |
| if __name__=='__main__': | |
| parser=argparse.ArgumentParser();parser.add_argument('--run-config',required=True);args=parser.parse_args() | |
| print(json.dumps(recover(args.run_config),indent=2)) | |