Download code/scripts/restore_remote_progress.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 6.01 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/restore_remote_progress.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/restore_remote_progress.py
-
curl -L -o restore_remote_progress.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/restore_remote_progress.py
6.01 kB
| """Overlay newer verified HF progress onto a disposable working copy.""" | |
| import hashlib,json,os,pathlib,shutil | |
| os.environ['HF_HOME']='/tmp/robottt-hf-cache';os.environ['HF_HUB_DISABLE_XET']='1' | |
| import torch | |
| from huggingface_hub import HfApi,hf_hub_download | |
| ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local' | |
| def sha(path): | |
| h=hashlib.sha256() | |
| with path.open('rb') as f: | |
| while b:=f.read(8*1024*1024):h.update(b) | |
| return h.hexdigest() | |
| def main(): | |
| config=json.loads((LOCAL/'hf-backup.json').read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token) | |
| revision=api.repo_info(config['repo_id'],repo_type='model').sha | |
| names=set(api.list_repo_files(config['repo_id'],repo_type='model',revision=revision)) | |
| def get(name):return pathlib.Path(hf_hub_download(config['repo_id'],name,repo_type='model',revision=revision,token=token,local_dir='/tmp/robottt-remote-progress',cache_dir='/tmp/robottt-hf-cache/hub')) | |
| manifest=json.loads(get('backup-manifest.json').read_text());restored=[] | |
| for name in ('comparison-lite','comparison-reference'): | |
| remote=manifest.get('checkpoints',{}).get(name) | |
| if not remote:continue | |
| out=LOCAL/'runs'/name;checkpoint=out/'checkpoints/latest.pt' | |
| local_step=torch.load(checkpoint,map_location='cpu',weights_only=True)['step'] if checkpoint.exists() else -1 | |
| if remote['step']<=local_step:continue | |
| downloaded=get(remote['path']);saved=torch.load(downloaded,map_location='cpu',weights_only=True) | |
| if saved['step']!=remote['step']:raise ValueError('HF manifest/checkpoint step mismatch') | |
| rp=LOCAL/(name+'.json');remote_config=get('progress/run-configs/'+name+'.json') | |
| if sha(remote_config)!=saved['provenance']['run_config']:raise ValueError('HF config/checkpoint provenance mismatch') | |
| checkpoint.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(downloaded,checkpoint);shutil.copyfile(remote_config,rp) | |
| journal_name='runs/'+name+'/ledger/events.jsonl';journal=out/'ledger/events.jsonl';journal.parent.mkdir(exist_ok=True) | |
| if journal_name in names:shutil.copyfile(get(journal_name),journal) | |
| else:journal.write_text('') | |
| # Checkpoint counters are authoritative if a concurrent log snapshot | |
| # ended just before the checkpoint commit. Explicitly audit reconciliation. | |
| records=[];attempted={k:0 for k in saved['attempted']};completed={k:0 for k in saved['completed']} | |
| for line in journal.read_text().splitlines(): | |
| try:r=json.loads(line) | |
| except json.JSONDecodeError:break | |
| scope=r.get('scope') | |
| if isinstance(scope,str) and scope.isdigit() and int(scope)>saved['step']:break | |
| if r.get('step',0)>saved['step']:break | |
| if r['event'] in ('reserve','complete'): | |
| counts=attempted if r['event']=='reserve' else completed;limits=saved['attempted'] if r['event']=='reserve' else saved['completed'] | |
| if counts[r['kind']]+1>limits[r['kind']]:break | |
| counts[r['kind']]+=1 | |
| records.append(r) | |
| added=0 | |
| for kind in attempted: | |
| for event,counts,target in [('reserve',attempted,saved['attempted']),('complete',completed,saved['completed'])]: | |
| missing=target[kind]-counts[kind] | |
| if missing<0:raise ValueError('Journal exceeds saved boundary') | |
| if kind=='optimizer_update': | |
| present={int(r['scope']) for r in records if r.get('event')==event and r.get('kind')==kind} | |
| scopes=[str(i) for i in range(1,saved['step']+1) if i not in present] | |
| if len(scopes)!=missing:raise ValueError('Optimizer counter/scope mismatch') | |
| else:scopes=['train' if kind=='model_load' else str(saved['step'])]*missing | |
| for scope in scopes:records.append(dict(event=event,kind=kind,scope=scope,restored_from_verified_checkpoint=True));added+=1 | |
| journal.write_text(''.join(json.dumps(r)+'\n' for r in records)) | |
| for filename in ('status.json','model_audit.json','recovery-transaction.json'): | |
| remote_name='runs/'+name+'/'+filename | |
| if remote_name in names and filename!='recovery-transaction.json':shutil.copyfile(get(remote_name),out/filename) | |
| done='runs/'+name+'/completed.json' | |
| if done in names: | |
| record=json.loads(get(done).read_text()) | |
| if record['updates']==saved['step'] and record['checkpoint']['sha256']==sha(checkpoint):shutil.copyfile(get(done),out/'completed.json') | |
| restored.append(dict(run=name,step=saved['step'],reconciled_counter_events=added)) | |
| packed='preprocessing/prefix-contracts.jsonl' | |
| if packed in names: | |
| contracts=LOCAL/'prefix-contracts';contracts.mkdir(exist_ok=True) | |
| for line in get(packed).read_text().splitlines(): | |
| r=json.loads(line);identity={k:v for k,v in r.items() if k!='prefix_tensor_sha256'};key=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest();path=contracts/(key+'.json') | |
| if path.exists() and json.loads(path.read_text())!=r:raise ValueError('Restored prefix contract differs') | |
| if not path.exists():path.write_text(json.dumps(r,indent=2)) | |
| # Evaluation outputs are immutable and each rollout checks its own identity. | |
| for name in names: | |
| if name.startswith('evaluations/') and name.endswith('.json'): | |
| target=LOCAL/name;target.parent.mkdir(parents=True,exist_ok=True) | |
| if not target.exists():shutil.copyfile(get(name),target) | |
| marker=LOCAL/'.workspace-ready.json';state=json.loads(marker.read_text());state['remote_restored']=True;marker.write_text(json.dumps(state)) | |
| report=dict(revision=revision,restored=restored);(LOCAL/'audit/remote-progress-restore.json').write_text(json.dumps(report,indent=2));print(json.dumps(report),flush=True) | |
| if __name__=='__main__':main() | |