"""Overlay newer verified HF progress onto a disposable working copy.""" import hashlib,json,os,pathlib,shutil os.environ['HF_HOME']='/tmp/robottt-hf-cache';os.environ['HF_HUB_DISABLE_XET']='1' import torch from huggingface_hub import HfApi,hf_hub_download ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local' def sha(path): h=hashlib.sha256() with path.open('rb') as f: while b:=f.read(8*1024*1024):h.update(b) return h.hexdigest() def main(): config=json.loads((LOCAL/'hf-backup.json').read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token) revision=api.repo_info(config['repo_id'],repo_type='model').sha names=set(api.list_repo_files(config['repo_id'],repo_type='model',revision=revision)) def get(name):return pathlib.Path(hf_hub_download(config['repo_id'],name,repo_type='model',revision=revision,token=token,local_dir='/tmp/robottt-remote-progress',cache_dir='/tmp/robottt-hf-cache/hub')) manifest=json.loads(get('backup-manifest.json').read_text());restored=[] for name in ('comparison-lite','comparison-reference'): remote=manifest.get('checkpoints',{}).get(name) if not remote:continue out=LOCAL/'runs'/name;checkpoint=out/'checkpoints/latest.pt' local_step=torch.load(checkpoint,map_location='cpu',weights_only=True)['step'] if checkpoint.exists() else -1 if remote['step']<=local_step:continue downloaded=get(remote['path']);saved=torch.load(downloaded,map_location='cpu',weights_only=True) if saved['step']!=remote['step']:raise ValueError('HF manifest/checkpoint step mismatch') rp=LOCAL/(name+'.json');remote_config=get('progress/run-configs/'+name+'.json') if sha(remote_config)!=saved['provenance']['run_config']:raise ValueError('HF config/checkpoint provenance mismatch') checkpoint.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(downloaded,checkpoint);shutil.copyfile(remote_config,rp) journal_name='runs/'+name+'/ledger/events.jsonl';journal=out/'ledger/events.jsonl';journal.parent.mkdir(exist_ok=True) if journal_name in names:shutil.copyfile(get(journal_name),journal) else:journal.write_text('') # Checkpoint counters are authoritative if a concurrent log snapshot # ended just before the checkpoint commit. Explicitly audit reconciliation. records=[];attempted={k:0 for k in saved['attempted']};completed={k:0 for k in saved['completed']} for line in journal.read_text().splitlines(): try:r=json.loads(line) except json.JSONDecodeError:break scope=r.get('scope') if isinstance(scope,str) and scope.isdigit() and int(scope)>saved['step']:break if r.get('step',0)>saved['step']:break if r['event'] in ('reserve','complete'): counts=attempted if r['event']=='reserve' else completed;limits=saved['attempted'] if r['event']=='reserve' else saved['completed'] if counts[r['kind']]+1>limits[r['kind']]:break counts[r['kind']]+=1 records.append(r) added=0 for kind in attempted: for event,counts,target in [('reserve',attempted,saved['attempted']),('complete',completed,saved['completed'])]: missing=target[kind]-counts[kind] if missing<0:raise ValueError('Journal exceeds saved boundary') if kind=='optimizer_update': present={int(r['scope']) for r in records if r.get('event')==event and r.get('kind')==kind} scopes=[str(i) for i in range(1,saved['step']+1) if i not in present] if len(scopes)!=missing:raise ValueError('Optimizer counter/scope mismatch') else:scopes=['train' if kind=='model_load' else str(saved['step'])]*missing for scope in scopes:records.append(dict(event=event,kind=kind,scope=scope,restored_from_verified_checkpoint=True));added+=1 journal.write_text(''.join(json.dumps(r)+'\n' for r in records)) for filename in ('status.json','model_audit.json','recovery-transaction.json'): remote_name='runs/'+name+'/'+filename if remote_name in names and filename!='recovery-transaction.json':shutil.copyfile(get(remote_name),out/filename) done='runs/'+name+'/completed.json' if done in names: record=json.loads(get(done).read_text()) if record['updates']==saved['step'] and record['checkpoint']['sha256']==sha(checkpoint):shutil.copyfile(get(done),out/'completed.json') restored.append(dict(run=name,step=saved['step'],reconciled_counter_events=added)) packed='preprocessing/prefix-contracts.jsonl' if packed in names: contracts=LOCAL/'prefix-contracts';contracts.mkdir(exist_ok=True) for line in get(packed).read_text().splitlines(): r=json.loads(line);identity={k:v for k,v in r.items() if k!='prefix_tensor_sha256'};key=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest();path=contracts/(key+'.json') if path.exists() and json.loads(path.read_text())!=r:raise ValueError('Restored prefix contract differs') if not path.exists():path.write_text(json.dumps(r,indent=2)) # Evaluation outputs are immutable and each rollout checks its own identity. for name in names: if name.startswith('evaluations/') and name.endswith('.json'): target=LOCAL/name;target.parent.mkdir(parents=True,exist_ok=True) if not target.exists():shutil.copyfile(get(name),target) marker=LOCAL/'.workspace-ready.json';state=json.loads(marker.read_text());state['remote_restored']=True;marker.write_text(json.dumps(state)) report=dict(revision=revision,restored=restored);(LOCAL/'audit/remote-progress-restore.json').write_text(json.dumps(report,indent=2));print(json.dumps(report),flush=True) if __name__=='__main__':main()