File size: 6,009 Bytes
a02e3b4 3f8ab35 a02e3b4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Overlay newer verified HF progress onto a disposable working copy."""
import hashlib,json,os,pathlib,shutil
os.environ['HF_HOME']='/tmp/robottt-hf-cache';os.environ['HF_HUB_DISABLE_XET']='1'
import torch
from huggingface_hub import HfApi,hf_hub_download
ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local'
def sha(path):
h=hashlib.sha256()
with path.open('rb') as f:
while b:=f.read(8*1024*1024):h.update(b)
return h.hexdigest()
def main():
config=json.loads((LOCAL/'hf-backup.json').read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token)
revision=api.repo_info(config['repo_id'],repo_type='model').sha
names=set(api.list_repo_files(config['repo_id'],repo_type='model',revision=revision))
def get(name):return pathlib.Path(hf_hub_download(config['repo_id'],name,repo_type='model',revision=revision,token=token,local_dir='/tmp/robottt-remote-progress',cache_dir='/tmp/robottt-hf-cache/hub'))
manifest=json.loads(get('backup-manifest.json').read_text());restored=[]
for name in ('comparison-lite','comparison-reference'):
remote=manifest.get('checkpoints',{}).get(name)
if not remote:continue
out=LOCAL/'runs'/name;checkpoint=out/'checkpoints/latest.pt'
local_step=torch.load(checkpoint,map_location='cpu',weights_only=True)['step'] if checkpoint.exists() else -1
if remote['step']<=local_step:continue
downloaded=get(remote['path']);saved=torch.load(downloaded,map_location='cpu',weights_only=True)
if saved['step']!=remote['step']:raise ValueError('HF manifest/checkpoint step mismatch')
rp=LOCAL/(name+'.json');remote_config=get('progress/run-configs/'+name+'.json')
if sha(remote_config)!=saved['provenance']['run_config']:raise ValueError('HF config/checkpoint provenance mismatch')
checkpoint.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(downloaded,checkpoint);shutil.copyfile(remote_config,rp)
journal_name='runs/'+name+'/ledger/events.jsonl';journal=out/'ledger/events.jsonl';journal.parent.mkdir(exist_ok=True)
if journal_name in names:shutil.copyfile(get(journal_name),journal)
else:journal.write_text('')
# Checkpoint counters are authoritative if a concurrent log snapshot
# ended just before the checkpoint commit. Explicitly audit reconciliation.
records=[];attempted={k:0 for k in saved['attempted']};completed={k:0 for k in saved['completed']}
for line in journal.read_text().splitlines():
try:r=json.loads(line)
except json.JSONDecodeError:break
scope=r.get('scope')
if isinstance(scope,str) and scope.isdigit() and int(scope)>saved['step']:break
if r.get('step',0)>saved['step']:break
if r['event'] in ('reserve','complete'):
counts=attempted if r['event']=='reserve' else completed;limits=saved['attempted'] if r['event']=='reserve' else saved['completed']
if counts[r['kind']]+1>limits[r['kind']]:break
counts[r['kind']]+=1
records.append(r)
added=0
for kind in attempted:
for event,counts,target in [('reserve',attempted,saved['attempted']),('complete',completed,saved['completed'])]:
missing=target[kind]-counts[kind]
if missing<0:raise ValueError('Journal exceeds saved boundary')
if kind=='optimizer_update':
present={int(r['scope']) for r in records if r.get('event')==event and r.get('kind')==kind}
scopes=[str(i) for i in range(1,saved['step']+1) if i not in present]
if len(scopes)!=missing:raise ValueError('Optimizer counter/scope mismatch')
else:scopes=['train' if kind=='model_load' else str(saved['step'])]*missing
for scope in scopes:records.append(dict(event=event,kind=kind,scope=scope,restored_from_verified_checkpoint=True));added+=1
journal.write_text(''.join(json.dumps(r)+'\n' for r in records))
for filename in ('status.json','model_audit.json','recovery-transaction.json'):
remote_name='runs/'+name+'/'+filename
if remote_name in names and filename!='recovery-transaction.json':shutil.copyfile(get(remote_name),out/filename)
done='runs/'+name+'/completed.json'
if done in names:
record=json.loads(get(done).read_text())
if record['updates']==saved['step'] and record['checkpoint']['sha256']==sha(checkpoint):shutil.copyfile(get(done),out/'completed.json')
restored.append(dict(run=name,step=saved['step'],reconciled_counter_events=added))
packed='preprocessing/prefix-contracts.jsonl'
if packed in names:
contracts=LOCAL/'prefix-contracts';contracts.mkdir(exist_ok=True)
for line in get(packed).read_text().splitlines():
r=json.loads(line);identity={k:v for k,v in r.items() if k!='prefix_tensor_sha256'};key=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest();path=contracts/(key+'.json')
if path.exists() and json.loads(path.read_text())!=r:raise ValueError('Restored prefix contract differs')
if not path.exists():path.write_text(json.dumps(r,indent=2))
# Evaluation outputs are immutable and each rollout checks its own identity.
for name in names:
if name.startswith('evaluations/') and name.endswith('.json'):
target=LOCAL/name;target.parent.mkdir(parents=True,exist_ok=True)
if not target.exists():shutil.copyfile(get(name),target)
marker=LOCAL/'.workspace-ready.json';state=json.loads(marker.read_text());state['remote_restored']=True;marker.write_text(json.dumps(state))
report=dict(revision=revision,restored=restored);(LOCAL/'audit/remote-progress-restore.json').write_text(json.dumps(report,indent=2));print(json.dumps(report),flush=True)
if __name__=='__main__':main()
|