robottt-pi05-base-comparison / code /scripts /backup_huggingface.py
jisue's picture
Experiment backup 2026-11-11T10:42:21.225426+00:00
f599749 verified
Raw History Blame Contribute Delete
12 kB
"""Back up explicitly selected experiment artifacts to an authorized private HF repo."""
import argparse,fcntl,hashlib,json,os,pathlib,pickletools,shutil,time,zipfile
from datetime import datetime,timezone
os.environ['HF_HOME']='/tmp/robottt-hf-cache'
os.environ['HF_XET_CACHE']='/tmp/robottt-hf-cache/xet'
os.environ.setdefault('HF_XET_CHUNK_CACHE_SIZE_BYTES','0')
from huggingface_hub import HfApi
ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local'
def checkpoint_step(path):
with zipfile.ZipFile(path) as archive:
name=next(n for n in archive.namelist() if n.endswith('/data.pkl'))
if archive.getinfo(name).file_size>64*1024*1024:raise ValueError('Unexpected checkpoint metadata size')
expect=False
for opcode,arg,_ in pickletools.genops(archive.read(name)):
if expect:
if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue
if opcode.name in ('BININT','BININT1','BININT2','LONG1','LONG4','INT') and type(arg) is int:return arg
expect=False
if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='step':expect=True
return None
def checkpoint_config_hash(path):
with zipfile.ZipFile(path) as archive:
name=next(n for n in archive.namelist() if n.endswith('/data.pkl'));expect=False
for opcode,arg,_ in pickletools.genops(archive.read(name)):
if expect:
if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue
if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and isinstance(arg,str) and len(arg)==64:return arg
raise ValueError('Unexpected checkpoint config provenance encoding')
if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='run_config':expect=True
raise ValueError('Checkpoint config provenance missing')
def snapshot(stage):
files=[]
def copy(source,target):
source=pathlib.Path(source)
if not source.is_file():return
dest=stage/target;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,dest);files.append(target)
for folder in ('src','scripts','tests'):
for path in sorted((ROOT/folder).rglob('*.py')):copy(path,'code/'+str(path.relative_to(ROOT)))
for pattern in ('configs/*.json','configs/environment-linux-cu126.txt','provenance/base_comparison_assets.json','provenance/base-tensor-equality.json','provenance/shared-base-verification.json','provenance/base-counterpart-download.json','provenance/opentau_reference_core.json','provenance/persistence-layout.json','docs/base_comparison.md','HANDOFF.md','pyproject.toml','provenance/pre-restart-remote-verification.json','provenance/pre-restart-backup.json'):
for path in ROOT.glob(pattern):copy(path,str(path.relative_to(ROOT)))
for name in ('comparison-status.json','common-artifacts.json','libero-eval-plan.json','comparison-results.json','monitor-status.json','monitor-history.jsonl','hf-assets-backup-status.json','hf-runtime-backup-status.json','runtime-archive.json','runtime-support.json','automation-plan.json','completion-watch-status.json','completion-watch-state.json','completion-watch-events.jsonl','legacy-l18-status.json','legacy-l18-results.json','legacy-l18-notification.json','hf-legacy-assets-status.json','legacy-libero-status.json','legacy-comparison-final.json','legacy-libero-plan.json','legacy-libero-notification.json'):
copy(LOCAL/name,'progress/'+name)
for path in LOCAL.glob('libero-plus-*.json'):
copy(path,'progress/'+path.name)
copy(pathlib.Path('/tmp/robottt-comparison/prefix-cache/status.json'),'progress/prefix-status.json')
for name in ('comparison-lite.json','comparison-reference.json'):copy(LOCAL/name,'progress/run-configs/'+name)
for path in LOCAL.glob('legacy-libero-*.json'):
if path.name not in ('legacy-libero-status.json','legacy-libero-plan.json','legacy-libero-notification.json'):
copy(path,'progress/run-configs/'+path.name)
copy(LOCAL/'ondemand-prefix/asset-index.json','preprocessing/ondemand-asset-index.json')
contracts=sorted((LOCAL/'prefix-contracts').glob('*.json'))
if contracts:
target=stage/'preprocessing/prefix-contracts.jsonl';target.parent.mkdir(parents=True,exist_ok=True)
with target.open('w') as output:
for path in contracts:output.write(json.dumps(json.loads(path.read_text()))+'\n')
files.append('preprocessing/prefix-contracts.jsonl')
for path in (ROOT/'licenses').glob('*'):
if path.is_file():copy(path,'licenses/'+path.name)
copy(ROOT/'NOTICE.md','NOTICE.md')
for name in ('data-contract.json','runtime-profile-parity.json','simulator-smoke.json','cuda-smoke.json','unittest.log','environment-final-freeze.txt','ondemand-unittest.log','recovery-unittest.log','recovery-integration.json','recovery-replayed-loss-parity.json','simulator-timing.json','query-render-parity.json','runtime-restore-integration.json','runtime-support-restore.json','legacy-l18-runtime-validation.json','legacy-live-preprocessing-validation.json'):
copy(LOCAL/'audit'/name,'validation/'+name)
for path in (LOCAL/'audit').glob('query-render-parity-task*.json'):copy(path,'validation/'+path.name)
copy(LOCAL/'assets/processed-libero/normalization.json','preprocessing/normalization.json')
copy(LOCAL/'assets/paligemma_tokenizer.model','preprocessing/paligemma_tokenizer.model')
checkpoints={};runs=LOCAL/'runs'
for run in sorted(runs.iterdir()) if runs.exists() else []:
if not run.is_dir():continue
for name in ('status.json','model_audit.json','completed.json','recovery-transaction.json'):
copy(run/name,'runs/'+run.name+'/'+name)
if not run.name.startswith(('comparison-','legacy-ft-')):continue
path=run/'checkpoints/latest.pt'
if path.is_file():
relative='runs/'+run.name+'/checkpoints/latest.pt';dest=stage/relative;dest.parent.mkdir(parents=True,exist_ok=True)
if dest.exists():dest.unlink()
os.link(path,dest)
if run.name.startswith(('comparison-','legacy-ft-')):
source_config=LOCAL/(run.name+'.json')
if hashlib.sha256(source_config.read_bytes()).hexdigest()!=checkpoint_config_hash(dest):raise ValueError('Concurrent config/checkpoint migration; retry backup')
copy(source_config,'progress/run-configs/'+run.name+'.json')
copy(run/'ledger/events.jsonl','runs/'+run.name+'/ledger/events.jsonl')
checkpoints[run.name]=dict(step=checkpoint_step(dest),bytes=dest.stat().st_size,path=relative)
files.append(relative)
for name in ('protocol-comparison-lite','protocol-comparison-reference','protocol-legacy-ft-l18'):
folder=LOCAL/name
if folder.exists():
for path in folder.iterdir():
if path.suffix not in ('.json','.pt'):continue
if name=='protocol-comparison-reference' and path.name=='bank.pt':continue
target=stage/'protocols'/name/path.name;target.parent.mkdir(parents=True,exist_ok=True)
if target.exists():target.unlink()
os.link(path,target);files.append(str(target.relative_to(stage)))
evaluations=LOCAL/'evaluations'
if evaluations.exists():
for path in evaluations.rglob('*.json'):copy(path,'evaluations/'+str(path.relative_to(evaluations)))
for path in evaluations.glob('legacy-ft-*/*/sequence.pt'):copy(path,'evaluations/'+str(path.relative_to(evaluations)))
now=datetime.now(timezone.utc).isoformat()
assets_status=LOCAL/'hf-assets-backup-status.json'
assets_uploaded=assets_status.exists() and json.loads(assets_status.read_text()).get('state')=='uploaded'
result=dict(updated_utc=now,checkpoints=checkpoints,base_weights_uploaded=assets_uploaded,base_model_sha256='df9bac8b5c5a7ca023fc2773daf60993843379148bc626f5b10ad9d5bb0da47b',files=files)
(stage/'backup-manifest.json').write_text(json.dumps(result,indent=2))
(stage/'README.md').write_text('''---
tags:
- robotics
- pi05
- test-time-training
---
# RoboTTT: common official-base comparison
Public experiment backup, as authorized by the repository owner. Progress is updated approximately every five minutes while the backup worker runs.
See `progress/comparison-status.json` and `backup-manifest.json` for current status and durable checkpoint steps. `smoke-*` runs are two-update integration checks, **not** completed 3000-update experiments. Main runs are named `comparison-lite` and `comparison-reference`.
Both conditions use the same converted official pi05_base, Train episodes 0..59 and Dev 60..69, T32/stride1/S8, and saved schedule/noise. The comparison is between a four-layer lightweight TTT design (561,540 added trainable parameters) and the pinned OpenTau core on all 18 layers (85,319,824). This is a new common pipeline, not a reproduction of the historical full OpenTau trainer.
Extension/optimizer checkpoints, seeded protocols, preprocessing contracts, code and progress are backed up every five minutes. The converted base and exact source/processed data are mirrored separately; see `progress/hf-assets-backup-status.json`. Exact base provenance and hashes are in `provenance/base_comparison_assets.json`.
VLM prefixes are generated on demand with a bounded 2 GiB RAM cache. No large prefix tensor files are written to ephemeral storage. Sequence tensor-hash contracts are shared and checked between both methods.
The exact Python environment is stored as `environment/runtime-env.tar.gz` with a checksum manifest. Its runtime directory is disposable and restored from this archive after a restart, avoiding tens of thousands of persistent small files. Check `progress/hf-runtime-backup-status.json` for upload completion. Resume using `python3 scripts/resume_comparison.py`; saved optimizer/RNG state is preserved and unsaved interrupted work is archived and rolled back explicitly.
Flow-matching loss and simulator success are separate metrics; neither should be inferred from a smoke checkpoint. Final metrics appear in `progress/comparison-results.json` only after paired evaluations complete.
''')
return result
def main():
parser=argparse.ArgumentParser();parser.add_argument('--config',default=str(LOCAL/'hf-backup.json'));parser.add_argument('--watch',action='store_true');a=parser.parse_args()
config=json.loads(pathlib.Path(a.config).read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token)
lock=(LOCAL/'hf-backup.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
stage=LOCAL/'hf-backup-staging';stage.mkdir(exist_ok=True)
while True:
try:
manifest=snapshot(stage)
commit=api.upload_folder(repo_id=config['repo_id'],repo_type='model',folder_path=stage,commit_message='Experiment backup '+manifest['updated_utc'],ignore_patterns=['preprocessing/prefix-contracts/**','progress/prefix-status.json','runs/smoke-*/checkpoints/*.pt','protocols/protocol-comparison-reference/bank.pt'])
state=dict(state='uploaded',repo_id=config['repo_id'],updated_utc=manifest['updated_utc'],commit_oid=commit.oid if hasattr(commit,'oid') else commit.commit_oid,checkpoints=manifest['checkpoints'])
(LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True)
except Exception as error:
state=dict(state='upload_failed',updated_utc=datetime.now(timezone.utc).isoformat(),error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]'))
(LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True)
if not a.watch:raise SystemExit(1)
if not a.watch:return
time.sleep(config.get('interval_seconds',300))
if __name__=='__main__':main()