"""Back up explicitly selected experiment artifacts to an authorized private HF repo.""" import argparse,fcntl,hashlib,json,os,pathlib,pickletools,shutil,time,zipfile from datetime import datetime,timezone os.environ['HF_HOME']='/tmp/robottt-hf-cache' os.environ['HF_XET_CACHE']='/tmp/robottt-hf-cache/xet' os.environ.setdefault('HF_XET_CHUNK_CACHE_SIZE_BYTES','0') from huggingface_hub import HfApi ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local' def checkpoint_step(path): with zipfile.ZipFile(path) as archive: name=next(n for n in archive.namelist() if n.endswith('/data.pkl')) if archive.getinfo(name).file_size>64*1024*1024:raise ValueError('Unexpected checkpoint metadata size') expect=False for opcode,arg,_ in pickletools.genops(archive.read(name)): if expect: if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue if opcode.name in ('BININT','BININT1','BININT2','LONG1','LONG4','INT') and type(arg) is int:return arg expect=False if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='step':expect=True return None def checkpoint_config_hash(path): with zipfile.ZipFile(path) as archive: name=next(n for n in archive.namelist() if n.endswith('/data.pkl'));expect=False for opcode,arg,_ in pickletools.genops(archive.read(name)): if expect: if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and isinstance(arg,str) and len(arg)==64:return arg raise ValueError('Unexpected checkpoint config provenance encoding') if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='run_config':expect=True raise ValueError('Checkpoint config provenance missing') def snapshot(stage): files=[] def copy(source,target): source=pathlib.Path(source) if not source.is_file():return dest=stage/target;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,dest);files.append(target) for folder in ('src','scripts','tests'): for path in sorted((ROOT/folder).rglob('*.py')):copy(path,'code/'+str(path.relative_to(ROOT))) for pattern in ('configs/*.json','configs/environment-linux-cu126.txt','provenance/base_comparison_assets.json','provenance/base-tensor-equality.json','provenance/shared-base-verification.json','provenance/base-counterpart-download.json','provenance/opentau_reference_core.json','provenance/persistence-layout.json','docs/base_comparison.md','HANDOFF.md','pyproject.toml','provenance/pre-restart-remote-verification.json','provenance/pre-restart-backup.json'): for path in ROOT.glob(pattern):copy(path,str(path.relative_to(ROOT))) for name in ('comparison-status.json','common-artifacts.json','libero-eval-plan.json','comparison-results.json','monitor-status.json','monitor-history.jsonl','hf-assets-backup-status.json','hf-runtime-backup-status.json','runtime-archive.json','runtime-support.json','automation-plan.json','completion-watch-status.json','completion-watch-state.json','completion-watch-events.jsonl','legacy-l18-status.json','legacy-l18-results.json','legacy-l18-notification.json','hf-legacy-assets-status.json','legacy-libero-status.json','legacy-comparison-final.json','legacy-libero-plan.json','legacy-libero-notification.json'): copy(LOCAL/name,'progress/'+name) for path in LOCAL.glob('libero-plus-*.json'): copy(path,'progress/'+path.name) copy(pathlib.Path('/tmp/robottt-comparison/prefix-cache/status.json'),'progress/prefix-status.json') for name in ('comparison-lite.json','comparison-reference.json'):copy(LOCAL/name,'progress/run-configs/'+name) for path in LOCAL.glob('legacy-libero-*.json'): if path.name not in ('legacy-libero-status.json','legacy-libero-plan.json','legacy-libero-notification.json'): copy(path,'progress/run-configs/'+path.name) copy(LOCAL/'ondemand-prefix/asset-index.json','preprocessing/ondemand-asset-index.json') contracts=sorted((LOCAL/'prefix-contracts').glob('*.json')) if contracts: target=stage/'preprocessing/prefix-contracts.jsonl';target.parent.mkdir(parents=True,exist_ok=True) with target.open('w') as output: for path in contracts:output.write(json.dumps(json.loads(path.read_text()))+'\n') files.append('preprocessing/prefix-contracts.jsonl') for path in (ROOT/'licenses').glob('*'): if path.is_file():copy(path,'licenses/'+path.name) copy(ROOT/'NOTICE.md','NOTICE.md') for name in ('data-contract.json','runtime-profile-parity.json','simulator-smoke.json','cuda-smoke.json','unittest.log','environment-final-freeze.txt','ondemand-unittest.log','recovery-unittest.log','recovery-integration.json','recovery-replayed-loss-parity.json','simulator-timing.json','query-render-parity.json','runtime-restore-integration.json','runtime-support-restore.json','legacy-l18-runtime-validation.json','legacy-live-preprocessing-validation.json'): copy(LOCAL/'audit'/name,'validation/'+name) for path in (LOCAL/'audit').glob('query-render-parity-task*.json'):copy(path,'validation/'+path.name) copy(LOCAL/'assets/processed-libero/normalization.json','preprocessing/normalization.json') copy(LOCAL/'assets/paligemma_tokenizer.model','preprocessing/paligemma_tokenizer.model') checkpoints={};runs=LOCAL/'runs' for run in sorted(runs.iterdir()) if runs.exists() else []: if not run.is_dir():continue for name in ('status.json','model_audit.json','completed.json','recovery-transaction.json'): copy(run/name,'runs/'+run.name+'/'+name) if not run.name.startswith(('comparison-','legacy-ft-')):continue path=run/'checkpoints/latest.pt' if path.is_file(): relative='runs/'+run.name+'/checkpoints/latest.pt';dest=stage/relative;dest.parent.mkdir(parents=True,exist_ok=True) if dest.exists():dest.unlink() os.link(path,dest) if run.name.startswith(('comparison-','legacy-ft-')): source_config=LOCAL/(run.name+'.json') if hashlib.sha256(source_config.read_bytes()).hexdigest()!=checkpoint_config_hash(dest):raise ValueError('Concurrent config/checkpoint migration; retry backup') copy(source_config,'progress/run-configs/'+run.name+'.json') copy(run/'ledger/events.jsonl','runs/'+run.name+'/ledger/events.jsonl') checkpoints[run.name]=dict(step=checkpoint_step(dest),bytes=dest.stat().st_size,path=relative) files.append(relative) for name in ('protocol-comparison-lite','protocol-comparison-reference','protocol-legacy-ft-l18'): folder=LOCAL/name if folder.exists(): for path in folder.iterdir(): if path.suffix not in ('.json','.pt'):continue if name=='protocol-comparison-reference' and path.name=='bank.pt':continue target=stage/'protocols'/name/path.name;target.parent.mkdir(parents=True,exist_ok=True) if target.exists():target.unlink() os.link(path,target);files.append(str(target.relative_to(stage))) evaluations=LOCAL/'evaluations' if evaluations.exists(): for path in evaluations.rglob('*.json'):copy(path,'evaluations/'+str(path.relative_to(evaluations))) for path in evaluations.glob('legacy-ft-*/*/sequence.pt'):copy(path,'evaluations/'+str(path.relative_to(evaluations))) now=datetime.now(timezone.utc).isoformat() assets_status=LOCAL/'hf-assets-backup-status.json' assets_uploaded=assets_status.exists() and json.loads(assets_status.read_text()).get('state')=='uploaded' result=dict(updated_utc=now,checkpoints=checkpoints,base_weights_uploaded=assets_uploaded,base_model_sha256='df9bac8b5c5a7ca023fc2773daf60993843379148bc626f5b10ad9d5bb0da47b',files=files) (stage/'backup-manifest.json').write_text(json.dumps(result,indent=2)) (stage/'README.md').write_text('''--- tags: - robotics - pi05 - test-time-training --- # RoboTTT: common official-base comparison Public experiment backup, as authorized by the repository owner. Progress is updated approximately every five minutes while the backup worker runs. See `progress/comparison-status.json` and `backup-manifest.json` for current status and durable checkpoint steps. `smoke-*` runs are two-update integration checks, **not** completed 3000-update experiments. Main runs are named `comparison-lite` and `comparison-reference`. Both conditions use the same converted official pi05_base, Train episodes 0..59 and Dev 60..69, T32/stride1/S8, and saved schedule/noise. The comparison is between a four-layer lightweight TTT design (561,540 added trainable parameters) and the pinned OpenTau core on all 18 layers (85,319,824). This is a new common pipeline, not a reproduction of the historical full OpenTau trainer. Extension/optimizer checkpoints, seeded protocols, preprocessing contracts, code and progress are backed up every five minutes. The converted base and exact source/processed data are mirrored separately; see `progress/hf-assets-backup-status.json`. Exact base provenance and hashes are in `provenance/base_comparison_assets.json`. VLM prefixes are generated on demand with a bounded 2 GiB RAM cache. No large prefix tensor files are written to ephemeral storage. Sequence tensor-hash contracts are shared and checked between both methods. The exact Python environment is stored as `environment/runtime-env.tar.gz` with a checksum manifest. Its runtime directory is disposable and restored from this archive after a restart, avoiding tens of thousands of persistent small files. Check `progress/hf-runtime-backup-status.json` for upload completion. Resume using `python3 scripts/resume_comparison.py`; saved optimizer/RNG state is preserved and unsaved interrupted work is archived and rolled back explicitly. Flow-matching loss and simulator success are separate metrics; neither should be inferred from a smoke checkpoint. Final metrics appear in `progress/comparison-results.json` only after paired evaluations complete. ''') return result def main(): parser=argparse.ArgumentParser();parser.add_argument('--config',default=str(LOCAL/'hf-backup.json'));parser.add_argument('--watch',action='store_true');a=parser.parse_args() config=json.loads(pathlib.Path(a.config).read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token) lock=(LOCAL/'hf-backup.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) stage=LOCAL/'hf-backup-staging';stage.mkdir(exist_ok=True) while True: try: manifest=snapshot(stage) commit=api.upload_folder(repo_id=config['repo_id'],repo_type='model',folder_path=stage,commit_message='Experiment backup '+manifest['updated_utc'],ignore_patterns=['preprocessing/prefix-contracts/**','progress/prefix-status.json','runs/smoke-*/checkpoints/*.pt','protocols/protocol-comparison-reference/bank.pt']) state=dict(state='uploaded',repo_id=config['repo_id'],updated_utc=manifest['updated_utc'],commit_oid=commit.oid if hasattr(commit,'oid') else commit.commit_oid,checkpoints=manifest['checkpoints']) (LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True) except Exception as error: state=dict(state='upload_failed',updated_utc=datetime.now(timezone.utc).isoformat(),error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]')) (LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True) if not a.watch:raise SystemExit(1) if not a.watch:return time.sleep(config.get('interval_seconds',300)) if __name__=='__main__':main()