Download code/scripts/backup_huggingface.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 12 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/backup_huggingface.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/backup_huggingface.py
-
curl -L -o backup_huggingface.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/backup_huggingface.py
12 kB
| """Back up explicitly selected experiment artifacts to an authorized private HF repo.""" | |
| import argparse,fcntl,hashlib,json,os,pathlib,pickletools,shutil,time,zipfile | |
| from datetime import datetime,timezone | |
| os.environ['HF_HOME']='/tmp/robottt-hf-cache' | |
| os.environ['HF_XET_CACHE']='/tmp/robottt-hf-cache/xet' | |
| os.environ.setdefault('HF_XET_CHUNK_CACHE_SIZE_BYTES','0') | |
| from huggingface_hub import HfApi | |
| ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local' | |
| def checkpoint_step(path): | |
| with zipfile.ZipFile(path) as archive: | |
| name=next(n for n in archive.namelist() if n.endswith('/data.pkl')) | |
| if archive.getinfo(name).file_size>64*1024*1024:raise ValueError('Unexpected checkpoint metadata size') | |
| expect=False | |
| for opcode,arg,_ in pickletools.genops(archive.read(name)): | |
| if expect: | |
| if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue | |
| if opcode.name in ('BININT','BININT1','BININT2','LONG1','LONG4','INT') and type(arg) is int:return arg | |
| expect=False | |
| if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='step':expect=True | |
| return None | |
| def checkpoint_config_hash(path): | |
| with zipfile.ZipFile(path) as archive: | |
| name=next(n for n in archive.namelist() if n.endswith('/data.pkl'));expect=False | |
| for opcode,arg,_ in pickletools.genops(archive.read(name)): | |
| if expect: | |
| if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue | |
| if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and isinstance(arg,str) and len(arg)==64:return arg | |
| raise ValueError('Unexpected checkpoint config provenance encoding') | |
| if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='run_config':expect=True | |
| raise ValueError('Checkpoint config provenance missing') | |
| def snapshot(stage): | |
| files=[] | |
| def copy(source,target): | |
| source=pathlib.Path(source) | |
| if not source.is_file():return | |
| dest=stage/target;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,dest);files.append(target) | |
| for folder in ('src','scripts','tests'): | |
| for path in sorted((ROOT/folder).rglob('*.py')):copy(path,'code/'+str(path.relative_to(ROOT))) | |
| for pattern in ('configs/*.json','configs/environment-linux-cu126.txt','provenance/base_comparison_assets.json','provenance/base-tensor-equality.json','provenance/shared-base-verification.json','provenance/base-counterpart-download.json','provenance/opentau_reference_core.json','provenance/persistence-layout.json','docs/base_comparison.md','HANDOFF.md','pyproject.toml','provenance/pre-restart-remote-verification.json','provenance/pre-restart-backup.json'): | |
| for path in ROOT.glob(pattern):copy(path,str(path.relative_to(ROOT))) | |
| for name in ('comparison-status.json','common-artifacts.json','libero-eval-plan.json','comparison-results.json','monitor-status.json','monitor-history.jsonl','hf-assets-backup-status.json','hf-runtime-backup-status.json','runtime-archive.json','runtime-support.json','automation-plan.json','completion-watch-status.json','completion-watch-state.json','completion-watch-events.jsonl','legacy-l18-status.json','legacy-l18-results.json','legacy-l18-notification.json','hf-legacy-assets-status.json','legacy-libero-status.json','legacy-comparison-final.json','legacy-libero-plan.json','legacy-libero-notification.json'): | |
| copy(LOCAL/name,'progress/'+name) | |
| for path in LOCAL.glob('libero-plus-*.json'): | |
| copy(path,'progress/'+path.name) | |
| copy(pathlib.Path('/tmp/robottt-comparison/prefix-cache/status.json'),'progress/prefix-status.json') | |
| for name in ('comparison-lite.json','comparison-reference.json'):copy(LOCAL/name,'progress/run-configs/'+name) | |
| for path in LOCAL.glob('legacy-libero-*.json'): | |
| if path.name not in ('legacy-libero-status.json','legacy-libero-plan.json','legacy-libero-notification.json'): | |
| copy(path,'progress/run-configs/'+path.name) | |
| copy(LOCAL/'ondemand-prefix/asset-index.json','preprocessing/ondemand-asset-index.json') | |
| contracts=sorted((LOCAL/'prefix-contracts').glob('*.json')) | |
| if contracts: | |
| target=stage/'preprocessing/prefix-contracts.jsonl';target.parent.mkdir(parents=True,exist_ok=True) | |
| with target.open('w') as output: | |
| for path in contracts:output.write(json.dumps(json.loads(path.read_text()))+'\n') | |
| files.append('preprocessing/prefix-contracts.jsonl') | |
| for path in (ROOT/'licenses').glob('*'): | |
| if path.is_file():copy(path,'licenses/'+path.name) | |
| copy(ROOT/'NOTICE.md','NOTICE.md') | |
| for name in ('data-contract.json','runtime-profile-parity.json','simulator-smoke.json','cuda-smoke.json','unittest.log','environment-final-freeze.txt','ondemand-unittest.log','recovery-unittest.log','recovery-integration.json','recovery-replayed-loss-parity.json','simulator-timing.json','query-render-parity.json','runtime-restore-integration.json','runtime-support-restore.json','legacy-l18-runtime-validation.json','legacy-live-preprocessing-validation.json'): | |
| copy(LOCAL/'audit'/name,'validation/'+name) | |
| for path in (LOCAL/'audit').glob('query-render-parity-task*.json'):copy(path,'validation/'+path.name) | |
| copy(LOCAL/'assets/processed-libero/normalization.json','preprocessing/normalization.json') | |
| copy(LOCAL/'assets/paligemma_tokenizer.model','preprocessing/paligemma_tokenizer.model') | |
| checkpoints={};runs=LOCAL/'runs' | |
| for run in sorted(runs.iterdir()) if runs.exists() else []: | |
| if not run.is_dir():continue | |
| for name in ('status.json','model_audit.json','completed.json','recovery-transaction.json'): | |
| copy(run/name,'runs/'+run.name+'/'+name) | |
| if not run.name.startswith(('comparison-','legacy-ft-')):continue | |
| path=run/'checkpoints/latest.pt' | |
| if path.is_file(): | |
| relative='runs/'+run.name+'/checkpoints/latest.pt';dest=stage/relative;dest.parent.mkdir(parents=True,exist_ok=True) | |
| if dest.exists():dest.unlink() | |
| os.link(path,dest) | |
| if run.name.startswith(('comparison-','legacy-ft-')): | |
| source_config=LOCAL/(run.name+'.json') | |
| if hashlib.sha256(source_config.read_bytes()).hexdigest()!=checkpoint_config_hash(dest):raise ValueError('Concurrent config/checkpoint migration; retry backup') | |
| copy(source_config,'progress/run-configs/'+run.name+'.json') | |
| copy(run/'ledger/events.jsonl','runs/'+run.name+'/ledger/events.jsonl') | |
| checkpoints[run.name]=dict(step=checkpoint_step(dest),bytes=dest.stat().st_size,path=relative) | |
| files.append(relative) | |
| for name in ('protocol-comparison-lite','protocol-comparison-reference','protocol-legacy-ft-l18'): | |
| folder=LOCAL/name | |
| if folder.exists(): | |
| for path in folder.iterdir(): | |
| if path.suffix not in ('.json','.pt'):continue | |
| if name=='protocol-comparison-reference' and path.name=='bank.pt':continue | |
| target=stage/'protocols'/name/path.name;target.parent.mkdir(parents=True,exist_ok=True) | |
| if target.exists():target.unlink() | |
| os.link(path,target);files.append(str(target.relative_to(stage))) | |
| evaluations=LOCAL/'evaluations' | |
| if evaluations.exists(): | |
| for path in evaluations.rglob('*.json'):copy(path,'evaluations/'+str(path.relative_to(evaluations))) | |
| for path in evaluations.glob('legacy-ft-*/*/sequence.pt'):copy(path,'evaluations/'+str(path.relative_to(evaluations))) | |
| now=datetime.now(timezone.utc).isoformat() | |
| assets_status=LOCAL/'hf-assets-backup-status.json' | |
| assets_uploaded=assets_status.exists() and json.loads(assets_status.read_text()).get('state')=='uploaded' | |
| result=dict(updated_utc=now,checkpoints=checkpoints,base_weights_uploaded=assets_uploaded,base_model_sha256='df9bac8b5c5a7ca023fc2773daf60993843379148bc626f5b10ad9d5bb0da47b',files=files) | |
| (stage/'backup-manifest.json').write_text(json.dumps(result,indent=2)) | |
| (stage/'README.md').write_text('''--- | |
| tags: | |
| - robotics | |
| - pi05 | |
| - test-time-training | |
| --- | |
| # RoboTTT: common official-base comparison | |
| Public experiment backup, as authorized by the repository owner. Progress is updated approximately every five minutes while the backup worker runs. | |
| See `progress/comparison-status.json` and `backup-manifest.json` for current status and durable checkpoint steps. `smoke-*` runs are two-update integration checks, **not** completed 3000-update experiments. Main runs are named `comparison-lite` and `comparison-reference`. | |
| Both conditions use the same converted official pi05_base, Train episodes 0..59 and Dev 60..69, T32/stride1/S8, and saved schedule/noise. The comparison is between a four-layer lightweight TTT design (561,540 added trainable parameters) and the pinned OpenTau core on all 18 layers (85,319,824). This is a new common pipeline, not a reproduction of the historical full OpenTau trainer. | |
| Extension/optimizer checkpoints, seeded protocols, preprocessing contracts, code and progress are backed up every five minutes. The converted base and exact source/processed data are mirrored separately; see `progress/hf-assets-backup-status.json`. Exact base provenance and hashes are in `provenance/base_comparison_assets.json`. | |
| VLM prefixes are generated on demand with a bounded 2 GiB RAM cache. No large prefix tensor files are written to ephemeral storage. Sequence tensor-hash contracts are shared and checked between both methods. | |
| The exact Python environment is stored as `environment/runtime-env.tar.gz` with a checksum manifest. Its runtime directory is disposable and restored from this archive after a restart, avoiding tens of thousands of persistent small files. Check `progress/hf-runtime-backup-status.json` for upload completion. Resume using `python3 scripts/resume_comparison.py`; saved optimizer/RNG state is preserved and unsaved interrupted work is archived and rolled back explicitly. | |
| Flow-matching loss and simulator success are separate metrics; neither should be inferred from a smoke checkpoint. Final metrics appear in `progress/comparison-results.json` only after paired evaluations complete. | |
| ''') | |
| return result | |
| def main(): | |
| parser=argparse.ArgumentParser();parser.add_argument('--config',default=str(LOCAL/'hf-backup.json'));parser.add_argument('--watch',action='store_true');a=parser.parse_args() | |
| config=json.loads(pathlib.Path(a.config).read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token) | |
| lock=(LOCAL/'hf-backup.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| stage=LOCAL/'hf-backup-staging';stage.mkdir(exist_ok=True) | |
| while True: | |
| try: | |
| manifest=snapshot(stage) | |
| commit=api.upload_folder(repo_id=config['repo_id'],repo_type='model',folder_path=stage,commit_message='Experiment backup '+manifest['updated_utc'],ignore_patterns=['preprocessing/prefix-contracts/**','progress/prefix-status.json','runs/smoke-*/checkpoints/*.pt','protocols/protocol-comparison-reference/bank.pt']) | |
| state=dict(state='uploaded',repo_id=config['repo_id'],updated_utc=manifest['updated_utc'],commit_oid=commit.oid if hasattr(commit,'oid') else commit.commit_oid,checkpoints=manifest['checkpoints']) | |
| (LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True) | |
| except Exception as error: | |
| state=dict(state='upload_failed',updated_utc=datetime.now(timezone.utc).isoformat(),error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]')) | |
| (LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True) | |
| if not a.watch:raise SystemExit(1) | |
| if not a.watch:return | |
| time.sleep(config.get('interval_seconds',300)) | |
| if __name__=='__main__':main() | |