File size: 12,016 Bytes
0f5d220 3f8ab35 0f5d220 87d5fb6 0f5d220 3f8ab35 0f5d220 add1d5f 0f5d220 4c004d7 0f5d220 70b9640 0f5d220 f599749 18ff84d 07a447a 70b9640 07a447a 498f5ae 18ff84d 70b9640 0f5d220 331e8f5 0f5d220 07a447a 0f5d220 6e6c22f 0f5d220 6e6c22f 3f8ab35 0f5d220 6e6c22f add1d5f ebaa80e add1d5f 0f5d220 70b9640 0f5d220 daf1162 0f5d220 add1d5f 0f5d220 daf1162 0f5d220 331e8f5 0f5d220 ebaa80e 0f5d220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | """Back up explicitly selected experiment artifacts to an authorized private HF repo."""
import argparse,fcntl,hashlib,json,os,pathlib,pickletools,shutil,time,zipfile
from datetime import datetime,timezone
os.environ['HF_HOME']='/tmp/robottt-hf-cache'
os.environ['HF_XET_CACHE']='/tmp/robottt-hf-cache/xet'
os.environ.setdefault('HF_XET_CHUNK_CACHE_SIZE_BYTES','0')
from huggingface_hub import HfApi
ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local'
def checkpoint_step(path):
with zipfile.ZipFile(path) as archive:
name=next(n for n in archive.namelist() if n.endswith('/data.pkl'))
if archive.getinfo(name).file_size>64*1024*1024:raise ValueError('Unexpected checkpoint metadata size')
expect=False
for opcode,arg,_ in pickletools.genops(archive.read(name)):
if expect:
if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue
if opcode.name in ('BININT','BININT1','BININT2','LONG1','LONG4','INT') and type(arg) is int:return arg
expect=False
if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='step':expect=True
return None
def checkpoint_config_hash(path):
with zipfile.ZipFile(path) as archive:
name=next(n for n in archive.namelist() if n.endswith('/data.pkl'));expect=False
for opcode,arg,_ in pickletools.genops(archive.read(name)):
if expect:
if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue
if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and isinstance(arg,str) and len(arg)==64:return arg
raise ValueError('Unexpected checkpoint config provenance encoding')
if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='run_config':expect=True
raise ValueError('Checkpoint config provenance missing')
def snapshot(stage):
files=[]
def copy(source,target):
source=pathlib.Path(source)
if not source.is_file():return
dest=stage/target;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,dest);files.append(target)
for folder in ('src','scripts','tests'):
for path in sorted((ROOT/folder).rglob('*.py')):copy(path,'code/'+str(path.relative_to(ROOT)))
for pattern in ('configs/*.json','configs/environment-linux-cu126.txt','provenance/base_comparison_assets.json','provenance/base-tensor-equality.json','provenance/shared-base-verification.json','provenance/base-counterpart-download.json','provenance/opentau_reference_core.json','provenance/persistence-layout.json','docs/base_comparison.md','HANDOFF.md','pyproject.toml','provenance/pre-restart-remote-verification.json','provenance/pre-restart-backup.json'):
for path in ROOT.glob(pattern):copy(path,str(path.relative_to(ROOT)))
for name in ('comparison-status.json','common-artifacts.json','libero-eval-plan.json','comparison-results.json','monitor-status.json','monitor-history.jsonl','hf-assets-backup-status.json','hf-runtime-backup-status.json','runtime-archive.json','runtime-support.json','automation-plan.json','completion-watch-status.json','completion-watch-state.json','completion-watch-events.jsonl','legacy-l18-status.json','legacy-l18-results.json','legacy-l18-notification.json','hf-legacy-assets-status.json','legacy-libero-status.json','legacy-comparison-final.json','legacy-libero-plan.json','legacy-libero-notification.json'):
copy(LOCAL/name,'progress/'+name)
for path in LOCAL.glob('libero-plus-*.json'):
copy(path,'progress/'+path.name)
copy(pathlib.Path('/tmp/robottt-comparison/prefix-cache/status.json'),'progress/prefix-status.json')
for name in ('comparison-lite.json','comparison-reference.json'):copy(LOCAL/name,'progress/run-configs/'+name)
for path in LOCAL.glob('legacy-libero-*.json'):
if path.name not in ('legacy-libero-status.json','legacy-libero-plan.json','legacy-libero-notification.json'):
copy(path,'progress/run-configs/'+path.name)
copy(LOCAL/'ondemand-prefix/asset-index.json','preprocessing/ondemand-asset-index.json')
contracts=sorted((LOCAL/'prefix-contracts').glob('*.json'))
if contracts:
target=stage/'preprocessing/prefix-contracts.jsonl';target.parent.mkdir(parents=True,exist_ok=True)
with target.open('w') as output:
for path in contracts:output.write(json.dumps(json.loads(path.read_text()))+'\n')
files.append('preprocessing/prefix-contracts.jsonl')
for path in (ROOT/'licenses').glob('*'):
if path.is_file():copy(path,'licenses/'+path.name)
copy(ROOT/'NOTICE.md','NOTICE.md')
for name in ('data-contract.json','runtime-profile-parity.json','simulator-smoke.json','cuda-smoke.json','unittest.log','environment-final-freeze.txt','ondemand-unittest.log','recovery-unittest.log','recovery-integration.json','recovery-replayed-loss-parity.json','simulator-timing.json','query-render-parity.json','runtime-restore-integration.json','runtime-support-restore.json','legacy-l18-runtime-validation.json','legacy-live-preprocessing-validation.json'):
copy(LOCAL/'audit'/name,'validation/'+name)
for path in (LOCAL/'audit').glob('query-render-parity-task*.json'):copy(path,'validation/'+path.name)
copy(LOCAL/'assets/processed-libero/normalization.json','preprocessing/normalization.json')
copy(LOCAL/'assets/paligemma_tokenizer.model','preprocessing/paligemma_tokenizer.model')
checkpoints={};runs=LOCAL/'runs'
for run in sorted(runs.iterdir()) if runs.exists() else []:
if not run.is_dir():continue
for name in ('status.json','model_audit.json','completed.json','recovery-transaction.json'):
copy(run/name,'runs/'+run.name+'/'+name)
if not run.name.startswith(('comparison-','legacy-ft-')):continue
path=run/'checkpoints/latest.pt'
if path.is_file():
relative='runs/'+run.name+'/checkpoints/latest.pt';dest=stage/relative;dest.parent.mkdir(parents=True,exist_ok=True)
if dest.exists():dest.unlink()
os.link(path,dest)
if run.name.startswith(('comparison-','legacy-ft-')):
source_config=LOCAL/(run.name+'.json')
if hashlib.sha256(source_config.read_bytes()).hexdigest()!=checkpoint_config_hash(dest):raise ValueError('Concurrent config/checkpoint migration; retry backup')
copy(source_config,'progress/run-configs/'+run.name+'.json')
copy(run/'ledger/events.jsonl','runs/'+run.name+'/ledger/events.jsonl')
checkpoints[run.name]=dict(step=checkpoint_step(dest),bytes=dest.stat().st_size,path=relative)
files.append(relative)
for name in ('protocol-comparison-lite','protocol-comparison-reference','protocol-legacy-ft-l18'):
folder=LOCAL/name
if folder.exists():
for path in folder.iterdir():
if path.suffix not in ('.json','.pt'):continue
if name=='protocol-comparison-reference' and path.name=='bank.pt':continue
target=stage/'protocols'/name/path.name;target.parent.mkdir(parents=True,exist_ok=True)
if target.exists():target.unlink()
os.link(path,target);files.append(str(target.relative_to(stage)))
evaluations=LOCAL/'evaluations'
if evaluations.exists():
for path in evaluations.rglob('*.json'):copy(path,'evaluations/'+str(path.relative_to(evaluations)))
for path in evaluations.glob('legacy-ft-*/*/sequence.pt'):copy(path,'evaluations/'+str(path.relative_to(evaluations)))
now=datetime.now(timezone.utc).isoformat()
assets_status=LOCAL/'hf-assets-backup-status.json'
assets_uploaded=assets_status.exists() and json.loads(assets_status.read_text()).get('state')=='uploaded'
result=dict(updated_utc=now,checkpoints=checkpoints,base_weights_uploaded=assets_uploaded,base_model_sha256='df9bac8b5c5a7ca023fc2773daf60993843379148bc626f5b10ad9d5bb0da47b',files=files)
(stage/'backup-manifest.json').write_text(json.dumps(result,indent=2))
(stage/'README.md').write_text('''---
tags:
- robotics
- pi05
- test-time-training
---
# RoboTTT: common official-base comparison
Public experiment backup, as authorized by the repository owner. Progress is updated approximately every five minutes while the backup worker runs.
See `progress/comparison-status.json` and `backup-manifest.json` for current status and durable checkpoint steps. `smoke-*` runs are two-update integration checks, **not** completed 3000-update experiments. Main runs are named `comparison-lite` and `comparison-reference`.
Both conditions use the same converted official pi05_base, Train episodes 0..59 and Dev 60..69, T32/stride1/S8, and saved schedule/noise. The comparison is between a four-layer lightweight TTT design (561,540 added trainable parameters) and the pinned OpenTau core on all 18 layers (85,319,824). This is a new common pipeline, not a reproduction of the historical full OpenTau trainer.
Extension/optimizer checkpoints, seeded protocols, preprocessing contracts, code and progress are backed up every five minutes. The converted base and exact source/processed data are mirrored separately; see `progress/hf-assets-backup-status.json`. Exact base provenance and hashes are in `provenance/base_comparison_assets.json`.
VLM prefixes are generated on demand with a bounded 2 GiB RAM cache. No large prefix tensor files are written to ephemeral storage. Sequence tensor-hash contracts are shared and checked between both methods.
The exact Python environment is stored as `environment/runtime-env.tar.gz` with a checksum manifest. Its runtime directory is disposable and restored from this archive after a restart, avoiding tens of thousands of persistent small files. Check `progress/hf-runtime-backup-status.json` for upload completion. Resume using `python3 scripts/resume_comparison.py`; saved optimizer/RNG state is preserved and unsaved interrupted work is archived and rolled back explicitly.
Flow-matching loss and simulator success are separate metrics; neither should be inferred from a smoke checkpoint. Final metrics appear in `progress/comparison-results.json` only after paired evaluations complete.
''')
return result
def main():
parser=argparse.ArgumentParser();parser.add_argument('--config',default=str(LOCAL/'hf-backup.json'));parser.add_argument('--watch',action='store_true');a=parser.parse_args()
config=json.loads(pathlib.Path(a.config).read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token)
lock=(LOCAL/'hf-backup.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
stage=LOCAL/'hf-backup-staging';stage.mkdir(exist_ok=True)
while True:
try:
manifest=snapshot(stage)
commit=api.upload_folder(repo_id=config['repo_id'],repo_type='model',folder_path=stage,commit_message='Experiment backup '+manifest['updated_utc'],ignore_patterns=['preprocessing/prefix-contracts/**','progress/prefix-status.json','runs/smoke-*/checkpoints/*.pt','protocols/protocol-comparison-reference/bank.pt'])
state=dict(state='uploaded',repo_id=config['repo_id'],updated_utc=manifest['updated_utc'],commit_oid=commit.oid if hasattr(commit,'oid') else commit.commit_oid,checkpoints=manifest['checkpoints'])
(LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True)
except Exception as error:
state=dict(state='upload_failed',updated_utc=datetime.now(timezone.utc).isoformat(),error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]'))
(LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True)
if not a.watch:raise SystemExit(1)
if not a.watch:return
time.sleep(config.get('interval_seconds',300))
if __name__=='__main__':main()
|