File size: 12,016 Bytes
0f5d220
3f8ab35
0f5d220
87d5fb6
 
 
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3f8ab35
 
 
 
 
 
 
 
 
 
 
0f5d220
 
 
 
 
 
add1d5f
0f5d220
4c004d7
0f5d220
70b9640
0f5d220
f599749
 
18ff84d
07a447a
70b9640
 
 
07a447a
498f5ae
 
 
 
 
 
18ff84d
 
 
70b9640
0f5d220
331e8f5
0f5d220
 
 
 
 
07a447a
0f5d220
6e6c22f
0f5d220
 
 
 
 
6e6c22f
3f8ab35
 
 
 
0f5d220
 
6e6c22f
add1d5f
 
 
 
ebaa80e
add1d5f
 
 
0f5d220
 
 
70b9640
0f5d220
daf1162
 
 
0f5d220
 
 
 
 
 
 
 
 
add1d5f
0f5d220
 
 
 
 
daf1162
 
 
0f5d220
331e8f5
 
0f5d220
 
 
 
 
 
 
 
 
 
 
 
ebaa80e
0f5d220
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
"""Back up explicitly selected experiment artifacts to an authorized private HF repo."""
import argparse,fcntl,hashlib,json,os,pathlib,pickletools,shutil,time,zipfile
from datetime import datetime,timezone
os.environ['HF_HOME']='/tmp/robottt-hf-cache'
os.environ['HF_XET_CACHE']='/tmp/robottt-hf-cache/xet'
os.environ.setdefault('HF_XET_CHUNK_CACHE_SIZE_BYTES','0')
from huggingface_hub import HfApi

ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local'

def checkpoint_step(path):
    with zipfile.ZipFile(path) as archive:
        name=next(n for n in archive.namelist() if n.endswith('/data.pkl'))
        if archive.getinfo(name).file_size>64*1024*1024:raise ValueError('Unexpected checkpoint metadata size')
        expect=False
        for opcode,arg,_ in pickletools.genops(archive.read(name)):
            if expect:
                if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue
                if opcode.name in ('BININT','BININT1','BININT2','LONG1','LONG4','INT') and type(arg) is int:return arg
                expect=False
            if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='step':expect=True
    return None

def checkpoint_config_hash(path):
    with zipfile.ZipFile(path) as archive:
        name=next(n for n in archive.namelist() if n.endswith('/data.pkl'));expect=False
        for opcode,arg,_ in pickletools.genops(archive.read(name)):
            if expect:
                if opcode.name in ('BINPUT','LONG_BINPUT','MEMOIZE','PUT'):continue
                if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and isinstance(arg,str) and len(arg)==64:return arg
                raise ValueError('Unexpected checkpoint config provenance encoding')
            if opcode.name in ('BINUNICODE','SHORT_BINUNICODE','UNICODE') and arg=='run_config':expect=True
    raise ValueError('Checkpoint config provenance missing')

def snapshot(stage):
    files=[]
    def copy(source,target):
        source=pathlib.Path(source)
        if not source.is_file():return
        dest=stage/target;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,dest);files.append(target)
    for folder in ('src','scripts','tests'):
        for path in sorted((ROOT/folder).rglob('*.py')):copy(path,'code/'+str(path.relative_to(ROOT)))
    for pattern in ('configs/*.json','configs/environment-linux-cu126.txt','provenance/base_comparison_assets.json','provenance/base-tensor-equality.json','provenance/shared-base-verification.json','provenance/base-counterpart-download.json','provenance/opentau_reference_core.json','provenance/persistence-layout.json','docs/base_comparison.md','HANDOFF.md','pyproject.toml','provenance/pre-restart-remote-verification.json','provenance/pre-restart-backup.json'):
        for path in ROOT.glob(pattern):copy(path,str(path.relative_to(ROOT)))
    for name in ('comparison-status.json','common-artifacts.json','libero-eval-plan.json','comparison-results.json','monitor-status.json','monitor-history.jsonl','hf-assets-backup-status.json','hf-runtime-backup-status.json','runtime-archive.json','runtime-support.json','automation-plan.json','completion-watch-status.json','completion-watch-state.json','completion-watch-events.jsonl','legacy-l18-status.json','legacy-l18-results.json','legacy-l18-notification.json','hf-legacy-assets-status.json','legacy-libero-status.json','legacy-comparison-final.json','legacy-libero-plan.json','legacy-libero-notification.json'):
        copy(LOCAL/name,'progress/'+name)
    for path in LOCAL.glob('libero-plus-*.json'):
        copy(path,'progress/'+path.name)
    copy(pathlib.Path('/tmp/robottt-comparison/prefix-cache/status.json'),'progress/prefix-status.json')
    for name in ('comparison-lite.json','comparison-reference.json'):copy(LOCAL/name,'progress/run-configs/'+name)
    for path in LOCAL.glob('legacy-libero-*.json'):
        if path.name not in ('legacy-libero-status.json','legacy-libero-plan.json','legacy-libero-notification.json'):
            copy(path,'progress/run-configs/'+path.name)
    copy(LOCAL/'ondemand-prefix/asset-index.json','preprocessing/ondemand-asset-index.json')
    contracts=sorted((LOCAL/'prefix-contracts').glob('*.json'))
    if contracts:
        target=stage/'preprocessing/prefix-contracts.jsonl';target.parent.mkdir(parents=True,exist_ok=True)
        with target.open('w') as output:
            for path in contracts:output.write(json.dumps(json.loads(path.read_text()))+'\n')
        files.append('preprocessing/prefix-contracts.jsonl')
    for path in (ROOT/'licenses').glob('*'):
        if path.is_file():copy(path,'licenses/'+path.name)
    copy(ROOT/'NOTICE.md','NOTICE.md')
    for name in ('data-contract.json','runtime-profile-parity.json','simulator-smoke.json','cuda-smoke.json','unittest.log','environment-final-freeze.txt','ondemand-unittest.log','recovery-unittest.log','recovery-integration.json','recovery-replayed-loss-parity.json','simulator-timing.json','query-render-parity.json','runtime-restore-integration.json','runtime-support-restore.json','legacy-l18-runtime-validation.json','legacy-live-preprocessing-validation.json'):
        copy(LOCAL/'audit'/name,'validation/'+name)
    for path in (LOCAL/'audit').glob('query-render-parity-task*.json'):copy(path,'validation/'+path.name)
    copy(LOCAL/'assets/processed-libero/normalization.json','preprocessing/normalization.json')
    copy(LOCAL/'assets/paligemma_tokenizer.model','preprocessing/paligemma_tokenizer.model')
    checkpoints={};runs=LOCAL/'runs'
    for run in sorted(runs.iterdir()) if runs.exists() else []:
        if not run.is_dir():continue
        for name in ('status.json','model_audit.json','completed.json','recovery-transaction.json'):
            copy(run/name,'runs/'+run.name+'/'+name)
        if not run.name.startswith(('comparison-','legacy-ft-')):continue
        path=run/'checkpoints/latest.pt'
        if path.is_file():
            relative='runs/'+run.name+'/checkpoints/latest.pt';dest=stage/relative;dest.parent.mkdir(parents=True,exist_ok=True)
            if dest.exists():dest.unlink()
            os.link(path,dest)
            if run.name.startswith(('comparison-','legacy-ft-')):
                source_config=LOCAL/(run.name+'.json')
                if hashlib.sha256(source_config.read_bytes()).hexdigest()!=checkpoint_config_hash(dest):raise ValueError('Concurrent config/checkpoint migration; retry backup')
                copy(source_config,'progress/run-configs/'+run.name+'.json')
                copy(run/'ledger/events.jsonl','runs/'+run.name+'/ledger/events.jsonl')
            checkpoints[run.name]=dict(step=checkpoint_step(dest),bytes=dest.stat().st_size,path=relative)
            files.append(relative)
    for name in ('protocol-comparison-lite','protocol-comparison-reference','protocol-legacy-ft-l18'):
        folder=LOCAL/name
        if folder.exists():
            for path in folder.iterdir():
                if path.suffix not in ('.json','.pt'):continue
                if name=='protocol-comparison-reference' and path.name=='bank.pt':continue
                target=stage/'protocols'/name/path.name;target.parent.mkdir(parents=True,exist_ok=True)
                if target.exists():target.unlink()
                os.link(path,target);files.append(str(target.relative_to(stage)))
    evaluations=LOCAL/'evaluations'
    if evaluations.exists():
        for path in evaluations.rglob('*.json'):copy(path,'evaluations/'+str(path.relative_to(evaluations)))
        for path in evaluations.glob('legacy-ft-*/*/sequence.pt'):copy(path,'evaluations/'+str(path.relative_to(evaluations)))
    now=datetime.now(timezone.utc).isoformat()
    assets_status=LOCAL/'hf-assets-backup-status.json'
    assets_uploaded=assets_status.exists() and json.loads(assets_status.read_text()).get('state')=='uploaded'
    result=dict(updated_utc=now,checkpoints=checkpoints,base_weights_uploaded=assets_uploaded,base_model_sha256='df9bac8b5c5a7ca023fc2773daf60993843379148bc626f5b10ad9d5bb0da47b',files=files)
    (stage/'backup-manifest.json').write_text(json.dumps(result,indent=2))
    (stage/'README.md').write_text('''---
tags:
- robotics
- pi05
- test-time-training
---
# RoboTTT: common official-base comparison

Public experiment backup, as authorized by the repository owner. Progress is updated approximately every five minutes while the backup worker runs.

See `progress/comparison-status.json` and `backup-manifest.json` for current status and durable checkpoint steps. `smoke-*` runs are two-update integration checks, **not** completed 3000-update experiments. Main runs are named `comparison-lite` and `comparison-reference`.

Both conditions use the same converted official pi05_base, Train episodes 0..59 and Dev 60..69, T32/stride1/S8, and saved schedule/noise. The comparison is between a four-layer lightweight TTT design (561,540 added trainable parameters) and the pinned OpenTau core on all 18 layers (85,319,824). This is a new common pipeline, not a reproduction of the historical full OpenTau trainer.

Extension/optimizer checkpoints, seeded protocols, preprocessing contracts, code and progress are backed up every five minutes. The converted base and exact source/processed data are mirrored separately; see `progress/hf-assets-backup-status.json`. Exact base provenance and hashes are in `provenance/base_comparison_assets.json`.

VLM prefixes are generated on demand with a bounded 2 GiB RAM cache. No large prefix tensor files are written to ephemeral storage. Sequence tensor-hash contracts are shared and checked between both methods.

The exact Python environment is stored as `environment/runtime-env.tar.gz` with a checksum manifest. Its runtime directory is disposable and restored from this archive after a restart, avoiding tens of thousands of persistent small files. Check `progress/hf-runtime-backup-status.json` for upload completion. Resume using `python3 scripts/resume_comparison.py`; saved optimizer/RNG state is preserved and unsaved interrupted work is archived and rolled back explicitly.

Flow-matching loss and simulator success are separate metrics; neither should be inferred from a smoke checkpoint. Final metrics appear in `progress/comparison-results.json` only after paired evaluations complete.
''')
    return result

def main():
    parser=argparse.ArgumentParser();parser.add_argument('--config',default=str(LOCAL/'hf-backup.json'));parser.add_argument('--watch',action='store_true');a=parser.parse_args()
    config=json.loads(pathlib.Path(a.config).read_text());token=pathlib.Path(config['token_file']).read_text().strip();api=HfApi(token=token)
    lock=(LOCAL/'hf-backup.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
    stage=LOCAL/'hf-backup-staging';stage.mkdir(exist_ok=True)
    while True:
        try:
            manifest=snapshot(stage)
            commit=api.upload_folder(repo_id=config['repo_id'],repo_type='model',folder_path=stage,commit_message='Experiment backup '+manifest['updated_utc'],ignore_patterns=['preprocessing/prefix-contracts/**','progress/prefix-status.json','runs/smoke-*/checkpoints/*.pt','protocols/protocol-comparison-reference/bank.pt'])
            state=dict(state='uploaded',repo_id=config['repo_id'],updated_utc=manifest['updated_utc'],commit_oid=commit.oid if hasattr(commit,'oid') else commit.commit_oid,checkpoints=manifest['checkpoints'])
            (LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True)
        except Exception as error:
            state=dict(state='upload_failed',updated_utc=datetime.now(timezone.utc).isoformat(),error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]'))
            (LOCAL/'hf-backup-status.json').write_text(json.dumps(state,indent=2));print(json.dumps(state),flush=True)
            if not a.watch:raise SystemExit(1)
        if not a.watch:return
        time.sleep(config.get('interval_seconds',300))

if __name__=='__main__':main()