robottt-pi05-base-comparison / code /scripts /backup_legacy_assets.py
jisue's picture
Experiment backup 2026-11-11T06:43:55.654465+00:00
6e6c22f verified
Raw History Blame Contribute Delete
2.61 kB
"""Mirror verified FT ablation inputs once, excluding regenerable prefix tensors."""
import json
import os
from pathlib import Path
import shutil
from datetime import datetime, timezone
from huggingface_hub import HfApi
ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'
def main():
config = json.loads((LOCAL / 'hf-backup.json').read_text())
token = Path(config['token_file']).read_text().strip()
api = HfApi(token=token)
stage = LOCAL / 'hf-legacy-assets-staging'
status = LOCAL / 'hf-legacy-assets-status.json'
stage.mkdir(exist_ok=True)
def add(source, relative):
destination = stage / relative
destination.parent.mkdir(parents=True, exist_ok=True)
if destination.exists():
return
try:
os.link(source, destination)
except OSError:
shutil.copyfile(source, destination)
add(LOCAL / 'ft-base-reproduction/converted/pi05_libero_pytorch/model.safetensors', 'legacy-ft/assets/pi05_libero_pytorch/model.safetensors')
add(Path('/root/incoming/l4_learned_team_handoff_v1.tar'), 'legacy-ft/original-handoff.tar')
for name in ['asset-index.json', 'normalization.json', 'original-file-verification.json', 'protocol/selection.json']:
add(LOCAL / 'legacy-ft-data' / name, 'legacy-ft/data/' + name)
for source in (LOCAL / 'legacy-ft-data/processed').rglob('*'):
if source.is_file():
add(source, 'legacy-ft/data/processed/' + str(source.relative_to(LOCAL / 'legacy-ft-data/processed')))
add(LOCAL / 'legacy-ft-prefix/asset-index.json', 'legacy-ft/assets/prefix-index.json')
files = [p for p in stage.rglob('*') if p.is_file()]
total = sum(p.stat().st_size for p in files)
status.write_text(json.dumps(dict(state='uploading', bytes=total, files=len(files)), indent=2))
try:
commit = api.upload_folder(repo_id=config['repo_id'], folder_path=stage, repo_type='model',
commit_message='Mirror exact original L4 handoff, FT checkpoint and hash-matched processed inputs')
status.write_text(json.dumps(dict(state='uploaded', bytes=total, files=len(files), commit_oid=commit.oid,
updated_utc=datetime.now(timezone.utc).isoformat()), indent=2))
shutil.rmtree(stage)
print('LEGACY_ASSETS_BACKED_UP', commit.oid, flush=True)
except Exception as error:
status.write_text(json.dumps(dict(state='failed', error=str(error).replace(token, '[redacted]')), indent=2))
raise
if __name__ == '__main__':
main()