File size: 1,968 Bytes
daf1162 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | """One-time authorized public asset mirror using hardlinks, with no tensor copies."""
import json
import os
from pathlib import Path
from datetime import datetime,timezone
from huggingface_hub import HfApi
ROOT=Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local'
config=json.loads((LOCAL/'hf-backup.json').read_text())
token=Path(config['token_file']).read_text().strip()
stage=LOCAL/'hf-assets-staging';stage.mkdir(exist_ok=True)
status=LOCAL/'hf-assets-backup-status.json'
def link(path,target):
dest=stage/target;dest.parent.mkdir(parents=True,exist_ok=True)
if dest.exists():
if os.stat(dest).st_ino==os.stat(path).st_ino:return
dest.unlink()
os.link(path,dest)
try:
base=LOCAL/'assets/pi05_base_pytorch/model.safetensors'
link(base,'assets/pi05_base_pytorch/model.safetensors')
for folder,name in [(LOCAL/'assets/processed-libero','processed-libero'),(LOCAL/'assets/libero','libero-source')]:
for path in sorted(folder.rglob('*')):
if path.is_file() and path.suffix in ('.npy','.npz','.parquet','.json','.jsonl'):
link(path,'assets/'+name+'/'+str(path.relative_to(folder)))
total=sum(p.stat().st_size for p in stage.rglob('*') if p.is_file())
status.write_text(json.dumps(dict(state='uploading',bytes=total,updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
api=HfApi(token=token)
commit=api.upload_folder(repo_id=config['repo_id'],repo_type='model',folder_path=stage,commit_message='Mirror official-base derivative and exact processed LIBERO assets')
status.write_text(json.dumps(dict(state='uploaded',bytes=total,commit_oid=commit.oid,updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
except Exception as error:
status.write_text(json.dumps(dict(state='failed',error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]'),updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
raise SystemExit(1)
|