File size: 1,968 Bytes
daf1162
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
"""One-time authorized public asset mirror using hardlinks, with no tensor copies."""
import json
import os
from pathlib import Path
from datetime import datetime,timezone
from huggingface_hub import HfApi

ROOT=Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local'
config=json.loads((LOCAL/'hf-backup.json').read_text())
token=Path(config['token_file']).read_text().strip()
stage=LOCAL/'hf-assets-staging';stage.mkdir(exist_ok=True)
status=LOCAL/'hf-assets-backup-status.json'

def link(path,target):
    dest=stage/target;dest.parent.mkdir(parents=True,exist_ok=True)
    if dest.exists():
        if os.stat(dest).st_ino==os.stat(path).st_ino:return
        dest.unlink()
    os.link(path,dest)

try:
    base=LOCAL/'assets/pi05_base_pytorch/model.safetensors'
    link(base,'assets/pi05_base_pytorch/model.safetensors')
    for folder,name in [(LOCAL/'assets/processed-libero','processed-libero'),(LOCAL/'assets/libero','libero-source')]:
        for path in sorted(folder.rglob('*')):
            if path.is_file() and path.suffix in ('.npy','.npz','.parquet','.json','.jsonl'):
                link(path,'assets/'+name+'/'+str(path.relative_to(folder)))
    total=sum(p.stat().st_size for p in stage.rglob('*') if p.is_file())
    status.write_text(json.dumps(dict(state='uploading',bytes=total,updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
    api=HfApi(token=token)
    commit=api.upload_folder(repo_id=config['repo_id'],repo_type='model',folder_path=stage,commit_message='Mirror official-base derivative and exact processed LIBERO assets')
    status.write_text(json.dumps(dict(state='uploaded',bytes=total,commit_oid=commit.oid,updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
except Exception as error:
    status.write_text(json.dumps(dict(state='failed',error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]'),updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
    raise SystemExit(1)