File size: 2,606 Bytes
6e6c22f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | """Mirror verified FT ablation inputs once, excluding regenerable prefix tensors."""
import json
import os
from pathlib import Path
import shutil
from datetime import datetime, timezone
from huggingface_hub import HfApi
ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'
def main():
config = json.loads((LOCAL / 'hf-backup.json').read_text())
token = Path(config['token_file']).read_text().strip()
api = HfApi(token=token)
stage = LOCAL / 'hf-legacy-assets-staging'
status = LOCAL / 'hf-legacy-assets-status.json'
stage.mkdir(exist_ok=True)
def add(source, relative):
destination = stage / relative
destination.parent.mkdir(parents=True, exist_ok=True)
if destination.exists():
return
try:
os.link(source, destination)
except OSError:
shutil.copyfile(source, destination)
add(LOCAL / 'ft-base-reproduction/converted/pi05_libero_pytorch/model.safetensors', 'legacy-ft/assets/pi05_libero_pytorch/model.safetensors')
add(Path('/root/incoming/l4_learned_team_handoff_v1.tar'), 'legacy-ft/original-handoff.tar')
for name in ['asset-index.json', 'normalization.json', 'original-file-verification.json', 'protocol/selection.json']:
add(LOCAL / 'legacy-ft-data' / name, 'legacy-ft/data/' + name)
for source in (LOCAL / 'legacy-ft-data/processed').rglob('*'):
if source.is_file():
add(source, 'legacy-ft/data/processed/' + str(source.relative_to(LOCAL / 'legacy-ft-data/processed')))
add(LOCAL / 'legacy-ft-prefix/asset-index.json', 'legacy-ft/assets/prefix-index.json')
files = [p for p in stage.rglob('*') if p.is_file()]
total = sum(p.stat().st_size for p in files)
status.write_text(json.dumps(dict(state='uploading', bytes=total, files=len(files)), indent=2))
try:
commit = api.upload_folder(repo_id=config['repo_id'], folder_path=stage, repo_type='model',
commit_message='Mirror exact original L4 handoff, FT checkpoint and hash-matched processed inputs')
status.write_text(json.dumps(dict(state='uploaded', bytes=total, files=len(files), commit_oid=commit.oid,
updated_utc=datetime.now(timezone.utc).isoformat()), indent=2))
shutil.rmtree(stage)
print('LEGACY_ASSETS_BACKED_UP', commit.oid, flush=True)
except Exception as error:
status.write_text(json.dumps(dict(state='failed', error=str(error).replace(token, '[redacted]')), indent=2))
raise
if __name__ == '__main__':
main()
|