File size: 2,606 Bytes
6e6c22f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
"""Mirror verified FT ablation inputs once, excluding regenerable prefix tensors."""
import json
import os
from pathlib import Path
import shutil
from datetime import datetime, timezone
from huggingface_hub import HfApi

ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'


def main():
    config = json.loads((LOCAL / 'hf-backup.json').read_text())
    token = Path(config['token_file']).read_text().strip()
    api = HfApi(token=token)
    stage = LOCAL / 'hf-legacy-assets-staging'
    status = LOCAL / 'hf-legacy-assets-status.json'
    stage.mkdir(exist_ok=True)
    def add(source, relative):
        destination = stage / relative
        destination.parent.mkdir(parents=True, exist_ok=True)
        if destination.exists():
            return
        try:
            os.link(source, destination)
        except OSError:
            shutil.copyfile(source, destination)
    add(LOCAL / 'ft-base-reproduction/converted/pi05_libero_pytorch/model.safetensors', 'legacy-ft/assets/pi05_libero_pytorch/model.safetensors')
    add(Path('/root/incoming/l4_learned_team_handoff_v1.tar'), 'legacy-ft/original-handoff.tar')
    for name in ['asset-index.json', 'normalization.json', 'original-file-verification.json', 'protocol/selection.json']:
        add(LOCAL / 'legacy-ft-data' / name, 'legacy-ft/data/' + name)
    for source in (LOCAL / 'legacy-ft-data/processed').rglob('*'):
        if source.is_file():
            add(source, 'legacy-ft/data/processed/' + str(source.relative_to(LOCAL / 'legacy-ft-data/processed')))
    add(LOCAL / 'legacy-ft-prefix/asset-index.json', 'legacy-ft/assets/prefix-index.json')
    files = [p for p in stage.rglob('*') if p.is_file()]
    total = sum(p.stat().st_size for p in files)
    status.write_text(json.dumps(dict(state='uploading', bytes=total, files=len(files)), indent=2))
    try:
        commit = api.upload_folder(repo_id=config['repo_id'], folder_path=stage, repo_type='model',
                                   commit_message='Mirror exact original L4 handoff, FT checkpoint and hash-matched processed inputs')
        status.write_text(json.dumps(dict(state='uploaded', bytes=total, files=len(files), commit_oid=commit.oid,
                                         updated_utc=datetime.now(timezone.utc).isoformat()), indent=2))
        shutil.rmtree(stage)
        print('LEGACY_ASSETS_BACKED_UP', commit.oid, flush=True)
    except Exception as error:
        status.write_text(json.dumps(dict(state='failed', error=str(error).replace(token, '[redacted]')), indent=2))
        raise


if __name__ == '__main__':
    main()