File size: 2,050 Bytes
6de2e12
 
 
 
 
 
 
 
 
 
 
 
 
 
2cdcdbd
6de2e12
2cdcdbd
 
 
 
 
 
6de2e12
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
"""Upload the completed runtime archive without blocking checkpoint backups."""
import json,pathlib,time
from datetime import datetime,timezone
from huggingface_hub import HfApi,CommitOperationAdd

root=pathlib.Path(__file__).resolve().parents[1];local=root/'.release_local'
config=json.loads((local/'hf-backup.json').read_text());token=pathlib.Path(config['token_file']).read_text().strip()
archive=local/'runtime-env.tar.gz';metadata=local/'runtime-archive.json';status=local/'hf-runtime-backup-status.json'
try:
    deadline=time.monotonic()+3600
    while not (archive.exists() and metadata.exists()):
        if time.monotonic()>deadline:raise TimeoutError('Runtime archive did not finish')
        time.sleep(10)
    status.write_text(json.dumps(dict(state='uploading',bytes=archive.stat().st_size,updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
    operations=[
        CommitOperationAdd(path_in_repo='environment/runtime-env.tar.gz',path_or_fileobj=str(archive)),
        CommitOperationAdd(path_in_repo='environment/runtime-archive.json',path_or_fileobj=str(metadata))]
    support=local/'runtime-support.tar.gz'
    if support.exists():
        operations.extend([CommitOperationAdd(path_in_repo='environment/runtime-support.tar.gz',path_or_fileobj=str(support)),CommitOperationAdd(path_in_repo='environment/runtime-support.json',path_or_fileobj=str(local/'runtime-support.json'))])
    commit=HfApi(token=token).create_commit(repo_id=config['repo_id'],repo_type='model',commit_message='Back up the exact runtime as inode-efficient archives',operations=operations)
    status.write_text(json.dumps(dict(state='uploaded',commit_oid=commit.oid,bytes=archive.stat().st_size,includes_python_and_renderer_support=support.exists(),updated_utc=datetime.now(timezone.utc).isoformat()),indent=2))
except Exception as error:
    status.write_text(json.dumps(dict(state='failed',error=(type(error).__name__+': '+str(error)).replace(token,'[redacted]'),updated_utc=datetime.now(timezone.utc).isoformat()),indent=2));raise SystemExit(1)