RealArDor/NanoJev-SimuNRGTrans / source /scripts /upload_unified_private_release.py
RealArDor's picture
download
raw
9.34 kB
#!/usr/bin/env python3
"""Upload and verify the explicitly prepared private unified development release.
Credentials are read into memory from an explicit .env path, never printed or
stored by this script. Only the two declared development repositories can be
created or updated. Public repositories are observed solely as unchanged guards.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import time
os.environ['HF_HUB_DISABLE_PROGRESS_BARS'] = '1'
os.environ['HF_HUB_DISABLE_TELEMETRY'] = '1'
os.environ['HF_HUB_VERBOSITY'] = 'error'
os.environ['RUST_LOG'] = 'error'
from dotenv import dotenv_values
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.errors import RepositoryNotFoundError
REPOSITORIES = {'model': 'C-Tianyu/NanoJev-dev', 'dataset': 'C-Tianyu/NanoJev-Data-dev'}
PUBLIC = {'model': 'C-Tianyu/NanoJev', 'dataset': 'C-Tianyu/NanoJev-Data'}
TAG = 'unified-games-v1'
def sha(path):
with Path(path).open('rb') as handle:
return hashlib.file_digest(handle, 'sha256').hexdigest()
def blob_sha(path):
digest = hashlib.sha1(b'blob ' + str(path.stat().st_size).encode() + b'\0')
with path.open('rb') as handle:
for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b''): digest.update(chunk)
return digest.hexdigest()
def save(path, data):
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + '.pending')
temporary.write_text(json.dumps(data, indent=2) + '\n')
os.replace(temporary, path)
def log(**data):
print(json.dumps(data), flush=True)
def upload_one(api, token, kind, folder, receipts):
rid = REPOSITORIES[kind]
manifest = json.loads((folder / 'SHA256_MANIFEST.json').read_text())
assert manifest['kind'] == kind and manifest['private'] is True
rows = manifest['files'] + [{'path': 'SHA256_MANIFEST.json', 'bytes': (folder / 'SHA256_MANIFEST.json').stat().st_size,
'sha256': sha(folder / 'SHA256_MANIFEST.json')}]
names = [row['path'] for row in rows]
assert len(names) == len(set(names)) and 'README.md' in names
for row in rows:
relative = Path(row['path']); path = folder / relative
assert not relative.is_absolute() and '..' not in relative.parts and path.is_file() and not path.is_symlink()
assert sha(path) == row['sha256'] and path.stat().st_size == row['bytes'], row['path']
try:
before = api.repo_info(rid, repo_type=kind, files_metadata=True)
assert before.private is True, 'Only private development repositories may be updated'
except RepositoryNotFoundError:
api.create_repo(rid, repo_type=kind, private=True, exist_ok=False)
before = api.repo_info(rid, repo_type=kind, files_metadata=True)
assert before.private is True
log(phase='upload_started', repo=rid, private=True, files=len(rows), bytes=sum(r['bytes'] for r in rows))
commit = api.upload_folder(repo_id=rid, repo_type=kind, folder_path=folder,
allow_patterns=names, ignore_patterns=['.cache/**'], parent_commit=before.sha,
commit_message='Add complete unified game model development release' if kind == 'model' else
'Add complete mixed game supervision and verified recordings')
after = api.repo_info(rid, repo_type=kind, revision=commit.oid, files_metadata=True)
assert after.private is True
remote = {item.rfilename: item for item in after.siblings}
verified = []
for row in rows:
item = remote[row['path']]
assert item.size == row['bytes'], row['path']
if item.lfs is not None:
assert item.lfs.sha256 == row['sha256'], row['path']
method = 'remote_lfs_sha256'
else:
assert item.blob_id == blob_sha(folder / row['path']), row['path']
method = 'remote_git_blob_identity_of_local_sha256_verified_bytes'
verified.append({**row, 'remote_verification': method})
for item in before.siblings:
if item.rfilename not in names:
assert item.rfilename in remote and remote[item.rfilename].blob_id == item.blob_id
samples = ['README.md', 'SHA256_MANIFEST.json', 'TRAINING_RECIPE.md']
samples += ['config.json', 'backbone_config/config.json', 'training_initialization/config.json'] if kind == 'model' else [
'DATA_STATISTICS.json', 'unified/hard/manifest.json', 'unified/soft/manifest.json',
'configs/hard_navigation_demo_v1_cases.jsonl']
expected = {row['path']: row for row in rows}
downloads = []
for name in samples:
path = hf_hub_download(rid, filename=name, repo_type=kind, revision=commit.oid, token=token,
cache_dir=receipts / 'readback_cache')
assert sha(path) == expected[name]['sha256'], name
downloads.append({'path': name, 'sha256': sha(path)})
refs = api.list_repo_refs(rid, repo_type=kind)
existing_tag = next((item for item in refs.tags if item.name == TAG), None)
if existing_tag is None:
api.create_tag(rid, repo_type=kind, tag=TAG, revision=commit.oid,
tag_message='Verified private unified game development release')
else:
assert existing_tag.target_commit == commit.oid, 'An existing immutable release tag cannot be moved'
tagged = api.repo_info(rid, repo_type=kind, revision=TAG)
assert tagged.sha == commit.oid and tagged.private is True
result = {'repository': rid, 'repo_type': kind, 'url': 'https://huggingface.co/' + ('datasets/' if kind == 'dataset' else '') + rid,
'revision': commit.oid, 'tag': TAG, 'private': True, 'previous_revision': before.sha,
'files': verified, 'uploaded_files': len(rows), 'uploaded_bytes': sum(r['bytes'] for r in rows),
'all_uploaded_hashes_verified': True, 'previous_unrelated_files_preserved': True,
'authenticated_readback': downloads, 'full_weights_redownloaded': False,
'package_manifest_sha256': sha(folder / 'SHA256_MANIFEST.json')}
save(receipts / (kind + '_upload_receipt.json'), result)
log(phase='verified', repo=rid, revision=commit.oid, private=True, files=len(rows), tag=TAG)
return result
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--env', type=Path, required=True)
parser.add_argument('--packages', type=Path, default=Path('runs/hf_unified_release_v1/packages_final'))
parser.add_argument('--inventory', type=Path, default=Path('runs/hf_unified_release_v1/account_inventory.json'))
parser.add_argument('--receipts', type=Path, default=Path('runs/hf_unified_release_v1'))
parser.add_argument('--output', type=Path, default=Path('results/huggingface_unified_dev_release.json'))
args = parser.parse_args(argv)
values = dotenv_values(args.env)
tokens = {value for value in values.values() if isinstance(value, str) and value.startswith('hf_')}
assert len(tokens) == 1, 'Expected exactly one Hugging Face credential in the supplied .env'
token = tokens.pop()
api = HfApi(token=token)
assert api.whoami()['name'] == 'C-Tianyu'
inventory = json.loads(args.inventory.read_text())
guards = {item['repo_type']: item for item in inventory['repositories'] if item['repo_id'] in PUBLIC.values()}
assert set(guards) == set(PUBLIC)
for kind, rid in PUBLIC.items():
current = api.repo_info(rid, repo_type=kind)
assert current.sha == guards[kind]['revision'] and current.private is False
started = time.monotonic()
results = {}
with ThreadPoolExecutor(max_workers=2) as pool:
futures = {pool.submit(upload_one, api, token, kind, args.packages / kind, args.receipts): kind for kind in REPOSITORIES}
for future in as_completed(futures): results[futures[future]] = future.result()
for kind, rid in PUBLIC.items():
current = api.repo_info(rid, repo_type=kind)
assert current.sha == guards[kind]['revision'] and current.private is False
result = {'schema': 'nanojev-unified-private-hf-release-v1', 'created_at': datetime.now(timezone.utc).isoformat(),
'passed': True, 'account': 'C-Tianyu', 'phase': 'private_development_review',
'public_repositories_unchanged': {kind: {'repository': item['repo_id'], 'revision': item['revision']} for kind, item in guards.items()},
'selected_checkpoint_sha256': 'f68c47d66998231b86b7e91b4ed5e82ae23acf104c8b7cd6d165c3ac7b7ffe1b',
'selected_checkpoint_step': 400, 'initial_checkpoint_included': True,
'model': results['model'], 'dataset': results['dataset'], 'elapsed_seconds': time.monotonic() - started}
save(args.output, result)
log(phase='release_complete', output=str(args.output), model_revision=results['model']['revision'],
dataset_revision=results['dataset']['revision'], private=True)
if __name__ == '__main__':
try:
main()
except Exception as error:
# Exception text and HTTP payloads may include signed URLs: report only safe metadata.
log(phase='failed', error_type=type(error).__name__,
http_status=getattr(getattr(error, 'response', None), 'status_code', None))
raise SystemExit(1)

Xet Storage Details

Size:
9.34 kB
·
Xet hash:
b01ccd673550093895b286c365dd887472b4d97690bff7daa5ad3b21c853551d

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.