File size: 4,004 Bytes
4f03424 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | """Publish the explicit release inventory; never upload workspace caches or credentials."""
import argparse
import hashlib
import json
import os
from pathlib import Path
from scripts.common import write_json,sha256
from scripts.release import validate_manifest,validate_evaluation
def main():
ap=argparse.ArgumentParser()
ap.add_argument('platform',choices=('huggingface','modelscope'))
ap.add_argument('--folder',type=Path)
ap.add_argument('--dry-run',action='store_true')
args=ap.parse_args(); folder=args.folder or Path('release')/args.platform
owner='ixim' if args.platform=='huggingface' else 'iximbox'
repo=f'{owner}/Image21-MLX-8bit'
validate_evaluation(folder/'evaluation')
rows=validate_manifest(folder); names=[x['path'] for x in rows]+['MANIFEST.json']
remote_rows=rows+[dict(path='MANIFEST.json',size=(folder/'MANIFEST.json').stat().st_size,
sha256=sha256(folder/'MANIFEST.json'))]
print(f'Validated {len(rows)} files: {repo}',flush=True)
if args.dry_run: return
if args.platform=='huggingface':
from huggingface_hub import HfApi,get_token
if not get_token(): raise RuntimeError('Run python -m scripts.login huggingface locally')
api=HfApi()
if api.whoami()['name']!=owner: raise ValueError('Wrong HF account')
api.create_repo(repo_id=repo,repo_type='model',private=False,exist_ok=True)
result=api.upload_folder(repo_id=repo,repo_type='model',folder_path=folder,allow_patterns=names,
commit_message='Release Image21-MLX 8-bit with reproducible evaluation')
info=api.model_info(repo,files_metadata=True)
remote={x.rfilename:x for x in info.siblings}
for row in remote_rows:
if row['path'] not in remote or remote[row['path']].size!=row['size']:
raise ValueError(f'Remote size mismatch: {row["path"]}')
lfs=remote[row['path']].lfs
if lfs is not None and lfs.sha256!=row['sha256']: raise ValueError('Remote LFS hash mismatch')
if lfs is None:
data=(folder/row['path']).read_bytes()
blob=hashlib.sha1(f'blob {len(data)}\0'.encode()+data).hexdigest()
if remote[row['path']].blob_id!=blob: raise ValueError('Remote Git blob hash mismatch')
record=dict(repo_id=repo,url=f'https://huggingface.co/{repo}',revision=info.sha,
commit_url=str(result.commit_url),verified_files=len(remote_rows))
else:
from modelscope_hub import HubApi
from modelscope_hub.errors import NotExistError
api=HubApi()
if api.whoami().username!=owner: raise ValueError('Wrong ModelScope account')
# create only on an explicit not-found response; never hide authentication failures.
try: api.get_repo(repo,repo_type='model')
except NotExistError:
api.create_repo(repo,repo_type='model',visibility='public',license='other',
description='Image21-MLX 8-bit. Built with Qwen. Research/evaluation only.')
result=api.upload_folder(repo_id=repo,repo_type='model',folder_path=folder,allow_patterns=names,
max_workers=4,commit_message='Release verified Image21-MLX 8-bit')
remote={x.path:x for x in api.list_repo_files(repo,repo_type='model')}
for row in remote_rows:
item=remote.get(row['path'])
if item is None or item.size!=row['size']: raise ValueError(f'Remote size mismatch: {row["path"]}')
if item.sha256 and item.sha256!=row['sha256']: raise ValueError('Remote SHA256 mismatch')
record=dict(repo_id=repo,url=f'https://modelscope.cn/models/{repo}',
verified_files=len(remote_rows),remote_verification='file sizes and available SHA256')
write_json(Path('artifacts')/f'publication-{args.platform}.json',record)
print(json.dumps(record,indent=2))
if __name__=='__main__': main()
|