scripts/upload.py

"""Publish the explicit release inventory; never upload workspace caches or credentials."""
import argparse
import hashlib
import json
import os
from pathlib import Path
from scripts.common import write_json,sha256
from scripts.release import validate_manifest,validate_evaluation

def main():
    ap=argparse.ArgumentParser()
    ap.add_argument('platform',choices=('huggingface','modelscope'))
    ap.add_argument('--folder',type=Path)
    ap.add_argument('--dry-run',action='store_true')
    args=ap.parse_args(); folder=args.folder or Path('release')/args.platform
    owner='ixim' if args.platform=='huggingface' else 'iximbox'
    repo=f'{owner}/Image21-MLX-8bit'
    validate_evaluation(folder/'evaluation')
    rows=validate_manifest(folder); names=[x['path'] for x in rows]+['MANIFEST.json']
    remote_rows=rows+[dict(path='MANIFEST.json',size=(folder/'MANIFEST.json').stat().st_size,
                           sha256=sha256(folder/'MANIFEST.json'))]
    print(f'Validated {len(rows)} files: {repo}',flush=True)
    if args.dry_run: return
    if args.platform=='huggingface':
        from huggingface_hub import HfApi,get_token
        if not get_token(): raise RuntimeError('Run python -m scripts.login huggingface locally')
        api=HfApi()
        if api.whoami()['name']!=owner: raise ValueError('Wrong HF account')
        api.create_repo(repo_id=repo,repo_type='model',private=False,exist_ok=True)
        result=api.upload_folder(repo_id=repo,repo_type='model',folder_path=folder,allow_patterns=names,
                                 commit_message='Release Image21-MLX 8-bit with reproducible evaluation')
        info=api.model_info(repo,files_metadata=True)
        remote={x.rfilename:x for x in info.siblings}
        for row in remote_rows:
            if row['path'] not in remote or remote[row['path']].size!=row['size']:
                raise ValueError(f'Remote size mismatch: {row["path"]}')
            lfs=remote[row['path']].lfs
            if lfs is not None and lfs.sha256!=row['sha256']: raise ValueError('Remote LFS hash mismatch')
            if lfs is None:
                data=(folder/row['path']).read_bytes()
                blob=hashlib.sha1(f'blob {len(data)}\0'.encode()+data).hexdigest()
                if remote[row['path']].blob_id!=blob: raise ValueError('Remote Git blob hash mismatch')
        record=dict(repo_id=repo,url=f'https://huggingface.co/{repo}',revision=info.sha,
                    commit_url=str(result.commit_url),verified_files=len(remote_rows))
    else:
        from modelscope_hub import HubApi
        from modelscope_hub.errors import NotExistError
        api=HubApi()
        if api.whoami().username!=owner: raise ValueError('Wrong ModelScope account')
        # create only on an explicit not-found response; never hide authentication failures.
        try: api.get_repo(repo,repo_type='model')
        except NotExistError:
            api.create_repo(repo,repo_type='model',visibility='public',license='other',
                            description='Image21-MLX 8-bit. Built with Qwen. Research/evaluation only.')
        result=api.upload_folder(repo_id=repo,repo_type='model',folder_path=folder,allow_patterns=names,
                                 max_workers=4,commit_message='Release verified Image21-MLX 8-bit')
        remote={x.path:x for x in api.list_repo_files(repo,repo_type='model')}
        for row in remote_rows:
            item=remote.get(row['path'])
            if item is None or item.size!=row['size']: raise ValueError(f'Remote size mismatch: {row["path"]}')
            if item.sha256 and item.sha256!=row['sha256']: raise ValueError('Remote SHA256 mismatch')
        record=dict(repo_id=repo,url=f'https://modelscope.cn/models/{repo}',
                    verified_files=len(remote_rows),remote_verification='file sizes and available SHA256')
    write_json(Path('artifacts')/f'publication-{args.platform}.json',record)
    print(json.dumps(record,indent=2))

if __name__=='__main__': main()