#!/usr/bin/env python3 """Sync ``hf_upload/`` to a Hugging Face model repo in a single commit. Unlike a plain upload, this also *deletes* files that exist on the Hub but are no longer part of the package (e.g. the old ``mage_vl``/``mage_flow`` vendor paths after the capability-slot rename). Nothing is deleted unless it is genuinely absent from the local package. The token is read from ``HF_TOKEN`` at run time and never written to disk. Usage: HF_TOKEN=... python scripts/sync_to_hf.py --repo Frankenstein-Labs/NOEON HF_TOKEN=... python scripts/sync_to_hf.py --repo ... --dry-run """ from __future__ import annotations import argparse import json import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parent.parent LOCAL = ROOT / "hf_upload" def remote_files(repo: str) -> set[str]: url = f"https://huggingface.co/api/models/{repo}" data = json.load(urllib.request.urlopen(url)) return {s["rfilename"] for s in data.get("siblings", [])} def local_files() -> set[str]: return {str(p.relative_to(LOCAL)) for p in LOCAL.rglob("*") if p.is_file()} def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--repo", required=True) ap.add_argument("--dry-run", action="store_true") ap.add_argument("--message", default="NOEON: capability-slot rename + provenance") args = ap.parse_args() if not LOCAL.exists(): raise SystemExit("hf_upload/ not found; run scripts/build_hf_repo.py first") remote = remote_files(args.repo) local = local_files() to_add = sorted(local - remote) to_delete = sorted(remote - local) print(f"remote={len(remote)} local={len(local)} " f"add/update={len(to_add)} delete={len(to_delete)}") for f in to_delete: print(" delete:", f) if args.dry_run: return 0 import os from huggingface_hub import HfApi token = os.environ.get("HF_TOKEN") if not token: raise SystemExit("HF_TOKEN is not set") api = HfApi(token=token) api.upload_folder( repo_id=args.repo, repo_type="model", folder_path=str(LOCAL), commit_message=args.message, delete_patterns=to_delete or None, ) after = remote_files(args.repo) missing = sorted(local - after) print(f"\nremote after: {len(after)} files") print("missing:", missing[:20]) print("OK" if not missing else "CHECK") return 0 if not missing else 1 if __name__ == "__main__": raise SystemExit(main())