NOEON / scripts /sync_to_hf.py
Frankenstein-Labs's picture
NOEON: official identity (Frankenstein-Labs) + premium model card; proprietary-weights structure
67aa1dc verified
Raw History Blame Contribute Delete
2.51 kB
#!/usr/bin/env python3
"""Sync ``hf_upload/`` to a Hugging Face model repo in a single commit.
Unlike a plain upload, this also *deletes* files that exist on the Hub but are no
longer part of the package (e.g. the old ``mage_vl``/``mage_flow`` vendor paths
after the capability-slot rename). Nothing is deleted unless it is genuinely
absent from the local package.
The token is read from ``HF_TOKEN`` at run time and never written to disk.
Usage:
HF_TOKEN=... python scripts/sync_to_hf.py --repo Frankenstein-Labs/NOEON
HF_TOKEN=... python scripts/sync_to_hf.py --repo ... --dry-run
"""
from __future__ import annotations
import argparse
import json
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
LOCAL = ROOT / "hf_upload"
def remote_files(repo: str) -> set[str]:
url = f"https://huggingface.co/api/models/{repo}"
data = json.load(urllib.request.urlopen(url))
return {s["rfilename"] for s in data.get("siblings", [])}
def local_files() -> set[str]:
return {str(p.relative_to(LOCAL)) for p in LOCAL.rglob("*") if p.is_file()}
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--repo", required=True)
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--message", default="NOEON: capability-slot rename + provenance")
args = ap.parse_args()
if not LOCAL.exists():
raise SystemExit("hf_upload/ not found; run scripts/build_hf_repo.py first")
remote = remote_files(args.repo)
local = local_files()
to_add = sorted(local - remote)
to_delete = sorted(remote - local)
print(f"remote={len(remote)} local={len(local)} "
f"add/update={len(to_add)} delete={len(to_delete)}")
for f in to_delete:
print(" delete:", f)
if args.dry_run:
return 0
import os
from huggingface_hub import HfApi
token = os.environ.get("HF_TOKEN")
if not token:
raise SystemExit("HF_TOKEN is not set")
api = HfApi(token=token)
api.upload_folder(
repo_id=args.repo,
repo_type="model",
folder_path=str(LOCAL),
commit_message=args.message,
delete_patterns=to_delete or None,
)
after = remote_files(args.repo)
missing = sorted(local - after)
print(f"\nremote after: {len(after)} files")
print("missing:", missing[:20])
print("OK" if not missing else "CHECK")
return 0 if not missing else 1
if __name__ == "__main__":
raise SystemExit(main())