| |
| """Mirror this repository's source files into the Hugging Face Space and org card. |
| |
| Design note, because the safety property here is the whole point: |
| |
| This script only ever UPLOADS files it finds in the TARGETS directories. |
| It never deletes anything on Hugging Face, and it refuses to upload into |
| any path owned by the weekly benchmark job. |
| |
| The Space holds two kinds of file. Source files -- index.html, the harness, |
| the README -- are authored here and this repo is their home. Generated files |
| -- results/, runs/, leaderboard*.csv -- are written on Hugging Face by the |
| weekly job and have no copy here at all. A sync that deleted whatever it did |
| not recognise would erase the run history the weekly job exists to accumulate, |
| and it would do it silently, at whatever hour the job happens to run. So the |
| sync is strictly additive, and PROTECTED below is a hard stop rather than a |
| convention: if a file ever appears in space/runs/ the sync aborts instead of |
| overwriting a week of measurements. |
| |
| Usage: |
| python tools/sync.py --dry-run # print the plan, touch nothing |
| python tools/sync.py # upload changed files (needs HF_TOKEN) |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import os |
| import sys |
| from pathlib import Path |
|
|
| REPO_ROOT = Path(__file__).resolve().parent.parent |
|
|
| |
| |
| TARGETS = [ |
| ("space", "Infergauge-load/llm-performance-and-load", "space"), |
| ("org-card", "Infergauge-load/README", "space"), |
| |
| |
| |
| ("archive-space", "Infergauge-load/llm-performance-and-load-weekly-runs", "space"), |
| ] |
|
|
| |
| |
| PROTECTED = ("results/", "runs/", "weeks/", "leaderboard", "thumbnail.png") |
|
|
| |
| IGNORE_NAMES = {".DS_Store", "__pycache__", ".gitkeep"} |
| IGNORE_SUFFIXES = (".pyc", ".swp", "~") |
|
|
|
|
| def git_blob_sha(data: bytes) -> str: |
| """The object id Hugging Face reports for a small file, so we can compare |
| local and remote without downloading anything.""" |
| return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() |
|
|
|
|
| def local_files(root: Path): |
| """Every file under root, as (path-in-repo, absolute path).""" |
| out = [] |
| for p in sorted(root.rglob("*")): |
| if not p.is_file(): |
| continue |
| if p.name in IGNORE_NAMES or p.name.endswith(IGNORE_SUFFIXES): |
| continue |
| if any(part in IGNORE_NAMES for part in p.parts): |
| continue |
| out.append((p.relative_to(root).as_posix(), p)) |
| return out |
|
|
|
|
| def check_protected(entries) -> list: |
| return [rel for rel, _ in entries if rel.startswith(PROTECTED)] |
|
|
|
|
| def remote_shas(api, repo_id: str, repo_type: str) -> dict: |
| """path -> object id for what is currently on Hugging Face.""" |
| from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError |
| try: |
| info = api.repo_info(repo_id=repo_id, repo_type=repo_type, files_metadata=True) |
| except (RepositoryNotFoundError, EntryNotFoundError): |
| return {} |
| return {s.rfilename: (s.blob_id or "") for s in (info.siblings or [])} |
|
|
|
|
| def main() -> int: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--dry-run", action="store_true", |
| help="print what would be uploaded and exit") |
| args = ap.parse_args() |
|
|
| plans = [] |
| for subdir, repo_id, repo_type in TARGETS: |
| root = REPO_ROOT / subdir |
| if not root.is_dir(): |
| print(f"!! {subdir}/ is missing", file=sys.stderr) |
| return 2 |
| entries = local_files(root) |
| if not entries: |
| print(f"!! {subdir}/ is empty -- refusing to run, this is how you " |
| f"accidentally blank a Space", file=sys.stderr) |
| return 2 |
| bad = check_protected(entries) |
| if bad: |
| print(f"!! {subdir}/ contains paths owned by the weekly job: " |
| f"{', '.join(bad)}", file=sys.stderr) |
| print(" Those files are written on Hugging Face and have no copy here. " |
| "Remove them from the repo; syncing them would overwrite " |
| "measurements.", file=sys.stderr) |
| return 2 |
| plans.append((subdir, repo_id, repo_type, entries)) |
|
|
| if args.dry_run: |
| for subdir, repo_id, repo_type, entries in plans: |
| print(f"\n{subdir}/ -> {repo_id} ({repo_type})") |
| for rel, path in entries: |
| data = path.read_bytes() |
| print(f" {rel:28} {len(data):>7} bytes {git_blob_sha(data)[:10]}") |
| print("\nDry run: nothing was uploaded. " |
| "Generated paths are never touched: " + ", ".join(PROTECTED)) |
| return 0 |
|
|
| token = os.environ.get("HF_TOKEN") |
| if not token: |
| print("!! HF_TOKEN is not set.", file=sys.stderr) |
| return 2 |
|
|
| from huggingface_hub import HfApi |
| api = HfApi(token=token) |
|
|
| total_up = total_same = 0 |
| for subdir, repo_id, repo_type, entries in plans: |
| remote = remote_shas(api, repo_id, repo_type) |
| changed = [] |
| for rel, path in entries: |
| data = path.read_bytes() |
| if remote.get(rel) == git_blob_sha(data): |
| total_same += 1 |
| continue |
| changed.append((rel, path)) |
|
|
| if not changed: |
| print(f"{subdir}/ -> {repo_id}: already in sync " |
| f"({len(entries)} files)") |
| continue |
|
|
| print(f"{subdir}/ -> {repo_id}: uploading {len(changed)} file(s)") |
| for rel, _ in changed: |
| print(f" {rel}") |
| api.upload_folder( |
| folder_path=str(REPO_ROOT / subdir), |
| repo_id=repo_id, |
| repo_type=repo_type, |
| allow_patterns=[rel for rel, _ in changed], |
| commit_message=_commit_message(len(changed)), |
| ) |
| total_up += len(changed) |
|
|
| print(f"\nDone. {total_up} uploaded, {total_same} already current. " |
| f"Nothing was deleted.") |
| return 0 |
|
|
|
|
| def _commit_message(n: int) -> str: |
| sha = os.environ.get("GITHUB_SHA", "")[:7] |
| where = f" (github {sha})" if sha else "" |
| return f"Sync {n} file(s) from source repo{where}" |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
