File size: 2,250 Bytes
a10f8a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Hugging Face Hub sync for code pull and checkpoint push (Kaggle-friendly)."""
import os
import subprocess
from pathlib import Path
from typing import Optional
def clone_or_pull_repo(
repo_id: str,
local_dir: str,
token: Optional[str] = None,
revision: Optional[str] = None,
) -> str:
"""
Clone HF repo via git if missing; otherwise git pull.
Requires `git` and `git-lfs` on the machine (Kaggle has both).
"""
token = token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
local = Path(local_dir)
local.parent.mkdir(parents=True, exist_ok=True)
if local.is_dir() and (local / ".git").is_dir():
subprocess.run(["git", "-C", str(local), "pull"], check=False)
return str(local)
clone_url = f"https://huggingface.co/{repo_id}"
if token:
clone_url = f"https://user:{token}@huggingface.co/{repo_id}"
cmd = ["git", "clone", clone_url, str(local)]
if revision:
cmd.extend(["--branch", revision])
subprocess.run(cmd, check=True)
return str(local)
def snapshot_download_repo(repo_id: str, local_dir: str, token: Optional[str] = None) -> str:
"""Download repo snapshot without git (good for read-only code pull)."""
from huggingface_hub import snapshot_download
token = token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
path = snapshot_download(
repo_id=repo_id,
local_dir=local_dir,
token=token,
local_dir_use_symlinks=False,
)
return path
def push_folder_to_hub(
folder_path: str,
repo_id: str,
path_in_repo: str = "runs",
token: Optional[str] = None,
commit_message: str = "Upload training artifacts",
) -> str:
"""Upload a local folder (checkpoints, metrics) to an HF model/dataset repo."""
from huggingface_hub import HfApi
token = token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
api = HfApi(token=token)
api.upload_folder(
folder_path=folder_path,
path_in_repo=path_in_repo,
repo_id=repo_id,
repo_type="model",
commit_message=commit_message,
)
return f"https://huggingface.co/{repo_id}/tree/main/{path_in_repo}"
|