File size: 3,208 Bytes
7399b6f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | """Upload the code and the asset archives to a Hugging Face repo.
Requires a login first (this script will NOT prompt, so it is safe to run unattended):
hf auth login # or: huggingface-cli login
Then:
python scripts/push_to_hf.py --repo yqi19/mp_yam_code # dry run, lists what goes
python scripts/push_to_hf.py --repo yqi19/mp_yam_code --push # actually upload
Code and assets go up as separate commits so a failed 3 GB asset upload does not lose the code.
"""
import argparse
import os
import sys
ap = argparse.ArgumentParser()
ap.add_argument("--repo", default="yqi19/mp_yam_code")
ap.add_argument("--repo-type", default="model", choices=["model", "dataset"])
ap.add_argument("--stage", default="/home/yu/internship_yu/hf_stage")
ap.add_argument("--push", action="store_true", help="without this, only report what would go")
ap.add_argument("--skip-assets", action="store_true")
a = ap.parse_args()
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# what counts as "the code": everything that is ours, none of the multi-GB outputs
CODE_ALLOW = ["source/**", "scripts/**", "README.md", "pyproject.toml", "setup.py",
"outputs/frames/overview.png"]
CODE_IGNORE = ["**/__pycache__/**", "**/*.pyc", "outputs/tasks/**", "outputs/sheets/**",
"**/.git/**", "**/*.mp4"]
def human(n):
for u in ("B", "KB", "MB", "GB"):
if n < 1024 or u == "GB":
return f"{n:.1f} {u}"
n /= 1024
def main():
from huggingface_hub import HfApi
api = HfApi()
try:
who = api.whoami()["name"]
except Exception:
sys.exit("[hf] not logged in. Run: hf auth login (then re-run this script)")
print(f"[hf] authenticated as {who}, target {a.repo} ({a.repo_type})")
assets = []
if not a.skip_assets:
for fn in ("robotwin_usd.zip", "robotwin_source_meshes.zip"):
p = os.path.join(a.stage, fn)
if os.path.exists(p):
assets.append((p, f"assets/{fn}", os.path.getsize(p)))
else:
print(f"[hf] missing {p} -- skipping")
print(f"[hf] code: {REPO_ROOT} (allow={CODE_ALLOW})")
for _p, dest, size in assets:
print(f"[hf] asset: {dest} {human(size)}")
if not a.push:
print("\n[hf] DRY RUN -- nothing uploaded. Re-run with --push to upload.")
return 0
api.create_repo(a.repo, repo_type=a.repo_type, exist_ok=True)
print("[hf] uploading code ...")
api.upload_folder(folder_path=REPO_ROOT, repo_id=a.repo, repo_type=a.repo_type,
allow_patterns=CODE_ALLOW, ignore_patterns=CODE_IGNORE,
commit_message="YAM bimanual task suite: env, solvers, tasks, converters")
for p, dest, size in assets:
print(f"[hf] uploading {dest} ({human(size)}) -- this is slow, it is LFS ...")
api.upload_file(path_or_fileobj=p, path_in_repo=dest, repo_id=a.repo,
repo_type=a.repo_type,
commit_message=f"assets: {os.path.basename(p)}")
print(f"[hf] done -> https://huggingface.co/{a.repo}")
return 0
if __name__ == "__main__":
sys.exit(main())
|