Download utils/hf_backup.py from matCercola18/DiffusionControl: direct link, hf CLI and curl.
- Browser
- Download file 3.65 kB
-
https://huggingface.co/matCercola18/DiffusionControl/resolve/main/utils/hf_backup.py
- Command line
-
hf download hf://matCercola18/DiffusionControl/utils/hf_backup.py
-
curl -L -o hf_backup.py https://huggingface.co/matCercola18/DiffusionControl/resolve/main/utils/hf_backup.py
3.65 kB
| """Back up every DiffusionControl file that git does not track (models, results, data, logs, figures) to the HF repo | |
| matCercola18/DiffusionControl, paths mirroring this directory. Uploads what is missing or whose content differs (sha256 for | |
| LFS files, git blob sha1 for small ones), in commits of <= 2 GB, then re-checks each commit. Skips envs, caches, the separate | |
| paper repo (paper_icml/), *.stale and *.pyc. HF keeps the old version of a replaced file in its history. | |
| usage: python utils/hf_backup.py [--dry] ["commit message"] (needs `hf auth login` once per machine)""" | |
| import os, sys, subprocess, hashlib | |
| from concurrent.futures import ThreadPoolExecutor | |
| from huggingface_hub import HfApi, CommitOperationAdd | |
| from huggingface_hub.hf_api import RepoFile | |
| REPO = "matCercola18/DiffusionControl" | |
| os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) | |
| SKIP_DIRS = {"myenv", ".cache", ".claude", "wandb", "__pycache__", ".git", "paper_icml"} | |
| BATCH = 2e9 | |
| args = [a for a in sys.argv[1:] if a != "--dry"] | |
| DRY, MSG = "--dry" in sys.argv, (args[0] if args else "Backup of files not tracked by git") | |
| api = HfApi() | |
| def chunks(f): | |
| with open(f, "rb") as fh: | |
| yield from iter(lambda: fh.read(1 << 22), b"") | |
| def same(f, p): # local file f == remote entry p | |
| if p is None or p.size != os.path.getsize(f): return False | |
| if p.lfs is not None: | |
| h = hashlib.sha256() | |
| for b in chunks(f): h.update(b) | |
| return h.hexdigest() == p.lfs.sha256 | |
| h = hashlib.sha1(b"blob %d\0" % p.size) | |
| for b in chunks(f): h.update(b) | |
| return h.hexdigest() == p.blob_id | |
| def progress(msg): | |
| print(msg, flush=True) | |
| subprocess.run(["runs", "set", "hf_backup", "running", msg[:100]], capture_output=True) | |
| tracked = set(subprocess.run(["git", "ls-files", "-z", "."], capture_output=True).stdout.decode().split("\0")) | |
| local = [] | |
| for dp, dn, fn in os.walk("."): | |
| dn[:] = [d for d in dn if d not in SKIP_DIRS] | |
| local += [p for p in (os.path.join(dp, f)[2:] for f in fn) if p not in tracked and not p.endswith((".stale", ".pyc")) and os.path.isfile(p)] | |
| local.sort() | |
| remote = {p.path: p for p in api.list_repo_tree(REPO, recursive=True) if isinstance(p, RepoFile)} | |
| with ThreadPoolExecutor(3) as ex: flags = list(ex.map(lambda f: same(f, remote.get(f)), local)) | |
| todo = [f for f, ok in zip(local, flags) if not ok] | |
| changed = [f for f in todo if f in remote] | |
| print(f"{len(local)} untracked local files, {len(todo)} to upload ({sum(map(os.path.getsize, todo)) / 1e9:.2f} GB), {len(changed)} of them replace a different remote file", flush=True) | |
| for f in changed[:20]: print(" changed:", f) | |
| if DRY or not todo: sys.exit(0) | |
| batches, cur, n = [], [], 0 | |
| for f in todo: | |
| s = os.path.getsize(f) | |
| if cur and n + s > BATCH: batches.append(cur); cur, n = [], 0 | |
| cur.append(f); n += s | |
| batches.append(cur) | |
| bad_all = [] | |
| for i, b in enumerate(batches, 1): | |
| progress(f"commit {i}/{len(batches)}: {len(b)} files, {sum(map(os.path.getsize, b)) / 1e9:.2f} GB") | |
| c = api.create_commit(REPO, operations=[CommitOperationAdd(path_in_repo=f, path_or_fileobj=f) for f in b], commit_message=f"{MSG} ({i}/{len(batches)})") | |
| info = {p.path: p for j in range(0, len(b), 200) for p in api.get_paths_info(REPO, b[j:j + 200], revision=c.oid)} | |
| with ThreadPoolExecutor(3) as ex: ok = list(ex.map(lambda f: same(f, info.get(f)), b)) | |
| bad = [f for f, k in zip(b, ok) if not k]; bad_all += bad | |
| progress(f"commit {c.oid[:8]} verified {len(b) - len(bad)}/{len(b)}" + (f"; bad: {bad[:5]}" if bad else "")) | |
| print("DONE, bad:", bad_all, flush=True) | |
| sys.exit(1 if bad_all else 0) | |