"""Back up every DiffusionControl file that git does not track (models, results, data, logs, figures) to the HF repo matCercola18/DiffusionControl, paths mirroring this directory. Uploads what is missing or whose content differs (sha256 for LFS files, git blob sha1 for small ones), in commits of <= 2 GB, then re-checks each commit. Skips envs, caches, the separate paper repo (paper_icml/), *.stale and *.pyc. HF keeps the old version of a replaced file in its history. usage: python utils/hf_backup.py [--dry] ["commit message"] (needs `hf auth login` once per machine)""" import os, sys, subprocess, hashlib from concurrent.futures import ThreadPoolExecutor from huggingface_hub import HfApi, CommitOperationAdd from huggingface_hub.hf_api import RepoFile REPO = "matCercola18/DiffusionControl" os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) SKIP_DIRS = {"myenv", ".cache", ".claude", "wandb", "__pycache__", ".git", "paper_icml"} BATCH = 2e9 args = [a for a in sys.argv[1:] if a != "--dry"] DRY, MSG = "--dry" in sys.argv, (args[0] if args else "Backup of files not tracked by git") api = HfApi() def chunks(f): with open(f, "rb") as fh: yield from iter(lambda: fh.read(1 << 22), b"") def same(f, p): # local file f == remote entry p if p is None or p.size != os.path.getsize(f): return False if p.lfs is not None: h = hashlib.sha256() for b in chunks(f): h.update(b) return h.hexdigest() == p.lfs.sha256 h = hashlib.sha1(b"blob %d\0" % p.size) for b in chunks(f): h.update(b) return h.hexdigest() == p.blob_id def progress(msg): print(msg, flush=True) subprocess.run(["runs", "set", "hf_backup", "running", msg[:100]], capture_output=True) tracked = set(subprocess.run(["git", "ls-files", "-z", "."], capture_output=True).stdout.decode().split("\0")) local = [] for dp, dn, fn in os.walk("."): dn[:] = [d for d in dn if d not in SKIP_DIRS] local += [p for p in (os.path.join(dp, f)[2:] for f in fn) if p not in tracked and not p.endswith((".stale", ".pyc")) and os.path.isfile(p)] local.sort() remote = {p.path: p for p in api.list_repo_tree(REPO, recursive=True) if isinstance(p, RepoFile)} with ThreadPoolExecutor(3) as ex: flags = list(ex.map(lambda f: same(f, remote.get(f)), local)) todo = [f for f, ok in zip(local, flags) if not ok] changed = [f for f in todo if f in remote] print(f"{len(local)} untracked local files, {len(todo)} to upload ({sum(map(os.path.getsize, todo)) / 1e9:.2f} GB), {len(changed)} of them replace a different remote file", flush=True) for f in changed[:20]: print(" changed:", f) if DRY or not todo: sys.exit(0) batches, cur, n = [], [], 0 for f in todo: s = os.path.getsize(f) if cur and n + s > BATCH: batches.append(cur); cur, n = [], 0 cur.append(f); n += s batches.append(cur) bad_all = [] for i, b in enumerate(batches, 1): progress(f"commit {i}/{len(batches)}: {len(b)} files, {sum(map(os.path.getsize, b)) / 1e9:.2f} GB") c = api.create_commit(REPO, operations=[CommitOperationAdd(path_in_repo=f, path_or_fileobj=f) for f in b], commit_message=f"{MSG} ({i}/{len(batches)})") info = {p.path: p for j in range(0, len(b), 200) for p in api.get_paths_info(REPO, b[j:j + 200], revision=c.oid)} with ThreadPoolExecutor(3) as ex: ok = list(ex.map(lambda f: same(f, info.get(f)), b)) bad = [f for f, k in zip(b, ok) if not k]; bad_all += bad progress(f"commit {c.oid[:8]} verified {len(b) - len(bad)}/{len(b)}" + (f"; bad: {bad[:5]}" if bad else "")) print("DONE, bad:", bad_all, flush=True) sys.exit(1 if bad_all else 0)