DiffusionControl / utils /hf_backup.py
matCercola18's picture
Backup of untracked models, results, data and logs (2026-10-26) (6/6)
dfae7d7 verified
Raw History Blame Contribute Delete
3.65 kB
"""Back up every DiffusionControl file that git does not track (models, results, data, logs, figures) to the HF repo
matCercola18/DiffusionControl, paths mirroring this directory. Uploads what is missing or whose content differs (sha256 for
LFS files, git blob sha1 for small ones), in commits of <= 2 GB, then re-checks each commit. Skips envs, caches, the separate
paper repo (paper_icml/), *.stale and *.pyc. HF keeps the old version of a replaced file in its history.
usage: python utils/hf_backup.py [--dry] ["commit message"] (needs `hf auth login` once per machine)"""
import os, sys, subprocess, hashlib
from concurrent.futures import ThreadPoolExecutor
from huggingface_hub import HfApi, CommitOperationAdd
from huggingface_hub.hf_api import RepoFile
REPO = "matCercola18/DiffusionControl"
os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
SKIP_DIRS = {"myenv", ".cache", ".claude", "wandb", "__pycache__", ".git", "paper_icml"}
BATCH = 2e9
args = [a for a in sys.argv[1:] if a != "--dry"]
DRY, MSG = "--dry" in sys.argv, (args[0] if args else "Backup of files not tracked by git")
api = HfApi()
def chunks(f):
with open(f, "rb") as fh:
yield from iter(lambda: fh.read(1 << 22), b"")
def same(f, p): # local file f == remote entry p
if p is None or p.size != os.path.getsize(f): return False
if p.lfs is not None:
h = hashlib.sha256()
for b in chunks(f): h.update(b)
return h.hexdigest() == p.lfs.sha256
h = hashlib.sha1(b"blob %d\0" % p.size)
for b in chunks(f): h.update(b)
return h.hexdigest() == p.blob_id
def progress(msg):
print(msg, flush=True)
subprocess.run(["runs", "set", "hf_backup", "running", msg[:100]], capture_output=True)
tracked = set(subprocess.run(["git", "ls-files", "-z", "."], capture_output=True).stdout.decode().split("\0"))
local = []
for dp, dn, fn in os.walk("."):
dn[:] = [d for d in dn if d not in SKIP_DIRS]
local += [p for p in (os.path.join(dp, f)[2:] for f in fn) if p not in tracked and not p.endswith((".stale", ".pyc")) and os.path.isfile(p)]
local.sort()
remote = {p.path: p for p in api.list_repo_tree(REPO, recursive=True) if isinstance(p, RepoFile)}
with ThreadPoolExecutor(3) as ex: flags = list(ex.map(lambda f: same(f, remote.get(f)), local))
todo = [f for f, ok in zip(local, flags) if not ok]
changed = [f for f in todo if f in remote]
print(f"{len(local)} untracked local files, {len(todo)} to upload ({sum(map(os.path.getsize, todo)) / 1e9:.2f} GB), {len(changed)} of them replace a different remote file", flush=True)
for f in changed[:20]: print(" changed:", f)
if DRY or not todo: sys.exit(0)
batches, cur, n = [], [], 0
for f in todo:
s = os.path.getsize(f)
if cur and n + s > BATCH: batches.append(cur); cur, n = [], 0
cur.append(f); n += s
batches.append(cur)
bad_all = []
for i, b in enumerate(batches, 1):
progress(f"commit {i}/{len(batches)}: {len(b)} files, {sum(map(os.path.getsize, b)) / 1e9:.2f} GB")
c = api.create_commit(REPO, operations=[CommitOperationAdd(path_in_repo=f, path_or_fileobj=f) for f in b], commit_message=f"{MSG} ({i}/{len(batches)})")
info = {p.path: p for j in range(0, len(b), 200) for p in api.get_paths_info(REPO, b[j:j + 200], revision=c.oid)}
with ThreadPoolExecutor(3) as ex: ok = list(ex.map(lambda f: same(f, info.get(f)), b))
bad = [f for f, k in zip(b, ok) if not k]; bad_all += bad
progress(f"commit {c.oid[:8]} verified {len(b) - len(bad)}/{len(b)}" + (f"; bad: {bad[:5]}" if bad else ""))
print("DONE, bad:", bad_all, flush=True)
sys.exit(1 if bad_all else 0)