Spaces:
Running
Running
File size: 14,814 Bytes
7103d1c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 | """Move data and artefacts between this repository and a private Hugging Face dataset repo (plan §0.6: the Mac and
Colab share one store).
python -m experiments.hub_sync push --bundles data,embeddings,cache,packs,runs
python -m experiments.hub_sync pull --bundles data,embeddings,cache,packs,runs
python -m experiments.hub_sync status --bundles data,embeddings,cache,packs,runs # read-only comparison
Bundles are defined in configs/base.yaml (`hub.bundles`): include/exclude patterns relative to the repository root
(fnmatch, `*` crosses folders), or `archive: <folder>` for a folder of many small files, which travels as one
tar.gz under archives/. Pull never replaces a local file that differs from the hub's copy and was modified after
the hub's last commit of it (it is kept and reported; push it). Push uploads only files whose content differs from the hub's copy (git blob sha1 for small
files, sha256 for LFS files), in commits of bounded size, and never deletes anything on the hub. runs/log.csv is
merged rather than overwritten (push_log / pull_log): the hub keeps the union of every machine's rows, and each
machine appends the rows it lacks. Credentials come from the logged-in Hugging Face client (HF_TOKEN or `hf auth
login`); none is read or printed here. Symlinked folders (a git worktree) are followed.
"""
from __future__ import annotations
import argparse
import csv
import fnmatch
import hashlib
import io
import os
import tarfile
import time
from pathlib import Path
from typing import Any
REPO = Path(__file__).resolve().parent.parent
LOG_IN_HUB = "logs/runs_log.csv"
MAX_COMMIT_BYTES = 512 * 1024 ** 2 # a failed upload (network timeout) loses at most one such chunk
MAX_COMMIT_FILES = 500
COMMIT_RETRIES = 4
RETRY_WAIT_S = 30.0
JUNK_FILES = {".DS_Store", "Thumbs.db"}
def git_blob_sha1_bytes(data: bytes) -> str:
return hashlib.sha1(b"blob %d\x00" % len(data) + data).hexdigest()
def _file_hashes(path: Path) -> tuple[str, str]:
"""(git blob sha1, sha256) of a file, streamed."""
size = path.stat().st_size
g, s = hashlib.sha1(b"blob %d\x00" % size), hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
g.update(chunk)
s.update(chunk)
return g.hexdigest(), s.hexdigest()
def _matches(rel: str, patterns: list[str]) -> bool:
return any(fnmatch.fnmatch(rel, p) or fnmatch.fnmatch(rel, p.rstrip("/*") + "/*") for p in patterns)
def select(root: Path, bundle: dict[str, Any]) -> list[str]:
"""Repository-relative paths (posix) of the bundle's files, following symlinked folders."""
include, exclude = bundle.get("include", []), bundle.get("exclude", [])
out = []
for dirpath, dirnames, filenames in os.walk(root, followlinks=True):
rel_dir = Path(dirpath).relative_to(root).as_posix()
dirnames[:] = [d for d in dirnames if d not in (".git", ".venv", "__pycache__")
and not _matches(f"{rel_dir}/{d}".lstrip("./"), exclude)]
for name in filenames:
if name in JUNK_FILES:
continue
rel = f"{rel_dir}/{name}".lstrip("./") if rel_dir != "." else name
if _matches(rel, include) and not _matches(rel, exclude):
out.append(rel)
return sorted(out)
def _remote(api: Any, repo_id: str) -> dict[str, Any]:
try:
return {e.path: e for e in api.list_repo_tree(repo_id, repo_type="dataset", recursive=True, expand=True)
if hasattr(e, "size")}
except Exception: # an empty repo has no tree yet
return {}
def _same(path: Path, r: Any) -> bool:
"""A local file has the content of the hub's entry (git blob sha1 for small files, sha256 for LFS files)."""
if r.size != path.stat().st_size:
return False
g, s = _file_hashes(path)
return getattr(r, "blob_id", None) == g or bool(getattr(r, "lfs", None) and r.lfs.sha256 == s)
def _literal(rel: str) -> str:
"""An fnmatch pattern matching exactly this path (snapshot_download filters with fnmatch)."""
return "".join(f"[{c}]" if c in "*?[" else c for c in rel)
def newer_local(root: Path, remote: dict[str, Any], bundle: dict[str, Any]) -> list[str]:
"""Bundle files on the hub whose local copy differs and was modified after the hub's last commit of the file: a
pull must not replace them (2026-10-20: a push during a running job put a partial file on the hub, and a later
pull replaced the finished local file with it). Push them instead. An older local copy is replaced as before."""
include, exclude = bundle.get("include", []), bundle.get("exclude", [])
keep = []
for rel, r in sorted(remote.items()):
p = root / rel
if not (_matches(rel, include) and not _matches(rel, exclude)) or not p.is_file():
continue
date = getattr(getattr(r, "last_commit", None), "date", None)
if date is not None and p.stat().st_mtime > date.timestamp() and not _same(p, r):
keep.append(rel)
return keep
def _archive(folder: Path) -> bytes:
import gzip
buf = io.BytesIO()
# gzip writes the current time into its header; mtime=0 keeps the archive byte-identical for identical files
with gzip.GzipFile(fileobj=buf, mode="wb", compresslevel=6, mtime=0) as gz, tarfile.open(fileobj=gz, mode="w") as tar:
for p in sorted(folder.rglob("*")):
if p.is_file():
info = tar.gettarinfo(str(p), arcname=p.relative_to(folder).as_posix())
info.mtime, info.uid, info.gid, info.uname, info.gname = 0, 0, 0, "", "" # same bytes for the same files
with open(p, "rb") as f:
tar.addfile(info, f)
return buf.getvalue()
def _plan(api: Any, repo_id: str, root: Path, bundles: dict[str, Any], names: list[str]
) -> tuple[list[tuple[str, Any, int, bool]], int]:
"""Files whose content differs from the hub's copy: [(path in repo, local source, size, missing on hub)], and the
number of files that already match. Archives are staged under .cache/hub_sync so the hub's chunked upload works."""
remote = _remote(api, repo_id)
todo: list[tuple[str, Any, int, bool]] = []
matched = 0
for name in names:
b = bundles[name]
if "archive" in b:
folder = root / b["archive"]
if not folder.is_dir():
continue
data = _archive(folder)
path = f"archives/{Path(b['archive']).name}.tar.gz"
r = remote.get(path)
if r is None or getattr(r, "blob_id", None) != git_blob_sha1_bytes(data) and not (
getattr(r, "lfs", None) and r.lfs.sha256 == hashlib.sha256(data).hexdigest()):
staged = root / ".cache" / "hub_sync" / path # a file, so the hub's chunked upload can be used
staged.parent.mkdir(parents=True, exist_ok=True)
staged.write_bytes(data)
todo.append((path, str(staged), len(data), r is None))
else:
matched += 1
continue
for rel in select(root, b):
p = (root / rel).resolve()
r = remote.get(rel)
if r is not None and _same(p, r):
matched += 1
continue
todo.append((rel, str(p), p.stat().st_size, r is None))
return todo, matched
def status(api: Any, repo_id: str, root: Path, bundles: dict[str, Any], names: list[str]) -> dict[str, Any]:
"""Read-only: which local bundle files differ from (or are missing on) the hub. Nothing is uploaded."""
todo, matched = _plan(api, repo_id, root, bundles, names)
return {"matched": matched, "differs": len(todo), "missing_on_hub": sum(m for *_, m in todo),
"bytes": sum(t[2] for t in todo), "files": [t[0] for t in todo]}
def push(api: Any, repo_id: str, root: Path, bundles: dict[str, Any], names: list[str]) -> dict[str, Any]:
from huggingface_hub import CommitOperationAdd
api.create_repo(repo_id, repo_type="dataset", private=True, exist_ok=True)
todo, _ = _plan(api, repo_id, root, bundles, names)
ops = [CommitOperationAdd(path_in_repo=rel, path_or_fileobj=src) for rel, src, _, _ in todo]
sizes = [size for _, _, size, _ in todo]
print(f"[hub_sync] plan: {len(ops)} files, {sum(sizes) / 1e9:.2f} GB to upload", flush=True)
commits, start = 0, 0
while start < len(ops): # bounded commits: a failed upload loses one chunk, not everything
end, total = start, 0
while end < len(ops) and end - start < MAX_COMMIT_FILES and (total + sizes[end] <= MAX_COMMIT_BYTES or end == start):
total += sizes[end]
end += 1
for attempt in range(COMMIT_RETRIES + 1): # network timeouts happen on long uploads: retry the chunk
try:
api.create_commit(repo_id=repo_id, repo_type="dataset", operations=ops[start:end],
commit_message=f"sync {','.join(names)}: {end - start} files")
break
except Exception as e:
if attempt == COMMIT_RETRIES:
raise
wait = RETRY_WAIT_S * 2 ** attempt
print(f"[hub_sync] commit of {end - start} files failed ({type(e).__name__}); retry {attempt + 1}/"
f"{COMMIT_RETRIES} in {wait:.0f}s", flush=True)
time.sleep(wait)
commits += 1
print(f"[hub_sync] committed {end}/{len(ops)} files", flush=True)
start = end
return {"uploaded": len(ops), "bytes": sum(sizes), "commits": commits}
def _read_log(text: str) -> tuple[list[str], list[dict[str, str]]]:
r = csv.DictReader(io.StringIO(text))
return list(r.fieldnames or []), list(r)
def _key(row: dict[str, str]) -> tuple[str, str]:
return row.get("timestamp", ""), row.get("run_id", "")
def push_log(api: Any, repo_id: str, root: Path) -> int:
"""Union of the hub's run log and this machine's runs/log.csv, written back to the hub; returns rows added."""
from huggingface_hub import CommitOperationAdd
local_path = root / "runs" / "log.csv"
if not local_path.is_file():
return 0
fields, local = _read_log(local_path.read_text(encoding="utf-8"))
hub_fields, hub = fields, []
try:
hub_fields, hub = _read_log(Path(api.hf_hub_download(repo_id, LOG_IN_HUB, repo_type="dataset",
local_dir=root / ".cache" / "hub_sync")).read_text())
except Exception:
pass
have = {_key(r) for r in hub}
new = [r for r in local if _key(r) not in have]
if not new and hub:
return 0
rows = sorted(hub + new, key=_key)
buf = io.StringIO()
w = csv.DictWriter(buf, fieldnames=hub_fields or fields)
w.writeheader()
w.writerows(rows)
api.create_commit(repo_id=repo_id, repo_type="dataset", commit_message=f"run log: +{len(new)} rows",
operations=[CommitOperationAdd(path_in_repo=LOG_IN_HUB,
path_or_fileobj=io.BytesIO(buf.getvalue().encode()))])
return len(new)
def pull_log(api: Any, repo_id: str, root: Path) -> int:
"""Append to runs/log.csv the hub's rows this machine lacks (never rewrites existing rows); returns rows added."""
try:
path = Path(api.hf_hub_download(repo_id, LOG_IN_HUB, repo_type="dataset", local_dir=root / ".cache" / "hub_sync"))
except Exception:
return 0
_, hub = _read_log(path.read_text(encoding="utf-8"))
local_path = root / "runs" / "log.csv"
fields, local = _read_log(local_path.read_text(encoding="utf-8"))
have = {_key(r) for r in local}
new = [r for r in hub if _key(r) not in have]
if new:
with open(local_path, "a", newline="", encoding="utf-8") as f:
csv.DictWriter(f, fieldnames=fields, extrasaction="ignore").writerows(new)
return len(new)
def _snapshot(repo_id: str, root: Path, include: list[str], exclude: list[str]) -> None:
from huggingface_hub import snapshot_download
snapshot_download(repo_id, repo_type="dataset", local_dir=root, allow_patterns=include,
ignore_patterns=exclude or None)
def pull(api: Any, repo_id: str, root: Path, bundles: dict[str, Any], names: list[str]) -> dict[str, Any]:
"""Download the bundles. A local file newer than the hub's differing copy is kept and reported (newer_local)."""
pulled, kept = [], []
remote = None
for name in names:
b = bundles[name]
if "archive" in b:
path = f"archives/{Path(b['archive']).name}.tar.gz"
local = Path(api.hf_hub_download(repo_id, path, repo_type="dataset", local_dir=root / ".cache" / "hub_sync"))
dest = root / b["archive"]
dest.mkdir(parents=True, exist_ok=True)
with tarfile.open(local, mode="r:gz") as tar:
tar.extractall(dest, filter="data") # the "data" filter refuses absolute paths and links out of dest
pulled.append(name)
continue
remote = _remote(api, repo_id) if remote is None else remote
keep = newer_local(root, remote, b)
kept += keep
_snapshot(repo_id, root, list(b.get("include", [])), list(b.get("exclude", [])) + [_literal(k) for k in keep])
pulled.append(name)
if kept:
print(f"[hub_sync] kept {len(kept)} local files newer than the hub's copy (push them): {kept[:20]}", flush=True)
return {"pulled": pulled, "kept_local": kept}
def main() -> None:
import yaml
from huggingface_hub import HfApi
ap = argparse.ArgumentParser()
ap.add_argument("action", choices=["push", "pull", "status"])
ap.add_argument("--bundles", default="data,embeddings,cache,packs,runs")
ap.add_argument("--config", default="configs/base.yaml")
args = ap.parse_args()
hub = yaml.safe_load((REPO / args.config).read_text(encoding="utf-8"))["hub"]
names = [n.strip() for n in args.bundles.split(",") if n.strip()]
api = HfApi()
if args.action == "status":
s = status(api, hub["repo"], REPO, hub["bundles"], names)
print({k: v for k, v in s.items() if k != "files"}, flush=True)
for f in s["files"][:50]:
print(" differs:", f)
elif args.action == "push":
print(push(api, hub["repo"], REPO, hub["bundles"], names), flush=True)
if "runs" in names:
print({"log_rows_added_to_hub": push_log(api, hub["repo"], REPO)})
else:
print(pull(api, hub["repo"], REPO, hub["bundles"], names), flush=True)
if "runs" in names:
print({"log_rows_added_here": pull_log(api, hub["repo"], REPO)})
if __name__ == "__main__":
main()
|