#!/usr/bin/env python3 """Download the original public checkpoint at one immutable Hugging Face revision.""" import concurrent.futures import hashlib import json import pathlib import time import urllib.request ROOT = pathlib.Path(__file__).resolve().parents[1] REPO = "google/embeddinggemma-2" REVISION = "f9972ac22e696a105c482202646979e33e4b5a1e" url = f"https://huggingface.co/api/models/{REPO}/revision/{REVISION}?blobs=true" with urllib.request.urlopen(url, timeout=120) as response: metadata = json.load(response) assert metadata["sha"] == REVISION (ROOT / "logs/source-api.json").write_text(json.dumps(metadata, indent=2) + "\n") def fetch(entry): name = entry["rfilename"] path = ROOT / "source" / name path.parent.mkdir(parents=True, exist_ok=True) expected = entry.get("lfs", {}).get("sha256") for attempt in range(4): try: digest = hashlib.sha256() with urllib.request.urlopen(f"https://huggingface.co/{REPO}/resolve/{REVISION}/{name}", timeout=180) as response, path.with_suffix(path.suffix + ".part").open("wb") as target: while block := response.read(8 * 1024 * 1024): digest.update(block) target.write(block) sha = digest.hexdigest() if expected: assert sha == expected, (name, sha, expected) path.with_suffix(path.suffix + ".part").replace(path) print(f"{name}: {path.stat().st_size} bytes sha256={sha}", flush=True) return dict(path=name, size_bytes=path.stat().st_size, sha256=sha, upstream_lfs_sha256=expected) except Exception: if attempt == 3: raise time.sleep(2 ** attempt) with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: files = sorted(executor.map(fetch, metadata["siblings"]), key=lambda item: item["path"]) (ROOT / "logs/source-manifest.json").write_text(json.dumps(dict(repo_id=REPO, revision=REVISION, files=files), indent=2) + "\n") (ROOT / "logs/source-SHA256SUMS").write_text("".join(f"{item['sha256']} {item['path']}\n" for item in files))