embeddinggemma-2-GGUF / reproduce /scripts /download_source.py
homerquan's picture
Add standard Q4_K_M text model and Q8_0 vision/audio companion
336c50f verified
Raw History Blame Contribute Delete
2.16 kB
#!/usr/bin/env python3
"""Download the original public checkpoint at one immutable Hugging Face revision."""
import concurrent.futures
import hashlib
import json
import pathlib
import time
import urllib.request
ROOT = pathlib.Path(__file__).resolve().parents[1]
REPO = "google/embeddinggemma-2"
REVISION = "f9972ac22e696a105c482202646979e33e4b5a1e"
url = f"https://huggingface.co/api/models/{REPO}/revision/{REVISION}?blobs=true"
with urllib.request.urlopen(url, timeout=120) as response:
metadata = json.load(response)
assert metadata["sha"] == REVISION
(ROOT / "logs/source-api.json").write_text(json.dumps(metadata, indent=2) + "\n")
def fetch(entry):
name = entry["rfilename"]
path = ROOT / "source" / name
path.parent.mkdir(parents=True, exist_ok=True)
expected = entry.get("lfs", {}).get("sha256")
for attempt in range(4):
try:
digest = hashlib.sha256()
with urllib.request.urlopen(f"https://huggingface.co/{REPO}/resolve/{REVISION}/{name}", timeout=180) as response, path.with_suffix(path.suffix + ".part").open("wb") as target:
while block := response.read(8 * 1024 * 1024):
digest.update(block)
target.write(block)
sha = digest.hexdigest()
if expected:
assert sha == expected, (name, sha, expected)
path.with_suffix(path.suffix + ".part").replace(path)
print(f"{name}: {path.stat().st_size} bytes sha256={sha}", flush=True)
return dict(path=name, size_bytes=path.stat().st_size, sha256=sha,
upstream_lfs_sha256=expected)
except Exception:
if attempt == 3:
raise
time.sleep(2 ** attempt)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
files = sorted(executor.map(fetch, metadata["siblings"]), key=lambda item: item["path"])
(ROOT / "logs/source-manifest.json").write_text(json.dumps(dict(repo_id=REPO, revision=REVISION, files=files), indent=2) + "\n")
(ROOT / "logs/source-SHA256SUMS").write_text("".join(f"{item['sha256']} {item['path']}\n" for item in files))