File size: 1,130 Bytes
d4eb935 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | """Download the exact Kev adapter and Qwen base revisions used by this conversion."""
from __future__ import annotations
import hashlib
import json
from dataclasses import replace
from pathlib import Path
from huggingface_hub import snapshot_download
from kev.checkpoint import Checkpoint, LoadOptions
ROOT = Path(__file__).resolve().parent
LOCK_PATH = ROOT / "assets.lock.json"
def load_model():
lock = json.loads(LOCK_PATH.read_text())
checkpoint = lock["checkpoint"]
base = lock["base"]
checkpoint_path = snapshot_download(checkpoint["repo"], revision=checkpoint["revision"])
for name, expected in checkpoint["files"].items():
actual = hashlib.sha256((Path(checkpoint_path) / name).read_bytes()).hexdigest()
if actual != expected:
raise ValueError(f"trained Kev file hash mismatch: {name}")
base_path = snapshot_download(base["repo"], revision=base["revision"])
loader = Checkpoint(checkpoint_path)
loader.meta = replace(loader.meta, base=base_path, base_revision=None)
tokenizer, model = loader.load("cpu", LoadOptions())
return lock, tokenizer, model
|