""" model_files.py Keeps the local model files in step with the Hugging Face model repo, which the training repo republishes whenever it retrains. The six files are one unit: a retrain can rebuild tokenizer.json (new cards shift vocab_size), and a tokenizer paired with checkpoints from a different release fails to load or silently misaligns token ids. So an update is all-or-nothing: 1. Ask the Hub for the commit sha of MODEL_REVISION (default "main"). 2. If it matches the sha recorded in data/model_version.json and every file exists locally, do nothing. 3. Otherwise download all six files pinned to that sha into a staging folder (data/.model-snapshot/; unchanged files are skipped via the Hub's etag metadata). 4. Check that they belong together (validate_release): the tokenizer loads with this code's format_version, the text embeddings have one row per tokenizer card, no checkpoint knows more cards than the tokenizer, and card_tiers.json has as many staples as this code's staple targets assume (training.metrics.STAPLE_TOP_N). (A checkpoint may know fewer: new cards are appended to the tokenizer, and the server keeps them away from the model.) This catches a release caught mid-upload, or one built from mismatched files, before it can replace working files. 5. Only then copy them into the paths the training code loads from and record the new sha. If the Hub can't be reached, or the release is incomplete or fails the check, existing local files are kept (with a warning); with no local files it's an error. The Docker image pins MTG_MODEL_REVISION to a specific commit, so a deploy only changes model when that pin is bumped; local development follows "main". Run directly to sync without starting the server: python -m server.model_files """ from __future__ import annotations import json import os import shutil from datetime import datetime, timezone from pathlib import Path MODEL_REPO = os.environ.get("MTG_MODEL_REPO", "nsaroiu/mtg-deck-completion") MODEL_REVISION = os.environ.get("MTG_MODEL_REVISION", "main") # Hub filename -> local directory the training code expects it in (see README). MODEL_FILES = { "tokenizer.json": ".", "tokenizer_text_embeddings.npy": ".", "deepsets_softmax_checkpoint.pt": "data", "set_transformer_softmax_checkpoint.pt": "data", "prune_checkpoint_inject_lowdensity.pt": "data", "card_tiers.json": "data", } VERSION_FILE = Path("data/model_version.json") STAGING_DIR = Path("data/.model-snapshot") def local_path(filename: str) -> Path: return Path(MODEL_FILES[filename]) / filename def files_present() -> bool: return all(local_path(f).exists() for f in MODEL_FILES) def read_version() -> dict | None: try: return json.loads(VERSION_FILE.read_text()) except (OSError, ValueError): return None def _remote_version(repo_id: str, revision: str) -> dict: from huggingface_hub import HfApi info = HfApi().model_info(repo_id, revision=revision) last_modified = info.last_modified.astimezone(timezone.utc).isoformat() if info.last_modified else None files = [s.rfilename for s in info.siblings or []] return {"repo": repo_id, "revision": revision, "sha": info.sha, "last_modified": last_modified, "files": files} def _download_snapshot(repo_id: str, sha: str) -> Path: from huggingface_hub import snapshot_download return Path(snapshot_download(repo_id, revision=sha, allow_patterns=list(MODEL_FILES), local_dir=STAGING_DIR)) CHECKPOINTS = ("deepsets_softmax_checkpoint.pt", "set_transformer_softmax_checkpoint.pt", "prune_checkpoint_inject_lowdensity.pt") def validate_release(snapshot: Path) -> None: """Raises ValueError unless the downloaded files fit together and this code can read them. Loading the checkpoints into models is left to the server (and the Docker build's selfcheck): an architecture change shows up there, while this catches mismatched files cheaply.""" import numpy as np import torch from tokenizer.mtg_tokenizer import CardTokenizer try: vocab_size = CardTokenizer.load_pretrained(str(snapshot / "tokenizer.json")).vocab_size except Exception as exc: raise ValueError(f"tokenizer.json doesn't load ({exc})") from exc mismatched = [] # files that don't fit the tokenizer's vocab rows = np.load(snapshot / "tokenizer_text_embeddings.npy", mmap_mode="r").shape[0] if rows != vocab_size: mismatched.append(f"tokenizer_text_embeddings.npy has {rows} rows") for name in CHECKPOINTS: ckpt = torch.load(snapshot / name, map_location="cpu", weights_only=False) ckpt_vocab = ckpt.get("vocab_size") if not isinstance(ckpt_vocab, int) or ckpt_vocab > vocab_size: mismatched.append(f"{name} has vocab_size {ckpt_vocab}") problems = [f"tokenizer.json has {vocab_size} cards, but " + "; ".join(mismatched)] if mismatched else [] tiers = json.loads((snapshot / "card_tiers.json").read_text()) if not isinstance(tiers, dict): problems.append("card_tiers.json isn't a JSON object") else: # The staple top-up's per-color targets (ensemble.py) are calibrated # to a staple set of exactly STAPLE_TOP_N cards. from training.metrics import STAPLE_TOP_N staples = sum(tier == "staple" for tier in tiers.values()) if staples != STAPLE_TOP_N: problems.append(f"card_tiers.json has {staples} staples where this code expects {STAPLE_TOP_N}") if problems: raise ValueError("; ".join(problems)) def _install(snapshot: Path) -> None: for filename in MODEL_FILES: dest = local_path(filename) dest.parent.mkdir(parents=True, exist_ok=True) tmp = dest.with_name(dest.name + ".partial") shutil.copyfile(snapshot / filename, tmp) os.replace(tmp, dest) # atomic per file; all sources are already complete def sync_model_files(repo_id: str = MODEL_REPO, revision: str = MODEL_REVISION, check_remote: bool = True) -> dict | None: """Brings local model files up to date with the Hub (see module docstring) and returns the installed version record, or None when the files' origin is unknown (present locally but never synced). check_remote=False skips the Hub check entirely (MTG_MODEL_UPDATE=0 or HF_HUB_OFFLINE=1 do the same when called from the server).""" local = read_version() if not check_remote: if not files_present(): raise RuntimeError("Model files are missing and the update check is disabled.") return local try: remote = _remote_version(repo_id, revision) except Exception as exc: # network down, Hub error, offline mode if files_present(): print(f"Couldn't check {repo_id} for model updates ({exc.__class__.__name__}); using local files.") return local raise RuntimeError(f"Model files are missing and {repo_id} can't be reached: {exc}") from exc if local and local.get("sha") == remote["sha"] and local.get("repo") == repo_id and files_present(): return local def reject(problem: str) -> dict | None: message = f"Model release {repo_id}@{remote['sha'][:7]} {problem}" if not files_present(): raise RuntimeError(f"{message}, and there are no local model files to fall back on.") print(f"{message}; keeping the current files.") return local # Checked against the release's own file list, not the staging folder: # staging is reused across releases, so a file dropped from a new # release would otherwise be masked by the previous release's copy. release_files = set(remote.pop("files")) missing = [f for f in MODEL_FILES if f not in release_files] if missing: return reject(f"is missing {', '.join(missing)}") print(f"Installing model {repo_id}@{remote['sha'][:7]} ...") snapshot = _download_snapshot(repo_id, remote["sha"]) try: validate_release(snapshot) except Exception as exc: return reject(f"failed validation ({exc})") _install(snapshot) remote["installed_at"] = datetime.now(timezone.utc).isoformat() VERSION_FILE.parent.mkdir(parents=True, exist_ok=True) VERSION_FILE.write_text(json.dumps(remote, indent=2) + "\n") return remote def update_check_enabled() -> bool: return os.environ.get("MTG_MODEL_UPDATE", "1") != "0" and os.environ.get("HF_HUB_OFFLINE", "0") in ("", "0") if __name__ == "__main__": version = sync_model_files(check_remote=update_check_enabled()) print(f"Model files ready: {version['repo']}@{version['sha'][:7]}" if version else "Model files ready (version unknown).")