| from __future__ import annotations
|
|
|
| import json
|
| from collections.abc import Callable
|
| from pathlib import Path
|
| from typing import Any
|
|
|
| import httpx
|
| import pytest
|
| from fitcheck import config_parser
|
| from fitcheck.config_parser import (
|
| UnsupportedModelError,
|
| _reported_param_count,
|
| fetch_model_config,
|
| )
|
| from huggingface_hub.errors import GatedRepoError
|
|
|
|
|
| @pytest.mark.network
|
| def test_fetch_model_config_parses_llama_31_8b() -> None:
|
| config = fetch_model_config("meta-llama/Llama-3.1-8B")
|
|
|
| assert config.name == "Llama-3.1-8B"
|
| assert config.hidden_size == 4096
|
| assert config.num_layers == 32
|
| assert config.num_attention_heads == 32
|
| assert config.num_kv_heads == 8
|
| assert config.intermediate_size == 14336
|
| assert config.vocab_size == 128256
|
| assert config.head_dim == 128
|
| assert config.tie_word_embeddings is False
|
| assert config.num_params == pytest.approx(8_030_261_248, rel=0.00001)
|
|
|
|
|
| def test_fetch_model_config_parses_llama_31_8b_offline(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(llama_31_8b_config)
|
|
|
| config = fetch_model_config("meta-llama/Llama-3.1-8B")
|
|
|
| assert config.name == "Llama-3.1-8B"
|
| assert config.num_kv_heads == 8
|
| assert config.head_dim == 128
|
| assert config.num_params == pytest.approx(8_030_261_248, rel=0.00001)
|
|
|
|
|
| def test_fetch_model_config_defaults_missing_kv_heads_to_mha(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| mha_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(mha_config)
|
|
|
| config = fetch_model_config("openai-community/gpt2")
|
|
|
| assert config.num_kv_heads == config.num_attention_heads == 12
|
| assert config.head_dim == 64
|
| expected_attention_params = 4 * config.hidden_size**2
|
| expected_mlp_params = 3 * config.hidden_size * config.intermediate_size
|
| expected_norm_params = 2 * config.hidden_size
|
| expected_embedding_params = config.vocab_size * config.hidden_size
|
| expected = (
|
| expected_embedding_params
|
| + config.num_layers
|
| * (expected_attention_params + expected_mlp_params + expected_norm_params)
|
| + config.hidden_size
|
| + expected_embedding_params
|
| )
|
| assert config.num_params == expected
|
|
|
|
|
| def test_fetch_model_config_handles_mqa(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| mqa_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(mqa_config)
|
|
|
| config = fetch_model_config("tiiuae/falcon-7b")
|
|
|
| assert config.num_kv_heads == 1
|
| assert config.num_attention_heads == 16
|
| assert config.head_dim == 128
|
|
|
|
|
| def test_fetch_model_config_handles_tied_embeddings(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| tied_embeddings_config: dict[str, Any],
|
| ) -> None:
|
| """tie_word_embeddings=True must count the embedding table once, not twice."""
|
| fake_config_download(tied_embeddings_config)
|
|
|
| tied = fetch_model_config("fake-org/tied-model")
|
| assert tied.tie_word_embeddings is True
|
|
|
| untied_config = dict(tied_embeddings_config, tie_word_embeddings=False)
|
| fake_config_download(untied_config)
|
| untied = fetch_model_config("fake-org/untied-model")
|
| assert untied.tie_word_embeddings is False
|
|
|
| embedding_params = tied_embeddings_config["vocab_size"] * tied_embeddings_config["hidden_size"]
|
| assert untied.num_params - tied.num_params == embedding_params
|
|
|
|
|
| _GEMMA_2_9B_PARAMS = 9_241_404_928
|
|
|
|
|
| def test_declared_head_dim_wins_over_hidden_size_over_heads(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| """Gemma-2-9B declares head_dim 256; hidden_size // heads would give 224."""
|
| fake_config_download(gemma_2_9b_config)
|
|
|
| config = fetch_model_config("google/gemma-2-9b")
|
|
|
| assert config.head_dim == 256
|
| assert config.head_dim != config.hidden_size // config.num_attention_heads
|
|
|
|
|
| def test_head_dim_falls_back_to_hidden_size_over_heads_when_absent(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| without_head_dim = {
|
| key: value for key, value in gemma_2_9b_config.items() if key != "head_dim"
|
| }
|
| fake_config_download(without_head_dim)
|
|
|
| config = fetch_model_config("fake-org/no-head-dim")
|
|
|
| assert config.head_dim == 3584 // 16 == 224
|
|
|
|
|
| def test_declared_head_dim_lifts_the_divisibility_requirement(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| """Divisibility constrains deriving head_dim, not the model itself."""
|
| indivisible = dict(gemma_2_9b_config, hidden_size=3585)
|
| fake_config_download(indivisible)
|
|
|
| config = fetch_model_config("fake-org/indivisible")
|
|
|
| assert config.hidden_size % config.num_attention_heads != 0
|
| assert config.head_dim == 256
|
|
|
|
|
| def test_absent_tie_word_embeddings_means_tied_for_gemma(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| assert "tie_word_embeddings" not in gemma_2_9b_config
|
| fake_config_download(gemma_2_9b_config)
|
|
|
| config = fetch_model_config("google/gemma-2-9b")
|
|
|
| assert config.tie_word_embeddings is True
|
|
|
|
|
| def test_absent_tie_word_embeddings_defaults_untied_for_unknown_architecture(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| """Unrecognised model_type keeps the conservative (over-counting) default."""
|
| unknown = dict(gemma_2_9b_config, model_type="not-a-real-architecture")
|
| fake_config_download(unknown)
|
|
|
| config = fetch_model_config("fake-org/unknown-arch")
|
|
|
| assert config.tie_word_embeddings is False
|
|
|
|
|
| def test_an_explicit_tie_word_embeddings_beats_the_architecture_default(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(dict(gemma_2_9b_config, tie_word_embeddings=False))
|
|
|
| config = fetch_model_config("fake-org/gemma-untied")
|
|
|
| assert config.tie_word_embeddings is False
|
|
|
|
|
| def test_gemma_2_9b_counts_924b_not_993b(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| """Both traps compound: head_dim 224 + untied would over-count by 7.4%."""
|
| fake_config_download(gemma_2_9b_config)
|
|
|
| config = fetch_model_config("google/gemma-2-9b")
|
|
|
| assert config.num_params == _GEMMA_2_9B_PARAMS
|
| assert config.num_params / 1e9 == pytest.approx(9.24, abs=0.005)
|
| assert config.num_params < 9_500_000_000
|
|
|
|
|
| def test_attention_params_use_head_dim_not_hidden_size_squared(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| gemma_2_9b_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(gemma_2_9b_config)
|
| config = fetch_model_config("google/gemma-2-9b")
|
|
|
| q_out = config.num_attention_heads * config.head_dim
|
| kv_out = config.num_kv_heads * config.head_dim
|
| expected_attention = 2 * config.hidden_size * q_out + 2 * config.hidden_size * kv_out
|
|
|
| embedding = config.vocab_size * config.hidden_size
|
| mlp = 3 * config.hidden_size * config.intermediate_size
|
| norms = 2 * config.hidden_size
|
| expected = (
|
| embedding
|
| + config.num_layers * (expected_attention + mlp + norms)
|
| + config.hidden_size
|
| )
|
|
|
| assert q_out == 4096 != config.hidden_size
|
| assert config.num_params == expected
|
|
|
|
|
| def test_generalized_attention_formula_leaves_llama_untouched(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| """n_h * d_k == h for Llama, so 2h*n_h*d_k + 2h*n_kv*d_k == 2h^2 + 2h*n_kv*d_k."""
|
| fake_config_download(llama_31_8b_config)
|
|
|
| config = fetch_model_config("meta-llama/Llama-3.1-8B")
|
|
|
| assert config.num_attention_heads * config.head_dim == config.hidden_size
|
| assert config.num_params == 8_030_261_248
|
|
|
|
|
| @pytest.mark.parametrize("missing_field", ["hidden_size", "vocab_size", "num_hidden_layers"])
|
| def test_fetch_model_config_missing_required_field_raises(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| llama_31_8b_config: dict[str, Any],
|
| missing_field: str,
|
| ) -> None:
|
| broken_config = dict(llama_31_8b_config)
|
| del broken_config[missing_field]
|
| fake_config_download(broken_config)
|
|
|
| with pytest.raises(ValueError, match=missing_field):
|
| fetch_model_config("fake-org/broken-model")
|
|
|
|
|
| def test_fetch_model_config_missing_intermediate_size_falls_back_to_4h(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| llama_31_8b_config: dict[str, Any],
|
| capsys: pytest.CaptureFixture[str],
|
| ) -> None:
|
| config_without_ffn_size = dict(llama_31_8b_config)
|
| del config_without_ffn_size["intermediate_size"]
|
| fake_config_download(config_without_ffn_size)
|
|
|
| config = fetch_model_config("fake-org/tied-model")
|
|
|
| assert config.intermediate_size == 4 * config.hidden_size
|
| captured = capsys.readouterr()
|
| assert captured.out == ""
|
| assert "'intermediate_size' not in config.json" in captured.err
|
| assert "4 x hidden_size" in captured.err
|
|
|
|
|
| def test_fetch_model_config_hidden_size_not_divisible_by_heads_raises(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| bad_config = dict(llama_31_8b_config, num_attention_heads=33)
|
| fake_config_download(bad_config)
|
|
|
| with pytest.raises(ValueError, match="divisible"):
|
| fetch_model_config("fake-org/bad-heads-model")
|
|
|
|
|
| def test_fetch_model_config_kv_heads_exceeding_attention_heads_raises(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| bad_config = dict(llama_31_8b_config, num_key_value_heads=64)
|
| fake_config_download(bad_config)
|
|
|
| with pytest.raises(ValueError, match="cannot exceed"):
|
| fetch_model_config("fake-org/bad-kv-heads-model")
|
|
|
|
|
| @pytest.mark.parametrize("declared", ["false", "true", 0, 1, [], "yes"])
|
| def test_fetch_model_config_rejects_a_non_boolean_tie_word_embeddings(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| declared: object,
|
| ) -> None:
|
| |
| fake_config_download(dict(llama_31_8b_config, tie_word_embeddings=declared))
|
|
|
| with pytest.raises(ValueError, match="'tie_word_embeddings' must be true or false"):
|
| fetch_model_config("fake-org/string-tie-flag")
|
|
|
|
|
| @pytest.mark.parametrize("root", [[{"hidden_size": 4096}], "config", 7, None])
|
| def test_fetch_model_config_rejects_a_non_object_json_root(
|
| fake_config_download: Callable[..., None], root: object
|
| ) -> None:
|
| fake_config_download(root)
|
|
|
| with pytest.raises(ValueError, match="must be a JSON object"):
|
| fetch_model_config("fake-org/not-an-object")
|
|
|
|
|
| @pytest.mark.parametrize("model_id", ["", " "])
|
| def test_fetch_model_config_empty_model_id_raises(model_id: str) -> None:
|
| with pytest.raises(ValueError, match="non-empty"):
|
| fetch_model_config(model_id)
|
|
|
|
|
| def _install_gated(monkeypatch: pytest.MonkeyPatch) -> None:
|
| def _raise_gated(*, repo_id: str, filename: str, token: str | None = None) -> str:
|
| fake_response = httpx.Response(
|
| 403, request=httpx.Request("GET", f"https://huggingface.co/{repo_id}")
|
| )
|
| raise GatedRepoError("gated", response=fake_response)
|
|
|
| monkeypatch.setattr("fitcheck.config_parser.hf_hub_download", _raise_gated)
|
|
|
|
|
| def _clear_token_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
| monkeypatch.delenv("HF_TOKEN", raising=False)
|
| monkeypatch.delenv("HUGGING_FACE_HUB_TOKEN", raising=False)
|
|
|
|
|
| def test_fetch_model_config_gated_repo_raises_actionable_runtime_error(
|
| monkeypatch: pytest.MonkeyPatch,
|
| ) -> None:
|
| _clear_token_env(monkeypatch)
|
| _install_gated(monkeypatch)
|
|
|
| with pytest.raises(RuntimeError, match="gated"):
|
| fetch_model_config("meta-llama/some-gated-model")
|
|
|
|
|
| def test_gated_error_without_a_token_points_at_login_and_hf_token(
|
| monkeypatch: pytest.MonkeyPatch,
|
| ) -> None:
|
| _clear_token_env(monkeypatch)
|
| _install_gated(monkeypatch)
|
|
|
| with pytest.raises(RuntimeError, match="no token was found") as excinfo:
|
| fetch_model_config("meta-llama/some-gated-model")
|
|
|
| message = str(excinfo.value)
|
| assert "hf auth login" in message
|
| assert "HF_TOKEN" in message
|
|
|
|
|
| def test_gated_error_with_a_token_blames_access_not_authentication(
|
| monkeypatch: pytest.MonkeyPatch,
|
| ) -> None:
|
| """A token that is present but unapproved is a different problem, and a
|
| 'run hf auth login' message sends the user in the wrong direction."""
|
| _clear_token_env(monkeypatch)
|
| monkeypatch.setenv("HF_TOKEN", "hf_secret_value")
|
| _install_gated(monkeypatch)
|
|
|
| with pytest.raises(RuntimeError, match="does not grant access") as excinfo:
|
| fetch_model_config("meta-llama/some-gated-model")
|
|
|
| assert "hf auth login" not in str(excinfo.value)
|
|
|
|
|
| def test_gated_error_never_echoes_the_token(monkeypatch: pytest.MonkeyPatch) -> None:
|
| _clear_token_env(monkeypatch)
|
| monkeypatch.setenv("HF_TOKEN", "hf_secret_value")
|
| _install_gated(monkeypatch)
|
|
|
| with pytest.raises(RuntimeError) as excinfo:
|
| fetch_model_config("meta-llama/some-gated-model")
|
|
|
| assert "hf_secret_value" not in str(excinfo.value)
|
| assert "hf_secret_value" not in repr(excinfo.value)
|
|
|
|
|
| @pytest.mark.parametrize(
|
| "variable", ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]
|
| )
|
| def test_environment_token_is_forwarded_to_the_hub(
|
| monkeypatch: pytest.MonkeyPatch, tmp_path: Path, variable: str
|
| ) -> None:
|
| """Kaggle and Colab export a token instead of caching a CLI login."""
|
| _clear_token_env(monkeypatch)
|
| monkeypatch.setenv(variable, " hf_from_env ")
|
| seen: dict[str, Any] = {}
|
| config_path = tmp_path / "config.json"
|
| config_path.write_text(json.dumps(_MINIMAL_CONFIG), encoding="utf-8")
|
|
|
| def _capture(*, repo_id: str, filename: str, token: str | None = None) -> str:
|
| seen["token"] = token
|
| return str(config_path)
|
|
|
| monkeypatch.setattr("fitcheck.config_parser.hf_hub_download", _capture)
|
| fetch_model_config("some/model")
|
|
|
| assert seen["token"] == "hf_from_env"
|
|
|
|
|
| def test_no_environment_token_forwards_none(
|
| monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
| ) -> None:
|
| """None lets huggingface_hub fall back to a cached `hf auth login`."""
|
| _clear_token_env(monkeypatch)
|
| seen: dict[str, Any] = {}
|
| config_path = tmp_path / "config.json"
|
| config_path.write_text(json.dumps(_MINIMAL_CONFIG), encoding="utf-8")
|
|
|
| def _capture(*, repo_id: str, filename: str, token: str | None = None) -> str:
|
| seen["token"] = token
|
| return str(config_path)
|
|
|
| monkeypatch.setattr("fitcheck.config_parser.hf_hub_download", _capture)
|
| fetch_model_config("some/model")
|
|
|
| assert seen["token"] is None
|
|
|
|
|
| _MINIMAL_CONFIG = {
|
| "hidden_size": 64,
|
| "num_hidden_layers": 2,
|
| "num_attention_heads": 4,
|
| "num_key_value_heads": 2,
|
| "intermediate_size": 128,
|
| "vocab_size": 100,
|
| "tie_word_embeddings": False,
|
| }
|
|
|
|
|
| _MIXTRAL_8X7B_CONFIG = {
|
| "model_type": "mixtral",
|
| "hidden_size": 4096,
|
| "num_hidden_layers": 32,
|
| "num_attention_heads": 32,
|
| "num_key_value_heads": 8,
|
| "intermediate_size": 14336,
|
| "vocab_size": 32000,
|
| "num_local_experts": 8,
|
| "num_experts_per_tok": 2,
|
| }
|
|
|
| _QWEN3_30B_A3B_CONFIG = {
|
| "model_type": "qwen3_moe",
|
| "hidden_size": 2048,
|
| "num_hidden_layers": 48,
|
| "num_attention_heads": 32,
|
| "num_key_value_heads": 4,
|
| "head_dim": 128,
|
| "intermediate_size": 6144,
|
| "moe_intermediate_size": 768,
|
| "vocab_size": 151936,
|
| "num_experts": 128,
|
| "num_experts_per_tok": 8,
|
| }
|
|
|
| _GPT_OSS_20B_CONFIG = {
|
| "model_type": "gpt_oss",
|
| "hidden_size": 2880,
|
| "num_hidden_layers": 24,
|
| "num_attention_heads": 64,
|
| "num_key_value_heads": 8,
|
| "head_dim": 64,
|
| "intermediate_size": 2880,
|
| "vocab_size": 201088,
|
| "num_local_experts": 32,
|
| "num_experts_per_tok": 4,
|
| }
|
|
|
| _DEEPSEEK_V2_LITE_CONFIG = {
|
| "model_type": "deepseek_v2",
|
| "hidden_size": 2048,
|
| "num_hidden_layers": 27,
|
| "num_attention_heads": 16,
|
| "num_key_value_heads": 16,
|
| "intermediate_size": 10944,
|
| "moe_intermediate_size": 1408,
|
| "vocab_size": 102400,
|
| "n_routed_experts": 64,
|
| "num_experts_per_tok": 6,
|
| }
|
|
|
| _SMOLVLM_CONFIG = {
|
| "model_type": "idefics3",
|
| "text_config": {
|
| "model_type": "llama",
|
| "hidden_size": 2048,
|
| "num_hidden_layers": 24,
|
| "num_attention_heads": 32,
|
| "num_key_value_heads": 32,
|
| "intermediate_size": 8192,
|
| "vocab_size": 49280,
|
| },
|
| "vision_config": {"hidden_size": 1152, "num_hidden_layers": 27},
|
| }
|
|
|
|
|
| @pytest.mark.parametrize(
|
| ("model_id", "config", "expected"),
|
| [
|
| ("mistralai/Mixtral-8x7B-v0.1", _MIXTRAL_8X7B_CONFIG, "Mixture-of-Experts"),
|
| ("Qwen/Qwen3-30B-A3B", _QWEN3_30B_A3B_CONFIG, "Mixture-of-Experts"),
|
| ("openai/gpt-oss-20b", _GPT_OSS_20B_CONFIG, "Mixture-of-Experts"),
|
| ("deepseek-ai/DeepSeek-V2-Lite", _DEEPSEEK_V2_LITE_CONFIG, "Mixture-of-Experts"),
|
| ("HuggingFaceTB/SmolVLM-Instruct", _SMOLVLM_CONFIG, "nests"),
|
| ],
|
| )
|
| def test_unsupported_architectures_are_refused(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| model_id: str,
|
| config: dict[str, Any],
|
| expected: str,
|
| ) -> None:
|
| fake_config_download(config)
|
|
|
| with pytest.raises(UnsupportedModelError, match=expected) as excinfo:
|
| fetch_model_config(model_id)
|
|
|
| message = str(excinfo.value)
|
| assert model_id in message
|
| assert "OOM" in message
|
|
|
|
|
| @pytest.mark.parametrize(
|
| ("model_id", "config", "declared_key"),
|
| [
|
| ("mistralai/Mixtral-8x7B-v0.1", _MIXTRAL_8X7B_CONFIG, "num_local_experts"),
|
| ("Qwen/Qwen3-30B-A3B", _QWEN3_30B_A3B_CONFIG, "num_experts"),
|
| ("openai/gpt-oss-20b", _GPT_OSS_20B_CONFIG, "num_local_experts"),
|
| ("deepseek-ai/DeepSeek-V2-Lite", _DEEPSEEK_V2_LITE_CONFIG, "n_routed_experts"),
|
| ],
|
| )
|
| def test_moe_refusal_names_the_key_it_saw(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| model_id: str,
|
| config: dict[str, Any],
|
| declared_key: str,
|
| ) -> None:
|
| fake_config_download(config)
|
|
|
| with pytest.raises(UnsupportedModelError) as excinfo:
|
| fetch_model_config(model_id)
|
|
|
| assert declared_key in str(excinfo.value)
|
| assert "num_experts_per_tok" in str(excinfo.value)
|
|
|
|
|
| def test_nested_multimodal_refusal_says_nested_not_malformed(
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| ) -> None:
|
| fake_config_download(_SMOLVLM_CONFIG)
|
|
|
| with pytest.raises(UnsupportedModelError) as excinfo:
|
| fetch_model_config("HuggingFaceTB/SmolVLM-Instruct")
|
|
|
| message = str(excinfo.value)
|
| assert "text_config" in message
|
| assert "nested, not missing" in message
|
| assert "must be a positive integer" not in message
|
|
|
|
|
| def test_unsupported_model_error_is_a_value_error() -> None:
|
| assert issubclass(UnsupportedModelError, ValueError)
|
|
|
|
|
| @pytest.mark.parametrize(
|
| ("model_id", "config"),
|
| [
|
| ("meta-llama/Llama-3.1-8B", "llama_31_8b_config"),
|
| ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", "mqa_config"),
|
| ("openai-community/gpt2", "mha_config"),
|
| ("google/gemma-2-9b", "gemma_2_9b_config"),
|
| ("HuggingFaceTB/SmolLM2-1.7B", "tied_embeddings_config"),
|
| ],
|
| )
|
| def test_supported_dense_models_are_not_refused(
|
| request: pytest.FixtureRequest,
|
| fake_config_download: Callable[[dict[str, Any]], None],
|
| model_id: str,
|
| config: str,
|
| ) -> None:
|
| fake_config_download(request.getfixturevalue(config))
|
|
|
| assert fetch_model_config(model_id).num_params > 0
|
|
|
|
|
| def test_a_vision_tower_alongside_top_level_dims_passes_the_parse_gate(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(
|
| dict(llama_31_8b_config, vision_config={"hidden_size": 1280, "depth": 32})
|
| )
|
| assert fetch_model_config("Qwen/Qwen2.5-VL-7B-Instruct").num_params > 0
|
|
|
|
|
|
|
| _LLAMA_31_8B_PARAMS = 8_030_261_248
|
|
|
| _PHI_2_CONFIG = {
|
| "model_type": "phi",
|
| "hidden_size": 2560,
|
| "num_hidden_layers": 32,
|
| "num_attention_heads": 32,
|
| "intermediate_size": 10240,
|
| "vocab_size": 51200,
|
| "tie_word_embeddings": False,
|
| }
|
| _PHI_2_DERIVED = 3_617_753_600
|
| _PHI_2_HUB = 2_779_683_840
|
|
|
|
|
| def test_hub_count_is_preferred_over_the_derived_one(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| hub_count = _LLAMA_31_8B_PARAMS + 57_344
|
|
|
| fake_config_download(llama_31_8b_config, hub_param_count=hub_count)
|
|
|
| assert fetch_model_config("meta-llama/Llama-3.1-8B").num_params == hub_count
|
|
|
|
|
| def test_hub_count_agreeing_exactly_leaves_the_golden_number_alone(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(llama_31_8b_config, hub_param_count=_LLAMA_31_8B_PARAMS)
|
|
|
| assert fetch_model_config("meta-llama/Llama-3.1-8B").num_params == _LLAMA_31_8B_PARAMS
|
|
|
|
|
| def test_offline_fallback_keeps_the_derived_count(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| ) -> None:
|
| fake_config_download(llama_31_8b_config, hub_param_count=None)
|
|
|
| assert fetch_model_config("meta-llama/Llama-3.1-8B").num_params == _LLAMA_31_8B_PARAMS
|
|
|
|
|
| def test_offline_fallback_warns_that_the_count_is_derived(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| capsys: pytest.CaptureFixture[str],
|
| ) -> None:
|
| fake_config_download(llama_31_8b_config, hub_param_count=None)
|
|
|
| fetch_model_config("meta-llama/Llama-3.1-8B")
|
|
|
| stderr = capsys.readouterr().err
|
| assert "derived from config.json" in stderr
|
| assert "SwiGLU" in stderr
|
|
|
|
|
| def test_phi_2_style_over_count_is_refused(
|
| fake_config_download: Callable[..., None],
|
| ) -> None:
|
| fake_config_download(_PHI_2_CONFIG, hub_param_count=_PHI_2_HUB)
|
|
|
| with pytest.raises(UnsupportedModelError) as excinfo:
|
| fetch_model_config("microsoft/phi-2")
|
|
|
| message = str(excinfo.value)
|
| assert "microsoft/phi-2" in message
|
| assert f"{_PHI_2_HUB:,}" in message
|
| assert f"{_PHI_2_DERIVED:,}" in message
|
| assert "+30.1%" in message
|
| assert "over-counts" in message
|
|
|
|
|
| _QWEN2_5_VL_7B_CONFIG = {
|
| "model_type": "qwen2_5_vl",
|
| "hidden_size": 3584,
|
| "num_hidden_layers": 28,
|
| "num_attention_heads": 28,
|
| "num_key_value_heads": 4,
|
| "intermediate_size": 18944,
|
| "vocab_size": 152064,
|
| "tie_word_embeddings": False,
|
| "vision_config": {"hidden_size": 1280, "depth": 32, "intermediate_size": 3420},
|
| }
|
| _QWEN2_5_VL_7B_DERIVED = 7_615_487_488
|
| _QWEN2_5_VL_7B_HUB = 8_292_166_656
|
|
|
|
|
| def test_flat_multimodal_is_refused_by_the_hub_cross_check(
|
| fake_config_download: Callable[..., None],
|
| ) -> None:
|
| fake_config_download(
|
| _QWEN2_5_VL_7B_CONFIG, hub_param_count=_QWEN2_5_VL_7B_HUB
|
| )
|
|
|
| with pytest.raises(UnsupportedModelError) as excinfo:
|
| fetch_model_config("Qwen/Qwen2.5-VL-7B-Instruct")
|
|
|
| message = str(excinfo.value)
|
| assert f"{_QWEN2_5_VL_7B_DERIVED:,}" in message
|
| assert "-8.2%" in message
|
| assert "OOM" in message
|
|
|
|
|
| def test_disagreement_refusal_does_not_average_the_two(
|
| fake_config_download: Callable[..., None],
|
| ) -> None:
|
| fake_config_download(_PHI_2_CONFIG, hub_param_count=_PHI_2_HUB)
|
|
|
| with pytest.raises(UnsupportedModelError, match="refuses instead of averaging"):
|
| fetch_model_config("microsoft/phi-2")
|
|
|
|
|
| @pytest.mark.parametrize(
|
| ("hub_count", "refused"),
|
| [
|
| (int(_LLAMA_31_8B_PARAMS / 1.02) + 1, False), |
| (int(_LLAMA_31_8B_PARAMS / 1.021), True), |
| (int(_LLAMA_31_8B_PARAMS / 0.98) - 1, False),
|
| (int(_LLAMA_31_8B_PARAMS / 0.979), True),
|
| ],
|
| )
|
| def test_two_percent_is_the_boundary(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| hub_count: int,
|
| refused: bool,
|
| ) -> None:
|
| fake_config_download(llama_31_8b_config, hub_param_count=hub_count)
|
|
|
| if refused:
|
| with pytest.raises(UnsupportedModelError):
|
| fetch_model_config("meta-llama/Llama-3.1-8B")
|
| else:
|
| assert fetch_model_config("meta-llama/Llama-3.1-8B").num_params == hub_count
|
|
|
|
|
| def test_moe_is_refused_before_the_hub_is_asked(
|
| fake_config_download: Callable[..., None],
|
| monkeypatch: pytest.MonkeyPatch,
|
| ) -> None:
|
| """The parse gate runs first, so a refused model costs no metadata call."""
|
| calls: list[str] = []
|
|
|
| def _record(model_id: str, token: str | None) -> int | None:
|
| calls.append(model_id)
|
| return None
|
|
|
| fake_config_download(_MIXTRAL_8X7B_CONFIG)
|
| monkeypatch.setattr("fitcheck.config_parser._reported_param_count", _record)
|
|
|
| with pytest.raises(UnsupportedModelError):
|
| fetch_model_config("mistralai/Mixtral-8x7B-v0.1")
|
|
|
| assert calls == []
|
|
|
|
|
| class _FakeSafetensors:
|
| def __init__(self, total: Any) -> None:
|
| self.total = total
|
|
|
|
|
| class _FakeModelInfo:
|
| def __init__(self, safetensors: Any) -> None:
|
| self.safetensors = safetensors
|
|
|
|
|
| def _install_fake_api(monkeypatch: pytest.MonkeyPatch, result: Any) -> dict[str, Any]:
|
| seen: dict[str, Any] = {}
|
|
|
| class _FakeApi:
|
| def __init__(self, token: str | None = None) -> None:
|
| seen["token"] = token
|
|
|
| def model_info(self, model_id: str, expand: list[str] | None = None) -> Any:
|
| seen["model_id"] = model_id
|
| seen["expand"] = expand
|
| if isinstance(result, Exception):
|
| raise result
|
| return result
|
|
|
| monkeypatch.setattr("fitcheck.config_parser.HfApi", _FakeApi)
|
| return seen
|
|
|
|
|
| def test_reported_param_count_reads_safetensors_total(
|
| monkeypatch: pytest.MonkeyPatch,
|
| ) -> None:
|
| seen = _install_fake_api(
|
| monkeypatch, _FakeModelInfo(_FakeSafetensors(_LLAMA_31_8B_PARAMS))
|
| )
|
|
|
| assert (
|
| _reported_param_count("meta-llama/Llama-3.1-8B", "hf_tok") == _LLAMA_31_8B_PARAMS
|
| )
|
| assert seen["token"] == "hf_tok"
|
| assert seen["expand"] == ["safetensors"]
|
|
|
|
|
| @pytest.mark.parametrize(
|
| "result",
|
| [
|
| OSError("no network"),
|
| RuntimeError("hub is down"),
|
| _FakeModelInfo(None),
|
| _FakeModelInfo(_FakeSafetensors(None)),
|
| _FakeModelInfo(_FakeSafetensors(0)),
|
| _FakeModelInfo(_FakeSafetensors(True)),
|
| ],
|
| )
|
| def test_reported_param_count_returns_none_when_the_hub_cannot_answer(
|
| monkeypatch: pytest.MonkeyPatch, result: Any
|
| ) -> None:
|
| _install_fake_api(monkeypatch, result)
|
|
|
| assert _reported_param_count("some/model", None) is None
|
|
|
|
|
| def test_the_hub_cross_check_downloads_no_weights(
|
| fake_config_download: Callable[..., None],
|
| llama_31_8b_config: dict[str, Any],
|
| monkeypatch: pytest.MonkeyPatch,
|
| ) -> None:
|
| """The whole premise: a metadata call plus config.json, and no other file."""
|
| fake_config_download(llama_31_8b_config)
|
| _install_fake_api(monkeypatch, _FakeModelInfo(_FakeSafetensors(_LLAMA_31_8B_PARAMS)))
|
| monkeypatch.setattr(
|
| "fitcheck.config_parser._reported_param_count", _reported_param_count
|
| )
|
| downloaded: list[str] = []
|
| stubbed = config_parser.hf_hub_download
|
|
|
| def _record(*, repo_id: str, filename: str, token: str | None = None) -> str:
|
| downloaded.append(filename)
|
| return stubbed(repo_id=repo_id, filename=filename, token=token)
|
|
|
| monkeypatch.setattr("fitcheck.config_parser.hf_hub_download", _record)
|
|
|
| assert fetch_model_config("meta-llama/Llama-3.1-8B").num_params == _LLAMA_31_8B_PARAMS
|
| assert downloaded == ["config.json"]
|
|
|
|
|
| @pytest.mark.network
|
| @pytest.mark.parametrize(
|
| ("model_id", "expected"),
|
| [
|
| ("meta-llama/Llama-3.1-8B", 8_030_261_248),
|
| ("HuggingFaceTB/SmolLM2-1.7B", 1_711_376_384),
|
| ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", 1_100_048_384),
|
| ("Qwen/Qwen2.5-1.5B", 1_543_714_304),
|
| ],
|
| )
|
| def test_num_params_matches_the_hub_for_dense_models(
|
| model_id: str, expected: int
|
| ) -> None:
|
| assert fetch_model_config(model_id).num_params == expected
|
|
|
|
|
| @pytest.mark.network
|
| @pytest.mark.parametrize(
|
| "model_id", ["microsoft/phi-2", "Qwen/Qwen2.5-VL-7B-Instruct"]
|
| )
|
| def test_architectures_outside_the_formula_are_refused_against_the_real_hub(
|
| model_id: str,
|
| ) -> None:
|
| with pytest.raises(UnsupportedModelError, match="tolerance"):
|
| fetch_model_config(model_id)
|
|
|
