Spaces:
Sleeping
Sleeping
Deploy TinyMetatron SLM read-only demo (build-time trained checkpoint, deploy-mode gating)
3deca49 verified Download tests/test_model.py from Quantum927/tinymetatron-slm: direct link, hf CLI and curl.
- Browser
- Download file 4.13 kB
-
https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tests/test_model.py
- Command line
-
hf download hf://spaces/Quantum927/tinymetatron-slm/tests/test_model.py
-
curl -L -o test_model.py https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tests/test_model.py
4.13 kB
| """ | |
| tests/test_model.py | |
| ================== | |
| ECOSYSTEM pytest suite for the TinyMetatron SLM (contract section 2). | |
| Covers (contract section 5 / test descriptions): | |
| * forward(input_ids[B,32]) -> logits shape (B, 32, 291) | |
| * total_params in [5e6, 7e6] via param_count() | |
| * generate(...) returns ids in [0, 291) | |
| * one forward + backward produces a finite loss | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import sys | |
| import math | |
| _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if _REPO_ROOT not in sys.path: | |
| sys.path.insert(0, _REPO_ROOT) | |
| import pytest | |
| import torch | |
| import torch.nn.functional as F | |
| from config import CONFIG | |
| from tinymetatron_model import TinyMetatron | |
| def model() -> TinyMetatron: | |
| torch.manual_seed(CONFIG["seed"]) | |
| m = TinyMetatron(CONFIG) | |
| return m | |
| # ββ forward shape βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def test_forward_shape(model: TinyMetatron) -> None: | |
| B, L = 2, CONFIG["seq_len"] # seq_len == 32 | |
| input_ids = torch.randint(0, CONFIG["vocab_size"], (B, L)) | |
| logits, aux = model(input_ids) | |
| assert logits.shape == (B, L, CONFIG["vocab_size"]), ( | |
| f"logits shape {tuple(logits.shape)} != {(B, L, CONFIG['vocab_size'])}") | |
| assert aux.shape == (), f"aux_loss must be scalar, got {tuple(aux.shape)}" | |
| # ββ parameter budget ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def test_param_count_in_budget(model: TinyMetatron) -> None: | |
| pc = model.param_count() | |
| total = pc["total"] | |
| assert isinstance(total, int) | |
| assert 5_000_000 <= total <= 7_000_000, ( | |
| f"total params {total} outside [5e6, 7e6]") | |
| assert pc["in_5_7M_budget"] is True | |
| # active MoE per token must be a small fraction of the total. | |
| assert pc["active_moe_per_token"] < total | |
| # ββ generate returns ids in [0, vocab_size) ββββββββββββββββββββββββββββββββββββ | |
| def test_generate_ids_in_vocab(model: TinyMetatron) -> None: | |
| B = 2 | |
| prompt_len = 4 | |
| prompt = torch.randint(0, CONFIG["vocab_size"], (B, prompt_len)) | |
| out = model.generate(prompt, max_length=5, temperature=0.7) | |
| assert out.shape[0] == B | |
| # at least the prompt, at most prompt + max_length | |
| assert prompt_len <= out.shape[1] <= prompt_len + 5 | |
| assert int(out.min().item()) >= 0 | |
| assert int(out.max().item()) < CONFIG["vocab_size"] | |
| # ββ forward + backward produces finite loss βββββββββββββββββββββββββββββββββββ | |
| def test_forward_backward_finite_loss(model: TinyMetatron) -> None: | |
| B, L = 2, CONFIG["seq_len"] | |
| input_ids = torch.randint(0, CONFIG["vocab_size"], (B, L)) | |
| target = torch.randint(0, CONFIG["vocab_size"], (B, L)) | |
| logits, aux = model(input_ids) | |
| ce = F.cross_entropy(logits.view(-1, CONFIG["vocab_size"]), | |
| target.view(-1)) | |
| total = ce + CONFIG["moe_aux_loss_weight"] * aux | |
| assert math.isfinite(float(total.item())), ( | |
| f"total loss not finite: {total.item()}") | |
| total.backward() | |
| # At least one parameter must have a non-None gradient after backward. | |
| grads = [p.grad for p in model.parameters() if p.grad is not None] | |
| assert len(grads) > 0, "no gradients produced after backward" | |
| # All present grads must be finite. | |
| for g in grads: | |
| assert torch.isfinite(g).all().item(), "non-finite gradient encountered" | |
| # ββ from_config builds an equivalent model ββββββββββββββββββββββββββββββββββββ | |
| def test_from_config(model: TinyMetatron) -> None: | |
| m2 = TinyMetatron.from_config() | |
| pc2 = m2.param_count() | |
| assert pc2["total"] == model.param_count()["total"] | |
| assert 5_000_000 <= pc2["total"] <= 7_000_000 |