import copy import json from pathlib import Path import pytest import yaml from jsonschema.validators import Draft202012Validator from dlb.registry import load_registry @pytest.fixture def registry(): return load_registry(Path("configs/experiments.yaml")) @pytest.fixture def schema_validator(): schema = json.loads(Path("configs/schema.json").read_text()) return Draft202012Validator(schema) @pytest.fixture def canonical_registry_document(): return yaml.safe_load(Path("configs/experiments.yaml").read_text()) def test_registry_contains_full_scope(registry): assert set(registry.models) == { "flm", "langflow", "duo", "mdlm", "candi", "rdlm", "fmlm", "duo_dcd", "duo_di4c", "mdlm_sdtt", "mdlm_di4c", } assert registry.step_grids["many"] == [1, 2, 4, 8, 16, 32, 1024] assert registry.step_grids["few"] == [1, 2, 4, 8, 16, 32] assert registry.step_grids["fixed_1024"] == [1024] assert registry.models["rdlm"].step_override == [1000, 1024] assert registry.models["flm"].step_override is None def test_unsupported_cells_have_reason(registry): for model in registry.models.values(): for support in model.datasets.values(): if support.status == "unsupported": assert len(support.reason) >= 20 def test_supported_cells_keep_provenance_separate_from_status(registry): supported = registry.models["duo"].datasets["lm1b"] assert supported.status == "supported" assert supported.provenance == "reference_reproduction" assert supported.reason is None def test_loader_rejects_unsupported_cell_with_provenance(tmp_path): registry_config = tmp_path / "invalid.yaml" registry_config.write_text( """ step_grids: many: [1, 2, 4, 8, 16, 32, 1024] few: [1, 2, 4, 8, 16, 32] models: flm: category: many environment: dlb-flm adapter: flm source: flm datasets: lm1b: status: unsupported reason: Unsupported for a clearly documented project reason. provenance: official """.strip() ) with pytest.raises(ValueError): load_registry(registry_config) def test_loader_rejects_wrong_adapter_for_known_model(tmp_path): registry_config = tmp_path / "invalid.yaml" registry_config.write_text( Path("configs/experiments.yaml") .read_text() .replace(" adapter: flm", " adapter: wrong-adapter", 1) ) with pytest.raises(ValueError): load_registry(registry_config) def test_json_schema_accepts_canonical_registry( schema_validator, canonical_registry_document ): assert list(schema_validator.iter_errors(canonical_registry_document)) == [] @pytest.mark.parametrize( ("status", "provenance", "reason"), [ ("unsupported", "official", "A documented unsupported reason."), ("unsupported", None, None), ("supported", None, None), ("supported", "official", "A reason must not accompany support."), ], ) def test_json_schema_rejects_invalid_support_combinations( schema_validator, canonical_registry_document, status, provenance, reason ): document = copy.deepcopy(canonical_registry_document) support = document["models"]["flm"]["datasets"]["lm1b"] support.clear() support["status"] = status if provenance is not None: support["provenance"] = provenance if reason is not None: support["reason"] = reason assert list(schema_validator.iter_errors(document))