File size: 3,570 Bytes
784f572 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | import copy
import json
from pathlib import Path
import pytest
import yaml
from jsonschema.validators import Draft202012Validator
from dlb.registry import load_registry
@pytest.fixture
def registry():
return load_registry(Path("configs/experiments.yaml"))
@pytest.fixture
def schema_validator():
schema = json.loads(Path("configs/schema.json").read_text())
return Draft202012Validator(schema)
@pytest.fixture
def canonical_registry_document():
return yaml.safe_load(Path("configs/experiments.yaml").read_text())
def test_registry_contains_full_scope(registry):
assert set(registry.models) == {
"flm",
"langflow",
"duo",
"mdlm",
"candi",
"rdlm",
"fmlm",
"duo_dcd",
"duo_di4c",
"mdlm_sdtt",
"mdlm_di4c",
}
assert registry.step_grids["many"] == [1, 2, 4, 8, 16, 32, 1024]
assert registry.step_grids["few"] == [1, 2, 4, 8, 16, 32]
assert registry.step_grids["fixed_1024"] == [1024]
assert registry.models["rdlm"].step_override == [1000, 1024]
assert registry.models["flm"].step_override is None
def test_unsupported_cells_have_reason(registry):
for model in registry.models.values():
for support in model.datasets.values():
if support.status == "unsupported":
assert len(support.reason) >= 20
def test_supported_cells_keep_provenance_separate_from_status(registry):
supported = registry.models["duo"].datasets["lm1b"]
assert supported.status == "supported"
assert supported.provenance == "reference_reproduction"
assert supported.reason is None
def test_loader_rejects_unsupported_cell_with_provenance(tmp_path):
registry_config = tmp_path / "invalid.yaml"
registry_config.write_text(
"""
step_grids:
many: [1, 2, 4, 8, 16, 32, 1024]
few: [1, 2, 4, 8, 16, 32]
models:
flm:
category: many
environment: dlb-flm
adapter: flm
source: flm
datasets:
lm1b:
status: unsupported
reason: Unsupported for a clearly documented project reason.
provenance: official
""".strip()
)
with pytest.raises(ValueError):
load_registry(registry_config)
def test_loader_rejects_wrong_adapter_for_known_model(tmp_path):
registry_config = tmp_path / "invalid.yaml"
registry_config.write_text(
Path("configs/experiments.yaml")
.read_text()
.replace(" adapter: flm", " adapter: wrong-adapter", 1)
)
with pytest.raises(ValueError):
load_registry(registry_config)
def test_json_schema_accepts_canonical_registry(
schema_validator, canonical_registry_document
):
assert list(schema_validator.iter_errors(canonical_registry_document)) == []
@pytest.mark.parametrize(
("status", "provenance", "reason"),
[
("unsupported", "official", "A documented unsupported reason."),
("unsupported", None, None),
("supported", None, None),
("supported", "official", "A reason must not accompany support."),
],
)
def test_json_schema_rejects_invalid_support_combinations(
schema_validator, canonical_registry_document, status, provenance, reason
):
document = copy.deepcopy(canonical_registry_document)
support = document["models"]["flm"]["datasets"]["lm1b"]
support.clear()
support["status"] = status
if provenance is not None:
support["provenance"] = provenance
if reason is not None:
support["reason"] = reason
assert list(schema_validator.iter_errors(document))
|