decision_boundary-dev / backend /tests /test_models.py
nhamid289's picture
Deploy decision_boundary: two-tab panel, dataset popup, All settings popup
cf8ab7a verified
Raw History Blame Contribute Delete
6.32 kB
"""Check models.json against scikit-learn, and that every model can be built and fitted."""
import numpy as np
import pytest
import json
from pathlib import Path
from logic import ALL_PARAMS, ESTIMATORS, MODEL_SPECS, Dataset, ModelSettings, describe_model, init_model, model_arguments, train_model
SPECS = list(MODEL_SPECS.values())
PARAMS = [(spec, param) for spec in SPECS for param in spec["params"]]
KINDS = {"number", "integer", "choice"}
# A small dataset with two classes
DATASET = Dataset(
inputs=np.array([[0.0, 0.0], [0.5, 1.0], [1.0, 0.2], [0.2, 0.8], [3.0, 3.0], [3.5, 2.5], [2.8, 3.6], [3.2, 2.9]]),
outputs=np.array(["a", "a", "a", "a", "b", "b", "b", "b"]),
)
def ids(items):
return [f"{spec['id']}.{param['name']}" if param else spec["id"] for spec, param in items]
@pytest.mark.parametrize("spec", SPECS, ids=[spec["id"] for spec in SPECS])
def test_model_spec_is_complete(spec):
assert spec["estimator"] in ESTIMATORS
assert spec["label"]
estimator_params = ESTIMATORS[spec["estimator"]]().get_params()
for name in spec["fixed"]:
assert name in estimator_params, f"fixed argument {name} isn't a {spec['estimator']} parameter"
@pytest.mark.parametrize("spec,param", PARAMS, ids=ids(PARAMS))
def test_setting_matches_scikit_learn(spec, param):
assert param["name"] in ESTIMATORS[spec["estimator"]]().get_params()
assert param["kind"] in KINDS
assert param["label"]
if param["kind"] == "choice":
assert param["default"] in [option["value"] for option in param["options"]]
elif param["default"] is None:
assert "nullable" in param
else:
assert param["min"] <= param["default"] <= param["max"]
others = {p["name"]: p for p in spec["params"]}
for other, allowed in param.get("when", {}).items():
assert others[other]["kind"] == "choice"
assert set(allowed) <= {option["value"] for option in others[other]["options"]}
@pytest.mark.parametrize("spec", SPECS, ids=[spec["id"] for spec in SPECS])
def test_model_fits_with_defaults(spec):
model = train_model(init_model(ModelSettings(type=spec["id"], params={})), DATASET)
assert set(model.predict(DATASET.inputs)) <= {"a", "b"}
CHOICES = [(spec, param, option["value"]) for spec, param in PARAMS if param["kind"] == "choice" for option in param["options"]]
@pytest.mark.parametrize("spec,param,value", CHOICES, ids=[f"{s['id']}.{p['name']}={v}" for s, p, v in CHOICES])
def test_model_fits_with_each_choice(spec, param, value):
train_model(init_model(ModelSettings(type=spec["id"], params={param["name"]: value})), DATASET)
def test_settings_that_dont_apply_are_left_out():
svm = MODEL_SPECS["svm"]
assert "gamma" not in model_arguments(svm, {"kernel": "linear", "gamma": 5.0})
assert model_arguments(svm, {"kernel": "rbf", "gamma": 5.0})["gamma"] == 5.0
def test_invalid_settings_are_rejected():
knn, tree = MODEL_SPECS["knn"], MODEL_SPECS["decision_tree"]
with pytest.raises(ValueError, match="between 1 and 30"):
model_arguments(knn, {"n_neighbors": 31})
with pytest.raises(ValueError, match="whole number"):
model_arguments(knn, {"n_neighbors": 2.5})
with pytest.raises(ValueError, match="Unknown settings"):
model_arguments(knn, {"not_a_setting": 10})
with pytest.raises(ValueError, match="needs a value"):
model_arguments(knn, {"n_neighbors": None})
assert model_arguments(tree, {"max_depth": None})["max_depth"] is None
# Whole numbers may arrive as floats from JavaScript
assert model_arguments(knn, {"n_neighbors": 3.0})["n_neighbors"] == 3
@pytest.mark.parametrize("spec", SPECS, ids=[spec["id"] for spec in SPECS])
def test_results_are_plain_json(spec):
model = train_model(init_model(ModelSettings(type=spec["id"], params={})), DATASET)
results = describe_model(model, DATASET, ["x", "y"])
assert json.loads(json.dumps(results)) == results
assert [item["name"] for item in results["metrics"]] == ["Training accuracy", "Confusion matrix (rows: true, columns: predicted)"]
for item in results["metrics"] + results["parameters"]:
assert item["kind"] in {"scalar", "vector", "matrix", "text"}
# k-NN only stores the training points, so it has nothing to report
assert bool(results["parameters"]) == (spec["id"] != "knn")
OTHER_PARAMS = [(spec, param) for spec in SPECS for param in ALL_PARAMS[spec["id"]]]
def test_generated_settings_are_current():
"""Regenerate with the scikit-learn Pyodide ships if this fails (see the script)."""
import sklearn
from scripts.generate_sklearn_params import generate
generated = json.loads((Path(__file__).parent.parent / "src" / "sklearn_params.json").read_text())
if generated["sklearn_version"] != sklearn.__version__:
pytest.skip(f"generated with scikit-learn {generated['sklearn_version']}, running {sklearn.__version__}")
assert generate() == generated
@pytest.mark.parametrize("spec,param", OTHER_PARAMS, ids=ids(OTHER_PARAMS))
def test_generated_setting_fits_at_its_default(spec, param):
assert param["name"] in ESTIMATORS[spec["estimator"]]().get_params()
shown = {p["name"] for p in spec["params"]}
if param["name"] in shown:
return
model = init_model(ModelSettings(type=spec["id"], params={param["name"]: param["default"]}))
train_model(model, DATASET)
def test_other_settings_are_passed_to_scikit_learn():
svm, tree = MODEL_SPECS["svm"], MODEL_SPECS["decision_tree"]
assert model_arguments(svm, {"shrinking": False})["shrinking"] is False
assert model_arguments(svm, {"degree": 4.0})["degree"] == 4
assert isinstance(model_arguments(svm, {"degree": 4.0})["degree"], int)
assert model_arguments(tree, {"max_features": "sqrt"})["max_features"] == "sqrt"
# Fixed arguments are defaults students can change
logistic = MODEL_SPECS["logistic_regression"]
assert model_arguments(logistic, {})["max_iter"] == 1000
assert model_arguments(logistic, {"max_iter": 50.0})["max_iter"] == 50
assert model_arguments(tree, {"random_state": None})["random_state"] is None
# scikit-learn checks the values when fitting
with pytest.raises(ValueError, match="degree"):
train_model(init_model(ModelSettings(type="svm", params={"degree": -1})), DATASET)