"""Check models.json against scikit-learn, and that every model can be built and fitted.""" import numpy as np import pytest import json from pathlib import Path from logic import ALL_PARAMS, ESTIMATORS, MODEL_SPECS, Dataset, ModelSettings, describe_model, init_model, model_arguments, train_model SPECS = list(MODEL_SPECS.values()) PARAMS = [(spec, param) for spec in SPECS for param in spec["params"]] KINDS = {"number", "integer", "choice"} # A small dataset with two classes DATASET = Dataset( inputs=np.array([[0.0, 0.0], [0.5, 1.0], [1.0, 0.2], [0.2, 0.8], [3.0, 3.0], [3.5, 2.5], [2.8, 3.6], [3.2, 2.9]]), outputs=np.array(["a", "a", "a", "a", "b", "b", "b", "b"]), ) def ids(items): return [f"{spec['id']}.{param['name']}" if param else spec["id"] for spec, param in items] @pytest.mark.parametrize("spec", SPECS, ids=[spec["id"] for spec in SPECS]) def test_model_spec_is_complete(spec): assert spec["estimator"] in ESTIMATORS assert spec["label"] estimator_params = ESTIMATORS[spec["estimator"]]().get_params() for name in spec["fixed"]: assert name in estimator_params, f"fixed argument {name} isn't a {spec['estimator']} parameter" @pytest.mark.parametrize("spec,param", PARAMS, ids=ids(PARAMS)) def test_setting_matches_scikit_learn(spec, param): assert param["name"] in ESTIMATORS[spec["estimator"]]().get_params() assert param["kind"] in KINDS assert param["label"] if param["kind"] == "choice": assert param["default"] in [option["value"] for option in param["options"]] elif param["default"] is None: assert "nullable" in param else: assert param["min"] <= param["default"] <= param["max"] others = {p["name"]: p for p in spec["params"]} for other, allowed in param.get("when", {}).items(): assert others[other]["kind"] == "choice" assert set(allowed) <= {option["value"] for option in others[other]["options"]} @pytest.mark.parametrize("spec", SPECS, ids=[spec["id"] for spec in SPECS]) def test_model_fits_with_defaults(spec): model = train_model(init_model(ModelSettings(type=spec["id"], params={})), DATASET) assert set(model.predict(DATASET.inputs)) <= {"a", "b"} CHOICES = [(spec, param, option["value"]) for spec, param in PARAMS if param["kind"] == "choice" for option in param["options"]] @pytest.mark.parametrize("spec,param,value", CHOICES, ids=[f"{s['id']}.{p['name']}={v}" for s, p, v in CHOICES]) def test_model_fits_with_each_choice(spec, param, value): train_model(init_model(ModelSettings(type=spec["id"], params={param["name"]: value})), DATASET) def test_settings_that_dont_apply_are_left_out(): svm = MODEL_SPECS["svm"] assert "gamma" not in model_arguments(svm, {"kernel": "linear", "gamma": 5.0}) assert model_arguments(svm, {"kernel": "rbf", "gamma": 5.0})["gamma"] == 5.0 def test_invalid_settings_are_rejected(): knn, tree = MODEL_SPECS["knn"], MODEL_SPECS["decision_tree"] with pytest.raises(ValueError, match="between 1 and 30"): model_arguments(knn, {"n_neighbors": 31}) with pytest.raises(ValueError, match="whole number"): model_arguments(knn, {"n_neighbors": 2.5}) with pytest.raises(ValueError, match="Unknown settings"): model_arguments(knn, {"not_a_setting": 10}) with pytest.raises(ValueError, match="needs a value"): model_arguments(knn, {"n_neighbors": None}) assert model_arguments(tree, {"max_depth": None})["max_depth"] is None # Whole numbers may arrive as floats from JavaScript assert model_arguments(knn, {"n_neighbors": 3.0})["n_neighbors"] == 3 @pytest.mark.parametrize("spec", SPECS, ids=[spec["id"] for spec in SPECS]) def test_results_are_plain_json(spec): model = train_model(init_model(ModelSettings(type=spec["id"], params={})), DATASET) results = describe_model(model, DATASET, ["x", "y"]) assert json.loads(json.dumps(results)) == results assert [item["name"] for item in results["metrics"]] == ["Training accuracy", "Confusion matrix (rows: true, columns: predicted)"] for item in results["metrics"] + results["parameters"]: assert item["kind"] in {"scalar", "vector", "matrix", "text"} # k-NN only stores the training points, so it has nothing to report assert bool(results["parameters"]) == (spec["id"] != "knn") OTHER_PARAMS = [(spec, param) for spec in SPECS for param in ALL_PARAMS[spec["id"]]] def test_generated_settings_are_current(): """Regenerate with the scikit-learn Pyodide ships if this fails (see the script).""" import sklearn from scripts.generate_sklearn_params import generate generated = json.loads((Path(__file__).parent.parent / "src" / "sklearn_params.json").read_text()) if generated["sklearn_version"] != sklearn.__version__: pytest.skip(f"generated with scikit-learn {generated['sklearn_version']}, running {sklearn.__version__}") assert generate() == generated @pytest.mark.parametrize("spec,param", OTHER_PARAMS, ids=ids(OTHER_PARAMS)) def test_generated_setting_fits_at_its_default(spec, param): assert param["name"] in ESTIMATORS[spec["estimator"]]().get_params() shown = {p["name"] for p in spec["params"]} if param["name"] in shown: return model = init_model(ModelSettings(type=spec["id"], params={param["name"]: param["default"]})) train_model(model, DATASET) def test_other_settings_are_passed_to_scikit_learn(): svm, tree = MODEL_SPECS["svm"], MODEL_SPECS["decision_tree"] assert model_arguments(svm, {"shrinking": False})["shrinking"] is False assert model_arguments(svm, {"degree": 4.0})["degree"] == 4 assert isinstance(model_arguments(svm, {"degree": 4.0})["degree"], int) assert model_arguments(tree, {"max_features": "sqrt"})["max_features"] == "sqrt" # Fixed arguments are defaults students can change logistic = MODEL_SPECS["logistic_regression"] assert model_arguments(logistic, {})["max_iter"] == 1000 assert model_arguments(logistic, {"max_iter": 50.0})["max_iter"] == 50 assert model_arguments(tree, {"random_state": None})["random_state"] is None # scikit-learn checks the values when fitting with pytest.raises(ValueError, match="degree"): train_model(init_model(ModelSettings(type="svm", params={"degree": -1})), DATASET)