Spaces:
Running
Running
Download backend/tests/test_models.py from nhamid289/decision_boundary-dev: direct link, hf CLI and curl.
- Browser
- Download file 6.32 kB
-
https://huggingface.co/spaces/nhamid289/decision_boundary-dev/resolve/main/backend/tests/test_models.py
- Command line
-
hf download hf://spaces/nhamid289/decision_boundary-dev/backend/tests/test_models.py
-
curl -L -o test_models.py https://huggingface.co/spaces/nhamid289/decision_boundary-dev/resolve/main/backend/tests/test_models.py
6.32 kB
| """Check models.json against scikit-learn, and that every model can be built and fitted.""" | |
| import numpy as np | |
| import pytest | |
| import json | |
| from pathlib import Path | |
| from logic import ALL_PARAMS, ESTIMATORS, MODEL_SPECS, Dataset, ModelSettings, describe_model, init_model, model_arguments, train_model | |
| SPECS = list(MODEL_SPECS.values()) | |
| PARAMS = [(spec, param) for spec in SPECS for param in spec["params"]] | |
| KINDS = {"number", "integer", "choice"} | |
| # A small dataset with two classes | |
| DATASET = Dataset( | |
| inputs=np.array([[0.0, 0.0], [0.5, 1.0], [1.0, 0.2], [0.2, 0.8], [3.0, 3.0], [3.5, 2.5], [2.8, 3.6], [3.2, 2.9]]), | |
| outputs=np.array(["a", "a", "a", "a", "b", "b", "b", "b"]), | |
| ) | |
| def ids(items): | |
| return [f"{spec['id']}.{param['name']}" if param else spec["id"] for spec, param in items] | |
| def test_model_spec_is_complete(spec): | |
| assert spec["estimator"] in ESTIMATORS | |
| assert spec["label"] | |
| estimator_params = ESTIMATORS[spec["estimator"]]().get_params() | |
| for name in spec["fixed"]: | |
| assert name in estimator_params, f"fixed argument {name} isn't a {spec['estimator']} parameter" | |
| def test_setting_matches_scikit_learn(spec, param): | |
| assert param["name"] in ESTIMATORS[spec["estimator"]]().get_params() | |
| assert param["kind"] in KINDS | |
| assert param["label"] | |
| if param["kind"] == "choice": | |
| assert param["default"] in [option["value"] for option in param["options"]] | |
| elif param["default"] is None: | |
| assert "nullable" in param | |
| else: | |
| assert param["min"] <= param["default"] <= param["max"] | |
| others = {p["name"]: p for p in spec["params"]} | |
| for other, allowed in param.get("when", {}).items(): | |
| assert others[other]["kind"] == "choice" | |
| assert set(allowed) <= {option["value"] for option in others[other]["options"]} | |
| def test_model_fits_with_defaults(spec): | |
| model = train_model(init_model(ModelSettings(type=spec["id"], params={})), DATASET) | |
| assert set(model.predict(DATASET.inputs)) <= {"a", "b"} | |
| CHOICES = [(spec, param, option["value"]) for spec, param in PARAMS if param["kind"] == "choice" for option in param["options"]] | |
| def test_model_fits_with_each_choice(spec, param, value): | |
| train_model(init_model(ModelSettings(type=spec["id"], params={param["name"]: value})), DATASET) | |
| def test_settings_that_dont_apply_are_left_out(): | |
| svm = MODEL_SPECS["svm"] | |
| assert "gamma" not in model_arguments(svm, {"kernel": "linear", "gamma": 5.0}) | |
| assert model_arguments(svm, {"kernel": "rbf", "gamma": 5.0})["gamma"] == 5.0 | |
| def test_invalid_settings_are_rejected(): | |
| knn, tree = MODEL_SPECS["knn"], MODEL_SPECS["decision_tree"] | |
| with pytest.raises(ValueError, match="between 1 and 30"): | |
| model_arguments(knn, {"n_neighbors": 31}) | |
| with pytest.raises(ValueError, match="whole number"): | |
| model_arguments(knn, {"n_neighbors": 2.5}) | |
| with pytest.raises(ValueError, match="Unknown settings"): | |
| model_arguments(knn, {"not_a_setting": 10}) | |
| with pytest.raises(ValueError, match="needs a value"): | |
| model_arguments(knn, {"n_neighbors": None}) | |
| assert model_arguments(tree, {"max_depth": None})["max_depth"] is None | |
| # Whole numbers may arrive as floats from JavaScript | |
| assert model_arguments(knn, {"n_neighbors": 3.0})["n_neighbors"] == 3 | |
| def test_results_are_plain_json(spec): | |
| model = train_model(init_model(ModelSettings(type=spec["id"], params={})), DATASET) | |
| results = describe_model(model, DATASET, ["x", "y"]) | |
| assert json.loads(json.dumps(results)) == results | |
| assert [item["name"] for item in results["metrics"]] == ["Training accuracy", "Confusion matrix (rows: true, columns: predicted)"] | |
| for item in results["metrics"] + results["parameters"]: | |
| assert item["kind"] in {"scalar", "vector", "matrix", "text"} | |
| # k-NN only stores the training points, so it has nothing to report | |
| assert bool(results["parameters"]) == (spec["id"] != "knn") | |
| OTHER_PARAMS = [(spec, param) for spec in SPECS for param in ALL_PARAMS[spec["id"]]] | |
| def test_generated_settings_are_current(): | |
| """Regenerate with the scikit-learn Pyodide ships if this fails (see the script).""" | |
| import sklearn | |
| from scripts.generate_sklearn_params import generate | |
| generated = json.loads((Path(__file__).parent.parent / "src" / "sklearn_params.json").read_text()) | |
| if generated["sklearn_version"] != sklearn.__version__: | |
| pytest.skip(f"generated with scikit-learn {generated['sklearn_version']}, running {sklearn.__version__}") | |
| assert generate() == generated | |
| def test_generated_setting_fits_at_its_default(spec, param): | |
| assert param["name"] in ESTIMATORS[spec["estimator"]]().get_params() | |
| shown = {p["name"] for p in spec["params"]} | |
| if param["name"] in shown: | |
| return | |
| model = init_model(ModelSettings(type=spec["id"], params={param["name"]: param["default"]})) | |
| train_model(model, DATASET) | |
| def test_other_settings_are_passed_to_scikit_learn(): | |
| svm, tree = MODEL_SPECS["svm"], MODEL_SPECS["decision_tree"] | |
| assert model_arguments(svm, {"shrinking": False})["shrinking"] is False | |
| assert model_arguments(svm, {"degree": 4.0})["degree"] == 4 | |
| assert isinstance(model_arguments(svm, {"degree": 4.0})["degree"], int) | |
| assert model_arguments(tree, {"max_features": "sqrt"})["max_features"] == "sqrt" | |
| # Fixed arguments are defaults students can change | |
| logistic = MODEL_SPECS["logistic_regression"] | |
| assert model_arguments(logistic, {})["max_iter"] == 1000 | |
| assert model_arguments(logistic, {"max_iter": 50.0})["max_iter"] == 50 | |
| assert model_arguments(tree, {"random_state": None})["random_state"] is None | |
| # scikit-learn checks the values when fitting | |
| with pytest.raises(ValueError, match="degree"): | |
| train_model(init_model(ModelSettings(type="svm", params={"degree": -1})), DATASET) | |