nhamid289's picture
Deploy decision_boundary: two-tab panel, dataset popup, All settings popup
cf8ab7a verified
Raw History Blame Contribute Delete
10.3 kB
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
import numpy as np
from sklearn.base import BaseEstimator
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier, export_text
ESTIMATORS = {
"LogisticRegression": LogisticRegression,
"KNeighborsClassifier": KNeighborsClassifier,
"SVC": SVC,
"DecisionTreeClassifier": DecisionTreeClassifier,
"RandomForestClassifier": RandomForestClassifier,
"GradientBoostingClassifier": GradientBoostingClassifier,
}
# The models students can choose and their settings, shared with the frontend
MODEL_SPECS = {
model["id"]: model
for model in json.loads((Path(__file__).parent / "models.json").read_text())["models"]
}
# Every scikit-learn setting of each model, by model id (generated by
# scripts/generate_sklearn_params.py); models.json picks the ones shown first
ALL_PARAMS = json.loads((Path(__file__).parent / "sklearn_params.json").read_text())["models"]
@dataclass
class Dataset:
inputs: np.ndarray
outputs: np.ndarray
@dataclass
class ModelSettings:
# A model id from models.json
type: str
# Setting name -> value; missing settings use their defaults
params: dict[str, Any]
@dataclass
class DecisionBoundarySettings:
xmin: float
xmax: float
ymin: float
ymax: float
resolution: int = 100
@dataclass
class Projection:
"""Linear map from plot coordinates z back to inputs: x = origin + z @ components."""
origin: np.ndarray
components: np.ndarray
def lift(self, points: np.ndarray) -> np.ndarray:
return self.origin + points @ self.components
@dataclass
class DecisionBoundaryResult:
x: list[float]
y: list[float]
labels: list[list[str]]
def model_arguments(spec: dict, params: dict[str, Any]) -> dict[str, Any]:
"""Keyword arguments for the estimator: its fixed ones, any other scikit-learn
settings given (checked by scikit-learn itself), plus each models.json setting
that applies (see "when"), from params or the setting's default."""
shown = {param["name"] for param in spec["params"]}
others = {param["name"]: param for param in ALL_PARAMS.get(spec["id"], []) if param["name"] not in shown}
unknown = set(params) - shown - set(others)
if unknown:
raise ValueError(f"Unknown settings for {spec['id']}: {', '.join(sorted(unknown))}")
values = {param["name"]: params.get(param["name"], param["default"]) for param in spec["params"]}
arguments = dict(spec["fixed"])
for name, value in params.items():
if name in others:
arguments[name] = _whole_if_integer(others[name], value)
for param in spec["params"]:
when = param.get("when", {})
if all(values[other] in allowed for other, allowed in when.items()):
arguments[param["name"]] = _checked_value(param, values[param["name"]])
return arguments
def _whole_if_integer(param: dict, value: Any) -> Any:
# Whole numbers may arrive as floats from JavaScript
integer = param["kind"] == "integer" or param.get("number", {}).get("integer", False)
if integer and isinstance(value, float) and value.is_integer():
return int(value)
return value
def _checked_value(param: dict, value: Any) -> Any:
label = param["label"]
if param["kind"] == "choice":
if value not in [option["value"] for option in param["options"]]:
raise ValueError(f"{label} can't be {value!r}")
return value
if value is None:
if "nullable" not in param:
raise ValueError(f"{label} needs a value")
return None
number = float(value)
if param["kind"] == "integer":
if not number.is_integer():
raise ValueError(f"{label} must be a whole number")
number = int(number)
if not param["min"] <= number <= param["max"]:
raise ValueError(f"{label} must be between {param['min']} and {param['max']}")
return number
def init_model(settings: ModelSettings) -> BaseEstimator:
spec = MODEL_SPECS.get(settings.type)
if spec is None:
raise ValueError(f"Unsupported model type: {settings.type}")
return ESTIMATORS[spec["estimator"]](**model_arguments(spec, settings.params))
def train_model(model: BaseEstimator, dataset: Dataset) -> BaseEstimator:
model.fit(dataset.inputs, dataset.outputs)
return model
# Results shown after fitting, as {name, kind, value} items:
# scalar: a number or string
# vector: {labels, values}, one value per label
# matrix: {rows, columns, values}
# text: a block of text (shown collapsed)
def _number(value: Any) -> int | float:
"""A plain Python number; whole-number types (counts, depths) stay integers."""
return int(value) if isinstance(value, (int, np.integer)) else float(value)
def _scalar(name: str, value: Any) -> dict:
return {"name": name, "kind": "scalar", "value": value if isinstance(value, str) else _number(value)}
def _vector(name: str, labels: list[str], values: Any) -> dict:
return {"name": name, "kind": "vector", "value": {"labels": list(labels), "values": [_number(v) for v in values]}}
def _matrix(name: str, rows: list[str], columns: list[str], values: Any) -> dict:
return {"name": name, "kind": "matrix", "value": {"rows": list(rows), "columns": list(columns), "values": [[_number(v) for v in row] for row in values]}}
def _text(name: str, value: str) -> dict:
return {"name": name, "kind": "text", "value": value}
def describe_model(model: BaseEstimator, dataset: Dataset, input_names: list[str]) -> dict:
"""Metrics on the training data, and what the model learned."""
classes = [str(c) for c in model.classes_]
predictions = model.predict(dataset.inputs)
metrics = [
_scalar("Training accuracy", np.mean(predictions == dataset.outputs)),
_matrix("Confusion matrix (rows: true, columns: predicted)", classes, classes,
confusion_matrix(dataset.outputs, predictions, labels=model.classes_)),
]
return {"metrics": metrics, "parameters": _learned_parameters(model, classes, input_names)}
def _learned_parameters(model: BaseEstimator, classes: list[str], input_names: list[str]) -> list[dict]:
if isinstance(model, LogisticRegression):
return _linear_parameters(model.coef_, model.intercept_, classes, input_names)
if isinstance(model, KNeighborsClassifier):
# Nothing is learned beyond storing the training points
return []
if isinstance(model, SVC):
items = [_vector("Support vectors per class", classes, model.n_support_)]
if model.kernel == "linear":
# One-vs-one: one boundary per pair of classes
pairs = [f"{a} vs {b}" for i, a in enumerate(classes) for b in classes[i + 1:]]
items += _linear_parameters(model.coef_, model.intercept_, pairs, input_names)
if len(classes) == 2:
items.append(_scalar("Margin width", 2 / np.linalg.norm(model.coef_[0])))
return items
if isinstance(model, DecisionTreeClassifier):
return [
_scalar("Depth", int(model.get_depth())),
_scalar("Leaves", int(model.get_n_leaves())),
_vector("Feature importances", input_names, model.feature_importances_),
_text("Tree", export_text(model, feature_names=input_names)),
]
if isinstance(model, RandomForestClassifier):
return [
_scalar("Trees", int(len(model.estimators_))),
_scalar("Average depth", np.mean([tree.get_depth() for tree in model.estimators_])),
_vector("Feature importances", input_names, model.feature_importances_),
]
if isinstance(model, GradientBoostingClassifier):
return [
_scalar("Stages", int(model.n_estimators_)),
_scalar("Final training loss", model.train_score_[-1]),
_vector("Feature importances", input_names, model.feature_importances_),
]
return []
def _linear_parameters(coef: np.ndarray, intercept: np.ndarray, rows: list[str], input_names: list[str]) -> list[dict]:
"""Weights and biases of linear boundaries; rows names each boundary when there are several."""
if len(coef) > 1:
return [_matrix("Weights", rows, input_names, coef), _vector("Bias", rows, intercept)]
# Terms far smaller than the largest are numerical noise; show them as 0
scale = max(np.max(np.abs(coef[0])), abs(intercept[0]))
def term(value: float) -> str:
value = 0.0 if abs(value) < 1e-4 * scale else value
return f"{'−' if value < 0 else '+'} {abs(value):.3g}"
terms = " ".join(f"{term(w)}·{name}" for w, name in zip(coef[0], input_names))
equation = f"{terms.removeprefix('+ ')} {term(intercept[0])} = 0"
bias = intercept[0]
return [_vector("Weights", input_names, coef[0]), _scalar("Bias", bias), _scalar("Boundary", equation)]
def get_decision_boundary_values(
model: BaseEstimator,
settings: DecisionBoundarySettings,
inverse_projection: Callable[[np.ndarray], np.ndarray] | None = None,
) -> DecisionBoundaryResult:
if (settings.xmin >= settings.xmax) or (settings.ymin >= settings.ymax):
raise ValueError("Invalid boundary settings: min values must be less than max values.")
x_values = np.linspace(settings.xmin, settings.xmax, settings.resolution)
y_values = np.linspace(settings.ymin, settings.ymax, settings.resolution)
xx, yy = np.meshgrid(x_values, y_values)
grid_points = np.c_[xx.ravel(), yy.ravel()]
try:
if inverse_projection is not None:
grid_points = inverse_projection(grid_points)
predictions = model.predict(grid_points)
except Exception as e:
raise ValueError(f"Error generating predictions: {e}")
labels = predictions.reshape(xx.shape).tolist()
return DecisionBoundaryResult(
x=x_values.tolist(),
y=y_values.tolist(),
labels=labels,
)