Add loan-advisor-agent
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +13 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/msvcp140.dll +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/vcomp140.dll +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__init__.py +54 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.lib +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.pyx +2 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/meson.build +6 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__init__.py +150 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_config.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_distributor_init.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/base.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/exceptions.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/tempita.py +62 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/version.py +16 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_config.py +407 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.lib +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_distributor_init.py +22 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.lib +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.pyx +116 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__init__.py +33 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/link.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/loss.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.lib +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pxd +101 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pyx.tp +1505 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/link.py +282 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/loss.py +1216 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/meson.build +23 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_link.py +111 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_loss.py +1358 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/_min_dependencies.py +74 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/base.py +1340 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/calibration.py +1709 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/__init__.py +64 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_affinity_propagation.py +607 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_agglomerative.py +1335 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bicluster.py +622 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_birch.py +730 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bisect_k_means.py +543 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan.py +512 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan_inner.cp312-win_amd64.lib +0 -0
.gitattributes
CHANGED
|
@@ -663,3 +663,16 @@ loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_stats.cp312-win_amd64.py
|
|
| 663 |
loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_stats_pythran.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 664 |
loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_unuran/unuran_wrapper.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 665 |
loan-advisor-agent/.venv/Lib/site-packages/scipy.libs/libscipy_openblas-64eda39e79589aedb16f58e5547eb599.dll filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 663 |
loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_stats_pythran.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 664 |
loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_unuran/unuran_wrapper.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 665 |
loan-advisor-agent/.venv/Lib/site-packages/scipy.libs/libscipy_openblas-64eda39e79589aedb16f58e5547eb599.dll filter=lfs diff=lfs merge=lfs -text
|
| 666 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/msvcp140.dll filter=lfs diff=lfs merge=lfs -text
|
| 667 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/vcomp140.dll filter=lfs diff=lfs merge=lfs -text
|
| 668 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 669 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 670 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_hdbscan/_reachability.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 671 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_hdbscan/_tree.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 672 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_hierarchical_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 673 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_common.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 674 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_elkan.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 675 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_lloyd.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 676 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/_svmlight_format_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 677 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/china.jpg filter=lfs diff=lfs merge=lfs -text
|
| 678 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/flower.jpg filter=lfs diff=lfs merge=lfs -text
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/msvcp140.dll
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0f885b509a685d2bbfa652fed26b5fb31d88fbdab0a978c641d1c7b8aa460aa9
|
| 3 |
+
size 557728
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/vcomp140.dll
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:55aba23cdcd6484fbb06f4155b8ca75adfce7a881f10afd0c49457165e677164
|
| 3 |
+
size 193152
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__init__.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module to give helpful messages to the user that did not
|
| 2 |
+
compile scikit-learn properly.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
# Authors: The scikit-learn developers
|
| 6 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 7 |
+
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
INPLACE_MSG = """
|
| 11 |
+
It appears that you are importing a local scikit-learn source tree. For
|
| 12 |
+
this, you need to have an inplace install. Maybe you are in the source
|
| 13 |
+
directory and you need to try from another location."""
|
| 14 |
+
|
| 15 |
+
STANDARD_MSG = """
|
| 16 |
+
If you have used an installer, please check that it is suited for your
|
| 17 |
+
Python version, your operating system and your platform."""
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def raise_build_error(e):
|
| 21 |
+
# Raise a comprehensible error and list the contents of the
|
| 22 |
+
# directory to help debugging on the mailing list.
|
| 23 |
+
local_dir = os.path.split(__file__)[0]
|
| 24 |
+
msg = STANDARD_MSG
|
| 25 |
+
if local_dir == "sklearn/__check_build":
|
| 26 |
+
# Picking up the local install: this will work only if the
|
| 27 |
+
# install is an 'inplace build'
|
| 28 |
+
msg = INPLACE_MSG
|
| 29 |
+
dir_content = list()
|
| 30 |
+
for i, filename in enumerate(os.listdir(local_dir)):
|
| 31 |
+
if (i + 1) % 3:
|
| 32 |
+
dir_content.append(filename.ljust(26))
|
| 33 |
+
else:
|
| 34 |
+
dir_content.append(filename + "\n")
|
| 35 |
+
raise ImportError(
|
| 36 |
+
"""%s
|
| 37 |
+
___________________________________________________________________________
|
| 38 |
+
Contents of %s:
|
| 39 |
+
%s
|
| 40 |
+
___________________________________________________________________________
|
| 41 |
+
It seems that scikit-learn has not been built correctly.
|
| 42 |
+
|
| 43 |
+
If you have installed scikit-learn from source, please do not forget
|
| 44 |
+
to build the package before using it. For detailed instructions, see:
|
| 45 |
+
https://scikit-learn.org/dev/developers/advanced_installation.html#building-from-source
|
| 46 |
+
%s"""
|
| 47 |
+
% (e, local_dir, "".join(dir_content).strip(), msg)
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
try:
|
| 52 |
+
from sklearn.__check_build._check_build import check_build # noqa: F401
|
| 53 |
+
except ImportError as e:
|
| 54 |
+
raise_build_error(e)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (2.26 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.lib
ADDED
|
Binary file (2.1 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.pyd
ADDED
|
Binary file (28.7 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.pyx
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def check_build():
|
| 2 |
+
return
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/meson.build
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
py.extension_module(
|
| 2 |
+
'_check_build',
|
| 3 |
+
cython_gen.process('_check_build.pyx'),
|
| 4 |
+
install: true,
|
| 5 |
+
subdir: 'sklearn/__check_build',
|
| 6 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__init__.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configure global settings and get information about the working environment."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
# Machine learning module for Python
|
| 7 |
+
# ==================================
|
| 8 |
+
#
|
| 9 |
+
# sklearn is a Python module integrating classical machine
|
| 10 |
+
# learning algorithms in the tightly-knit world of scientific Python
|
| 11 |
+
# packages (numpy, scipy, matplotlib).
|
| 12 |
+
#
|
| 13 |
+
# It aims to provide simple and efficient solutions to learning problems
|
| 14 |
+
# that are accessible to everybody and reusable in various contexts:
|
| 15 |
+
# machine-learning as a versatile tool for science and engineering.
|
| 16 |
+
#
|
| 17 |
+
# See https://scikit-learn.org for complete documentation.
|
| 18 |
+
|
| 19 |
+
import importlib as _importlib
|
| 20 |
+
import logging
|
| 21 |
+
import os
|
| 22 |
+
import random
|
| 23 |
+
|
| 24 |
+
from sklearn._config import config_context, get_config, set_config
|
| 25 |
+
|
| 26 |
+
logger = logging.getLogger(__name__)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
# PEP0440 compatible formatted version, see:
|
| 30 |
+
# https://www.python.org/dev/peps/pep-0440/
|
| 31 |
+
#
|
| 32 |
+
# Generic release markers:
|
| 33 |
+
# X.Y.0 # For first release after an increment in Y
|
| 34 |
+
# X.Y.Z # For bugfix releases
|
| 35 |
+
#
|
| 36 |
+
# Admissible pre-release markers:
|
| 37 |
+
# X.Y.ZaN # Alpha release
|
| 38 |
+
# X.Y.ZbN # Beta release
|
| 39 |
+
# X.Y.ZrcN # Release Candidate
|
| 40 |
+
# X.Y.Z # Final release
|
| 41 |
+
#
|
| 42 |
+
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
|
| 43 |
+
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
|
| 44 |
+
#
|
| 45 |
+
__version__ = "1.8.0"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# On OSX, we can get a runtime error due to multiple OpenMP libraries loaded
|
| 49 |
+
# simultaneously. This can happen for instance when calling BLAS inside a
|
| 50 |
+
# prange. Setting the following environment variable allows multiple OpenMP
|
| 51 |
+
# libraries to be loaded. It should not degrade performances since we manually
|
| 52 |
+
# take care of potential over-subcription performance issues, in sections of
|
| 53 |
+
# the code where nested OpenMP loops can happen, by dynamically reconfiguring
|
| 54 |
+
# the inner OpenMP runtime to temporarily disable it while under the scope of
|
| 55 |
+
# the outer OpenMP parallel section.
|
| 56 |
+
os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "True")
|
| 57 |
+
|
| 58 |
+
# Workaround issue discovered in intel-openmp 2019.5:
|
| 59 |
+
# https://github.com/ContinuumIO/anaconda-issues/issues/11294
|
| 60 |
+
os.environ.setdefault("KMP_INIT_AT_FORK", "FALSE")
|
| 61 |
+
|
| 62 |
+
# `_distributor_init` allows distributors to run custom init code.
|
| 63 |
+
# For instance, for the Windows wheel, this is used to pre-load the
|
| 64 |
+
# vcomp shared library runtime for OpenMP embedded in the sklearn/.libs
|
| 65 |
+
# sub-folder.
|
| 66 |
+
# It is necessary to do this prior to importing show_versions as the
|
| 67 |
+
# later is linked to the OpenMP runtime to make it possible to introspect
|
| 68 |
+
# it and importing it first would fail if the OpenMP dll cannot be found.
|
| 69 |
+
from sklearn import __check_build, _distributor_init # noqa: E402 F401
|
| 70 |
+
from sklearn.base import clone # noqa: E402
|
| 71 |
+
from sklearn.utils._show_versions import show_versions # noqa: E402
|
| 72 |
+
|
| 73 |
+
_submodules = [
|
| 74 |
+
"calibration",
|
| 75 |
+
"cluster",
|
| 76 |
+
"covariance",
|
| 77 |
+
"cross_decomposition",
|
| 78 |
+
"datasets",
|
| 79 |
+
"decomposition",
|
| 80 |
+
"dummy",
|
| 81 |
+
"ensemble",
|
| 82 |
+
"exceptions",
|
| 83 |
+
"experimental",
|
| 84 |
+
"externals",
|
| 85 |
+
"feature_extraction",
|
| 86 |
+
"feature_selection",
|
| 87 |
+
"frozen",
|
| 88 |
+
"gaussian_process",
|
| 89 |
+
"inspection",
|
| 90 |
+
"isotonic",
|
| 91 |
+
"kernel_approximation",
|
| 92 |
+
"kernel_ridge",
|
| 93 |
+
"linear_model",
|
| 94 |
+
"manifold",
|
| 95 |
+
"metrics",
|
| 96 |
+
"mixture",
|
| 97 |
+
"model_selection",
|
| 98 |
+
"multiclass",
|
| 99 |
+
"multioutput",
|
| 100 |
+
"naive_bayes",
|
| 101 |
+
"neighbors",
|
| 102 |
+
"neural_network",
|
| 103 |
+
"pipeline",
|
| 104 |
+
"preprocessing",
|
| 105 |
+
"random_projection",
|
| 106 |
+
"semi_supervised",
|
| 107 |
+
"svm",
|
| 108 |
+
"tree",
|
| 109 |
+
"discriminant_analysis",
|
| 110 |
+
"impute",
|
| 111 |
+
"compose",
|
| 112 |
+
]
|
| 113 |
+
|
| 114 |
+
__all__ = _submodules + [
|
| 115 |
+
# Non-modules:
|
| 116 |
+
"clone",
|
| 117 |
+
"get_config",
|
| 118 |
+
"set_config",
|
| 119 |
+
"config_context",
|
| 120 |
+
"show_versions",
|
| 121 |
+
]
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def __dir__():
|
| 125 |
+
return __all__
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def __getattr__(name):
|
| 129 |
+
if name in _submodules:
|
| 130 |
+
return _importlib.import_module(f"sklearn.{name}")
|
| 131 |
+
else:
|
| 132 |
+
try:
|
| 133 |
+
return globals()[name]
|
| 134 |
+
except KeyError:
|
| 135 |
+
raise AttributeError(f"Module 'sklearn' has no attribute '{name}'")
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def setup_module(module):
|
| 139 |
+
"""Fixture for the tests to assure globally controllable seeding of RNGs"""
|
| 140 |
+
|
| 141 |
+
import numpy as np
|
| 142 |
+
|
| 143 |
+
# Check if a random seed exists in the environment, if not create one.
|
| 144 |
+
_random_seed = os.environ.get("SKLEARN_SEED", None)
|
| 145 |
+
if _random_seed is None:
|
| 146 |
+
_random_seed = np.random.uniform() * np.iinfo(np.int32).max
|
| 147 |
+
_random_seed = int(_random_seed)
|
| 148 |
+
print("I: Seeding RNGs with %r" % _random_seed)
|
| 149 |
+
np.random.seed(_random_seed)
|
| 150 |
+
random.seed(_random_seed)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (2.96 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_config.cpython-312.pyc
ADDED
|
Binary file (14.2 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_distributor_init.cpython-312.pyc
ADDED
|
Binary file (1.11 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/base.cpython-312.pyc
ADDED
|
Binary file (51 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/exceptions.cpython-312.pyc
ADDED
|
Binary file (9.49 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/tempita.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import os
|
| 8 |
+
|
| 9 |
+
from Cython import Tempita as tempita
|
| 10 |
+
|
| 11 |
+
# XXX: If this import ever fails (does it really?), vendor either
|
| 12 |
+
# cython.tempita or numpy/npy_tempita.
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def process_tempita(fromfile, outfile=None):
|
| 16 |
+
"""Process tempita templated file and write out the result.
|
| 17 |
+
|
| 18 |
+
The template file is expected to end in `.c.tp` or `.pyx.tp`:
|
| 19 |
+
E.g. processing `template.c.in` generates `template.c`.
|
| 20 |
+
|
| 21 |
+
"""
|
| 22 |
+
with open(fromfile, "r", encoding="utf-8") as f:
|
| 23 |
+
template_content = f.read()
|
| 24 |
+
|
| 25 |
+
template = tempita.Template(template_content)
|
| 26 |
+
content = template.substitute()
|
| 27 |
+
|
| 28 |
+
with open(outfile, "w", encoding="utf-8") as f:
|
| 29 |
+
f.write(content)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def main():
|
| 33 |
+
parser = argparse.ArgumentParser()
|
| 34 |
+
parser.add_argument("infile", type=str, help="Path to the input file")
|
| 35 |
+
parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory")
|
| 36 |
+
parser.add_argument(
|
| 37 |
+
"-i",
|
| 38 |
+
"--ignore",
|
| 39 |
+
type=str,
|
| 40 |
+
help=(
|
| 41 |
+
"An ignored input - may be useful to add a "
|
| 42 |
+
"dependency between custom targets"
|
| 43 |
+
),
|
| 44 |
+
)
|
| 45 |
+
args = parser.parse_args()
|
| 46 |
+
|
| 47 |
+
if not args.infile.endswith(".tp"):
|
| 48 |
+
raise ValueError(f"Unexpected extension: {args.infile}")
|
| 49 |
+
|
| 50 |
+
if not args.outdir:
|
| 51 |
+
raise ValueError("Missing `--outdir` argument to tempita.py")
|
| 52 |
+
|
| 53 |
+
outdir_abs = os.path.join(os.getcwd(), args.outdir)
|
| 54 |
+
outfile = os.path.join(
|
| 55 |
+
outdir_abs, os.path.splitext(os.path.split(args.infile)[1])[0]
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
process_tempita(args.infile, outfile)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
if __name__ == "__main__":
|
| 62 |
+
main()
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/version.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Extract version number from __init__.py"""
|
| 3 |
+
|
| 4 |
+
# Authors: The scikit-learn developers
|
| 5 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
|
| 9 |
+
sklearn_init = os.path.join(os.path.dirname(__file__), "../__init__.py")
|
| 10 |
+
|
| 11 |
+
data = open(sklearn_init).readlines()
|
| 12 |
+
version_line = next(line for line in data if line.startswith("__version__"))
|
| 13 |
+
|
| 14 |
+
version = version_line.strip().split(" = ")[1].replace('"', "").replace("'", "")
|
| 15 |
+
|
| 16 |
+
print(version)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_config.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Global configuration state and functions for management"""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import threading
|
| 8 |
+
from contextlib import contextmanager as contextmanager
|
| 9 |
+
|
| 10 |
+
_global_config = {
|
| 11 |
+
"assume_finite": bool(os.environ.get("SKLEARN_ASSUME_FINITE", False)),
|
| 12 |
+
"working_memory": int(os.environ.get("SKLEARN_WORKING_MEMORY", 1024)),
|
| 13 |
+
"print_changed_only": True,
|
| 14 |
+
"display": "diagram",
|
| 15 |
+
"pairwise_dist_chunk_size": int(
|
| 16 |
+
os.environ.get("SKLEARN_PAIRWISE_DIST_CHUNK_SIZE", 256)
|
| 17 |
+
),
|
| 18 |
+
"enable_cython_pairwise_dist": True,
|
| 19 |
+
"array_api_dispatch": False,
|
| 20 |
+
"transform_output": "default",
|
| 21 |
+
"enable_metadata_routing": False,
|
| 22 |
+
"skip_parameter_validation": False,
|
| 23 |
+
}
|
| 24 |
+
_threadlocal = threading.local()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _get_threadlocal_config():
|
| 28 |
+
"""Get a threadlocal **mutable** configuration. If the configuration
|
| 29 |
+
does not exist, copy the default global configuration."""
|
| 30 |
+
if not hasattr(_threadlocal, "global_config"):
|
| 31 |
+
_threadlocal.global_config = _global_config.copy()
|
| 32 |
+
return _threadlocal.global_config
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def get_config():
|
| 36 |
+
"""Retrieve the current scikit-learn configuration.
|
| 37 |
+
|
| 38 |
+
This reflects the effective global configurations as established by default upon
|
| 39 |
+
library import, or modified via :func:`set_config` or :func:`config_context`.
|
| 40 |
+
|
| 41 |
+
Returns
|
| 42 |
+
-------
|
| 43 |
+
config : dict
|
| 44 |
+
Keys are parameter names that can be passed to :func:`set_config`.
|
| 45 |
+
|
| 46 |
+
See Also
|
| 47 |
+
--------
|
| 48 |
+
config_context : Context manager for global scikit-learn configuration.
|
| 49 |
+
set_config : Set global scikit-learn configuration.
|
| 50 |
+
|
| 51 |
+
Examples
|
| 52 |
+
--------
|
| 53 |
+
>>> import sklearn
|
| 54 |
+
>>> config = sklearn.get_config()
|
| 55 |
+
>>> config.keys()
|
| 56 |
+
dict_keys([...])
|
| 57 |
+
"""
|
| 58 |
+
# Return a copy of the threadlocal configuration so that users will
|
| 59 |
+
# not be able to modify the configuration with the returned dict.
|
| 60 |
+
return _get_threadlocal_config().copy()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def set_config(
|
| 64 |
+
assume_finite=None,
|
| 65 |
+
working_memory=None,
|
| 66 |
+
print_changed_only=None,
|
| 67 |
+
display=None,
|
| 68 |
+
pairwise_dist_chunk_size=None,
|
| 69 |
+
enable_cython_pairwise_dist=None,
|
| 70 |
+
array_api_dispatch=None,
|
| 71 |
+
transform_output=None,
|
| 72 |
+
enable_metadata_routing=None,
|
| 73 |
+
skip_parameter_validation=None,
|
| 74 |
+
):
|
| 75 |
+
"""Set global scikit-learn configuration.
|
| 76 |
+
|
| 77 |
+
These settings control the behaviour of scikit-learn functions during a library
|
| 78 |
+
usage session. Global configuration defaults (as described in the parameter list
|
| 79 |
+
below) take effect when scikit-learn is imported.
|
| 80 |
+
|
| 81 |
+
This function can be used to modify the global scikit-learn configuration at
|
| 82 |
+
runtime. Passing `None` as an argument (the default) leaves the corresponding
|
| 83 |
+
setting unchanged. This allows users to selectively update the global configuration
|
| 84 |
+
values without affecting the others.
|
| 85 |
+
|
| 86 |
+
.. versionadded:: 0.19
|
| 87 |
+
|
| 88 |
+
Parameters
|
| 89 |
+
----------
|
| 90 |
+
assume_finite : bool, default=None
|
| 91 |
+
If True, validation for finiteness will be skipped,
|
| 92 |
+
saving time, but leading to potential crashes. If
|
| 93 |
+
False, validation for finiteness will be performed,
|
| 94 |
+
avoiding error. Global default: False.
|
| 95 |
+
|
| 96 |
+
.. versionadded:: 0.19
|
| 97 |
+
|
| 98 |
+
working_memory : int, default=None
|
| 99 |
+
If set, scikit-learn will attempt to limit the size of temporary arrays
|
| 100 |
+
to this number of MiB (per job when parallelised), often saving both
|
| 101 |
+
computation time and memory on expensive operations that can be
|
| 102 |
+
performed in chunks. Global default: 1024.
|
| 103 |
+
|
| 104 |
+
.. versionadded:: 0.20
|
| 105 |
+
|
| 106 |
+
print_changed_only : bool, default=None
|
| 107 |
+
If True, only the parameters that were set to non-default
|
| 108 |
+
values will be printed when printing an estimator. For example,
|
| 109 |
+
``print(SVC())`` while True will only print 'SVC()' while the default
|
| 110 |
+
behaviour would be to print 'SVC(C=1.0, cache_size=200, ...)' with
|
| 111 |
+
all the non-changed parameters. Global default: True.
|
| 112 |
+
|
| 113 |
+
.. versionadded:: 0.21
|
| 114 |
+
.. versionchanged:: 0.23
|
| 115 |
+
Global default configuration changed from False to True.
|
| 116 |
+
|
| 117 |
+
display : {'text', 'diagram'}, default=None
|
| 118 |
+
If 'diagram', estimators will be displayed as a diagram in a Jupyter
|
| 119 |
+
lab or notebook context. If 'text', estimators will be displayed as
|
| 120 |
+
text. Global default: 'diagram'.
|
| 121 |
+
|
| 122 |
+
.. versionadded:: 0.23
|
| 123 |
+
|
| 124 |
+
pairwise_dist_chunk_size : int, default=None
|
| 125 |
+
The number of row vectors per chunk for the accelerated pairwise-
|
| 126 |
+
distances reduction backend. Global default: 256 (suitable for most of
|
| 127 |
+
modern laptops' caches and architectures).
|
| 128 |
+
|
| 129 |
+
Intended for easier benchmarking and testing of scikit-learn internals.
|
| 130 |
+
End users are not expected to benefit from customizing this configuration
|
| 131 |
+
setting.
|
| 132 |
+
|
| 133 |
+
.. versionadded:: 1.1
|
| 134 |
+
|
| 135 |
+
enable_cython_pairwise_dist : bool, default=None
|
| 136 |
+
Use the accelerated pairwise-distances reduction backend when
|
| 137 |
+
possible. Global default: True.
|
| 138 |
+
|
| 139 |
+
Intended for easier benchmarking and testing of scikit-learn internals.
|
| 140 |
+
End users are not expected to benefit from customizing this configuration
|
| 141 |
+
setting.
|
| 142 |
+
|
| 143 |
+
.. versionadded:: 1.1
|
| 144 |
+
|
| 145 |
+
array_api_dispatch : bool, default=None
|
| 146 |
+
Use Array API dispatching when inputs follow the Array API standard.
|
| 147 |
+
Global default: False.
|
| 148 |
+
|
| 149 |
+
See the :ref:`User Guide <array_api>` for more details.
|
| 150 |
+
|
| 151 |
+
.. versionadded:: 1.2
|
| 152 |
+
|
| 153 |
+
transform_output : str, default=None
|
| 154 |
+
Configure output of `transform` and `fit_transform`.
|
| 155 |
+
|
| 156 |
+
See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py`
|
| 157 |
+
for an example on how to use the API.
|
| 158 |
+
|
| 159 |
+
- `"default"`: Default output format of a transformer
|
| 160 |
+
- `"pandas"`: DataFrame output
|
| 161 |
+
- `"polars"`: Polars output
|
| 162 |
+
- `None`: Transform configuration is unchanged
|
| 163 |
+
|
| 164 |
+
Global default: "default".
|
| 165 |
+
|
| 166 |
+
.. versionadded:: 1.2
|
| 167 |
+
.. versionadded:: 1.4
|
| 168 |
+
`"polars"` option was added.
|
| 169 |
+
|
| 170 |
+
enable_metadata_routing : bool, default=None
|
| 171 |
+
Enable metadata routing. By default this feature is disabled.
|
| 172 |
+
|
| 173 |
+
Refer to :ref:`metadata routing user guide <metadata_routing>` for more
|
| 174 |
+
details.
|
| 175 |
+
|
| 176 |
+
- `True`: Metadata routing is enabled
|
| 177 |
+
- `False`: Metadata routing is disabled, use the old syntax.
|
| 178 |
+
- `None`: Configuration is unchanged
|
| 179 |
+
|
| 180 |
+
Global default: False.
|
| 181 |
+
|
| 182 |
+
.. versionadded:: 1.3
|
| 183 |
+
|
| 184 |
+
skip_parameter_validation : bool, default=None
|
| 185 |
+
If `True`, disable the validation of the hyper-parameters' types and values in
|
| 186 |
+
the fit method of estimators and for arguments passed to public helper
|
| 187 |
+
functions. It can save time in some situations but can lead to low level
|
| 188 |
+
crashes and exceptions with confusing error messages.
|
| 189 |
+
Global default: False.
|
| 190 |
+
|
| 191 |
+
Note that for data parameters, such as `X` and `y`, only type validation is
|
| 192 |
+
skipped but validation with `check_array` will continue to run.
|
| 193 |
+
|
| 194 |
+
.. versionadded:: 1.3
|
| 195 |
+
|
| 196 |
+
See Also
|
| 197 |
+
--------
|
| 198 |
+
config_context : Context manager for global scikit-learn configuration.
|
| 199 |
+
get_config : Retrieve current values of the global configuration.
|
| 200 |
+
|
| 201 |
+
Examples
|
| 202 |
+
--------
|
| 203 |
+
>>> from sklearn import set_config
|
| 204 |
+
>>> set_config(display='diagram') # doctest: +SKIP
|
| 205 |
+
"""
|
| 206 |
+
local_config = _get_threadlocal_config()
|
| 207 |
+
|
| 208 |
+
if assume_finite is not None:
|
| 209 |
+
local_config["assume_finite"] = assume_finite
|
| 210 |
+
if working_memory is not None:
|
| 211 |
+
local_config["working_memory"] = working_memory
|
| 212 |
+
if print_changed_only is not None:
|
| 213 |
+
local_config["print_changed_only"] = print_changed_only
|
| 214 |
+
if display is not None:
|
| 215 |
+
local_config["display"] = display
|
| 216 |
+
if pairwise_dist_chunk_size is not None:
|
| 217 |
+
local_config["pairwise_dist_chunk_size"] = pairwise_dist_chunk_size
|
| 218 |
+
if enable_cython_pairwise_dist is not None:
|
| 219 |
+
local_config["enable_cython_pairwise_dist"] = enable_cython_pairwise_dist
|
| 220 |
+
if array_api_dispatch is not None:
|
| 221 |
+
from sklearn.utils._array_api import _check_array_api_dispatch
|
| 222 |
+
|
| 223 |
+
_check_array_api_dispatch(array_api_dispatch)
|
| 224 |
+
local_config["array_api_dispatch"] = array_api_dispatch
|
| 225 |
+
if transform_output is not None:
|
| 226 |
+
local_config["transform_output"] = transform_output
|
| 227 |
+
if enable_metadata_routing is not None:
|
| 228 |
+
local_config["enable_metadata_routing"] = enable_metadata_routing
|
| 229 |
+
if skip_parameter_validation is not None:
|
| 230 |
+
local_config["skip_parameter_validation"] = skip_parameter_validation
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
@contextmanager
|
| 234 |
+
def config_context(
|
| 235 |
+
*,
|
| 236 |
+
assume_finite=None,
|
| 237 |
+
working_memory=None,
|
| 238 |
+
print_changed_only=None,
|
| 239 |
+
display=None,
|
| 240 |
+
pairwise_dist_chunk_size=None,
|
| 241 |
+
enable_cython_pairwise_dist=None,
|
| 242 |
+
array_api_dispatch=None,
|
| 243 |
+
transform_output=None,
|
| 244 |
+
enable_metadata_routing=None,
|
| 245 |
+
skip_parameter_validation=None,
|
| 246 |
+
):
|
| 247 |
+
"""Context manager to temporarily change the global scikit-learn configuration.
|
| 248 |
+
|
| 249 |
+
This context manager can be used to apply scikit-learn configuration changes within
|
| 250 |
+
the scope of the with statement. Once the context exits, the global configuration is
|
| 251 |
+
restored again.
|
| 252 |
+
|
| 253 |
+
The default global configurations (which take effect when scikit-learn is imported)
|
| 254 |
+
are defined below in the parameter list.
|
| 255 |
+
|
| 256 |
+
Parameters
|
| 257 |
+
----------
|
| 258 |
+
assume_finite : bool, default=None
|
| 259 |
+
If True, validation for finiteness will be skipped,
|
| 260 |
+
saving time, but leading to potential crashes. If
|
| 261 |
+
False, validation for finiteness will be performed,
|
| 262 |
+
avoiding error. If None, the existing configuration won't change.
|
| 263 |
+
Global default: False.
|
| 264 |
+
|
| 265 |
+
working_memory : int, default=None
|
| 266 |
+
If set, scikit-learn will attempt to limit the size of temporary arrays
|
| 267 |
+
to this number of MiB (per job when parallelised), often saving both
|
| 268 |
+
computation time and memory on expensive operations that can be
|
| 269 |
+
performed in chunks. If None, the existing configuration won't change.
|
| 270 |
+
Global default: 1024.
|
| 271 |
+
|
| 272 |
+
print_changed_only : bool, default=None
|
| 273 |
+
If True, only the parameters that were set to non-default
|
| 274 |
+
values will be printed when printing an estimator. For example,
|
| 275 |
+
``print(SVC())`` while True will only print 'SVC()', but would print
|
| 276 |
+
'SVC(C=1.0, cache_size=200, ...)' with all the non-changed parameters
|
| 277 |
+
when False. If None, the existing configuration won't change.
|
| 278 |
+
Global default: True.
|
| 279 |
+
|
| 280 |
+
.. versionchanged:: 0.23
|
| 281 |
+
Global default configuration changed from False to True.
|
| 282 |
+
|
| 283 |
+
display : {'text', 'diagram'}, default=None
|
| 284 |
+
If 'diagram', estimators will be displayed as a diagram in a Jupyter
|
| 285 |
+
lab or notebook context. If 'text', estimators will be displayed as
|
| 286 |
+
text. If None, the existing configuration won't change.
|
| 287 |
+
Global default: 'diagram'.
|
| 288 |
+
|
| 289 |
+
.. versionadded:: 0.23
|
| 290 |
+
|
| 291 |
+
pairwise_dist_chunk_size : int, default=None
|
| 292 |
+
The number of row vectors per chunk for the accelerated pairwise-
|
| 293 |
+
distances reduction backend. Global default: 256 (suitable for most of
|
| 294 |
+
modern laptops' caches and architectures).
|
| 295 |
+
|
| 296 |
+
Intended for easier benchmarking and testing of scikit-learn internals.
|
| 297 |
+
End users are not expected to benefit from customizing this configuration
|
| 298 |
+
setting.
|
| 299 |
+
|
| 300 |
+
.. versionadded:: 1.1
|
| 301 |
+
|
| 302 |
+
enable_cython_pairwise_dist : bool, default=None
|
| 303 |
+
Use the accelerated pairwise-distances reduction backend when
|
| 304 |
+
possible. Global default: True.
|
| 305 |
+
|
| 306 |
+
Intended for easier benchmarking and testing of scikit-learn internals.
|
| 307 |
+
End users are not expected to benefit from customizing this configuration
|
| 308 |
+
setting.
|
| 309 |
+
|
| 310 |
+
.. versionadded:: 1.1
|
| 311 |
+
|
| 312 |
+
array_api_dispatch : bool, default=None
|
| 313 |
+
Use Array API dispatching when inputs follow the Array API standard.
|
| 314 |
+
Global default: False.
|
| 315 |
+
|
| 316 |
+
See the :ref:`User Guide <array_api>` for more details.
|
| 317 |
+
|
| 318 |
+
.. versionadded:: 1.2
|
| 319 |
+
|
| 320 |
+
transform_output : str, default=None
|
| 321 |
+
Configure output of `transform` and `fit_transform`.
|
| 322 |
+
|
| 323 |
+
See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py`
|
| 324 |
+
for an example on how to use the API.
|
| 325 |
+
|
| 326 |
+
- `"default"`: Default output format of a transformer
|
| 327 |
+
- `"pandas"`: DataFrame output
|
| 328 |
+
- `"polars"`: Polars output
|
| 329 |
+
- `None`: Transform configuration is unchanged
|
| 330 |
+
|
| 331 |
+
Global default: "default".
|
| 332 |
+
|
| 333 |
+
.. versionadded:: 1.2
|
| 334 |
+
.. versionadded:: 1.4
|
| 335 |
+
`"polars"` option was added.
|
| 336 |
+
|
| 337 |
+
enable_metadata_routing : bool, default=None
|
| 338 |
+
Enable metadata routing. By default this feature is disabled.
|
| 339 |
+
|
| 340 |
+
Refer to :ref:`metadata routing user guide <metadata_routing>` for more
|
| 341 |
+
details.
|
| 342 |
+
|
| 343 |
+
- `True`: Metadata routing is enabled
|
| 344 |
+
- `False`: Metadata routing is disabled, use the old syntax.
|
| 345 |
+
- `None`: Configuration is unchanged
|
| 346 |
+
|
| 347 |
+
Global default: False.
|
| 348 |
+
|
| 349 |
+
.. versionadded:: 1.3
|
| 350 |
+
|
| 351 |
+
skip_parameter_validation : bool, default=None
|
| 352 |
+
If `True`, disable the validation of the hyper-parameters' types and values in
|
| 353 |
+
the fit method of estimators and for arguments passed to public helper
|
| 354 |
+
functions. It can save time in some situations but can lead to low level
|
| 355 |
+
crashes and exceptions with confusing error messages.
|
| 356 |
+
Global default: False.
|
| 357 |
+
|
| 358 |
+
Note that for data parameters, such as `X` and `y`, only type validation is
|
| 359 |
+
skipped but validation with `check_array` will continue to run.
|
| 360 |
+
|
| 361 |
+
.. versionadded:: 1.3
|
| 362 |
+
|
| 363 |
+
Yields
|
| 364 |
+
------
|
| 365 |
+
None.
|
| 366 |
+
|
| 367 |
+
See Also
|
| 368 |
+
--------
|
| 369 |
+
set_config : Set global scikit-learn configuration.
|
| 370 |
+
get_config : Retrieve current values of the global configuration.
|
| 371 |
+
|
| 372 |
+
Notes
|
| 373 |
+
-----
|
| 374 |
+
All settings, not just those presently modified, will be returned to
|
| 375 |
+
their previous values when the context manager is exited.
|
| 376 |
+
|
| 377 |
+
Examples
|
| 378 |
+
--------
|
| 379 |
+
>>> import sklearn
|
| 380 |
+
>>> from sklearn.utils.validation import assert_all_finite
|
| 381 |
+
>>> with sklearn.config_context(assume_finite=True):
|
| 382 |
+
... assert_all_finite([float('nan')])
|
| 383 |
+
>>> with sklearn.config_context(assume_finite=True):
|
| 384 |
+
... with sklearn.config_context(assume_finite=False):
|
| 385 |
+
... assert_all_finite([float('nan')])
|
| 386 |
+
Traceback (most recent call last):
|
| 387 |
+
...
|
| 388 |
+
ValueError: Input contains NaN...
|
| 389 |
+
"""
|
| 390 |
+
old_config = get_config()
|
| 391 |
+
set_config(
|
| 392 |
+
assume_finite=assume_finite,
|
| 393 |
+
working_memory=working_memory,
|
| 394 |
+
print_changed_only=print_changed_only,
|
| 395 |
+
display=display,
|
| 396 |
+
pairwise_dist_chunk_size=pairwise_dist_chunk_size,
|
| 397 |
+
enable_cython_pairwise_dist=enable_cython_pairwise_dist,
|
| 398 |
+
array_api_dispatch=array_api_dispatch,
|
| 399 |
+
transform_output=transform_output,
|
| 400 |
+
enable_metadata_routing=enable_metadata_routing,
|
| 401 |
+
skip_parameter_validation=skip_parameter_validation,
|
| 402 |
+
)
|
| 403 |
+
|
| 404 |
+
try:
|
| 405 |
+
yield
|
| 406 |
+
finally:
|
| 407 |
+
set_config(**old_config)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.lib
ADDED
|
Binary file (2.07 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d0c2c7fdd27c3deededaf164b441e9622d275d46bac4b6409f2f3abd03b1aa64
|
| 3 |
+
size 129024
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_distributor_init.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
'''Helper to preload vcomp140.dll and msvcp140.dll to prevent
|
| 3 |
+
"not found" errors.
|
| 4 |
+
|
| 5 |
+
Once vcomp140.dll and msvcp140.dll are
|
| 6 |
+
preloaded, the namespace is made available to any subsequent
|
| 7 |
+
vcomp140.dll and msvcp140.dll. This is
|
| 8 |
+
created as part of the scripts that build the wheel.
|
| 9 |
+
'''
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
import os
|
| 13 |
+
import os.path as op
|
| 14 |
+
from ctypes import WinDLL
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
if os.name == "nt":
|
| 18 |
+
libs_path = op.join(op.dirname(__file__), ".libs")
|
| 19 |
+
vcomp140_dll_filename = op.join(libs_path, "vcomp140.dll")
|
| 20 |
+
msvcp140_dll_filename = op.join(libs_path, "msvcp140.dll")
|
| 21 |
+
WinDLL(op.abspath(vcomp140_dll_filename))
|
| 22 |
+
WinDLL(op.abspath(msvcp140_dll_filename))
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.lib
ADDED
|
Binary file (2.05 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.pyd
ADDED
|
Binary file (91.6 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.pyx
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Authors: The scikit-learn developers
|
| 2 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 3 |
+
|
| 4 |
+
# Uses the pool adjacent violators algorithm (PAVA), with the
|
| 5 |
+
# enhancement of searching for the longest decreasing subsequence to
|
| 6 |
+
# pool at each step.
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
from cython cimport floating
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _inplace_contiguous_isotonic_regression(floating[::1] y, floating[::1] w):
|
| 13 |
+
cdef:
|
| 14 |
+
Py_ssize_t n = y.shape[0], i, k
|
| 15 |
+
floating prev_y, sum_wy, sum_w
|
| 16 |
+
Py_ssize_t[::1] target = np.arange(n, dtype=np.intp)
|
| 17 |
+
|
| 18 |
+
# target describes a list of blocks. At any time, if [i..j] (inclusive) is
|
| 19 |
+
# an active block, then target[i] := j and target[j] := i.
|
| 20 |
+
|
| 21 |
+
# For "active" indices (block starts):
|
| 22 |
+
# w[i] := sum{w_orig[j], j=[i..target[i]]}
|
| 23 |
+
# y[i] := sum{y_orig[j]*w_orig[j], j=[i..target[i]]} / w[i]
|
| 24 |
+
|
| 25 |
+
with nogil:
|
| 26 |
+
i = 0
|
| 27 |
+
while i < n:
|
| 28 |
+
k = target[i] + 1
|
| 29 |
+
if k == n:
|
| 30 |
+
break
|
| 31 |
+
if y[i] < y[k]:
|
| 32 |
+
i = k
|
| 33 |
+
continue
|
| 34 |
+
sum_wy = w[i] * y[i]
|
| 35 |
+
sum_w = w[i]
|
| 36 |
+
while True:
|
| 37 |
+
# We are within a decreasing subsequence.
|
| 38 |
+
prev_y = y[k]
|
| 39 |
+
sum_wy += w[k] * y[k]
|
| 40 |
+
sum_w += w[k]
|
| 41 |
+
k = target[k] + 1
|
| 42 |
+
if k == n or prev_y < y[k]:
|
| 43 |
+
# Non-singleton decreasing subsequence is finished,
|
| 44 |
+
# update first entry.
|
| 45 |
+
y[i] = sum_wy / sum_w
|
| 46 |
+
w[i] = sum_w
|
| 47 |
+
target[i] = k - 1
|
| 48 |
+
target[k - 1] = i
|
| 49 |
+
if i > 0:
|
| 50 |
+
# Backtrack if we can. This makes the algorithm
|
| 51 |
+
# single-pass and ensures O(n) complexity.
|
| 52 |
+
i = target[i - 1]
|
| 53 |
+
# Otherwise, restart from the same point.
|
| 54 |
+
break
|
| 55 |
+
# Reconstruct the solution.
|
| 56 |
+
i = 0
|
| 57 |
+
while i < n:
|
| 58 |
+
k = target[i] + 1
|
| 59 |
+
y[i + 1 : k] = y[i]
|
| 60 |
+
i = k
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _make_unique(const floating[::1] X,
|
| 64 |
+
const floating[::1] y,
|
| 65 |
+
const floating[::1] sample_weights):
|
| 66 |
+
"""Average targets for duplicate X, drop duplicates.
|
| 67 |
+
|
| 68 |
+
Aggregates duplicate X values into a single X value where
|
| 69 |
+
the target y is a (sample_weighted) average of the individual
|
| 70 |
+
targets.
|
| 71 |
+
|
| 72 |
+
Assumes that X is ordered, so that all duplicates follow each other.
|
| 73 |
+
"""
|
| 74 |
+
unique_values = len(np.unique(X))
|
| 75 |
+
|
| 76 |
+
if floating is float:
|
| 77 |
+
dtype = np.float32
|
| 78 |
+
else:
|
| 79 |
+
dtype = np.float64
|
| 80 |
+
|
| 81 |
+
cdef floating[::1] y_out = np.empty(unique_values, dtype=dtype)
|
| 82 |
+
cdef floating[::1] x_out = np.empty_like(y_out)
|
| 83 |
+
cdef floating[::1] weights_out = np.empty_like(y_out)
|
| 84 |
+
|
| 85 |
+
cdef floating current_x = X[0]
|
| 86 |
+
cdef floating current_y = 0
|
| 87 |
+
cdef floating current_weight = 0
|
| 88 |
+
cdef int i = 0
|
| 89 |
+
cdef int j
|
| 90 |
+
cdef floating x
|
| 91 |
+
cdef int n_samples = len(X)
|
| 92 |
+
cdef floating eps = np.finfo(dtype).resolution
|
| 93 |
+
|
| 94 |
+
for j in range(n_samples):
|
| 95 |
+
x = X[j]
|
| 96 |
+
if x - current_x >= eps:
|
| 97 |
+
# next unique value
|
| 98 |
+
x_out[i] = current_x
|
| 99 |
+
weights_out[i] = current_weight
|
| 100 |
+
y_out[i] = current_y / current_weight
|
| 101 |
+
i += 1
|
| 102 |
+
current_x = x
|
| 103 |
+
current_weight = sample_weights[j]
|
| 104 |
+
current_y = y[j] * sample_weights[j]
|
| 105 |
+
else:
|
| 106 |
+
current_weight += sample_weights[j]
|
| 107 |
+
current_y += y[j] * sample_weights[j]
|
| 108 |
+
|
| 109 |
+
x_out[i] = current_x
|
| 110 |
+
weights_out[i] = current_weight
|
| 111 |
+
y_out[i] = current_y / current_weight
|
| 112 |
+
return(
|
| 113 |
+
np.asarray(x_out[:i+1]),
|
| 114 |
+
np.asarray(y_out[:i+1]),
|
| 115 |
+
np.asarray(weights_out[:i+1]),
|
| 116 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__init__.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
The :mod:`sklearn._loss` module includes loss function classes suitable for
|
| 3 |
+
fitting classification and regression tasks.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
# Authors: The scikit-learn developers
|
| 7 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 8 |
+
|
| 9 |
+
from sklearn._loss.loss import (
|
| 10 |
+
AbsoluteError,
|
| 11 |
+
HalfBinomialLoss,
|
| 12 |
+
HalfGammaLoss,
|
| 13 |
+
HalfMultinomialLoss,
|
| 14 |
+
HalfPoissonLoss,
|
| 15 |
+
HalfSquaredError,
|
| 16 |
+
HalfTweedieLoss,
|
| 17 |
+
HalfTweedieLossIdentity,
|
| 18 |
+
HuberLoss,
|
| 19 |
+
PinballLoss,
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"AbsoluteError",
|
| 24 |
+
"HalfBinomialLoss",
|
| 25 |
+
"HalfGammaLoss",
|
| 26 |
+
"HalfMultinomialLoss",
|
| 27 |
+
"HalfPoissonLoss",
|
| 28 |
+
"HalfSquaredError",
|
| 29 |
+
"HalfTweedieLoss",
|
| 30 |
+
"HalfTweedieLossIdentity",
|
| 31 |
+
"HuberLoss",
|
| 32 |
+
"PinballLoss",
|
| 33 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (663 Bytes). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/link.cpython-312.pyc
ADDED
|
Binary file (11.2 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/loss.cpython-312.pyc
ADDED
|
Binary file (44.4 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.lib
ADDED
|
Binary file (1.98 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cc0cde6619e704aae36a7ea864ff3540019b0bd38924b4ab6bdb42623dac5c1e
|
| 3 |
+
size 1311232
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pxd
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Fused types for input like y_true, raw_prediction, sample_weights.
|
| 2 |
+
ctypedef fused floating_in:
|
| 3 |
+
double
|
| 4 |
+
float
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
# Fused types for output like gradient and hessian
|
| 8 |
+
# We use a different fused types for input (floating_in) and output (floating_out), such
|
| 9 |
+
# that input and output can have different dtypes in the same function call. A single
|
| 10 |
+
# fused type can only take on one single value (type) for all arguments in one function
|
| 11 |
+
# call.
|
| 12 |
+
ctypedef fused floating_out:
|
| 13 |
+
double
|
| 14 |
+
float
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
# Struct to return 2 doubles
|
| 18 |
+
ctypedef struct double_pair:
|
| 19 |
+
double val1
|
| 20 |
+
double val2
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
# C base class for loss functions
|
| 24 |
+
cdef class CyLossFunction:
|
| 25 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 26 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 27 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
cdef class CyHalfSquaredError(CyLossFunction):
|
| 31 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 32 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 33 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
cdef class CyAbsoluteError(CyLossFunction):
|
| 37 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 38 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 39 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
cdef class CyPinballLoss(CyLossFunction):
|
| 43 |
+
cdef readonly double quantile # readonly makes it accessible from Python
|
| 44 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 45 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 46 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
cdef class CyHuberLoss(CyLossFunction):
|
| 50 |
+
cdef public double delta # public makes it accessible from Python
|
| 51 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 52 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 53 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
cdef class CyHalfPoissonLoss(CyLossFunction):
|
| 57 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 58 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 59 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
cdef class CyHalfGammaLoss(CyLossFunction):
|
| 63 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 64 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 65 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
cdef class CyHalfTweedieLoss(CyLossFunction):
|
| 69 |
+
cdef readonly double power # readonly makes it accessible from Python
|
| 70 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 71 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 72 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
cdef class CyHalfTweedieLossIdentity(CyLossFunction):
|
| 76 |
+
cdef readonly double power # readonly makes it accessible from Python
|
| 77 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 78 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 79 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
cdef class CyHalfBinomialLoss(CyLossFunction):
|
| 83 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 84 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 85 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
cdef class CyExponentialLoss(CyLossFunction):
|
| 89 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
|
| 90 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
|
| 91 |
+
cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
cdef class CyHalfMultinomialLoss():
|
| 95 |
+
cdef void cy_gradient(
|
| 96 |
+
self,
|
| 97 |
+
const floating_in y_true,
|
| 98 |
+
const floating_in[::1] raw_prediction,
|
| 99 |
+
const floating_in sample_weight,
|
| 100 |
+
floating_out[::1] gradient_out,
|
| 101 |
+
) noexcept nogil
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pyx.tp
ADDED
|
@@ -0,0 +1,1505 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{py:
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
Template file to easily generate loops over samples using Tempita
|
| 5 |
+
(https://github.com/cython/cython/blob/master/Cython/Tempita/_tempita.py).
|
| 6 |
+
|
| 7 |
+
Generated file: _loss.pyx
|
| 8 |
+
|
| 9 |
+
Each loss class is generated by a cdef functions on single samples.
|
| 10 |
+
The keywords between double braces are substituted during the build.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
doc_HalfSquaredError = (
|
| 14 |
+
"""Half Squared Error with identity link.
|
| 15 |
+
|
| 16 |
+
Domain:
|
| 17 |
+
y_true and y_pred all real numbers
|
| 18 |
+
|
| 19 |
+
Link:
|
| 20 |
+
y_pred = raw_prediction
|
| 21 |
+
"""
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
doc_AbsoluteError = (
|
| 25 |
+
"""Absolute Error with identity link.
|
| 26 |
+
|
| 27 |
+
Domain:
|
| 28 |
+
y_true and y_pred all real numbers
|
| 29 |
+
|
| 30 |
+
Link:
|
| 31 |
+
y_pred = raw_prediction
|
| 32 |
+
"""
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
doc_PinballLoss = (
|
| 36 |
+
"""Quantile Loss aka Pinball Loss with identity link.
|
| 37 |
+
|
| 38 |
+
Domain:
|
| 39 |
+
y_true and y_pred all real numbers
|
| 40 |
+
quantile in (0, 1)
|
| 41 |
+
|
| 42 |
+
Link:
|
| 43 |
+
y_pred = raw_prediction
|
| 44 |
+
|
| 45 |
+
Note: 2 * cPinballLoss(quantile=0.5) equals cAbsoluteError()
|
| 46 |
+
"""
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
doc_HuberLoss = (
|
| 50 |
+
"""Huber Loss with identity link.
|
| 51 |
+
|
| 52 |
+
Domain:
|
| 53 |
+
y_true and y_pred all real numbers
|
| 54 |
+
delta in positive real numbers
|
| 55 |
+
|
| 56 |
+
Link:
|
| 57 |
+
y_pred = raw_prediction
|
| 58 |
+
"""
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
doc_HalfPoissonLoss = (
|
| 62 |
+
"""Half Poisson deviance loss with log-link.
|
| 63 |
+
|
| 64 |
+
Domain:
|
| 65 |
+
y_true in non-negative real numbers
|
| 66 |
+
y_pred in positive real numbers
|
| 67 |
+
|
| 68 |
+
Link:
|
| 69 |
+
y_pred = exp(raw_prediction)
|
| 70 |
+
|
| 71 |
+
Half Poisson deviance with log-link is
|
| 72 |
+
y_true * log(y_true/y_pred) + y_pred - y_true
|
| 73 |
+
= y_true * log(y_true) - y_true * raw_prediction
|
| 74 |
+
+ exp(raw_prediction) - y_true
|
| 75 |
+
|
| 76 |
+
Dropping constant terms, this gives:
|
| 77 |
+
exp(raw_prediction) - y_true * raw_prediction
|
| 78 |
+
"""
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
doc_HalfGammaLoss = (
|
| 82 |
+
"""Half Gamma deviance loss with log-link.
|
| 83 |
+
|
| 84 |
+
Domain:
|
| 85 |
+
y_true and y_pred in positive real numbers
|
| 86 |
+
|
| 87 |
+
Link:
|
| 88 |
+
y_pred = exp(raw_prediction)
|
| 89 |
+
|
| 90 |
+
Half Gamma deviance with log-link is
|
| 91 |
+
log(y_pred/y_true) + y_true/y_pred - 1
|
| 92 |
+
= raw_prediction - log(y_true) + y_true * exp(-raw_prediction) - 1
|
| 93 |
+
|
| 94 |
+
Dropping constant terms, this gives:
|
| 95 |
+
raw_prediction + y_true * exp(-raw_prediction)
|
| 96 |
+
"""
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
doc_HalfTweedieLoss = (
|
| 100 |
+
"""Half Tweedie deviance loss with log-link.
|
| 101 |
+
|
| 102 |
+
Domain:
|
| 103 |
+
y_true in real numbers if p <= 0
|
| 104 |
+
y_true in non-negative real numbers if 0 < p < 2
|
| 105 |
+
y_true in positive real numbers if p >= 2
|
| 106 |
+
y_pred and power in positive real numbers
|
| 107 |
+
|
| 108 |
+
Link:
|
| 109 |
+
y_pred = exp(raw_prediction)
|
| 110 |
+
|
| 111 |
+
Half Tweedie deviance with log-link and p=power is
|
| 112 |
+
max(y_true, 0)**(2-p) / (1-p) / (2-p)
|
| 113 |
+
- y_true * y_pred**(1-p) / (1-p)
|
| 114 |
+
+ y_pred**(2-p) / (2-p)
|
| 115 |
+
= max(y_true, 0)**(2-p) / (1-p) / (2-p)
|
| 116 |
+
- y_true * exp((1-p) * raw_prediction) / (1-p)
|
| 117 |
+
+ exp((2-p) * raw_prediction) / (2-p)
|
| 118 |
+
|
| 119 |
+
Dropping constant terms, this gives:
|
| 120 |
+
exp((2-p) * raw_prediction) / (2-p)
|
| 121 |
+
- y_true * exp((1-p) * raw_prediction) / (1-p)
|
| 122 |
+
|
| 123 |
+
Notes:
|
| 124 |
+
- Poisson with p=1 and Gamma with p=2 have different terms dropped such
|
| 125 |
+
that cHalfTweedieLoss is not continuous in p=power at p=1 and p=2.
|
| 126 |
+
- While the Tweedie distribution only exists for p<=0 or p>=1, the range
|
| 127 |
+
0<p<1 still gives a strictly consistent scoring function for the
|
| 128 |
+
expectation.
|
| 129 |
+
"""
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
doc_HalfTweedieLossIdentity = (
|
| 133 |
+
"""Half Tweedie deviance loss with identity link.
|
| 134 |
+
|
| 135 |
+
Domain:
|
| 136 |
+
y_true in real numbers if p <= 0
|
| 137 |
+
y_true in non-negative real numbers if 0 < p < 2
|
| 138 |
+
y_true in positive real numbers if p >= 2
|
| 139 |
+
y_pred and power in positive real numbers, y_pred may be negative for p=0.
|
| 140 |
+
|
| 141 |
+
Link:
|
| 142 |
+
y_pred = raw_prediction
|
| 143 |
+
|
| 144 |
+
Half Tweedie deviance with identity link and p=power is
|
| 145 |
+
max(y_true, 0)**(2-p) / (1-p) / (2-p)
|
| 146 |
+
- y_true * y_pred**(1-p) / (1-p)
|
| 147 |
+
+ y_pred**(2-p) / (2-p)
|
| 148 |
+
|
| 149 |
+
Notes:
|
| 150 |
+
- Here, we do not drop constant terms in contrast to the version with log-link.
|
| 151 |
+
"""
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
doc_HalfBinomialLoss = (
|
| 155 |
+
"""Half Binomial deviance loss with logit link.
|
| 156 |
+
|
| 157 |
+
Domain:
|
| 158 |
+
y_true in [0, 1]
|
| 159 |
+
y_pred in (0, 1), i.e. boundaries excluded
|
| 160 |
+
|
| 161 |
+
Link:
|
| 162 |
+
y_pred = expit(raw_prediction)
|
| 163 |
+
"""
|
| 164 |
+
)
|
| 165 |
+
|
| 166 |
+
doc_ExponentialLoss = (
|
| 167 |
+
""""Exponential loss with (half) logit link
|
| 168 |
+
|
| 169 |
+
Domain:
|
| 170 |
+
y_true in [0, 1]
|
| 171 |
+
y_pred in (0, 1), i.e. boundaries excluded
|
| 172 |
+
|
| 173 |
+
Link:
|
| 174 |
+
y_pred = expit(2 * raw_prediction)
|
| 175 |
+
"""
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
# loss class name, docstring, param,
|
| 179 |
+
# cy_loss, cy_loss_grad,
|
| 180 |
+
# cy_grad, cy_grad_hess,
|
| 181 |
+
class_list = [
|
| 182 |
+
("CyHalfSquaredError", doc_HalfSquaredError, None,
|
| 183 |
+
"closs_half_squared_error", None,
|
| 184 |
+
"cgradient_half_squared_error", "cgrad_hess_half_squared_error"),
|
| 185 |
+
("CyAbsoluteError", doc_AbsoluteError, None,
|
| 186 |
+
"closs_absolute_error", None,
|
| 187 |
+
"cgradient_absolute_error", "cgrad_hess_absolute_error"),
|
| 188 |
+
("CyPinballLoss", doc_PinballLoss, "quantile",
|
| 189 |
+
"closs_pinball_loss", None,
|
| 190 |
+
"cgradient_pinball_loss", "cgrad_hess_pinball_loss"),
|
| 191 |
+
("CyHuberLoss", doc_HuberLoss, "delta",
|
| 192 |
+
"closs_huber_loss", None,
|
| 193 |
+
"cgradient_huber_loss", "cgrad_hess_huber_loss"),
|
| 194 |
+
("CyHalfPoissonLoss", doc_HalfPoissonLoss, None,
|
| 195 |
+
"closs_half_poisson", "closs_grad_half_poisson",
|
| 196 |
+
"cgradient_half_poisson", "cgrad_hess_half_poisson"),
|
| 197 |
+
("CyHalfGammaLoss", doc_HalfGammaLoss, None,
|
| 198 |
+
"closs_half_gamma", "closs_grad_half_gamma",
|
| 199 |
+
"cgradient_half_gamma", "cgrad_hess_half_gamma"),
|
| 200 |
+
("CyHalfTweedieLoss", doc_HalfTweedieLoss, "power",
|
| 201 |
+
"closs_half_tweedie", "closs_grad_half_tweedie",
|
| 202 |
+
"cgradient_half_tweedie", "cgrad_hess_half_tweedie"),
|
| 203 |
+
("CyHalfTweedieLossIdentity", doc_HalfTweedieLossIdentity, "power",
|
| 204 |
+
"closs_half_tweedie_identity", "closs_grad_half_tweedie_identity",
|
| 205 |
+
"cgradient_half_tweedie_identity", "cgrad_hess_half_tweedie_identity"),
|
| 206 |
+
("CyHalfBinomialLoss", doc_HalfBinomialLoss, None,
|
| 207 |
+
"closs_half_binomial", "closs_grad_half_binomial",
|
| 208 |
+
"cgradient_half_binomial", "cgrad_hess_half_binomial"),
|
| 209 |
+
("CyExponentialLoss", doc_ExponentialLoss, None,
|
| 210 |
+
"closs_exponential", "closs_grad_exponential",
|
| 211 |
+
"cgradient_exponential", "cgrad_hess_exponential"),
|
| 212 |
+
]
|
| 213 |
+
}}
|
| 214 |
+
|
| 215 |
+
# Design:
|
| 216 |
+
# See https://github.com/scikit-learn/scikit-learn/issues/15123 for reasons.
|
| 217 |
+
# a) Merge link functions into loss functions for speed and numerical
|
| 218 |
+
# stability, i.e. use raw_prediction instead of y_pred in signature.
|
| 219 |
+
# b) Pure C functions (nogil) calculate single points (single sample)
|
| 220 |
+
# c) Wrap C functions in a loop to get Python functions operating on ndarrays.
|
| 221 |
+
# - Write loops manually---use Tempita for this.
|
| 222 |
+
# Reason: There is still some performance overhead when using a wrapper
|
| 223 |
+
# function "wrap" that carries out the loop and gets as argument a function
|
| 224 |
+
# pointer to one of the C functions from b), e.g.
|
| 225 |
+
# wrap(closs_half_poisson, y_true, ...)
|
| 226 |
+
# - Pass n_threads as argument to prange and propagate option to all callers.
|
| 227 |
+
# d) Provide classes (Cython extension types) per loss (names start with Cy) in
|
| 228 |
+
# order to have semantical structured objects.
|
| 229 |
+
# - Member functions for single points just call the C function from b).
|
| 230 |
+
# These are used e.g. in SGD `_plain_sgd`.
|
| 231 |
+
# - Member functions operating on ndarrays, see c), looping over calls to C
|
| 232 |
+
# functions from b).
|
| 233 |
+
# e) Provide convenience Python classes that compose from these extension types
|
| 234 |
+
# elsewhere (see loss.py)
|
| 235 |
+
# - Example: loss.gradient calls CyLoss.gradient but does some input
|
| 236 |
+
# checking like None -> np.empty().
|
| 237 |
+
#
|
| 238 |
+
# Note: We require 1-dim ndarrays to be contiguous.
|
| 239 |
+
|
| 240 |
+
from cython.parallel import parallel, prange
|
| 241 |
+
import numpy as np
|
| 242 |
+
|
| 243 |
+
from libc.math cimport exp, fabs, log, log1p, pow
|
| 244 |
+
from libc.stdlib cimport malloc, free
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
# -------------------------------------
|
| 248 |
+
# Helper functions
|
| 249 |
+
# -------------------------------------
|
| 250 |
+
# Numerically stable version of log(1 + exp(x)) for double precision, see Eq. (10) of
|
| 251 |
+
# https://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf
|
| 252 |
+
# Note: The only important cutoff is at x = 18. All others are to save computation
|
| 253 |
+
# time. Compared to the reference, we add the additional case distinction x <= -2 in
|
| 254 |
+
# order to use log instead of log1p for improved performance. As with the other
|
| 255 |
+
# cutoffs, this is accurate within machine precision of double.
|
| 256 |
+
cdef inline double log1pexp(double x) noexcept nogil:
|
| 257 |
+
if x <= -37:
|
| 258 |
+
return exp(x)
|
| 259 |
+
elif x <= -2:
|
| 260 |
+
return log1p(exp(x))
|
| 261 |
+
elif x <= 18:
|
| 262 |
+
return log(1. + exp(x))
|
| 263 |
+
elif x <= 33.3:
|
| 264 |
+
return x + exp(-x)
|
| 265 |
+
else:
|
| 266 |
+
return x
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
cdef inline double_pair sum_exp_minus_max(
|
| 270 |
+
const int i,
|
| 271 |
+
const floating_in[:, :] raw_prediction, # IN
|
| 272 |
+
floating_out *p # OUT
|
| 273 |
+
) noexcept nogil:
|
| 274 |
+
# Thread local buffers are used to store part of the results via p.
|
| 275 |
+
# The results are stored as follows:
|
| 276 |
+
# p[k] = exp(raw_prediction_i_k - max_value) for k = 0 to n_classes-1
|
| 277 |
+
# return.val1 = max_value = max(raw_prediction_i_k, k = 0 to n_classes-1)
|
| 278 |
+
# return.val2 = sum_exps = sum(p[k], k = 0 to n_classes-1) = sum of exponentials
|
| 279 |
+
# len(p) must be n_classes
|
| 280 |
+
# Notes:
|
| 281 |
+
# - We return the max value and sum of exps (stored in p) as a double_pair.
|
| 282 |
+
# - i needs to be passed (and stays constant) because otherwise Cython does
|
| 283 |
+
# not generate optimal code, see
|
| 284 |
+
# https://github.com/scikit-learn/scikit-learn/issues/17299
|
| 285 |
+
# - We do not normalize p by calculating p[k] = p[k] / sum_exps.
|
| 286 |
+
# This helps to save one loop over k.
|
| 287 |
+
cdef:
|
| 288 |
+
int k
|
| 289 |
+
int n_classes = raw_prediction.shape[1]
|
| 290 |
+
double_pair max_value_and_sum_exps # val1 = max_value, val2 = sum_exps
|
| 291 |
+
|
| 292 |
+
max_value_and_sum_exps.val1 = raw_prediction[i, 0]
|
| 293 |
+
max_value_and_sum_exps.val2 = 0
|
| 294 |
+
for k in range(1, n_classes):
|
| 295 |
+
# Compute max value of array for numerical stability
|
| 296 |
+
if max_value_and_sum_exps.val1 < raw_prediction[i, k]:
|
| 297 |
+
max_value_and_sum_exps.val1 = raw_prediction[i, k]
|
| 298 |
+
|
| 299 |
+
for k in range(n_classes):
|
| 300 |
+
p[k] = exp(raw_prediction[i, k] - max_value_and_sum_exps.val1)
|
| 301 |
+
max_value_and_sum_exps.val2 += p[k]
|
| 302 |
+
|
| 303 |
+
return max_value_and_sum_exps
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
# -------------------------------------
|
| 307 |
+
# Single point inline C functions
|
| 308 |
+
# -------------------------------------
|
| 309 |
+
# Half Squared Error
|
| 310 |
+
cdef inline double closs_half_squared_error(
|
| 311 |
+
double y_true,
|
| 312 |
+
double raw_prediction
|
| 313 |
+
) noexcept nogil:
|
| 314 |
+
return 0.5 * (raw_prediction - y_true) * (raw_prediction - y_true)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
cdef inline double cgradient_half_squared_error(
|
| 318 |
+
double y_true,
|
| 319 |
+
double raw_prediction
|
| 320 |
+
) noexcept nogil:
|
| 321 |
+
return raw_prediction - y_true
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
cdef inline double_pair cgrad_hess_half_squared_error(
|
| 325 |
+
double y_true,
|
| 326 |
+
double raw_prediction
|
| 327 |
+
) noexcept nogil:
|
| 328 |
+
cdef double_pair gh
|
| 329 |
+
gh.val1 = raw_prediction - y_true # gradient
|
| 330 |
+
gh.val2 = 1. # hessian
|
| 331 |
+
return gh
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
# Absolute Error
|
| 335 |
+
cdef inline double closs_absolute_error(
|
| 336 |
+
double y_true,
|
| 337 |
+
double raw_prediction
|
| 338 |
+
) noexcept nogil:
|
| 339 |
+
return fabs(raw_prediction - y_true)
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
cdef inline double cgradient_absolute_error(
|
| 343 |
+
double y_true,
|
| 344 |
+
double raw_prediction
|
| 345 |
+
) noexcept nogil:
|
| 346 |
+
return 1. if raw_prediction > y_true else -1.
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
cdef inline double_pair cgrad_hess_absolute_error(
|
| 350 |
+
double y_true,
|
| 351 |
+
double raw_prediction
|
| 352 |
+
) noexcept nogil:
|
| 353 |
+
cdef double_pair gh
|
| 354 |
+
# Note that exact hessian = 0 almost everywhere. Optimization routines like
|
| 355 |
+
# in HGBT, however, need a hessian > 0. Therefore, we assign 1.
|
| 356 |
+
gh.val1 = 1. if raw_prediction > y_true else -1. # gradient
|
| 357 |
+
gh.val2 = 1. # hessian
|
| 358 |
+
return gh
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
# Quantile Loss / Pinball Loss
|
| 362 |
+
cdef inline double closs_pinball_loss(
|
| 363 |
+
double y_true,
|
| 364 |
+
double raw_prediction,
|
| 365 |
+
double quantile
|
| 366 |
+
) noexcept nogil:
|
| 367 |
+
return (quantile * (y_true - raw_prediction) if y_true >= raw_prediction
|
| 368 |
+
else (1. - quantile) * (raw_prediction - y_true))
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
cdef inline double cgradient_pinball_loss(
|
| 372 |
+
double y_true,
|
| 373 |
+
double raw_prediction,
|
| 374 |
+
double quantile
|
| 375 |
+
) noexcept nogil:
|
| 376 |
+
return -quantile if y_true >=raw_prediction else 1. - quantile
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
cdef inline double_pair cgrad_hess_pinball_loss(
|
| 380 |
+
double y_true,
|
| 381 |
+
double raw_prediction,
|
| 382 |
+
double quantile
|
| 383 |
+
) noexcept nogil:
|
| 384 |
+
cdef double_pair gh
|
| 385 |
+
# Note that exact hessian = 0 almost everywhere. Optimization routines like
|
| 386 |
+
# in HGBT, however, need a hessian > 0. Therefore, we assign 1.
|
| 387 |
+
gh.val1 = -quantile if y_true >=raw_prediction else 1. - quantile # gradient
|
| 388 |
+
gh.val2 = 1. # hessian
|
| 389 |
+
return gh
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
# Huber Loss
|
| 393 |
+
cdef inline double closs_huber_loss(
|
| 394 |
+
double y_true,
|
| 395 |
+
double raw_prediction,
|
| 396 |
+
double delta,
|
| 397 |
+
) noexcept nogil:
|
| 398 |
+
cdef double abserr = fabs(y_true - raw_prediction)
|
| 399 |
+
if abserr <= delta:
|
| 400 |
+
return 0.5 * abserr**2
|
| 401 |
+
else:
|
| 402 |
+
return delta * (abserr - 0.5 * delta)
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
cdef inline double cgradient_huber_loss(
|
| 406 |
+
double y_true,
|
| 407 |
+
double raw_prediction,
|
| 408 |
+
double delta,
|
| 409 |
+
) noexcept nogil:
|
| 410 |
+
cdef double res = raw_prediction - y_true
|
| 411 |
+
if fabs(res) <= delta:
|
| 412 |
+
return res
|
| 413 |
+
else:
|
| 414 |
+
return delta if res >=0 else -delta
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
cdef inline double_pair cgrad_hess_huber_loss(
|
| 418 |
+
double y_true,
|
| 419 |
+
double raw_prediction,
|
| 420 |
+
double delta,
|
| 421 |
+
) noexcept nogil:
|
| 422 |
+
cdef double_pair gh
|
| 423 |
+
gh.val2 = raw_prediction - y_true # used as temporary
|
| 424 |
+
if fabs(gh.val2) <= delta:
|
| 425 |
+
gh.val1 = gh.val2 # gradient
|
| 426 |
+
gh.val2 = 1 # hessian
|
| 427 |
+
else:
|
| 428 |
+
gh.val1 = delta if gh.val2 >=0 else -delta # gradient
|
| 429 |
+
gh.val2 = 0 # hessian
|
| 430 |
+
return gh
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
# Half Poisson Deviance with Log-Link, dropping constant terms
|
| 434 |
+
cdef inline double closs_half_poisson(
|
| 435 |
+
double y_true,
|
| 436 |
+
double raw_prediction
|
| 437 |
+
) noexcept nogil:
|
| 438 |
+
return exp(raw_prediction) - y_true * raw_prediction
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
cdef inline double cgradient_half_poisson(
|
| 442 |
+
double y_true,
|
| 443 |
+
double raw_prediction
|
| 444 |
+
) noexcept nogil:
|
| 445 |
+
# y_pred - y_true
|
| 446 |
+
return exp(raw_prediction) - y_true
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
cdef inline double_pair closs_grad_half_poisson(
|
| 450 |
+
double y_true,
|
| 451 |
+
double raw_prediction
|
| 452 |
+
) noexcept nogil:
|
| 453 |
+
cdef double_pair lg
|
| 454 |
+
lg.val2 = exp(raw_prediction) # used as temporary
|
| 455 |
+
lg.val1 = lg.val2 - y_true * raw_prediction # loss
|
| 456 |
+
lg.val2 -= y_true # gradient
|
| 457 |
+
return lg
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
cdef inline double_pair cgrad_hess_half_poisson(
|
| 461 |
+
double y_true,
|
| 462 |
+
double raw_prediction
|
| 463 |
+
) noexcept nogil:
|
| 464 |
+
cdef double_pair gh
|
| 465 |
+
gh.val2 = exp(raw_prediction) # hessian
|
| 466 |
+
gh.val1 = gh.val2 - y_true # gradient
|
| 467 |
+
return gh
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
# Half Gamma Deviance with Log-Link, dropping constant terms
|
| 471 |
+
cdef inline double closs_half_gamma(
|
| 472 |
+
double y_true,
|
| 473 |
+
double raw_prediction
|
| 474 |
+
) noexcept nogil:
|
| 475 |
+
return raw_prediction + y_true * exp(-raw_prediction)
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
cdef inline double cgradient_half_gamma(
|
| 479 |
+
double y_true,
|
| 480 |
+
double raw_prediction
|
| 481 |
+
) noexcept nogil:
|
| 482 |
+
return 1. - y_true * exp(-raw_prediction)
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
cdef inline double_pair closs_grad_half_gamma(
|
| 486 |
+
double y_true,
|
| 487 |
+
double raw_prediction
|
| 488 |
+
) noexcept nogil:
|
| 489 |
+
cdef double_pair lg
|
| 490 |
+
lg.val2 = exp(-raw_prediction) # used as temporary
|
| 491 |
+
lg.val1 = raw_prediction + y_true * lg.val2 # loss
|
| 492 |
+
lg.val2 = 1. - y_true * lg.val2 # gradient
|
| 493 |
+
return lg
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
cdef inline double_pair cgrad_hess_half_gamma(
|
| 497 |
+
double y_true,
|
| 498 |
+
double raw_prediction
|
| 499 |
+
) noexcept nogil:
|
| 500 |
+
cdef double_pair gh
|
| 501 |
+
gh.val2 = exp(-raw_prediction) # used as temporary
|
| 502 |
+
gh.val1 = 1. - y_true * gh.val2 # gradient
|
| 503 |
+
gh.val2 *= y_true # hessian
|
| 504 |
+
return gh
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
# Half Tweedie Deviance with Log-Link, dropping constant terms
|
| 508 |
+
# Note that by dropping constants this is no longer continuous in parameter power.
|
| 509 |
+
cdef inline double closs_half_tweedie(
|
| 510 |
+
double y_true,
|
| 511 |
+
double raw_prediction,
|
| 512 |
+
double power
|
| 513 |
+
) noexcept nogil:
|
| 514 |
+
if power == 0.:
|
| 515 |
+
return closs_half_squared_error(y_true, exp(raw_prediction))
|
| 516 |
+
elif power == 1.:
|
| 517 |
+
return closs_half_poisson(y_true, raw_prediction)
|
| 518 |
+
elif power == 2.:
|
| 519 |
+
return closs_half_gamma(y_true, raw_prediction)
|
| 520 |
+
else:
|
| 521 |
+
return (exp((2. - power) * raw_prediction) / (2. - power)
|
| 522 |
+
- y_true * exp((1. - power) * raw_prediction) / (1. - power))
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
cdef inline double cgradient_half_tweedie(
|
| 526 |
+
double y_true,
|
| 527 |
+
double raw_prediction,
|
| 528 |
+
double power
|
| 529 |
+
) noexcept nogil:
|
| 530 |
+
cdef double exp1
|
| 531 |
+
if power == 0.:
|
| 532 |
+
exp1 = exp(raw_prediction)
|
| 533 |
+
return exp1 * (exp1 - y_true)
|
| 534 |
+
elif power == 1.:
|
| 535 |
+
return cgradient_half_poisson(y_true, raw_prediction)
|
| 536 |
+
elif power == 2.:
|
| 537 |
+
return cgradient_half_gamma(y_true, raw_prediction)
|
| 538 |
+
else:
|
| 539 |
+
return (exp((2. - power) * raw_prediction)
|
| 540 |
+
- y_true * exp((1. - power) * raw_prediction))
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
cdef inline double_pair closs_grad_half_tweedie(
|
| 544 |
+
double y_true,
|
| 545 |
+
double raw_prediction,
|
| 546 |
+
double power
|
| 547 |
+
) noexcept nogil:
|
| 548 |
+
cdef double_pair lg
|
| 549 |
+
cdef double exp1, exp2
|
| 550 |
+
if power == 0.:
|
| 551 |
+
exp1 = exp(raw_prediction)
|
| 552 |
+
lg.val1 = closs_half_squared_error(y_true, exp1) # loss
|
| 553 |
+
lg.val2 = exp1 * (exp1 - y_true) # gradient
|
| 554 |
+
elif power == 1.:
|
| 555 |
+
return closs_grad_half_poisson(y_true, raw_prediction)
|
| 556 |
+
elif power == 2.:
|
| 557 |
+
return closs_grad_half_gamma(y_true, raw_prediction)
|
| 558 |
+
else:
|
| 559 |
+
exp1 = exp((1. - power) * raw_prediction)
|
| 560 |
+
exp2 = exp((2. - power) * raw_prediction)
|
| 561 |
+
lg.val1 = exp2 / (2. - power) - y_true * exp1 / (1. - power) # loss
|
| 562 |
+
lg.val2 = exp2 - y_true * exp1 # gradient
|
| 563 |
+
return lg
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
cdef inline double_pair cgrad_hess_half_tweedie(
|
| 567 |
+
double y_true,
|
| 568 |
+
double raw_prediction,
|
| 569 |
+
double power
|
| 570 |
+
) noexcept nogil:
|
| 571 |
+
cdef double_pair gh
|
| 572 |
+
cdef double exp1, exp2
|
| 573 |
+
if power == 0.:
|
| 574 |
+
exp1 = exp(raw_prediction)
|
| 575 |
+
gh.val1 = exp1 * (exp1 - y_true) # gradient
|
| 576 |
+
gh.val2 = exp1 * (2 * exp1 - y_true) # hessian
|
| 577 |
+
elif power == 1.:
|
| 578 |
+
return cgrad_hess_half_poisson(y_true, raw_prediction)
|
| 579 |
+
elif power == 2.:
|
| 580 |
+
return cgrad_hess_half_gamma(y_true, raw_prediction)
|
| 581 |
+
else:
|
| 582 |
+
exp1 = exp((1. - power) * raw_prediction)
|
| 583 |
+
exp2 = exp((2. - power) * raw_prediction)
|
| 584 |
+
gh.val1 = exp2 - y_true * exp1 # gradient
|
| 585 |
+
gh.val2 = (2. - power) * exp2 - (1. - power) * y_true * exp1 # hessian
|
| 586 |
+
return gh
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
# Half Tweedie Deviance with identity link, without dropping constant terms!
|
| 590 |
+
# Therefore, best loss value is zero.
|
| 591 |
+
cdef inline double closs_half_tweedie_identity(
|
| 592 |
+
double y_true,
|
| 593 |
+
double raw_prediction,
|
| 594 |
+
double power
|
| 595 |
+
) noexcept nogil:
|
| 596 |
+
cdef double tmp
|
| 597 |
+
if power == 0.:
|
| 598 |
+
return closs_half_squared_error(y_true, raw_prediction)
|
| 599 |
+
elif power == 1.:
|
| 600 |
+
if y_true == 0:
|
| 601 |
+
return raw_prediction
|
| 602 |
+
else:
|
| 603 |
+
return y_true * log(y_true/raw_prediction) + raw_prediction - y_true
|
| 604 |
+
elif power == 2.:
|
| 605 |
+
return log(raw_prediction/y_true) + y_true/raw_prediction - 1.
|
| 606 |
+
else:
|
| 607 |
+
tmp = pow(raw_prediction, 1. - power)
|
| 608 |
+
tmp = raw_prediction * tmp / (2. - power) - y_true * tmp / (1. - power)
|
| 609 |
+
if y_true > 0:
|
| 610 |
+
tmp += pow(y_true, 2. - power) / ((1. - power) * (2. - power))
|
| 611 |
+
return tmp
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
cdef inline double cgradient_half_tweedie_identity(
|
| 615 |
+
double y_true,
|
| 616 |
+
double raw_prediction,
|
| 617 |
+
double power
|
| 618 |
+
) noexcept nogil:
|
| 619 |
+
if power == 0.:
|
| 620 |
+
return raw_prediction - y_true
|
| 621 |
+
elif power == 1.:
|
| 622 |
+
return 1. - y_true / raw_prediction
|
| 623 |
+
elif power == 2.:
|
| 624 |
+
return (raw_prediction - y_true) / (raw_prediction * raw_prediction)
|
| 625 |
+
else:
|
| 626 |
+
return pow(raw_prediction, -power) * (raw_prediction - y_true)
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
cdef inline double_pair closs_grad_half_tweedie_identity(
|
| 630 |
+
double y_true,
|
| 631 |
+
double raw_prediction,
|
| 632 |
+
double power
|
| 633 |
+
) noexcept nogil:
|
| 634 |
+
cdef double_pair lg
|
| 635 |
+
cdef double tmp
|
| 636 |
+
if power == 0.:
|
| 637 |
+
lg.val2 = raw_prediction - y_true # gradient
|
| 638 |
+
lg.val1 = 0.5 * lg.val2 * lg.val2 # loss
|
| 639 |
+
elif power == 1.:
|
| 640 |
+
if y_true == 0:
|
| 641 |
+
lg.val1 = raw_prediction
|
| 642 |
+
else:
|
| 643 |
+
lg.val1 = (y_true * log(y_true/raw_prediction) # loss
|
| 644 |
+
+ raw_prediction - y_true)
|
| 645 |
+
lg.val2 = 1. - y_true / raw_prediction # gradient
|
| 646 |
+
elif power == 2.:
|
| 647 |
+
lg.val1 = log(raw_prediction/y_true) + y_true/raw_prediction - 1. # loss
|
| 648 |
+
tmp = raw_prediction * raw_prediction
|
| 649 |
+
lg.val2 = (raw_prediction - y_true) / tmp # gradient
|
| 650 |
+
else:
|
| 651 |
+
tmp = pow(raw_prediction, 1. - power)
|
| 652 |
+
lg.val1 = (raw_prediction * tmp / (2. - power) # loss
|
| 653 |
+
- y_true * tmp / (1. - power))
|
| 654 |
+
if y_true > 0:
|
| 655 |
+
lg.val1 += (pow(y_true, 2. - power)
|
| 656 |
+
/ ((1. - power) * (2. - power)))
|
| 657 |
+
lg.val2 = tmp * (1. - y_true / raw_prediction) # gradient
|
| 658 |
+
return lg
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
cdef inline double_pair cgrad_hess_half_tweedie_identity(
|
| 662 |
+
double y_true,
|
| 663 |
+
double raw_prediction,
|
| 664 |
+
double power
|
| 665 |
+
) noexcept nogil:
|
| 666 |
+
cdef double_pair gh
|
| 667 |
+
cdef double tmp
|
| 668 |
+
if power == 0.:
|
| 669 |
+
gh.val1 = raw_prediction - y_true # gradient
|
| 670 |
+
gh.val2 = 1. # hessian
|
| 671 |
+
elif power == 1.:
|
| 672 |
+
gh.val1 = 1. - y_true / raw_prediction # gradient
|
| 673 |
+
gh.val2 = y_true / (raw_prediction * raw_prediction) # hessian
|
| 674 |
+
elif power == 2.:
|
| 675 |
+
tmp = raw_prediction * raw_prediction
|
| 676 |
+
gh.val1 = (raw_prediction - y_true) / tmp # gradient
|
| 677 |
+
gh.val2 = (-1. + 2. * y_true / raw_prediction) / tmp # hessian
|
| 678 |
+
else:
|
| 679 |
+
tmp = pow(raw_prediction, -power)
|
| 680 |
+
gh.val1 = tmp * (raw_prediction - y_true) # gradient
|
| 681 |
+
gh.val2 = tmp * ((1. - power) + power * y_true / raw_prediction) # hessian
|
| 682 |
+
return gh
|
| 683 |
+
|
| 684 |
+
|
| 685 |
+
# Half Binomial deviance with logit-link, aka log-loss or binary cross entropy
|
| 686 |
+
cdef inline double closs_half_binomial(
|
| 687 |
+
double y_true,
|
| 688 |
+
double raw_prediction
|
| 689 |
+
) noexcept nogil:
|
| 690 |
+
# log1p(exp(raw_prediction)) - y_true * raw_prediction
|
| 691 |
+
return log1pexp(raw_prediction) - y_true * raw_prediction
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
cdef inline double cgradient_half_binomial(
|
| 695 |
+
double y_true,
|
| 696 |
+
double raw_prediction
|
| 697 |
+
) noexcept nogil:
|
| 698 |
+
# gradient = y_pred - y_true = expit(raw_prediction) - y_true
|
| 699 |
+
# Numerically more stable, see http://fa.bianp.net/blog/2019/evaluate_logistic/
|
| 700 |
+
# if raw_prediction < 0:
|
| 701 |
+
# exp_tmp = exp(raw_prediction)
|
| 702 |
+
# return ((1 - y_true) * exp_tmp - y_true) / (1 + exp_tmp)
|
| 703 |
+
# else:
|
| 704 |
+
# exp_tmp = exp(-raw_prediction)
|
| 705 |
+
# return ((1 - y_true) - y_true * exp_tmp) / (1 + exp_tmp)
|
| 706 |
+
# Note that optimal speed would be achieved, at the cost of precision, by
|
| 707 |
+
# return expit(raw_prediction) - y_true
|
| 708 |
+
# i.e. no "if else" and an own inline implementation of expit instead of
|
| 709 |
+
# from scipy.special.cython_special cimport expit
|
| 710 |
+
# The case distinction raw_prediction < 0 in the stable implementation does not
|
| 711 |
+
# provide significant better precision apart from protecting overflow of exp(..).
|
| 712 |
+
# The branch (if else), however, can incur runtime costs of up to 30%.
|
| 713 |
+
# Instead, we help branch prediction by almost always ending in the first if clause
|
| 714 |
+
# and making the second branch (else) a bit simpler. This has the exact same
|
| 715 |
+
# precision but is faster than the stable implementation.
|
| 716 |
+
# As branching criteria, we use the same cutoff as in log1pexp. Note that the
|
| 717 |
+
# maximal value to get gradient = -1 with y_true = 1 is -37.439198610162731
|
| 718 |
+
# (based on mpmath), and scipy.special.logit(np.finfo(float).eps) ~ -36.04365.
|
| 719 |
+
cdef double exp_tmp
|
| 720 |
+
if raw_prediction > -37:
|
| 721 |
+
exp_tmp = exp(-raw_prediction)
|
| 722 |
+
return ((1 - y_true) - y_true * exp_tmp) / (1 + exp_tmp)
|
| 723 |
+
else:
|
| 724 |
+
# expit(raw_prediction) = exp(raw_prediction) for raw_prediction <= -37
|
| 725 |
+
return exp(raw_prediction) - y_true
|
| 726 |
+
|
| 727 |
+
|
| 728 |
+
cdef inline double_pair closs_grad_half_binomial(
|
| 729 |
+
double y_true,
|
| 730 |
+
double raw_prediction
|
| 731 |
+
) noexcept nogil:
|
| 732 |
+
cdef double_pair lg
|
| 733 |
+
# Same if else conditions as in log1pexp.
|
| 734 |
+
if raw_prediction <= -37:
|
| 735 |
+
lg.val2 = exp(raw_prediction) # used as temporary
|
| 736 |
+
lg.val1 = lg.val2 - y_true * raw_prediction # loss
|
| 737 |
+
lg.val2 -= y_true # gradient
|
| 738 |
+
elif raw_prediction <= -2:
|
| 739 |
+
lg.val2 = exp(raw_prediction) # used as temporary
|
| 740 |
+
lg.val1 = log1p(lg.val2) - y_true * raw_prediction # loss
|
| 741 |
+
lg.val2 = ((1 - y_true) * lg.val2 - y_true) / (1 + lg.val2) # gradient
|
| 742 |
+
elif raw_prediction <= 18:
|
| 743 |
+
lg.val2 = exp(-raw_prediction) # used as temporary
|
| 744 |
+
# log1p(exp(x)) = log(1 + exp(x)) = x + log1p(exp(-x))
|
| 745 |
+
lg.val1 = log1p(lg.val2) + (1 - y_true) * raw_prediction # loss
|
| 746 |
+
lg.val2 = ((1 - y_true) - y_true * lg.val2) / (1 + lg.val2) # gradient
|
| 747 |
+
else:
|
| 748 |
+
lg.val2 = exp(-raw_prediction) # used as temporary
|
| 749 |
+
lg.val1 = lg.val2 + (1 - y_true) * raw_prediction # loss
|
| 750 |
+
lg.val2 = ((1 - y_true) - y_true * lg.val2) / (1 + lg.val2) # gradient
|
| 751 |
+
return lg
|
| 752 |
+
|
| 753 |
+
|
| 754 |
+
cdef inline double_pair cgrad_hess_half_binomial(
|
| 755 |
+
double y_true,
|
| 756 |
+
double raw_prediction
|
| 757 |
+
) noexcept nogil:
|
| 758 |
+
# with y_pred = expit(raw)
|
| 759 |
+
# hessian = y_pred * (1 - y_pred) = exp( raw) / (1 + exp( raw))**2
|
| 760 |
+
# = exp(-raw) / (1 + exp(-raw))**2
|
| 761 |
+
cdef double_pair gh
|
| 762 |
+
# See comment in cgradient_half_binomial.
|
| 763 |
+
if raw_prediction > -37:
|
| 764 |
+
gh.val2 = exp(-raw_prediction) # used as temporary
|
| 765 |
+
gh.val1 = ((1 - y_true) - y_true * gh.val2) / (1 + gh.val2) # gradient
|
| 766 |
+
gh.val2 = gh.val2 / (1 + gh.val2)**2 # hessian
|
| 767 |
+
else:
|
| 768 |
+
gh.val2 = exp(raw_prediction) # = 1. order Taylor in exp(raw_prediction)
|
| 769 |
+
gh.val1 = gh.val2 - y_true
|
| 770 |
+
return gh
|
| 771 |
+
|
| 772 |
+
|
| 773 |
+
# Exponential loss with (half) logit-link, aka boosting loss
|
| 774 |
+
cdef inline double closs_exponential(
|
| 775 |
+
double y_true,
|
| 776 |
+
double raw_prediction
|
| 777 |
+
) noexcept nogil:
|
| 778 |
+
cdef double tmp = exp(raw_prediction)
|
| 779 |
+
return y_true / tmp + (1 - y_true) * tmp
|
| 780 |
+
|
| 781 |
+
|
| 782 |
+
cdef inline double cgradient_exponential(
|
| 783 |
+
double y_true,
|
| 784 |
+
double raw_prediction
|
| 785 |
+
) noexcept nogil:
|
| 786 |
+
cdef double tmp = exp(raw_prediction)
|
| 787 |
+
return -y_true / tmp + (1 - y_true) * tmp
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
cdef inline double_pair closs_grad_exponential(
|
| 791 |
+
double y_true,
|
| 792 |
+
double raw_prediction
|
| 793 |
+
) noexcept nogil:
|
| 794 |
+
cdef double_pair lg
|
| 795 |
+
lg.val2 = exp(raw_prediction) # used as temporary
|
| 796 |
+
|
| 797 |
+
lg.val1 = y_true / lg.val2 + (1 - y_true) * lg.val2 # loss
|
| 798 |
+
lg.val2 = -y_true / lg.val2 + (1 - y_true) * lg.val2 # gradient
|
| 799 |
+
return lg
|
| 800 |
+
|
| 801 |
+
|
| 802 |
+
cdef inline double_pair cgrad_hess_exponential(
|
| 803 |
+
double y_true,
|
| 804 |
+
double raw_prediction
|
| 805 |
+
) noexcept nogil:
|
| 806 |
+
# Note that hessian = loss
|
| 807 |
+
cdef double_pair gh
|
| 808 |
+
gh.val2 = exp(raw_prediction) # used as temporary
|
| 809 |
+
|
| 810 |
+
gh.val1 = -y_true / gh.val2 + (1 - y_true) * gh.val2 # gradient
|
| 811 |
+
gh.val2 = y_true / gh.val2 + (1 - y_true) * gh.val2 # hessian
|
| 812 |
+
return gh
|
| 813 |
+
|
| 814 |
+
|
| 815 |
+
# ---------------------------------------------------
|
| 816 |
+
# Extension Types for Loss Functions of 1-dim targets
|
| 817 |
+
# ---------------------------------------------------
|
| 818 |
+
cdef class CyLossFunction:
|
| 819 |
+
"""Base class for convex loss functions."""
|
| 820 |
+
|
| 821 |
+
def __reduce__(self):
|
| 822 |
+
return (self.__class__, ())
|
| 823 |
+
|
| 824 |
+
cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil:
|
| 825 |
+
"""Compute the loss for a single sample.
|
| 826 |
+
|
| 827 |
+
Parameters
|
| 828 |
+
----------
|
| 829 |
+
y_true : double
|
| 830 |
+
Observed, true target value.
|
| 831 |
+
raw_prediction : double
|
| 832 |
+
Raw prediction value (in link space).
|
| 833 |
+
|
| 834 |
+
Returns
|
| 835 |
+
-------
|
| 836 |
+
double
|
| 837 |
+
The loss evaluated at `y_true` and `raw_prediction`.
|
| 838 |
+
"""
|
| 839 |
+
pass
|
| 840 |
+
|
| 841 |
+
cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil:
|
| 842 |
+
"""Compute gradient of loss w.r.t. raw_prediction for a single sample.
|
| 843 |
+
|
| 844 |
+
Parameters
|
| 845 |
+
----------
|
| 846 |
+
y_true : double
|
| 847 |
+
Observed, true target value.
|
| 848 |
+
raw_prediction : double
|
| 849 |
+
Raw prediction value (in link space).
|
| 850 |
+
|
| 851 |
+
Returns
|
| 852 |
+
-------
|
| 853 |
+
double
|
| 854 |
+
The derivative of the loss function w.r.t. `raw_prediction`.
|
| 855 |
+
"""
|
| 856 |
+
pass
|
| 857 |
+
|
| 858 |
+
cdef double_pair cy_grad_hess(
|
| 859 |
+
self, double y_true, double raw_prediction
|
| 860 |
+
) noexcept nogil:
|
| 861 |
+
"""Compute gradient and hessian.
|
| 862 |
+
|
| 863 |
+
Gradient and hessian of loss w.r.t. raw_prediction for a single sample.
|
| 864 |
+
|
| 865 |
+
This is usually diagonal in raw_prediction_i and raw_prediction_j.
|
| 866 |
+
Therefore, we return the diagonal element i=j.
|
| 867 |
+
|
| 868 |
+
For a loss with a non-canonical link, this might implement the diagonal
|
| 869 |
+
of the Fisher matrix (=expected hessian) instead of the hessian.
|
| 870 |
+
|
| 871 |
+
Parameters
|
| 872 |
+
----------
|
| 873 |
+
y_true : double
|
| 874 |
+
Observed, true target value.
|
| 875 |
+
raw_prediction : double
|
| 876 |
+
Raw prediction value (in link space).
|
| 877 |
+
|
| 878 |
+
Returns
|
| 879 |
+
-------
|
| 880 |
+
double_pair
|
| 881 |
+
Gradient and hessian of the loss function w.r.t. `raw_prediction`.
|
| 882 |
+
"""
|
| 883 |
+
pass
|
| 884 |
+
|
| 885 |
+
def loss(
|
| 886 |
+
self,
|
| 887 |
+
const floating_in[::1] y_true, # IN
|
| 888 |
+
const floating_in[::1] raw_prediction, # IN
|
| 889 |
+
const floating_in[::1] sample_weight, # IN
|
| 890 |
+
floating_out[::1] loss_out, # OUT
|
| 891 |
+
int n_threads=1
|
| 892 |
+
):
|
| 893 |
+
"""Compute the point-wise loss value for each input.
|
| 894 |
+
|
| 895 |
+
The point-wise loss is written to `loss_out` and no array is returned.
|
| 896 |
+
|
| 897 |
+
Parameters
|
| 898 |
+
----------
|
| 899 |
+
y_true : array of shape (n_samples,)
|
| 900 |
+
Observed, true target values.
|
| 901 |
+
raw_prediction : array of shape (n_samples,)
|
| 902 |
+
Raw prediction values (in link space).
|
| 903 |
+
sample_weight : array of shape (n_samples,) or None
|
| 904 |
+
Sample weights.
|
| 905 |
+
loss_out : array of shape (n_samples,)
|
| 906 |
+
A location into which the result is stored.
|
| 907 |
+
n_threads : int
|
| 908 |
+
Number of threads used by OpenMP (if any).
|
| 909 |
+
"""
|
| 910 |
+
pass
|
| 911 |
+
|
| 912 |
+
def gradient(
|
| 913 |
+
self,
|
| 914 |
+
const floating_in[::1] y_true, # IN
|
| 915 |
+
const floating_in[::1] raw_prediction, # IN
|
| 916 |
+
const floating_in[::1] sample_weight, # IN
|
| 917 |
+
floating_out[::1] gradient_out, # OUT
|
| 918 |
+
int n_threads=1
|
| 919 |
+
):
|
| 920 |
+
"""Compute gradient of loss w.r.t raw_prediction for each input.
|
| 921 |
+
|
| 922 |
+
The gradient is written to `gradient_out` and no array is returned.
|
| 923 |
+
|
| 924 |
+
Parameters
|
| 925 |
+
----------
|
| 926 |
+
y_true : array of shape (n_samples,)
|
| 927 |
+
Observed, true target values.
|
| 928 |
+
raw_prediction : array of shape (n_samples,)
|
| 929 |
+
Raw prediction values (in link space).
|
| 930 |
+
sample_weight : array of shape (n_samples,) or None
|
| 931 |
+
Sample weights.
|
| 932 |
+
gradient_out : array of shape (n_samples,)
|
| 933 |
+
A location into which the result is stored.
|
| 934 |
+
n_threads : int
|
| 935 |
+
Number of threads used by OpenMP (if any).
|
| 936 |
+
"""
|
| 937 |
+
pass
|
| 938 |
+
|
| 939 |
+
def loss_gradient(
|
| 940 |
+
self,
|
| 941 |
+
const floating_in[::1] y_true, # IN
|
| 942 |
+
const floating_in[::1] raw_prediction, # IN
|
| 943 |
+
const floating_in[::1] sample_weight, # IN
|
| 944 |
+
floating_out[::1] loss_out, # OUT
|
| 945 |
+
floating_out[::1] gradient_out, # OUT
|
| 946 |
+
int n_threads=1
|
| 947 |
+
):
|
| 948 |
+
"""Compute loss and gradient of loss w.r.t raw_prediction.
|
| 949 |
+
|
| 950 |
+
The loss and gradient are written to `loss_out` and `gradient_out` and no arrays
|
| 951 |
+
are returned.
|
| 952 |
+
|
| 953 |
+
Parameters
|
| 954 |
+
----------
|
| 955 |
+
y_true : array of shape (n_samples,)
|
| 956 |
+
Observed, true target values.
|
| 957 |
+
raw_prediction : array of shape (n_samples,)
|
| 958 |
+
Raw prediction values (in link space).
|
| 959 |
+
sample_weight : array of shape (n_samples,) or None
|
| 960 |
+
Sample weights.
|
| 961 |
+
loss_out : array of shape (n_samples,) or None
|
| 962 |
+
A location into which the element-wise loss is stored.
|
| 963 |
+
gradient_out : array of shape (n_samples,)
|
| 964 |
+
A location into which the gradient is stored.
|
| 965 |
+
n_threads : int
|
| 966 |
+
Number of threads used by OpenMP (if any).
|
| 967 |
+
"""
|
| 968 |
+
self.loss(y_true, raw_prediction, sample_weight, loss_out, n_threads)
|
| 969 |
+
self.gradient(y_true, raw_prediction, sample_weight, gradient_out, n_threads)
|
| 970 |
+
|
| 971 |
+
def gradient_hessian(
|
| 972 |
+
self,
|
| 973 |
+
const floating_in[::1] y_true, # IN
|
| 974 |
+
const floating_in[::1] raw_prediction, # IN
|
| 975 |
+
const floating_in[::1] sample_weight, # IN
|
| 976 |
+
floating_out[::1] gradient_out, # OUT
|
| 977 |
+
floating_out[::1] hessian_out, # OUT
|
| 978 |
+
int n_threads=1
|
| 979 |
+
):
|
| 980 |
+
"""Compute gradient and hessian of loss w.r.t raw_prediction.
|
| 981 |
+
|
| 982 |
+
The gradient and hessian are written to `gradient_out` and `hessian_out` and no
|
| 983 |
+
arrays are returned.
|
| 984 |
+
|
| 985 |
+
Parameters
|
| 986 |
+
----------
|
| 987 |
+
y_true : array of shape (n_samples,)
|
| 988 |
+
Observed, true target values.
|
| 989 |
+
raw_prediction : array of shape (n_samples,)
|
| 990 |
+
Raw prediction values (in link space).
|
| 991 |
+
sample_weight : array of shape (n_samples,) or None
|
| 992 |
+
Sample weights.
|
| 993 |
+
gradient_out : array of shape (n_samples,)
|
| 994 |
+
A location into which the gradient is stored.
|
| 995 |
+
hessian_out : array of shape (n_samples,)
|
| 996 |
+
A location into which the hessian is stored.
|
| 997 |
+
n_threads : int
|
| 998 |
+
Number of threads used by OpenMP (if any).
|
| 999 |
+
"""
|
| 1000 |
+
pass
|
| 1001 |
+
|
| 1002 |
+
|
| 1003 |
+
{{for name, docstring, param, closs, closs_grad, cgrad, cgrad_hess, in class_list}}
|
| 1004 |
+
{{py:
|
| 1005 |
+
if param is None:
|
| 1006 |
+
with_param = ""
|
| 1007 |
+
else:
|
| 1008 |
+
with_param = ", self." + param
|
| 1009 |
+
}}
|
| 1010 |
+
|
| 1011 |
+
cdef class {{name}}(CyLossFunction):
|
| 1012 |
+
"""{{docstring}}"""
|
| 1013 |
+
|
| 1014 |
+
{{if param is not None}}
|
| 1015 |
+
def __init__(self, {{param}}):
|
| 1016 |
+
self.{{param}} = {{param}}
|
| 1017 |
+
{{endif}}
|
| 1018 |
+
|
| 1019 |
+
{{if param is not None}}
|
| 1020 |
+
def __reduce__(self):
|
| 1021 |
+
return (self.__class__, (self.{{param}},))
|
| 1022 |
+
{{endif}}
|
| 1023 |
+
|
| 1024 |
+
cdef inline double cy_loss(self, double y_true, double raw_prediction) noexcept nogil:
|
| 1025 |
+
return {{closs}}(y_true, raw_prediction{{with_param}})
|
| 1026 |
+
|
| 1027 |
+
cdef inline double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil:
|
| 1028 |
+
return {{cgrad}}(y_true, raw_prediction{{with_param}})
|
| 1029 |
+
|
| 1030 |
+
cdef inline double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil:
|
| 1031 |
+
return {{cgrad_hess}}(y_true, raw_prediction{{with_param}})
|
| 1032 |
+
|
| 1033 |
+
def loss(
|
| 1034 |
+
self,
|
| 1035 |
+
const floating_in[::1] y_true, # IN
|
| 1036 |
+
const floating_in[::1] raw_prediction, # IN
|
| 1037 |
+
const floating_in[::1] sample_weight, # IN
|
| 1038 |
+
floating_out[::1] loss_out, # OUT
|
| 1039 |
+
int n_threads=1
|
| 1040 |
+
):
|
| 1041 |
+
cdef:
|
| 1042 |
+
int i
|
| 1043 |
+
int n_samples = y_true.shape[0]
|
| 1044 |
+
|
| 1045 |
+
if sample_weight is None:
|
| 1046 |
+
for i in prange(
|
| 1047 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1048 |
+
):
|
| 1049 |
+
loss_out[i] = {{closs}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1050 |
+
else:
|
| 1051 |
+
for i in prange(
|
| 1052 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1053 |
+
):
|
| 1054 |
+
loss_out[i] = sample_weight[i] * {{closs}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1055 |
+
|
| 1056 |
+
{{if closs_grad is not None}}
|
| 1057 |
+
def loss_gradient(
|
| 1058 |
+
self,
|
| 1059 |
+
const floating_in[::1] y_true, # IN
|
| 1060 |
+
const floating_in[::1] raw_prediction, # IN
|
| 1061 |
+
const floating_in[::1] sample_weight, # IN
|
| 1062 |
+
floating_out[::1] loss_out, # OUT
|
| 1063 |
+
floating_out[::1] gradient_out, # OUT
|
| 1064 |
+
int n_threads=1
|
| 1065 |
+
):
|
| 1066 |
+
cdef:
|
| 1067 |
+
int i
|
| 1068 |
+
int n_samples = y_true.shape[0]
|
| 1069 |
+
double_pair dbl2
|
| 1070 |
+
|
| 1071 |
+
if sample_weight is None:
|
| 1072 |
+
for i in prange(
|
| 1073 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1074 |
+
):
|
| 1075 |
+
dbl2 = {{closs_grad}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1076 |
+
loss_out[i] = dbl2.val1
|
| 1077 |
+
gradient_out[i] = dbl2.val2
|
| 1078 |
+
else:
|
| 1079 |
+
for i in prange(
|
| 1080 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1081 |
+
):
|
| 1082 |
+
dbl2 = {{closs_grad}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1083 |
+
loss_out[i] = sample_weight[i] * dbl2.val1
|
| 1084 |
+
gradient_out[i] = sample_weight[i] * dbl2.val2
|
| 1085 |
+
|
| 1086 |
+
{{endif}}
|
| 1087 |
+
|
| 1088 |
+
def gradient(
|
| 1089 |
+
self,
|
| 1090 |
+
const floating_in[::1] y_true, # IN
|
| 1091 |
+
const floating_in[::1] raw_prediction, # IN
|
| 1092 |
+
const floating_in[::1] sample_weight, # IN
|
| 1093 |
+
floating_out[::1] gradient_out, # OUT
|
| 1094 |
+
int n_threads=1
|
| 1095 |
+
):
|
| 1096 |
+
cdef:
|
| 1097 |
+
int i
|
| 1098 |
+
int n_samples = y_true.shape[0]
|
| 1099 |
+
|
| 1100 |
+
if sample_weight is None:
|
| 1101 |
+
for i in prange(
|
| 1102 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1103 |
+
):
|
| 1104 |
+
gradient_out[i] = {{cgrad}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1105 |
+
else:
|
| 1106 |
+
for i in prange(
|
| 1107 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1108 |
+
):
|
| 1109 |
+
gradient_out[i] = sample_weight[i] * {{cgrad}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1110 |
+
|
| 1111 |
+
def gradient_hessian(
|
| 1112 |
+
self,
|
| 1113 |
+
const floating_in[::1] y_true, # IN
|
| 1114 |
+
const floating_in[::1] raw_prediction, # IN
|
| 1115 |
+
const floating_in[::1] sample_weight, # IN
|
| 1116 |
+
floating_out[::1] gradient_out, # OUT
|
| 1117 |
+
floating_out[::1] hessian_out, # OUT
|
| 1118 |
+
int n_threads=1
|
| 1119 |
+
):
|
| 1120 |
+
cdef:
|
| 1121 |
+
int i
|
| 1122 |
+
int n_samples = y_true.shape[0]
|
| 1123 |
+
double_pair dbl2
|
| 1124 |
+
|
| 1125 |
+
if sample_weight is None:
|
| 1126 |
+
for i in prange(
|
| 1127 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1128 |
+
):
|
| 1129 |
+
dbl2 = {{cgrad_hess}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1130 |
+
gradient_out[i] = dbl2.val1
|
| 1131 |
+
hessian_out[i] = dbl2.val2
|
| 1132 |
+
else:
|
| 1133 |
+
for i in prange(
|
| 1134 |
+
n_samples, schedule='static', nogil=True, num_threads=n_threads
|
| 1135 |
+
):
|
| 1136 |
+
dbl2 = {{cgrad_hess}}(y_true[i], raw_prediction[i]{{with_param}})
|
| 1137 |
+
gradient_out[i] = sample_weight[i] * dbl2.val1
|
| 1138 |
+
hessian_out[i] = sample_weight[i] * dbl2.val2
|
| 1139 |
+
|
| 1140 |
+
{{endfor}}
|
| 1141 |
+
|
| 1142 |
+
|
| 1143 |
+
# The multinomial deviance loss is also known as categorical cross-entropy or
|
| 1144 |
+
# multinomial log-likelihood.
|
| 1145 |
+
# Here, we do not inherit from CyLossFunction as its cy_gradient method deviates
|
| 1146 |
+
# from the API.
|
| 1147 |
+
cdef class CyHalfMultinomialLoss():
|
| 1148 |
+
"""Half Multinomial deviance loss with multinomial logit link.
|
| 1149 |
+
|
| 1150 |
+
Domain:
|
| 1151 |
+
y_true in {0, 1, 2, 3, .., n_classes - 1}
|
| 1152 |
+
y_pred in (0, 1)**n_classes, i.e. interval with boundaries excluded
|
| 1153 |
+
|
| 1154 |
+
Link:
|
| 1155 |
+
y_pred = softmax(raw_prediction)
|
| 1156 |
+
|
| 1157 |
+
Note: Label encoding is built-in, i.e. {0, 1, 2, 3, .., n_classes - 1} is
|
| 1158 |
+
mapped to (y_true == k) for k = 0 .. n_classes - 1 which is either 0 or 1.
|
| 1159 |
+
"""
|
| 1160 |
+
|
| 1161 |
+
# Here we deviate from the CyLossFunction API. SAG/SAGA needs direct access to
|
| 1162 |
+
# sample-wise gradients which we provide here.
|
| 1163 |
+
cdef inline void cy_gradient(
|
| 1164 |
+
self,
|
| 1165 |
+
const floating_in y_true,
|
| 1166 |
+
const floating_in[::1] raw_prediction, # IN
|
| 1167 |
+
const floating_in sample_weight,
|
| 1168 |
+
floating_out[::1] gradient_out, # OUT
|
| 1169 |
+
) noexcept nogil:
|
| 1170 |
+
"""Compute gradient of loss w.r.t. `raw_prediction` for a single sample.
|
| 1171 |
+
|
| 1172 |
+
The gradient of the multinomial logistic loss with respect to a class k,
|
| 1173 |
+
and for one sample is:
|
| 1174 |
+
grad_k = - sw * (p[k] - (y==k))
|
| 1175 |
+
|
| 1176 |
+
where:
|
| 1177 |
+
p[k] = proba[k] = exp(raw_prediction[k] - logsumexp(raw_prediction))
|
| 1178 |
+
sw = sample_weight
|
| 1179 |
+
|
| 1180 |
+
Parameters
|
| 1181 |
+
----------
|
| 1182 |
+
y_true : double
|
| 1183 |
+
Observed, true target value.
|
| 1184 |
+
raw_prediction : array of shape (n_classes,)
|
| 1185 |
+
Raw prediction values (in link space).
|
| 1186 |
+
sample_weight : double
|
| 1187 |
+
Sample weight.
|
| 1188 |
+
gradient_out : array of shape (n_classs,)
|
| 1189 |
+
A location into which the gradient is stored.
|
| 1190 |
+
|
| 1191 |
+
Returns
|
| 1192 |
+
-------
|
| 1193 |
+
gradient : double
|
| 1194 |
+
The derivative of the loss function w.r.t. `raw_prediction`.
|
| 1195 |
+
"""
|
| 1196 |
+
cdef:
|
| 1197 |
+
int k
|
| 1198 |
+
int n_classes = raw_prediction.shape[0]
|
| 1199 |
+
double_pair max_value_and_sum_exps
|
| 1200 |
+
const floating_in[:, :] raw = raw_prediction[None, :]
|
| 1201 |
+
|
| 1202 |
+
max_value_and_sum_exps = sum_exp_minus_max(0, raw, &gradient_out[0])
|
| 1203 |
+
for k in range(n_classes):
|
| 1204 |
+
# gradient_out[k] = p_k = y_pred_k = prob of class k
|
| 1205 |
+
gradient_out[k] /= max_value_and_sum_exps.val2
|
| 1206 |
+
# gradient_k = (p_k - (y_true == k)) * sw
|
| 1207 |
+
gradient_out[k] = (gradient_out[k] - (y_true == k)) * sample_weight
|
| 1208 |
+
|
| 1209 |
+
def _test_cy_gradient(
|
| 1210 |
+
self,
|
| 1211 |
+
const floating_in[::1] y_true, # IN
|
| 1212 |
+
const floating_in[:, ::1] raw_prediction, # IN
|
| 1213 |
+
const floating_in[::1] sample_weight, # IN
|
| 1214 |
+
):
|
| 1215 |
+
"""For testing only."""
|
| 1216 |
+
cdef:
|
| 1217 |
+
int i, k
|
| 1218 |
+
int n_samples = y_true.shape[0]
|
| 1219 |
+
int n_classes = raw_prediction.shape[1]
|
| 1220 |
+
floating_in [:, ::1] gradient_out
|
| 1221 |
+
gradient = np.empty((n_samples, n_classes), dtype=np.float64)
|
| 1222 |
+
gradient_out = gradient
|
| 1223 |
+
|
| 1224 |
+
for i in range(n_samples):
|
| 1225 |
+
self.cy_gradient(
|
| 1226 |
+
y_true=y_true[i],
|
| 1227 |
+
raw_prediction=raw_prediction[i, :],
|
| 1228 |
+
sample_weight=1.0 if sample_weight is None else sample_weight[i],
|
| 1229 |
+
gradient_out=gradient_out[i, :],
|
| 1230 |
+
)
|
| 1231 |
+
return gradient
|
| 1232 |
+
|
| 1233 |
+
# Note that we do not assume memory alignment/contiguity of 2d arrays.
|
| 1234 |
+
# There seems to be little benefit in doing so. Benchmarks proofing the
|
| 1235 |
+
# opposite are welcome.
|
| 1236 |
+
def loss(
|
| 1237 |
+
self,
|
| 1238 |
+
const floating_in[::1] y_true, # IN
|
| 1239 |
+
const floating_in[:, :] raw_prediction, # IN
|
| 1240 |
+
const floating_in[::1] sample_weight, # IN
|
| 1241 |
+
floating_out[::1] loss_out, # OUT
|
| 1242 |
+
int n_threads=1
|
| 1243 |
+
):
|
| 1244 |
+
cdef:
|
| 1245 |
+
int i, k
|
| 1246 |
+
int n_samples = y_true.shape[0]
|
| 1247 |
+
int n_classes = raw_prediction.shape[1]
|
| 1248 |
+
floating_in max_value, sum_exps
|
| 1249 |
+
floating_in* p # temporary buffer
|
| 1250 |
+
double_pair max_value_and_sum_exps
|
| 1251 |
+
|
| 1252 |
+
# We assume n_samples > n_classes. In this case having the inner loop
|
| 1253 |
+
# over n_classes is a good default.
|
| 1254 |
+
# TODO: If every memoryview is contiguous and raw_prediction is
|
| 1255 |
+
# f-contiguous, can we write a better algo (loops) to improve
|
| 1256 |
+
# performance?
|
| 1257 |
+
if sample_weight is None:
|
| 1258 |
+
# inner loop over n_classes
|
| 1259 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1260 |
+
# Define private buffer variables as each thread might use its
|
| 1261 |
+
# own.
|
| 1262 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1263 |
+
|
| 1264 |
+
for i in prange(n_samples, schedule='static'):
|
| 1265 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1266 |
+
max_value = max_value_and_sum_exps.val1
|
| 1267 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1268 |
+
loss_out[i] = log(sum_exps) + max_value
|
| 1269 |
+
|
| 1270 |
+
# label encoded y_true
|
| 1271 |
+
k = int(y_true[i])
|
| 1272 |
+
loss_out[i] -= raw_prediction[i, k]
|
| 1273 |
+
|
| 1274 |
+
free(p)
|
| 1275 |
+
else:
|
| 1276 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1277 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1278 |
+
|
| 1279 |
+
for i in prange(n_samples, schedule='static'):
|
| 1280 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1281 |
+
max_value = max_value_and_sum_exps.val1
|
| 1282 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1283 |
+
loss_out[i] = log(sum_exps) + max_value
|
| 1284 |
+
|
| 1285 |
+
# label encoded y_true
|
| 1286 |
+
k = int(y_true[i])
|
| 1287 |
+
loss_out[i] -= raw_prediction[i, k]
|
| 1288 |
+
|
| 1289 |
+
loss_out[i] *= sample_weight[i]
|
| 1290 |
+
|
| 1291 |
+
free(p)
|
| 1292 |
+
|
| 1293 |
+
def loss_gradient(
|
| 1294 |
+
self,
|
| 1295 |
+
const floating_in[::1] y_true, # IN
|
| 1296 |
+
const floating_in[:, :] raw_prediction, # IN
|
| 1297 |
+
const floating_in[::1] sample_weight, # IN
|
| 1298 |
+
floating_out[::1] loss_out, # OUT
|
| 1299 |
+
floating_out[:, :] gradient_out, # OUT
|
| 1300 |
+
int n_threads=1
|
| 1301 |
+
):
|
| 1302 |
+
cdef:
|
| 1303 |
+
int i, k
|
| 1304 |
+
int n_samples = y_true.shape[0]
|
| 1305 |
+
int n_classes = raw_prediction.shape[1]
|
| 1306 |
+
floating_in max_value, sum_exps
|
| 1307 |
+
floating_in* p # temporary buffer
|
| 1308 |
+
double_pair max_value_and_sum_exps
|
| 1309 |
+
|
| 1310 |
+
if sample_weight is None:
|
| 1311 |
+
# inner loop over n_classes
|
| 1312 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1313 |
+
# Define private buffer variables as each thread might use its
|
| 1314 |
+
# own.
|
| 1315 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1316 |
+
|
| 1317 |
+
for i in prange(n_samples, schedule='static'):
|
| 1318 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1319 |
+
max_value = max_value_and_sum_exps.val1
|
| 1320 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1321 |
+
loss_out[i] = log(sum_exps) + max_value
|
| 1322 |
+
|
| 1323 |
+
for k in range(n_classes):
|
| 1324 |
+
# label decode y_true
|
| 1325 |
+
if y_true[i] == k:
|
| 1326 |
+
loss_out[i] -= raw_prediction[i, k]
|
| 1327 |
+
p[k] /= sum_exps # p_k = y_pred_k = prob of class k
|
| 1328 |
+
# gradient_k = p_k - (y_true == k)
|
| 1329 |
+
gradient_out[i, k] = p[k] - (y_true[i] == k)
|
| 1330 |
+
|
| 1331 |
+
free(p)
|
| 1332 |
+
else:
|
| 1333 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1334 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1335 |
+
|
| 1336 |
+
for i in prange(n_samples, schedule='static'):
|
| 1337 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1338 |
+
max_value = max_value_and_sum_exps.val1
|
| 1339 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1340 |
+
loss_out[i] = log(sum_exps) + max_value
|
| 1341 |
+
|
| 1342 |
+
for k in range(n_classes):
|
| 1343 |
+
# label decode y_true
|
| 1344 |
+
if y_true[i] == k:
|
| 1345 |
+
loss_out[i] -= raw_prediction[i, k]
|
| 1346 |
+
p[k] /= sum_exps # p_k = y_pred_k = prob of class k
|
| 1347 |
+
# gradient_k = (p_k - (y_true == k)) * sw
|
| 1348 |
+
gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i]
|
| 1349 |
+
|
| 1350 |
+
loss_out[i] *= sample_weight[i]
|
| 1351 |
+
|
| 1352 |
+
free(p)
|
| 1353 |
+
|
| 1354 |
+
def gradient(
|
| 1355 |
+
self,
|
| 1356 |
+
const floating_in[::1] y_true, # IN
|
| 1357 |
+
const floating_in[:, :] raw_prediction, # IN
|
| 1358 |
+
const floating_in[::1] sample_weight, # IN
|
| 1359 |
+
floating_out[:, :] gradient_out, # OUT
|
| 1360 |
+
int n_threads=1
|
| 1361 |
+
):
|
| 1362 |
+
cdef:
|
| 1363 |
+
int i, k
|
| 1364 |
+
int n_samples = y_true.shape[0]
|
| 1365 |
+
int n_classes = raw_prediction.shape[1]
|
| 1366 |
+
floating_in sum_exps
|
| 1367 |
+
floating_in* p # temporary buffer
|
| 1368 |
+
double_pair max_value_and_sum_exps
|
| 1369 |
+
|
| 1370 |
+
if sample_weight is None:
|
| 1371 |
+
# inner loop over n_classes
|
| 1372 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1373 |
+
# Define private buffer variables as each thread might use its
|
| 1374 |
+
# own.
|
| 1375 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1376 |
+
|
| 1377 |
+
for i in prange(n_samples, schedule='static'):
|
| 1378 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1379 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1380 |
+
|
| 1381 |
+
for k in range(n_classes):
|
| 1382 |
+
p[k] /= sum_exps # p_k = y_pred_k = prob of class k
|
| 1383 |
+
# gradient_k = y_pred_k - (y_true == k)
|
| 1384 |
+
gradient_out[i, k] = p[k] - (y_true[i] == k)
|
| 1385 |
+
|
| 1386 |
+
free(p)
|
| 1387 |
+
else:
|
| 1388 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1389 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1390 |
+
|
| 1391 |
+
for i in prange(n_samples, schedule='static'):
|
| 1392 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1393 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1394 |
+
|
| 1395 |
+
for k in range(n_classes):
|
| 1396 |
+
p[k] /= sum_exps # p_k = y_pred_k = prob of class k
|
| 1397 |
+
# gradient_k = (p_k - (y_true == k)) * sw
|
| 1398 |
+
gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i]
|
| 1399 |
+
|
| 1400 |
+
free(p)
|
| 1401 |
+
|
| 1402 |
+
def gradient_hessian(
|
| 1403 |
+
self,
|
| 1404 |
+
const floating_in[::1] y_true, # IN
|
| 1405 |
+
const floating_in[:, :] raw_prediction, # IN
|
| 1406 |
+
const floating_in[::1] sample_weight, # IN
|
| 1407 |
+
floating_out[:, :] gradient_out, # OUT
|
| 1408 |
+
floating_out[:, :] hessian_out, # OUT
|
| 1409 |
+
int n_threads=1
|
| 1410 |
+
):
|
| 1411 |
+
cdef:
|
| 1412 |
+
int i, k
|
| 1413 |
+
int n_samples = y_true.shape[0]
|
| 1414 |
+
int n_classes = raw_prediction.shape[1]
|
| 1415 |
+
floating_in sum_exps
|
| 1416 |
+
floating_in* p # temporary buffer
|
| 1417 |
+
double_pair max_value_and_sum_exps
|
| 1418 |
+
|
| 1419 |
+
if sample_weight is None:
|
| 1420 |
+
# inner loop over n_classes
|
| 1421 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1422 |
+
# Define private buffer variables as each thread might use its
|
| 1423 |
+
# own.
|
| 1424 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1425 |
+
|
| 1426 |
+
for i in prange(n_samples, schedule='static'):
|
| 1427 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1428 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1429 |
+
|
| 1430 |
+
for k in range(n_classes):
|
| 1431 |
+
p[k] /= sum_exps # p_k = y_pred_k = prob of class k
|
| 1432 |
+
# hessian_k = p_k * (1 - p_k)
|
| 1433 |
+
# gradient_k = p_k - (y_true == k)
|
| 1434 |
+
gradient_out[i, k] = p[k] - (y_true[i] == k)
|
| 1435 |
+
hessian_out[i, k] = p[k] * (1. - p[k])
|
| 1436 |
+
|
| 1437 |
+
free(p)
|
| 1438 |
+
else:
|
| 1439 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1440 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1441 |
+
|
| 1442 |
+
for i in prange(n_samples, schedule='static'):
|
| 1443 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1444 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1445 |
+
|
| 1446 |
+
for k in range(n_classes):
|
| 1447 |
+
p[k] /= sum_exps # p_k = y_pred_k = prob of class k
|
| 1448 |
+
# gradient_k = (p_k - (y_true == k)) * sw
|
| 1449 |
+
# hessian_k = p_k * (1 - p_k) * sw
|
| 1450 |
+
gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i]
|
| 1451 |
+
hessian_out[i, k] = (p[k] * (1. - p[k])) * sample_weight[i]
|
| 1452 |
+
|
| 1453 |
+
free(p)
|
| 1454 |
+
|
| 1455 |
+
# This method simplifies the implementation of hessp in linear models,
|
| 1456 |
+
# i.e. the matrix-vector product of the full hessian, not only of the
|
| 1457 |
+
# diagonal (in the classes) approximation as implemented above.
|
| 1458 |
+
def gradient_proba(
|
| 1459 |
+
self,
|
| 1460 |
+
const floating_in[::1] y_true, # IN
|
| 1461 |
+
const floating_in[:, :] raw_prediction, # IN
|
| 1462 |
+
const floating_in[::1] sample_weight, # IN
|
| 1463 |
+
floating_out[:, :] gradient_out, # OUT
|
| 1464 |
+
floating_out[:, :] proba_out, # OUT
|
| 1465 |
+
int n_threads=1
|
| 1466 |
+
):
|
| 1467 |
+
cdef:
|
| 1468 |
+
int i, k
|
| 1469 |
+
int n_samples = y_true.shape[0]
|
| 1470 |
+
int n_classes = raw_prediction.shape[1]
|
| 1471 |
+
floating_in sum_exps
|
| 1472 |
+
floating_in* p # temporary buffer
|
| 1473 |
+
double_pair max_value_and_sum_exps
|
| 1474 |
+
|
| 1475 |
+
if sample_weight is None:
|
| 1476 |
+
# inner loop over n_classes
|
| 1477 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1478 |
+
# Define private buffer variables as each thread might use its
|
| 1479 |
+
# own.
|
| 1480 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1481 |
+
|
| 1482 |
+
for i in prange(n_samples, schedule='static'):
|
| 1483 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1484 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1485 |
+
|
| 1486 |
+
for k in range(n_classes):
|
| 1487 |
+
proba_out[i, k] = p[k] / sum_exps # y_pred_k = prob of class k
|
| 1488 |
+
# gradient_k = y_pred_k - (y_true == k)
|
| 1489 |
+
gradient_out[i, k] = proba_out[i, k] - (y_true[i] == k)
|
| 1490 |
+
|
| 1491 |
+
free(p)
|
| 1492 |
+
else:
|
| 1493 |
+
with nogil, parallel(num_threads=n_threads):
|
| 1494 |
+
p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
|
| 1495 |
+
|
| 1496 |
+
for i in prange(n_samples, schedule='static'):
|
| 1497 |
+
max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
|
| 1498 |
+
sum_exps = max_value_and_sum_exps.val2
|
| 1499 |
+
|
| 1500 |
+
for k in range(n_classes):
|
| 1501 |
+
proba_out[i, k] = p[k] / sum_exps # y_pred_k = prob of class k
|
| 1502 |
+
# gradient_k = (p_k - (y_true == k)) * sw
|
| 1503 |
+
gradient_out[i, k] = (proba_out[i, k] - (y_true[i] == k)) * sample_weight[i]
|
| 1504 |
+
|
| 1505 |
+
free(p)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/link.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Module contains classes for invertible (and differentiable) link functions.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
# Authors: The scikit-learn developers
|
| 6 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 7 |
+
|
| 8 |
+
from abc import ABC, abstractmethod
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
from scipy.special import expit, logit
|
| 13 |
+
from scipy.stats import gmean
|
| 14 |
+
|
| 15 |
+
from sklearn.utils.extmath import softmax
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class Interval:
|
| 20 |
+
low: float
|
| 21 |
+
high: float
|
| 22 |
+
low_inclusive: bool
|
| 23 |
+
high_inclusive: bool
|
| 24 |
+
|
| 25 |
+
def __post_init__(self):
|
| 26 |
+
"""Check that low <= high"""
|
| 27 |
+
if self.low > self.high:
|
| 28 |
+
raise ValueError(
|
| 29 |
+
f"One must have low <= high; got low={self.low}, high={self.high}."
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
def includes(self, x):
|
| 33 |
+
"""Test whether all values of x are in interval range.
|
| 34 |
+
|
| 35 |
+
Parameters
|
| 36 |
+
----------
|
| 37 |
+
x : ndarray
|
| 38 |
+
Array whose elements are tested to be in interval range.
|
| 39 |
+
|
| 40 |
+
Returns
|
| 41 |
+
-------
|
| 42 |
+
result : bool
|
| 43 |
+
"""
|
| 44 |
+
if self.low_inclusive:
|
| 45 |
+
low = np.greater_equal(x, self.low)
|
| 46 |
+
else:
|
| 47 |
+
low = np.greater(x, self.low)
|
| 48 |
+
|
| 49 |
+
if not np.all(low):
|
| 50 |
+
return False
|
| 51 |
+
|
| 52 |
+
if self.high_inclusive:
|
| 53 |
+
high = np.less_equal(x, self.high)
|
| 54 |
+
else:
|
| 55 |
+
high = np.less(x, self.high)
|
| 56 |
+
|
| 57 |
+
# Note: np.all returns numpy.bool_
|
| 58 |
+
return bool(np.all(high))
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _inclusive_low_high(interval, dtype=np.float64):
|
| 62 |
+
"""Generate values low and high to be within the interval range.
|
| 63 |
+
|
| 64 |
+
This is used in tests only.
|
| 65 |
+
|
| 66 |
+
Returns
|
| 67 |
+
-------
|
| 68 |
+
low, high : tuple
|
| 69 |
+
The returned values low and high lie within the interval.
|
| 70 |
+
"""
|
| 71 |
+
eps = 10 * np.finfo(dtype).eps
|
| 72 |
+
if interval.low == -np.inf:
|
| 73 |
+
low = -1e10
|
| 74 |
+
elif interval.low < 0:
|
| 75 |
+
low = interval.low * (1 - eps) + eps
|
| 76 |
+
else:
|
| 77 |
+
low = interval.low * (1 + eps) + eps
|
| 78 |
+
|
| 79 |
+
if interval.high == np.inf:
|
| 80 |
+
high = 1e10
|
| 81 |
+
elif interval.high < 0:
|
| 82 |
+
high = interval.high * (1 + eps) - eps
|
| 83 |
+
else:
|
| 84 |
+
high = interval.high * (1 - eps) - eps
|
| 85 |
+
|
| 86 |
+
return low, high
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
class BaseLink(ABC):
|
| 90 |
+
"""Abstract base class for differentiable, invertible link functions.
|
| 91 |
+
|
| 92 |
+
Convention:
|
| 93 |
+
- link function g: raw_prediction = g(y_pred)
|
| 94 |
+
- inverse link h: y_pred = h(raw_prediction)
|
| 95 |
+
|
| 96 |
+
For (generalized) linear models, `raw_prediction = X @ coef` is the so
|
| 97 |
+
called linear predictor, and `y_pred = h(raw_prediction)` is the predicted
|
| 98 |
+
conditional (on X) expected value of the target `y_true`.
|
| 99 |
+
|
| 100 |
+
The methods are not implemented as staticmethods in case a link function needs
|
| 101 |
+
parameters.
|
| 102 |
+
"""
|
| 103 |
+
|
| 104 |
+
is_multiclass = False # used for testing only
|
| 105 |
+
|
| 106 |
+
# Usually, raw_prediction may be any real number and y_pred is an open
|
| 107 |
+
# interval.
|
| 108 |
+
# interval_raw_prediction = Interval(-np.inf, np.inf, False, False)
|
| 109 |
+
interval_y_pred = Interval(-np.inf, np.inf, False, False)
|
| 110 |
+
|
| 111 |
+
@abstractmethod
|
| 112 |
+
def link(self, y_pred, out=None):
|
| 113 |
+
"""Compute the link function g(y_pred).
|
| 114 |
+
|
| 115 |
+
The link function maps (predicted) target values to raw predictions,
|
| 116 |
+
i.e. `g(y_pred) = raw_prediction`.
|
| 117 |
+
|
| 118 |
+
Parameters
|
| 119 |
+
----------
|
| 120 |
+
y_pred : array
|
| 121 |
+
Predicted target values.
|
| 122 |
+
out : array
|
| 123 |
+
A location into which the result is stored. If provided, it must
|
| 124 |
+
have a shape that the inputs broadcast to. If not provided or None,
|
| 125 |
+
a freshly-allocated array is returned.
|
| 126 |
+
|
| 127 |
+
Returns
|
| 128 |
+
-------
|
| 129 |
+
out : array
|
| 130 |
+
Output array, element-wise link function.
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
@abstractmethod
|
| 134 |
+
def inverse(self, raw_prediction, out=None):
|
| 135 |
+
"""Compute the inverse link function h(raw_prediction).
|
| 136 |
+
|
| 137 |
+
The inverse link function maps raw predictions to predicted target
|
| 138 |
+
values, i.e. `h(raw_prediction) = y_pred`.
|
| 139 |
+
|
| 140 |
+
Parameters
|
| 141 |
+
----------
|
| 142 |
+
raw_prediction : array
|
| 143 |
+
Raw prediction values (in link space).
|
| 144 |
+
out : array
|
| 145 |
+
A location into which the result is stored. If provided, it must
|
| 146 |
+
have a shape that the inputs broadcast to. If not provided or None,
|
| 147 |
+
a freshly-allocated array is returned.
|
| 148 |
+
|
| 149 |
+
Returns
|
| 150 |
+
-------
|
| 151 |
+
out : array
|
| 152 |
+
Output array, element-wise inverse link function.
|
| 153 |
+
"""
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class IdentityLink(BaseLink):
|
| 157 |
+
"""The identity link function g(x)=x."""
|
| 158 |
+
|
| 159 |
+
def link(self, y_pred, out=None):
|
| 160 |
+
if out is not None:
|
| 161 |
+
np.copyto(out, y_pred)
|
| 162 |
+
return out
|
| 163 |
+
else:
|
| 164 |
+
return y_pred
|
| 165 |
+
|
| 166 |
+
inverse = link
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
class LogLink(BaseLink):
|
| 170 |
+
"""The log link function g(x)=log(x)."""
|
| 171 |
+
|
| 172 |
+
interval_y_pred = Interval(0, np.inf, False, False)
|
| 173 |
+
|
| 174 |
+
def link(self, y_pred, out=None):
|
| 175 |
+
return np.log(y_pred, out=out)
|
| 176 |
+
|
| 177 |
+
def inverse(self, raw_prediction, out=None):
|
| 178 |
+
return np.exp(raw_prediction, out=out)
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
class LogitLink(BaseLink):
|
| 182 |
+
"""The logit link function g(x)=logit(x)."""
|
| 183 |
+
|
| 184 |
+
interval_y_pred = Interval(0, 1, False, False)
|
| 185 |
+
|
| 186 |
+
def link(self, y_pred, out=None):
|
| 187 |
+
return logit(y_pred, out=out)
|
| 188 |
+
|
| 189 |
+
def inverse(self, raw_prediction, out=None):
|
| 190 |
+
return expit(raw_prediction, out=out)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
class HalfLogitLink(BaseLink):
|
| 194 |
+
"""Half the logit link function g(x)=1/2 * logit(x).
|
| 195 |
+
|
| 196 |
+
Used for the exponential loss.
|
| 197 |
+
"""
|
| 198 |
+
|
| 199 |
+
interval_y_pred = Interval(0, 1, False, False)
|
| 200 |
+
|
| 201 |
+
def link(self, y_pred, out=None):
|
| 202 |
+
out = logit(y_pred, out=out)
|
| 203 |
+
out *= 0.5
|
| 204 |
+
return out
|
| 205 |
+
|
| 206 |
+
def inverse(self, raw_prediction, out=None):
|
| 207 |
+
return expit(2 * raw_prediction, out)
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class MultinomialLogit(BaseLink):
|
| 211 |
+
"""The symmetric multinomial logit function.
|
| 212 |
+
|
| 213 |
+
Convention:
|
| 214 |
+
- y_pred.shape = raw_prediction.shape = (n_samples, n_classes)
|
| 215 |
+
|
| 216 |
+
Notes:
|
| 217 |
+
- The inverse link h is the softmax function.
|
| 218 |
+
- The sum is over the second axis, i.e. axis=1 (n_classes).
|
| 219 |
+
|
| 220 |
+
We have to choose additional constraints in order to make
|
| 221 |
+
|
| 222 |
+
y_pred[k] = exp(raw_pred[k]) / sum(exp(raw_pred[k]), k=0..n_classes-1)
|
| 223 |
+
|
| 224 |
+
for n_classes classes identifiable and invertible.
|
| 225 |
+
We choose the symmetric side constraint where the geometric mean response
|
| 226 |
+
is set as reference category, see [2]:
|
| 227 |
+
|
| 228 |
+
The symmetric multinomial logit link function for a single data point is
|
| 229 |
+
then defined as
|
| 230 |
+
|
| 231 |
+
raw_prediction[k] = g(y_pred[k]) = log(y_pred[k]/gmean(y_pred))
|
| 232 |
+
= log(y_pred[k]) - mean(log(y_pred)).
|
| 233 |
+
|
| 234 |
+
Note that this is equivalent to the definition in [1] and implies mean
|
| 235 |
+
centered raw predictions:
|
| 236 |
+
|
| 237 |
+
sum(raw_prediction[k], k=0..n_classes-1) = 0.
|
| 238 |
+
|
| 239 |
+
For linear models with raw_prediction = X @ coef, this corresponds to
|
| 240 |
+
sum(coef[k], k=0..n_classes-1) = 0, i.e. the sum over classes for every
|
| 241 |
+
feature is zero.
|
| 242 |
+
|
| 243 |
+
Reference
|
| 244 |
+
---------
|
| 245 |
+
.. [1] Friedman, Jerome; Hastie, Trevor; Tibshirani, Robert. "Additive
|
| 246 |
+
logistic regression: a statistical view of boosting" Ann. Statist.
|
| 247 |
+
28 (2000), no. 2, 337--407. doi:10.1214/aos/1016218223.
|
| 248 |
+
https://projecteuclid.org/euclid.aos/1016218223
|
| 249 |
+
|
| 250 |
+
.. [2] Zahid, Faisal Maqbool and Gerhard Tutz. "Ridge estimation for
|
| 251 |
+
multinomial logit models with symmetric side constraints."
|
| 252 |
+
Computational Statistics 28 (2013): 1017-1034.
|
| 253 |
+
http://epub.ub.uni-muenchen.de/11001/1/tr067.pdf
|
| 254 |
+
"""
|
| 255 |
+
|
| 256 |
+
is_multiclass = True
|
| 257 |
+
interval_y_pred = Interval(0, 1, False, False)
|
| 258 |
+
|
| 259 |
+
def symmetrize_raw_prediction(self, raw_prediction):
|
| 260 |
+
return raw_prediction - np.mean(raw_prediction, axis=1)[:, np.newaxis]
|
| 261 |
+
|
| 262 |
+
def link(self, y_pred, out=None):
|
| 263 |
+
# geometric mean as reference category
|
| 264 |
+
gm = gmean(y_pred, axis=1)
|
| 265 |
+
return np.log(y_pred / gm[:, np.newaxis], out=out)
|
| 266 |
+
|
| 267 |
+
def inverse(self, raw_prediction, out=None):
|
| 268 |
+
if out is None:
|
| 269 |
+
return softmax(raw_prediction, copy=True)
|
| 270 |
+
else:
|
| 271 |
+
np.copyto(out, raw_prediction)
|
| 272 |
+
softmax(out, copy=False)
|
| 273 |
+
return out
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
_LINKS = {
|
| 277 |
+
"identity": IdentityLink,
|
| 278 |
+
"log": LogLink,
|
| 279 |
+
"logit": LogitLink,
|
| 280 |
+
"half_logit": HalfLogitLink,
|
| 281 |
+
"multinomial_logit": MultinomialLogit,
|
| 282 |
+
}
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/loss.py
ADDED
|
@@ -0,0 +1,1216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
This module contains loss classes suitable for fitting.
|
| 3 |
+
|
| 4 |
+
It is not part of the public API.
|
| 5 |
+
Specific losses are used for regression, binary classification or multiclass
|
| 6 |
+
classification.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
# Authors: The scikit-learn developers
|
| 10 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 11 |
+
|
| 12 |
+
# Goals:
|
| 13 |
+
# - Provide a common private module for loss functions/classes.
|
| 14 |
+
# - To be used in:
|
| 15 |
+
# - LogisticRegression
|
| 16 |
+
# - PoissonRegressor, GammaRegressor, TweedieRegressor
|
| 17 |
+
# - HistGradientBoostingRegressor, HistGradientBoostingClassifier
|
| 18 |
+
# - GradientBoostingRegressor, GradientBoostingClassifier
|
| 19 |
+
# - SGDRegressor, SGDClassifier
|
| 20 |
+
# - Replace link module of GLMs.
|
| 21 |
+
|
| 22 |
+
import numbers
|
| 23 |
+
|
| 24 |
+
import numpy as np
|
| 25 |
+
from scipy.special import xlogy
|
| 26 |
+
|
| 27 |
+
from sklearn._loss._loss import (
|
| 28 |
+
CyAbsoluteError,
|
| 29 |
+
CyExponentialLoss,
|
| 30 |
+
CyHalfBinomialLoss,
|
| 31 |
+
CyHalfGammaLoss,
|
| 32 |
+
CyHalfMultinomialLoss,
|
| 33 |
+
CyHalfPoissonLoss,
|
| 34 |
+
CyHalfSquaredError,
|
| 35 |
+
CyHalfTweedieLoss,
|
| 36 |
+
CyHalfTweedieLossIdentity,
|
| 37 |
+
CyHuberLoss,
|
| 38 |
+
CyPinballLoss,
|
| 39 |
+
)
|
| 40 |
+
from sklearn._loss.link import (
|
| 41 |
+
HalfLogitLink,
|
| 42 |
+
IdentityLink,
|
| 43 |
+
Interval,
|
| 44 |
+
LogitLink,
|
| 45 |
+
LogLink,
|
| 46 |
+
MultinomialLogit,
|
| 47 |
+
)
|
| 48 |
+
from sklearn.utils import check_scalar
|
| 49 |
+
from sklearn.utils.stats import _weighted_percentile
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# Note: The shape of raw_prediction for multiclass classifications are
|
| 53 |
+
# - GradientBoostingClassifier: (n_samples, n_classes)
|
| 54 |
+
# - HistGradientBoostingClassifier: (n_classes, n_samples)
|
| 55 |
+
#
|
| 56 |
+
# Note: Instead of inheritance like
|
| 57 |
+
#
|
| 58 |
+
# class BaseLoss(BaseLink, CyLossFunction):
|
| 59 |
+
# ...
|
| 60 |
+
#
|
| 61 |
+
# # Note: Naturally, we would inherit in the following order
|
| 62 |
+
# # class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss)
|
| 63 |
+
# # But because of https://github.com/cython/cython/issues/4350 we set BaseLoss as
|
| 64 |
+
# # the last one. This, of course, changes the MRO.
|
| 65 |
+
# class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss):
|
| 66 |
+
#
|
| 67 |
+
# we use composition. This way we improve maintainability by avoiding the above
|
| 68 |
+
# mentioned Cython edge case and have easier to understand code (which method calls
|
| 69 |
+
# which code).
|
| 70 |
+
class BaseLoss:
|
| 71 |
+
"""Base class for a loss function of 1-dimensional targets.
|
| 72 |
+
|
| 73 |
+
Conventions:
|
| 74 |
+
|
| 75 |
+
- y_true.shape = sample_weight.shape = (n_samples,)
|
| 76 |
+
- y_pred.shape = raw_prediction.shape = (n_samples,)
|
| 77 |
+
- If is_multiclass is true (multiclass classification), then
|
| 78 |
+
y_pred.shape = raw_prediction.shape = (n_samples, n_classes)
|
| 79 |
+
Note that this corresponds to the return value of decision_function.
|
| 80 |
+
|
| 81 |
+
y_true, y_pred, sample_weight and raw_prediction must either be all float64
|
| 82 |
+
or all float32.
|
| 83 |
+
gradient and hessian must be either both float64 or both float32.
|
| 84 |
+
|
| 85 |
+
Note that y_pred = link.inverse(raw_prediction).
|
| 86 |
+
|
| 87 |
+
Specific loss classes can inherit specific link classes to satisfy
|
| 88 |
+
BaseLink's abstractmethods.
|
| 89 |
+
|
| 90 |
+
Parameters
|
| 91 |
+
----------
|
| 92 |
+
sample_weight : {None, ndarray}
|
| 93 |
+
If sample_weight is None, the hessian might be constant.
|
| 94 |
+
n_classes : {None, int}
|
| 95 |
+
The number of classes for classification, else None.
|
| 96 |
+
|
| 97 |
+
Attributes
|
| 98 |
+
----------
|
| 99 |
+
closs: CyLossFunction
|
| 100 |
+
link : BaseLink
|
| 101 |
+
interval_y_true : Interval
|
| 102 |
+
Valid interval for y_true
|
| 103 |
+
interval_y_pred : Interval
|
| 104 |
+
Valid Interval for y_pred
|
| 105 |
+
differentiable : bool
|
| 106 |
+
Indicates whether or not loss function is differentiable in
|
| 107 |
+
raw_prediction everywhere.
|
| 108 |
+
need_update_leaves_values : bool
|
| 109 |
+
Indicates whether decision trees in gradient boosting need to uptade
|
| 110 |
+
leave values after having been fit to the (negative) gradients.
|
| 111 |
+
approx_hessian : bool
|
| 112 |
+
Indicates whether the hessian is approximated or exact. If,
|
| 113 |
+
approximated, it should be larger or equal to the exact one.
|
| 114 |
+
constant_hessian : bool
|
| 115 |
+
Indicates whether the hessian is one for this loss.
|
| 116 |
+
is_multiclass : bool
|
| 117 |
+
Indicates whether n_classes > 2 is allowed.
|
| 118 |
+
"""
|
| 119 |
+
|
| 120 |
+
# For gradient boosted decision trees:
|
| 121 |
+
# This variable indicates whether the loss requires the leaves values to
|
| 122 |
+
# be updated once the tree has been trained. The trees are trained to
|
| 123 |
+
# predict a Newton-Raphson step (see grower._finalize_leaf()). But for
|
| 124 |
+
# some losses (e.g. least absolute deviation) we need to adjust the tree
|
| 125 |
+
# values to account for the "line search" of the gradient descent
|
| 126 |
+
# procedure. See the original paper Greedy Function Approximation: A
|
| 127 |
+
# Gradient Boosting Machine by Friedman
|
| 128 |
+
# (https://statweb.stanford.edu/~jhf/ftp/trebst.pdf) for the theory.
|
| 129 |
+
differentiable = True
|
| 130 |
+
need_update_leaves_values = False
|
| 131 |
+
is_multiclass = False
|
| 132 |
+
|
| 133 |
+
def __init__(self, closs, link, n_classes=None):
|
| 134 |
+
self.closs = closs
|
| 135 |
+
self.link = link
|
| 136 |
+
self.approx_hessian = False
|
| 137 |
+
self.constant_hessian = False
|
| 138 |
+
self.n_classes = n_classes
|
| 139 |
+
self.interval_y_true = Interval(-np.inf, np.inf, False, False)
|
| 140 |
+
self.interval_y_pred = self.link.interval_y_pred
|
| 141 |
+
|
| 142 |
+
def in_y_true_range(self, y):
|
| 143 |
+
"""Return True if y is in the valid range of y_true.
|
| 144 |
+
|
| 145 |
+
Parameters
|
| 146 |
+
----------
|
| 147 |
+
y : ndarray
|
| 148 |
+
"""
|
| 149 |
+
return self.interval_y_true.includes(y)
|
| 150 |
+
|
| 151 |
+
def in_y_pred_range(self, y):
|
| 152 |
+
"""Return True if y is in the valid range of y_pred.
|
| 153 |
+
|
| 154 |
+
Parameters
|
| 155 |
+
----------
|
| 156 |
+
y : ndarray
|
| 157 |
+
"""
|
| 158 |
+
return self.interval_y_pred.includes(y)
|
| 159 |
+
|
| 160 |
+
def loss(
|
| 161 |
+
self,
|
| 162 |
+
y_true,
|
| 163 |
+
raw_prediction,
|
| 164 |
+
sample_weight=None,
|
| 165 |
+
loss_out=None,
|
| 166 |
+
n_threads=1,
|
| 167 |
+
):
|
| 168 |
+
"""Compute the pointwise loss value for each input.
|
| 169 |
+
|
| 170 |
+
Parameters
|
| 171 |
+
----------
|
| 172 |
+
y_true : C-contiguous array of shape (n_samples,)
|
| 173 |
+
Observed, true target values.
|
| 174 |
+
raw_prediction : C-contiguous array of shape (n_samples,) or array of \
|
| 175 |
+
shape (n_samples, n_classes)
|
| 176 |
+
Raw prediction values (in link space).
|
| 177 |
+
sample_weight : None or C-contiguous array of shape (n_samples,)
|
| 178 |
+
Sample weights.
|
| 179 |
+
loss_out : None or C-contiguous array of shape (n_samples,)
|
| 180 |
+
A location into which the result is stored. If None, a new array
|
| 181 |
+
might be created.
|
| 182 |
+
n_threads : int, default=1
|
| 183 |
+
Might use openmp thread parallelism.
|
| 184 |
+
|
| 185 |
+
Returns
|
| 186 |
+
-------
|
| 187 |
+
loss : array of shape (n_samples,)
|
| 188 |
+
Element-wise loss function.
|
| 189 |
+
"""
|
| 190 |
+
if loss_out is None:
|
| 191 |
+
loss_out = np.empty_like(y_true)
|
| 192 |
+
# Be graceful to shape (n_samples, 1) -> (n_samples,)
|
| 193 |
+
if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
|
| 194 |
+
raw_prediction = raw_prediction.squeeze(1)
|
| 195 |
+
|
| 196 |
+
self.closs.loss(
|
| 197 |
+
y_true=y_true,
|
| 198 |
+
raw_prediction=raw_prediction,
|
| 199 |
+
sample_weight=sample_weight,
|
| 200 |
+
loss_out=loss_out,
|
| 201 |
+
n_threads=n_threads,
|
| 202 |
+
)
|
| 203 |
+
return loss_out
|
| 204 |
+
|
| 205 |
+
def loss_gradient(
|
| 206 |
+
self,
|
| 207 |
+
y_true,
|
| 208 |
+
raw_prediction,
|
| 209 |
+
sample_weight=None,
|
| 210 |
+
loss_out=None,
|
| 211 |
+
gradient_out=None,
|
| 212 |
+
n_threads=1,
|
| 213 |
+
):
|
| 214 |
+
"""Compute loss and gradient w.r.t. raw_prediction for each input.
|
| 215 |
+
|
| 216 |
+
Parameters
|
| 217 |
+
----------
|
| 218 |
+
y_true : C-contiguous array of shape (n_samples,)
|
| 219 |
+
Observed, true target values.
|
| 220 |
+
raw_prediction : C-contiguous array of shape (n_samples,) or array of \
|
| 221 |
+
shape (n_samples, n_classes)
|
| 222 |
+
Raw prediction values (in link space).
|
| 223 |
+
sample_weight : None or C-contiguous array of shape (n_samples,)
|
| 224 |
+
Sample weights.
|
| 225 |
+
loss_out : None or C-contiguous array of shape (n_samples,)
|
| 226 |
+
A location into which the loss is stored. If None, a new array
|
| 227 |
+
might be created.
|
| 228 |
+
gradient_out : None or C-contiguous array of shape (n_samples,) or array \
|
| 229 |
+
of shape (n_samples, n_classes)
|
| 230 |
+
A location into which the gradient is stored. If None, a new array
|
| 231 |
+
might be created.
|
| 232 |
+
n_threads : int, default=1
|
| 233 |
+
Might use openmp thread parallelism.
|
| 234 |
+
|
| 235 |
+
Returns
|
| 236 |
+
-------
|
| 237 |
+
loss : array of shape (n_samples,)
|
| 238 |
+
Element-wise loss function.
|
| 239 |
+
|
| 240 |
+
gradient : array of shape (n_samples,) or (n_samples, n_classes)
|
| 241 |
+
Element-wise gradients.
|
| 242 |
+
"""
|
| 243 |
+
if loss_out is None:
|
| 244 |
+
if gradient_out is None:
|
| 245 |
+
loss_out = np.empty_like(y_true)
|
| 246 |
+
gradient_out = np.empty_like(raw_prediction)
|
| 247 |
+
else:
|
| 248 |
+
loss_out = np.empty_like(y_true, dtype=gradient_out.dtype)
|
| 249 |
+
elif gradient_out is None:
|
| 250 |
+
gradient_out = np.empty_like(raw_prediction, dtype=loss_out.dtype)
|
| 251 |
+
|
| 252 |
+
# Be graceful to shape (n_samples, 1) -> (n_samples,)
|
| 253 |
+
if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
|
| 254 |
+
raw_prediction = raw_prediction.squeeze(1)
|
| 255 |
+
if gradient_out.ndim == 2 and gradient_out.shape[1] == 1:
|
| 256 |
+
gradient_out = gradient_out.squeeze(1)
|
| 257 |
+
|
| 258 |
+
self.closs.loss_gradient(
|
| 259 |
+
y_true=y_true,
|
| 260 |
+
raw_prediction=raw_prediction,
|
| 261 |
+
sample_weight=sample_weight,
|
| 262 |
+
loss_out=loss_out,
|
| 263 |
+
gradient_out=gradient_out,
|
| 264 |
+
n_threads=n_threads,
|
| 265 |
+
)
|
| 266 |
+
return loss_out, gradient_out
|
| 267 |
+
|
| 268 |
+
def gradient(
|
| 269 |
+
self,
|
| 270 |
+
y_true,
|
| 271 |
+
raw_prediction,
|
| 272 |
+
sample_weight=None,
|
| 273 |
+
gradient_out=None,
|
| 274 |
+
n_threads=1,
|
| 275 |
+
):
|
| 276 |
+
"""Compute gradient of loss w.r.t raw_prediction for each input.
|
| 277 |
+
|
| 278 |
+
Parameters
|
| 279 |
+
----------
|
| 280 |
+
y_true : C-contiguous array of shape (n_samples,)
|
| 281 |
+
Observed, true target values.
|
| 282 |
+
raw_prediction : C-contiguous array of shape (n_samples,) or array of \
|
| 283 |
+
shape (n_samples, n_classes)
|
| 284 |
+
Raw prediction values (in link space).
|
| 285 |
+
sample_weight : None or C-contiguous array of shape (n_samples,)
|
| 286 |
+
Sample weights.
|
| 287 |
+
gradient_out : None or C-contiguous array of shape (n_samples,) or array \
|
| 288 |
+
of shape (n_samples, n_classes)
|
| 289 |
+
A location into which the result is stored. If None, a new array
|
| 290 |
+
might be created.
|
| 291 |
+
n_threads : int, default=1
|
| 292 |
+
Might use openmp thread parallelism.
|
| 293 |
+
|
| 294 |
+
Returns
|
| 295 |
+
-------
|
| 296 |
+
gradient : array of shape (n_samples,) or (n_samples, n_classes)
|
| 297 |
+
Element-wise gradients.
|
| 298 |
+
"""
|
| 299 |
+
if gradient_out is None:
|
| 300 |
+
gradient_out = np.empty_like(raw_prediction)
|
| 301 |
+
|
| 302 |
+
# Be graceful to shape (n_samples, 1) -> (n_samples,)
|
| 303 |
+
if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
|
| 304 |
+
raw_prediction = raw_prediction.squeeze(1)
|
| 305 |
+
if gradient_out.ndim == 2 and gradient_out.shape[1] == 1:
|
| 306 |
+
gradient_out = gradient_out.squeeze(1)
|
| 307 |
+
|
| 308 |
+
self.closs.gradient(
|
| 309 |
+
y_true=y_true,
|
| 310 |
+
raw_prediction=raw_prediction,
|
| 311 |
+
sample_weight=sample_weight,
|
| 312 |
+
gradient_out=gradient_out,
|
| 313 |
+
n_threads=n_threads,
|
| 314 |
+
)
|
| 315 |
+
return gradient_out
|
| 316 |
+
|
| 317 |
+
def gradient_hessian(
|
| 318 |
+
self,
|
| 319 |
+
y_true,
|
| 320 |
+
raw_prediction,
|
| 321 |
+
sample_weight=None,
|
| 322 |
+
gradient_out=None,
|
| 323 |
+
hessian_out=None,
|
| 324 |
+
n_threads=1,
|
| 325 |
+
):
|
| 326 |
+
"""Compute gradient and hessian of loss w.r.t raw_prediction.
|
| 327 |
+
|
| 328 |
+
Parameters
|
| 329 |
+
----------
|
| 330 |
+
y_true : C-contiguous array of shape (n_samples,)
|
| 331 |
+
Observed, true target values.
|
| 332 |
+
raw_prediction : C-contiguous array of shape (n_samples,) or array of \
|
| 333 |
+
shape (n_samples, n_classes)
|
| 334 |
+
Raw prediction values (in link space).
|
| 335 |
+
sample_weight : None or C-contiguous array of shape (n_samples,)
|
| 336 |
+
Sample weights.
|
| 337 |
+
gradient_out : None or C-contiguous array of shape (n_samples,) or array \
|
| 338 |
+
of shape (n_samples, n_classes)
|
| 339 |
+
A location into which the gradient is stored. If None, a new array
|
| 340 |
+
might be created.
|
| 341 |
+
hessian_out : None or C-contiguous array of shape (n_samples,) or array \
|
| 342 |
+
of shape (n_samples, n_classes)
|
| 343 |
+
A location into which the hessian is stored. If None, a new array
|
| 344 |
+
might be created.
|
| 345 |
+
n_threads : int, default=1
|
| 346 |
+
Might use openmp thread parallelism.
|
| 347 |
+
|
| 348 |
+
Returns
|
| 349 |
+
-------
|
| 350 |
+
gradient : arrays of shape (n_samples,) or (n_samples, n_classes)
|
| 351 |
+
Element-wise gradients.
|
| 352 |
+
|
| 353 |
+
hessian : arrays of shape (n_samples,) or (n_samples, n_classes)
|
| 354 |
+
Element-wise hessians.
|
| 355 |
+
"""
|
| 356 |
+
if gradient_out is None:
|
| 357 |
+
if hessian_out is None:
|
| 358 |
+
gradient_out = np.empty_like(raw_prediction)
|
| 359 |
+
hessian_out = np.empty_like(raw_prediction)
|
| 360 |
+
else:
|
| 361 |
+
gradient_out = np.empty_like(hessian_out)
|
| 362 |
+
elif hessian_out is None:
|
| 363 |
+
hessian_out = np.empty_like(gradient_out)
|
| 364 |
+
|
| 365 |
+
# Be graceful to shape (n_samples, 1) -> (n_samples,)
|
| 366 |
+
if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
|
| 367 |
+
raw_prediction = raw_prediction.squeeze(1)
|
| 368 |
+
if gradient_out.ndim == 2 and gradient_out.shape[1] == 1:
|
| 369 |
+
gradient_out = gradient_out.squeeze(1)
|
| 370 |
+
if hessian_out.ndim == 2 and hessian_out.shape[1] == 1:
|
| 371 |
+
hessian_out = hessian_out.squeeze(1)
|
| 372 |
+
|
| 373 |
+
self.closs.gradient_hessian(
|
| 374 |
+
y_true=y_true,
|
| 375 |
+
raw_prediction=raw_prediction,
|
| 376 |
+
sample_weight=sample_weight,
|
| 377 |
+
gradient_out=gradient_out,
|
| 378 |
+
hessian_out=hessian_out,
|
| 379 |
+
n_threads=n_threads,
|
| 380 |
+
)
|
| 381 |
+
return gradient_out, hessian_out
|
| 382 |
+
|
| 383 |
+
def __call__(self, y_true, raw_prediction, sample_weight=None, n_threads=1):
|
| 384 |
+
"""Compute the weighted average loss.
|
| 385 |
+
|
| 386 |
+
Parameters
|
| 387 |
+
----------
|
| 388 |
+
y_true : C-contiguous array of shape (n_samples,)
|
| 389 |
+
Observed, true target values.
|
| 390 |
+
raw_prediction : C-contiguous array of shape (n_samples,) or array of \
|
| 391 |
+
shape (n_samples, n_classes)
|
| 392 |
+
Raw prediction values (in link space).
|
| 393 |
+
sample_weight : None or C-contiguous array of shape (n_samples,)
|
| 394 |
+
Sample weights.
|
| 395 |
+
n_threads : int, default=1
|
| 396 |
+
Might use openmp thread parallelism.
|
| 397 |
+
|
| 398 |
+
Returns
|
| 399 |
+
-------
|
| 400 |
+
loss : float
|
| 401 |
+
Mean or averaged loss function.
|
| 402 |
+
"""
|
| 403 |
+
return np.average(
|
| 404 |
+
self.loss(
|
| 405 |
+
y_true=y_true,
|
| 406 |
+
raw_prediction=raw_prediction,
|
| 407 |
+
sample_weight=None,
|
| 408 |
+
loss_out=None,
|
| 409 |
+
n_threads=n_threads,
|
| 410 |
+
),
|
| 411 |
+
weights=sample_weight,
|
| 412 |
+
)
|
| 413 |
+
|
| 414 |
+
def fit_intercept_only(self, y_true, sample_weight=None):
|
| 415 |
+
"""Compute raw_prediction of an intercept-only model.
|
| 416 |
+
|
| 417 |
+
This can be used as initial estimates of predictions, i.e. before the
|
| 418 |
+
first iteration in fit.
|
| 419 |
+
|
| 420 |
+
Parameters
|
| 421 |
+
----------
|
| 422 |
+
y_true : array-like of shape (n_samples,)
|
| 423 |
+
Observed, true target values.
|
| 424 |
+
sample_weight : None or array of shape (n_samples,)
|
| 425 |
+
Sample weights.
|
| 426 |
+
|
| 427 |
+
Returns
|
| 428 |
+
-------
|
| 429 |
+
raw_prediction : numpy scalar or array of shape (n_classes,)
|
| 430 |
+
Raw predictions of an intercept-only model.
|
| 431 |
+
"""
|
| 432 |
+
# As default, take weighted average of the target over the samples
|
| 433 |
+
# axis=0 and then transform into link-scale (raw_prediction).
|
| 434 |
+
y_pred = np.average(y_true, weights=sample_weight, axis=0)
|
| 435 |
+
eps = 10 * np.finfo(y_pred.dtype).eps
|
| 436 |
+
|
| 437 |
+
if self.interval_y_pred.low == -np.inf:
|
| 438 |
+
a_min = None
|
| 439 |
+
elif self.interval_y_pred.low_inclusive:
|
| 440 |
+
a_min = self.interval_y_pred.low
|
| 441 |
+
else:
|
| 442 |
+
a_min = self.interval_y_pred.low + eps
|
| 443 |
+
|
| 444 |
+
if self.interval_y_pred.high == np.inf:
|
| 445 |
+
a_max = None
|
| 446 |
+
elif self.interval_y_pred.high_inclusive:
|
| 447 |
+
a_max = self.interval_y_pred.high
|
| 448 |
+
else:
|
| 449 |
+
a_max = self.interval_y_pred.high - eps
|
| 450 |
+
|
| 451 |
+
if a_min is None and a_max is None:
|
| 452 |
+
return self.link.link(y_pred)
|
| 453 |
+
else:
|
| 454 |
+
return self.link.link(np.clip(y_pred, a_min, a_max))
|
| 455 |
+
|
| 456 |
+
def constant_to_optimal_zero(self, y_true, sample_weight=None):
|
| 457 |
+
"""Calculate term dropped in loss.
|
| 458 |
+
|
| 459 |
+
With this term added, the loss of perfect predictions is zero.
|
| 460 |
+
|
| 461 |
+
Parameters
|
| 462 |
+
----------
|
| 463 |
+
y_true : array-like of shape (n_samples,)
|
| 464 |
+
Observed, true target values.
|
| 465 |
+
|
| 466 |
+
sample_weight : None or array of shape (n_samples,), default=None
|
| 467 |
+
Sample weights.
|
| 468 |
+
|
| 469 |
+
Returns
|
| 470 |
+
-------
|
| 471 |
+
constant : ndarray of shape (n_samples,)
|
| 472 |
+
Constant value to be added to raw predictions so that the loss
|
| 473 |
+
of perfect predictions becomes zero.
|
| 474 |
+
"""
|
| 475 |
+
return np.zeros_like(y_true)
|
| 476 |
+
|
| 477 |
+
def init_gradient_and_hessian(self, n_samples, dtype=np.float64, order="F"):
|
| 478 |
+
"""Initialize arrays for gradients and hessians.
|
| 479 |
+
|
| 480 |
+
Unless hessians are constant, arrays are initialized with undefined values.
|
| 481 |
+
|
| 482 |
+
Parameters
|
| 483 |
+
----------
|
| 484 |
+
n_samples : int
|
| 485 |
+
The number of samples, usually passed to `fit()`.
|
| 486 |
+
dtype : {np.float64, np.float32}, default=np.float64
|
| 487 |
+
The dtype of the arrays gradient and hessian.
|
| 488 |
+
order : {'C', 'F'}, default='F'
|
| 489 |
+
Order of the arrays gradient and hessian. The default 'F' makes the arrays
|
| 490 |
+
contiguous along samples.
|
| 491 |
+
|
| 492 |
+
Returns
|
| 493 |
+
-------
|
| 494 |
+
gradient : C-contiguous array of shape (n_samples,) or array of shape \
|
| 495 |
+
(n_samples, n_classes)
|
| 496 |
+
Empty array (allocated but not initialized) to be used as argument
|
| 497 |
+
gradient_out.
|
| 498 |
+
hessian : C-contiguous array of shape (n_samples,), array of shape
|
| 499 |
+
(n_samples, n_classes) or shape (1,)
|
| 500 |
+
Empty (allocated but not initialized) array to be used as argument
|
| 501 |
+
hessian_out.
|
| 502 |
+
If constant_hessian is True (e.g. `HalfSquaredError`), the array is
|
| 503 |
+
initialized to ``1``.
|
| 504 |
+
"""
|
| 505 |
+
if dtype not in (np.float32, np.float64):
|
| 506 |
+
raise ValueError(
|
| 507 |
+
"Valid options for 'dtype' are np.float32 and np.float64. "
|
| 508 |
+
f"Got dtype={dtype} instead."
|
| 509 |
+
)
|
| 510 |
+
|
| 511 |
+
if self.is_multiclass:
|
| 512 |
+
shape = (n_samples, self.n_classes)
|
| 513 |
+
else:
|
| 514 |
+
shape = (n_samples,)
|
| 515 |
+
gradient = np.empty(shape=shape, dtype=dtype, order=order)
|
| 516 |
+
|
| 517 |
+
if self.constant_hessian:
|
| 518 |
+
# If the hessians are constant, we consider them equal to 1.
|
| 519 |
+
# - This is correct for HalfSquaredError
|
| 520 |
+
# - For AbsoluteError, hessians are actually 0, but they are
|
| 521 |
+
# always ignored anyway.
|
| 522 |
+
hessian = np.ones(shape=(1,), dtype=dtype)
|
| 523 |
+
else:
|
| 524 |
+
hessian = np.empty(shape=shape, dtype=dtype, order=order)
|
| 525 |
+
|
| 526 |
+
return gradient, hessian
|
| 527 |
+
|
| 528 |
+
|
| 529 |
+
# Note: Naturally, we would inherit in the following order
|
| 530 |
+
# class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss)
|
| 531 |
+
# But because of https://github.com/cython/cython/issues/4350 we
|
| 532 |
+
# set BaseLoss as the last one. This, of course, changes the MRO.
|
| 533 |
+
class HalfSquaredError(BaseLoss):
|
| 534 |
+
"""Half squared error with identity link, for regression.
|
| 535 |
+
|
| 536 |
+
Domain:
|
| 537 |
+
y_true and y_pred all real numbers
|
| 538 |
+
|
| 539 |
+
Link:
|
| 540 |
+
y_pred = raw_prediction
|
| 541 |
+
|
| 542 |
+
For a given sample x_i, half squared error is defined as::
|
| 543 |
+
|
| 544 |
+
loss(x_i) = 0.5 * (y_true_i - raw_prediction_i)**2
|
| 545 |
+
|
| 546 |
+
The factor of 0.5 simplifies the computation of gradients and results in a
|
| 547 |
+
unit hessian (and is consistent with what is done in LightGBM). It is also
|
| 548 |
+
half the Normal distribution deviance.
|
| 549 |
+
"""
|
| 550 |
+
|
| 551 |
+
def __init__(self, sample_weight=None):
|
| 552 |
+
super().__init__(closs=CyHalfSquaredError(), link=IdentityLink())
|
| 553 |
+
self.constant_hessian = sample_weight is None
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
class AbsoluteError(BaseLoss):
|
| 557 |
+
"""Absolute error with identity link, for regression.
|
| 558 |
+
|
| 559 |
+
Domain:
|
| 560 |
+
y_true and y_pred all real numbers
|
| 561 |
+
|
| 562 |
+
Link:
|
| 563 |
+
y_pred = raw_prediction
|
| 564 |
+
|
| 565 |
+
For a given sample x_i, the absolute error is defined as::
|
| 566 |
+
|
| 567 |
+
loss(x_i) = |y_true_i - raw_prediction_i|
|
| 568 |
+
|
| 569 |
+
Note that the exact hessian = 0 almost everywhere (except at one point, therefore
|
| 570 |
+
differentiable = False). Optimization routines like in HGBT, however, need a
|
| 571 |
+
hessian > 0. Therefore, we assign 1.
|
| 572 |
+
"""
|
| 573 |
+
|
| 574 |
+
differentiable = False
|
| 575 |
+
need_update_leaves_values = True
|
| 576 |
+
|
| 577 |
+
def __init__(self, sample_weight=None):
|
| 578 |
+
super().__init__(closs=CyAbsoluteError(), link=IdentityLink())
|
| 579 |
+
self.approx_hessian = True
|
| 580 |
+
self.constant_hessian = sample_weight is None
|
| 581 |
+
|
| 582 |
+
def fit_intercept_only(self, y_true, sample_weight=None):
|
| 583 |
+
"""Compute raw_prediction of an intercept-only model.
|
| 584 |
+
|
| 585 |
+
This is the weighted median of the target, i.e. over the samples
|
| 586 |
+
axis=0.
|
| 587 |
+
"""
|
| 588 |
+
if sample_weight is None:
|
| 589 |
+
return np.median(y_true, axis=0)
|
| 590 |
+
else:
|
| 591 |
+
return _weighted_percentile(y_true, sample_weight, 50)
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
class PinballLoss(BaseLoss):
|
| 595 |
+
"""Quantile loss aka pinball loss, for regression.
|
| 596 |
+
|
| 597 |
+
Domain:
|
| 598 |
+
y_true and y_pred all real numbers
|
| 599 |
+
quantile in (0, 1)
|
| 600 |
+
|
| 601 |
+
Link:
|
| 602 |
+
y_pred = raw_prediction
|
| 603 |
+
|
| 604 |
+
For a given sample x_i, the pinball loss is defined as::
|
| 605 |
+
|
| 606 |
+
loss(x_i) = rho_{quantile}(y_true_i - raw_prediction_i)
|
| 607 |
+
|
| 608 |
+
rho_{quantile}(u) = u * (quantile - 1_{u<0})
|
| 609 |
+
= -u *(1 - quantile) if u < 0
|
| 610 |
+
u * quantile if u >= 0
|
| 611 |
+
|
| 612 |
+
Note: 2 * PinballLoss(quantile=0.5) equals AbsoluteError().
|
| 613 |
+
|
| 614 |
+
Note that the exact hessian = 0 almost everywhere (except at one point, therefore
|
| 615 |
+
differentiable = False). Optimization routines like in HGBT, however, need a
|
| 616 |
+
hessian > 0. Therefore, we assign 1.
|
| 617 |
+
|
| 618 |
+
Additional Attributes
|
| 619 |
+
---------------------
|
| 620 |
+
quantile : float
|
| 621 |
+
The quantile level of the quantile to be estimated. Must be in range (0, 1).
|
| 622 |
+
"""
|
| 623 |
+
|
| 624 |
+
differentiable = False
|
| 625 |
+
need_update_leaves_values = True
|
| 626 |
+
|
| 627 |
+
def __init__(self, sample_weight=None, quantile=0.5):
|
| 628 |
+
check_scalar(
|
| 629 |
+
quantile,
|
| 630 |
+
"quantile",
|
| 631 |
+
target_type=numbers.Real,
|
| 632 |
+
min_val=0,
|
| 633 |
+
max_val=1,
|
| 634 |
+
include_boundaries="neither",
|
| 635 |
+
)
|
| 636 |
+
super().__init__(
|
| 637 |
+
closs=CyPinballLoss(quantile=float(quantile)),
|
| 638 |
+
link=IdentityLink(),
|
| 639 |
+
)
|
| 640 |
+
self.approx_hessian = True
|
| 641 |
+
self.constant_hessian = sample_weight is None
|
| 642 |
+
|
| 643 |
+
def fit_intercept_only(self, y_true, sample_weight=None):
|
| 644 |
+
"""Compute raw_prediction of an intercept-only model.
|
| 645 |
+
|
| 646 |
+
This is the weighted median of the target, i.e. over the samples
|
| 647 |
+
axis=0.
|
| 648 |
+
"""
|
| 649 |
+
if sample_weight is None:
|
| 650 |
+
return np.percentile(y_true, 100 * self.closs.quantile, axis=0)
|
| 651 |
+
else:
|
| 652 |
+
return _weighted_percentile(
|
| 653 |
+
y_true, sample_weight, 100 * self.closs.quantile
|
| 654 |
+
)
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
class HuberLoss(BaseLoss):
|
| 658 |
+
"""Huber loss, for regression.
|
| 659 |
+
|
| 660 |
+
Domain:
|
| 661 |
+
y_true and y_pred all real numbers
|
| 662 |
+
quantile in (0, 1)
|
| 663 |
+
|
| 664 |
+
Link:
|
| 665 |
+
y_pred = raw_prediction
|
| 666 |
+
|
| 667 |
+
For a given sample x_i, the Huber loss is defined as::
|
| 668 |
+
|
| 669 |
+
loss(x_i) = 1/2 * abserr**2 if abserr <= delta
|
| 670 |
+
delta * (abserr - delta/2) if abserr > delta
|
| 671 |
+
|
| 672 |
+
abserr = |y_true_i - raw_prediction_i|
|
| 673 |
+
delta = quantile(abserr, self.quantile)
|
| 674 |
+
|
| 675 |
+
Note: HuberLoss(quantile=1) equals HalfSquaredError and HuberLoss(quantile=0)
|
| 676 |
+
equals delta * (AbsoluteError() - delta/2).
|
| 677 |
+
|
| 678 |
+
Additional Attributes
|
| 679 |
+
---------------------
|
| 680 |
+
quantile : float
|
| 681 |
+
The quantile level which defines the breaking point `delta` to distinguish
|
| 682 |
+
between absolute error and squared error. Must be in range (0, 1).
|
| 683 |
+
|
| 684 |
+
Reference
|
| 685 |
+
---------
|
| 686 |
+
.. [1] Friedman, J.H. (2001). :doi:`Greedy function approximation: A gradient
|
| 687 |
+
boosting machine <10.1214/aos/1013203451>`.
|
| 688 |
+
Annals of Statistics, 29, 1189-1232.
|
| 689 |
+
"""
|
| 690 |
+
|
| 691 |
+
differentiable = False
|
| 692 |
+
need_update_leaves_values = True
|
| 693 |
+
|
| 694 |
+
def __init__(self, sample_weight=None, quantile=0.9, delta=0.5):
|
| 695 |
+
check_scalar(
|
| 696 |
+
quantile,
|
| 697 |
+
"quantile",
|
| 698 |
+
target_type=numbers.Real,
|
| 699 |
+
min_val=0,
|
| 700 |
+
max_val=1,
|
| 701 |
+
include_boundaries="neither",
|
| 702 |
+
)
|
| 703 |
+
self.quantile = quantile # This is better stored outside of Cython.
|
| 704 |
+
super().__init__(
|
| 705 |
+
closs=CyHuberLoss(delta=float(delta)),
|
| 706 |
+
link=IdentityLink(),
|
| 707 |
+
)
|
| 708 |
+
self.approx_hessian = True
|
| 709 |
+
self.constant_hessian = False
|
| 710 |
+
|
| 711 |
+
def fit_intercept_only(self, y_true, sample_weight=None):
|
| 712 |
+
"""Compute raw_prediction of an intercept-only model.
|
| 713 |
+
|
| 714 |
+
This is the weighted median of the target, i.e. over the samples
|
| 715 |
+
axis=0.
|
| 716 |
+
"""
|
| 717 |
+
# See formula before algo 4 in Friedman (2001), but we apply it to y_true,
|
| 718 |
+
# not to the residual y_true - raw_prediction. An estimator like
|
| 719 |
+
# HistGradientBoostingRegressor might then call it on the residual, e.g.
|
| 720 |
+
# fit_intercept_only(y_true - raw_prediction).
|
| 721 |
+
if sample_weight is None:
|
| 722 |
+
median = np.percentile(y_true, 50, axis=0)
|
| 723 |
+
else:
|
| 724 |
+
median = _weighted_percentile(y_true, sample_weight, 50)
|
| 725 |
+
diff = y_true - median
|
| 726 |
+
term = np.sign(diff) * np.minimum(self.closs.delta, np.abs(diff))
|
| 727 |
+
return median + np.average(term, weights=sample_weight)
|
| 728 |
+
|
| 729 |
+
|
| 730 |
+
class HalfPoissonLoss(BaseLoss):
|
| 731 |
+
"""Half Poisson deviance loss with log-link, for regression.
|
| 732 |
+
|
| 733 |
+
Domain:
|
| 734 |
+
y_true in non-negative real numbers
|
| 735 |
+
y_pred in positive real numbers
|
| 736 |
+
|
| 737 |
+
Link:
|
| 738 |
+
y_pred = exp(raw_prediction)
|
| 739 |
+
|
| 740 |
+
For a given sample x_i, half the Poisson deviance is defined as::
|
| 741 |
+
|
| 742 |
+
loss(x_i) = y_true_i * log(y_true_i/exp(raw_prediction_i))
|
| 743 |
+
- y_true_i + exp(raw_prediction_i)
|
| 744 |
+
|
| 745 |
+
Half the Poisson deviance is actually the negative log-likelihood up to
|
| 746 |
+
constant terms (not involving raw_prediction) and simplifies the
|
| 747 |
+
computation of the gradients.
|
| 748 |
+
We also skip the constant term `y_true_i * log(y_true_i) - y_true_i`.
|
| 749 |
+
"""
|
| 750 |
+
|
| 751 |
+
def __init__(self, sample_weight=None):
|
| 752 |
+
super().__init__(closs=CyHalfPoissonLoss(), link=LogLink())
|
| 753 |
+
self.interval_y_true = Interval(0, np.inf, True, False)
|
| 754 |
+
|
| 755 |
+
def constant_to_optimal_zero(self, y_true, sample_weight=None):
|
| 756 |
+
term = xlogy(y_true, y_true) - y_true
|
| 757 |
+
if sample_weight is not None:
|
| 758 |
+
term *= sample_weight
|
| 759 |
+
return term
|
| 760 |
+
|
| 761 |
+
|
| 762 |
+
class HalfGammaLoss(BaseLoss):
|
| 763 |
+
"""Half Gamma deviance loss with log-link, for regression.
|
| 764 |
+
|
| 765 |
+
Domain:
|
| 766 |
+
y_true and y_pred in positive real numbers
|
| 767 |
+
|
| 768 |
+
Link:
|
| 769 |
+
y_pred = exp(raw_prediction)
|
| 770 |
+
|
| 771 |
+
For a given sample x_i, half Gamma deviance loss is defined as::
|
| 772 |
+
|
| 773 |
+
loss(x_i) = log(exp(raw_prediction_i)/y_true_i)
|
| 774 |
+
+ y_true/exp(raw_prediction_i) - 1
|
| 775 |
+
|
| 776 |
+
Half the Gamma deviance is actually proportional to the negative log-
|
| 777 |
+
likelihood up to constant terms (not involving raw_prediction) and
|
| 778 |
+
simplifies the computation of the gradients.
|
| 779 |
+
We also skip the constant term `-log(y_true_i) - 1`.
|
| 780 |
+
"""
|
| 781 |
+
|
| 782 |
+
def __init__(self, sample_weight=None):
|
| 783 |
+
super().__init__(closs=CyHalfGammaLoss(), link=LogLink())
|
| 784 |
+
self.interval_y_true = Interval(0, np.inf, False, False)
|
| 785 |
+
|
| 786 |
+
def constant_to_optimal_zero(self, y_true, sample_weight=None):
|
| 787 |
+
term = -np.log(y_true) - 1
|
| 788 |
+
if sample_weight is not None:
|
| 789 |
+
term *= sample_weight
|
| 790 |
+
return term
|
| 791 |
+
|
| 792 |
+
|
| 793 |
+
class HalfTweedieLoss(BaseLoss):
|
| 794 |
+
"""Half Tweedie deviance loss with log-link, for regression.
|
| 795 |
+
|
| 796 |
+
Domain:
|
| 797 |
+
y_true in real numbers for power <= 0
|
| 798 |
+
y_true in non-negative real numbers for 0 < power < 2
|
| 799 |
+
y_true in positive real numbers for 2 <= power
|
| 800 |
+
y_pred in positive real numbers
|
| 801 |
+
power in real numbers
|
| 802 |
+
|
| 803 |
+
Link:
|
| 804 |
+
y_pred = exp(raw_prediction)
|
| 805 |
+
|
| 806 |
+
For a given sample x_i, half Tweedie deviance loss with p=power is defined
|
| 807 |
+
as::
|
| 808 |
+
|
| 809 |
+
loss(x_i) = max(y_true_i, 0)**(2-p) / (1-p) / (2-p)
|
| 810 |
+
- y_true_i * exp(raw_prediction_i)**(1-p) / (1-p)
|
| 811 |
+
+ exp(raw_prediction_i)**(2-p) / (2-p)
|
| 812 |
+
|
| 813 |
+
Taking the limits for p=0, 1, 2 gives HalfSquaredError with a log link,
|
| 814 |
+
HalfPoissonLoss and HalfGammaLoss.
|
| 815 |
+
|
| 816 |
+
We also skip constant terms, but those are different for p=0, 1, 2.
|
| 817 |
+
Therefore, the loss is not continuous in `power`.
|
| 818 |
+
|
| 819 |
+
Note furthermore that although no Tweedie distribution exists for
|
| 820 |
+
0 < power < 1, it still gives a strictly consistent scoring function for
|
| 821 |
+
the expectation.
|
| 822 |
+
"""
|
| 823 |
+
|
| 824 |
+
def __init__(self, sample_weight=None, power=1.5):
|
| 825 |
+
super().__init__(
|
| 826 |
+
closs=CyHalfTweedieLoss(power=float(power)),
|
| 827 |
+
link=LogLink(),
|
| 828 |
+
)
|
| 829 |
+
if self.closs.power <= 0:
|
| 830 |
+
self.interval_y_true = Interval(-np.inf, np.inf, False, False)
|
| 831 |
+
elif self.closs.power < 2:
|
| 832 |
+
self.interval_y_true = Interval(0, np.inf, True, False)
|
| 833 |
+
else:
|
| 834 |
+
self.interval_y_true = Interval(0, np.inf, False, False)
|
| 835 |
+
|
| 836 |
+
def constant_to_optimal_zero(self, y_true, sample_weight=None):
|
| 837 |
+
if self.closs.power == 0:
|
| 838 |
+
return HalfSquaredError().constant_to_optimal_zero(
|
| 839 |
+
y_true=y_true, sample_weight=sample_weight
|
| 840 |
+
)
|
| 841 |
+
elif self.closs.power == 1:
|
| 842 |
+
return HalfPoissonLoss().constant_to_optimal_zero(
|
| 843 |
+
y_true=y_true, sample_weight=sample_weight
|
| 844 |
+
)
|
| 845 |
+
elif self.closs.power == 2:
|
| 846 |
+
return HalfGammaLoss().constant_to_optimal_zero(
|
| 847 |
+
y_true=y_true, sample_weight=sample_weight
|
| 848 |
+
)
|
| 849 |
+
else:
|
| 850 |
+
p = self.closs.power
|
| 851 |
+
term = np.power(np.maximum(y_true, 0), 2 - p) / (1 - p) / (2 - p)
|
| 852 |
+
if sample_weight is not None:
|
| 853 |
+
term *= sample_weight
|
| 854 |
+
return term
|
| 855 |
+
|
| 856 |
+
|
| 857 |
+
class HalfTweedieLossIdentity(BaseLoss):
|
| 858 |
+
"""Half Tweedie deviance loss with identity link, for regression.
|
| 859 |
+
|
| 860 |
+
Domain:
|
| 861 |
+
y_true in real numbers for power <= 0
|
| 862 |
+
y_true in non-negative real numbers for 0 < power < 2
|
| 863 |
+
y_true in positive real numbers for 2 <= power
|
| 864 |
+
y_pred in positive real numbers for power != 0
|
| 865 |
+
y_pred in real numbers for power = 0
|
| 866 |
+
power in real numbers
|
| 867 |
+
|
| 868 |
+
Link:
|
| 869 |
+
y_pred = raw_prediction
|
| 870 |
+
|
| 871 |
+
For a given sample x_i, half Tweedie deviance loss with p=power is defined
|
| 872 |
+
as::
|
| 873 |
+
|
| 874 |
+
loss(x_i) = max(y_true_i, 0)**(2-p) / (1-p) / (2-p)
|
| 875 |
+
- y_true_i * raw_prediction_i**(1-p) / (1-p)
|
| 876 |
+
+ raw_prediction_i**(2-p) / (2-p)
|
| 877 |
+
|
| 878 |
+
Note that the minimum value of this loss is 0.
|
| 879 |
+
|
| 880 |
+
Note furthermore that although no Tweedie distribution exists for
|
| 881 |
+
0 < power < 1, it still gives a strictly consistent scoring function for
|
| 882 |
+
the expectation.
|
| 883 |
+
"""
|
| 884 |
+
|
| 885 |
+
def __init__(self, sample_weight=None, power=1.5):
|
| 886 |
+
super().__init__(
|
| 887 |
+
closs=CyHalfTweedieLossIdentity(power=float(power)),
|
| 888 |
+
link=IdentityLink(),
|
| 889 |
+
)
|
| 890 |
+
if self.closs.power <= 0:
|
| 891 |
+
self.interval_y_true = Interval(-np.inf, np.inf, False, False)
|
| 892 |
+
elif self.closs.power < 2:
|
| 893 |
+
self.interval_y_true = Interval(0, np.inf, True, False)
|
| 894 |
+
else:
|
| 895 |
+
self.interval_y_true = Interval(0, np.inf, False, False)
|
| 896 |
+
|
| 897 |
+
if self.closs.power == 0:
|
| 898 |
+
self.interval_y_pred = Interval(-np.inf, np.inf, False, False)
|
| 899 |
+
else:
|
| 900 |
+
self.interval_y_pred = Interval(0, np.inf, False, False)
|
| 901 |
+
|
| 902 |
+
|
| 903 |
+
class HalfBinomialLoss(BaseLoss):
|
| 904 |
+
"""Half Binomial deviance loss with logit link, for binary classification.
|
| 905 |
+
|
| 906 |
+
This is also know as binary cross entropy, log-loss and logistic loss.
|
| 907 |
+
|
| 908 |
+
Domain:
|
| 909 |
+
y_true in [0, 1], i.e. regression on the unit interval
|
| 910 |
+
y_pred in (0, 1), i.e. boundaries excluded
|
| 911 |
+
|
| 912 |
+
Link:
|
| 913 |
+
y_pred = expit(raw_prediction)
|
| 914 |
+
|
| 915 |
+
For a given sample x_i, half Binomial deviance is defined as the negative
|
| 916 |
+
log-likelihood of the Binomial/Bernoulli distribution and can be expressed
|
| 917 |
+
as::
|
| 918 |
+
|
| 919 |
+
loss(x_i) = log(1 + exp(raw_pred_i)) - y_true_i * raw_pred_i
|
| 920 |
+
|
| 921 |
+
See The Elements of Statistical Learning, by Hastie, Tibshirani, Friedman,
|
| 922 |
+
section 4.4.1 (about logistic regression).
|
| 923 |
+
|
| 924 |
+
Note that the formulation works for classification, y = {0, 1}, as well as
|
| 925 |
+
logistic regression, y = [0, 1].
|
| 926 |
+
If you add `constant_to_optimal_zero` to the loss, you get half the
|
| 927 |
+
Bernoulli/binomial deviance.
|
| 928 |
+
|
| 929 |
+
More details: Inserting the predicted probability y_pred = expit(raw_prediction)
|
| 930 |
+
in the loss gives the well known::
|
| 931 |
+
|
| 932 |
+
loss(x_i) = - y_true_i * log(y_pred_i) - (1 - y_true_i) * log(1 - y_pred_i)
|
| 933 |
+
"""
|
| 934 |
+
|
| 935 |
+
def __init__(self, sample_weight=None):
|
| 936 |
+
super().__init__(
|
| 937 |
+
closs=CyHalfBinomialLoss(),
|
| 938 |
+
link=LogitLink(),
|
| 939 |
+
n_classes=2,
|
| 940 |
+
)
|
| 941 |
+
self.interval_y_true = Interval(0, 1, True, True)
|
| 942 |
+
|
| 943 |
+
def constant_to_optimal_zero(self, y_true, sample_weight=None):
|
| 944 |
+
# This is non-zero only if y_true is neither 0 nor 1.
|
| 945 |
+
term = xlogy(y_true, y_true) + xlogy(1 - y_true, 1 - y_true)
|
| 946 |
+
if sample_weight is not None:
|
| 947 |
+
term *= sample_weight
|
| 948 |
+
return term
|
| 949 |
+
|
| 950 |
+
def predict_proba(self, raw_prediction):
|
| 951 |
+
"""Predict probabilities.
|
| 952 |
+
|
| 953 |
+
Parameters
|
| 954 |
+
----------
|
| 955 |
+
raw_prediction : array of shape (n_samples,) or (n_samples, 1)
|
| 956 |
+
Raw prediction values (in link space).
|
| 957 |
+
|
| 958 |
+
Returns
|
| 959 |
+
-------
|
| 960 |
+
proba : array of shape (n_samples, 2)
|
| 961 |
+
Element-wise class probabilities.
|
| 962 |
+
"""
|
| 963 |
+
# Be graceful to shape (n_samples, 1) -> (n_samples,)
|
| 964 |
+
if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
|
| 965 |
+
raw_prediction = raw_prediction.squeeze(1)
|
| 966 |
+
proba = np.empty((raw_prediction.shape[0], 2), dtype=raw_prediction.dtype)
|
| 967 |
+
proba[:, 1] = self.link.inverse(raw_prediction)
|
| 968 |
+
proba[:, 0] = 1 - proba[:, 1]
|
| 969 |
+
return proba
|
| 970 |
+
|
| 971 |
+
|
| 972 |
+
class HalfMultinomialLoss(BaseLoss):
|
| 973 |
+
"""Categorical cross-entropy loss, for multiclass classification.
|
| 974 |
+
|
| 975 |
+
Domain:
|
| 976 |
+
y_true in {0, 1, 2, 3, .., n_classes - 1}
|
| 977 |
+
y_pred has n_classes elements, each element in (0, 1)
|
| 978 |
+
|
| 979 |
+
Link:
|
| 980 |
+
y_pred = softmax(raw_prediction)
|
| 981 |
+
|
| 982 |
+
Note: We assume y_true to be already label encoded. The inverse link is
|
| 983 |
+
softmax. But the full link function is the symmetric multinomial logit
|
| 984 |
+
function.
|
| 985 |
+
|
| 986 |
+
For a given sample x_i, the categorical cross-entropy loss is defined as
|
| 987 |
+
the negative log-likelihood of the multinomial distribution, it
|
| 988 |
+
generalizes the binary cross-entropy to more than 2 classes::
|
| 989 |
+
|
| 990 |
+
loss_i = log(sum(exp(raw_pred_{i, k}), k=0..n_classes-1))
|
| 991 |
+
- sum(y_true_{i, k} * raw_pred_{i, k}, k=0..n_classes-1)
|
| 992 |
+
|
| 993 |
+
See [1].
|
| 994 |
+
|
| 995 |
+
Note that for the hessian, we calculate only the diagonal part in the
|
| 996 |
+
classes: If the full hessian for classes k and l and sample i is H_i_k_l,
|
| 997 |
+
we calculate H_i_k_k, i.e. k=l.
|
| 998 |
+
|
| 999 |
+
Parameters
|
| 1000 |
+
----------
|
| 1001 |
+
sample_weight : {None, ndarray}
|
| 1002 |
+
If sample_weight is None, the hessian might be constant.
|
| 1003 |
+
|
| 1004 |
+
n_classes : {None, int}
|
| 1005 |
+
The number of classes for classification, else None.
|
| 1006 |
+
|
| 1007 |
+
References
|
| 1008 |
+
----------
|
| 1009 |
+
.. [1] :arxiv:`Simon, Noah, J. Friedman and T. Hastie.
|
| 1010 |
+
"A Blockwise Descent Algorithm for Group-penalized Multiresponse and
|
| 1011 |
+
Multinomial Regression".
|
| 1012 |
+
<1311.6529>`
|
| 1013 |
+
"""
|
| 1014 |
+
|
| 1015 |
+
is_multiclass = True
|
| 1016 |
+
|
| 1017 |
+
def __init__(self, sample_weight=None, n_classes=3):
|
| 1018 |
+
super().__init__(
|
| 1019 |
+
closs=CyHalfMultinomialLoss(),
|
| 1020 |
+
link=MultinomialLogit(),
|
| 1021 |
+
n_classes=n_classes,
|
| 1022 |
+
)
|
| 1023 |
+
self.interval_y_true = Interval(0, np.inf, True, False)
|
| 1024 |
+
self.interval_y_pred = Interval(0, 1, False, False)
|
| 1025 |
+
|
| 1026 |
+
def in_y_true_range(self, y):
|
| 1027 |
+
"""Return True if y is in the valid range of y_true.
|
| 1028 |
+
|
| 1029 |
+
Parameters
|
| 1030 |
+
----------
|
| 1031 |
+
y : ndarray
|
| 1032 |
+
"""
|
| 1033 |
+
return self.interval_y_true.includes(y) and np.all(y.astype(int) == y)
|
| 1034 |
+
|
| 1035 |
+
def fit_intercept_only(self, y_true, sample_weight=None):
|
| 1036 |
+
"""Compute raw_prediction of an intercept-only model.
|
| 1037 |
+
|
| 1038 |
+
This is the softmax of the weighted average of the target, i.e. over
|
| 1039 |
+
the samples axis=0.
|
| 1040 |
+
|
| 1041 |
+
Parameters
|
| 1042 |
+
----------
|
| 1043 |
+
y_true : array-like of shape (n_samples,)
|
| 1044 |
+
Observed, true target values.
|
| 1045 |
+
|
| 1046 |
+
sample_weight : None or array of shape (n_samples,), default=None
|
| 1047 |
+
Sample weights.
|
| 1048 |
+
|
| 1049 |
+
Returns
|
| 1050 |
+
-------
|
| 1051 |
+
raw_prediction : numpy scalar or array of shape (n_classes,)
|
| 1052 |
+
Raw predictions of an intercept-only model.
|
| 1053 |
+
"""
|
| 1054 |
+
out = np.zeros(self.n_classes, dtype=y_true.dtype)
|
| 1055 |
+
eps = np.finfo(y_true.dtype).eps
|
| 1056 |
+
for k in range(self.n_classes):
|
| 1057 |
+
out[k] = np.average(y_true == k, weights=sample_weight, axis=0)
|
| 1058 |
+
out[k] = np.clip(out[k], eps, 1 - eps)
|
| 1059 |
+
return self.link.link(out[None, :]).reshape(-1)
|
| 1060 |
+
|
| 1061 |
+
def predict_proba(self, raw_prediction):
|
| 1062 |
+
"""Predict probabilities.
|
| 1063 |
+
|
| 1064 |
+
Parameters
|
| 1065 |
+
----------
|
| 1066 |
+
raw_prediction : array of shape (n_samples, n_classes)
|
| 1067 |
+
Raw prediction values (in link space).
|
| 1068 |
+
|
| 1069 |
+
Returns
|
| 1070 |
+
-------
|
| 1071 |
+
proba : array of shape (n_samples, n_classes)
|
| 1072 |
+
Element-wise class probabilities.
|
| 1073 |
+
"""
|
| 1074 |
+
return self.link.inverse(raw_prediction)
|
| 1075 |
+
|
| 1076 |
+
def gradient_proba(
|
| 1077 |
+
self,
|
| 1078 |
+
y_true,
|
| 1079 |
+
raw_prediction,
|
| 1080 |
+
sample_weight=None,
|
| 1081 |
+
gradient_out=None,
|
| 1082 |
+
proba_out=None,
|
| 1083 |
+
n_threads=1,
|
| 1084 |
+
):
|
| 1085 |
+
"""Compute gradient and class probabilities fow raw_prediction.
|
| 1086 |
+
|
| 1087 |
+
Parameters
|
| 1088 |
+
----------
|
| 1089 |
+
y_true : C-contiguous array of shape (n_samples,)
|
| 1090 |
+
Observed, true target values.
|
| 1091 |
+
raw_prediction : array of shape (n_samples, n_classes)
|
| 1092 |
+
Raw prediction values (in link space).
|
| 1093 |
+
sample_weight : None or C-contiguous array of shape (n_samples,)
|
| 1094 |
+
Sample weights.
|
| 1095 |
+
gradient_out : None or array of shape (n_samples, n_classes)
|
| 1096 |
+
A location into which the gradient is stored. If None, a new array
|
| 1097 |
+
might be created.
|
| 1098 |
+
proba_out : None or array of shape (n_samples, n_classes)
|
| 1099 |
+
A location into which the class probabilities are stored. If None,
|
| 1100 |
+
a new array might be created.
|
| 1101 |
+
n_threads : int, default=1
|
| 1102 |
+
Might use openmp thread parallelism.
|
| 1103 |
+
|
| 1104 |
+
Returns
|
| 1105 |
+
-------
|
| 1106 |
+
gradient : array of shape (n_samples, n_classes)
|
| 1107 |
+
Element-wise gradients.
|
| 1108 |
+
|
| 1109 |
+
proba : array of shape (n_samples, n_classes)
|
| 1110 |
+
Element-wise class probabilities.
|
| 1111 |
+
"""
|
| 1112 |
+
if gradient_out is None:
|
| 1113 |
+
if proba_out is None:
|
| 1114 |
+
gradient_out = np.empty_like(raw_prediction)
|
| 1115 |
+
proba_out = np.empty_like(raw_prediction)
|
| 1116 |
+
else:
|
| 1117 |
+
gradient_out = np.empty_like(proba_out)
|
| 1118 |
+
elif proba_out is None:
|
| 1119 |
+
proba_out = np.empty_like(gradient_out)
|
| 1120 |
+
|
| 1121 |
+
self.closs.gradient_proba(
|
| 1122 |
+
y_true=y_true,
|
| 1123 |
+
raw_prediction=raw_prediction,
|
| 1124 |
+
sample_weight=sample_weight,
|
| 1125 |
+
gradient_out=gradient_out,
|
| 1126 |
+
proba_out=proba_out,
|
| 1127 |
+
n_threads=n_threads,
|
| 1128 |
+
)
|
| 1129 |
+
return gradient_out, proba_out
|
| 1130 |
+
|
| 1131 |
+
|
| 1132 |
+
class ExponentialLoss(BaseLoss):
|
| 1133 |
+
"""Exponential loss with (half) logit link, for binary classification.
|
| 1134 |
+
|
| 1135 |
+
This is also know as boosting loss.
|
| 1136 |
+
|
| 1137 |
+
Domain:
|
| 1138 |
+
y_true in [0, 1], i.e. regression on the unit interval
|
| 1139 |
+
y_pred in (0, 1), i.e. boundaries excluded
|
| 1140 |
+
|
| 1141 |
+
Link:
|
| 1142 |
+
y_pred = expit(2 * raw_prediction)
|
| 1143 |
+
|
| 1144 |
+
For a given sample x_i, the exponential loss is defined as::
|
| 1145 |
+
|
| 1146 |
+
loss(x_i) = y_true_i * exp(-raw_pred_i)) + (1 - y_true_i) * exp(raw_pred_i)
|
| 1147 |
+
|
| 1148 |
+
See:
|
| 1149 |
+
- J. Friedman, T. Hastie, R. Tibshirani.
|
| 1150 |
+
"Additive logistic regression: a statistical view of boosting (With discussion
|
| 1151 |
+
and a rejoinder by the authors)." Ann. Statist. 28 (2) 337 - 407, April 2000.
|
| 1152 |
+
https://doi.org/10.1214/aos/1016218223
|
| 1153 |
+
- A. Buja, W. Stuetzle, Y. Shen. (2005).
|
| 1154 |
+
"Loss Functions for Binary Class Probability Estimation and Classification:
|
| 1155 |
+
Structure and Applications."
|
| 1156 |
+
|
| 1157 |
+
Note that the formulation works for classification, y = {0, 1}, as well as
|
| 1158 |
+
"exponential logistic" regression, y = [0, 1].
|
| 1159 |
+
Note that this is a proper scoring rule, but without it's canonical link.
|
| 1160 |
+
|
| 1161 |
+
More details: Inserting the predicted probability
|
| 1162 |
+
y_pred = expit(2 * raw_prediction) in the loss gives::
|
| 1163 |
+
|
| 1164 |
+
loss(x_i) = y_true_i * sqrt((1 - y_pred_i) / y_pred_i)
|
| 1165 |
+
+ (1 - y_true_i) * sqrt(y_pred_i / (1 - y_pred_i))
|
| 1166 |
+
"""
|
| 1167 |
+
|
| 1168 |
+
def __init__(self, sample_weight=None):
|
| 1169 |
+
super().__init__(
|
| 1170 |
+
closs=CyExponentialLoss(),
|
| 1171 |
+
link=HalfLogitLink(),
|
| 1172 |
+
n_classes=2,
|
| 1173 |
+
)
|
| 1174 |
+
self.interval_y_true = Interval(0, 1, True, True)
|
| 1175 |
+
|
| 1176 |
+
def constant_to_optimal_zero(self, y_true, sample_weight=None):
|
| 1177 |
+
# This is non-zero only if y_true is neither 0 nor 1.
|
| 1178 |
+
term = -2 * np.sqrt(y_true * (1 - y_true))
|
| 1179 |
+
if sample_weight is not None:
|
| 1180 |
+
term *= sample_weight
|
| 1181 |
+
return term
|
| 1182 |
+
|
| 1183 |
+
def predict_proba(self, raw_prediction):
|
| 1184 |
+
"""Predict probabilities.
|
| 1185 |
+
|
| 1186 |
+
Parameters
|
| 1187 |
+
----------
|
| 1188 |
+
raw_prediction : array of shape (n_samples,) or (n_samples, 1)
|
| 1189 |
+
Raw prediction values (in link space).
|
| 1190 |
+
|
| 1191 |
+
Returns
|
| 1192 |
+
-------
|
| 1193 |
+
proba : array of shape (n_samples, 2)
|
| 1194 |
+
Element-wise class probabilities.
|
| 1195 |
+
"""
|
| 1196 |
+
# Be graceful to shape (n_samples, 1) -> (n_samples,)
|
| 1197 |
+
if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
|
| 1198 |
+
raw_prediction = raw_prediction.squeeze(1)
|
| 1199 |
+
proba = np.empty((raw_prediction.shape[0], 2), dtype=raw_prediction.dtype)
|
| 1200 |
+
proba[:, 1] = self.link.inverse(raw_prediction)
|
| 1201 |
+
proba[:, 0] = 1 - proba[:, 1]
|
| 1202 |
+
return proba
|
| 1203 |
+
|
| 1204 |
+
|
| 1205 |
+
_LOSSES = {
|
| 1206 |
+
"squared_error": HalfSquaredError,
|
| 1207 |
+
"absolute_error": AbsoluteError,
|
| 1208 |
+
"pinball_loss": PinballLoss,
|
| 1209 |
+
"huber_loss": HuberLoss,
|
| 1210 |
+
"poisson_loss": HalfPoissonLoss,
|
| 1211 |
+
"gamma_loss": HalfGammaLoss,
|
| 1212 |
+
"tweedie_loss": HalfTweedieLoss,
|
| 1213 |
+
"binomial_loss": HalfBinomialLoss,
|
| 1214 |
+
"multinomial_loss": HalfMultinomialLoss,
|
| 1215 |
+
"exponential_loss": ExponentialLoss,
|
| 1216 |
+
}
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/meson.build
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# .pyx is generated, so this is needed to make Cython compilation work
|
| 2 |
+
_loss_cython_tree = [
|
| 3 |
+
fs.copyfile('_loss.pxd')
|
| 4 |
+
]
|
| 5 |
+
|
| 6 |
+
_loss_pyx = custom_target(
|
| 7 |
+
'_loss_pyx',
|
| 8 |
+
output: '_loss.pyx',
|
| 9 |
+
input: '_loss.pyx.tp',
|
| 10 |
+
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
| 11 |
+
# TODO in principle this should go in py.exension_module below. This is
|
| 12 |
+
# temporary work-around for dependency issue with .pyx.tp files. For more
|
| 13 |
+
# details, see https://github.com/mesonbuild/meson/issues/13212
|
| 14 |
+
depends: _loss_cython_tree,
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
py.extension_module(
|
| 18 |
+
'_loss',
|
| 19 |
+
cython_gen.process(_loss_pyx),
|
| 20 |
+
dependencies: [openmp_dep],
|
| 21 |
+
install: true,
|
| 22 |
+
subdir: 'sklearn/_loss',
|
| 23 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_link.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import pytest
|
| 3 |
+
from numpy.testing import assert_allclose, assert_array_equal
|
| 4 |
+
|
| 5 |
+
from sklearn._loss.link import (
|
| 6 |
+
_LINKS,
|
| 7 |
+
HalfLogitLink,
|
| 8 |
+
Interval,
|
| 9 |
+
MultinomialLogit,
|
| 10 |
+
_inclusive_low_high,
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
LINK_FUNCTIONS = list(_LINKS.values())
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def test_interval_raises():
|
| 17 |
+
"""Test that interval with low > high raises ValueError."""
|
| 18 |
+
with pytest.raises(
|
| 19 |
+
ValueError, match="One must have low <= high; got low=1, high=0."
|
| 20 |
+
):
|
| 21 |
+
Interval(1, 0, False, False)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@pytest.mark.parametrize(
|
| 25 |
+
"interval",
|
| 26 |
+
[
|
| 27 |
+
Interval(0, 1, False, False),
|
| 28 |
+
Interval(0, 1, False, True),
|
| 29 |
+
Interval(0, 1, True, False),
|
| 30 |
+
Interval(0, 1, True, True),
|
| 31 |
+
Interval(-np.inf, np.inf, False, False),
|
| 32 |
+
Interval(-np.inf, np.inf, False, True),
|
| 33 |
+
Interval(-np.inf, np.inf, True, False),
|
| 34 |
+
Interval(-np.inf, np.inf, True, True),
|
| 35 |
+
Interval(-10, -1, False, False),
|
| 36 |
+
Interval(-10, -1, False, True),
|
| 37 |
+
Interval(-10, -1, True, False),
|
| 38 |
+
Interval(-10, -1, True, True),
|
| 39 |
+
],
|
| 40 |
+
)
|
| 41 |
+
def test_is_in_range(interval):
|
| 42 |
+
# make sure low and high are always within the interval, used for linspace
|
| 43 |
+
low, high = _inclusive_low_high(interval)
|
| 44 |
+
|
| 45 |
+
x = np.linspace(low, high, num=10)
|
| 46 |
+
assert interval.includes(x)
|
| 47 |
+
|
| 48 |
+
# x contains lower bound
|
| 49 |
+
assert interval.includes(np.r_[x, interval.low]) == interval.low_inclusive
|
| 50 |
+
|
| 51 |
+
# x contains upper bound
|
| 52 |
+
assert interval.includes(np.r_[x, interval.high]) == interval.high_inclusive
|
| 53 |
+
|
| 54 |
+
# x contains upper and lower bound
|
| 55 |
+
assert interval.includes(np.r_[x, interval.low, interval.high]) == (
|
| 56 |
+
interval.low_inclusive and interval.high_inclusive
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@pytest.mark.parametrize("link", LINK_FUNCTIONS)
|
| 61 |
+
def test_link_inverse_identity(link, global_random_seed):
|
| 62 |
+
# Test that link of inverse gives identity.
|
| 63 |
+
rng = np.random.RandomState(global_random_seed)
|
| 64 |
+
link = link()
|
| 65 |
+
n_samples, n_classes = 100, None
|
| 66 |
+
# The values for `raw_prediction` are limited from -20 to 20 because in the
|
| 67 |
+
# class `LogitLink` the term `expit(x)` comes very close to 1 for large
|
| 68 |
+
# positive x and therefore loses precision.
|
| 69 |
+
if link.is_multiclass:
|
| 70 |
+
n_classes = 10
|
| 71 |
+
raw_prediction = rng.uniform(low=-20, high=20, size=(n_samples, n_classes))
|
| 72 |
+
if isinstance(link, MultinomialLogit):
|
| 73 |
+
raw_prediction = link.symmetrize_raw_prediction(raw_prediction)
|
| 74 |
+
elif isinstance(link, HalfLogitLink):
|
| 75 |
+
raw_prediction = rng.uniform(low=-10, high=10, size=(n_samples))
|
| 76 |
+
else:
|
| 77 |
+
raw_prediction = rng.uniform(low=-20, high=20, size=(n_samples))
|
| 78 |
+
|
| 79 |
+
assert_allclose(link.link(link.inverse(raw_prediction)), raw_prediction)
|
| 80 |
+
y_pred = link.inverse(raw_prediction)
|
| 81 |
+
assert_allclose(link.inverse(link.link(y_pred)), y_pred)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@pytest.mark.parametrize("link", LINK_FUNCTIONS)
|
| 85 |
+
def test_link_out_argument(link):
|
| 86 |
+
# Test that out argument gets assigned the result.
|
| 87 |
+
rng = np.random.RandomState(42)
|
| 88 |
+
link = link()
|
| 89 |
+
n_samples, n_classes = 100, None
|
| 90 |
+
if link.is_multiclass:
|
| 91 |
+
n_classes = 10
|
| 92 |
+
raw_prediction = rng.normal(loc=0, scale=10, size=(n_samples, n_classes))
|
| 93 |
+
if isinstance(link, MultinomialLogit):
|
| 94 |
+
raw_prediction = link.symmetrize_raw_prediction(raw_prediction)
|
| 95 |
+
else:
|
| 96 |
+
# So far, the valid interval of raw_prediction is (-inf, inf) and
|
| 97 |
+
# we do not need to distinguish.
|
| 98 |
+
raw_prediction = rng.uniform(low=-10, high=10, size=(n_samples))
|
| 99 |
+
|
| 100 |
+
y_pred = link.inverse(raw_prediction, out=None)
|
| 101 |
+
out = np.empty_like(raw_prediction)
|
| 102 |
+
y_pred_2 = link.inverse(raw_prediction, out=out)
|
| 103 |
+
assert_allclose(y_pred, out)
|
| 104 |
+
assert_array_equal(out, y_pred_2)
|
| 105 |
+
assert np.shares_memory(out, y_pred_2)
|
| 106 |
+
|
| 107 |
+
out = np.empty_like(y_pred)
|
| 108 |
+
raw_prediction_2 = link.link(y_pred, out=out)
|
| 109 |
+
assert_allclose(raw_prediction, out)
|
| 110 |
+
assert_array_equal(out, raw_prediction_2)
|
| 111 |
+
assert np.shares_memory(out, raw_prediction_2)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_loss.py
ADDED
|
@@ -0,0 +1,1358 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pickle
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import pytest
|
| 5 |
+
from numpy.testing import assert_allclose, assert_array_equal
|
| 6 |
+
from pytest import approx
|
| 7 |
+
from scipy.optimize import (
|
| 8 |
+
LinearConstraint,
|
| 9 |
+
minimize,
|
| 10 |
+
minimize_scalar,
|
| 11 |
+
newton,
|
| 12 |
+
)
|
| 13 |
+
from scipy.special import logsumexp
|
| 14 |
+
|
| 15 |
+
from sklearn._loss.link import IdentityLink, _inclusive_low_high
|
| 16 |
+
from sklearn._loss.loss import (
|
| 17 |
+
_LOSSES,
|
| 18 |
+
AbsoluteError,
|
| 19 |
+
BaseLoss,
|
| 20 |
+
HalfBinomialLoss,
|
| 21 |
+
HalfGammaLoss,
|
| 22 |
+
HalfMultinomialLoss,
|
| 23 |
+
HalfPoissonLoss,
|
| 24 |
+
HalfSquaredError,
|
| 25 |
+
HalfTweedieLoss,
|
| 26 |
+
HalfTweedieLossIdentity,
|
| 27 |
+
HuberLoss,
|
| 28 |
+
PinballLoss,
|
| 29 |
+
)
|
| 30 |
+
from sklearn.utils import assert_all_finite
|
| 31 |
+
from sklearn.utils._testing import create_memmap_backed_data, skip_if_32bit
|
| 32 |
+
|
| 33 |
+
ALL_LOSSES = list(_LOSSES.values())
|
| 34 |
+
|
| 35 |
+
LOSS_INSTANCES = [loss() for loss in ALL_LOSSES]
|
| 36 |
+
# HalfTweedieLoss(power=1.5) is already there as default
|
| 37 |
+
LOSS_INSTANCES += [
|
| 38 |
+
PinballLoss(quantile=0.25),
|
| 39 |
+
HuberLoss(quantile=0.75),
|
| 40 |
+
HalfTweedieLoss(power=-1.5),
|
| 41 |
+
HalfTweedieLoss(power=0),
|
| 42 |
+
HalfTweedieLoss(power=1),
|
| 43 |
+
HalfTweedieLoss(power=2),
|
| 44 |
+
HalfTweedieLoss(power=3.0),
|
| 45 |
+
HalfTweedieLossIdentity(power=0),
|
| 46 |
+
HalfTweedieLossIdentity(power=1),
|
| 47 |
+
HalfTweedieLossIdentity(power=2),
|
| 48 |
+
HalfTweedieLossIdentity(power=3.0),
|
| 49 |
+
]
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def loss_instance_name(param):
|
| 53 |
+
if isinstance(param, BaseLoss):
|
| 54 |
+
loss = param
|
| 55 |
+
name = loss.__class__.__name__
|
| 56 |
+
if isinstance(loss, PinballLoss):
|
| 57 |
+
name += f"(quantile={loss.closs.quantile})"
|
| 58 |
+
elif isinstance(loss, HuberLoss):
|
| 59 |
+
name += f"(quantile={loss.quantile}"
|
| 60 |
+
elif hasattr(loss, "closs") and hasattr(loss.closs, "power"):
|
| 61 |
+
name += f"(power={loss.closs.power})"
|
| 62 |
+
return name
|
| 63 |
+
else:
|
| 64 |
+
return str(param)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def random_y_true_raw_prediction(
|
| 68 |
+
loss, n_samples, y_bound=(-100, 100), raw_bound=(-5, 5), seed=42
|
| 69 |
+
):
|
| 70 |
+
"""Random generate y_true and raw_prediction in valid range."""
|
| 71 |
+
rng = np.random.RandomState(seed)
|
| 72 |
+
if loss.is_multiclass:
|
| 73 |
+
raw_prediction = np.empty((n_samples, loss.n_classes))
|
| 74 |
+
raw_prediction.flat[:] = rng.uniform(
|
| 75 |
+
low=raw_bound[0],
|
| 76 |
+
high=raw_bound[1],
|
| 77 |
+
size=n_samples * loss.n_classes,
|
| 78 |
+
)
|
| 79 |
+
y_true = np.arange(n_samples).astype(float) % loss.n_classes
|
| 80 |
+
else:
|
| 81 |
+
# If link is identity, we must respect the interval of y_pred:
|
| 82 |
+
if isinstance(loss.link, IdentityLink):
|
| 83 |
+
low, high = _inclusive_low_high(loss.interval_y_pred)
|
| 84 |
+
low = np.amax([low, raw_bound[0]])
|
| 85 |
+
high = np.amin([high, raw_bound[1]])
|
| 86 |
+
raw_bound = (low, high)
|
| 87 |
+
raw_prediction = rng.uniform(
|
| 88 |
+
low=raw_bound[0], high=raw_bound[1], size=n_samples
|
| 89 |
+
)
|
| 90 |
+
# generate a y_true in valid range
|
| 91 |
+
low, high = _inclusive_low_high(loss.interval_y_true)
|
| 92 |
+
low = max(low, y_bound[0])
|
| 93 |
+
high = min(high, y_bound[1])
|
| 94 |
+
y_true = rng.uniform(low, high, size=n_samples)
|
| 95 |
+
# set some values at special boundaries
|
| 96 |
+
if loss.interval_y_true.low == 0 and loss.interval_y_true.low_inclusive:
|
| 97 |
+
y_true[:: (n_samples // 3)] = 0
|
| 98 |
+
if loss.interval_y_true.high == 1 and loss.interval_y_true.high_inclusive:
|
| 99 |
+
y_true[1 :: (n_samples // 3)] = 1
|
| 100 |
+
|
| 101 |
+
return y_true, raw_prediction
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def numerical_derivative(func, x, eps):
|
| 105 |
+
"""Helper function for numerical (first) derivatives."""
|
| 106 |
+
# For numerical derivatives, see
|
| 107 |
+
# https://en.wikipedia.org/wiki/Numerical_differentiation
|
| 108 |
+
# https://en.wikipedia.org/wiki/Finite_difference_coefficient
|
| 109 |
+
# We use central finite differences of accuracy 4.
|
| 110 |
+
h = np.full_like(x, fill_value=eps)
|
| 111 |
+
f_minus_2h = func(x - 2 * h)
|
| 112 |
+
f_minus_1h = func(x - h)
|
| 113 |
+
f_plus_1h = func(x + h)
|
| 114 |
+
f_plus_2h = func(x + 2 * h)
|
| 115 |
+
return (-f_plus_2h + 8 * f_plus_1h - 8 * f_minus_1h + f_minus_2h) / (12.0 * eps)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 119 |
+
def test_loss_boundary(loss):
|
| 120 |
+
"""Test interval ranges of y_true and y_pred in losses."""
|
| 121 |
+
# make sure low and high are always within the interval, used for linspace
|
| 122 |
+
if loss.is_multiclass:
|
| 123 |
+
n_classes = 3 # default value
|
| 124 |
+
y_true = np.tile(np.linspace(0, n_classes - 1, num=n_classes), 3)
|
| 125 |
+
else:
|
| 126 |
+
low, high = _inclusive_low_high(loss.interval_y_true)
|
| 127 |
+
y_true = np.linspace(low, high, num=10)
|
| 128 |
+
|
| 129 |
+
# add boundaries if they are included
|
| 130 |
+
if loss.interval_y_true.low_inclusive:
|
| 131 |
+
y_true = np.r_[y_true, loss.interval_y_true.low]
|
| 132 |
+
if loss.interval_y_true.high_inclusive:
|
| 133 |
+
y_true = np.r_[y_true, loss.interval_y_true.high]
|
| 134 |
+
|
| 135 |
+
assert loss.in_y_true_range(y_true)
|
| 136 |
+
|
| 137 |
+
n = y_true.shape[0]
|
| 138 |
+
low, high = _inclusive_low_high(loss.interval_y_pred)
|
| 139 |
+
if loss.is_multiclass:
|
| 140 |
+
y_pred = np.empty((n, n_classes))
|
| 141 |
+
y_pred[:, 0] = np.linspace(low, high, num=n)
|
| 142 |
+
y_pred[:, 1] = 0.5 * (1 - y_pred[:, 0])
|
| 143 |
+
y_pred[:, 2] = 0.5 * (1 - y_pred[:, 0])
|
| 144 |
+
else:
|
| 145 |
+
y_pred = np.linspace(low, high, num=n)
|
| 146 |
+
|
| 147 |
+
assert loss.in_y_pred_range(y_pred)
|
| 148 |
+
|
| 149 |
+
# calculating losses should not fail
|
| 150 |
+
raw_prediction = loss.link.link(y_pred)
|
| 151 |
+
loss.loss(y_true=y_true, raw_prediction=raw_prediction)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
# Fixture to test valid value ranges.
|
| 155 |
+
Y_COMMON_PARAMS = [
|
| 156 |
+
# (loss, [y success], [y fail])
|
| 157 |
+
(HalfSquaredError(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
|
| 158 |
+
(AbsoluteError(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
|
| 159 |
+
(PinballLoss(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
|
| 160 |
+
(HuberLoss(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
|
| 161 |
+
(HalfPoissonLoss(), [0.1, 100], [-np.inf, -3, -0.1, np.inf]),
|
| 162 |
+
(HalfGammaLoss(), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
|
| 163 |
+
(HalfTweedieLoss(power=-3), [0.1, 100], [-np.inf, np.inf]),
|
| 164 |
+
(HalfTweedieLoss(power=0), [0.1, 100], [-np.inf, np.inf]),
|
| 165 |
+
(HalfTweedieLoss(power=1.5), [0.1, 100], [-np.inf, -3, -0.1, np.inf]),
|
| 166 |
+
(HalfTweedieLoss(power=2), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
|
| 167 |
+
(HalfTweedieLoss(power=3), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
|
| 168 |
+
(HalfTweedieLossIdentity(power=-3), [0.1, 100], [-np.inf, np.inf]),
|
| 169 |
+
(HalfTweedieLossIdentity(power=0), [-3, -0.1, 0, 0.1, 100], [-np.inf, np.inf]),
|
| 170 |
+
(HalfTweedieLossIdentity(power=1.5), [0.1, 100], [-np.inf, -3, -0.1, np.inf]),
|
| 171 |
+
(HalfTweedieLossIdentity(power=2), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
|
| 172 |
+
(HalfTweedieLossIdentity(power=3), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
|
| 173 |
+
(HalfBinomialLoss(), [0.1, 0.5, 0.9], [-np.inf, -1, 2, np.inf]),
|
| 174 |
+
(HalfMultinomialLoss(), [], [-np.inf, -1, 1.1, np.inf]),
|
| 175 |
+
]
|
| 176 |
+
# y_pred and y_true do not always have the same domain (valid value range).
|
| 177 |
+
# Hence, we define extra sets of parameters for each of them.
|
| 178 |
+
Y_TRUE_PARAMS = [ # type: ignore[var-annotated]
|
| 179 |
+
# (loss, [y success], [y fail])
|
| 180 |
+
(HalfPoissonLoss(), [0], []),
|
| 181 |
+
(HuberLoss(), [0], []),
|
| 182 |
+
(HalfTweedieLoss(power=-3), [-100, -0.1, 0], []),
|
| 183 |
+
(HalfTweedieLoss(power=0), [-100, 0], []),
|
| 184 |
+
(HalfTweedieLoss(power=1.5), [0], []),
|
| 185 |
+
(HalfTweedieLossIdentity(power=-3), [-100, -0.1, 0], []),
|
| 186 |
+
(HalfTweedieLossIdentity(power=0), [-100, 0], []),
|
| 187 |
+
(HalfTweedieLossIdentity(power=1.5), [0], []),
|
| 188 |
+
(HalfBinomialLoss(), [0, 1], []),
|
| 189 |
+
(HalfMultinomialLoss(), [0.0, 1.0, 2], []),
|
| 190 |
+
]
|
| 191 |
+
Y_PRED_PARAMS = [
|
| 192 |
+
# (loss, [y success], [y fail])
|
| 193 |
+
(HalfPoissonLoss(), [], [0]),
|
| 194 |
+
(HalfTweedieLoss(power=-3), [], [-3, -0.1, 0]),
|
| 195 |
+
(HalfTweedieLoss(power=0), [], [-3, -0.1, 0]),
|
| 196 |
+
(HalfTweedieLoss(power=1.5), [], [0]),
|
| 197 |
+
(HalfTweedieLossIdentity(power=-3), [], [-3, -0.1, 0]),
|
| 198 |
+
(HalfTweedieLossIdentity(power=0), [-3, -0.1, 0], []),
|
| 199 |
+
(HalfTweedieLossIdentity(power=1.5), [], [0]),
|
| 200 |
+
(HalfBinomialLoss(), [], [0, 1]),
|
| 201 |
+
(HalfMultinomialLoss(), [0.1, 0.5], [0, 1]),
|
| 202 |
+
]
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
@pytest.mark.parametrize(
|
| 206 |
+
"loss, y_true_success, y_true_fail",
|
| 207 |
+
Y_COMMON_PARAMS + Y_TRUE_PARAMS, # type: ignore[operator]
|
| 208 |
+
)
|
| 209 |
+
def test_loss_boundary_y_true(loss, y_true_success, y_true_fail):
|
| 210 |
+
"""Test boundaries of y_true for loss functions."""
|
| 211 |
+
for y in y_true_success:
|
| 212 |
+
assert loss.in_y_true_range(np.array([y]))
|
| 213 |
+
for y in y_true_fail:
|
| 214 |
+
assert not loss.in_y_true_range(np.array([y]))
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
@pytest.mark.parametrize(
|
| 218 |
+
"loss, y_pred_success, y_pred_fail",
|
| 219 |
+
Y_COMMON_PARAMS + Y_PRED_PARAMS, # type: ignore[operator]
|
| 220 |
+
)
|
| 221 |
+
def test_loss_boundary_y_pred(loss, y_pred_success, y_pred_fail):
|
| 222 |
+
"""Test boundaries of y_pred for loss functions."""
|
| 223 |
+
for y in y_pred_success:
|
| 224 |
+
assert loss.in_y_pred_range(np.array([y]))
|
| 225 |
+
for y in y_pred_fail:
|
| 226 |
+
assert not loss.in_y_pred_range(np.array([y]))
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
@pytest.mark.parametrize(
|
| 230 |
+
"loss, y_true, raw_prediction, loss_true, gradient_true, hessian_true",
|
| 231 |
+
[
|
| 232 |
+
(HalfSquaredError(), 1.0, 5.0, 8, 4, 1),
|
| 233 |
+
(AbsoluteError(), 1.0, 5.0, 4.0, 1.0, None),
|
| 234 |
+
(PinballLoss(quantile=0.5), 1.0, 5.0, 2, 0.5, None),
|
| 235 |
+
(PinballLoss(quantile=0.25), 1.0, 5.0, 4 * (1 - 0.25), 1 - 0.25, None),
|
| 236 |
+
(PinballLoss(quantile=0.25), 5.0, 1.0, 4 * 0.25, -0.25, None),
|
| 237 |
+
(HuberLoss(quantile=0.5, delta=3), 1.0, 5.0, 3 * (4 - 3 / 2), None, None),
|
| 238 |
+
(HuberLoss(quantile=0.5, delta=3), 1.0, 3.0, 0.5 * 2**2, None, None),
|
| 239 |
+
(HalfPoissonLoss(), 2.0, np.log(4), 4 - 2 * np.log(4), 4 - 2, 4),
|
| 240 |
+
(HalfGammaLoss(), 2.0, np.log(4), np.log(4) + 2 / 4, 1 - 2 / 4, 2 / 4),
|
| 241 |
+
(HalfTweedieLoss(power=3), 2.0, np.log(4), -1 / 4 + 1 / 4**2, None, None),
|
| 242 |
+
(HalfTweedieLossIdentity(power=1), 2.0, 4.0, 2 - 2 * np.log(2), None, None),
|
| 243 |
+
(HalfTweedieLossIdentity(power=2), 2.0, 4.0, np.log(2) - 1 / 2, None, None),
|
| 244 |
+
(
|
| 245 |
+
HalfTweedieLossIdentity(power=3),
|
| 246 |
+
2.0,
|
| 247 |
+
4.0,
|
| 248 |
+
-1 / 4 + 1 / 4**2 + 1 / 2 / 2,
|
| 249 |
+
None,
|
| 250 |
+
None,
|
| 251 |
+
),
|
| 252 |
+
(
|
| 253 |
+
HalfBinomialLoss(),
|
| 254 |
+
0.25,
|
| 255 |
+
np.log(4),
|
| 256 |
+
np.log1p(4) - 0.25 * np.log(4),
|
| 257 |
+
None,
|
| 258 |
+
None,
|
| 259 |
+
),
|
| 260 |
+
# Extreme log loss cases, checked with mpmath:
|
| 261 |
+
# import mpmath as mp
|
| 262 |
+
#
|
| 263 |
+
# # Stolen from scipy
|
| 264 |
+
# def mpf2float(x):
|
| 265 |
+
# return float(mp.nstr(x, 17, min_fixed=0, max_fixed=0))
|
| 266 |
+
#
|
| 267 |
+
# def mp_logloss(y_true, raw):
|
| 268 |
+
# with mp.workdps(100):
|
| 269 |
+
# y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw))
|
| 270 |
+
# out = mp.log1p(mp.exp(raw)) - y_true * raw
|
| 271 |
+
# return mpf2float(out)
|
| 272 |
+
#
|
| 273 |
+
# def mp_gradient(y_true, raw):
|
| 274 |
+
# with mp.workdps(100):
|
| 275 |
+
# y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw))
|
| 276 |
+
# out = mp.mpf(1) / (mp.mpf(1) + mp.exp(-raw)) - y_true
|
| 277 |
+
# return mpf2float(out)
|
| 278 |
+
#
|
| 279 |
+
# def mp_hessian(y_true, raw):
|
| 280 |
+
# with mp.workdps(100):
|
| 281 |
+
# y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw))
|
| 282 |
+
# p = mp.mpf(1) / (mp.mpf(1) + mp.exp(-raw))
|
| 283 |
+
# out = p * (mp.mpf(1) - p)
|
| 284 |
+
# return mpf2float(out)
|
| 285 |
+
#
|
| 286 |
+
# y, raw = 0.0, 37.
|
| 287 |
+
# mp_logloss(y, raw), mp_gradient(y, raw), mp_hessian(y, raw)
|
| 288 |
+
(HalfBinomialLoss(), 0.0, -1e20, 0, 0, 0),
|
| 289 |
+
(HalfBinomialLoss(), 1.0, -1e20, 1e20, -1, 0),
|
| 290 |
+
(HalfBinomialLoss(), 0.0, -1e3, 0, 0, 0),
|
| 291 |
+
(HalfBinomialLoss(), 1.0, -1e3, 1e3, -1, 0),
|
| 292 |
+
(HalfBinomialLoss(), 1.0, -37.5, 37.5, -1, 0),
|
| 293 |
+
(HalfBinomialLoss(), 1.0, -37.0, 37, 1e-16 - 1, 8.533047625744065e-17),
|
| 294 |
+
(HalfBinomialLoss(), 0.0, -37.0, *[8.533047625744065e-17] * 3),
|
| 295 |
+
(HalfBinomialLoss(), 1.0, -36.9, 36.9, 1e-16 - 1, 9.430476078526806e-17),
|
| 296 |
+
(HalfBinomialLoss(), 0.0, -36.9, *[9.430476078526806e-17] * 3),
|
| 297 |
+
(HalfBinomialLoss(), 0.0, 37.0, 37, 1 - 1e-16, 8.533047625744065e-17),
|
| 298 |
+
(HalfBinomialLoss(), 1.0, 37.0, *[8.533047625744066e-17] * 3),
|
| 299 |
+
(HalfBinomialLoss(), 0.0, 37.5, 37.5, 1, 5.175555005801868e-17),
|
| 300 |
+
(HalfBinomialLoss(), 0.0, 232.8, 232.8, 1, 1.4287342391028437e-101),
|
| 301 |
+
(HalfBinomialLoss(), 1.0, 1e20, 0, 0, 0),
|
| 302 |
+
(HalfBinomialLoss(), 0.0, 1e20, 1e20, 1, 0),
|
| 303 |
+
(
|
| 304 |
+
HalfBinomialLoss(),
|
| 305 |
+
1.0,
|
| 306 |
+
232.8,
|
| 307 |
+
0,
|
| 308 |
+
-1.4287342391028437e-101,
|
| 309 |
+
1.4287342391028437e-101,
|
| 310 |
+
),
|
| 311 |
+
(HalfBinomialLoss(), 1.0, 232.9, 0, 0, 0),
|
| 312 |
+
(HalfBinomialLoss(), 1.0, 1e3, 0, 0, 0),
|
| 313 |
+
(HalfBinomialLoss(), 0.0, 1e3, 1e3, 1, 0),
|
| 314 |
+
(
|
| 315 |
+
HalfMultinomialLoss(n_classes=3),
|
| 316 |
+
0.0,
|
| 317 |
+
[0.2, 0.5, 0.3],
|
| 318 |
+
logsumexp([0.2, 0.5, 0.3]) - 0.2,
|
| 319 |
+
None,
|
| 320 |
+
None,
|
| 321 |
+
),
|
| 322 |
+
(
|
| 323 |
+
HalfMultinomialLoss(n_classes=3),
|
| 324 |
+
1.0,
|
| 325 |
+
[0.2, 0.5, 0.3],
|
| 326 |
+
logsumexp([0.2, 0.5, 0.3]) - 0.5,
|
| 327 |
+
None,
|
| 328 |
+
None,
|
| 329 |
+
),
|
| 330 |
+
(
|
| 331 |
+
HalfMultinomialLoss(n_classes=3),
|
| 332 |
+
2.0,
|
| 333 |
+
[0.2, 0.5, 0.3],
|
| 334 |
+
logsumexp([0.2, 0.5, 0.3]) - 0.3,
|
| 335 |
+
None,
|
| 336 |
+
None,
|
| 337 |
+
),
|
| 338 |
+
(
|
| 339 |
+
HalfMultinomialLoss(n_classes=3),
|
| 340 |
+
2.0,
|
| 341 |
+
[1e4, 0, 7e-7],
|
| 342 |
+
logsumexp([1e4, 0, 7e-7]) - (7e-7),
|
| 343 |
+
None,
|
| 344 |
+
None,
|
| 345 |
+
),
|
| 346 |
+
],
|
| 347 |
+
ids=loss_instance_name,
|
| 348 |
+
)
|
| 349 |
+
def test_loss_on_specific_values(
|
| 350 |
+
loss, y_true, raw_prediction, loss_true, gradient_true, hessian_true
|
| 351 |
+
):
|
| 352 |
+
"""Test losses, gradients and hessians at specific values."""
|
| 353 |
+
loss1 = loss(y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction]))
|
| 354 |
+
grad1 = loss.gradient(
|
| 355 |
+
y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction])
|
| 356 |
+
)
|
| 357 |
+
loss2, grad2 = loss.loss_gradient(
|
| 358 |
+
y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction])
|
| 359 |
+
)
|
| 360 |
+
grad3, hess = loss.gradient_hessian(
|
| 361 |
+
y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction])
|
| 362 |
+
)
|
| 363 |
+
|
| 364 |
+
assert loss1 == approx(loss_true, rel=1e-15, abs=1e-15)
|
| 365 |
+
assert loss2 == approx(loss_true, rel=1e-15, abs=1e-15)
|
| 366 |
+
|
| 367 |
+
if gradient_true is not None:
|
| 368 |
+
assert grad1 == approx(gradient_true, rel=1e-15, abs=1e-15)
|
| 369 |
+
assert grad2 == approx(gradient_true, rel=1e-15, abs=1e-15)
|
| 370 |
+
assert grad3 == approx(gradient_true, rel=1e-15, abs=1e-15)
|
| 371 |
+
|
| 372 |
+
if hessian_true is not None:
|
| 373 |
+
assert hess == approx(hessian_true, rel=1e-15, abs=1e-15)
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
@pytest.mark.parametrize("loss", ALL_LOSSES)
|
| 377 |
+
@pytest.mark.parametrize("readonly_memmap", [False, True])
|
| 378 |
+
@pytest.mark.parametrize("dtype_in", [np.float32, np.float64])
|
| 379 |
+
@pytest.mark.parametrize("dtype_out", [np.float32, np.float64])
|
| 380 |
+
@pytest.mark.parametrize("sample_weight", [None, 1])
|
| 381 |
+
@pytest.mark.parametrize("out1", [None, 1])
|
| 382 |
+
@pytest.mark.parametrize("out2", [None, 1])
|
| 383 |
+
@pytest.mark.parametrize("n_threads", [1, 2])
|
| 384 |
+
def test_loss_dtype(
|
| 385 |
+
loss, readonly_memmap, dtype_in, dtype_out, sample_weight, out1, out2, n_threads
|
| 386 |
+
):
|
| 387 |
+
"""Test acceptance of dtypes, readonly and writeable arrays in loss functions.
|
| 388 |
+
|
| 389 |
+
Check that loss accepts if all input arrays are either all float32 or all
|
| 390 |
+
float64, and all output arrays are either all float32 or all float64.
|
| 391 |
+
|
| 392 |
+
Also check that input arrays can be readonly, e.g. memory mapped.
|
| 393 |
+
"""
|
| 394 |
+
loss = loss()
|
| 395 |
+
# generate a y_true and raw_prediction in valid range
|
| 396 |
+
n_samples = 5
|
| 397 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 398 |
+
loss=loss,
|
| 399 |
+
n_samples=n_samples,
|
| 400 |
+
y_bound=(-100, 100),
|
| 401 |
+
raw_bound=(-10, 10),
|
| 402 |
+
seed=42,
|
| 403 |
+
)
|
| 404 |
+
y_true = y_true.astype(dtype_in)
|
| 405 |
+
raw_prediction = raw_prediction.astype(dtype_in)
|
| 406 |
+
|
| 407 |
+
if sample_weight is not None:
|
| 408 |
+
sample_weight = np.array([2.0] * n_samples, dtype=dtype_in)
|
| 409 |
+
if out1 is not None:
|
| 410 |
+
out1 = np.empty_like(y_true, dtype=dtype_out)
|
| 411 |
+
if out2 is not None:
|
| 412 |
+
out2 = np.empty_like(raw_prediction, dtype=dtype_out)
|
| 413 |
+
|
| 414 |
+
if readonly_memmap:
|
| 415 |
+
y_true = create_memmap_backed_data(y_true)
|
| 416 |
+
raw_prediction = create_memmap_backed_data(raw_prediction)
|
| 417 |
+
if sample_weight is not None:
|
| 418 |
+
sample_weight = create_memmap_backed_data(sample_weight)
|
| 419 |
+
|
| 420 |
+
l = loss.loss(
|
| 421 |
+
y_true=y_true,
|
| 422 |
+
raw_prediction=raw_prediction,
|
| 423 |
+
sample_weight=sample_weight,
|
| 424 |
+
loss_out=out1,
|
| 425 |
+
n_threads=n_threads,
|
| 426 |
+
)
|
| 427 |
+
assert l is out1 if out1 is not None else True
|
| 428 |
+
g = loss.gradient(
|
| 429 |
+
y_true=y_true,
|
| 430 |
+
raw_prediction=raw_prediction,
|
| 431 |
+
sample_weight=sample_weight,
|
| 432 |
+
gradient_out=out2,
|
| 433 |
+
n_threads=n_threads,
|
| 434 |
+
)
|
| 435 |
+
assert g is out2 if out2 is not None else True
|
| 436 |
+
l, g = loss.loss_gradient(
|
| 437 |
+
y_true=y_true,
|
| 438 |
+
raw_prediction=raw_prediction,
|
| 439 |
+
sample_weight=sample_weight,
|
| 440 |
+
loss_out=out1,
|
| 441 |
+
gradient_out=out2,
|
| 442 |
+
n_threads=n_threads,
|
| 443 |
+
)
|
| 444 |
+
assert l is out1 if out1 is not None else True
|
| 445 |
+
assert g is out2 if out2 is not None else True
|
| 446 |
+
if out1 is not None and loss.is_multiclass:
|
| 447 |
+
out1 = np.empty_like(raw_prediction, dtype=dtype_out)
|
| 448 |
+
g, h = loss.gradient_hessian(
|
| 449 |
+
y_true=y_true,
|
| 450 |
+
raw_prediction=raw_prediction,
|
| 451 |
+
sample_weight=sample_weight,
|
| 452 |
+
gradient_out=out1,
|
| 453 |
+
hessian_out=out2,
|
| 454 |
+
n_threads=n_threads,
|
| 455 |
+
)
|
| 456 |
+
assert g is out1 if out1 is not None else True
|
| 457 |
+
assert h is out2 if out2 is not None else True
|
| 458 |
+
loss(y_true=y_true, raw_prediction=raw_prediction, sample_weight=sample_weight)
|
| 459 |
+
loss.fit_intercept_only(y_true=y_true, sample_weight=sample_weight)
|
| 460 |
+
loss.constant_to_optimal_zero(y_true=y_true, sample_weight=sample_weight)
|
| 461 |
+
if hasattr(loss, "predict_proba"):
|
| 462 |
+
loss.predict_proba(raw_prediction=raw_prediction)
|
| 463 |
+
if hasattr(loss, "gradient_proba"):
|
| 464 |
+
g, p = loss.gradient_proba(
|
| 465 |
+
y_true=y_true,
|
| 466 |
+
raw_prediction=raw_prediction,
|
| 467 |
+
sample_weight=sample_weight,
|
| 468 |
+
gradient_out=out1,
|
| 469 |
+
proba_out=out2,
|
| 470 |
+
n_threads=n_threads,
|
| 471 |
+
)
|
| 472 |
+
assert g is out1 if out1 is not None else True
|
| 473 |
+
assert p is out2 if out2 is not None else True
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 477 |
+
@pytest.mark.parametrize("sample_weight", [None, "range"])
|
| 478 |
+
def test_loss_same_as_C_functions(loss, sample_weight):
|
| 479 |
+
"""Test that Python and Cython functions return same results."""
|
| 480 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 481 |
+
loss=loss,
|
| 482 |
+
n_samples=20,
|
| 483 |
+
y_bound=(-100, 100),
|
| 484 |
+
raw_bound=(-10, 10),
|
| 485 |
+
seed=42,
|
| 486 |
+
)
|
| 487 |
+
if sample_weight == "range":
|
| 488 |
+
sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
|
| 489 |
+
|
| 490 |
+
out_l1 = np.empty_like(y_true)
|
| 491 |
+
out_l2 = np.empty_like(y_true)
|
| 492 |
+
out_g1 = np.empty_like(raw_prediction)
|
| 493 |
+
out_g2 = np.empty_like(raw_prediction)
|
| 494 |
+
out_h1 = np.empty_like(raw_prediction)
|
| 495 |
+
out_h2 = np.empty_like(raw_prediction)
|
| 496 |
+
loss.loss(
|
| 497 |
+
y_true=y_true,
|
| 498 |
+
raw_prediction=raw_prediction,
|
| 499 |
+
sample_weight=sample_weight,
|
| 500 |
+
loss_out=out_l1,
|
| 501 |
+
)
|
| 502 |
+
loss.closs.loss(
|
| 503 |
+
y_true=y_true,
|
| 504 |
+
raw_prediction=raw_prediction,
|
| 505 |
+
sample_weight=sample_weight,
|
| 506 |
+
loss_out=out_l2,
|
| 507 |
+
)
|
| 508 |
+
assert_allclose(out_l1, out_l2)
|
| 509 |
+
loss.gradient(
|
| 510 |
+
y_true=y_true,
|
| 511 |
+
raw_prediction=raw_prediction,
|
| 512 |
+
sample_weight=sample_weight,
|
| 513 |
+
gradient_out=out_g1,
|
| 514 |
+
)
|
| 515 |
+
loss.closs.gradient(
|
| 516 |
+
y_true=y_true,
|
| 517 |
+
raw_prediction=raw_prediction,
|
| 518 |
+
sample_weight=sample_weight,
|
| 519 |
+
gradient_out=out_g2,
|
| 520 |
+
)
|
| 521 |
+
assert_allclose(out_g1, out_g2)
|
| 522 |
+
loss.closs.loss_gradient(
|
| 523 |
+
y_true=y_true,
|
| 524 |
+
raw_prediction=raw_prediction,
|
| 525 |
+
sample_weight=sample_weight,
|
| 526 |
+
loss_out=out_l1,
|
| 527 |
+
gradient_out=out_g1,
|
| 528 |
+
)
|
| 529 |
+
loss.closs.loss_gradient(
|
| 530 |
+
y_true=y_true,
|
| 531 |
+
raw_prediction=raw_prediction,
|
| 532 |
+
sample_weight=sample_weight,
|
| 533 |
+
loss_out=out_l2,
|
| 534 |
+
gradient_out=out_g2,
|
| 535 |
+
)
|
| 536 |
+
assert_allclose(out_l1, out_l2)
|
| 537 |
+
assert_allclose(out_g1, out_g2)
|
| 538 |
+
loss.gradient_hessian(
|
| 539 |
+
y_true=y_true,
|
| 540 |
+
raw_prediction=raw_prediction,
|
| 541 |
+
sample_weight=sample_weight,
|
| 542 |
+
gradient_out=out_g1,
|
| 543 |
+
hessian_out=out_h1,
|
| 544 |
+
)
|
| 545 |
+
loss.closs.gradient_hessian(
|
| 546 |
+
y_true=y_true,
|
| 547 |
+
raw_prediction=raw_prediction,
|
| 548 |
+
sample_weight=sample_weight,
|
| 549 |
+
gradient_out=out_g2,
|
| 550 |
+
hessian_out=out_h2,
|
| 551 |
+
)
|
| 552 |
+
assert_allclose(out_g1, out_g2)
|
| 553 |
+
assert_allclose(out_h1, out_h2)
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 557 |
+
@pytest.mark.parametrize("sample_weight", [None, "range"])
|
| 558 |
+
def test_loss_gradients_are_the_same(loss, sample_weight, global_random_seed):
|
| 559 |
+
"""Test that loss and gradient are the same across different functions.
|
| 560 |
+
|
| 561 |
+
Also test that output arguments contain correct results.
|
| 562 |
+
"""
|
| 563 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 564 |
+
loss=loss,
|
| 565 |
+
n_samples=20,
|
| 566 |
+
y_bound=(-100, 100),
|
| 567 |
+
raw_bound=(-10, 10),
|
| 568 |
+
seed=global_random_seed,
|
| 569 |
+
)
|
| 570 |
+
if sample_weight == "range":
|
| 571 |
+
sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
|
| 572 |
+
|
| 573 |
+
out_l1 = np.empty_like(y_true)
|
| 574 |
+
out_l2 = np.empty_like(y_true)
|
| 575 |
+
out_g1 = np.empty_like(raw_prediction)
|
| 576 |
+
out_g2 = np.empty_like(raw_prediction)
|
| 577 |
+
out_g3 = np.empty_like(raw_prediction)
|
| 578 |
+
out_h3 = np.empty_like(raw_prediction)
|
| 579 |
+
|
| 580 |
+
l1 = loss.loss(
|
| 581 |
+
y_true=y_true,
|
| 582 |
+
raw_prediction=raw_prediction,
|
| 583 |
+
sample_weight=sample_weight,
|
| 584 |
+
loss_out=out_l1,
|
| 585 |
+
)
|
| 586 |
+
g1 = loss.gradient(
|
| 587 |
+
y_true=y_true,
|
| 588 |
+
raw_prediction=raw_prediction,
|
| 589 |
+
sample_weight=sample_weight,
|
| 590 |
+
gradient_out=out_g1,
|
| 591 |
+
)
|
| 592 |
+
l2, g2 = loss.loss_gradient(
|
| 593 |
+
y_true=y_true,
|
| 594 |
+
raw_prediction=raw_prediction,
|
| 595 |
+
sample_weight=sample_weight,
|
| 596 |
+
loss_out=out_l2,
|
| 597 |
+
gradient_out=out_g2,
|
| 598 |
+
)
|
| 599 |
+
g3, h3 = loss.gradient_hessian(
|
| 600 |
+
y_true=y_true,
|
| 601 |
+
raw_prediction=raw_prediction,
|
| 602 |
+
sample_weight=sample_weight,
|
| 603 |
+
gradient_out=out_g3,
|
| 604 |
+
hessian_out=out_h3,
|
| 605 |
+
)
|
| 606 |
+
assert_allclose(l1, l2)
|
| 607 |
+
assert_array_equal(l1, out_l1)
|
| 608 |
+
assert np.shares_memory(l1, out_l1)
|
| 609 |
+
assert_array_equal(l2, out_l2)
|
| 610 |
+
assert np.shares_memory(l2, out_l2)
|
| 611 |
+
assert_allclose(g1, g2)
|
| 612 |
+
assert_allclose(g1, g3)
|
| 613 |
+
assert_array_equal(g1, out_g1)
|
| 614 |
+
assert np.shares_memory(g1, out_g1)
|
| 615 |
+
assert_array_equal(g2, out_g2)
|
| 616 |
+
assert np.shares_memory(g2, out_g2)
|
| 617 |
+
assert_array_equal(g3, out_g3)
|
| 618 |
+
assert np.shares_memory(g3, out_g3)
|
| 619 |
+
|
| 620 |
+
if hasattr(loss, "gradient_proba"):
|
| 621 |
+
assert loss.is_multiclass # only for HalfMultinomialLoss
|
| 622 |
+
out_g4 = np.empty_like(raw_prediction)
|
| 623 |
+
out_proba = np.empty_like(raw_prediction)
|
| 624 |
+
g4, proba = loss.gradient_proba(
|
| 625 |
+
y_true=y_true,
|
| 626 |
+
raw_prediction=raw_prediction,
|
| 627 |
+
sample_weight=sample_weight,
|
| 628 |
+
gradient_out=out_g4,
|
| 629 |
+
proba_out=out_proba,
|
| 630 |
+
)
|
| 631 |
+
assert_allclose(g1, out_g4)
|
| 632 |
+
assert_allclose(g1, g4)
|
| 633 |
+
assert_allclose(proba, out_proba)
|
| 634 |
+
assert_allclose(np.sum(proba, axis=1), 1, rtol=1e-11)
|
| 635 |
+
|
| 636 |
+
|
| 637 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 638 |
+
@pytest.mark.parametrize("sample_weight", ["ones", "random"])
|
| 639 |
+
def test_sample_weight_multiplies(loss, sample_weight, global_random_seed):
|
| 640 |
+
"""Test sample weights in loss, gradients and hessians.
|
| 641 |
+
|
| 642 |
+
Make sure that passing sample weights to loss, gradient and hessian
|
| 643 |
+
computation methods is equivalent to multiplying by the weights.
|
| 644 |
+
"""
|
| 645 |
+
n_samples = 100
|
| 646 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 647 |
+
loss=loss,
|
| 648 |
+
n_samples=n_samples,
|
| 649 |
+
y_bound=(-100, 100),
|
| 650 |
+
raw_bound=(-5, 5),
|
| 651 |
+
seed=global_random_seed,
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
if sample_weight == "ones":
|
| 655 |
+
sample_weight = np.ones(shape=n_samples, dtype=np.float64)
|
| 656 |
+
else:
|
| 657 |
+
rng = np.random.RandomState(global_random_seed)
|
| 658 |
+
sample_weight = rng.normal(size=n_samples).astype(np.float64)
|
| 659 |
+
|
| 660 |
+
assert_allclose(
|
| 661 |
+
loss.loss(
|
| 662 |
+
y_true=y_true,
|
| 663 |
+
raw_prediction=raw_prediction,
|
| 664 |
+
sample_weight=sample_weight,
|
| 665 |
+
),
|
| 666 |
+
sample_weight
|
| 667 |
+
* loss.loss(
|
| 668 |
+
y_true=y_true,
|
| 669 |
+
raw_prediction=raw_prediction,
|
| 670 |
+
sample_weight=None,
|
| 671 |
+
),
|
| 672 |
+
)
|
| 673 |
+
|
| 674 |
+
losses, gradient = loss.loss_gradient(
|
| 675 |
+
y_true=y_true,
|
| 676 |
+
raw_prediction=raw_prediction,
|
| 677 |
+
sample_weight=None,
|
| 678 |
+
)
|
| 679 |
+
losses_sw, gradient_sw = loss.loss_gradient(
|
| 680 |
+
y_true=y_true,
|
| 681 |
+
raw_prediction=raw_prediction,
|
| 682 |
+
sample_weight=sample_weight,
|
| 683 |
+
)
|
| 684 |
+
assert_allclose(losses * sample_weight, losses_sw)
|
| 685 |
+
if not loss.is_multiclass:
|
| 686 |
+
assert_allclose(gradient * sample_weight, gradient_sw)
|
| 687 |
+
else:
|
| 688 |
+
assert_allclose(gradient * sample_weight[:, None], gradient_sw)
|
| 689 |
+
|
| 690 |
+
gradient, hessian = loss.gradient_hessian(
|
| 691 |
+
y_true=y_true,
|
| 692 |
+
raw_prediction=raw_prediction,
|
| 693 |
+
sample_weight=None,
|
| 694 |
+
)
|
| 695 |
+
gradient_sw, hessian_sw = loss.gradient_hessian(
|
| 696 |
+
y_true=y_true,
|
| 697 |
+
raw_prediction=raw_prediction,
|
| 698 |
+
sample_weight=sample_weight,
|
| 699 |
+
)
|
| 700 |
+
if not loss.is_multiclass:
|
| 701 |
+
assert_allclose(gradient * sample_weight, gradient_sw)
|
| 702 |
+
assert_allclose(hessian * sample_weight, hessian_sw)
|
| 703 |
+
else:
|
| 704 |
+
assert_allclose(gradient * sample_weight[:, None], gradient_sw)
|
| 705 |
+
assert_allclose(hessian * sample_weight[:, None], hessian_sw)
|
| 706 |
+
|
| 707 |
+
|
| 708 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 709 |
+
def test_graceful_squeezing(loss):
|
| 710 |
+
"""Test that reshaped raw_prediction gives same results."""
|
| 711 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 712 |
+
loss=loss,
|
| 713 |
+
n_samples=20,
|
| 714 |
+
y_bound=(-100, 100),
|
| 715 |
+
raw_bound=(-10, 10),
|
| 716 |
+
seed=42,
|
| 717 |
+
)
|
| 718 |
+
|
| 719 |
+
if raw_prediction.ndim == 1:
|
| 720 |
+
raw_prediction_2d = raw_prediction[:, None]
|
| 721 |
+
assert_allclose(
|
| 722 |
+
loss.loss(y_true=y_true, raw_prediction=raw_prediction_2d),
|
| 723 |
+
loss.loss(y_true=y_true, raw_prediction=raw_prediction),
|
| 724 |
+
)
|
| 725 |
+
assert_allclose(
|
| 726 |
+
loss.loss_gradient(y_true=y_true, raw_prediction=raw_prediction_2d),
|
| 727 |
+
loss.loss_gradient(y_true=y_true, raw_prediction=raw_prediction),
|
| 728 |
+
)
|
| 729 |
+
assert_allclose(
|
| 730 |
+
loss.gradient(y_true=y_true, raw_prediction=raw_prediction_2d),
|
| 731 |
+
loss.gradient(y_true=y_true, raw_prediction=raw_prediction),
|
| 732 |
+
)
|
| 733 |
+
assert_allclose(
|
| 734 |
+
loss.gradient_hessian(y_true=y_true, raw_prediction=raw_prediction_2d),
|
| 735 |
+
loss.gradient_hessian(y_true=y_true, raw_prediction=raw_prediction),
|
| 736 |
+
)
|
| 737 |
+
|
| 738 |
+
|
| 739 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 740 |
+
@pytest.mark.parametrize("sample_weight", [None, "range"])
|
| 741 |
+
def test_loss_of_perfect_prediction(loss, sample_weight):
|
| 742 |
+
"""Test value of perfect predictions.
|
| 743 |
+
|
| 744 |
+
Loss of y_pred = y_true plus constant_to_optimal_zero should sums up to
|
| 745 |
+
zero.
|
| 746 |
+
"""
|
| 747 |
+
if not loss.is_multiclass:
|
| 748 |
+
# Use small values such that exp(value) is not nan.
|
| 749 |
+
raw_prediction = np.array([-10, -0.1, 0, 0.1, 3, 10])
|
| 750 |
+
# If link is identity, we must respect the interval of y_pred:
|
| 751 |
+
if isinstance(loss.link, IdentityLink):
|
| 752 |
+
eps = 1e-10
|
| 753 |
+
low = loss.interval_y_pred.low
|
| 754 |
+
if not loss.interval_y_pred.low_inclusive:
|
| 755 |
+
low = low + eps
|
| 756 |
+
high = loss.interval_y_pred.high
|
| 757 |
+
if not loss.interval_y_pred.high_inclusive:
|
| 758 |
+
high = high - eps
|
| 759 |
+
raw_prediction = np.clip(raw_prediction, low, high)
|
| 760 |
+
y_true = loss.link.inverse(raw_prediction)
|
| 761 |
+
else:
|
| 762 |
+
# HalfMultinomialLoss
|
| 763 |
+
y_true = np.arange(loss.n_classes).astype(float)
|
| 764 |
+
# raw_prediction with entries -exp(10), but +exp(10) on the diagonal
|
| 765 |
+
# this is close enough to np.inf which would produce nan
|
| 766 |
+
raw_prediction = np.full(
|
| 767 |
+
shape=(loss.n_classes, loss.n_classes),
|
| 768 |
+
fill_value=-np.exp(10),
|
| 769 |
+
dtype=float,
|
| 770 |
+
)
|
| 771 |
+
raw_prediction.flat[:: loss.n_classes + 1] = np.exp(10)
|
| 772 |
+
|
| 773 |
+
if sample_weight == "range":
|
| 774 |
+
sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
|
| 775 |
+
|
| 776 |
+
loss_value = loss.loss(
|
| 777 |
+
y_true=y_true,
|
| 778 |
+
raw_prediction=raw_prediction,
|
| 779 |
+
sample_weight=sample_weight,
|
| 780 |
+
)
|
| 781 |
+
constant_term = loss.constant_to_optimal_zero(
|
| 782 |
+
y_true=y_true, sample_weight=sample_weight
|
| 783 |
+
)
|
| 784 |
+
# Comparing loss_value + constant_term to zero would result in large
|
| 785 |
+
# round-off errors.
|
| 786 |
+
assert_allclose(loss_value, -constant_term, atol=1e-14, rtol=1e-15)
|
| 787 |
+
|
| 788 |
+
|
| 789 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 790 |
+
@pytest.mark.parametrize("sample_weight", [None, "range"])
|
| 791 |
+
def test_gradients_hessians_numerically(loss, sample_weight, global_random_seed):
|
| 792 |
+
"""Test gradients and hessians with numerical derivatives.
|
| 793 |
+
|
| 794 |
+
Gradient should equal the numerical derivatives of the loss function.
|
| 795 |
+
Hessians should equal the numerical derivatives of gradients.
|
| 796 |
+
"""
|
| 797 |
+
n_samples = 20
|
| 798 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 799 |
+
loss=loss,
|
| 800 |
+
n_samples=n_samples,
|
| 801 |
+
y_bound=(-100, 100),
|
| 802 |
+
raw_bound=(-5, 5),
|
| 803 |
+
seed=global_random_seed,
|
| 804 |
+
)
|
| 805 |
+
|
| 806 |
+
if sample_weight == "range":
|
| 807 |
+
sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
|
| 808 |
+
|
| 809 |
+
g, h = loss.gradient_hessian(
|
| 810 |
+
y_true=y_true,
|
| 811 |
+
raw_prediction=raw_prediction,
|
| 812 |
+
sample_weight=sample_weight,
|
| 813 |
+
)
|
| 814 |
+
|
| 815 |
+
assert g.shape == raw_prediction.shape
|
| 816 |
+
assert h.shape == raw_prediction.shape
|
| 817 |
+
|
| 818 |
+
if not loss.is_multiclass:
|
| 819 |
+
|
| 820 |
+
def loss_func(x):
|
| 821 |
+
return loss.loss(
|
| 822 |
+
y_true=y_true,
|
| 823 |
+
raw_prediction=x,
|
| 824 |
+
sample_weight=sample_weight,
|
| 825 |
+
)
|
| 826 |
+
|
| 827 |
+
g_numeric = numerical_derivative(loss_func, raw_prediction, eps=1e-6)
|
| 828 |
+
assert_allclose(g, g_numeric, rtol=5e-6, atol=1e-10)
|
| 829 |
+
|
| 830 |
+
def grad_func(x):
|
| 831 |
+
return loss.gradient(
|
| 832 |
+
y_true=y_true,
|
| 833 |
+
raw_prediction=x,
|
| 834 |
+
sample_weight=sample_weight,
|
| 835 |
+
)
|
| 836 |
+
|
| 837 |
+
h_numeric = numerical_derivative(grad_func, raw_prediction, eps=1e-6)
|
| 838 |
+
if loss.approx_hessian:
|
| 839 |
+
# TODO: What could we test if loss.approx_hessian?
|
| 840 |
+
pass
|
| 841 |
+
else:
|
| 842 |
+
assert_allclose(h, h_numeric, rtol=5e-6, atol=1e-10)
|
| 843 |
+
else:
|
| 844 |
+
# For multiclass loss, we should only change the predictions of the
|
| 845 |
+
# class for which the derivative is taken for, e.g. offset[:, k] = eps
|
| 846 |
+
# for class k.
|
| 847 |
+
# As a softmax is computed, offsetting the whole array by a constant
|
| 848 |
+
# would have no effect on the probabilities, and thus on the loss.
|
| 849 |
+
for k in range(loss.n_classes):
|
| 850 |
+
|
| 851 |
+
def loss_func(x):
|
| 852 |
+
raw = raw_prediction.copy()
|
| 853 |
+
raw[:, k] = x
|
| 854 |
+
return loss.loss(
|
| 855 |
+
y_true=y_true,
|
| 856 |
+
raw_prediction=raw,
|
| 857 |
+
sample_weight=sample_weight,
|
| 858 |
+
)
|
| 859 |
+
|
| 860 |
+
g_numeric = numerical_derivative(loss_func, raw_prediction[:, k], eps=1e-5)
|
| 861 |
+
assert_allclose(g[:, k], g_numeric, rtol=5e-6, atol=1e-10)
|
| 862 |
+
|
| 863 |
+
def grad_func(x):
|
| 864 |
+
raw = raw_prediction.copy()
|
| 865 |
+
raw[:, k] = x
|
| 866 |
+
return loss.gradient(
|
| 867 |
+
y_true=y_true,
|
| 868 |
+
raw_prediction=raw,
|
| 869 |
+
sample_weight=sample_weight,
|
| 870 |
+
)[:, k]
|
| 871 |
+
|
| 872 |
+
h_numeric = numerical_derivative(grad_func, raw_prediction[:, k], eps=1e-6)
|
| 873 |
+
if loss.approx_hessian:
|
| 874 |
+
# TODO: What could we test if loss.approx_hessian?
|
| 875 |
+
pass
|
| 876 |
+
else:
|
| 877 |
+
assert_allclose(h[:, k], h_numeric, rtol=5e-6, atol=1e-10)
|
| 878 |
+
|
| 879 |
+
|
| 880 |
+
@pytest.mark.parametrize(
|
| 881 |
+
"loss, x0, y_true",
|
| 882 |
+
[
|
| 883 |
+
("squared_error", -2.0, 42),
|
| 884 |
+
("squared_error", 117.0, 1.05),
|
| 885 |
+
("squared_error", 0.0, 0.0),
|
| 886 |
+
# The argmin of binomial_loss for y_true=0 and y_true=1 is resp.
|
| 887 |
+
# -inf and +inf due to logit, cf. "complete separation". Therefore, we
|
| 888 |
+
# use 0 < y_true < 1.
|
| 889 |
+
("binomial_loss", 0.3, 0.1),
|
| 890 |
+
("binomial_loss", -12, 0.2),
|
| 891 |
+
("binomial_loss", 30, 0.9),
|
| 892 |
+
("poisson_loss", 12.0, 1.0),
|
| 893 |
+
("poisson_loss", 0.0, 2.0),
|
| 894 |
+
("poisson_loss", -22.0, 10.0),
|
| 895 |
+
],
|
| 896 |
+
)
|
| 897 |
+
@skip_if_32bit
|
| 898 |
+
def test_derivatives(loss, x0, y_true):
|
| 899 |
+
"""Test that gradients are zero at the minimum of the loss.
|
| 900 |
+
|
| 901 |
+
We check this on a single value/sample using Halley's method with the
|
| 902 |
+
first and second order derivatives computed by the Loss instance.
|
| 903 |
+
Note that methods of Loss instances operate on arrays while the newton
|
| 904 |
+
root finder expects a scalar or a one-element array for this purpose.
|
| 905 |
+
"""
|
| 906 |
+
loss = _LOSSES[loss](sample_weight=None)
|
| 907 |
+
y_true = np.array([y_true], dtype=np.float64)
|
| 908 |
+
x0 = np.array([x0], dtype=np.float64)
|
| 909 |
+
|
| 910 |
+
def func(x: np.ndarray) -> np.ndarray:
|
| 911 |
+
"""Compute loss plus constant term.
|
| 912 |
+
|
| 913 |
+
The constant term is such that the minimum function value is zero,
|
| 914 |
+
which is required by the Newton method.
|
| 915 |
+
"""
|
| 916 |
+
return loss.loss(
|
| 917 |
+
y_true=y_true, raw_prediction=x
|
| 918 |
+
) + loss.constant_to_optimal_zero(y_true=y_true)
|
| 919 |
+
|
| 920 |
+
def fprime(x: np.ndarray) -> np.ndarray:
|
| 921 |
+
return loss.gradient(y_true=y_true, raw_prediction=x)
|
| 922 |
+
|
| 923 |
+
def fprime2(x: np.ndarray) -> np.ndarray:
|
| 924 |
+
return loss.gradient_hessian(y_true=y_true, raw_prediction=x)[1]
|
| 925 |
+
|
| 926 |
+
optimum = newton(
|
| 927 |
+
func,
|
| 928 |
+
x0=x0,
|
| 929 |
+
fprime=fprime,
|
| 930 |
+
fprime2=fprime2,
|
| 931 |
+
maxiter=100,
|
| 932 |
+
tol=5e-8,
|
| 933 |
+
)
|
| 934 |
+
|
| 935 |
+
# Need to ravel arrays because assert_allclose requires matching
|
| 936 |
+
# dimensions.
|
| 937 |
+
y_true = y_true.ravel()
|
| 938 |
+
optimum = optimum.ravel()
|
| 939 |
+
assert_allclose(loss.link.inverse(optimum), y_true)
|
| 940 |
+
assert_allclose(func(optimum), 0, atol=1e-14)
|
| 941 |
+
assert_allclose(loss.gradient(y_true=y_true, raw_prediction=optimum), 0, atol=5e-7)
|
| 942 |
+
|
| 943 |
+
|
| 944 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 945 |
+
@pytest.mark.parametrize("sample_weight", [None, "range"])
|
| 946 |
+
def test_loss_intercept_only(loss, sample_weight):
|
| 947 |
+
"""Test that fit_intercept_only returns the argmin of the loss.
|
| 948 |
+
|
| 949 |
+
Also test that the gradient is zero at the minimum.
|
| 950 |
+
"""
|
| 951 |
+
n_samples = 50
|
| 952 |
+
if not loss.is_multiclass:
|
| 953 |
+
y_true = loss.link.inverse(np.linspace(-4, 4, num=n_samples))
|
| 954 |
+
else:
|
| 955 |
+
y_true = np.arange(n_samples).astype(np.float64) % loss.n_classes
|
| 956 |
+
y_true[::5] = 0 # exceedance of class 0
|
| 957 |
+
|
| 958 |
+
if sample_weight == "range":
|
| 959 |
+
sample_weight = np.linspace(0.1, 2, num=n_samples)
|
| 960 |
+
|
| 961 |
+
a = loss.fit_intercept_only(y_true=y_true, sample_weight=sample_weight)
|
| 962 |
+
|
| 963 |
+
# find minimum by optimization
|
| 964 |
+
def fun(x):
|
| 965 |
+
if not loss.is_multiclass:
|
| 966 |
+
raw_prediction = np.full(shape=(n_samples), fill_value=x)
|
| 967 |
+
else:
|
| 968 |
+
raw_prediction = np.ascontiguousarray(
|
| 969 |
+
np.broadcast_to(x, shape=(n_samples, loss.n_classes))
|
| 970 |
+
)
|
| 971 |
+
return loss(
|
| 972 |
+
y_true=y_true,
|
| 973 |
+
raw_prediction=raw_prediction,
|
| 974 |
+
sample_weight=sample_weight,
|
| 975 |
+
)
|
| 976 |
+
|
| 977 |
+
if not loss.is_multiclass:
|
| 978 |
+
opt = minimize_scalar(fun, tol=1e-7, options={"maxiter": 100})
|
| 979 |
+
grad = loss.gradient(
|
| 980 |
+
y_true=y_true,
|
| 981 |
+
raw_prediction=np.full_like(y_true, a),
|
| 982 |
+
sample_weight=sample_weight,
|
| 983 |
+
)
|
| 984 |
+
assert a.shape == tuple() # scalar
|
| 985 |
+
assert a.dtype == y_true.dtype
|
| 986 |
+
assert_all_finite(a)
|
| 987 |
+
a == approx(opt.x, rel=1e-7)
|
| 988 |
+
grad.sum() == approx(0, abs=1e-12)
|
| 989 |
+
else:
|
| 990 |
+
# The constraint corresponds to sum(raw_prediction) = 0. Without it, we would
|
| 991 |
+
# need to apply loss.symmetrize_raw_prediction to opt.x before comparing.
|
| 992 |
+
opt = minimize(
|
| 993 |
+
fun,
|
| 994 |
+
np.zeros((loss.n_classes)),
|
| 995 |
+
tol=1e-13,
|
| 996 |
+
options={"maxiter": 100},
|
| 997 |
+
method="SLSQP",
|
| 998 |
+
constraints=LinearConstraint(np.ones((1, loss.n_classes)), 0, 0),
|
| 999 |
+
)
|
| 1000 |
+
grad = loss.gradient(
|
| 1001 |
+
y_true=y_true,
|
| 1002 |
+
raw_prediction=np.tile(a, (n_samples, 1)),
|
| 1003 |
+
sample_weight=sample_weight,
|
| 1004 |
+
)
|
| 1005 |
+
assert a.dtype == y_true.dtype
|
| 1006 |
+
assert_all_finite(a)
|
| 1007 |
+
assert_allclose(a, opt.x, rtol=5e-6, atol=1e-12)
|
| 1008 |
+
assert_allclose(grad.sum(axis=0), 0, atol=1e-12)
|
| 1009 |
+
|
| 1010 |
+
|
| 1011 |
+
@pytest.mark.parametrize(
|
| 1012 |
+
"loss, func, random_dist",
|
| 1013 |
+
[
|
| 1014 |
+
(HalfSquaredError(), np.mean, "normal"),
|
| 1015 |
+
(AbsoluteError(), np.median, "normal"),
|
| 1016 |
+
(PinballLoss(quantile=0.25), lambda x: np.percentile(x, q=25), "normal"),
|
| 1017 |
+
(HalfPoissonLoss(), np.mean, "poisson"),
|
| 1018 |
+
(HalfGammaLoss(), np.mean, "exponential"),
|
| 1019 |
+
(HalfTweedieLoss(), np.mean, "exponential"),
|
| 1020 |
+
(HalfBinomialLoss(), np.mean, "binomial"),
|
| 1021 |
+
],
|
| 1022 |
+
)
|
| 1023 |
+
def test_specific_fit_intercept_only(loss, func, random_dist, global_random_seed):
|
| 1024 |
+
"""Test that fit_intercept_only returns the correct functional.
|
| 1025 |
+
|
| 1026 |
+
We test the functional for specific, meaningful distributions, e.g.
|
| 1027 |
+
squared error estimates the expectation of a probability distribution.
|
| 1028 |
+
"""
|
| 1029 |
+
rng = np.random.RandomState(global_random_seed)
|
| 1030 |
+
if random_dist == "binomial":
|
| 1031 |
+
y_train = rng.binomial(1, 0.5, size=100)
|
| 1032 |
+
else:
|
| 1033 |
+
y_train = getattr(rng, random_dist)(size=100)
|
| 1034 |
+
baseline_prediction = loss.fit_intercept_only(y_true=y_train)
|
| 1035 |
+
# Make sure baseline prediction is the expected functional=func, e.g. mean
|
| 1036 |
+
# or median.
|
| 1037 |
+
assert_all_finite(baseline_prediction)
|
| 1038 |
+
assert baseline_prediction == approx(loss.link.link(func(y_train)))
|
| 1039 |
+
assert loss.link.inverse(baseline_prediction) == approx(func(y_train))
|
| 1040 |
+
if isinstance(loss, IdentityLink):
|
| 1041 |
+
assert_allclose(loss.link.inverse(baseline_prediction), baseline_prediction)
|
| 1042 |
+
|
| 1043 |
+
# Test baseline at boundary
|
| 1044 |
+
if loss.interval_y_true.low_inclusive:
|
| 1045 |
+
y_train.fill(loss.interval_y_true.low)
|
| 1046 |
+
baseline_prediction = loss.fit_intercept_only(y_true=y_train)
|
| 1047 |
+
assert_all_finite(baseline_prediction)
|
| 1048 |
+
if loss.interval_y_true.high_inclusive:
|
| 1049 |
+
y_train.fill(loss.interval_y_true.high)
|
| 1050 |
+
baseline_prediction = loss.fit_intercept_only(y_true=y_train)
|
| 1051 |
+
assert_all_finite(baseline_prediction)
|
| 1052 |
+
|
| 1053 |
+
|
| 1054 |
+
def test_multinomial_loss_fit_intercept_only():
|
| 1055 |
+
"""Test that fit_intercept_only returns the mean functional for CCE."""
|
| 1056 |
+
rng = np.random.RandomState(0)
|
| 1057 |
+
n_classes = 4
|
| 1058 |
+
loss = HalfMultinomialLoss(n_classes=n_classes)
|
| 1059 |
+
# Same logic as test_specific_fit_intercept_only. Here inverse link
|
| 1060 |
+
# function = softmax and link function = log - symmetry term.
|
| 1061 |
+
y_train = rng.randint(0, n_classes + 1, size=100).astype(np.float64)
|
| 1062 |
+
baseline_prediction = loss.fit_intercept_only(y_true=y_train)
|
| 1063 |
+
assert baseline_prediction.shape == (n_classes,)
|
| 1064 |
+
p = np.zeros(n_classes, dtype=y_train.dtype)
|
| 1065 |
+
for k in range(n_classes):
|
| 1066 |
+
p[k] = (y_train == k).mean()
|
| 1067 |
+
assert_allclose(baseline_prediction, np.log(p) - np.mean(np.log(p)))
|
| 1068 |
+
assert_allclose(baseline_prediction[None, :], loss.link.link(p[None, :]))
|
| 1069 |
+
|
| 1070 |
+
for y_train in (np.zeros(shape=10), np.ones(shape=10)):
|
| 1071 |
+
y_train = y_train.astype(np.float64)
|
| 1072 |
+
baseline_prediction = loss.fit_intercept_only(y_true=y_train)
|
| 1073 |
+
assert baseline_prediction.dtype == y_train.dtype
|
| 1074 |
+
assert_all_finite(baseline_prediction)
|
| 1075 |
+
|
| 1076 |
+
|
| 1077 |
+
def test_multinomial_cy_gradient(global_random_seed):
|
| 1078 |
+
"""Test that Multinomial cy_gradient gives the same result as gradient.
|
| 1079 |
+
|
| 1080 |
+
CyHalfMultinomialLoss does not inherit from CyLossFunction and has a different API.
|
| 1081 |
+
As a consequence, the functions like `loss` and `gradient` do not rely on `cy_loss`
|
| 1082 |
+
and `cy_gradient`.
|
| 1083 |
+
"""
|
| 1084 |
+
n_samples = 100
|
| 1085 |
+
n_classes = 5
|
| 1086 |
+
loss = HalfMultinomialLoss(n_classes=n_classes)
|
| 1087 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 1088 |
+
loss=loss,
|
| 1089 |
+
n_samples=n_samples,
|
| 1090 |
+
seed=global_random_seed,
|
| 1091 |
+
)
|
| 1092 |
+
sample_weight = np.linspace(0.1, 2, num=n_samples)
|
| 1093 |
+
|
| 1094 |
+
grad1 = loss.closs._test_cy_gradient(
|
| 1095 |
+
y_true=y_true,
|
| 1096 |
+
raw_prediction=raw_prediction, # needs to be C-contiguous
|
| 1097 |
+
sample_weight=sample_weight,
|
| 1098 |
+
)
|
| 1099 |
+
grad2 = loss.gradient(
|
| 1100 |
+
y_true=y_true,
|
| 1101 |
+
raw_prediction=raw_prediction,
|
| 1102 |
+
sample_weight=sample_weight,
|
| 1103 |
+
)
|
| 1104 |
+
assert_allclose(grad1, grad2)
|
| 1105 |
+
|
| 1106 |
+
|
| 1107 |
+
def test_binomial_and_multinomial_loss(global_random_seed):
|
| 1108 |
+
"""Test that multinomial loss with n_classes = 2 is the same as binomial loss."""
|
| 1109 |
+
rng = np.random.RandomState(global_random_seed)
|
| 1110 |
+
n_samples = 20
|
| 1111 |
+
binom = HalfBinomialLoss()
|
| 1112 |
+
multinom = HalfMultinomialLoss(n_classes=2)
|
| 1113 |
+
y_train = rng.randint(0, 2, size=n_samples).astype(np.float64)
|
| 1114 |
+
raw_prediction = rng.normal(size=n_samples)
|
| 1115 |
+
raw_multinom = np.empty((n_samples, 2))
|
| 1116 |
+
raw_multinom[:, 0] = -0.5 * raw_prediction
|
| 1117 |
+
raw_multinom[:, 1] = 0.5 * raw_prediction
|
| 1118 |
+
assert_allclose(
|
| 1119 |
+
binom.loss(y_true=y_train, raw_prediction=raw_prediction),
|
| 1120 |
+
multinom.loss(y_true=y_train, raw_prediction=raw_multinom),
|
| 1121 |
+
)
|
| 1122 |
+
|
| 1123 |
+
|
| 1124 |
+
@pytest.mark.parametrize("y_true", (np.array([0.0, 0, 0]), np.array([1.0, 1, 1])))
|
| 1125 |
+
@pytest.mark.parametrize("y_pred", (np.array([-5.0, -5, -5]), np.array([3.0, 3, 3])))
|
| 1126 |
+
def test_binomial_vs_alternative_formulation(y_true, y_pred, global_dtype):
|
| 1127 |
+
"""Test that both formulations of the binomial deviance agree.
|
| 1128 |
+
|
| 1129 |
+
Often, the binomial deviance or log loss is written in terms of a variable
|
| 1130 |
+
z in {-1, +1}, but we use y in {0, 1}, hence z = 2 * y - 1.
|
| 1131 |
+
ESL II Eq. (10.18):
|
| 1132 |
+
|
| 1133 |
+
-loglike(z, f) = log(1 + exp(-2 * z * f))
|
| 1134 |
+
|
| 1135 |
+
Note:
|
| 1136 |
+
- ESL 2*f = raw_prediction, hence the factor 2 of ESL disappears.
|
| 1137 |
+
- Deviance = -2*loglike + .., but HalfBinomialLoss is half of the
|
| 1138 |
+
deviance, hence the factor of 2 cancels in the comparison.
|
| 1139 |
+
"""
|
| 1140 |
+
|
| 1141 |
+
def alt_loss(y, raw_pred):
|
| 1142 |
+
z = 2 * y - 1
|
| 1143 |
+
return np.mean(np.log(1 + np.exp(-z * raw_pred)))
|
| 1144 |
+
|
| 1145 |
+
def alt_gradient(y, raw_pred):
|
| 1146 |
+
# alternative gradient formula according to ESL
|
| 1147 |
+
z = 2 * y - 1
|
| 1148 |
+
return -z / (1 + np.exp(z * raw_pred))
|
| 1149 |
+
|
| 1150 |
+
bin_loss = HalfBinomialLoss()
|
| 1151 |
+
|
| 1152 |
+
y_true = y_true.astype(global_dtype)
|
| 1153 |
+
y_pred = y_pred.astype(global_dtype)
|
| 1154 |
+
datum = (y_true, y_pred)
|
| 1155 |
+
|
| 1156 |
+
assert bin_loss(*datum) == approx(alt_loss(*datum))
|
| 1157 |
+
assert_allclose(bin_loss.gradient(*datum), alt_gradient(*datum))
|
| 1158 |
+
|
| 1159 |
+
|
| 1160 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 1161 |
+
def test_predict_proba(loss, global_random_seed):
|
| 1162 |
+
"""Test that predict_proba and gradient_proba work as expected."""
|
| 1163 |
+
n_samples = 20
|
| 1164 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 1165 |
+
loss=loss,
|
| 1166 |
+
n_samples=n_samples,
|
| 1167 |
+
y_bound=(-100, 100),
|
| 1168 |
+
raw_bound=(-5, 5),
|
| 1169 |
+
seed=global_random_seed,
|
| 1170 |
+
)
|
| 1171 |
+
|
| 1172 |
+
if hasattr(loss, "predict_proba"):
|
| 1173 |
+
proba = loss.predict_proba(raw_prediction)
|
| 1174 |
+
assert proba.shape == (n_samples, loss.n_classes)
|
| 1175 |
+
assert np.sum(proba, axis=1) == approx(1, rel=1e-11)
|
| 1176 |
+
|
| 1177 |
+
if hasattr(loss, "gradient_proba"):
|
| 1178 |
+
for grad, proba in (
|
| 1179 |
+
(None, None),
|
| 1180 |
+
(None, np.empty_like(raw_prediction)),
|
| 1181 |
+
(np.empty_like(raw_prediction), None),
|
| 1182 |
+
(np.empty_like(raw_prediction), np.empty_like(raw_prediction)),
|
| 1183 |
+
):
|
| 1184 |
+
grad, proba = loss.gradient_proba(
|
| 1185 |
+
y_true=y_true,
|
| 1186 |
+
raw_prediction=raw_prediction,
|
| 1187 |
+
sample_weight=None,
|
| 1188 |
+
gradient_out=grad,
|
| 1189 |
+
proba_out=proba,
|
| 1190 |
+
)
|
| 1191 |
+
assert proba.shape == (n_samples, loss.n_classes)
|
| 1192 |
+
assert np.sum(proba, axis=1) == approx(1, rel=1e-11)
|
| 1193 |
+
assert_allclose(
|
| 1194 |
+
grad,
|
| 1195 |
+
loss.gradient(
|
| 1196 |
+
y_true=y_true,
|
| 1197 |
+
raw_prediction=raw_prediction,
|
| 1198 |
+
sample_weight=None,
|
| 1199 |
+
gradient_out=None,
|
| 1200 |
+
),
|
| 1201 |
+
)
|
| 1202 |
+
|
| 1203 |
+
|
| 1204 |
+
@pytest.mark.parametrize("loss", ALL_LOSSES)
|
| 1205 |
+
@pytest.mark.parametrize("sample_weight", [None, "range"])
|
| 1206 |
+
@pytest.mark.parametrize("dtype", (np.float32, np.float64))
|
| 1207 |
+
@pytest.mark.parametrize("order", ("C", "F"))
|
| 1208 |
+
def test_init_gradient_and_hessians(loss, sample_weight, dtype, order):
|
| 1209 |
+
"""Test that init_gradient_and_hessian works as expected.
|
| 1210 |
+
|
| 1211 |
+
passing sample_weight to a loss correctly influences the constant_hessian
|
| 1212 |
+
attribute, and consequently the shape of the hessian array.
|
| 1213 |
+
"""
|
| 1214 |
+
n_samples = 5
|
| 1215 |
+
if sample_weight == "range":
|
| 1216 |
+
sample_weight = np.ones(n_samples)
|
| 1217 |
+
loss = loss(sample_weight=sample_weight)
|
| 1218 |
+
gradient, hessian = loss.init_gradient_and_hessian(
|
| 1219 |
+
n_samples=n_samples,
|
| 1220 |
+
dtype=dtype,
|
| 1221 |
+
order=order,
|
| 1222 |
+
)
|
| 1223 |
+
if loss.constant_hessian:
|
| 1224 |
+
assert gradient.shape == (n_samples,)
|
| 1225 |
+
assert hessian.shape == (1,)
|
| 1226 |
+
elif loss.is_multiclass:
|
| 1227 |
+
assert gradient.shape == (n_samples, loss.n_classes)
|
| 1228 |
+
assert hessian.shape == (n_samples, loss.n_classes)
|
| 1229 |
+
else:
|
| 1230 |
+
assert hessian.shape == (n_samples,)
|
| 1231 |
+
assert hessian.shape == (n_samples,)
|
| 1232 |
+
|
| 1233 |
+
assert gradient.dtype == dtype
|
| 1234 |
+
assert hessian.dtype == dtype
|
| 1235 |
+
|
| 1236 |
+
if order == "C":
|
| 1237 |
+
assert gradient.flags.c_contiguous
|
| 1238 |
+
assert hessian.flags.c_contiguous
|
| 1239 |
+
else:
|
| 1240 |
+
assert gradient.flags.f_contiguous
|
| 1241 |
+
assert hessian.flags.f_contiguous
|
| 1242 |
+
|
| 1243 |
+
|
| 1244 |
+
@pytest.mark.parametrize("loss", ALL_LOSSES)
|
| 1245 |
+
@pytest.mark.parametrize(
|
| 1246 |
+
"params, err_msg",
|
| 1247 |
+
[
|
| 1248 |
+
(
|
| 1249 |
+
{"dtype": np.int64},
|
| 1250 |
+
f"Valid options for 'dtype' are .* Got dtype={np.int64} instead.",
|
| 1251 |
+
),
|
| 1252 |
+
],
|
| 1253 |
+
)
|
| 1254 |
+
def test_init_gradient_and_hessian_raises(loss, params, err_msg):
|
| 1255 |
+
"""Test that init_gradient_and_hessian raises errors for invalid input."""
|
| 1256 |
+
loss = loss()
|
| 1257 |
+
with pytest.raises((ValueError, TypeError), match=err_msg):
|
| 1258 |
+
gradient, hessian = loss.init_gradient_and_hessian(n_samples=5, **params)
|
| 1259 |
+
|
| 1260 |
+
|
| 1261 |
+
@pytest.mark.parametrize(
|
| 1262 |
+
"loss, params, err_type, err_msg",
|
| 1263 |
+
[
|
| 1264 |
+
(
|
| 1265 |
+
PinballLoss,
|
| 1266 |
+
{"quantile": None},
|
| 1267 |
+
TypeError,
|
| 1268 |
+
"quantile must be an instance of float, not NoneType.",
|
| 1269 |
+
),
|
| 1270 |
+
(
|
| 1271 |
+
PinballLoss,
|
| 1272 |
+
{"quantile": 0},
|
| 1273 |
+
ValueError,
|
| 1274 |
+
"quantile == 0, must be > 0.",
|
| 1275 |
+
),
|
| 1276 |
+
(PinballLoss, {"quantile": 1.1}, ValueError, "quantile == 1.1, must be < 1."),
|
| 1277 |
+
(
|
| 1278 |
+
HuberLoss,
|
| 1279 |
+
{"quantile": None},
|
| 1280 |
+
TypeError,
|
| 1281 |
+
"quantile must be an instance of float, not NoneType.",
|
| 1282 |
+
),
|
| 1283 |
+
(
|
| 1284 |
+
HuberLoss,
|
| 1285 |
+
{"quantile": 0},
|
| 1286 |
+
ValueError,
|
| 1287 |
+
"quantile == 0, must be > 0.",
|
| 1288 |
+
),
|
| 1289 |
+
(HuberLoss, {"quantile": 1.1}, ValueError, "quantile == 1.1, must be < 1."),
|
| 1290 |
+
],
|
| 1291 |
+
)
|
| 1292 |
+
def test_loss_init_parameter_validation(loss, params, err_type, err_msg):
|
| 1293 |
+
"""Test that loss raises errors for invalid input."""
|
| 1294 |
+
with pytest.raises(err_type, match=err_msg):
|
| 1295 |
+
loss(**params)
|
| 1296 |
+
|
| 1297 |
+
|
| 1298 |
+
@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
|
| 1299 |
+
def test_loss_pickle(loss):
|
| 1300 |
+
"""Test that losses can be pickled."""
|
| 1301 |
+
n_samples = 20
|
| 1302 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 1303 |
+
loss=loss,
|
| 1304 |
+
n_samples=n_samples,
|
| 1305 |
+
y_bound=(-100, 100),
|
| 1306 |
+
raw_bound=(-5, 5),
|
| 1307 |
+
seed=42,
|
| 1308 |
+
)
|
| 1309 |
+
pickled_loss = pickle.dumps(loss)
|
| 1310 |
+
unpickled_loss = pickle.loads(pickled_loss)
|
| 1311 |
+
assert loss(y_true=y_true, raw_prediction=raw_prediction) == approx(
|
| 1312 |
+
unpickled_loss(y_true=y_true, raw_prediction=raw_prediction)
|
| 1313 |
+
)
|
| 1314 |
+
|
| 1315 |
+
|
| 1316 |
+
@pytest.mark.parametrize("p", [-1.5, 0, 1, 1.5, 2, 3])
|
| 1317 |
+
def test_tweedie_log_identity_consistency(p):
|
| 1318 |
+
"""Test for identical losses when only the link function is different."""
|
| 1319 |
+
half_tweedie_log = HalfTweedieLoss(power=p)
|
| 1320 |
+
half_tweedie_identity = HalfTweedieLossIdentity(power=p)
|
| 1321 |
+
n_samples = 10
|
| 1322 |
+
y_true, raw_prediction = random_y_true_raw_prediction(
|
| 1323 |
+
loss=half_tweedie_log, n_samples=n_samples, seed=42
|
| 1324 |
+
)
|
| 1325 |
+
y_pred = half_tweedie_log.link.inverse(raw_prediction) # exp(raw_prediction)
|
| 1326 |
+
|
| 1327 |
+
# Let's compare the loss values, up to some constant term that is dropped
|
| 1328 |
+
# in HalfTweedieLoss but not in HalfTweedieLossIdentity.
|
| 1329 |
+
loss_log = half_tweedie_log.loss(
|
| 1330 |
+
y_true=y_true, raw_prediction=raw_prediction
|
| 1331 |
+
) + half_tweedie_log.constant_to_optimal_zero(y_true)
|
| 1332 |
+
loss_identity = half_tweedie_identity.loss(
|
| 1333 |
+
y_true=y_true, raw_prediction=y_pred
|
| 1334 |
+
) + half_tweedie_identity.constant_to_optimal_zero(y_true)
|
| 1335 |
+
# Note that HalfTweedieLoss ignores different constant terms than
|
| 1336 |
+
# HalfTweedieLossIdentity. Constant terms means terms not depending on
|
| 1337 |
+
# raw_prediction. By adding these terms, `constant_to_optimal_zero`, both losses
|
| 1338 |
+
# give the same values.
|
| 1339 |
+
assert_allclose(loss_log, loss_identity)
|
| 1340 |
+
|
| 1341 |
+
# For gradients and hessians, the constant terms do not matter. We have, however,
|
| 1342 |
+
# to account for the chain rule, i.e. with x=raw_prediction
|
| 1343 |
+
# gradient_log(x) = d/dx loss_log(x)
|
| 1344 |
+
# = d/dx loss_identity(exp(x))
|
| 1345 |
+
# = exp(x) * gradient_identity(exp(x))
|
| 1346 |
+
# Similarly,
|
| 1347 |
+
# hessian_log(x) = exp(x) * gradient_identity(exp(x))
|
| 1348 |
+
# + exp(x)**2 * hessian_identity(x)
|
| 1349 |
+
gradient_log, hessian_log = half_tweedie_log.gradient_hessian(
|
| 1350 |
+
y_true=y_true, raw_prediction=raw_prediction
|
| 1351 |
+
)
|
| 1352 |
+
gradient_identity, hessian_identity = half_tweedie_identity.gradient_hessian(
|
| 1353 |
+
y_true=y_true, raw_prediction=y_pred
|
| 1354 |
+
)
|
| 1355 |
+
assert_allclose(gradient_log, y_pred * gradient_identity)
|
| 1356 |
+
assert_allclose(
|
| 1357 |
+
hessian_log, y_pred * gradient_identity + y_pred**2 * hessian_identity
|
| 1358 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_min_dependencies.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""All minimum dependencies for scikit-learn."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
from collections import defaultdict
|
| 8 |
+
|
| 9 |
+
# scipy and cython should by in sync with pyproject.toml
|
| 10 |
+
NUMPY_MIN_VERSION = "1.24.1"
|
| 11 |
+
SCIPY_MIN_VERSION = "1.10.0"
|
| 12 |
+
JOBLIB_MIN_VERSION = "1.3.0"
|
| 13 |
+
THREADPOOLCTL_MIN_VERSION = "3.2.0"
|
| 14 |
+
PYTEST_MIN_VERSION = "7.1.2"
|
| 15 |
+
CYTHON_MIN_VERSION = "3.1.2"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# 'build' and 'install' is included to have structured metadata for CI.
|
| 19 |
+
# It will NOT be included in setup's extras_require
|
| 20 |
+
# The values are (version_spec, comma separated tags)
|
| 21 |
+
dependent_packages = {
|
| 22 |
+
"numpy": (NUMPY_MIN_VERSION, "build, install"),
|
| 23 |
+
"scipy": (SCIPY_MIN_VERSION, "build, install"),
|
| 24 |
+
"joblib": (JOBLIB_MIN_VERSION, "install"),
|
| 25 |
+
"threadpoolctl": (THREADPOOLCTL_MIN_VERSION, "install"),
|
| 26 |
+
"cython": (CYTHON_MIN_VERSION, "build"),
|
| 27 |
+
"meson-python": ("0.17.1", "build"),
|
| 28 |
+
"matplotlib": ("3.6.1", "benchmark, docs, examples, tests"),
|
| 29 |
+
"scikit-image": ("0.22.0", "docs, examples"),
|
| 30 |
+
"pandas": ("1.5.0", "benchmark, docs, examples, tests"),
|
| 31 |
+
"seaborn": ("0.13.0", "docs, examples"),
|
| 32 |
+
"memory_profiler": ("0.57.0", "benchmark, docs"),
|
| 33 |
+
"pytest": (PYTEST_MIN_VERSION, "tests"),
|
| 34 |
+
"pytest-cov": ("2.9.0", "tests"),
|
| 35 |
+
"ruff": ("0.11.7", "tests"),
|
| 36 |
+
"mypy": ("1.15", "tests"),
|
| 37 |
+
"pyamg": ("5.0.0", "tests"),
|
| 38 |
+
"polars": ("0.20.30", "docs, tests"),
|
| 39 |
+
"pyarrow": ("12.0.0", "tests"),
|
| 40 |
+
"sphinx": ("7.3.7", "docs"),
|
| 41 |
+
"sphinx-copybutton": ("0.5.2", "docs"),
|
| 42 |
+
"sphinx-gallery": ("0.17.1", "docs"),
|
| 43 |
+
"numpydoc": ("1.2.0", "docs, tests"),
|
| 44 |
+
"Pillow": ("10.1.0", "docs"),
|
| 45 |
+
"pooch": ("1.8.0", "docs, examples, tests"),
|
| 46 |
+
"sphinx-prompt": ("1.4.0", "docs"),
|
| 47 |
+
"sphinxext-opengraph": ("0.9.1", "docs"),
|
| 48 |
+
"plotly": ("5.18.0", "docs, examples"),
|
| 49 |
+
"sphinxcontrib-sass": ("0.3.4", "docs"),
|
| 50 |
+
"sphinx-remove-toctrees": ("1.0.0.post1", "docs"),
|
| 51 |
+
"sphinx-design": ("0.6.0", "docs"),
|
| 52 |
+
"pydata-sphinx-theme": ("0.15.3", "docs"),
|
| 53 |
+
"towncrier": ("24.8.0", "docs"),
|
| 54 |
+
# XXX: Pin conda-lock to the latest released version (needs manual update
|
| 55 |
+
# from time to time)
|
| 56 |
+
"conda-lock": ("3.0.1", "maintenance"),
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
# create inverse mapping for setuptools
|
| 61 |
+
tag_to_packages: dict = defaultdict(list)
|
| 62 |
+
for package, (min_version, extras) in dependent_packages.items():
|
| 63 |
+
for extra in extras.split(", "):
|
| 64 |
+
tag_to_packages[extra].append("{}>={}".format(package, min_version))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# Used by CI to get the min dependencies
|
| 68 |
+
if __name__ == "__main__":
|
| 69 |
+
parser = argparse.ArgumentParser(description="Get min dependencies for a package")
|
| 70 |
+
|
| 71 |
+
parser.add_argument("package", choices=dependent_packages)
|
| 72 |
+
args = parser.parse_args()
|
| 73 |
+
min_version = dependent_packages[args.package][0]
|
| 74 |
+
print(min_version)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/base.py
ADDED
|
@@ -0,0 +1,1340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Base classes for all estimators and various utility functions."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
import copy
|
| 7 |
+
import functools
|
| 8 |
+
import inspect
|
| 9 |
+
import platform
|
| 10 |
+
import re
|
| 11 |
+
import warnings
|
| 12 |
+
from collections import defaultdict
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
|
| 16 |
+
from sklearn import __version__
|
| 17 |
+
from sklearn._config import config_context, get_config
|
| 18 |
+
from sklearn.exceptions import InconsistentVersionWarning
|
| 19 |
+
from sklearn.utils._metadata_requests import _MetadataRequester, _routing_enabled
|
| 20 |
+
from sklearn.utils._missing import is_pandas_na, is_scalar_nan
|
| 21 |
+
from sklearn.utils._param_validation import validate_parameter_constraints
|
| 22 |
+
from sklearn.utils._repr_html.base import ReprHTMLMixin, _HTMLDocumentationLinkMixin
|
| 23 |
+
from sklearn.utils._repr_html.estimator import estimator_html_repr
|
| 24 |
+
from sklearn.utils._repr_html.params import ParamsDict
|
| 25 |
+
from sklearn.utils._set_output import _SetOutputMixin
|
| 26 |
+
from sklearn.utils._tags import (
|
| 27 |
+
ClassifierTags,
|
| 28 |
+
RegressorTags,
|
| 29 |
+
Tags,
|
| 30 |
+
TargetTags,
|
| 31 |
+
TransformerTags,
|
| 32 |
+
get_tags,
|
| 33 |
+
)
|
| 34 |
+
from sklearn.utils.fixes import _IS_32BIT
|
| 35 |
+
from sklearn.utils.validation import (
|
| 36 |
+
_check_feature_names_in,
|
| 37 |
+
_generate_get_feature_names_out,
|
| 38 |
+
_is_fitted,
|
| 39 |
+
check_array,
|
| 40 |
+
check_is_fitted,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def clone(estimator, *, safe=True):
|
| 45 |
+
"""Construct a new unfitted estimator with the same parameters.
|
| 46 |
+
|
| 47 |
+
Clone does a deep copy of the model in an estimator
|
| 48 |
+
without actually copying attached data. It returns a new estimator
|
| 49 |
+
with the same parameters that has not been fitted on any data.
|
| 50 |
+
|
| 51 |
+
.. versionchanged:: 1.3
|
| 52 |
+
Delegates to `estimator.__sklearn_clone__` if the method exists.
|
| 53 |
+
|
| 54 |
+
Parameters
|
| 55 |
+
----------
|
| 56 |
+
estimator : {list, tuple, set} of estimator instance or a single \
|
| 57 |
+
estimator instance
|
| 58 |
+
The estimator or group of estimators to be cloned.
|
| 59 |
+
safe : bool, default=True
|
| 60 |
+
If safe is False, clone will fall back to a deep copy on objects
|
| 61 |
+
that are not estimators. Ignored if `estimator.__sklearn_clone__`
|
| 62 |
+
exists.
|
| 63 |
+
|
| 64 |
+
Returns
|
| 65 |
+
-------
|
| 66 |
+
estimator : object
|
| 67 |
+
The deep copy of the input, an estimator if input is an estimator.
|
| 68 |
+
|
| 69 |
+
Notes
|
| 70 |
+
-----
|
| 71 |
+
If the estimator's `random_state` parameter is an integer (or if the
|
| 72 |
+
estimator doesn't have a `random_state` parameter), an *exact clone* is
|
| 73 |
+
returned: the clone and the original estimator will give the exact same
|
| 74 |
+
results. Otherwise, *statistical clone* is returned: the clone might
|
| 75 |
+
return different results from the original estimator. More details can be
|
| 76 |
+
found in :ref:`randomness`.
|
| 77 |
+
|
| 78 |
+
Examples
|
| 79 |
+
--------
|
| 80 |
+
>>> from sklearn.base import clone
|
| 81 |
+
>>> from sklearn.linear_model import LogisticRegression
|
| 82 |
+
>>> X = [[-1, 0], [0, 1], [0, -1], [1, 0]]
|
| 83 |
+
>>> y = [0, 0, 1, 1]
|
| 84 |
+
>>> classifier = LogisticRegression().fit(X, y)
|
| 85 |
+
>>> cloned_classifier = clone(classifier)
|
| 86 |
+
>>> hasattr(classifier, "classes_")
|
| 87 |
+
True
|
| 88 |
+
>>> hasattr(cloned_classifier, "classes_")
|
| 89 |
+
False
|
| 90 |
+
>>> classifier is cloned_classifier
|
| 91 |
+
False
|
| 92 |
+
"""
|
| 93 |
+
if hasattr(estimator, "__sklearn_clone__") and not inspect.isclass(estimator):
|
| 94 |
+
return estimator.__sklearn_clone__()
|
| 95 |
+
return _clone_parametrized(estimator, safe=safe)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def _clone_parametrized(estimator, *, safe=True):
|
| 99 |
+
"""Default implementation of clone. See :func:`sklearn.base.clone` for details."""
|
| 100 |
+
|
| 101 |
+
estimator_type = type(estimator)
|
| 102 |
+
if estimator_type is dict:
|
| 103 |
+
return {k: clone(v, safe=safe) for k, v in estimator.items()}
|
| 104 |
+
elif estimator_type in (list, tuple, set, frozenset):
|
| 105 |
+
return estimator_type([clone(e, safe=safe) for e in estimator])
|
| 106 |
+
elif not hasattr(estimator, "get_params") or isinstance(estimator, type):
|
| 107 |
+
if not safe:
|
| 108 |
+
return copy.deepcopy(estimator)
|
| 109 |
+
else:
|
| 110 |
+
if isinstance(estimator, type):
|
| 111 |
+
raise TypeError(
|
| 112 |
+
"Cannot clone object. "
|
| 113 |
+
"You should provide an instance of "
|
| 114 |
+
"scikit-learn estimator instead of a class."
|
| 115 |
+
)
|
| 116 |
+
else:
|
| 117 |
+
raise TypeError(
|
| 118 |
+
"Cannot clone object '%s' (type %s): "
|
| 119 |
+
"it does not seem to be a scikit-learn "
|
| 120 |
+
"estimator as it does not implement a "
|
| 121 |
+
"'get_params' method." % (repr(estimator), type(estimator))
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
klass = estimator.__class__
|
| 125 |
+
new_object_params = estimator.get_params(deep=False)
|
| 126 |
+
for name, param in new_object_params.items():
|
| 127 |
+
new_object_params[name] = clone(param, safe=False)
|
| 128 |
+
|
| 129 |
+
new_object = klass(**new_object_params)
|
| 130 |
+
try:
|
| 131 |
+
new_object._metadata_request = copy.deepcopy(estimator._metadata_request)
|
| 132 |
+
except AttributeError:
|
| 133 |
+
pass
|
| 134 |
+
|
| 135 |
+
params_set = new_object.get_params(deep=False)
|
| 136 |
+
|
| 137 |
+
# quick sanity check of the parameters of the clone
|
| 138 |
+
for name in new_object_params:
|
| 139 |
+
param1 = new_object_params[name]
|
| 140 |
+
param2 = params_set[name]
|
| 141 |
+
if param1 is not param2:
|
| 142 |
+
raise RuntimeError(
|
| 143 |
+
"Cannot clone object %s, as the constructor "
|
| 144 |
+
"either does not set or modifies parameter %s" % (estimator, name)
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
# _sklearn_output_config is used by `set_output` to configure the output
|
| 148 |
+
# container of an estimator.
|
| 149 |
+
if hasattr(estimator, "_sklearn_output_config"):
|
| 150 |
+
new_object._sklearn_output_config = copy.deepcopy(
|
| 151 |
+
estimator._sklearn_output_config
|
| 152 |
+
)
|
| 153 |
+
return new_object
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class BaseEstimator(ReprHTMLMixin, _HTMLDocumentationLinkMixin, _MetadataRequester):
|
| 157 |
+
"""Base class for all estimators in scikit-learn.
|
| 158 |
+
|
| 159 |
+
Inheriting from this class provides default implementations of:
|
| 160 |
+
|
| 161 |
+
- setting and getting parameters used by `GridSearchCV` and friends;
|
| 162 |
+
- textual and HTML representation displayed in terminals and IDEs;
|
| 163 |
+
- estimator serialization;
|
| 164 |
+
- parameters validation;
|
| 165 |
+
- data validation;
|
| 166 |
+
- feature names validation.
|
| 167 |
+
|
| 168 |
+
Read more in the :ref:`User Guide <rolling_your_own_estimator>`.
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
Notes
|
| 172 |
+
-----
|
| 173 |
+
All estimators should specify all the parameters that can be set
|
| 174 |
+
at the class level in their ``__init__`` as explicit keyword
|
| 175 |
+
arguments (no ``*args`` or ``**kwargs``).
|
| 176 |
+
|
| 177 |
+
Examples
|
| 178 |
+
--------
|
| 179 |
+
>>> import numpy as np
|
| 180 |
+
>>> from sklearn.base import BaseEstimator
|
| 181 |
+
>>> class MyEstimator(BaseEstimator):
|
| 182 |
+
... def __init__(self, *, param=1):
|
| 183 |
+
... self.param = param
|
| 184 |
+
... def fit(self, X, y=None):
|
| 185 |
+
... self.is_fitted_ = True
|
| 186 |
+
... return self
|
| 187 |
+
... def predict(self, X):
|
| 188 |
+
... return np.full(shape=X.shape[0], fill_value=self.param)
|
| 189 |
+
>>> estimator = MyEstimator(param=2)
|
| 190 |
+
>>> estimator.get_params()
|
| 191 |
+
{'param': 2}
|
| 192 |
+
>>> X = np.array([[1, 2], [2, 3], [3, 4]])
|
| 193 |
+
>>> y = np.array([1, 0, 1])
|
| 194 |
+
>>> estimator.fit(X, y).predict(X)
|
| 195 |
+
array([2, 2, 2])
|
| 196 |
+
>>> estimator.set_params(param=3).fit(X, y).predict(X)
|
| 197 |
+
array([3, 3, 3])
|
| 198 |
+
"""
|
| 199 |
+
|
| 200 |
+
def __dir__(self):
|
| 201 |
+
# Filters conditional methods that should be hidden based
|
| 202 |
+
# on the `available_if` decorator
|
| 203 |
+
with warnings.catch_warnings():
|
| 204 |
+
warnings.filterwarnings("ignore", category=FutureWarning)
|
| 205 |
+
return [attr for attr in super().__dir__() if hasattr(self, attr)]
|
| 206 |
+
|
| 207 |
+
_html_repr = estimator_html_repr
|
| 208 |
+
|
| 209 |
+
@classmethod
|
| 210 |
+
def _get_param_names(cls):
|
| 211 |
+
"""Get parameter names for the estimator"""
|
| 212 |
+
# fetch the constructor or the original constructor before
|
| 213 |
+
# deprecation wrapping if any
|
| 214 |
+
init = getattr(cls.__init__, "deprecated_original", cls.__init__)
|
| 215 |
+
if init is object.__init__:
|
| 216 |
+
# No explicit constructor to introspect
|
| 217 |
+
return []
|
| 218 |
+
|
| 219 |
+
# introspect the constructor arguments to find the model parameters
|
| 220 |
+
# to represent
|
| 221 |
+
init_signature = inspect.signature(init)
|
| 222 |
+
# Consider the constructor parameters excluding 'self'
|
| 223 |
+
parameters = [
|
| 224 |
+
p
|
| 225 |
+
for p in init_signature.parameters.values()
|
| 226 |
+
if p.name != "self" and p.kind != p.VAR_KEYWORD
|
| 227 |
+
]
|
| 228 |
+
for p in parameters:
|
| 229 |
+
if p.kind == p.VAR_POSITIONAL:
|
| 230 |
+
raise RuntimeError(
|
| 231 |
+
"scikit-learn estimators should always "
|
| 232 |
+
"specify their parameters in the signature"
|
| 233 |
+
" of their __init__ (no varargs)."
|
| 234 |
+
" %s with constructor %s doesn't "
|
| 235 |
+
" follow this convention." % (cls, init_signature)
|
| 236 |
+
)
|
| 237 |
+
# Extract and sort argument names excluding 'self'
|
| 238 |
+
return sorted([p.name for p in parameters])
|
| 239 |
+
|
| 240 |
+
def get_params(self, deep=True):
|
| 241 |
+
"""
|
| 242 |
+
Get parameters for this estimator.
|
| 243 |
+
|
| 244 |
+
Parameters
|
| 245 |
+
----------
|
| 246 |
+
deep : bool, default=True
|
| 247 |
+
If True, will return the parameters for this estimator and
|
| 248 |
+
contained subobjects that are estimators.
|
| 249 |
+
|
| 250 |
+
Returns
|
| 251 |
+
-------
|
| 252 |
+
params : dict
|
| 253 |
+
Parameter names mapped to their values.
|
| 254 |
+
"""
|
| 255 |
+
out = dict()
|
| 256 |
+
for key in self._get_param_names():
|
| 257 |
+
value = getattr(self, key)
|
| 258 |
+
if deep and hasattr(value, "get_params") and not isinstance(value, type):
|
| 259 |
+
deep_items = value.get_params().items()
|
| 260 |
+
out.update((key + "__" + k, val) for k, val in deep_items)
|
| 261 |
+
out[key] = value
|
| 262 |
+
return out
|
| 263 |
+
|
| 264 |
+
def _get_params_html(self, deep=True, doc_link=""):
|
| 265 |
+
"""
|
| 266 |
+
Get parameters for this estimator with a specific HTML representation.
|
| 267 |
+
|
| 268 |
+
Parameters
|
| 269 |
+
----------
|
| 270 |
+
deep : bool, default=True
|
| 271 |
+
If True, will return the parameters for this estimator and
|
| 272 |
+
contained subobjects that are estimators.
|
| 273 |
+
|
| 274 |
+
doc_link : str
|
| 275 |
+
URL to the estimator documentation.
|
| 276 |
+
Used for linking to the estimator's parameters documentation
|
| 277 |
+
available in HTML displays.
|
| 278 |
+
|
| 279 |
+
Returns
|
| 280 |
+
-------
|
| 281 |
+
params : ParamsDict
|
| 282 |
+
Parameter names mapped to their values. We return a `ParamsDict`
|
| 283 |
+
dictionary, which renders a specific HTML representation in table
|
| 284 |
+
form.
|
| 285 |
+
"""
|
| 286 |
+
out = self.get_params(deep=deep)
|
| 287 |
+
|
| 288 |
+
init_func = getattr(self.__init__, "deprecated_original", self.__init__)
|
| 289 |
+
init_default_params = inspect.signature(init_func).parameters
|
| 290 |
+
init_default_params = {
|
| 291 |
+
name: param.default for name, param in init_default_params.items()
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
def is_non_default(param_name, param_value):
|
| 295 |
+
"""Finds the parameters that have been set by the user."""
|
| 296 |
+
if param_name not in init_default_params:
|
| 297 |
+
# happens if k is part of a **kwargs
|
| 298 |
+
return True
|
| 299 |
+
if init_default_params[param_name] == inspect._empty:
|
| 300 |
+
# k has no default value
|
| 301 |
+
return True
|
| 302 |
+
# avoid calling repr on nested estimators
|
| 303 |
+
if isinstance(param_value, BaseEstimator) and type(param_value) is not type(
|
| 304 |
+
init_default_params[param_name]
|
| 305 |
+
):
|
| 306 |
+
return True
|
| 307 |
+
if is_pandas_na(param_value) and not is_pandas_na(
|
| 308 |
+
init_default_params[param_name]
|
| 309 |
+
):
|
| 310 |
+
return True
|
| 311 |
+
if not np.array_equal(
|
| 312 |
+
param_value, init_default_params[param_name]
|
| 313 |
+
) and not (
|
| 314 |
+
is_scalar_nan(init_default_params[param_name])
|
| 315 |
+
and is_scalar_nan(param_value)
|
| 316 |
+
):
|
| 317 |
+
return True
|
| 318 |
+
|
| 319 |
+
return False
|
| 320 |
+
|
| 321 |
+
# reorder the parameters from `self.get_params` using the `__init__`
|
| 322 |
+
# signature
|
| 323 |
+
remaining_params = [name for name in out if name not in init_default_params]
|
| 324 |
+
ordered_out = {name: out[name] for name in init_default_params if name in out}
|
| 325 |
+
ordered_out.update({name: out[name] for name in remaining_params})
|
| 326 |
+
|
| 327 |
+
non_default_ls = tuple(
|
| 328 |
+
[name for name, value in ordered_out.items() if is_non_default(name, value)]
|
| 329 |
+
)
|
| 330 |
+
|
| 331 |
+
return ParamsDict(
|
| 332 |
+
params=ordered_out,
|
| 333 |
+
non_default=non_default_ls,
|
| 334 |
+
estimator_class=self.__class__,
|
| 335 |
+
doc_link=doc_link,
|
| 336 |
+
)
|
| 337 |
+
|
| 338 |
+
def set_params(self, **params):
|
| 339 |
+
"""Set the parameters of this estimator.
|
| 340 |
+
|
| 341 |
+
The method works on simple estimators as well as on nested objects
|
| 342 |
+
(such as :class:`~sklearn.pipeline.Pipeline`). The latter have
|
| 343 |
+
parameters of the form ``<component>__<parameter>`` so that it's
|
| 344 |
+
possible to update each component of a nested object.
|
| 345 |
+
|
| 346 |
+
Parameters
|
| 347 |
+
----------
|
| 348 |
+
**params : dict
|
| 349 |
+
Estimator parameters.
|
| 350 |
+
|
| 351 |
+
Returns
|
| 352 |
+
-------
|
| 353 |
+
self : estimator instance
|
| 354 |
+
Estimator instance.
|
| 355 |
+
"""
|
| 356 |
+
if not params:
|
| 357 |
+
# Simple optimization to gain speed (inspect is slow)
|
| 358 |
+
return self
|
| 359 |
+
valid_params = self.get_params(deep=True)
|
| 360 |
+
|
| 361 |
+
nested_params = defaultdict(dict) # grouped by prefix
|
| 362 |
+
for key, value in params.items():
|
| 363 |
+
key, delim, sub_key = key.partition("__")
|
| 364 |
+
if key not in valid_params:
|
| 365 |
+
local_valid_params = self._get_param_names()
|
| 366 |
+
raise ValueError(
|
| 367 |
+
f"Invalid parameter {key!r} for estimator {self}. "
|
| 368 |
+
f"Valid parameters are: {local_valid_params!r}."
|
| 369 |
+
)
|
| 370 |
+
|
| 371 |
+
if delim:
|
| 372 |
+
nested_params[key][sub_key] = value
|
| 373 |
+
else:
|
| 374 |
+
setattr(self, key, value)
|
| 375 |
+
valid_params[key] = value
|
| 376 |
+
|
| 377 |
+
for key, sub_params in nested_params.items():
|
| 378 |
+
valid_params[key].set_params(**sub_params)
|
| 379 |
+
|
| 380 |
+
return self
|
| 381 |
+
|
| 382 |
+
def __sklearn_clone__(self):
|
| 383 |
+
return _clone_parametrized(self)
|
| 384 |
+
|
| 385 |
+
def __repr__(self, N_CHAR_MAX=700):
|
| 386 |
+
# N_CHAR_MAX is the (approximate) maximum number of non-blank
|
| 387 |
+
# characters to render. We pass it as an optional parameter to ease
|
| 388 |
+
# the tests.
|
| 389 |
+
|
| 390 |
+
from sklearn.utils._pprint import _EstimatorPrettyPrinter
|
| 391 |
+
|
| 392 |
+
N_MAX_ELEMENTS_TO_SHOW = 30 # number of elements to show in sequences
|
| 393 |
+
|
| 394 |
+
# use ellipsis for sequences with a lot of elements
|
| 395 |
+
pp = _EstimatorPrettyPrinter(
|
| 396 |
+
compact=True,
|
| 397 |
+
indent=1,
|
| 398 |
+
indent_at_name=True,
|
| 399 |
+
n_max_elements_to_show=N_MAX_ELEMENTS_TO_SHOW,
|
| 400 |
+
)
|
| 401 |
+
|
| 402 |
+
repr_ = pp.pformat(self)
|
| 403 |
+
|
| 404 |
+
# Use bruteforce ellipsis when there are a lot of non-blank characters
|
| 405 |
+
n_nonblank = len("".join(repr_.split()))
|
| 406 |
+
if n_nonblank > N_CHAR_MAX:
|
| 407 |
+
lim = N_CHAR_MAX // 2 # apprx number of chars to keep on both ends
|
| 408 |
+
regex = r"^(\s*\S){%d}" % lim
|
| 409 |
+
# The regex '^(\s*\S){%d}' % n
|
| 410 |
+
# matches from the start of the string until the nth non-blank
|
| 411 |
+
# character:
|
| 412 |
+
# - ^ matches the start of string
|
| 413 |
+
# - (pattern){n} matches n repetitions of pattern
|
| 414 |
+
# - \s*\S matches a non-blank char following zero or more blanks
|
| 415 |
+
left_lim = re.match(regex, repr_).end()
|
| 416 |
+
right_lim = re.match(regex, repr_[::-1]).end()
|
| 417 |
+
|
| 418 |
+
if "\n" in repr_[left_lim:-right_lim]:
|
| 419 |
+
# The left side and right side aren't on the same line.
|
| 420 |
+
# To avoid weird cuts, e.g.:
|
| 421 |
+
# categoric...ore',
|
| 422 |
+
# we need to start the right side with an appropriate newline
|
| 423 |
+
# character so that it renders properly as:
|
| 424 |
+
# categoric...
|
| 425 |
+
# handle_unknown='ignore',
|
| 426 |
+
# so we add [^\n]*\n which matches until the next \n
|
| 427 |
+
regex += r"[^\n]*\n"
|
| 428 |
+
right_lim = re.match(regex, repr_[::-1]).end()
|
| 429 |
+
|
| 430 |
+
ellipsis = "..."
|
| 431 |
+
if left_lim + len(ellipsis) < len(repr_) - right_lim:
|
| 432 |
+
# Only add ellipsis if it results in a shorter repr
|
| 433 |
+
repr_ = repr_[:left_lim] + "..." + repr_[-right_lim:]
|
| 434 |
+
|
| 435 |
+
return repr_
|
| 436 |
+
|
| 437 |
+
def __getstate__(self):
|
| 438 |
+
if getattr(self, "__slots__", None):
|
| 439 |
+
raise TypeError(
|
| 440 |
+
"You cannot use `__slots__` in objects inheriting from "
|
| 441 |
+
"`sklearn.base.BaseEstimator`."
|
| 442 |
+
)
|
| 443 |
+
|
| 444 |
+
try:
|
| 445 |
+
state = super().__getstate__()
|
| 446 |
+
if state is None:
|
| 447 |
+
# For Python 3.11+, empty instance (no `__slots__`,
|
| 448 |
+
# and `__dict__`) will return a state equal to `None`.
|
| 449 |
+
state = self.__dict__.copy()
|
| 450 |
+
except AttributeError:
|
| 451 |
+
# Python < 3.11
|
| 452 |
+
state = self.__dict__.copy()
|
| 453 |
+
|
| 454 |
+
if type(self).__module__.startswith("sklearn."):
|
| 455 |
+
return dict(state.items(), _sklearn_version=__version__)
|
| 456 |
+
else:
|
| 457 |
+
return state
|
| 458 |
+
|
| 459 |
+
def __setstate__(self, state):
|
| 460 |
+
if type(self).__module__.startswith("sklearn."):
|
| 461 |
+
pickle_version = state.pop("_sklearn_version", "pre-0.18")
|
| 462 |
+
if pickle_version != __version__:
|
| 463 |
+
warnings.warn(
|
| 464 |
+
InconsistentVersionWarning(
|
| 465 |
+
estimator_name=self.__class__.__name__,
|
| 466 |
+
current_sklearn_version=__version__,
|
| 467 |
+
original_sklearn_version=pickle_version,
|
| 468 |
+
),
|
| 469 |
+
)
|
| 470 |
+
try:
|
| 471 |
+
super().__setstate__(state)
|
| 472 |
+
except AttributeError:
|
| 473 |
+
self.__dict__.update(state)
|
| 474 |
+
|
| 475 |
+
def __sklearn_tags__(self):
|
| 476 |
+
return Tags(
|
| 477 |
+
estimator_type=None,
|
| 478 |
+
target_tags=TargetTags(required=False),
|
| 479 |
+
transformer_tags=None,
|
| 480 |
+
regressor_tags=None,
|
| 481 |
+
classifier_tags=None,
|
| 482 |
+
)
|
| 483 |
+
|
| 484 |
+
def _validate_params(self):
|
| 485 |
+
"""Validate types and values of constructor parameters
|
| 486 |
+
|
| 487 |
+
The expected type and values must be defined in the `_parameter_constraints`
|
| 488 |
+
class attribute, which is a dictionary `param_name: list of constraints`. See
|
| 489 |
+
the docstring of `validate_parameter_constraints` for a description of the
|
| 490 |
+
accepted constraints.
|
| 491 |
+
"""
|
| 492 |
+
validate_parameter_constraints(
|
| 493 |
+
self._parameter_constraints,
|
| 494 |
+
self.get_params(deep=False),
|
| 495 |
+
caller_name=self.__class__.__name__,
|
| 496 |
+
)
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
class ClassifierMixin:
|
| 500 |
+
"""Mixin class for all classifiers in scikit-learn.
|
| 501 |
+
|
| 502 |
+
This mixin defines the following functionality:
|
| 503 |
+
|
| 504 |
+
- set estimator type to `"classifier"` through the `estimator_type` tag;
|
| 505 |
+
- `score` method that default to :func:`~sklearn.metrics.accuracy_score`.
|
| 506 |
+
- enforce that `fit` requires `y` to be passed through the `requires_y` tag,
|
| 507 |
+
which is done by setting the classifier type tag.
|
| 508 |
+
|
| 509 |
+
Read more in the :ref:`User Guide <rolling_your_own_estimator>`.
|
| 510 |
+
|
| 511 |
+
Examples
|
| 512 |
+
--------
|
| 513 |
+
>>> import numpy as np
|
| 514 |
+
>>> from sklearn.base import BaseEstimator, ClassifierMixin
|
| 515 |
+
>>> # Mixin classes should always be on the left-hand side for a correct MRO
|
| 516 |
+
>>> class MyEstimator(ClassifierMixin, BaseEstimator):
|
| 517 |
+
... def __init__(self, *, param=1):
|
| 518 |
+
... self.param = param
|
| 519 |
+
... def fit(self, X, y=None):
|
| 520 |
+
... self.is_fitted_ = True
|
| 521 |
+
... return self
|
| 522 |
+
... def predict(self, X):
|
| 523 |
+
... return np.full(shape=X.shape[0], fill_value=self.param)
|
| 524 |
+
>>> estimator = MyEstimator(param=1)
|
| 525 |
+
>>> X = np.array([[1, 2], [2, 3], [3, 4]])
|
| 526 |
+
>>> y = np.array([1, 0, 1])
|
| 527 |
+
>>> estimator.fit(X, y).predict(X)
|
| 528 |
+
array([1, 1, 1])
|
| 529 |
+
>>> estimator.score(X, y)
|
| 530 |
+
0.66...
|
| 531 |
+
"""
|
| 532 |
+
|
| 533 |
+
def __sklearn_tags__(self):
|
| 534 |
+
tags = super().__sklearn_tags__()
|
| 535 |
+
tags.estimator_type = "classifier"
|
| 536 |
+
tags.classifier_tags = ClassifierTags()
|
| 537 |
+
tags.target_tags.required = True
|
| 538 |
+
return tags
|
| 539 |
+
|
| 540 |
+
def score(self, X, y, sample_weight=None):
|
| 541 |
+
"""
|
| 542 |
+
Return :ref:`accuracy <accuracy_score>` on provided data and labels.
|
| 543 |
+
|
| 544 |
+
In multi-label classification, this is the subset accuracy
|
| 545 |
+
which is a harsh metric since you require for each sample that
|
| 546 |
+
each label set be correctly predicted.
|
| 547 |
+
|
| 548 |
+
Parameters
|
| 549 |
+
----------
|
| 550 |
+
X : array-like of shape (n_samples, n_features)
|
| 551 |
+
Test samples.
|
| 552 |
+
|
| 553 |
+
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
|
| 554 |
+
True labels for `X`.
|
| 555 |
+
|
| 556 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 557 |
+
Sample weights.
|
| 558 |
+
|
| 559 |
+
Returns
|
| 560 |
+
-------
|
| 561 |
+
score : float
|
| 562 |
+
Mean accuracy of ``self.predict(X)`` w.r.t. `y`.
|
| 563 |
+
"""
|
| 564 |
+
from sklearn.metrics import accuracy_score
|
| 565 |
+
|
| 566 |
+
return accuracy_score(y, self.predict(X), sample_weight=sample_weight)
|
| 567 |
+
|
| 568 |
+
|
| 569 |
+
class RegressorMixin:
|
| 570 |
+
"""Mixin class for all regression estimators in scikit-learn.
|
| 571 |
+
|
| 572 |
+
This mixin defines the following functionality:
|
| 573 |
+
|
| 574 |
+
- set estimator type to `"regressor"` through the `estimator_type` tag;
|
| 575 |
+
- `score` method that default to :func:`~sklearn.metrics.r2_score`.
|
| 576 |
+
- enforce that `fit` requires `y` to be passed through the `requires_y` tag,
|
| 577 |
+
which is done by setting the regressor type tag.
|
| 578 |
+
|
| 579 |
+
Read more in the :ref:`User Guide <rolling_your_own_estimator>`.
|
| 580 |
+
|
| 581 |
+
Examples
|
| 582 |
+
--------
|
| 583 |
+
>>> import numpy as np
|
| 584 |
+
>>> from sklearn.base import BaseEstimator, RegressorMixin
|
| 585 |
+
>>> # Mixin classes should always be on the left-hand side for a correct MRO
|
| 586 |
+
>>> class MyEstimator(RegressorMixin, BaseEstimator):
|
| 587 |
+
... def __init__(self, *, param=1):
|
| 588 |
+
... self.param = param
|
| 589 |
+
... def fit(self, X, y=None):
|
| 590 |
+
... self.is_fitted_ = True
|
| 591 |
+
... return self
|
| 592 |
+
... def predict(self, X):
|
| 593 |
+
... return np.full(shape=X.shape[0], fill_value=self.param)
|
| 594 |
+
>>> estimator = MyEstimator(param=0)
|
| 595 |
+
>>> X = np.array([[1, 2], [2, 3], [3, 4]])
|
| 596 |
+
>>> y = np.array([-1, 0, 1])
|
| 597 |
+
>>> estimator.fit(X, y).predict(X)
|
| 598 |
+
array([0, 0, 0])
|
| 599 |
+
>>> estimator.score(X, y)
|
| 600 |
+
0.0
|
| 601 |
+
"""
|
| 602 |
+
|
| 603 |
+
def __sklearn_tags__(self):
|
| 604 |
+
tags = super().__sklearn_tags__()
|
| 605 |
+
tags.estimator_type = "regressor"
|
| 606 |
+
tags.regressor_tags = RegressorTags()
|
| 607 |
+
tags.target_tags.required = True
|
| 608 |
+
return tags
|
| 609 |
+
|
| 610 |
+
def score(self, X, y, sample_weight=None):
|
| 611 |
+
"""Return :ref:`coefficient of determination <r2_score>` on test data.
|
| 612 |
+
|
| 613 |
+
The coefficient of determination, :math:`R^2`, is defined as
|
| 614 |
+
:math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual
|
| 615 |
+
sum of squares ``((y_true - y_pred)** 2).sum()`` and :math:`v`
|
| 616 |
+
is the total sum of squares ``((y_true - y_true.mean()) ** 2).sum()``.
|
| 617 |
+
The best possible score is 1.0 and it can be negative (because the
|
| 618 |
+
model can be arbitrarily worse). A constant model that always predicts
|
| 619 |
+
the expected value of `y`, disregarding the input features, would get
|
| 620 |
+
a :math:`R^2` score of 0.0.
|
| 621 |
+
|
| 622 |
+
Parameters
|
| 623 |
+
----------
|
| 624 |
+
X : array-like of shape (n_samples, n_features)
|
| 625 |
+
Test samples. For some estimators this may be a precomputed
|
| 626 |
+
kernel matrix or a list of generic objects instead with shape
|
| 627 |
+
``(n_samples, n_samples_fitted)``, where ``n_samples_fitted``
|
| 628 |
+
is the number of samples used in the fitting for the estimator.
|
| 629 |
+
|
| 630 |
+
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
|
| 631 |
+
True values for `X`.
|
| 632 |
+
|
| 633 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 634 |
+
Sample weights.
|
| 635 |
+
|
| 636 |
+
Returns
|
| 637 |
+
-------
|
| 638 |
+
score : float
|
| 639 |
+
:math:`R^2` of ``self.predict(X)`` w.r.t. `y`.
|
| 640 |
+
|
| 641 |
+
Notes
|
| 642 |
+
-----
|
| 643 |
+
The :math:`R^2` score used when calling ``score`` on a regressor uses
|
| 644 |
+
``multioutput='uniform_average'`` from version 0.23 to keep consistent
|
| 645 |
+
with default value of :func:`~sklearn.metrics.r2_score`.
|
| 646 |
+
This influences the ``score`` method of all the multioutput
|
| 647 |
+
regressors (except for
|
| 648 |
+
:class:`~sklearn.multioutput.MultiOutputRegressor`).
|
| 649 |
+
"""
|
| 650 |
+
|
| 651 |
+
from sklearn.metrics import r2_score
|
| 652 |
+
|
| 653 |
+
y_pred = self.predict(X)
|
| 654 |
+
return r2_score(y, y_pred, sample_weight=sample_weight)
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
class ClusterMixin:
|
| 658 |
+
"""Mixin class for all cluster estimators in scikit-learn.
|
| 659 |
+
|
| 660 |
+
- set estimator type to `"clusterer"` through the `estimator_type` tag;
|
| 661 |
+
- `fit_predict` method returning the cluster labels associated to each sample.
|
| 662 |
+
|
| 663 |
+
Examples
|
| 664 |
+
--------
|
| 665 |
+
>>> import numpy as np
|
| 666 |
+
>>> from sklearn.base import BaseEstimator, ClusterMixin
|
| 667 |
+
>>> class MyClusterer(ClusterMixin, BaseEstimator):
|
| 668 |
+
... def fit(self, X, y=None):
|
| 669 |
+
... self.labels_ = np.ones(shape=(len(X),), dtype=np.int64)
|
| 670 |
+
... return self
|
| 671 |
+
>>> X = [[1, 2], [2, 3], [3, 4]]
|
| 672 |
+
>>> MyClusterer().fit_predict(X)
|
| 673 |
+
array([1, 1, 1])
|
| 674 |
+
"""
|
| 675 |
+
|
| 676 |
+
def __sklearn_tags__(self):
|
| 677 |
+
tags = super().__sklearn_tags__()
|
| 678 |
+
tags.estimator_type = "clusterer"
|
| 679 |
+
if tags.transformer_tags is not None:
|
| 680 |
+
tags.transformer_tags.preserves_dtype = []
|
| 681 |
+
return tags
|
| 682 |
+
|
| 683 |
+
def fit_predict(self, X, y=None, **kwargs):
|
| 684 |
+
"""
|
| 685 |
+
Perform clustering on `X` and returns cluster labels.
|
| 686 |
+
|
| 687 |
+
Parameters
|
| 688 |
+
----------
|
| 689 |
+
X : array-like of shape (n_samples, n_features)
|
| 690 |
+
Input data.
|
| 691 |
+
|
| 692 |
+
y : Ignored
|
| 693 |
+
Not used, present for API consistency by convention.
|
| 694 |
+
|
| 695 |
+
**kwargs : dict
|
| 696 |
+
Arguments to be passed to ``fit``.
|
| 697 |
+
|
| 698 |
+
.. versionadded:: 1.4
|
| 699 |
+
|
| 700 |
+
Returns
|
| 701 |
+
-------
|
| 702 |
+
labels : ndarray of shape (n_samples,), dtype=np.int64
|
| 703 |
+
Cluster labels.
|
| 704 |
+
"""
|
| 705 |
+
# non-optimized default implementation; override when a better
|
| 706 |
+
# method is possible for a given clustering algorithm
|
| 707 |
+
self.fit(X, **kwargs)
|
| 708 |
+
return self.labels_
|
| 709 |
+
|
| 710 |
+
|
| 711 |
+
class BiclusterMixin:
|
| 712 |
+
"""Mixin class for all bicluster estimators in scikit-learn.
|
| 713 |
+
|
| 714 |
+
This mixin defines the following functionality:
|
| 715 |
+
|
| 716 |
+
- `biclusters_` property that returns the row and column indicators;
|
| 717 |
+
- `get_indices` method that returns the row and column indices of a bicluster;
|
| 718 |
+
- `get_shape` method that returns the shape of a bicluster;
|
| 719 |
+
- `get_submatrix` method that returns the submatrix corresponding to a bicluster.
|
| 720 |
+
|
| 721 |
+
Examples
|
| 722 |
+
--------
|
| 723 |
+
>>> import numpy as np
|
| 724 |
+
>>> from sklearn.base import BaseEstimator, BiclusterMixin
|
| 725 |
+
>>> class DummyBiClustering(BiclusterMixin, BaseEstimator):
|
| 726 |
+
... def fit(self, X, y=None):
|
| 727 |
+
... self.rows_ = np.ones(shape=(1, X.shape[0]), dtype=bool)
|
| 728 |
+
... self.columns_ = np.ones(shape=(1, X.shape[1]), dtype=bool)
|
| 729 |
+
... return self
|
| 730 |
+
>>> X = np.array([[1, 1], [2, 1], [1, 0],
|
| 731 |
+
... [4, 7], [3, 5], [3, 6]])
|
| 732 |
+
>>> bicluster = DummyBiClustering().fit(X)
|
| 733 |
+
>>> hasattr(bicluster, "biclusters_")
|
| 734 |
+
True
|
| 735 |
+
>>> bicluster.get_indices(0)
|
| 736 |
+
(array([0, 1, 2, 3, 4, 5]), array([0, 1]))
|
| 737 |
+
"""
|
| 738 |
+
|
| 739 |
+
@property
|
| 740 |
+
def biclusters_(self):
|
| 741 |
+
"""Convenient way to get row and column indicators together.
|
| 742 |
+
|
| 743 |
+
Returns the ``rows_`` and ``columns_`` members.
|
| 744 |
+
"""
|
| 745 |
+
return self.rows_, self.columns_
|
| 746 |
+
|
| 747 |
+
def get_indices(self, i):
|
| 748 |
+
"""Row and column indices of the `i`'th bicluster.
|
| 749 |
+
|
| 750 |
+
Only works if ``rows_`` and ``columns_`` attributes exist.
|
| 751 |
+
|
| 752 |
+
Parameters
|
| 753 |
+
----------
|
| 754 |
+
i : int
|
| 755 |
+
The index of the cluster.
|
| 756 |
+
|
| 757 |
+
Returns
|
| 758 |
+
-------
|
| 759 |
+
row_ind : ndarray, dtype=np.intp
|
| 760 |
+
Indices of rows in the dataset that belong to the bicluster.
|
| 761 |
+
col_ind : ndarray, dtype=np.intp
|
| 762 |
+
Indices of columns in the dataset that belong to the bicluster.
|
| 763 |
+
"""
|
| 764 |
+
rows = self.rows_[i]
|
| 765 |
+
columns = self.columns_[i]
|
| 766 |
+
return np.nonzero(rows)[0], np.nonzero(columns)[0]
|
| 767 |
+
|
| 768 |
+
def get_shape(self, i):
|
| 769 |
+
"""Shape of the `i`'th bicluster.
|
| 770 |
+
|
| 771 |
+
Parameters
|
| 772 |
+
----------
|
| 773 |
+
i : int
|
| 774 |
+
The index of the cluster.
|
| 775 |
+
|
| 776 |
+
Returns
|
| 777 |
+
-------
|
| 778 |
+
n_rows : int
|
| 779 |
+
Number of rows in the bicluster.
|
| 780 |
+
|
| 781 |
+
n_cols : int
|
| 782 |
+
Number of columns in the bicluster.
|
| 783 |
+
"""
|
| 784 |
+
indices = self.get_indices(i)
|
| 785 |
+
return tuple(len(i) for i in indices)
|
| 786 |
+
|
| 787 |
+
def get_submatrix(self, i, data):
|
| 788 |
+
"""Return the submatrix corresponding to bicluster `i`.
|
| 789 |
+
|
| 790 |
+
Parameters
|
| 791 |
+
----------
|
| 792 |
+
i : int
|
| 793 |
+
The index of the cluster.
|
| 794 |
+
data : array-like of shape (n_samples, n_features)
|
| 795 |
+
The data.
|
| 796 |
+
|
| 797 |
+
Returns
|
| 798 |
+
-------
|
| 799 |
+
submatrix : ndarray of shape (n_rows, n_cols)
|
| 800 |
+
The submatrix corresponding to bicluster `i`.
|
| 801 |
+
|
| 802 |
+
Notes
|
| 803 |
+
-----
|
| 804 |
+
Works with sparse matrices. Only works if ``rows_`` and
|
| 805 |
+
``columns_`` attributes exist.
|
| 806 |
+
"""
|
| 807 |
+
|
| 808 |
+
data = check_array(data, accept_sparse="csr")
|
| 809 |
+
row_ind, col_ind = self.get_indices(i)
|
| 810 |
+
return data[row_ind[:, np.newaxis], col_ind]
|
| 811 |
+
|
| 812 |
+
|
| 813 |
+
class TransformerMixin(_SetOutputMixin):
|
| 814 |
+
"""Mixin class for all transformers in scikit-learn.
|
| 815 |
+
|
| 816 |
+
This mixin defines the following functionality:
|
| 817 |
+
|
| 818 |
+
- a `fit_transform` method that delegates to `fit` and `transform`;
|
| 819 |
+
- a `set_output` method to output `X` as a specific container type.
|
| 820 |
+
|
| 821 |
+
If :term:`get_feature_names_out` is defined, then :class:`BaseEstimator` will
|
| 822 |
+
automatically wrap `transform` and `fit_transform` to follow the `set_output`
|
| 823 |
+
API. See the :ref:`developer_api_set_output` for details.
|
| 824 |
+
|
| 825 |
+
:class:`OneToOneFeatureMixin` and
|
| 826 |
+
:class:`ClassNamePrefixFeaturesOutMixin` are helpful mixins for
|
| 827 |
+
defining :term:`get_feature_names_out`.
|
| 828 |
+
|
| 829 |
+
Examples
|
| 830 |
+
--------
|
| 831 |
+
>>> import numpy as np
|
| 832 |
+
>>> from sklearn.base import BaseEstimator, TransformerMixin
|
| 833 |
+
>>> class MyTransformer(TransformerMixin, BaseEstimator):
|
| 834 |
+
... def __init__(self, *, param=1):
|
| 835 |
+
... self.param = param
|
| 836 |
+
... def fit(self, X, y=None):
|
| 837 |
+
... return self
|
| 838 |
+
... def transform(self, X):
|
| 839 |
+
... return np.full(shape=len(X), fill_value=self.param)
|
| 840 |
+
>>> transformer = MyTransformer()
|
| 841 |
+
>>> X = [[1, 2], [2, 3], [3, 4]]
|
| 842 |
+
>>> transformer.fit_transform(X)
|
| 843 |
+
array([1, 1, 1])
|
| 844 |
+
"""
|
| 845 |
+
|
| 846 |
+
def __sklearn_tags__(self):
|
| 847 |
+
tags = super().__sklearn_tags__()
|
| 848 |
+
tags.transformer_tags = TransformerTags()
|
| 849 |
+
return tags
|
| 850 |
+
|
| 851 |
+
def fit_transform(self, X, y=None, **fit_params):
|
| 852 |
+
"""
|
| 853 |
+
Fit to data, then transform it.
|
| 854 |
+
|
| 855 |
+
Fits transformer to `X` and `y` with optional parameters `fit_params`
|
| 856 |
+
and returns a transformed version of `X`.
|
| 857 |
+
|
| 858 |
+
Parameters
|
| 859 |
+
----------
|
| 860 |
+
X : array-like of shape (n_samples, n_features)
|
| 861 |
+
Input samples.
|
| 862 |
+
|
| 863 |
+
y : array-like of shape (n_samples,) or (n_samples, n_outputs), \
|
| 864 |
+
default=None
|
| 865 |
+
Target values (None for unsupervised transformations).
|
| 866 |
+
|
| 867 |
+
**fit_params : dict
|
| 868 |
+
Additional fit parameters.
|
| 869 |
+
Pass only if the estimator accepts additional params in its `fit` method.
|
| 870 |
+
|
| 871 |
+
Returns
|
| 872 |
+
-------
|
| 873 |
+
X_new : ndarray array of shape (n_samples, n_features_new)
|
| 874 |
+
Transformed array.
|
| 875 |
+
"""
|
| 876 |
+
# non-optimized default implementation; override when a better
|
| 877 |
+
# method is possible for a given clustering algorithm
|
| 878 |
+
|
| 879 |
+
# we do not route parameters here, since consumers don't route. But
|
| 880 |
+
# since it's possible for a `transform` method to also consume
|
| 881 |
+
# metadata, we check if that's the case, and we raise a warning telling
|
| 882 |
+
# users that they should implement a custom `fit_transform` method
|
| 883 |
+
# to forward metadata to `transform` as well.
|
| 884 |
+
#
|
| 885 |
+
# For that, we calculate routing and check if anything would be routed
|
| 886 |
+
# to `transform` if we were to route them.
|
| 887 |
+
if _routing_enabled():
|
| 888 |
+
transform_params = self.get_metadata_routing().consumes(
|
| 889 |
+
method="transform", params=fit_params.keys()
|
| 890 |
+
)
|
| 891 |
+
if transform_params:
|
| 892 |
+
warnings.warn(
|
| 893 |
+
(
|
| 894 |
+
f"This object ({self.__class__.__name__}) has a `transform`"
|
| 895 |
+
" method which consumes metadata, but `fit_transform` does not"
|
| 896 |
+
" forward metadata to `transform`. Please implement a custom"
|
| 897 |
+
" `fit_transform` method to forward metadata to `transform` as"
|
| 898 |
+
" well. Alternatively, you can explicitly do"
|
| 899 |
+
" `set_transform_request`and set all values to `False` to"
|
| 900 |
+
" disable metadata routed to `transform`, if that's an option."
|
| 901 |
+
),
|
| 902 |
+
UserWarning,
|
| 903 |
+
)
|
| 904 |
+
|
| 905 |
+
if y is None:
|
| 906 |
+
# fit method of arity 1 (unsupervised transformation)
|
| 907 |
+
return self.fit(X, **fit_params).transform(X)
|
| 908 |
+
else:
|
| 909 |
+
# fit method of arity 2 (supervised transformation)
|
| 910 |
+
return self.fit(X, y, **fit_params).transform(X)
|
| 911 |
+
|
| 912 |
+
|
| 913 |
+
class OneToOneFeatureMixin:
|
| 914 |
+
"""Provides `get_feature_names_out` for simple transformers.
|
| 915 |
+
|
| 916 |
+
This mixin assumes there's a 1-to-1 correspondence between input features
|
| 917 |
+
and output features, such as :class:`~sklearn.preprocessing.StandardScaler`.
|
| 918 |
+
|
| 919 |
+
Examples
|
| 920 |
+
--------
|
| 921 |
+
>>> import numpy as np
|
| 922 |
+
>>> from sklearn.base import OneToOneFeatureMixin, BaseEstimator
|
| 923 |
+
>>> class MyEstimator(OneToOneFeatureMixin, BaseEstimator):
|
| 924 |
+
... def fit(self, X, y=None):
|
| 925 |
+
... self.n_features_in_ = X.shape[1]
|
| 926 |
+
... return self
|
| 927 |
+
>>> X = np.array([[1, 2], [3, 4]])
|
| 928 |
+
>>> MyEstimator().fit(X).get_feature_names_out()
|
| 929 |
+
array(['x0', 'x1'], dtype=object)
|
| 930 |
+
"""
|
| 931 |
+
|
| 932 |
+
def get_feature_names_out(self, input_features=None):
|
| 933 |
+
"""Get output feature names for transformation.
|
| 934 |
+
|
| 935 |
+
Parameters
|
| 936 |
+
----------
|
| 937 |
+
input_features : array-like of str or None, default=None
|
| 938 |
+
Input features.
|
| 939 |
+
|
| 940 |
+
- If `input_features` is `None`, then `feature_names_in_` is
|
| 941 |
+
used as feature names in. If `feature_names_in_` is not defined,
|
| 942 |
+
then the following input feature names are generated:
|
| 943 |
+
`["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
|
| 944 |
+
- If `input_features` is an array-like, then `input_features` must
|
| 945 |
+
match `feature_names_in_` if `feature_names_in_` is defined.
|
| 946 |
+
|
| 947 |
+
Returns
|
| 948 |
+
-------
|
| 949 |
+
feature_names_out : ndarray of str objects
|
| 950 |
+
Same as input features.
|
| 951 |
+
"""
|
| 952 |
+
# Note that passing attributes="n_features_in_" forces check_is_fitted
|
| 953 |
+
# to check if the attribute is present. Otherwise it will pass on
|
| 954 |
+
# stateless estimators (requires_fit=False)
|
| 955 |
+
check_is_fitted(self, attributes="n_features_in_")
|
| 956 |
+
return _check_feature_names_in(self, input_features)
|
| 957 |
+
|
| 958 |
+
|
| 959 |
+
class ClassNamePrefixFeaturesOutMixin:
|
| 960 |
+
"""Mixin class for transformers that generate their own names by prefixing.
|
| 961 |
+
|
| 962 |
+
This mixin is useful when the transformer needs to generate its own feature
|
| 963 |
+
names out, such as :class:`~sklearn.decomposition.PCA`. For example, if
|
| 964 |
+
:class:`~sklearn.decomposition.PCA` outputs 3 features, then the generated feature
|
| 965 |
+
names out are: `["pca0", "pca1", "pca2"]`.
|
| 966 |
+
|
| 967 |
+
This mixin assumes that a `_n_features_out` attribute is defined when the
|
| 968 |
+
transformer is fitted. `_n_features_out` is the number of output features
|
| 969 |
+
that the transformer will return in `transform` of `fit_transform`.
|
| 970 |
+
|
| 971 |
+
Examples
|
| 972 |
+
--------
|
| 973 |
+
>>> import numpy as np
|
| 974 |
+
>>> from sklearn.base import ClassNamePrefixFeaturesOutMixin, BaseEstimator
|
| 975 |
+
>>> class MyEstimator(ClassNamePrefixFeaturesOutMixin, BaseEstimator):
|
| 976 |
+
... def fit(self, X, y=None):
|
| 977 |
+
... self._n_features_out = X.shape[1]
|
| 978 |
+
... return self
|
| 979 |
+
>>> X = np.array([[1, 2], [3, 4]])
|
| 980 |
+
>>> MyEstimator().fit(X).get_feature_names_out()
|
| 981 |
+
array(['myestimator0', 'myestimator1'], dtype=object)
|
| 982 |
+
"""
|
| 983 |
+
|
| 984 |
+
def get_feature_names_out(self, input_features=None):
|
| 985 |
+
"""Get output feature names for transformation.
|
| 986 |
+
|
| 987 |
+
The feature names out will prefixed by the lowercased class name. For
|
| 988 |
+
example, if the transformer outputs 3 features, then the feature names
|
| 989 |
+
out are: `["class_name0", "class_name1", "class_name2"]`.
|
| 990 |
+
|
| 991 |
+
Parameters
|
| 992 |
+
----------
|
| 993 |
+
input_features : array-like of str or None, default=None
|
| 994 |
+
Only used to validate feature names with the names seen in `fit`.
|
| 995 |
+
|
| 996 |
+
Returns
|
| 997 |
+
-------
|
| 998 |
+
feature_names_out : ndarray of str objects
|
| 999 |
+
Transformed feature names.
|
| 1000 |
+
"""
|
| 1001 |
+
check_is_fitted(self, "_n_features_out")
|
| 1002 |
+
return _generate_get_feature_names_out(
|
| 1003 |
+
self, self._n_features_out, input_features=input_features
|
| 1004 |
+
)
|
| 1005 |
+
|
| 1006 |
+
|
| 1007 |
+
class DensityMixin:
|
| 1008 |
+
"""Mixin class for all density estimators in scikit-learn.
|
| 1009 |
+
|
| 1010 |
+
This mixin defines the following functionality:
|
| 1011 |
+
|
| 1012 |
+
- sets estimator type to `"density_estimator"` through the `estimator_type` tag;
|
| 1013 |
+
- `score` method that default that do no-op.
|
| 1014 |
+
|
| 1015 |
+
Examples
|
| 1016 |
+
--------
|
| 1017 |
+
>>> from sklearn.base import DensityMixin
|
| 1018 |
+
>>> class MyEstimator(DensityMixin):
|
| 1019 |
+
... def fit(self, X, y=None):
|
| 1020 |
+
... self.is_fitted_ = True
|
| 1021 |
+
... return self
|
| 1022 |
+
>>> estimator = MyEstimator()
|
| 1023 |
+
>>> hasattr(estimator, "score")
|
| 1024 |
+
True
|
| 1025 |
+
"""
|
| 1026 |
+
|
| 1027 |
+
def __sklearn_tags__(self):
|
| 1028 |
+
tags = super().__sklearn_tags__()
|
| 1029 |
+
tags.estimator_type = "density_estimator"
|
| 1030 |
+
return tags
|
| 1031 |
+
|
| 1032 |
+
def score(self, X, y=None):
|
| 1033 |
+
"""Return the score of the model on the data `X`.
|
| 1034 |
+
|
| 1035 |
+
Parameters
|
| 1036 |
+
----------
|
| 1037 |
+
X : array-like of shape (n_samples, n_features)
|
| 1038 |
+
Test samples.
|
| 1039 |
+
|
| 1040 |
+
y : Ignored
|
| 1041 |
+
Not used, present for API consistency by convention.
|
| 1042 |
+
|
| 1043 |
+
Returns
|
| 1044 |
+
-------
|
| 1045 |
+
score : float
|
| 1046 |
+
"""
|
| 1047 |
+
pass
|
| 1048 |
+
|
| 1049 |
+
|
| 1050 |
+
class OutlierMixin:
|
| 1051 |
+
"""Mixin class for all outlier detection estimators in scikit-learn.
|
| 1052 |
+
|
| 1053 |
+
This mixin defines the following functionality:
|
| 1054 |
+
|
| 1055 |
+
- set estimator type to `"outlier_detector"` through the `estimator_type` tag;
|
| 1056 |
+
- `fit_predict` method that default to `fit` and `predict`.
|
| 1057 |
+
|
| 1058 |
+
Examples
|
| 1059 |
+
--------
|
| 1060 |
+
>>> import numpy as np
|
| 1061 |
+
>>> from sklearn.base import BaseEstimator, OutlierMixin
|
| 1062 |
+
>>> class MyEstimator(OutlierMixin):
|
| 1063 |
+
... def fit(self, X, y=None):
|
| 1064 |
+
... self.is_fitted_ = True
|
| 1065 |
+
... return self
|
| 1066 |
+
... def predict(self, X):
|
| 1067 |
+
... return np.ones(shape=len(X))
|
| 1068 |
+
>>> estimator = MyEstimator()
|
| 1069 |
+
>>> X = np.array([[1, 2], [2, 3], [3, 4]])
|
| 1070 |
+
>>> estimator.fit_predict(X)
|
| 1071 |
+
array([1., 1., 1.])
|
| 1072 |
+
"""
|
| 1073 |
+
|
| 1074 |
+
def __sklearn_tags__(self):
|
| 1075 |
+
tags = super().__sklearn_tags__()
|
| 1076 |
+
tags.estimator_type = "outlier_detector"
|
| 1077 |
+
return tags
|
| 1078 |
+
|
| 1079 |
+
def fit_predict(self, X, y=None, **kwargs):
|
| 1080 |
+
"""Perform fit on X and returns labels for X.
|
| 1081 |
+
|
| 1082 |
+
Returns -1 for outliers and 1 for inliers.
|
| 1083 |
+
|
| 1084 |
+
Parameters
|
| 1085 |
+
----------
|
| 1086 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 1087 |
+
The input samples.
|
| 1088 |
+
|
| 1089 |
+
y : Ignored
|
| 1090 |
+
Not used, present for API consistency by convention.
|
| 1091 |
+
|
| 1092 |
+
**kwargs : dict
|
| 1093 |
+
Arguments to be passed to ``fit``.
|
| 1094 |
+
|
| 1095 |
+
.. versionadded:: 1.4
|
| 1096 |
+
|
| 1097 |
+
Returns
|
| 1098 |
+
-------
|
| 1099 |
+
y : ndarray of shape (n_samples,)
|
| 1100 |
+
1 for inliers, -1 for outliers.
|
| 1101 |
+
"""
|
| 1102 |
+
# we do not route parameters here, since consumers don't route. But
|
| 1103 |
+
# since it's possible for a `predict` method to also consume
|
| 1104 |
+
# metadata, we check if that's the case, and we raise a warning telling
|
| 1105 |
+
# users that they should implement a custom `fit_predict` method
|
| 1106 |
+
# to forward metadata to `predict` as well.
|
| 1107 |
+
#
|
| 1108 |
+
# For that, we calculate routing and check if anything would be routed
|
| 1109 |
+
# to `predict` if we were to route them.
|
| 1110 |
+
if _routing_enabled():
|
| 1111 |
+
transform_params = self.get_metadata_routing().consumes(
|
| 1112 |
+
method="predict", params=kwargs.keys()
|
| 1113 |
+
)
|
| 1114 |
+
if transform_params:
|
| 1115 |
+
warnings.warn(
|
| 1116 |
+
(
|
| 1117 |
+
f"This object ({self.__class__.__name__}) has a `predict` "
|
| 1118 |
+
"method which consumes metadata, but `fit_predict` does not "
|
| 1119 |
+
"forward metadata to `predict`. Please implement a custom "
|
| 1120 |
+
"`fit_predict` method to forward metadata to `predict` as well."
|
| 1121 |
+
"Alternatively, you can explicitly do `set_predict_request`"
|
| 1122 |
+
"and set all values to `False` to disable metadata routed to "
|
| 1123 |
+
"`predict`, if that's an option."
|
| 1124 |
+
),
|
| 1125 |
+
UserWarning,
|
| 1126 |
+
)
|
| 1127 |
+
|
| 1128 |
+
# override for transductive outlier detectors like LocalOulierFactor
|
| 1129 |
+
return self.fit(X, **kwargs).predict(X)
|
| 1130 |
+
|
| 1131 |
+
|
| 1132 |
+
class MetaEstimatorMixin:
|
| 1133 |
+
"""Mixin class for all meta estimators in scikit-learn.
|
| 1134 |
+
|
| 1135 |
+
This mixin is empty, and only exists to indicate that the estimator is a
|
| 1136 |
+
meta-estimator.
|
| 1137 |
+
|
| 1138 |
+
.. versionchanged:: 1.6
|
| 1139 |
+
The `_required_parameters` is now removed and is unnecessary since tests are
|
| 1140 |
+
refactored and don't use this anymore.
|
| 1141 |
+
|
| 1142 |
+
Examples
|
| 1143 |
+
--------
|
| 1144 |
+
>>> from sklearn.base import MetaEstimatorMixin
|
| 1145 |
+
>>> from sklearn.datasets import load_iris
|
| 1146 |
+
>>> from sklearn.linear_model import LogisticRegression
|
| 1147 |
+
>>> class MyEstimator(MetaEstimatorMixin):
|
| 1148 |
+
... def __init__(self, *, estimator=None):
|
| 1149 |
+
... self.estimator = estimator
|
| 1150 |
+
... def fit(self, X, y=None):
|
| 1151 |
+
... if self.estimator is None:
|
| 1152 |
+
... self.estimator_ = LogisticRegression()
|
| 1153 |
+
... else:
|
| 1154 |
+
... self.estimator_ = self.estimator
|
| 1155 |
+
... return self
|
| 1156 |
+
>>> X, y = load_iris(return_X_y=True)
|
| 1157 |
+
>>> estimator = MyEstimator().fit(X, y)
|
| 1158 |
+
>>> estimator.estimator_
|
| 1159 |
+
LogisticRegression()
|
| 1160 |
+
"""
|
| 1161 |
+
|
| 1162 |
+
|
| 1163 |
+
class MultiOutputMixin:
|
| 1164 |
+
"""Mixin to mark estimators that support multioutput."""
|
| 1165 |
+
|
| 1166 |
+
def __sklearn_tags__(self):
|
| 1167 |
+
tags = super().__sklearn_tags__()
|
| 1168 |
+
tags.target_tags.multi_output = True
|
| 1169 |
+
return tags
|
| 1170 |
+
|
| 1171 |
+
|
| 1172 |
+
class _UnstableArchMixin:
|
| 1173 |
+
"""Mark estimators that are non-determinstic on 32bit or PowerPC"""
|
| 1174 |
+
|
| 1175 |
+
def __sklearn_tags__(self):
|
| 1176 |
+
tags = super().__sklearn_tags__()
|
| 1177 |
+
tags.non_deterministic = _IS_32BIT or platform.machine().startswith(
|
| 1178 |
+
("ppc", "powerpc")
|
| 1179 |
+
)
|
| 1180 |
+
return tags
|
| 1181 |
+
|
| 1182 |
+
|
| 1183 |
+
def is_classifier(estimator):
|
| 1184 |
+
"""Return True if the given estimator is (probably) a classifier.
|
| 1185 |
+
|
| 1186 |
+
Parameters
|
| 1187 |
+
----------
|
| 1188 |
+
estimator : estimator instance
|
| 1189 |
+
Estimator object to test.
|
| 1190 |
+
|
| 1191 |
+
Returns
|
| 1192 |
+
-------
|
| 1193 |
+
out : bool
|
| 1194 |
+
True if estimator is a classifier and False otherwise.
|
| 1195 |
+
|
| 1196 |
+
Examples
|
| 1197 |
+
--------
|
| 1198 |
+
>>> from sklearn.base import is_classifier
|
| 1199 |
+
>>> from sklearn.cluster import KMeans
|
| 1200 |
+
>>> from sklearn.svm import SVC, SVR
|
| 1201 |
+
>>> classifier = SVC()
|
| 1202 |
+
>>> regressor = SVR()
|
| 1203 |
+
>>> kmeans = KMeans()
|
| 1204 |
+
>>> is_classifier(classifier)
|
| 1205 |
+
True
|
| 1206 |
+
>>> is_classifier(regressor)
|
| 1207 |
+
False
|
| 1208 |
+
>>> is_classifier(kmeans)
|
| 1209 |
+
False
|
| 1210 |
+
"""
|
| 1211 |
+
return get_tags(estimator).estimator_type == "classifier"
|
| 1212 |
+
|
| 1213 |
+
|
| 1214 |
+
def is_regressor(estimator):
|
| 1215 |
+
"""Return True if the given estimator is (probably) a regressor.
|
| 1216 |
+
|
| 1217 |
+
Parameters
|
| 1218 |
+
----------
|
| 1219 |
+
estimator : estimator instance
|
| 1220 |
+
Estimator object to test.
|
| 1221 |
+
|
| 1222 |
+
Returns
|
| 1223 |
+
-------
|
| 1224 |
+
out : bool
|
| 1225 |
+
True if estimator is a regressor and False otherwise.
|
| 1226 |
+
|
| 1227 |
+
Examples
|
| 1228 |
+
--------
|
| 1229 |
+
>>> from sklearn.base import is_regressor
|
| 1230 |
+
>>> from sklearn.cluster import KMeans
|
| 1231 |
+
>>> from sklearn.svm import SVC, SVR
|
| 1232 |
+
>>> classifier = SVC()
|
| 1233 |
+
>>> regressor = SVR()
|
| 1234 |
+
>>> kmeans = KMeans()
|
| 1235 |
+
>>> is_regressor(classifier)
|
| 1236 |
+
False
|
| 1237 |
+
>>> is_regressor(regressor)
|
| 1238 |
+
True
|
| 1239 |
+
>>> is_regressor(kmeans)
|
| 1240 |
+
False
|
| 1241 |
+
"""
|
| 1242 |
+
return get_tags(estimator).estimator_type == "regressor"
|
| 1243 |
+
|
| 1244 |
+
|
| 1245 |
+
def is_clusterer(estimator):
|
| 1246 |
+
"""Return True if the given estimator is (probably) a clusterer.
|
| 1247 |
+
|
| 1248 |
+
.. versionadded:: 1.6
|
| 1249 |
+
|
| 1250 |
+
Parameters
|
| 1251 |
+
----------
|
| 1252 |
+
estimator : estimator instance
|
| 1253 |
+
Estimator object to test.
|
| 1254 |
+
|
| 1255 |
+
Returns
|
| 1256 |
+
-------
|
| 1257 |
+
out : bool
|
| 1258 |
+
True if estimator is a clusterer and False otherwise.
|
| 1259 |
+
|
| 1260 |
+
Examples
|
| 1261 |
+
--------
|
| 1262 |
+
>>> from sklearn.base import is_clusterer
|
| 1263 |
+
>>> from sklearn.cluster import KMeans
|
| 1264 |
+
>>> from sklearn.svm import SVC, SVR
|
| 1265 |
+
>>> classifier = SVC()
|
| 1266 |
+
>>> regressor = SVR()
|
| 1267 |
+
>>> kmeans = KMeans()
|
| 1268 |
+
>>> is_clusterer(classifier)
|
| 1269 |
+
False
|
| 1270 |
+
>>> is_clusterer(regressor)
|
| 1271 |
+
False
|
| 1272 |
+
>>> is_clusterer(kmeans)
|
| 1273 |
+
True
|
| 1274 |
+
"""
|
| 1275 |
+
return get_tags(estimator).estimator_type == "clusterer"
|
| 1276 |
+
|
| 1277 |
+
|
| 1278 |
+
def is_outlier_detector(estimator):
|
| 1279 |
+
"""Return True if the given estimator is (probably) an outlier detector.
|
| 1280 |
+
|
| 1281 |
+
Parameters
|
| 1282 |
+
----------
|
| 1283 |
+
estimator : estimator instance
|
| 1284 |
+
Estimator object to test.
|
| 1285 |
+
|
| 1286 |
+
Returns
|
| 1287 |
+
-------
|
| 1288 |
+
out : bool
|
| 1289 |
+
True if estimator is an outlier detector and False otherwise.
|
| 1290 |
+
"""
|
| 1291 |
+
return get_tags(estimator).estimator_type == "outlier_detector"
|
| 1292 |
+
|
| 1293 |
+
|
| 1294 |
+
def _fit_context(*, prefer_skip_nested_validation):
|
| 1295 |
+
"""Decorator to run the fit methods of estimators within context managers.
|
| 1296 |
+
|
| 1297 |
+
Parameters
|
| 1298 |
+
----------
|
| 1299 |
+
prefer_skip_nested_validation : bool
|
| 1300 |
+
If True, the validation of parameters of inner estimators or functions
|
| 1301 |
+
called during fit will be skipped.
|
| 1302 |
+
|
| 1303 |
+
This is useful to avoid validating many times the parameters passed by the
|
| 1304 |
+
user from the public facing API. It's also useful to avoid validating
|
| 1305 |
+
parameters that we pass internally to inner functions that are guaranteed to
|
| 1306 |
+
be valid by the test suite.
|
| 1307 |
+
|
| 1308 |
+
It should be set to True for most estimators, except for those that receive
|
| 1309 |
+
non-validated objects as parameters, such as meta-estimators that are given
|
| 1310 |
+
estimator objects.
|
| 1311 |
+
|
| 1312 |
+
Returns
|
| 1313 |
+
-------
|
| 1314 |
+
decorated_fit : method
|
| 1315 |
+
The decorated fit method.
|
| 1316 |
+
"""
|
| 1317 |
+
|
| 1318 |
+
def decorator(fit_method):
|
| 1319 |
+
@functools.wraps(fit_method)
|
| 1320 |
+
def wrapper(estimator, *args, **kwargs):
|
| 1321 |
+
global_skip_validation = get_config()["skip_parameter_validation"]
|
| 1322 |
+
|
| 1323 |
+
# we don't want to validate again for each call to partial_fit
|
| 1324 |
+
partial_fit_and_fitted = (
|
| 1325 |
+
fit_method.__name__ == "partial_fit" and _is_fitted(estimator)
|
| 1326 |
+
)
|
| 1327 |
+
|
| 1328 |
+
if not global_skip_validation and not partial_fit_and_fitted:
|
| 1329 |
+
estimator._validate_params()
|
| 1330 |
+
|
| 1331 |
+
with config_context(
|
| 1332 |
+
skip_parameter_validation=(
|
| 1333 |
+
prefer_skip_nested_validation or global_skip_validation
|
| 1334 |
+
)
|
| 1335 |
+
):
|
| 1336 |
+
return fit_method(estimator, *args, **kwargs)
|
| 1337 |
+
|
| 1338 |
+
return wrapper
|
| 1339 |
+
|
| 1340 |
+
return decorator
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/calibration.py
ADDED
|
@@ -0,0 +1,1709 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Methods for calibrating predicted probabilities."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
import warnings
|
| 7 |
+
from functools import partial
|
| 8 |
+
from inspect import signature
|
| 9 |
+
from math import log
|
| 10 |
+
from numbers import Integral, Real
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
from scipy.optimize import minimize, minimize_scalar
|
| 14 |
+
from scipy.special import expit
|
| 15 |
+
|
| 16 |
+
from sklearn._loss import HalfBinomialLoss, HalfMultinomialLoss
|
| 17 |
+
from sklearn.base import (
|
| 18 |
+
BaseEstimator,
|
| 19 |
+
ClassifierMixin,
|
| 20 |
+
MetaEstimatorMixin,
|
| 21 |
+
RegressorMixin,
|
| 22 |
+
_fit_context,
|
| 23 |
+
clone,
|
| 24 |
+
)
|
| 25 |
+
from sklearn.externals import array_api_extra as xpx
|
| 26 |
+
from sklearn.frozen import FrozenEstimator
|
| 27 |
+
from sklearn.isotonic import IsotonicRegression
|
| 28 |
+
from sklearn.model_selection import LeaveOneOut, check_cv, cross_val_predict
|
| 29 |
+
from sklearn.preprocessing import LabelEncoder, label_binarize
|
| 30 |
+
from sklearn.svm import LinearSVC
|
| 31 |
+
from sklearn.utils import Bunch, _safe_indexing, column_or_1d, get_tags, indexable
|
| 32 |
+
from sklearn.utils._array_api import (
|
| 33 |
+
_convert_to_numpy,
|
| 34 |
+
_half_multinomial_loss,
|
| 35 |
+
_is_numpy_namespace,
|
| 36 |
+
get_namespace,
|
| 37 |
+
get_namespace_and_device,
|
| 38 |
+
move_to,
|
| 39 |
+
)
|
| 40 |
+
from sklearn.utils._param_validation import (
|
| 41 |
+
HasMethods,
|
| 42 |
+
Interval,
|
| 43 |
+
StrOptions,
|
| 44 |
+
validate_params,
|
| 45 |
+
)
|
| 46 |
+
from sklearn.utils._plotting import (
|
| 47 |
+
_BinaryClassifierCurveDisplayMixin,
|
| 48 |
+
_validate_style_kwargs,
|
| 49 |
+
)
|
| 50 |
+
from sklearn.utils._response import _get_response_values, _process_predict_proba
|
| 51 |
+
from sklearn.utils.extmath import softmax
|
| 52 |
+
from sklearn.utils.metadata_routing import (
|
| 53 |
+
MetadataRouter,
|
| 54 |
+
MethodMapping,
|
| 55 |
+
_routing_enabled,
|
| 56 |
+
process_routing,
|
| 57 |
+
)
|
| 58 |
+
from sklearn.utils.multiclass import check_classification_targets
|
| 59 |
+
from sklearn.utils.parallel import Parallel, delayed
|
| 60 |
+
from sklearn.utils.validation import (
|
| 61 |
+
_check_method_params,
|
| 62 |
+
_check_pos_label_consistency,
|
| 63 |
+
_check_response_method,
|
| 64 |
+
_check_sample_weight,
|
| 65 |
+
_num_samples,
|
| 66 |
+
check_array,
|
| 67 |
+
check_consistent_length,
|
| 68 |
+
check_is_fitted,
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
class CalibratedClassifierCV(ClassifierMixin, MetaEstimatorMixin, BaseEstimator):
|
| 73 |
+
"""Calibrate probabilities using isotonic, sigmoid, or temperature scaling.
|
| 74 |
+
|
| 75 |
+
This class uses cross-validation to both estimate the parameters of a
|
| 76 |
+
classifier and subsequently calibrate a classifier. With
|
| 77 |
+
`ensemble=True`, for each cv split it
|
| 78 |
+
fits a copy of the base estimator to the training subset, and calibrates it
|
| 79 |
+
using the testing subset. For prediction, predicted probabilities are
|
| 80 |
+
averaged across these individual calibrated classifiers. When
|
| 81 |
+
`ensemble=False`, cross-validation is used to obtain unbiased predictions,
|
| 82 |
+
via :func:`~sklearn.model_selection.cross_val_predict`, which are then
|
| 83 |
+
used for calibration. For prediction, the base estimator, trained using all
|
| 84 |
+
the data, is used. This is the prediction method implemented when
|
| 85 |
+
`probabilities=True` for :class:`~sklearn.svm.SVC` and :class:`~sklearn.svm.NuSVC`
|
| 86 |
+
estimators (see :ref:`User Guide <scores_probabilities>` for details).
|
| 87 |
+
|
| 88 |
+
Already fitted classifiers can be calibrated by wrapping the model in a
|
| 89 |
+
:class:`~sklearn.frozen.FrozenEstimator`. In this case all provided
|
| 90 |
+
data is used for calibration. The user has to take care manually that data
|
| 91 |
+
for model fitting and calibration are disjoint.
|
| 92 |
+
|
| 93 |
+
The calibration is based on the :term:`decision_function` method of the
|
| 94 |
+
`estimator` if it exists, else on :term:`predict_proba`.
|
| 95 |
+
|
| 96 |
+
Read more in the :ref:`User Guide <calibration>`.
|
| 97 |
+
In order to learn more on the CalibratedClassifierCV class, see the
|
| 98 |
+
following calibration examples:
|
| 99 |
+
:ref:`sphx_glr_auto_examples_calibration_plot_calibration.py`,
|
| 100 |
+
:ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py`, and
|
| 101 |
+
:ref:`sphx_glr_auto_examples_calibration_plot_calibration_multiclass.py`.
|
| 102 |
+
|
| 103 |
+
Parameters
|
| 104 |
+
----------
|
| 105 |
+
estimator : estimator instance, default=None
|
| 106 |
+
The classifier whose output need to be calibrated to provide more
|
| 107 |
+
accurate `predict_proba` outputs. The default classifier is
|
| 108 |
+
a :class:`~sklearn.svm.LinearSVC`.
|
| 109 |
+
|
| 110 |
+
.. versionadded:: 1.2
|
| 111 |
+
|
| 112 |
+
method : {'sigmoid', 'isotonic', 'temperature'}, default='sigmoid'
|
| 113 |
+
The method to use for calibration. Can be:
|
| 114 |
+
|
| 115 |
+
- 'sigmoid', which corresponds to Platt's method (i.e. a binary logistic
|
| 116 |
+
regression model).
|
| 117 |
+
- 'isotonic', which is a non-parametric approach.
|
| 118 |
+
- 'temperature', temperature scaling.
|
| 119 |
+
|
| 120 |
+
Sigmoid and isotonic calibration methods natively support only binary
|
| 121 |
+
classifiers and extend to multi-class classification using a One-vs-Rest (OvR)
|
| 122 |
+
strategy with post-hoc renormalization, i.e., adjusting the probabilities after
|
| 123 |
+
calibration to ensure they sum up to 1.
|
| 124 |
+
|
| 125 |
+
In contrast, temperature scaling naturally supports multi-class calibration by
|
| 126 |
+
applying `softmax(classifier_logits/T)` with a value of `T` (temperature)
|
| 127 |
+
that optimizes the log loss.
|
| 128 |
+
|
| 129 |
+
For very uncalibrated classifiers on very imbalanced datasets, sigmoid
|
| 130 |
+
calibration might be preferred because it fits an additional intercept
|
| 131 |
+
parameter. This helps shift decision boundaries appropriately when the
|
| 132 |
+
classifier being calibrated is biased towards the majority class.
|
| 133 |
+
|
| 134 |
+
Isotonic calibration is not recommended when the number of calibration samples
|
| 135 |
+
is too low ``(≪1000)`` since it then tends to overfit.
|
| 136 |
+
|
| 137 |
+
.. versionchanged:: 1.8
|
| 138 |
+
Added option 'temperature'.
|
| 139 |
+
|
| 140 |
+
cv : int, cross-validation generator, or iterable, default=None
|
| 141 |
+
Determines the cross-validation splitting strategy.
|
| 142 |
+
Possible inputs for cv are:
|
| 143 |
+
|
| 144 |
+
- None, to use the default 5-fold cross-validation,
|
| 145 |
+
- integer, to specify the number of folds.
|
| 146 |
+
- :term:`CV splitter`,
|
| 147 |
+
- An iterable yielding (train, test) splits as arrays of indices.
|
| 148 |
+
|
| 149 |
+
For integer/None inputs, if ``y`` is binary or multiclass,
|
| 150 |
+
:class:`~sklearn.model_selection.StratifiedKFold` is used. If ``y`` is
|
| 151 |
+
neither binary nor multiclass, :class:`~sklearn.model_selection.KFold`
|
| 152 |
+
is used.
|
| 153 |
+
|
| 154 |
+
Refer to the :ref:`User Guide <cross_validation>` for the various
|
| 155 |
+
cross-validation strategies that can be used here.
|
| 156 |
+
|
| 157 |
+
.. versionchanged:: 0.22
|
| 158 |
+
``cv`` default value if None changed from 3-fold to 5-fold.
|
| 159 |
+
|
| 160 |
+
n_jobs : int, default=None
|
| 161 |
+
Number of jobs to run in parallel.
|
| 162 |
+
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
|
| 163 |
+
``-1`` means using all processors.
|
| 164 |
+
|
| 165 |
+
Base estimator clones are fitted in parallel across cross-validation
|
| 166 |
+
iterations.
|
| 167 |
+
|
| 168 |
+
See :term:`Glossary <n_jobs>` for more details.
|
| 169 |
+
|
| 170 |
+
.. versionadded:: 0.24
|
| 171 |
+
|
| 172 |
+
ensemble : bool, or "auto", default="auto"
|
| 173 |
+
Determines how the calibrator is fitted.
|
| 174 |
+
|
| 175 |
+
"auto" will use `False` if the `estimator` is a
|
| 176 |
+
:class:`~sklearn.frozen.FrozenEstimator`, and `True` otherwise.
|
| 177 |
+
|
| 178 |
+
If `True`, the `estimator` is fitted using training data, and
|
| 179 |
+
calibrated using testing data, for each `cv` fold. The final estimator
|
| 180 |
+
is an ensemble of `n_cv` fitted classifier and calibrator pairs, where
|
| 181 |
+
`n_cv` is the number of cross-validation folds. The output is the
|
| 182 |
+
average predicted probabilities of all pairs.
|
| 183 |
+
|
| 184 |
+
If `False`, `cv` is used to compute unbiased predictions, via
|
| 185 |
+
:func:`~sklearn.model_selection.cross_val_predict`, which are then
|
| 186 |
+
used for calibration. At prediction time, the classifier used is the
|
| 187 |
+
`estimator` trained on all the data.
|
| 188 |
+
Note that this method is also internally implemented in
|
| 189 |
+
:mod:`sklearn.svm` estimators with the `probabilities=True` parameter.
|
| 190 |
+
|
| 191 |
+
.. versionadded:: 0.24
|
| 192 |
+
|
| 193 |
+
.. versionchanged:: 1.6
|
| 194 |
+
`"auto"` option is added and is the default.
|
| 195 |
+
|
| 196 |
+
Attributes
|
| 197 |
+
----------
|
| 198 |
+
classes_ : ndarray of shape (n_classes,)
|
| 199 |
+
The class labels.
|
| 200 |
+
|
| 201 |
+
n_features_in_ : int
|
| 202 |
+
Number of features seen during :term:`fit`. Only defined if the
|
| 203 |
+
underlying estimator exposes such an attribute when fit.
|
| 204 |
+
|
| 205 |
+
.. versionadded:: 0.24
|
| 206 |
+
|
| 207 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 208 |
+
Names of features seen during :term:`fit`. Only defined if the
|
| 209 |
+
underlying estimator exposes such an attribute when fit.
|
| 210 |
+
|
| 211 |
+
.. versionadded:: 1.0
|
| 212 |
+
|
| 213 |
+
calibrated_classifiers_ : list (len() equal to cv or 1 if `ensemble=False`)
|
| 214 |
+
The list of classifier and calibrator pairs.
|
| 215 |
+
|
| 216 |
+
- When `ensemble=True`, `n_cv` fitted `estimator` and calibrator pairs.
|
| 217 |
+
`n_cv` is the number of cross-validation folds.
|
| 218 |
+
- When `ensemble=False`, the `estimator`, fitted on all the data, and fitted
|
| 219 |
+
calibrator.
|
| 220 |
+
|
| 221 |
+
.. versionchanged:: 0.24
|
| 222 |
+
Single calibrated classifier case when `ensemble=False`.
|
| 223 |
+
|
| 224 |
+
See Also
|
| 225 |
+
--------
|
| 226 |
+
calibration_curve : Compute true and predicted probabilities
|
| 227 |
+
for a calibration curve.
|
| 228 |
+
|
| 229 |
+
References
|
| 230 |
+
----------
|
| 231 |
+
.. [1] B. Zadrozny & C. Elkan.
|
| 232 |
+
`Obtaining calibrated probability estimates from decision trees
|
| 233 |
+
and naive Bayesian classifiers
|
| 234 |
+
<https://cseweb.ucsd.edu/~elkan/calibrated.pdf>`_, ICML 2001.
|
| 235 |
+
|
| 236 |
+
.. [2] B. Zadrozny & C. Elkan.
|
| 237 |
+
`Transforming Classifier Scores into Accurate Multiclass
|
| 238 |
+
Probability Estimates
|
| 239 |
+
<https://web.archive.org/web/20060720141520id_/http://www.research.ibm.com:80/people/z/zadrozny/kdd2002-Transf.pdf>`_,
|
| 240 |
+
KDD 2002.
|
| 241 |
+
|
| 242 |
+
.. [3] J. Platt. `Probabilistic Outputs for Support Vector Machines
|
| 243 |
+
and Comparisons to Regularized Likelihood Methods
|
| 244 |
+
<https://www.researchgate.net/profile/John-Platt-2/publication/2594015_Probabilistic_Outputs_for_Support_Vector_Machines_and_Comparisons_to_Regularized_Likelihood_Methods/links/004635154cff5262d6000000/Probabilistic-Outputs-for-Support-Vector-Machines-and-Comparisons-to-Regularized-Likelihood-Methods.pdf>`_,
|
| 245 |
+
1999.
|
| 246 |
+
|
| 247 |
+
.. [4] A. Niculescu-Mizil & R. Caruana.
|
| 248 |
+
`Predicting Good Probabilities with Supervised Learning
|
| 249 |
+
<https://www.cs.cornell.edu/~alexn/papers/calibration.icml05.crc.rev3.pdf>`_,
|
| 250 |
+
ICML 2005.
|
| 251 |
+
|
| 252 |
+
.. [5] Chuan Guo, Geoff Pleiss, Yu Sun, Kilian Q. Weinberger.
|
| 253 |
+
:doi:`On Calibration of Modern Neural Networks<10.48550/arXiv.1706.04599>`.
|
| 254 |
+
Proceedings of the 34th International Conference on Machine Learning,
|
| 255 |
+
PMLR 70:1321-1330, 2017.
|
| 256 |
+
|
| 257 |
+
Examples
|
| 258 |
+
--------
|
| 259 |
+
>>> from sklearn.datasets import make_classification
|
| 260 |
+
>>> from sklearn.naive_bayes import GaussianNB
|
| 261 |
+
>>> from sklearn.calibration import CalibratedClassifierCV
|
| 262 |
+
>>> X, y = make_classification(n_samples=100, n_features=2,
|
| 263 |
+
... n_redundant=0, random_state=42)
|
| 264 |
+
>>> base_clf = GaussianNB()
|
| 265 |
+
>>> calibrated_clf = CalibratedClassifierCV(base_clf, cv=3)
|
| 266 |
+
>>> calibrated_clf.fit(X, y)
|
| 267 |
+
CalibratedClassifierCV(...)
|
| 268 |
+
>>> len(calibrated_clf.calibrated_classifiers_)
|
| 269 |
+
3
|
| 270 |
+
>>> calibrated_clf.predict_proba(X)[:5, :]
|
| 271 |
+
array([[0.110, 0.889],
|
| 272 |
+
[0.072, 0.927],
|
| 273 |
+
[0.928, 0.072],
|
| 274 |
+
[0.928, 0.072],
|
| 275 |
+
[0.072, 0.928]])
|
| 276 |
+
>>> from sklearn.model_selection import train_test_split
|
| 277 |
+
>>> X, y = make_classification(n_samples=100, n_features=2,
|
| 278 |
+
... n_redundant=0, random_state=42)
|
| 279 |
+
>>> X_train, X_calib, y_train, y_calib = train_test_split(
|
| 280 |
+
... X, y, random_state=42
|
| 281 |
+
... )
|
| 282 |
+
>>> base_clf = GaussianNB()
|
| 283 |
+
>>> base_clf.fit(X_train, y_train)
|
| 284 |
+
GaussianNB()
|
| 285 |
+
>>> from sklearn.frozen import FrozenEstimator
|
| 286 |
+
>>> calibrated_clf = CalibratedClassifierCV(FrozenEstimator(base_clf))
|
| 287 |
+
>>> calibrated_clf.fit(X_calib, y_calib)
|
| 288 |
+
CalibratedClassifierCV(...)
|
| 289 |
+
>>> len(calibrated_clf.calibrated_classifiers_)
|
| 290 |
+
1
|
| 291 |
+
>>> calibrated_clf.predict_proba([[-0.5, 0.5]])
|
| 292 |
+
array([[0.936, 0.063]])
|
| 293 |
+
"""
|
| 294 |
+
|
| 295 |
+
_parameter_constraints: dict = {
|
| 296 |
+
"estimator": [
|
| 297 |
+
HasMethods(["fit", "predict_proba"]),
|
| 298 |
+
HasMethods(["fit", "decision_function"]),
|
| 299 |
+
None,
|
| 300 |
+
],
|
| 301 |
+
"method": [StrOptions({"isotonic", "sigmoid", "temperature"})],
|
| 302 |
+
"cv": ["cv_object"],
|
| 303 |
+
"n_jobs": [Integral, None],
|
| 304 |
+
"ensemble": ["boolean", StrOptions({"auto"})],
|
| 305 |
+
}
|
| 306 |
+
|
| 307 |
+
def __init__(
|
| 308 |
+
self,
|
| 309 |
+
estimator=None,
|
| 310 |
+
*,
|
| 311 |
+
method="sigmoid",
|
| 312 |
+
cv=None,
|
| 313 |
+
n_jobs=None,
|
| 314 |
+
ensemble="auto",
|
| 315 |
+
):
|
| 316 |
+
self.estimator = estimator
|
| 317 |
+
self.method = method
|
| 318 |
+
self.cv = cv
|
| 319 |
+
self.n_jobs = n_jobs
|
| 320 |
+
self.ensemble = ensemble
|
| 321 |
+
|
| 322 |
+
def _get_estimator(self):
|
| 323 |
+
"""Resolve which estimator to return (default is LinearSVC)"""
|
| 324 |
+
if self.estimator is None:
|
| 325 |
+
# we want all classifiers that don't expose a random_state
|
| 326 |
+
# to be deterministic (and we don't want to expose this one).
|
| 327 |
+
estimator = LinearSVC(random_state=0)
|
| 328 |
+
if _routing_enabled():
|
| 329 |
+
estimator.set_fit_request(sample_weight=True)
|
| 330 |
+
else:
|
| 331 |
+
estimator = self.estimator
|
| 332 |
+
|
| 333 |
+
return estimator
|
| 334 |
+
|
| 335 |
+
@_fit_context(
|
| 336 |
+
# CalibratedClassifierCV.estimator is not validated yet
|
| 337 |
+
prefer_skip_nested_validation=False
|
| 338 |
+
)
|
| 339 |
+
def fit(self, X, y, sample_weight=None, **fit_params):
|
| 340 |
+
"""Fit the calibrated model.
|
| 341 |
+
|
| 342 |
+
Parameters
|
| 343 |
+
----------
|
| 344 |
+
X : array-like of shape (n_samples, n_features)
|
| 345 |
+
Training data.
|
| 346 |
+
|
| 347 |
+
y : array-like of shape (n_samples,)
|
| 348 |
+
Target values.
|
| 349 |
+
|
| 350 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 351 |
+
Sample weights. If None, then samples are equally weighted.
|
| 352 |
+
|
| 353 |
+
**fit_params : dict
|
| 354 |
+
Parameters to pass to the `fit` method of the underlying
|
| 355 |
+
classifier.
|
| 356 |
+
|
| 357 |
+
Returns
|
| 358 |
+
-------
|
| 359 |
+
self : object
|
| 360 |
+
Returns an instance of self.
|
| 361 |
+
"""
|
| 362 |
+
check_classification_targets(y)
|
| 363 |
+
X, y = indexable(X, y)
|
| 364 |
+
estimator = self._get_estimator()
|
| 365 |
+
|
| 366 |
+
_ensemble = self.ensemble
|
| 367 |
+
if _ensemble == "auto":
|
| 368 |
+
_ensemble = not isinstance(estimator, FrozenEstimator)
|
| 369 |
+
|
| 370 |
+
self.calibrated_classifiers_ = []
|
| 371 |
+
|
| 372 |
+
# Set `classes_` using all `y`
|
| 373 |
+
label_encoder_ = LabelEncoder().fit(y)
|
| 374 |
+
self.classes_ = label_encoder_.classes_
|
| 375 |
+
if self.method == "temperature" and isinstance(y[0], str):
|
| 376 |
+
# for temperature scaling if `y` contains strings then encode it
|
| 377 |
+
# right here to avoid fitting LabelEncoder again within the
|
| 378 |
+
# `_fit_calibrator` function.
|
| 379 |
+
y = label_encoder_.transform(y=y)
|
| 380 |
+
|
| 381 |
+
if _routing_enabled():
|
| 382 |
+
routed_params = process_routing(
|
| 383 |
+
self,
|
| 384 |
+
"fit",
|
| 385 |
+
sample_weight=sample_weight,
|
| 386 |
+
**fit_params,
|
| 387 |
+
)
|
| 388 |
+
else:
|
| 389 |
+
# sample_weight checks
|
| 390 |
+
fit_parameters = signature(estimator.fit).parameters
|
| 391 |
+
supports_sw = "sample_weight" in fit_parameters
|
| 392 |
+
if sample_weight is not None and not supports_sw:
|
| 393 |
+
estimator_name = type(estimator).__name__
|
| 394 |
+
warnings.warn(
|
| 395 |
+
f"Since {estimator_name} does not appear to accept"
|
| 396 |
+
" sample_weight, sample weights will only be used for the"
|
| 397 |
+
" calibration itself. This can be caused by a limitation of"
|
| 398 |
+
" the current scikit-learn API. See the following issue for"
|
| 399 |
+
" more details:"
|
| 400 |
+
" https://github.com/scikit-learn/scikit-learn/issues/21134."
|
| 401 |
+
" Be warned that the result of the calibration is likely to be"
|
| 402 |
+
" incorrect."
|
| 403 |
+
)
|
| 404 |
+
routed_params = Bunch()
|
| 405 |
+
routed_params.splitter = Bunch(split={}) # no routing for splitter
|
| 406 |
+
routed_params.estimator = Bunch(fit=fit_params)
|
| 407 |
+
if sample_weight is not None and supports_sw:
|
| 408 |
+
routed_params.estimator.fit["sample_weight"] = sample_weight
|
| 409 |
+
|
| 410 |
+
xp, is_array_api, device_ = get_namespace_and_device(X)
|
| 411 |
+
if is_array_api:
|
| 412 |
+
y, sample_weight = move_to(y, sample_weight, xp=xp, device=device_)
|
| 413 |
+
# Check that each cross-validation fold can have at least one
|
| 414 |
+
# example per class
|
| 415 |
+
if isinstance(self.cv, int):
|
| 416 |
+
n_folds = self.cv
|
| 417 |
+
elif hasattr(self.cv, "n_splits"):
|
| 418 |
+
n_folds = self.cv.n_splits
|
| 419 |
+
else:
|
| 420 |
+
n_folds = None
|
| 421 |
+
if n_folds and xp.any(xp.unique_counts(y)[1] < n_folds):
|
| 422 |
+
raise ValueError(
|
| 423 |
+
f"Requesting {n_folds}-fold "
|
| 424 |
+
"cross-validation but provided less than "
|
| 425 |
+
f"{n_folds} examples for at least one class."
|
| 426 |
+
)
|
| 427 |
+
if isinstance(self.cv, LeaveOneOut):
|
| 428 |
+
raise ValueError(
|
| 429 |
+
"LeaveOneOut cross-validation does not allow"
|
| 430 |
+
"all classes to be present in test splits. "
|
| 431 |
+
"Please use a cross-validation generator that allows "
|
| 432 |
+
"all classes to appear in every test and train split."
|
| 433 |
+
)
|
| 434 |
+
cv = check_cv(self.cv, y, classifier=True)
|
| 435 |
+
|
| 436 |
+
if _ensemble:
|
| 437 |
+
parallel = Parallel(n_jobs=self.n_jobs)
|
| 438 |
+
self.calibrated_classifiers_ = parallel(
|
| 439 |
+
delayed(_fit_classifier_calibrator_pair)(
|
| 440 |
+
clone(estimator),
|
| 441 |
+
X,
|
| 442 |
+
y,
|
| 443 |
+
train=train,
|
| 444 |
+
test=test,
|
| 445 |
+
method=self.method,
|
| 446 |
+
classes=self.classes_,
|
| 447 |
+
xp=xp,
|
| 448 |
+
sample_weight=sample_weight,
|
| 449 |
+
fit_params=routed_params.estimator.fit,
|
| 450 |
+
)
|
| 451 |
+
for train, test in cv.split(X, y, **routed_params.splitter.split)
|
| 452 |
+
)
|
| 453 |
+
else:
|
| 454 |
+
this_estimator = clone(estimator)
|
| 455 |
+
method_name = _check_response_method(
|
| 456 |
+
this_estimator,
|
| 457 |
+
["decision_function", "predict_proba"],
|
| 458 |
+
).__name__
|
| 459 |
+
predictions = cross_val_predict(
|
| 460 |
+
estimator=this_estimator,
|
| 461 |
+
X=X,
|
| 462 |
+
y=y,
|
| 463 |
+
cv=cv,
|
| 464 |
+
method=method_name,
|
| 465 |
+
n_jobs=self.n_jobs,
|
| 466 |
+
params=routed_params.estimator.fit,
|
| 467 |
+
)
|
| 468 |
+
if self.classes_.shape[0] == 2:
|
| 469 |
+
# Ensure shape (n_samples, 1) in the binary case
|
| 470 |
+
if method_name == "predict_proba":
|
| 471 |
+
# Select the probability column of the positive class
|
| 472 |
+
predictions = _process_predict_proba(
|
| 473 |
+
y_pred=predictions,
|
| 474 |
+
target_type="binary",
|
| 475 |
+
classes=self.classes_,
|
| 476 |
+
pos_label=self.classes_[1],
|
| 477 |
+
)
|
| 478 |
+
predictions = predictions.reshape(-1, 1)
|
| 479 |
+
|
| 480 |
+
if sample_weight is not None:
|
| 481 |
+
# Check that the sample_weight dtype is consistent with the
|
| 482 |
+
# predictions to avoid unintentional upcasts.
|
| 483 |
+
sample_weight = _check_sample_weight(
|
| 484 |
+
sample_weight, predictions, dtype=predictions.dtype
|
| 485 |
+
)
|
| 486 |
+
|
| 487 |
+
this_estimator.fit(X, y, **routed_params.estimator.fit)
|
| 488 |
+
# Note: Here we don't pass on fit_params because the supported
|
| 489 |
+
# calibrators don't support fit_params anyway
|
| 490 |
+
calibrated_classifier = _fit_calibrator(
|
| 491 |
+
this_estimator,
|
| 492 |
+
predictions,
|
| 493 |
+
y,
|
| 494 |
+
self.classes_,
|
| 495 |
+
self.method,
|
| 496 |
+
xp=xp,
|
| 497 |
+
sample_weight=sample_weight,
|
| 498 |
+
)
|
| 499 |
+
self.calibrated_classifiers_.append(calibrated_classifier)
|
| 500 |
+
|
| 501 |
+
first_clf = self.calibrated_classifiers_[0].estimator
|
| 502 |
+
if hasattr(first_clf, "n_features_in_"):
|
| 503 |
+
self.n_features_in_ = first_clf.n_features_in_
|
| 504 |
+
if hasattr(first_clf, "feature_names_in_"):
|
| 505 |
+
self.feature_names_in_ = first_clf.feature_names_in_
|
| 506 |
+
return self
|
| 507 |
+
|
| 508 |
+
def predict_proba(self, X):
|
| 509 |
+
"""Calibrated probabilities of classification.
|
| 510 |
+
|
| 511 |
+
This function returns calibrated probabilities of classification
|
| 512 |
+
according to each class on an array of test vectors X.
|
| 513 |
+
|
| 514 |
+
Parameters
|
| 515 |
+
----------
|
| 516 |
+
X : array-like of shape (n_samples, n_features)
|
| 517 |
+
The samples, as accepted by `estimator.predict_proba`.
|
| 518 |
+
|
| 519 |
+
Returns
|
| 520 |
+
-------
|
| 521 |
+
C : ndarray of shape (n_samples, n_classes)
|
| 522 |
+
The predicted probas.
|
| 523 |
+
"""
|
| 524 |
+
check_is_fitted(self)
|
| 525 |
+
# Compute the arithmetic mean of the predictions of the calibrated
|
| 526 |
+
# classifiers
|
| 527 |
+
xp, _, device_ = get_namespace_and_device(X)
|
| 528 |
+
mean_proba = xp.zeros((_num_samples(X), self.classes_.shape[0]), device=device_)
|
| 529 |
+
for calibrated_classifier in self.calibrated_classifiers_:
|
| 530 |
+
proba = calibrated_classifier.predict_proba(X)
|
| 531 |
+
mean_proba += proba
|
| 532 |
+
|
| 533 |
+
mean_proba /= len(self.calibrated_classifiers_)
|
| 534 |
+
|
| 535 |
+
return mean_proba
|
| 536 |
+
|
| 537 |
+
def predict(self, X):
|
| 538 |
+
"""Predict the target of new samples.
|
| 539 |
+
|
| 540 |
+
The predicted class is the class that has the highest probability,
|
| 541 |
+
and can thus be different from the prediction of the uncalibrated classifier.
|
| 542 |
+
|
| 543 |
+
Parameters
|
| 544 |
+
----------
|
| 545 |
+
X : array-like of shape (n_samples, n_features)
|
| 546 |
+
The samples, as accepted by `estimator.predict`.
|
| 547 |
+
|
| 548 |
+
Returns
|
| 549 |
+
-------
|
| 550 |
+
C : ndarray of shape (n_samples,)
|
| 551 |
+
The predicted class.
|
| 552 |
+
"""
|
| 553 |
+
xp, _ = get_namespace(X)
|
| 554 |
+
check_is_fitted(self)
|
| 555 |
+
class_indices = xp.argmax(self.predict_proba(X), axis=1)
|
| 556 |
+
if isinstance(self.classes_[0], str):
|
| 557 |
+
class_indices = _convert_to_numpy(class_indices, xp=xp)
|
| 558 |
+
|
| 559 |
+
return self.classes_[class_indices]
|
| 560 |
+
|
| 561 |
+
def get_metadata_routing(self):
|
| 562 |
+
"""Get metadata routing of this object.
|
| 563 |
+
|
| 564 |
+
Please check :ref:`User Guide <metadata_routing>` on how the routing
|
| 565 |
+
mechanism works.
|
| 566 |
+
|
| 567 |
+
Returns
|
| 568 |
+
-------
|
| 569 |
+
routing : MetadataRouter
|
| 570 |
+
A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
|
| 571 |
+
routing information.
|
| 572 |
+
"""
|
| 573 |
+
router = (
|
| 574 |
+
MetadataRouter(owner=self)
|
| 575 |
+
.add_self_request(self)
|
| 576 |
+
.add(
|
| 577 |
+
estimator=self._get_estimator(),
|
| 578 |
+
method_mapping=MethodMapping().add(caller="fit", callee="fit"),
|
| 579 |
+
)
|
| 580 |
+
.add(
|
| 581 |
+
splitter=self.cv,
|
| 582 |
+
method_mapping=MethodMapping().add(caller="fit", callee="split"),
|
| 583 |
+
)
|
| 584 |
+
)
|
| 585 |
+
return router
|
| 586 |
+
|
| 587 |
+
def __sklearn_tags__(self):
|
| 588 |
+
tags = super().__sklearn_tags__()
|
| 589 |
+
estimator_tags = get_tags(self._get_estimator())
|
| 590 |
+
tags.input_tags.sparse = estimator_tags.input_tags.sparse
|
| 591 |
+
tags.array_api_support = (
|
| 592 |
+
estimator_tags.array_api_support and self.method == "temperature"
|
| 593 |
+
)
|
| 594 |
+
return tags
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
def _fit_classifier_calibrator_pair(
|
| 598 |
+
estimator,
|
| 599 |
+
X,
|
| 600 |
+
y,
|
| 601 |
+
train,
|
| 602 |
+
test,
|
| 603 |
+
method,
|
| 604 |
+
classes,
|
| 605 |
+
xp,
|
| 606 |
+
sample_weight=None,
|
| 607 |
+
fit_params=None,
|
| 608 |
+
):
|
| 609 |
+
"""Fit a classifier/calibration pair on a given train/test split.
|
| 610 |
+
|
| 611 |
+
Fit the classifier on the train set, compute its predictions on the test
|
| 612 |
+
set and use the predictions as input to fit the calibrator along with the
|
| 613 |
+
test labels.
|
| 614 |
+
|
| 615 |
+
Parameters
|
| 616 |
+
----------
|
| 617 |
+
estimator : estimator instance
|
| 618 |
+
Cloned base estimator.
|
| 619 |
+
|
| 620 |
+
X : array-like, shape (n_samples, n_features)
|
| 621 |
+
Sample data.
|
| 622 |
+
|
| 623 |
+
y : array-like, shape (n_samples,)
|
| 624 |
+
Targets.
|
| 625 |
+
|
| 626 |
+
train : ndarray, shape (n_train_indices,)
|
| 627 |
+
Indices of the training subset.
|
| 628 |
+
|
| 629 |
+
test : ndarray, shape (n_test_indices,)
|
| 630 |
+
Indices of the testing subset.
|
| 631 |
+
|
| 632 |
+
method : {'sigmoid', 'isotonic', 'temperature'}
|
| 633 |
+
Method to use for calibration.
|
| 634 |
+
|
| 635 |
+
classes : ndarray, shape (n_classes,)
|
| 636 |
+
The target classes.
|
| 637 |
+
|
| 638 |
+
xp : namespace
|
| 639 |
+
Array API namespace.
|
| 640 |
+
|
| 641 |
+
sample_weight : array-like, default=None
|
| 642 |
+
Sample weights for `X`.
|
| 643 |
+
|
| 644 |
+
fit_params : dict, default=None
|
| 645 |
+
Parameters to pass to the `fit` method of the underlying
|
| 646 |
+
classifier.
|
| 647 |
+
|
| 648 |
+
Returns
|
| 649 |
+
-------
|
| 650 |
+
calibrated_classifier : _CalibratedClassifier instance
|
| 651 |
+
"""
|
| 652 |
+
fit_params_train = _check_method_params(X, params=fit_params, indices=train)
|
| 653 |
+
X_train, y_train = _safe_indexing(X, train), _safe_indexing(y, train)
|
| 654 |
+
X_test, y_test = _safe_indexing(X, test), _safe_indexing(y, test)
|
| 655 |
+
|
| 656 |
+
estimator.fit(X_train, y_train, **fit_params_train)
|
| 657 |
+
|
| 658 |
+
predictions, _ = _get_response_values(
|
| 659 |
+
estimator,
|
| 660 |
+
X_test,
|
| 661 |
+
response_method=["decision_function", "predict_proba"],
|
| 662 |
+
)
|
| 663 |
+
if predictions.ndim == 1:
|
| 664 |
+
# Reshape binary output from `(n_samples,)` to `(n_samples, 1)`
|
| 665 |
+
predictions = predictions.reshape(-1, 1)
|
| 666 |
+
|
| 667 |
+
if sample_weight is not None:
|
| 668 |
+
# Check that the sample_weight dtype is consistent with the predictions
|
| 669 |
+
# to avoid unintentional upcasts.
|
| 670 |
+
sample_weight = _check_sample_weight(sample_weight, X, dtype=predictions.dtype)
|
| 671 |
+
sw_test = _safe_indexing(sample_weight, test)
|
| 672 |
+
else:
|
| 673 |
+
sw_test = None
|
| 674 |
+
calibrated_classifier = _fit_calibrator(
|
| 675 |
+
estimator,
|
| 676 |
+
predictions,
|
| 677 |
+
y_test,
|
| 678 |
+
classes,
|
| 679 |
+
method,
|
| 680 |
+
xp=xp,
|
| 681 |
+
sample_weight=sw_test,
|
| 682 |
+
)
|
| 683 |
+
return calibrated_classifier
|
| 684 |
+
|
| 685 |
+
|
| 686 |
+
def _fit_calibrator(clf, predictions, y, classes, method, xp, sample_weight=None):
|
| 687 |
+
"""Fit calibrator(s) and return a `_CalibratedClassifier`
|
| 688 |
+
instance.
|
| 689 |
+
|
| 690 |
+
A separate calibrator is fitted for each of the `n_classes`
|
| 691 |
+
(i.e. `len(clf.classes_)`). However, if `n_classes` is 2 or if
|
| 692 |
+
`method` is 'temperature', only one calibrator is fitted.
|
| 693 |
+
|
| 694 |
+
Parameters
|
| 695 |
+
----------
|
| 696 |
+
clf : estimator instance
|
| 697 |
+
Fitted classifier.
|
| 698 |
+
|
| 699 |
+
predictions : array-like, shape (n_samples, n_classes) or (n_samples, 1) \
|
| 700 |
+
when binary.
|
| 701 |
+
Raw predictions returned by the un-calibrated base classifier.
|
| 702 |
+
|
| 703 |
+
y : array-like, shape (n_samples,)
|
| 704 |
+
The targets. For `method="temperature"`, `y` needs to be label encoded.
|
| 705 |
+
|
| 706 |
+
classes : ndarray, shape (n_classes,)
|
| 707 |
+
All the prediction classes.
|
| 708 |
+
|
| 709 |
+
method : {'sigmoid', 'isotonic', 'temperature'}
|
| 710 |
+
The method to use for calibration.
|
| 711 |
+
|
| 712 |
+
xp : namespace
|
| 713 |
+
Array API namespace.
|
| 714 |
+
|
| 715 |
+
sample_weight : ndarray, shape (n_samples,), default=None
|
| 716 |
+
Sample weights. If None, then samples are equally weighted.
|
| 717 |
+
|
| 718 |
+
Returns
|
| 719 |
+
-------
|
| 720 |
+
pipeline : _CalibratedClassifier instance
|
| 721 |
+
"""
|
| 722 |
+
calibrators = []
|
| 723 |
+
|
| 724 |
+
if method in ("isotonic", "sigmoid"):
|
| 725 |
+
Y = label_binarize(y, classes=classes)
|
| 726 |
+
label_encoder = LabelEncoder().fit(classes)
|
| 727 |
+
pos_class_indices = label_encoder.transform(clf.classes_)
|
| 728 |
+
for class_idx, this_pred in zip(pos_class_indices, predictions.T):
|
| 729 |
+
if method == "isotonic":
|
| 730 |
+
calibrator = IsotonicRegression(out_of_bounds="clip")
|
| 731 |
+
else: # "sigmoid"
|
| 732 |
+
calibrator = _SigmoidCalibration()
|
| 733 |
+
calibrator.fit(this_pred, Y[:, class_idx], sample_weight)
|
| 734 |
+
calibrators.append(calibrator)
|
| 735 |
+
elif method == "temperature":
|
| 736 |
+
if classes.shape[0] == 2 and predictions.shape[-1] == 1:
|
| 737 |
+
response_method_name = _check_response_method(
|
| 738 |
+
clf,
|
| 739 |
+
["decision_function", "predict_proba"],
|
| 740 |
+
).__name__
|
| 741 |
+
if response_method_name == "predict_proba":
|
| 742 |
+
predictions = xp.concat([1 - predictions, predictions], axis=1)
|
| 743 |
+
calibrator = _TemperatureScaling()
|
| 744 |
+
calibrator.fit(predictions, y, sample_weight)
|
| 745 |
+
calibrators.append(calibrator)
|
| 746 |
+
|
| 747 |
+
pipeline = _CalibratedClassifier(clf, calibrators, method=method, classes=classes)
|
| 748 |
+
return pipeline
|
| 749 |
+
|
| 750 |
+
|
| 751 |
+
class _CalibratedClassifier:
|
| 752 |
+
"""Pipeline-like chaining a fitted classifier and its fitted calibrators.
|
| 753 |
+
|
| 754 |
+
Parameters
|
| 755 |
+
----------
|
| 756 |
+
estimator : estimator instance
|
| 757 |
+
Fitted classifier.
|
| 758 |
+
|
| 759 |
+
calibrators : list of fitted estimator instances
|
| 760 |
+
List of fitted calibrators (either 'IsotonicRegression' or
|
| 761 |
+
'_SigmoidCalibration'). The number of calibrators equals the number of
|
| 762 |
+
classes. However, if there are 2 classes, the list contains only one
|
| 763 |
+
fitted calibrator.
|
| 764 |
+
|
| 765 |
+
classes : array-like of shape (n_classes,)
|
| 766 |
+
All the prediction classes.
|
| 767 |
+
|
| 768 |
+
method : {'sigmoid', 'isotonic'}, default='sigmoid'
|
| 769 |
+
The method to use for calibration. Can be 'sigmoid' which
|
| 770 |
+
corresponds to Platt's method or 'isotonic' which is a
|
| 771 |
+
non-parametric approach based on isotonic regression.
|
| 772 |
+
"""
|
| 773 |
+
|
| 774 |
+
def __init__(self, estimator, calibrators, *, classes, method="sigmoid"):
|
| 775 |
+
self.estimator = estimator
|
| 776 |
+
self.calibrators = calibrators
|
| 777 |
+
self.classes = classes
|
| 778 |
+
self.method = method
|
| 779 |
+
|
| 780 |
+
def predict_proba(self, X):
|
| 781 |
+
"""Calculate calibrated probabilities.
|
| 782 |
+
|
| 783 |
+
Calculates classification calibrated probabilities
|
| 784 |
+
for each class, in a one-vs-all manner, for `X`.
|
| 785 |
+
|
| 786 |
+
Parameters
|
| 787 |
+
----------
|
| 788 |
+
X : ndarray of shape (n_samples, n_features)
|
| 789 |
+
The sample data.
|
| 790 |
+
|
| 791 |
+
Returns
|
| 792 |
+
-------
|
| 793 |
+
proba : array, shape (n_samples, n_classes)
|
| 794 |
+
The predicted probabilities. Can be exact zeros.
|
| 795 |
+
"""
|
| 796 |
+
predictions, _ = _get_response_values(
|
| 797 |
+
self.estimator,
|
| 798 |
+
X,
|
| 799 |
+
response_method=["decision_function", "predict_proba"],
|
| 800 |
+
)
|
| 801 |
+
if predictions.ndim == 1:
|
| 802 |
+
# Reshape binary output from `(n_samples,)` to `(n_samples, 1)`
|
| 803 |
+
predictions = predictions.reshape(-1, 1)
|
| 804 |
+
|
| 805 |
+
n_classes = self.classes.shape[0]
|
| 806 |
+
|
| 807 |
+
proba = np.zeros((_num_samples(X), n_classes))
|
| 808 |
+
|
| 809 |
+
if self.method in ("sigmoid", "isotonic"):
|
| 810 |
+
label_encoder = LabelEncoder().fit(self.classes)
|
| 811 |
+
pos_class_indices = label_encoder.transform(self.estimator.classes_)
|
| 812 |
+
for class_idx, this_pred, calibrator in zip(
|
| 813 |
+
pos_class_indices, predictions.T, self.calibrators
|
| 814 |
+
):
|
| 815 |
+
if n_classes == 2:
|
| 816 |
+
# When binary, `predictions` consists only of predictions for
|
| 817 |
+
# clf.classes_[1] but `pos_class_indices` = 0
|
| 818 |
+
class_idx += 1
|
| 819 |
+
proba[:, class_idx] = calibrator.predict(this_pred)
|
| 820 |
+
# Normalize the probabilities
|
| 821 |
+
if n_classes == 2:
|
| 822 |
+
proba[:, 0] = 1.0 - proba[:, 1]
|
| 823 |
+
else:
|
| 824 |
+
denominator = np.sum(proba, axis=1)[:, np.newaxis]
|
| 825 |
+
# In the edge case where for each class calibrator returns a zero
|
| 826 |
+
# probability for a given sample, use the uniform distribution
|
| 827 |
+
# instead.
|
| 828 |
+
uniform_proba = np.full_like(proba, 1 / n_classes)
|
| 829 |
+
proba = np.divide(
|
| 830 |
+
proba, denominator, out=uniform_proba, where=denominator != 0
|
| 831 |
+
)
|
| 832 |
+
elif self.method == "temperature":
|
| 833 |
+
xp, _ = get_namespace(predictions)
|
| 834 |
+
if n_classes == 2 and predictions.shape[-1] == 1:
|
| 835 |
+
response_method_name = _check_response_method(
|
| 836 |
+
self.estimator,
|
| 837 |
+
["decision_function", "predict_proba"],
|
| 838 |
+
).__name__
|
| 839 |
+
if response_method_name == "predict_proba":
|
| 840 |
+
predictions = xp.concat([1 - predictions, predictions], axis=1)
|
| 841 |
+
proba = self.calibrators[0].predict(predictions)
|
| 842 |
+
|
| 843 |
+
# Deal with cases where the predicted probability minimally exceeds 1.0
|
| 844 |
+
proba[(1.0 < proba) & (proba <= 1.0 + 1e-5)] = 1.0
|
| 845 |
+
|
| 846 |
+
return proba
|
| 847 |
+
|
| 848 |
+
|
| 849 |
+
# The max_abs_prediction_threshold was approximated using
|
| 850 |
+
# logit(np.finfo(np.float64).eps) which is about -36
|
| 851 |
+
def _sigmoid_calibration(
|
| 852 |
+
predictions, y, sample_weight=None, max_abs_prediction_threshold=30
|
| 853 |
+
):
|
| 854 |
+
"""Probability Calibration with sigmoid method (Platt 2000)
|
| 855 |
+
|
| 856 |
+
Parameters
|
| 857 |
+
----------
|
| 858 |
+
predictions : ndarray of shape (n_samples,)
|
| 859 |
+
The decision function or predict proba for the samples.
|
| 860 |
+
|
| 861 |
+
y : ndarray of shape (n_samples,)
|
| 862 |
+
The targets.
|
| 863 |
+
|
| 864 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 865 |
+
Sample weights. If None, then samples are equally weighted.
|
| 866 |
+
|
| 867 |
+
Returns
|
| 868 |
+
-------
|
| 869 |
+
a : float
|
| 870 |
+
The slope.
|
| 871 |
+
|
| 872 |
+
b : float
|
| 873 |
+
The intercept.
|
| 874 |
+
|
| 875 |
+
References
|
| 876 |
+
----------
|
| 877 |
+
Platt, "Probabilistic Outputs for Support Vector Machines"
|
| 878 |
+
"""
|
| 879 |
+
predictions = column_or_1d(predictions)
|
| 880 |
+
y = column_or_1d(y)
|
| 881 |
+
|
| 882 |
+
F = predictions # F follows Platt's notations
|
| 883 |
+
|
| 884 |
+
scale_constant = 1.0
|
| 885 |
+
max_prediction = np.max(np.abs(F))
|
| 886 |
+
|
| 887 |
+
# If the predictions have large values we scale them in order to bring
|
| 888 |
+
# them within a suitable range. This has no effect on the final
|
| 889 |
+
# (prediction) result because linear models like Logisitic Regression
|
| 890 |
+
# without a penalty are invariant to multiplying the features by a
|
| 891 |
+
# constant.
|
| 892 |
+
if max_prediction >= max_abs_prediction_threshold:
|
| 893 |
+
scale_constant = max_prediction
|
| 894 |
+
# We rescale the features in a copy: inplace rescaling could confuse
|
| 895 |
+
# the caller and make the code harder to reason about.
|
| 896 |
+
F = F / scale_constant
|
| 897 |
+
|
| 898 |
+
# Bayesian priors (see Platt end of section 2.2):
|
| 899 |
+
# It corresponds to the number of samples, taking into account the
|
| 900 |
+
# `sample_weight`.
|
| 901 |
+
mask_negative_samples = y <= 0
|
| 902 |
+
if sample_weight is not None:
|
| 903 |
+
prior0 = (sample_weight[mask_negative_samples]).sum()
|
| 904 |
+
prior1 = (sample_weight[~mask_negative_samples]).sum()
|
| 905 |
+
else:
|
| 906 |
+
prior0 = float(np.sum(mask_negative_samples))
|
| 907 |
+
prior1 = y.shape[0] - prior0
|
| 908 |
+
T = np.zeros_like(y, dtype=predictions.dtype)
|
| 909 |
+
T[y > 0] = (prior1 + 1.0) / (prior1 + 2.0)
|
| 910 |
+
T[y <= 0] = 1.0 / (prior0 + 2.0)
|
| 911 |
+
|
| 912 |
+
bin_loss = HalfBinomialLoss()
|
| 913 |
+
|
| 914 |
+
def loss_grad(AB):
|
| 915 |
+
# .astype below is needed to ensure y_true and raw_prediction have the
|
| 916 |
+
# same dtype. With result = np.float64(0) * np.array([1, 2], dtype=np.float32)
|
| 917 |
+
# - in Numpy 2, result.dtype is float64
|
| 918 |
+
# - in Numpy<2, result.dtype is float32
|
| 919 |
+
raw_prediction = -(AB[0] * F + AB[1]).astype(dtype=predictions.dtype)
|
| 920 |
+
l, g = bin_loss.loss_gradient(
|
| 921 |
+
y_true=T,
|
| 922 |
+
raw_prediction=raw_prediction,
|
| 923 |
+
sample_weight=sample_weight,
|
| 924 |
+
)
|
| 925 |
+
loss = l.sum()
|
| 926 |
+
# TODO: Remove casting to np.float64 when minimum supported SciPy is 1.11.2
|
| 927 |
+
# With SciPy >= 1.11.2, the LBFGS implementation will cast to float64
|
| 928 |
+
# https://github.com/scipy/scipy/pull/18825.
|
| 929 |
+
# Here we cast to float64 to support SciPy < 1.11.2
|
| 930 |
+
grad = np.asarray([-g @ F, -g.sum()], dtype=np.float64)
|
| 931 |
+
return loss, grad
|
| 932 |
+
|
| 933 |
+
AB0 = np.array([0.0, log((prior0 + 1.0) / (prior1 + 1.0))])
|
| 934 |
+
|
| 935 |
+
opt_result = minimize(
|
| 936 |
+
loss_grad,
|
| 937 |
+
AB0,
|
| 938 |
+
method="L-BFGS-B",
|
| 939 |
+
jac=True,
|
| 940 |
+
options={
|
| 941 |
+
"gtol": 1e-6,
|
| 942 |
+
"ftol": 64 * np.finfo(float).eps,
|
| 943 |
+
},
|
| 944 |
+
)
|
| 945 |
+
AB_ = opt_result.x
|
| 946 |
+
|
| 947 |
+
# The tuned multiplicative parameter is converted back to the original
|
| 948 |
+
# input feature scale. The offset parameter does not need rescaling since
|
| 949 |
+
# we did not rescale the outcome variable.
|
| 950 |
+
return AB_[0] / scale_constant, AB_[1]
|
| 951 |
+
|
| 952 |
+
|
| 953 |
+
def _convert_to_logits(decision_values, eps=1e-12, xp=None):
|
| 954 |
+
"""Convert decision_function values to 2D and predict_proba values to logits.
|
| 955 |
+
|
| 956 |
+
This function ensures that the output of `decision_function` is
|
| 957 |
+
converted into a (n_samples, n_classes) array. For binary classification,
|
| 958 |
+
each row contains logits for the negative and positive classes as (-x, x).
|
| 959 |
+
|
| 960 |
+
If `predict_proba` is provided instead, it is converted into
|
| 961 |
+
log-probabilities using `numpy.log`.
|
| 962 |
+
|
| 963 |
+
Parameters
|
| 964 |
+
----------
|
| 965 |
+
decision_values : array-like of shape (n_samples,) or (n_samples, 1) \
|
| 966 |
+
or (n_samples, n_classes).
|
| 967 |
+
|
| 968 |
+
The decision function values or probability estimates.
|
| 969 |
+
- If shape is (n_samples,), converts to (n_samples, 2) with (-x, x).
|
| 970 |
+
- If shape is (n_samples, 1), converts to (n_samples, 2) with (-x, x).
|
| 971 |
+
- If shape is (n_samples, n_classes), returns unchanged.
|
| 972 |
+
- For probability estimates, returns `numpy.log(decision_values + eps)`.
|
| 973 |
+
|
| 974 |
+
eps : float
|
| 975 |
+
Small positive value added to avoid log(0).
|
| 976 |
+
|
| 977 |
+
Returns
|
| 978 |
+
-------
|
| 979 |
+
logits : ndarray of shape (n_samples, n_classes)
|
| 980 |
+
"""
|
| 981 |
+
xp, _, device_ = get_namespace_and_device(decision_values, xp=xp)
|
| 982 |
+
decision_values = check_array(
|
| 983 |
+
decision_values, dtype=[xp.float64, xp.float32], ensure_2d=False
|
| 984 |
+
)
|
| 985 |
+
if (decision_values.ndim == 2) and (decision_values.shape[1] > 1):
|
| 986 |
+
# Check if it is the output of predict_proba
|
| 987 |
+
entries_zero_to_one = xp.all((decision_values >= 0) & (decision_values <= 1))
|
| 988 |
+
# TODO: simplify once upstream issue is addressed
|
| 989 |
+
# https://github.com/data-apis/array-api-extra/issues/478
|
| 990 |
+
row_sums_to_one = xp.all(
|
| 991 |
+
xpx.isclose(
|
| 992 |
+
xp.sum(decision_values, axis=1),
|
| 993 |
+
xp.asarray(1.0, device=device_, dtype=decision_values.dtype),
|
| 994 |
+
)
|
| 995 |
+
)
|
| 996 |
+
|
| 997 |
+
if entries_zero_to_one and row_sums_to_one:
|
| 998 |
+
logits = xp.log(decision_values + eps)
|
| 999 |
+
else:
|
| 1000 |
+
logits = decision_values
|
| 1001 |
+
|
| 1002 |
+
elif (decision_values.ndim == 2) and (decision_values.shape[1] == 1):
|
| 1003 |
+
logits = xp.concat([-decision_values, decision_values], axis=1)
|
| 1004 |
+
|
| 1005 |
+
elif decision_values.ndim == 1:
|
| 1006 |
+
decision_values = xp.reshape(decision_values, (-1, 1))
|
| 1007 |
+
logits = xp.concat([-decision_values, decision_values], axis=1)
|
| 1008 |
+
|
| 1009 |
+
return logits
|
| 1010 |
+
|
| 1011 |
+
|
| 1012 |
+
class _SigmoidCalibration(RegressorMixin, BaseEstimator):
|
| 1013 |
+
"""Sigmoid regression model.
|
| 1014 |
+
|
| 1015 |
+
Attributes
|
| 1016 |
+
----------
|
| 1017 |
+
a_ : float
|
| 1018 |
+
The slope.
|
| 1019 |
+
|
| 1020 |
+
b_ : float
|
| 1021 |
+
The intercept.
|
| 1022 |
+
"""
|
| 1023 |
+
|
| 1024 |
+
def fit(self, X, y, sample_weight=None):
|
| 1025 |
+
"""Fit the model using X, y as training data.
|
| 1026 |
+
|
| 1027 |
+
Parameters
|
| 1028 |
+
----------
|
| 1029 |
+
X : array-like of shape (n_samples,)
|
| 1030 |
+
Training data.
|
| 1031 |
+
|
| 1032 |
+
y : array-like of shape (n_samples,)
|
| 1033 |
+
Training target.
|
| 1034 |
+
|
| 1035 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 1036 |
+
Sample weights. If None, then samples are equally weighted.
|
| 1037 |
+
|
| 1038 |
+
Returns
|
| 1039 |
+
-------
|
| 1040 |
+
self : object
|
| 1041 |
+
Returns an instance of self.
|
| 1042 |
+
"""
|
| 1043 |
+
X = column_or_1d(X)
|
| 1044 |
+
y = column_or_1d(y)
|
| 1045 |
+
X, y = indexable(X, y)
|
| 1046 |
+
|
| 1047 |
+
self.a_, self.b_ = _sigmoid_calibration(X, y, sample_weight)
|
| 1048 |
+
return self
|
| 1049 |
+
|
| 1050 |
+
def predict(self, T):
|
| 1051 |
+
"""Predict new data by linear interpolation.
|
| 1052 |
+
|
| 1053 |
+
Parameters
|
| 1054 |
+
----------
|
| 1055 |
+
T : array-like of shape (n_samples,)
|
| 1056 |
+
Data to predict from.
|
| 1057 |
+
|
| 1058 |
+
Returns
|
| 1059 |
+
-------
|
| 1060 |
+
T_ : ndarray of shape (n_samples,)
|
| 1061 |
+
The predicted data.
|
| 1062 |
+
"""
|
| 1063 |
+
T = column_or_1d(T)
|
| 1064 |
+
return expit(-(self.a_ * T + self.b_))
|
| 1065 |
+
|
| 1066 |
+
|
| 1067 |
+
class _TemperatureScaling(RegressorMixin, BaseEstimator):
|
| 1068 |
+
"""Temperature scaling model.
|
| 1069 |
+
|
| 1070 |
+
Attributes
|
| 1071 |
+
----------
|
| 1072 |
+
beta_ : float
|
| 1073 |
+
The optimized inverse temperature.
|
| 1074 |
+
"""
|
| 1075 |
+
|
| 1076 |
+
def fit(self, X, y, sample_weight=None):
|
| 1077 |
+
"""Fit the model using X, y as training data.
|
| 1078 |
+
|
| 1079 |
+
Parameters
|
| 1080 |
+
----------
|
| 1081 |
+
X : ndarray of shape (n_samples,) or (n_samples, n_classes)
|
| 1082 |
+
Training data.
|
| 1083 |
+
|
| 1084 |
+
This should be the output of `decision_function` or `predict_proba`.
|
| 1085 |
+
If the input appears to be probabilities (i.e., values between 0 and 1
|
| 1086 |
+
that sum to 1 across classes), it will be converted to logits using
|
| 1087 |
+
`np.log(p + eps)`.
|
| 1088 |
+
|
| 1089 |
+
Binary decision function outputs (1D) will be converted to two-class
|
| 1090 |
+
logits of the form (-x, x). For shapes of the form (n_samples, 1), the
|
| 1091 |
+
same process applies.
|
| 1092 |
+
|
| 1093 |
+
y : array-like of shape (n_samples,)
|
| 1094 |
+
Training target.
|
| 1095 |
+
|
| 1096 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 1097 |
+
Sample weights. If None, then samples are equally weighted.
|
| 1098 |
+
|
| 1099 |
+
Returns
|
| 1100 |
+
-------
|
| 1101 |
+
self : object
|
| 1102 |
+
Returns an instance of self.
|
| 1103 |
+
"""
|
| 1104 |
+
xp, _, xp_device = get_namespace_and_device(X, y)
|
| 1105 |
+
X, y = indexable(X, y)
|
| 1106 |
+
check_consistent_length(X, y)
|
| 1107 |
+
logits = _convert_to_logits(X) # guarantees xp.float64 or xp.float32
|
| 1108 |
+
|
| 1109 |
+
dtype_ = logits.dtype
|
| 1110 |
+
labels = column_or_1d(y, dtype=dtype_)
|
| 1111 |
+
|
| 1112 |
+
if sample_weight is not None:
|
| 1113 |
+
sample_weight = _check_sample_weight(sample_weight, labels, dtype=dtype_)
|
| 1114 |
+
|
| 1115 |
+
if _is_numpy_namespace(xp):
|
| 1116 |
+
multinomial_loss = HalfMultinomialLoss(n_classes=logits.shape[1])
|
| 1117 |
+
else:
|
| 1118 |
+
multinomial_loss = partial(_half_multinomial_loss, xp=xp)
|
| 1119 |
+
|
| 1120 |
+
def log_loss(log_beta=0.0):
|
| 1121 |
+
"""Compute the log loss as a parameter of the inverse temperature
|
| 1122 |
+
(beta).
|
| 1123 |
+
|
| 1124 |
+
Parameters
|
| 1125 |
+
----------
|
| 1126 |
+
log_beta : float
|
| 1127 |
+
The current logarithm of the inverse temperature value during
|
| 1128 |
+
optimisation.
|
| 1129 |
+
|
| 1130 |
+
Returns
|
| 1131 |
+
-------
|
| 1132 |
+
negative_log_likelihood_loss : float
|
| 1133 |
+
The negative log likelihood loss.
|
| 1134 |
+
|
| 1135 |
+
"""
|
| 1136 |
+
# TODO: numpy 2.0
|
| 1137 |
+
# Ensure raw_prediction has the same dtype as labels using .astype().
|
| 1138 |
+
# Without this, dtype promotion rules differ across NumPy versions:
|
| 1139 |
+
#
|
| 1140 |
+
# beta = np.float64(0)
|
| 1141 |
+
# logits = np.array([1, 2], dtype=np.float32)
|
| 1142 |
+
#
|
| 1143 |
+
# result = beta * logits
|
| 1144 |
+
# - NumPy < 2: result.dtype is float32
|
| 1145 |
+
# - NumPy 2+: result.dtype is float64
|
| 1146 |
+
#
|
| 1147 |
+
# This can cause dtype mismatch errors downstream (e.g., buffer dtype).
|
| 1148 |
+
log_beta = xp.asarray(log_beta, dtype=dtype_, device=xp_device)
|
| 1149 |
+
raw_prediction = xp.exp(log_beta) * logits
|
| 1150 |
+
return multinomial_loss(labels, raw_prediction, sample_weight)
|
| 1151 |
+
|
| 1152 |
+
xatol = 64 * xp.finfo(dtype_).eps
|
| 1153 |
+
log_beta_minimizer = minimize_scalar(
|
| 1154 |
+
log_loss,
|
| 1155 |
+
bounds=(-10.0, 10.0),
|
| 1156 |
+
options={
|
| 1157 |
+
"xatol": xatol,
|
| 1158 |
+
},
|
| 1159 |
+
)
|
| 1160 |
+
|
| 1161 |
+
if not log_beta_minimizer.success: # pragma: no cover
|
| 1162 |
+
raise RuntimeError(
|
| 1163 |
+
"Temperature scaling fails to optimize during calibration. "
|
| 1164 |
+
"Reason from `scipy.optimize.minimize_scalar`: "
|
| 1165 |
+
f"{log_beta_minimizer.message}"
|
| 1166 |
+
)
|
| 1167 |
+
|
| 1168 |
+
self.beta_ = xp.exp(
|
| 1169 |
+
xp.asarray(log_beta_minimizer.x, dtype=dtype_, device=xp_device)
|
| 1170 |
+
)
|
| 1171 |
+
|
| 1172 |
+
return self
|
| 1173 |
+
|
| 1174 |
+
def predict(self, X):
|
| 1175 |
+
"""Predict new data by linear interpolation.
|
| 1176 |
+
|
| 1177 |
+
Parameters
|
| 1178 |
+
----------
|
| 1179 |
+
X : ndarray of shape (n_samples,) or (n_samples, n_classes)
|
| 1180 |
+
Data to predict from.
|
| 1181 |
+
|
| 1182 |
+
This should be the output of `decision_function` or `predict_proba`.
|
| 1183 |
+
If the input appears to be probabilities (i.e., values between 0 and 1
|
| 1184 |
+
that sum to 1 across classes), it will be converted to logits using
|
| 1185 |
+
`np.log(p + eps)`.
|
| 1186 |
+
|
| 1187 |
+
Binary decision function outputs (1D) will be converted to two-class
|
| 1188 |
+
logits of the form (-x, x). For shapes of the form (n_samples, 1), the
|
| 1189 |
+
same process applies.
|
| 1190 |
+
|
| 1191 |
+
Returns
|
| 1192 |
+
-------
|
| 1193 |
+
X_ : ndarray of shape (n_samples, n_classes)
|
| 1194 |
+
The predicted data.
|
| 1195 |
+
"""
|
| 1196 |
+
logits = _convert_to_logits(X)
|
| 1197 |
+
return softmax(self.beta_ * logits)
|
| 1198 |
+
|
| 1199 |
+
def __sklearn_tags__(self):
|
| 1200 |
+
tags = super().__sklearn_tags__()
|
| 1201 |
+
tags.input_tags.one_d_array = True
|
| 1202 |
+
tags.input_tags.two_d_array = False
|
| 1203 |
+
return tags
|
| 1204 |
+
|
| 1205 |
+
|
| 1206 |
+
@validate_params(
|
| 1207 |
+
{
|
| 1208 |
+
"y_true": ["array-like"],
|
| 1209 |
+
"y_prob": ["array-like"],
|
| 1210 |
+
"pos_label": [Real, str, "boolean", None],
|
| 1211 |
+
"n_bins": [Interval(Integral, 1, None, closed="left")],
|
| 1212 |
+
"strategy": [StrOptions({"uniform", "quantile"})],
|
| 1213 |
+
},
|
| 1214 |
+
prefer_skip_nested_validation=True,
|
| 1215 |
+
)
|
| 1216 |
+
def calibration_curve(
|
| 1217 |
+
y_true,
|
| 1218 |
+
y_prob,
|
| 1219 |
+
*,
|
| 1220 |
+
pos_label=None,
|
| 1221 |
+
n_bins=5,
|
| 1222 |
+
strategy="uniform",
|
| 1223 |
+
):
|
| 1224 |
+
"""Compute true and predicted probabilities for a calibration curve.
|
| 1225 |
+
|
| 1226 |
+
The method assumes the inputs come from a binary classifier, and
|
| 1227 |
+
discretize the [0, 1] interval into bins.
|
| 1228 |
+
|
| 1229 |
+
Calibration curves may also be referred to as reliability diagrams.
|
| 1230 |
+
|
| 1231 |
+
Read more in the :ref:`User Guide <calibration>`.
|
| 1232 |
+
|
| 1233 |
+
Parameters
|
| 1234 |
+
----------
|
| 1235 |
+
y_true : array-like of shape (n_samples,)
|
| 1236 |
+
True targets.
|
| 1237 |
+
|
| 1238 |
+
y_prob : array-like of shape (n_samples,)
|
| 1239 |
+
Probabilities of the positive class.
|
| 1240 |
+
|
| 1241 |
+
pos_label : int, float, bool or str, default=None
|
| 1242 |
+
The label of the positive class.
|
| 1243 |
+
|
| 1244 |
+
.. versionadded:: 1.1
|
| 1245 |
+
|
| 1246 |
+
n_bins : int, default=5
|
| 1247 |
+
Number of bins to discretize the [0, 1] interval. A bigger number
|
| 1248 |
+
requires more data. Bins with no samples (i.e. without
|
| 1249 |
+
corresponding values in `y_prob`) will not be returned, thus the
|
| 1250 |
+
returned arrays may have less than `n_bins` values.
|
| 1251 |
+
|
| 1252 |
+
strategy : {'uniform', 'quantile'}, default='uniform'
|
| 1253 |
+
Strategy used to define the widths of the bins.
|
| 1254 |
+
|
| 1255 |
+
uniform
|
| 1256 |
+
The bins have identical widths.
|
| 1257 |
+
quantile
|
| 1258 |
+
The bins have the same number of samples and depend on `y_prob`.
|
| 1259 |
+
|
| 1260 |
+
Returns
|
| 1261 |
+
-------
|
| 1262 |
+
prob_true : ndarray of shape (n_bins,) or smaller
|
| 1263 |
+
The proportion of samples whose class is the positive class, in each
|
| 1264 |
+
bin (fraction of positives).
|
| 1265 |
+
|
| 1266 |
+
prob_pred : ndarray of shape (n_bins,) or smaller
|
| 1267 |
+
The mean predicted probability in each bin.
|
| 1268 |
+
|
| 1269 |
+
See Also
|
| 1270 |
+
--------
|
| 1271 |
+
CalibrationDisplay.from_predictions : Plot calibration curve using true
|
| 1272 |
+
and predicted labels.
|
| 1273 |
+
CalibrationDisplay.from_estimator : Plot calibration curve using an
|
| 1274 |
+
estimator and data.
|
| 1275 |
+
|
| 1276 |
+
References
|
| 1277 |
+
----------
|
| 1278 |
+
Alexandru Niculescu-Mizil and Rich Caruana (2005) Predicting Good
|
| 1279 |
+
Probabilities With Supervised Learning, in Proceedings of the 22nd
|
| 1280 |
+
International Conference on Machine Learning (ICML).
|
| 1281 |
+
See section 4 (Qualitative Analysis of Predictions).
|
| 1282 |
+
|
| 1283 |
+
Examples
|
| 1284 |
+
--------
|
| 1285 |
+
>>> import numpy as np
|
| 1286 |
+
>>> from sklearn.calibration import calibration_curve
|
| 1287 |
+
>>> y_true = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1])
|
| 1288 |
+
>>> y_pred = np.array([0.1, 0.2, 0.3, 0.4, 0.65, 0.7, 0.8, 0.9, 1.])
|
| 1289 |
+
>>> prob_true, prob_pred = calibration_curve(y_true, y_pred, n_bins=3)
|
| 1290 |
+
>>> prob_true
|
| 1291 |
+
array([0. , 0.5, 1. ])
|
| 1292 |
+
>>> prob_pred
|
| 1293 |
+
array([0.2 , 0.525, 0.85 ])
|
| 1294 |
+
"""
|
| 1295 |
+
y_true = column_or_1d(y_true)
|
| 1296 |
+
y_prob = column_or_1d(y_prob)
|
| 1297 |
+
check_consistent_length(y_true, y_prob)
|
| 1298 |
+
pos_label = _check_pos_label_consistency(pos_label, y_true)
|
| 1299 |
+
|
| 1300 |
+
if y_prob.min() < 0 or y_prob.max() > 1:
|
| 1301 |
+
raise ValueError("y_prob has values outside [0, 1].")
|
| 1302 |
+
|
| 1303 |
+
labels = np.unique(y_true)
|
| 1304 |
+
if len(labels) > 2:
|
| 1305 |
+
raise ValueError(
|
| 1306 |
+
f"Only binary classification is supported. Provided labels {labels}."
|
| 1307 |
+
)
|
| 1308 |
+
y_true = y_true == pos_label
|
| 1309 |
+
|
| 1310 |
+
if strategy == "quantile": # Determine bin edges by distribution of data
|
| 1311 |
+
quantiles = np.linspace(0, 1, n_bins + 1)
|
| 1312 |
+
bins = np.percentile(y_prob, quantiles * 100)
|
| 1313 |
+
elif strategy == "uniform":
|
| 1314 |
+
bins = np.linspace(0.0, 1.0, n_bins + 1)
|
| 1315 |
+
else:
|
| 1316 |
+
raise ValueError(
|
| 1317 |
+
"Invalid entry to 'strategy' input. Strategy "
|
| 1318 |
+
"must be either 'quantile' or 'uniform'."
|
| 1319 |
+
)
|
| 1320 |
+
|
| 1321 |
+
binids = np.searchsorted(bins[1:-1], y_prob)
|
| 1322 |
+
|
| 1323 |
+
bin_sums = np.bincount(binids, weights=y_prob, minlength=len(bins))
|
| 1324 |
+
bin_true = np.bincount(binids, weights=y_true, minlength=len(bins))
|
| 1325 |
+
bin_total = np.bincount(binids, minlength=len(bins))
|
| 1326 |
+
|
| 1327 |
+
nonzero = bin_total != 0
|
| 1328 |
+
prob_true = bin_true[nonzero] / bin_total[nonzero]
|
| 1329 |
+
prob_pred = bin_sums[nonzero] / bin_total[nonzero]
|
| 1330 |
+
|
| 1331 |
+
return prob_true, prob_pred
|
| 1332 |
+
|
| 1333 |
+
|
| 1334 |
+
class CalibrationDisplay(_BinaryClassifierCurveDisplayMixin):
|
| 1335 |
+
"""Calibration curve (also known as reliability diagram) visualization.
|
| 1336 |
+
|
| 1337 |
+
It is recommended to use
|
| 1338 |
+
:func:`~sklearn.calibration.CalibrationDisplay.from_estimator` or
|
| 1339 |
+
:func:`~sklearn.calibration.CalibrationDisplay.from_predictions`
|
| 1340 |
+
to create a `CalibrationDisplay`. All parameters are stored as attributes.
|
| 1341 |
+
|
| 1342 |
+
Read more about calibration in the :ref:`User Guide <calibration>` and
|
| 1343 |
+
more about the scikit-learn visualization API in :ref:`visualizations`.
|
| 1344 |
+
|
| 1345 |
+
For an example on how to use the visualization, see
|
| 1346 |
+
:ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py`.
|
| 1347 |
+
|
| 1348 |
+
.. versionadded:: 1.0
|
| 1349 |
+
|
| 1350 |
+
Parameters
|
| 1351 |
+
----------
|
| 1352 |
+
prob_true : ndarray of shape (n_bins,)
|
| 1353 |
+
The proportion of samples whose class is the positive class (fraction
|
| 1354 |
+
of positives), in each bin.
|
| 1355 |
+
|
| 1356 |
+
prob_pred : ndarray of shape (n_bins,)
|
| 1357 |
+
The mean predicted probability in each bin.
|
| 1358 |
+
|
| 1359 |
+
y_prob : ndarray of shape (n_samples,)
|
| 1360 |
+
Probability estimates for the positive class, for each sample.
|
| 1361 |
+
|
| 1362 |
+
estimator_name : str, default=None
|
| 1363 |
+
Name of estimator. If None, the estimator name is not shown.
|
| 1364 |
+
|
| 1365 |
+
pos_label : int, float, bool or str, default=None
|
| 1366 |
+
The positive class when calibration curve computed.
|
| 1367 |
+
If not `None`, this value is displayed in the x- and y-axes labels.
|
| 1368 |
+
|
| 1369 |
+
.. versionadded:: 1.1
|
| 1370 |
+
|
| 1371 |
+
Attributes
|
| 1372 |
+
----------
|
| 1373 |
+
line_ : matplotlib Artist
|
| 1374 |
+
Calibration curve.
|
| 1375 |
+
|
| 1376 |
+
ax_ : matplotlib Axes
|
| 1377 |
+
Axes with calibration curve.
|
| 1378 |
+
|
| 1379 |
+
figure_ : matplotlib Figure
|
| 1380 |
+
Figure containing the curve.
|
| 1381 |
+
|
| 1382 |
+
See Also
|
| 1383 |
+
--------
|
| 1384 |
+
calibration_curve : Compute true and predicted probabilities for a
|
| 1385 |
+
calibration curve.
|
| 1386 |
+
CalibrationDisplay.from_predictions : Plot calibration curve using true
|
| 1387 |
+
and predicted labels.
|
| 1388 |
+
CalibrationDisplay.from_estimator : Plot calibration curve using an
|
| 1389 |
+
estimator and data.
|
| 1390 |
+
|
| 1391 |
+
Examples
|
| 1392 |
+
--------
|
| 1393 |
+
>>> from sklearn.datasets import make_classification
|
| 1394 |
+
>>> from sklearn.model_selection import train_test_split
|
| 1395 |
+
>>> from sklearn.linear_model import LogisticRegression
|
| 1396 |
+
>>> from sklearn.calibration import calibration_curve, CalibrationDisplay
|
| 1397 |
+
>>> X, y = make_classification(random_state=0)
|
| 1398 |
+
>>> X_train, X_test, y_train, y_test = train_test_split(
|
| 1399 |
+
... X, y, random_state=0)
|
| 1400 |
+
>>> clf = LogisticRegression(random_state=0)
|
| 1401 |
+
>>> clf.fit(X_train, y_train)
|
| 1402 |
+
LogisticRegression(random_state=0)
|
| 1403 |
+
>>> y_prob = clf.predict_proba(X_test)[:, 1]
|
| 1404 |
+
>>> prob_true, prob_pred = calibration_curve(y_test, y_prob, n_bins=10)
|
| 1405 |
+
>>> disp = CalibrationDisplay(prob_true, prob_pred, y_prob)
|
| 1406 |
+
>>> disp.plot()
|
| 1407 |
+
<...>
|
| 1408 |
+
"""
|
| 1409 |
+
|
| 1410 |
+
def __init__(
|
| 1411 |
+
self, prob_true, prob_pred, y_prob, *, estimator_name=None, pos_label=None
|
| 1412 |
+
):
|
| 1413 |
+
self.prob_true = prob_true
|
| 1414 |
+
self.prob_pred = prob_pred
|
| 1415 |
+
self.y_prob = y_prob
|
| 1416 |
+
self.estimator_name = estimator_name
|
| 1417 |
+
self.pos_label = pos_label
|
| 1418 |
+
|
| 1419 |
+
def plot(self, *, ax=None, name=None, ref_line=True, **kwargs):
|
| 1420 |
+
"""Plot visualization.
|
| 1421 |
+
|
| 1422 |
+
Extra keyword arguments will be passed to
|
| 1423 |
+
:func:`matplotlib.pyplot.plot`.
|
| 1424 |
+
|
| 1425 |
+
Parameters
|
| 1426 |
+
----------
|
| 1427 |
+
ax : Matplotlib Axes, default=None
|
| 1428 |
+
Axes object to plot on. If `None`, a new figure and axes is
|
| 1429 |
+
created.
|
| 1430 |
+
|
| 1431 |
+
name : str, default=None
|
| 1432 |
+
Name for labeling curve. If `None`, use `estimator_name` if
|
| 1433 |
+
not `None`, otherwise no labeling is shown.
|
| 1434 |
+
|
| 1435 |
+
ref_line : bool, default=True
|
| 1436 |
+
If `True`, plots a reference line representing a perfectly
|
| 1437 |
+
calibrated classifier.
|
| 1438 |
+
|
| 1439 |
+
**kwargs : dict
|
| 1440 |
+
Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`.
|
| 1441 |
+
|
| 1442 |
+
Returns
|
| 1443 |
+
-------
|
| 1444 |
+
display : :class:`~sklearn.calibration.CalibrationDisplay`
|
| 1445 |
+
Object that stores computed values.
|
| 1446 |
+
"""
|
| 1447 |
+
self.ax_, self.figure_, name = self._validate_plot_params(ax=ax, name=name)
|
| 1448 |
+
|
| 1449 |
+
info_pos_label = (
|
| 1450 |
+
f"(Positive class: {self.pos_label})" if self.pos_label is not None else ""
|
| 1451 |
+
)
|
| 1452 |
+
|
| 1453 |
+
default_line_kwargs = {"marker": "s", "linestyle": "-"}
|
| 1454 |
+
if name is not None:
|
| 1455 |
+
default_line_kwargs["label"] = name
|
| 1456 |
+
line_kwargs = _validate_style_kwargs(default_line_kwargs, kwargs)
|
| 1457 |
+
|
| 1458 |
+
ref_line_label = "Perfectly calibrated"
|
| 1459 |
+
existing_ref_line = ref_line_label in self.ax_.get_legend_handles_labels()[1]
|
| 1460 |
+
if ref_line and not existing_ref_line:
|
| 1461 |
+
self.ax_.plot([0, 1], [0, 1], "k:", label=ref_line_label)
|
| 1462 |
+
self.line_ = self.ax_.plot(self.prob_pred, self.prob_true, **line_kwargs)[0]
|
| 1463 |
+
|
| 1464 |
+
# We always have to show the legend for at least the reference line
|
| 1465 |
+
self.ax_.legend(loc="lower right")
|
| 1466 |
+
|
| 1467 |
+
xlabel = f"Mean predicted probability {info_pos_label}"
|
| 1468 |
+
ylabel = f"Fraction of positives {info_pos_label}"
|
| 1469 |
+
self.ax_.set(xlabel=xlabel, ylabel=ylabel)
|
| 1470 |
+
|
| 1471 |
+
return self
|
| 1472 |
+
|
| 1473 |
+
@classmethod
|
| 1474 |
+
def from_estimator(
|
| 1475 |
+
cls,
|
| 1476 |
+
estimator,
|
| 1477 |
+
X,
|
| 1478 |
+
y,
|
| 1479 |
+
*,
|
| 1480 |
+
n_bins=5,
|
| 1481 |
+
strategy="uniform",
|
| 1482 |
+
pos_label=None,
|
| 1483 |
+
name=None,
|
| 1484 |
+
ax=None,
|
| 1485 |
+
ref_line=True,
|
| 1486 |
+
**kwargs,
|
| 1487 |
+
):
|
| 1488 |
+
"""Plot calibration curve using a binary classifier and data.
|
| 1489 |
+
|
| 1490 |
+
A calibration curve, also known as a reliability diagram, uses inputs
|
| 1491 |
+
from a binary classifier and plots the average predicted probability
|
| 1492 |
+
for each bin against the fraction of positive classes, on the
|
| 1493 |
+
y-axis.
|
| 1494 |
+
|
| 1495 |
+
Extra keyword arguments will be passed to
|
| 1496 |
+
:func:`matplotlib.pyplot.plot`.
|
| 1497 |
+
|
| 1498 |
+
Read more about calibration in the :ref:`User Guide <calibration>` and
|
| 1499 |
+
more about the scikit-learn visualization API in :ref:`visualizations`.
|
| 1500 |
+
|
| 1501 |
+
.. versionadded:: 1.0
|
| 1502 |
+
|
| 1503 |
+
Parameters
|
| 1504 |
+
----------
|
| 1505 |
+
estimator : estimator instance
|
| 1506 |
+
Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
|
| 1507 |
+
in which the last estimator is a classifier. The classifier must
|
| 1508 |
+
have a :term:`predict_proba` method.
|
| 1509 |
+
|
| 1510 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 1511 |
+
Input values.
|
| 1512 |
+
|
| 1513 |
+
y : array-like of shape (n_samples,)
|
| 1514 |
+
Binary target values.
|
| 1515 |
+
|
| 1516 |
+
n_bins : int, default=5
|
| 1517 |
+
Number of bins to discretize the [0, 1] interval into when
|
| 1518 |
+
calculating the calibration curve. A bigger number requires more
|
| 1519 |
+
data.
|
| 1520 |
+
|
| 1521 |
+
strategy : {'uniform', 'quantile'}, default='uniform'
|
| 1522 |
+
Strategy used to define the widths of the bins.
|
| 1523 |
+
|
| 1524 |
+
- `'uniform'`: The bins have identical widths.
|
| 1525 |
+
- `'quantile'`: The bins have the same number of samples and depend
|
| 1526 |
+
on predicted probabilities.
|
| 1527 |
+
|
| 1528 |
+
pos_label : int, float, bool or str, default=None
|
| 1529 |
+
The positive class when computing the calibration curve.
|
| 1530 |
+
By default, `estimators.classes_[1]` is considered as the
|
| 1531 |
+
positive class.
|
| 1532 |
+
|
| 1533 |
+
.. versionadded:: 1.1
|
| 1534 |
+
|
| 1535 |
+
name : str, default=None
|
| 1536 |
+
Name for labeling curve. If `None`, the name of the estimator is
|
| 1537 |
+
used.
|
| 1538 |
+
|
| 1539 |
+
ax : matplotlib axes, default=None
|
| 1540 |
+
Axes object to plot on. If `None`, a new figure and axes is
|
| 1541 |
+
created.
|
| 1542 |
+
|
| 1543 |
+
ref_line : bool, default=True
|
| 1544 |
+
If `True`, plots a reference line representing a perfectly
|
| 1545 |
+
calibrated classifier.
|
| 1546 |
+
|
| 1547 |
+
**kwargs : dict
|
| 1548 |
+
Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`.
|
| 1549 |
+
|
| 1550 |
+
Returns
|
| 1551 |
+
-------
|
| 1552 |
+
display : :class:`~sklearn.calibration.CalibrationDisplay`.
|
| 1553 |
+
Object that stores computed values.
|
| 1554 |
+
|
| 1555 |
+
See Also
|
| 1556 |
+
--------
|
| 1557 |
+
CalibrationDisplay.from_predictions : Plot calibration curve using true
|
| 1558 |
+
and predicted labels.
|
| 1559 |
+
|
| 1560 |
+
Examples
|
| 1561 |
+
--------
|
| 1562 |
+
>>> import matplotlib.pyplot as plt
|
| 1563 |
+
>>> from sklearn.datasets import make_classification
|
| 1564 |
+
>>> from sklearn.model_selection import train_test_split
|
| 1565 |
+
>>> from sklearn.linear_model import LogisticRegression
|
| 1566 |
+
>>> from sklearn.calibration import CalibrationDisplay
|
| 1567 |
+
>>> X, y = make_classification(random_state=0)
|
| 1568 |
+
>>> X_train, X_test, y_train, y_test = train_test_split(
|
| 1569 |
+
... X, y, random_state=0)
|
| 1570 |
+
>>> clf = LogisticRegression(random_state=0)
|
| 1571 |
+
>>> clf.fit(X_train, y_train)
|
| 1572 |
+
LogisticRegression(random_state=0)
|
| 1573 |
+
>>> disp = CalibrationDisplay.from_estimator(clf, X_test, y_test)
|
| 1574 |
+
>>> plt.show()
|
| 1575 |
+
"""
|
| 1576 |
+
y_prob, pos_label, name = cls._validate_and_get_response_values(
|
| 1577 |
+
estimator,
|
| 1578 |
+
X,
|
| 1579 |
+
y,
|
| 1580 |
+
response_method="predict_proba",
|
| 1581 |
+
pos_label=pos_label,
|
| 1582 |
+
name=name,
|
| 1583 |
+
)
|
| 1584 |
+
|
| 1585 |
+
return cls.from_predictions(
|
| 1586 |
+
y,
|
| 1587 |
+
y_prob,
|
| 1588 |
+
n_bins=n_bins,
|
| 1589 |
+
strategy=strategy,
|
| 1590 |
+
pos_label=pos_label,
|
| 1591 |
+
name=name,
|
| 1592 |
+
ref_line=ref_line,
|
| 1593 |
+
ax=ax,
|
| 1594 |
+
**kwargs,
|
| 1595 |
+
)
|
| 1596 |
+
|
| 1597 |
+
@classmethod
|
| 1598 |
+
def from_predictions(
|
| 1599 |
+
cls,
|
| 1600 |
+
y_true,
|
| 1601 |
+
y_prob,
|
| 1602 |
+
*,
|
| 1603 |
+
n_bins=5,
|
| 1604 |
+
strategy="uniform",
|
| 1605 |
+
pos_label=None,
|
| 1606 |
+
name=None,
|
| 1607 |
+
ax=None,
|
| 1608 |
+
ref_line=True,
|
| 1609 |
+
**kwargs,
|
| 1610 |
+
):
|
| 1611 |
+
"""Plot calibration curve using true labels and predicted probabilities.
|
| 1612 |
+
|
| 1613 |
+
Calibration curve, also known as reliability diagram, uses inputs
|
| 1614 |
+
from a binary classifier and plots the average predicted probability
|
| 1615 |
+
for each bin against the fraction of positive classes, on the
|
| 1616 |
+
y-axis.
|
| 1617 |
+
|
| 1618 |
+
Extra keyword arguments will be passed to
|
| 1619 |
+
:func:`matplotlib.pyplot.plot`.
|
| 1620 |
+
|
| 1621 |
+
Read more about calibration in the :ref:`User Guide <calibration>` and
|
| 1622 |
+
more about the scikit-learn visualization API in :ref:`visualizations`.
|
| 1623 |
+
|
| 1624 |
+
.. versionadded:: 1.0
|
| 1625 |
+
|
| 1626 |
+
Parameters
|
| 1627 |
+
----------
|
| 1628 |
+
y_true : array-like of shape (n_samples,)
|
| 1629 |
+
True labels.
|
| 1630 |
+
|
| 1631 |
+
y_prob : array-like of shape (n_samples,)
|
| 1632 |
+
The predicted probabilities of the positive class.
|
| 1633 |
+
|
| 1634 |
+
n_bins : int, default=5
|
| 1635 |
+
Number of bins to discretize the [0, 1] interval into when
|
| 1636 |
+
calculating the calibration curve. A bigger number requires more
|
| 1637 |
+
data.
|
| 1638 |
+
|
| 1639 |
+
strategy : {'uniform', 'quantile'}, default='uniform'
|
| 1640 |
+
Strategy used to define the widths of the bins.
|
| 1641 |
+
|
| 1642 |
+
- `'uniform'`: The bins have identical widths.
|
| 1643 |
+
- `'quantile'`: The bins have the same number of samples and depend
|
| 1644 |
+
on predicted probabilities.
|
| 1645 |
+
|
| 1646 |
+
pos_label : int, float, bool or str, default=None
|
| 1647 |
+
The positive class when computing the calibration curve.
|
| 1648 |
+
When `pos_label=None`, if `y_true` is in {-1, 1} or {0, 1},
|
| 1649 |
+
`pos_label` is set to 1, otherwise an error will be raised.
|
| 1650 |
+
|
| 1651 |
+
.. versionadded:: 1.1
|
| 1652 |
+
|
| 1653 |
+
name : str, default=None
|
| 1654 |
+
Name for labeling curve.
|
| 1655 |
+
|
| 1656 |
+
ax : matplotlib axes, default=None
|
| 1657 |
+
Axes object to plot on. If `None`, a new figure and axes is
|
| 1658 |
+
created.
|
| 1659 |
+
|
| 1660 |
+
ref_line : bool, default=True
|
| 1661 |
+
If `True`, plots a reference line representing a perfectly
|
| 1662 |
+
calibrated classifier.
|
| 1663 |
+
|
| 1664 |
+
**kwargs : dict
|
| 1665 |
+
Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`.
|
| 1666 |
+
|
| 1667 |
+
Returns
|
| 1668 |
+
-------
|
| 1669 |
+
display : :class:`~sklearn.calibration.CalibrationDisplay`.
|
| 1670 |
+
Object that stores computed values.
|
| 1671 |
+
|
| 1672 |
+
See Also
|
| 1673 |
+
--------
|
| 1674 |
+
CalibrationDisplay.from_estimator : Plot calibration curve using an
|
| 1675 |
+
estimator and data.
|
| 1676 |
+
|
| 1677 |
+
Examples
|
| 1678 |
+
--------
|
| 1679 |
+
>>> import matplotlib.pyplot as plt
|
| 1680 |
+
>>> from sklearn.datasets import make_classification
|
| 1681 |
+
>>> from sklearn.model_selection import train_test_split
|
| 1682 |
+
>>> from sklearn.linear_model import LogisticRegression
|
| 1683 |
+
>>> from sklearn.calibration import CalibrationDisplay
|
| 1684 |
+
>>> X, y = make_classification(random_state=0)
|
| 1685 |
+
>>> X_train, X_test, y_train, y_test = train_test_split(
|
| 1686 |
+
... X, y, random_state=0)
|
| 1687 |
+
>>> clf = LogisticRegression(random_state=0)
|
| 1688 |
+
>>> clf.fit(X_train, y_train)
|
| 1689 |
+
LogisticRegression(random_state=0)
|
| 1690 |
+
>>> y_prob = clf.predict_proba(X_test)[:, 1]
|
| 1691 |
+
>>> disp = CalibrationDisplay.from_predictions(y_test, y_prob)
|
| 1692 |
+
>>> plt.show()
|
| 1693 |
+
"""
|
| 1694 |
+
pos_label_validated, name = cls._validate_from_predictions_params(
|
| 1695 |
+
y_true, y_prob, sample_weight=None, pos_label=pos_label, name=name
|
| 1696 |
+
)
|
| 1697 |
+
|
| 1698 |
+
prob_true, prob_pred = calibration_curve(
|
| 1699 |
+
y_true, y_prob, n_bins=n_bins, strategy=strategy, pos_label=pos_label
|
| 1700 |
+
)
|
| 1701 |
+
|
| 1702 |
+
disp = cls(
|
| 1703 |
+
prob_true=prob_true,
|
| 1704 |
+
prob_pred=prob_pred,
|
| 1705 |
+
y_prob=y_prob,
|
| 1706 |
+
estimator_name=name,
|
| 1707 |
+
pos_label=pos_label_validated,
|
| 1708 |
+
)
|
| 1709 |
+
return disp.plot(ax=ax, ref_line=ref_line, **kwargs)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/__init__.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Popular unsupervised clustering algorithms."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
from sklearn.cluster._affinity_propagation import (
|
| 7 |
+
AffinityPropagation,
|
| 8 |
+
affinity_propagation,
|
| 9 |
+
)
|
| 10 |
+
from sklearn.cluster._agglomerative import (
|
| 11 |
+
AgglomerativeClustering,
|
| 12 |
+
FeatureAgglomeration,
|
| 13 |
+
linkage_tree,
|
| 14 |
+
ward_tree,
|
| 15 |
+
)
|
| 16 |
+
from sklearn.cluster._bicluster import SpectralBiclustering, SpectralCoclustering
|
| 17 |
+
from sklearn.cluster._birch import Birch
|
| 18 |
+
from sklearn.cluster._bisect_k_means import BisectingKMeans
|
| 19 |
+
from sklearn.cluster._dbscan import DBSCAN, dbscan
|
| 20 |
+
from sklearn.cluster._hdbscan.hdbscan import HDBSCAN
|
| 21 |
+
from sklearn.cluster._kmeans import KMeans, MiniBatchKMeans, k_means, kmeans_plusplus
|
| 22 |
+
from sklearn.cluster._mean_shift import (
|
| 23 |
+
MeanShift,
|
| 24 |
+
estimate_bandwidth,
|
| 25 |
+
get_bin_seeds,
|
| 26 |
+
mean_shift,
|
| 27 |
+
)
|
| 28 |
+
from sklearn.cluster._optics import (
|
| 29 |
+
OPTICS,
|
| 30 |
+
cluster_optics_dbscan,
|
| 31 |
+
cluster_optics_xi,
|
| 32 |
+
compute_optics_graph,
|
| 33 |
+
)
|
| 34 |
+
from sklearn.cluster._spectral import SpectralClustering, spectral_clustering
|
| 35 |
+
|
| 36 |
+
__all__ = [
|
| 37 |
+
"DBSCAN",
|
| 38 |
+
"HDBSCAN",
|
| 39 |
+
"OPTICS",
|
| 40 |
+
"AffinityPropagation",
|
| 41 |
+
"AgglomerativeClustering",
|
| 42 |
+
"Birch",
|
| 43 |
+
"BisectingKMeans",
|
| 44 |
+
"FeatureAgglomeration",
|
| 45 |
+
"KMeans",
|
| 46 |
+
"MeanShift",
|
| 47 |
+
"MiniBatchKMeans",
|
| 48 |
+
"SpectralBiclustering",
|
| 49 |
+
"SpectralClustering",
|
| 50 |
+
"SpectralCoclustering",
|
| 51 |
+
"affinity_propagation",
|
| 52 |
+
"cluster_optics_dbscan",
|
| 53 |
+
"cluster_optics_xi",
|
| 54 |
+
"compute_optics_graph",
|
| 55 |
+
"dbscan",
|
| 56 |
+
"estimate_bandwidth",
|
| 57 |
+
"get_bin_seeds",
|
| 58 |
+
"k_means",
|
| 59 |
+
"kmeans_plusplus",
|
| 60 |
+
"linkage_tree",
|
| 61 |
+
"mean_shift",
|
| 62 |
+
"spectral_clustering",
|
| 63 |
+
"ward_tree",
|
| 64 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_affinity_propagation.py
ADDED
|
@@ -0,0 +1,607 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Affinity Propagation clustering algorithm."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
import warnings
|
| 7 |
+
from numbers import Integral, Real
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
from sklearn._config import config_context
|
| 12 |
+
from sklearn.base import BaseEstimator, ClusterMixin, _fit_context
|
| 13 |
+
from sklearn.exceptions import ConvergenceWarning
|
| 14 |
+
from sklearn.metrics import euclidean_distances, pairwise_distances_argmin
|
| 15 |
+
from sklearn.utils import check_random_state
|
| 16 |
+
from sklearn.utils._param_validation import Interval, StrOptions, validate_params
|
| 17 |
+
from sklearn.utils.validation import check_is_fitted, validate_data
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _equal_similarities_and_preferences(S, preference):
|
| 21 |
+
def all_equal_preferences():
|
| 22 |
+
return np.all(preference == preference.flat[0])
|
| 23 |
+
|
| 24 |
+
def all_equal_similarities():
|
| 25 |
+
# Create mask to ignore diagonal of S
|
| 26 |
+
mask = np.ones(S.shape, dtype=bool)
|
| 27 |
+
np.fill_diagonal(mask, 0)
|
| 28 |
+
|
| 29 |
+
return np.all(S[mask].flat == S[mask].flat[0])
|
| 30 |
+
|
| 31 |
+
return all_equal_preferences() and all_equal_similarities()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _affinity_propagation(
|
| 35 |
+
S,
|
| 36 |
+
*,
|
| 37 |
+
preference,
|
| 38 |
+
convergence_iter,
|
| 39 |
+
max_iter,
|
| 40 |
+
damping,
|
| 41 |
+
verbose,
|
| 42 |
+
return_n_iter,
|
| 43 |
+
random_state,
|
| 44 |
+
):
|
| 45 |
+
"""Main affinity propagation algorithm."""
|
| 46 |
+
n_samples = S.shape[0]
|
| 47 |
+
if n_samples == 1 or _equal_similarities_and_preferences(S, preference):
|
| 48 |
+
# It makes no sense to run the algorithm in this case, so return 1 or
|
| 49 |
+
# n_samples clusters, depending on preferences
|
| 50 |
+
warnings.warn(
|
| 51 |
+
"All samples have mutually equal similarities. "
|
| 52 |
+
"Returning arbitrary cluster center(s)."
|
| 53 |
+
)
|
| 54 |
+
if preference.flat[0] > S.flat[n_samples - 1]:
|
| 55 |
+
return (
|
| 56 |
+
(np.arange(n_samples), np.arange(n_samples), 0)
|
| 57 |
+
if return_n_iter
|
| 58 |
+
else (np.arange(n_samples), np.arange(n_samples))
|
| 59 |
+
)
|
| 60 |
+
else:
|
| 61 |
+
return (
|
| 62 |
+
(np.array([0]), np.array([0] * n_samples), 0)
|
| 63 |
+
if return_n_iter
|
| 64 |
+
else (np.array([0]), np.array([0] * n_samples))
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
# Place preference on the diagonal of S
|
| 68 |
+
S.flat[:: (n_samples + 1)] = preference
|
| 69 |
+
|
| 70 |
+
A = np.zeros((n_samples, n_samples))
|
| 71 |
+
R = np.zeros((n_samples, n_samples)) # Initialize messages
|
| 72 |
+
# Intermediate results
|
| 73 |
+
tmp = np.zeros((n_samples, n_samples))
|
| 74 |
+
|
| 75 |
+
# Remove degeneracies
|
| 76 |
+
S += (
|
| 77 |
+
np.finfo(S.dtype).eps * S + np.finfo(S.dtype).tiny * 100
|
| 78 |
+
) * random_state.standard_normal(size=(n_samples, n_samples))
|
| 79 |
+
|
| 80 |
+
# Execute parallel affinity propagation updates
|
| 81 |
+
e = np.zeros((n_samples, convergence_iter))
|
| 82 |
+
|
| 83 |
+
ind = np.arange(n_samples)
|
| 84 |
+
|
| 85 |
+
for it in range(max_iter):
|
| 86 |
+
# tmp = A + S; compute responsibilities
|
| 87 |
+
np.add(A, S, tmp)
|
| 88 |
+
I = np.argmax(tmp, axis=1)
|
| 89 |
+
Y = tmp[ind, I] # np.max(A + S, axis=1)
|
| 90 |
+
tmp[ind, I] = -np.inf
|
| 91 |
+
Y2 = np.max(tmp, axis=1)
|
| 92 |
+
|
| 93 |
+
# tmp = Rnew
|
| 94 |
+
np.subtract(S, Y[:, None], tmp)
|
| 95 |
+
tmp[ind, I] = S[ind, I] - Y2
|
| 96 |
+
|
| 97 |
+
# Damping
|
| 98 |
+
tmp *= 1 - damping
|
| 99 |
+
R *= damping
|
| 100 |
+
R += tmp
|
| 101 |
+
|
| 102 |
+
# tmp = Rp; compute availabilities
|
| 103 |
+
np.maximum(R, 0, out=tmp)
|
| 104 |
+
tmp.flat[:: n_samples + 1] = R.flat[:: n_samples + 1]
|
| 105 |
+
|
| 106 |
+
# tmp = -Anew
|
| 107 |
+
tmp -= np.sum(tmp, axis=0)
|
| 108 |
+
dA = np.diag(tmp).copy()
|
| 109 |
+
tmp.clip(0, np.inf, tmp)
|
| 110 |
+
tmp.flat[:: n_samples + 1] = dA
|
| 111 |
+
|
| 112 |
+
# Damping
|
| 113 |
+
tmp *= 1 - damping
|
| 114 |
+
A *= damping
|
| 115 |
+
A -= tmp
|
| 116 |
+
|
| 117 |
+
# Check for convergence
|
| 118 |
+
E = (np.diag(A) + np.diag(R)) > 0
|
| 119 |
+
e[:, it % convergence_iter] = E
|
| 120 |
+
K = np.sum(E, axis=0)
|
| 121 |
+
|
| 122 |
+
if it >= convergence_iter:
|
| 123 |
+
se = np.sum(e, axis=1)
|
| 124 |
+
unconverged = np.sum((se == convergence_iter) + (se == 0)) != n_samples
|
| 125 |
+
if (not unconverged and (K > 0)) or (it == max_iter):
|
| 126 |
+
never_converged = False
|
| 127 |
+
if verbose:
|
| 128 |
+
print("Converged after %d iterations." % it)
|
| 129 |
+
break
|
| 130 |
+
else:
|
| 131 |
+
never_converged = True
|
| 132 |
+
if verbose:
|
| 133 |
+
print("Did not converge")
|
| 134 |
+
|
| 135 |
+
I = np.flatnonzero(E)
|
| 136 |
+
K = I.size # Identify exemplars
|
| 137 |
+
|
| 138 |
+
if K > 0:
|
| 139 |
+
if never_converged:
|
| 140 |
+
warnings.warn(
|
| 141 |
+
(
|
| 142 |
+
"Affinity propagation did not converge, this model "
|
| 143 |
+
"may return degenerate cluster centers and labels."
|
| 144 |
+
),
|
| 145 |
+
ConvergenceWarning,
|
| 146 |
+
)
|
| 147 |
+
c = np.argmax(S[:, I], axis=1)
|
| 148 |
+
c[I] = np.arange(K) # Identify clusters
|
| 149 |
+
# Refine the final set of exemplars and clusters and return results
|
| 150 |
+
for k in range(K):
|
| 151 |
+
ii = np.asarray(c == k).nonzero()[0]
|
| 152 |
+
j = np.argmax(np.sum(S[ii[:, np.newaxis], ii], axis=0))
|
| 153 |
+
I[k] = ii[j]
|
| 154 |
+
|
| 155 |
+
c = np.argmax(S[:, I], axis=1)
|
| 156 |
+
c[I] = np.arange(K)
|
| 157 |
+
labels = I[c]
|
| 158 |
+
# Reduce labels to a sorted, gapless, list
|
| 159 |
+
cluster_centers_indices = np.unique(labels)
|
| 160 |
+
labels = np.searchsorted(cluster_centers_indices, labels)
|
| 161 |
+
else:
|
| 162 |
+
warnings.warn(
|
| 163 |
+
(
|
| 164 |
+
"Affinity propagation did not converge and this model "
|
| 165 |
+
"will not have any cluster centers."
|
| 166 |
+
),
|
| 167 |
+
ConvergenceWarning,
|
| 168 |
+
)
|
| 169 |
+
labels = np.array([-1] * n_samples)
|
| 170 |
+
cluster_centers_indices = []
|
| 171 |
+
|
| 172 |
+
if return_n_iter:
|
| 173 |
+
return cluster_centers_indices, labels, it + 1
|
| 174 |
+
else:
|
| 175 |
+
return cluster_centers_indices, labels
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
###############################################################################
|
| 179 |
+
# Public API
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
@validate_params(
|
| 183 |
+
{
|
| 184 |
+
"S": ["array-like"],
|
| 185 |
+
"return_n_iter": ["boolean"],
|
| 186 |
+
},
|
| 187 |
+
prefer_skip_nested_validation=False,
|
| 188 |
+
)
|
| 189 |
+
def affinity_propagation(
|
| 190 |
+
S,
|
| 191 |
+
*,
|
| 192 |
+
preference=None,
|
| 193 |
+
convergence_iter=15,
|
| 194 |
+
max_iter=200,
|
| 195 |
+
damping=0.5,
|
| 196 |
+
copy=True,
|
| 197 |
+
verbose=False,
|
| 198 |
+
return_n_iter=False,
|
| 199 |
+
random_state=None,
|
| 200 |
+
):
|
| 201 |
+
"""Perform Affinity Propagation Clustering of data.
|
| 202 |
+
|
| 203 |
+
Read more in the :ref:`User Guide <affinity_propagation>`.
|
| 204 |
+
|
| 205 |
+
Parameters
|
| 206 |
+
----------
|
| 207 |
+
S : array-like of shape (n_samples, n_samples)
|
| 208 |
+
Matrix of similarities between points.
|
| 209 |
+
|
| 210 |
+
preference : array-like of shape (n_samples,) or float, default=None
|
| 211 |
+
Preferences for each point - points with larger values of
|
| 212 |
+
preferences are more likely to be chosen as exemplars. The number of
|
| 213 |
+
exemplars, i.e. of clusters, is influenced by the input preferences
|
| 214 |
+
value. If the preferences are not passed as arguments, they will be
|
| 215 |
+
set to the median of the input similarities (resulting in a moderate
|
| 216 |
+
number of clusters). For a smaller amount of clusters, this can be set
|
| 217 |
+
to the minimum value of the similarities.
|
| 218 |
+
|
| 219 |
+
convergence_iter : int, default=15
|
| 220 |
+
Number of iterations with no change in the number
|
| 221 |
+
of estimated clusters that stops the convergence.
|
| 222 |
+
|
| 223 |
+
max_iter : int, default=200
|
| 224 |
+
Maximum number of iterations.
|
| 225 |
+
|
| 226 |
+
damping : float, default=0.5
|
| 227 |
+
Damping factor between 0.5 and 1.
|
| 228 |
+
|
| 229 |
+
copy : bool, default=True
|
| 230 |
+
If copy is False, the affinity matrix is modified inplace by the
|
| 231 |
+
algorithm, for memory efficiency.
|
| 232 |
+
|
| 233 |
+
verbose : bool, default=False
|
| 234 |
+
The verbosity level.
|
| 235 |
+
|
| 236 |
+
return_n_iter : bool, default=False
|
| 237 |
+
Whether or not to return the number of iterations.
|
| 238 |
+
|
| 239 |
+
random_state : int, RandomState instance or None, default=None
|
| 240 |
+
Pseudo-random number generator to control the starting state.
|
| 241 |
+
Use an int for reproducible results across function calls.
|
| 242 |
+
See the :term:`Glossary <random_state>`.
|
| 243 |
+
|
| 244 |
+
.. versionadded:: 0.23
|
| 245 |
+
this parameter was previously hardcoded as 0.
|
| 246 |
+
|
| 247 |
+
Returns
|
| 248 |
+
-------
|
| 249 |
+
cluster_centers_indices : ndarray of shape (n_clusters,)
|
| 250 |
+
Index of clusters centers.
|
| 251 |
+
|
| 252 |
+
labels : ndarray of shape (n_samples,)
|
| 253 |
+
Cluster labels for each point.
|
| 254 |
+
|
| 255 |
+
n_iter : int
|
| 256 |
+
Number of iterations run. Returned only if `return_n_iter` is
|
| 257 |
+
set to True.
|
| 258 |
+
|
| 259 |
+
Notes
|
| 260 |
+
-----
|
| 261 |
+
For an example usage,
|
| 262 |
+
see :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`.
|
| 263 |
+
You may also check out,
|
| 264 |
+
:ref:`sphx_glr_auto_examples_applications_plot_stock_market.py`
|
| 265 |
+
|
| 266 |
+
When the algorithm does not converge, it will still return an array of
|
| 267 |
+
``cluster_center_indices`` and labels if there are any exemplars/clusters,
|
| 268 |
+
however they may be degenerate and should be used with caution.
|
| 269 |
+
|
| 270 |
+
When all training samples have equal similarities and equal preferences,
|
| 271 |
+
the assignment of cluster centers and labels depends on the preference.
|
| 272 |
+
If the preference is smaller than the similarities, a single cluster center
|
| 273 |
+
and label ``0`` for every sample will be returned. Otherwise, every
|
| 274 |
+
training sample becomes its own cluster center and is assigned a unique
|
| 275 |
+
label.
|
| 276 |
+
|
| 277 |
+
References
|
| 278 |
+
----------
|
| 279 |
+
Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages
|
| 280 |
+
Between Data Points", Science Feb. 2007
|
| 281 |
+
|
| 282 |
+
Examples
|
| 283 |
+
--------
|
| 284 |
+
>>> import numpy as np
|
| 285 |
+
>>> from sklearn.cluster import affinity_propagation
|
| 286 |
+
>>> from sklearn.metrics.pairwise import euclidean_distances
|
| 287 |
+
>>> X = np.array([[1, 2], [1, 4], [1, 0],
|
| 288 |
+
... [4, 2], [4, 4], [4, 0]])
|
| 289 |
+
>>> S = -euclidean_distances(X, squared=True)
|
| 290 |
+
>>> cluster_centers_indices, labels = affinity_propagation(S, random_state=0)
|
| 291 |
+
>>> cluster_centers_indices
|
| 292 |
+
array([0, 3])
|
| 293 |
+
>>> labels
|
| 294 |
+
array([0, 0, 0, 1, 1, 1])
|
| 295 |
+
"""
|
| 296 |
+
estimator = AffinityPropagation(
|
| 297 |
+
damping=damping,
|
| 298 |
+
max_iter=max_iter,
|
| 299 |
+
convergence_iter=convergence_iter,
|
| 300 |
+
copy=copy,
|
| 301 |
+
preference=preference,
|
| 302 |
+
affinity="precomputed",
|
| 303 |
+
verbose=verbose,
|
| 304 |
+
random_state=random_state,
|
| 305 |
+
).fit(S)
|
| 306 |
+
|
| 307 |
+
if return_n_iter:
|
| 308 |
+
return estimator.cluster_centers_indices_, estimator.labels_, estimator.n_iter_
|
| 309 |
+
return estimator.cluster_centers_indices_, estimator.labels_
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
class AffinityPropagation(ClusterMixin, BaseEstimator):
|
| 313 |
+
"""Perform Affinity Propagation Clustering of data.
|
| 314 |
+
|
| 315 |
+
Read more in the :ref:`User Guide <affinity_propagation>`.
|
| 316 |
+
|
| 317 |
+
Parameters
|
| 318 |
+
----------
|
| 319 |
+
damping : float, default=0.5
|
| 320 |
+
Damping factor in the range `[0.5, 1.0)` is the extent to
|
| 321 |
+
which the current value is maintained relative to
|
| 322 |
+
incoming values (weighted 1 - damping). This in order
|
| 323 |
+
to avoid numerical oscillations when updating these
|
| 324 |
+
values (messages).
|
| 325 |
+
|
| 326 |
+
max_iter : int, default=200
|
| 327 |
+
Maximum number of iterations.
|
| 328 |
+
|
| 329 |
+
convergence_iter : int, default=15
|
| 330 |
+
Number of iterations with no change in the number
|
| 331 |
+
of estimated clusters that stops the convergence.
|
| 332 |
+
|
| 333 |
+
copy : bool, default=True
|
| 334 |
+
Make a copy of input data.
|
| 335 |
+
|
| 336 |
+
preference : array-like of shape (n_samples,) or float, default=None
|
| 337 |
+
Preferences for each point - points with larger values of
|
| 338 |
+
preferences are more likely to be chosen as exemplars. The number
|
| 339 |
+
of exemplars, ie of clusters, is influenced by the input
|
| 340 |
+
preferences value. If the preferences are not passed as arguments,
|
| 341 |
+
they will be set to the median of the input similarities.
|
| 342 |
+
|
| 343 |
+
affinity : {'euclidean', 'precomputed'}, default='euclidean'
|
| 344 |
+
Which affinity to use. At the moment 'precomputed' and
|
| 345 |
+
``euclidean`` are supported. 'euclidean' uses the
|
| 346 |
+
negative squared euclidean distance between points.
|
| 347 |
+
|
| 348 |
+
verbose : bool, default=False
|
| 349 |
+
Whether to be verbose.
|
| 350 |
+
|
| 351 |
+
random_state : int, RandomState instance or None, default=None
|
| 352 |
+
Pseudo-random number generator to control the starting state.
|
| 353 |
+
Use an int for reproducible results across function calls.
|
| 354 |
+
See the :term:`Glossary <random_state>`.
|
| 355 |
+
|
| 356 |
+
.. versionadded:: 0.23
|
| 357 |
+
this parameter was previously hardcoded as 0.
|
| 358 |
+
|
| 359 |
+
Attributes
|
| 360 |
+
----------
|
| 361 |
+
cluster_centers_indices_ : ndarray of shape (n_clusters,)
|
| 362 |
+
Indices of cluster centers.
|
| 363 |
+
|
| 364 |
+
cluster_centers_ : ndarray of shape (n_clusters, n_features)
|
| 365 |
+
Cluster centers (if affinity != ``precomputed``).
|
| 366 |
+
|
| 367 |
+
labels_ : ndarray of shape (n_samples,)
|
| 368 |
+
Labels of each point.
|
| 369 |
+
|
| 370 |
+
affinity_matrix_ : ndarray of shape (n_samples, n_samples)
|
| 371 |
+
Stores the affinity matrix used in ``fit``.
|
| 372 |
+
|
| 373 |
+
n_iter_ : int
|
| 374 |
+
Number of iterations taken to converge.
|
| 375 |
+
|
| 376 |
+
n_features_in_ : int
|
| 377 |
+
Number of features seen during :term:`fit`.
|
| 378 |
+
|
| 379 |
+
.. versionadded:: 0.24
|
| 380 |
+
|
| 381 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 382 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 383 |
+
has feature names that are all strings.
|
| 384 |
+
|
| 385 |
+
.. versionadded:: 1.0
|
| 386 |
+
|
| 387 |
+
See Also
|
| 388 |
+
--------
|
| 389 |
+
AgglomerativeClustering : Recursively merges the pair of
|
| 390 |
+
clusters that minimally increases a given linkage distance.
|
| 391 |
+
FeatureAgglomeration : Similar to AgglomerativeClustering,
|
| 392 |
+
but recursively merges features instead of samples.
|
| 393 |
+
KMeans : K-Means clustering.
|
| 394 |
+
MiniBatchKMeans : Mini-Batch K-Means clustering.
|
| 395 |
+
MeanShift : Mean shift clustering using a flat kernel.
|
| 396 |
+
SpectralClustering : Apply clustering to a projection
|
| 397 |
+
of the normalized Laplacian.
|
| 398 |
+
|
| 399 |
+
Notes
|
| 400 |
+
-----
|
| 401 |
+
The algorithmic complexity of affinity propagation is quadratic
|
| 402 |
+
in the number of points.
|
| 403 |
+
|
| 404 |
+
When the algorithm does not converge, it will still return an array of
|
| 405 |
+
``cluster_center_indices`` and labels if there are any exemplars/clusters,
|
| 406 |
+
however they may be degenerate and should be used with caution.
|
| 407 |
+
|
| 408 |
+
When ``fit`` does not converge, ``cluster_centers_`` is still populated
|
| 409 |
+
however it may be degenerate. In such a case, proceed with caution.
|
| 410 |
+
If ``fit`` does not converge and fails to produce any ``cluster_centers_``
|
| 411 |
+
then ``predict`` will label every sample as ``-1``.
|
| 412 |
+
|
| 413 |
+
When all training samples have equal similarities and equal preferences,
|
| 414 |
+
the assignment of cluster centers and labels depends on the preference.
|
| 415 |
+
If the preference is smaller than the similarities, ``fit`` will result in
|
| 416 |
+
a single cluster center and label ``0`` for every sample. Otherwise, every
|
| 417 |
+
training sample becomes its own cluster center and is assigned a unique
|
| 418 |
+
label.
|
| 419 |
+
|
| 420 |
+
References
|
| 421 |
+
----------
|
| 422 |
+
|
| 423 |
+
Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages
|
| 424 |
+
Between Data Points", Science Feb. 2007
|
| 425 |
+
|
| 426 |
+
Examples
|
| 427 |
+
--------
|
| 428 |
+
>>> from sklearn.cluster import AffinityPropagation
|
| 429 |
+
>>> import numpy as np
|
| 430 |
+
>>> X = np.array([[1, 2], [1, 4], [1, 0],
|
| 431 |
+
... [4, 2], [4, 4], [4, 0]])
|
| 432 |
+
>>> clustering = AffinityPropagation(random_state=5).fit(X)
|
| 433 |
+
>>> clustering
|
| 434 |
+
AffinityPropagation(random_state=5)
|
| 435 |
+
>>> clustering.labels_
|
| 436 |
+
array([0, 0, 0, 1, 1, 1])
|
| 437 |
+
>>> clustering.predict([[0, 0], [4, 4]])
|
| 438 |
+
array([0, 1])
|
| 439 |
+
>>> clustering.cluster_centers_
|
| 440 |
+
array([[1, 2],
|
| 441 |
+
[4, 2]])
|
| 442 |
+
|
| 443 |
+
For an example usage,
|
| 444 |
+
see :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`.
|
| 445 |
+
|
| 446 |
+
For a comparison of Affinity Propagation with other clustering algorithms, see
|
| 447 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
|
| 448 |
+
"""
|
| 449 |
+
|
| 450 |
+
_parameter_constraints: dict = {
|
| 451 |
+
"damping": [Interval(Real, 0.5, 1.0, closed="left")],
|
| 452 |
+
"max_iter": [Interval(Integral, 1, None, closed="left")],
|
| 453 |
+
"convergence_iter": [Interval(Integral, 1, None, closed="left")],
|
| 454 |
+
"copy": ["boolean"],
|
| 455 |
+
"preference": [
|
| 456 |
+
"array-like",
|
| 457 |
+
Interval(Real, None, None, closed="neither"),
|
| 458 |
+
None,
|
| 459 |
+
],
|
| 460 |
+
"affinity": [StrOptions({"euclidean", "precomputed"})],
|
| 461 |
+
"verbose": ["verbose"],
|
| 462 |
+
"random_state": ["random_state"],
|
| 463 |
+
}
|
| 464 |
+
|
| 465 |
+
def __init__(
|
| 466 |
+
self,
|
| 467 |
+
*,
|
| 468 |
+
damping=0.5,
|
| 469 |
+
max_iter=200,
|
| 470 |
+
convergence_iter=15,
|
| 471 |
+
copy=True,
|
| 472 |
+
preference=None,
|
| 473 |
+
affinity="euclidean",
|
| 474 |
+
verbose=False,
|
| 475 |
+
random_state=None,
|
| 476 |
+
):
|
| 477 |
+
self.damping = damping
|
| 478 |
+
self.max_iter = max_iter
|
| 479 |
+
self.convergence_iter = convergence_iter
|
| 480 |
+
self.copy = copy
|
| 481 |
+
self.verbose = verbose
|
| 482 |
+
self.preference = preference
|
| 483 |
+
self.affinity = affinity
|
| 484 |
+
self.random_state = random_state
|
| 485 |
+
|
| 486 |
+
def __sklearn_tags__(self):
|
| 487 |
+
tags = super().__sklearn_tags__()
|
| 488 |
+
tags.input_tags.pairwise = self.affinity == "precomputed"
|
| 489 |
+
tags.input_tags.sparse = self.affinity != "precomputed"
|
| 490 |
+
return tags
|
| 491 |
+
|
| 492 |
+
@_fit_context(prefer_skip_nested_validation=True)
|
| 493 |
+
def fit(self, X, y=None):
|
| 494 |
+
"""Fit the clustering from features, or affinity matrix.
|
| 495 |
+
|
| 496 |
+
Parameters
|
| 497 |
+
----------
|
| 498 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
|
| 499 |
+
array-like of shape (n_samples, n_samples)
|
| 500 |
+
Training instances to cluster, or similarities / affinities between
|
| 501 |
+
instances if ``affinity='precomputed'``. If a sparse feature matrix
|
| 502 |
+
is provided, it will be converted into a sparse ``csr_matrix``.
|
| 503 |
+
|
| 504 |
+
y : Ignored
|
| 505 |
+
Not used, present here for API consistency by convention.
|
| 506 |
+
|
| 507 |
+
Returns
|
| 508 |
+
-------
|
| 509 |
+
self
|
| 510 |
+
Returns the instance itself.
|
| 511 |
+
"""
|
| 512 |
+
if self.affinity == "precomputed":
|
| 513 |
+
X = validate_data(self, X, copy=self.copy, force_writeable=True)
|
| 514 |
+
self.affinity_matrix_ = X
|
| 515 |
+
else: # self.affinity == "euclidean"
|
| 516 |
+
X = validate_data(self, X, accept_sparse="csr")
|
| 517 |
+
self.affinity_matrix_ = -euclidean_distances(X, squared=True)
|
| 518 |
+
|
| 519 |
+
if self.affinity_matrix_.shape[0] != self.affinity_matrix_.shape[1]:
|
| 520 |
+
raise ValueError(
|
| 521 |
+
"The matrix of similarities must be a square array. "
|
| 522 |
+
f"Got {self.affinity_matrix_.shape} instead."
|
| 523 |
+
)
|
| 524 |
+
|
| 525 |
+
if self.preference is None:
|
| 526 |
+
preference = np.median(self.affinity_matrix_)
|
| 527 |
+
else:
|
| 528 |
+
preference = self.preference
|
| 529 |
+
preference = np.asarray(preference)
|
| 530 |
+
|
| 531 |
+
random_state = check_random_state(self.random_state)
|
| 532 |
+
|
| 533 |
+
(
|
| 534 |
+
self.cluster_centers_indices_,
|
| 535 |
+
self.labels_,
|
| 536 |
+
self.n_iter_,
|
| 537 |
+
) = _affinity_propagation(
|
| 538 |
+
self.affinity_matrix_,
|
| 539 |
+
max_iter=self.max_iter,
|
| 540 |
+
convergence_iter=self.convergence_iter,
|
| 541 |
+
preference=preference,
|
| 542 |
+
damping=self.damping,
|
| 543 |
+
verbose=self.verbose,
|
| 544 |
+
return_n_iter=True,
|
| 545 |
+
random_state=random_state,
|
| 546 |
+
)
|
| 547 |
+
|
| 548 |
+
if self.affinity != "precomputed":
|
| 549 |
+
self.cluster_centers_ = X[self.cluster_centers_indices_].copy()
|
| 550 |
+
|
| 551 |
+
return self
|
| 552 |
+
|
| 553 |
+
def predict(self, X):
|
| 554 |
+
"""Predict the closest cluster each sample in X belongs to.
|
| 555 |
+
|
| 556 |
+
Parameters
|
| 557 |
+
----------
|
| 558 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 559 |
+
New data to predict. If a sparse matrix is provided, it will be
|
| 560 |
+
converted into a sparse ``csr_matrix``.
|
| 561 |
+
|
| 562 |
+
Returns
|
| 563 |
+
-------
|
| 564 |
+
labels : ndarray of shape (n_samples,)
|
| 565 |
+
Cluster labels.
|
| 566 |
+
"""
|
| 567 |
+
check_is_fitted(self)
|
| 568 |
+
X = validate_data(self, X, reset=False, accept_sparse="csr")
|
| 569 |
+
if not hasattr(self, "cluster_centers_"):
|
| 570 |
+
raise ValueError(
|
| 571 |
+
"Predict method is not supported when affinity='precomputed'."
|
| 572 |
+
)
|
| 573 |
+
|
| 574 |
+
if self.cluster_centers_.shape[0] > 0:
|
| 575 |
+
with config_context(assume_finite=True):
|
| 576 |
+
return pairwise_distances_argmin(X, self.cluster_centers_)
|
| 577 |
+
else:
|
| 578 |
+
warnings.warn(
|
| 579 |
+
(
|
| 580 |
+
"This model does not have any cluster centers "
|
| 581 |
+
"because affinity propagation did not converge. "
|
| 582 |
+
"Labeling every sample as '-1'."
|
| 583 |
+
),
|
| 584 |
+
ConvergenceWarning,
|
| 585 |
+
)
|
| 586 |
+
return np.array([-1] * X.shape[0])
|
| 587 |
+
|
| 588 |
+
def fit_predict(self, X, y=None):
|
| 589 |
+
"""Fit clustering from features/affinity matrix; return cluster labels.
|
| 590 |
+
|
| 591 |
+
Parameters
|
| 592 |
+
----------
|
| 593 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
|
| 594 |
+
array-like of shape (n_samples, n_samples)
|
| 595 |
+
Training instances to cluster, or similarities / affinities between
|
| 596 |
+
instances if ``affinity='precomputed'``. If a sparse feature matrix
|
| 597 |
+
is provided, it will be converted into a sparse ``csr_matrix``.
|
| 598 |
+
|
| 599 |
+
y : Ignored
|
| 600 |
+
Not used, present here for API consistency by convention.
|
| 601 |
+
|
| 602 |
+
Returns
|
| 603 |
+
-------
|
| 604 |
+
labels : ndarray of shape (n_samples,)
|
| 605 |
+
Cluster labels.
|
| 606 |
+
"""
|
| 607 |
+
return super().fit_predict(X, y)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_agglomerative.py
ADDED
|
@@ -0,0 +1,1335 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hierarchical Agglomerative Clustering
|
| 2 |
+
|
| 3 |
+
These routines perform some hierarchical agglomerative clustering of some
|
| 4 |
+
input data.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
# Authors: The scikit-learn developers
|
| 8 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 9 |
+
|
| 10 |
+
import warnings
|
| 11 |
+
from heapq import heapify, heappop, heappush, heappushpop
|
| 12 |
+
from numbers import Integral, Real
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
from scipy import sparse
|
| 16 |
+
from scipy.sparse.csgraph import connected_components
|
| 17 |
+
|
| 18 |
+
from sklearn.base import (
|
| 19 |
+
BaseEstimator,
|
| 20 |
+
ClassNamePrefixFeaturesOutMixin,
|
| 21 |
+
ClusterMixin,
|
| 22 |
+
_fit_context,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
# mypy error: Module 'sklearn.cluster' has no attribute '_hierarchical_fast'
|
| 26 |
+
from sklearn.cluster import ( # type: ignore[attr-defined]
|
| 27 |
+
_hierarchical_fast as _hierarchical,
|
| 28 |
+
)
|
| 29 |
+
from sklearn.cluster._feature_agglomeration import AgglomerationTransform
|
| 30 |
+
from sklearn.metrics import DistanceMetric
|
| 31 |
+
from sklearn.metrics._dist_metrics import METRIC_MAPPING64
|
| 32 |
+
from sklearn.metrics.pairwise import _VALID_METRICS, paired_distances
|
| 33 |
+
from sklearn.utils import check_array
|
| 34 |
+
from sklearn.utils._fast_dict import IntFloatDict
|
| 35 |
+
from sklearn.utils._param_validation import (
|
| 36 |
+
HasMethods,
|
| 37 |
+
Interval,
|
| 38 |
+
StrOptions,
|
| 39 |
+
validate_params,
|
| 40 |
+
)
|
| 41 |
+
from sklearn.utils.graph import _fix_connected_components
|
| 42 |
+
from sklearn.utils.validation import check_memory, validate_data
|
| 43 |
+
|
| 44 |
+
###############################################################################
|
| 45 |
+
# For non fully-connected graphs
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _fix_connectivity(X, connectivity, affinity):
|
| 49 |
+
"""
|
| 50 |
+
Fixes the connectivity matrix.
|
| 51 |
+
|
| 52 |
+
The different steps are:
|
| 53 |
+
|
| 54 |
+
- copies it
|
| 55 |
+
- makes it symmetric
|
| 56 |
+
- converts it to LIL if necessary
|
| 57 |
+
- completes it if necessary.
|
| 58 |
+
|
| 59 |
+
Parameters
|
| 60 |
+
----------
|
| 61 |
+
X : array-like of shape (n_samples, n_features)
|
| 62 |
+
Feature matrix representing `n_samples` samples to be clustered.
|
| 63 |
+
|
| 64 |
+
connectivity : sparse matrix, default=None
|
| 65 |
+
Connectivity matrix. Defines for each sample the neighboring samples
|
| 66 |
+
following a given structure of the data. The matrix is assumed to
|
| 67 |
+
be symmetric and only the upper triangular half is used.
|
| 68 |
+
Default is `None`, i.e, the Ward algorithm is unstructured.
|
| 69 |
+
|
| 70 |
+
affinity : {"euclidean", "precomputed"}, default="euclidean"
|
| 71 |
+
Which affinity to use. At the moment `precomputed` and
|
| 72 |
+
``euclidean`` are supported. `euclidean` uses the
|
| 73 |
+
negative squared Euclidean distance between points.
|
| 74 |
+
|
| 75 |
+
Returns
|
| 76 |
+
-------
|
| 77 |
+
connectivity : sparse matrix
|
| 78 |
+
The fixed connectivity matrix.
|
| 79 |
+
|
| 80 |
+
n_connected_components : int
|
| 81 |
+
The number of connected components in the graph.
|
| 82 |
+
"""
|
| 83 |
+
n_samples = X.shape[0]
|
| 84 |
+
if connectivity.shape[0] != n_samples or connectivity.shape[1] != n_samples:
|
| 85 |
+
raise ValueError(
|
| 86 |
+
"Wrong shape for connectivity matrix: %s when X is %s"
|
| 87 |
+
% (connectivity.shape, X.shape)
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
# Make the connectivity matrix symmetric:
|
| 91 |
+
connectivity = connectivity + connectivity.T
|
| 92 |
+
|
| 93 |
+
# Convert connectivity matrix to LIL
|
| 94 |
+
if not sparse.issparse(connectivity):
|
| 95 |
+
connectivity = sparse.lil_matrix(connectivity)
|
| 96 |
+
|
| 97 |
+
# `connectivity` is a sparse matrix at this point
|
| 98 |
+
if connectivity.format != "lil":
|
| 99 |
+
connectivity = connectivity.tolil()
|
| 100 |
+
|
| 101 |
+
# Compute the number of nodes
|
| 102 |
+
n_connected_components, labels = connected_components(connectivity)
|
| 103 |
+
|
| 104 |
+
if n_connected_components > 1:
|
| 105 |
+
warnings.warn(
|
| 106 |
+
"the number of connected components of the "
|
| 107 |
+
"connectivity matrix is %d > 1. Completing it to avoid "
|
| 108 |
+
"stopping the tree early." % n_connected_components,
|
| 109 |
+
stacklevel=2,
|
| 110 |
+
)
|
| 111 |
+
# XXX: Can we do without completing the matrix?
|
| 112 |
+
connectivity = _fix_connected_components(
|
| 113 |
+
X=X,
|
| 114 |
+
graph=connectivity,
|
| 115 |
+
n_connected_components=n_connected_components,
|
| 116 |
+
component_labels=labels,
|
| 117 |
+
metric=affinity,
|
| 118 |
+
mode="connectivity",
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
return connectivity, n_connected_components
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _single_linkage_tree(
|
| 125 |
+
connectivity,
|
| 126 |
+
n_samples,
|
| 127 |
+
n_nodes,
|
| 128 |
+
n_clusters,
|
| 129 |
+
n_connected_components,
|
| 130 |
+
return_distance,
|
| 131 |
+
):
|
| 132 |
+
"""
|
| 133 |
+
Perform single linkage clustering on sparse data via the minimum
|
| 134 |
+
spanning tree from scipy.sparse.csgraph, then using union-find to label.
|
| 135 |
+
The parent array is then generated by walking through the tree.
|
| 136 |
+
"""
|
| 137 |
+
from scipy.sparse.csgraph import minimum_spanning_tree
|
| 138 |
+
|
| 139 |
+
# explicitly cast connectivity to ensure safety
|
| 140 |
+
connectivity = connectivity.astype(np.float64, copy=False)
|
| 141 |
+
|
| 142 |
+
# Ensure zero distances aren't ignored by setting them to "epsilon"
|
| 143 |
+
epsilon_value = np.finfo(dtype=connectivity.data.dtype).eps
|
| 144 |
+
connectivity.data[connectivity.data == 0] = epsilon_value
|
| 145 |
+
|
| 146 |
+
# Use scipy.sparse.csgraph to generate a minimum spanning tree
|
| 147 |
+
mst = minimum_spanning_tree(connectivity.tocsr())
|
| 148 |
+
|
| 149 |
+
# Convert the graph to scipy.cluster.hierarchy array format
|
| 150 |
+
mst = mst.tocoo()
|
| 151 |
+
|
| 152 |
+
# Undo the epsilon values
|
| 153 |
+
mst.data[mst.data == epsilon_value] = 0
|
| 154 |
+
|
| 155 |
+
mst_array = np.vstack([mst.row, mst.col, mst.data]).T
|
| 156 |
+
|
| 157 |
+
# Sort edges of the min_spanning_tree by weight
|
| 158 |
+
mst_array = mst_array[np.argsort(mst_array.T[2], kind="mergesort"), :]
|
| 159 |
+
|
| 160 |
+
# Convert edge list into standard hierarchical clustering format
|
| 161 |
+
single_linkage_tree = _hierarchical._single_linkage_label(mst_array)
|
| 162 |
+
children_ = single_linkage_tree[:, :2].astype(int)
|
| 163 |
+
|
| 164 |
+
# Compute parents
|
| 165 |
+
parent = np.arange(n_nodes, dtype=np.intp)
|
| 166 |
+
for i, (left, right) in enumerate(children_, n_samples):
|
| 167 |
+
if n_clusters is not None and i >= n_nodes:
|
| 168 |
+
break
|
| 169 |
+
if left < n_nodes:
|
| 170 |
+
parent[left] = i
|
| 171 |
+
if right < n_nodes:
|
| 172 |
+
parent[right] = i
|
| 173 |
+
|
| 174 |
+
if return_distance:
|
| 175 |
+
distances = single_linkage_tree[:, 2]
|
| 176 |
+
return children_, n_connected_components, n_samples, parent, distances
|
| 177 |
+
return children_, n_connected_components, n_samples, parent
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
###############################################################################
|
| 181 |
+
# Hierarchical tree building functions
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
@validate_params(
|
| 185 |
+
{
|
| 186 |
+
"X": ["array-like"],
|
| 187 |
+
"connectivity": ["array-like", "sparse matrix", None],
|
| 188 |
+
"n_clusters": [Interval(Integral, 1, None, closed="left"), None],
|
| 189 |
+
"return_distance": ["boolean"],
|
| 190 |
+
},
|
| 191 |
+
prefer_skip_nested_validation=True,
|
| 192 |
+
)
|
| 193 |
+
def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False):
|
| 194 |
+
"""Ward clustering based on a Feature matrix.
|
| 195 |
+
|
| 196 |
+
Recursively merges the pair of clusters that minimally increases
|
| 197 |
+
within-cluster variance.
|
| 198 |
+
|
| 199 |
+
The inertia matrix uses a Heapq-based representation.
|
| 200 |
+
|
| 201 |
+
This is the structured version, that takes into account some topological
|
| 202 |
+
structure between samples.
|
| 203 |
+
|
| 204 |
+
Read more in the :ref:`User Guide <hierarchical_clustering>`.
|
| 205 |
+
|
| 206 |
+
Parameters
|
| 207 |
+
----------
|
| 208 |
+
X : array-like of shape (n_samples, n_features)
|
| 209 |
+
Feature matrix representing `n_samples` samples to be clustered.
|
| 210 |
+
|
| 211 |
+
connectivity : {array-like, sparse matrix}, default=None
|
| 212 |
+
Connectivity matrix. Defines for each sample the neighboring samples
|
| 213 |
+
following a given structure of the data. The matrix is assumed to
|
| 214 |
+
be symmetric and only the upper triangular half is used.
|
| 215 |
+
Default is None, i.e, the Ward algorithm is unstructured.
|
| 216 |
+
|
| 217 |
+
n_clusters : int, default=None
|
| 218 |
+
`n_clusters` should be less than `n_samples`. Stop early the
|
| 219 |
+
construction of the tree at `n_clusters.` This is useful to decrease
|
| 220 |
+
computation time if the number of clusters is not small compared to the
|
| 221 |
+
number of samples. In this case, the complete tree is not computed, thus
|
| 222 |
+
the 'children' output is of limited use, and the 'parents' output should
|
| 223 |
+
rather be used. This option is valid only when specifying a connectivity
|
| 224 |
+
matrix.
|
| 225 |
+
|
| 226 |
+
return_distance : bool, default=False
|
| 227 |
+
If `True`, return the distance between the clusters.
|
| 228 |
+
|
| 229 |
+
Returns
|
| 230 |
+
-------
|
| 231 |
+
children : ndarray of shape (n_nodes-1, 2)
|
| 232 |
+
The children of each non-leaf node. Values less than `n_samples`
|
| 233 |
+
correspond to leaves of the tree which are the original samples.
|
| 234 |
+
A node `i` greater than or equal to `n_samples` is a non-leaf
|
| 235 |
+
node and has children `children_[i - n_samples]`. Alternatively
|
| 236 |
+
at the i-th iteration, children[i][0] and children[i][1]
|
| 237 |
+
are merged to form node `n_samples + i`.
|
| 238 |
+
|
| 239 |
+
n_connected_components : int
|
| 240 |
+
The number of connected components in the graph.
|
| 241 |
+
|
| 242 |
+
n_leaves : int
|
| 243 |
+
The number of leaves in the tree.
|
| 244 |
+
|
| 245 |
+
parents : ndarray of shape (n_nodes,) or None
|
| 246 |
+
The parent of each node. Only returned when a connectivity matrix
|
| 247 |
+
is specified, elsewhere 'None' is returned.
|
| 248 |
+
|
| 249 |
+
distances : ndarray of shape (n_nodes-1,)
|
| 250 |
+
Only returned if `return_distance` is set to `True` (for compatibility).
|
| 251 |
+
The distances between the centers of the nodes. `distances[i]`
|
| 252 |
+
corresponds to a weighted Euclidean distance between
|
| 253 |
+
the nodes `children[i, 1]` and `children[i, 2]`. If the nodes refer to
|
| 254 |
+
leaves of the tree, then `distances[i]` is their unweighted Euclidean
|
| 255 |
+
distance. Distances are updated in the following way
|
| 256 |
+
(from scipy.hierarchy.linkage):
|
| 257 |
+
|
| 258 |
+
The new entry :math:`d(u,v)` is computed as follows,
|
| 259 |
+
|
| 260 |
+
.. math::
|
| 261 |
+
|
| 262 |
+
d(u,v) = \\sqrt{\\frac{|v|+|s|}
|
| 263 |
+
{T}d(v,s)^2
|
| 264 |
+
+ \\frac{|v|+|t|}
|
| 265 |
+
{T}d(v,t)^2
|
| 266 |
+
- \\frac{|v|}
|
| 267 |
+
{T}d(s,t)^2}
|
| 268 |
+
|
| 269 |
+
where :math:`u` is the newly joined cluster consisting of
|
| 270 |
+
clusters :math:`s` and :math:`t`, :math:`v` is an unused
|
| 271 |
+
cluster in the forest, :math:`T=|v|+|s|+|t|`, and
|
| 272 |
+
:math:`|*|` is the cardinality of its argument. This is also
|
| 273 |
+
known as the incremental algorithm.
|
| 274 |
+
|
| 275 |
+
Examples
|
| 276 |
+
--------
|
| 277 |
+
>>> import numpy as np
|
| 278 |
+
>>> from sklearn.cluster import ward_tree
|
| 279 |
+
>>> X = np.array([[1, 2], [1, 4], [1, 0],
|
| 280 |
+
... [4, 2], [4, 4], [4, 0]])
|
| 281 |
+
>>> children, n_connected_components, n_leaves, parents = ward_tree(X)
|
| 282 |
+
>>> children
|
| 283 |
+
array([[0, 1],
|
| 284 |
+
[3, 5],
|
| 285 |
+
[2, 6],
|
| 286 |
+
[4, 7],
|
| 287 |
+
[8, 9]])
|
| 288 |
+
>>> n_connected_components
|
| 289 |
+
1
|
| 290 |
+
>>> n_leaves
|
| 291 |
+
6
|
| 292 |
+
"""
|
| 293 |
+
X = np.asarray(X)
|
| 294 |
+
if X.ndim == 1:
|
| 295 |
+
X = np.reshape(X, (-1, 1))
|
| 296 |
+
n_samples, n_features = X.shape
|
| 297 |
+
|
| 298 |
+
if connectivity is None:
|
| 299 |
+
from scipy.cluster import hierarchy # imports PIL
|
| 300 |
+
|
| 301 |
+
if n_clusters is not None:
|
| 302 |
+
warnings.warn(
|
| 303 |
+
(
|
| 304 |
+
"Partial build of the tree is implemented "
|
| 305 |
+
"only for structured clustering (i.e. with "
|
| 306 |
+
"explicit connectivity). The algorithm "
|
| 307 |
+
"will build the full tree and only "
|
| 308 |
+
"retain the lower branches required "
|
| 309 |
+
"for the specified number of clusters"
|
| 310 |
+
),
|
| 311 |
+
stacklevel=2,
|
| 312 |
+
)
|
| 313 |
+
X = np.require(X, requirements="W")
|
| 314 |
+
out = hierarchy.ward(X)
|
| 315 |
+
children_ = out[:, :2].astype(np.intp)
|
| 316 |
+
|
| 317 |
+
if return_distance:
|
| 318 |
+
distances = out[:, 2]
|
| 319 |
+
return children_, 1, n_samples, None, distances
|
| 320 |
+
else:
|
| 321 |
+
return children_, 1, n_samples, None
|
| 322 |
+
|
| 323 |
+
connectivity, n_connected_components = _fix_connectivity(
|
| 324 |
+
X, connectivity, affinity="euclidean"
|
| 325 |
+
)
|
| 326 |
+
if n_clusters is None:
|
| 327 |
+
n_nodes = 2 * n_samples - 1
|
| 328 |
+
else:
|
| 329 |
+
if n_clusters > n_samples:
|
| 330 |
+
raise ValueError(
|
| 331 |
+
"Cannot provide more clusters than samples. "
|
| 332 |
+
"%i n_clusters was asked, and there are %i "
|
| 333 |
+
"samples." % (n_clusters, n_samples)
|
| 334 |
+
)
|
| 335 |
+
n_nodes = 2 * n_samples - n_clusters
|
| 336 |
+
|
| 337 |
+
# create inertia matrix
|
| 338 |
+
coord_row = []
|
| 339 |
+
coord_col = []
|
| 340 |
+
A = []
|
| 341 |
+
for ind, row in enumerate(connectivity.rows):
|
| 342 |
+
A.append(row)
|
| 343 |
+
# We keep only the upper triangular for the moments
|
| 344 |
+
# Generator expressions are faster than arrays on the following
|
| 345 |
+
row = [i for i in row if i < ind]
|
| 346 |
+
coord_row.extend(
|
| 347 |
+
len(row)
|
| 348 |
+
* [
|
| 349 |
+
ind,
|
| 350 |
+
]
|
| 351 |
+
)
|
| 352 |
+
coord_col.extend(row)
|
| 353 |
+
|
| 354 |
+
coord_row = np.array(coord_row, dtype=np.intp, order="C")
|
| 355 |
+
coord_col = np.array(coord_col, dtype=np.intp, order="C")
|
| 356 |
+
|
| 357 |
+
# build moments as a list
|
| 358 |
+
moments_1 = np.zeros(n_nodes, order="C")
|
| 359 |
+
moments_1[:n_samples] = 1
|
| 360 |
+
moments_2 = np.zeros((n_nodes, n_features), order="C")
|
| 361 |
+
moments_2[:n_samples] = X
|
| 362 |
+
inertia = np.empty(len(coord_row), dtype=np.float64, order="C")
|
| 363 |
+
_hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col, inertia)
|
| 364 |
+
inertia = list(zip(inertia, coord_row, coord_col))
|
| 365 |
+
heapify(inertia)
|
| 366 |
+
|
| 367 |
+
# prepare the main fields
|
| 368 |
+
parent = np.arange(n_nodes, dtype=np.intp)
|
| 369 |
+
used_node = np.ones(n_nodes, dtype=bool)
|
| 370 |
+
children = []
|
| 371 |
+
if return_distance:
|
| 372 |
+
distances = np.empty(n_nodes - n_samples)
|
| 373 |
+
|
| 374 |
+
not_visited = np.empty(n_nodes, dtype=bool, order="C")
|
| 375 |
+
|
| 376 |
+
# recursive merge loop
|
| 377 |
+
for k in range(n_samples, n_nodes):
|
| 378 |
+
# identify the merge
|
| 379 |
+
while True:
|
| 380 |
+
inert, i, j = heappop(inertia)
|
| 381 |
+
if used_node[i] and used_node[j]:
|
| 382 |
+
break
|
| 383 |
+
parent[i], parent[j] = k, k
|
| 384 |
+
children.append((i, j))
|
| 385 |
+
used_node[i] = used_node[j] = False
|
| 386 |
+
if return_distance: # store inertia value
|
| 387 |
+
distances[k - n_samples] = inert
|
| 388 |
+
|
| 389 |
+
# update the moments
|
| 390 |
+
moments_1[k] = moments_1[i] + moments_1[j]
|
| 391 |
+
moments_2[k] = moments_2[i] + moments_2[j]
|
| 392 |
+
|
| 393 |
+
# update the structure matrix A and the inertia matrix
|
| 394 |
+
coord_col = []
|
| 395 |
+
not_visited.fill(1)
|
| 396 |
+
not_visited[k] = 0
|
| 397 |
+
_hierarchical._get_parents(A[i], coord_col, parent, not_visited)
|
| 398 |
+
_hierarchical._get_parents(A[j], coord_col, parent, not_visited)
|
| 399 |
+
# List comprehension is faster than a for loop
|
| 400 |
+
[A[col].append(k) for col in coord_col]
|
| 401 |
+
A.append(coord_col)
|
| 402 |
+
coord_col = np.array(coord_col, dtype=np.intp, order="C")
|
| 403 |
+
coord_row = np.empty(coord_col.shape, dtype=np.intp, order="C")
|
| 404 |
+
coord_row.fill(k)
|
| 405 |
+
n_additions = len(coord_row)
|
| 406 |
+
ini = np.empty(n_additions, dtype=np.float64, order="C")
|
| 407 |
+
|
| 408 |
+
_hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col, ini)
|
| 409 |
+
|
| 410 |
+
# List comprehension is faster than a for loop
|
| 411 |
+
[heappush(inertia, (ini[idx], k, coord_col[idx])) for idx in range(n_additions)]
|
| 412 |
+
|
| 413 |
+
# Separate leaves in children (empty lists up to now)
|
| 414 |
+
n_leaves = n_samples
|
| 415 |
+
# sort children to get consistent output with unstructured version
|
| 416 |
+
children = [c[::-1] for c in children]
|
| 417 |
+
children = np.array(children) # return numpy array for efficient caching
|
| 418 |
+
|
| 419 |
+
if return_distance:
|
| 420 |
+
# 2 is scaling factor to compare w/ unstructured version
|
| 421 |
+
distances = np.sqrt(2.0 * distances)
|
| 422 |
+
return children, n_connected_components, n_leaves, parent, distances
|
| 423 |
+
else:
|
| 424 |
+
return children, n_connected_components, n_leaves, parent
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
# single average and complete linkage
|
| 428 |
+
def linkage_tree(
|
| 429 |
+
X,
|
| 430 |
+
connectivity=None,
|
| 431 |
+
n_clusters=None,
|
| 432 |
+
linkage="complete",
|
| 433 |
+
affinity="euclidean",
|
| 434 |
+
return_distance=False,
|
| 435 |
+
):
|
| 436 |
+
"""Linkage agglomerative clustering based on a Feature matrix.
|
| 437 |
+
|
| 438 |
+
The inertia matrix uses a Heapq-based representation.
|
| 439 |
+
|
| 440 |
+
This is the structured version, that takes into account some topological
|
| 441 |
+
structure between samples.
|
| 442 |
+
|
| 443 |
+
Read more in the :ref:`User Guide <hierarchical_clustering>`.
|
| 444 |
+
|
| 445 |
+
Parameters
|
| 446 |
+
----------
|
| 447 |
+
X : array-like of shape (n_samples, n_features)
|
| 448 |
+
Feature matrix representing `n_samples` samples to be clustered.
|
| 449 |
+
|
| 450 |
+
connectivity : sparse matrix, default=None
|
| 451 |
+
Connectivity matrix. Defines for each sample the neighboring samples
|
| 452 |
+
following a given structure of the data. The matrix is assumed to
|
| 453 |
+
be symmetric and only the upper triangular half is used.
|
| 454 |
+
Default is `None`, i.e, the Ward algorithm is unstructured.
|
| 455 |
+
|
| 456 |
+
n_clusters : int, default=None
|
| 457 |
+
Stop early the construction of the tree at `n_clusters`. This is
|
| 458 |
+
useful to decrease computation time if the number of clusters is
|
| 459 |
+
not small compared to the number of samples. In this case, the
|
| 460 |
+
complete tree is not computed, thus the 'children' output is of
|
| 461 |
+
limited use, and the 'parents' output should rather be used.
|
| 462 |
+
This option is valid only when specifying a connectivity matrix.
|
| 463 |
+
|
| 464 |
+
linkage : {"average", "complete", "single"}, default="complete"
|
| 465 |
+
Which linkage criteria to use. The linkage criterion determines which
|
| 466 |
+
distance to use between sets of observation.
|
| 467 |
+
- "average" uses the average of the distances of each observation of
|
| 468 |
+
the two sets.
|
| 469 |
+
- "complete" or maximum linkage uses the maximum distances between
|
| 470 |
+
all observations of the two sets.
|
| 471 |
+
- "single" uses the minimum of the distances between all
|
| 472 |
+
observations of the two sets.
|
| 473 |
+
|
| 474 |
+
affinity : str or callable, default='euclidean'
|
| 475 |
+
Which metric to use. Can be 'euclidean', 'manhattan', or any
|
| 476 |
+
distance known to paired distance (see metric.pairwise).
|
| 477 |
+
|
| 478 |
+
return_distance : bool, default=False
|
| 479 |
+
Whether or not to return the distances between the clusters.
|
| 480 |
+
|
| 481 |
+
Returns
|
| 482 |
+
-------
|
| 483 |
+
children : ndarray of shape (n_nodes-1, 2)
|
| 484 |
+
The children of each non-leaf node. Values less than `n_samples`
|
| 485 |
+
correspond to leaves of the tree which are the original samples.
|
| 486 |
+
A node `i` greater than or equal to `n_samples` is a non-leaf
|
| 487 |
+
node and has children `children_[i - n_samples]`. Alternatively
|
| 488 |
+
at the i-th iteration, children[i][0] and children[i][1]
|
| 489 |
+
are merged to form node `n_samples + i`.
|
| 490 |
+
|
| 491 |
+
n_connected_components : int
|
| 492 |
+
The number of connected components in the graph.
|
| 493 |
+
|
| 494 |
+
n_leaves : int
|
| 495 |
+
The number of leaves in the tree.
|
| 496 |
+
|
| 497 |
+
parents : ndarray of shape (n_nodes, ) or None
|
| 498 |
+
The parent of each node. Only returned when a connectivity matrix
|
| 499 |
+
is specified, elsewhere 'None' is returned.
|
| 500 |
+
|
| 501 |
+
distances : ndarray of shape (n_nodes-1,)
|
| 502 |
+
Returned when `return_distance` is set to `True`.
|
| 503 |
+
|
| 504 |
+
distances[i] refers to the distance between children[i][0] and
|
| 505 |
+
children[i][1] when they are merged.
|
| 506 |
+
|
| 507 |
+
See Also
|
| 508 |
+
--------
|
| 509 |
+
ward_tree : Hierarchical clustering with ward linkage.
|
| 510 |
+
"""
|
| 511 |
+
X = np.asarray(X)
|
| 512 |
+
if X.ndim == 1:
|
| 513 |
+
X = np.reshape(X, (-1, 1))
|
| 514 |
+
n_samples, n_features = X.shape
|
| 515 |
+
|
| 516 |
+
linkage_choices = {
|
| 517 |
+
"complete": _hierarchical.max_merge,
|
| 518 |
+
"average": _hierarchical.average_merge,
|
| 519 |
+
"single": None,
|
| 520 |
+
} # Single linkage is handled differently
|
| 521 |
+
try:
|
| 522 |
+
join_func = linkage_choices[linkage]
|
| 523 |
+
except KeyError as e:
|
| 524 |
+
raise ValueError(
|
| 525 |
+
"Unknown linkage option, linkage should be one of %s, but %s was given"
|
| 526 |
+
% (linkage_choices.keys(), linkage)
|
| 527 |
+
) from e
|
| 528 |
+
|
| 529 |
+
if affinity == "cosine" and np.any(~np.any(X, axis=1)):
|
| 530 |
+
raise ValueError("Cosine affinity cannot be used when X contains zero vectors")
|
| 531 |
+
|
| 532 |
+
if connectivity is None:
|
| 533 |
+
from scipy.cluster import hierarchy # imports PIL
|
| 534 |
+
|
| 535 |
+
if n_clusters is not None:
|
| 536 |
+
warnings.warn(
|
| 537 |
+
(
|
| 538 |
+
"Partial build of the tree is implemented "
|
| 539 |
+
"only for structured clustering (i.e. with "
|
| 540 |
+
"explicit connectivity). The algorithm "
|
| 541 |
+
"will build the full tree and only "
|
| 542 |
+
"retain the lower branches required "
|
| 543 |
+
"for the specified number of clusters"
|
| 544 |
+
),
|
| 545 |
+
stacklevel=2,
|
| 546 |
+
)
|
| 547 |
+
|
| 548 |
+
if affinity == "precomputed":
|
| 549 |
+
# for the linkage function of hierarchy to work on precomputed
|
| 550 |
+
# data, provide as first argument an ndarray of the shape returned
|
| 551 |
+
# by sklearn.metrics.pairwise_distances.
|
| 552 |
+
if X.shape[0] != X.shape[1]:
|
| 553 |
+
raise ValueError(
|
| 554 |
+
f"Distance matrix should be square, got matrix of shape {X.shape}"
|
| 555 |
+
)
|
| 556 |
+
i, j = np.triu_indices(X.shape[0], k=1)
|
| 557 |
+
X = X[i, j]
|
| 558 |
+
elif affinity == "l2":
|
| 559 |
+
# Translate to something understood by scipy
|
| 560 |
+
affinity = "euclidean"
|
| 561 |
+
elif affinity in ("l1", "manhattan"):
|
| 562 |
+
affinity = "cityblock"
|
| 563 |
+
elif callable(affinity):
|
| 564 |
+
X = affinity(X)
|
| 565 |
+
i, j = np.triu_indices(X.shape[0], k=1)
|
| 566 |
+
X = X[i, j]
|
| 567 |
+
if (
|
| 568 |
+
linkage == "single"
|
| 569 |
+
and affinity != "precomputed"
|
| 570 |
+
and not callable(affinity)
|
| 571 |
+
and affinity in METRIC_MAPPING64
|
| 572 |
+
):
|
| 573 |
+
# We need the fast cythonized metric from neighbors
|
| 574 |
+
dist_metric = DistanceMetric.get_metric(affinity)
|
| 575 |
+
|
| 576 |
+
# The Cython routines used require contiguous arrays
|
| 577 |
+
X = np.ascontiguousarray(X, dtype=np.double)
|
| 578 |
+
|
| 579 |
+
mst = _hierarchical.mst_linkage_core(X, dist_metric)
|
| 580 |
+
# Sort edges of the min_spanning_tree by weight
|
| 581 |
+
mst = mst[np.argsort(mst.T[2], kind="mergesort"), :]
|
| 582 |
+
|
| 583 |
+
# Convert edge list into standard hierarchical clustering format
|
| 584 |
+
out = _hierarchical.single_linkage_label(mst)
|
| 585 |
+
else:
|
| 586 |
+
out = hierarchy.linkage(X, method=linkage, metric=affinity)
|
| 587 |
+
children_ = out[:, :2].astype(int, copy=False)
|
| 588 |
+
|
| 589 |
+
if return_distance:
|
| 590 |
+
distances = out[:, 2]
|
| 591 |
+
return children_, 1, n_samples, None, distances
|
| 592 |
+
return children_, 1, n_samples, None
|
| 593 |
+
|
| 594 |
+
connectivity, n_connected_components = _fix_connectivity(
|
| 595 |
+
X, connectivity, affinity=affinity
|
| 596 |
+
)
|
| 597 |
+
connectivity = connectivity.tocoo()
|
| 598 |
+
# Put the diagonal to zero
|
| 599 |
+
diag_mask = connectivity.row != connectivity.col
|
| 600 |
+
connectivity.row = connectivity.row[diag_mask]
|
| 601 |
+
connectivity.col = connectivity.col[diag_mask]
|
| 602 |
+
connectivity.data = connectivity.data[diag_mask]
|
| 603 |
+
del diag_mask
|
| 604 |
+
|
| 605 |
+
if affinity == "precomputed":
|
| 606 |
+
distances = X[connectivity.row, connectivity.col].astype(np.float64, copy=False)
|
| 607 |
+
else:
|
| 608 |
+
# FIXME We compute all the distances, while we could have only computed
|
| 609 |
+
# the "interesting" distances
|
| 610 |
+
distances = paired_distances(
|
| 611 |
+
X[connectivity.row], X[connectivity.col], metric=affinity
|
| 612 |
+
)
|
| 613 |
+
connectivity.data = distances
|
| 614 |
+
|
| 615 |
+
if n_clusters is None:
|
| 616 |
+
n_nodes = 2 * n_samples - 1
|
| 617 |
+
else:
|
| 618 |
+
assert n_clusters <= n_samples
|
| 619 |
+
n_nodes = 2 * n_samples - n_clusters
|
| 620 |
+
|
| 621 |
+
if linkage == "single":
|
| 622 |
+
return _single_linkage_tree(
|
| 623 |
+
connectivity,
|
| 624 |
+
n_samples,
|
| 625 |
+
n_nodes,
|
| 626 |
+
n_clusters,
|
| 627 |
+
n_connected_components,
|
| 628 |
+
return_distance,
|
| 629 |
+
)
|
| 630 |
+
|
| 631 |
+
if return_distance:
|
| 632 |
+
distances = np.empty(n_nodes - n_samples)
|
| 633 |
+
# create inertia heap and connection matrix
|
| 634 |
+
A = np.empty(n_nodes, dtype=object)
|
| 635 |
+
inertia = list()
|
| 636 |
+
|
| 637 |
+
# LIL seems to the best format to access the rows quickly,
|
| 638 |
+
# without the numpy overhead of slicing CSR indices and data.
|
| 639 |
+
connectivity = connectivity.tolil()
|
| 640 |
+
# We are storing the graph in a list of IntFloatDict
|
| 641 |
+
for ind, (data, row) in enumerate(zip(connectivity.data, connectivity.rows)):
|
| 642 |
+
A[ind] = IntFloatDict(
|
| 643 |
+
np.asarray(row, dtype=np.intp), np.asarray(data, dtype=np.float64)
|
| 644 |
+
)
|
| 645 |
+
# We keep only the upper triangular for the heap
|
| 646 |
+
# Generator expressions are faster than arrays on the following
|
| 647 |
+
inertia.extend(
|
| 648 |
+
_hierarchical.WeightedEdge(d, ind, r) for r, d in zip(row, data) if r < ind
|
| 649 |
+
)
|
| 650 |
+
del connectivity
|
| 651 |
+
|
| 652 |
+
heapify(inertia)
|
| 653 |
+
|
| 654 |
+
# prepare the main fields
|
| 655 |
+
parent = np.arange(n_nodes, dtype=np.intp)
|
| 656 |
+
used_node = np.ones(n_nodes, dtype=np.intp)
|
| 657 |
+
children = []
|
| 658 |
+
|
| 659 |
+
# recursive merge loop
|
| 660 |
+
for k in range(n_samples, n_nodes):
|
| 661 |
+
# identify the merge
|
| 662 |
+
while True:
|
| 663 |
+
edge = heappop(inertia)
|
| 664 |
+
if used_node[edge.a] and used_node[edge.b]:
|
| 665 |
+
break
|
| 666 |
+
i = edge.a
|
| 667 |
+
j = edge.b
|
| 668 |
+
|
| 669 |
+
if return_distance:
|
| 670 |
+
# store distances
|
| 671 |
+
distances[k - n_samples] = edge.weight
|
| 672 |
+
|
| 673 |
+
parent[i] = parent[j] = k
|
| 674 |
+
children.append((i, j))
|
| 675 |
+
# Keep track of the number of elements per cluster
|
| 676 |
+
n_i = used_node[i]
|
| 677 |
+
n_j = used_node[j]
|
| 678 |
+
used_node[k] = n_i + n_j
|
| 679 |
+
used_node[i] = used_node[j] = False
|
| 680 |
+
|
| 681 |
+
# update the structure matrix A and the inertia matrix
|
| 682 |
+
# a clever 'min', or 'max' operation between A[i] and A[j]
|
| 683 |
+
coord_col = join_func(A[i], A[j], used_node, n_i, n_j)
|
| 684 |
+
for col, d in coord_col:
|
| 685 |
+
A[col].append(k, d)
|
| 686 |
+
# Here we use the information from coord_col (containing the
|
| 687 |
+
# distances) to update the heap
|
| 688 |
+
heappush(inertia, _hierarchical.WeightedEdge(d, k, col))
|
| 689 |
+
A[k] = coord_col
|
| 690 |
+
# Clear A[i] and A[j] to save memory
|
| 691 |
+
A[i] = A[j] = 0
|
| 692 |
+
|
| 693 |
+
# Separate leaves in children (empty lists up to now)
|
| 694 |
+
n_leaves = n_samples
|
| 695 |
+
|
| 696 |
+
# # return numpy array for efficient caching
|
| 697 |
+
children = np.array(children)[:, ::-1]
|
| 698 |
+
|
| 699 |
+
if return_distance:
|
| 700 |
+
return children, n_connected_components, n_leaves, parent, distances
|
| 701 |
+
return children, n_connected_components, n_leaves, parent
|
| 702 |
+
|
| 703 |
+
|
| 704 |
+
# Matching names to tree-building strategies
|
| 705 |
+
def _complete_linkage(*args, **kwargs):
|
| 706 |
+
kwargs["linkage"] = "complete"
|
| 707 |
+
return linkage_tree(*args, **kwargs)
|
| 708 |
+
|
| 709 |
+
|
| 710 |
+
def _average_linkage(*args, **kwargs):
|
| 711 |
+
kwargs["linkage"] = "average"
|
| 712 |
+
return linkage_tree(*args, **kwargs)
|
| 713 |
+
|
| 714 |
+
|
| 715 |
+
def _single_linkage(*args, **kwargs):
|
| 716 |
+
kwargs["linkage"] = "single"
|
| 717 |
+
return linkage_tree(*args, **kwargs)
|
| 718 |
+
|
| 719 |
+
|
| 720 |
+
_TREE_BUILDERS = dict(
|
| 721 |
+
ward=ward_tree,
|
| 722 |
+
complete=_complete_linkage,
|
| 723 |
+
average=_average_linkage,
|
| 724 |
+
single=_single_linkage,
|
| 725 |
+
)
|
| 726 |
+
|
| 727 |
+
###############################################################################
|
| 728 |
+
# Functions for cutting hierarchical clustering tree
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
def _hc_cut(n_clusters, children, n_leaves):
|
| 732 |
+
"""Function cutting the ward tree for a given number of clusters.
|
| 733 |
+
|
| 734 |
+
Parameters
|
| 735 |
+
----------
|
| 736 |
+
n_clusters : int or ndarray
|
| 737 |
+
The number of clusters to form.
|
| 738 |
+
|
| 739 |
+
children : ndarray of shape (n_nodes-1, 2)
|
| 740 |
+
The children of each non-leaf node. Values less than `n_samples`
|
| 741 |
+
correspond to leaves of the tree which are the original samples.
|
| 742 |
+
A node `i` greater than or equal to `n_samples` is a non-leaf
|
| 743 |
+
node and has children `children_[i - n_samples]`. Alternatively
|
| 744 |
+
at the i-th iteration, children[i][0] and children[i][1]
|
| 745 |
+
are merged to form node `n_samples + i`.
|
| 746 |
+
|
| 747 |
+
n_leaves : int
|
| 748 |
+
Number of leaves of the tree.
|
| 749 |
+
|
| 750 |
+
Returns
|
| 751 |
+
-------
|
| 752 |
+
labels : array [n_samples]
|
| 753 |
+
Cluster labels for each point.
|
| 754 |
+
"""
|
| 755 |
+
if n_clusters > n_leaves:
|
| 756 |
+
raise ValueError(
|
| 757 |
+
"Cannot extract more clusters than samples: "
|
| 758 |
+
f"{n_clusters} clusters were given for a tree with {n_leaves} leaves."
|
| 759 |
+
)
|
| 760 |
+
# In this function, we store nodes as a heap to avoid recomputing
|
| 761 |
+
# the max of the nodes: the first element is always the smallest
|
| 762 |
+
# We use negated indices as heaps work on smallest elements, and we
|
| 763 |
+
# are interested in largest elements
|
| 764 |
+
# children[-1] is the root of the tree
|
| 765 |
+
nodes = [-(max(children[-1]) + 1)]
|
| 766 |
+
for _ in range(n_clusters - 1):
|
| 767 |
+
# As we have a heap, nodes[0] is the smallest element
|
| 768 |
+
these_children = children[-nodes[0] - n_leaves]
|
| 769 |
+
# Insert the 2 children and remove the largest node
|
| 770 |
+
heappush(nodes, -these_children[0])
|
| 771 |
+
heappushpop(nodes, -these_children[1])
|
| 772 |
+
label = np.zeros(n_leaves, dtype=np.intp)
|
| 773 |
+
for i, node in enumerate(nodes):
|
| 774 |
+
label[_hierarchical._hc_get_descendent(-node, children, n_leaves)] = i
|
| 775 |
+
return label
|
| 776 |
+
|
| 777 |
+
|
| 778 |
+
###############################################################################
|
| 779 |
+
|
| 780 |
+
|
| 781 |
+
class AgglomerativeClustering(ClusterMixin, BaseEstimator):
|
| 782 |
+
"""
|
| 783 |
+
Agglomerative Clustering.
|
| 784 |
+
|
| 785 |
+
Recursively merges pair of clusters of sample data; uses linkage distance.
|
| 786 |
+
|
| 787 |
+
Read more in the :ref:`User Guide <hierarchical_clustering>`.
|
| 788 |
+
|
| 789 |
+
Parameters
|
| 790 |
+
----------
|
| 791 |
+
n_clusters : int or None, default=2
|
| 792 |
+
The number of clusters to find. It must be ``None`` if
|
| 793 |
+
``distance_threshold`` is not ``None``.
|
| 794 |
+
|
| 795 |
+
metric : str or callable, default="euclidean"
|
| 796 |
+
Metric used to compute the linkage. Can be "euclidean", "l1", "l2",
|
| 797 |
+
"manhattan", "cosine", or "precomputed". If linkage is "ward", only
|
| 798 |
+
"euclidean" is accepted. If "precomputed", a distance matrix is needed
|
| 799 |
+
as input for the fit method. If connectivity is None, linkage is
|
| 800 |
+
"single" and affinity is not "precomputed" any valid pairwise distance
|
| 801 |
+
metric can be assigned.
|
| 802 |
+
|
| 803 |
+
For an example of agglomerative clustering with different metrics, see
|
| 804 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering_metrics.py`.
|
| 805 |
+
|
| 806 |
+
.. versionadded:: 1.2
|
| 807 |
+
|
| 808 |
+
memory : str or object with the joblib.Memory interface, default=None
|
| 809 |
+
Used to cache the output of the computation of the tree.
|
| 810 |
+
By default, no caching is done. If a string is given, it is the
|
| 811 |
+
path to the caching directory.
|
| 812 |
+
|
| 813 |
+
connectivity : array-like, sparse matrix, or callable, default=None
|
| 814 |
+
Connectivity matrix. Defines for each sample the neighboring
|
| 815 |
+
samples following a given structure of the data.
|
| 816 |
+
This can be a connectivity matrix itself or a callable that transforms
|
| 817 |
+
the data into a connectivity matrix, such as derived from
|
| 818 |
+
`kneighbors_graph`. Default is ``None``, i.e, the
|
| 819 |
+
hierarchical clustering algorithm is unstructured.
|
| 820 |
+
|
| 821 |
+
For an example of connectivity matrix using
|
| 822 |
+
:class:`~sklearn.neighbors.kneighbors_graph`, see
|
| 823 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_ward_structured_vs_unstructured.py`.
|
| 824 |
+
|
| 825 |
+
compute_full_tree : 'auto' or bool, default='auto'
|
| 826 |
+
Stop early the construction of the tree at ``n_clusters``. This is
|
| 827 |
+
useful to decrease computation time if the number of clusters is not
|
| 828 |
+
small compared to the number of samples. This option is useful only
|
| 829 |
+
when specifying a connectivity matrix. Note also that when varying the
|
| 830 |
+
number of clusters and using caching, it may be advantageous to compute
|
| 831 |
+
the full tree. It must be ``True`` if ``distance_threshold`` is not
|
| 832 |
+
``None``. By default `compute_full_tree` is "auto", which is equivalent
|
| 833 |
+
to `True` when `distance_threshold` is not `None` or that `n_clusters`
|
| 834 |
+
is inferior to the maximum between 100 or `0.02 * n_samples`.
|
| 835 |
+
Otherwise, "auto" is equivalent to `False`.
|
| 836 |
+
|
| 837 |
+
linkage : {'ward', 'complete', 'average', 'single'}, default='ward'
|
| 838 |
+
Which linkage criterion to use. The linkage criterion determines which
|
| 839 |
+
distance to use between sets of observation. The algorithm will merge
|
| 840 |
+
the pairs of cluster that minimize this criterion.
|
| 841 |
+
|
| 842 |
+
- 'ward' minimizes the variance of the clusters being merged.
|
| 843 |
+
- 'average' uses the average of the distances of each observation of
|
| 844 |
+
the two sets.
|
| 845 |
+
- 'complete' or 'maximum' linkage uses the maximum distances between
|
| 846 |
+
all observations of the two sets.
|
| 847 |
+
- 'single' uses the minimum of the distances between all observations
|
| 848 |
+
of the two sets.
|
| 849 |
+
|
| 850 |
+
.. versionadded:: 0.20
|
| 851 |
+
Added the 'single' option
|
| 852 |
+
|
| 853 |
+
For examples comparing different `linkage` criteria, see
|
| 854 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_linkage_comparison.py`.
|
| 855 |
+
|
| 856 |
+
distance_threshold : float, default=None
|
| 857 |
+
The linkage distance threshold at or above which clusters will not be
|
| 858 |
+
merged. If not ``None``, ``n_clusters`` must be ``None`` and
|
| 859 |
+
``compute_full_tree`` must be ``True``.
|
| 860 |
+
|
| 861 |
+
.. versionadded:: 0.21
|
| 862 |
+
|
| 863 |
+
compute_distances : bool, default=False
|
| 864 |
+
Computes distances between clusters even if `distance_threshold` is not
|
| 865 |
+
used. This can be used to make dendrogram visualization, but introduces
|
| 866 |
+
a computational and memory overhead.
|
| 867 |
+
|
| 868 |
+
.. versionadded:: 0.24
|
| 869 |
+
|
| 870 |
+
For an example of dendrogram visualization, see
|
| 871 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_dendrogram.py`.
|
| 872 |
+
|
| 873 |
+
Attributes
|
| 874 |
+
----------
|
| 875 |
+
n_clusters_ : int
|
| 876 |
+
The number of clusters found by the algorithm. If
|
| 877 |
+
``distance_threshold=None``, it will be equal to the given
|
| 878 |
+
``n_clusters``.
|
| 879 |
+
|
| 880 |
+
labels_ : ndarray of shape (n_samples)
|
| 881 |
+
Cluster labels for each point.
|
| 882 |
+
|
| 883 |
+
n_leaves_ : int
|
| 884 |
+
Number of leaves in the hierarchical tree.
|
| 885 |
+
|
| 886 |
+
n_connected_components_ : int
|
| 887 |
+
The estimated number of connected components in the graph.
|
| 888 |
+
|
| 889 |
+
.. versionadded:: 0.21
|
| 890 |
+
``n_connected_components_`` was added to replace ``n_components_``.
|
| 891 |
+
|
| 892 |
+
n_features_in_ : int
|
| 893 |
+
Number of features seen during :term:`fit`.
|
| 894 |
+
|
| 895 |
+
.. versionadded:: 0.24
|
| 896 |
+
|
| 897 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 898 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 899 |
+
has feature names that are all strings.
|
| 900 |
+
|
| 901 |
+
.. versionadded:: 1.0
|
| 902 |
+
|
| 903 |
+
children_ : array-like of shape (n_samples-1, 2)
|
| 904 |
+
The children of each non-leaf node. Values less than `n_samples`
|
| 905 |
+
correspond to leaves of the tree which are the original samples.
|
| 906 |
+
A node `i` greater than or equal to `n_samples` is a non-leaf
|
| 907 |
+
node and has children `children_[i - n_samples]`. Alternatively
|
| 908 |
+
at the i-th iteration, children[i][0] and children[i][1]
|
| 909 |
+
are merged to form node `n_samples + i`.
|
| 910 |
+
|
| 911 |
+
distances_ : array-like of shape (n_nodes-1,)
|
| 912 |
+
Distances between nodes in the corresponding place in `children_`.
|
| 913 |
+
Only computed if `distance_threshold` is used or `compute_distances`
|
| 914 |
+
is set to `True`.
|
| 915 |
+
|
| 916 |
+
See Also
|
| 917 |
+
--------
|
| 918 |
+
FeatureAgglomeration : Agglomerative clustering but for features instead of
|
| 919 |
+
samples.
|
| 920 |
+
ward_tree : Hierarchical clustering with ward linkage.
|
| 921 |
+
|
| 922 |
+
Examples
|
| 923 |
+
--------
|
| 924 |
+
>>> from sklearn.cluster import AgglomerativeClustering
|
| 925 |
+
>>> import numpy as np
|
| 926 |
+
>>> X = np.array([[1, 2], [1, 4], [1, 0],
|
| 927 |
+
... [4, 2], [4, 4], [4, 0]])
|
| 928 |
+
>>> clustering = AgglomerativeClustering().fit(X)
|
| 929 |
+
>>> clustering
|
| 930 |
+
AgglomerativeClustering()
|
| 931 |
+
>>> clustering.labels_
|
| 932 |
+
array([1, 1, 1, 0, 0, 0])
|
| 933 |
+
|
| 934 |
+
For a comparison of Agglomerative clustering with other clustering algorithms, see
|
| 935 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
|
| 936 |
+
"""
|
| 937 |
+
|
| 938 |
+
_parameter_constraints: dict = {
|
| 939 |
+
"n_clusters": [Interval(Integral, 1, None, closed="left"), None],
|
| 940 |
+
"metric": [
|
| 941 |
+
StrOptions(set(_VALID_METRICS) | {"precomputed"}),
|
| 942 |
+
callable,
|
| 943 |
+
],
|
| 944 |
+
"memory": [str, HasMethods("cache"), None],
|
| 945 |
+
"connectivity": ["array-like", "sparse matrix", callable, None],
|
| 946 |
+
"compute_full_tree": [StrOptions({"auto"}), "boolean"],
|
| 947 |
+
"linkage": [StrOptions(set(_TREE_BUILDERS.keys()))],
|
| 948 |
+
"distance_threshold": [Interval(Real, 0, None, closed="left"), None],
|
| 949 |
+
"compute_distances": ["boolean"],
|
| 950 |
+
}
|
| 951 |
+
|
| 952 |
+
def __init__(
|
| 953 |
+
self,
|
| 954 |
+
n_clusters=2,
|
| 955 |
+
*,
|
| 956 |
+
metric="euclidean",
|
| 957 |
+
memory=None,
|
| 958 |
+
connectivity=None,
|
| 959 |
+
compute_full_tree="auto",
|
| 960 |
+
linkage="ward",
|
| 961 |
+
distance_threshold=None,
|
| 962 |
+
compute_distances=False,
|
| 963 |
+
):
|
| 964 |
+
self.n_clusters = n_clusters
|
| 965 |
+
self.distance_threshold = distance_threshold
|
| 966 |
+
self.memory = memory
|
| 967 |
+
self.connectivity = connectivity
|
| 968 |
+
self.compute_full_tree = compute_full_tree
|
| 969 |
+
self.linkage = linkage
|
| 970 |
+
self.metric = metric
|
| 971 |
+
self.compute_distances = compute_distances
|
| 972 |
+
|
| 973 |
+
@_fit_context(prefer_skip_nested_validation=True)
|
| 974 |
+
def fit(self, X, y=None):
|
| 975 |
+
"""Fit the hierarchical clustering from features, or distance matrix.
|
| 976 |
+
|
| 977 |
+
Parameters
|
| 978 |
+
----------
|
| 979 |
+
X : array-like, shape (n_samples, n_features) or \
|
| 980 |
+
(n_samples, n_samples)
|
| 981 |
+
Training instances to cluster, or distances between instances if
|
| 982 |
+
``metric='precomputed'``.
|
| 983 |
+
|
| 984 |
+
y : Ignored
|
| 985 |
+
Not used, present here for API consistency by convention.
|
| 986 |
+
|
| 987 |
+
Returns
|
| 988 |
+
-------
|
| 989 |
+
self : object
|
| 990 |
+
Returns the fitted instance.
|
| 991 |
+
"""
|
| 992 |
+
X = validate_data(self, X, ensure_min_samples=2)
|
| 993 |
+
return self._fit(X)
|
| 994 |
+
|
| 995 |
+
def _fit(self, X):
|
| 996 |
+
"""Fit without validation
|
| 997 |
+
|
| 998 |
+
Parameters
|
| 999 |
+
----------
|
| 1000 |
+
X : ndarray of shape (n_samples, n_features) or (n_samples, n_samples)
|
| 1001 |
+
Training instances to cluster, or distances between instances if
|
| 1002 |
+
``metric='precomputed'``.
|
| 1003 |
+
|
| 1004 |
+
Returns
|
| 1005 |
+
-------
|
| 1006 |
+
self : object
|
| 1007 |
+
Returns the fitted instance.
|
| 1008 |
+
"""
|
| 1009 |
+
memory = check_memory(self.memory)
|
| 1010 |
+
|
| 1011 |
+
if not ((self.n_clusters is None) ^ (self.distance_threshold is None)):
|
| 1012 |
+
raise ValueError(
|
| 1013 |
+
"Exactly one of n_clusters and "
|
| 1014 |
+
"distance_threshold has to be set, and the other "
|
| 1015 |
+
"needs to be None."
|
| 1016 |
+
)
|
| 1017 |
+
|
| 1018 |
+
if self.distance_threshold is not None and not self.compute_full_tree:
|
| 1019 |
+
raise ValueError(
|
| 1020 |
+
"compute_full_tree must be True if distance_threshold is set."
|
| 1021 |
+
)
|
| 1022 |
+
|
| 1023 |
+
if self.linkage == "ward" and self.metric != "euclidean":
|
| 1024 |
+
raise ValueError(
|
| 1025 |
+
f"{self.metric} was provided as metric. Ward can only "
|
| 1026 |
+
"work with euclidean distances."
|
| 1027 |
+
)
|
| 1028 |
+
|
| 1029 |
+
tree_builder = _TREE_BUILDERS[self.linkage]
|
| 1030 |
+
|
| 1031 |
+
connectivity = self.connectivity
|
| 1032 |
+
if self.connectivity is not None:
|
| 1033 |
+
if callable(self.connectivity):
|
| 1034 |
+
connectivity = self.connectivity(X)
|
| 1035 |
+
connectivity = check_array(
|
| 1036 |
+
connectivity, accept_sparse=["csr", "coo", "lil"]
|
| 1037 |
+
)
|
| 1038 |
+
|
| 1039 |
+
n_samples = len(X)
|
| 1040 |
+
compute_full_tree = self.compute_full_tree
|
| 1041 |
+
if self.connectivity is None:
|
| 1042 |
+
compute_full_tree = True
|
| 1043 |
+
if compute_full_tree == "auto":
|
| 1044 |
+
if self.distance_threshold is not None:
|
| 1045 |
+
compute_full_tree = True
|
| 1046 |
+
else:
|
| 1047 |
+
# Early stopping is likely to give a speed up only for
|
| 1048 |
+
# a large number of clusters. The actual threshold
|
| 1049 |
+
# implemented here is heuristic
|
| 1050 |
+
compute_full_tree = self.n_clusters < max(100, 0.02 * n_samples)
|
| 1051 |
+
n_clusters = self.n_clusters
|
| 1052 |
+
if compute_full_tree:
|
| 1053 |
+
n_clusters = None
|
| 1054 |
+
|
| 1055 |
+
# Construct the tree
|
| 1056 |
+
kwargs = {}
|
| 1057 |
+
if self.linkage != "ward":
|
| 1058 |
+
kwargs["linkage"] = self.linkage
|
| 1059 |
+
kwargs["affinity"] = self.metric
|
| 1060 |
+
|
| 1061 |
+
distance_threshold = self.distance_threshold
|
| 1062 |
+
|
| 1063 |
+
return_distance = (distance_threshold is not None) or self.compute_distances
|
| 1064 |
+
|
| 1065 |
+
out = memory.cache(tree_builder)(
|
| 1066 |
+
X,
|
| 1067 |
+
connectivity=connectivity,
|
| 1068 |
+
n_clusters=n_clusters,
|
| 1069 |
+
return_distance=return_distance,
|
| 1070 |
+
**kwargs,
|
| 1071 |
+
)
|
| 1072 |
+
(self.children_, self.n_connected_components_, self.n_leaves_, parents) = out[
|
| 1073 |
+
:4
|
| 1074 |
+
]
|
| 1075 |
+
|
| 1076 |
+
if return_distance:
|
| 1077 |
+
self.distances_ = out[-1]
|
| 1078 |
+
|
| 1079 |
+
if self.distance_threshold is not None: # distance_threshold is used
|
| 1080 |
+
self.n_clusters_ = (
|
| 1081 |
+
np.count_nonzero(self.distances_ >= distance_threshold) + 1
|
| 1082 |
+
)
|
| 1083 |
+
else: # n_clusters is used
|
| 1084 |
+
self.n_clusters_ = self.n_clusters
|
| 1085 |
+
|
| 1086 |
+
# Cut the tree
|
| 1087 |
+
if compute_full_tree:
|
| 1088 |
+
self.labels_ = _hc_cut(self.n_clusters_, self.children_, self.n_leaves_)
|
| 1089 |
+
else:
|
| 1090 |
+
labels = _hierarchical.hc_get_heads(parents, copy=False)
|
| 1091 |
+
# copy to avoid holding a reference on the original array
|
| 1092 |
+
labels = np.copy(labels[:n_samples])
|
| 1093 |
+
# Reassign cluster numbers
|
| 1094 |
+
self.labels_ = np.searchsorted(np.unique(labels), labels)
|
| 1095 |
+
return self
|
| 1096 |
+
|
| 1097 |
+
def fit_predict(self, X, y=None):
|
| 1098 |
+
"""Fit and return the result of each sample's clustering assignment.
|
| 1099 |
+
|
| 1100 |
+
In addition to fitting, this method also return the result of the
|
| 1101 |
+
clustering assignment for each sample in the training set.
|
| 1102 |
+
|
| 1103 |
+
Parameters
|
| 1104 |
+
----------
|
| 1105 |
+
X : array-like of shape (n_samples, n_features) or \
|
| 1106 |
+
(n_samples, n_samples)
|
| 1107 |
+
Training instances to cluster, or distances between instances if
|
| 1108 |
+
``affinity='precomputed'``.
|
| 1109 |
+
|
| 1110 |
+
y : Ignored
|
| 1111 |
+
Not used, present here for API consistency by convention.
|
| 1112 |
+
|
| 1113 |
+
Returns
|
| 1114 |
+
-------
|
| 1115 |
+
labels : ndarray of shape (n_samples,)
|
| 1116 |
+
Cluster labels.
|
| 1117 |
+
"""
|
| 1118 |
+
return super().fit_predict(X, y)
|
| 1119 |
+
|
| 1120 |
+
|
| 1121 |
+
class FeatureAgglomeration(
|
| 1122 |
+
ClassNamePrefixFeaturesOutMixin, AgglomerationTransform, AgglomerativeClustering
|
| 1123 |
+
):
|
| 1124 |
+
"""Agglomerate features.
|
| 1125 |
+
|
| 1126 |
+
Recursively merges pair of clusters of features.
|
| 1127 |
+
|
| 1128 |
+
Refer to
|
| 1129 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_feature_agglomeration_vs_univariate_selection.py`
|
| 1130 |
+
for an example comparison of :class:`FeatureAgglomeration` strategy with a
|
| 1131 |
+
univariate feature selection strategy (based on ANOVA).
|
| 1132 |
+
|
| 1133 |
+
Read more in the :ref:`User Guide <hierarchical_clustering>`.
|
| 1134 |
+
|
| 1135 |
+
Parameters
|
| 1136 |
+
----------
|
| 1137 |
+
n_clusters : int or None, default=2
|
| 1138 |
+
The number of clusters to find. It must be ``None`` if
|
| 1139 |
+
``distance_threshold`` is not ``None``.
|
| 1140 |
+
|
| 1141 |
+
metric : str or callable, default="euclidean"
|
| 1142 |
+
Metric used to compute the linkage. Can be "euclidean", "l1", "l2",
|
| 1143 |
+
"manhattan", "cosine", or "precomputed". If linkage is "ward", only
|
| 1144 |
+
"euclidean" is accepted. If "precomputed", a distance matrix is needed
|
| 1145 |
+
as input for the fit method.
|
| 1146 |
+
|
| 1147 |
+
.. versionadded:: 1.2
|
| 1148 |
+
|
| 1149 |
+
memory : str or object with the joblib.Memory interface, default=None
|
| 1150 |
+
Used to cache the output of the computation of the tree.
|
| 1151 |
+
By default, no caching is done. If a string is given, it is the
|
| 1152 |
+
path to the caching directory.
|
| 1153 |
+
|
| 1154 |
+
connectivity : array-like, sparse matrix, or callable, default=None
|
| 1155 |
+
Connectivity matrix. Defines for each feature the neighboring
|
| 1156 |
+
features following a given structure of the data.
|
| 1157 |
+
This can be a connectivity matrix itself or a callable that transforms
|
| 1158 |
+
the data into a connectivity matrix, such as derived from
|
| 1159 |
+
`kneighbors_graph`. Default is `None`, i.e, the
|
| 1160 |
+
hierarchical clustering algorithm is unstructured.
|
| 1161 |
+
|
| 1162 |
+
compute_full_tree : 'auto' or bool, default='auto'
|
| 1163 |
+
Stop early the construction of the tree at `n_clusters`. This is useful
|
| 1164 |
+
to decrease computation time if the number of clusters is not small
|
| 1165 |
+
compared to the number of features. This option is useful only when
|
| 1166 |
+
specifying a connectivity matrix. Note also that when varying the
|
| 1167 |
+
number of clusters and using caching, it may be advantageous to compute
|
| 1168 |
+
the full tree. It must be ``True`` if ``distance_threshold`` is not
|
| 1169 |
+
``None``. By default `compute_full_tree` is "auto", which is equivalent
|
| 1170 |
+
to `True` when `distance_threshold` is not `None` or that `n_clusters`
|
| 1171 |
+
is inferior to the maximum between 100 or `0.02 * n_samples`.
|
| 1172 |
+
Otherwise, "auto" is equivalent to `False`.
|
| 1173 |
+
|
| 1174 |
+
linkage : {"ward", "complete", "average", "single"}, default="ward"
|
| 1175 |
+
Which linkage criterion to use. The linkage criterion determines which
|
| 1176 |
+
distance to use between sets of features. The algorithm will merge
|
| 1177 |
+
the pairs of cluster that minimize this criterion.
|
| 1178 |
+
|
| 1179 |
+
- "ward" minimizes the variance of the clusters being merged.
|
| 1180 |
+
- "complete" or maximum linkage uses the maximum distances between
|
| 1181 |
+
all features of the two sets.
|
| 1182 |
+
- "average" uses the average of the distances of each feature of
|
| 1183 |
+
the two sets.
|
| 1184 |
+
- "single" uses the minimum of the distances between all features
|
| 1185 |
+
of the two sets.
|
| 1186 |
+
|
| 1187 |
+
pooling_func : callable, default=np.mean
|
| 1188 |
+
This combines the values of agglomerated features into a single
|
| 1189 |
+
value, and should accept an array of shape [M, N] and the keyword
|
| 1190 |
+
argument `axis=1`, and reduce it to an array of size [M].
|
| 1191 |
+
|
| 1192 |
+
distance_threshold : float, default=None
|
| 1193 |
+
The linkage distance threshold at or above which clusters will not be
|
| 1194 |
+
merged. If not ``None``, ``n_clusters`` must be ``None`` and
|
| 1195 |
+
``compute_full_tree`` must be ``True``.
|
| 1196 |
+
|
| 1197 |
+
.. versionadded:: 0.21
|
| 1198 |
+
|
| 1199 |
+
compute_distances : bool, default=False
|
| 1200 |
+
Computes distances between clusters even if `distance_threshold` is not
|
| 1201 |
+
used. This can be used to make dendrogram visualization, but introduces
|
| 1202 |
+
a computational and memory overhead.
|
| 1203 |
+
|
| 1204 |
+
.. versionadded:: 0.24
|
| 1205 |
+
|
| 1206 |
+
Attributes
|
| 1207 |
+
----------
|
| 1208 |
+
n_clusters_ : int
|
| 1209 |
+
The number of clusters found by the algorithm. If
|
| 1210 |
+
``distance_threshold=None``, it will be equal to the given
|
| 1211 |
+
``n_clusters``.
|
| 1212 |
+
|
| 1213 |
+
labels_ : array-like of (n_features,)
|
| 1214 |
+
Cluster labels for each feature.
|
| 1215 |
+
|
| 1216 |
+
n_leaves_ : int
|
| 1217 |
+
Number of leaves in the hierarchical tree.
|
| 1218 |
+
|
| 1219 |
+
n_connected_components_ : int
|
| 1220 |
+
The estimated number of connected components in the graph.
|
| 1221 |
+
|
| 1222 |
+
.. versionadded:: 0.21
|
| 1223 |
+
``n_connected_components_`` was added to replace ``n_components_``.
|
| 1224 |
+
|
| 1225 |
+
n_features_in_ : int
|
| 1226 |
+
Number of features seen during :term:`fit`.
|
| 1227 |
+
|
| 1228 |
+
.. versionadded:: 0.24
|
| 1229 |
+
|
| 1230 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 1231 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 1232 |
+
has feature names that are all strings.
|
| 1233 |
+
|
| 1234 |
+
.. versionadded:: 1.0
|
| 1235 |
+
|
| 1236 |
+
children_ : array-like of shape (n_nodes-1, 2)
|
| 1237 |
+
The children of each non-leaf node. Values less than `n_features`
|
| 1238 |
+
correspond to leaves of the tree which are the original samples.
|
| 1239 |
+
A node `i` greater than or equal to `n_features` is a non-leaf
|
| 1240 |
+
node and has children `children_[i - n_features]`. Alternatively
|
| 1241 |
+
at the i-th iteration, children[i][0] and children[i][1]
|
| 1242 |
+
are merged to form node `n_features + i`.
|
| 1243 |
+
|
| 1244 |
+
distances_ : array-like of shape (n_nodes-1,)
|
| 1245 |
+
Distances between nodes in the corresponding place in `children_`.
|
| 1246 |
+
Only computed if `distance_threshold` is used or `compute_distances`
|
| 1247 |
+
is set to `True`.
|
| 1248 |
+
|
| 1249 |
+
See Also
|
| 1250 |
+
--------
|
| 1251 |
+
AgglomerativeClustering : Agglomerative clustering samples instead of
|
| 1252 |
+
features.
|
| 1253 |
+
ward_tree : Hierarchical clustering with ward linkage.
|
| 1254 |
+
|
| 1255 |
+
Examples
|
| 1256 |
+
--------
|
| 1257 |
+
>>> import numpy as np
|
| 1258 |
+
>>> from sklearn import datasets, cluster
|
| 1259 |
+
>>> digits = datasets.load_digits()
|
| 1260 |
+
>>> images = digits.images
|
| 1261 |
+
>>> X = np.reshape(images, (len(images), -1))
|
| 1262 |
+
>>> agglo = cluster.FeatureAgglomeration(n_clusters=32)
|
| 1263 |
+
>>> agglo.fit(X)
|
| 1264 |
+
FeatureAgglomeration(n_clusters=32)
|
| 1265 |
+
>>> X_reduced = agglo.transform(X)
|
| 1266 |
+
>>> X_reduced.shape
|
| 1267 |
+
(1797, 32)
|
| 1268 |
+
"""
|
| 1269 |
+
|
| 1270 |
+
_parameter_constraints: dict = {
|
| 1271 |
+
"n_clusters": [Interval(Integral, 1, None, closed="left"), None],
|
| 1272 |
+
"metric": [
|
| 1273 |
+
StrOptions(set(_VALID_METRICS) | {"precomputed"}),
|
| 1274 |
+
callable,
|
| 1275 |
+
],
|
| 1276 |
+
"memory": [str, HasMethods("cache"), None],
|
| 1277 |
+
"connectivity": ["array-like", "sparse matrix", callable, None],
|
| 1278 |
+
"compute_full_tree": [StrOptions({"auto"}), "boolean"],
|
| 1279 |
+
"linkage": [StrOptions(set(_TREE_BUILDERS.keys()))],
|
| 1280 |
+
"pooling_func": [callable],
|
| 1281 |
+
"distance_threshold": [Interval(Real, 0, None, closed="left"), None],
|
| 1282 |
+
"compute_distances": ["boolean"],
|
| 1283 |
+
}
|
| 1284 |
+
|
| 1285 |
+
def __init__(
|
| 1286 |
+
self,
|
| 1287 |
+
n_clusters=2,
|
| 1288 |
+
*,
|
| 1289 |
+
metric="euclidean",
|
| 1290 |
+
memory=None,
|
| 1291 |
+
connectivity=None,
|
| 1292 |
+
compute_full_tree="auto",
|
| 1293 |
+
linkage="ward",
|
| 1294 |
+
pooling_func=np.mean,
|
| 1295 |
+
distance_threshold=None,
|
| 1296 |
+
compute_distances=False,
|
| 1297 |
+
):
|
| 1298 |
+
super().__init__(
|
| 1299 |
+
n_clusters=n_clusters,
|
| 1300 |
+
memory=memory,
|
| 1301 |
+
connectivity=connectivity,
|
| 1302 |
+
compute_full_tree=compute_full_tree,
|
| 1303 |
+
linkage=linkage,
|
| 1304 |
+
metric=metric,
|
| 1305 |
+
distance_threshold=distance_threshold,
|
| 1306 |
+
compute_distances=compute_distances,
|
| 1307 |
+
)
|
| 1308 |
+
self.pooling_func = pooling_func
|
| 1309 |
+
|
| 1310 |
+
@_fit_context(prefer_skip_nested_validation=True)
|
| 1311 |
+
def fit(self, X, y=None):
|
| 1312 |
+
"""Fit the hierarchical clustering on the data.
|
| 1313 |
+
|
| 1314 |
+
Parameters
|
| 1315 |
+
----------
|
| 1316 |
+
X : array-like of shape (n_samples, n_features)
|
| 1317 |
+
The data.
|
| 1318 |
+
|
| 1319 |
+
y : Ignored
|
| 1320 |
+
Not used, present here for API consistency by convention.
|
| 1321 |
+
|
| 1322 |
+
Returns
|
| 1323 |
+
-------
|
| 1324 |
+
self : object
|
| 1325 |
+
Returns the transformer.
|
| 1326 |
+
"""
|
| 1327 |
+
X = validate_data(self, X, ensure_min_features=2)
|
| 1328 |
+
super()._fit(X.T)
|
| 1329 |
+
self._n_features_out = self.n_clusters_
|
| 1330 |
+
return self
|
| 1331 |
+
|
| 1332 |
+
@property
|
| 1333 |
+
def fit_predict(self):
|
| 1334 |
+
"""Fit and return the result of each sample's clustering assignment."""
|
| 1335 |
+
raise AttributeError
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bicluster.py
ADDED
|
@@ -0,0 +1,622 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Spectral biclustering algorithms."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
from abc import ABCMeta, abstractmethod
|
| 7 |
+
from numbers import Integral
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
from scipy.linalg import norm
|
| 11 |
+
from scipy.sparse import dia_matrix, issparse
|
| 12 |
+
from scipy.sparse.linalg import eigsh, svds
|
| 13 |
+
|
| 14 |
+
from sklearn.base import BaseEstimator, BiclusterMixin, _fit_context
|
| 15 |
+
from sklearn.cluster._kmeans import KMeans, MiniBatchKMeans
|
| 16 |
+
from sklearn.utils import check_random_state, check_scalar
|
| 17 |
+
from sklearn.utils._param_validation import Interval, StrOptions
|
| 18 |
+
from sklearn.utils.extmath import _randomized_svd, make_nonnegative, safe_sparse_dot
|
| 19 |
+
from sklearn.utils.validation import assert_all_finite, validate_data
|
| 20 |
+
|
| 21 |
+
__all__ = ["SpectralBiclustering", "SpectralCoclustering"]
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _scale_normalize(X):
|
| 25 |
+
"""Normalize ``X`` by scaling rows and columns independently.
|
| 26 |
+
|
| 27 |
+
Returns the normalized matrix and the row and column scaling
|
| 28 |
+
factors.
|
| 29 |
+
"""
|
| 30 |
+
X = make_nonnegative(X)
|
| 31 |
+
row_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=1))).squeeze()
|
| 32 |
+
col_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=0))).squeeze()
|
| 33 |
+
row_diag = np.where(np.isnan(row_diag), 0, row_diag)
|
| 34 |
+
col_diag = np.where(np.isnan(col_diag), 0, col_diag)
|
| 35 |
+
if issparse(X):
|
| 36 |
+
n_rows, n_cols = X.shape
|
| 37 |
+
r = dia_matrix((row_diag, [0]), shape=(n_rows, n_rows))
|
| 38 |
+
c = dia_matrix((col_diag, [0]), shape=(n_cols, n_cols))
|
| 39 |
+
an = r @ X @ c
|
| 40 |
+
else:
|
| 41 |
+
an = row_diag[:, np.newaxis] * X * col_diag
|
| 42 |
+
return an, row_diag, col_diag
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _bistochastic_normalize(X, max_iter=1000, tol=1e-5):
|
| 46 |
+
"""Normalize rows and columns of ``X`` simultaneously so that all
|
| 47 |
+
rows sum to one constant and all columns sum to a different
|
| 48 |
+
constant.
|
| 49 |
+
"""
|
| 50 |
+
# According to paper, this can also be done more efficiently with
|
| 51 |
+
# deviation reduction and balancing algorithms.
|
| 52 |
+
X = make_nonnegative(X)
|
| 53 |
+
X_scaled = X
|
| 54 |
+
for _ in range(max_iter):
|
| 55 |
+
X_new, _, _ = _scale_normalize(X_scaled)
|
| 56 |
+
if issparse(X):
|
| 57 |
+
dist = norm(X_scaled.data - X.data)
|
| 58 |
+
else:
|
| 59 |
+
dist = norm(X_scaled - X_new)
|
| 60 |
+
X_scaled = X_new
|
| 61 |
+
if dist is not None and dist < tol:
|
| 62 |
+
break
|
| 63 |
+
return X_scaled
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _log_normalize(X):
|
| 67 |
+
"""Normalize ``X`` according to Kluger's log-interactions scheme."""
|
| 68 |
+
X = make_nonnegative(X, min_value=1)
|
| 69 |
+
if issparse(X):
|
| 70 |
+
raise ValueError(
|
| 71 |
+
"Cannot compute log of a sparse matrix,"
|
| 72 |
+
" because log(x) diverges to -infinity as x"
|
| 73 |
+
" goes to 0."
|
| 74 |
+
)
|
| 75 |
+
L = np.log(X)
|
| 76 |
+
row_avg = L.mean(axis=1)[:, np.newaxis]
|
| 77 |
+
col_avg = L.mean(axis=0)
|
| 78 |
+
avg = L.mean()
|
| 79 |
+
return L - row_avg - col_avg + avg
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class BaseSpectral(BiclusterMixin, BaseEstimator, metaclass=ABCMeta):
|
| 83 |
+
"""Base class for spectral biclustering."""
|
| 84 |
+
|
| 85 |
+
_parameter_constraints: dict = {
|
| 86 |
+
"svd_method": [StrOptions({"randomized", "arpack"})],
|
| 87 |
+
"n_svd_vecs": [Interval(Integral, 0, None, closed="left"), None],
|
| 88 |
+
"mini_batch": ["boolean"],
|
| 89 |
+
"init": [StrOptions({"k-means++", "random"}), np.ndarray],
|
| 90 |
+
"n_init": [Interval(Integral, 1, None, closed="left")],
|
| 91 |
+
"random_state": ["random_state"],
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
@abstractmethod
|
| 95 |
+
def __init__(
|
| 96 |
+
self,
|
| 97 |
+
n_clusters=3,
|
| 98 |
+
svd_method="randomized",
|
| 99 |
+
n_svd_vecs=None,
|
| 100 |
+
mini_batch=False,
|
| 101 |
+
init="k-means++",
|
| 102 |
+
n_init=10,
|
| 103 |
+
random_state=None,
|
| 104 |
+
):
|
| 105 |
+
self.n_clusters = n_clusters
|
| 106 |
+
self.svd_method = svd_method
|
| 107 |
+
self.n_svd_vecs = n_svd_vecs
|
| 108 |
+
self.mini_batch = mini_batch
|
| 109 |
+
self.init = init
|
| 110 |
+
self.n_init = n_init
|
| 111 |
+
self.random_state = random_state
|
| 112 |
+
|
| 113 |
+
@abstractmethod
|
| 114 |
+
def _check_parameters(self, n_samples):
|
| 115 |
+
"""Validate parameters depending on the input data."""
|
| 116 |
+
|
| 117 |
+
@_fit_context(prefer_skip_nested_validation=True)
|
| 118 |
+
def fit(self, X, y=None):
|
| 119 |
+
"""Create a biclustering for X.
|
| 120 |
+
|
| 121 |
+
Parameters
|
| 122 |
+
----------
|
| 123 |
+
X : array-like of shape (n_samples, n_features)
|
| 124 |
+
Training data.
|
| 125 |
+
|
| 126 |
+
y : Ignored
|
| 127 |
+
Not used, present for API consistency by convention.
|
| 128 |
+
|
| 129 |
+
Returns
|
| 130 |
+
-------
|
| 131 |
+
self : object
|
| 132 |
+
SpectralBiclustering instance.
|
| 133 |
+
"""
|
| 134 |
+
X = validate_data(self, X, accept_sparse="csr", dtype=np.float64)
|
| 135 |
+
self._check_parameters(X.shape[0])
|
| 136 |
+
self._fit(X)
|
| 137 |
+
return self
|
| 138 |
+
|
| 139 |
+
def _svd(self, array, n_components, n_discard):
|
| 140 |
+
"""Returns first `n_components` left and right singular
|
| 141 |
+
vectors u and v, discarding the first `n_discard`.
|
| 142 |
+
"""
|
| 143 |
+
if self.svd_method == "randomized":
|
| 144 |
+
kwargs = {}
|
| 145 |
+
if self.n_svd_vecs is not None:
|
| 146 |
+
kwargs["n_oversamples"] = self.n_svd_vecs
|
| 147 |
+
u, _, vt = _randomized_svd(
|
| 148 |
+
array, n_components, random_state=self.random_state, **kwargs
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
elif self.svd_method == "arpack":
|
| 152 |
+
u, _, vt = svds(array, k=n_components, ncv=self.n_svd_vecs)
|
| 153 |
+
if np.any(np.isnan(vt)):
|
| 154 |
+
# some eigenvalues of A * A.T are negative, causing
|
| 155 |
+
# sqrt() to be np.nan. This causes some vectors in vt
|
| 156 |
+
# to be np.nan.
|
| 157 |
+
A = safe_sparse_dot(array.T, array)
|
| 158 |
+
random_state = check_random_state(self.random_state)
|
| 159 |
+
# initialize with [-1,1] as in ARPACK
|
| 160 |
+
v0 = random_state.uniform(-1, 1, A.shape[0])
|
| 161 |
+
_, v = eigsh(A, ncv=self.n_svd_vecs, v0=v0)
|
| 162 |
+
vt = v.T
|
| 163 |
+
if np.any(np.isnan(u)):
|
| 164 |
+
A = safe_sparse_dot(array, array.T)
|
| 165 |
+
random_state = check_random_state(self.random_state)
|
| 166 |
+
# initialize with [-1,1] as in ARPACK
|
| 167 |
+
v0 = random_state.uniform(-1, 1, A.shape[0])
|
| 168 |
+
_, u = eigsh(A, ncv=self.n_svd_vecs, v0=v0)
|
| 169 |
+
|
| 170 |
+
assert_all_finite(u)
|
| 171 |
+
assert_all_finite(vt)
|
| 172 |
+
u = u[:, n_discard:]
|
| 173 |
+
vt = vt[n_discard:]
|
| 174 |
+
return u, vt.T
|
| 175 |
+
|
| 176 |
+
def _k_means(self, data, n_clusters):
|
| 177 |
+
if self.mini_batch:
|
| 178 |
+
model = MiniBatchKMeans(
|
| 179 |
+
n_clusters,
|
| 180 |
+
init=self.init,
|
| 181 |
+
n_init=self.n_init,
|
| 182 |
+
random_state=self.random_state,
|
| 183 |
+
)
|
| 184 |
+
else:
|
| 185 |
+
model = KMeans(
|
| 186 |
+
n_clusters,
|
| 187 |
+
init=self.init,
|
| 188 |
+
n_init=self.n_init,
|
| 189 |
+
random_state=self.random_state,
|
| 190 |
+
)
|
| 191 |
+
model.fit(data)
|
| 192 |
+
centroid = model.cluster_centers_
|
| 193 |
+
labels = model.labels_
|
| 194 |
+
return centroid, labels
|
| 195 |
+
|
| 196 |
+
def __sklearn_tags__(self):
|
| 197 |
+
tags = super().__sklearn_tags__()
|
| 198 |
+
tags.input_tags.sparse = True
|
| 199 |
+
return tags
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class SpectralCoclustering(BaseSpectral):
|
| 203 |
+
"""Spectral Co-Clustering algorithm (Dhillon, 2001) [1]_.
|
| 204 |
+
|
| 205 |
+
Clusters rows and columns of an array `X` to solve the relaxed
|
| 206 |
+
normalized cut of the bipartite graph created from `X` as follows:
|
| 207 |
+
the edge between row vertex `i` and column vertex `j` has weight
|
| 208 |
+
`X[i, j]`.
|
| 209 |
+
|
| 210 |
+
The resulting bicluster structure is block-diagonal, since each
|
| 211 |
+
row and each column belongs to exactly one bicluster.
|
| 212 |
+
|
| 213 |
+
Supports sparse matrices, as long as they are nonnegative.
|
| 214 |
+
|
| 215 |
+
Read more in the :ref:`User Guide <spectral_coclustering>`.
|
| 216 |
+
|
| 217 |
+
Parameters
|
| 218 |
+
----------
|
| 219 |
+
n_clusters : int, default=3
|
| 220 |
+
The number of biclusters to find.
|
| 221 |
+
|
| 222 |
+
svd_method : {'randomized', 'arpack'}, default='randomized'
|
| 223 |
+
Selects the algorithm for finding singular vectors. May be
|
| 224 |
+
'randomized' or 'arpack'. If 'randomized', use
|
| 225 |
+
:func:`sklearn.utils.extmath.randomized_svd`, which may be faster
|
| 226 |
+
for large matrices. If 'arpack', use
|
| 227 |
+
:func:`scipy.sparse.linalg.svds`, which is more accurate, but
|
| 228 |
+
possibly slower in some cases.
|
| 229 |
+
|
| 230 |
+
n_svd_vecs : int, default=None
|
| 231 |
+
Number of vectors to use in calculating the SVD. Corresponds
|
| 232 |
+
to `ncv` when `svd_method=arpack` and `n_oversamples` when
|
| 233 |
+
`svd_method` is 'randomized`.
|
| 234 |
+
|
| 235 |
+
mini_batch : bool, default=False
|
| 236 |
+
Whether to use mini-batch k-means, which is faster but may get
|
| 237 |
+
different results.
|
| 238 |
+
|
| 239 |
+
init : {'k-means++', 'random'}, or ndarray of shape \
|
| 240 |
+
(n_clusters, n_features), default='k-means++'
|
| 241 |
+
Method for initialization of k-means algorithm; defaults to
|
| 242 |
+
'k-means++'.
|
| 243 |
+
|
| 244 |
+
n_init : int, default=10
|
| 245 |
+
Number of random initializations that are tried with the
|
| 246 |
+
k-means algorithm.
|
| 247 |
+
|
| 248 |
+
If mini-batch k-means is used, the best initialization is
|
| 249 |
+
chosen and the algorithm runs once. Otherwise, the algorithm
|
| 250 |
+
is run for each initialization and the best solution chosen.
|
| 251 |
+
|
| 252 |
+
random_state : int, RandomState instance, default=None
|
| 253 |
+
Used for randomizing the singular value decomposition and the k-means
|
| 254 |
+
initialization. Use an int to make the randomness deterministic.
|
| 255 |
+
See :term:`Glossary <random_state>`.
|
| 256 |
+
|
| 257 |
+
Attributes
|
| 258 |
+
----------
|
| 259 |
+
rows_ : array-like of shape (n_row_clusters, n_rows)
|
| 260 |
+
Results of the clustering. `rows[i, r]` is True if
|
| 261 |
+
cluster `i` contains row `r`. Available only after calling ``fit``.
|
| 262 |
+
|
| 263 |
+
columns_ : array-like of shape (n_column_clusters, n_columns)
|
| 264 |
+
Results of the clustering, like `rows`.
|
| 265 |
+
|
| 266 |
+
row_labels_ : array-like of shape (n_rows,)
|
| 267 |
+
The bicluster label of each row.
|
| 268 |
+
|
| 269 |
+
column_labels_ : array-like of shape (n_cols,)
|
| 270 |
+
The bicluster label of each column.
|
| 271 |
+
|
| 272 |
+
biclusters_ : tuple of two ndarrays
|
| 273 |
+
The tuple contains the `rows_` and `columns_` arrays.
|
| 274 |
+
|
| 275 |
+
n_features_in_ : int
|
| 276 |
+
Number of features seen during :term:`fit`.
|
| 277 |
+
|
| 278 |
+
.. versionadded:: 0.24
|
| 279 |
+
|
| 280 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 281 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 282 |
+
has feature names that are all strings.
|
| 283 |
+
|
| 284 |
+
.. versionadded:: 1.0
|
| 285 |
+
|
| 286 |
+
See Also
|
| 287 |
+
--------
|
| 288 |
+
SpectralBiclustering : Partitions rows and columns under the assumption
|
| 289 |
+
that the data has an underlying checkerboard structure.
|
| 290 |
+
|
| 291 |
+
References
|
| 292 |
+
----------
|
| 293 |
+
.. [1] :doi:`Dhillon, Inderjit S, 2001. Co-clustering documents and words using
|
| 294 |
+
bipartite spectral graph partitioning.
|
| 295 |
+
<10.1145/502512.502550>`
|
| 296 |
+
|
| 297 |
+
Examples
|
| 298 |
+
--------
|
| 299 |
+
>>> from sklearn.cluster import SpectralCoclustering
|
| 300 |
+
>>> import numpy as np
|
| 301 |
+
>>> X = np.array([[1, 1], [2, 1], [1, 0],
|
| 302 |
+
... [4, 7], [3, 5], [3, 6]])
|
| 303 |
+
>>> clustering = SpectralCoclustering(n_clusters=2, random_state=0).fit(X)
|
| 304 |
+
>>> clustering.row_labels_ #doctest: +SKIP
|
| 305 |
+
array([0, 1, 1, 0, 0, 0], dtype=int32)
|
| 306 |
+
>>> clustering.column_labels_ #doctest: +SKIP
|
| 307 |
+
array([0, 0], dtype=int32)
|
| 308 |
+
>>> clustering
|
| 309 |
+
SpectralCoclustering(n_clusters=2, random_state=0)
|
| 310 |
+
|
| 311 |
+
For a more detailed example, see the following:
|
| 312 |
+
:ref:`sphx_glr_auto_examples_bicluster_plot_spectral_coclustering.py`.
|
| 313 |
+
"""
|
| 314 |
+
|
| 315 |
+
_parameter_constraints: dict = {
|
| 316 |
+
**BaseSpectral._parameter_constraints,
|
| 317 |
+
"n_clusters": [Interval(Integral, 1, None, closed="left")],
|
| 318 |
+
}
|
| 319 |
+
|
| 320 |
+
def __init__(
|
| 321 |
+
self,
|
| 322 |
+
n_clusters=3,
|
| 323 |
+
*,
|
| 324 |
+
svd_method="randomized",
|
| 325 |
+
n_svd_vecs=None,
|
| 326 |
+
mini_batch=False,
|
| 327 |
+
init="k-means++",
|
| 328 |
+
n_init=10,
|
| 329 |
+
random_state=None,
|
| 330 |
+
):
|
| 331 |
+
super().__init__(
|
| 332 |
+
n_clusters, svd_method, n_svd_vecs, mini_batch, init, n_init, random_state
|
| 333 |
+
)
|
| 334 |
+
|
| 335 |
+
def _check_parameters(self, n_samples):
|
| 336 |
+
if self.n_clusters > n_samples:
|
| 337 |
+
raise ValueError(
|
| 338 |
+
f"n_clusters should be <= n_samples={n_samples}. Got"
|
| 339 |
+
f" {self.n_clusters} instead."
|
| 340 |
+
)
|
| 341 |
+
|
| 342 |
+
def _fit(self, X):
|
| 343 |
+
normalized_data, row_diag, col_diag = _scale_normalize(X)
|
| 344 |
+
n_sv = 1 + int(np.ceil(np.log2(self.n_clusters)))
|
| 345 |
+
u, v = self._svd(normalized_data, n_sv, n_discard=1)
|
| 346 |
+
z = np.vstack((row_diag[:, np.newaxis] * u, col_diag[:, np.newaxis] * v))
|
| 347 |
+
|
| 348 |
+
_, labels = self._k_means(z, self.n_clusters)
|
| 349 |
+
|
| 350 |
+
n_rows = X.shape[0]
|
| 351 |
+
self.row_labels_ = labels[:n_rows]
|
| 352 |
+
self.column_labels_ = labels[n_rows:]
|
| 353 |
+
|
| 354 |
+
self.rows_ = np.vstack([self.row_labels_ == c for c in range(self.n_clusters)])
|
| 355 |
+
self.columns_ = np.vstack(
|
| 356 |
+
[self.column_labels_ == c for c in range(self.n_clusters)]
|
| 357 |
+
)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
class SpectralBiclustering(BaseSpectral):
|
| 361 |
+
"""Spectral biclustering (Kluger, 2003) [1]_.
|
| 362 |
+
|
| 363 |
+
Partitions rows and columns under the assumption that the data has
|
| 364 |
+
an underlying checkerboard structure. For instance, if there are
|
| 365 |
+
two row partitions and three column partitions, each row will
|
| 366 |
+
belong to three biclusters, and each column will belong to two
|
| 367 |
+
biclusters. The outer product of the corresponding row and column
|
| 368 |
+
label vectors gives this checkerboard structure.
|
| 369 |
+
|
| 370 |
+
Read more in the :ref:`User Guide <spectral_biclustering>`.
|
| 371 |
+
|
| 372 |
+
Parameters
|
| 373 |
+
----------
|
| 374 |
+
n_clusters : int or tuple (n_row_clusters, n_column_clusters), default=3
|
| 375 |
+
The number of row and column clusters in the checkerboard
|
| 376 |
+
structure.
|
| 377 |
+
|
| 378 |
+
method : {'bistochastic', 'scale', 'log'}, default='bistochastic'
|
| 379 |
+
Method of normalizing and converting singular vectors into
|
| 380 |
+
biclusters. May be one of 'scale', 'bistochastic', or 'log'.
|
| 381 |
+
The authors recommend using 'log'. If the data is sparse,
|
| 382 |
+
however, log normalization will not work, which is why the
|
| 383 |
+
default is 'bistochastic'.
|
| 384 |
+
|
| 385 |
+
.. warning::
|
| 386 |
+
if `method='log'`, the data must not be sparse.
|
| 387 |
+
|
| 388 |
+
n_components : int, default=6
|
| 389 |
+
Number of singular vectors to check.
|
| 390 |
+
|
| 391 |
+
n_best : int, default=3
|
| 392 |
+
Number of best singular vectors to which to project the data
|
| 393 |
+
for clustering.
|
| 394 |
+
|
| 395 |
+
svd_method : {'randomized', 'arpack'}, default='randomized'
|
| 396 |
+
Selects the algorithm for finding singular vectors. May be
|
| 397 |
+
'randomized' or 'arpack'. If 'randomized', uses
|
| 398 |
+
:func:`~sklearn.utils.extmath.randomized_svd`, which may be faster
|
| 399 |
+
for large matrices. If 'arpack', uses
|
| 400 |
+
`scipy.sparse.linalg.svds`, which is more accurate, but
|
| 401 |
+
possibly slower in some cases.
|
| 402 |
+
|
| 403 |
+
n_svd_vecs : int, default=None
|
| 404 |
+
Number of vectors to use in calculating the SVD. Corresponds
|
| 405 |
+
to `ncv` when `svd_method=arpack` and `n_oversamples` when
|
| 406 |
+
`svd_method` is 'randomized`.
|
| 407 |
+
|
| 408 |
+
mini_batch : bool, default=False
|
| 409 |
+
Whether to use mini-batch k-means, which is faster but may get
|
| 410 |
+
different results.
|
| 411 |
+
|
| 412 |
+
init : {'k-means++', 'random'} or ndarray of shape (n_clusters, n_features), \
|
| 413 |
+
default='k-means++'
|
| 414 |
+
Method for initialization of k-means algorithm; defaults to
|
| 415 |
+
'k-means++'.
|
| 416 |
+
|
| 417 |
+
n_init : int, default=10
|
| 418 |
+
Number of random initializations that are tried with the
|
| 419 |
+
k-means algorithm.
|
| 420 |
+
|
| 421 |
+
If mini-batch k-means is used, the best initialization is
|
| 422 |
+
chosen and the algorithm runs once. Otherwise, the algorithm
|
| 423 |
+
is run for each initialization and the best solution chosen.
|
| 424 |
+
|
| 425 |
+
random_state : int, RandomState instance, default=None
|
| 426 |
+
Used for randomizing the singular value decomposition and the k-means
|
| 427 |
+
initialization. Use an int to make the randomness deterministic.
|
| 428 |
+
See :term:`Glossary <random_state>`.
|
| 429 |
+
|
| 430 |
+
Attributes
|
| 431 |
+
----------
|
| 432 |
+
rows_ : array-like of shape (n_row_clusters, n_rows)
|
| 433 |
+
Results of the clustering. `rows[i, r]` is True if
|
| 434 |
+
cluster `i` contains row `r`. Available only after calling ``fit``.
|
| 435 |
+
|
| 436 |
+
columns_ : array-like of shape (n_column_clusters, n_columns)
|
| 437 |
+
Results of the clustering, like `rows`.
|
| 438 |
+
|
| 439 |
+
row_labels_ : array-like of shape (n_rows,)
|
| 440 |
+
Row partition labels.
|
| 441 |
+
|
| 442 |
+
column_labels_ : array-like of shape (n_cols,)
|
| 443 |
+
Column partition labels.
|
| 444 |
+
|
| 445 |
+
biclusters_ : tuple of two ndarrays
|
| 446 |
+
The tuple contains the `rows_` and `columns_` arrays.
|
| 447 |
+
|
| 448 |
+
n_features_in_ : int
|
| 449 |
+
Number of features seen during :term:`fit`.
|
| 450 |
+
|
| 451 |
+
.. versionadded:: 0.24
|
| 452 |
+
|
| 453 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 454 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 455 |
+
has feature names that are all strings.
|
| 456 |
+
|
| 457 |
+
.. versionadded:: 1.0
|
| 458 |
+
|
| 459 |
+
See Also
|
| 460 |
+
--------
|
| 461 |
+
SpectralCoclustering : Clusters rows and columns of an array `X` to solve the
|
| 462 |
+
relaxed normalized cut of the bipartite graph created from `X`.
|
| 463 |
+
|
| 464 |
+
References
|
| 465 |
+
----------
|
| 466 |
+
|
| 467 |
+
.. [1] :doi:`Kluger, Yuval, et. al., 2003. Spectral biclustering of microarray
|
| 468 |
+
data: coclustering genes and conditions.
|
| 469 |
+
<10.1101/gr.648603>`
|
| 470 |
+
|
| 471 |
+
Examples
|
| 472 |
+
--------
|
| 473 |
+
>>> from sklearn.cluster import SpectralBiclustering
|
| 474 |
+
>>> import numpy as np
|
| 475 |
+
>>> X = np.array([[1, 1], [2, 1], [1, 0],
|
| 476 |
+
... [4, 7], [3, 5], [3, 6]])
|
| 477 |
+
>>> clustering = SpectralBiclustering(n_clusters=2, random_state=0).fit(X)
|
| 478 |
+
>>> clustering.row_labels_
|
| 479 |
+
array([1, 1, 1, 0, 0, 0], dtype=int32)
|
| 480 |
+
>>> clustering.column_labels_
|
| 481 |
+
array([1, 0], dtype=int32)
|
| 482 |
+
>>> clustering
|
| 483 |
+
SpectralBiclustering(n_clusters=2, random_state=0)
|
| 484 |
+
|
| 485 |
+
For a more detailed example, see
|
| 486 |
+
:ref:`sphx_glr_auto_examples_bicluster_plot_spectral_biclustering.py`
|
| 487 |
+
"""
|
| 488 |
+
|
| 489 |
+
_parameter_constraints: dict = {
|
| 490 |
+
**BaseSpectral._parameter_constraints,
|
| 491 |
+
"n_clusters": [Interval(Integral, 1, None, closed="left"), tuple],
|
| 492 |
+
"method": [StrOptions({"bistochastic", "scale", "log"})],
|
| 493 |
+
"n_components": [Interval(Integral, 1, None, closed="left")],
|
| 494 |
+
"n_best": [Interval(Integral, 1, None, closed="left")],
|
| 495 |
+
}
|
| 496 |
+
|
| 497 |
+
def __init__(
|
| 498 |
+
self,
|
| 499 |
+
n_clusters=3,
|
| 500 |
+
*,
|
| 501 |
+
method="bistochastic",
|
| 502 |
+
n_components=6,
|
| 503 |
+
n_best=3,
|
| 504 |
+
svd_method="randomized",
|
| 505 |
+
n_svd_vecs=None,
|
| 506 |
+
mini_batch=False,
|
| 507 |
+
init="k-means++",
|
| 508 |
+
n_init=10,
|
| 509 |
+
random_state=None,
|
| 510 |
+
):
|
| 511 |
+
super().__init__(
|
| 512 |
+
n_clusters, svd_method, n_svd_vecs, mini_batch, init, n_init, random_state
|
| 513 |
+
)
|
| 514 |
+
self.method = method
|
| 515 |
+
self.n_components = n_components
|
| 516 |
+
self.n_best = n_best
|
| 517 |
+
|
| 518 |
+
def _check_parameters(self, n_samples):
|
| 519 |
+
if isinstance(self.n_clusters, Integral):
|
| 520 |
+
if self.n_clusters > n_samples:
|
| 521 |
+
raise ValueError(
|
| 522 |
+
f"n_clusters should be <= n_samples={n_samples}. Got"
|
| 523 |
+
f" {self.n_clusters} instead."
|
| 524 |
+
)
|
| 525 |
+
else: # tuple
|
| 526 |
+
try:
|
| 527 |
+
n_row_clusters, n_column_clusters = self.n_clusters
|
| 528 |
+
check_scalar(
|
| 529 |
+
n_row_clusters,
|
| 530 |
+
"n_row_clusters",
|
| 531 |
+
target_type=Integral,
|
| 532 |
+
min_val=1,
|
| 533 |
+
max_val=n_samples,
|
| 534 |
+
)
|
| 535 |
+
check_scalar(
|
| 536 |
+
n_column_clusters,
|
| 537 |
+
"n_column_clusters",
|
| 538 |
+
target_type=Integral,
|
| 539 |
+
min_val=1,
|
| 540 |
+
max_val=n_samples,
|
| 541 |
+
)
|
| 542 |
+
except (ValueError, TypeError) as e:
|
| 543 |
+
raise ValueError(
|
| 544 |
+
"Incorrect parameter n_clusters has value:"
|
| 545 |
+
f" {self.n_clusters}. It should either be a single integer"
|
| 546 |
+
" or an iterable with two integers:"
|
| 547 |
+
" (n_row_clusters, n_column_clusters)"
|
| 548 |
+
" And the values are should be in the"
|
| 549 |
+
" range: (1, n_samples)"
|
| 550 |
+
) from e
|
| 551 |
+
|
| 552 |
+
if self.n_best > self.n_components:
|
| 553 |
+
raise ValueError(
|
| 554 |
+
f"n_best={self.n_best} must be <= n_components={self.n_components}."
|
| 555 |
+
)
|
| 556 |
+
|
| 557 |
+
def _fit(self, X):
|
| 558 |
+
n_sv = self.n_components
|
| 559 |
+
if self.method == "bistochastic":
|
| 560 |
+
normalized_data = _bistochastic_normalize(X)
|
| 561 |
+
n_sv += 1
|
| 562 |
+
elif self.method == "scale":
|
| 563 |
+
normalized_data, _, _ = _scale_normalize(X)
|
| 564 |
+
n_sv += 1
|
| 565 |
+
elif self.method == "log":
|
| 566 |
+
normalized_data = _log_normalize(X)
|
| 567 |
+
n_discard = 0 if self.method == "log" else 1
|
| 568 |
+
u, v = self._svd(normalized_data, n_sv, n_discard)
|
| 569 |
+
ut = u.T
|
| 570 |
+
vt = v.T
|
| 571 |
+
|
| 572 |
+
try:
|
| 573 |
+
n_row_clusters, n_col_clusters = self.n_clusters
|
| 574 |
+
except TypeError:
|
| 575 |
+
n_row_clusters = n_col_clusters = self.n_clusters
|
| 576 |
+
|
| 577 |
+
best_ut = self._fit_best_piecewise(ut, self.n_best, n_row_clusters)
|
| 578 |
+
|
| 579 |
+
best_vt = self._fit_best_piecewise(vt, self.n_best, n_col_clusters)
|
| 580 |
+
|
| 581 |
+
self.row_labels_ = self._project_and_cluster(X, best_vt.T, n_row_clusters)
|
| 582 |
+
|
| 583 |
+
self.column_labels_ = self._project_and_cluster(X.T, best_ut.T, n_col_clusters)
|
| 584 |
+
|
| 585 |
+
self.rows_ = np.vstack(
|
| 586 |
+
[
|
| 587 |
+
self.row_labels_ == label
|
| 588 |
+
for label in range(n_row_clusters)
|
| 589 |
+
for _ in range(n_col_clusters)
|
| 590 |
+
]
|
| 591 |
+
)
|
| 592 |
+
self.columns_ = np.vstack(
|
| 593 |
+
[
|
| 594 |
+
self.column_labels_ == label
|
| 595 |
+
for _ in range(n_row_clusters)
|
| 596 |
+
for label in range(n_col_clusters)
|
| 597 |
+
]
|
| 598 |
+
)
|
| 599 |
+
|
| 600 |
+
def _fit_best_piecewise(self, vectors, n_best, n_clusters):
|
| 601 |
+
"""Find the ``n_best`` vectors that are best approximated by piecewise
|
| 602 |
+
constant vectors.
|
| 603 |
+
|
| 604 |
+
The piecewise vectors are found by k-means; the best is chosen
|
| 605 |
+
according to Euclidean distance.
|
| 606 |
+
|
| 607 |
+
"""
|
| 608 |
+
|
| 609 |
+
def make_piecewise(v):
|
| 610 |
+
centroid, labels = self._k_means(v.reshape(-1, 1), n_clusters)
|
| 611 |
+
return centroid[labels].ravel()
|
| 612 |
+
|
| 613 |
+
piecewise_vectors = np.apply_along_axis(make_piecewise, axis=1, arr=vectors)
|
| 614 |
+
dists = np.apply_along_axis(norm, axis=1, arr=(vectors - piecewise_vectors))
|
| 615 |
+
result = vectors[np.argsort(dists)[:n_best]]
|
| 616 |
+
return result
|
| 617 |
+
|
| 618 |
+
def _project_and_cluster(self, data, vectors, n_clusters):
|
| 619 |
+
"""Project ``data`` to ``vectors`` and cluster the result."""
|
| 620 |
+
projected = safe_sparse_dot(data, vectors)
|
| 621 |
+
_, labels = self._k_means(projected, n_clusters)
|
| 622 |
+
return labels
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_birch.py
ADDED
|
@@ -0,0 +1,730 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Authors: The scikit-learn developers
|
| 2 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 3 |
+
|
| 4 |
+
import warnings
|
| 5 |
+
from math import sqrt
|
| 6 |
+
from numbers import Integral, Real
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
from scipy import sparse
|
| 10 |
+
|
| 11 |
+
from sklearn._config import config_context
|
| 12 |
+
from sklearn.base import (
|
| 13 |
+
BaseEstimator,
|
| 14 |
+
ClassNamePrefixFeaturesOutMixin,
|
| 15 |
+
ClusterMixin,
|
| 16 |
+
TransformerMixin,
|
| 17 |
+
_fit_context,
|
| 18 |
+
)
|
| 19 |
+
from sklearn.cluster import AgglomerativeClustering
|
| 20 |
+
from sklearn.exceptions import ConvergenceWarning
|
| 21 |
+
from sklearn.metrics import pairwise_distances_argmin
|
| 22 |
+
from sklearn.metrics.pairwise import euclidean_distances
|
| 23 |
+
from sklearn.utils._param_validation import Interval
|
| 24 |
+
from sklearn.utils.extmath import row_norms
|
| 25 |
+
from sklearn.utils.validation import check_is_fitted, validate_data
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _iterate_sparse_X(X):
|
| 29 |
+
"""This little hack returns a densified row when iterating over a sparse
|
| 30 |
+
matrix, instead of constructing a sparse matrix for every row that is
|
| 31 |
+
expensive.
|
| 32 |
+
"""
|
| 33 |
+
n_samples = X.shape[0]
|
| 34 |
+
X_indices = X.indices
|
| 35 |
+
X_data = X.data
|
| 36 |
+
X_indptr = X.indptr
|
| 37 |
+
|
| 38 |
+
for i in range(n_samples):
|
| 39 |
+
row = np.zeros(X.shape[1])
|
| 40 |
+
startptr, endptr = X_indptr[i], X_indptr[i + 1]
|
| 41 |
+
nonzero_indices = X_indices[startptr:endptr]
|
| 42 |
+
row[nonzero_indices] = X_data[startptr:endptr]
|
| 43 |
+
yield row
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _split_node(node, threshold, branching_factor):
|
| 47 |
+
"""The node has to be split if there is no place for a new subcluster
|
| 48 |
+
in the node.
|
| 49 |
+
1. Two empty nodes and two empty subclusters are initialized.
|
| 50 |
+
2. The pair of distant subclusters are found.
|
| 51 |
+
3. The properties of the empty subclusters and nodes are updated
|
| 52 |
+
according to the nearest distance between the subclusters to the
|
| 53 |
+
pair of distant subclusters.
|
| 54 |
+
4. The two nodes are set as children to the two subclusters.
|
| 55 |
+
"""
|
| 56 |
+
new_subcluster1 = _CFSubcluster()
|
| 57 |
+
new_subcluster2 = _CFSubcluster()
|
| 58 |
+
new_node1 = _CFNode(
|
| 59 |
+
threshold=threshold,
|
| 60 |
+
branching_factor=branching_factor,
|
| 61 |
+
is_leaf=node.is_leaf,
|
| 62 |
+
n_features=node.n_features,
|
| 63 |
+
dtype=node.init_centroids_.dtype,
|
| 64 |
+
)
|
| 65 |
+
new_node2 = _CFNode(
|
| 66 |
+
threshold=threshold,
|
| 67 |
+
branching_factor=branching_factor,
|
| 68 |
+
is_leaf=node.is_leaf,
|
| 69 |
+
n_features=node.n_features,
|
| 70 |
+
dtype=node.init_centroids_.dtype,
|
| 71 |
+
)
|
| 72 |
+
new_subcluster1.child_ = new_node1
|
| 73 |
+
new_subcluster2.child_ = new_node2
|
| 74 |
+
|
| 75 |
+
if node.is_leaf:
|
| 76 |
+
if node.prev_leaf_ is not None:
|
| 77 |
+
node.prev_leaf_.next_leaf_ = new_node1
|
| 78 |
+
new_node1.prev_leaf_ = node.prev_leaf_
|
| 79 |
+
new_node1.next_leaf_ = new_node2
|
| 80 |
+
new_node2.prev_leaf_ = new_node1
|
| 81 |
+
new_node2.next_leaf_ = node.next_leaf_
|
| 82 |
+
if node.next_leaf_ is not None:
|
| 83 |
+
node.next_leaf_.prev_leaf_ = new_node2
|
| 84 |
+
|
| 85 |
+
dist = euclidean_distances(
|
| 86 |
+
node.centroids_, Y_norm_squared=node.squared_norm_, squared=True
|
| 87 |
+
)
|
| 88 |
+
n_clusters = dist.shape[0]
|
| 89 |
+
|
| 90 |
+
farthest_idx = np.unravel_index(dist.argmax(), (n_clusters, n_clusters))
|
| 91 |
+
node1_dist, node2_dist = dist[(farthest_idx,)]
|
| 92 |
+
|
| 93 |
+
node1_closer = node1_dist < node2_dist
|
| 94 |
+
# make sure node1 is closest to itself even if all distances are equal.
|
| 95 |
+
# This can only happen when all node.centroids_ are duplicates leading to all
|
| 96 |
+
# distances between centroids being zero.
|
| 97 |
+
node1_closer[farthest_idx[0]] = True
|
| 98 |
+
|
| 99 |
+
for idx, subcluster in enumerate(node.subclusters_):
|
| 100 |
+
if node1_closer[idx]:
|
| 101 |
+
new_node1.append_subcluster(subcluster)
|
| 102 |
+
new_subcluster1.update(subcluster)
|
| 103 |
+
else:
|
| 104 |
+
new_node2.append_subcluster(subcluster)
|
| 105 |
+
new_subcluster2.update(subcluster)
|
| 106 |
+
return new_subcluster1, new_subcluster2
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
class _CFNode:
|
| 110 |
+
"""Each node in a CFTree is called a CFNode.
|
| 111 |
+
|
| 112 |
+
The CFNode can have a maximum of branching_factor
|
| 113 |
+
number of CFSubclusters.
|
| 114 |
+
|
| 115 |
+
Parameters
|
| 116 |
+
----------
|
| 117 |
+
threshold : float
|
| 118 |
+
Threshold needed for a new subcluster to enter a CFSubcluster.
|
| 119 |
+
|
| 120 |
+
branching_factor : int
|
| 121 |
+
Maximum number of CF subclusters in each node.
|
| 122 |
+
|
| 123 |
+
is_leaf : bool
|
| 124 |
+
We need to know if the CFNode is a leaf or not, in order to
|
| 125 |
+
retrieve the final subclusters.
|
| 126 |
+
|
| 127 |
+
n_features : int
|
| 128 |
+
The number of features.
|
| 129 |
+
|
| 130 |
+
Attributes
|
| 131 |
+
----------
|
| 132 |
+
subclusters_ : list
|
| 133 |
+
List of subclusters for a particular CFNode.
|
| 134 |
+
|
| 135 |
+
prev_leaf_ : _CFNode
|
| 136 |
+
Useful only if is_leaf is True.
|
| 137 |
+
|
| 138 |
+
next_leaf_ : _CFNode
|
| 139 |
+
next_leaf. Useful only if is_leaf is True.
|
| 140 |
+
the final subclusters.
|
| 141 |
+
|
| 142 |
+
init_centroids_ : ndarray of shape (branching_factor + 1, n_features)
|
| 143 |
+
Manipulate ``init_centroids_`` throughout rather than centroids_ since
|
| 144 |
+
the centroids are just a view of the ``init_centroids_`` .
|
| 145 |
+
|
| 146 |
+
init_sq_norm_ : ndarray of shape (branching_factor + 1,)
|
| 147 |
+
manipulate init_sq_norm_ throughout. similar to ``init_centroids_``.
|
| 148 |
+
|
| 149 |
+
centroids_ : ndarray of shape (branching_factor + 1, n_features)
|
| 150 |
+
View of ``init_centroids_``.
|
| 151 |
+
|
| 152 |
+
squared_norm_ : ndarray of shape (branching_factor + 1,)
|
| 153 |
+
View of ``init_sq_norm_``.
|
| 154 |
+
|
| 155 |
+
"""
|
| 156 |
+
|
| 157 |
+
def __init__(self, *, threshold, branching_factor, is_leaf, n_features, dtype):
|
| 158 |
+
self.threshold = threshold
|
| 159 |
+
self.branching_factor = branching_factor
|
| 160 |
+
self.is_leaf = is_leaf
|
| 161 |
+
self.n_features = n_features
|
| 162 |
+
|
| 163 |
+
# The list of subclusters, centroids and squared norms
|
| 164 |
+
# to manipulate throughout.
|
| 165 |
+
self.subclusters_ = []
|
| 166 |
+
self.init_centroids_ = np.zeros((branching_factor + 1, n_features), dtype=dtype)
|
| 167 |
+
self.init_sq_norm_ = np.zeros((branching_factor + 1), dtype)
|
| 168 |
+
self.squared_norm_ = []
|
| 169 |
+
self.prev_leaf_ = None
|
| 170 |
+
self.next_leaf_ = None
|
| 171 |
+
|
| 172 |
+
def append_subcluster(self, subcluster):
|
| 173 |
+
n_samples = len(self.subclusters_)
|
| 174 |
+
self.subclusters_.append(subcluster)
|
| 175 |
+
self.init_centroids_[n_samples] = subcluster.centroid_
|
| 176 |
+
self.init_sq_norm_[n_samples] = subcluster.sq_norm_
|
| 177 |
+
|
| 178 |
+
# Keep centroids and squared norm as views. In this way
|
| 179 |
+
# if we change init_centroids and init_sq_norm_, it is
|
| 180 |
+
# sufficient,
|
| 181 |
+
self.centroids_ = self.init_centroids_[: n_samples + 1, :]
|
| 182 |
+
self.squared_norm_ = self.init_sq_norm_[: n_samples + 1]
|
| 183 |
+
|
| 184 |
+
def update_split_subclusters(self, subcluster, new_subcluster1, new_subcluster2):
|
| 185 |
+
"""Remove a subcluster from a node and update it with the
|
| 186 |
+
split subclusters.
|
| 187 |
+
"""
|
| 188 |
+
ind = self.subclusters_.index(subcluster)
|
| 189 |
+
self.subclusters_[ind] = new_subcluster1
|
| 190 |
+
self.init_centroids_[ind] = new_subcluster1.centroid_
|
| 191 |
+
self.init_sq_norm_[ind] = new_subcluster1.sq_norm_
|
| 192 |
+
self.append_subcluster(new_subcluster2)
|
| 193 |
+
|
| 194 |
+
def insert_cf_subcluster(self, subcluster):
|
| 195 |
+
"""Insert a new subcluster into the node."""
|
| 196 |
+
if not self.subclusters_:
|
| 197 |
+
self.append_subcluster(subcluster)
|
| 198 |
+
return False
|
| 199 |
+
|
| 200 |
+
threshold = self.threshold
|
| 201 |
+
branching_factor = self.branching_factor
|
| 202 |
+
# We need to find the closest subcluster among all the
|
| 203 |
+
# subclusters so that we can insert our new subcluster.
|
| 204 |
+
dist_matrix = np.dot(self.centroids_, subcluster.centroid_)
|
| 205 |
+
dist_matrix *= -2.0
|
| 206 |
+
dist_matrix += self.squared_norm_
|
| 207 |
+
closest_index = np.argmin(dist_matrix)
|
| 208 |
+
closest_subcluster = self.subclusters_[closest_index]
|
| 209 |
+
|
| 210 |
+
# If the subcluster has a child, we need a recursive strategy.
|
| 211 |
+
if closest_subcluster.child_ is not None:
|
| 212 |
+
split_child = closest_subcluster.child_.insert_cf_subcluster(subcluster)
|
| 213 |
+
|
| 214 |
+
if not split_child:
|
| 215 |
+
# If it is determined that the child need not be split, we
|
| 216 |
+
# can just update the closest_subcluster
|
| 217 |
+
closest_subcluster.update(subcluster)
|
| 218 |
+
self.init_centroids_[closest_index] = self.subclusters_[
|
| 219 |
+
closest_index
|
| 220 |
+
].centroid_
|
| 221 |
+
self.init_sq_norm_[closest_index] = self.subclusters_[
|
| 222 |
+
closest_index
|
| 223 |
+
].sq_norm_
|
| 224 |
+
return False
|
| 225 |
+
|
| 226 |
+
# things not too good. we need to redistribute the subclusters in
|
| 227 |
+
# our child node, and add a new subcluster in the parent
|
| 228 |
+
# subcluster to accommodate the new child.
|
| 229 |
+
else:
|
| 230 |
+
new_subcluster1, new_subcluster2 = _split_node(
|
| 231 |
+
closest_subcluster.child_,
|
| 232 |
+
threshold,
|
| 233 |
+
branching_factor,
|
| 234 |
+
)
|
| 235 |
+
self.update_split_subclusters(
|
| 236 |
+
closest_subcluster, new_subcluster1, new_subcluster2
|
| 237 |
+
)
|
| 238 |
+
|
| 239 |
+
if len(self.subclusters_) > self.branching_factor:
|
| 240 |
+
return True
|
| 241 |
+
return False
|
| 242 |
+
|
| 243 |
+
# good to go!
|
| 244 |
+
else:
|
| 245 |
+
merged = closest_subcluster.merge_subcluster(subcluster, self.threshold)
|
| 246 |
+
if merged:
|
| 247 |
+
self.init_centroids_[closest_index] = closest_subcluster.centroid_
|
| 248 |
+
self.init_sq_norm_[closest_index] = closest_subcluster.sq_norm_
|
| 249 |
+
return False
|
| 250 |
+
|
| 251 |
+
# not close to any other subclusters, and we still
|
| 252 |
+
# have space, so add.
|
| 253 |
+
elif len(self.subclusters_) < self.branching_factor:
|
| 254 |
+
self.append_subcluster(subcluster)
|
| 255 |
+
return False
|
| 256 |
+
|
| 257 |
+
# We do not have enough space nor is it closer to an
|
| 258 |
+
# other subcluster. We need to split.
|
| 259 |
+
else:
|
| 260 |
+
self.append_subcluster(subcluster)
|
| 261 |
+
return True
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
class _CFSubcluster:
|
| 265 |
+
"""Each subcluster in a CFNode is called a CFSubcluster.
|
| 266 |
+
|
| 267 |
+
A CFSubcluster can have a CFNode has its child.
|
| 268 |
+
|
| 269 |
+
Parameters
|
| 270 |
+
----------
|
| 271 |
+
linear_sum : ndarray of shape (n_features,), default=None
|
| 272 |
+
Sample. This is kept optional to allow initialization of empty
|
| 273 |
+
subclusters.
|
| 274 |
+
|
| 275 |
+
Attributes
|
| 276 |
+
----------
|
| 277 |
+
n_samples_ : int
|
| 278 |
+
Number of samples that belong to each subcluster.
|
| 279 |
+
|
| 280 |
+
linear_sum_ : ndarray
|
| 281 |
+
Linear sum of all the samples in a subcluster. Prevents holding
|
| 282 |
+
all sample data in memory.
|
| 283 |
+
|
| 284 |
+
squared_sum_ : float
|
| 285 |
+
Sum of the squared l2 norms of all samples belonging to a subcluster.
|
| 286 |
+
|
| 287 |
+
centroid_ : ndarray of shape (branching_factor + 1, n_features)
|
| 288 |
+
Centroid of the subcluster. Prevent recomputing of centroids when
|
| 289 |
+
``CFNode.centroids_`` is called.
|
| 290 |
+
|
| 291 |
+
child_ : _CFNode
|
| 292 |
+
Child Node of the subcluster. Once a given _CFNode is set as the child
|
| 293 |
+
of the _CFNode, it is set to ``self.child_``.
|
| 294 |
+
|
| 295 |
+
sq_norm_ : ndarray of shape (branching_factor + 1,)
|
| 296 |
+
Squared norm of the subcluster. Used to prevent recomputing when
|
| 297 |
+
pairwise minimum distances are computed.
|
| 298 |
+
"""
|
| 299 |
+
|
| 300 |
+
def __init__(self, *, linear_sum=None):
|
| 301 |
+
if linear_sum is None:
|
| 302 |
+
self.n_samples_ = 0
|
| 303 |
+
self.squared_sum_ = 0.0
|
| 304 |
+
self.centroid_ = self.linear_sum_ = 0
|
| 305 |
+
else:
|
| 306 |
+
self.n_samples_ = 1
|
| 307 |
+
self.centroid_ = self.linear_sum_ = linear_sum
|
| 308 |
+
self.squared_sum_ = self.sq_norm_ = np.dot(
|
| 309 |
+
self.linear_sum_, self.linear_sum_
|
| 310 |
+
)
|
| 311 |
+
self.child_ = None
|
| 312 |
+
|
| 313 |
+
def update(self, subcluster):
|
| 314 |
+
self.n_samples_ += subcluster.n_samples_
|
| 315 |
+
self.linear_sum_ += subcluster.linear_sum_
|
| 316 |
+
self.squared_sum_ += subcluster.squared_sum_
|
| 317 |
+
self.centroid_ = self.linear_sum_ / self.n_samples_
|
| 318 |
+
self.sq_norm_ = np.dot(self.centroid_, self.centroid_)
|
| 319 |
+
|
| 320 |
+
def merge_subcluster(self, nominee_cluster, threshold):
|
| 321 |
+
"""Check if a cluster is worthy enough to be merged. If
|
| 322 |
+
yes then merge.
|
| 323 |
+
"""
|
| 324 |
+
new_ss = self.squared_sum_ + nominee_cluster.squared_sum_
|
| 325 |
+
new_ls = self.linear_sum_ + nominee_cluster.linear_sum_
|
| 326 |
+
new_n = self.n_samples_ + nominee_cluster.n_samples_
|
| 327 |
+
new_centroid = (1 / new_n) * new_ls
|
| 328 |
+
new_sq_norm = np.dot(new_centroid, new_centroid)
|
| 329 |
+
|
| 330 |
+
# The squared radius of the cluster is defined:
|
| 331 |
+
# r^2 = sum_i ||x_i - c||^2 / n
|
| 332 |
+
# with x_i the n points assigned to the cluster and c its centroid:
|
| 333 |
+
# c = sum_i x_i / n
|
| 334 |
+
# This can be expanded to:
|
| 335 |
+
# r^2 = sum_i ||x_i||^2 / n - 2 < sum_i x_i / n, c> + n ||c||^2 / n
|
| 336 |
+
# and therefore simplifies to:
|
| 337 |
+
# r^2 = sum_i ||x_i||^2 / n - ||c||^2
|
| 338 |
+
sq_radius = new_ss / new_n - new_sq_norm
|
| 339 |
+
|
| 340 |
+
if sq_radius <= threshold**2:
|
| 341 |
+
(
|
| 342 |
+
self.n_samples_,
|
| 343 |
+
self.linear_sum_,
|
| 344 |
+
self.squared_sum_,
|
| 345 |
+
self.centroid_,
|
| 346 |
+
self.sq_norm_,
|
| 347 |
+
) = (new_n, new_ls, new_ss, new_centroid, new_sq_norm)
|
| 348 |
+
return True
|
| 349 |
+
return False
|
| 350 |
+
|
| 351 |
+
@property
|
| 352 |
+
def radius(self):
|
| 353 |
+
"""Return radius of the subcluster"""
|
| 354 |
+
# Because of numerical issues, this could become negative
|
| 355 |
+
sq_radius = self.squared_sum_ / self.n_samples_ - self.sq_norm_
|
| 356 |
+
return sqrt(max(0, sq_radius))
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
class Birch(
|
| 360 |
+
ClassNamePrefixFeaturesOutMixin, ClusterMixin, TransformerMixin, BaseEstimator
|
| 361 |
+
):
|
| 362 |
+
"""Implements the BIRCH clustering algorithm.
|
| 363 |
+
|
| 364 |
+
It is a memory-efficient, online-learning algorithm provided as an
|
| 365 |
+
alternative to :class:`MiniBatchKMeans`. It constructs a tree
|
| 366 |
+
data structure with the cluster centroids being read off the leaf.
|
| 367 |
+
These can be either the final cluster centroids or can be provided as input
|
| 368 |
+
to another clustering algorithm such as :class:`AgglomerativeClustering`.
|
| 369 |
+
|
| 370 |
+
Read more in the :ref:`User Guide <birch>`.
|
| 371 |
+
|
| 372 |
+
.. versionadded:: 0.16
|
| 373 |
+
|
| 374 |
+
Parameters
|
| 375 |
+
----------
|
| 376 |
+
threshold : float, default=0.5
|
| 377 |
+
The radius of the subcluster obtained by merging a new sample and the
|
| 378 |
+
closest subcluster should be lesser than the threshold. Otherwise a new
|
| 379 |
+
subcluster is started. Setting this value to be very low promotes
|
| 380 |
+
splitting and vice-versa.
|
| 381 |
+
|
| 382 |
+
branching_factor : int, default=50
|
| 383 |
+
Maximum number of CF subclusters in each node. If a new samples enters
|
| 384 |
+
such that the number of subclusters exceed the branching_factor then
|
| 385 |
+
that node is split into two nodes with the subclusters redistributed
|
| 386 |
+
in each. The parent subcluster of that node is removed and two new
|
| 387 |
+
subclusters are added as parents of the 2 split nodes.
|
| 388 |
+
|
| 389 |
+
n_clusters : int, instance of sklearn.cluster model or None, default=3
|
| 390 |
+
Number of clusters after the final clustering step, which treats the
|
| 391 |
+
subclusters from the leaves as new samples.
|
| 392 |
+
|
| 393 |
+
- `None` : the final clustering step is not performed and the
|
| 394 |
+
subclusters are returned as they are.
|
| 395 |
+
|
| 396 |
+
- :mod:`sklearn.cluster` Estimator : If a model is provided, the model
|
| 397 |
+
is fit treating the subclusters as new samples and the initial data
|
| 398 |
+
is mapped to the label of the closest subcluster.
|
| 399 |
+
|
| 400 |
+
- `int` : the model fit is :class:`AgglomerativeClustering` with
|
| 401 |
+
`n_clusters` set to be equal to the int.
|
| 402 |
+
|
| 403 |
+
compute_labels : bool, default=True
|
| 404 |
+
Whether or not to compute labels for each fit.
|
| 405 |
+
|
| 406 |
+
Attributes
|
| 407 |
+
----------
|
| 408 |
+
root_ : _CFNode
|
| 409 |
+
Root of the CFTree.
|
| 410 |
+
|
| 411 |
+
dummy_leaf_ : _CFNode
|
| 412 |
+
Start pointer to all the leaves.
|
| 413 |
+
|
| 414 |
+
subcluster_centers_ : ndarray
|
| 415 |
+
Centroids of all subclusters read directly from the leaves.
|
| 416 |
+
|
| 417 |
+
subcluster_labels_ : ndarray
|
| 418 |
+
Labels assigned to the centroids of the subclusters after
|
| 419 |
+
they are clustered globally.
|
| 420 |
+
|
| 421 |
+
labels_ : ndarray of shape (n_samples,)
|
| 422 |
+
Array of labels assigned to the input data.
|
| 423 |
+
if partial_fit is used instead of fit, they are assigned to the
|
| 424 |
+
last batch of data.
|
| 425 |
+
|
| 426 |
+
n_features_in_ : int
|
| 427 |
+
Number of features seen during :term:`fit`.
|
| 428 |
+
|
| 429 |
+
.. versionadded:: 0.24
|
| 430 |
+
|
| 431 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 432 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 433 |
+
has feature names that are all strings.
|
| 434 |
+
|
| 435 |
+
.. versionadded:: 1.0
|
| 436 |
+
|
| 437 |
+
See Also
|
| 438 |
+
--------
|
| 439 |
+
MiniBatchKMeans : Alternative implementation that does incremental updates
|
| 440 |
+
of the centers' positions using mini-batches.
|
| 441 |
+
|
| 442 |
+
Notes
|
| 443 |
+
-----
|
| 444 |
+
The tree data structure consists of nodes with each node consisting of
|
| 445 |
+
a number of subclusters. The maximum number of subclusters in a node
|
| 446 |
+
is determined by the branching factor. Each subcluster maintains a
|
| 447 |
+
linear sum, squared sum and the number of samples in that subcluster.
|
| 448 |
+
In addition, each subcluster can also have a node as its child, if the
|
| 449 |
+
subcluster is not a member of a leaf node.
|
| 450 |
+
|
| 451 |
+
For a new point entering the root, it is merged with the subcluster closest
|
| 452 |
+
to it and the linear sum, squared sum and the number of samples of that
|
| 453 |
+
subcluster are updated. This is done recursively till the properties of
|
| 454 |
+
the leaf node are updated.
|
| 455 |
+
|
| 456 |
+
See :ref:`sphx_glr_auto_examples_cluster_plot_birch_vs_minibatchkmeans.py` for a
|
| 457 |
+
comparison with :class:`~sklearn.cluster.MiniBatchKMeans`.
|
| 458 |
+
|
| 459 |
+
References
|
| 460 |
+
----------
|
| 461 |
+
* Tian Zhang, Raghu Ramakrishnan, Maron Livny
|
| 462 |
+
BIRCH: An efficient data clustering method for large databases.
|
| 463 |
+
https://www.cs.sfu.ca/CourseCentral/459/han/papers/zhang96.pdf
|
| 464 |
+
|
| 465 |
+
* Roberto Perdisci
|
| 466 |
+
JBirch - Java implementation of BIRCH clustering algorithm
|
| 467 |
+
https://code.google.com/archive/p/jbirch
|
| 468 |
+
|
| 469 |
+
Examples
|
| 470 |
+
--------
|
| 471 |
+
>>> from sklearn.cluster import Birch
|
| 472 |
+
>>> X = [[0, 1], [0.3, 1], [-0.3, 1], [0, -1], [0.3, -1], [-0.3, -1]]
|
| 473 |
+
>>> brc = Birch(n_clusters=None)
|
| 474 |
+
>>> brc.fit(X)
|
| 475 |
+
Birch(n_clusters=None)
|
| 476 |
+
>>> brc.predict(X)
|
| 477 |
+
array([0, 0, 0, 1, 1, 1])
|
| 478 |
+
|
| 479 |
+
For a comparison of the BIRCH clustering algorithm with other clustering algorithms,
|
| 480 |
+
see :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
|
| 481 |
+
"""
|
| 482 |
+
|
| 483 |
+
_parameter_constraints: dict = {
|
| 484 |
+
"threshold": [Interval(Real, 0.0, None, closed="neither")],
|
| 485 |
+
"branching_factor": [Interval(Integral, 1, None, closed="neither")],
|
| 486 |
+
"n_clusters": [None, ClusterMixin, Interval(Integral, 1, None, closed="left")],
|
| 487 |
+
"compute_labels": ["boolean"],
|
| 488 |
+
}
|
| 489 |
+
|
| 490 |
+
def __init__(
|
| 491 |
+
self,
|
| 492 |
+
*,
|
| 493 |
+
threshold=0.5,
|
| 494 |
+
branching_factor=50,
|
| 495 |
+
n_clusters=3,
|
| 496 |
+
compute_labels=True,
|
| 497 |
+
):
|
| 498 |
+
self.threshold = threshold
|
| 499 |
+
self.branching_factor = branching_factor
|
| 500 |
+
self.n_clusters = n_clusters
|
| 501 |
+
self.compute_labels = compute_labels
|
| 502 |
+
|
| 503 |
+
@_fit_context(prefer_skip_nested_validation=True)
|
| 504 |
+
def fit(self, X, y=None):
|
| 505 |
+
"""
|
| 506 |
+
Build a CF Tree for the input data.
|
| 507 |
+
|
| 508 |
+
Parameters
|
| 509 |
+
----------
|
| 510 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 511 |
+
Input data.
|
| 512 |
+
|
| 513 |
+
y : Ignored
|
| 514 |
+
Not used, present here for API consistency by convention.
|
| 515 |
+
|
| 516 |
+
Returns
|
| 517 |
+
-------
|
| 518 |
+
self
|
| 519 |
+
Fitted estimator.
|
| 520 |
+
"""
|
| 521 |
+
return self._fit(X, partial=False)
|
| 522 |
+
|
| 523 |
+
def _fit(self, X, partial):
|
| 524 |
+
has_root = getattr(self, "root_", None)
|
| 525 |
+
first_call = not (partial and has_root)
|
| 526 |
+
|
| 527 |
+
X = validate_data(
|
| 528 |
+
self,
|
| 529 |
+
X,
|
| 530 |
+
accept_sparse="csr",
|
| 531 |
+
reset=first_call,
|
| 532 |
+
dtype=[np.float64, np.float32],
|
| 533 |
+
)
|
| 534 |
+
threshold = self.threshold
|
| 535 |
+
branching_factor = self.branching_factor
|
| 536 |
+
|
| 537 |
+
n_samples, n_features = X.shape
|
| 538 |
+
|
| 539 |
+
# If partial_fit is called for the first time or fit is called, we
|
| 540 |
+
# start a new tree.
|
| 541 |
+
if first_call:
|
| 542 |
+
# The first root is the leaf. Manipulate this object throughout.
|
| 543 |
+
self.root_ = _CFNode(
|
| 544 |
+
threshold=threshold,
|
| 545 |
+
branching_factor=branching_factor,
|
| 546 |
+
is_leaf=True,
|
| 547 |
+
n_features=n_features,
|
| 548 |
+
dtype=X.dtype,
|
| 549 |
+
)
|
| 550 |
+
|
| 551 |
+
# To enable getting back subclusters.
|
| 552 |
+
self.dummy_leaf_ = _CFNode(
|
| 553 |
+
threshold=threshold,
|
| 554 |
+
branching_factor=branching_factor,
|
| 555 |
+
is_leaf=True,
|
| 556 |
+
n_features=n_features,
|
| 557 |
+
dtype=X.dtype,
|
| 558 |
+
)
|
| 559 |
+
self.dummy_leaf_.next_leaf_ = self.root_
|
| 560 |
+
self.root_.prev_leaf_ = self.dummy_leaf_
|
| 561 |
+
|
| 562 |
+
# Cannot vectorize. Enough to convince to use cython.
|
| 563 |
+
if not sparse.issparse(X):
|
| 564 |
+
iter_func = iter
|
| 565 |
+
else:
|
| 566 |
+
iter_func = _iterate_sparse_X
|
| 567 |
+
|
| 568 |
+
for sample in iter_func(X):
|
| 569 |
+
subcluster = _CFSubcluster(linear_sum=sample)
|
| 570 |
+
split = self.root_.insert_cf_subcluster(subcluster)
|
| 571 |
+
|
| 572 |
+
if split:
|
| 573 |
+
new_subcluster1, new_subcluster2 = _split_node(
|
| 574 |
+
self.root_, threshold, branching_factor
|
| 575 |
+
)
|
| 576 |
+
del self.root_
|
| 577 |
+
self.root_ = _CFNode(
|
| 578 |
+
threshold=threshold,
|
| 579 |
+
branching_factor=branching_factor,
|
| 580 |
+
is_leaf=False,
|
| 581 |
+
n_features=n_features,
|
| 582 |
+
dtype=X.dtype,
|
| 583 |
+
)
|
| 584 |
+
self.root_.append_subcluster(new_subcluster1)
|
| 585 |
+
self.root_.append_subcluster(new_subcluster2)
|
| 586 |
+
|
| 587 |
+
centroids = np.concatenate([leaf.centroids_ for leaf in self._get_leaves()])
|
| 588 |
+
self.subcluster_centers_ = centroids
|
| 589 |
+
self._n_features_out = self.subcluster_centers_.shape[0]
|
| 590 |
+
|
| 591 |
+
self._global_clustering(X)
|
| 592 |
+
return self
|
| 593 |
+
|
| 594 |
+
def _get_leaves(self):
|
| 595 |
+
"""
|
| 596 |
+
Retrieve the leaves of the CF Node.
|
| 597 |
+
|
| 598 |
+
Returns
|
| 599 |
+
-------
|
| 600 |
+
leaves : list of shape (n_leaves,)
|
| 601 |
+
List of the leaf nodes.
|
| 602 |
+
"""
|
| 603 |
+
leaf_ptr = self.dummy_leaf_.next_leaf_
|
| 604 |
+
leaves = []
|
| 605 |
+
while leaf_ptr is not None:
|
| 606 |
+
leaves.append(leaf_ptr)
|
| 607 |
+
leaf_ptr = leaf_ptr.next_leaf_
|
| 608 |
+
return leaves
|
| 609 |
+
|
| 610 |
+
@_fit_context(prefer_skip_nested_validation=True)
|
| 611 |
+
def partial_fit(self, X=None, y=None):
|
| 612 |
+
"""
|
| 613 |
+
Online learning. Prevents rebuilding of CFTree from scratch.
|
| 614 |
+
|
| 615 |
+
Parameters
|
| 616 |
+
----------
|
| 617 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features), \
|
| 618 |
+
default=None
|
| 619 |
+
Input data. If X is not provided, only the global clustering
|
| 620 |
+
step is done.
|
| 621 |
+
|
| 622 |
+
y : Ignored
|
| 623 |
+
Not used, present here for API consistency by convention.
|
| 624 |
+
|
| 625 |
+
Returns
|
| 626 |
+
-------
|
| 627 |
+
self
|
| 628 |
+
Fitted estimator.
|
| 629 |
+
"""
|
| 630 |
+
if X is None:
|
| 631 |
+
# Perform just the final global clustering step.
|
| 632 |
+
self._global_clustering()
|
| 633 |
+
return self
|
| 634 |
+
else:
|
| 635 |
+
return self._fit(X, partial=True)
|
| 636 |
+
|
| 637 |
+
def predict(self, X):
|
| 638 |
+
"""
|
| 639 |
+
Predict data using the ``centroids_`` of subclusters.
|
| 640 |
+
|
| 641 |
+
Avoid computation of the row norms of X.
|
| 642 |
+
|
| 643 |
+
Parameters
|
| 644 |
+
----------
|
| 645 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 646 |
+
Input data.
|
| 647 |
+
|
| 648 |
+
Returns
|
| 649 |
+
-------
|
| 650 |
+
labels : ndarray of shape(n_samples,)
|
| 651 |
+
Labelled data.
|
| 652 |
+
"""
|
| 653 |
+
check_is_fitted(self)
|
| 654 |
+
X = validate_data(self, X, accept_sparse="csr", reset=False)
|
| 655 |
+
return self._predict(X)
|
| 656 |
+
|
| 657 |
+
def _predict(self, X):
|
| 658 |
+
"""Predict data using the ``centroids_`` of subclusters."""
|
| 659 |
+
kwargs = {"Y_norm_squared": self._subcluster_norms}
|
| 660 |
+
|
| 661 |
+
with config_context(assume_finite=True):
|
| 662 |
+
argmin = pairwise_distances_argmin(
|
| 663 |
+
X, self.subcluster_centers_, metric_kwargs=kwargs
|
| 664 |
+
)
|
| 665 |
+
return self.subcluster_labels_[argmin]
|
| 666 |
+
|
| 667 |
+
def transform(self, X):
|
| 668 |
+
"""
|
| 669 |
+
Transform X into subcluster centroids dimension.
|
| 670 |
+
|
| 671 |
+
Each dimension represents the distance from the sample point to each
|
| 672 |
+
cluster centroid.
|
| 673 |
+
|
| 674 |
+
Parameters
|
| 675 |
+
----------
|
| 676 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 677 |
+
Input data.
|
| 678 |
+
|
| 679 |
+
Returns
|
| 680 |
+
-------
|
| 681 |
+
X_trans : {array-like, sparse matrix} of shape (n_samples, n_clusters)
|
| 682 |
+
Transformed data.
|
| 683 |
+
"""
|
| 684 |
+
check_is_fitted(self)
|
| 685 |
+
X = validate_data(self, X, accept_sparse="csr", reset=False)
|
| 686 |
+
with config_context(assume_finite=True):
|
| 687 |
+
return euclidean_distances(X, self.subcluster_centers_)
|
| 688 |
+
|
| 689 |
+
def _global_clustering(self, X=None):
|
| 690 |
+
"""
|
| 691 |
+
Global clustering for the subclusters obtained after fitting
|
| 692 |
+
"""
|
| 693 |
+
clusterer = self.n_clusters
|
| 694 |
+
centroids = self.subcluster_centers_
|
| 695 |
+
compute_labels = (X is not None) and self.compute_labels
|
| 696 |
+
|
| 697 |
+
# Preprocessing for the global clustering.
|
| 698 |
+
not_enough_centroids = False
|
| 699 |
+
if isinstance(clusterer, Integral):
|
| 700 |
+
clusterer = AgglomerativeClustering(n_clusters=self.n_clusters)
|
| 701 |
+
# There is no need to perform the global clustering step.
|
| 702 |
+
if len(centroids) < self.n_clusters:
|
| 703 |
+
not_enough_centroids = True
|
| 704 |
+
|
| 705 |
+
# To use in predict to avoid recalculation.
|
| 706 |
+
self._subcluster_norms = row_norms(self.subcluster_centers_, squared=True)
|
| 707 |
+
|
| 708 |
+
if clusterer is None or not_enough_centroids:
|
| 709 |
+
self.subcluster_labels_ = np.arange(len(centroids))
|
| 710 |
+
if not_enough_centroids:
|
| 711 |
+
warnings.warn(
|
| 712 |
+
"Number of subclusters found (%d) by BIRCH is less "
|
| 713 |
+
"than (%d). Decrease the threshold."
|
| 714 |
+
% (len(centroids), self.n_clusters),
|
| 715 |
+
ConvergenceWarning,
|
| 716 |
+
)
|
| 717 |
+
else:
|
| 718 |
+
# The global clustering step that clusters the subclusters of
|
| 719 |
+
# the leaves. It assumes the centroids of the subclusters as
|
| 720 |
+
# samples and finds the final centroids.
|
| 721 |
+
self.subcluster_labels_ = clusterer.fit_predict(self.subcluster_centers_)
|
| 722 |
+
|
| 723 |
+
if compute_labels:
|
| 724 |
+
self.labels_ = self._predict(X)
|
| 725 |
+
|
| 726 |
+
def __sklearn_tags__(self):
|
| 727 |
+
tags = super().__sklearn_tags__()
|
| 728 |
+
tags.transformer_tags.preserves_dtype = ["float64", "float32"]
|
| 729 |
+
tags.input_tags.sparse = True
|
| 730 |
+
return tags
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bisect_k_means.py
ADDED
|
@@ -0,0 +1,543 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bisecting K-means clustering."""
|
| 2 |
+
|
| 3 |
+
# Authors: The scikit-learn developers
|
| 4 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 5 |
+
|
| 6 |
+
import warnings
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import scipy.sparse as sp
|
| 10 |
+
|
| 11 |
+
from sklearn.base import _fit_context
|
| 12 |
+
from sklearn.cluster._k_means_common import _inertia_dense, _inertia_sparse
|
| 13 |
+
from sklearn.cluster._kmeans import (
|
| 14 |
+
_BaseKMeans,
|
| 15 |
+
_kmeans_single_elkan,
|
| 16 |
+
_kmeans_single_lloyd,
|
| 17 |
+
_labels_inertia_threadpool_limit,
|
| 18 |
+
)
|
| 19 |
+
from sklearn.utils._openmp_helpers import _openmp_effective_n_threads
|
| 20 |
+
from sklearn.utils._param_validation import Integral, Interval, StrOptions
|
| 21 |
+
from sklearn.utils.extmath import row_norms
|
| 22 |
+
from sklearn.utils.validation import (
|
| 23 |
+
_check_sample_weight,
|
| 24 |
+
check_is_fitted,
|
| 25 |
+
check_random_state,
|
| 26 |
+
validate_data,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class _BisectingTree:
|
| 31 |
+
"""Tree structure representing the hierarchical clusters of BisectingKMeans."""
|
| 32 |
+
|
| 33 |
+
def __init__(self, center, indices, score):
|
| 34 |
+
"""Create a new cluster node in the tree.
|
| 35 |
+
|
| 36 |
+
The node holds the center of this cluster and the indices of the data points
|
| 37 |
+
that belong to it.
|
| 38 |
+
"""
|
| 39 |
+
self.center = center
|
| 40 |
+
self.indices = indices
|
| 41 |
+
self.score = score
|
| 42 |
+
|
| 43 |
+
self.left = None
|
| 44 |
+
self.right = None
|
| 45 |
+
|
| 46 |
+
def split(self, labels, centers, scores):
|
| 47 |
+
"""Split the cluster node into two subclusters."""
|
| 48 |
+
self.left = _BisectingTree(
|
| 49 |
+
indices=self.indices[labels == 0], center=centers[0], score=scores[0]
|
| 50 |
+
)
|
| 51 |
+
self.right = _BisectingTree(
|
| 52 |
+
indices=self.indices[labels == 1], center=centers[1], score=scores[1]
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
# reset the indices attribute to save memory
|
| 56 |
+
self.indices = None
|
| 57 |
+
|
| 58 |
+
def get_cluster_to_bisect(self):
|
| 59 |
+
"""Return the cluster node to bisect next.
|
| 60 |
+
|
| 61 |
+
It's based on the score of the cluster, which can be either the number of
|
| 62 |
+
data points assigned to that cluster or the inertia of that cluster
|
| 63 |
+
(see `bisecting_strategy` for details).
|
| 64 |
+
"""
|
| 65 |
+
max_score = None
|
| 66 |
+
|
| 67 |
+
for cluster_leaf in self.iter_leaves():
|
| 68 |
+
if max_score is None or cluster_leaf.score > max_score:
|
| 69 |
+
max_score = cluster_leaf.score
|
| 70 |
+
best_cluster_leaf = cluster_leaf
|
| 71 |
+
|
| 72 |
+
return best_cluster_leaf
|
| 73 |
+
|
| 74 |
+
def iter_leaves(self):
|
| 75 |
+
"""Iterate over all the cluster leaves in the tree."""
|
| 76 |
+
if self.left is None:
|
| 77 |
+
yield self
|
| 78 |
+
else:
|
| 79 |
+
yield from self.left.iter_leaves()
|
| 80 |
+
yield from self.right.iter_leaves()
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class BisectingKMeans(_BaseKMeans):
|
| 84 |
+
"""Bisecting K-Means clustering.
|
| 85 |
+
|
| 86 |
+
Read more in the :ref:`User Guide <bisect_k_means>`.
|
| 87 |
+
|
| 88 |
+
.. versionadded:: 1.1
|
| 89 |
+
|
| 90 |
+
Parameters
|
| 91 |
+
----------
|
| 92 |
+
n_clusters : int, default=8
|
| 93 |
+
The number of clusters to form as well as the number of
|
| 94 |
+
centroids to generate.
|
| 95 |
+
|
| 96 |
+
init : {'k-means++', 'random'} or callable, default='random'
|
| 97 |
+
Method for initialization:
|
| 98 |
+
|
| 99 |
+
'k-means++' : selects initial cluster centers for k-mean
|
| 100 |
+
clustering in a smart way to speed up convergence. See section
|
| 101 |
+
Notes in k_init for more details.
|
| 102 |
+
|
| 103 |
+
'random': choose `n_clusters` observations (rows) at random from data
|
| 104 |
+
for the initial centroids.
|
| 105 |
+
|
| 106 |
+
If a callable is passed, it should take arguments X, n_clusters and a
|
| 107 |
+
random state and return an initialization.
|
| 108 |
+
|
| 109 |
+
n_init : int, default=1
|
| 110 |
+
Number of time the inner k-means algorithm will be run with different
|
| 111 |
+
centroid seeds in each bisection.
|
| 112 |
+
That will result producing for each bisection best output of n_init
|
| 113 |
+
consecutive runs in terms of inertia.
|
| 114 |
+
|
| 115 |
+
random_state : int, RandomState instance or None, default=None
|
| 116 |
+
Determines random number generation for centroid initialization
|
| 117 |
+
in inner K-Means. Use an int to make the randomness deterministic.
|
| 118 |
+
See :term:`Glossary <random_state>`.
|
| 119 |
+
|
| 120 |
+
max_iter : int, default=300
|
| 121 |
+
Maximum number of iterations of the inner k-means algorithm at each
|
| 122 |
+
bisection.
|
| 123 |
+
|
| 124 |
+
verbose : int, default=0
|
| 125 |
+
Verbosity mode.
|
| 126 |
+
|
| 127 |
+
tol : float, default=1e-4
|
| 128 |
+
Relative tolerance with regards to Frobenius norm of the difference
|
| 129 |
+
in the cluster centers of two consecutive iterations to declare
|
| 130 |
+
convergence. Used in inner k-means algorithm at each bisection to pick
|
| 131 |
+
best possible clusters.
|
| 132 |
+
|
| 133 |
+
copy_x : bool, default=True
|
| 134 |
+
When pre-computing distances it is more numerically accurate to center
|
| 135 |
+
the data first. If copy_x is True (default), then the original data is
|
| 136 |
+
not modified. If False, the original data is modified, and put back
|
| 137 |
+
before the function returns, but small numerical differences may be
|
| 138 |
+
introduced by subtracting and then adding the data mean. Note that if
|
| 139 |
+
the original data is not C-contiguous, a copy will be made even if
|
| 140 |
+
copy_x is False. If the original data is sparse, but not in CSR format,
|
| 141 |
+
a copy will be made even if copy_x is False.
|
| 142 |
+
|
| 143 |
+
algorithm : {"lloyd", "elkan"}, default="lloyd"
|
| 144 |
+
Inner K-means algorithm used in bisection.
|
| 145 |
+
The classical EM-style algorithm is `"lloyd"`.
|
| 146 |
+
The `"elkan"` variation can be more efficient on some datasets with
|
| 147 |
+
well-defined clusters, by using the triangle inequality. However it's
|
| 148 |
+
more memory intensive due to the allocation of an extra array of shape
|
| 149 |
+
`(n_samples, n_clusters)`.
|
| 150 |
+
|
| 151 |
+
bisecting_strategy : {"biggest_inertia", "largest_cluster"},\
|
| 152 |
+
default="biggest_inertia"
|
| 153 |
+
Defines how bisection should be performed:
|
| 154 |
+
|
| 155 |
+
- "biggest_inertia" means that BisectingKMeans will always check
|
| 156 |
+
all calculated cluster for cluster with biggest SSE
|
| 157 |
+
(Sum of squared errors) and bisect it. This approach concentrates on
|
| 158 |
+
precision, but may be costly in terms of execution time (especially for
|
| 159 |
+
larger amount of data points).
|
| 160 |
+
|
| 161 |
+
- "largest_cluster" - BisectingKMeans will always split cluster with
|
| 162 |
+
largest amount of points assigned to it from all clusters
|
| 163 |
+
previously calculated. That should work faster than picking by SSE
|
| 164 |
+
('biggest_inertia') and may produce similar results in most cases.
|
| 165 |
+
|
| 166 |
+
Attributes
|
| 167 |
+
----------
|
| 168 |
+
cluster_centers_ : ndarray of shape (n_clusters, n_features)
|
| 169 |
+
Coordinates of cluster centers. If the algorithm stops before fully
|
| 170 |
+
converging (see ``tol`` and ``max_iter``), these will not be
|
| 171 |
+
consistent with ``labels_``.
|
| 172 |
+
|
| 173 |
+
labels_ : ndarray of shape (n_samples,)
|
| 174 |
+
Labels of each point.
|
| 175 |
+
|
| 176 |
+
inertia_ : float
|
| 177 |
+
Sum of squared distances of samples to their closest cluster center,
|
| 178 |
+
weighted by the sample weights if provided.
|
| 179 |
+
|
| 180 |
+
n_features_in_ : int
|
| 181 |
+
Number of features seen during :term:`fit`.
|
| 182 |
+
|
| 183 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 184 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 185 |
+
has feature names that are all strings.
|
| 186 |
+
|
| 187 |
+
See Also
|
| 188 |
+
--------
|
| 189 |
+
KMeans : Original implementation of K-Means algorithm.
|
| 190 |
+
|
| 191 |
+
Notes
|
| 192 |
+
-----
|
| 193 |
+
It might be inefficient when n_cluster is less than 3, due to unnecessary
|
| 194 |
+
calculations for that case.
|
| 195 |
+
|
| 196 |
+
Examples
|
| 197 |
+
--------
|
| 198 |
+
>>> from sklearn.cluster import BisectingKMeans
|
| 199 |
+
>>> import numpy as np
|
| 200 |
+
>>> X = np.array([[1, 1], [10, 1], [3, 1],
|
| 201 |
+
... [10, 0], [2, 1], [10, 2],
|
| 202 |
+
... [10, 8], [10, 9], [10, 10]])
|
| 203 |
+
>>> bisect_means = BisectingKMeans(n_clusters=3, random_state=0).fit(X)
|
| 204 |
+
>>> bisect_means.labels_
|
| 205 |
+
array([0, 2, 0, 2, 0, 2, 1, 1, 1], dtype=int32)
|
| 206 |
+
>>> bisect_means.predict([[0, 0], [12, 3]])
|
| 207 |
+
array([0, 2], dtype=int32)
|
| 208 |
+
>>> bisect_means.cluster_centers_
|
| 209 |
+
array([[ 2., 1.],
|
| 210 |
+
[10., 9.],
|
| 211 |
+
[10., 1.]])
|
| 212 |
+
|
| 213 |
+
For a comparison between BisectingKMeans and K-Means refer to example
|
| 214 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_bisect_kmeans.py`.
|
| 215 |
+
"""
|
| 216 |
+
|
| 217 |
+
_parameter_constraints: dict = {
|
| 218 |
+
**_BaseKMeans._parameter_constraints,
|
| 219 |
+
"init": [StrOptions({"k-means++", "random"}), callable],
|
| 220 |
+
"n_init": [Interval(Integral, 1, None, closed="left")],
|
| 221 |
+
"copy_x": ["boolean"],
|
| 222 |
+
"algorithm": [StrOptions({"lloyd", "elkan"})],
|
| 223 |
+
"bisecting_strategy": [StrOptions({"biggest_inertia", "largest_cluster"})],
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
def __init__(
|
| 227 |
+
self,
|
| 228 |
+
n_clusters=8,
|
| 229 |
+
*,
|
| 230 |
+
init="random",
|
| 231 |
+
n_init=1,
|
| 232 |
+
random_state=None,
|
| 233 |
+
max_iter=300,
|
| 234 |
+
verbose=0,
|
| 235 |
+
tol=1e-4,
|
| 236 |
+
copy_x=True,
|
| 237 |
+
algorithm="lloyd",
|
| 238 |
+
bisecting_strategy="biggest_inertia",
|
| 239 |
+
):
|
| 240 |
+
super().__init__(
|
| 241 |
+
n_clusters=n_clusters,
|
| 242 |
+
init=init,
|
| 243 |
+
max_iter=max_iter,
|
| 244 |
+
verbose=verbose,
|
| 245 |
+
random_state=random_state,
|
| 246 |
+
tol=tol,
|
| 247 |
+
n_init=n_init,
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
self.copy_x = copy_x
|
| 251 |
+
self.algorithm = algorithm
|
| 252 |
+
self.bisecting_strategy = bisecting_strategy
|
| 253 |
+
|
| 254 |
+
def _warn_mkl_vcomp(self, n_active_threads):
|
| 255 |
+
"""Warn when vcomp and mkl are both present"""
|
| 256 |
+
warnings.warn(
|
| 257 |
+
"BisectingKMeans is known to have a memory leak on Windows "
|
| 258 |
+
"with MKL, when there are less chunks than available "
|
| 259 |
+
"threads. You can avoid it by setting the environment"
|
| 260 |
+
f" variable OMP_NUM_THREADS={n_active_threads}."
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
def _inertia_per_cluster(self, X, centers, labels, sample_weight):
|
| 264 |
+
"""Calculate the sum of squared errors (inertia) per cluster.
|
| 265 |
+
|
| 266 |
+
Parameters
|
| 267 |
+
----------
|
| 268 |
+
X : {ndarray, csr_matrix} of shape (n_samples, n_features)
|
| 269 |
+
The input samples.
|
| 270 |
+
|
| 271 |
+
centers : ndarray of shape (n_clusters=2, n_features)
|
| 272 |
+
The cluster centers.
|
| 273 |
+
|
| 274 |
+
labels : ndarray of shape (n_samples,)
|
| 275 |
+
Index of the cluster each sample belongs to.
|
| 276 |
+
|
| 277 |
+
sample_weight : ndarray of shape (n_samples,)
|
| 278 |
+
The weights for each observation in X.
|
| 279 |
+
|
| 280 |
+
Returns
|
| 281 |
+
-------
|
| 282 |
+
inertia_per_cluster : ndarray of shape (n_clusters=2,)
|
| 283 |
+
Sum of squared errors (inertia) for each cluster.
|
| 284 |
+
"""
|
| 285 |
+
n_clusters = centers.shape[0] # = 2 since centers comes from a bisection
|
| 286 |
+
_inertia = _inertia_sparse if sp.issparse(X) else _inertia_dense
|
| 287 |
+
|
| 288 |
+
inertia_per_cluster = np.empty(n_clusters)
|
| 289 |
+
for label in range(n_clusters):
|
| 290 |
+
inertia_per_cluster[label] = _inertia(
|
| 291 |
+
X, sample_weight, centers, labels, self._n_threads, single_label=label
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
return inertia_per_cluster
|
| 295 |
+
|
| 296 |
+
def _bisect(self, X, x_squared_norms, sample_weight, cluster_to_bisect):
|
| 297 |
+
"""Split a cluster into 2 subsclusters.
|
| 298 |
+
|
| 299 |
+
Parameters
|
| 300 |
+
----------
|
| 301 |
+
X : {ndarray, csr_matrix} of shape (n_samples, n_features)
|
| 302 |
+
Training instances to cluster.
|
| 303 |
+
|
| 304 |
+
x_squared_norms : ndarray of shape (n_samples,)
|
| 305 |
+
Squared euclidean norm of each data point.
|
| 306 |
+
|
| 307 |
+
sample_weight : ndarray of shape (n_samples,)
|
| 308 |
+
The weights for each observation in X.
|
| 309 |
+
|
| 310 |
+
cluster_to_bisect : _BisectingTree node object
|
| 311 |
+
The cluster node to split.
|
| 312 |
+
"""
|
| 313 |
+
X = X[cluster_to_bisect.indices]
|
| 314 |
+
x_squared_norms = x_squared_norms[cluster_to_bisect.indices]
|
| 315 |
+
sample_weight = sample_weight[cluster_to_bisect.indices]
|
| 316 |
+
|
| 317 |
+
best_inertia = None
|
| 318 |
+
|
| 319 |
+
# Split samples in X into 2 clusters.
|
| 320 |
+
# Repeating `n_init` times to obtain best clusters
|
| 321 |
+
for _ in range(self.n_init):
|
| 322 |
+
centers_init = self._init_centroids(
|
| 323 |
+
X,
|
| 324 |
+
x_squared_norms=x_squared_norms,
|
| 325 |
+
init=self.init,
|
| 326 |
+
random_state=self._random_state,
|
| 327 |
+
n_centroids=2,
|
| 328 |
+
sample_weight=sample_weight,
|
| 329 |
+
)
|
| 330 |
+
|
| 331 |
+
labels, inertia, centers, _ = self._kmeans_single(
|
| 332 |
+
X,
|
| 333 |
+
sample_weight,
|
| 334 |
+
centers_init,
|
| 335 |
+
max_iter=self.max_iter,
|
| 336 |
+
verbose=self.verbose,
|
| 337 |
+
tol=self.tol,
|
| 338 |
+
n_threads=self._n_threads,
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
# allow small tolerance on the inertia to accommodate for
|
| 342 |
+
# non-deterministic rounding errors due to parallel computation
|
| 343 |
+
if best_inertia is None or inertia < best_inertia * (1 - 1e-6):
|
| 344 |
+
best_labels = labels
|
| 345 |
+
best_centers = centers
|
| 346 |
+
best_inertia = inertia
|
| 347 |
+
|
| 348 |
+
if self.verbose:
|
| 349 |
+
print(f"New centroids from bisection: {best_centers}")
|
| 350 |
+
|
| 351 |
+
if self.bisecting_strategy == "biggest_inertia":
|
| 352 |
+
scores = self._inertia_per_cluster(
|
| 353 |
+
X, best_centers, best_labels, sample_weight
|
| 354 |
+
)
|
| 355 |
+
else: # bisecting_strategy == "largest_cluster"
|
| 356 |
+
# Using minlength to make sure that we have the counts for both labels even
|
| 357 |
+
# if all samples are labelled 0.
|
| 358 |
+
scores = np.bincount(best_labels, minlength=2)
|
| 359 |
+
|
| 360 |
+
cluster_to_bisect.split(best_labels, best_centers, scores)
|
| 361 |
+
|
| 362 |
+
@_fit_context(prefer_skip_nested_validation=True)
|
| 363 |
+
def fit(self, X, y=None, sample_weight=None):
|
| 364 |
+
"""Compute bisecting k-means clustering.
|
| 365 |
+
|
| 366 |
+
Parameters
|
| 367 |
+
----------
|
| 368 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 369 |
+
|
| 370 |
+
Training instances to cluster.
|
| 371 |
+
|
| 372 |
+
.. note:: The data will be converted to C ordering,
|
| 373 |
+
which will cause a memory copy
|
| 374 |
+
if the given data is not C-contiguous.
|
| 375 |
+
|
| 376 |
+
y : Ignored
|
| 377 |
+
Not used, present here for API consistency by convention.
|
| 378 |
+
|
| 379 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 380 |
+
The weights for each observation in X. If None, all observations
|
| 381 |
+
are assigned equal weight. `sample_weight` is not used during
|
| 382 |
+
initialization if `init` is a callable.
|
| 383 |
+
|
| 384 |
+
Returns
|
| 385 |
+
-------
|
| 386 |
+
self
|
| 387 |
+
Fitted estimator.
|
| 388 |
+
"""
|
| 389 |
+
X = validate_data(
|
| 390 |
+
self,
|
| 391 |
+
X,
|
| 392 |
+
accept_sparse="csr",
|
| 393 |
+
dtype=[np.float64, np.float32],
|
| 394 |
+
order="C",
|
| 395 |
+
copy=self.copy_x,
|
| 396 |
+
accept_large_sparse=False,
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
self._check_params_vs_input(X)
|
| 400 |
+
|
| 401 |
+
self._random_state = check_random_state(self.random_state)
|
| 402 |
+
sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
|
| 403 |
+
self._n_threads = _openmp_effective_n_threads()
|
| 404 |
+
|
| 405 |
+
if self.algorithm == "lloyd" or self.n_clusters == 1:
|
| 406 |
+
self._kmeans_single = _kmeans_single_lloyd
|
| 407 |
+
self._check_mkl_vcomp(X, X.shape[0])
|
| 408 |
+
else:
|
| 409 |
+
self._kmeans_single = _kmeans_single_elkan
|
| 410 |
+
|
| 411 |
+
# Subtract of mean of X for more accurate distance computations
|
| 412 |
+
if not sp.issparse(X):
|
| 413 |
+
self._X_mean = X.mean(axis=0)
|
| 414 |
+
X -= self._X_mean
|
| 415 |
+
|
| 416 |
+
# Initialize the hierarchical clusters tree
|
| 417 |
+
self._bisecting_tree = _BisectingTree(
|
| 418 |
+
indices=np.arange(X.shape[0]),
|
| 419 |
+
center=X.mean(axis=0),
|
| 420 |
+
score=0,
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
x_squared_norms = row_norms(X, squared=True)
|
| 424 |
+
|
| 425 |
+
for _ in range(self.n_clusters - 1):
|
| 426 |
+
# Chose cluster to bisect
|
| 427 |
+
cluster_to_bisect = self._bisecting_tree.get_cluster_to_bisect()
|
| 428 |
+
|
| 429 |
+
# Split this cluster into 2 subclusters
|
| 430 |
+
self._bisect(X, x_squared_norms, sample_weight, cluster_to_bisect)
|
| 431 |
+
|
| 432 |
+
# Aggregate final labels and centers from the bisecting tree
|
| 433 |
+
self.labels_ = np.full(X.shape[0], -1, dtype=np.int32)
|
| 434 |
+
self.cluster_centers_ = np.empty((self.n_clusters, X.shape[1]), dtype=X.dtype)
|
| 435 |
+
|
| 436 |
+
for i, cluster_node in enumerate(self._bisecting_tree.iter_leaves()):
|
| 437 |
+
self.labels_[cluster_node.indices] = i
|
| 438 |
+
self.cluster_centers_[i] = cluster_node.center
|
| 439 |
+
cluster_node.label = i # label final clusters for future prediction
|
| 440 |
+
cluster_node.indices = None # release memory
|
| 441 |
+
|
| 442 |
+
# Restore original data
|
| 443 |
+
if not sp.issparse(X):
|
| 444 |
+
X += self._X_mean
|
| 445 |
+
self.cluster_centers_ += self._X_mean
|
| 446 |
+
|
| 447 |
+
_inertia = _inertia_sparse if sp.issparse(X) else _inertia_dense
|
| 448 |
+
self.inertia_ = _inertia(
|
| 449 |
+
X, sample_weight, self.cluster_centers_, self.labels_, self._n_threads
|
| 450 |
+
)
|
| 451 |
+
|
| 452 |
+
self._n_features_out = self.cluster_centers_.shape[0]
|
| 453 |
+
|
| 454 |
+
return self
|
| 455 |
+
|
| 456 |
+
def predict(self, X):
|
| 457 |
+
"""Predict which cluster each sample in X belongs to.
|
| 458 |
+
|
| 459 |
+
Prediction is made by going down the hierarchical tree
|
| 460 |
+
in searching of closest leaf cluster.
|
| 461 |
+
|
| 462 |
+
In the vector quantization literature, `cluster_centers_` is called
|
| 463 |
+
the code book and each value returned by `predict` is the index of
|
| 464 |
+
the closest code in the code book.
|
| 465 |
+
|
| 466 |
+
Parameters
|
| 467 |
+
----------
|
| 468 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
| 469 |
+
New data to predict.
|
| 470 |
+
|
| 471 |
+
Returns
|
| 472 |
+
-------
|
| 473 |
+
labels : ndarray of shape (n_samples,)
|
| 474 |
+
Index of the cluster each sample belongs to.
|
| 475 |
+
"""
|
| 476 |
+
check_is_fitted(self)
|
| 477 |
+
|
| 478 |
+
X = self._check_test_data(X)
|
| 479 |
+
x_squared_norms = row_norms(X, squared=True)
|
| 480 |
+
|
| 481 |
+
# sample weights are unused but necessary in cython helpers
|
| 482 |
+
sample_weight = np.ones_like(x_squared_norms)
|
| 483 |
+
|
| 484 |
+
labels = self._predict_recursive(X, sample_weight, self._bisecting_tree)
|
| 485 |
+
|
| 486 |
+
return labels
|
| 487 |
+
|
| 488 |
+
def _predict_recursive(self, X, sample_weight, cluster_node):
|
| 489 |
+
"""Predict recursively by going down the hierarchical tree.
|
| 490 |
+
|
| 491 |
+
Parameters
|
| 492 |
+
----------
|
| 493 |
+
X : {ndarray, csr_matrix} of shape (n_samples, n_features)
|
| 494 |
+
The data points, currently assigned to `cluster_node`, to predict between
|
| 495 |
+
the subclusters of this node.
|
| 496 |
+
|
| 497 |
+
sample_weight : ndarray of shape (n_samples,)
|
| 498 |
+
The weights for each observation in X.
|
| 499 |
+
|
| 500 |
+
cluster_node : _BisectingTree node object
|
| 501 |
+
The cluster node of the hierarchical tree.
|
| 502 |
+
|
| 503 |
+
Returns
|
| 504 |
+
-------
|
| 505 |
+
labels : ndarray of shape (n_samples,)
|
| 506 |
+
Index of the cluster each sample belongs to.
|
| 507 |
+
"""
|
| 508 |
+
if cluster_node.left is None:
|
| 509 |
+
# This cluster has no subcluster. Labels are just the label of the cluster.
|
| 510 |
+
return np.full(X.shape[0], cluster_node.label, dtype=np.int32)
|
| 511 |
+
|
| 512 |
+
# Determine if data points belong to the left or right subcluster
|
| 513 |
+
centers = np.vstack((cluster_node.left.center, cluster_node.right.center))
|
| 514 |
+
if hasattr(self, "_X_mean"):
|
| 515 |
+
centers += self._X_mean
|
| 516 |
+
|
| 517 |
+
cluster_labels = _labels_inertia_threadpool_limit(
|
| 518 |
+
X,
|
| 519 |
+
sample_weight,
|
| 520 |
+
centers,
|
| 521 |
+
self._n_threads,
|
| 522 |
+
return_inertia=False,
|
| 523 |
+
)
|
| 524 |
+
mask = cluster_labels == 0
|
| 525 |
+
|
| 526 |
+
# Compute the labels for each subset of the data points.
|
| 527 |
+
labels = np.full(X.shape[0], -1, dtype=np.int32)
|
| 528 |
+
|
| 529 |
+
labels[mask] = self._predict_recursive(
|
| 530 |
+
X[mask], sample_weight[mask], cluster_node.left
|
| 531 |
+
)
|
| 532 |
+
|
| 533 |
+
labels[~mask] = self._predict_recursive(
|
| 534 |
+
X[~mask], sample_weight[~mask], cluster_node.right
|
| 535 |
+
)
|
| 536 |
+
|
| 537 |
+
return labels
|
| 538 |
+
|
| 539 |
+
def __sklearn_tags__(self):
|
| 540 |
+
tags = super().__sklearn_tags__()
|
| 541 |
+
tags.input_tags.sparse = True
|
| 542 |
+
tags.transformer_tags.preserves_dtype = ["float64", "float32"]
|
| 543 |
+
return tags
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan.py
ADDED
|
@@ -0,0 +1,512 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
DBSCAN: Density-Based Spatial Clustering of Applications with Noise
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
# Authors: The scikit-learn developers
|
| 6 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 7 |
+
|
| 8 |
+
import warnings
|
| 9 |
+
from numbers import Integral, Real
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
from scipy import sparse
|
| 13 |
+
|
| 14 |
+
from sklearn.base import BaseEstimator, ClusterMixin, _fit_context
|
| 15 |
+
from sklearn.cluster._dbscan_inner import dbscan_inner
|
| 16 |
+
from sklearn.metrics.pairwise import _VALID_METRICS
|
| 17 |
+
from sklearn.neighbors import NearestNeighbors
|
| 18 |
+
from sklearn.utils._param_validation import Interval, StrOptions, validate_params
|
| 19 |
+
from sklearn.utils.validation import _check_sample_weight, validate_data
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@validate_params(
|
| 23 |
+
{
|
| 24 |
+
"X": ["array-like", "sparse matrix"],
|
| 25 |
+
"sample_weight": ["array-like", None],
|
| 26 |
+
},
|
| 27 |
+
prefer_skip_nested_validation=False,
|
| 28 |
+
)
|
| 29 |
+
def dbscan(
|
| 30 |
+
X,
|
| 31 |
+
eps=0.5,
|
| 32 |
+
*,
|
| 33 |
+
min_samples=5,
|
| 34 |
+
metric="minkowski",
|
| 35 |
+
metric_params=None,
|
| 36 |
+
algorithm="auto",
|
| 37 |
+
leaf_size=30,
|
| 38 |
+
p=2,
|
| 39 |
+
sample_weight=None,
|
| 40 |
+
n_jobs=None,
|
| 41 |
+
):
|
| 42 |
+
"""Perform DBSCAN clustering from vector array or distance matrix.
|
| 43 |
+
|
| 44 |
+
This function is a wrapper around :class:`~cluster.DBSCAN`, suitable for
|
| 45 |
+
quick, standalone clustering tasks. For estimator-based workflows, where
|
| 46 |
+
estimator attributes or pipeline integration is required, prefer
|
| 47 |
+
:class:`~cluster.DBSCAN`.
|
| 48 |
+
|
| 49 |
+
DBSCAN (Density-Based Spatial Clustering of Applications with Noise) is a
|
| 50 |
+
density-based clustering algorithm that groups together points that are
|
| 51 |
+
closely packed while marking points in low-density regions as outliers.
|
| 52 |
+
|
| 53 |
+
Read more in the :ref:`User Guide <dbscan>`.
|
| 54 |
+
|
| 55 |
+
Parameters
|
| 56 |
+
----------
|
| 57 |
+
X : {array-like, scipy sparse matrix} of shape (n_samples, n_features) or \
|
| 58 |
+
(n_samples, n_samples)
|
| 59 |
+
A feature array, or array of distances between samples if
|
| 60 |
+
``metric='precomputed'``. When using precomputed distances, X must
|
| 61 |
+
be a square symmetric matrix.
|
| 62 |
+
|
| 63 |
+
eps : float, default=0.5
|
| 64 |
+
The maximum distance between two samples for one to be considered
|
| 65 |
+
as in the neighborhood of the other. This is not a maximum bound
|
| 66 |
+
on the distances of points within a cluster. This is the most
|
| 67 |
+
important DBSCAN parameter to choose appropriately for your data set
|
| 68 |
+
and distance function. Smaller values result in more clusters,
|
| 69 |
+
while larger values result in fewer, larger clusters.
|
| 70 |
+
|
| 71 |
+
min_samples : int, default=5
|
| 72 |
+
The number of samples (or total weight) in a neighborhood for a point
|
| 73 |
+
to be considered as a core point. This includes the point itself.
|
| 74 |
+
Higher values yield fewer, denser clusters, while lower values yield
|
| 75 |
+
more, sparser clusters.
|
| 76 |
+
|
| 77 |
+
metric : str or callable, default='minkowski'
|
| 78 |
+
The metric to use when calculating distance between instances in a
|
| 79 |
+
feature array. If metric is a string or callable, it must be one of
|
| 80 |
+
the options allowed by :func:`sklearn.metrics.pairwise_distances` for
|
| 81 |
+
its metric parameter.
|
| 82 |
+
If metric is "precomputed", X is assumed to be a distance matrix and
|
| 83 |
+
must be square during fit.
|
| 84 |
+
X may be a :term:`sparse graph <sparse graph>`,
|
| 85 |
+
in which case only "nonzero" elements may be considered neighbors.
|
| 86 |
+
|
| 87 |
+
metric_params : dict, default=None
|
| 88 |
+
Additional keyword arguments for the metric function.
|
| 89 |
+
|
| 90 |
+
.. versionadded:: 0.19
|
| 91 |
+
|
| 92 |
+
algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
|
| 93 |
+
The algorithm to be used by the NearestNeighbors module
|
| 94 |
+
to compute pointwise distances and find nearest neighbors.
|
| 95 |
+
'auto' will attempt to decide the most appropriate algorithm
|
| 96 |
+
based on the values passed to :meth:`fit` method.
|
| 97 |
+
See :class:`~sklearn.neighbors.NearestNeighbors` documentation for
|
| 98 |
+
details.
|
| 99 |
+
|
| 100 |
+
leaf_size : int, default=30
|
| 101 |
+
Leaf size passed to BallTree or cKDTree. This can affect the speed
|
| 102 |
+
of the construction and query, as well as the memory required
|
| 103 |
+
to store the tree. The optimal value depends
|
| 104 |
+
on the nature of the problem. Generally, smaller leaf sizes
|
| 105 |
+
lead to faster queries but slower construction.
|
| 106 |
+
|
| 107 |
+
p : float, default=2
|
| 108 |
+
Power parameter for the Minkowski metric. When p = 1, this is equivalent
|
| 109 |
+
to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2.
|
| 110 |
+
For arbitrary p, minkowski_distance (l_p) is used. This parameter is expected
|
| 111 |
+
to be positive.
|
| 112 |
+
|
| 113 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 114 |
+
Weight of each sample, such that a sample with a weight of at least
|
| 115 |
+
``min_samples`` is by itself a core sample; a sample with negative
|
| 116 |
+
weight may inhibit its eps-neighbor from being core.
|
| 117 |
+
Note that weights are absolute, and default to 1.
|
| 118 |
+
|
| 119 |
+
n_jobs : int, default=None
|
| 120 |
+
The number of parallel jobs to run for neighbors search. ``None`` means
|
| 121 |
+
1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means
|
| 122 |
+
using all processors. See :term:`Glossary <n_jobs>` for more details.
|
| 123 |
+
If precomputed distances are used, parallel execution is not available
|
| 124 |
+
and thus n_jobs will have no effect.
|
| 125 |
+
|
| 126 |
+
Returns
|
| 127 |
+
-------
|
| 128 |
+
core_samples : ndarray of shape (n_core_samples,)
|
| 129 |
+
Indices of core samples.
|
| 130 |
+
|
| 131 |
+
labels : ndarray of shape (n_samples,)
|
| 132 |
+
Cluster labels for each point. Noisy samples are given the label -1.
|
| 133 |
+
Non-negative integers indicate cluster membership.
|
| 134 |
+
|
| 135 |
+
See Also
|
| 136 |
+
--------
|
| 137 |
+
DBSCAN : An estimator interface for this clustering algorithm.
|
| 138 |
+
OPTICS : A similar estimator interface clustering at multiple values of
|
| 139 |
+
eps. Our implementation is optimized for memory usage.
|
| 140 |
+
|
| 141 |
+
Notes
|
| 142 |
+
-----
|
| 143 |
+
For an example, see :ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py`.
|
| 144 |
+
|
| 145 |
+
This implementation bulk-computes all neighborhood queries, which increases
|
| 146 |
+
the memory complexity to O(n.d) where d is the average number of neighbors,
|
| 147 |
+
while original DBSCAN had memory complexity O(n). It may attract a higher
|
| 148 |
+
memory complexity when querying these nearest neighborhoods, depending
|
| 149 |
+
on the ``algorithm``.
|
| 150 |
+
|
| 151 |
+
One way to avoid the query complexity is to pre-compute sparse
|
| 152 |
+
neighborhoods in chunks using
|
| 153 |
+
:func:`NearestNeighbors.radius_neighbors_graph
|
| 154 |
+
<sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>` with
|
| 155 |
+
``mode='distance'``, then using ``metric='precomputed'`` here.
|
| 156 |
+
|
| 157 |
+
Another way to reduce memory and computation time is to remove
|
| 158 |
+
(near-)duplicate points and use ``sample_weight`` instead.
|
| 159 |
+
|
| 160 |
+
:class:`~sklearn.cluster.OPTICS` provides a similar clustering with lower
|
| 161 |
+
memory usage.
|
| 162 |
+
|
| 163 |
+
References
|
| 164 |
+
----------
|
| 165 |
+
Ester, M., H. P. Kriegel, J. Sander, and X. Xu, `"A Density-Based
|
| 166 |
+
Algorithm for Discovering Clusters in Large Spatial Databases with Noise"
|
| 167 |
+
<https://www.dbs.ifi.lmu.de/Publikationen/Papers/KDD-96.final.frame.pdf>`_.
|
| 168 |
+
In: Proceedings of the 2nd International Conference on Knowledge Discovery
|
| 169 |
+
and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
|
| 170 |
+
|
| 171 |
+
Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017).
|
| 172 |
+
:doi:`"DBSCAN revisited, revisited: why and how you should (still) use DBSCAN."
|
| 173 |
+
<10.1145/3068335>`
|
| 174 |
+
ACM Transactions on Database Systems (TODS), 42(3), 19.
|
| 175 |
+
|
| 176 |
+
Examples
|
| 177 |
+
--------
|
| 178 |
+
>>> from sklearn.cluster import dbscan
|
| 179 |
+
>>> X = [[1, 2], [2, 2], [2, 3], [8, 7], [8, 8], [25, 80]]
|
| 180 |
+
>>> core_samples, labels = dbscan(X, eps=3, min_samples=2)
|
| 181 |
+
>>> core_samples
|
| 182 |
+
array([0, 1, 2, 3, 4])
|
| 183 |
+
>>> labels
|
| 184 |
+
array([ 0, 0, 0, 1, 1, -1])
|
| 185 |
+
"""
|
| 186 |
+
|
| 187 |
+
est = DBSCAN(
|
| 188 |
+
eps=eps,
|
| 189 |
+
min_samples=min_samples,
|
| 190 |
+
metric=metric,
|
| 191 |
+
metric_params=metric_params,
|
| 192 |
+
algorithm=algorithm,
|
| 193 |
+
leaf_size=leaf_size,
|
| 194 |
+
p=p,
|
| 195 |
+
n_jobs=n_jobs,
|
| 196 |
+
)
|
| 197 |
+
est.fit(X, sample_weight=sample_weight)
|
| 198 |
+
return est.core_sample_indices_, est.labels_
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
class DBSCAN(ClusterMixin, BaseEstimator):
|
| 202 |
+
"""Perform DBSCAN clustering from vector array or distance matrix.
|
| 203 |
+
|
| 204 |
+
DBSCAN - Density-Based Spatial Clustering of Applications with Noise.
|
| 205 |
+
Finds core samples of high density and expands clusters from them.
|
| 206 |
+
This algorithm is particularly good for data which contains clusters of
|
| 207 |
+
similar density and can find clusters of arbitrary shape.
|
| 208 |
+
|
| 209 |
+
Unlike K-means, DBSCAN does not require specifying the number of clusters
|
| 210 |
+
in advance and can identify outliers as noise points.
|
| 211 |
+
|
| 212 |
+
This implementation has a worst case memory complexity of :math:`O({n}^2)`,
|
| 213 |
+
which can occur when the `eps` param is large and `min_samples` is low,
|
| 214 |
+
while the original DBSCAN only uses linear memory.
|
| 215 |
+
For further details, see the Notes below.
|
| 216 |
+
|
| 217 |
+
Read more in the :ref:`User Guide <dbscan>`.
|
| 218 |
+
|
| 219 |
+
Parameters
|
| 220 |
+
----------
|
| 221 |
+
eps : float, default=0.5
|
| 222 |
+
The maximum distance between two samples for one to be considered
|
| 223 |
+
as in the neighborhood of the other. This is not a maximum bound
|
| 224 |
+
on the distances of points within a cluster. This is the most
|
| 225 |
+
important DBSCAN parameter to choose appropriately for your data set
|
| 226 |
+
and distance function. Smaller values generally lead to more clusters.
|
| 227 |
+
|
| 228 |
+
min_samples : int, default=5
|
| 229 |
+
The number of samples (or total weight) in a neighborhood for a point to
|
| 230 |
+
be considered as a core point. This includes the point itself. If
|
| 231 |
+
`min_samples` is set to a higher value, DBSCAN will find denser clusters,
|
| 232 |
+
whereas if it is set to a lower value, the found clusters will be more
|
| 233 |
+
sparse.
|
| 234 |
+
|
| 235 |
+
metric : str, or callable, default='euclidean'
|
| 236 |
+
The metric to use when calculating distance between instances in a
|
| 237 |
+
feature array. If metric is a string or callable, it must be one of
|
| 238 |
+
the options allowed by :func:`sklearn.metrics.pairwise_distances` for
|
| 239 |
+
its metric parameter.
|
| 240 |
+
If metric is "precomputed", X is assumed to be a distance matrix and
|
| 241 |
+
must be square. X may be a :term:`sparse graph`, in which
|
| 242 |
+
case only "nonzero" elements may be considered neighbors for DBSCAN.
|
| 243 |
+
|
| 244 |
+
.. versionadded:: 0.17
|
| 245 |
+
metric *precomputed* to accept precomputed sparse matrix.
|
| 246 |
+
|
| 247 |
+
metric_params : dict, default=None
|
| 248 |
+
Additional keyword arguments for the metric function.
|
| 249 |
+
|
| 250 |
+
.. versionadded:: 0.19
|
| 251 |
+
|
| 252 |
+
algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
|
| 253 |
+
The algorithm to be used by the NearestNeighbors module
|
| 254 |
+
to compute pointwise distances and find nearest neighbors.
|
| 255 |
+
'auto' will attempt to decide the most appropriate algorithm
|
| 256 |
+
based on the values passed to :meth:`fit` method.
|
| 257 |
+
See :class:`~sklearn.neighbors.NearestNeighbors` documentation for
|
| 258 |
+
details.
|
| 259 |
+
|
| 260 |
+
leaf_size : int, default=30
|
| 261 |
+
Leaf size passed to BallTree or cKDTree. This can affect the speed
|
| 262 |
+
of the construction and query, as well as the memory required
|
| 263 |
+
to store the tree. The optimal value depends
|
| 264 |
+
on the nature of the problem.
|
| 265 |
+
|
| 266 |
+
p : float, default=None
|
| 267 |
+
The power of the Minkowski metric to be used to calculate distance
|
| 268 |
+
between points. If None, then ``p=2`` (equivalent to the Euclidean
|
| 269 |
+
distance). When p=1, this is equivalent to Manhattan distance.
|
| 270 |
+
|
| 271 |
+
n_jobs : int, default=None
|
| 272 |
+
The number of parallel jobs to run.
|
| 273 |
+
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
|
| 274 |
+
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
|
| 275 |
+
for more details.
|
| 276 |
+
|
| 277 |
+
Attributes
|
| 278 |
+
----------
|
| 279 |
+
core_sample_indices_ : ndarray of shape (n_core_samples,)
|
| 280 |
+
Indices of core samples.
|
| 281 |
+
|
| 282 |
+
components_ : ndarray of shape (n_core_samples, n_features)
|
| 283 |
+
Copy of each core sample found by training.
|
| 284 |
+
|
| 285 |
+
labels_ : ndarray of shape (n_samples,)
|
| 286 |
+
Cluster labels for each point in the dataset given to fit().
|
| 287 |
+
Noisy samples are given the label -1. Non-negative integers
|
| 288 |
+
indicate cluster membership.
|
| 289 |
+
|
| 290 |
+
n_features_in_ : int
|
| 291 |
+
Number of features seen during :term:`fit`.
|
| 292 |
+
|
| 293 |
+
.. versionadded:: 0.24
|
| 294 |
+
|
| 295 |
+
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
| 296 |
+
Names of features seen during :term:`fit`. Defined only when `X`
|
| 297 |
+
has feature names that are all strings.
|
| 298 |
+
|
| 299 |
+
.. versionadded:: 1.0
|
| 300 |
+
|
| 301 |
+
See Also
|
| 302 |
+
--------
|
| 303 |
+
OPTICS : A similar clustering at multiple values of eps. Our implementation
|
| 304 |
+
is optimized for memory usage.
|
| 305 |
+
|
| 306 |
+
Notes
|
| 307 |
+
-----
|
| 308 |
+
This implementation bulk-computes all neighborhood queries, which increases
|
| 309 |
+
the memory complexity to O(n.d) where d is the average number of neighbors,
|
| 310 |
+
while original DBSCAN had memory complexity O(n). It may attract a higher
|
| 311 |
+
memory complexity when querying these nearest neighborhoods, depending
|
| 312 |
+
on the ``algorithm``.
|
| 313 |
+
|
| 314 |
+
One way to avoid the query complexity is to pre-compute sparse
|
| 315 |
+
neighborhoods in chunks using
|
| 316 |
+
:func:`NearestNeighbors.radius_neighbors_graph
|
| 317 |
+
<sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>` with
|
| 318 |
+
``mode='distance'``, then using ``metric='precomputed'`` here.
|
| 319 |
+
|
| 320 |
+
Another way to reduce memory and computation time is to remove
|
| 321 |
+
(near-)duplicate points and use ``sample_weight`` instead.
|
| 322 |
+
|
| 323 |
+
:class:`~sklearn.cluster.OPTICS` provides a similar clustering with lower memory
|
| 324 |
+
usage.
|
| 325 |
+
|
| 326 |
+
References
|
| 327 |
+
----------
|
| 328 |
+
Ester, M., H. P. Kriegel, J. Sander, and X. Xu, `"A Density-Based
|
| 329 |
+
Algorithm for Discovering Clusters in Large Spatial Databases with Noise"
|
| 330 |
+
<https://www.dbs.ifi.lmu.de/Publikationen/Papers/KDD-96.final.frame.pdf>`_.
|
| 331 |
+
In: Proceedings of the 2nd International Conference on Knowledge Discovery
|
| 332 |
+
and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
|
| 333 |
+
|
| 334 |
+
Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017).
|
| 335 |
+
:doi:`"DBSCAN revisited, revisited: why and how you should (still) use DBSCAN."
|
| 336 |
+
<10.1145/3068335>`
|
| 337 |
+
ACM Transactions on Database Systems (TODS), 42(3), 19.
|
| 338 |
+
|
| 339 |
+
Examples
|
| 340 |
+
--------
|
| 341 |
+
>>> from sklearn.cluster import DBSCAN
|
| 342 |
+
>>> import numpy as np
|
| 343 |
+
>>> X = np.array([[1, 2], [2, 2], [2, 3],
|
| 344 |
+
... [8, 7], [8, 8], [25, 80]])
|
| 345 |
+
>>> clustering = DBSCAN(eps=3, min_samples=2).fit(X)
|
| 346 |
+
>>> clustering.labels_
|
| 347 |
+
array([ 0, 0, 0, 1, 1, -1])
|
| 348 |
+
>>> clustering
|
| 349 |
+
DBSCAN(eps=3, min_samples=2)
|
| 350 |
+
|
| 351 |
+
For an example, see
|
| 352 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py`.
|
| 353 |
+
|
| 354 |
+
For a comparison of DBSCAN with other clustering algorithms, see
|
| 355 |
+
:ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
|
| 356 |
+
"""
|
| 357 |
+
|
| 358 |
+
_parameter_constraints: dict = {
|
| 359 |
+
"eps": [Interval(Real, 0.0, None, closed="neither")],
|
| 360 |
+
"min_samples": [Interval(Integral, 1, None, closed="left")],
|
| 361 |
+
"metric": [
|
| 362 |
+
StrOptions(set(_VALID_METRICS) | {"precomputed"}),
|
| 363 |
+
callable,
|
| 364 |
+
],
|
| 365 |
+
"metric_params": [dict, None],
|
| 366 |
+
"algorithm": [StrOptions({"auto", "ball_tree", "kd_tree", "brute"})],
|
| 367 |
+
"leaf_size": [Interval(Integral, 1, None, closed="left")],
|
| 368 |
+
"p": [Interval(Real, 0.0, None, closed="left"), None],
|
| 369 |
+
"n_jobs": [Integral, None],
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
def __init__(
|
| 373 |
+
self,
|
| 374 |
+
eps=0.5,
|
| 375 |
+
*,
|
| 376 |
+
min_samples=5,
|
| 377 |
+
metric="euclidean",
|
| 378 |
+
metric_params=None,
|
| 379 |
+
algorithm="auto",
|
| 380 |
+
leaf_size=30,
|
| 381 |
+
p=None,
|
| 382 |
+
n_jobs=None,
|
| 383 |
+
):
|
| 384 |
+
self.eps = eps
|
| 385 |
+
self.min_samples = min_samples
|
| 386 |
+
self.metric = metric
|
| 387 |
+
self.metric_params = metric_params
|
| 388 |
+
self.algorithm = algorithm
|
| 389 |
+
self.leaf_size = leaf_size
|
| 390 |
+
self.p = p
|
| 391 |
+
self.n_jobs = n_jobs
|
| 392 |
+
|
| 393 |
+
@_fit_context(
|
| 394 |
+
# DBSCAN.metric is not validated yet
|
| 395 |
+
prefer_skip_nested_validation=False
|
| 396 |
+
)
|
| 397 |
+
def fit(self, X, y=None, sample_weight=None):
|
| 398 |
+
"""Perform DBSCAN clustering from features, or distance matrix.
|
| 399 |
+
|
| 400 |
+
Parameters
|
| 401 |
+
----------
|
| 402 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
|
| 403 |
+
(n_samples, n_samples)
|
| 404 |
+
Training instances to cluster, or distances between instances if
|
| 405 |
+
``metric='precomputed'``. If a sparse matrix is provided, it will
|
| 406 |
+
be converted into a sparse ``csr_matrix``.
|
| 407 |
+
|
| 408 |
+
y : Ignored
|
| 409 |
+
Not used, present here for API consistency by convention.
|
| 410 |
+
|
| 411 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 412 |
+
Weight of each sample, such that a sample with a weight of at least
|
| 413 |
+
``min_samples`` is by itself a core sample; a sample with a
|
| 414 |
+
negative weight may inhibit its eps-neighbor from being core.
|
| 415 |
+
Note that weights are absolute, and default to 1.
|
| 416 |
+
|
| 417 |
+
Returns
|
| 418 |
+
-------
|
| 419 |
+
self : object
|
| 420 |
+
Returns a fitted instance of self.
|
| 421 |
+
"""
|
| 422 |
+
X = validate_data(self, X, accept_sparse="csr")
|
| 423 |
+
|
| 424 |
+
if sample_weight is not None:
|
| 425 |
+
sample_weight = _check_sample_weight(sample_weight, X)
|
| 426 |
+
|
| 427 |
+
# Calculate neighborhood for all samples. This leaves the original
|
| 428 |
+
# point in, which needs to be considered later (i.e. point i is in the
|
| 429 |
+
# neighborhood of point i. While True, its useless information)
|
| 430 |
+
if self.metric == "precomputed" and sparse.issparse(X):
|
| 431 |
+
# set the diagonal to explicit values, as a point is its own
|
| 432 |
+
# neighbor
|
| 433 |
+
X = X.copy() # copy to avoid in-place modification
|
| 434 |
+
with warnings.catch_warnings():
|
| 435 |
+
warnings.simplefilter("ignore", sparse.SparseEfficiencyWarning)
|
| 436 |
+
X.setdiag(X.diagonal())
|
| 437 |
+
|
| 438 |
+
neighbors_model = NearestNeighbors(
|
| 439 |
+
radius=self.eps,
|
| 440 |
+
algorithm=self.algorithm,
|
| 441 |
+
leaf_size=self.leaf_size,
|
| 442 |
+
metric=self.metric,
|
| 443 |
+
metric_params=self.metric_params,
|
| 444 |
+
p=self.p,
|
| 445 |
+
n_jobs=self.n_jobs,
|
| 446 |
+
)
|
| 447 |
+
neighbors_model.fit(X)
|
| 448 |
+
# This has worst case O(n^2) memory complexity
|
| 449 |
+
neighborhoods = neighbors_model.radius_neighbors(X, return_distance=False)
|
| 450 |
+
|
| 451 |
+
if sample_weight is None:
|
| 452 |
+
n_neighbors = np.array([len(neighbors) for neighbors in neighborhoods])
|
| 453 |
+
else:
|
| 454 |
+
n_neighbors = np.array(
|
| 455 |
+
[np.sum(sample_weight[neighbors]) for neighbors in neighborhoods]
|
| 456 |
+
)
|
| 457 |
+
|
| 458 |
+
# Initially, all samples are noise.
|
| 459 |
+
labels = np.full(X.shape[0], -1, dtype=np.intp)
|
| 460 |
+
|
| 461 |
+
# A list of all core samples found.
|
| 462 |
+
core_samples = np.asarray(n_neighbors >= self.min_samples, dtype=np.uint8)
|
| 463 |
+
dbscan_inner(core_samples, neighborhoods, labels)
|
| 464 |
+
|
| 465 |
+
self.core_sample_indices_ = np.where(core_samples)[0]
|
| 466 |
+
self.labels_ = labels
|
| 467 |
+
|
| 468 |
+
if len(self.core_sample_indices_):
|
| 469 |
+
# fix for scipy sparse indexing issue
|
| 470 |
+
self.components_ = X[self.core_sample_indices_].copy()
|
| 471 |
+
else:
|
| 472 |
+
# no core samples
|
| 473 |
+
self.components_ = np.empty((0, X.shape[1]))
|
| 474 |
+
return self
|
| 475 |
+
|
| 476 |
+
def fit_predict(self, X, y=None, sample_weight=None):
|
| 477 |
+
"""Compute clusters from a data or distance matrix and predict labels.
|
| 478 |
+
|
| 479 |
+
This method fits the model and returns the cluster labels in a single step.
|
| 480 |
+
It is equivalent to calling fit(X).labels_.
|
| 481 |
+
|
| 482 |
+
Parameters
|
| 483 |
+
----------
|
| 484 |
+
X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
|
| 485 |
+
(n_samples, n_samples)
|
| 486 |
+
Training instances to cluster, or distances between instances if
|
| 487 |
+
``metric='precomputed'``. If a sparse matrix is provided, it will
|
| 488 |
+
be converted into a sparse ``csr_matrix``.
|
| 489 |
+
|
| 490 |
+
y : Ignored
|
| 491 |
+
Not used, present here for API consistency by convention.
|
| 492 |
+
|
| 493 |
+
sample_weight : array-like of shape (n_samples,), default=None
|
| 494 |
+
Weight of each sample, such that a sample with a weight of at least
|
| 495 |
+
``min_samples`` is by itself a core sample; a sample with a
|
| 496 |
+
negative weight may inhibit its eps-neighbor from being core.
|
| 497 |
+
Note that weights are absolute, and default to 1.
|
| 498 |
+
|
| 499 |
+
Returns
|
| 500 |
+
-------
|
| 501 |
+
labels : ndarray of shape (n_samples,)
|
| 502 |
+
Cluster labels. Noisy samples are given the label -1.
|
| 503 |
+
Non-negative integers indicate cluster membership.
|
| 504 |
+
"""
|
| 505 |
+
self.fit(X, sample_weight=sample_weight)
|
| 506 |
+
return self.labels_
|
| 507 |
+
|
| 508 |
+
def __sklearn_tags__(self):
|
| 509 |
+
tags = super().__sklearn_tags__()
|
| 510 |
+
tags.input_tags.pairwise = self.metric == "precomputed"
|
| 511 |
+
tags.input_tags.sparse = True
|
| 512 |
+
return tags
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan_inner.cp312-win_amd64.lib
ADDED
|
Binary file (2.12 kB). View file
|
|
|