techprotrade commited on
Commit
c91c831
·
verified ·
1 Parent(s): 4fb4a7d

Add loan-advisor-agent

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +13 -0
  2. loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/msvcp140.dll +3 -0
  3. loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/vcomp140.dll +3 -0
  4. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__init__.py +54 -0
  5. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__pycache__/__init__.cpython-312.pyc +0 -0
  6. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.lib +0 -0
  7. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.pyd +0 -0
  8. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.pyx +2 -0
  9. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/meson.build +6 -0
  10. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__init__.py +150 -0
  11. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/__init__.cpython-312.pyc +0 -0
  12. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_config.cpython-312.pyc +0 -0
  13. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_distributor_init.cpython-312.pyc +0 -0
  14. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/base.cpython-312.pyc +0 -0
  15. loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/exceptions.cpython-312.pyc +0 -0
  16. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/__init__.py +0 -0
  17. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/tempita.py +62 -0
  18. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/version.py +16 -0
  19. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_config.py +407 -0
  20. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.lib +0 -0
  21. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.pyd +3 -0
  22. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_distributor_init.py +22 -0
  23. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.lib +0 -0
  24. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.pyd +0 -0
  25. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.pyx +116 -0
  26. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__init__.py +33 -0
  27. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/__init__.cpython-312.pyc +0 -0
  28. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/link.cpython-312.pyc +0 -0
  29. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/loss.cpython-312.pyc +0 -0
  30. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.lib +0 -0
  31. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.pyd +3 -0
  32. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pxd +101 -0
  33. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pyx.tp +1505 -0
  34. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/link.py +282 -0
  35. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/loss.py +1216 -0
  36. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/meson.build +23 -0
  37. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/__init__.py +0 -0
  38. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_link.py +111 -0
  39. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_loss.py +1358 -0
  40. loan-advisor-agent/.venv/Lib/site-packages/sklearn/_min_dependencies.py +74 -0
  41. loan-advisor-agent/.venv/Lib/site-packages/sklearn/base.py +1340 -0
  42. loan-advisor-agent/.venv/Lib/site-packages/sklearn/calibration.py +1709 -0
  43. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/__init__.py +64 -0
  44. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_affinity_propagation.py +607 -0
  45. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_agglomerative.py +1335 -0
  46. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bicluster.py +622 -0
  47. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_birch.py +730 -0
  48. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bisect_k_means.py +543 -0
  49. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan.py +512 -0
  50. loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan_inner.cp312-win_amd64.lib +0 -0
.gitattributes CHANGED
@@ -663,3 +663,16 @@ loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_stats.cp312-win_amd64.py
663
  loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_stats_pythran.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
664
  loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_unuran/unuran_wrapper.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
665
  loan-advisor-agent/.venv/Lib/site-packages/scipy.libs/libscipy_openblas-64eda39e79589aedb16f58e5547eb599.dll filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
663
  loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_stats_pythran.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
664
  loan-advisor-agent/.venv/Lib/site-packages/scipy/stats/_unuran/unuran_wrapper.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
665
  loan-advisor-agent/.venv/Lib/site-packages/scipy.libs/libscipy_openblas-64eda39e79589aedb16f58e5547eb599.dll filter=lfs diff=lfs merge=lfs -text
666
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/msvcp140.dll filter=lfs diff=lfs merge=lfs -text
667
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/vcomp140.dll filter=lfs diff=lfs merge=lfs -text
668
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
669
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
670
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_hdbscan/_reachability.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
671
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_hdbscan/_tree.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
672
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_hierarchical_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
673
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_common.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
674
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_elkan.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
675
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_lloyd.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
676
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/_svmlight_format_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
677
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/china.jpg filter=lfs diff=lfs merge=lfs -text
678
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/flower.jpg filter=lfs diff=lfs merge=lfs -text
loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/msvcp140.dll ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f885b509a685d2bbfa652fed26b5fb31d88fbdab0a978c641d1c7b8aa460aa9
3
+ size 557728
loan-advisor-agent/.venv/Lib/site-packages/sklearn/.libs/vcomp140.dll ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55aba23cdcd6484fbb06f4155b8ca75adfce7a881f10afd0c49457165e677164
3
+ size 193152
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__init__.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Module to give helpful messages to the user that did not
2
+ compile scikit-learn properly.
3
+ """
4
+
5
+ # Authors: The scikit-learn developers
6
+ # SPDX-License-Identifier: BSD-3-Clause
7
+
8
+ import os
9
+
10
+ INPLACE_MSG = """
11
+ It appears that you are importing a local scikit-learn source tree. For
12
+ this, you need to have an inplace install. Maybe you are in the source
13
+ directory and you need to try from another location."""
14
+
15
+ STANDARD_MSG = """
16
+ If you have used an installer, please check that it is suited for your
17
+ Python version, your operating system and your platform."""
18
+
19
+
20
+ def raise_build_error(e):
21
+ # Raise a comprehensible error and list the contents of the
22
+ # directory to help debugging on the mailing list.
23
+ local_dir = os.path.split(__file__)[0]
24
+ msg = STANDARD_MSG
25
+ if local_dir == "sklearn/__check_build":
26
+ # Picking up the local install: this will work only if the
27
+ # install is an 'inplace build'
28
+ msg = INPLACE_MSG
29
+ dir_content = list()
30
+ for i, filename in enumerate(os.listdir(local_dir)):
31
+ if (i + 1) % 3:
32
+ dir_content.append(filename.ljust(26))
33
+ else:
34
+ dir_content.append(filename + "\n")
35
+ raise ImportError(
36
+ """%s
37
+ ___________________________________________________________________________
38
+ Contents of %s:
39
+ %s
40
+ ___________________________________________________________________________
41
+ It seems that scikit-learn has not been built correctly.
42
+
43
+ If you have installed scikit-learn from source, please do not forget
44
+ to build the package before using it. For detailed instructions, see:
45
+ https://scikit-learn.org/dev/developers/advanced_installation.html#building-from-source
46
+ %s"""
47
+ % (e, local_dir, "".join(dir_content).strip(), msg)
48
+ )
49
+
50
+
51
+ try:
52
+ from sklearn.__check_build._check_build import check_build # noqa: F401
53
+ except ImportError as e:
54
+ raise_build_error(e)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (2.26 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.lib ADDED
Binary file (2.1 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.cp312-win_amd64.pyd ADDED
Binary file (28.7 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/_check_build.pyx ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ def check_build():
2
+ return
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__check_build/meson.build ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ py.extension_module(
2
+ '_check_build',
3
+ cython_gen.process('_check_build.pyx'),
4
+ install: true,
5
+ subdir: 'sklearn/__check_build',
6
+ )
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__init__.py ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Configure global settings and get information about the working environment."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ # Machine learning module for Python
7
+ # ==================================
8
+ #
9
+ # sklearn is a Python module integrating classical machine
10
+ # learning algorithms in the tightly-knit world of scientific Python
11
+ # packages (numpy, scipy, matplotlib).
12
+ #
13
+ # It aims to provide simple and efficient solutions to learning problems
14
+ # that are accessible to everybody and reusable in various contexts:
15
+ # machine-learning as a versatile tool for science and engineering.
16
+ #
17
+ # See https://scikit-learn.org for complete documentation.
18
+
19
+ import importlib as _importlib
20
+ import logging
21
+ import os
22
+ import random
23
+
24
+ from sklearn._config import config_context, get_config, set_config
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+
29
+ # PEP0440 compatible formatted version, see:
30
+ # https://www.python.org/dev/peps/pep-0440/
31
+ #
32
+ # Generic release markers:
33
+ # X.Y.0 # For first release after an increment in Y
34
+ # X.Y.Z # For bugfix releases
35
+ #
36
+ # Admissible pre-release markers:
37
+ # X.Y.ZaN # Alpha release
38
+ # X.Y.ZbN # Beta release
39
+ # X.Y.ZrcN # Release Candidate
40
+ # X.Y.Z # Final release
41
+ #
42
+ # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
43
+ # 'X.Y.dev0' is the canonical version of 'X.Y.dev'
44
+ #
45
+ __version__ = "1.8.0"
46
+
47
+
48
+ # On OSX, we can get a runtime error due to multiple OpenMP libraries loaded
49
+ # simultaneously. This can happen for instance when calling BLAS inside a
50
+ # prange. Setting the following environment variable allows multiple OpenMP
51
+ # libraries to be loaded. It should not degrade performances since we manually
52
+ # take care of potential over-subcription performance issues, in sections of
53
+ # the code where nested OpenMP loops can happen, by dynamically reconfiguring
54
+ # the inner OpenMP runtime to temporarily disable it while under the scope of
55
+ # the outer OpenMP parallel section.
56
+ os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "True")
57
+
58
+ # Workaround issue discovered in intel-openmp 2019.5:
59
+ # https://github.com/ContinuumIO/anaconda-issues/issues/11294
60
+ os.environ.setdefault("KMP_INIT_AT_FORK", "FALSE")
61
+
62
+ # `_distributor_init` allows distributors to run custom init code.
63
+ # For instance, for the Windows wheel, this is used to pre-load the
64
+ # vcomp shared library runtime for OpenMP embedded in the sklearn/.libs
65
+ # sub-folder.
66
+ # It is necessary to do this prior to importing show_versions as the
67
+ # later is linked to the OpenMP runtime to make it possible to introspect
68
+ # it and importing it first would fail if the OpenMP dll cannot be found.
69
+ from sklearn import __check_build, _distributor_init # noqa: E402 F401
70
+ from sklearn.base import clone # noqa: E402
71
+ from sklearn.utils._show_versions import show_versions # noqa: E402
72
+
73
+ _submodules = [
74
+ "calibration",
75
+ "cluster",
76
+ "covariance",
77
+ "cross_decomposition",
78
+ "datasets",
79
+ "decomposition",
80
+ "dummy",
81
+ "ensemble",
82
+ "exceptions",
83
+ "experimental",
84
+ "externals",
85
+ "feature_extraction",
86
+ "feature_selection",
87
+ "frozen",
88
+ "gaussian_process",
89
+ "inspection",
90
+ "isotonic",
91
+ "kernel_approximation",
92
+ "kernel_ridge",
93
+ "linear_model",
94
+ "manifold",
95
+ "metrics",
96
+ "mixture",
97
+ "model_selection",
98
+ "multiclass",
99
+ "multioutput",
100
+ "naive_bayes",
101
+ "neighbors",
102
+ "neural_network",
103
+ "pipeline",
104
+ "preprocessing",
105
+ "random_projection",
106
+ "semi_supervised",
107
+ "svm",
108
+ "tree",
109
+ "discriminant_analysis",
110
+ "impute",
111
+ "compose",
112
+ ]
113
+
114
+ __all__ = _submodules + [
115
+ # Non-modules:
116
+ "clone",
117
+ "get_config",
118
+ "set_config",
119
+ "config_context",
120
+ "show_versions",
121
+ ]
122
+
123
+
124
+ def __dir__():
125
+ return __all__
126
+
127
+
128
+ def __getattr__(name):
129
+ if name in _submodules:
130
+ return _importlib.import_module(f"sklearn.{name}")
131
+ else:
132
+ try:
133
+ return globals()[name]
134
+ except KeyError:
135
+ raise AttributeError(f"Module 'sklearn' has no attribute '{name}'")
136
+
137
+
138
+ def setup_module(module):
139
+ """Fixture for the tests to assure globally controllable seeding of RNGs"""
140
+
141
+ import numpy as np
142
+
143
+ # Check if a random seed exists in the environment, if not create one.
144
+ _random_seed = os.environ.get("SKLEARN_SEED", None)
145
+ if _random_seed is None:
146
+ _random_seed = np.random.uniform() * np.iinfo(np.int32).max
147
+ _random_seed = int(_random_seed)
148
+ print("I: Seeding RNGs with %r" % _random_seed)
149
+ np.random.seed(_random_seed)
150
+ random.seed(_random_seed)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (2.96 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_config.cpython-312.pyc ADDED
Binary file (14.2 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/_distributor_init.cpython-312.pyc ADDED
Binary file (1.11 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/base.cpython-312.pyc ADDED
Binary file (51 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/__pycache__/exceptions.cpython-312.pyc ADDED
Binary file (9.49 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/__init__.py ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/tempita.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ import argparse
7
+ import os
8
+
9
+ from Cython import Tempita as tempita
10
+
11
+ # XXX: If this import ever fails (does it really?), vendor either
12
+ # cython.tempita or numpy/npy_tempita.
13
+
14
+
15
+ def process_tempita(fromfile, outfile=None):
16
+ """Process tempita templated file and write out the result.
17
+
18
+ The template file is expected to end in `.c.tp` or `.pyx.tp`:
19
+ E.g. processing `template.c.in` generates `template.c`.
20
+
21
+ """
22
+ with open(fromfile, "r", encoding="utf-8") as f:
23
+ template_content = f.read()
24
+
25
+ template = tempita.Template(template_content)
26
+ content = template.substitute()
27
+
28
+ with open(outfile, "w", encoding="utf-8") as f:
29
+ f.write(content)
30
+
31
+
32
+ def main():
33
+ parser = argparse.ArgumentParser()
34
+ parser.add_argument("infile", type=str, help="Path to the input file")
35
+ parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory")
36
+ parser.add_argument(
37
+ "-i",
38
+ "--ignore",
39
+ type=str,
40
+ help=(
41
+ "An ignored input - may be useful to add a "
42
+ "dependency between custom targets"
43
+ ),
44
+ )
45
+ args = parser.parse_args()
46
+
47
+ if not args.infile.endswith(".tp"):
48
+ raise ValueError(f"Unexpected extension: {args.infile}")
49
+
50
+ if not args.outdir:
51
+ raise ValueError("Missing `--outdir` argument to tempita.py")
52
+
53
+ outdir_abs = os.path.join(os.getcwd(), args.outdir)
54
+ outfile = os.path.join(
55
+ outdir_abs, os.path.splitext(os.path.split(args.infile)[1])[0]
56
+ )
57
+
58
+ process_tempita(args.infile, outfile)
59
+
60
+
61
+ if __name__ == "__main__":
62
+ main()
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_build_utils/version.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Extract version number from __init__.py"""
3
+
4
+ # Authors: The scikit-learn developers
5
+ # SPDX-License-Identifier: BSD-3-Clause
6
+
7
+ import os
8
+
9
+ sklearn_init = os.path.join(os.path.dirname(__file__), "../__init__.py")
10
+
11
+ data = open(sklearn_init).readlines()
12
+ version_line = next(line for line in data if line.startswith("__version__"))
13
+
14
+ version = version_line.strip().split(" = ")[1].replace('"', "").replace("'", "")
15
+
16
+ print(version)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_config.py ADDED
@@ -0,0 +1,407 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Global configuration state and functions for management"""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ import os
7
+ import threading
8
+ from contextlib import contextmanager as contextmanager
9
+
10
+ _global_config = {
11
+ "assume_finite": bool(os.environ.get("SKLEARN_ASSUME_FINITE", False)),
12
+ "working_memory": int(os.environ.get("SKLEARN_WORKING_MEMORY", 1024)),
13
+ "print_changed_only": True,
14
+ "display": "diagram",
15
+ "pairwise_dist_chunk_size": int(
16
+ os.environ.get("SKLEARN_PAIRWISE_DIST_CHUNK_SIZE", 256)
17
+ ),
18
+ "enable_cython_pairwise_dist": True,
19
+ "array_api_dispatch": False,
20
+ "transform_output": "default",
21
+ "enable_metadata_routing": False,
22
+ "skip_parameter_validation": False,
23
+ }
24
+ _threadlocal = threading.local()
25
+
26
+
27
+ def _get_threadlocal_config():
28
+ """Get a threadlocal **mutable** configuration. If the configuration
29
+ does not exist, copy the default global configuration."""
30
+ if not hasattr(_threadlocal, "global_config"):
31
+ _threadlocal.global_config = _global_config.copy()
32
+ return _threadlocal.global_config
33
+
34
+
35
+ def get_config():
36
+ """Retrieve the current scikit-learn configuration.
37
+
38
+ This reflects the effective global configurations as established by default upon
39
+ library import, or modified via :func:`set_config` or :func:`config_context`.
40
+
41
+ Returns
42
+ -------
43
+ config : dict
44
+ Keys are parameter names that can be passed to :func:`set_config`.
45
+
46
+ See Also
47
+ --------
48
+ config_context : Context manager for global scikit-learn configuration.
49
+ set_config : Set global scikit-learn configuration.
50
+
51
+ Examples
52
+ --------
53
+ >>> import sklearn
54
+ >>> config = sklearn.get_config()
55
+ >>> config.keys()
56
+ dict_keys([...])
57
+ """
58
+ # Return a copy of the threadlocal configuration so that users will
59
+ # not be able to modify the configuration with the returned dict.
60
+ return _get_threadlocal_config().copy()
61
+
62
+
63
+ def set_config(
64
+ assume_finite=None,
65
+ working_memory=None,
66
+ print_changed_only=None,
67
+ display=None,
68
+ pairwise_dist_chunk_size=None,
69
+ enable_cython_pairwise_dist=None,
70
+ array_api_dispatch=None,
71
+ transform_output=None,
72
+ enable_metadata_routing=None,
73
+ skip_parameter_validation=None,
74
+ ):
75
+ """Set global scikit-learn configuration.
76
+
77
+ These settings control the behaviour of scikit-learn functions during a library
78
+ usage session. Global configuration defaults (as described in the parameter list
79
+ below) take effect when scikit-learn is imported.
80
+
81
+ This function can be used to modify the global scikit-learn configuration at
82
+ runtime. Passing `None` as an argument (the default) leaves the corresponding
83
+ setting unchanged. This allows users to selectively update the global configuration
84
+ values without affecting the others.
85
+
86
+ .. versionadded:: 0.19
87
+
88
+ Parameters
89
+ ----------
90
+ assume_finite : bool, default=None
91
+ If True, validation for finiteness will be skipped,
92
+ saving time, but leading to potential crashes. If
93
+ False, validation for finiteness will be performed,
94
+ avoiding error. Global default: False.
95
+
96
+ .. versionadded:: 0.19
97
+
98
+ working_memory : int, default=None
99
+ If set, scikit-learn will attempt to limit the size of temporary arrays
100
+ to this number of MiB (per job when parallelised), often saving both
101
+ computation time and memory on expensive operations that can be
102
+ performed in chunks. Global default: 1024.
103
+
104
+ .. versionadded:: 0.20
105
+
106
+ print_changed_only : bool, default=None
107
+ If True, only the parameters that were set to non-default
108
+ values will be printed when printing an estimator. For example,
109
+ ``print(SVC())`` while True will only print 'SVC()' while the default
110
+ behaviour would be to print 'SVC(C=1.0, cache_size=200, ...)' with
111
+ all the non-changed parameters. Global default: True.
112
+
113
+ .. versionadded:: 0.21
114
+ .. versionchanged:: 0.23
115
+ Global default configuration changed from False to True.
116
+
117
+ display : {'text', 'diagram'}, default=None
118
+ If 'diagram', estimators will be displayed as a diagram in a Jupyter
119
+ lab or notebook context. If 'text', estimators will be displayed as
120
+ text. Global default: 'diagram'.
121
+
122
+ .. versionadded:: 0.23
123
+
124
+ pairwise_dist_chunk_size : int, default=None
125
+ The number of row vectors per chunk for the accelerated pairwise-
126
+ distances reduction backend. Global default: 256 (suitable for most of
127
+ modern laptops' caches and architectures).
128
+
129
+ Intended for easier benchmarking and testing of scikit-learn internals.
130
+ End users are not expected to benefit from customizing this configuration
131
+ setting.
132
+
133
+ .. versionadded:: 1.1
134
+
135
+ enable_cython_pairwise_dist : bool, default=None
136
+ Use the accelerated pairwise-distances reduction backend when
137
+ possible. Global default: True.
138
+
139
+ Intended for easier benchmarking and testing of scikit-learn internals.
140
+ End users are not expected to benefit from customizing this configuration
141
+ setting.
142
+
143
+ .. versionadded:: 1.1
144
+
145
+ array_api_dispatch : bool, default=None
146
+ Use Array API dispatching when inputs follow the Array API standard.
147
+ Global default: False.
148
+
149
+ See the :ref:`User Guide <array_api>` for more details.
150
+
151
+ .. versionadded:: 1.2
152
+
153
+ transform_output : str, default=None
154
+ Configure output of `transform` and `fit_transform`.
155
+
156
+ See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py`
157
+ for an example on how to use the API.
158
+
159
+ - `"default"`: Default output format of a transformer
160
+ - `"pandas"`: DataFrame output
161
+ - `"polars"`: Polars output
162
+ - `None`: Transform configuration is unchanged
163
+
164
+ Global default: "default".
165
+
166
+ .. versionadded:: 1.2
167
+ .. versionadded:: 1.4
168
+ `"polars"` option was added.
169
+
170
+ enable_metadata_routing : bool, default=None
171
+ Enable metadata routing. By default this feature is disabled.
172
+
173
+ Refer to :ref:`metadata routing user guide <metadata_routing>` for more
174
+ details.
175
+
176
+ - `True`: Metadata routing is enabled
177
+ - `False`: Metadata routing is disabled, use the old syntax.
178
+ - `None`: Configuration is unchanged
179
+
180
+ Global default: False.
181
+
182
+ .. versionadded:: 1.3
183
+
184
+ skip_parameter_validation : bool, default=None
185
+ If `True`, disable the validation of the hyper-parameters' types and values in
186
+ the fit method of estimators and for arguments passed to public helper
187
+ functions. It can save time in some situations but can lead to low level
188
+ crashes and exceptions with confusing error messages.
189
+ Global default: False.
190
+
191
+ Note that for data parameters, such as `X` and `y`, only type validation is
192
+ skipped but validation with `check_array` will continue to run.
193
+
194
+ .. versionadded:: 1.3
195
+
196
+ See Also
197
+ --------
198
+ config_context : Context manager for global scikit-learn configuration.
199
+ get_config : Retrieve current values of the global configuration.
200
+
201
+ Examples
202
+ --------
203
+ >>> from sklearn import set_config
204
+ >>> set_config(display='diagram') # doctest: +SKIP
205
+ """
206
+ local_config = _get_threadlocal_config()
207
+
208
+ if assume_finite is not None:
209
+ local_config["assume_finite"] = assume_finite
210
+ if working_memory is not None:
211
+ local_config["working_memory"] = working_memory
212
+ if print_changed_only is not None:
213
+ local_config["print_changed_only"] = print_changed_only
214
+ if display is not None:
215
+ local_config["display"] = display
216
+ if pairwise_dist_chunk_size is not None:
217
+ local_config["pairwise_dist_chunk_size"] = pairwise_dist_chunk_size
218
+ if enable_cython_pairwise_dist is not None:
219
+ local_config["enable_cython_pairwise_dist"] = enable_cython_pairwise_dist
220
+ if array_api_dispatch is not None:
221
+ from sklearn.utils._array_api import _check_array_api_dispatch
222
+
223
+ _check_array_api_dispatch(array_api_dispatch)
224
+ local_config["array_api_dispatch"] = array_api_dispatch
225
+ if transform_output is not None:
226
+ local_config["transform_output"] = transform_output
227
+ if enable_metadata_routing is not None:
228
+ local_config["enable_metadata_routing"] = enable_metadata_routing
229
+ if skip_parameter_validation is not None:
230
+ local_config["skip_parameter_validation"] = skip_parameter_validation
231
+
232
+
233
+ @contextmanager
234
+ def config_context(
235
+ *,
236
+ assume_finite=None,
237
+ working_memory=None,
238
+ print_changed_only=None,
239
+ display=None,
240
+ pairwise_dist_chunk_size=None,
241
+ enable_cython_pairwise_dist=None,
242
+ array_api_dispatch=None,
243
+ transform_output=None,
244
+ enable_metadata_routing=None,
245
+ skip_parameter_validation=None,
246
+ ):
247
+ """Context manager to temporarily change the global scikit-learn configuration.
248
+
249
+ This context manager can be used to apply scikit-learn configuration changes within
250
+ the scope of the with statement. Once the context exits, the global configuration is
251
+ restored again.
252
+
253
+ The default global configurations (which take effect when scikit-learn is imported)
254
+ are defined below in the parameter list.
255
+
256
+ Parameters
257
+ ----------
258
+ assume_finite : bool, default=None
259
+ If True, validation for finiteness will be skipped,
260
+ saving time, but leading to potential crashes. If
261
+ False, validation for finiteness will be performed,
262
+ avoiding error. If None, the existing configuration won't change.
263
+ Global default: False.
264
+
265
+ working_memory : int, default=None
266
+ If set, scikit-learn will attempt to limit the size of temporary arrays
267
+ to this number of MiB (per job when parallelised), often saving both
268
+ computation time and memory on expensive operations that can be
269
+ performed in chunks. If None, the existing configuration won't change.
270
+ Global default: 1024.
271
+
272
+ print_changed_only : bool, default=None
273
+ If True, only the parameters that were set to non-default
274
+ values will be printed when printing an estimator. For example,
275
+ ``print(SVC())`` while True will only print 'SVC()', but would print
276
+ 'SVC(C=1.0, cache_size=200, ...)' with all the non-changed parameters
277
+ when False. If None, the existing configuration won't change.
278
+ Global default: True.
279
+
280
+ .. versionchanged:: 0.23
281
+ Global default configuration changed from False to True.
282
+
283
+ display : {'text', 'diagram'}, default=None
284
+ If 'diagram', estimators will be displayed as a diagram in a Jupyter
285
+ lab or notebook context. If 'text', estimators will be displayed as
286
+ text. If None, the existing configuration won't change.
287
+ Global default: 'diagram'.
288
+
289
+ .. versionadded:: 0.23
290
+
291
+ pairwise_dist_chunk_size : int, default=None
292
+ The number of row vectors per chunk for the accelerated pairwise-
293
+ distances reduction backend. Global default: 256 (suitable for most of
294
+ modern laptops' caches and architectures).
295
+
296
+ Intended for easier benchmarking and testing of scikit-learn internals.
297
+ End users are not expected to benefit from customizing this configuration
298
+ setting.
299
+
300
+ .. versionadded:: 1.1
301
+
302
+ enable_cython_pairwise_dist : bool, default=None
303
+ Use the accelerated pairwise-distances reduction backend when
304
+ possible. Global default: True.
305
+
306
+ Intended for easier benchmarking and testing of scikit-learn internals.
307
+ End users are not expected to benefit from customizing this configuration
308
+ setting.
309
+
310
+ .. versionadded:: 1.1
311
+
312
+ array_api_dispatch : bool, default=None
313
+ Use Array API dispatching when inputs follow the Array API standard.
314
+ Global default: False.
315
+
316
+ See the :ref:`User Guide <array_api>` for more details.
317
+
318
+ .. versionadded:: 1.2
319
+
320
+ transform_output : str, default=None
321
+ Configure output of `transform` and `fit_transform`.
322
+
323
+ See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py`
324
+ for an example on how to use the API.
325
+
326
+ - `"default"`: Default output format of a transformer
327
+ - `"pandas"`: DataFrame output
328
+ - `"polars"`: Polars output
329
+ - `None`: Transform configuration is unchanged
330
+
331
+ Global default: "default".
332
+
333
+ .. versionadded:: 1.2
334
+ .. versionadded:: 1.4
335
+ `"polars"` option was added.
336
+
337
+ enable_metadata_routing : bool, default=None
338
+ Enable metadata routing. By default this feature is disabled.
339
+
340
+ Refer to :ref:`metadata routing user guide <metadata_routing>` for more
341
+ details.
342
+
343
+ - `True`: Metadata routing is enabled
344
+ - `False`: Metadata routing is disabled, use the old syntax.
345
+ - `None`: Configuration is unchanged
346
+
347
+ Global default: False.
348
+
349
+ .. versionadded:: 1.3
350
+
351
+ skip_parameter_validation : bool, default=None
352
+ If `True`, disable the validation of the hyper-parameters' types and values in
353
+ the fit method of estimators and for arguments passed to public helper
354
+ functions. It can save time in some situations but can lead to low level
355
+ crashes and exceptions with confusing error messages.
356
+ Global default: False.
357
+
358
+ Note that for data parameters, such as `X` and `y`, only type validation is
359
+ skipped but validation with `check_array` will continue to run.
360
+
361
+ .. versionadded:: 1.3
362
+
363
+ Yields
364
+ ------
365
+ None.
366
+
367
+ See Also
368
+ --------
369
+ set_config : Set global scikit-learn configuration.
370
+ get_config : Retrieve current values of the global configuration.
371
+
372
+ Notes
373
+ -----
374
+ All settings, not just those presently modified, will be returned to
375
+ their previous values when the context manager is exited.
376
+
377
+ Examples
378
+ --------
379
+ >>> import sklearn
380
+ >>> from sklearn.utils.validation import assert_all_finite
381
+ >>> with sklearn.config_context(assume_finite=True):
382
+ ... assert_all_finite([float('nan')])
383
+ >>> with sklearn.config_context(assume_finite=True):
384
+ ... with sklearn.config_context(assume_finite=False):
385
+ ... assert_all_finite([float('nan')])
386
+ Traceback (most recent call last):
387
+ ...
388
+ ValueError: Input contains NaN...
389
+ """
390
+ old_config = get_config()
391
+ set_config(
392
+ assume_finite=assume_finite,
393
+ working_memory=working_memory,
394
+ print_changed_only=print_changed_only,
395
+ display=display,
396
+ pairwise_dist_chunk_size=pairwise_dist_chunk_size,
397
+ enable_cython_pairwise_dist=enable_cython_pairwise_dist,
398
+ array_api_dispatch=array_api_dispatch,
399
+ transform_output=transform_output,
400
+ enable_metadata_routing=enable_metadata_routing,
401
+ skip_parameter_validation=skip_parameter_validation,
402
+ )
403
+
404
+ try:
405
+ yield
406
+ finally:
407
+ set_config(**old_config)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.lib ADDED
Binary file (2.07 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_cyutility.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0c2c7fdd27c3deededaf164b441e9622d275d46bac4b6409f2f3abd03b1aa64
3
+ size 129024
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_distributor_init.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ '''Helper to preload vcomp140.dll and msvcp140.dll to prevent
3
+ "not found" errors.
4
+
5
+ Once vcomp140.dll and msvcp140.dll are
6
+ preloaded, the namespace is made available to any subsequent
7
+ vcomp140.dll and msvcp140.dll. This is
8
+ created as part of the scripts that build the wheel.
9
+ '''
10
+
11
+
12
+ import os
13
+ import os.path as op
14
+ from ctypes import WinDLL
15
+
16
+
17
+ if os.name == "nt":
18
+ libs_path = op.join(op.dirname(__file__), ".libs")
19
+ vcomp140_dll_filename = op.join(libs_path, "vcomp140.dll")
20
+ msvcp140_dll_filename = op.join(libs_path, "msvcp140.dll")
21
+ WinDLL(op.abspath(vcomp140_dll_filename))
22
+ WinDLL(op.abspath(msvcp140_dll_filename))
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.lib ADDED
Binary file (2.05 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.cp312-win_amd64.pyd ADDED
Binary file (91.6 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_isotonic.pyx ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Authors: The scikit-learn developers
2
+ # SPDX-License-Identifier: BSD-3-Clause
3
+
4
+ # Uses the pool adjacent violators algorithm (PAVA), with the
5
+ # enhancement of searching for the longest decreasing subsequence to
6
+ # pool at each step.
7
+
8
+ import numpy as np
9
+ from cython cimport floating
10
+
11
+
12
+ def _inplace_contiguous_isotonic_regression(floating[::1] y, floating[::1] w):
13
+ cdef:
14
+ Py_ssize_t n = y.shape[0], i, k
15
+ floating prev_y, sum_wy, sum_w
16
+ Py_ssize_t[::1] target = np.arange(n, dtype=np.intp)
17
+
18
+ # target describes a list of blocks. At any time, if [i..j] (inclusive) is
19
+ # an active block, then target[i] := j and target[j] := i.
20
+
21
+ # For "active" indices (block starts):
22
+ # w[i] := sum{w_orig[j], j=[i..target[i]]}
23
+ # y[i] := sum{y_orig[j]*w_orig[j], j=[i..target[i]]} / w[i]
24
+
25
+ with nogil:
26
+ i = 0
27
+ while i < n:
28
+ k = target[i] + 1
29
+ if k == n:
30
+ break
31
+ if y[i] < y[k]:
32
+ i = k
33
+ continue
34
+ sum_wy = w[i] * y[i]
35
+ sum_w = w[i]
36
+ while True:
37
+ # We are within a decreasing subsequence.
38
+ prev_y = y[k]
39
+ sum_wy += w[k] * y[k]
40
+ sum_w += w[k]
41
+ k = target[k] + 1
42
+ if k == n or prev_y < y[k]:
43
+ # Non-singleton decreasing subsequence is finished,
44
+ # update first entry.
45
+ y[i] = sum_wy / sum_w
46
+ w[i] = sum_w
47
+ target[i] = k - 1
48
+ target[k - 1] = i
49
+ if i > 0:
50
+ # Backtrack if we can. This makes the algorithm
51
+ # single-pass and ensures O(n) complexity.
52
+ i = target[i - 1]
53
+ # Otherwise, restart from the same point.
54
+ break
55
+ # Reconstruct the solution.
56
+ i = 0
57
+ while i < n:
58
+ k = target[i] + 1
59
+ y[i + 1 : k] = y[i]
60
+ i = k
61
+
62
+
63
+ def _make_unique(const floating[::1] X,
64
+ const floating[::1] y,
65
+ const floating[::1] sample_weights):
66
+ """Average targets for duplicate X, drop duplicates.
67
+
68
+ Aggregates duplicate X values into a single X value where
69
+ the target y is a (sample_weighted) average of the individual
70
+ targets.
71
+
72
+ Assumes that X is ordered, so that all duplicates follow each other.
73
+ """
74
+ unique_values = len(np.unique(X))
75
+
76
+ if floating is float:
77
+ dtype = np.float32
78
+ else:
79
+ dtype = np.float64
80
+
81
+ cdef floating[::1] y_out = np.empty(unique_values, dtype=dtype)
82
+ cdef floating[::1] x_out = np.empty_like(y_out)
83
+ cdef floating[::1] weights_out = np.empty_like(y_out)
84
+
85
+ cdef floating current_x = X[0]
86
+ cdef floating current_y = 0
87
+ cdef floating current_weight = 0
88
+ cdef int i = 0
89
+ cdef int j
90
+ cdef floating x
91
+ cdef int n_samples = len(X)
92
+ cdef floating eps = np.finfo(dtype).resolution
93
+
94
+ for j in range(n_samples):
95
+ x = X[j]
96
+ if x - current_x >= eps:
97
+ # next unique value
98
+ x_out[i] = current_x
99
+ weights_out[i] = current_weight
100
+ y_out[i] = current_y / current_weight
101
+ i += 1
102
+ current_x = x
103
+ current_weight = sample_weights[j]
104
+ current_y = y[j] * sample_weights[j]
105
+ else:
106
+ current_weight += sample_weights[j]
107
+ current_y += y[j] * sample_weights[j]
108
+
109
+ x_out[i] = current_x
110
+ weights_out[i] = current_weight
111
+ y_out[i] = current_y / current_weight
112
+ return(
113
+ np.asarray(x_out[:i+1]),
114
+ np.asarray(y_out[:i+1]),
115
+ np.asarray(weights_out[:i+1]),
116
+ )
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__init__.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ The :mod:`sklearn._loss` module includes loss function classes suitable for
3
+ fitting classification and regression tasks.
4
+ """
5
+
6
+ # Authors: The scikit-learn developers
7
+ # SPDX-License-Identifier: BSD-3-Clause
8
+
9
+ from sklearn._loss.loss import (
10
+ AbsoluteError,
11
+ HalfBinomialLoss,
12
+ HalfGammaLoss,
13
+ HalfMultinomialLoss,
14
+ HalfPoissonLoss,
15
+ HalfSquaredError,
16
+ HalfTweedieLoss,
17
+ HalfTweedieLossIdentity,
18
+ HuberLoss,
19
+ PinballLoss,
20
+ )
21
+
22
+ __all__ = [
23
+ "AbsoluteError",
24
+ "HalfBinomialLoss",
25
+ "HalfGammaLoss",
26
+ "HalfMultinomialLoss",
27
+ "HalfPoissonLoss",
28
+ "HalfSquaredError",
29
+ "HalfTweedieLoss",
30
+ "HalfTweedieLossIdentity",
31
+ "HuberLoss",
32
+ "PinballLoss",
33
+ ]
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (663 Bytes). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/link.cpython-312.pyc ADDED
Binary file (11.2 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/__pycache__/loss.cpython-312.pyc ADDED
Binary file (44.4 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.lib ADDED
Binary file (1.98 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc0cde6619e704aae36a7ea864ff3540019b0bd38924b4ab6bdb42623dac5c1e
3
+ size 1311232
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pxd ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Fused types for input like y_true, raw_prediction, sample_weights.
2
+ ctypedef fused floating_in:
3
+ double
4
+ float
5
+
6
+
7
+ # Fused types for output like gradient and hessian
8
+ # We use a different fused types for input (floating_in) and output (floating_out), such
9
+ # that input and output can have different dtypes in the same function call. A single
10
+ # fused type can only take on one single value (type) for all arguments in one function
11
+ # call.
12
+ ctypedef fused floating_out:
13
+ double
14
+ float
15
+
16
+
17
+ # Struct to return 2 doubles
18
+ ctypedef struct double_pair:
19
+ double val1
20
+ double val2
21
+
22
+
23
+ # C base class for loss functions
24
+ cdef class CyLossFunction:
25
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
26
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
27
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
28
+
29
+
30
+ cdef class CyHalfSquaredError(CyLossFunction):
31
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
32
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
33
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
34
+
35
+
36
+ cdef class CyAbsoluteError(CyLossFunction):
37
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
38
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
39
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
40
+
41
+
42
+ cdef class CyPinballLoss(CyLossFunction):
43
+ cdef readonly double quantile # readonly makes it accessible from Python
44
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
45
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
46
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
47
+
48
+
49
+ cdef class CyHuberLoss(CyLossFunction):
50
+ cdef public double delta # public makes it accessible from Python
51
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
52
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
53
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
54
+
55
+
56
+ cdef class CyHalfPoissonLoss(CyLossFunction):
57
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
58
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
59
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
60
+
61
+
62
+ cdef class CyHalfGammaLoss(CyLossFunction):
63
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
64
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
65
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
66
+
67
+
68
+ cdef class CyHalfTweedieLoss(CyLossFunction):
69
+ cdef readonly double power # readonly makes it accessible from Python
70
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
71
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
72
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
73
+
74
+
75
+ cdef class CyHalfTweedieLossIdentity(CyLossFunction):
76
+ cdef readonly double power # readonly makes it accessible from Python
77
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
78
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
79
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
80
+
81
+
82
+ cdef class CyHalfBinomialLoss(CyLossFunction):
83
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
84
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
85
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
86
+
87
+
88
+ cdef class CyExponentialLoss(CyLossFunction):
89
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil
90
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil
91
+ cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil
92
+
93
+
94
+ cdef class CyHalfMultinomialLoss():
95
+ cdef void cy_gradient(
96
+ self,
97
+ const floating_in y_true,
98
+ const floating_in[::1] raw_prediction,
99
+ const floating_in sample_weight,
100
+ floating_out[::1] gradient_out,
101
+ ) noexcept nogil
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/_loss.pyx.tp ADDED
@@ -0,0 +1,1505 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{py:
2
+
3
+ """
4
+ Template file to easily generate loops over samples using Tempita
5
+ (https://github.com/cython/cython/blob/master/Cython/Tempita/_tempita.py).
6
+
7
+ Generated file: _loss.pyx
8
+
9
+ Each loss class is generated by a cdef functions on single samples.
10
+ The keywords between double braces are substituted during the build.
11
+ """
12
+
13
+ doc_HalfSquaredError = (
14
+ """Half Squared Error with identity link.
15
+
16
+ Domain:
17
+ y_true and y_pred all real numbers
18
+
19
+ Link:
20
+ y_pred = raw_prediction
21
+ """
22
+ )
23
+
24
+ doc_AbsoluteError = (
25
+ """Absolute Error with identity link.
26
+
27
+ Domain:
28
+ y_true and y_pred all real numbers
29
+
30
+ Link:
31
+ y_pred = raw_prediction
32
+ """
33
+ )
34
+
35
+ doc_PinballLoss = (
36
+ """Quantile Loss aka Pinball Loss with identity link.
37
+
38
+ Domain:
39
+ y_true and y_pred all real numbers
40
+ quantile in (0, 1)
41
+
42
+ Link:
43
+ y_pred = raw_prediction
44
+
45
+ Note: 2 * cPinballLoss(quantile=0.5) equals cAbsoluteError()
46
+ """
47
+ )
48
+
49
+ doc_HuberLoss = (
50
+ """Huber Loss with identity link.
51
+
52
+ Domain:
53
+ y_true and y_pred all real numbers
54
+ delta in positive real numbers
55
+
56
+ Link:
57
+ y_pred = raw_prediction
58
+ """
59
+ )
60
+
61
+ doc_HalfPoissonLoss = (
62
+ """Half Poisson deviance loss with log-link.
63
+
64
+ Domain:
65
+ y_true in non-negative real numbers
66
+ y_pred in positive real numbers
67
+
68
+ Link:
69
+ y_pred = exp(raw_prediction)
70
+
71
+ Half Poisson deviance with log-link is
72
+ y_true * log(y_true/y_pred) + y_pred - y_true
73
+ = y_true * log(y_true) - y_true * raw_prediction
74
+ + exp(raw_prediction) - y_true
75
+
76
+ Dropping constant terms, this gives:
77
+ exp(raw_prediction) - y_true * raw_prediction
78
+ """
79
+ )
80
+
81
+ doc_HalfGammaLoss = (
82
+ """Half Gamma deviance loss with log-link.
83
+
84
+ Domain:
85
+ y_true and y_pred in positive real numbers
86
+
87
+ Link:
88
+ y_pred = exp(raw_prediction)
89
+
90
+ Half Gamma deviance with log-link is
91
+ log(y_pred/y_true) + y_true/y_pred - 1
92
+ = raw_prediction - log(y_true) + y_true * exp(-raw_prediction) - 1
93
+
94
+ Dropping constant terms, this gives:
95
+ raw_prediction + y_true * exp(-raw_prediction)
96
+ """
97
+ )
98
+
99
+ doc_HalfTweedieLoss = (
100
+ """Half Tweedie deviance loss with log-link.
101
+
102
+ Domain:
103
+ y_true in real numbers if p <= 0
104
+ y_true in non-negative real numbers if 0 < p < 2
105
+ y_true in positive real numbers if p >= 2
106
+ y_pred and power in positive real numbers
107
+
108
+ Link:
109
+ y_pred = exp(raw_prediction)
110
+
111
+ Half Tweedie deviance with log-link and p=power is
112
+ max(y_true, 0)**(2-p) / (1-p) / (2-p)
113
+ - y_true * y_pred**(1-p) / (1-p)
114
+ + y_pred**(2-p) / (2-p)
115
+ = max(y_true, 0)**(2-p) / (1-p) / (2-p)
116
+ - y_true * exp((1-p) * raw_prediction) / (1-p)
117
+ + exp((2-p) * raw_prediction) / (2-p)
118
+
119
+ Dropping constant terms, this gives:
120
+ exp((2-p) * raw_prediction) / (2-p)
121
+ - y_true * exp((1-p) * raw_prediction) / (1-p)
122
+
123
+ Notes:
124
+ - Poisson with p=1 and Gamma with p=2 have different terms dropped such
125
+ that cHalfTweedieLoss is not continuous in p=power at p=1 and p=2.
126
+ - While the Tweedie distribution only exists for p<=0 or p>=1, the range
127
+ 0<p<1 still gives a strictly consistent scoring function for the
128
+ expectation.
129
+ """
130
+ )
131
+
132
+ doc_HalfTweedieLossIdentity = (
133
+ """Half Tweedie deviance loss with identity link.
134
+
135
+ Domain:
136
+ y_true in real numbers if p <= 0
137
+ y_true in non-negative real numbers if 0 < p < 2
138
+ y_true in positive real numbers if p >= 2
139
+ y_pred and power in positive real numbers, y_pred may be negative for p=0.
140
+
141
+ Link:
142
+ y_pred = raw_prediction
143
+
144
+ Half Tweedie deviance with identity link and p=power is
145
+ max(y_true, 0)**(2-p) / (1-p) / (2-p)
146
+ - y_true * y_pred**(1-p) / (1-p)
147
+ + y_pred**(2-p) / (2-p)
148
+
149
+ Notes:
150
+ - Here, we do not drop constant terms in contrast to the version with log-link.
151
+ """
152
+ )
153
+
154
+ doc_HalfBinomialLoss = (
155
+ """Half Binomial deviance loss with logit link.
156
+
157
+ Domain:
158
+ y_true in [0, 1]
159
+ y_pred in (0, 1), i.e. boundaries excluded
160
+
161
+ Link:
162
+ y_pred = expit(raw_prediction)
163
+ """
164
+ )
165
+
166
+ doc_ExponentialLoss = (
167
+ """"Exponential loss with (half) logit link
168
+
169
+ Domain:
170
+ y_true in [0, 1]
171
+ y_pred in (0, 1), i.e. boundaries excluded
172
+
173
+ Link:
174
+ y_pred = expit(2 * raw_prediction)
175
+ """
176
+ )
177
+
178
+ # loss class name, docstring, param,
179
+ # cy_loss, cy_loss_grad,
180
+ # cy_grad, cy_grad_hess,
181
+ class_list = [
182
+ ("CyHalfSquaredError", doc_HalfSquaredError, None,
183
+ "closs_half_squared_error", None,
184
+ "cgradient_half_squared_error", "cgrad_hess_half_squared_error"),
185
+ ("CyAbsoluteError", doc_AbsoluteError, None,
186
+ "closs_absolute_error", None,
187
+ "cgradient_absolute_error", "cgrad_hess_absolute_error"),
188
+ ("CyPinballLoss", doc_PinballLoss, "quantile",
189
+ "closs_pinball_loss", None,
190
+ "cgradient_pinball_loss", "cgrad_hess_pinball_loss"),
191
+ ("CyHuberLoss", doc_HuberLoss, "delta",
192
+ "closs_huber_loss", None,
193
+ "cgradient_huber_loss", "cgrad_hess_huber_loss"),
194
+ ("CyHalfPoissonLoss", doc_HalfPoissonLoss, None,
195
+ "closs_half_poisson", "closs_grad_half_poisson",
196
+ "cgradient_half_poisson", "cgrad_hess_half_poisson"),
197
+ ("CyHalfGammaLoss", doc_HalfGammaLoss, None,
198
+ "closs_half_gamma", "closs_grad_half_gamma",
199
+ "cgradient_half_gamma", "cgrad_hess_half_gamma"),
200
+ ("CyHalfTweedieLoss", doc_HalfTweedieLoss, "power",
201
+ "closs_half_tweedie", "closs_grad_half_tweedie",
202
+ "cgradient_half_tweedie", "cgrad_hess_half_tweedie"),
203
+ ("CyHalfTweedieLossIdentity", doc_HalfTweedieLossIdentity, "power",
204
+ "closs_half_tweedie_identity", "closs_grad_half_tweedie_identity",
205
+ "cgradient_half_tweedie_identity", "cgrad_hess_half_tweedie_identity"),
206
+ ("CyHalfBinomialLoss", doc_HalfBinomialLoss, None,
207
+ "closs_half_binomial", "closs_grad_half_binomial",
208
+ "cgradient_half_binomial", "cgrad_hess_half_binomial"),
209
+ ("CyExponentialLoss", doc_ExponentialLoss, None,
210
+ "closs_exponential", "closs_grad_exponential",
211
+ "cgradient_exponential", "cgrad_hess_exponential"),
212
+ ]
213
+ }}
214
+
215
+ # Design:
216
+ # See https://github.com/scikit-learn/scikit-learn/issues/15123 for reasons.
217
+ # a) Merge link functions into loss functions for speed and numerical
218
+ # stability, i.e. use raw_prediction instead of y_pred in signature.
219
+ # b) Pure C functions (nogil) calculate single points (single sample)
220
+ # c) Wrap C functions in a loop to get Python functions operating on ndarrays.
221
+ # - Write loops manually---use Tempita for this.
222
+ # Reason: There is still some performance overhead when using a wrapper
223
+ # function "wrap" that carries out the loop and gets as argument a function
224
+ # pointer to one of the C functions from b), e.g.
225
+ # wrap(closs_half_poisson, y_true, ...)
226
+ # - Pass n_threads as argument to prange and propagate option to all callers.
227
+ # d) Provide classes (Cython extension types) per loss (names start with Cy) in
228
+ # order to have semantical structured objects.
229
+ # - Member functions for single points just call the C function from b).
230
+ # These are used e.g. in SGD `_plain_sgd`.
231
+ # - Member functions operating on ndarrays, see c), looping over calls to C
232
+ # functions from b).
233
+ # e) Provide convenience Python classes that compose from these extension types
234
+ # elsewhere (see loss.py)
235
+ # - Example: loss.gradient calls CyLoss.gradient but does some input
236
+ # checking like None -> np.empty().
237
+ #
238
+ # Note: We require 1-dim ndarrays to be contiguous.
239
+
240
+ from cython.parallel import parallel, prange
241
+ import numpy as np
242
+
243
+ from libc.math cimport exp, fabs, log, log1p, pow
244
+ from libc.stdlib cimport malloc, free
245
+
246
+
247
+ # -------------------------------------
248
+ # Helper functions
249
+ # -------------------------------------
250
+ # Numerically stable version of log(1 + exp(x)) for double precision, see Eq. (10) of
251
+ # https://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf
252
+ # Note: The only important cutoff is at x = 18. All others are to save computation
253
+ # time. Compared to the reference, we add the additional case distinction x <= -2 in
254
+ # order to use log instead of log1p for improved performance. As with the other
255
+ # cutoffs, this is accurate within machine precision of double.
256
+ cdef inline double log1pexp(double x) noexcept nogil:
257
+ if x <= -37:
258
+ return exp(x)
259
+ elif x <= -2:
260
+ return log1p(exp(x))
261
+ elif x <= 18:
262
+ return log(1. + exp(x))
263
+ elif x <= 33.3:
264
+ return x + exp(-x)
265
+ else:
266
+ return x
267
+
268
+
269
+ cdef inline double_pair sum_exp_minus_max(
270
+ const int i,
271
+ const floating_in[:, :] raw_prediction, # IN
272
+ floating_out *p # OUT
273
+ ) noexcept nogil:
274
+ # Thread local buffers are used to store part of the results via p.
275
+ # The results are stored as follows:
276
+ # p[k] = exp(raw_prediction_i_k - max_value) for k = 0 to n_classes-1
277
+ # return.val1 = max_value = max(raw_prediction_i_k, k = 0 to n_classes-1)
278
+ # return.val2 = sum_exps = sum(p[k], k = 0 to n_classes-1) = sum of exponentials
279
+ # len(p) must be n_classes
280
+ # Notes:
281
+ # - We return the max value and sum of exps (stored in p) as a double_pair.
282
+ # - i needs to be passed (and stays constant) because otherwise Cython does
283
+ # not generate optimal code, see
284
+ # https://github.com/scikit-learn/scikit-learn/issues/17299
285
+ # - We do not normalize p by calculating p[k] = p[k] / sum_exps.
286
+ # This helps to save one loop over k.
287
+ cdef:
288
+ int k
289
+ int n_classes = raw_prediction.shape[1]
290
+ double_pair max_value_and_sum_exps # val1 = max_value, val2 = sum_exps
291
+
292
+ max_value_and_sum_exps.val1 = raw_prediction[i, 0]
293
+ max_value_and_sum_exps.val2 = 0
294
+ for k in range(1, n_classes):
295
+ # Compute max value of array for numerical stability
296
+ if max_value_and_sum_exps.val1 < raw_prediction[i, k]:
297
+ max_value_and_sum_exps.val1 = raw_prediction[i, k]
298
+
299
+ for k in range(n_classes):
300
+ p[k] = exp(raw_prediction[i, k] - max_value_and_sum_exps.val1)
301
+ max_value_and_sum_exps.val2 += p[k]
302
+
303
+ return max_value_and_sum_exps
304
+
305
+
306
+ # -------------------------------------
307
+ # Single point inline C functions
308
+ # -------------------------------------
309
+ # Half Squared Error
310
+ cdef inline double closs_half_squared_error(
311
+ double y_true,
312
+ double raw_prediction
313
+ ) noexcept nogil:
314
+ return 0.5 * (raw_prediction - y_true) * (raw_prediction - y_true)
315
+
316
+
317
+ cdef inline double cgradient_half_squared_error(
318
+ double y_true,
319
+ double raw_prediction
320
+ ) noexcept nogil:
321
+ return raw_prediction - y_true
322
+
323
+
324
+ cdef inline double_pair cgrad_hess_half_squared_error(
325
+ double y_true,
326
+ double raw_prediction
327
+ ) noexcept nogil:
328
+ cdef double_pair gh
329
+ gh.val1 = raw_prediction - y_true # gradient
330
+ gh.val2 = 1. # hessian
331
+ return gh
332
+
333
+
334
+ # Absolute Error
335
+ cdef inline double closs_absolute_error(
336
+ double y_true,
337
+ double raw_prediction
338
+ ) noexcept nogil:
339
+ return fabs(raw_prediction - y_true)
340
+
341
+
342
+ cdef inline double cgradient_absolute_error(
343
+ double y_true,
344
+ double raw_prediction
345
+ ) noexcept nogil:
346
+ return 1. if raw_prediction > y_true else -1.
347
+
348
+
349
+ cdef inline double_pair cgrad_hess_absolute_error(
350
+ double y_true,
351
+ double raw_prediction
352
+ ) noexcept nogil:
353
+ cdef double_pair gh
354
+ # Note that exact hessian = 0 almost everywhere. Optimization routines like
355
+ # in HGBT, however, need a hessian > 0. Therefore, we assign 1.
356
+ gh.val1 = 1. if raw_prediction > y_true else -1. # gradient
357
+ gh.val2 = 1. # hessian
358
+ return gh
359
+
360
+
361
+ # Quantile Loss / Pinball Loss
362
+ cdef inline double closs_pinball_loss(
363
+ double y_true,
364
+ double raw_prediction,
365
+ double quantile
366
+ ) noexcept nogil:
367
+ return (quantile * (y_true - raw_prediction) if y_true >= raw_prediction
368
+ else (1. - quantile) * (raw_prediction - y_true))
369
+
370
+
371
+ cdef inline double cgradient_pinball_loss(
372
+ double y_true,
373
+ double raw_prediction,
374
+ double quantile
375
+ ) noexcept nogil:
376
+ return -quantile if y_true >=raw_prediction else 1. - quantile
377
+
378
+
379
+ cdef inline double_pair cgrad_hess_pinball_loss(
380
+ double y_true,
381
+ double raw_prediction,
382
+ double quantile
383
+ ) noexcept nogil:
384
+ cdef double_pair gh
385
+ # Note that exact hessian = 0 almost everywhere. Optimization routines like
386
+ # in HGBT, however, need a hessian > 0. Therefore, we assign 1.
387
+ gh.val1 = -quantile if y_true >=raw_prediction else 1. - quantile # gradient
388
+ gh.val2 = 1. # hessian
389
+ return gh
390
+
391
+
392
+ # Huber Loss
393
+ cdef inline double closs_huber_loss(
394
+ double y_true,
395
+ double raw_prediction,
396
+ double delta,
397
+ ) noexcept nogil:
398
+ cdef double abserr = fabs(y_true - raw_prediction)
399
+ if abserr <= delta:
400
+ return 0.5 * abserr**2
401
+ else:
402
+ return delta * (abserr - 0.5 * delta)
403
+
404
+
405
+ cdef inline double cgradient_huber_loss(
406
+ double y_true,
407
+ double raw_prediction,
408
+ double delta,
409
+ ) noexcept nogil:
410
+ cdef double res = raw_prediction - y_true
411
+ if fabs(res) <= delta:
412
+ return res
413
+ else:
414
+ return delta if res >=0 else -delta
415
+
416
+
417
+ cdef inline double_pair cgrad_hess_huber_loss(
418
+ double y_true,
419
+ double raw_prediction,
420
+ double delta,
421
+ ) noexcept nogil:
422
+ cdef double_pair gh
423
+ gh.val2 = raw_prediction - y_true # used as temporary
424
+ if fabs(gh.val2) <= delta:
425
+ gh.val1 = gh.val2 # gradient
426
+ gh.val2 = 1 # hessian
427
+ else:
428
+ gh.val1 = delta if gh.val2 >=0 else -delta # gradient
429
+ gh.val2 = 0 # hessian
430
+ return gh
431
+
432
+
433
+ # Half Poisson Deviance with Log-Link, dropping constant terms
434
+ cdef inline double closs_half_poisson(
435
+ double y_true,
436
+ double raw_prediction
437
+ ) noexcept nogil:
438
+ return exp(raw_prediction) - y_true * raw_prediction
439
+
440
+
441
+ cdef inline double cgradient_half_poisson(
442
+ double y_true,
443
+ double raw_prediction
444
+ ) noexcept nogil:
445
+ # y_pred - y_true
446
+ return exp(raw_prediction) - y_true
447
+
448
+
449
+ cdef inline double_pair closs_grad_half_poisson(
450
+ double y_true,
451
+ double raw_prediction
452
+ ) noexcept nogil:
453
+ cdef double_pair lg
454
+ lg.val2 = exp(raw_prediction) # used as temporary
455
+ lg.val1 = lg.val2 - y_true * raw_prediction # loss
456
+ lg.val2 -= y_true # gradient
457
+ return lg
458
+
459
+
460
+ cdef inline double_pair cgrad_hess_half_poisson(
461
+ double y_true,
462
+ double raw_prediction
463
+ ) noexcept nogil:
464
+ cdef double_pair gh
465
+ gh.val2 = exp(raw_prediction) # hessian
466
+ gh.val1 = gh.val2 - y_true # gradient
467
+ return gh
468
+
469
+
470
+ # Half Gamma Deviance with Log-Link, dropping constant terms
471
+ cdef inline double closs_half_gamma(
472
+ double y_true,
473
+ double raw_prediction
474
+ ) noexcept nogil:
475
+ return raw_prediction + y_true * exp(-raw_prediction)
476
+
477
+
478
+ cdef inline double cgradient_half_gamma(
479
+ double y_true,
480
+ double raw_prediction
481
+ ) noexcept nogil:
482
+ return 1. - y_true * exp(-raw_prediction)
483
+
484
+
485
+ cdef inline double_pair closs_grad_half_gamma(
486
+ double y_true,
487
+ double raw_prediction
488
+ ) noexcept nogil:
489
+ cdef double_pair lg
490
+ lg.val2 = exp(-raw_prediction) # used as temporary
491
+ lg.val1 = raw_prediction + y_true * lg.val2 # loss
492
+ lg.val2 = 1. - y_true * lg.val2 # gradient
493
+ return lg
494
+
495
+
496
+ cdef inline double_pair cgrad_hess_half_gamma(
497
+ double y_true,
498
+ double raw_prediction
499
+ ) noexcept nogil:
500
+ cdef double_pair gh
501
+ gh.val2 = exp(-raw_prediction) # used as temporary
502
+ gh.val1 = 1. - y_true * gh.val2 # gradient
503
+ gh.val2 *= y_true # hessian
504
+ return gh
505
+
506
+
507
+ # Half Tweedie Deviance with Log-Link, dropping constant terms
508
+ # Note that by dropping constants this is no longer continuous in parameter power.
509
+ cdef inline double closs_half_tweedie(
510
+ double y_true,
511
+ double raw_prediction,
512
+ double power
513
+ ) noexcept nogil:
514
+ if power == 0.:
515
+ return closs_half_squared_error(y_true, exp(raw_prediction))
516
+ elif power == 1.:
517
+ return closs_half_poisson(y_true, raw_prediction)
518
+ elif power == 2.:
519
+ return closs_half_gamma(y_true, raw_prediction)
520
+ else:
521
+ return (exp((2. - power) * raw_prediction) / (2. - power)
522
+ - y_true * exp((1. - power) * raw_prediction) / (1. - power))
523
+
524
+
525
+ cdef inline double cgradient_half_tweedie(
526
+ double y_true,
527
+ double raw_prediction,
528
+ double power
529
+ ) noexcept nogil:
530
+ cdef double exp1
531
+ if power == 0.:
532
+ exp1 = exp(raw_prediction)
533
+ return exp1 * (exp1 - y_true)
534
+ elif power == 1.:
535
+ return cgradient_half_poisson(y_true, raw_prediction)
536
+ elif power == 2.:
537
+ return cgradient_half_gamma(y_true, raw_prediction)
538
+ else:
539
+ return (exp((2. - power) * raw_prediction)
540
+ - y_true * exp((1. - power) * raw_prediction))
541
+
542
+
543
+ cdef inline double_pair closs_grad_half_tweedie(
544
+ double y_true,
545
+ double raw_prediction,
546
+ double power
547
+ ) noexcept nogil:
548
+ cdef double_pair lg
549
+ cdef double exp1, exp2
550
+ if power == 0.:
551
+ exp1 = exp(raw_prediction)
552
+ lg.val1 = closs_half_squared_error(y_true, exp1) # loss
553
+ lg.val2 = exp1 * (exp1 - y_true) # gradient
554
+ elif power == 1.:
555
+ return closs_grad_half_poisson(y_true, raw_prediction)
556
+ elif power == 2.:
557
+ return closs_grad_half_gamma(y_true, raw_prediction)
558
+ else:
559
+ exp1 = exp((1. - power) * raw_prediction)
560
+ exp2 = exp((2. - power) * raw_prediction)
561
+ lg.val1 = exp2 / (2. - power) - y_true * exp1 / (1. - power) # loss
562
+ lg.val2 = exp2 - y_true * exp1 # gradient
563
+ return lg
564
+
565
+
566
+ cdef inline double_pair cgrad_hess_half_tweedie(
567
+ double y_true,
568
+ double raw_prediction,
569
+ double power
570
+ ) noexcept nogil:
571
+ cdef double_pair gh
572
+ cdef double exp1, exp2
573
+ if power == 0.:
574
+ exp1 = exp(raw_prediction)
575
+ gh.val1 = exp1 * (exp1 - y_true) # gradient
576
+ gh.val2 = exp1 * (2 * exp1 - y_true) # hessian
577
+ elif power == 1.:
578
+ return cgrad_hess_half_poisson(y_true, raw_prediction)
579
+ elif power == 2.:
580
+ return cgrad_hess_half_gamma(y_true, raw_prediction)
581
+ else:
582
+ exp1 = exp((1. - power) * raw_prediction)
583
+ exp2 = exp((2. - power) * raw_prediction)
584
+ gh.val1 = exp2 - y_true * exp1 # gradient
585
+ gh.val2 = (2. - power) * exp2 - (1. - power) * y_true * exp1 # hessian
586
+ return gh
587
+
588
+
589
+ # Half Tweedie Deviance with identity link, without dropping constant terms!
590
+ # Therefore, best loss value is zero.
591
+ cdef inline double closs_half_tweedie_identity(
592
+ double y_true,
593
+ double raw_prediction,
594
+ double power
595
+ ) noexcept nogil:
596
+ cdef double tmp
597
+ if power == 0.:
598
+ return closs_half_squared_error(y_true, raw_prediction)
599
+ elif power == 1.:
600
+ if y_true == 0:
601
+ return raw_prediction
602
+ else:
603
+ return y_true * log(y_true/raw_prediction) + raw_prediction - y_true
604
+ elif power == 2.:
605
+ return log(raw_prediction/y_true) + y_true/raw_prediction - 1.
606
+ else:
607
+ tmp = pow(raw_prediction, 1. - power)
608
+ tmp = raw_prediction * tmp / (2. - power) - y_true * tmp / (1. - power)
609
+ if y_true > 0:
610
+ tmp += pow(y_true, 2. - power) / ((1. - power) * (2. - power))
611
+ return tmp
612
+
613
+
614
+ cdef inline double cgradient_half_tweedie_identity(
615
+ double y_true,
616
+ double raw_prediction,
617
+ double power
618
+ ) noexcept nogil:
619
+ if power == 0.:
620
+ return raw_prediction - y_true
621
+ elif power == 1.:
622
+ return 1. - y_true / raw_prediction
623
+ elif power == 2.:
624
+ return (raw_prediction - y_true) / (raw_prediction * raw_prediction)
625
+ else:
626
+ return pow(raw_prediction, -power) * (raw_prediction - y_true)
627
+
628
+
629
+ cdef inline double_pair closs_grad_half_tweedie_identity(
630
+ double y_true,
631
+ double raw_prediction,
632
+ double power
633
+ ) noexcept nogil:
634
+ cdef double_pair lg
635
+ cdef double tmp
636
+ if power == 0.:
637
+ lg.val2 = raw_prediction - y_true # gradient
638
+ lg.val1 = 0.5 * lg.val2 * lg.val2 # loss
639
+ elif power == 1.:
640
+ if y_true == 0:
641
+ lg.val1 = raw_prediction
642
+ else:
643
+ lg.val1 = (y_true * log(y_true/raw_prediction) # loss
644
+ + raw_prediction - y_true)
645
+ lg.val2 = 1. - y_true / raw_prediction # gradient
646
+ elif power == 2.:
647
+ lg.val1 = log(raw_prediction/y_true) + y_true/raw_prediction - 1. # loss
648
+ tmp = raw_prediction * raw_prediction
649
+ lg.val2 = (raw_prediction - y_true) / tmp # gradient
650
+ else:
651
+ tmp = pow(raw_prediction, 1. - power)
652
+ lg.val1 = (raw_prediction * tmp / (2. - power) # loss
653
+ - y_true * tmp / (1. - power))
654
+ if y_true > 0:
655
+ lg.val1 += (pow(y_true, 2. - power)
656
+ / ((1. - power) * (2. - power)))
657
+ lg.val2 = tmp * (1. - y_true / raw_prediction) # gradient
658
+ return lg
659
+
660
+
661
+ cdef inline double_pair cgrad_hess_half_tweedie_identity(
662
+ double y_true,
663
+ double raw_prediction,
664
+ double power
665
+ ) noexcept nogil:
666
+ cdef double_pair gh
667
+ cdef double tmp
668
+ if power == 0.:
669
+ gh.val1 = raw_prediction - y_true # gradient
670
+ gh.val2 = 1. # hessian
671
+ elif power == 1.:
672
+ gh.val1 = 1. - y_true / raw_prediction # gradient
673
+ gh.val2 = y_true / (raw_prediction * raw_prediction) # hessian
674
+ elif power == 2.:
675
+ tmp = raw_prediction * raw_prediction
676
+ gh.val1 = (raw_prediction - y_true) / tmp # gradient
677
+ gh.val2 = (-1. + 2. * y_true / raw_prediction) / tmp # hessian
678
+ else:
679
+ tmp = pow(raw_prediction, -power)
680
+ gh.val1 = tmp * (raw_prediction - y_true) # gradient
681
+ gh.val2 = tmp * ((1. - power) + power * y_true / raw_prediction) # hessian
682
+ return gh
683
+
684
+
685
+ # Half Binomial deviance with logit-link, aka log-loss or binary cross entropy
686
+ cdef inline double closs_half_binomial(
687
+ double y_true,
688
+ double raw_prediction
689
+ ) noexcept nogil:
690
+ # log1p(exp(raw_prediction)) - y_true * raw_prediction
691
+ return log1pexp(raw_prediction) - y_true * raw_prediction
692
+
693
+
694
+ cdef inline double cgradient_half_binomial(
695
+ double y_true,
696
+ double raw_prediction
697
+ ) noexcept nogil:
698
+ # gradient = y_pred - y_true = expit(raw_prediction) - y_true
699
+ # Numerically more stable, see http://fa.bianp.net/blog/2019/evaluate_logistic/
700
+ # if raw_prediction < 0:
701
+ # exp_tmp = exp(raw_prediction)
702
+ # return ((1 - y_true) * exp_tmp - y_true) / (1 + exp_tmp)
703
+ # else:
704
+ # exp_tmp = exp(-raw_prediction)
705
+ # return ((1 - y_true) - y_true * exp_tmp) / (1 + exp_tmp)
706
+ # Note that optimal speed would be achieved, at the cost of precision, by
707
+ # return expit(raw_prediction) - y_true
708
+ # i.e. no "if else" and an own inline implementation of expit instead of
709
+ # from scipy.special.cython_special cimport expit
710
+ # The case distinction raw_prediction < 0 in the stable implementation does not
711
+ # provide significant better precision apart from protecting overflow of exp(..).
712
+ # The branch (if else), however, can incur runtime costs of up to 30%.
713
+ # Instead, we help branch prediction by almost always ending in the first if clause
714
+ # and making the second branch (else) a bit simpler. This has the exact same
715
+ # precision but is faster than the stable implementation.
716
+ # As branching criteria, we use the same cutoff as in log1pexp. Note that the
717
+ # maximal value to get gradient = -1 with y_true = 1 is -37.439198610162731
718
+ # (based on mpmath), and scipy.special.logit(np.finfo(float).eps) ~ -36.04365.
719
+ cdef double exp_tmp
720
+ if raw_prediction > -37:
721
+ exp_tmp = exp(-raw_prediction)
722
+ return ((1 - y_true) - y_true * exp_tmp) / (1 + exp_tmp)
723
+ else:
724
+ # expit(raw_prediction) = exp(raw_prediction) for raw_prediction <= -37
725
+ return exp(raw_prediction) - y_true
726
+
727
+
728
+ cdef inline double_pair closs_grad_half_binomial(
729
+ double y_true,
730
+ double raw_prediction
731
+ ) noexcept nogil:
732
+ cdef double_pair lg
733
+ # Same if else conditions as in log1pexp.
734
+ if raw_prediction <= -37:
735
+ lg.val2 = exp(raw_prediction) # used as temporary
736
+ lg.val1 = lg.val2 - y_true * raw_prediction # loss
737
+ lg.val2 -= y_true # gradient
738
+ elif raw_prediction <= -2:
739
+ lg.val2 = exp(raw_prediction) # used as temporary
740
+ lg.val1 = log1p(lg.val2) - y_true * raw_prediction # loss
741
+ lg.val2 = ((1 - y_true) * lg.val2 - y_true) / (1 + lg.val2) # gradient
742
+ elif raw_prediction <= 18:
743
+ lg.val2 = exp(-raw_prediction) # used as temporary
744
+ # log1p(exp(x)) = log(1 + exp(x)) = x + log1p(exp(-x))
745
+ lg.val1 = log1p(lg.val2) + (1 - y_true) * raw_prediction # loss
746
+ lg.val2 = ((1 - y_true) - y_true * lg.val2) / (1 + lg.val2) # gradient
747
+ else:
748
+ lg.val2 = exp(-raw_prediction) # used as temporary
749
+ lg.val1 = lg.val2 + (1 - y_true) * raw_prediction # loss
750
+ lg.val2 = ((1 - y_true) - y_true * lg.val2) / (1 + lg.val2) # gradient
751
+ return lg
752
+
753
+
754
+ cdef inline double_pair cgrad_hess_half_binomial(
755
+ double y_true,
756
+ double raw_prediction
757
+ ) noexcept nogil:
758
+ # with y_pred = expit(raw)
759
+ # hessian = y_pred * (1 - y_pred) = exp( raw) / (1 + exp( raw))**2
760
+ # = exp(-raw) / (1 + exp(-raw))**2
761
+ cdef double_pair gh
762
+ # See comment in cgradient_half_binomial.
763
+ if raw_prediction > -37:
764
+ gh.val2 = exp(-raw_prediction) # used as temporary
765
+ gh.val1 = ((1 - y_true) - y_true * gh.val2) / (1 + gh.val2) # gradient
766
+ gh.val2 = gh.val2 / (1 + gh.val2)**2 # hessian
767
+ else:
768
+ gh.val2 = exp(raw_prediction) # = 1. order Taylor in exp(raw_prediction)
769
+ gh.val1 = gh.val2 - y_true
770
+ return gh
771
+
772
+
773
+ # Exponential loss with (half) logit-link, aka boosting loss
774
+ cdef inline double closs_exponential(
775
+ double y_true,
776
+ double raw_prediction
777
+ ) noexcept nogil:
778
+ cdef double tmp = exp(raw_prediction)
779
+ return y_true / tmp + (1 - y_true) * tmp
780
+
781
+
782
+ cdef inline double cgradient_exponential(
783
+ double y_true,
784
+ double raw_prediction
785
+ ) noexcept nogil:
786
+ cdef double tmp = exp(raw_prediction)
787
+ return -y_true / tmp + (1 - y_true) * tmp
788
+
789
+
790
+ cdef inline double_pair closs_grad_exponential(
791
+ double y_true,
792
+ double raw_prediction
793
+ ) noexcept nogil:
794
+ cdef double_pair lg
795
+ lg.val2 = exp(raw_prediction) # used as temporary
796
+
797
+ lg.val1 = y_true / lg.val2 + (1 - y_true) * lg.val2 # loss
798
+ lg.val2 = -y_true / lg.val2 + (1 - y_true) * lg.val2 # gradient
799
+ return lg
800
+
801
+
802
+ cdef inline double_pair cgrad_hess_exponential(
803
+ double y_true,
804
+ double raw_prediction
805
+ ) noexcept nogil:
806
+ # Note that hessian = loss
807
+ cdef double_pair gh
808
+ gh.val2 = exp(raw_prediction) # used as temporary
809
+
810
+ gh.val1 = -y_true / gh.val2 + (1 - y_true) * gh.val2 # gradient
811
+ gh.val2 = y_true / gh.val2 + (1 - y_true) * gh.val2 # hessian
812
+ return gh
813
+
814
+
815
+ # ---------------------------------------------------
816
+ # Extension Types for Loss Functions of 1-dim targets
817
+ # ---------------------------------------------------
818
+ cdef class CyLossFunction:
819
+ """Base class for convex loss functions."""
820
+
821
+ def __reduce__(self):
822
+ return (self.__class__, ())
823
+
824
+ cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil:
825
+ """Compute the loss for a single sample.
826
+
827
+ Parameters
828
+ ----------
829
+ y_true : double
830
+ Observed, true target value.
831
+ raw_prediction : double
832
+ Raw prediction value (in link space).
833
+
834
+ Returns
835
+ -------
836
+ double
837
+ The loss evaluated at `y_true` and `raw_prediction`.
838
+ """
839
+ pass
840
+
841
+ cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil:
842
+ """Compute gradient of loss w.r.t. raw_prediction for a single sample.
843
+
844
+ Parameters
845
+ ----------
846
+ y_true : double
847
+ Observed, true target value.
848
+ raw_prediction : double
849
+ Raw prediction value (in link space).
850
+
851
+ Returns
852
+ -------
853
+ double
854
+ The derivative of the loss function w.r.t. `raw_prediction`.
855
+ """
856
+ pass
857
+
858
+ cdef double_pair cy_grad_hess(
859
+ self, double y_true, double raw_prediction
860
+ ) noexcept nogil:
861
+ """Compute gradient and hessian.
862
+
863
+ Gradient and hessian of loss w.r.t. raw_prediction for a single sample.
864
+
865
+ This is usually diagonal in raw_prediction_i and raw_prediction_j.
866
+ Therefore, we return the diagonal element i=j.
867
+
868
+ For a loss with a non-canonical link, this might implement the diagonal
869
+ of the Fisher matrix (=expected hessian) instead of the hessian.
870
+
871
+ Parameters
872
+ ----------
873
+ y_true : double
874
+ Observed, true target value.
875
+ raw_prediction : double
876
+ Raw prediction value (in link space).
877
+
878
+ Returns
879
+ -------
880
+ double_pair
881
+ Gradient and hessian of the loss function w.r.t. `raw_prediction`.
882
+ """
883
+ pass
884
+
885
+ def loss(
886
+ self,
887
+ const floating_in[::1] y_true, # IN
888
+ const floating_in[::1] raw_prediction, # IN
889
+ const floating_in[::1] sample_weight, # IN
890
+ floating_out[::1] loss_out, # OUT
891
+ int n_threads=1
892
+ ):
893
+ """Compute the point-wise loss value for each input.
894
+
895
+ The point-wise loss is written to `loss_out` and no array is returned.
896
+
897
+ Parameters
898
+ ----------
899
+ y_true : array of shape (n_samples,)
900
+ Observed, true target values.
901
+ raw_prediction : array of shape (n_samples,)
902
+ Raw prediction values (in link space).
903
+ sample_weight : array of shape (n_samples,) or None
904
+ Sample weights.
905
+ loss_out : array of shape (n_samples,)
906
+ A location into which the result is stored.
907
+ n_threads : int
908
+ Number of threads used by OpenMP (if any).
909
+ """
910
+ pass
911
+
912
+ def gradient(
913
+ self,
914
+ const floating_in[::1] y_true, # IN
915
+ const floating_in[::1] raw_prediction, # IN
916
+ const floating_in[::1] sample_weight, # IN
917
+ floating_out[::1] gradient_out, # OUT
918
+ int n_threads=1
919
+ ):
920
+ """Compute gradient of loss w.r.t raw_prediction for each input.
921
+
922
+ The gradient is written to `gradient_out` and no array is returned.
923
+
924
+ Parameters
925
+ ----------
926
+ y_true : array of shape (n_samples,)
927
+ Observed, true target values.
928
+ raw_prediction : array of shape (n_samples,)
929
+ Raw prediction values (in link space).
930
+ sample_weight : array of shape (n_samples,) or None
931
+ Sample weights.
932
+ gradient_out : array of shape (n_samples,)
933
+ A location into which the result is stored.
934
+ n_threads : int
935
+ Number of threads used by OpenMP (if any).
936
+ """
937
+ pass
938
+
939
+ def loss_gradient(
940
+ self,
941
+ const floating_in[::1] y_true, # IN
942
+ const floating_in[::1] raw_prediction, # IN
943
+ const floating_in[::1] sample_weight, # IN
944
+ floating_out[::1] loss_out, # OUT
945
+ floating_out[::1] gradient_out, # OUT
946
+ int n_threads=1
947
+ ):
948
+ """Compute loss and gradient of loss w.r.t raw_prediction.
949
+
950
+ The loss and gradient are written to `loss_out` and `gradient_out` and no arrays
951
+ are returned.
952
+
953
+ Parameters
954
+ ----------
955
+ y_true : array of shape (n_samples,)
956
+ Observed, true target values.
957
+ raw_prediction : array of shape (n_samples,)
958
+ Raw prediction values (in link space).
959
+ sample_weight : array of shape (n_samples,) or None
960
+ Sample weights.
961
+ loss_out : array of shape (n_samples,) or None
962
+ A location into which the element-wise loss is stored.
963
+ gradient_out : array of shape (n_samples,)
964
+ A location into which the gradient is stored.
965
+ n_threads : int
966
+ Number of threads used by OpenMP (if any).
967
+ """
968
+ self.loss(y_true, raw_prediction, sample_weight, loss_out, n_threads)
969
+ self.gradient(y_true, raw_prediction, sample_weight, gradient_out, n_threads)
970
+
971
+ def gradient_hessian(
972
+ self,
973
+ const floating_in[::1] y_true, # IN
974
+ const floating_in[::1] raw_prediction, # IN
975
+ const floating_in[::1] sample_weight, # IN
976
+ floating_out[::1] gradient_out, # OUT
977
+ floating_out[::1] hessian_out, # OUT
978
+ int n_threads=1
979
+ ):
980
+ """Compute gradient and hessian of loss w.r.t raw_prediction.
981
+
982
+ The gradient and hessian are written to `gradient_out` and `hessian_out` and no
983
+ arrays are returned.
984
+
985
+ Parameters
986
+ ----------
987
+ y_true : array of shape (n_samples,)
988
+ Observed, true target values.
989
+ raw_prediction : array of shape (n_samples,)
990
+ Raw prediction values (in link space).
991
+ sample_weight : array of shape (n_samples,) or None
992
+ Sample weights.
993
+ gradient_out : array of shape (n_samples,)
994
+ A location into which the gradient is stored.
995
+ hessian_out : array of shape (n_samples,)
996
+ A location into which the hessian is stored.
997
+ n_threads : int
998
+ Number of threads used by OpenMP (if any).
999
+ """
1000
+ pass
1001
+
1002
+
1003
+ {{for name, docstring, param, closs, closs_grad, cgrad, cgrad_hess, in class_list}}
1004
+ {{py:
1005
+ if param is None:
1006
+ with_param = ""
1007
+ else:
1008
+ with_param = ", self." + param
1009
+ }}
1010
+
1011
+ cdef class {{name}}(CyLossFunction):
1012
+ """{{docstring}}"""
1013
+
1014
+ {{if param is not None}}
1015
+ def __init__(self, {{param}}):
1016
+ self.{{param}} = {{param}}
1017
+ {{endif}}
1018
+
1019
+ {{if param is not None}}
1020
+ def __reduce__(self):
1021
+ return (self.__class__, (self.{{param}},))
1022
+ {{endif}}
1023
+
1024
+ cdef inline double cy_loss(self, double y_true, double raw_prediction) noexcept nogil:
1025
+ return {{closs}}(y_true, raw_prediction{{with_param}})
1026
+
1027
+ cdef inline double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil:
1028
+ return {{cgrad}}(y_true, raw_prediction{{with_param}})
1029
+
1030
+ cdef inline double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil:
1031
+ return {{cgrad_hess}}(y_true, raw_prediction{{with_param}})
1032
+
1033
+ def loss(
1034
+ self,
1035
+ const floating_in[::1] y_true, # IN
1036
+ const floating_in[::1] raw_prediction, # IN
1037
+ const floating_in[::1] sample_weight, # IN
1038
+ floating_out[::1] loss_out, # OUT
1039
+ int n_threads=1
1040
+ ):
1041
+ cdef:
1042
+ int i
1043
+ int n_samples = y_true.shape[0]
1044
+
1045
+ if sample_weight is None:
1046
+ for i in prange(
1047
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1048
+ ):
1049
+ loss_out[i] = {{closs}}(y_true[i], raw_prediction[i]{{with_param}})
1050
+ else:
1051
+ for i in prange(
1052
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1053
+ ):
1054
+ loss_out[i] = sample_weight[i] * {{closs}}(y_true[i], raw_prediction[i]{{with_param}})
1055
+
1056
+ {{if closs_grad is not None}}
1057
+ def loss_gradient(
1058
+ self,
1059
+ const floating_in[::1] y_true, # IN
1060
+ const floating_in[::1] raw_prediction, # IN
1061
+ const floating_in[::1] sample_weight, # IN
1062
+ floating_out[::1] loss_out, # OUT
1063
+ floating_out[::1] gradient_out, # OUT
1064
+ int n_threads=1
1065
+ ):
1066
+ cdef:
1067
+ int i
1068
+ int n_samples = y_true.shape[0]
1069
+ double_pair dbl2
1070
+
1071
+ if sample_weight is None:
1072
+ for i in prange(
1073
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1074
+ ):
1075
+ dbl2 = {{closs_grad}}(y_true[i], raw_prediction[i]{{with_param}})
1076
+ loss_out[i] = dbl2.val1
1077
+ gradient_out[i] = dbl2.val2
1078
+ else:
1079
+ for i in prange(
1080
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1081
+ ):
1082
+ dbl2 = {{closs_grad}}(y_true[i], raw_prediction[i]{{with_param}})
1083
+ loss_out[i] = sample_weight[i] * dbl2.val1
1084
+ gradient_out[i] = sample_weight[i] * dbl2.val2
1085
+
1086
+ {{endif}}
1087
+
1088
+ def gradient(
1089
+ self,
1090
+ const floating_in[::1] y_true, # IN
1091
+ const floating_in[::1] raw_prediction, # IN
1092
+ const floating_in[::1] sample_weight, # IN
1093
+ floating_out[::1] gradient_out, # OUT
1094
+ int n_threads=1
1095
+ ):
1096
+ cdef:
1097
+ int i
1098
+ int n_samples = y_true.shape[0]
1099
+
1100
+ if sample_weight is None:
1101
+ for i in prange(
1102
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1103
+ ):
1104
+ gradient_out[i] = {{cgrad}}(y_true[i], raw_prediction[i]{{with_param}})
1105
+ else:
1106
+ for i in prange(
1107
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1108
+ ):
1109
+ gradient_out[i] = sample_weight[i] * {{cgrad}}(y_true[i], raw_prediction[i]{{with_param}})
1110
+
1111
+ def gradient_hessian(
1112
+ self,
1113
+ const floating_in[::1] y_true, # IN
1114
+ const floating_in[::1] raw_prediction, # IN
1115
+ const floating_in[::1] sample_weight, # IN
1116
+ floating_out[::1] gradient_out, # OUT
1117
+ floating_out[::1] hessian_out, # OUT
1118
+ int n_threads=1
1119
+ ):
1120
+ cdef:
1121
+ int i
1122
+ int n_samples = y_true.shape[0]
1123
+ double_pair dbl2
1124
+
1125
+ if sample_weight is None:
1126
+ for i in prange(
1127
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1128
+ ):
1129
+ dbl2 = {{cgrad_hess}}(y_true[i], raw_prediction[i]{{with_param}})
1130
+ gradient_out[i] = dbl2.val1
1131
+ hessian_out[i] = dbl2.val2
1132
+ else:
1133
+ for i in prange(
1134
+ n_samples, schedule='static', nogil=True, num_threads=n_threads
1135
+ ):
1136
+ dbl2 = {{cgrad_hess}}(y_true[i], raw_prediction[i]{{with_param}})
1137
+ gradient_out[i] = sample_weight[i] * dbl2.val1
1138
+ hessian_out[i] = sample_weight[i] * dbl2.val2
1139
+
1140
+ {{endfor}}
1141
+
1142
+
1143
+ # The multinomial deviance loss is also known as categorical cross-entropy or
1144
+ # multinomial log-likelihood.
1145
+ # Here, we do not inherit from CyLossFunction as its cy_gradient method deviates
1146
+ # from the API.
1147
+ cdef class CyHalfMultinomialLoss():
1148
+ """Half Multinomial deviance loss with multinomial logit link.
1149
+
1150
+ Domain:
1151
+ y_true in {0, 1, 2, 3, .., n_classes - 1}
1152
+ y_pred in (0, 1)**n_classes, i.e. interval with boundaries excluded
1153
+
1154
+ Link:
1155
+ y_pred = softmax(raw_prediction)
1156
+
1157
+ Note: Label encoding is built-in, i.e. {0, 1, 2, 3, .., n_classes - 1} is
1158
+ mapped to (y_true == k) for k = 0 .. n_classes - 1 which is either 0 or 1.
1159
+ """
1160
+
1161
+ # Here we deviate from the CyLossFunction API. SAG/SAGA needs direct access to
1162
+ # sample-wise gradients which we provide here.
1163
+ cdef inline void cy_gradient(
1164
+ self,
1165
+ const floating_in y_true,
1166
+ const floating_in[::1] raw_prediction, # IN
1167
+ const floating_in sample_weight,
1168
+ floating_out[::1] gradient_out, # OUT
1169
+ ) noexcept nogil:
1170
+ """Compute gradient of loss w.r.t. `raw_prediction` for a single sample.
1171
+
1172
+ The gradient of the multinomial logistic loss with respect to a class k,
1173
+ and for one sample is:
1174
+ grad_k = - sw * (p[k] - (y==k))
1175
+
1176
+ where:
1177
+ p[k] = proba[k] = exp(raw_prediction[k] - logsumexp(raw_prediction))
1178
+ sw = sample_weight
1179
+
1180
+ Parameters
1181
+ ----------
1182
+ y_true : double
1183
+ Observed, true target value.
1184
+ raw_prediction : array of shape (n_classes,)
1185
+ Raw prediction values (in link space).
1186
+ sample_weight : double
1187
+ Sample weight.
1188
+ gradient_out : array of shape (n_classs,)
1189
+ A location into which the gradient is stored.
1190
+
1191
+ Returns
1192
+ -------
1193
+ gradient : double
1194
+ The derivative of the loss function w.r.t. `raw_prediction`.
1195
+ """
1196
+ cdef:
1197
+ int k
1198
+ int n_classes = raw_prediction.shape[0]
1199
+ double_pair max_value_and_sum_exps
1200
+ const floating_in[:, :] raw = raw_prediction[None, :]
1201
+
1202
+ max_value_and_sum_exps = sum_exp_minus_max(0, raw, &gradient_out[0])
1203
+ for k in range(n_classes):
1204
+ # gradient_out[k] = p_k = y_pred_k = prob of class k
1205
+ gradient_out[k] /= max_value_and_sum_exps.val2
1206
+ # gradient_k = (p_k - (y_true == k)) * sw
1207
+ gradient_out[k] = (gradient_out[k] - (y_true == k)) * sample_weight
1208
+
1209
+ def _test_cy_gradient(
1210
+ self,
1211
+ const floating_in[::1] y_true, # IN
1212
+ const floating_in[:, ::1] raw_prediction, # IN
1213
+ const floating_in[::1] sample_weight, # IN
1214
+ ):
1215
+ """For testing only."""
1216
+ cdef:
1217
+ int i, k
1218
+ int n_samples = y_true.shape[0]
1219
+ int n_classes = raw_prediction.shape[1]
1220
+ floating_in [:, ::1] gradient_out
1221
+ gradient = np.empty((n_samples, n_classes), dtype=np.float64)
1222
+ gradient_out = gradient
1223
+
1224
+ for i in range(n_samples):
1225
+ self.cy_gradient(
1226
+ y_true=y_true[i],
1227
+ raw_prediction=raw_prediction[i, :],
1228
+ sample_weight=1.0 if sample_weight is None else sample_weight[i],
1229
+ gradient_out=gradient_out[i, :],
1230
+ )
1231
+ return gradient
1232
+
1233
+ # Note that we do not assume memory alignment/contiguity of 2d arrays.
1234
+ # There seems to be little benefit in doing so. Benchmarks proofing the
1235
+ # opposite are welcome.
1236
+ def loss(
1237
+ self,
1238
+ const floating_in[::1] y_true, # IN
1239
+ const floating_in[:, :] raw_prediction, # IN
1240
+ const floating_in[::1] sample_weight, # IN
1241
+ floating_out[::1] loss_out, # OUT
1242
+ int n_threads=1
1243
+ ):
1244
+ cdef:
1245
+ int i, k
1246
+ int n_samples = y_true.shape[0]
1247
+ int n_classes = raw_prediction.shape[1]
1248
+ floating_in max_value, sum_exps
1249
+ floating_in* p # temporary buffer
1250
+ double_pair max_value_and_sum_exps
1251
+
1252
+ # We assume n_samples > n_classes. In this case having the inner loop
1253
+ # over n_classes is a good default.
1254
+ # TODO: If every memoryview is contiguous and raw_prediction is
1255
+ # f-contiguous, can we write a better algo (loops) to improve
1256
+ # performance?
1257
+ if sample_weight is None:
1258
+ # inner loop over n_classes
1259
+ with nogil, parallel(num_threads=n_threads):
1260
+ # Define private buffer variables as each thread might use its
1261
+ # own.
1262
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1263
+
1264
+ for i in prange(n_samples, schedule='static'):
1265
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1266
+ max_value = max_value_and_sum_exps.val1
1267
+ sum_exps = max_value_and_sum_exps.val2
1268
+ loss_out[i] = log(sum_exps) + max_value
1269
+
1270
+ # label encoded y_true
1271
+ k = int(y_true[i])
1272
+ loss_out[i] -= raw_prediction[i, k]
1273
+
1274
+ free(p)
1275
+ else:
1276
+ with nogil, parallel(num_threads=n_threads):
1277
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1278
+
1279
+ for i in prange(n_samples, schedule='static'):
1280
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1281
+ max_value = max_value_and_sum_exps.val1
1282
+ sum_exps = max_value_and_sum_exps.val2
1283
+ loss_out[i] = log(sum_exps) + max_value
1284
+
1285
+ # label encoded y_true
1286
+ k = int(y_true[i])
1287
+ loss_out[i] -= raw_prediction[i, k]
1288
+
1289
+ loss_out[i] *= sample_weight[i]
1290
+
1291
+ free(p)
1292
+
1293
+ def loss_gradient(
1294
+ self,
1295
+ const floating_in[::1] y_true, # IN
1296
+ const floating_in[:, :] raw_prediction, # IN
1297
+ const floating_in[::1] sample_weight, # IN
1298
+ floating_out[::1] loss_out, # OUT
1299
+ floating_out[:, :] gradient_out, # OUT
1300
+ int n_threads=1
1301
+ ):
1302
+ cdef:
1303
+ int i, k
1304
+ int n_samples = y_true.shape[0]
1305
+ int n_classes = raw_prediction.shape[1]
1306
+ floating_in max_value, sum_exps
1307
+ floating_in* p # temporary buffer
1308
+ double_pair max_value_and_sum_exps
1309
+
1310
+ if sample_weight is None:
1311
+ # inner loop over n_classes
1312
+ with nogil, parallel(num_threads=n_threads):
1313
+ # Define private buffer variables as each thread might use its
1314
+ # own.
1315
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1316
+
1317
+ for i in prange(n_samples, schedule='static'):
1318
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1319
+ max_value = max_value_and_sum_exps.val1
1320
+ sum_exps = max_value_and_sum_exps.val2
1321
+ loss_out[i] = log(sum_exps) + max_value
1322
+
1323
+ for k in range(n_classes):
1324
+ # label decode y_true
1325
+ if y_true[i] == k:
1326
+ loss_out[i] -= raw_prediction[i, k]
1327
+ p[k] /= sum_exps # p_k = y_pred_k = prob of class k
1328
+ # gradient_k = p_k - (y_true == k)
1329
+ gradient_out[i, k] = p[k] - (y_true[i] == k)
1330
+
1331
+ free(p)
1332
+ else:
1333
+ with nogil, parallel(num_threads=n_threads):
1334
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1335
+
1336
+ for i in prange(n_samples, schedule='static'):
1337
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1338
+ max_value = max_value_and_sum_exps.val1
1339
+ sum_exps = max_value_and_sum_exps.val2
1340
+ loss_out[i] = log(sum_exps) + max_value
1341
+
1342
+ for k in range(n_classes):
1343
+ # label decode y_true
1344
+ if y_true[i] == k:
1345
+ loss_out[i] -= raw_prediction[i, k]
1346
+ p[k] /= sum_exps # p_k = y_pred_k = prob of class k
1347
+ # gradient_k = (p_k - (y_true == k)) * sw
1348
+ gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i]
1349
+
1350
+ loss_out[i] *= sample_weight[i]
1351
+
1352
+ free(p)
1353
+
1354
+ def gradient(
1355
+ self,
1356
+ const floating_in[::1] y_true, # IN
1357
+ const floating_in[:, :] raw_prediction, # IN
1358
+ const floating_in[::1] sample_weight, # IN
1359
+ floating_out[:, :] gradient_out, # OUT
1360
+ int n_threads=1
1361
+ ):
1362
+ cdef:
1363
+ int i, k
1364
+ int n_samples = y_true.shape[0]
1365
+ int n_classes = raw_prediction.shape[1]
1366
+ floating_in sum_exps
1367
+ floating_in* p # temporary buffer
1368
+ double_pair max_value_and_sum_exps
1369
+
1370
+ if sample_weight is None:
1371
+ # inner loop over n_classes
1372
+ with nogil, parallel(num_threads=n_threads):
1373
+ # Define private buffer variables as each thread might use its
1374
+ # own.
1375
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1376
+
1377
+ for i in prange(n_samples, schedule='static'):
1378
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1379
+ sum_exps = max_value_and_sum_exps.val2
1380
+
1381
+ for k in range(n_classes):
1382
+ p[k] /= sum_exps # p_k = y_pred_k = prob of class k
1383
+ # gradient_k = y_pred_k - (y_true == k)
1384
+ gradient_out[i, k] = p[k] - (y_true[i] == k)
1385
+
1386
+ free(p)
1387
+ else:
1388
+ with nogil, parallel(num_threads=n_threads):
1389
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1390
+
1391
+ for i in prange(n_samples, schedule='static'):
1392
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1393
+ sum_exps = max_value_and_sum_exps.val2
1394
+
1395
+ for k in range(n_classes):
1396
+ p[k] /= sum_exps # p_k = y_pred_k = prob of class k
1397
+ # gradient_k = (p_k - (y_true == k)) * sw
1398
+ gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i]
1399
+
1400
+ free(p)
1401
+
1402
+ def gradient_hessian(
1403
+ self,
1404
+ const floating_in[::1] y_true, # IN
1405
+ const floating_in[:, :] raw_prediction, # IN
1406
+ const floating_in[::1] sample_weight, # IN
1407
+ floating_out[:, :] gradient_out, # OUT
1408
+ floating_out[:, :] hessian_out, # OUT
1409
+ int n_threads=1
1410
+ ):
1411
+ cdef:
1412
+ int i, k
1413
+ int n_samples = y_true.shape[0]
1414
+ int n_classes = raw_prediction.shape[1]
1415
+ floating_in sum_exps
1416
+ floating_in* p # temporary buffer
1417
+ double_pair max_value_and_sum_exps
1418
+
1419
+ if sample_weight is None:
1420
+ # inner loop over n_classes
1421
+ with nogil, parallel(num_threads=n_threads):
1422
+ # Define private buffer variables as each thread might use its
1423
+ # own.
1424
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1425
+
1426
+ for i in prange(n_samples, schedule='static'):
1427
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1428
+ sum_exps = max_value_and_sum_exps.val2
1429
+
1430
+ for k in range(n_classes):
1431
+ p[k] /= sum_exps # p_k = y_pred_k = prob of class k
1432
+ # hessian_k = p_k * (1 - p_k)
1433
+ # gradient_k = p_k - (y_true == k)
1434
+ gradient_out[i, k] = p[k] - (y_true[i] == k)
1435
+ hessian_out[i, k] = p[k] * (1. - p[k])
1436
+
1437
+ free(p)
1438
+ else:
1439
+ with nogil, parallel(num_threads=n_threads):
1440
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1441
+
1442
+ for i in prange(n_samples, schedule='static'):
1443
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1444
+ sum_exps = max_value_and_sum_exps.val2
1445
+
1446
+ for k in range(n_classes):
1447
+ p[k] /= sum_exps # p_k = y_pred_k = prob of class k
1448
+ # gradient_k = (p_k - (y_true == k)) * sw
1449
+ # hessian_k = p_k * (1 - p_k) * sw
1450
+ gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i]
1451
+ hessian_out[i, k] = (p[k] * (1. - p[k])) * sample_weight[i]
1452
+
1453
+ free(p)
1454
+
1455
+ # This method simplifies the implementation of hessp in linear models,
1456
+ # i.e. the matrix-vector product of the full hessian, not only of the
1457
+ # diagonal (in the classes) approximation as implemented above.
1458
+ def gradient_proba(
1459
+ self,
1460
+ const floating_in[::1] y_true, # IN
1461
+ const floating_in[:, :] raw_prediction, # IN
1462
+ const floating_in[::1] sample_weight, # IN
1463
+ floating_out[:, :] gradient_out, # OUT
1464
+ floating_out[:, :] proba_out, # OUT
1465
+ int n_threads=1
1466
+ ):
1467
+ cdef:
1468
+ int i, k
1469
+ int n_samples = y_true.shape[0]
1470
+ int n_classes = raw_prediction.shape[1]
1471
+ floating_in sum_exps
1472
+ floating_in* p # temporary buffer
1473
+ double_pair max_value_and_sum_exps
1474
+
1475
+ if sample_weight is None:
1476
+ # inner loop over n_classes
1477
+ with nogil, parallel(num_threads=n_threads):
1478
+ # Define private buffer variables as each thread might use its
1479
+ # own.
1480
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1481
+
1482
+ for i in prange(n_samples, schedule='static'):
1483
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1484
+ sum_exps = max_value_and_sum_exps.val2
1485
+
1486
+ for k in range(n_classes):
1487
+ proba_out[i, k] = p[k] / sum_exps # y_pred_k = prob of class k
1488
+ # gradient_k = y_pred_k - (y_true == k)
1489
+ gradient_out[i, k] = proba_out[i, k] - (y_true[i] == k)
1490
+
1491
+ free(p)
1492
+ else:
1493
+ with nogil, parallel(num_threads=n_threads):
1494
+ p = <floating_in *> malloc(sizeof(floating_in) * (n_classes))
1495
+
1496
+ for i in prange(n_samples, schedule='static'):
1497
+ max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p)
1498
+ sum_exps = max_value_and_sum_exps.val2
1499
+
1500
+ for k in range(n_classes):
1501
+ proba_out[i, k] = p[k] / sum_exps # y_pred_k = prob of class k
1502
+ # gradient_k = (p_k - (y_true == k)) * sw
1503
+ gradient_out[i, k] = (proba_out[i, k] - (y_true[i] == k)) * sample_weight[i]
1504
+
1505
+ free(p)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/link.py ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Module contains classes for invertible (and differentiable) link functions.
3
+ """
4
+
5
+ # Authors: The scikit-learn developers
6
+ # SPDX-License-Identifier: BSD-3-Clause
7
+
8
+ from abc import ABC, abstractmethod
9
+ from dataclasses import dataclass
10
+
11
+ import numpy as np
12
+ from scipy.special import expit, logit
13
+ from scipy.stats import gmean
14
+
15
+ from sklearn.utils.extmath import softmax
16
+
17
+
18
+ @dataclass
19
+ class Interval:
20
+ low: float
21
+ high: float
22
+ low_inclusive: bool
23
+ high_inclusive: bool
24
+
25
+ def __post_init__(self):
26
+ """Check that low <= high"""
27
+ if self.low > self.high:
28
+ raise ValueError(
29
+ f"One must have low <= high; got low={self.low}, high={self.high}."
30
+ )
31
+
32
+ def includes(self, x):
33
+ """Test whether all values of x are in interval range.
34
+
35
+ Parameters
36
+ ----------
37
+ x : ndarray
38
+ Array whose elements are tested to be in interval range.
39
+
40
+ Returns
41
+ -------
42
+ result : bool
43
+ """
44
+ if self.low_inclusive:
45
+ low = np.greater_equal(x, self.low)
46
+ else:
47
+ low = np.greater(x, self.low)
48
+
49
+ if not np.all(low):
50
+ return False
51
+
52
+ if self.high_inclusive:
53
+ high = np.less_equal(x, self.high)
54
+ else:
55
+ high = np.less(x, self.high)
56
+
57
+ # Note: np.all returns numpy.bool_
58
+ return bool(np.all(high))
59
+
60
+
61
+ def _inclusive_low_high(interval, dtype=np.float64):
62
+ """Generate values low and high to be within the interval range.
63
+
64
+ This is used in tests only.
65
+
66
+ Returns
67
+ -------
68
+ low, high : tuple
69
+ The returned values low and high lie within the interval.
70
+ """
71
+ eps = 10 * np.finfo(dtype).eps
72
+ if interval.low == -np.inf:
73
+ low = -1e10
74
+ elif interval.low < 0:
75
+ low = interval.low * (1 - eps) + eps
76
+ else:
77
+ low = interval.low * (1 + eps) + eps
78
+
79
+ if interval.high == np.inf:
80
+ high = 1e10
81
+ elif interval.high < 0:
82
+ high = interval.high * (1 + eps) - eps
83
+ else:
84
+ high = interval.high * (1 - eps) - eps
85
+
86
+ return low, high
87
+
88
+
89
+ class BaseLink(ABC):
90
+ """Abstract base class for differentiable, invertible link functions.
91
+
92
+ Convention:
93
+ - link function g: raw_prediction = g(y_pred)
94
+ - inverse link h: y_pred = h(raw_prediction)
95
+
96
+ For (generalized) linear models, `raw_prediction = X @ coef` is the so
97
+ called linear predictor, and `y_pred = h(raw_prediction)` is the predicted
98
+ conditional (on X) expected value of the target `y_true`.
99
+
100
+ The methods are not implemented as staticmethods in case a link function needs
101
+ parameters.
102
+ """
103
+
104
+ is_multiclass = False # used for testing only
105
+
106
+ # Usually, raw_prediction may be any real number and y_pred is an open
107
+ # interval.
108
+ # interval_raw_prediction = Interval(-np.inf, np.inf, False, False)
109
+ interval_y_pred = Interval(-np.inf, np.inf, False, False)
110
+
111
+ @abstractmethod
112
+ def link(self, y_pred, out=None):
113
+ """Compute the link function g(y_pred).
114
+
115
+ The link function maps (predicted) target values to raw predictions,
116
+ i.e. `g(y_pred) = raw_prediction`.
117
+
118
+ Parameters
119
+ ----------
120
+ y_pred : array
121
+ Predicted target values.
122
+ out : array
123
+ A location into which the result is stored. If provided, it must
124
+ have a shape that the inputs broadcast to. If not provided or None,
125
+ a freshly-allocated array is returned.
126
+
127
+ Returns
128
+ -------
129
+ out : array
130
+ Output array, element-wise link function.
131
+ """
132
+
133
+ @abstractmethod
134
+ def inverse(self, raw_prediction, out=None):
135
+ """Compute the inverse link function h(raw_prediction).
136
+
137
+ The inverse link function maps raw predictions to predicted target
138
+ values, i.e. `h(raw_prediction) = y_pred`.
139
+
140
+ Parameters
141
+ ----------
142
+ raw_prediction : array
143
+ Raw prediction values (in link space).
144
+ out : array
145
+ A location into which the result is stored. If provided, it must
146
+ have a shape that the inputs broadcast to. If not provided or None,
147
+ a freshly-allocated array is returned.
148
+
149
+ Returns
150
+ -------
151
+ out : array
152
+ Output array, element-wise inverse link function.
153
+ """
154
+
155
+
156
+ class IdentityLink(BaseLink):
157
+ """The identity link function g(x)=x."""
158
+
159
+ def link(self, y_pred, out=None):
160
+ if out is not None:
161
+ np.copyto(out, y_pred)
162
+ return out
163
+ else:
164
+ return y_pred
165
+
166
+ inverse = link
167
+
168
+
169
+ class LogLink(BaseLink):
170
+ """The log link function g(x)=log(x)."""
171
+
172
+ interval_y_pred = Interval(0, np.inf, False, False)
173
+
174
+ def link(self, y_pred, out=None):
175
+ return np.log(y_pred, out=out)
176
+
177
+ def inverse(self, raw_prediction, out=None):
178
+ return np.exp(raw_prediction, out=out)
179
+
180
+
181
+ class LogitLink(BaseLink):
182
+ """The logit link function g(x)=logit(x)."""
183
+
184
+ interval_y_pred = Interval(0, 1, False, False)
185
+
186
+ def link(self, y_pred, out=None):
187
+ return logit(y_pred, out=out)
188
+
189
+ def inverse(self, raw_prediction, out=None):
190
+ return expit(raw_prediction, out=out)
191
+
192
+
193
+ class HalfLogitLink(BaseLink):
194
+ """Half the logit link function g(x)=1/2 * logit(x).
195
+
196
+ Used for the exponential loss.
197
+ """
198
+
199
+ interval_y_pred = Interval(0, 1, False, False)
200
+
201
+ def link(self, y_pred, out=None):
202
+ out = logit(y_pred, out=out)
203
+ out *= 0.5
204
+ return out
205
+
206
+ def inverse(self, raw_prediction, out=None):
207
+ return expit(2 * raw_prediction, out)
208
+
209
+
210
+ class MultinomialLogit(BaseLink):
211
+ """The symmetric multinomial logit function.
212
+
213
+ Convention:
214
+ - y_pred.shape = raw_prediction.shape = (n_samples, n_classes)
215
+
216
+ Notes:
217
+ - The inverse link h is the softmax function.
218
+ - The sum is over the second axis, i.e. axis=1 (n_classes).
219
+
220
+ We have to choose additional constraints in order to make
221
+
222
+ y_pred[k] = exp(raw_pred[k]) / sum(exp(raw_pred[k]), k=0..n_classes-1)
223
+
224
+ for n_classes classes identifiable and invertible.
225
+ We choose the symmetric side constraint where the geometric mean response
226
+ is set as reference category, see [2]:
227
+
228
+ The symmetric multinomial logit link function for a single data point is
229
+ then defined as
230
+
231
+ raw_prediction[k] = g(y_pred[k]) = log(y_pred[k]/gmean(y_pred))
232
+ = log(y_pred[k]) - mean(log(y_pred)).
233
+
234
+ Note that this is equivalent to the definition in [1] and implies mean
235
+ centered raw predictions:
236
+
237
+ sum(raw_prediction[k], k=0..n_classes-1) = 0.
238
+
239
+ For linear models with raw_prediction = X @ coef, this corresponds to
240
+ sum(coef[k], k=0..n_classes-1) = 0, i.e. the sum over classes for every
241
+ feature is zero.
242
+
243
+ Reference
244
+ ---------
245
+ .. [1] Friedman, Jerome; Hastie, Trevor; Tibshirani, Robert. "Additive
246
+ logistic regression: a statistical view of boosting" Ann. Statist.
247
+ 28 (2000), no. 2, 337--407. doi:10.1214/aos/1016218223.
248
+ https://projecteuclid.org/euclid.aos/1016218223
249
+
250
+ .. [2] Zahid, Faisal Maqbool and Gerhard Tutz. "Ridge estimation for
251
+ multinomial logit models with symmetric side constraints."
252
+ Computational Statistics 28 (2013): 1017-1034.
253
+ http://epub.ub.uni-muenchen.de/11001/1/tr067.pdf
254
+ """
255
+
256
+ is_multiclass = True
257
+ interval_y_pred = Interval(0, 1, False, False)
258
+
259
+ def symmetrize_raw_prediction(self, raw_prediction):
260
+ return raw_prediction - np.mean(raw_prediction, axis=1)[:, np.newaxis]
261
+
262
+ def link(self, y_pred, out=None):
263
+ # geometric mean as reference category
264
+ gm = gmean(y_pred, axis=1)
265
+ return np.log(y_pred / gm[:, np.newaxis], out=out)
266
+
267
+ def inverse(self, raw_prediction, out=None):
268
+ if out is None:
269
+ return softmax(raw_prediction, copy=True)
270
+ else:
271
+ np.copyto(out, raw_prediction)
272
+ softmax(out, copy=False)
273
+ return out
274
+
275
+
276
+ _LINKS = {
277
+ "identity": IdentityLink,
278
+ "log": LogLink,
279
+ "logit": LogitLink,
280
+ "half_logit": HalfLogitLink,
281
+ "multinomial_logit": MultinomialLogit,
282
+ }
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/loss.py ADDED
@@ -0,0 +1,1216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ This module contains loss classes suitable for fitting.
3
+
4
+ It is not part of the public API.
5
+ Specific losses are used for regression, binary classification or multiclass
6
+ classification.
7
+ """
8
+
9
+ # Authors: The scikit-learn developers
10
+ # SPDX-License-Identifier: BSD-3-Clause
11
+
12
+ # Goals:
13
+ # - Provide a common private module for loss functions/classes.
14
+ # - To be used in:
15
+ # - LogisticRegression
16
+ # - PoissonRegressor, GammaRegressor, TweedieRegressor
17
+ # - HistGradientBoostingRegressor, HistGradientBoostingClassifier
18
+ # - GradientBoostingRegressor, GradientBoostingClassifier
19
+ # - SGDRegressor, SGDClassifier
20
+ # - Replace link module of GLMs.
21
+
22
+ import numbers
23
+
24
+ import numpy as np
25
+ from scipy.special import xlogy
26
+
27
+ from sklearn._loss._loss import (
28
+ CyAbsoluteError,
29
+ CyExponentialLoss,
30
+ CyHalfBinomialLoss,
31
+ CyHalfGammaLoss,
32
+ CyHalfMultinomialLoss,
33
+ CyHalfPoissonLoss,
34
+ CyHalfSquaredError,
35
+ CyHalfTweedieLoss,
36
+ CyHalfTweedieLossIdentity,
37
+ CyHuberLoss,
38
+ CyPinballLoss,
39
+ )
40
+ from sklearn._loss.link import (
41
+ HalfLogitLink,
42
+ IdentityLink,
43
+ Interval,
44
+ LogitLink,
45
+ LogLink,
46
+ MultinomialLogit,
47
+ )
48
+ from sklearn.utils import check_scalar
49
+ from sklearn.utils.stats import _weighted_percentile
50
+
51
+
52
+ # Note: The shape of raw_prediction for multiclass classifications are
53
+ # - GradientBoostingClassifier: (n_samples, n_classes)
54
+ # - HistGradientBoostingClassifier: (n_classes, n_samples)
55
+ #
56
+ # Note: Instead of inheritance like
57
+ #
58
+ # class BaseLoss(BaseLink, CyLossFunction):
59
+ # ...
60
+ #
61
+ # # Note: Naturally, we would inherit in the following order
62
+ # # class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss)
63
+ # # But because of https://github.com/cython/cython/issues/4350 we set BaseLoss as
64
+ # # the last one. This, of course, changes the MRO.
65
+ # class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss):
66
+ #
67
+ # we use composition. This way we improve maintainability by avoiding the above
68
+ # mentioned Cython edge case and have easier to understand code (which method calls
69
+ # which code).
70
+ class BaseLoss:
71
+ """Base class for a loss function of 1-dimensional targets.
72
+
73
+ Conventions:
74
+
75
+ - y_true.shape = sample_weight.shape = (n_samples,)
76
+ - y_pred.shape = raw_prediction.shape = (n_samples,)
77
+ - If is_multiclass is true (multiclass classification), then
78
+ y_pred.shape = raw_prediction.shape = (n_samples, n_classes)
79
+ Note that this corresponds to the return value of decision_function.
80
+
81
+ y_true, y_pred, sample_weight and raw_prediction must either be all float64
82
+ or all float32.
83
+ gradient and hessian must be either both float64 or both float32.
84
+
85
+ Note that y_pred = link.inverse(raw_prediction).
86
+
87
+ Specific loss classes can inherit specific link classes to satisfy
88
+ BaseLink's abstractmethods.
89
+
90
+ Parameters
91
+ ----------
92
+ sample_weight : {None, ndarray}
93
+ If sample_weight is None, the hessian might be constant.
94
+ n_classes : {None, int}
95
+ The number of classes for classification, else None.
96
+
97
+ Attributes
98
+ ----------
99
+ closs: CyLossFunction
100
+ link : BaseLink
101
+ interval_y_true : Interval
102
+ Valid interval for y_true
103
+ interval_y_pred : Interval
104
+ Valid Interval for y_pred
105
+ differentiable : bool
106
+ Indicates whether or not loss function is differentiable in
107
+ raw_prediction everywhere.
108
+ need_update_leaves_values : bool
109
+ Indicates whether decision trees in gradient boosting need to uptade
110
+ leave values after having been fit to the (negative) gradients.
111
+ approx_hessian : bool
112
+ Indicates whether the hessian is approximated or exact. If,
113
+ approximated, it should be larger or equal to the exact one.
114
+ constant_hessian : bool
115
+ Indicates whether the hessian is one for this loss.
116
+ is_multiclass : bool
117
+ Indicates whether n_classes > 2 is allowed.
118
+ """
119
+
120
+ # For gradient boosted decision trees:
121
+ # This variable indicates whether the loss requires the leaves values to
122
+ # be updated once the tree has been trained. The trees are trained to
123
+ # predict a Newton-Raphson step (see grower._finalize_leaf()). But for
124
+ # some losses (e.g. least absolute deviation) we need to adjust the tree
125
+ # values to account for the "line search" of the gradient descent
126
+ # procedure. See the original paper Greedy Function Approximation: A
127
+ # Gradient Boosting Machine by Friedman
128
+ # (https://statweb.stanford.edu/~jhf/ftp/trebst.pdf) for the theory.
129
+ differentiable = True
130
+ need_update_leaves_values = False
131
+ is_multiclass = False
132
+
133
+ def __init__(self, closs, link, n_classes=None):
134
+ self.closs = closs
135
+ self.link = link
136
+ self.approx_hessian = False
137
+ self.constant_hessian = False
138
+ self.n_classes = n_classes
139
+ self.interval_y_true = Interval(-np.inf, np.inf, False, False)
140
+ self.interval_y_pred = self.link.interval_y_pred
141
+
142
+ def in_y_true_range(self, y):
143
+ """Return True if y is in the valid range of y_true.
144
+
145
+ Parameters
146
+ ----------
147
+ y : ndarray
148
+ """
149
+ return self.interval_y_true.includes(y)
150
+
151
+ def in_y_pred_range(self, y):
152
+ """Return True if y is in the valid range of y_pred.
153
+
154
+ Parameters
155
+ ----------
156
+ y : ndarray
157
+ """
158
+ return self.interval_y_pred.includes(y)
159
+
160
+ def loss(
161
+ self,
162
+ y_true,
163
+ raw_prediction,
164
+ sample_weight=None,
165
+ loss_out=None,
166
+ n_threads=1,
167
+ ):
168
+ """Compute the pointwise loss value for each input.
169
+
170
+ Parameters
171
+ ----------
172
+ y_true : C-contiguous array of shape (n_samples,)
173
+ Observed, true target values.
174
+ raw_prediction : C-contiguous array of shape (n_samples,) or array of \
175
+ shape (n_samples, n_classes)
176
+ Raw prediction values (in link space).
177
+ sample_weight : None or C-contiguous array of shape (n_samples,)
178
+ Sample weights.
179
+ loss_out : None or C-contiguous array of shape (n_samples,)
180
+ A location into which the result is stored. If None, a new array
181
+ might be created.
182
+ n_threads : int, default=1
183
+ Might use openmp thread parallelism.
184
+
185
+ Returns
186
+ -------
187
+ loss : array of shape (n_samples,)
188
+ Element-wise loss function.
189
+ """
190
+ if loss_out is None:
191
+ loss_out = np.empty_like(y_true)
192
+ # Be graceful to shape (n_samples, 1) -> (n_samples,)
193
+ if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
194
+ raw_prediction = raw_prediction.squeeze(1)
195
+
196
+ self.closs.loss(
197
+ y_true=y_true,
198
+ raw_prediction=raw_prediction,
199
+ sample_weight=sample_weight,
200
+ loss_out=loss_out,
201
+ n_threads=n_threads,
202
+ )
203
+ return loss_out
204
+
205
+ def loss_gradient(
206
+ self,
207
+ y_true,
208
+ raw_prediction,
209
+ sample_weight=None,
210
+ loss_out=None,
211
+ gradient_out=None,
212
+ n_threads=1,
213
+ ):
214
+ """Compute loss and gradient w.r.t. raw_prediction for each input.
215
+
216
+ Parameters
217
+ ----------
218
+ y_true : C-contiguous array of shape (n_samples,)
219
+ Observed, true target values.
220
+ raw_prediction : C-contiguous array of shape (n_samples,) or array of \
221
+ shape (n_samples, n_classes)
222
+ Raw prediction values (in link space).
223
+ sample_weight : None or C-contiguous array of shape (n_samples,)
224
+ Sample weights.
225
+ loss_out : None or C-contiguous array of shape (n_samples,)
226
+ A location into which the loss is stored. If None, a new array
227
+ might be created.
228
+ gradient_out : None or C-contiguous array of shape (n_samples,) or array \
229
+ of shape (n_samples, n_classes)
230
+ A location into which the gradient is stored. If None, a new array
231
+ might be created.
232
+ n_threads : int, default=1
233
+ Might use openmp thread parallelism.
234
+
235
+ Returns
236
+ -------
237
+ loss : array of shape (n_samples,)
238
+ Element-wise loss function.
239
+
240
+ gradient : array of shape (n_samples,) or (n_samples, n_classes)
241
+ Element-wise gradients.
242
+ """
243
+ if loss_out is None:
244
+ if gradient_out is None:
245
+ loss_out = np.empty_like(y_true)
246
+ gradient_out = np.empty_like(raw_prediction)
247
+ else:
248
+ loss_out = np.empty_like(y_true, dtype=gradient_out.dtype)
249
+ elif gradient_out is None:
250
+ gradient_out = np.empty_like(raw_prediction, dtype=loss_out.dtype)
251
+
252
+ # Be graceful to shape (n_samples, 1) -> (n_samples,)
253
+ if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
254
+ raw_prediction = raw_prediction.squeeze(1)
255
+ if gradient_out.ndim == 2 and gradient_out.shape[1] == 1:
256
+ gradient_out = gradient_out.squeeze(1)
257
+
258
+ self.closs.loss_gradient(
259
+ y_true=y_true,
260
+ raw_prediction=raw_prediction,
261
+ sample_weight=sample_weight,
262
+ loss_out=loss_out,
263
+ gradient_out=gradient_out,
264
+ n_threads=n_threads,
265
+ )
266
+ return loss_out, gradient_out
267
+
268
+ def gradient(
269
+ self,
270
+ y_true,
271
+ raw_prediction,
272
+ sample_weight=None,
273
+ gradient_out=None,
274
+ n_threads=1,
275
+ ):
276
+ """Compute gradient of loss w.r.t raw_prediction for each input.
277
+
278
+ Parameters
279
+ ----------
280
+ y_true : C-contiguous array of shape (n_samples,)
281
+ Observed, true target values.
282
+ raw_prediction : C-contiguous array of shape (n_samples,) or array of \
283
+ shape (n_samples, n_classes)
284
+ Raw prediction values (in link space).
285
+ sample_weight : None or C-contiguous array of shape (n_samples,)
286
+ Sample weights.
287
+ gradient_out : None or C-contiguous array of shape (n_samples,) or array \
288
+ of shape (n_samples, n_classes)
289
+ A location into which the result is stored. If None, a new array
290
+ might be created.
291
+ n_threads : int, default=1
292
+ Might use openmp thread parallelism.
293
+
294
+ Returns
295
+ -------
296
+ gradient : array of shape (n_samples,) or (n_samples, n_classes)
297
+ Element-wise gradients.
298
+ """
299
+ if gradient_out is None:
300
+ gradient_out = np.empty_like(raw_prediction)
301
+
302
+ # Be graceful to shape (n_samples, 1) -> (n_samples,)
303
+ if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
304
+ raw_prediction = raw_prediction.squeeze(1)
305
+ if gradient_out.ndim == 2 and gradient_out.shape[1] == 1:
306
+ gradient_out = gradient_out.squeeze(1)
307
+
308
+ self.closs.gradient(
309
+ y_true=y_true,
310
+ raw_prediction=raw_prediction,
311
+ sample_weight=sample_weight,
312
+ gradient_out=gradient_out,
313
+ n_threads=n_threads,
314
+ )
315
+ return gradient_out
316
+
317
+ def gradient_hessian(
318
+ self,
319
+ y_true,
320
+ raw_prediction,
321
+ sample_weight=None,
322
+ gradient_out=None,
323
+ hessian_out=None,
324
+ n_threads=1,
325
+ ):
326
+ """Compute gradient and hessian of loss w.r.t raw_prediction.
327
+
328
+ Parameters
329
+ ----------
330
+ y_true : C-contiguous array of shape (n_samples,)
331
+ Observed, true target values.
332
+ raw_prediction : C-contiguous array of shape (n_samples,) or array of \
333
+ shape (n_samples, n_classes)
334
+ Raw prediction values (in link space).
335
+ sample_weight : None or C-contiguous array of shape (n_samples,)
336
+ Sample weights.
337
+ gradient_out : None or C-contiguous array of shape (n_samples,) or array \
338
+ of shape (n_samples, n_classes)
339
+ A location into which the gradient is stored. If None, a new array
340
+ might be created.
341
+ hessian_out : None or C-contiguous array of shape (n_samples,) or array \
342
+ of shape (n_samples, n_classes)
343
+ A location into which the hessian is stored. If None, a new array
344
+ might be created.
345
+ n_threads : int, default=1
346
+ Might use openmp thread parallelism.
347
+
348
+ Returns
349
+ -------
350
+ gradient : arrays of shape (n_samples,) or (n_samples, n_classes)
351
+ Element-wise gradients.
352
+
353
+ hessian : arrays of shape (n_samples,) or (n_samples, n_classes)
354
+ Element-wise hessians.
355
+ """
356
+ if gradient_out is None:
357
+ if hessian_out is None:
358
+ gradient_out = np.empty_like(raw_prediction)
359
+ hessian_out = np.empty_like(raw_prediction)
360
+ else:
361
+ gradient_out = np.empty_like(hessian_out)
362
+ elif hessian_out is None:
363
+ hessian_out = np.empty_like(gradient_out)
364
+
365
+ # Be graceful to shape (n_samples, 1) -> (n_samples,)
366
+ if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
367
+ raw_prediction = raw_prediction.squeeze(1)
368
+ if gradient_out.ndim == 2 and gradient_out.shape[1] == 1:
369
+ gradient_out = gradient_out.squeeze(1)
370
+ if hessian_out.ndim == 2 and hessian_out.shape[1] == 1:
371
+ hessian_out = hessian_out.squeeze(1)
372
+
373
+ self.closs.gradient_hessian(
374
+ y_true=y_true,
375
+ raw_prediction=raw_prediction,
376
+ sample_weight=sample_weight,
377
+ gradient_out=gradient_out,
378
+ hessian_out=hessian_out,
379
+ n_threads=n_threads,
380
+ )
381
+ return gradient_out, hessian_out
382
+
383
+ def __call__(self, y_true, raw_prediction, sample_weight=None, n_threads=1):
384
+ """Compute the weighted average loss.
385
+
386
+ Parameters
387
+ ----------
388
+ y_true : C-contiguous array of shape (n_samples,)
389
+ Observed, true target values.
390
+ raw_prediction : C-contiguous array of shape (n_samples,) or array of \
391
+ shape (n_samples, n_classes)
392
+ Raw prediction values (in link space).
393
+ sample_weight : None or C-contiguous array of shape (n_samples,)
394
+ Sample weights.
395
+ n_threads : int, default=1
396
+ Might use openmp thread parallelism.
397
+
398
+ Returns
399
+ -------
400
+ loss : float
401
+ Mean or averaged loss function.
402
+ """
403
+ return np.average(
404
+ self.loss(
405
+ y_true=y_true,
406
+ raw_prediction=raw_prediction,
407
+ sample_weight=None,
408
+ loss_out=None,
409
+ n_threads=n_threads,
410
+ ),
411
+ weights=sample_weight,
412
+ )
413
+
414
+ def fit_intercept_only(self, y_true, sample_weight=None):
415
+ """Compute raw_prediction of an intercept-only model.
416
+
417
+ This can be used as initial estimates of predictions, i.e. before the
418
+ first iteration in fit.
419
+
420
+ Parameters
421
+ ----------
422
+ y_true : array-like of shape (n_samples,)
423
+ Observed, true target values.
424
+ sample_weight : None or array of shape (n_samples,)
425
+ Sample weights.
426
+
427
+ Returns
428
+ -------
429
+ raw_prediction : numpy scalar or array of shape (n_classes,)
430
+ Raw predictions of an intercept-only model.
431
+ """
432
+ # As default, take weighted average of the target over the samples
433
+ # axis=0 and then transform into link-scale (raw_prediction).
434
+ y_pred = np.average(y_true, weights=sample_weight, axis=0)
435
+ eps = 10 * np.finfo(y_pred.dtype).eps
436
+
437
+ if self.interval_y_pred.low == -np.inf:
438
+ a_min = None
439
+ elif self.interval_y_pred.low_inclusive:
440
+ a_min = self.interval_y_pred.low
441
+ else:
442
+ a_min = self.interval_y_pred.low + eps
443
+
444
+ if self.interval_y_pred.high == np.inf:
445
+ a_max = None
446
+ elif self.interval_y_pred.high_inclusive:
447
+ a_max = self.interval_y_pred.high
448
+ else:
449
+ a_max = self.interval_y_pred.high - eps
450
+
451
+ if a_min is None and a_max is None:
452
+ return self.link.link(y_pred)
453
+ else:
454
+ return self.link.link(np.clip(y_pred, a_min, a_max))
455
+
456
+ def constant_to_optimal_zero(self, y_true, sample_weight=None):
457
+ """Calculate term dropped in loss.
458
+
459
+ With this term added, the loss of perfect predictions is zero.
460
+
461
+ Parameters
462
+ ----------
463
+ y_true : array-like of shape (n_samples,)
464
+ Observed, true target values.
465
+
466
+ sample_weight : None or array of shape (n_samples,), default=None
467
+ Sample weights.
468
+
469
+ Returns
470
+ -------
471
+ constant : ndarray of shape (n_samples,)
472
+ Constant value to be added to raw predictions so that the loss
473
+ of perfect predictions becomes zero.
474
+ """
475
+ return np.zeros_like(y_true)
476
+
477
+ def init_gradient_and_hessian(self, n_samples, dtype=np.float64, order="F"):
478
+ """Initialize arrays for gradients and hessians.
479
+
480
+ Unless hessians are constant, arrays are initialized with undefined values.
481
+
482
+ Parameters
483
+ ----------
484
+ n_samples : int
485
+ The number of samples, usually passed to `fit()`.
486
+ dtype : {np.float64, np.float32}, default=np.float64
487
+ The dtype of the arrays gradient and hessian.
488
+ order : {'C', 'F'}, default='F'
489
+ Order of the arrays gradient and hessian. The default 'F' makes the arrays
490
+ contiguous along samples.
491
+
492
+ Returns
493
+ -------
494
+ gradient : C-contiguous array of shape (n_samples,) or array of shape \
495
+ (n_samples, n_classes)
496
+ Empty array (allocated but not initialized) to be used as argument
497
+ gradient_out.
498
+ hessian : C-contiguous array of shape (n_samples,), array of shape
499
+ (n_samples, n_classes) or shape (1,)
500
+ Empty (allocated but not initialized) array to be used as argument
501
+ hessian_out.
502
+ If constant_hessian is True (e.g. `HalfSquaredError`), the array is
503
+ initialized to ``1``.
504
+ """
505
+ if dtype not in (np.float32, np.float64):
506
+ raise ValueError(
507
+ "Valid options for 'dtype' are np.float32 and np.float64. "
508
+ f"Got dtype={dtype} instead."
509
+ )
510
+
511
+ if self.is_multiclass:
512
+ shape = (n_samples, self.n_classes)
513
+ else:
514
+ shape = (n_samples,)
515
+ gradient = np.empty(shape=shape, dtype=dtype, order=order)
516
+
517
+ if self.constant_hessian:
518
+ # If the hessians are constant, we consider them equal to 1.
519
+ # - This is correct for HalfSquaredError
520
+ # - For AbsoluteError, hessians are actually 0, but they are
521
+ # always ignored anyway.
522
+ hessian = np.ones(shape=(1,), dtype=dtype)
523
+ else:
524
+ hessian = np.empty(shape=shape, dtype=dtype, order=order)
525
+
526
+ return gradient, hessian
527
+
528
+
529
+ # Note: Naturally, we would inherit in the following order
530
+ # class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss)
531
+ # But because of https://github.com/cython/cython/issues/4350 we
532
+ # set BaseLoss as the last one. This, of course, changes the MRO.
533
+ class HalfSquaredError(BaseLoss):
534
+ """Half squared error with identity link, for regression.
535
+
536
+ Domain:
537
+ y_true and y_pred all real numbers
538
+
539
+ Link:
540
+ y_pred = raw_prediction
541
+
542
+ For a given sample x_i, half squared error is defined as::
543
+
544
+ loss(x_i) = 0.5 * (y_true_i - raw_prediction_i)**2
545
+
546
+ The factor of 0.5 simplifies the computation of gradients and results in a
547
+ unit hessian (and is consistent with what is done in LightGBM). It is also
548
+ half the Normal distribution deviance.
549
+ """
550
+
551
+ def __init__(self, sample_weight=None):
552
+ super().__init__(closs=CyHalfSquaredError(), link=IdentityLink())
553
+ self.constant_hessian = sample_weight is None
554
+
555
+
556
+ class AbsoluteError(BaseLoss):
557
+ """Absolute error with identity link, for regression.
558
+
559
+ Domain:
560
+ y_true and y_pred all real numbers
561
+
562
+ Link:
563
+ y_pred = raw_prediction
564
+
565
+ For a given sample x_i, the absolute error is defined as::
566
+
567
+ loss(x_i) = |y_true_i - raw_prediction_i|
568
+
569
+ Note that the exact hessian = 0 almost everywhere (except at one point, therefore
570
+ differentiable = False). Optimization routines like in HGBT, however, need a
571
+ hessian > 0. Therefore, we assign 1.
572
+ """
573
+
574
+ differentiable = False
575
+ need_update_leaves_values = True
576
+
577
+ def __init__(self, sample_weight=None):
578
+ super().__init__(closs=CyAbsoluteError(), link=IdentityLink())
579
+ self.approx_hessian = True
580
+ self.constant_hessian = sample_weight is None
581
+
582
+ def fit_intercept_only(self, y_true, sample_weight=None):
583
+ """Compute raw_prediction of an intercept-only model.
584
+
585
+ This is the weighted median of the target, i.e. over the samples
586
+ axis=0.
587
+ """
588
+ if sample_weight is None:
589
+ return np.median(y_true, axis=0)
590
+ else:
591
+ return _weighted_percentile(y_true, sample_weight, 50)
592
+
593
+
594
+ class PinballLoss(BaseLoss):
595
+ """Quantile loss aka pinball loss, for regression.
596
+
597
+ Domain:
598
+ y_true and y_pred all real numbers
599
+ quantile in (0, 1)
600
+
601
+ Link:
602
+ y_pred = raw_prediction
603
+
604
+ For a given sample x_i, the pinball loss is defined as::
605
+
606
+ loss(x_i) = rho_{quantile}(y_true_i - raw_prediction_i)
607
+
608
+ rho_{quantile}(u) = u * (quantile - 1_{u<0})
609
+ = -u *(1 - quantile) if u < 0
610
+ u * quantile if u >= 0
611
+
612
+ Note: 2 * PinballLoss(quantile=0.5) equals AbsoluteError().
613
+
614
+ Note that the exact hessian = 0 almost everywhere (except at one point, therefore
615
+ differentiable = False). Optimization routines like in HGBT, however, need a
616
+ hessian > 0. Therefore, we assign 1.
617
+
618
+ Additional Attributes
619
+ ---------------------
620
+ quantile : float
621
+ The quantile level of the quantile to be estimated. Must be in range (0, 1).
622
+ """
623
+
624
+ differentiable = False
625
+ need_update_leaves_values = True
626
+
627
+ def __init__(self, sample_weight=None, quantile=0.5):
628
+ check_scalar(
629
+ quantile,
630
+ "quantile",
631
+ target_type=numbers.Real,
632
+ min_val=0,
633
+ max_val=1,
634
+ include_boundaries="neither",
635
+ )
636
+ super().__init__(
637
+ closs=CyPinballLoss(quantile=float(quantile)),
638
+ link=IdentityLink(),
639
+ )
640
+ self.approx_hessian = True
641
+ self.constant_hessian = sample_weight is None
642
+
643
+ def fit_intercept_only(self, y_true, sample_weight=None):
644
+ """Compute raw_prediction of an intercept-only model.
645
+
646
+ This is the weighted median of the target, i.e. over the samples
647
+ axis=0.
648
+ """
649
+ if sample_weight is None:
650
+ return np.percentile(y_true, 100 * self.closs.quantile, axis=0)
651
+ else:
652
+ return _weighted_percentile(
653
+ y_true, sample_weight, 100 * self.closs.quantile
654
+ )
655
+
656
+
657
+ class HuberLoss(BaseLoss):
658
+ """Huber loss, for regression.
659
+
660
+ Domain:
661
+ y_true and y_pred all real numbers
662
+ quantile in (0, 1)
663
+
664
+ Link:
665
+ y_pred = raw_prediction
666
+
667
+ For a given sample x_i, the Huber loss is defined as::
668
+
669
+ loss(x_i) = 1/2 * abserr**2 if abserr <= delta
670
+ delta * (abserr - delta/2) if abserr > delta
671
+
672
+ abserr = |y_true_i - raw_prediction_i|
673
+ delta = quantile(abserr, self.quantile)
674
+
675
+ Note: HuberLoss(quantile=1) equals HalfSquaredError and HuberLoss(quantile=0)
676
+ equals delta * (AbsoluteError() - delta/2).
677
+
678
+ Additional Attributes
679
+ ---------------------
680
+ quantile : float
681
+ The quantile level which defines the breaking point `delta` to distinguish
682
+ between absolute error and squared error. Must be in range (0, 1).
683
+
684
+ Reference
685
+ ---------
686
+ .. [1] Friedman, J.H. (2001). :doi:`Greedy function approximation: A gradient
687
+ boosting machine <10.1214/aos/1013203451>`.
688
+ Annals of Statistics, 29, 1189-1232.
689
+ """
690
+
691
+ differentiable = False
692
+ need_update_leaves_values = True
693
+
694
+ def __init__(self, sample_weight=None, quantile=0.9, delta=0.5):
695
+ check_scalar(
696
+ quantile,
697
+ "quantile",
698
+ target_type=numbers.Real,
699
+ min_val=0,
700
+ max_val=1,
701
+ include_boundaries="neither",
702
+ )
703
+ self.quantile = quantile # This is better stored outside of Cython.
704
+ super().__init__(
705
+ closs=CyHuberLoss(delta=float(delta)),
706
+ link=IdentityLink(),
707
+ )
708
+ self.approx_hessian = True
709
+ self.constant_hessian = False
710
+
711
+ def fit_intercept_only(self, y_true, sample_weight=None):
712
+ """Compute raw_prediction of an intercept-only model.
713
+
714
+ This is the weighted median of the target, i.e. over the samples
715
+ axis=0.
716
+ """
717
+ # See formula before algo 4 in Friedman (2001), but we apply it to y_true,
718
+ # not to the residual y_true - raw_prediction. An estimator like
719
+ # HistGradientBoostingRegressor might then call it on the residual, e.g.
720
+ # fit_intercept_only(y_true - raw_prediction).
721
+ if sample_weight is None:
722
+ median = np.percentile(y_true, 50, axis=0)
723
+ else:
724
+ median = _weighted_percentile(y_true, sample_weight, 50)
725
+ diff = y_true - median
726
+ term = np.sign(diff) * np.minimum(self.closs.delta, np.abs(diff))
727
+ return median + np.average(term, weights=sample_weight)
728
+
729
+
730
+ class HalfPoissonLoss(BaseLoss):
731
+ """Half Poisson deviance loss with log-link, for regression.
732
+
733
+ Domain:
734
+ y_true in non-negative real numbers
735
+ y_pred in positive real numbers
736
+
737
+ Link:
738
+ y_pred = exp(raw_prediction)
739
+
740
+ For a given sample x_i, half the Poisson deviance is defined as::
741
+
742
+ loss(x_i) = y_true_i * log(y_true_i/exp(raw_prediction_i))
743
+ - y_true_i + exp(raw_prediction_i)
744
+
745
+ Half the Poisson deviance is actually the negative log-likelihood up to
746
+ constant terms (not involving raw_prediction) and simplifies the
747
+ computation of the gradients.
748
+ We also skip the constant term `y_true_i * log(y_true_i) - y_true_i`.
749
+ """
750
+
751
+ def __init__(self, sample_weight=None):
752
+ super().__init__(closs=CyHalfPoissonLoss(), link=LogLink())
753
+ self.interval_y_true = Interval(0, np.inf, True, False)
754
+
755
+ def constant_to_optimal_zero(self, y_true, sample_weight=None):
756
+ term = xlogy(y_true, y_true) - y_true
757
+ if sample_weight is not None:
758
+ term *= sample_weight
759
+ return term
760
+
761
+
762
+ class HalfGammaLoss(BaseLoss):
763
+ """Half Gamma deviance loss with log-link, for regression.
764
+
765
+ Domain:
766
+ y_true and y_pred in positive real numbers
767
+
768
+ Link:
769
+ y_pred = exp(raw_prediction)
770
+
771
+ For a given sample x_i, half Gamma deviance loss is defined as::
772
+
773
+ loss(x_i) = log(exp(raw_prediction_i)/y_true_i)
774
+ + y_true/exp(raw_prediction_i) - 1
775
+
776
+ Half the Gamma deviance is actually proportional to the negative log-
777
+ likelihood up to constant terms (not involving raw_prediction) and
778
+ simplifies the computation of the gradients.
779
+ We also skip the constant term `-log(y_true_i) - 1`.
780
+ """
781
+
782
+ def __init__(self, sample_weight=None):
783
+ super().__init__(closs=CyHalfGammaLoss(), link=LogLink())
784
+ self.interval_y_true = Interval(0, np.inf, False, False)
785
+
786
+ def constant_to_optimal_zero(self, y_true, sample_weight=None):
787
+ term = -np.log(y_true) - 1
788
+ if sample_weight is not None:
789
+ term *= sample_weight
790
+ return term
791
+
792
+
793
+ class HalfTweedieLoss(BaseLoss):
794
+ """Half Tweedie deviance loss with log-link, for regression.
795
+
796
+ Domain:
797
+ y_true in real numbers for power <= 0
798
+ y_true in non-negative real numbers for 0 < power < 2
799
+ y_true in positive real numbers for 2 <= power
800
+ y_pred in positive real numbers
801
+ power in real numbers
802
+
803
+ Link:
804
+ y_pred = exp(raw_prediction)
805
+
806
+ For a given sample x_i, half Tweedie deviance loss with p=power is defined
807
+ as::
808
+
809
+ loss(x_i) = max(y_true_i, 0)**(2-p) / (1-p) / (2-p)
810
+ - y_true_i * exp(raw_prediction_i)**(1-p) / (1-p)
811
+ + exp(raw_prediction_i)**(2-p) / (2-p)
812
+
813
+ Taking the limits for p=0, 1, 2 gives HalfSquaredError with a log link,
814
+ HalfPoissonLoss and HalfGammaLoss.
815
+
816
+ We also skip constant terms, but those are different for p=0, 1, 2.
817
+ Therefore, the loss is not continuous in `power`.
818
+
819
+ Note furthermore that although no Tweedie distribution exists for
820
+ 0 < power < 1, it still gives a strictly consistent scoring function for
821
+ the expectation.
822
+ """
823
+
824
+ def __init__(self, sample_weight=None, power=1.5):
825
+ super().__init__(
826
+ closs=CyHalfTweedieLoss(power=float(power)),
827
+ link=LogLink(),
828
+ )
829
+ if self.closs.power <= 0:
830
+ self.interval_y_true = Interval(-np.inf, np.inf, False, False)
831
+ elif self.closs.power < 2:
832
+ self.interval_y_true = Interval(0, np.inf, True, False)
833
+ else:
834
+ self.interval_y_true = Interval(0, np.inf, False, False)
835
+
836
+ def constant_to_optimal_zero(self, y_true, sample_weight=None):
837
+ if self.closs.power == 0:
838
+ return HalfSquaredError().constant_to_optimal_zero(
839
+ y_true=y_true, sample_weight=sample_weight
840
+ )
841
+ elif self.closs.power == 1:
842
+ return HalfPoissonLoss().constant_to_optimal_zero(
843
+ y_true=y_true, sample_weight=sample_weight
844
+ )
845
+ elif self.closs.power == 2:
846
+ return HalfGammaLoss().constant_to_optimal_zero(
847
+ y_true=y_true, sample_weight=sample_weight
848
+ )
849
+ else:
850
+ p = self.closs.power
851
+ term = np.power(np.maximum(y_true, 0), 2 - p) / (1 - p) / (2 - p)
852
+ if sample_weight is not None:
853
+ term *= sample_weight
854
+ return term
855
+
856
+
857
+ class HalfTweedieLossIdentity(BaseLoss):
858
+ """Half Tweedie deviance loss with identity link, for regression.
859
+
860
+ Domain:
861
+ y_true in real numbers for power <= 0
862
+ y_true in non-negative real numbers for 0 < power < 2
863
+ y_true in positive real numbers for 2 <= power
864
+ y_pred in positive real numbers for power != 0
865
+ y_pred in real numbers for power = 0
866
+ power in real numbers
867
+
868
+ Link:
869
+ y_pred = raw_prediction
870
+
871
+ For a given sample x_i, half Tweedie deviance loss with p=power is defined
872
+ as::
873
+
874
+ loss(x_i) = max(y_true_i, 0)**(2-p) / (1-p) / (2-p)
875
+ - y_true_i * raw_prediction_i**(1-p) / (1-p)
876
+ + raw_prediction_i**(2-p) / (2-p)
877
+
878
+ Note that the minimum value of this loss is 0.
879
+
880
+ Note furthermore that although no Tweedie distribution exists for
881
+ 0 < power < 1, it still gives a strictly consistent scoring function for
882
+ the expectation.
883
+ """
884
+
885
+ def __init__(self, sample_weight=None, power=1.5):
886
+ super().__init__(
887
+ closs=CyHalfTweedieLossIdentity(power=float(power)),
888
+ link=IdentityLink(),
889
+ )
890
+ if self.closs.power <= 0:
891
+ self.interval_y_true = Interval(-np.inf, np.inf, False, False)
892
+ elif self.closs.power < 2:
893
+ self.interval_y_true = Interval(0, np.inf, True, False)
894
+ else:
895
+ self.interval_y_true = Interval(0, np.inf, False, False)
896
+
897
+ if self.closs.power == 0:
898
+ self.interval_y_pred = Interval(-np.inf, np.inf, False, False)
899
+ else:
900
+ self.interval_y_pred = Interval(0, np.inf, False, False)
901
+
902
+
903
+ class HalfBinomialLoss(BaseLoss):
904
+ """Half Binomial deviance loss with logit link, for binary classification.
905
+
906
+ This is also know as binary cross entropy, log-loss and logistic loss.
907
+
908
+ Domain:
909
+ y_true in [0, 1], i.e. regression on the unit interval
910
+ y_pred in (0, 1), i.e. boundaries excluded
911
+
912
+ Link:
913
+ y_pred = expit(raw_prediction)
914
+
915
+ For a given sample x_i, half Binomial deviance is defined as the negative
916
+ log-likelihood of the Binomial/Bernoulli distribution and can be expressed
917
+ as::
918
+
919
+ loss(x_i) = log(1 + exp(raw_pred_i)) - y_true_i * raw_pred_i
920
+
921
+ See The Elements of Statistical Learning, by Hastie, Tibshirani, Friedman,
922
+ section 4.4.1 (about logistic regression).
923
+
924
+ Note that the formulation works for classification, y = {0, 1}, as well as
925
+ logistic regression, y = [0, 1].
926
+ If you add `constant_to_optimal_zero` to the loss, you get half the
927
+ Bernoulli/binomial deviance.
928
+
929
+ More details: Inserting the predicted probability y_pred = expit(raw_prediction)
930
+ in the loss gives the well known::
931
+
932
+ loss(x_i) = - y_true_i * log(y_pred_i) - (1 - y_true_i) * log(1 - y_pred_i)
933
+ """
934
+
935
+ def __init__(self, sample_weight=None):
936
+ super().__init__(
937
+ closs=CyHalfBinomialLoss(),
938
+ link=LogitLink(),
939
+ n_classes=2,
940
+ )
941
+ self.interval_y_true = Interval(0, 1, True, True)
942
+
943
+ def constant_to_optimal_zero(self, y_true, sample_weight=None):
944
+ # This is non-zero only if y_true is neither 0 nor 1.
945
+ term = xlogy(y_true, y_true) + xlogy(1 - y_true, 1 - y_true)
946
+ if sample_weight is not None:
947
+ term *= sample_weight
948
+ return term
949
+
950
+ def predict_proba(self, raw_prediction):
951
+ """Predict probabilities.
952
+
953
+ Parameters
954
+ ----------
955
+ raw_prediction : array of shape (n_samples,) or (n_samples, 1)
956
+ Raw prediction values (in link space).
957
+
958
+ Returns
959
+ -------
960
+ proba : array of shape (n_samples, 2)
961
+ Element-wise class probabilities.
962
+ """
963
+ # Be graceful to shape (n_samples, 1) -> (n_samples,)
964
+ if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
965
+ raw_prediction = raw_prediction.squeeze(1)
966
+ proba = np.empty((raw_prediction.shape[0], 2), dtype=raw_prediction.dtype)
967
+ proba[:, 1] = self.link.inverse(raw_prediction)
968
+ proba[:, 0] = 1 - proba[:, 1]
969
+ return proba
970
+
971
+
972
+ class HalfMultinomialLoss(BaseLoss):
973
+ """Categorical cross-entropy loss, for multiclass classification.
974
+
975
+ Domain:
976
+ y_true in {0, 1, 2, 3, .., n_classes - 1}
977
+ y_pred has n_classes elements, each element in (0, 1)
978
+
979
+ Link:
980
+ y_pred = softmax(raw_prediction)
981
+
982
+ Note: We assume y_true to be already label encoded. The inverse link is
983
+ softmax. But the full link function is the symmetric multinomial logit
984
+ function.
985
+
986
+ For a given sample x_i, the categorical cross-entropy loss is defined as
987
+ the negative log-likelihood of the multinomial distribution, it
988
+ generalizes the binary cross-entropy to more than 2 classes::
989
+
990
+ loss_i = log(sum(exp(raw_pred_{i, k}), k=0..n_classes-1))
991
+ - sum(y_true_{i, k} * raw_pred_{i, k}, k=0..n_classes-1)
992
+
993
+ See [1].
994
+
995
+ Note that for the hessian, we calculate only the diagonal part in the
996
+ classes: If the full hessian for classes k and l and sample i is H_i_k_l,
997
+ we calculate H_i_k_k, i.e. k=l.
998
+
999
+ Parameters
1000
+ ----------
1001
+ sample_weight : {None, ndarray}
1002
+ If sample_weight is None, the hessian might be constant.
1003
+
1004
+ n_classes : {None, int}
1005
+ The number of classes for classification, else None.
1006
+
1007
+ References
1008
+ ----------
1009
+ .. [1] :arxiv:`Simon, Noah, J. Friedman and T. Hastie.
1010
+ "A Blockwise Descent Algorithm for Group-penalized Multiresponse and
1011
+ Multinomial Regression".
1012
+ <1311.6529>`
1013
+ """
1014
+
1015
+ is_multiclass = True
1016
+
1017
+ def __init__(self, sample_weight=None, n_classes=3):
1018
+ super().__init__(
1019
+ closs=CyHalfMultinomialLoss(),
1020
+ link=MultinomialLogit(),
1021
+ n_classes=n_classes,
1022
+ )
1023
+ self.interval_y_true = Interval(0, np.inf, True, False)
1024
+ self.interval_y_pred = Interval(0, 1, False, False)
1025
+
1026
+ def in_y_true_range(self, y):
1027
+ """Return True if y is in the valid range of y_true.
1028
+
1029
+ Parameters
1030
+ ----------
1031
+ y : ndarray
1032
+ """
1033
+ return self.interval_y_true.includes(y) and np.all(y.astype(int) == y)
1034
+
1035
+ def fit_intercept_only(self, y_true, sample_weight=None):
1036
+ """Compute raw_prediction of an intercept-only model.
1037
+
1038
+ This is the softmax of the weighted average of the target, i.e. over
1039
+ the samples axis=0.
1040
+
1041
+ Parameters
1042
+ ----------
1043
+ y_true : array-like of shape (n_samples,)
1044
+ Observed, true target values.
1045
+
1046
+ sample_weight : None or array of shape (n_samples,), default=None
1047
+ Sample weights.
1048
+
1049
+ Returns
1050
+ -------
1051
+ raw_prediction : numpy scalar or array of shape (n_classes,)
1052
+ Raw predictions of an intercept-only model.
1053
+ """
1054
+ out = np.zeros(self.n_classes, dtype=y_true.dtype)
1055
+ eps = np.finfo(y_true.dtype).eps
1056
+ for k in range(self.n_classes):
1057
+ out[k] = np.average(y_true == k, weights=sample_weight, axis=0)
1058
+ out[k] = np.clip(out[k], eps, 1 - eps)
1059
+ return self.link.link(out[None, :]).reshape(-1)
1060
+
1061
+ def predict_proba(self, raw_prediction):
1062
+ """Predict probabilities.
1063
+
1064
+ Parameters
1065
+ ----------
1066
+ raw_prediction : array of shape (n_samples, n_classes)
1067
+ Raw prediction values (in link space).
1068
+
1069
+ Returns
1070
+ -------
1071
+ proba : array of shape (n_samples, n_classes)
1072
+ Element-wise class probabilities.
1073
+ """
1074
+ return self.link.inverse(raw_prediction)
1075
+
1076
+ def gradient_proba(
1077
+ self,
1078
+ y_true,
1079
+ raw_prediction,
1080
+ sample_weight=None,
1081
+ gradient_out=None,
1082
+ proba_out=None,
1083
+ n_threads=1,
1084
+ ):
1085
+ """Compute gradient and class probabilities fow raw_prediction.
1086
+
1087
+ Parameters
1088
+ ----------
1089
+ y_true : C-contiguous array of shape (n_samples,)
1090
+ Observed, true target values.
1091
+ raw_prediction : array of shape (n_samples, n_classes)
1092
+ Raw prediction values (in link space).
1093
+ sample_weight : None or C-contiguous array of shape (n_samples,)
1094
+ Sample weights.
1095
+ gradient_out : None or array of shape (n_samples, n_classes)
1096
+ A location into which the gradient is stored. If None, a new array
1097
+ might be created.
1098
+ proba_out : None or array of shape (n_samples, n_classes)
1099
+ A location into which the class probabilities are stored. If None,
1100
+ a new array might be created.
1101
+ n_threads : int, default=1
1102
+ Might use openmp thread parallelism.
1103
+
1104
+ Returns
1105
+ -------
1106
+ gradient : array of shape (n_samples, n_classes)
1107
+ Element-wise gradients.
1108
+
1109
+ proba : array of shape (n_samples, n_classes)
1110
+ Element-wise class probabilities.
1111
+ """
1112
+ if gradient_out is None:
1113
+ if proba_out is None:
1114
+ gradient_out = np.empty_like(raw_prediction)
1115
+ proba_out = np.empty_like(raw_prediction)
1116
+ else:
1117
+ gradient_out = np.empty_like(proba_out)
1118
+ elif proba_out is None:
1119
+ proba_out = np.empty_like(gradient_out)
1120
+
1121
+ self.closs.gradient_proba(
1122
+ y_true=y_true,
1123
+ raw_prediction=raw_prediction,
1124
+ sample_weight=sample_weight,
1125
+ gradient_out=gradient_out,
1126
+ proba_out=proba_out,
1127
+ n_threads=n_threads,
1128
+ )
1129
+ return gradient_out, proba_out
1130
+
1131
+
1132
+ class ExponentialLoss(BaseLoss):
1133
+ """Exponential loss with (half) logit link, for binary classification.
1134
+
1135
+ This is also know as boosting loss.
1136
+
1137
+ Domain:
1138
+ y_true in [0, 1], i.e. regression on the unit interval
1139
+ y_pred in (0, 1), i.e. boundaries excluded
1140
+
1141
+ Link:
1142
+ y_pred = expit(2 * raw_prediction)
1143
+
1144
+ For a given sample x_i, the exponential loss is defined as::
1145
+
1146
+ loss(x_i) = y_true_i * exp(-raw_pred_i)) + (1 - y_true_i) * exp(raw_pred_i)
1147
+
1148
+ See:
1149
+ - J. Friedman, T. Hastie, R. Tibshirani.
1150
+ "Additive logistic regression: a statistical view of boosting (With discussion
1151
+ and a rejoinder by the authors)." Ann. Statist. 28 (2) 337 - 407, April 2000.
1152
+ https://doi.org/10.1214/aos/1016218223
1153
+ - A. Buja, W. Stuetzle, Y. Shen. (2005).
1154
+ "Loss Functions for Binary Class Probability Estimation and Classification:
1155
+ Structure and Applications."
1156
+
1157
+ Note that the formulation works for classification, y = {0, 1}, as well as
1158
+ "exponential logistic" regression, y = [0, 1].
1159
+ Note that this is a proper scoring rule, but without it's canonical link.
1160
+
1161
+ More details: Inserting the predicted probability
1162
+ y_pred = expit(2 * raw_prediction) in the loss gives::
1163
+
1164
+ loss(x_i) = y_true_i * sqrt((1 - y_pred_i) / y_pred_i)
1165
+ + (1 - y_true_i) * sqrt(y_pred_i / (1 - y_pred_i))
1166
+ """
1167
+
1168
+ def __init__(self, sample_weight=None):
1169
+ super().__init__(
1170
+ closs=CyExponentialLoss(),
1171
+ link=HalfLogitLink(),
1172
+ n_classes=2,
1173
+ )
1174
+ self.interval_y_true = Interval(0, 1, True, True)
1175
+
1176
+ def constant_to_optimal_zero(self, y_true, sample_weight=None):
1177
+ # This is non-zero only if y_true is neither 0 nor 1.
1178
+ term = -2 * np.sqrt(y_true * (1 - y_true))
1179
+ if sample_weight is not None:
1180
+ term *= sample_weight
1181
+ return term
1182
+
1183
+ def predict_proba(self, raw_prediction):
1184
+ """Predict probabilities.
1185
+
1186
+ Parameters
1187
+ ----------
1188
+ raw_prediction : array of shape (n_samples,) or (n_samples, 1)
1189
+ Raw prediction values (in link space).
1190
+
1191
+ Returns
1192
+ -------
1193
+ proba : array of shape (n_samples, 2)
1194
+ Element-wise class probabilities.
1195
+ """
1196
+ # Be graceful to shape (n_samples, 1) -> (n_samples,)
1197
+ if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1:
1198
+ raw_prediction = raw_prediction.squeeze(1)
1199
+ proba = np.empty((raw_prediction.shape[0], 2), dtype=raw_prediction.dtype)
1200
+ proba[:, 1] = self.link.inverse(raw_prediction)
1201
+ proba[:, 0] = 1 - proba[:, 1]
1202
+ return proba
1203
+
1204
+
1205
+ _LOSSES = {
1206
+ "squared_error": HalfSquaredError,
1207
+ "absolute_error": AbsoluteError,
1208
+ "pinball_loss": PinballLoss,
1209
+ "huber_loss": HuberLoss,
1210
+ "poisson_loss": HalfPoissonLoss,
1211
+ "gamma_loss": HalfGammaLoss,
1212
+ "tweedie_loss": HalfTweedieLoss,
1213
+ "binomial_loss": HalfBinomialLoss,
1214
+ "multinomial_loss": HalfMultinomialLoss,
1215
+ "exponential_loss": ExponentialLoss,
1216
+ }
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/meson.build ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # .pyx is generated, so this is needed to make Cython compilation work
2
+ _loss_cython_tree = [
3
+ fs.copyfile('_loss.pxd')
4
+ ]
5
+
6
+ _loss_pyx = custom_target(
7
+ '_loss_pyx',
8
+ output: '_loss.pyx',
9
+ input: '_loss.pyx.tp',
10
+ command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
11
+ # TODO in principle this should go in py.exension_module below. This is
12
+ # temporary work-around for dependency issue with .pyx.tp files. For more
13
+ # details, see https://github.com/mesonbuild/meson/issues/13212
14
+ depends: _loss_cython_tree,
15
+ )
16
+
17
+ py.extension_module(
18
+ '_loss',
19
+ cython_gen.process(_loss_pyx),
20
+ dependencies: [openmp_dep],
21
+ install: true,
22
+ subdir: 'sklearn/_loss',
23
+ )
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/__init__.py ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_link.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import pytest
3
+ from numpy.testing import assert_allclose, assert_array_equal
4
+
5
+ from sklearn._loss.link import (
6
+ _LINKS,
7
+ HalfLogitLink,
8
+ Interval,
9
+ MultinomialLogit,
10
+ _inclusive_low_high,
11
+ )
12
+
13
+ LINK_FUNCTIONS = list(_LINKS.values())
14
+
15
+
16
+ def test_interval_raises():
17
+ """Test that interval with low > high raises ValueError."""
18
+ with pytest.raises(
19
+ ValueError, match="One must have low <= high; got low=1, high=0."
20
+ ):
21
+ Interval(1, 0, False, False)
22
+
23
+
24
+ @pytest.mark.parametrize(
25
+ "interval",
26
+ [
27
+ Interval(0, 1, False, False),
28
+ Interval(0, 1, False, True),
29
+ Interval(0, 1, True, False),
30
+ Interval(0, 1, True, True),
31
+ Interval(-np.inf, np.inf, False, False),
32
+ Interval(-np.inf, np.inf, False, True),
33
+ Interval(-np.inf, np.inf, True, False),
34
+ Interval(-np.inf, np.inf, True, True),
35
+ Interval(-10, -1, False, False),
36
+ Interval(-10, -1, False, True),
37
+ Interval(-10, -1, True, False),
38
+ Interval(-10, -1, True, True),
39
+ ],
40
+ )
41
+ def test_is_in_range(interval):
42
+ # make sure low and high are always within the interval, used for linspace
43
+ low, high = _inclusive_low_high(interval)
44
+
45
+ x = np.linspace(low, high, num=10)
46
+ assert interval.includes(x)
47
+
48
+ # x contains lower bound
49
+ assert interval.includes(np.r_[x, interval.low]) == interval.low_inclusive
50
+
51
+ # x contains upper bound
52
+ assert interval.includes(np.r_[x, interval.high]) == interval.high_inclusive
53
+
54
+ # x contains upper and lower bound
55
+ assert interval.includes(np.r_[x, interval.low, interval.high]) == (
56
+ interval.low_inclusive and interval.high_inclusive
57
+ )
58
+
59
+
60
+ @pytest.mark.parametrize("link", LINK_FUNCTIONS)
61
+ def test_link_inverse_identity(link, global_random_seed):
62
+ # Test that link of inverse gives identity.
63
+ rng = np.random.RandomState(global_random_seed)
64
+ link = link()
65
+ n_samples, n_classes = 100, None
66
+ # The values for `raw_prediction` are limited from -20 to 20 because in the
67
+ # class `LogitLink` the term `expit(x)` comes very close to 1 for large
68
+ # positive x and therefore loses precision.
69
+ if link.is_multiclass:
70
+ n_classes = 10
71
+ raw_prediction = rng.uniform(low=-20, high=20, size=(n_samples, n_classes))
72
+ if isinstance(link, MultinomialLogit):
73
+ raw_prediction = link.symmetrize_raw_prediction(raw_prediction)
74
+ elif isinstance(link, HalfLogitLink):
75
+ raw_prediction = rng.uniform(low=-10, high=10, size=(n_samples))
76
+ else:
77
+ raw_prediction = rng.uniform(low=-20, high=20, size=(n_samples))
78
+
79
+ assert_allclose(link.link(link.inverse(raw_prediction)), raw_prediction)
80
+ y_pred = link.inverse(raw_prediction)
81
+ assert_allclose(link.inverse(link.link(y_pred)), y_pred)
82
+
83
+
84
+ @pytest.mark.parametrize("link", LINK_FUNCTIONS)
85
+ def test_link_out_argument(link):
86
+ # Test that out argument gets assigned the result.
87
+ rng = np.random.RandomState(42)
88
+ link = link()
89
+ n_samples, n_classes = 100, None
90
+ if link.is_multiclass:
91
+ n_classes = 10
92
+ raw_prediction = rng.normal(loc=0, scale=10, size=(n_samples, n_classes))
93
+ if isinstance(link, MultinomialLogit):
94
+ raw_prediction = link.symmetrize_raw_prediction(raw_prediction)
95
+ else:
96
+ # So far, the valid interval of raw_prediction is (-inf, inf) and
97
+ # we do not need to distinguish.
98
+ raw_prediction = rng.uniform(low=-10, high=10, size=(n_samples))
99
+
100
+ y_pred = link.inverse(raw_prediction, out=None)
101
+ out = np.empty_like(raw_prediction)
102
+ y_pred_2 = link.inverse(raw_prediction, out=out)
103
+ assert_allclose(y_pred, out)
104
+ assert_array_equal(out, y_pred_2)
105
+ assert np.shares_memory(out, y_pred_2)
106
+
107
+ out = np.empty_like(y_pred)
108
+ raw_prediction_2 = link.link(y_pred, out=out)
109
+ assert_allclose(raw_prediction, out)
110
+ assert_array_equal(out, raw_prediction_2)
111
+ assert np.shares_memory(out, raw_prediction_2)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_loss/tests/test_loss.py ADDED
@@ -0,0 +1,1358 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pickle
2
+
3
+ import numpy as np
4
+ import pytest
5
+ from numpy.testing import assert_allclose, assert_array_equal
6
+ from pytest import approx
7
+ from scipy.optimize import (
8
+ LinearConstraint,
9
+ minimize,
10
+ minimize_scalar,
11
+ newton,
12
+ )
13
+ from scipy.special import logsumexp
14
+
15
+ from sklearn._loss.link import IdentityLink, _inclusive_low_high
16
+ from sklearn._loss.loss import (
17
+ _LOSSES,
18
+ AbsoluteError,
19
+ BaseLoss,
20
+ HalfBinomialLoss,
21
+ HalfGammaLoss,
22
+ HalfMultinomialLoss,
23
+ HalfPoissonLoss,
24
+ HalfSquaredError,
25
+ HalfTweedieLoss,
26
+ HalfTweedieLossIdentity,
27
+ HuberLoss,
28
+ PinballLoss,
29
+ )
30
+ from sklearn.utils import assert_all_finite
31
+ from sklearn.utils._testing import create_memmap_backed_data, skip_if_32bit
32
+
33
+ ALL_LOSSES = list(_LOSSES.values())
34
+
35
+ LOSS_INSTANCES = [loss() for loss in ALL_LOSSES]
36
+ # HalfTweedieLoss(power=1.5) is already there as default
37
+ LOSS_INSTANCES += [
38
+ PinballLoss(quantile=0.25),
39
+ HuberLoss(quantile=0.75),
40
+ HalfTweedieLoss(power=-1.5),
41
+ HalfTweedieLoss(power=0),
42
+ HalfTweedieLoss(power=1),
43
+ HalfTweedieLoss(power=2),
44
+ HalfTweedieLoss(power=3.0),
45
+ HalfTweedieLossIdentity(power=0),
46
+ HalfTweedieLossIdentity(power=1),
47
+ HalfTweedieLossIdentity(power=2),
48
+ HalfTweedieLossIdentity(power=3.0),
49
+ ]
50
+
51
+
52
+ def loss_instance_name(param):
53
+ if isinstance(param, BaseLoss):
54
+ loss = param
55
+ name = loss.__class__.__name__
56
+ if isinstance(loss, PinballLoss):
57
+ name += f"(quantile={loss.closs.quantile})"
58
+ elif isinstance(loss, HuberLoss):
59
+ name += f"(quantile={loss.quantile}"
60
+ elif hasattr(loss, "closs") and hasattr(loss.closs, "power"):
61
+ name += f"(power={loss.closs.power})"
62
+ return name
63
+ else:
64
+ return str(param)
65
+
66
+
67
+ def random_y_true_raw_prediction(
68
+ loss, n_samples, y_bound=(-100, 100), raw_bound=(-5, 5), seed=42
69
+ ):
70
+ """Random generate y_true and raw_prediction in valid range."""
71
+ rng = np.random.RandomState(seed)
72
+ if loss.is_multiclass:
73
+ raw_prediction = np.empty((n_samples, loss.n_classes))
74
+ raw_prediction.flat[:] = rng.uniform(
75
+ low=raw_bound[0],
76
+ high=raw_bound[1],
77
+ size=n_samples * loss.n_classes,
78
+ )
79
+ y_true = np.arange(n_samples).astype(float) % loss.n_classes
80
+ else:
81
+ # If link is identity, we must respect the interval of y_pred:
82
+ if isinstance(loss.link, IdentityLink):
83
+ low, high = _inclusive_low_high(loss.interval_y_pred)
84
+ low = np.amax([low, raw_bound[0]])
85
+ high = np.amin([high, raw_bound[1]])
86
+ raw_bound = (low, high)
87
+ raw_prediction = rng.uniform(
88
+ low=raw_bound[0], high=raw_bound[1], size=n_samples
89
+ )
90
+ # generate a y_true in valid range
91
+ low, high = _inclusive_low_high(loss.interval_y_true)
92
+ low = max(low, y_bound[0])
93
+ high = min(high, y_bound[1])
94
+ y_true = rng.uniform(low, high, size=n_samples)
95
+ # set some values at special boundaries
96
+ if loss.interval_y_true.low == 0 and loss.interval_y_true.low_inclusive:
97
+ y_true[:: (n_samples // 3)] = 0
98
+ if loss.interval_y_true.high == 1 and loss.interval_y_true.high_inclusive:
99
+ y_true[1 :: (n_samples // 3)] = 1
100
+
101
+ return y_true, raw_prediction
102
+
103
+
104
+ def numerical_derivative(func, x, eps):
105
+ """Helper function for numerical (first) derivatives."""
106
+ # For numerical derivatives, see
107
+ # https://en.wikipedia.org/wiki/Numerical_differentiation
108
+ # https://en.wikipedia.org/wiki/Finite_difference_coefficient
109
+ # We use central finite differences of accuracy 4.
110
+ h = np.full_like(x, fill_value=eps)
111
+ f_minus_2h = func(x - 2 * h)
112
+ f_minus_1h = func(x - h)
113
+ f_plus_1h = func(x + h)
114
+ f_plus_2h = func(x + 2 * h)
115
+ return (-f_plus_2h + 8 * f_plus_1h - 8 * f_minus_1h + f_minus_2h) / (12.0 * eps)
116
+
117
+
118
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
119
+ def test_loss_boundary(loss):
120
+ """Test interval ranges of y_true and y_pred in losses."""
121
+ # make sure low and high are always within the interval, used for linspace
122
+ if loss.is_multiclass:
123
+ n_classes = 3 # default value
124
+ y_true = np.tile(np.linspace(0, n_classes - 1, num=n_classes), 3)
125
+ else:
126
+ low, high = _inclusive_low_high(loss.interval_y_true)
127
+ y_true = np.linspace(low, high, num=10)
128
+
129
+ # add boundaries if they are included
130
+ if loss.interval_y_true.low_inclusive:
131
+ y_true = np.r_[y_true, loss.interval_y_true.low]
132
+ if loss.interval_y_true.high_inclusive:
133
+ y_true = np.r_[y_true, loss.interval_y_true.high]
134
+
135
+ assert loss.in_y_true_range(y_true)
136
+
137
+ n = y_true.shape[0]
138
+ low, high = _inclusive_low_high(loss.interval_y_pred)
139
+ if loss.is_multiclass:
140
+ y_pred = np.empty((n, n_classes))
141
+ y_pred[:, 0] = np.linspace(low, high, num=n)
142
+ y_pred[:, 1] = 0.5 * (1 - y_pred[:, 0])
143
+ y_pred[:, 2] = 0.5 * (1 - y_pred[:, 0])
144
+ else:
145
+ y_pred = np.linspace(low, high, num=n)
146
+
147
+ assert loss.in_y_pred_range(y_pred)
148
+
149
+ # calculating losses should not fail
150
+ raw_prediction = loss.link.link(y_pred)
151
+ loss.loss(y_true=y_true, raw_prediction=raw_prediction)
152
+
153
+
154
+ # Fixture to test valid value ranges.
155
+ Y_COMMON_PARAMS = [
156
+ # (loss, [y success], [y fail])
157
+ (HalfSquaredError(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
158
+ (AbsoluteError(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
159
+ (PinballLoss(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
160
+ (HuberLoss(), [-100, 0, 0.1, 100], [-np.inf, np.inf]),
161
+ (HalfPoissonLoss(), [0.1, 100], [-np.inf, -3, -0.1, np.inf]),
162
+ (HalfGammaLoss(), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
163
+ (HalfTweedieLoss(power=-3), [0.1, 100], [-np.inf, np.inf]),
164
+ (HalfTweedieLoss(power=0), [0.1, 100], [-np.inf, np.inf]),
165
+ (HalfTweedieLoss(power=1.5), [0.1, 100], [-np.inf, -3, -0.1, np.inf]),
166
+ (HalfTweedieLoss(power=2), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
167
+ (HalfTweedieLoss(power=3), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
168
+ (HalfTweedieLossIdentity(power=-3), [0.1, 100], [-np.inf, np.inf]),
169
+ (HalfTweedieLossIdentity(power=0), [-3, -0.1, 0, 0.1, 100], [-np.inf, np.inf]),
170
+ (HalfTweedieLossIdentity(power=1.5), [0.1, 100], [-np.inf, -3, -0.1, np.inf]),
171
+ (HalfTweedieLossIdentity(power=2), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
172
+ (HalfTweedieLossIdentity(power=3), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]),
173
+ (HalfBinomialLoss(), [0.1, 0.5, 0.9], [-np.inf, -1, 2, np.inf]),
174
+ (HalfMultinomialLoss(), [], [-np.inf, -1, 1.1, np.inf]),
175
+ ]
176
+ # y_pred and y_true do not always have the same domain (valid value range).
177
+ # Hence, we define extra sets of parameters for each of them.
178
+ Y_TRUE_PARAMS = [ # type: ignore[var-annotated]
179
+ # (loss, [y success], [y fail])
180
+ (HalfPoissonLoss(), [0], []),
181
+ (HuberLoss(), [0], []),
182
+ (HalfTweedieLoss(power=-3), [-100, -0.1, 0], []),
183
+ (HalfTweedieLoss(power=0), [-100, 0], []),
184
+ (HalfTweedieLoss(power=1.5), [0], []),
185
+ (HalfTweedieLossIdentity(power=-3), [-100, -0.1, 0], []),
186
+ (HalfTweedieLossIdentity(power=0), [-100, 0], []),
187
+ (HalfTweedieLossIdentity(power=1.5), [0], []),
188
+ (HalfBinomialLoss(), [0, 1], []),
189
+ (HalfMultinomialLoss(), [0.0, 1.0, 2], []),
190
+ ]
191
+ Y_PRED_PARAMS = [
192
+ # (loss, [y success], [y fail])
193
+ (HalfPoissonLoss(), [], [0]),
194
+ (HalfTweedieLoss(power=-3), [], [-3, -0.1, 0]),
195
+ (HalfTweedieLoss(power=0), [], [-3, -0.1, 0]),
196
+ (HalfTweedieLoss(power=1.5), [], [0]),
197
+ (HalfTweedieLossIdentity(power=-3), [], [-3, -0.1, 0]),
198
+ (HalfTweedieLossIdentity(power=0), [-3, -0.1, 0], []),
199
+ (HalfTweedieLossIdentity(power=1.5), [], [0]),
200
+ (HalfBinomialLoss(), [], [0, 1]),
201
+ (HalfMultinomialLoss(), [0.1, 0.5], [0, 1]),
202
+ ]
203
+
204
+
205
+ @pytest.mark.parametrize(
206
+ "loss, y_true_success, y_true_fail",
207
+ Y_COMMON_PARAMS + Y_TRUE_PARAMS, # type: ignore[operator]
208
+ )
209
+ def test_loss_boundary_y_true(loss, y_true_success, y_true_fail):
210
+ """Test boundaries of y_true for loss functions."""
211
+ for y in y_true_success:
212
+ assert loss.in_y_true_range(np.array([y]))
213
+ for y in y_true_fail:
214
+ assert not loss.in_y_true_range(np.array([y]))
215
+
216
+
217
+ @pytest.mark.parametrize(
218
+ "loss, y_pred_success, y_pred_fail",
219
+ Y_COMMON_PARAMS + Y_PRED_PARAMS, # type: ignore[operator]
220
+ )
221
+ def test_loss_boundary_y_pred(loss, y_pred_success, y_pred_fail):
222
+ """Test boundaries of y_pred for loss functions."""
223
+ for y in y_pred_success:
224
+ assert loss.in_y_pred_range(np.array([y]))
225
+ for y in y_pred_fail:
226
+ assert not loss.in_y_pred_range(np.array([y]))
227
+
228
+
229
+ @pytest.mark.parametrize(
230
+ "loss, y_true, raw_prediction, loss_true, gradient_true, hessian_true",
231
+ [
232
+ (HalfSquaredError(), 1.0, 5.0, 8, 4, 1),
233
+ (AbsoluteError(), 1.0, 5.0, 4.0, 1.0, None),
234
+ (PinballLoss(quantile=0.5), 1.0, 5.0, 2, 0.5, None),
235
+ (PinballLoss(quantile=0.25), 1.0, 5.0, 4 * (1 - 0.25), 1 - 0.25, None),
236
+ (PinballLoss(quantile=0.25), 5.0, 1.0, 4 * 0.25, -0.25, None),
237
+ (HuberLoss(quantile=0.5, delta=3), 1.0, 5.0, 3 * (4 - 3 / 2), None, None),
238
+ (HuberLoss(quantile=0.5, delta=3), 1.0, 3.0, 0.5 * 2**2, None, None),
239
+ (HalfPoissonLoss(), 2.0, np.log(4), 4 - 2 * np.log(4), 4 - 2, 4),
240
+ (HalfGammaLoss(), 2.0, np.log(4), np.log(4) + 2 / 4, 1 - 2 / 4, 2 / 4),
241
+ (HalfTweedieLoss(power=3), 2.0, np.log(4), -1 / 4 + 1 / 4**2, None, None),
242
+ (HalfTweedieLossIdentity(power=1), 2.0, 4.0, 2 - 2 * np.log(2), None, None),
243
+ (HalfTweedieLossIdentity(power=2), 2.0, 4.0, np.log(2) - 1 / 2, None, None),
244
+ (
245
+ HalfTweedieLossIdentity(power=3),
246
+ 2.0,
247
+ 4.0,
248
+ -1 / 4 + 1 / 4**2 + 1 / 2 / 2,
249
+ None,
250
+ None,
251
+ ),
252
+ (
253
+ HalfBinomialLoss(),
254
+ 0.25,
255
+ np.log(4),
256
+ np.log1p(4) - 0.25 * np.log(4),
257
+ None,
258
+ None,
259
+ ),
260
+ # Extreme log loss cases, checked with mpmath:
261
+ # import mpmath as mp
262
+ #
263
+ # # Stolen from scipy
264
+ # def mpf2float(x):
265
+ # return float(mp.nstr(x, 17, min_fixed=0, max_fixed=0))
266
+ #
267
+ # def mp_logloss(y_true, raw):
268
+ # with mp.workdps(100):
269
+ # y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw))
270
+ # out = mp.log1p(mp.exp(raw)) - y_true * raw
271
+ # return mpf2float(out)
272
+ #
273
+ # def mp_gradient(y_true, raw):
274
+ # with mp.workdps(100):
275
+ # y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw))
276
+ # out = mp.mpf(1) / (mp.mpf(1) + mp.exp(-raw)) - y_true
277
+ # return mpf2float(out)
278
+ #
279
+ # def mp_hessian(y_true, raw):
280
+ # with mp.workdps(100):
281
+ # y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw))
282
+ # p = mp.mpf(1) / (mp.mpf(1) + mp.exp(-raw))
283
+ # out = p * (mp.mpf(1) - p)
284
+ # return mpf2float(out)
285
+ #
286
+ # y, raw = 0.0, 37.
287
+ # mp_logloss(y, raw), mp_gradient(y, raw), mp_hessian(y, raw)
288
+ (HalfBinomialLoss(), 0.0, -1e20, 0, 0, 0),
289
+ (HalfBinomialLoss(), 1.0, -1e20, 1e20, -1, 0),
290
+ (HalfBinomialLoss(), 0.0, -1e3, 0, 0, 0),
291
+ (HalfBinomialLoss(), 1.0, -1e3, 1e3, -1, 0),
292
+ (HalfBinomialLoss(), 1.0, -37.5, 37.5, -1, 0),
293
+ (HalfBinomialLoss(), 1.0, -37.0, 37, 1e-16 - 1, 8.533047625744065e-17),
294
+ (HalfBinomialLoss(), 0.0, -37.0, *[8.533047625744065e-17] * 3),
295
+ (HalfBinomialLoss(), 1.0, -36.9, 36.9, 1e-16 - 1, 9.430476078526806e-17),
296
+ (HalfBinomialLoss(), 0.0, -36.9, *[9.430476078526806e-17] * 3),
297
+ (HalfBinomialLoss(), 0.0, 37.0, 37, 1 - 1e-16, 8.533047625744065e-17),
298
+ (HalfBinomialLoss(), 1.0, 37.0, *[8.533047625744066e-17] * 3),
299
+ (HalfBinomialLoss(), 0.0, 37.5, 37.5, 1, 5.175555005801868e-17),
300
+ (HalfBinomialLoss(), 0.0, 232.8, 232.8, 1, 1.4287342391028437e-101),
301
+ (HalfBinomialLoss(), 1.0, 1e20, 0, 0, 0),
302
+ (HalfBinomialLoss(), 0.0, 1e20, 1e20, 1, 0),
303
+ (
304
+ HalfBinomialLoss(),
305
+ 1.0,
306
+ 232.8,
307
+ 0,
308
+ -1.4287342391028437e-101,
309
+ 1.4287342391028437e-101,
310
+ ),
311
+ (HalfBinomialLoss(), 1.0, 232.9, 0, 0, 0),
312
+ (HalfBinomialLoss(), 1.0, 1e3, 0, 0, 0),
313
+ (HalfBinomialLoss(), 0.0, 1e3, 1e3, 1, 0),
314
+ (
315
+ HalfMultinomialLoss(n_classes=3),
316
+ 0.0,
317
+ [0.2, 0.5, 0.3],
318
+ logsumexp([0.2, 0.5, 0.3]) - 0.2,
319
+ None,
320
+ None,
321
+ ),
322
+ (
323
+ HalfMultinomialLoss(n_classes=3),
324
+ 1.0,
325
+ [0.2, 0.5, 0.3],
326
+ logsumexp([0.2, 0.5, 0.3]) - 0.5,
327
+ None,
328
+ None,
329
+ ),
330
+ (
331
+ HalfMultinomialLoss(n_classes=3),
332
+ 2.0,
333
+ [0.2, 0.5, 0.3],
334
+ logsumexp([0.2, 0.5, 0.3]) - 0.3,
335
+ None,
336
+ None,
337
+ ),
338
+ (
339
+ HalfMultinomialLoss(n_classes=3),
340
+ 2.0,
341
+ [1e4, 0, 7e-7],
342
+ logsumexp([1e4, 0, 7e-7]) - (7e-7),
343
+ None,
344
+ None,
345
+ ),
346
+ ],
347
+ ids=loss_instance_name,
348
+ )
349
+ def test_loss_on_specific_values(
350
+ loss, y_true, raw_prediction, loss_true, gradient_true, hessian_true
351
+ ):
352
+ """Test losses, gradients and hessians at specific values."""
353
+ loss1 = loss(y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction]))
354
+ grad1 = loss.gradient(
355
+ y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction])
356
+ )
357
+ loss2, grad2 = loss.loss_gradient(
358
+ y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction])
359
+ )
360
+ grad3, hess = loss.gradient_hessian(
361
+ y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction])
362
+ )
363
+
364
+ assert loss1 == approx(loss_true, rel=1e-15, abs=1e-15)
365
+ assert loss2 == approx(loss_true, rel=1e-15, abs=1e-15)
366
+
367
+ if gradient_true is not None:
368
+ assert grad1 == approx(gradient_true, rel=1e-15, abs=1e-15)
369
+ assert grad2 == approx(gradient_true, rel=1e-15, abs=1e-15)
370
+ assert grad3 == approx(gradient_true, rel=1e-15, abs=1e-15)
371
+
372
+ if hessian_true is not None:
373
+ assert hess == approx(hessian_true, rel=1e-15, abs=1e-15)
374
+
375
+
376
+ @pytest.mark.parametrize("loss", ALL_LOSSES)
377
+ @pytest.mark.parametrize("readonly_memmap", [False, True])
378
+ @pytest.mark.parametrize("dtype_in", [np.float32, np.float64])
379
+ @pytest.mark.parametrize("dtype_out", [np.float32, np.float64])
380
+ @pytest.mark.parametrize("sample_weight", [None, 1])
381
+ @pytest.mark.parametrize("out1", [None, 1])
382
+ @pytest.mark.parametrize("out2", [None, 1])
383
+ @pytest.mark.parametrize("n_threads", [1, 2])
384
+ def test_loss_dtype(
385
+ loss, readonly_memmap, dtype_in, dtype_out, sample_weight, out1, out2, n_threads
386
+ ):
387
+ """Test acceptance of dtypes, readonly and writeable arrays in loss functions.
388
+
389
+ Check that loss accepts if all input arrays are either all float32 or all
390
+ float64, and all output arrays are either all float32 or all float64.
391
+
392
+ Also check that input arrays can be readonly, e.g. memory mapped.
393
+ """
394
+ loss = loss()
395
+ # generate a y_true and raw_prediction in valid range
396
+ n_samples = 5
397
+ y_true, raw_prediction = random_y_true_raw_prediction(
398
+ loss=loss,
399
+ n_samples=n_samples,
400
+ y_bound=(-100, 100),
401
+ raw_bound=(-10, 10),
402
+ seed=42,
403
+ )
404
+ y_true = y_true.astype(dtype_in)
405
+ raw_prediction = raw_prediction.astype(dtype_in)
406
+
407
+ if sample_weight is not None:
408
+ sample_weight = np.array([2.0] * n_samples, dtype=dtype_in)
409
+ if out1 is not None:
410
+ out1 = np.empty_like(y_true, dtype=dtype_out)
411
+ if out2 is not None:
412
+ out2 = np.empty_like(raw_prediction, dtype=dtype_out)
413
+
414
+ if readonly_memmap:
415
+ y_true = create_memmap_backed_data(y_true)
416
+ raw_prediction = create_memmap_backed_data(raw_prediction)
417
+ if sample_weight is not None:
418
+ sample_weight = create_memmap_backed_data(sample_weight)
419
+
420
+ l = loss.loss(
421
+ y_true=y_true,
422
+ raw_prediction=raw_prediction,
423
+ sample_weight=sample_weight,
424
+ loss_out=out1,
425
+ n_threads=n_threads,
426
+ )
427
+ assert l is out1 if out1 is not None else True
428
+ g = loss.gradient(
429
+ y_true=y_true,
430
+ raw_prediction=raw_prediction,
431
+ sample_weight=sample_weight,
432
+ gradient_out=out2,
433
+ n_threads=n_threads,
434
+ )
435
+ assert g is out2 if out2 is not None else True
436
+ l, g = loss.loss_gradient(
437
+ y_true=y_true,
438
+ raw_prediction=raw_prediction,
439
+ sample_weight=sample_weight,
440
+ loss_out=out1,
441
+ gradient_out=out2,
442
+ n_threads=n_threads,
443
+ )
444
+ assert l is out1 if out1 is not None else True
445
+ assert g is out2 if out2 is not None else True
446
+ if out1 is not None and loss.is_multiclass:
447
+ out1 = np.empty_like(raw_prediction, dtype=dtype_out)
448
+ g, h = loss.gradient_hessian(
449
+ y_true=y_true,
450
+ raw_prediction=raw_prediction,
451
+ sample_weight=sample_weight,
452
+ gradient_out=out1,
453
+ hessian_out=out2,
454
+ n_threads=n_threads,
455
+ )
456
+ assert g is out1 if out1 is not None else True
457
+ assert h is out2 if out2 is not None else True
458
+ loss(y_true=y_true, raw_prediction=raw_prediction, sample_weight=sample_weight)
459
+ loss.fit_intercept_only(y_true=y_true, sample_weight=sample_weight)
460
+ loss.constant_to_optimal_zero(y_true=y_true, sample_weight=sample_weight)
461
+ if hasattr(loss, "predict_proba"):
462
+ loss.predict_proba(raw_prediction=raw_prediction)
463
+ if hasattr(loss, "gradient_proba"):
464
+ g, p = loss.gradient_proba(
465
+ y_true=y_true,
466
+ raw_prediction=raw_prediction,
467
+ sample_weight=sample_weight,
468
+ gradient_out=out1,
469
+ proba_out=out2,
470
+ n_threads=n_threads,
471
+ )
472
+ assert g is out1 if out1 is not None else True
473
+ assert p is out2 if out2 is not None else True
474
+
475
+
476
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
477
+ @pytest.mark.parametrize("sample_weight", [None, "range"])
478
+ def test_loss_same_as_C_functions(loss, sample_weight):
479
+ """Test that Python and Cython functions return same results."""
480
+ y_true, raw_prediction = random_y_true_raw_prediction(
481
+ loss=loss,
482
+ n_samples=20,
483
+ y_bound=(-100, 100),
484
+ raw_bound=(-10, 10),
485
+ seed=42,
486
+ )
487
+ if sample_weight == "range":
488
+ sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
489
+
490
+ out_l1 = np.empty_like(y_true)
491
+ out_l2 = np.empty_like(y_true)
492
+ out_g1 = np.empty_like(raw_prediction)
493
+ out_g2 = np.empty_like(raw_prediction)
494
+ out_h1 = np.empty_like(raw_prediction)
495
+ out_h2 = np.empty_like(raw_prediction)
496
+ loss.loss(
497
+ y_true=y_true,
498
+ raw_prediction=raw_prediction,
499
+ sample_weight=sample_weight,
500
+ loss_out=out_l1,
501
+ )
502
+ loss.closs.loss(
503
+ y_true=y_true,
504
+ raw_prediction=raw_prediction,
505
+ sample_weight=sample_weight,
506
+ loss_out=out_l2,
507
+ )
508
+ assert_allclose(out_l1, out_l2)
509
+ loss.gradient(
510
+ y_true=y_true,
511
+ raw_prediction=raw_prediction,
512
+ sample_weight=sample_weight,
513
+ gradient_out=out_g1,
514
+ )
515
+ loss.closs.gradient(
516
+ y_true=y_true,
517
+ raw_prediction=raw_prediction,
518
+ sample_weight=sample_weight,
519
+ gradient_out=out_g2,
520
+ )
521
+ assert_allclose(out_g1, out_g2)
522
+ loss.closs.loss_gradient(
523
+ y_true=y_true,
524
+ raw_prediction=raw_prediction,
525
+ sample_weight=sample_weight,
526
+ loss_out=out_l1,
527
+ gradient_out=out_g1,
528
+ )
529
+ loss.closs.loss_gradient(
530
+ y_true=y_true,
531
+ raw_prediction=raw_prediction,
532
+ sample_weight=sample_weight,
533
+ loss_out=out_l2,
534
+ gradient_out=out_g2,
535
+ )
536
+ assert_allclose(out_l1, out_l2)
537
+ assert_allclose(out_g1, out_g2)
538
+ loss.gradient_hessian(
539
+ y_true=y_true,
540
+ raw_prediction=raw_prediction,
541
+ sample_weight=sample_weight,
542
+ gradient_out=out_g1,
543
+ hessian_out=out_h1,
544
+ )
545
+ loss.closs.gradient_hessian(
546
+ y_true=y_true,
547
+ raw_prediction=raw_prediction,
548
+ sample_weight=sample_weight,
549
+ gradient_out=out_g2,
550
+ hessian_out=out_h2,
551
+ )
552
+ assert_allclose(out_g1, out_g2)
553
+ assert_allclose(out_h1, out_h2)
554
+
555
+
556
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
557
+ @pytest.mark.parametrize("sample_weight", [None, "range"])
558
+ def test_loss_gradients_are_the_same(loss, sample_weight, global_random_seed):
559
+ """Test that loss and gradient are the same across different functions.
560
+
561
+ Also test that output arguments contain correct results.
562
+ """
563
+ y_true, raw_prediction = random_y_true_raw_prediction(
564
+ loss=loss,
565
+ n_samples=20,
566
+ y_bound=(-100, 100),
567
+ raw_bound=(-10, 10),
568
+ seed=global_random_seed,
569
+ )
570
+ if sample_weight == "range":
571
+ sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
572
+
573
+ out_l1 = np.empty_like(y_true)
574
+ out_l2 = np.empty_like(y_true)
575
+ out_g1 = np.empty_like(raw_prediction)
576
+ out_g2 = np.empty_like(raw_prediction)
577
+ out_g3 = np.empty_like(raw_prediction)
578
+ out_h3 = np.empty_like(raw_prediction)
579
+
580
+ l1 = loss.loss(
581
+ y_true=y_true,
582
+ raw_prediction=raw_prediction,
583
+ sample_weight=sample_weight,
584
+ loss_out=out_l1,
585
+ )
586
+ g1 = loss.gradient(
587
+ y_true=y_true,
588
+ raw_prediction=raw_prediction,
589
+ sample_weight=sample_weight,
590
+ gradient_out=out_g1,
591
+ )
592
+ l2, g2 = loss.loss_gradient(
593
+ y_true=y_true,
594
+ raw_prediction=raw_prediction,
595
+ sample_weight=sample_weight,
596
+ loss_out=out_l2,
597
+ gradient_out=out_g2,
598
+ )
599
+ g3, h3 = loss.gradient_hessian(
600
+ y_true=y_true,
601
+ raw_prediction=raw_prediction,
602
+ sample_weight=sample_weight,
603
+ gradient_out=out_g3,
604
+ hessian_out=out_h3,
605
+ )
606
+ assert_allclose(l1, l2)
607
+ assert_array_equal(l1, out_l1)
608
+ assert np.shares_memory(l1, out_l1)
609
+ assert_array_equal(l2, out_l2)
610
+ assert np.shares_memory(l2, out_l2)
611
+ assert_allclose(g1, g2)
612
+ assert_allclose(g1, g3)
613
+ assert_array_equal(g1, out_g1)
614
+ assert np.shares_memory(g1, out_g1)
615
+ assert_array_equal(g2, out_g2)
616
+ assert np.shares_memory(g2, out_g2)
617
+ assert_array_equal(g3, out_g3)
618
+ assert np.shares_memory(g3, out_g3)
619
+
620
+ if hasattr(loss, "gradient_proba"):
621
+ assert loss.is_multiclass # only for HalfMultinomialLoss
622
+ out_g4 = np.empty_like(raw_prediction)
623
+ out_proba = np.empty_like(raw_prediction)
624
+ g4, proba = loss.gradient_proba(
625
+ y_true=y_true,
626
+ raw_prediction=raw_prediction,
627
+ sample_weight=sample_weight,
628
+ gradient_out=out_g4,
629
+ proba_out=out_proba,
630
+ )
631
+ assert_allclose(g1, out_g4)
632
+ assert_allclose(g1, g4)
633
+ assert_allclose(proba, out_proba)
634
+ assert_allclose(np.sum(proba, axis=1), 1, rtol=1e-11)
635
+
636
+
637
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
638
+ @pytest.mark.parametrize("sample_weight", ["ones", "random"])
639
+ def test_sample_weight_multiplies(loss, sample_weight, global_random_seed):
640
+ """Test sample weights in loss, gradients and hessians.
641
+
642
+ Make sure that passing sample weights to loss, gradient and hessian
643
+ computation methods is equivalent to multiplying by the weights.
644
+ """
645
+ n_samples = 100
646
+ y_true, raw_prediction = random_y_true_raw_prediction(
647
+ loss=loss,
648
+ n_samples=n_samples,
649
+ y_bound=(-100, 100),
650
+ raw_bound=(-5, 5),
651
+ seed=global_random_seed,
652
+ )
653
+
654
+ if sample_weight == "ones":
655
+ sample_weight = np.ones(shape=n_samples, dtype=np.float64)
656
+ else:
657
+ rng = np.random.RandomState(global_random_seed)
658
+ sample_weight = rng.normal(size=n_samples).astype(np.float64)
659
+
660
+ assert_allclose(
661
+ loss.loss(
662
+ y_true=y_true,
663
+ raw_prediction=raw_prediction,
664
+ sample_weight=sample_weight,
665
+ ),
666
+ sample_weight
667
+ * loss.loss(
668
+ y_true=y_true,
669
+ raw_prediction=raw_prediction,
670
+ sample_weight=None,
671
+ ),
672
+ )
673
+
674
+ losses, gradient = loss.loss_gradient(
675
+ y_true=y_true,
676
+ raw_prediction=raw_prediction,
677
+ sample_weight=None,
678
+ )
679
+ losses_sw, gradient_sw = loss.loss_gradient(
680
+ y_true=y_true,
681
+ raw_prediction=raw_prediction,
682
+ sample_weight=sample_weight,
683
+ )
684
+ assert_allclose(losses * sample_weight, losses_sw)
685
+ if not loss.is_multiclass:
686
+ assert_allclose(gradient * sample_weight, gradient_sw)
687
+ else:
688
+ assert_allclose(gradient * sample_weight[:, None], gradient_sw)
689
+
690
+ gradient, hessian = loss.gradient_hessian(
691
+ y_true=y_true,
692
+ raw_prediction=raw_prediction,
693
+ sample_weight=None,
694
+ )
695
+ gradient_sw, hessian_sw = loss.gradient_hessian(
696
+ y_true=y_true,
697
+ raw_prediction=raw_prediction,
698
+ sample_weight=sample_weight,
699
+ )
700
+ if not loss.is_multiclass:
701
+ assert_allclose(gradient * sample_weight, gradient_sw)
702
+ assert_allclose(hessian * sample_weight, hessian_sw)
703
+ else:
704
+ assert_allclose(gradient * sample_weight[:, None], gradient_sw)
705
+ assert_allclose(hessian * sample_weight[:, None], hessian_sw)
706
+
707
+
708
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
709
+ def test_graceful_squeezing(loss):
710
+ """Test that reshaped raw_prediction gives same results."""
711
+ y_true, raw_prediction = random_y_true_raw_prediction(
712
+ loss=loss,
713
+ n_samples=20,
714
+ y_bound=(-100, 100),
715
+ raw_bound=(-10, 10),
716
+ seed=42,
717
+ )
718
+
719
+ if raw_prediction.ndim == 1:
720
+ raw_prediction_2d = raw_prediction[:, None]
721
+ assert_allclose(
722
+ loss.loss(y_true=y_true, raw_prediction=raw_prediction_2d),
723
+ loss.loss(y_true=y_true, raw_prediction=raw_prediction),
724
+ )
725
+ assert_allclose(
726
+ loss.loss_gradient(y_true=y_true, raw_prediction=raw_prediction_2d),
727
+ loss.loss_gradient(y_true=y_true, raw_prediction=raw_prediction),
728
+ )
729
+ assert_allclose(
730
+ loss.gradient(y_true=y_true, raw_prediction=raw_prediction_2d),
731
+ loss.gradient(y_true=y_true, raw_prediction=raw_prediction),
732
+ )
733
+ assert_allclose(
734
+ loss.gradient_hessian(y_true=y_true, raw_prediction=raw_prediction_2d),
735
+ loss.gradient_hessian(y_true=y_true, raw_prediction=raw_prediction),
736
+ )
737
+
738
+
739
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
740
+ @pytest.mark.parametrize("sample_weight", [None, "range"])
741
+ def test_loss_of_perfect_prediction(loss, sample_weight):
742
+ """Test value of perfect predictions.
743
+
744
+ Loss of y_pred = y_true plus constant_to_optimal_zero should sums up to
745
+ zero.
746
+ """
747
+ if not loss.is_multiclass:
748
+ # Use small values such that exp(value) is not nan.
749
+ raw_prediction = np.array([-10, -0.1, 0, 0.1, 3, 10])
750
+ # If link is identity, we must respect the interval of y_pred:
751
+ if isinstance(loss.link, IdentityLink):
752
+ eps = 1e-10
753
+ low = loss.interval_y_pred.low
754
+ if not loss.interval_y_pred.low_inclusive:
755
+ low = low + eps
756
+ high = loss.interval_y_pred.high
757
+ if not loss.interval_y_pred.high_inclusive:
758
+ high = high - eps
759
+ raw_prediction = np.clip(raw_prediction, low, high)
760
+ y_true = loss.link.inverse(raw_prediction)
761
+ else:
762
+ # HalfMultinomialLoss
763
+ y_true = np.arange(loss.n_classes).astype(float)
764
+ # raw_prediction with entries -exp(10), but +exp(10) on the diagonal
765
+ # this is close enough to np.inf which would produce nan
766
+ raw_prediction = np.full(
767
+ shape=(loss.n_classes, loss.n_classes),
768
+ fill_value=-np.exp(10),
769
+ dtype=float,
770
+ )
771
+ raw_prediction.flat[:: loss.n_classes + 1] = np.exp(10)
772
+
773
+ if sample_weight == "range":
774
+ sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
775
+
776
+ loss_value = loss.loss(
777
+ y_true=y_true,
778
+ raw_prediction=raw_prediction,
779
+ sample_weight=sample_weight,
780
+ )
781
+ constant_term = loss.constant_to_optimal_zero(
782
+ y_true=y_true, sample_weight=sample_weight
783
+ )
784
+ # Comparing loss_value + constant_term to zero would result in large
785
+ # round-off errors.
786
+ assert_allclose(loss_value, -constant_term, atol=1e-14, rtol=1e-15)
787
+
788
+
789
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
790
+ @pytest.mark.parametrize("sample_weight", [None, "range"])
791
+ def test_gradients_hessians_numerically(loss, sample_weight, global_random_seed):
792
+ """Test gradients and hessians with numerical derivatives.
793
+
794
+ Gradient should equal the numerical derivatives of the loss function.
795
+ Hessians should equal the numerical derivatives of gradients.
796
+ """
797
+ n_samples = 20
798
+ y_true, raw_prediction = random_y_true_raw_prediction(
799
+ loss=loss,
800
+ n_samples=n_samples,
801
+ y_bound=(-100, 100),
802
+ raw_bound=(-5, 5),
803
+ seed=global_random_seed,
804
+ )
805
+
806
+ if sample_weight == "range":
807
+ sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0])
808
+
809
+ g, h = loss.gradient_hessian(
810
+ y_true=y_true,
811
+ raw_prediction=raw_prediction,
812
+ sample_weight=sample_weight,
813
+ )
814
+
815
+ assert g.shape == raw_prediction.shape
816
+ assert h.shape == raw_prediction.shape
817
+
818
+ if not loss.is_multiclass:
819
+
820
+ def loss_func(x):
821
+ return loss.loss(
822
+ y_true=y_true,
823
+ raw_prediction=x,
824
+ sample_weight=sample_weight,
825
+ )
826
+
827
+ g_numeric = numerical_derivative(loss_func, raw_prediction, eps=1e-6)
828
+ assert_allclose(g, g_numeric, rtol=5e-6, atol=1e-10)
829
+
830
+ def grad_func(x):
831
+ return loss.gradient(
832
+ y_true=y_true,
833
+ raw_prediction=x,
834
+ sample_weight=sample_weight,
835
+ )
836
+
837
+ h_numeric = numerical_derivative(grad_func, raw_prediction, eps=1e-6)
838
+ if loss.approx_hessian:
839
+ # TODO: What could we test if loss.approx_hessian?
840
+ pass
841
+ else:
842
+ assert_allclose(h, h_numeric, rtol=5e-6, atol=1e-10)
843
+ else:
844
+ # For multiclass loss, we should only change the predictions of the
845
+ # class for which the derivative is taken for, e.g. offset[:, k] = eps
846
+ # for class k.
847
+ # As a softmax is computed, offsetting the whole array by a constant
848
+ # would have no effect on the probabilities, and thus on the loss.
849
+ for k in range(loss.n_classes):
850
+
851
+ def loss_func(x):
852
+ raw = raw_prediction.copy()
853
+ raw[:, k] = x
854
+ return loss.loss(
855
+ y_true=y_true,
856
+ raw_prediction=raw,
857
+ sample_weight=sample_weight,
858
+ )
859
+
860
+ g_numeric = numerical_derivative(loss_func, raw_prediction[:, k], eps=1e-5)
861
+ assert_allclose(g[:, k], g_numeric, rtol=5e-6, atol=1e-10)
862
+
863
+ def grad_func(x):
864
+ raw = raw_prediction.copy()
865
+ raw[:, k] = x
866
+ return loss.gradient(
867
+ y_true=y_true,
868
+ raw_prediction=raw,
869
+ sample_weight=sample_weight,
870
+ )[:, k]
871
+
872
+ h_numeric = numerical_derivative(grad_func, raw_prediction[:, k], eps=1e-6)
873
+ if loss.approx_hessian:
874
+ # TODO: What could we test if loss.approx_hessian?
875
+ pass
876
+ else:
877
+ assert_allclose(h[:, k], h_numeric, rtol=5e-6, atol=1e-10)
878
+
879
+
880
+ @pytest.mark.parametrize(
881
+ "loss, x0, y_true",
882
+ [
883
+ ("squared_error", -2.0, 42),
884
+ ("squared_error", 117.0, 1.05),
885
+ ("squared_error", 0.0, 0.0),
886
+ # The argmin of binomial_loss for y_true=0 and y_true=1 is resp.
887
+ # -inf and +inf due to logit, cf. "complete separation". Therefore, we
888
+ # use 0 < y_true < 1.
889
+ ("binomial_loss", 0.3, 0.1),
890
+ ("binomial_loss", -12, 0.2),
891
+ ("binomial_loss", 30, 0.9),
892
+ ("poisson_loss", 12.0, 1.0),
893
+ ("poisson_loss", 0.0, 2.0),
894
+ ("poisson_loss", -22.0, 10.0),
895
+ ],
896
+ )
897
+ @skip_if_32bit
898
+ def test_derivatives(loss, x0, y_true):
899
+ """Test that gradients are zero at the minimum of the loss.
900
+
901
+ We check this on a single value/sample using Halley's method with the
902
+ first and second order derivatives computed by the Loss instance.
903
+ Note that methods of Loss instances operate on arrays while the newton
904
+ root finder expects a scalar or a one-element array for this purpose.
905
+ """
906
+ loss = _LOSSES[loss](sample_weight=None)
907
+ y_true = np.array([y_true], dtype=np.float64)
908
+ x0 = np.array([x0], dtype=np.float64)
909
+
910
+ def func(x: np.ndarray) -> np.ndarray:
911
+ """Compute loss plus constant term.
912
+
913
+ The constant term is such that the minimum function value is zero,
914
+ which is required by the Newton method.
915
+ """
916
+ return loss.loss(
917
+ y_true=y_true, raw_prediction=x
918
+ ) + loss.constant_to_optimal_zero(y_true=y_true)
919
+
920
+ def fprime(x: np.ndarray) -> np.ndarray:
921
+ return loss.gradient(y_true=y_true, raw_prediction=x)
922
+
923
+ def fprime2(x: np.ndarray) -> np.ndarray:
924
+ return loss.gradient_hessian(y_true=y_true, raw_prediction=x)[1]
925
+
926
+ optimum = newton(
927
+ func,
928
+ x0=x0,
929
+ fprime=fprime,
930
+ fprime2=fprime2,
931
+ maxiter=100,
932
+ tol=5e-8,
933
+ )
934
+
935
+ # Need to ravel arrays because assert_allclose requires matching
936
+ # dimensions.
937
+ y_true = y_true.ravel()
938
+ optimum = optimum.ravel()
939
+ assert_allclose(loss.link.inverse(optimum), y_true)
940
+ assert_allclose(func(optimum), 0, atol=1e-14)
941
+ assert_allclose(loss.gradient(y_true=y_true, raw_prediction=optimum), 0, atol=5e-7)
942
+
943
+
944
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
945
+ @pytest.mark.parametrize("sample_weight", [None, "range"])
946
+ def test_loss_intercept_only(loss, sample_weight):
947
+ """Test that fit_intercept_only returns the argmin of the loss.
948
+
949
+ Also test that the gradient is zero at the minimum.
950
+ """
951
+ n_samples = 50
952
+ if not loss.is_multiclass:
953
+ y_true = loss.link.inverse(np.linspace(-4, 4, num=n_samples))
954
+ else:
955
+ y_true = np.arange(n_samples).astype(np.float64) % loss.n_classes
956
+ y_true[::5] = 0 # exceedance of class 0
957
+
958
+ if sample_weight == "range":
959
+ sample_weight = np.linspace(0.1, 2, num=n_samples)
960
+
961
+ a = loss.fit_intercept_only(y_true=y_true, sample_weight=sample_weight)
962
+
963
+ # find minimum by optimization
964
+ def fun(x):
965
+ if not loss.is_multiclass:
966
+ raw_prediction = np.full(shape=(n_samples), fill_value=x)
967
+ else:
968
+ raw_prediction = np.ascontiguousarray(
969
+ np.broadcast_to(x, shape=(n_samples, loss.n_classes))
970
+ )
971
+ return loss(
972
+ y_true=y_true,
973
+ raw_prediction=raw_prediction,
974
+ sample_weight=sample_weight,
975
+ )
976
+
977
+ if not loss.is_multiclass:
978
+ opt = minimize_scalar(fun, tol=1e-7, options={"maxiter": 100})
979
+ grad = loss.gradient(
980
+ y_true=y_true,
981
+ raw_prediction=np.full_like(y_true, a),
982
+ sample_weight=sample_weight,
983
+ )
984
+ assert a.shape == tuple() # scalar
985
+ assert a.dtype == y_true.dtype
986
+ assert_all_finite(a)
987
+ a == approx(opt.x, rel=1e-7)
988
+ grad.sum() == approx(0, abs=1e-12)
989
+ else:
990
+ # The constraint corresponds to sum(raw_prediction) = 0. Without it, we would
991
+ # need to apply loss.symmetrize_raw_prediction to opt.x before comparing.
992
+ opt = minimize(
993
+ fun,
994
+ np.zeros((loss.n_classes)),
995
+ tol=1e-13,
996
+ options={"maxiter": 100},
997
+ method="SLSQP",
998
+ constraints=LinearConstraint(np.ones((1, loss.n_classes)), 0, 0),
999
+ )
1000
+ grad = loss.gradient(
1001
+ y_true=y_true,
1002
+ raw_prediction=np.tile(a, (n_samples, 1)),
1003
+ sample_weight=sample_weight,
1004
+ )
1005
+ assert a.dtype == y_true.dtype
1006
+ assert_all_finite(a)
1007
+ assert_allclose(a, opt.x, rtol=5e-6, atol=1e-12)
1008
+ assert_allclose(grad.sum(axis=0), 0, atol=1e-12)
1009
+
1010
+
1011
+ @pytest.mark.parametrize(
1012
+ "loss, func, random_dist",
1013
+ [
1014
+ (HalfSquaredError(), np.mean, "normal"),
1015
+ (AbsoluteError(), np.median, "normal"),
1016
+ (PinballLoss(quantile=0.25), lambda x: np.percentile(x, q=25), "normal"),
1017
+ (HalfPoissonLoss(), np.mean, "poisson"),
1018
+ (HalfGammaLoss(), np.mean, "exponential"),
1019
+ (HalfTweedieLoss(), np.mean, "exponential"),
1020
+ (HalfBinomialLoss(), np.mean, "binomial"),
1021
+ ],
1022
+ )
1023
+ def test_specific_fit_intercept_only(loss, func, random_dist, global_random_seed):
1024
+ """Test that fit_intercept_only returns the correct functional.
1025
+
1026
+ We test the functional for specific, meaningful distributions, e.g.
1027
+ squared error estimates the expectation of a probability distribution.
1028
+ """
1029
+ rng = np.random.RandomState(global_random_seed)
1030
+ if random_dist == "binomial":
1031
+ y_train = rng.binomial(1, 0.5, size=100)
1032
+ else:
1033
+ y_train = getattr(rng, random_dist)(size=100)
1034
+ baseline_prediction = loss.fit_intercept_only(y_true=y_train)
1035
+ # Make sure baseline prediction is the expected functional=func, e.g. mean
1036
+ # or median.
1037
+ assert_all_finite(baseline_prediction)
1038
+ assert baseline_prediction == approx(loss.link.link(func(y_train)))
1039
+ assert loss.link.inverse(baseline_prediction) == approx(func(y_train))
1040
+ if isinstance(loss, IdentityLink):
1041
+ assert_allclose(loss.link.inverse(baseline_prediction), baseline_prediction)
1042
+
1043
+ # Test baseline at boundary
1044
+ if loss.interval_y_true.low_inclusive:
1045
+ y_train.fill(loss.interval_y_true.low)
1046
+ baseline_prediction = loss.fit_intercept_only(y_true=y_train)
1047
+ assert_all_finite(baseline_prediction)
1048
+ if loss.interval_y_true.high_inclusive:
1049
+ y_train.fill(loss.interval_y_true.high)
1050
+ baseline_prediction = loss.fit_intercept_only(y_true=y_train)
1051
+ assert_all_finite(baseline_prediction)
1052
+
1053
+
1054
+ def test_multinomial_loss_fit_intercept_only():
1055
+ """Test that fit_intercept_only returns the mean functional for CCE."""
1056
+ rng = np.random.RandomState(0)
1057
+ n_classes = 4
1058
+ loss = HalfMultinomialLoss(n_classes=n_classes)
1059
+ # Same logic as test_specific_fit_intercept_only. Here inverse link
1060
+ # function = softmax and link function = log - symmetry term.
1061
+ y_train = rng.randint(0, n_classes + 1, size=100).astype(np.float64)
1062
+ baseline_prediction = loss.fit_intercept_only(y_true=y_train)
1063
+ assert baseline_prediction.shape == (n_classes,)
1064
+ p = np.zeros(n_classes, dtype=y_train.dtype)
1065
+ for k in range(n_classes):
1066
+ p[k] = (y_train == k).mean()
1067
+ assert_allclose(baseline_prediction, np.log(p) - np.mean(np.log(p)))
1068
+ assert_allclose(baseline_prediction[None, :], loss.link.link(p[None, :]))
1069
+
1070
+ for y_train in (np.zeros(shape=10), np.ones(shape=10)):
1071
+ y_train = y_train.astype(np.float64)
1072
+ baseline_prediction = loss.fit_intercept_only(y_true=y_train)
1073
+ assert baseline_prediction.dtype == y_train.dtype
1074
+ assert_all_finite(baseline_prediction)
1075
+
1076
+
1077
+ def test_multinomial_cy_gradient(global_random_seed):
1078
+ """Test that Multinomial cy_gradient gives the same result as gradient.
1079
+
1080
+ CyHalfMultinomialLoss does not inherit from CyLossFunction and has a different API.
1081
+ As a consequence, the functions like `loss` and `gradient` do not rely on `cy_loss`
1082
+ and `cy_gradient`.
1083
+ """
1084
+ n_samples = 100
1085
+ n_classes = 5
1086
+ loss = HalfMultinomialLoss(n_classes=n_classes)
1087
+ y_true, raw_prediction = random_y_true_raw_prediction(
1088
+ loss=loss,
1089
+ n_samples=n_samples,
1090
+ seed=global_random_seed,
1091
+ )
1092
+ sample_weight = np.linspace(0.1, 2, num=n_samples)
1093
+
1094
+ grad1 = loss.closs._test_cy_gradient(
1095
+ y_true=y_true,
1096
+ raw_prediction=raw_prediction, # needs to be C-contiguous
1097
+ sample_weight=sample_weight,
1098
+ )
1099
+ grad2 = loss.gradient(
1100
+ y_true=y_true,
1101
+ raw_prediction=raw_prediction,
1102
+ sample_weight=sample_weight,
1103
+ )
1104
+ assert_allclose(grad1, grad2)
1105
+
1106
+
1107
+ def test_binomial_and_multinomial_loss(global_random_seed):
1108
+ """Test that multinomial loss with n_classes = 2 is the same as binomial loss."""
1109
+ rng = np.random.RandomState(global_random_seed)
1110
+ n_samples = 20
1111
+ binom = HalfBinomialLoss()
1112
+ multinom = HalfMultinomialLoss(n_classes=2)
1113
+ y_train = rng.randint(0, 2, size=n_samples).astype(np.float64)
1114
+ raw_prediction = rng.normal(size=n_samples)
1115
+ raw_multinom = np.empty((n_samples, 2))
1116
+ raw_multinom[:, 0] = -0.5 * raw_prediction
1117
+ raw_multinom[:, 1] = 0.5 * raw_prediction
1118
+ assert_allclose(
1119
+ binom.loss(y_true=y_train, raw_prediction=raw_prediction),
1120
+ multinom.loss(y_true=y_train, raw_prediction=raw_multinom),
1121
+ )
1122
+
1123
+
1124
+ @pytest.mark.parametrize("y_true", (np.array([0.0, 0, 0]), np.array([1.0, 1, 1])))
1125
+ @pytest.mark.parametrize("y_pred", (np.array([-5.0, -5, -5]), np.array([3.0, 3, 3])))
1126
+ def test_binomial_vs_alternative_formulation(y_true, y_pred, global_dtype):
1127
+ """Test that both formulations of the binomial deviance agree.
1128
+
1129
+ Often, the binomial deviance or log loss is written in terms of a variable
1130
+ z in {-1, +1}, but we use y in {0, 1}, hence z = 2 * y - 1.
1131
+ ESL II Eq. (10.18):
1132
+
1133
+ -loglike(z, f) = log(1 + exp(-2 * z * f))
1134
+
1135
+ Note:
1136
+ - ESL 2*f = raw_prediction, hence the factor 2 of ESL disappears.
1137
+ - Deviance = -2*loglike + .., but HalfBinomialLoss is half of the
1138
+ deviance, hence the factor of 2 cancels in the comparison.
1139
+ """
1140
+
1141
+ def alt_loss(y, raw_pred):
1142
+ z = 2 * y - 1
1143
+ return np.mean(np.log(1 + np.exp(-z * raw_pred)))
1144
+
1145
+ def alt_gradient(y, raw_pred):
1146
+ # alternative gradient formula according to ESL
1147
+ z = 2 * y - 1
1148
+ return -z / (1 + np.exp(z * raw_pred))
1149
+
1150
+ bin_loss = HalfBinomialLoss()
1151
+
1152
+ y_true = y_true.astype(global_dtype)
1153
+ y_pred = y_pred.astype(global_dtype)
1154
+ datum = (y_true, y_pred)
1155
+
1156
+ assert bin_loss(*datum) == approx(alt_loss(*datum))
1157
+ assert_allclose(bin_loss.gradient(*datum), alt_gradient(*datum))
1158
+
1159
+
1160
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
1161
+ def test_predict_proba(loss, global_random_seed):
1162
+ """Test that predict_proba and gradient_proba work as expected."""
1163
+ n_samples = 20
1164
+ y_true, raw_prediction = random_y_true_raw_prediction(
1165
+ loss=loss,
1166
+ n_samples=n_samples,
1167
+ y_bound=(-100, 100),
1168
+ raw_bound=(-5, 5),
1169
+ seed=global_random_seed,
1170
+ )
1171
+
1172
+ if hasattr(loss, "predict_proba"):
1173
+ proba = loss.predict_proba(raw_prediction)
1174
+ assert proba.shape == (n_samples, loss.n_classes)
1175
+ assert np.sum(proba, axis=1) == approx(1, rel=1e-11)
1176
+
1177
+ if hasattr(loss, "gradient_proba"):
1178
+ for grad, proba in (
1179
+ (None, None),
1180
+ (None, np.empty_like(raw_prediction)),
1181
+ (np.empty_like(raw_prediction), None),
1182
+ (np.empty_like(raw_prediction), np.empty_like(raw_prediction)),
1183
+ ):
1184
+ grad, proba = loss.gradient_proba(
1185
+ y_true=y_true,
1186
+ raw_prediction=raw_prediction,
1187
+ sample_weight=None,
1188
+ gradient_out=grad,
1189
+ proba_out=proba,
1190
+ )
1191
+ assert proba.shape == (n_samples, loss.n_classes)
1192
+ assert np.sum(proba, axis=1) == approx(1, rel=1e-11)
1193
+ assert_allclose(
1194
+ grad,
1195
+ loss.gradient(
1196
+ y_true=y_true,
1197
+ raw_prediction=raw_prediction,
1198
+ sample_weight=None,
1199
+ gradient_out=None,
1200
+ ),
1201
+ )
1202
+
1203
+
1204
+ @pytest.mark.parametrize("loss", ALL_LOSSES)
1205
+ @pytest.mark.parametrize("sample_weight", [None, "range"])
1206
+ @pytest.mark.parametrize("dtype", (np.float32, np.float64))
1207
+ @pytest.mark.parametrize("order", ("C", "F"))
1208
+ def test_init_gradient_and_hessians(loss, sample_weight, dtype, order):
1209
+ """Test that init_gradient_and_hessian works as expected.
1210
+
1211
+ passing sample_weight to a loss correctly influences the constant_hessian
1212
+ attribute, and consequently the shape of the hessian array.
1213
+ """
1214
+ n_samples = 5
1215
+ if sample_weight == "range":
1216
+ sample_weight = np.ones(n_samples)
1217
+ loss = loss(sample_weight=sample_weight)
1218
+ gradient, hessian = loss.init_gradient_and_hessian(
1219
+ n_samples=n_samples,
1220
+ dtype=dtype,
1221
+ order=order,
1222
+ )
1223
+ if loss.constant_hessian:
1224
+ assert gradient.shape == (n_samples,)
1225
+ assert hessian.shape == (1,)
1226
+ elif loss.is_multiclass:
1227
+ assert gradient.shape == (n_samples, loss.n_classes)
1228
+ assert hessian.shape == (n_samples, loss.n_classes)
1229
+ else:
1230
+ assert hessian.shape == (n_samples,)
1231
+ assert hessian.shape == (n_samples,)
1232
+
1233
+ assert gradient.dtype == dtype
1234
+ assert hessian.dtype == dtype
1235
+
1236
+ if order == "C":
1237
+ assert gradient.flags.c_contiguous
1238
+ assert hessian.flags.c_contiguous
1239
+ else:
1240
+ assert gradient.flags.f_contiguous
1241
+ assert hessian.flags.f_contiguous
1242
+
1243
+
1244
+ @pytest.mark.parametrize("loss", ALL_LOSSES)
1245
+ @pytest.mark.parametrize(
1246
+ "params, err_msg",
1247
+ [
1248
+ (
1249
+ {"dtype": np.int64},
1250
+ f"Valid options for 'dtype' are .* Got dtype={np.int64} instead.",
1251
+ ),
1252
+ ],
1253
+ )
1254
+ def test_init_gradient_and_hessian_raises(loss, params, err_msg):
1255
+ """Test that init_gradient_and_hessian raises errors for invalid input."""
1256
+ loss = loss()
1257
+ with pytest.raises((ValueError, TypeError), match=err_msg):
1258
+ gradient, hessian = loss.init_gradient_and_hessian(n_samples=5, **params)
1259
+
1260
+
1261
+ @pytest.mark.parametrize(
1262
+ "loss, params, err_type, err_msg",
1263
+ [
1264
+ (
1265
+ PinballLoss,
1266
+ {"quantile": None},
1267
+ TypeError,
1268
+ "quantile must be an instance of float, not NoneType.",
1269
+ ),
1270
+ (
1271
+ PinballLoss,
1272
+ {"quantile": 0},
1273
+ ValueError,
1274
+ "quantile == 0, must be > 0.",
1275
+ ),
1276
+ (PinballLoss, {"quantile": 1.1}, ValueError, "quantile == 1.1, must be < 1."),
1277
+ (
1278
+ HuberLoss,
1279
+ {"quantile": None},
1280
+ TypeError,
1281
+ "quantile must be an instance of float, not NoneType.",
1282
+ ),
1283
+ (
1284
+ HuberLoss,
1285
+ {"quantile": 0},
1286
+ ValueError,
1287
+ "quantile == 0, must be > 0.",
1288
+ ),
1289
+ (HuberLoss, {"quantile": 1.1}, ValueError, "quantile == 1.1, must be < 1."),
1290
+ ],
1291
+ )
1292
+ def test_loss_init_parameter_validation(loss, params, err_type, err_msg):
1293
+ """Test that loss raises errors for invalid input."""
1294
+ with pytest.raises(err_type, match=err_msg):
1295
+ loss(**params)
1296
+
1297
+
1298
+ @pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name)
1299
+ def test_loss_pickle(loss):
1300
+ """Test that losses can be pickled."""
1301
+ n_samples = 20
1302
+ y_true, raw_prediction = random_y_true_raw_prediction(
1303
+ loss=loss,
1304
+ n_samples=n_samples,
1305
+ y_bound=(-100, 100),
1306
+ raw_bound=(-5, 5),
1307
+ seed=42,
1308
+ )
1309
+ pickled_loss = pickle.dumps(loss)
1310
+ unpickled_loss = pickle.loads(pickled_loss)
1311
+ assert loss(y_true=y_true, raw_prediction=raw_prediction) == approx(
1312
+ unpickled_loss(y_true=y_true, raw_prediction=raw_prediction)
1313
+ )
1314
+
1315
+
1316
+ @pytest.mark.parametrize("p", [-1.5, 0, 1, 1.5, 2, 3])
1317
+ def test_tweedie_log_identity_consistency(p):
1318
+ """Test for identical losses when only the link function is different."""
1319
+ half_tweedie_log = HalfTweedieLoss(power=p)
1320
+ half_tweedie_identity = HalfTweedieLossIdentity(power=p)
1321
+ n_samples = 10
1322
+ y_true, raw_prediction = random_y_true_raw_prediction(
1323
+ loss=half_tweedie_log, n_samples=n_samples, seed=42
1324
+ )
1325
+ y_pred = half_tweedie_log.link.inverse(raw_prediction) # exp(raw_prediction)
1326
+
1327
+ # Let's compare the loss values, up to some constant term that is dropped
1328
+ # in HalfTweedieLoss but not in HalfTweedieLossIdentity.
1329
+ loss_log = half_tweedie_log.loss(
1330
+ y_true=y_true, raw_prediction=raw_prediction
1331
+ ) + half_tweedie_log.constant_to_optimal_zero(y_true)
1332
+ loss_identity = half_tweedie_identity.loss(
1333
+ y_true=y_true, raw_prediction=y_pred
1334
+ ) + half_tweedie_identity.constant_to_optimal_zero(y_true)
1335
+ # Note that HalfTweedieLoss ignores different constant terms than
1336
+ # HalfTweedieLossIdentity. Constant terms means terms not depending on
1337
+ # raw_prediction. By adding these terms, `constant_to_optimal_zero`, both losses
1338
+ # give the same values.
1339
+ assert_allclose(loss_log, loss_identity)
1340
+
1341
+ # For gradients and hessians, the constant terms do not matter. We have, however,
1342
+ # to account for the chain rule, i.e. with x=raw_prediction
1343
+ # gradient_log(x) = d/dx loss_log(x)
1344
+ # = d/dx loss_identity(exp(x))
1345
+ # = exp(x) * gradient_identity(exp(x))
1346
+ # Similarly,
1347
+ # hessian_log(x) = exp(x) * gradient_identity(exp(x))
1348
+ # + exp(x)**2 * hessian_identity(x)
1349
+ gradient_log, hessian_log = half_tweedie_log.gradient_hessian(
1350
+ y_true=y_true, raw_prediction=raw_prediction
1351
+ )
1352
+ gradient_identity, hessian_identity = half_tweedie_identity.gradient_hessian(
1353
+ y_true=y_true, raw_prediction=y_pred
1354
+ )
1355
+ assert_allclose(gradient_log, y_pred * gradient_identity)
1356
+ assert_allclose(
1357
+ hessian_log, y_pred * gradient_identity + y_pred**2 * hessian_identity
1358
+ )
loan-advisor-agent/.venv/Lib/site-packages/sklearn/_min_dependencies.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """All minimum dependencies for scikit-learn."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ import argparse
7
+ from collections import defaultdict
8
+
9
+ # scipy and cython should by in sync with pyproject.toml
10
+ NUMPY_MIN_VERSION = "1.24.1"
11
+ SCIPY_MIN_VERSION = "1.10.0"
12
+ JOBLIB_MIN_VERSION = "1.3.0"
13
+ THREADPOOLCTL_MIN_VERSION = "3.2.0"
14
+ PYTEST_MIN_VERSION = "7.1.2"
15
+ CYTHON_MIN_VERSION = "3.1.2"
16
+
17
+
18
+ # 'build' and 'install' is included to have structured metadata for CI.
19
+ # It will NOT be included in setup's extras_require
20
+ # The values are (version_spec, comma separated tags)
21
+ dependent_packages = {
22
+ "numpy": (NUMPY_MIN_VERSION, "build, install"),
23
+ "scipy": (SCIPY_MIN_VERSION, "build, install"),
24
+ "joblib": (JOBLIB_MIN_VERSION, "install"),
25
+ "threadpoolctl": (THREADPOOLCTL_MIN_VERSION, "install"),
26
+ "cython": (CYTHON_MIN_VERSION, "build"),
27
+ "meson-python": ("0.17.1", "build"),
28
+ "matplotlib": ("3.6.1", "benchmark, docs, examples, tests"),
29
+ "scikit-image": ("0.22.0", "docs, examples"),
30
+ "pandas": ("1.5.0", "benchmark, docs, examples, tests"),
31
+ "seaborn": ("0.13.0", "docs, examples"),
32
+ "memory_profiler": ("0.57.0", "benchmark, docs"),
33
+ "pytest": (PYTEST_MIN_VERSION, "tests"),
34
+ "pytest-cov": ("2.9.0", "tests"),
35
+ "ruff": ("0.11.7", "tests"),
36
+ "mypy": ("1.15", "tests"),
37
+ "pyamg": ("5.0.0", "tests"),
38
+ "polars": ("0.20.30", "docs, tests"),
39
+ "pyarrow": ("12.0.0", "tests"),
40
+ "sphinx": ("7.3.7", "docs"),
41
+ "sphinx-copybutton": ("0.5.2", "docs"),
42
+ "sphinx-gallery": ("0.17.1", "docs"),
43
+ "numpydoc": ("1.2.0", "docs, tests"),
44
+ "Pillow": ("10.1.0", "docs"),
45
+ "pooch": ("1.8.0", "docs, examples, tests"),
46
+ "sphinx-prompt": ("1.4.0", "docs"),
47
+ "sphinxext-opengraph": ("0.9.1", "docs"),
48
+ "plotly": ("5.18.0", "docs, examples"),
49
+ "sphinxcontrib-sass": ("0.3.4", "docs"),
50
+ "sphinx-remove-toctrees": ("1.0.0.post1", "docs"),
51
+ "sphinx-design": ("0.6.0", "docs"),
52
+ "pydata-sphinx-theme": ("0.15.3", "docs"),
53
+ "towncrier": ("24.8.0", "docs"),
54
+ # XXX: Pin conda-lock to the latest released version (needs manual update
55
+ # from time to time)
56
+ "conda-lock": ("3.0.1", "maintenance"),
57
+ }
58
+
59
+
60
+ # create inverse mapping for setuptools
61
+ tag_to_packages: dict = defaultdict(list)
62
+ for package, (min_version, extras) in dependent_packages.items():
63
+ for extra in extras.split(", "):
64
+ tag_to_packages[extra].append("{}>={}".format(package, min_version))
65
+
66
+
67
+ # Used by CI to get the min dependencies
68
+ if __name__ == "__main__":
69
+ parser = argparse.ArgumentParser(description="Get min dependencies for a package")
70
+
71
+ parser.add_argument("package", choices=dependent_packages)
72
+ args = parser.parse_args()
73
+ min_version = dependent_packages[args.package][0]
74
+ print(min_version)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/base.py ADDED
@@ -0,0 +1,1340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Base classes for all estimators and various utility functions."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ import copy
7
+ import functools
8
+ import inspect
9
+ import platform
10
+ import re
11
+ import warnings
12
+ from collections import defaultdict
13
+
14
+ import numpy as np
15
+
16
+ from sklearn import __version__
17
+ from sklearn._config import config_context, get_config
18
+ from sklearn.exceptions import InconsistentVersionWarning
19
+ from sklearn.utils._metadata_requests import _MetadataRequester, _routing_enabled
20
+ from sklearn.utils._missing import is_pandas_na, is_scalar_nan
21
+ from sklearn.utils._param_validation import validate_parameter_constraints
22
+ from sklearn.utils._repr_html.base import ReprHTMLMixin, _HTMLDocumentationLinkMixin
23
+ from sklearn.utils._repr_html.estimator import estimator_html_repr
24
+ from sklearn.utils._repr_html.params import ParamsDict
25
+ from sklearn.utils._set_output import _SetOutputMixin
26
+ from sklearn.utils._tags import (
27
+ ClassifierTags,
28
+ RegressorTags,
29
+ Tags,
30
+ TargetTags,
31
+ TransformerTags,
32
+ get_tags,
33
+ )
34
+ from sklearn.utils.fixes import _IS_32BIT
35
+ from sklearn.utils.validation import (
36
+ _check_feature_names_in,
37
+ _generate_get_feature_names_out,
38
+ _is_fitted,
39
+ check_array,
40
+ check_is_fitted,
41
+ )
42
+
43
+
44
+ def clone(estimator, *, safe=True):
45
+ """Construct a new unfitted estimator with the same parameters.
46
+
47
+ Clone does a deep copy of the model in an estimator
48
+ without actually copying attached data. It returns a new estimator
49
+ with the same parameters that has not been fitted on any data.
50
+
51
+ .. versionchanged:: 1.3
52
+ Delegates to `estimator.__sklearn_clone__` if the method exists.
53
+
54
+ Parameters
55
+ ----------
56
+ estimator : {list, tuple, set} of estimator instance or a single \
57
+ estimator instance
58
+ The estimator or group of estimators to be cloned.
59
+ safe : bool, default=True
60
+ If safe is False, clone will fall back to a deep copy on objects
61
+ that are not estimators. Ignored if `estimator.__sklearn_clone__`
62
+ exists.
63
+
64
+ Returns
65
+ -------
66
+ estimator : object
67
+ The deep copy of the input, an estimator if input is an estimator.
68
+
69
+ Notes
70
+ -----
71
+ If the estimator's `random_state` parameter is an integer (or if the
72
+ estimator doesn't have a `random_state` parameter), an *exact clone* is
73
+ returned: the clone and the original estimator will give the exact same
74
+ results. Otherwise, *statistical clone* is returned: the clone might
75
+ return different results from the original estimator. More details can be
76
+ found in :ref:`randomness`.
77
+
78
+ Examples
79
+ --------
80
+ >>> from sklearn.base import clone
81
+ >>> from sklearn.linear_model import LogisticRegression
82
+ >>> X = [[-1, 0], [0, 1], [0, -1], [1, 0]]
83
+ >>> y = [0, 0, 1, 1]
84
+ >>> classifier = LogisticRegression().fit(X, y)
85
+ >>> cloned_classifier = clone(classifier)
86
+ >>> hasattr(classifier, "classes_")
87
+ True
88
+ >>> hasattr(cloned_classifier, "classes_")
89
+ False
90
+ >>> classifier is cloned_classifier
91
+ False
92
+ """
93
+ if hasattr(estimator, "__sklearn_clone__") and not inspect.isclass(estimator):
94
+ return estimator.__sklearn_clone__()
95
+ return _clone_parametrized(estimator, safe=safe)
96
+
97
+
98
+ def _clone_parametrized(estimator, *, safe=True):
99
+ """Default implementation of clone. See :func:`sklearn.base.clone` for details."""
100
+
101
+ estimator_type = type(estimator)
102
+ if estimator_type is dict:
103
+ return {k: clone(v, safe=safe) for k, v in estimator.items()}
104
+ elif estimator_type in (list, tuple, set, frozenset):
105
+ return estimator_type([clone(e, safe=safe) for e in estimator])
106
+ elif not hasattr(estimator, "get_params") or isinstance(estimator, type):
107
+ if not safe:
108
+ return copy.deepcopy(estimator)
109
+ else:
110
+ if isinstance(estimator, type):
111
+ raise TypeError(
112
+ "Cannot clone object. "
113
+ "You should provide an instance of "
114
+ "scikit-learn estimator instead of a class."
115
+ )
116
+ else:
117
+ raise TypeError(
118
+ "Cannot clone object '%s' (type %s): "
119
+ "it does not seem to be a scikit-learn "
120
+ "estimator as it does not implement a "
121
+ "'get_params' method." % (repr(estimator), type(estimator))
122
+ )
123
+
124
+ klass = estimator.__class__
125
+ new_object_params = estimator.get_params(deep=False)
126
+ for name, param in new_object_params.items():
127
+ new_object_params[name] = clone(param, safe=False)
128
+
129
+ new_object = klass(**new_object_params)
130
+ try:
131
+ new_object._metadata_request = copy.deepcopy(estimator._metadata_request)
132
+ except AttributeError:
133
+ pass
134
+
135
+ params_set = new_object.get_params(deep=False)
136
+
137
+ # quick sanity check of the parameters of the clone
138
+ for name in new_object_params:
139
+ param1 = new_object_params[name]
140
+ param2 = params_set[name]
141
+ if param1 is not param2:
142
+ raise RuntimeError(
143
+ "Cannot clone object %s, as the constructor "
144
+ "either does not set or modifies parameter %s" % (estimator, name)
145
+ )
146
+
147
+ # _sklearn_output_config is used by `set_output` to configure the output
148
+ # container of an estimator.
149
+ if hasattr(estimator, "_sklearn_output_config"):
150
+ new_object._sklearn_output_config = copy.deepcopy(
151
+ estimator._sklearn_output_config
152
+ )
153
+ return new_object
154
+
155
+
156
+ class BaseEstimator(ReprHTMLMixin, _HTMLDocumentationLinkMixin, _MetadataRequester):
157
+ """Base class for all estimators in scikit-learn.
158
+
159
+ Inheriting from this class provides default implementations of:
160
+
161
+ - setting and getting parameters used by `GridSearchCV` and friends;
162
+ - textual and HTML representation displayed in terminals and IDEs;
163
+ - estimator serialization;
164
+ - parameters validation;
165
+ - data validation;
166
+ - feature names validation.
167
+
168
+ Read more in the :ref:`User Guide <rolling_your_own_estimator>`.
169
+
170
+
171
+ Notes
172
+ -----
173
+ All estimators should specify all the parameters that can be set
174
+ at the class level in their ``__init__`` as explicit keyword
175
+ arguments (no ``*args`` or ``**kwargs``).
176
+
177
+ Examples
178
+ --------
179
+ >>> import numpy as np
180
+ >>> from sklearn.base import BaseEstimator
181
+ >>> class MyEstimator(BaseEstimator):
182
+ ... def __init__(self, *, param=1):
183
+ ... self.param = param
184
+ ... def fit(self, X, y=None):
185
+ ... self.is_fitted_ = True
186
+ ... return self
187
+ ... def predict(self, X):
188
+ ... return np.full(shape=X.shape[0], fill_value=self.param)
189
+ >>> estimator = MyEstimator(param=2)
190
+ >>> estimator.get_params()
191
+ {'param': 2}
192
+ >>> X = np.array([[1, 2], [2, 3], [3, 4]])
193
+ >>> y = np.array([1, 0, 1])
194
+ >>> estimator.fit(X, y).predict(X)
195
+ array([2, 2, 2])
196
+ >>> estimator.set_params(param=3).fit(X, y).predict(X)
197
+ array([3, 3, 3])
198
+ """
199
+
200
+ def __dir__(self):
201
+ # Filters conditional methods that should be hidden based
202
+ # on the `available_if` decorator
203
+ with warnings.catch_warnings():
204
+ warnings.filterwarnings("ignore", category=FutureWarning)
205
+ return [attr for attr in super().__dir__() if hasattr(self, attr)]
206
+
207
+ _html_repr = estimator_html_repr
208
+
209
+ @classmethod
210
+ def _get_param_names(cls):
211
+ """Get parameter names for the estimator"""
212
+ # fetch the constructor or the original constructor before
213
+ # deprecation wrapping if any
214
+ init = getattr(cls.__init__, "deprecated_original", cls.__init__)
215
+ if init is object.__init__:
216
+ # No explicit constructor to introspect
217
+ return []
218
+
219
+ # introspect the constructor arguments to find the model parameters
220
+ # to represent
221
+ init_signature = inspect.signature(init)
222
+ # Consider the constructor parameters excluding 'self'
223
+ parameters = [
224
+ p
225
+ for p in init_signature.parameters.values()
226
+ if p.name != "self" and p.kind != p.VAR_KEYWORD
227
+ ]
228
+ for p in parameters:
229
+ if p.kind == p.VAR_POSITIONAL:
230
+ raise RuntimeError(
231
+ "scikit-learn estimators should always "
232
+ "specify their parameters in the signature"
233
+ " of their __init__ (no varargs)."
234
+ " %s with constructor %s doesn't "
235
+ " follow this convention." % (cls, init_signature)
236
+ )
237
+ # Extract and sort argument names excluding 'self'
238
+ return sorted([p.name for p in parameters])
239
+
240
+ def get_params(self, deep=True):
241
+ """
242
+ Get parameters for this estimator.
243
+
244
+ Parameters
245
+ ----------
246
+ deep : bool, default=True
247
+ If True, will return the parameters for this estimator and
248
+ contained subobjects that are estimators.
249
+
250
+ Returns
251
+ -------
252
+ params : dict
253
+ Parameter names mapped to their values.
254
+ """
255
+ out = dict()
256
+ for key in self._get_param_names():
257
+ value = getattr(self, key)
258
+ if deep and hasattr(value, "get_params") and not isinstance(value, type):
259
+ deep_items = value.get_params().items()
260
+ out.update((key + "__" + k, val) for k, val in deep_items)
261
+ out[key] = value
262
+ return out
263
+
264
+ def _get_params_html(self, deep=True, doc_link=""):
265
+ """
266
+ Get parameters for this estimator with a specific HTML representation.
267
+
268
+ Parameters
269
+ ----------
270
+ deep : bool, default=True
271
+ If True, will return the parameters for this estimator and
272
+ contained subobjects that are estimators.
273
+
274
+ doc_link : str
275
+ URL to the estimator documentation.
276
+ Used for linking to the estimator's parameters documentation
277
+ available in HTML displays.
278
+
279
+ Returns
280
+ -------
281
+ params : ParamsDict
282
+ Parameter names mapped to their values. We return a `ParamsDict`
283
+ dictionary, which renders a specific HTML representation in table
284
+ form.
285
+ """
286
+ out = self.get_params(deep=deep)
287
+
288
+ init_func = getattr(self.__init__, "deprecated_original", self.__init__)
289
+ init_default_params = inspect.signature(init_func).parameters
290
+ init_default_params = {
291
+ name: param.default for name, param in init_default_params.items()
292
+ }
293
+
294
+ def is_non_default(param_name, param_value):
295
+ """Finds the parameters that have been set by the user."""
296
+ if param_name not in init_default_params:
297
+ # happens if k is part of a **kwargs
298
+ return True
299
+ if init_default_params[param_name] == inspect._empty:
300
+ # k has no default value
301
+ return True
302
+ # avoid calling repr on nested estimators
303
+ if isinstance(param_value, BaseEstimator) and type(param_value) is not type(
304
+ init_default_params[param_name]
305
+ ):
306
+ return True
307
+ if is_pandas_na(param_value) and not is_pandas_na(
308
+ init_default_params[param_name]
309
+ ):
310
+ return True
311
+ if not np.array_equal(
312
+ param_value, init_default_params[param_name]
313
+ ) and not (
314
+ is_scalar_nan(init_default_params[param_name])
315
+ and is_scalar_nan(param_value)
316
+ ):
317
+ return True
318
+
319
+ return False
320
+
321
+ # reorder the parameters from `self.get_params` using the `__init__`
322
+ # signature
323
+ remaining_params = [name for name in out if name not in init_default_params]
324
+ ordered_out = {name: out[name] for name in init_default_params if name in out}
325
+ ordered_out.update({name: out[name] for name in remaining_params})
326
+
327
+ non_default_ls = tuple(
328
+ [name for name, value in ordered_out.items() if is_non_default(name, value)]
329
+ )
330
+
331
+ return ParamsDict(
332
+ params=ordered_out,
333
+ non_default=non_default_ls,
334
+ estimator_class=self.__class__,
335
+ doc_link=doc_link,
336
+ )
337
+
338
+ def set_params(self, **params):
339
+ """Set the parameters of this estimator.
340
+
341
+ The method works on simple estimators as well as on nested objects
342
+ (such as :class:`~sklearn.pipeline.Pipeline`). The latter have
343
+ parameters of the form ``<component>__<parameter>`` so that it's
344
+ possible to update each component of a nested object.
345
+
346
+ Parameters
347
+ ----------
348
+ **params : dict
349
+ Estimator parameters.
350
+
351
+ Returns
352
+ -------
353
+ self : estimator instance
354
+ Estimator instance.
355
+ """
356
+ if not params:
357
+ # Simple optimization to gain speed (inspect is slow)
358
+ return self
359
+ valid_params = self.get_params(deep=True)
360
+
361
+ nested_params = defaultdict(dict) # grouped by prefix
362
+ for key, value in params.items():
363
+ key, delim, sub_key = key.partition("__")
364
+ if key not in valid_params:
365
+ local_valid_params = self._get_param_names()
366
+ raise ValueError(
367
+ f"Invalid parameter {key!r} for estimator {self}. "
368
+ f"Valid parameters are: {local_valid_params!r}."
369
+ )
370
+
371
+ if delim:
372
+ nested_params[key][sub_key] = value
373
+ else:
374
+ setattr(self, key, value)
375
+ valid_params[key] = value
376
+
377
+ for key, sub_params in nested_params.items():
378
+ valid_params[key].set_params(**sub_params)
379
+
380
+ return self
381
+
382
+ def __sklearn_clone__(self):
383
+ return _clone_parametrized(self)
384
+
385
+ def __repr__(self, N_CHAR_MAX=700):
386
+ # N_CHAR_MAX is the (approximate) maximum number of non-blank
387
+ # characters to render. We pass it as an optional parameter to ease
388
+ # the tests.
389
+
390
+ from sklearn.utils._pprint import _EstimatorPrettyPrinter
391
+
392
+ N_MAX_ELEMENTS_TO_SHOW = 30 # number of elements to show in sequences
393
+
394
+ # use ellipsis for sequences with a lot of elements
395
+ pp = _EstimatorPrettyPrinter(
396
+ compact=True,
397
+ indent=1,
398
+ indent_at_name=True,
399
+ n_max_elements_to_show=N_MAX_ELEMENTS_TO_SHOW,
400
+ )
401
+
402
+ repr_ = pp.pformat(self)
403
+
404
+ # Use bruteforce ellipsis when there are a lot of non-blank characters
405
+ n_nonblank = len("".join(repr_.split()))
406
+ if n_nonblank > N_CHAR_MAX:
407
+ lim = N_CHAR_MAX // 2 # apprx number of chars to keep on both ends
408
+ regex = r"^(\s*\S){%d}" % lim
409
+ # The regex '^(\s*\S){%d}' % n
410
+ # matches from the start of the string until the nth non-blank
411
+ # character:
412
+ # - ^ matches the start of string
413
+ # - (pattern){n} matches n repetitions of pattern
414
+ # - \s*\S matches a non-blank char following zero or more blanks
415
+ left_lim = re.match(regex, repr_).end()
416
+ right_lim = re.match(regex, repr_[::-1]).end()
417
+
418
+ if "\n" in repr_[left_lim:-right_lim]:
419
+ # The left side and right side aren't on the same line.
420
+ # To avoid weird cuts, e.g.:
421
+ # categoric...ore',
422
+ # we need to start the right side with an appropriate newline
423
+ # character so that it renders properly as:
424
+ # categoric...
425
+ # handle_unknown='ignore',
426
+ # so we add [^\n]*\n which matches until the next \n
427
+ regex += r"[^\n]*\n"
428
+ right_lim = re.match(regex, repr_[::-1]).end()
429
+
430
+ ellipsis = "..."
431
+ if left_lim + len(ellipsis) < len(repr_) - right_lim:
432
+ # Only add ellipsis if it results in a shorter repr
433
+ repr_ = repr_[:left_lim] + "..." + repr_[-right_lim:]
434
+
435
+ return repr_
436
+
437
+ def __getstate__(self):
438
+ if getattr(self, "__slots__", None):
439
+ raise TypeError(
440
+ "You cannot use `__slots__` in objects inheriting from "
441
+ "`sklearn.base.BaseEstimator`."
442
+ )
443
+
444
+ try:
445
+ state = super().__getstate__()
446
+ if state is None:
447
+ # For Python 3.11+, empty instance (no `__slots__`,
448
+ # and `__dict__`) will return a state equal to `None`.
449
+ state = self.__dict__.copy()
450
+ except AttributeError:
451
+ # Python < 3.11
452
+ state = self.__dict__.copy()
453
+
454
+ if type(self).__module__.startswith("sklearn."):
455
+ return dict(state.items(), _sklearn_version=__version__)
456
+ else:
457
+ return state
458
+
459
+ def __setstate__(self, state):
460
+ if type(self).__module__.startswith("sklearn."):
461
+ pickle_version = state.pop("_sklearn_version", "pre-0.18")
462
+ if pickle_version != __version__:
463
+ warnings.warn(
464
+ InconsistentVersionWarning(
465
+ estimator_name=self.__class__.__name__,
466
+ current_sklearn_version=__version__,
467
+ original_sklearn_version=pickle_version,
468
+ ),
469
+ )
470
+ try:
471
+ super().__setstate__(state)
472
+ except AttributeError:
473
+ self.__dict__.update(state)
474
+
475
+ def __sklearn_tags__(self):
476
+ return Tags(
477
+ estimator_type=None,
478
+ target_tags=TargetTags(required=False),
479
+ transformer_tags=None,
480
+ regressor_tags=None,
481
+ classifier_tags=None,
482
+ )
483
+
484
+ def _validate_params(self):
485
+ """Validate types and values of constructor parameters
486
+
487
+ The expected type and values must be defined in the `_parameter_constraints`
488
+ class attribute, which is a dictionary `param_name: list of constraints`. See
489
+ the docstring of `validate_parameter_constraints` for a description of the
490
+ accepted constraints.
491
+ """
492
+ validate_parameter_constraints(
493
+ self._parameter_constraints,
494
+ self.get_params(deep=False),
495
+ caller_name=self.__class__.__name__,
496
+ )
497
+
498
+
499
+ class ClassifierMixin:
500
+ """Mixin class for all classifiers in scikit-learn.
501
+
502
+ This mixin defines the following functionality:
503
+
504
+ - set estimator type to `"classifier"` through the `estimator_type` tag;
505
+ - `score` method that default to :func:`~sklearn.metrics.accuracy_score`.
506
+ - enforce that `fit` requires `y` to be passed through the `requires_y` tag,
507
+ which is done by setting the classifier type tag.
508
+
509
+ Read more in the :ref:`User Guide <rolling_your_own_estimator>`.
510
+
511
+ Examples
512
+ --------
513
+ >>> import numpy as np
514
+ >>> from sklearn.base import BaseEstimator, ClassifierMixin
515
+ >>> # Mixin classes should always be on the left-hand side for a correct MRO
516
+ >>> class MyEstimator(ClassifierMixin, BaseEstimator):
517
+ ... def __init__(self, *, param=1):
518
+ ... self.param = param
519
+ ... def fit(self, X, y=None):
520
+ ... self.is_fitted_ = True
521
+ ... return self
522
+ ... def predict(self, X):
523
+ ... return np.full(shape=X.shape[0], fill_value=self.param)
524
+ >>> estimator = MyEstimator(param=1)
525
+ >>> X = np.array([[1, 2], [2, 3], [3, 4]])
526
+ >>> y = np.array([1, 0, 1])
527
+ >>> estimator.fit(X, y).predict(X)
528
+ array([1, 1, 1])
529
+ >>> estimator.score(X, y)
530
+ 0.66...
531
+ """
532
+
533
+ def __sklearn_tags__(self):
534
+ tags = super().__sklearn_tags__()
535
+ tags.estimator_type = "classifier"
536
+ tags.classifier_tags = ClassifierTags()
537
+ tags.target_tags.required = True
538
+ return tags
539
+
540
+ def score(self, X, y, sample_weight=None):
541
+ """
542
+ Return :ref:`accuracy <accuracy_score>` on provided data and labels.
543
+
544
+ In multi-label classification, this is the subset accuracy
545
+ which is a harsh metric since you require for each sample that
546
+ each label set be correctly predicted.
547
+
548
+ Parameters
549
+ ----------
550
+ X : array-like of shape (n_samples, n_features)
551
+ Test samples.
552
+
553
+ y : array-like of shape (n_samples,) or (n_samples, n_outputs)
554
+ True labels for `X`.
555
+
556
+ sample_weight : array-like of shape (n_samples,), default=None
557
+ Sample weights.
558
+
559
+ Returns
560
+ -------
561
+ score : float
562
+ Mean accuracy of ``self.predict(X)`` w.r.t. `y`.
563
+ """
564
+ from sklearn.metrics import accuracy_score
565
+
566
+ return accuracy_score(y, self.predict(X), sample_weight=sample_weight)
567
+
568
+
569
+ class RegressorMixin:
570
+ """Mixin class for all regression estimators in scikit-learn.
571
+
572
+ This mixin defines the following functionality:
573
+
574
+ - set estimator type to `"regressor"` through the `estimator_type` tag;
575
+ - `score` method that default to :func:`~sklearn.metrics.r2_score`.
576
+ - enforce that `fit` requires `y` to be passed through the `requires_y` tag,
577
+ which is done by setting the regressor type tag.
578
+
579
+ Read more in the :ref:`User Guide <rolling_your_own_estimator>`.
580
+
581
+ Examples
582
+ --------
583
+ >>> import numpy as np
584
+ >>> from sklearn.base import BaseEstimator, RegressorMixin
585
+ >>> # Mixin classes should always be on the left-hand side for a correct MRO
586
+ >>> class MyEstimator(RegressorMixin, BaseEstimator):
587
+ ... def __init__(self, *, param=1):
588
+ ... self.param = param
589
+ ... def fit(self, X, y=None):
590
+ ... self.is_fitted_ = True
591
+ ... return self
592
+ ... def predict(self, X):
593
+ ... return np.full(shape=X.shape[0], fill_value=self.param)
594
+ >>> estimator = MyEstimator(param=0)
595
+ >>> X = np.array([[1, 2], [2, 3], [3, 4]])
596
+ >>> y = np.array([-1, 0, 1])
597
+ >>> estimator.fit(X, y).predict(X)
598
+ array([0, 0, 0])
599
+ >>> estimator.score(X, y)
600
+ 0.0
601
+ """
602
+
603
+ def __sklearn_tags__(self):
604
+ tags = super().__sklearn_tags__()
605
+ tags.estimator_type = "regressor"
606
+ tags.regressor_tags = RegressorTags()
607
+ tags.target_tags.required = True
608
+ return tags
609
+
610
+ def score(self, X, y, sample_weight=None):
611
+ """Return :ref:`coefficient of determination <r2_score>` on test data.
612
+
613
+ The coefficient of determination, :math:`R^2`, is defined as
614
+ :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual
615
+ sum of squares ``((y_true - y_pred)** 2).sum()`` and :math:`v`
616
+ is the total sum of squares ``((y_true - y_true.mean()) ** 2).sum()``.
617
+ The best possible score is 1.0 and it can be negative (because the
618
+ model can be arbitrarily worse). A constant model that always predicts
619
+ the expected value of `y`, disregarding the input features, would get
620
+ a :math:`R^2` score of 0.0.
621
+
622
+ Parameters
623
+ ----------
624
+ X : array-like of shape (n_samples, n_features)
625
+ Test samples. For some estimators this may be a precomputed
626
+ kernel matrix or a list of generic objects instead with shape
627
+ ``(n_samples, n_samples_fitted)``, where ``n_samples_fitted``
628
+ is the number of samples used in the fitting for the estimator.
629
+
630
+ y : array-like of shape (n_samples,) or (n_samples, n_outputs)
631
+ True values for `X`.
632
+
633
+ sample_weight : array-like of shape (n_samples,), default=None
634
+ Sample weights.
635
+
636
+ Returns
637
+ -------
638
+ score : float
639
+ :math:`R^2` of ``self.predict(X)`` w.r.t. `y`.
640
+
641
+ Notes
642
+ -----
643
+ The :math:`R^2` score used when calling ``score`` on a regressor uses
644
+ ``multioutput='uniform_average'`` from version 0.23 to keep consistent
645
+ with default value of :func:`~sklearn.metrics.r2_score`.
646
+ This influences the ``score`` method of all the multioutput
647
+ regressors (except for
648
+ :class:`~sklearn.multioutput.MultiOutputRegressor`).
649
+ """
650
+
651
+ from sklearn.metrics import r2_score
652
+
653
+ y_pred = self.predict(X)
654
+ return r2_score(y, y_pred, sample_weight=sample_weight)
655
+
656
+
657
+ class ClusterMixin:
658
+ """Mixin class for all cluster estimators in scikit-learn.
659
+
660
+ - set estimator type to `"clusterer"` through the `estimator_type` tag;
661
+ - `fit_predict` method returning the cluster labels associated to each sample.
662
+
663
+ Examples
664
+ --------
665
+ >>> import numpy as np
666
+ >>> from sklearn.base import BaseEstimator, ClusterMixin
667
+ >>> class MyClusterer(ClusterMixin, BaseEstimator):
668
+ ... def fit(self, X, y=None):
669
+ ... self.labels_ = np.ones(shape=(len(X),), dtype=np.int64)
670
+ ... return self
671
+ >>> X = [[1, 2], [2, 3], [3, 4]]
672
+ >>> MyClusterer().fit_predict(X)
673
+ array([1, 1, 1])
674
+ """
675
+
676
+ def __sklearn_tags__(self):
677
+ tags = super().__sklearn_tags__()
678
+ tags.estimator_type = "clusterer"
679
+ if tags.transformer_tags is not None:
680
+ tags.transformer_tags.preserves_dtype = []
681
+ return tags
682
+
683
+ def fit_predict(self, X, y=None, **kwargs):
684
+ """
685
+ Perform clustering on `X` and returns cluster labels.
686
+
687
+ Parameters
688
+ ----------
689
+ X : array-like of shape (n_samples, n_features)
690
+ Input data.
691
+
692
+ y : Ignored
693
+ Not used, present for API consistency by convention.
694
+
695
+ **kwargs : dict
696
+ Arguments to be passed to ``fit``.
697
+
698
+ .. versionadded:: 1.4
699
+
700
+ Returns
701
+ -------
702
+ labels : ndarray of shape (n_samples,), dtype=np.int64
703
+ Cluster labels.
704
+ """
705
+ # non-optimized default implementation; override when a better
706
+ # method is possible for a given clustering algorithm
707
+ self.fit(X, **kwargs)
708
+ return self.labels_
709
+
710
+
711
+ class BiclusterMixin:
712
+ """Mixin class for all bicluster estimators in scikit-learn.
713
+
714
+ This mixin defines the following functionality:
715
+
716
+ - `biclusters_` property that returns the row and column indicators;
717
+ - `get_indices` method that returns the row and column indices of a bicluster;
718
+ - `get_shape` method that returns the shape of a bicluster;
719
+ - `get_submatrix` method that returns the submatrix corresponding to a bicluster.
720
+
721
+ Examples
722
+ --------
723
+ >>> import numpy as np
724
+ >>> from sklearn.base import BaseEstimator, BiclusterMixin
725
+ >>> class DummyBiClustering(BiclusterMixin, BaseEstimator):
726
+ ... def fit(self, X, y=None):
727
+ ... self.rows_ = np.ones(shape=(1, X.shape[0]), dtype=bool)
728
+ ... self.columns_ = np.ones(shape=(1, X.shape[1]), dtype=bool)
729
+ ... return self
730
+ >>> X = np.array([[1, 1], [2, 1], [1, 0],
731
+ ... [4, 7], [3, 5], [3, 6]])
732
+ >>> bicluster = DummyBiClustering().fit(X)
733
+ >>> hasattr(bicluster, "biclusters_")
734
+ True
735
+ >>> bicluster.get_indices(0)
736
+ (array([0, 1, 2, 3, 4, 5]), array([0, 1]))
737
+ """
738
+
739
+ @property
740
+ def biclusters_(self):
741
+ """Convenient way to get row and column indicators together.
742
+
743
+ Returns the ``rows_`` and ``columns_`` members.
744
+ """
745
+ return self.rows_, self.columns_
746
+
747
+ def get_indices(self, i):
748
+ """Row and column indices of the `i`'th bicluster.
749
+
750
+ Only works if ``rows_`` and ``columns_`` attributes exist.
751
+
752
+ Parameters
753
+ ----------
754
+ i : int
755
+ The index of the cluster.
756
+
757
+ Returns
758
+ -------
759
+ row_ind : ndarray, dtype=np.intp
760
+ Indices of rows in the dataset that belong to the bicluster.
761
+ col_ind : ndarray, dtype=np.intp
762
+ Indices of columns in the dataset that belong to the bicluster.
763
+ """
764
+ rows = self.rows_[i]
765
+ columns = self.columns_[i]
766
+ return np.nonzero(rows)[0], np.nonzero(columns)[0]
767
+
768
+ def get_shape(self, i):
769
+ """Shape of the `i`'th bicluster.
770
+
771
+ Parameters
772
+ ----------
773
+ i : int
774
+ The index of the cluster.
775
+
776
+ Returns
777
+ -------
778
+ n_rows : int
779
+ Number of rows in the bicluster.
780
+
781
+ n_cols : int
782
+ Number of columns in the bicluster.
783
+ """
784
+ indices = self.get_indices(i)
785
+ return tuple(len(i) for i in indices)
786
+
787
+ def get_submatrix(self, i, data):
788
+ """Return the submatrix corresponding to bicluster `i`.
789
+
790
+ Parameters
791
+ ----------
792
+ i : int
793
+ The index of the cluster.
794
+ data : array-like of shape (n_samples, n_features)
795
+ The data.
796
+
797
+ Returns
798
+ -------
799
+ submatrix : ndarray of shape (n_rows, n_cols)
800
+ The submatrix corresponding to bicluster `i`.
801
+
802
+ Notes
803
+ -----
804
+ Works with sparse matrices. Only works if ``rows_`` and
805
+ ``columns_`` attributes exist.
806
+ """
807
+
808
+ data = check_array(data, accept_sparse="csr")
809
+ row_ind, col_ind = self.get_indices(i)
810
+ return data[row_ind[:, np.newaxis], col_ind]
811
+
812
+
813
+ class TransformerMixin(_SetOutputMixin):
814
+ """Mixin class for all transformers in scikit-learn.
815
+
816
+ This mixin defines the following functionality:
817
+
818
+ - a `fit_transform` method that delegates to `fit` and `transform`;
819
+ - a `set_output` method to output `X` as a specific container type.
820
+
821
+ If :term:`get_feature_names_out` is defined, then :class:`BaseEstimator` will
822
+ automatically wrap `transform` and `fit_transform` to follow the `set_output`
823
+ API. See the :ref:`developer_api_set_output` for details.
824
+
825
+ :class:`OneToOneFeatureMixin` and
826
+ :class:`ClassNamePrefixFeaturesOutMixin` are helpful mixins for
827
+ defining :term:`get_feature_names_out`.
828
+
829
+ Examples
830
+ --------
831
+ >>> import numpy as np
832
+ >>> from sklearn.base import BaseEstimator, TransformerMixin
833
+ >>> class MyTransformer(TransformerMixin, BaseEstimator):
834
+ ... def __init__(self, *, param=1):
835
+ ... self.param = param
836
+ ... def fit(self, X, y=None):
837
+ ... return self
838
+ ... def transform(self, X):
839
+ ... return np.full(shape=len(X), fill_value=self.param)
840
+ >>> transformer = MyTransformer()
841
+ >>> X = [[1, 2], [2, 3], [3, 4]]
842
+ >>> transformer.fit_transform(X)
843
+ array([1, 1, 1])
844
+ """
845
+
846
+ def __sklearn_tags__(self):
847
+ tags = super().__sklearn_tags__()
848
+ tags.transformer_tags = TransformerTags()
849
+ return tags
850
+
851
+ def fit_transform(self, X, y=None, **fit_params):
852
+ """
853
+ Fit to data, then transform it.
854
+
855
+ Fits transformer to `X` and `y` with optional parameters `fit_params`
856
+ and returns a transformed version of `X`.
857
+
858
+ Parameters
859
+ ----------
860
+ X : array-like of shape (n_samples, n_features)
861
+ Input samples.
862
+
863
+ y : array-like of shape (n_samples,) or (n_samples, n_outputs), \
864
+ default=None
865
+ Target values (None for unsupervised transformations).
866
+
867
+ **fit_params : dict
868
+ Additional fit parameters.
869
+ Pass only if the estimator accepts additional params in its `fit` method.
870
+
871
+ Returns
872
+ -------
873
+ X_new : ndarray array of shape (n_samples, n_features_new)
874
+ Transformed array.
875
+ """
876
+ # non-optimized default implementation; override when a better
877
+ # method is possible for a given clustering algorithm
878
+
879
+ # we do not route parameters here, since consumers don't route. But
880
+ # since it's possible for a `transform` method to also consume
881
+ # metadata, we check if that's the case, and we raise a warning telling
882
+ # users that they should implement a custom `fit_transform` method
883
+ # to forward metadata to `transform` as well.
884
+ #
885
+ # For that, we calculate routing and check if anything would be routed
886
+ # to `transform` if we were to route them.
887
+ if _routing_enabled():
888
+ transform_params = self.get_metadata_routing().consumes(
889
+ method="transform", params=fit_params.keys()
890
+ )
891
+ if transform_params:
892
+ warnings.warn(
893
+ (
894
+ f"This object ({self.__class__.__name__}) has a `transform`"
895
+ " method which consumes metadata, but `fit_transform` does not"
896
+ " forward metadata to `transform`. Please implement a custom"
897
+ " `fit_transform` method to forward metadata to `transform` as"
898
+ " well. Alternatively, you can explicitly do"
899
+ " `set_transform_request`and set all values to `False` to"
900
+ " disable metadata routed to `transform`, if that's an option."
901
+ ),
902
+ UserWarning,
903
+ )
904
+
905
+ if y is None:
906
+ # fit method of arity 1 (unsupervised transformation)
907
+ return self.fit(X, **fit_params).transform(X)
908
+ else:
909
+ # fit method of arity 2 (supervised transformation)
910
+ return self.fit(X, y, **fit_params).transform(X)
911
+
912
+
913
+ class OneToOneFeatureMixin:
914
+ """Provides `get_feature_names_out` for simple transformers.
915
+
916
+ This mixin assumes there's a 1-to-1 correspondence between input features
917
+ and output features, such as :class:`~sklearn.preprocessing.StandardScaler`.
918
+
919
+ Examples
920
+ --------
921
+ >>> import numpy as np
922
+ >>> from sklearn.base import OneToOneFeatureMixin, BaseEstimator
923
+ >>> class MyEstimator(OneToOneFeatureMixin, BaseEstimator):
924
+ ... def fit(self, X, y=None):
925
+ ... self.n_features_in_ = X.shape[1]
926
+ ... return self
927
+ >>> X = np.array([[1, 2], [3, 4]])
928
+ >>> MyEstimator().fit(X).get_feature_names_out()
929
+ array(['x0', 'x1'], dtype=object)
930
+ """
931
+
932
+ def get_feature_names_out(self, input_features=None):
933
+ """Get output feature names for transformation.
934
+
935
+ Parameters
936
+ ----------
937
+ input_features : array-like of str or None, default=None
938
+ Input features.
939
+
940
+ - If `input_features` is `None`, then `feature_names_in_` is
941
+ used as feature names in. If `feature_names_in_` is not defined,
942
+ then the following input feature names are generated:
943
+ `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
944
+ - If `input_features` is an array-like, then `input_features` must
945
+ match `feature_names_in_` if `feature_names_in_` is defined.
946
+
947
+ Returns
948
+ -------
949
+ feature_names_out : ndarray of str objects
950
+ Same as input features.
951
+ """
952
+ # Note that passing attributes="n_features_in_" forces check_is_fitted
953
+ # to check if the attribute is present. Otherwise it will pass on
954
+ # stateless estimators (requires_fit=False)
955
+ check_is_fitted(self, attributes="n_features_in_")
956
+ return _check_feature_names_in(self, input_features)
957
+
958
+
959
+ class ClassNamePrefixFeaturesOutMixin:
960
+ """Mixin class for transformers that generate their own names by prefixing.
961
+
962
+ This mixin is useful when the transformer needs to generate its own feature
963
+ names out, such as :class:`~sklearn.decomposition.PCA`. For example, if
964
+ :class:`~sklearn.decomposition.PCA` outputs 3 features, then the generated feature
965
+ names out are: `["pca0", "pca1", "pca2"]`.
966
+
967
+ This mixin assumes that a `_n_features_out` attribute is defined when the
968
+ transformer is fitted. `_n_features_out` is the number of output features
969
+ that the transformer will return in `transform` of `fit_transform`.
970
+
971
+ Examples
972
+ --------
973
+ >>> import numpy as np
974
+ >>> from sklearn.base import ClassNamePrefixFeaturesOutMixin, BaseEstimator
975
+ >>> class MyEstimator(ClassNamePrefixFeaturesOutMixin, BaseEstimator):
976
+ ... def fit(self, X, y=None):
977
+ ... self._n_features_out = X.shape[1]
978
+ ... return self
979
+ >>> X = np.array([[1, 2], [3, 4]])
980
+ >>> MyEstimator().fit(X).get_feature_names_out()
981
+ array(['myestimator0', 'myestimator1'], dtype=object)
982
+ """
983
+
984
+ def get_feature_names_out(self, input_features=None):
985
+ """Get output feature names for transformation.
986
+
987
+ The feature names out will prefixed by the lowercased class name. For
988
+ example, if the transformer outputs 3 features, then the feature names
989
+ out are: `["class_name0", "class_name1", "class_name2"]`.
990
+
991
+ Parameters
992
+ ----------
993
+ input_features : array-like of str or None, default=None
994
+ Only used to validate feature names with the names seen in `fit`.
995
+
996
+ Returns
997
+ -------
998
+ feature_names_out : ndarray of str objects
999
+ Transformed feature names.
1000
+ """
1001
+ check_is_fitted(self, "_n_features_out")
1002
+ return _generate_get_feature_names_out(
1003
+ self, self._n_features_out, input_features=input_features
1004
+ )
1005
+
1006
+
1007
+ class DensityMixin:
1008
+ """Mixin class for all density estimators in scikit-learn.
1009
+
1010
+ This mixin defines the following functionality:
1011
+
1012
+ - sets estimator type to `"density_estimator"` through the `estimator_type` tag;
1013
+ - `score` method that default that do no-op.
1014
+
1015
+ Examples
1016
+ --------
1017
+ >>> from sklearn.base import DensityMixin
1018
+ >>> class MyEstimator(DensityMixin):
1019
+ ... def fit(self, X, y=None):
1020
+ ... self.is_fitted_ = True
1021
+ ... return self
1022
+ >>> estimator = MyEstimator()
1023
+ >>> hasattr(estimator, "score")
1024
+ True
1025
+ """
1026
+
1027
+ def __sklearn_tags__(self):
1028
+ tags = super().__sklearn_tags__()
1029
+ tags.estimator_type = "density_estimator"
1030
+ return tags
1031
+
1032
+ def score(self, X, y=None):
1033
+ """Return the score of the model on the data `X`.
1034
+
1035
+ Parameters
1036
+ ----------
1037
+ X : array-like of shape (n_samples, n_features)
1038
+ Test samples.
1039
+
1040
+ y : Ignored
1041
+ Not used, present for API consistency by convention.
1042
+
1043
+ Returns
1044
+ -------
1045
+ score : float
1046
+ """
1047
+ pass
1048
+
1049
+
1050
+ class OutlierMixin:
1051
+ """Mixin class for all outlier detection estimators in scikit-learn.
1052
+
1053
+ This mixin defines the following functionality:
1054
+
1055
+ - set estimator type to `"outlier_detector"` through the `estimator_type` tag;
1056
+ - `fit_predict` method that default to `fit` and `predict`.
1057
+
1058
+ Examples
1059
+ --------
1060
+ >>> import numpy as np
1061
+ >>> from sklearn.base import BaseEstimator, OutlierMixin
1062
+ >>> class MyEstimator(OutlierMixin):
1063
+ ... def fit(self, X, y=None):
1064
+ ... self.is_fitted_ = True
1065
+ ... return self
1066
+ ... def predict(self, X):
1067
+ ... return np.ones(shape=len(X))
1068
+ >>> estimator = MyEstimator()
1069
+ >>> X = np.array([[1, 2], [2, 3], [3, 4]])
1070
+ >>> estimator.fit_predict(X)
1071
+ array([1., 1., 1.])
1072
+ """
1073
+
1074
+ def __sklearn_tags__(self):
1075
+ tags = super().__sklearn_tags__()
1076
+ tags.estimator_type = "outlier_detector"
1077
+ return tags
1078
+
1079
+ def fit_predict(self, X, y=None, **kwargs):
1080
+ """Perform fit on X and returns labels for X.
1081
+
1082
+ Returns -1 for outliers and 1 for inliers.
1083
+
1084
+ Parameters
1085
+ ----------
1086
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
1087
+ The input samples.
1088
+
1089
+ y : Ignored
1090
+ Not used, present for API consistency by convention.
1091
+
1092
+ **kwargs : dict
1093
+ Arguments to be passed to ``fit``.
1094
+
1095
+ .. versionadded:: 1.4
1096
+
1097
+ Returns
1098
+ -------
1099
+ y : ndarray of shape (n_samples,)
1100
+ 1 for inliers, -1 for outliers.
1101
+ """
1102
+ # we do not route parameters here, since consumers don't route. But
1103
+ # since it's possible for a `predict` method to also consume
1104
+ # metadata, we check if that's the case, and we raise a warning telling
1105
+ # users that they should implement a custom `fit_predict` method
1106
+ # to forward metadata to `predict` as well.
1107
+ #
1108
+ # For that, we calculate routing and check if anything would be routed
1109
+ # to `predict` if we were to route them.
1110
+ if _routing_enabled():
1111
+ transform_params = self.get_metadata_routing().consumes(
1112
+ method="predict", params=kwargs.keys()
1113
+ )
1114
+ if transform_params:
1115
+ warnings.warn(
1116
+ (
1117
+ f"This object ({self.__class__.__name__}) has a `predict` "
1118
+ "method which consumes metadata, but `fit_predict` does not "
1119
+ "forward metadata to `predict`. Please implement a custom "
1120
+ "`fit_predict` method to forward metadata to `predict` as well."
1121
+ "Alternatively, you can explicitly do `set_predict_request`"
1122
+ "and set all values to `False` to disable metadata routed to "
1123
+ "`predict`, if that's an option."
1124
+ ),
1125
+ UserWarning,
1126
+ )
1127
+
1128
+ # override for transductive outlier detectors like LocalOulierFactor
1129
+ return self.fit(X, **kwargs).predict(X)
1130
+
1131
+
1132
+ class MetaEstimatorMixin:
1133
+ """Mixin class for all meta estimators in scikit-learn.
1134
+
1135
+ This mixin is empty, and only exists to indicate that the estimator is a
1136
+ meta-estimator.
1137
+
1138
+ .. versionchanged:: 1.6
1139
+ The `_required_parameters` is now removed and is unnecessary since tests are
1140
+ refactored and don't use this anymore.
1141
+
1142
+ Examples
1143
+ --------
1144
+ >>> from sklearn.base import MetaEstimatorMixin
1145
+ >>> from sklearn.datasets import load_iris
1146
+ >>> from sklearn.linear_model import LogisticRegression
1147
+ >>> class MyEstimator(MetaEstimatorMixin):
1148
+ ... def __init__(self, *, estimator=None):
1149
+ ... self.estimator = estimator
1150
+ ... def fit(self, X, y=None):
1151
+ ... if self.estimator is None:
1152
+ ... self.estimator_ = LogisticRegression()
1153
+ ... else:
1154
+ ... self.estimator_ = self.estimator
1155
+ ... return self
1156
+ >>> X, y = load_iris(return_X_y=True)
1157
+ >>> estimator = MyEstimator().fit(X, y)
1158
+ >>> estimator.estimator_
1159
+ LogisticRegression()
1160
+ """
1161
+
1162
+
1163
+ class MultiOutputMixin:
1164
+ """Mixin to mark estimators that support multioutput."""
1165
+
1166
+ def __sklearn_tags__(self):
1167
+ tags = super().__sklearn_tags__()
1168
+ tags.target_tags.multi_output = True
1169
+ return tags
1170
+
1171
+
1172
+ class _UnstableArchMixin:
1173
+ """Mark estimators that are non-determinstic on 32bit or PowerPC"""
1174
+
1175
+ def __sklearn_tags__(self):
1176
+ tags = super().__sklearn_tags__()
1177
+ tags.non_deterministic = _IS_32BIT or platform.machine().startswith(
1178
+ ("ppc", "powerpc")
1179
+ )
1180
+ return tags
1181
+
1182
+
1183
+ def is_classifier(estimator):
1184
+ """Return True if the given estimator is (probably) a classifier.
1185
+
1186
+ Parameters
1187
+ ----------
1188
+ estimator : estimator instance
1189
+ Estimator object to test.
1190
+
1191
+ Returns
1192
+ -------
1193
+ out : bool
1194
+ True if estimator is a classifier and False otherwise.
1195
+
1196
+ Examples
1197
+ --------
1198
+ >>> from sklearn.base import is_classifier
1199
+ >>> from sklearn.cluster import KMeans
1200
+ >>> from sklearn.svm import SVC, SVR
1201
+ >>> classifier = SVC()
1202
+ >>> regressor = SVR()
1203
+ >>> kmeans = KMeans()
1204
+ >>> is_classifier(classifier)
1205
+ True
1206
+ >>> is_classifier(regressor)
1207
+ False
1208
+ >>> is_classifier(kmeans)
1209
+ False
1210
+ """
1211
+ return get_tags(estimator).estimator_type == "classifier"
1212
+
1213
+
1214
+ def is_regressor(estimator):
1215
+ """Return True if the given estimator is (probably) a regressor.
1216
+
1217
+ Parameters
1218
+ ----------
1219
+ estimator : estimator instance
1220
+ Estimator object to test.
1221
+
1222
+ Returns
1223
+ -------
1224
+ out : bool
1225
+ True if estimator is a regressor and False otherwise.
1226
+
1227
+ Examples
1228
+ --------
1229
+ >>> from sklearn.base import is_regressor
1230
+ >>> from sklearn.cluster import KMeans
1231
+ >>> from sklearn.svm import SVC, SVR
1232
+ >>> classifier = SVC()
1233
+ >>> regressor = SVR()
1234
+ >>> kmeans = KMeans()
1235
+ >>> is_regressor(classifier)
1236
+ False
1237
+ >>> is_regressor(regressor)
1238
+ True
1239
+ >>> is_regressor(kmeans)
1240
+ False
1241
+ """
1242
+ return get_tags(estimator).estimator_type == "regressor"
1243
+
1244
+
1245
+ def is_clusterer(estimator):
1246
+ """Return True if the given estimator is (probably) a clusterer.
1247
+
1248
+ .. versionadded:: 1.6
1249
+
1250
+ Parameters
1251
+ ----------
1252
+ estimator : estimator instance
1253
+ Estimator object to test.
1254
+
1255
+ Returns
1256
+ -------
1257
+ out : bool
1258
+ True if estimator is a clusterer and False otherwise.
1259
+
1260
+ Examples
1261
+ --------
1262
+ >>> from sklearn.base import is_clusterer
1263
+ >>> from sklearn.cluster import KMeans
1264
+ >>> from sklearn.svm import SVC, SVR
1265
+ >>> classifier = SVC()
1266
+ >>> regressor = SVR()
1267
+ >>> kmeans = KMeans()
1268
+ >>> is_clusterer(classifier)
1269
+ False
1270
+ >>> is_clusterer(regressor)
1271
+ False
1272
+ >>> is_clusterer(kmeans)
1273
+ True
1274
+ """
1275
+ return get_tags(estimator).estimator_type == "clusterer"
1276
+
1277
+
1278
+ def is_outlier_detector(estimator):
1279
+ """Return True if the given estimator is (probably) an outlier detector.
1280
+
1281
+ Parameters
1282
+ ----------
1283
+ estimator : estimator instance
1284
+ Estimator object to test.
1285
+
1286
+ Returns
1287
+ -------
1288
+ out : bool
1289
+ True if estimator is an outlier detector and False otherwise.
1290
+ """
1291
+ return get_tags(estimator).estimator_type == "outlier_detector"
1292
+
1293
+
1294
+ def _fit_context(*, prefer_skip_nested_validation):
1295
+ """Decorator to run the fit methods of estimators within context managers.
1296
+
1297
+ Parameters
1298
+ ----------
1299
+ prefer_skip_nested_validation : bool
1300
+ If True, the validation of parameters of inner estimators or functions
1301
+ called during fit will be skipped.
1302
+
1303
+ This is useful to avoid validating many times the parameters passed by the
1304
+ user from the public facing API. It's also useful to avoid validating
1305
+ parameters that we pass internally to inner functions that are guaranteed to
1306
+ be valid by the test suite.
1307
+
1308
+ It should be set to True for most estimators, except for those that receive
1309
+ non-validated objects as parameters, such as meta-estimators that are given
1310
+ estimator objects.
1311
+
1312
+ Returns
1313
+ -------
1314
+ decorated_fit : method
1315
+ The decorated fit method.
1316
+ """
1317
+
1318
+ def decorator(fit_method):
1319
+ @functools.wraps(fit_method)
1320
+ def wrapper(estimator, *args, **kwargs):
1321
+ global_skip_validation = get_config()["skip_parameter_validation"]
1322
+
1323
+ # we don't want to validate again for each call to partial_fit
1324
+ partial_fit_and_fitted = (
1325
+ fit_method.__name__ == "partial_fit" and _is_fitted(estimator)
1326
+ )
1327
+
1328
+ if not global_skip_validation and not partial_fit_and_fitted:
1329
+ estimator._validate_params()
1330
+
1331
+ with config_context(
1332
+ skip_parameter_validation=(
1333
+ prefer_skip_nested_validation or global_skip_validation
1334
+ )
1335
+ ):
1336
+ return fit_method(estimator, *args, **kwargs)
1337
+
1338
+ return wrapper
1339
+
1340
+ return decorator
loan-advisor-agent/.venv/Lib/site-packages/sklearn/calibration.py ADDED
@@ -0,0 +1,1709 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Methods for calibrating predicted probabilities."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ import warnings
7
+ from functools import partial
8
+ from inspect import signature
9
+ from math import log
10
+ from numbers import Integral, Real
11
+
12
+ import numpy as np
13
+ from scipy.optimize import minimize, minimize_scalar
14
+ from scipy.special import expit
15
+
16
+ from sklearn._loss import HalfBinomialLoss, HalfMultinomialLoss
17
+ from sklearn.base import (
18
+ BaseEstimator,
19
+ ClassifierMixin,
20
+ MetaEstimatorMixin,
21
+ RegressorMixin,
22
+ _fit_context,
23
+ clone,
24
+ )
25
+ from sklearn.externals import array_api_extra as xpx
26
+ from sklearn.frozen import FrozenEstimator
27
+ from sklearn.isotonic import IsotonicRegression
28
+ from sklearn.model_selection import LeaveOneOut, check_cv, cross_val_predict
29
+ from sklearn.preprocessing import LabelEncoder, label_binarize
30
+ from sklearn.svm import LinearSVC
31
+ from sklearn.utils import Bunch, _safe_indexing, column_or_1d, get_tags, indexable
32
+ from sklearn.utils._array_api import (
33
+ _convert_to_numpy,
34
+ _half_multinomial_loss,
35
+ _is_numpy_namespace,
36
+ get_namespace,
37
+ get_namespace_and_device,
38
+ move_to,
39
+ )
40
+ from sklearn.utils._param_validation import (
41
+ HasMethods,
42
+ Interval,
43
+ StrOptions,
44
+ validate_params,
45
+ )
46
+ from sklearn.utils._plotting import (
47
+ _BinaryClassifierCurveDisplayMixin,
48
+ _validate_style_kwargs,
49
+ )
50
+ from sklearn.utils._response import _get_response_values, _process_predict_proba
51
+ from sklearn.utils.extmath import softmax
52
+ from sklearn.utils.metadata_routing import (
53
+ MetadataRouter,
54
+ MethodMapping,
55
+ _routing_enabled,
56
+ process_routing,
57
+ )
58
+ from sklearn.utils.multiclass import check_classification_targets
59
+ from sklearn.utils.parallel import Parallel, delayed
60
+ from sklearn.utils.validation import (
61
+ _check_method_params,
62
+ _check_pos_label_consistency,
63
+ _check_response_method,
64
+ _check_sample_weight,
65
+ _num_samples,
66
+ check_array,
67
+ check_consistent_length,
68
+ check_is_fitted,
69
+ )
70
+
71
+
72
+ class CalibratedClassifierCV(ClassifierMixin, MetaEstimatorMixin, BaseEstimator):
73
+ """Calibrate probabilities using isotonic, sigmoid, or temperature scaling.
74
+
75
+ This class uses cross-validation to both estimate the parameters of a
76
+ classifier and subsequently calibrate a classifier. With
77
+ `ensemble=True`, for each cv split it
78
+ fits a copy of the base estimator to the training subset, and calibrates it
79
+ using the testing subset. For prediction, predicted probabilities are
80
+ averaged across these individual calibrated classifiers. When
81
+ `ensemble=False`, cross-validation is used to obtain unbiased predictions,
82
+ via :func:`~sklearn.model_selection.cross_val_predict`, which are then
83
+ used for calibration. For prediction, the base estimator, trained using all
84
+ the data, is used. This is the prediction method implemented when
85
+ `probabilities=True` for :class:`~sklearn.svm.SVC` and :class:`~sklearn.svm.NuSVC`
86
+ estimators (see :ref:`User Guide <scores_probabilities>` for details).
87
+
88
+ Already fitted classifiers can be calibrated by wrapping the model in a
89
+ :class:`~sklearn.frozen.FrozenEstimator`. In this case all provided
90
+ data is used for calibration. The user has to take care manually that data
91
+ for model fitting and calibration are disjoint.
92
+
93
+ The calibration is based on the :term:`decision_function` method of the
94
+ `estimator` if it exists, else on :term:`predict_proba`.
95
+
96
+ Read more in the :ref:`User Guide <calibration>`.
97
+ In order to learn more on the CalibratedClassifierCV class, see the
98
+ following calibration examples:
99
+ :ref:`sphx_glr_auto_examples_calibration_plot_calibration.py`,
100
+ :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py`, and
101
+ :ref:`sphx_glr_auto_examples_calibration_plot_calibration_multiclass.py`.
102
+
103
+ Parameters
104
+ ----------
105
+ estimator : estimator instance, default=None
106
+ The classifier whose output need to be calibrated to provide more
107
+ accurate `predict_proba` outputs. The default classifier is
108
+ a :class:`~sklearn.svm.LinearSVC`.
109
+
110
+ .. versionadded:: 1.2
111
+
112
+ method : {'sigmoid', 'isotonic', 'temperature'}, default='sigmoid'
113
+ The method to use for calibration. Can be:
114
+
115
+ - 'sigmoid', which corresponds to Platt's method (i.e. a binary logistic
116
+ regression model).
117
+ - 'isotonic', which is a non-parametric approach.
118
+ - 'temperature', temperature scaling.
119
+
120
+ Sigmoid and isotonic calibration methods natively support only binary
121
+ classifiers and extend to multi-class classification using a One-vs-Rest (OvR)
122
+ strategy with post-hoc renormalization, i.e., adjusting the probabilities after
123
+ calibration to ensure they sum up to 1.
124
+
125
+ In contrast, temperature scaling naturally supports multi-class calibration by
126
+ applying `softmax(classifier_logits/T)` with a value of `T` (temperature)
127
+ that optimizes the log loss.
128
+
129
+ For very uncalibrated classifiers on very imbalanced datasets, sigmoid
130
+ calibration might be preferred because it fits an additional intercept
131
+ parameter. This helps shift decision boundaries appropriately when the
132
+ classifier being calibrated is biased towards the majority class.
133
+
134
+ Isotonic calibration is not recommended when the number of calibration samples
135
+ is too low ``(≪1000)`` since it then tends to overfit.
136
+
137
+ .. versionchanged:: 1.8
138
+ Added option 'temperature'.
139
+
140
+ cv : int, cross-validation generator, or iterable, default=None
141
+ Determines the cross-validation splitting strategy.
142
+ Possible inputs for cv are:
143
+
144
+ - None, to use the default 5-fold cross-validation,
145
+ - integer, to specify the number of folds.
146
+ - :term:`CV splitter`,
147
+ - An iterable yielding (train, test) splits as arrays of indices.
148
+
149
+ For integer/None inputs, if ``y`` is binary or multiclass,
150
+ :class:`~sklearn.model_selection.StratifiedKFold` is used. If ``y`` is
151
+ neither binary nor multiclass, :class:`~sklearn.model_selection.KFold`
152
+ is used.
153
+
154
+ Refer to the :ref:`User Guide <cross_validation>` for the various
155
+ cross-validation strategies that can be used here.
156
+
157
+ .. versionchanged:: 0.22
158
+ ``cv`` default value if None changed from 3-fold to 5-fold.
159
+
160
+ n_jobs : int, default=None
161
+ Number of jobs to run in parallel.
162
+ ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
163
+ ``-1`` means using all processors.
164
+
165
+ Base estimator clones are fitted in parallel across cross-validation
166
+ iterations.
167
+
168
+ See :term:`Glossary <n_jobs>` for more details.
169
+
170
+ .. versionadded:: 0.24
171
+
172
+ ensemble : bool, or "auto", default="auto"
173
+ Determines how the calibrator is fitted.
174
+
175
+ "auto" will use `False` if the `estimator` is a
176
+ :class:`~sklearn.frozen.FrozenEstimator`, and `True` otherwise.
177
+
178
+ If `True`, the `estimator` is fitted using training data, and
179
+ calibrated using testing data, for each `cv` fold. The final estimator
180
+ is an ensemble of `n_cv` fitted classifier and calibrator pairs, where
181
+ `n_cv` is the number of cross-validation folds. The output is the
182
+ average predicted probabilities of all pairs.
183
+
184
+ If `False`, `cv` is used to compute unbiased predictions, via
185
+ :func:`~sklearn.model_selection.cross_val_predict`, which are then
186
+ used for calibration. At prediction time, the classifier used is the
187
+ `estimator` trained on all the data.
188
+ Note that this method is also internally implemented in
189
+ :mod:`sklearn.svm` estimators with the `probabilities=True` parameter.
190
+
191
+ .. versionadded:: 0.24
192
+
193
+ .. versionchanged:: 1.6
194
+ `"auto"` option is added and is the default.
195
+
196
+ Attributes
197
+ ----------
198
+ classes_ : ndarray of shape (n_classes,)
199
+ The class labels.
200
+
201
+ n_features_in_ : int
202
+ Number of features seen during :term:`fit`. Only defined if the
203
+ underlying estimator exposes such an attribute when fit.
204
+
205
+ .. versionadded:: 0.24
206
+
207
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
208
+ Names of features seen during :term:`fit`. Only defined if the
209
+ underlying estimator exposes such an attribute when fit.
210
+
211
+ .. versionadded:: 1.0
212
+
213
+ calibrated_classifiers_ : list (len() equal to cv or 1 if `ensemble=False`)
214
+ The list of classifier and calibrator pairs.
215
+
216
+ - When `ensemble=True`, `n_cv` fitted `estimator` and calibrator pairs.
217
+ `n_cv` is the number of cross-validation folds.
218
+ - When `ensemble=False`, the `estimator`, fitted on all the data, and fitted
219
+ calibrator.
220
+
221
+ .. versionchanged:: 0.24
222
+ Single calibrated classifier case when `ensemble=False`.
223
+
224
+ See Also
225
+ --------
226
+ calibration_curve : Compute true and predicted probabilities
227
+ for a calibration curve.
228
+
229
+ References
230
+ ----------
231
+ .. [1] B. Zadrozny & C. Elkan.
232
+ `Obtaining calibrated probability estimates from decision trees
233
+ and naive Bayesian classifiers
234
+ <https://cseweb.ucsd.edu/~elkan/calibrated.pdf>`_, ICML 2001.
235
+
236
+ .. [2] B. Zadrozny & C. Elkan.
237
+ `Transforming Classifier Scores into Accurate Multiclass
238
+ Probability Estimates
239
+ <https://web.archive.org/web/20060720141520id_/http://www.research.ibm.com:80/people/z/zadrozny/kdd2002-Transf.pdf>`_,
240
+ KDD 2002.
241
+
242
+ .. [3] J. Platt. `Probabilistic Outputs for Support Vector Machines
243
+ and Comparisons to Regularized Likelihood Methods
244
+ <https://www.researchgate.net/profile/John-Platt-2/publication/2594015_Probabilistic_Outputs_for_Support_Vector_Machines_and_Comparisons_to_Regularized_Likelihood_Methods/links/004635154cff5262d6000000/Probabilistic-Outputs-for-Support-Vector-Machines-and-Comparisons-to-Regularized-Likelihood-Methods.pdf>`_,
245
+ 1999.
246
+
247
+ .. [4] A. Niculescu-Mizil & R. Caruana.
248
+ `Predicting Good Probabilities with Supervised Learning
249
+ <https://www.cs.cornell.edu/~alexn/papers/calibration.icml05.crc.rev3.pdf>`_,
250
+ ICML 2005.
251
+
252
+ .. [5] Chuan Guo, Geoff Pleiss, Yu Sun, Kilian Q. Weinberger.
253
+ :doi:`On Calibration of Modern Neural Networks<10.48550/arXiv.1706.04599>`.
254
+ Proceedings of the 34th International Conference on Machine Learning,
255
+ PMLR 70:1321-1330, 2017.
256
+
257
+ Examples
258
+ --------
259
+ >>> from sklearn.datasets import make_classification
260
+ >>> from sklearn.naive_bayes import GaussianNB
261
+ >>> from sklearn.calibration import CalibratedClassifierCV
262
+ >>> X, y = make_classification(n_samples=100, n_features=2,
263
+ ... n_redundant=0, random_state=42)
264
+ >>> base_clf = GaussianNB()
265
+ >>> calibrated_clf = CalibratedClassifierCV(base_clf, cv=3)
266
+ >>> calibrated_clf.fit(X, y)
267
+ CalibratedClassifierCV(...)
268
+ >>> len(calibrated_clf.calibrated_classifiers_)
269
+ 3
270
+ >>> calibrated_clf.predict_proba(X)[:5, :]
271
+ array([[0.110, 0.889],
272
+ [0.072, 0.927],
273
+ [0.928, 0.072],
274
+ [0.928, 0.072],
275
+ [0.072, 0.928]])
276
+ >>> from sklearn.model_selection import train_test_split
277
+ >>> X, y = make_classification(n_samples=100, n_features=2,
278
+ ... n_redundant=0, random_state=42)
279
+ >>> X_train, X_calib, y_train, y_calib = train_test_split(
280
+ ... X, y, random_state=42
281
+ ... )
282
+ >>> base_clf = GaussianNB()
283
+ >>> base_clf.fit(X_train, y_train)
284
+ GaussianNB()
285
+ >>> from sklearn.frozen import FrozenEstimator
286
+ >>> calibrated_clf = CalibratedClassifierCV(FrozenEstimator(base_clf))
287
+ >>> calibrated_clf.fit(X_calib, y_calib)
288
+ CalibratedClassifierCV(...)
289
+ >>> len(calibrated_clf.calibrated_classifiers_)
290
+ 1
291
+ >>> calibrated_clf.predict_proba([[-0.5, 0.5]])
292
+ array([[0.936, 0.063]])
293
+ """
294
+
295
+ _parameter_constraints: dict = {
296
+ "estimator": [
297
+ HasMethods(["fit", "predict_proba"]),
298
+ HasMethods(["fit", "decision_function"]),
299
+ None,
300
+ ],
301
+ "method": [StrOptions({"isotonic", "sigmoid", "temperature"})],
302
+ "cv": ["cv_object"],
303
+ "n_jobs": [Integral, None],
304
+ "ensemble": ["boolean", StrOptions({"auto"})],
305
+ }
306
+
307
+ def __init__(
308
+ self,
309
+ estimator=None,
310
+ *,
311
+ method="sigmoid",
312
+ cv=None,
313
+ n_jobs=None,
314
+ ensemble="auto",
315
+ ):
316
+ self.estimator = estimator
317
+ self.method = method
318
+ self.cv = cv
319
+ self.n_jobs = n_jobs
320
+ self.ensemble = ensemble
321
+
322
+ def _get_estimator(self):
323
+ """Resolve which estimator to return (default is LinearSVC)"""
324
+ if self.estimator is None:
325
+ # we want all classifiers that don't expose a random_state
326
+ # to be deterministic (and we don't want to expose this one).
327
+ estimator = LinearSVC(random_state=0)
328
+ if _routing_enabled():
329
+ estimator.set_fit_request(sample_weight=True)
330
+ else:
331
+ estimator = self.estimator
332
+
333
+ return estimator
334
+
335
+ @_fit_context(
336
+ # CalibratedClassifierCV.estimator is not validated yet
337
+ prefer_skip_nested_validation=False
338
+ )
339
+ def fit(self, X, y, sample_weight=None, **fit_params):
340
+ """Fit the calibrated model.
341
+
342
+ Parameters
343
+ ----------
344
+ X : array-like of shape (n_samples, n_features)
345
+ Training data.
346
+
347
+ y : array-like of shape (n_samples,)
348
+ Target values.
349
+
350
+ sample_weight : array-like of shape (n_samples,), default=None
351
+ Sample weights. If None, then samples are equally weighted.
352
+
353
+ **fit_params : dict
354
+ Parameters to pass to the `fit` method of the underlying
355
+ classifier.
356
+
357
+ Returns
358
+ -------
359
+ self : object
360
+ Returns an instance of self.
361
+ """
362
+ check_classification_targets(y)
363
+ X, y = indexable(X, y)
364
+ estimator = self._get_estimator()
365
+
366
+ _ensemble = self.ensemble
367
+ if _ensemble == "auto":
368
+ _ensemble = not isinstance(estimator, FrozenEstimator)
369
+
370
+ self.calibrated_classifiers_ = []
371
+
372
+ # Set `classes_` using all `y`
373
+ label_encoder_ = LabelEncoder().fit(y)
374
+ self.classes_ = label_encoder_.classes_
375
+ if self.method == "temperature" and isinstance(y[0], str):
376
+ # for temperature scaling if `y` contains strings then encode it
377
+ # right here to avoid fitting LabelEncoder again within the
378
+ # `_fit_calibrator` function.
379
+ y = label_encoder_.transform(y=y)
380
+
381
+ if _routing_enabled():
382
+ routed_params = process_routing(
383
+ self,
384
+ "fit",
385
+ sample_weight=sample_weight,
386
+ **fit_params,
387
+ )
388
+ else:
389
+ # sample_weight checks
390
+ fit_parameters = signature(estimator.fit).parameters
391
+ supports_sw = "sample_weight" in fit_parameters
392
+ if sample_weight is not None and not supports_sw:
393
+ estimator_name = type(estimator).__name__
394
+ warnings.warn(
395
+ f"Since {estimator_name} does not appear to accept"
396
+ " sample_weight, sample weights will only be used for the"
397
+ " calibration itself. This can be caused by a limitation of"
398
+ " the current scikit-learn API. See the following issue for"
399
+ " more details:"
400
+ " https://github.com/scikit-learn/scikit-learn/issues/21134."
401
+ " Be warned that the result of the calibration is likely to be"
402
+ " incorrect."
403
+ )
404
+ routed_params = Bunch()
405
+ routed_params.splitter = Bunch(split={}) # no routing for splitter
406
+ routed_params.estimator = Bunch(fit=fit_params)
407
+ if sample_weight is not None and supports_sw:
408
+ routed_params.estimator.fit["sample_weight"] = sample_weight
409
+
410
+ xp, is_array_api, device_ = get_namespace_and_device(X)
411
+ if is_array_api:
412
+ y, sample_weight = move_to(y, sample_weight, xp=xp, device=device_)
413
+ # Check that each cross-validation fold can have at least one
414
+ # example per class
415
+ if isinstance(self.cv, int):
416
+ n_folds = self.cv
417
+ elif hasattr(self.cv, "n_splits"):
418
+ n_folds = self.cv.n_splits
419
+ else:
420
+ n_folds = None
421
+ if n_folds and xp.any(xp.unique_counts(y)[1] < n_folds):
422
+ raise ValueError(
423
+ f"Requesting {n_folds}-fold "
424
+ "cross-validation but provided less than "
425
+ f"{n_folds} examples for at least one class."
426
+ )
427
+ if isinstance(self.cv, LeaveOneOut):
428
+ raise ValueError(
429
+ "LeaveOneOut cross-validation does not allow"
430
+ "all classes to be present in test splits. "
431
+ "Please use a cross-validation generator that allows "
432
+ "all classes to appear in every test and train split."
433
+ )
434
+ cv = check_cv(self.cv, y, classifier=True)
435
+
436
+ if _ensemble:
437
+ parallel = Parallel(n_jobs=self.n_jobs)
438
+ self.calibrated_classifiers_ = parallel(
439
+ delayed(_fit_classifier_calibrator_pair)(
440
+ clone(estimator),
441
+ X,
442
+ y,
443
+ train=train,
444
+ test=test,
445
+ method=self.method,
446
+ classes=self.classes_,
447
+ xp=xp,
448
+ sample_weight=sample_weight,
449
+ fit_params=routed_params.estimator.fit,
450
+ )
451
+ for train, test in cv.split(X, y, **routed_params.splitter.split)
452
+ )
453
+ else:
454
+ this_estimator = clone(estimator)
455
+ method_name = _check_response_method(
456
+ this_estimator,
457
+ ["decision_function", "predict_proba"],
458
+ ).__name__
459
+ predictions = cross_val_predict(
460
+ estimator=this_estimator,
461
+ X=X,
462
+ y=y,
463
+ cv=cv,
464
+ method=method_name,
465
+ n_jobs=self.n_jobs,
466
+ params=routed_params.estimator.fit,
467
+ )
468
+ if self.classes_.shape[0] == 2:
469
+ # Ensure shape (n_samples, 1) in the binary case
470
+ if method_name == "predict_proba":
471
+ # Select the probability column of the positive class
472
+ predictions = _process_predict_proba(
473
+ y_pred=predictions,
474
+ target_type="binary",
475
+ classes=self.classes_,
476
+ pos_label=self.classes_[1],
477
+ )
478
+ predictions = predictions.reshape(-1, 1)
479
+
480
+ if sample_weight is not None:
481
+ # Check that the sample_weight dtype is consistent with the
482
+ # predictions to avoid unintentional upcasts.
483
+ sample_weight = _check_sample_weight(
484
+ sample_weight, predictions, dtype=predictions.dtype
485
+ )
486
+
487
+ this_estimator.fit(X, y, **routed_params.estimator.fit)
488
+ # Note: Here we don't pass on fit_params because the supported
489
+ # calibrators don't support fit_params anyway
490
+ calibrated_classifier = _fit_calibrator(
491
+ this_estimator,
492
+ predictions,
493
+ y,
494
+ self.classes_,
495
+ self.method,
496
+ xp=xp,
497
+ sample_weight=sample_weight,
498
+ )
499
+ self.calibrated_classifiers_.append(calibrated_classifier)
500
+
501
+ first_clf = self.calibrated_classifiers_[0].estimator
502
+ if hasattr(first_clf, "n_features_in_"):
503
+ self.n_features_in_ = first_clf.n_features_in_
504
+ if hasattr(first_clf, "feature_names_in_"):
505
+ self.feature_names_in_ = first_clf.feature_names_in_
506
+ return self
507
+
508
+ def predict_proba(self, X):
509
+ """Calibrated probabilities of classification.
510
+
511
+ This function returns calibrated probabilities of classification
512
+ according to each class on an array of test vectors X.
513
+
514
+ Parameters
515
+ ----------
516
+ X : array-like of shape (n_samples, n_features)
517
+ The samples, as accepted by `estimator.predict_proba`.
518
+
519
+ Returns
520
+ -------
521
+ C : ndarray of shape (n_samples, n_classes)
522
+ The predicted probas.
523
+ """
524
+ check_is_fitted(self)
525
+ # Compute the arithmetic mean of the predictions of the calibrated
526
+ # classifiers
527
+ xp, _, device_ = get_namespace_and_device(X)
528
+ mean_proba = xp.zeros((_num_samples(X), self.classes_.shape[0]), device=device_)
529
+ for calibrated_classifier in self.calibrated_classifiers_:
530
+ proba = calibrated_classifier.predict_proba(X)
531
+ mean_proba += proba
532
+
533
+ mean_proba /= len(self.calibrated_classifiers_)
534
+
535
+ return mean_proba
536
+
537
+ def predict(self, X):
538
+ """Predict the target of new samples.
539
+
540
+ The predicted class is the class that has the highest probability,
541
+ and can thus be different from the prediction of the uncalibrated classifier.
542
+
543
+ Parameters
544
+ ----------
545
+ X : array-like of shape (n_samples, n_features)
546
+ The samples, as accepted by `estimator.predict`.
547
+
548
+ Returns
549
+ -------
550
+ C : ndarray of shape (n_samples,)
551
+ The predicted class.
552
+ """
553
+ xp, _ = get_namespace(X)
554
+ check_is_fitted(self)
555
+ class_indices = xp.argmax(self.predict_proba(X), axis=1)
556
+ if isinstance(self.classes_[0], str):
557
+ class_indices = _convert_to_numpy(class_indices, xp=xp)
558
+
559
+ return self.classes_[class_indices]
560
+
561
+ def get_metadata_routing(self):
562
+ """Get metadata routing of this object.
563
+
564
+ Please check :ref:`User Guide <metadata_routing>` on how the routing
565
+ mechanism works.
566
+
567
+ Returns
568
+ -------
569
+ routing : MetadataRouter
570
+ A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
571
+ routing information.
572
+ """
573
+ router = (
574
+ MetadataRouter(owner=self)
575
+ .add_self_request(self)
576
+ .add(
577
+ estimator=self._get_estimator(),
578
+ method_mapping=MethodMapping().add(caller="fit", callee="fit"),
579
+ )
580
+ .add(
581
+ splitter=self.cv,
582
+ method_mapping=MethodMapping().add(caller="fit", callee="split"),
583
+ )
584
+ )
585
+ return router
586
+
587
+ def __sklearn_tags__(self):
588
+ tags = super().__sklearn_tags__()
589
+ estimator_tags = get_tags(self._get_estimator())
590
+ tags.input_tags.sparse = estimator_tags.input_tags.sparse
591
+ tags.array_api_support = (
592
+ estimator_tags.array_api_support and self.method == "temperature"
593
+ )
594
+ return tags
595
+
596
+
597
+ def _fit_classifier_calibrator_pair(
598
+ estimator,
599
+ X,
600
+ y,
601
+ train,
602
+ test,
603
+ method,
604
+ classes,
605
+ xp,
606
+ sample_weight=None,
607
+ fit_params=None,
608
+ ):
609
+ """Fit a classifier/calibration pair on a given train/test split.
610
+
611
+ Fit the classifier on the train set, compute its predictions on the test
612
+ set and use the predictions as input to fit the calibrator along with the
613
+ test labels.
614
+
615
+ Parameters
616
+ ----------
617
+ estimator : estimator instance
618
+ Cloned base estimator.
619
+
620
+ X : array-like, shape (n_samples, n_features)
621
+ Sample data.
622
+
623
+ y : array-like, shape (n_samples,)
624
+ Targets.
625
+
626
+ train : ndarray, shape (n_train_indices,)
627
+ Indices of the training subset.
628
+
629
+ test : ndarray, shape (n_test_indices,)
630
+ Indices of the testing subset.
631
+
632
+ method : {'sigmoid', 'isotonic', 'temperature'}
633
+ Method to use for calibration.
634
+
635
+ classes : ndarray, shape (n_classes,)
636
+ The target classes.
637
+
638
+ xp : namespace
639
+ Array API namespace.
640
+
641
+ sample_weight : array-like, default=None
642
+ Sample weights for `X`.
643
+
644
+ fit_params : dict, default=None
645
+ Parameters to pass to the `fit` method of the underlying
646
+ classifier.
647
+
648
+ Returns
649
+ -------
650
+ calibrated_classifier : _CalibratedClassifier instance
651
+ """
652
+ fit_params_train = _check_method_params(X, params=fit_params, indices=train)
653
+ X_train, y_train = _safe_indexing(X, train), _safe_indexing(y, train)
654
+ X_test, y_test = _safe_indexing(X, test), _safe_indexing(y, test)
655
+
656
+ estimator.fit(X_train, y_train, **fit_params_train)
657
+
658
+ predictions, _ = _get_response_values(
659
+ estimator,
660
+ X_test,
661
+ response_method=["decision_function", "predict_proba"],
662
+ )
663
+ if predictions.ndim == 1:
664
+ # Reshape binary output from `(n_samples,)` to `(n_samples, 1)`
665
+ predictions = predictions.reshape(-1, 1)
666
+
667
+ if sample_weight is not None:
668
+ # Check that the sample_weight dtype is consistent with the predictions
669
+ # to avoid unintentional upcasts.
670
+ sample_weight = _check_sample_weight(sample_weight, X, dtype=predictions.dtype)
671
+ sw_test = _safe_indexing(sample_weight, test)
672
+ else:
673
+ sw_test = None
674
+ calibrated_classifier = _fit_calibrator(
675
+ estimator,
676
+ predictions,
677
+ y_test,
678
+ classes,
679
+ method,
680
+ xp=xp,
681
+ sample_weight=sw_test,
682
+ )
683
+ return calibrated_classifier
684
+
685
+
686
+ def _fit_calibrator(clf, predictions, y, classes, method, xp, sample_weight=None):
687
+ """Fit calibrator(s) and return a `_CalibratedClassifier`
688
+ instance.
689
+
690
+ A separate calibrator is fitted for each of the `n_classes`
691
+ (i.e. `len(clf.classes_)`). However, if `n_classes` is 2 or if
692
+ `method` is 'temperature', only one calibrator is fitted.
693
+
694
+ Parameters
695
+ ----------
696
+ clf : estimator instance
697
+ Fitted classifier.
698
+
699
+ predictions : array-like, shape (n_samples, n_classes) or (n_samples, 1) \
700
+ when binary.
701
+ Raw predictions returned by the un-calibrated base classifier.
702
+
703
+ y : array-like, shape (n_samples,)
704
+ The targets. For `method="temperature"`, `y` needs to be label encoded.
705
+
706
+ classes : ndarray, shape (n_classes,)
707
+ All the prediction classes.
708
+
709
+ method : {'sigmoid', 'isotonic', 'temperature'}
710
+ The method to use for calibration.
711
+
712
+ xp : namespace
713
+ Array API namespace.
714
+
715
+ sample_weight : ndarray, shape (n_samples,), default=None
716
+ Sample weights. If None, then samples are equally weighted.
717
+
718
+ Returns
719
+ -------
720
+ pipeline : _CalibratedClassifier instance
721
+ """
722
+ calibrators = []
723
+
724
+ if method in ("isotonic", "sigmoid"):
725
+ Y = label_binarize(y, classes=classes)
726
+ label_encoder = LabelEncoder().fit(classes)
727
+ pos_class_indices = label_encoder.transform(clf.classes_)
728
+ for class_idx, this_pred in zip(pos_class_indices, predictions.T):
729
+ if method == "isotonic":
730
+ calibrator = IsotonicRegression(out_of_bounds="clip")
731
+ else: # "sigmoid"
732
+ calibrator = _SigmoidCalibration()
733
+ calibrator.fit(this_pred, Y[:, class_idx], sample_weight)
734
+ calibrators.append(calibrator)
735
+ elif method == "temperature":
736
+ if classes.shape[0] == 2 and predictions.shape[-1] == 1:
737
+ response_method_name = _check_response_method(
738
+ clf,
739
+ ["decision_function", "predict_proba"],
740
+ ).__name__
741
+ if response_method_name == "predict_proba":
742
+ predictions = xp.concat([1 - predictions, predictions], axis=1)
743
+ calibrator = _TemperatureScaling()
744
+ calibrator.fit(predictions, y, sample_weight)
745
+ calibrators.append(calibrator)
746
+
747
+ pipeline = _CalibratedClassifier(clf, calibrators, method=method, classes=classes)
748
+ return pipeline
749
+
750
+
751
+ class _CalibratedClassifier:
752
+ """Pipeline-like chaining a fitted classifier and its fitted calibrators.
753
+
754
+ Parameters
755
+ ----------
756
+ estimator : estimator instance
757
+ Fitted classifier.
758
+
759
+ calibrators : list of fitted estimator instances
760
+ List of fitted calibrators (either 'IsotonicRegression' or
761
+ '_SigmoidCalibration'). The number of calibrators equals the number of
762
+ classes. However, if there are 2 classes, the list contains only one
763
+ fitted calibrator.
764
+
765
+ classes : array-like of shape (n_classes,)
766
+ All the prediction classes.
767
+
768
+ method : {'sigmoid', 'isotonic'}, default='sigmoid'
769
+ The method to use for calibration. Can be 'sigmoid' which
770
+ corresponds to Platt's method or 'isotonic' which is a
771
+ non-parametric approach based on isotonic regression.
772
+ """
773
+
774
+ def __init__(self, estimator, calibrators, *, classes, method="sigmoid"):
775
+ self.estimator = estimator
776
+ self.calibrators = calibrators
777
+ self.classes = classes
778
+ self.method = method
779
+
780
+ def predict_proba(self, X):
781
+ """Calculate calibrated probabilities.
782
+
783
+ Calculates classification calibrated probabilities
784
+ for each class, in a one-vs-all manner, for `X`.
785
+
786
+ Parameters
787
+ ----------
788
+ X : ndarray of shape (n_samples, n_features)
789
+ The sample data.
790
+
791
+ Returns
792
+ -------
793
+ proba : array, shape (n_samples, n_classes)
794
+ The predicted probabilities. Can be exact zeros.
795
+ """
796
+ predictions, _ = _get_response_values(
797
+ self.estimator,
798
+ X,
799
+ response_method=["decision_function", "predict_proba"],
800
+ )
801
+ if predictions.ndim == 1:
802
+ # Reshape binary output from `(n_samples,)` to `(n_samples, 1)`
803
+ predictions = predictions.reshape(-1, 1)
804
+
805
+ n_classes = self.classes.shape[0]
806
+
807
+ proba = np.zeros((_num_samples(X), n_classes))
808
+
809
+ if self.method in ("sigmoid", "isotonic"):
810
+ label_encoder = LabelEncoder().fit(self.classes)
811
+ pos_class_indices = label_encoder.transform(self.estimator.classes_)
812
+ for class_idx, this_pred, calibrator in zip(
813
+ pos_class_indices, predictions.T, self.calibrators
814
+ ):
815
+ if n_classes == 2:
816
+ # When binary, `predictions` consists only of predictions for
817
+ # clf.classes_[1] but `pos_class_indices` = 0
818
+ class_idx += 1
819
+ proba[:, class_idx] = calibrator.predict(this_pred)
820
+ # Normalize the probabilities
821
+ if n_classes == 2:
822
+ proba[:, 0] = 1.0 - proba[:, 1]
823
+ else:
824
+ denominator = np.sum(proba, axis=1)[:, np.newaxis]
825
+ # In the edge case where for each class calibrator returns a zero
826
+ # probability for a given sample, use the uniform distribution
827
+ # instead.
828
+ uniform_proba = np.full_like(proba, 1 / n_classes)
829
+ proba = np.divide(
830
+ proba, denominator, out=uniform_proba, where=denominator != 0
831
+ )
832
+ elif self.method == "temperature":
833
+ xp, _ = get_namespace(predictions)
834
+ if n_classes == 2 and predictions.shape[-1] == 1:
835
+ response_method_name = _check_response_method(
836
+ self.estimator,
837
+ ["decision_function", "predict_proba"],
838
+ ).__name__
839
+ if response_method_name == "predict_proba":
840
+ predictions = xp.concat([1 - predictions, predictions], axis=1)
841
+ proba = self.calibrators[0].predict(predictions)
842
+
843
+ # Deal with cases where the predicted probability minimally exceeds 1.0
844
+ proba[(1.0 < proba) & (proba <= 1.0 + 1e-5)] = 1.0
845
+
846
+ return proba
847
+
848
+
849
+ # The max_abs_prediction_threshold was approximated using
850
+ # logit(np.finfo(np.float64).eps) which is about -36
851
+ def _sigmoid_calibration(
852
+ predictions, y, sample_weight=None, max_abs_prediction_threshold=30
853
+ ):
854
+ """Probability Calibration with sigmoid method (Platt 2000)
855
+
856
+ Parameters
857
+ ----------
858
+ predictions : ndarray of shape (n_samples,)
859
+ The decision function or predict proba for the samples.
860
+
861
+ y : ndarray of shape (n_samples,)
862
+ The targets.
863
+
864
+ sample_weight : array-like of shape (n_samples,), default=None
865
+ Sample weights. If None, then samples are equally weighted.
866
+
867
+ Returns
868
+ -------
869
+ a : float
870
+ The slope.
871
+
872
+ b : float
873
+ The intercept.
874
+
875
+ References
876
+ ----------
877
+ Platt, "Probabilistic Outputs for Support Vector Machines"
878
+ """
879
+ predictions = column_or_1d(predictions)
880
+ y = column_or_1d(y)
881
+
882
+ F = predictions # F follows Platt's notations
883
+
884
+ scale_constant = 1.0
885
+ max_prediction = np.max(np.abs(F))
886
+
887
+ # If the predictions have large values we scale them in order to bring
888
+ # them within a suitable range. This has no effect on the final
889
+ # (prediction) result because linear models like Logisitic Regression
890
+ # without a penalty are invariant to multiplying the features by a
891
+ # constant.
892
+ if max_prediction >= max_abs_prediction_threshold:
893
+ scale_constant = max_prediction
894
+ # We rescale the features in a copy: inplace rescaling could confuse
895
+ # the caller and make the code harder to reason about.
896
+ F = F / scale_constant
897
+
898
+ # Bayesian priors (see Platt end of section 2.2):
899
+ # It corresponds to the number of samples, taking into account the
900
+ # `sample_weight`.
901
+ mask_negative_samples = y <= 0
902
+ if sample_weight is not None:
903
+ prior0 = (sample_weight[mask_negative_samples]).sum()
904
+ prior1 = (sample_weight[~mask_negative_samples]).sum()
905
+ else:
906
+ prior0 = float(np.sum(mask_negative_samples))
907
+ prior1 = y.shape[0] - prior0
908
+ T = np.zeros_like(y, dtype=predictions.dtype)
909
+ T[y > 0] = (prior1 + 1.0) / (prior1 + 2.0)
910
+ T[y <= 0] = 1.0 / (prior0 + 2.0)
911
+
912
+ bin_loss = HalfBinomialLoss()
913
+
914
+ def loss_grad(AB):
915
+ # .astype below is needed to ensure y_true and raw_prediction have the
916
+ # same dtype. With result = np.float64(0) * np.array([1, 2], dtype=np.float32)
917
+ # - in Numpy 2, result.dtype is float64
918
+ # - in Numpy<2, result.dtype is float32
919
+ raw_prediction = -(AB[0] * F + AB[1]).astype(dtype=predictions.dtype)
920
+ l, g = bin_loss.loss_gradient(
921
+ y_true=T,
922
+ raw_prediction=raw_prediction,
923
+ sample_weight=sample_weight,
924
+ )
925
+ loss = l.sum()
926
+ # TODO: Remove casting to np.float64 when minimum supported SciPy is 1.11.2
927
+ # With SciPy >= 1.11.2, the LBFGS implementation will cast to float64
928
+ # https://github.com/scipy/scipy/pull/18825.
929
+ # Here we cast to float64 to support SciPy < 1.11.2
930
+ grad = np.asarray([-g @ F, -g.sum()], dtype=np.float64)
931
+ return loss, grad
932
+
933
+ AB0 = np.array([0.0, log((prior0 + 1.0) / (prior1 + 1.0))])
934
+
935
+ opt_result = minimize(
936
+ loss_grad,
937
+ AB0,
938
+ method="L-BFGS-B",
939
+ jac=True,
940
+ options={
941
+ "gtol": 1e-6,
942
+ "ftol": 64 * np.finfo(float).eps,
943
+ },
944
+ )
945
+ AB_ = opt_result.x
946
+
947
+ # The tuned multiplicative parameter is converted back to the original
948
+ # input feature scale. The offset parameter does not need rescaling since
949
+ # we did not rescale the outcome variable.
950
+ return AB_[0] / scale_constant, AB_[1]
951
+
952
+
953
+ def _convert_to_logits(decision_values, eps=1e-12, xp=None):
954
+ """Convert decision_function values to 2D and predict_proba values to logits.
955
+
956
+ This function ensures that the output of `decision_function` is
957
+ converted into a (n_samples, n_classes) array. For binary classification,
958
+ each row contains logits for the negative and positive classes as (-x, x).
959
+
960
+ If `predict_proba` is provided instead, it is converted into
961
+ log-probabilities using `numpy.log`.
962
+
963
+ Parameters
964
+ ----------
965
+ decision_values : array-like of shape (n_samples,) or (n_samples, 1) \
966
+ or (n_samples, n_classes).
967
+
968
+ The decision function values or probability estimates.
969
+ - If shape is (n_samples,), converts to (n_samples, 2) with (-x, x).
970
+ - If shape is (n_samples, 1), converts to (n_samples, 2) with (-x, x).
971
+ - If shape is (n_samples, n_classes), returns unchanged.
972
+ - For probability estimates, returns `numpy.log(decision_values + eps)`.
973
+
974
+ eps : float
975
+ Small positive value added to avoid log(0).
976
+
977
+ Returns
978
+ -------
979
+ logits : ndarray of shape (n_samples, n_classes)
980
+ """
981
+ xp, _, device_ = get_namespace_and_device(decision_values, xp=xp)
982
+ decision_values = check_array(
983
+ decision_values, dtype=[xp.float64, xp.float32], ensure_2d=False
984
+ )
985
+ if (decision_values.ndim == 2) and (decision_values.shape[1] > 1):
986
+ # Check if it is the output of predict_proba
987
+ entries_zero_to_one = xp.all((decision_values >= 0) & (decision_values <= 1))
988
+ # TODO: simplify once upstream issue is addressed
989
+ # https://github.com/data-apis/array-api-extra/issues/478
990
+ row_sums_to_one = xp.all(
991
+ xpx.isclose(
992
+ xp.sum(decision_values, axis=1),
993
+ xp.asarray(1.0, device=device_, dtype=decision_values.dtype),
994
+ )
995
+ )
996
+
997
+ if entries_zero_to_one and row_sums_to_one:
998
+ logits = xp.log(decision_values + eps)
999
+ else:
1000
+ logits = decision_values
1001
+
1002
+ elif (decision_values.ndim == 2) and (decision_values.shape[1] == 1):
1003
+ logits = xp.concat([-decision_values, decision_values], axis=1)
1004
+
1005
+ elif decision_values.ndim == 1:
1006
+ decision_values = xp.reshape(decision_values, (-1, 1))
1007
+ logits = xp.concat([-decision_values, decision_values], axis=1)
1008
+
1009
+ return logits
1010
+
1011
+
1012
+ class _SigmoidCalibration(RegressorMixin, BaseEstimator):
1013
+ """Sigmoid regression model.
1014
+
1015
+ Attributes
1016
+ ----------
1017
+ a_ : float
1018
+ The slope.
1019
+
1020
+ b_ : float
1021
+ The intercept.
1022
+ """
1023
+
1024
+ def fit(self, X, y, sample_weight=None):
1025
+ """Fit the model using X, y as training data.
1026
+
1027
+ Parameters
1028
+ ----------
1029
+ X : array-like of shape (n_samples,)
1030
+ Training data.
1031
+
1032
+ y : array-like of shape (n_samples,)
1033
+ Training target.
1034
+
1035
+ sample_weight : array-like of shape (n_samples,), default=None
1036
+ Sample weights. If None, then samples are equally weighted.
1037
+
1038
+ Returns
1039
+ -------
1040
+ self : object
1041
+ Returns an instance of self.
1042
+ """
1043
+ X = column_or_1d(X)
1044
+ y = column_or_1d(y)
1045
+ X, y = indexable(X, y)
1046
+
1047
+ self.a_, self.b_ = _sigmoid_calibration(X, y, sample_weight)
1048
+ return self
1049
+
1050
+ def predict(self, T):
1051
+ """Predict new data by linear interpolation.
1052
+
1053
+ Parameters
1054
+ ----------
1055
+ T : array-like of shape (n_samples,)
1056
+ Data to predict from.
1057
+
1058
+ Returns
1059
+ -------
1060
+ T_ : ndarray of shape (n_samples,)
1061
+ The predicted data.
1062
+ """
1063
+ T = column_or_1d(T)
1064
+ return expit(-(self.a_ * T + self.b_))
1065
+
1066
+
1067
+ class _TemperatureScaling(RegressorMixin, BaseEstimator):
1068
+ """Temperature scaling model.
1069
+
1070
+ Attributes
1071
+ ----------
1072
+ beta_ : float
1073
+ The optimized inverse temperature.
1074
+ """
1075
+
1076
+ def fit(self, X, y, sample_weight=None):
1077
+ """Fit the model using X, y as training data.
1078
+
1079
+ Parameters
1080
+ ----------
1081
+ X : ndarray of shape (n_samples,) or (n_samples, n_classes)
1082
+ Training data.
1083
+
1084
+ This should be the output of `decision_function` or `predict_proba`.
1085
+ If the input appears to be probabilities (i.e., values between 0 and 1
1086
+ that sum to 1 across classes), it will be converted to logits using
1087
+ `np.log(p + eps)`.
1088
+
1089
+ Binary decision function outputs (1D) will be converted to two-class
1090
+ logits of the form (-x, x). For shapes of the form (n_samples, 1), the
1091
+ same process applies.
1092
+
1093
+ y : array-like of shape (n_samples,)
1094
+ Training target.
1095
+
1096
+ sample_weight : array-like of shape (n_samples,), default=None
1097
+ Sample weights. If None, then samples are equally weighted.
1098
+
1099
+ Returns
1100
+ -------
1101
+ self : object
1102
+ Returns an instance of self.
1103
+ """
1104
+ xp, _, xp_device = get_namespace_and_device(X, y)
1105
+ X, y = indexable(X, y)
1106
+ check_consistent_length(X, y)
1107
+ logits = _convert_to_logits(X) # guarantees xp.float64 or xp.float32
1108
+
1109
+ dtype_ = logits.dtype
1110
+ labels = column_or_1d(y, dtype=dtype_)
1111
+
1112
+ if sample_weight is not None:
1113
+ sample_weight = _check_sample_weight(sample_weight, labels, dtype=dtype_)
1114
+
1115
+ if _is_numpy_namespace(xp):
1116
+ multinomial_loss = HalfMultinomialLoss(n_classes=logits.shape[1])
1117
+ else:
1118
+ multinomial_loss = partial(_half_multinomial_loss, xp=xp)
1119
+
1120
+ def log_loss(log_beta=0.0):
1121
+ """Compute the log loss as a parameter of the inverse temperature
1122
+ (beta).
1123
+
1124
+ Parameters
1125
+ ----------
1126
+ log_beta : float
1127
+ The current logarithm of the inverse temperature value during
1128
+ optimisation.
1129
+
1130
+ Returns
1131
+ -------
1132
+ negative_log_likelihood_loss : float
1133
+ The negative log likelihood loss.
1134
+
1135
+ """
1136
+ # TODO: numpy 2.0
1137
+ # Ensure raw_prediction has the same dtype as labels using .astype().
1138
+ # Without this, dtype promotion rules differ across NumPy versions:
1139
+ #
1140
+ # beta = np.float64(0)
1141
+ # logits = np.array([1, 2], dtype=np.float32)
1142
+ #
1143
+ # result = beta * logits
1144
+ # - NumPy < 2: result.dtype is float32
1145
+ # - NumPy 2+: result.dtype is float64
1146
+ #
1147
+ # This can cause dtype mismatch errors downstream (e.g., buffer dtype).
1148
+ log_beta = xp.asarray(log_beta, dtype=dtype_, device=xp_device)
1149
+ raw_prediction = xp.exp(log_beta) * logits
1150
+ return multinomial_loss(labels, raw_prediction, sample_weight)
1151
+
1152
+ xatol = 64 * xp.finfo(dtype_).eps
1153
+ log_beta_minimizer = minimize_scalar(
1154
+ log_loss,
1155
+ bounds=(-10.0, 10.0),
1156
+ options={
1157
+ "xatol": xatol,
1158
+ },
1159
+ )
1160
+
1161
+ if not log_beta_minimizer.success: # pragma: no cover
1162
+ raise RuntimeError(
1163
+ "Temperature scaling fails to optimize during calibration. "
1164
+ "Reason from `scipy.optimize.minimize_scalar`: "
1165
+ f"{log_beta_minimizer.message}"
1166
+ )
1167
+
1168
+ self.beta_ = xp.exp(
1169
+ xp.asarray(log_beta_minimizer.x, dtype=dtype_, device=xp_device)
1170
+ )
1171
+
1172
+ return self
1173
+
1174
+ def predict(self, X):
1175
+ """Predict new data by linear interpolation.
1176
+
1177
+ Parameters
1178
+ ----------
1179
+ X : ndarray of shape (n_samples,) or (n_samples, n_classes)
1180
+ Data to predict from.
1181
+
1182
+ This should be the output of `decision_function` or `predict_proba`.
1183
+ If the input appears to be probabilities (i.e., values between 0 and 1
1184
+ that sum to 1 across classes), it will be converted to logits using
1185
+ `np.log(p + eps)`.
1186
+
1187
+ Binary decision function outputs (1D) will be converted to two-class
1188
+ logits of the form (-x, x). For shapes of the form (n_samples, 1), the
1189
+ same process applies.
1190
+
1191
+ Returns
1192
+ -------
1193
+ X_ : ndarray of shape (n_samples, n_classes)
1194
+ The predicted data.
1195
+ """
1196
+ logits = _convert_to_logits(X)
1197
+ return softmax(self.beta_ * logits)
1198
+
1199
+ def __sklearn_tags__(self):
1200
+ tags = super().__sklearn_tags__()
1201
+ tags.input_tags.one_d_array = True
1202
+ tags.input_tags.two_d_array = False
1203
+ return tags
1204
+
1205
+
1206
+ @validate_params(
1207
+ {
1208
+ "y_true": ["array-like"],
1209
+ "y_prob": ["array-like"],
1210
+ "pos_label": [Real, str, "boolean", None],
1211
+ "n_bins": [Interval(Integral, 1, None, closed="left")],
1212
+ "strategy": [StrOptions({"uniform", "quantile"})],
1213
+ },
1214
+ prefer_skip_nested_validation=True,
1215
+ )
1216
+ def calibration_curve(
1217
+ y_true,
1218
+ y_prob,
1219
+ *,
1220
+ pos_label=None,
1221
+ n_bins=5,
1222
+ strategy="uniform",
1223
+ ):
1224
+ """Compute true and predicted probabilities for a calibration curve.
1225
+
1226
+ The method assumes the inputs come from a binary classifier, and
1227
+ discretize the [0, 1] interval into bins.
1228
+
1229
+ Calibration curves may also be referred to as reliability diagrams.
1230
+
1231
+ Read more in the :ref:`User Guide <calibration>`.
1232
+
1233
+ Parameters
1234
+ ----------
1235
+ y_true : array-like of shape (n_samples,)
1236
+ True targets.
1237
+
1238
+ y_prob : array-like of shape (n_samples,)
1239
+ Probabilities of the positive class.
1240
+
1241
+ pos_label : int, float, bool or str, default=None
1242
+ The label of the positive class.
1243
+
1244
+ .. versionadded:: 1.1
1245
+
1246
+ n_bins : int, default=5
1247
+ Number of bins to discretize the [0, 1] interval. A bigger number
1248
+ requires more data. Bins with no samples (i.e. without
1249
+ corresponding values in `y_prob`) will not be returned, thus the
1250
+ returned arrays may have less than `n_bins` values.
1251
+
1252
+ strategy : {'uniform', 'quantile'}, default='uniform'
1253
+ Strategy used to define the widths of the bins.
1254
+
1255
+ uniform
1256
+ The bins have identical widths.
1257
+ quantile
1258
+ The bins have the same number of samples and depend on `y_prob`.
1259
+
1260
+ Returns
1261
+ -------
1262
+ prob_true : ndarray of shape (n_bins,) or smaller
1263
+ The proportion of samples whose class is the positive class, in each
1264
+ bin (fraction of positives).
1265
+
1266
+ prob_pred : ndarray of shape (n_bins,) or smaller
1267
+ The mean predicted probability in each bin.
1268
+
1269
+ See Also
1270
+ --------
1271
+ CalibrationDisplay.from_predictions : Plot calibration curve using true
1272
+ and predicted labels.
1273
+ CalibrationDisplay.from_estimator : Plot calibration curve using an
1274
+ estimator and data.
1275
+
1276
+ References
1277
+ ----------
1278
+ Alexandru Niculescu-Mizil and Rich Caruana (2005) Predicting Good
1279
+ Probabilities With Supervised Learning, in Proceedings of the 22nd
1280
+ International Conference on Machine Learning (ICML).
1281
+ See section 4 (Qualitative Analysis of Predictions).
1282
+
1283
+ Examples
1284
+ --------
1285
+ >>> import numpy as np
1286
+ >>> from sklearn.calibration import calibration_curve
1287
+ >>> y_true = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1])
1288
+ >>> y_pred = np.array([0.1, 0.2, 0.3, 0.4, 0.65, 0.7, 0.8, 0.9, 1.])
1289
+ >>> prob_true, prob_pred = calibration_curve(y_true, y_pred, n_bins=3)
1290
+ >>> prob_true
1291
+ array([0. , 0.5, 1. ])
1292
+ >>> prob_pred
1293
+ array([0.2 , 0.525, 0.85 ])
1294
+ """
1295
+ y_true = column_or_1d(y_true)
1296
+ y_prob = column_or_1d(y_prob)
1297
+ check_consistent_length(y_true, y_prob)
1298
+ pos_label = _check_pos_label_consistency(pos_label, y_true)
1299
+
1300
+ if y_prob.min() < 0 or y_prob.max() > 1:
1301
+ raise ValueError("y_prob has values outside [0, 1].")
1302
+
1303
+ labels = np.unique(y_true)
1304
+ if len(labels) > 2:
1305
+ raise ValueError(
1306
+ f"Only binary classification is supported. Provided labels {labels}."
1307
+ )
1308
+ y_true = y_true == pos_label
1309
+
1310
+ if strategy == "quantile": # Determine bin edges by distribution of data
1311
+ quantiles = np.linspace(0, 1, n_bins + 1)
1312
+ bins = np.percentile(y_prob, quantiles * 100)
1313
+ elif strategy == "uniform":
1314
+ bins = np.linspace(0.0, 1.0, n_bins + 1)
1315
+ else:
1316
+ raise ValueError(
1317
+ "Invalid entry to 'strategy' input. Strategy "
1318
+ "must be either 'quantile' or 'uniform'."
1319
+ )
1320
+
1321
+ binids = np.searchsorted(bins[1:-1], y_prob)
1322
+
1323
+ bin_sums = np.bincount(binids, weights=y_prob, minlength=len(bins))
1324
+ bin_true = np.bincount(binids, weights=y_true, minlength=len(bins))
1325
+ bin_total = np.bincount(binids, minlength=len(bins))
1326
+
1327
+ nonzero = bin_total != 0
1328
+ prob_true = bin_true[nonzero] / bin_total[nonzero]
1329
+ prob_pred = bin_sums[nonzero] / bin_total[nonzero]
1330
+
1331
+ return prob_true, prob_pred
1332
+
1333
+
1334
+ class CalibrationDisplay(_BinaryClassifierCurveDisplayMixin):
1335
+ """Calibration curve (also known as reliability diagram) visualization.
1336
+
1337
+ It is recommended to use
1338
+ :func:`~sklearn.calibration.CalibrationDisplay.from_estimator` or
1339
+ :func:`~sklearn.calibration.CalibrationDisplay.from_predictions`
1340
+ to create a `CalibrationDisplay`. All parameters are stored as attributes.
1341
+
1342
+ Read more about calibration in the :ref:`User Guide <calibration>` and
1343
+ more about the scikit-learn visualization API in :ref:`visualizations`.
1344
+
1345
+ For an example on how to use the visualization, see
1346
+ :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py`.
1347
+
1348
+ .. versionadded:: 1.0
1349
+
1350
+ Parameters
1351
+ ----------
1352
+ prob_true : ndarray of shape (n_bins,)
1353
+ The proportion of samples whose class is the positive class (fraction
1354
+ of positives), in each bin.
1355
+
1356
+ prob_pred : ndarray of shape (n_bins,)
1357
+ The mean predicted probability in each bin.
1358
+
1359
+ y_prob : ndarray of shape (n_samples,)
1360
+ Probability estimates for the positive class, for each sample.
1361
+
1362
+ estimator_name : str, default=None
1363
+ Name of estimator. If None, the estimator name is not shown.
1364
+
1365
+ pos_label : int, float, bool or str, default=None
1366
+ The positive class when calibration curve computed.
1367
+ If not `None`, this value is displayed in the x- and y-axes labels.
1368
+
1369
+ .. versionadded:: 1.1
1370
+
1371
+ Attributes
1372
+ ----------
1373
+ line_ : matplotlib Artist
1374
+ Calibration curve.
1375
+
1376
+ ax_ : matplotlib Axes
1377
+ Axes with calibration curve.
1378
+
1379
+ figure_ : matplotlib Figure
1380
+ Figure containing the curve.
1381
+
1382
+ See Also
1383
+ --------
1384
+ calibration_curve : Compute true and predicted probabilities for a
1385
+ calibration curve.
1386
+ CalibrationDisplay.from_predictions : Plot calibration curve using true
1387
+ and predicted labels.
1388
+ CalibrationDisplay.from_estimator : Plot calibration curve using an
1389
+ estimator and data.
1390
+
1391
+ Examples
1392
+ --------
1393
+ >>> from sklearn.datasets import make_classification
1394
+ >>> from sklearn.model_selection import train_test_split
1395
+ >>> from sklearn.linear_model import LogisticRegression
1396
+ >>> from sklearn.calibration import calibration_curve, CalibrationDisplay
1397
+ >>> X, y = make_classification(random_state=0)
1398
+ >>> X_train, X_test, y_train, y_test = train_test_split(
1399
+ ... X, y, random_state=0)
1400
+ >>> clf = LogisticRegression(random_state=0)
1401
+ >>> clf.fit(X_train, y_train)
1402
+ LogisticRegression(random_state=0)
1403
+ >>> y_prob = clf.predict_proba(X_test)[:, 1]
1404
+ >>> prob_true, prob_pred = calibration_curve(y_test, y_prob, n_bins=10)
1405
+ >>> disp = CalibrationDisplay(prob_true, prob_pred, y_prob)
1406
+ >>> disp.plot()
1407
+ <...>
1408
+ """
1409
+
1410
+ def __init__(
1411
+ self, prob_true, prob_pred, y_prob, *, estimator_name=None, pos_label=None
1412
+ ):
1413
+ self.prob_true = prob_true
1414
+ self.prob_pred = prob_pred
1415
+ self.y_prob = y_prob
1416
+ self.estimator_name = estimator_name
1417
+ self.pos_label = pos_label
1418
+
1419
+ def plot(self, *, ax=None, name=None, ref_line=True, **kwargs):
1420
+ """Plot visualization.
1421
+
1422
+ Extra keyword arguments will be passed to
1423
+ :func:`matplotlib.pyplot.plot`.
1424
+
1425
+ Parameters
1426
+ ----------
1427
+ ax : Matplotlib Axes, default=None
1428
+ Axes object to plot on. If `None`, a new figure and axes is
1429
+ created.
1430
+
1431
+ name : str, default=None
1432
+ Name for labeling curve. If `None`, use `estimator_name` if
1433
+ not `None`, otherwise no labeling is shown.
1434
+
1435
+ ref_line : bool, default=True
1436
+ If `True`, plots a reference line representing a perfectly
1437
+ calibrated classifier.
1438
+
1439
+ **kwargs : dict
1440
+ Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`.
1441
+
1442
+ Returns
1443
+ -------
1444
+ display : :class:`~sklearn.calibration.CalibrationDisplay`
1445
+ Object that stores computed values.
1446
+ """
1447
+ self.ax_, self.figure_, name = self._validate_plot_params(ax=ax, name=name)
1448
+
1449
+ info_pos_label = (
1450
+ f"(Positive class: {self.pos_label})" if self.pos_label is not None else ""
1451
+ )
1452
+
1453
+ default_line_kwargs = {"marker": "s", "linestyle": "-"}
1454
+ if name is not None:
1455
+ default_line_kwargs["label"] = name
1456
+ line_kwargs = _validate_style_kwargs(default_line_kwargs, kwargs)
1457
+
1458
+ ref_line_label = "Perfectly calibrated"
1459
+ existing_ref_line = ref_line_label in self.ax_.get_legend_handles_labels()[1]
1460
+ if ref_line and not existing_ref_line:
1461
+ self.ax_.plot([0, 1], [0, 1], "k:", label=ref_line_label)
1462
+ self.line_ = self.ax_.plot(self.prob_pred, self.prob_true, **line_kwargs)[0]
1463
+
1464
+ # We always have to show the legend for at least the reference line
1465
+ self.ax_.legend(loc="lower right")
1466
+
1467
+ xlabel = f"Mean predicted probability {info_pos_label}"
1468
+ ylabel = f"Fraction of positives {info_pos_label}"
1469
+ self.ax_.set(xlabel=xlabel, ylabel=ylabel)
1470
+
1471
+ return self
1472
+
1473
+ @classmethod
1474
+ def from_estimator(
1475
+ cls,
1476
+ estimator,
1477
+ X,
1478
+ y,
1479
+ *,
1480
+ n_bins=5,
1481
+ strategy="uniform",
1482
+ pos_label=None,
1483
+ name=None,
1484
+ ax=None,
1485
+ ref_line=True,
1486
+ **kwargs,
1487
+ ):
1488
+ """Plot calibration curve using a binary classifier and data.
1489
+
1490
+ A calibration curve, also known as a reliability diagram, uses inputs
1491
+ from a binary classifier and plots the average predicted probability
1492
+ for each bin against the fraction of positive classes, on the
1493
+ y-axis.
1494
+
1495
+ Extra keyword arguments will be passed to
1496
+ :func:`matplotlib.pyplot.plot`.
1497
+
1498
+ Read more about calibration in the :ref:`User Guide <calibration>` and
1499
+ more about the scikit-learn visualization API in :ref:`visualizations`.
1500
+
1501
+ .. versionadded:: 1.0
1502
+
1503
+ Parameters
1504
+ ----------
1505
+ estimator : estimator instance
1506
+ Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
1507
+ in which the last estimator is a classifier. The classifier must
1508
+ have a :term:`predict_proba` method.
1509
+
1510
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
1511
+ Input values.
1512
+
1513
+ y : array-like of shape (n_samples,)
1514
+ Binary target values.
1515
+
1516
+ n_bins : int, default=5
1517
+ Number of bins to discretize the [0, 1] interval into when
1518
+ calculating the calibration curve. A bigger number requires more
1519
+ data.
1520
+
1521
+ strategy : {'uniform', 'quantile'}, default='uniform'
1522
+ Strategy used to define the widths of the bins.
1523
+
1524
+ - `'uniform'`: The bins have identical widths.
1525
+ - `'quantile'`: The bins have the same number of samples and depend
1526
+ on predicted probabilities.
1527
+
1528
+ pos_label : int, float, bool or str, default=None
1529
+ The positive class when computing the calibration curve.
1530
+ By default, `estimators.classes_[1]` is considered as the
1531
+ positive class.
1532
+
1533
+ .. versionadded:: 1.1
1534
+
1535
+ name : str, default=None
1536
+ Name for labeling curve. If `None`, the name of the estimator is
1537
+ used.
1538
+
1539
+ ax : matplotlib axes, default=None
1540
+ Axes object to plot on. If `None`, a new figure and axes is
1541
+ created.
1542
+
1543
+ ref_line : bool, default=True
1544
+ If `True`, plots a reference line representing a perfectly
1545
+ calibrated classifier.
1546
+
1547
+ **kwargs : dict
1548
+ Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`.
1549
+
1550
+ Returns
1551
+ -------
1552
+ display : :class:`~sklearn.calibration.CalibrationDisplay`.
1553
+ Object that stores computed values.
1554
+
1555
+ See Also
1556
+ --------
1557
+ CalibrationDisplay.from_predictions : Plot calibration curve using true
1558
+ and predicted labels.
1559
+
1560
+ Examples
1561
+ --------
1562
+ >>> import matplotlib.pyplot as plt
1563
+ >>> from sklearn.datasets import make_classification
1564
+ >>> from sklearn.model_selection import train_test_split
1565
+ >>> from sklearn.linear_model import LogisticRegression
1566
+ >>> from sklearn.calibration import CalibrationDisplay
1567
+ >>> X, y = make_classification(random_state=0)
1568
+ >>> X_train, X_test, y_train, y_test = train_test_split(
1569
+ ... X, y, random_state=0)
1570
+ >>> clf = LogisticRegression(random_state=0)
1571
+ >>> clf.fit(X_train, y_train)
1572
+ LogisticRegression(random_state=0)
1573
+ >>> disp = CalibrationDisplay.from_estimator(clf, X_test, y_test)
1574
+ >>> plt.show()
1575
+ """
1576
+ y_prob, pos_label, name = cls._validate_and_get_response_values(
1577
+ estimator,
1578
+ X,
1579
+ y,
1580
+ response_method="predict_proba",
1581
+ pos_label=pos_label,
1582
+ name=name,
1583
+ )
1584
+
1585
+ return cls.from_predictions(
1586
+ y,
1587
+ y_prob,
1588
+ n_bins=n_bins,
1589
+ strategy=strategy,
1590
+ pos_label=pos_label,
1591
+ name=name,
1592
+ ref_line=ref_line,
1593
+ ax=ax,
1594
+ **kwargs,
1595
+ )
1596
+
1597
+ @classmethod
1598
+ def from_predictions(
1599
+ cls,
1600
+ y_true,
1601
+ y_prob,
1602
+ *,
1603
+ n_bins=5,
1604
+ strategy="uniform",
1605
+ pos_label=None,
1606
+ name=None,
1607
+ ax=None,
1608
+ ref_line=True,
1609
+ **kwargs,
1610
+ ):
1611
+ """Plot calibration curve using true labels and predicted probabilities.
1612
+
1613
+ Calibration curve, also known as reliability diagram, uses inputs
1614
+ from a binary classifier and plots the average predicted probability
1615
+ for each bin against the fraction of positive classes, on the
1616
+ y-axis.
1617
+
1618
+ Extra keyword arguments will be passed to
1619
+ :func:`matplotlib.pyplot.plot`.
1620
+
1621
+ Read more about calibration in the :ref:`User Guide <calibration>` and
1622
+ more about the scikit-learn visualization API in :ref:`visualizations`.
1623
+
1624
+ .. versionadded:: 1.0
1625
+
1626
+ Parameters
1627
+ ----------
1628
+ y_true : array-like of shape (n_samples,)
1629
+ True labels.
1630
+
1631
+ y_prob : array-like of shape (n_samples,)
1632
+ The predicted probabilities of the positive class.
1633
+
1634
+ n_bins : int, default=5
1635
+ Number of bins to discretize the [0, 1] interval into when
1636
+ calculating the calibration curve. A bigger number requires more
1637
+ data.
1638
+
1639
+ strategy : {'uniform', 'quantile'}, default='uniform'
1640
+ Strategy used to define the widths of the bins.
1641
+
1642
+ - `'uniform'`: The bins have identical widths.
1643
+ - `'quantile'`: The bins have the same number of samples and depend
1644
+ on predicted probabilities.
1645
+
1646
+ pos_label : int, float, bool or str, default=None
1647
+ The positive class when computing the calibration curve.
1648
+ When `pos_label=None`, if `y_true` is in {-1, 1} or {0, 1},
1649
+ `pos_label` is set to 1, otherwise an error will be raised.
1650
+
1651
+ .. versionadded:: 1.1
1652
+
1653
+ name : str, default=None
1654
+ Name for labeling curve.
1655
+
1656
+ ax : matplotlib axes, default=None
1657
+ Axes object to plot on. If `None`, a new figure and axes is
1658
+ created.
1659
+
1660
+ ref_line : bool, default=True
1661
+ If `True`, plots a reference line representing a perfectly
1662
+ calibrated classifier.
1663
+
1664
+ **kwargs : dict
1665
+ Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`.
1666
+
1667
+ Returns
1668
+ -------
1669
+ display : :class:`~sklearn.calibration.CalibrationDisplay`.
1670
+ Object that stores computed values.
1671
+
1672
+ See Also
1673
+ --------
1674
+ CalibrationDisplay.from_estimator : Plot calibration curve using an
1675
+ estimator and data.
1676
+
1677
+ Examples
1678
+ --------
1679
+ >>> import matplotlib.pyplot as plt
1680
+ >>> from sklearn.datasets import make_classification
1681
+ >>> from sklearn.model_selection import train_test_split
1682
+ >>> from sklearn.linear_model import LogisticRegression
1683
+ >>> from sklearn.calibration import CalibrationDisplay
1684
+ >>> X, y = make_classification(random_state=0)
1685
+ >>> X_train, X_test, y_train, y_test = train_test_split(
1686
+ ... X, y, random_state=0)
1687
+ >>> clf = LogisticRegression(random_state=0)
1688
+ >>> clf.fit(X_train, y_train)
1689
+ LogisticRegression(random_state=0)
1690
+ >>> y_prob = clf.predict_proba(X_test)[:, 1]
1691
+ >>> disp = CalibrationDisplay.from_predictions(y_test, y_prob)
1692
+ >>> plt.show()
1693
+ """
1694
+ pos_label_validated, name = cls._validate_from_predictions_params(
1695
+ y_true, y_prob, sample_weight=None, pos_label=pos_label, name=name
1696
+ )
1697
+
1698
+ prob_true, prob_pred = calibration_curve(
1699
+ y_true, y_prob, n_bins=n_bins, strategy=strategy, pos_label=pos_label
1700
+ )
1701
+
1702
+ disp = cls(
1703
+ prob_true=prob_true,
1704
+ prob_pred=prob_pred,
1705
+ y_prob=y_prob,
1706
+ estimator_name=name,
1707
+ pos_label=pos_label_validated,
1708
+ )
1709
+ return disp.plot(ax=ax, ref_line=ref_line, **kwargs)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/__init__.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Popular unsupervised clustering algorithms."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ from sklearn.cluster._affinity_propagation import (
7
+ AffinityPropagation,
8
+ affinity_propagation,
9
+ )
10
+ from sklearn.cluster._agglomerative import (
11
+ AgglomerativeClustering,
12
+ FeatureAgglomeration,
13
+ linkage_tree,
14
+ ward_tree,
15
+ )
16
+ from sklearn.cluster._bicluster import SpectralBiclustering, SpectralCoclustering
17
+ from sklearn.cluster._birch import Birch
18
+ from sklearn.cluster._bisect_k_means import BisectingKMeans
19
+ from sklearn.cluster._dbscan import DBSCAN, dbscan
20
+ from sklearn.cluster._hdbscan.hdbscan import HDBSCAN
21
+ from sklearn.cluster._kmeans import KMeans, MiniBatchKMeans, k_means, kmeans_plusplus
22
+ from sklearn.cluster._mean_shift import (
23
+ MeanShift,
24
+ estimate_bandwidth,
25
+ get_bin_seeds,
26
+ mean_shift,
27
+ )
28
+ from sklearn.cluster._optics import (
29
+ OPTICS,
30
+ cluster_optics_dbscan,
31
+ cluster_optics_xi,
32
+ compute_optics_graph,
33
+ )
34
+ from sklearn.cluster._spectral import SpectralClustering, spectral_clustering
35
+
36
+ __all__ = [
37
+ "DBSCAN",
38
+ "HDBSCAN",
39
+ "OPTICS",
40
+ "AffinityPropagation",
41
+ "AgglomerativeClustering",
42
+ "Birch",
43
+ "BisectingKMeans",
44
+ "FeatureAgglomeration",
45
+ "KMeans",
46
+ "MeanShift",
47
+ "MiniBatchKMeans",
48
+ "SpectralBiclustering",
49
+ "SpectralClustering",
50
+ "SpectralCoclustering",
51
+ "affinity_propagation",
52
+ "cluster_optics_dbscan",
53
+ "cluster_optics_xi",
54
+ "compute_optics_graph",
55
+ "dbscan",
56
+ "estimate_bandwidth",
57
+ "get_bin_seeds",
58
+ "k_means",
59
+ "kmeans_plusplus",
60
+ "linkage_tree",
61
+ "mean_shift",
62
+ "spectral_clustering",
63
+ "ward_tree",
64
+ ]
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_affinity_propagation.py ADDED
@@ -0,0 +1,607 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Affinity Propagation clustering algorithm."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ import warnings
7
+ from numbers import Integral, Real
8
+
9
+ import numpy as np
10
+
11
+ from sklearn._config import config_context
12
+ from sklearn.base import BaseEstimator, ClusterMixin, _fit_context
13
+ from sklearn.exceptions import ConvergenceWarning
14
+ from sklearn.metrics import euclidean_distances, pairwise_distances_argmin
15
+ from sklearn.utils import check_random_state
16
+ from sklearn.utils._param_validation import Interval, StrOptions, validate_params
17
+ from sklearn.utils.validation import check_is_fitted, validate_data
18
+
19
+
20
+ def _equal_similarities_and_preferences(S, preference):
21
+ def all_equal_preferences():
22
+ return np.all(preference == preference.flat[0])
23
+
24
+ def all_equal_similarities():
25
+ # Create mask to ignore diagonal of S
26
+ mask = np.ones(S.shape, dtype=bool)
27
+ np.fill_diagonal(mask, 0)
28
+
29
+ return np.all(S[mask].flat == S[mask].flat[0])
30
+
31
+ return all_equal_preferences() and all_equal_similarities()
32
+
33
+
34
+ def _affinity_propagation(
35
+ S,
36
+ *,
37
+ preference,
38
+ convergence_iter,
39
+ max_iter,
40
+ damping,
41
+ verbose,
42
+ return_n_iter,
43
+ random_state,
44
+ ):
45
+ """Main affinity propagation algorithm."""
46
+ n_samples = S.shape[0]
47
+ if n_samples == 1 or _equal_similarities_and_preferences(S, preference):
48
+ # It makes no sense to run the algorithm in this case, so return 1 or
49
+ # n_samples clusters, depending on preferences
50
+ warnings.warn(
51
+ "All samples have mutually equal similarities. "
52
+ "Returning arbitrary cluster center(s)."
53
+ )
54
+ if preference.flat[0] > S.flat[n_samples - 1]:
55
+ return (
56
+ (np.arange(n_samples), np.arange(n_samples), 0)
57
+ if return_n_iter
58
+ else (np.arange(n_samples), np.arange(n_samples))
59
+ )
60
+ else:
61
+ return (
62
+ (np.array([0]), np.array([0] * n_samples), 0)
63
+ if return_n_iter
64
+ else (np.array([0]), np.array([0] * n_samples))
65
+ )
66
+
67
+ # Place preference on the diagonal of S
68
+ S.flat[:: (n_samples + 1)] = preference
69
+
70
+ A = np.zeros((n_samples, n_samples))
71
+ R = np.zeros((n_samples, n_samples)) # Initialize messages
72
+ # Intermediate results
73
+ tmp = np.zeros((n_samples, n_samples))
74
+
75
+ # Remove degeneracies
76
+ S += (
77
+ np.finfo(S.dtype).eps * S + np.finfo(S.dtype).tiny * 100
78
+ ) * random_state.standard_normal(size=(n_samples, n_samples))
79
+
80
+ # Execute parallel affinity propagation updates
81
+ e = np.zeros((n_samples, convergence_iter))
82
+
83
+ ind = np.arange(n_samples)
84
+
85
+ for it in range(max_iter):
86
+ # tmp = A + S; compute responsibilities
87
+ np.add(A, S, tmp)
88
+ I = np.argmax(tmp, axis=1)
89
+ Y = tmp[ind, I] # np.max(A + S, axis=1)
90
+ tmp[ind, I] = -np.inf
91
+ Y2 = np.max(tmp, axis=1)
92
+
93
+ # tmp = Rnew
94
+ np.subtract(S, Y[:, None], tmp)
95
+ tmp[ind, I] = S[ind, I] - Y2
96
+
97
+ # Damping
98
+ tmp *= 1 - damping
99
+ R *= damping
100
+ R += tmp
101
+
102
+ # tmp = Rp; compute availabilities
103
+ np.maximum(R, 0, out=tmp)
104
+ tmp.flat[:: n_samples + 1] = R.flat[:: n_samples + 1]
105
+
106
+ # tmp = -Anew
107
+ tmp -= np.sum(tmp, axis=0)
108
+ dA = np.diag(tmp).copy()
109
+ tmp.clip(0, np.inf, tmp)
110
+ tmp.flat[:: n_samples + 1] = dA
111
+
112
+ # Damping
113
+ tmp *= 1 - damping
114
+ A *= damping
115
+ A -= tmp
116
+
117
+ # Check for convergence
118
+ E = (np.diag(A) + np.diag(R)) > 0
119
+ e[:, it % convergence_iter] = E
120
+ K = np.sum(E, axis=0)
121
+
122
+ if it >= convergence_iter:
123
+ se = np.sum(e, axis=1)
124
+ unconverged = np.sum((se == convergence_iter) + (se == 0)) != n_samples
125
+ if (not unconverged and (K > 0)) or (it == max_iter):
126
+ never_converged = False
127
+ if verbose:
128
+ print("Converged after %d iterations." % it)
129
+ break
130
+ else:
131
+ never_converged = True
132
+ if verbose:
133
+ print("Did not converge")
134
+
135
+ I = np.flatnonzero(E)
136
+ K = I.size # Identify exemplars
137
+
138
+ if K > 0:
139
+ if never_converged:
140
+ warnings.warn(
141
+ (
142
+ "Affinity propagation did not converge, this model "
143
+ "may return degenerate cluster centers and labels."
144
+ ),
145
+ ConvergenceWarning,
146
+ )
147
+ c = np.argmax(S[:, I], axis=1)
148
+ c[I] = np.arange(K) # Identify clusters
149
+ # Refine the final set of exemplars and clusters and return results
150
+ for k in range(K):
151
+ ii = np.asarray(c == k).nonzero()[0]
152
+ j = np.argmax(np.sum(S[ii[:, np.newaxis], ii], axis=0))
153
+ I[k] = ii[j]
154
+
155
+ c = np.argmax(S[:, I], axis=1)
156
+ c[I] = np.arange(K)
157
+ labels = I[c]
158
+ # Reduce labels to a sorted, gapless, list
159
+ cluster_centers_indices = np.unique(labels)
160
+ labels = np.searchsorted(cluster_centers_indices, labels)
161
+ else:
162
+ warnings.warn(
163
+ (
164
+ "Affinity propagation did not converge and this model "
165
+ "will not have any cluster centers."
166
+ ),
167
+ ConvergenceWarning,
168
+ )
169
+ labels = np.array([-1] * n_samples)
170
+ cluster_centers_indices = []
171
+
172
+ if return_n_iter:
173
+ return cluster_centers_indices, labels, it + 1
174
+ else:
175
+ return cluster_centers_indices, labels
176
+
177
+
178
+ ###############################################################################
179
+ # Public API
180
+
181
+
182
+ @validate_params(
183
+ {
184
+ "S": ["array-like"],
185
+ "return_n_iter": ["boolean"],
186
+ },
187
+ prefer_skip_nested_validation=False,
188
+ )
189
+ def affinity_propagation(
190
+ S,
191
+ *,
192
+ preference=None,
193
+ convergence_iter=15,
194
+ max_iter=200,
195
+ damping=0.5,
196
+ copy=True,
197
+ verbose=False,
198
+ return_n_iter=False,
199
+ random_state=None,
200
+ ):
201
+ """Perform Affinity Propagation Clustering of data.
202
+
203
+ Read more in the :ref:`User Guide <affinity_propagation>`.
204
+
205
+ Parameters
206
+ ----------
207
+ S : array-like of shape (n_samples, n_samples)
208
+ Matrix of similarities between points.
209
+
210
+ preference : array-like of shape (n_samples,) or float, default=None
211
+ Preferences for each point - points with larger values of
212
+ preferences are more likely to be chosen as exemplars. The number of
213
+ exemplars, i.e. of clusters, is influenced by the input preferences
214
+ value. If the preferences are not passed as arguments, they will be
215
+ set to the median of the input similarities (resulting in a moderate
216
+ number of clusters). For a smaller amount of clusters, this can be set
217
+ to the minimum value of the similarities.
218
+
219
+ convergence_iter : int, default=15
220
+ Number of iterations with no change in the number
221
+ of estimated clusters that stops the convergence.
222
+
223
+ max_iter : int, default=200
224
+ Maximum number of iterations.
225
+
226
+ damping : float, default=0.5
227
+ Damping factor between 0.5 and 1.
228
+
229
+ copy : bool, default=True
230
+ If copy is False, the affinity matrix is modified inplace by the
231
+ algorithm, for memory efficiency.
232
+
233
+ verbose : bool, default=False
234
+ The verbosity level.
235
+
236
+ return_n_iter : bool, default=False
237
+ Whether or not to return the number of iterations.
238
+
239
+ random_state : int, RandomState instance or None, default=None
240
+ Pseudo-random number generator to control the starting state.
241
+ Use an int for reproducible results across function calls.
242
+ See the :term:`Glossary <random_state>`.
243
+
244
+ .. versionadded:: 0.23
245
+ this parameter was previously hardcoded as 0.
246
+
247
+ Returns
248
+ -------
249
+ cluster_centers_indices : ndarray of shape (n_clusters,)
250
+ Index of clusters centers.
251
+
252
+ labels : ndarray of shape (n_samples,)
253
+ Cluster labels for each point.
254
+
255
+ n_iter : int
256
+ Number of iterations run. Returned only if `return_n_iter` is
257
+ set to True.
258
+
259
+ Notes
260
+ -----
261
+ For an example usage,
262
+ see :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`.
263
+ You may also check out,
264
+ :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py`
265
+
266
+ When the algorithm does not converge, it will still return an array of
267
+ ``cluster_center_indices`` and labels if there are any exemplars/clusters,
268
+ however they may be degenerate and should be used with caution.
269
+
270
+ When all training samples have equal similarities and equal preferences,
271
+ the assignment of cluster centers and labels depends on the preference.
272
+ If the preference is smaller than the similarities, a single cluster center
273
+ and label ``0`` for every sample will be returned. Otherwise, every
274
+ training sample becomes its own cluster center and is assigned a unique
275
+ label.
276
+
277
+ References
278
+ ----------
279
+ Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages
280
+ Between Data Points", Science Feb. 2007
281
+
282
+ Examples
283
+ --------
284
+ >>> import numpy as np
285
+ >>> from sklearn.cluster import affinity_propagation
286
+ >>> from sklearn.metrics.pairwise import euclidean_distances
287
+ >>> X = np.array([[1, 2], [1, 4], [1, 0],
288
+ ... [4, 2], [4, 4], [4, 0]])
289
+ >>> S = -euclidean_distances(X, squared=True)
290
+ >>> cluster_centers_indices, labels = affinity_propagation(S, random_state=0)
291
+ >>> cluster_centers_indices
292
+ array([0, 3])
293
+ >>> labels
294
+ array([0, 0, 0, 1, 1, 1])
295
+ """
296
+ estimator = AffinityPropagation(
297
+ damping=damping,
298
+ max_iter=max_iter,
299
+ convergence_iter=convergence_iter,
300
+ copy=copy,
301
+ preference=preference,
302
+ affinity="precomputed",
303
+ verbose=verbose,
304
+ random_state=random_state,
305
+ ).fit(S)
306
+
307
+ if return_n_iter:
308
+ return estimator.cluster_centers_indices_, estimator.labels_, estimator.n_iter_
309
+ return estimator.cluster_centers_indices_, estimator.labels_
310
+
311
+
312
+ class AffinityPropagation(ClusterMixin, BaseEstimator):
313
+ """Perform Affinity Propagation Clustering of data.
314
+
315
+ Read more in the :ref:`User Guide <affinity_propagation>`.
316
+
317
+ Parameters
318
+ ----------
319
+ damping : float, default=0.5
320
+ Damping factor in the range `[0.5, 1.0)` is the extent to
321
+ which the current value is maintained relative to
322
+ incoming values (weighted 1 - damping). This in order
323
+ to avoid numerical oscillations when updating these
324
+ values (messages).
325
+
326
+ max_iter : int, default=200
327
+ Maximum number of iterations.
328
+
329
+ convergence_iter : int, default=15
330
+ Number of iterations with no change in the number
331
+ of estimated clusters that stops the convergence.
332
+
333
+ copy : bool, default=True
334
+ Make a copy of input data.
335
+
336
+ preference : array-like of shape (n_samples,) or float, default=None
337
+ Preferences for each point - points with larger values of
338
+ preferences are more likely to be chosen as exemplars. The number
339
+ of exemplars, ie of clusters, is influenced by the input
340
+ preferences value. If the preferences are not passed as arguments,
341
+ they will be set to the median of the input similarities.
342
+
343
+ affinity : {'euclidean', 'precomputed'}, default='euclidean'
344
+ Which affinity to use. At the moment 'precomputed' and
345
+ ``euclidean`` are supported. 'euclidean' uses the
346
+ negative squared euclidean distance between points.
347
+
348
+ verbose : bool, default=False
349
+ Whether to be verbose.
350
+
351
+ random_state : int, RandomState instance or None, default=None
352
+ Pseudo-random number generator to control the starting state.
353
+ Use an int for reproducible results across function calls.
354
+ See the :term:`Glossary <random_state>`.
355
+
356
+ .. versionadded:: 0.23
357
+ this parameter was previously hardcoded as 0.
358
+
359
+ Attributes
360
+ ----------
361
+ cluster_centers_indices_ : ndarray of shape (n_clusters,)
362
+ Indices of cluster centers.
363
+
364
+ cluster_centers_ : ndarray of shape (n_clusters, n_features)
365
+ Cluster centers (if affinity != ``precomputed``).
366
+
367
+ labels_ : ndarray of shape (n_samples,)
368
+ Labels of each point.
369
+
370
+ affinity_matrix_ : ndarray of shape (n_samples, n_samples)
371
+ Stores the affinity matrix used in ``fit``.
372
+
373
+ n_iter_ : int
374
+ Number of iterations taken to converge.
375
+
376
+ n_features_in_ : int
377
+ Number of features seen during :term:`fit`.
378
+
379
+ .. versionadded:: 0.24
380
+
381
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
382
+ Names of features seen during :term:`fit`. Defined only when `X`
383
+ has feature names that are all strings.
384
+
385
+ .. versionadded:: 1.0
386
+
387
+ See Also
388
+ --------
389
+ AgglomerativeClustering : Recursively merges the pair of
390
+ clusters that minimally increases a given linkage distance.
391
+ FeatureAgglomeration : Similar to AgglomerativeClustering,
392
+ but recursively merges features instead of samples.
393
+ KMeans : K-Means clustering.
394
+ MiniBatchKMeans : Mini-Batch K-Means clustering.
395
+ MeanShift : Mean shift clustering using a flat kernel.
396
+ SpectralClustering : Apply clustering to a projection
397
+ of the normalized Laplacian.
398
+
399
+ Notes
400
+ -----
401
+ The algorithmic complexity of affinity propagation is quadratic
402
+ in the number of points.
403
+
404
+ When the algorithm does not converge, it will still return an array of
405
+ ``cluster_center_indices`` and labels if there are any exemplars/clusters,
406
+ however they may be degenerate and should be used with caution.
407
+
408
+ When ``fit`` does not converge, ``cluster_centers_`` is still populated
409
+ however it may be degenerate. In such a case, proceed with caution.
410
+ If ``fit`` does not converge and fails to produce any ``cluster_centers_``
411
+ then ``predict`` will label every sample as ``-1``.
412
+
413
+ When all training samples have equal similarities and equal preferences,
414
+ the assignment of cluster centers and labels depends on the preference.
415
+ If the preference is smaller than the similarities, ``fit`` will result in
416
+ a single cluster center and label ``0`` for every sample. Otherwise, every
417
+ training sample becomes its own cluster center and is assigned a unique
418
+ label.
419
+
420
+ References
421
+ ----------
422
+
423
+ Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages
424
+ Between Data Points", Science Feb. 2007
425
+
426
+ Examples
427
+ --------
428
+ >>> from sklearn.cluster import AffinityPropagation
429
+ >>> import numpy as np
430
+ >>> X = np.array([[1, 2], [1, 4], [1, 0],
431
+ ... [4, 2], [4, 4], [4, 0]])
432
+ >>> clustering = AffinityPropagation(random_state=5).fit(X)
433
+ >>> clustering
434
+ AffinityPropagation(random_state=5)
435
+ >>> clustering.labels_
436
+ array([0, 0, 0, 1, 1, 1])
437
+ >>> clustering.predict([[0, 0], [4, 4]])
438
+ array([0, 1])
439
+ >>> clustering.cluster_centers_
440
+ array([[1, 2],
441
+ [4, 2]])
442
+
443
+ For an example usage,
444
+ see :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`.
445
+
446
+ For a comparison of Affinity Propagation with other clustering algorithms, see
447
+ :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
448
+ """
449
+
450
+ _parameter_constraints: dict = {
451
+ "damping": [Interval(Real, 0.5, 1.0, closed="left")],
452
+ "max_iter": [Interval(Integral, 1, None, closed="left")],
453
+ "convergence_iter": [Interval(Integral, 1, None, closed="left")],
454
+ "copy": ["boolean"],
455
+ "preference": [
456
+ "array-like",
457
+ Interval(Real, None, None, closed="neither"),
458
+ None,
459
+ ],
460
+ "affinity": [StrOptions({"euclidean", "precomputed"})],
461
+ "verbose": ["verbose"],
462
+ "random_state": ["random_state"],
463
+ }
464
+
465
+ def __init__(
466
+ self,
467
+ *,
468
+ damping=0.5,
469
+ max_iter=200,
470
+ convergence_iter=15,
471
+ copy=True,
472
+ preference=None,
473
+ affinity="euclidean",
474
+ verbose=False,
475
+ random_state=None,
476
+ ):
477
+ self.damping = damping
478
+ self.max_iter = max_iter
479
+ self.convergence_iter = convergence_iter
480
+ self.copy = copy
481
+ self.verbose = verbose
482
+ self.preference = preference
483
+ self.affinity = affinity
484
+ self.random_state = random_state
485
+
486
+ def __sklearn_tags__(self):
487
+ tags = super().__sklearn_tags__()
488
+ tags.input_tags.pairwise = self.affinity == "precomputed"
489
+ tags.input_tags.sparse = self.affinity != "precomputed"
490
+ return tags
491
+
492
+ @_fit_context(prefer_skip_nested_validation=True)
493
+ def fit(self, X, y=None):
494
+ """Fit the clustering from features, or affinity matrix.
495
+
496
+ Parameters
497
+ ----------
498
+ X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
499
+ array-like of shape (n_samples, n_samples)
500
+ Training instances to cluster, or similarities / affinities between
501
+ instances if ``affinity='precomputed'``. If a sparse feature matrix
502
+ is provided, it will be converted into a sparse ``csr_matrix``.
503
+
504
+ y : Ignored
505
+ Not used, present here for API consistency by convention.
506
+
507
+ Returns
508
+ -------
509
+ self
510
+ Returns the instance itself.
511
+ """
512
+ if self.affinity == "precomputed":
513
+ X = validate_data(self, X, copy=self.copy, force_writeable=True)
514
+ self.affinity_matrix_ = X
515
+ else: # self.affinity == "euclidean"
516
+ X = validate_data(self, X, accept_sparse="csr")
517
+ self.affinity_matrix_ = -euclidean_distances(X, squared=True)
518
+
519
+ if self.affinity_matrix_.shape[0] != self.affinity_matrix_.shape[1]:
520
+ raise ValueError(
521
+ "The matrix of similarities must be a square array. "
522
+ f"Got {self.affinity_matrix_.shape} instead."
523
+ )
524
+
525
+ if self.preference is None:
526
+ preference = np.median(self.affinity_matrix_)
527
+ else:
528
+ preference = self.preference
529
+ preference = np.asarray(preference)
530
+
531
+ random_state = check_random_state(self.random_state)
532
+
533
+ (
534
+ self.cluster_centers_indices_,
535
+ self.labels_,
536
+ self.n_iter_,
537
+ ) = _affinity_propagation(
538
+ self.affinity_matrix_,
539
+ max_iter=self.max_iter,
540
+ convergence_iter=self.convergence_iter,
541
+ preference=preference,
542
+ damping=self.damping,
543
+ verbose=self.verbose,
544
+ return_n_iter=True,
545
+ random_state=random_state,
546
+ )
547
+
548
+ if self.affinity != "precomputed":
549
+ self.cluster_centers_ = X[self.cluster_centers_indices_].copy()
550
+
551
+ return self
552
+
553
+ def predict(self, X):
554
+ """Predict the closest cluster each sample in X belongs to.
555
+
556
+ Parameters
557
+ ----------
558
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
559
+ New data to predict. If a sparse matrix is provided, it will be
560
+ converted into a sparse ``csr_matrix``.
561
+
562
+ Returns
563
+ -------
564
+ labels : ndarray of shape (n_samples,)
565
+ Cluster labels.
566
+ """
567
+ check_is_fitted(self)
568
+ X = validate_data(self, X, reset=False, accept_sparse="csr")
569
+ if not hasattr(self, "cluster_centers_"):
570
+ raise ValueError(
571
+ "Predict method is not supported when affinity='precomputed'."
572
+ )
573
+
574
+ if self.cluster_centers_.shape[0] > 0:
575
+ with config_context(assume_finite=True):
576
+ return pairwise_distances_argmin(X, self.cluster_centers_)
577
+ else:
578
+ warnings.warn(
579
+ (
580
+ "This model does not have any cluster centers "
581
+ "because affinity propagation did not converge. "
582
+ "Labeling every sample as '-1'."
583
+ ),
584
+ ConvergenceWarning,
585
+ )
586
+ return np.array([-1] * X.shape[0])
587
+
588
+ def fit_predict(self, X, y=None):
589
+ """Fit clustering from features/affinity matrix; return cluster labels.
590
+
591
+ Parameters
592
+ ----------
593
+ X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
594
+ array-like of shape (n_samples, n_samples)
595
+ Training instances to cluster, or similarities / affinities between
596
+ instances if ``affinity='precomputed'``. If a sparse feature matrix
597
+ is provided, it will be converted into a sparse ``csr_matrix``.
598
+
599
+ y : Ignored
600
+ Not used, present here for API consistency by convention.
601
+
602
+ Returns
603
+ -------
604
+ labels : ndarray of shape (n_samples,)
605
+ Cluster labels.
606
+ """
607
+ return super().fit_predict(X, y)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_agglomerative.py ADDED
@@ -0,0 +1,1335 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Hierarchical Agglomerative Clustering
2
+
3
+ These routines perform some hierarchical agglomerative clustering of some
4
+ input data.
5
+ """
6
+
7
+ # Authors: The scikit-learn developers
8
+ # SPDX-License-Identifier: BSD-3-Clause
9
+
10
+ import warnings
11
+ from heapq import heapify, heappop, heappush, heappushpop
12
+ from numbers import Integral, Real
13
+
14
+ import numpy as np
15
+ from scipy import sparse
16
+ from scipy.sparse.csgraph import connected_components
17
+
18
+ from sklearn.base import (
19
+ BaseEstimator,
20
+ ClassNamePrefixFeaturesOutMixin,
21
+ ClusterMixin,
22
+ _fit_context,
23
+ )
24
+
25
+ # mypy error: Module 'sklearn.cluster' has no attribute '_hierarchical_fast'
26
+ from sklearn.cluster import ( # type: ignore[attr-defined]
27
+ _hierarchical_fast as _hierarchical,
28
+ )
29
+ from sklearn.cluster._feature_agglomeration import AgglomerationTransform
30
+ from sklearn.metrics import DistanceMetric
31
+ from sklearn.metrics._dist_metrics import METRIC_MAPPING64
32
+ from sklearn.metrics.pairwise import _VALID_METRICS, paired_distances
33
+ from sklearn.utils import check_array
34
+ from sklearn.utils._fast_dict import IntFloatDict
35
+ from sklearn.utils._param_validation import (
36
+ HasMethods,
37
+ Interval,
38
+ StrOptions,
39
+ validate_params,
40
+ )
41
+ from sklearn.utils.graph import _fix_connected_components
42
+ from sklearn.utils.validation import check_memory, validate_data
43
+
44
+ ###############################################################################
45
+ # For non fully-connected graphs
46
+
47
+
48
+ def _fix_connectivity(X, connectivity, affinity):
49
+ """
50
+ Fixes the connectivity matrix.
51
+
52
+ The different steps are:
53
+
54
+ - copies it
55
+ - makes it symmetric
56
+ - converts it to LIL if necessary
57
+ - completes it if necessary.
58
+
59
+ Parameters
60
+ ----------
61
+ X : array-like of shape (n_samples, n_features)
62
+ Feature matrix representing `n_samples` samples to be clustered.
63
+
64
+ connectivity : sparse matrix, default=None
65
+ Connectivity matrix. Defines for each sample the neighboring samples
66
+ following a given structure of the data. The matrix is assumed to
67
+ be symmetric and only the upper triangular half is used.
68
+ Default is `None`, i.e, the Ward algorithm is unstructured.
69
+
70
+ affinity : {"euclidean", "precomputed"}, default="euclidean"
71
+ Which affinity to use. At the moment `precomputed` and
72
+ ``euclidean`` are supported. `euclidean` uses the
73
+ negative squared Euclidean distance between points.
74
+
75
+ Returns
76
+ -------
77
+ connectivity : sparse matrix
78
+ The fixed connectivity matrix.
79
+
80
+ n_connected_components : int
81
+ The number of connected components in the graph.
82
+ """
83
+ n_samples = X.shape[0]
84
+ if connectivity.shape[0] != n_samples or connectivity.shape[1] != n_samples:
85
+ raise ValueError(
86
+ "Wrong shape for connectivity matrix: %s when X is %s"
87
+ % (connectivity.shape, X.shape)
88
+ )
89
+
90
+ # Make the connectivity matrix symmetric:
91
+ connectivity = connectivity + connectivity.T
92
+
93
+ # Convert connectivity matrix to LIL
94
+ if not sparse.issparse(connectivity):
95
+ connectivity = sparse.lil_matrix(connectivity)
96
+
97
+ # `connectivity` is a sparse matrix at this point
98
+ if connectivity.format != "lil":
99
+ connectivity = connectivity.tolil()
100
+
101
+ # Compute the number of nodes
102
+ n_connected_components, labels = connected_components(connectivity)
103
+
104
+ if n_connected_components > 1:
105
+ warnings.warn(
106
+ "the number of connected components of the "
107
+ "connectivity matrix is %d > 1. Completing it to avoid "
108
+ "stopping the tree early." % n_connected_components,
109
+ stacklevel=2,
110
+ )
111
+ # XXX: Can we do without completing the matrix?
112
+ connectivity = _fix_connected_components(
113
+ X=X,
114
+ graph=connectivity,
115
+ n_connected_components=n_connected_components,
116
+ component_labels=labels,
117
+ metric=affinity,
118
+ mode="connectivity",
119
+ )
120
+
121
+ return connectivity, n_connected_components
122
+
123
+
124
+ def _single_linkage_tree(
125
+ connectivity,
126
+ n_samples,
127
+ n_nodes,
128
+ n_clusters,
129
+ n_connected_components,
130
+ return_distance,
131
+ ):
132
+ """
133
+ Perform single linkage clustering on sparse data via the minimum
134
+ spanning tree from scipy.sparse.csgraph, then using union-find to label.
135
+ The parent array is then generated by walking through the tree.
136
+ """
137
+ from scipy.sparse.csgraph import minimum_spanning_tree
138
+
139
+ # explicitly cast connectivity to ensure safety
140
+ connectivity = connectivity.astype(np.float64, copy=False)
141
+
142
+ # Ensure zero distances aren't ignored by setting them to "epsilon"
143
+ epsilon_value = np.finfo(dtype=connectivity.data.dtype).eps
144
+ connectivity.data[connectivity.data == 0] = epsilon_value
145
+
146
+ # Use scipy.sparse.csgraph to generate a minimum spanning tree
147
+ mst = minimum_spanning_tree(connectivity.tocsr())
148
+
149
+ # Convert the graph to scipy.cluster.hierarchy array format
150
+ mst = mst.tocoo()
151
+
152
+ # Undo the epsilon values
153
+ mst.data[mst.data == epsilon_value] = 0
154
+
155
+ mst_array = np.vstack([mst.row, mst.col, mst.data]).T
156
+
157
+ # Sort edges of the min_spanning_tree by weight
158
+ mst_array = mst_array[np.argsort(mst_array.T[2], kind="mergesort"), :]
159
+
160
+ # Convert edge list into standard hierarchical clustering format
161
+ single_linkage_tree = _hierarchical._single_linkage_label(mst_array)
162
+ children_ = single_linkage_tree[:, :2].astype(int)
163
+
164
+ # Compute parents
165
+ parent = np.arange(n_nodes, dtype=np.intp)
166
+ for i, (left, right) in enumerate(children_, n_samples):
167
+ if n_clusters is not None and i >= n_nodes:
168
+ break
169
+ if left < n_nodes:
170
+ parent[left] = i
171
+ if right < n_nodes:
172
+ parent[right] = i
173
+
174
+ if return_distance:
175
+ distances = single_linkage_tree[:, 2]
176
+ return children_, n_connected_components, n_samples, parent, distances
177
+ return children_, n_connected_components, n_samples, parent
178
+
179
+
180
+ ###############################################################################
181
+ # Hierarchical tree building functions
182
+
183
+
184
+ @validate_params(
185
+ {
186
+ "X": ["array-like"],
187
+ "connectivity": ["array-like", "sparse matrix", None],
188
+ "n_clusters": [Interval(Integral, 1, None, closed="left"), None],
189
+ "return_distance": ["boolean"],
190
+ },
191
+ prefer_skip_nested_validation=True,
192
+ )
193
+ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False):
194
+ """Ward clustering based on a Feature matrix.
195
+
196
+ Recursively merges the pair of clusters that minimally increases
197
+ within-cluster variance.
198
+
199
+ The inertia matrix uses a Heapq-based representation.
200
+
201
+ This is the structured version, that takes into account some topological
202
+ structure between samples.
203
+
204
+ Read more in the :ref:`User Guide <hierarchical_clustering>`.
205
+
206
+ Parameters
207
+ ----------
208
+ X : array-like of shape (n_samples, n_features)
209
+ Feature matrix representing `n_samples` samples to be clustered.
210
+
211
+ connectivity : {array-like, sparse matrix}, default=None
212
+ Connectivity matrix. Defines for each sample the neighboring samples
213
+ following a given structure of the data. The matrix is assumed to
214
+ be symmetric and only the upper triangular half is used.
215
+ Default is None, i.e, the Ward algorithm is unstructured.
216
+
217
+ n_clusters : int, default=None
218
+ `n_clusters` should be less than `n_samples`. Stop early the
219
+ construction of the tree at `n_clusters.` This is useful to decrease
220
+ computation time if the number of clusters is not small compared to the
221
+ number of samples. In this case, the complete tree is not computed, thus
222
+ the 'children' output is of limited use, and the 'parents' output should
223
+ rather be used. This option is valid only when specifying a connectivity
224
+ matrix.
225
+
226
+ return_distance : bool, default=False
227
+ If `True`, return the distance between the clusters.
228
+
229
+ Returns
230
+ -------
231
+ children : ndarray of shape (n_nodes-1, 2)
232
+ The children of each non-leaf node. Values less than `n_samples`
233
+ correspond to leaves of the tree which are the original samples.
234
+ A node `i` greater than or equal to `n_samples` is a non-leaf
235
+ node and has children `children_[i - n_samples]`. Alternatively
236
+ at the i-th iteration, children[i][0] and children[i][1]
237
+ are merged to form node `n_samples + i`.
238
+
239
+ n_connected_components : int
240
+ The number of connected components in the graph.
241
+
242
+ n_leaves : int
243
+ The number of leaves in the tree.
244
+
245
+ parents : ndarray of shape (n_nodes,) or None
246
+ The parent of each node. Only returned when a connectivity matrix
247
+ is specified, elsewhere 'None' is returned.
248
+
249
+ distances : ndarray of shape (n_nodes-1,)
250
+ Only returned if `return_distance` is set to `True` (for compatibility).
251
+ The distances between the centers of the nodes. `distances[i]`
252
+ corresponds to a weighted Euclidean distance between
253
+ the nodes `children[i, 1]` and `children[i, 2]`. If the nodes refer to
254
+ leaves of the tree, then `distances[i]` is their unweighted Euclidean
255
+ distance. Distances are updated in the following way
256
+ (from scipy.hierarchy.linkage):
257
+
258
+ The new entry :math:`d(u,v)` is computed as follows,
259
+
260
+ .. math::
261
+
262
+ d(u,v) = \\sqrt{\\frac{|v|+|s|}
263
+ {T}d(v,s)^2
264
+ + \\frac{|v|+|t|}
265
+ {T}d(v,t)^2
266
+ - \\frac{|v|}
267
+ {T}d(s,t)^2}
268
+
269
+ where :math:`u` is the newly joined cluster consisting of
270
+ clusters :math:`s` and :math:`t`, :math:`v` is an unused
271
+ cluster in the forest, :math:`T=|v|+|s|+|t|`, and
272
+ :math:`|*|` is the cardinality of its argument. This is also
273
+ known as the incremental algorithm.
274
+
275
+ Examples
276
+ --------
277
+ >>> import numpy as np
278
+ >>> from sklearn.cluster import ward_tree
279
+ >>> X = np.array([[1, 2], [1, 4], [1, 0],
280
+ ... [4, 2], [4, 4], [4, 0]])
281
+ >>> children, n_connected_components, n_leaves, parents = ward_tree(X)
282
+ >>> children
283
+ array([[0, 1],
284
+ [3, 5],
285
+ [2, 6],
286
+ [4, 7],
287
+ [8, 9]])
288
+ >>> n_connected_components
289
+ 1
290
+ >>> n_leaves
291
+ 6
292
+ """
293
+ X = np.asarray(X)
294
+ if X.ndim == 1:
295
+ X = np.reshape(X, (-1, 1))
296
+ n_samples, n_features = X.shape
297
+
298
+ if connectivity is None:
299
+ from scipy.cluster import hierarchy # imports PIL
300
+
301
+ if n_clusters is not None:
302
+ warnings.warn(
303
+ (
304
+ "Partial build of the tree is implemented "
305
+ "only for structured clustering (i.e. with "
306
+ "explicit connectivity). The algorithm "
307
+ "will build the full tree and only "
308
+ "retain the lower branches required "
309
+ "for the specified number of clusters"
310
+ ),
311
+ stacklevel=2,
312
+ )
313
+ X = np.require(X, requirements="W")
314
+ out = hierarchy.ward(X)
315
+ children_ = out[:, :2].astype(np.intp)
316
+
317
+ if return_distance:
318
+ distances = out[:, 2]
319
+ return children_, 1, n_samples, None, distances
320
+ else:
321
+ return children_, 1, n_samples, None
322
+
323
+ connectivity, n_connected_components = _fix_connectivity(
324
+ X, connectivity, affinity="euclidean"
325
+ )
326
+ if n_clusters is None:
327
+ n_nodes = 2 * n_samples - 1
328
+ else:
329
+ if n_clusters > n_samples:
330
+ raise ValueError(
331
+ "Cannot provide more clusters than samples. "
332
+ "%i n_clusters was asked, and there are %i "
333
+ "samples." % (n_clusters, n_samples)
334
+ )
335
+ n_nodes = 2 * n_samples - n_clusters
336
+
337
+ # create inertia matrix
338
+ coord_row = []
339
+ coord_col = []
340
+ A = []
341
+ for ind, row in enumerate(connectivity.rows):
342
+ A.append(row)
343
+ # We keep only the upper triangular for the moments
344
+ # Generator expressions are faster than arrays on the following
345
+ row = [i for i in row if i < ind]
346
+ coord_row.extend(
347
+ len(row)
348
+ * [
349
+ ind,
350
+ ]
351
+ )
352
+ coord_col.extend(row)
353
+
354
+ coord_row = np.array(coord_row, dtype=np.intp, order="C")
355
+ coord_col = np.array(coord_col, dtype=np.intp, order="C")
356
+
357
+ # build moments as a list
358
+ moments_1 = np.zeros(n_nodes, order="C")
359
+ moments_1[:n_samples] = 1
360
+ moments_2 = np.zeros((n_nodes, n_features), order="C")
361
+ moments_2[:n_samples] = X
362
+ inertia = np.empty(len(coord_row), dtype=np.float64, order="C")
363
+ _hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col, inertia)
364
+ inertia = list(zip(inertia, coord_row, coord_col))
365
+ heapify(inertia)
366
+
367
+ # prepare the main fields
368
+ parent = np.arange(n_nodes, dtype=np.intp)
369
+ used_node = np.ones(n_nodes, dtype=bool)
370
+ children = []
371
+ if return_distance:
372
+ distances = np.empty(n_nodes - n_samples)
373
+
374
+ not_visited = np.empty(n_nodes, dtype=bool, order="C")
375
+
376
+ # recursive merge loop
377
+ for k in range(n_samples, n_nodes):
378
+ # identify the merge
379
+ while True:
380
+ inert, i, j = heappop(inertia)
381
+ if used_node[i] and used_node[j]:
382
+ break
383
+ parent[i], parent[j] = k, k
384
+ children.append((i, j))
385
+ used_node[i] = used_node[j] = False
386
+ if return_distance: # store inertia value
387
+ distances[k - n_samples] = inert
388
+
389
+ # update the moments
390
+ moments_1[k] = moments_1[i] + moments_1[j]
391
+ moments_2[k] = moments_2[i] + moments_2[j]
392
+
393
+ # update the structure matrix A and the inertia matrix
394
+ coord_col = []
395
+ not_visited.fill(1)
396
+ not_visited[k] = 0
397
+ _hierarchical._get_parents(A[i], coord_col, parent, not_visited)
398
+ _hierarchical._get_parents(A[j], coord_col, parent, not_visited)
399
+ # List comprehension is faster than a for loop
400
+ [A[col].append(k) for col in coord_col]
401
+ A.append(coord_col)
402
+ coord_col = np.array(coord_col, dtype=np.intp, order="C")
403
+ coord_row = np.empty(coord_col.shape, dtype=np.intp, order="C")
404
+ coord_row.fill(k)
405
+ n_additions = len(coord_row)
406
+ ini = np.empty(n_additions, dtype=np.float64, order="C")
407
+
408
+ _hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col, ini)
409
+
410
+ # List comprehension is faster than a for loop
411
+ [heappush(inertia, (ini[idx], k, coord_col[idx])) for idx in range(n_additions)]
412
+
413
+ # Separate leaves in children (empty lists up to now)
414
+ n_leaves = n_samples
415
+ # sort children to get consistent output with unstructured version
416
+ children = [c[::-1] for c in children]
417
+ children = np.array(children) # return numpy array for efficient caching
418
+
419
+ if return_distance:
420
+ # 2 is scaling factor to compare w/ unstructured version
421
+ distances = np.sqrt(2.0 * distances)
422
+ return children, n_connected_components, n_leaves, parent, distances
423
+ else:
424
+ return children, n_connected_components, n_leaves, parent
425
+
426
+
427
+ # single average and complete linkage
428
+ def linkage_tree(
429
+ X,
430
+ connectivity=None,
431
+ n_clusters=None,
432
+ linkage="complete",
433
+ affinity="euclidean",
434
+ return_distance=False,
435
+ ):
436
+ """Linkage agglomerative clustering based on a Feature matrix.
437
+
438
+ The inertia matrix uses a Heapq-based representation.
439
+
440
+ This is the structured version, that takes into account some topological
441
+ structure between samples.
442
+
443
+ Read more in the :ref:`User Guide <hierarchical_clustering>`.
444
+
445
+ Parameters
446
+ ----------
447
+ X : array-like of shape (n_samples, n_features)
448
+ Feature matrix representing `n_samples` samples to be clustered.
449
+
450
+ connectivity : sparse matrix, default=None
451
+ Connectivity matrix. Defines for each sample the neighboring samples
452
+ following a given structure of the data. The matrix is assumed to
453
+ be symmetric and only the upper triangular half is used.
454
+ Default is `None`, i.e, the Ward algorithm is unstructured.
455
+
456
+ n_clusters : int, default=None
457
+ Stop early the construction of the tree at `n_clusters`. This is
458
+ useful to decrease computation time if the number of clusters is
459
+ not small compared to the number of samples. In this case, the
460
+ complete tree is not computed, thus the 'children' output is of
461
+ limited use, and the 'parents' output should rather be used.
462
+ This option is valid only when specifying a connectivity matrix.
463
+
464
+ linkage : {"average", "complete", "single"}, default="complete"
465
+ Which linkage criteria to use. The linkage criterion determines which
466
+ distance to use between sets of observation.
467
+ - "average" uses the average of the distances of each observation of
468
+ the two sets.
469
+ - "complete" or maximum linkage uses the maximum distances between
470
+ all observations of the two sets.
471
+ - "single" uses the minimum of the distances between all
472
+ observations of the two sets.
473
+
474
+ affinity : str or callable, default='euclidean'
475
+ Which metric to use. Can be 'euclidean', 'manhattan', or any
476
+ distance known to paired distance (see metric.pairwise).
477
+
478
+ return_distance : bool, default=False
479
+ Whether or not to return the distances between the clusters.
480
+
481
+ Returns
482
+ -------
483
+ children : ndarray of shape (n_nodes-1, 2)
484
+ The children of each non-leaf node. Values less than `n_samples`
485
+ correspond to leaves of the tree which are the original samples.
486
+ A node `i` greater than or equal to `n_samples` is a non-leaf
487
+ node and has children `children_[i - n_samples]`. Alternatively
488
+ at the i-th iteration, children[i][0] and children[i][1]
489
+ are merged to form node `n_samples + i`.
490
+
491
+ n_connected_components : int
492
+ The number of connected components in the graph.
493
+
494
+ n_leaves : int
495
+ The number of leaves in the tree.
496
+
497
+ parents : ndarray of shape (n_nodes, ) or None
498
+ The parent of each node. Only returned when a connectivity matrix
499
+ is specified, elsewhere 'None' is returned.
500
+
501
+ distances : ndarray of shape (n_nodes-1,)
502
+ Returned when `return_distance` is set to `True`.
503
+
504
+ distances[i] refers to the distance between children[i][0] and
505
+ children[i][1] when they are merged.
506
+
507
+ See Also
508
+ --------
509
+ ward_tree : Hierarchical clustering with ward linkage.
510
+ """
511
+ X = np.asarray(X)
512
+ if X.ndim == 1:
513
+ X = np.reshape(X, (-1, 1))
514
+ n_samples, n_features = X.shape
515
+
516
+ linkage_choices = {
517
+ "complete": _hierarchical.max_merge,
518
+ "average": _hierarchical.average_merge,
519
+ "single": None,
520
+ } # Single linkage is handled differently
521
+ try:
522
+ join_func = linkage_choices[linkage]
523
+ except KeyError as e:
524
+ raise ValueError(
525
+ "Unknown linkage option, linkage should be one of %s, but %s was given"
526
+ % (linkage_choices.keys(), linkage)
527
+ ) from e
528
+
529
+ if affinity == "cosine" and np.any(~np.any(X, axis=1)):
530
+ raise ValueError("Cosine affinity cannot be used when X contains zero vectors")
531
+
532
+ if connectivity is None:
533
+ from scipy.cluster import hierarchy # imports PIL
534
+
535
+ if n_clusters is not None:
536
+ warnings.warn(
537
+ (
538
+ "Partial build of the tree is implemented "
539
+ "only for structured clustering (i.e. with "
540
+ "explicit connectivity). The algorithm "
541
+ "will build the full tree and only "
542
+ "retain the lower branches required "
543
+ "for the specified number of clusters"
544
+ ),
545
+ stacklevel=2,
546
+ )
547
+
548
+ if affinity == "precomputed":
549
+ # for the linkage function of hierarchy to work on precomputed
550
+ # data, provide as first argument an ndarray of the shape returned
551
+ # by sklearn.metrics.pairwise_distances.
552
+ if X.shape[0] != X.shape[1]:
553
+ raise ValueError(
554
+ f"Distance matrix should be square, got matrix of shape {X.shape}"
555
+ )
556
+ i, j = np.triu_indices(X.shape[0], k=1)
557
+ X = X[i, j]
558
+ elif affinity == "l2":
559
+ # Translate to something understood by scipy
560
+ affinity = "euclidean"
561
+ elif affinity in ("l1", "manhattan"):
562
+ affinity = "cityblock"
563
+ elif callable(affinity):
564
+ X = affinity(X)
565
+ i, j = np.triu_indices(X.shape[0], k=1)
566
+ X = X[i, j]
567
+ if (
568
+ linkage == "single"
569
+ and affinity != "precomputed"
570
+ and not callable(affinity)
571
+ and affinity in METRIC_MAPPING64
572
+ ):
573
+ # We need the fast cythonized metric from neighbors
574
+ dist_metric = DistanceMetric.get_metric(affinity)
575
+
576
+ # The Cython routines used require contiguous arrays
577
+ X = np.ascontiguousarray(X, dtype=np.double)
578
+
579
+ mst = _hierarchical.mst_linkage_core(X, dist_metric)
580
+ # Sort edges of the min_spanning_tree by weight
581
+ mst = mst[np.argsort(mst.T[2], kind="mergesort"), :]
582
+
583
+ # Convert edge list into standard hierarchical clustering format
584
+ out = _hierarchical.single_linkage_label(mst)
585
+ else:
586
+ out = hierarchy.linkage(X, method=linkage, metric=affinity)
587
+ children_ = out[:, :2].astype(int, copy=False)
588
+
589
+ if return_distance:
590
+ distances = out[:, 2]
591
+ return children_, 1, n_samples, None, distances
592
+ return children_, 1, n_samples, None
593
+
594
+ connectivity, n_connected_components = _fix_connectivity(
595
+ X, connectivity, affinity=affinity
596
+ )
597
+ connectivity = connectivity.tocoo()
598
+ # Put the diagonal to zero
599
+ diag_mask = connectivity.row != connectivity.col
600
+ connectivity.row = connectivity.row[diag_mask]
601
+ connectivity.col = connectivity.col[diag_mask]
602
+ connectivity.data = connectivity.data[diag_mask]
603
+ del diag_mask
604
+
605
+ if affinity == "precomputed":
606
+ distances = X[connectivity.row, connectivity.col].astype(np.float64, copy=False)
607
+ else:
608
+ # FIXME We compute all the distances, while we could have only computed
609
+ # the "interesting" distances
610
+ distances = paired_distances(
611
+ X[connectivity.row], X[connectivity.col], metric=affinity
612
+ )
613
+ connectivity.data = distances
614
+
615
+ if n_clusters is None:
616
+ n_nodes = 2 * n_samples - 1
617
+ else:
618
+ assert n_clusters <= n_samples
619
+ n_nodes = 2 * n_samples - n_clusters
620
+
621
+ if linkage == "single":
622
+ return _single_linkage_tree(
623
+ connectivity,
624
+ n_samples,
625
+ n_nodes,
626
+ n_clusters,
627
+ n_connected_components,
628
+ return_distance,
629
+ )
630
+
631
+ if return_distance:
632
+ distances = np.empty(n_nodes - n_samples)
633
+ # create inertia heap and connection matrix
634
+ A = np.empty(n_nodes, dtype=object)
635
+ inertia = list()
636
+
637
+ # LIL seems to the best format to access the rows quickly,
638
+ # without the numpy overhead of slicing CSR indices and data.
639
+ connectivity = connectivity.tolil()
640
+ # We are storing the graph in a list of IntFloatDict
641
+ for ind, (data, row) in enumerate(zip(connectivity.data, connectivity.rows)):
642
+ A[ind] = IntFloatDict(
643
+ np.asarray(row, dtype=np.intp), np.asarray(data, dtype=np.float64)
644
+ )
645
+ # We keep only the upper triangular for the heap
646
+ # Generator expressions are faster than arrays on the following
647
+ inertia.extend(
648
+ _hierarchical.WeightedEdge(d, ind, r) for r, d in zip(row, data) if r < ind
649
+ )
650
+ del connectivity
651
+
652
+ heapify(inertia)
653
+
654
+ # prepare the main fields
655
+ parent = np.arange(n_nodes, dtype=np.intp)
656
+ used_node = np.ones(n_nodes, dtype=np.intp)
657
+ children = []
658
+
659
+ # recursive merge loop
660
+ for k in range(n_samples, n_nodes):
661
+ # identify the merge
662
+ while True:
663
+ edge = heappop(inertia)
664
+ if used_node[edge.a] and used_node[edge.b]:
665
+ break
666
+ i = edge.a
667
+ j = edge.b
668
+
669
+ if return_distance:
670
+ # store distances
671
+ distances[k - n_samples] = edge.weight
672
+
673
+ parent[i] = parent[j] = k
674
+ children.append((i, j))
675
+ # Keep track of the number of elements per cluster
676
+ n_i = used_node[i]
677
+ n_j = used_node[j]
678
+ used_node[k] = n_i + n_j
679
+ used_node[i] = used_node[j] = False
680
+
681
+ # update the structure matrix A and the inertia matrix
682
+ # a clever 'min', or 'max' operation between A[i] and A[j]
683
+ coord_col = join_func(A[i], A[j], used_node, n_i, n_j)
684
+ for col, d in coord_col:
685
+ A[col].append(k, d)
686
+ # Here we use the information from coord_col (containing the
687
+ # distances) to update the heap
688
+ heappush(inertia, _hierarchical.WeightedEdge(d, k, col))
689
+ A[k] = coord_col
690
+ # Clear A[i] and A[j] to save memory
691
+ A[i] = A[j] = 0
692
+
693
+ # Separate leaves in children (empty lists up to now)
694
+ n_leaves = n_samples
695
+
696
+ # # return numpy array for efficient caching
697
+ children = np.array(children)[:, ::-1]
698
+
699
+ if return_distance:
700
+ return children, n_connected_components, n_leaves, parent, distances
701
+ return children, n_connected_components, n_leaves, parent
702
+
703
+
704
+ # Matching names to tree-building strategies
705
+ def _complete_linkage(*args, **kwargs):
706
+ kwargs["linkage"] = "complete"
707
+ return linkage_tree(*args, **kwargs)
708
+
709
+
710
+ def _average_linkage(*args, **kwargs):
711
+ kwargs["linkage"] = "average"
712
+ return linkage_tree(*args, **kwargs)
713
+
714
+
715
+ def _single_linkage(*args, **kwargs):
716
+ kwargs["linkage"] = "single"
717
+ return linkage_tree(*args, **kwargs)
718
+
719
+
720
+ _TREE_BUILDERS = dict(
721
+ ward=ward_tree,
722
+ complete=_complete_linkage,
723
+ average=_average_linkage,
724
+ single=_single_linkage,
725
+ )
726
+
727
+ ###############################################################################
728
+ # Functions for cutting hierarchical clustering tree
729
+
730
+
731
+ def _hc_cut(n_clusters, children, n_leaves):
732
+ """Function cutting the ward tree for a given number of clusters.
733
+
734
+ Parameters
735
+ ----------
736
+ n_clusters : int or ndarray
737
+ The number of clusters to form.
738
+
739
+ children : ndarray of shape (n_nodes-1, 2)
740
+ The children of each non-leaf node. Values less than `n_samples`
741
+ correspond to leaves of the tree which are the original samples.
742
+ A node `i` greater than or equal to `n_samples` is a non-leaf
743
+ node and has children `children_[i - n_samples]`. Alternatively
744
+ at the i-th iteration, children[i][0] and children[i][1]
745
+ are merged to form node `n_samples + i`.
746
+
747
+ n_leaves : int
748
+ Number of leaves of the tree.
749
+
750
+ Returns
751
+ -------
752
+ labels : array [n_samples]
753
+ Cluster labels for each point.
754
+ """
755
+ if n_clusters > n_leaves:
756
+ raise ValueError(
757
+ "Cannot extract more clusters than samples: "
758
+ f"{n_clusters} clusters were given for a tree with {n_leaves} leaves."
759
+ )
760
+ # In this function, we store nodes as a heap to avoid recomputing
761
+ # the max of the nodes: the first element is always the smallest
762
+ # We use negated indices as heaps work on smallest elements, and we
763
+ # are interested in largest elements
764
+ # children[-1] is the root of the tree
765
+ nodes = [-(max(children[-1]) + 1)]
766
+ for _ in range(n_clusters - 1):
767
+ # As we have a heap, nodes[0] is the smallest element
768
+ these_children = children[-nodes[0] - n_leaves]
769
+ # Insert the 2 children and remove the largest node
770
+ heappush(nodes, -these_children[0])
771
+ heappushpop(nodes, -these_children[1])
772
+ label = np.zeros(n_leaves, dtype=np.intp)
773
+ for i, node in enumerate(nodes):
774
+ label[_hierarchical._hc_get_descendent(-node, children, n_leaves)] = i
775
+ return label
776
+
777
+
778
+ ###############################################################################
779
+
780
+
781
+ class AgglomerativeClustering(ClusterMixin, BaseEstimator):
782
+ """
783
+ Agglomerative Clustering.
784
+
785
+ Recursively merges pair of clusters of sample data; uses linkage distance.
786
+
787
+ Read more in the :ref:`User Guide <hierarchical_clustering>`.
788
+
789
+ Parameters
790
+ ----------
791
+ n_clusters : int or None, default=2
792
+ The number of clusters to find. It must be ``None`` if
793
+ ``distance_threshold`` is not ``None``.
794
+
795
+ metric : str or callable, default="euclidean"
796
+ Metric used to compute the linkage. Can be "euclidean", "l1", "l2",
797
+ "manhattan", "cosine", or "precomputed". If linkage is "ward", only
798
+ "euclidean" is accepted. If "precomputed", a distance matrix is needed
799
+ as input for the fit method. If connectivity is None, linkage is
800
+ "single" and affinity is not "precomputed" any valid pairwise distance
801
+ metric can be assigned.
802
+
803
+ For an example of agglomerative clustering with different metrics, see
804
+ :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering_metrics.py`.
805
+
806
+ .. versionadded:: 1.2
807
+
808
+ memory : str or object with the joblib.Memory interface, default=None
809
+ Used to cache the output of the computation of the tree.
810
+ By default, no caching is done. If a string is given, it is the
811
+ path to the caching directory.
812
+
813
+ connectivity : array-like, sparse matrix, or callable, default=None
814
+ Connectivity matrix. Defines for each sample the neighboring
815
+ samples following a given structure of the data.
816
+ This can be a connectivity matrix itself or a callable that transforms
817
+ the data into a connectivity matrix, such as derived from
818
+ `kneighbors_graph`. Default is ``None``, i.e, the
819
+ hierarchical clustering algorithm is unstructured.
820
+
821
+ For an example of connectivity matrix using
822
+ :class:`~sklearn.neighbors.kneighbors_graph`, see
823
+ :ref:`sphx_glr_auto_examples_cluster_plot_ward_structured_vs_unstructured.py`.
824
+
825
+ compute_full_tree : 'auto' or bool, default='auto'
826
+ Stop early the construction of the tree at ``n_clusters``. This is
827
+ useful to decrease computation time if the number of clusters is not
828
+ small compared to the number of samples. This option is useful only
829
+ when specifying a connectivity matrix. Note also that when varying the
830
+ number of clusters and using caching, it may be advantageous to compute
831
+ the full tree. It must be ``True`` if ``distance_threshold`` is not
832
+ ``None``. By default `compute_full_tree` is "auto", which is equivalent
833
+ to `True` when `distance_threshold` is not `None` or that `n_clusters`
834
+ is inferior to the maximum between 100 or `0.02 * n_samples`.
835
+ Otherwise, "auto" is equivalent to `False`.
836
+
837
+ linkage : {'ward', 'complete', 'average', 'single'}, default='ward'
838
+ Which linkage criterion to use. The linkage criterion determines which
839
+ distance to use between sets of observation. The algorithm will merge
840
+ the pairs of cluster that minimize this criterion.
841
+
842
+ - 'ward' minimizes the variance of the clusters being merged.
843
+ - 'average' uses the average of the distances of each observation of
844
+ the two sets.
845
+ - 'complete' or 'maximum' linkage uses the maximum distances between
846
+ all observations of the two sets.
847
+ - 'single' uses the minimum of the distances between all observations
848
+ of the two sets.
849
+
850
+ .. versionadded:: 0.20
851
+ Added the 'single' option
852
+
853
+ For examples comparing different `linkage` criteria, see
854
+ :ref:`sphx_glr_auto_examples_cluster_plot_linkage_comparison.py`.
855
+
856
+ distance_threshold : float, default=None
857
+ The linkage distance threshold at or above which clusters will not be
858
+ merged. If not ``None``, ``n_clusters`` must be ``None`` and
859
+ ``compute_full_tree`` must be ``True``.
860
+
861
+ .. versionadded:: 0.21
862
+
863
+ compute_distances : bool, default=False
864
+ Computes distances between clusters even if `distance_threshold` is not
865
+ used. This can be used to make dendrogram visualization, but introduces
866
+ a computational and memory overhead.
867
+
868
+ .. versionadded:: 0.24
869
+
870
+ For an example of dendrogram visualization, see
871
+ :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_dendrogram.py`.
872
+
873
+ Attributes
874
+ ----------
875
+ n_clusters_ : int
876
+ The number of clusters found by the algorithm. If
877
+ ``distance_threshold=None``, it will be equal to the given
878
+ ``n_clusters``.
879
+
880
+ labels_ : ndarray of shape (n_samples)
881
+ Cluster labels for each point.
882
+
883
+ n_leaves_ : int
884
+ Number of leaves in the hierarchical tree.
885
+
886
+ n_connected_components_ : int
887
+ The estimated number of connected components in the graph.
888
+
889
+ .. versionadded:: 0.21
890
+ ``n_connected_components_`` was added to replace ``n_components_``.
891
+
892
+ n_features_in_ : int
893
+ Number of features seen during :term:`fit`.
894
+
895
+ .. versionadded:: 0.24
896
+
897
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
898
+ Names of features seen during :term:`fit`. Defined only when `X`
899
+ has feature names that are all strings.
900
+
901
+ .. versionadded:: 1.0
902
+
903
+ children_ : array-like of shape (n_samples-1, 2)
904
+ The children of each non-leaf node. Values less than `n_samples`
905
+ correspond to leaves of the tree which are the original samples.
906
+ A node `i` greater than or equal to `n_samples` is a non-leaf
907
+ node and has children `children_[i - n_samples]`. Alternatively
908
+ at the i-th iteration, children[i][0] and children[i][1]
909
+ are merged to form node `n_samples + i`.
910
+
911
+ distances_ : array-like of shape (n_nodes-1,)
912
+ Distances between nodes in the corresponding place in `children_`.
913
+ Only computed if `distance_threshold` is used or `compute_distances`
914
+ is set to `True`.
915
+
916
+ See Also
917
+ --------
918
+ FeatureAgglomeration : Agglomerative clustering but for features instead of
919
+ samples.
920
+ ward_tree : Hierarchical clustering with ward linkage.
921
+
922
+ Examples
923
+ --------
924
+ >>> from sklearn.cluster import AgglomerativeClustering
925
+ >>> import numpy as np
926
+ >>> X = np.array([[1, 2], [1, 4], [1, 0],
927
+ ... [4, 2], [4, 4], [4, 0]])
928
+ >>> clustering = AgglomerativeClustering().fit(X)
929
+ >>> clustering
930
+ AgglomerativeClustering()
931
+ >>> clustering.labels_
932
+ array([1, 1, 1, 0, 0, 0])
933
+
934
+ For a comparison of Agglomerative clustering with other clustering algorithms, see
935
+ :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
936
+ """
937
+
938
+ _parameter_constraints: dict = {
939
+ "n_clusters": [Interval(Integral, 1, None, closed="left"), None],
940
+ "metric": [
941
+ StrOptions(set(_VALID_METRICS) | {"precomputed"}),
942
+ callable,
943
+ ],
944
+ "memory": [str, HasMethods("cache"), None],
945
+ "connectivity": ["array-like", "sparse matrix", callable, None],
946
+ "compute_full_tree": [StrOptions({"auto"}), "boolean"],
947
+ "linkage": [StrOptions(set(_TREE_BUILDERS.keys()))],
948
+ "distance_threshold": [Interval(Real, 0, None, closed="left"), None],
949
+ "compute_distances": ["boolean"],
950
+ }
951
+
952
+ def __init__(
953
+ self,
954
+ n_clusters=2,
955
+ *,
956
+ metric="euclidean",
957
+ memory=None,
958
+ connectivity=None,
959
+ compute_full_tree="auto",
960
+ linkage="ward",
961
+ distance_threshold=None,
962
+ compute_distances=False,
963
+ ):
964
+ self.n_clusters = n_clusters
965
+ self.distance_threshold = distance_threshold
966
+ self.memory = memory
967
+ self.connectivity = connectivity
968
+ self.compute_full_tree = compute_full_tree
969
+ self.linkage = linkage
970
+ self.metric = metric
971
+ self.compute_distances = compute_distances
972
+
973
+ @_fit_context(prefer_skip_nested_validation=True)
974
+ def fit(self, X, y=None):
975
+ """Fit the hierarchical clustering from features, or distance matrix.
976
+
977
+ Parameters
978
+ ----------
979
+ X : array-like, shape (n_samples, n_features) or \
980
+ (n_samples, n_samples)
981
+ Training instances to cluster, or distances between instances if
982
+ ``metric='precomputed'``.
983
+
984
+ y : Ignored
985
+ Not used, present here for API consistency by convention.
986
+
987
+ Returns
988
+ -------
989
+ self : object
990
+ Returns the fitted instance.
991
+ """
992
+ X = validate_data(self, X, ensure_min_samples=2)
993
+ return self._fit(X)
994
+
995
+ def _fit(self, X):
996
+ """Fit without validation
997
+
998
+ Parameters
999
+ ----------
1000
+ X : ndarray of shape (n_samples, n_features) or (n_samples, n_samples)
1001
+ Training instances to cluster, or distances between instances if
1002
+ ``metric='precomputed'``.
1003
+
1004
+ Returns
1005
+ -------
1006
+ self : object
1007
+ Returns the fitted instance.
1008
+ """
1009
+ memory = check_memory(self.memory)
1010
+
1011
+ if not ((self.n_clusters is None) ^ (self.distance_threshold is None)):
1012
+ raise ValueError(
1013
+ "Exactly one of n_clusters and "
1014
+ "distance_threshold has to be set, and the other "
1015
+ "needs to be None."
1016
+ )
1017
+
1018
+ if self.distance_threshold is not None and not self.compute_full_tree:
1019
+ raise ValueError(
1020
+ "compute_full_tree must be True if distance_threshold is set."
1021
+ )
1022
+
1023
+ if self.linkage == "ward" and self.metric != "euclidean":
1024
+ raise ValueError(
1025
+ f"{self.metric} was provided as metric. Ward can only "
1026
+ "work with euclidean distances."
1027
+ )
1028
+
1029
+ tree_builder = _TREE_BUILDERS[self.linkage]
1030
+
1031
+ connectivity = self.connectivity
1032
+ if self.connectivity is not None:
1033
+ if callable(self.connectivity):
1034
+ connectivity = self.connectivity(X)
1035
+ connectivity = check_array(
1036
+ connectivity, accept_sparse=["csr", "coo", "lil"]
1037
+ )
1038
+
1039
+ n_samples = len(X)
1040
+ compute_full_tree = self.compute_full_tree
1041
+ if self.connectivity is None:
1042
+ compute_full_tree = True
1043
+ if compute_full_tree == "auto":
1044
+ if self.distance_threshold is not None:
1045
+ compute_full_tree = True
1046
+ else:
1047
+ # Early stopping is likely to give a speed up only for
1048
+ # a large number of clusters. The actual threshold
1049
+ # implemented here is heuristic
1050
+ compute_full_tree = self.n_clusters < max(100, 0.02 * n_samples)
1051
+ n_clusters = self.n_clusters
1052
+ if compute_full_tree:
1053
+ n_clusters = None
1054
+
1055
+ # Construct the tree
1056
+ kwargs = {}
1057
+ if self.linkage != "ward":
1058
+ kwargs["linkage"] = self.linkage
1059
+ kwargs["affinity"] = self.metric
1060
+
1061
+ distance_threshold = self.distance_threshold
1062
+
1063
+ return_distance = (distance_threshold is not None) or self.compute_distances
1064
+
1065
+ out = memory.cache(tree_builder)(
1066
+ X,
1067
+ connectivity=connectivity,
1068
+ n_clusters=n_clusters,
1069
+ return_distance=return_distance,
1070
+ **kwargs,
1071
+ )
1072
+ (self.children_, self.n_connected_components_, self.n_leaves_, parents) = out[
1073
+ :4
1074
+ ]
1075
+
1076
+ if return_distance:
1077
+ self.distances_ = out[-1]
1078
+
1079
+ if self.distance_threshold is not None: # distance_threshold is used
1080
+ self.n_clusters_ = (
1081
+ np.count_nonzero(self.distances_ >= distance_threshold) + 1
1082
+ )
1083
+ else: # n_clusters is used
1084
+ self.n_clusters_ = self.n_clusters
1085
+
1086
+ # Cut the tree
1087
+ if compute_full_tree:
1088
+ self.labels_ = _hc_cut(self.n_clusters_, self.children_, self.n_leaves_)
1089
+ else:
1090
+ labels = _hierarchical.hc_get_heads(parents, copy=False)
1091
+ # copy to avoid holding a reference on the original array
1092
+ labels = np.copy(labels[:n_samples])
1093
+ # Reassign cluster numbers
1094
+ self.labels_ = np.searchsorted(np.unique(labels), labels)
1095
+ return self
1096
+
1097
+ def fit_predict(self, X, y=None):
1098
+ """Fit and return the result of each sample's clustering assignment.
1099
+
1100
+ In addition to fitting, this method also return the result of the
1101
+ clustering assignment for each sample in the training set.
1102
+
1103
+ Parameters
1104
+ ----------
1105
+ X : array-like of shape (n_samples, n_features) or \
1106
+ (n_samples, n_samples)
1107
+ Training instances to cluster, or distances between instances if
1108
+ ``affinity='precomputed'``.
1109
+
1110
+ y : Ignored
1111
+ Not used, present here for API consistency by convention.
1112
+
1113
+ Returns
1114
+ -------
1115
+ labels : ndarray of shape (n_samples,)
1116
+ Cluster labels.
1117
+ """
1118
+ return super().fit_predict(X, y)
1119
+
1120
+
1121
+ class FeatureAgglomeration(
1122
+ ClassNamePrefixFeaturesOutMixin, AgglomerationTransform, AgglomerativeClustering
1123
+ ):
1124
+ """Agglomerate features.
1125
+
1126
+ Recursively merges pair of clusters of features.
1127
+
1128
+ Refer to
1129
+ :ref:`sphx_glr_auto_examples_cluster_plot_feature_agglomeration_vs_univariate_selection.py`
1130
+ for an example comparison of :class:`FeatureAgglomeration` strategy with a
1131
+ univariate feature selection strategy (based on ANOVA).
1132
+
1133
+ Read more in the :ref:`User Guide <hierarchical_clustering>`.
1134
+
1135
+ Parameters
1136
+ ----------
1137
+ n_clusters : int or None, default=2
1138
+ The number of clusters to find. It must be ``None`` if
1139
+ ``distance_threshold`` is not ``None``.
1140
+
1141
+ metric : str or callable, default="euclidean"
1142
+ Metric used to compute the linkage. Can be "euclidean", "l1", "l2",
1143
+ "manhattan", "cosine", or "precomputed". If linkage is "ward", only
1144
+ "euclidean" is accepted. If "precomputed", a distance matrix is needed
1145
+ as input for the fit method.
1146
+
1147
+ .. versionadded:: 1.2
1148
+
1149
+ memory : str or object with the joblib.Memory interface, default=None
1150
+ Used to cache the output of the computation of the tree.
1151
+ By default, no caching is done. If a string is given, it is the
1152
+ path to the caching directory.
1153
+
1154
+ connectivity : array-like, sparse matrix, or callable, default=None
1155
+ Connectivity matrix. Defines for each feature the neighboring
1156
+ features following a given structure of the data.
1157
+ This can be a connectivity matrix itself or a callable that transforms
1158
+ the data into a connectivity matrix, such as derived from
1159
+ `kneighbors_graph`. Default is `None`, i.e, the
1160
+ hierarchical clustering algorithm is unstructured.
1161
+
1162
+ compute_full_tree : 'auto' or bool, default='auto'
1163
+ Stop early the construction of the tree at `n_clusters`. This is useful
1164
+ to decrease computation time if the number of clusters is not small
1165
+ compared to the number of features. This option is useful only when
1166
+ specifying a connectivity matrix. Note also that when varying the
1167
+ number of clusters and using caching, it may be advantageous to compute
1168
+ the full tree. It must be ``True`` if ``distance_threshold`` is not
1169
+ ``None``. By default `compute_full_tree` is "auto", which is equivalent
1170
+ to `True` when `distance_threshold` is not `None` or that `n_clusters`
1171
+ is inferior to the maximum between 100 or `0.02 * n_samples`.
1172
+ Otherwise, "auto" is equivalent to `False`.
1173
+
1174
+ linkage : {"ward", "complete", "average", "single"}, default="ward"
1175
+ Which linkage criterion to use. The linkage criterion determines which
1176
+ distance to use between sets of features. The algorithm will merge
1177
+ the pairs of cluster that minimize this criterion.
1178
+
1179
+ - "ward" minimizes the variance of the clusters being merged.
1180
+ - "complete" or maximum linkage uses the maximum distances between
1181
+ all features of the two sets.
1182
+ - "average" uses the average of the distances of each feature of
1183
+ the two sets.
1184
+ - "single" uses the minimum of the distances between all features
1185
+ of the two sets.
1186
+
1187
+ pooling_func : callable, default=np.mean
1188
+ This combines the values of agglomerated features into a single
1189
+ value, and should accept an array of shape [M, N] and the keyword
1190
+ argument `axis=1`, and reduce it to an array of size [M].
1191
+
1192
+ distance_threshold : float, default=None
1193
+ The linkage distance threshold at or above which clusters will not be
1194
+ merged. If not ``None``, ``n_clusters`` must be ``None`` and
1195
+ ``compute_full_tree`` must be ``True``.
1196
+
1197
+ .. versionadded:: 0.21
1198
+
1199
+ compute_distances : bool, default=False
1200
+ Computes distances between clusters even if `distance_threshold` is not
1201
+ used. This can be used to make dendrogram visualization, but introduces
1202
+ a computational and memory overhead.
1203
+
1204
+ .. versionadded:: 0.24
1205
+
1206
+ Attributes
1207
+ ----------
1208
+ n_clusters_ : int
1209
+ The number of clusters found by the algorithm. If
1210
+ ``distance_threshold=None``, it will be equal to the given
1211
+ ``n_clusters``.
1212
+
1213
+ labels_ : array-like of (n_features,)
1214
+ Cluster labels for each feature.
1215
+
1216
+ n_leaves_ : int
1217
+ Number of leaves in the hierarchical tree.
1218
+
1219
+ n_connected_components_ : int
1220
+ The estimated number of connected components in the graph.
1221
+
1222
+ .. versionadded:: 0.21
1223
+ ``n_connected_components_`` was added to replace ``n_components_``.
1224
+
1225
+ n_features_in_ : int
1226
+ Number of features seen during :term:`fit`.
1227
+
1228
+ .. versionadded:: 0.24
1229
+
1230
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
1231
+ Names of features seen during :term:`fit`. Defined only when `X`
1232
+ has feature names that are all strings.
1233
+
1234
+ .. versionadded:: 1.0
1235
+
1236
+ children_ : array-like of shape (n_nodes-1, 2)
1237
+ The children of each non-leaf node. Values less than `n_features`
1238
+ correspond to leaves of the tree which are the original samples.
1239
+ A node `i` greater than or equal to `n_features` is a non-leaf
1240
+ node and has children `children_[i - n_features]`. Alternatively
1241
+ at the i-th iteration, children[i][0] and children[i][1]
1242
+ are merged to form node `n_features + i`.
1243
+
1244
+ distances_ : array-like of shape (n_nodes-1,)
1245
+ Distances between nodes in the corresponding place in `children_`.
1246
+ Only computed if `distance_threshold` is used or `compute_distances`
1247
+ is set to `True`.
1248
+
1249
+ See Also
1250
+ --------
1251
+ AgglomerativeClustering : Agglomerative clustering samples instead of
1252
+ features.
1253
+ ward_tree : Hierarchical clustering with ward linkage.
1254
+
1255
+ Examples
1256
+ --------
1257
+ >>> import numpy as np
1258
+ >>> from sklearn import datasets, cluster
1259
+ >>> digits = datasets.load_digits()
1260
+ >>> images = digits.images
1261
+ >>> X = np.reshape(images, (len(images), -1))
1262
+ >>> agglo = cluster.FeatureAgglomeration(n_clusters=32)
1263
+ >>> agglo.fit(X)
1264
+ FeatureAgglomeration(n_clusters=32)
1265
+ >>> X_reduced = agglo.transform(X)
1266
+ >>> X_reduced.shape
1267
+ (1797, 32)
1268
+ """
1269
+
1270
+ _parameter_constraints: dict = {
1271
+ "n_clusters": [Interval(Integral, 1, None, closed="left"), None],
1272
+ "metric": [
1273
+ StrOptions(set(_VALID_METRICS) | {"precomputed"}),
1274
+ callable,
1275
+ ],
1276
+ "memory": [str, HasMethods("cache"), None],
1277
+ "connectivity": ["array-like", "sparse matrix", callable, None],
1278
+ "compute_full_tree": [StrOptions({"auto"}), "boolean"],
1279
+ "linkage": [StrOptions(set(_TREE_BUILDERS.keys()))],
1280
+ "pooling_func": [callable],
1281
+ "distance_threshold": [Interval(Real, 0, None, closed="left"), None],
1282
+ "compute_distances": ["boolean"],
1283
+ }
1284
+
1285
+ def __init__(
1286
+ self,
1287
+ n_clusters=2,
1288
+ *,
1289
+ metric="euclidean",
1290
+ memory=None,
1291
+ connectivity=None,
1292
+ compute_full_tree="auto",
1293
+ linkage="ward",
1294
+ pooling_func=np.mean,
1295
+ distance_threshold=None,
1296
+ compute_distances=False,
1297
+ ):
1298
+ super().__init__(
1299
+ n_clusters=n_clusters,
1300
+ memory=memory,
1301
+ connectivity=connectivity,
1302
+ compute_full_tree=compute_full_tree,
1303
+ linkage=linkage,
1304
+ metric=metric,
1305
+ distance_threshold=distance_threshold,
1306
+ compute_distances=compute_distances,
1307
+ )
1308
+ self.pooling_func = pooling_func
1309
+
1310
+ @_fit_context(prefer_skip_nested_validation=True)
1311
+ def fit(self, X, y=None):
1312
+ """Fit the hierarchical clustering on the data.
1313
+
1314
+ Parameters
1315
+ ----------
1316
+ X : array-like of shape (n_samples, n_features)
1317
+ The data.
1318
+
1319
+ y : Ignored
1320
+ Not used, present here for API consistency by convention.
1321
+
1322
+ Returns
1323
+ -------
1324
+ self : object
1325
+ Returns the transformer.
1326
+ """
1327
+ X = validate_data(self, X, ensure_min_features=2)
1328
+ super()._fit(X.T)
1329
+ self._n_features_out = self.n_clusters_
1330
+ return self
1331
+
1332
+ @property
1333
+ def fit_predict(self):
1334
+ """Fit and return the result of each sample's clustering assignment."""
1335
+ raise AttributeError
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bicluster.py ADDED
@@ -0,0 +1,622 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Spectral biclustering algorithms."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ from abc import ABCMeta, abstractmethod
7
+ from numbers import Integral
8
+
9
+ import numpy as np
10
+ from scipy.linalg import norm
11
+ from scipy.sparse import dia_matrix, issparse
12
+ from scipy.sparse.linalg import eigsh, svds
13
+
14
+ from sklearn.base import BaseEstimator, BiclusterMixin, _fit_context
15
+ from sklearn.cluster._kmeans import KMeans, MiniBatchKMeans
16
+ from sklearn.utils import check_random_state, check_scalar
17
+ from sklearn.utils._param_validation import Interval, StrOptions
18
+ from sklearn.utils.extmath import _randomized_svd, make_nonnegative, safe_sparse_dot
19
+ from sklearn.utils.validation import assert_all_finite, validate_data
20
+
21
+ __all__ = ["SpectralBiclustering", "SpectralCoclustering"]
22
+
23
+
24
+ def _scale_normalize(X):
25
+ """Normalize ``X`` by scaling rows and columns independently.
26
+
27
+ Returns the normalized matrix and the row and column scaling
28
+ factors.
29
+ """
30
+ X = make_nonnegative(X)
31
+ row_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=1))).squeeze()
32
+ col_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=0))).squeeze()
33
+ row_diag = np.where(np.isnan(row_diag), 0, row_diag)
34
+ col_diag = np.where(np.isnan(col_diag), 0, col_diag)
35
+ if issparse(X):
36
+ n_rows, n_cols = X.shape
37
+ r = dia_matrix((row_diag, [0]), shape=(n_rows, n_rows))
38
+ c = dia_matrix((col_diag, [0]), shape=(n_cols, n_cols))
39
+ an = r @ X @ c
40
+ else:
41
+ an = row_diag[:, np.newaxis] * X * col_diag
42
+ return an, row_diag, col_diag
43
+
44
+
45
+ def _bistochastic_normalize(X, max_iter=1000, tol=1e-5):
46
+ """Normalize rows and columns of ``X`` simultaneously so that all
47
+ rows sum to one constant and all columns sum to a different
48
+ constant.
49
+ """
50
+ # According to paper, this can also be done more efficiently with
51
+ # deviation reduction and balancing algorithms.
52
+ X = make_nonnegative(X)
53
+ X_scaled = X
54
+ for _ in range(max_iter):
55
+ X_new, _, _ = _scale_normalize(X_scaled)
56
+ if issparse(X):
57
+ dist = norm(X_scaled.data - X.data)
58
+ else:
59
+ dist = norm(X_scaled - X_new)
60
+ X_scaled = X_new
61
+ if dist is not None and dist < tol:
62
+ break
63
+ return X_scaled
64
+
65
+
66
+ def _log_normalize(X):
67
+ """Normalize ``X`` according to Kluger's log-interactions scheme."""
68
+ X = make_nonnegative(X, min_value=1)
69
+ if issparse(X):
70
+ raise ValueError(
71
+ "Cannot compute log of a sparse matrix,"
72
+ " because log(x) diverges to -infinity as x"
73
+ " goes to 0."
74
+ )
75
+ L = np.log(X)
76
+ row_avg = L.mean(axis=1)[:, np.newaxis]
77
+ col_avg = L.mean(axis=0)
78
+ avg = L.mean()
79
+ return L - row_avg - col_avg + avg
80
+
81
+
82
+ class BaseSpectral(BiclusterMixin, BaseEstimator, metaclass=ABCMeta):
83
+ """Base class for spectral biclustering."""
84
+
85
+ _parameter_constraints: dict = {
86
+ "svd_method": [StrOptions({"randomized", "arpack"})],
87
+ "n_svd_vecs": [Interval(Integral, 0, None, closed="left"), None],
88
+ "mini_batch": ["boolean"],
89
+ "init": [StrOptions({"k-means++", "random"}), np.ndarray],
90
+ "n_init": [Interval(Integral, 1, None, closed="left")],
91
+ "random_state": ["random_state"],
92
+ }
93
+
94
+ @abstractmethod
95
+ def __init__(
96
+ self,
97
+ n_clusters=3,
98
+ svd_method="randomized",
99
+ n_svd_vecs=None,
100
+ mini_batch=False,
101
+ init="k-means++",
102
+ n_init=10,
103
+ random_state=None,
104
+ ):
105
+ self.n_clusters = n_clusters
106
+ self.svd_method = svd_method
107
+ self.n_svd_vecs = n_svd_vecs
108
+ self.mini_batch = mini_batch
109
+ self.init = init
110
+ self.n_init = n_init
111
+ self.random_state = random_state
112
+
113
+ @abstractmethod
114
+ def _check_parameters(self, n_samples):
115
+ """Validate parameters depending on the input data."""
116
+
117
+ @_fit_context(prefer_skip_nested_validation=True)
118
+ def fit(self, X, y=None):
119
+ """Create a biclustering for X.
120
+
121
+ Parameters
122
+ ----------
123
+ X : array-like of shape (n_samples, n_features)
124
+ Training data.
125
+
126
+ y : Ignored
127
+ Not used, present for API consistency by convention.
128
+
129
+ Returns
130
+ -------
131
+ self : object
132
+ SpectralBiclustering instance.
133
+ """
134
+ X = validate_data(self, X, accept_sparse="csr", dtype=np.float64)
135
+ self._check_parameters(X.shape[0])
136
+ self._fit(X)
137
+ return self
138
+
139
+ def _svd(self, array, n_components, n_discard):
140
+ """Returns first `n_components` left and right singular
141
+ vectors u and v, discarding the first `n_discard`.
142
+ """
143
+ if self.svd_method == "randomized":
144
+ kwargs = {}
145
+ if self.n_svd_vecs is not None:
146
+ kwargs["n_oversamples"] = self.n_svd_vecs
147
+ u, _, vt = _randomized_svd(
148
+ array, n_components, random_state=self.random_state, **kwargs
149
+ )
150
+
151
+ elif self.svd_method == "arpack":
152
+ u, _, vt = svds(array, k=n_components, ncv=self.n_svd_vecs)
153
+ if np.any(np.isnan(vt)):
154
+ # some eigenvalues of A * A.T are negative, causing
155
+ # sqrt() to be np.nan. This causes some vectors in vt
156
+ # to be np.nan.
157
+ A = safe_sparse_dot(array.T, array)
158
+ random_state = check_random_state(self.random_state)
159
+ # initialize with [-1,1] as in ARPACK
160
+ v0 = random_state.uniform(-1, 1, A.shape[0])
161
+ _, v = eigsh(A, ncv=self.n_svd_vecs, v0=v0)
162
+ vt = v.T
163
+ if np.any(np.isnan(u)):
164
+ A = safe_sparse_dot(array, array.T)
165
+ random_state = check_random_state(self.random_state)
166
+ # initialize with [-1,1] as in ARPACK
167
+ v0 = random_state.uniform(-1, 1, A.shape[0])
168
+ _, u = eigsh(A, ncv=self.n_svd_vecs, v0=v0)
169
+
170
+ assert_all_finite(u)
171
+ assert_all_finite(vt)
172
+ u = u[:, n_discard:]
173
+ vt = vt[n_discard:]
174
+ return u, vt.T
175
+
176
+ def _k_means(self, data, n_clusters):
177
+ if self.mini_batch:
178
+ model = MiniBatchKMeans(
179
+ n_clusters,
180
+ init=self.init,
181
+ n_init=self.n_init,
182
+ random_state=self.random_state,
183
+ )
184
+ else:
185
+ model = KMeans(
186
+ n_clusters,
187
+ init=self.init,
188
+ n_init=self.n_init,
189
+ random_state=self.random_state,
190
+ )
191
+ model.fit(data)
192
+ centroid = model.cluster_centers_
193
+ labels = model.labels_
194
+ return centroid, labels
195
+
196
+ def __sklearn_tags__(self):
197
+ tags = super().__sklearn_tags__()
198
+ tags.input_tags.sparse = True
199
+ return tags
200
+
201
+
202
+ class SpectralCoclustering(BaseSpectral):
203
+ """Spectral Co-Clustering algorithm (Dhillon, 2001) [1]_.
204
+
205
+ Clusters rows and columns of an array `X` to solve the relaxed
206
+ normalized cut of the bipartite graph created from `X` as follows:
207
+ the edge between row vertex `i` and column vertex `j` has weight
208
+ `X[i, j]`.
209
+
210
+ The resulting bicluster structure is block-diagonal, since each
211
+ row and each column belongs to exactly one bicluster.
212
+
213
+ Supports sparse matrices, as long as they are nonnegative.
214
+
215
+ Read more in the :ref:`User Guide <spectral_coclustering>`.
216
+
217
+ Parameters
218
+ ----------
219
+ n_clusters : int, default=3
220
+ The number of biclusters to find.
221
+
222
+ svd_method : {'randomized', 'arpack'}, default='randomized'
223
+ Selects the algorithm for finding singular vectors. May be
224
+ 'randomized' or 'arpack'. If 'randomized', use
225
+ :func:`sklearn.utils.extmath.randomized_svd`, which may be faster
226
+ for large matrices. If 'arpack', use
227
+ :func:`scipy.sparse.linalg.svds`, which is more accurate, but
228
+ possibly slower in some cases.
229
+
230
+ n_svd_vecs : int, default=None
231
+ Number of vectors to use in calculating the SVD. Corresponds
232
+ to `ncv` when `svd_method=arpack` and `n_oversamples` when
233
+ `svd_method` is 'randomized`.
234
+
235
+ mini_batch : bool, default=False
236
+ Whether to use mini-batch k-means, which is faster but may get
237
+ different results.
238
+
239
+ init : {'k-means++', 'random'}, or ndarray of shape \
240
+ (n_clusters, n_features), default='k-means++'
241
+ Method for initialization of k-means algorithm; defaults to
242
+ 'k-means++'.
243
+
244
+ n_init : int, default=10
245
+ Number of random initializations that are tried with the
246
+ k-means algorithm.
247
+
248
+ If mini-batch k-means is used, the best initialization is
249
+ chosen and the algorithm runs once. Otherwise, the algorithm
250
+ is run for each initialization and the best solution chosen.
251
+
252
+ random_state : int, RandomState instance, default=None
253
+ Used for randomizing the singular value decomposition and the k-means
254
+ initialization. Use an int to make the randomness deterministic.
255
+ See :term:`Glossary <random_state>`.
256
+
257
+ Attributes
258
+ ----------
259
+ rows_ : array-like of shape (n_row_clusters, n_rows)
260
+ Results of the clustering. `rows[i, r]` is True if
261
+ cluster `i` contains row `r`. Available only after calling ``fit``.
262
+
263
+ columns_ : array-like of shape (n_column_clusters, n_columns)
264
+ Results of the clustering, like `rows`.
265
+
266
+ row_labels_ : array-like of shape (n_rows,)
267
+ The bicluster label of each row.
268
+
269
+ column_labels_ : array-like of shape (n_cols,)
270
+ The bicluster label of each column.
271
+
272
+ biclusters_ : tuple of two ndarrays
273
+ The tuple contains the `rows_` and `columns_` arrays.
274
+
275
+ n_features_in_ : int
276
+ Number of features seen during :term:`fit`.
277
+
278
+ .. versionadded:: 0.24
279
+
280
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
281
+ Names of features seen during :term:`fit`. Defined only when `X`
282
+ has feature names that are all strings.
283
+
284
+ .. versionadded:: 1.0
285
+
286
+ See Also
287
+ --------
288
+ SpectralBiclustering : Partitions rows and columns under the assumption
289
+ that the data has an underlying checkerboard structure.
290
+
291
+ References
292
+ ----------
293
+ .. [1] :doi:`Dhillon, Inderjit S, 2001. Co-clustering documents and words using
294
+ bipartite spectral graph partitioning.
295
+ <10.1145/502512.502550>`
296
+
297
+ Examples
298
+ --------
299
+ >>> from sklearn.cluster import SpectralCoclustering
300
+ >>> import numpy as np
301
+ >>> X = np.array([[1, 1], [2, 1], [1, 0],
302
+ ... [4, 7], [3, 5], [3, 6]])
303
+ >>> clustering = SpectralCoclustering(n_clusters=2, random_state=0).fit(X)
304
+ >>> clustering.row_labels_ #doctest: +SKIP
305
+ array([0, 1, 1, 0, 0, 0], dtype=int32)
306
+ >>> clustering.column_labels_ #doctest: +SKIP
307
+ array([0, 0], dtype=int32)
308
+ >>> clustering
309
+ SpectralCoclustering(n_clusters=2, random_state=0)
310
+
311
+ For a more detailed example, see the following:
312
+ :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_coclustering.py`.
313
+ """
314
+
315
+ _parameter_constraints: dict = {
316
+ **BaseSpectral._parameter_constraints,
317
+ "n_clusters": [Interval(Integral, 1, None, closed="left")],
318
+ }
319
+
320
+ def __init__(
321
+ self,
322
+ n_clusters=3,
323
+ *,
324
+ svd_method="randomized",
325
+ n_svd_vecs=None,
326
+ mini_batch=False,
327
+ init="k-means++",
328
+ n_init=10,
329
+ random_state=None,
330
+ ):
331
+ super().__init__(
332
+ n_clusters, svd_method, n_svd_vecs, mini_batch, init, n_init, random_state
333
+ )
334
+
335
+ def _check_parameters(self, n_samples):
336
+ if self.n_clusters > n_samples:
337
+ raise ValueError(
338
+ f"n_clusters should be <= n_samples={n_samples}. Got"
339
+ f" {self.n_clusters} instead."
340
+ )
341
+
342
+ def _fit(self, X):
343
+ normalized_data, row_diag, col_diag = _scale_normalize(X)
344
+ n_sv = 1 + int(np.ceil(np.log2(self.n_clusters)))
345
+ u, v = self._svd(normalized_data, n_sv, n_discard=1)
346
+ z = np.vstack((row_diag[:, np.newaxis] * u, col_diag[:, np.newaxis] * v))
347
+
348
+ _, labels = self._k_means(z, self.n_clusters)
349
+
350
+ n_rows = X.shape[0]
351
+ self.row_labels_ = labels[:n_rows]
352
+ self.column_labels_ = labels[n_rows:]
353
+
354
+ self.rows_ = np.vstack([self.row_labels_ == c for c in range(self.n_clusters)])
355
+ self.columns_ = np.vstack(
356
+ [self.column_labels_ == c for c in range(self.n_clusters)]
357
+ )
358
+
359
+
360
+ class SpectralBiclustering(BaseSpectral):
361
+ """Spectral biclustering (Kluger, 2003) [1]_.
362
+
363
+ Partitions rows and columns under the assumption that the data has
364
+ an underlying checkerboard structure. For instance, if there are
365
+ two row partitions and three column partitions, each row will
366
+ belong to three biclusters, and each column will belong to two
367
+ biclusters. The outer product of the corresponding row and column
368
+ label vectors gives this checkerboard structure.
369
+
370
+ Read more in the :ref:`User Guide <spectral_biclustering>`.
371
+
372
+ Parameters
373
+ ----------
374
+ n_clusters : int or tuple (n_row_clusters, n_column_clusters), default=3
375
+ The number of row and column clusters in the checkerboard
376
+ structure.
377
+
378
+ method : {'bistochastic', 'scale', 'log'}, default='bistochastic'
379
+ Method of normalizing and converting singular vectors into
380
+ biclusters. May be one of 'scale', 'bistochastic', or 'log'.
381
+ The authors recommend using 'log'. If the data is sparse,
382
+ however, log normalization will not work, which is why the
383
+ default is 'bistochastic'.
384
+
385
+ .. warning::
386
+ if `method='log'`, the data must not be sparse.
387
+
388
+ n_components : int, default=6
389
+ Number of singular vectors to check.
390
+
391
+ n_best : int, default=3
392
+ Number of best singular vectors to which to project the data
393
+ for clustering.
394
+
395
+ svd_method : {'randomized', 'arpack'}, default='randomized'
396
+ Selects the algorithm for finding singular vectors. May be
397
+ 'randomized' or 'arpack'. If 'randomized', uses
398
+ :func:`~sklearn.utils.extmath.randomized_svd`, which may be faster
399
+ for large matrices. If 'arpack', uses
400
+ `scipy.sparse.linalg.svds`, which is more accurate, but
401
+ possibly slower in some cases.
402
+
403
+ n_svd_vecs : int, default=None
404
+ Number of vectors to use in calculating the SVD. Corresponds
405
+ to `ncv` when `svd_method=arpack` and `n_oversamples` when
406
+ `svd_method` is 'randomized`.
407
+
408
+ mini_batch : bool, default=False
409
+ Whether to use mini-batch k-means, which is faster but may get
410
+ different results.
411
+
412
+ init : {'k-means++', 'random'} or ndarray of shape (n_clusters, n_features), \
413
+ default='k-means++'
414
+ Method for initialization of k-means algorithm; defaults to
415
+ 'k-means++'.
416
+
417
+ n_init : int, default=10
418
+ Number of random initializations that are tried with the
419
+ k-means algorithm.
420
+
421
+ If mini-batch k-means is used, the best initialization is
422
+ chosen and the algorithm runs once. Otherwise, the algorithm
423
+ is run for each initialization and the best solution chosen.
424
+
425
+ random_state : int, RandomState instance, default=None
426
+ Used for randomizing the singular value decomposition and the k-means
427
+ initialization. Use an int to make the randomness deterministic.
428
+ See :term:`Glossary <random_state>`.
429
+
430
+ Attributes
431
+ ----------
432
+ rows_ : array-like of shape (n_row_clusters, n_rows)
433
+ Results of the clustering. `rows[i, r]` is True if
434
+ cluster `i` contains row `r`. Available only after calling ``fit``.
435
+
436
+ columns_ : array-like of shape (n_column_clusters, n_columns)
437
+ Results of the clustering, like `rows`.
438
+
439
+ row_labels_ : array-like of shape (n_rows,)
440
+ Row partition labels.
441
+
442
+ column_labels_ : array-like of shape (n_cols,)
443
+ Column partition labels.
444
+
445
+ biclusters_ : tuple of two ndarrays
446
+ The tuple contains the `rows_` and `columns_` arrays.
447
+
448
+ n_features_in_ : int
449
+ Number of features seen during :term:`fit`.
450
+
451
+ .. versionadded:: 0.24
452
+
453
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
454
+ Names of features seen during :term:`fit`. Defined only when `X`
455
+ has feature names that are all strings.
456
+
457
+ .. versionadded:: 1.0
458
+
459
+ See Also
460
+ --------
461
+ SpectralCoclustering : Clusters rows and columns of an array `X` to solve the
462
+ relaxed normalized cut of the bipartite graph created from `X`.
463
+
464
+ References
465
+ ----------
466
+
467
+ .. [1] :doi:`Kluger, Yuval, et. al., 2003. Spectral biclustering of microarray
468
+ data: coclustering genes and conditions.
469
+ <10.1101/gr.648603>`
470
+
471
+ Examples
472
+ --------
473
+ >>> from sklearn.cluster import SpectralBiclustering
474
+ >>> import numpy as np
475
+ >>> X = np.array([[1, 1], [2, 1], [1, 0],
476
+ ... [4, 7], [3, 5], [3, 6]])
477
+ >>> clustering = SpectralBiclustering(n_clusters=2, random_state=0).fit(X)
478
+ >>> clustering.row_labels_
479
+ array([1, 1, 1, 0, 0, 0], dtype=int32)
480
+ >>> clustering.column_labels_
481
+ array([1, 0], dtype=int32)
482
+ >>> clustering
483
+ SpectralBiclustering(n_clusters=2, random_state=0)
484
+
485
+ For a more detailed example, see
486
+ :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_biclustering.py`
487
+ """
488
+
489
+ _parameter_constraints: dict = {
490
+ **BaseSpectral._parameter_constraints,
491
+ "n_clusters": [Interval(Integral, 1, None, closed="left"), tuple],
492
+ "method": [StrOptions({"bistochastic", "scale", "log"})],
493
+ "n_components": [Interval(Integral, 1, None, closed="left")],
494
+ "n_best": [Interval(Integral, 1, None, closed="left")],
495
+ }
496
+
497
+ def __init__(
498
+ self,
499
+ n_clusters=3,
500
+ *,
501
+ method="bistochastic",
502
+ n_components=6,
503
+ n_best=3,
504
+ svd_method="randomized",
505
+ n_svd_vecs=None,
506
+ mini_batch=False,
507
+ init="k-means++",
508
+ n_init=10,
509
+ random_state=None,
510
+ ):
511
+ super().__init__(
512
+ n_clusters, svd_method, n_svd_vecs, mini_batch, init, n_init, random_state
513
+ )
514
+ self.method = method
515
+ self.n_components = n_components
516
+ self.n_best = n_best
517
+
518
+ def _check_parameters(self, n_samples):
519
+ if isinstance(self.n_clusters, Integral):
520
+ if self.n_clusters > n_samples:
521
+ raise ValueError(
522
+ f"n_clusters should be <= n_samples={n_samples}. Got"
523
+ f" {self.n_clusters} instead."
524
+ )
525
+ else: # tuple
526
+ try:
527
+ n_row_clusters, n_column_clusters = self.n_clusters
528
+ check_scalar(
529
+ n_row_clusters,
530
+ "n_row_clusters",
531
+ target_type=Integral,
532
+ min_val=1,
533
+ max_val=n_samples,
534
+ )
535
+ check_scalar(
536
+ n_column_clusters,
537
+ "n_column_clusters",
538
+ target_type=Integral,
539
+ min_val=1,
540
+ max_val=n_samples,
541
+ )
542
+ except (ValueError, TypeError) as e:
543
+ raise ValueError(
544
+ "Incorrect parameter n_clusters has value:"
545
+ f" {self.n_clusters}. It should either be a single integer"
546
+ " or an iterable with two integers:"
547
+ " (n_row_clusters, n_column_clusters)"
548
+ " And the values are should be in the"
549
+ " range: (1, n_samples)"
550
+ ) from e
551
+
552
+ if self.n_best > self.n_components:
553
+ raise ValueError(
554
+ f"n_best={self.n_best} must be <= n_components={self.n_components}."
555
+ )
556
+
557
+ def _fit(self, X):
558
+ n_sv = self.n_components
559
+ if self.method == "bistochastic":
560
+ normalized_data = _bistochastic_normalize(X)
561
+ n_sv += 1
562
+ elif self.method == "scale":
563
+ normalized_data, _, _ = _scale_normalize(X)
564
+ n_sv += 1
565
+ elif self.method == "log":
566
+ normalized_data = _log_normalize(X)
567
+ n_discard = 0 if self.method == "log" else 1
568
+ u, v = self._svd(normalized_data, n_sv, n_discard)
569
+ ut = u.T
570
+ vt = v.T
571
+
572
+ try:
573
+ n_row_clusters, n_col_clusters = self.n_clusters
574
+ except TypeError:
575
+ n_row_clusters = n_col_clusters = self.n_clusters
576
+
577
+ best_ut = self._fit_best_piecewise(ut, self.n_best, n_row_clusters)
578
+
579
+ best_vt = self._fit_best_piecewise(vt, self.n_best, n_col_clusters)
580
+
581
+ self.row_labels_ = self._project_and_cluster(X, best_vt.T, n_row_clusters)
582
+
583
+ self.column_labels_ = self._project_and_cluster(X.T, best_ut.T, n_col_clusters)
584
+
585
+ self.rows_ = np.vstack(
586
+ [
587
+ self.row_labels_ == label
588
+ for label in range(n_row_clusters)
589
+ for _ in range(n_col_clusters)
590
+ ]
591
+ )
592
+ self.columns_ = np.vstack(
593
+ [
594
+ self.column_labels_ == label
595
+ for _ in range(n_row_clusters)
596
+ for label in range(n_col_clusters)
597
+ ]
598
+ )
599
+
600
+ def _fit_best_piecewise(self, vectors, n_best, n_clusters):
601
+ """Find the ``n_best`` vectors that are best approximated by piecewise
602
+ constant vectors.
603
+
604
+ The piecewise vectors are found by k-means; the best is chosen
605
+ according to Euclidean distance.
606
+
607
+ """
608
+
609
+ def make_piecewise(v):
610
+ centroid, labels = self._k_means(v.reshape(-1, 1), n_clusters)
611
+ return centroid[labels].ravel()
612
+
613
+ piecewise_vectors = np.apply_along_axis(make_piecewise, axis=1, arr=vectors)
614
+ dists = np.apply_along_axis(norm, axis=1, arr=(vectors - piecewise_vectors))
615
+ result = vectors[np.argsort(dists)[:n_best]]
616
+ return result
617
+
618
+ def _project_and_cluster(self, data, vectors, n_clusters):
619
+ """Project ``data`` to ``vectors`` and cluster the result."""
620
+ projected = safe_sparse_dot(data, vectors)
621
+ _, labels = self._k_means(projected, n_clusters)
622
+ return labels
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_birch.py ADDED
@@ -0,0 +1,730 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Authors: The scikit-learn developers
2
+ # SPDX-License-Identifier: BSD-3-Clause
3
+
4
+ import warnings
5
+ from math import sqrt
6
+ from numbers import Integral, Real
7
+
8
+ import numpy as np
9
+ from scipy import sparse
10
+
11
+ from sklearn._config import config_context
12
+ from sklearn.base import (
13
+ BaseEstimator,
14
+ ClassNamePrefixFeaturesOutMixin,
15
+ ClusterMixin,
16
+ TransformerMixin,
17
+ _fit_context,
18
+ )
19
+ from sklearn.cluster import AgglomerativeClustering
20
+ from sklearn.exceptions import ConvergenceWarning
21
+ from sklearn.metrics import pairwise_distances_argmin
22
+ from sklearn.metrics.pairwise import euclidean_distances
23
+ from sklearn.utils._param_validation import Interval
24
+ from sklearn.utils.extmath import row_norms
25
+ from sklearn.utils.validation import check_is_fitted, validate_data
26
+
27
+
28
+ def _iterate_sparse_X(X):
29
+ """This little hack returns a densified row when iterating over a sparse
30
+ matrix, instead of constructing a sparse matrix for every row that is
31
+ expensive.
32
+ """
33
+ n_samples = X.shape[0]
34
+ X_indices = X.indices
35
+ X_data = X.data
36
+ X_indptr = X.indptr
37
+
38
+ for i in range(n_samples):
39
+ row = np.zeros(X.shape[1])
40
+ startptr, endptr = X_indptr[i], X_indptr[i + 1]
41
+ nonzero_indices = X_indices[startptr:endptr]
42
+ row[nonzero_indices] = X_data[startptr:endptr]
43
+ yield row
44
+
45
+
46
+ def _split_node(node, threshold, branching_factor):
47
+ """The node has to be split if there is no place for a new subcluster
48
+ in the node.
49
+ 1. Two empty nodes and two empty subclusters are initialized.
50
+ 2. The pair of distant subclusters are found.
51
+ 3. The properties of the empty subclusters and nodes are updated
52
+ according to the nearest distance between the subclusters to the
53
+ pair of distant subclusters.
54
+ 4. The two nodes are set as children to the two subclusters.
55
+ """
56
+ new_subcluster1 = _CFSubcluster()
57
+ new_subcluster2 = _CFSubcluster()
58
+ new_node1 = _CFNode(
59
+ threshold=threshold,
60
+ branching_factor=branching_factor,
61
+ is_leaf=node.is_leaf,
62
+ n_features=node.n_features,
63
+ dtype=node.init_centroids_.dtype,
64
+ )
65
+ new_node2 = _CFNode(
66
+ threshold=threshold,
67
+ branching_factor=branching_factor,
68
+ is_leaf=node.is_leaf,
69
+ n_features=node.n_features,
70
+ dtype=node.init_centroids_.dtype,
71
+ )
72
+ new_subcluster1.child_ = new_node1
73
+ new_subcluster2.child_ = new_node2
74
+
75
+ if node.is_leaf:
76
+ if node.prev_leaf_ is not None:
77
+ node.prev_leaf_.next_leaf_ = new_node1
78
+ new_node1.prev_leaf_ = node.prev_leaf_
79
+ new_node1.next_leaf_ = new_node2
80
+ new_node2.prev_leaf_ = new_node1
81
+ new_node2.next_leaf_ = node.next_leaf_
82
+ if node.next_leaf_ is not None:
83
+ node.next_leaf_.prev_leaf_ = new_node2
84
+
85
+ dist = euclidean_distances(
86
+ node.centroids_, Y_norm_squared=node.squared_norm_, squared=True
87
+ )
88
+ n_clusters = dist.shape[0]
89
+
90
+ farthest_idx = np.unravel_index(dist.argmax(), (n_clusters, n_clusters))
91
+ node1_dist, node2_dist = dist[(farthest_idx,)]
92
+
93
+ node1_closer = node1_dist < node2_dist
94
+ # make sure node1 is closest to itself even if all distances are equal.
95
+ # This can only happen when all node.centroids_ are duplicates leading to all
96
+ # distances between centroids being zero.
97
+ node1_closer[farthest_idx[0]] = True
98
+
99
+ for idx, subcluster in enumerate(node.subclusters_):
100
+ if node1_closer[idx]:
101
+ new_node1.append_subcluster(subcluster)
102
+ new_subcluster1.update(subcluster)
103
+ else:
104
+ new_node2.append_subcluster(subcluster)
105
+ new_subcluster2.update(subcluster)
106
+ return new_subcluster1, new_subcluster2
107
+
108
+
109
+ class _CFNode:
110
+ """Each node in a CFTree is called a CFNode.
111
+
112
+ The CFNode can have a maximum of branching_factor
113
+ number of CFSubclusters.
114
+
115
+ Parameters
116
+ ----------
117
+ threshold : float
118
+ Threshold needed for a new subcluster to enter a CFSubcluster.
119
+
120
+ branching_factor : int
121
+ Maximum number of CF subclusters in each node.
122
+
123
+ is_leaf : bool
124
+ We need to know if the CFNode is a leaf or not, in order to
125
+ retrieve the final subclusters.
126
+
127
+ n_features : int
128
+ The number of features.
129
+
130
+ Attributes
131
+ ----------
132
+ subclusters_ : list
133
+ List of subclusters for a particular CFNode.
134
+
135
+ prev_leaf_ : _CFNode
136
+ Useful only if is_leaf is True.
137
+
138
+ next_leaf_ : _CFNode
139
+ next_leaf. Useful only if is_leaf is True.
140
+ the final subclusters.
141
+
142
+ init_centroids_ : ndarray of shape (branching_factor + 1, n_features)
143
+ Manipulate ``init_centroids_`` throughout rather than centroids_ since
144
+ the centroids are just a view of the ``init_centroids_`` .
145
+
146
+ init_sq_norm_ : ndarray of shape (branching_factor + 1,)
147
+ manipulate init_sq_norm_ throughout. similar to ``init_centroids_``.
148
+
149
+ centroids_ : ndarray of shape (branching_factor + 1, n_features)
150
+ View of ``init_centroids_``.
151
+
152
+ squared_norm_ : ndarray of shape (branching_factor + 1,)
153
+ View of ``init_sq_norm_``.
154
+
155
+ """
156
+
157
+ def __init__(self, *, threshold, branching_factor, is_leaf, n_features, dtype):
158
+ self.threshold = threshold
159
+ self.branching_factor = branching_factor
160
+ self.is_leaf = is_leaf
161
+ self.n_features = n_features
162
+
163
+ # The list of subclusters, centroids and squared norms
164
+ # to manipulate throughout.
165
+ self.subclusters_ = []
166
+ self.init_centroids_ = np.zeros((branching_factor + 1, n_features), dtype=dtype)
167
+ self.init_sq_norm_ = np.zeros((branching_factor + 1), dtype)
168
+ self.squared_norm_ = []
169
+ self.prev_leaf_ = None
170
+ self.next_leaf_ = None
171
+
172
+ def append_subcluster(self, subcluster):
173
+ n_samples = len(self.subclusters_)
174
+ self.subclusters_.append(subcluster)
175
+ self.init_centroids_[n_samples] = subcluster.centroid_
176
+ self.init_sq_norm_[n_samples] = subcluster.sq_norm_
177
+
178
+ # Keep centroids and squared norm as views. In this way
179
+ # if we change init_centroids and init_sq_norm_, it is
180
+ # sufficient,
181
+ self.centroids_ = self.init_centroids_[: n_samples + 1, :]
182
+ self.squared_norm_ = self.init_sq_norm_[: n_samples + 1]
183
+
184
+ def update_split_subclusters(self, subcluster, new_subcluster1, new_subcluster2):
185
+ """Remove a subcluster from a node and update it with the
186
+ split subclusters.
187
+ """
188
+ ind = self.subclusters_.index(subcluster)
189
+ self.subclusters_[ind] = new_subcluster1
190
+ self.init_centroids_[ind] = new_subcluster1.centroid_
191
+ self.init_sq_norm_[ind] = new_subcluster1.sq_norm_
192
+ self.append_subcluster(new_subcluster2)
193
+
194
+ def insert_cf_subcluster(self, subcluster):
195
+ """Insert a new subcluster into the node."""
196
+ if not self.subclusters_:
197
+ self.append_subcluster(subcluster)
198
+ return False
199
+
200
+ threshold = self.threshold
201
+ branching_factor = self.branching_factor
202
+ # We need to find the closest subcluster among all the
203
+ # subclusters so that we can insert our new subcluster.
204
+ dist_matrix = np.dot(self.centroids_, subcluster.centroid_)
205
+ dist_matrix *= -2.0
206
+ dist_matrix += self.squared_norm_
207
+ closest_index = np.argmin(dist_matrix)
208
+ closest_subcluster = self.subclusters_[closest_index]
209
+
210
+ # If the subcluster has a child, we need a recursive strategy.
211
+ if closest_subcluster.child_ is not None:
212
+ split_child = closest_subcluster.child_.insert_cf_subcluster(subcluster)
213
+
214
+ if not split_child:
215
+ # If it is determined that the child need not be split, we
216
+ # can just update the closest_subcluster
217
+ closest_subcluster.update(subcluster)
218
+ self.init_centroids_[closest_index] = self.subclusters_[
219
+ closest_index
220
+ ].centroid_
221
+ self.init_sq_norm_[closest_index] = self.subclusters_[
222
+ closest_index
223
+ ].sq_norm_
224
+ return False
225
+
226
+ # things not too good. we need to redistribute the subclusters in
227
+ # our child node, and add a new subcluster in the parent
228
+ # subcluster to accommodate the new child.
229
+ else:
230
+ new_subcluster1, new_subcluster2 = _split_node(
231
+ closest_subcluster.child_,
232
+ threshold,
233
+ branching_factor,
234
+ )
235
+ self.update_split_subclusters(
236
+ closest_subcluster, new_subcluster1, new_subcluster2
237
+ )
238
+
239
+ if len(self.subclusters_) > self.branching_factor:
240
+ return True
241
+ return False
242
+
243
+ # good to go!
244
+ else:
245
+ merged = closest_subcluster.merge_subcluster(subcluster, self.threshold)
246
+ if merged:
247
+ self.init_centroids_[closest_index] = closest_subcluster.centroid_
248
+ self.init_sq_norm_[closest_index] = closest_subcluster.sq_norm_
249
+ return False
250
+
251
+ # not close to any other subclusters, and we still
252
+ # have space, so add.
253
+ elif len(self.subclusters_) < self.branching_factor:
254
+ self.append_subcluster(subcluster)
255
+ return False
256
+
257
+ # We do not have enough space nor is it closer to an
258
+ # other subcluster. We need to split.
259
+ else:
260
+ self.append_subcluster(subcluster)
261
+ return True
262
+
263
+
264
+ class _CFSubcluster:
265
+ """Each subcluster in a CFNode is called a CFSubcluster.
266
+
267
+ A CFSubcluster can have a CFNode has its child.
268
+
269
+ Parameters
270
+ ----------
271
+ linear_sum : ndarray of shape (n_features,), default=None
272
+ Sample. This is kept optional to allow initialization of empty
273
+ subclusters.
274
+
275
+ Attributes
276
+ ----------
277
+ n_samples_ : int
278
+ Number of samples that belong to each subcluster.
279
+
280
+ linear_sum_ : ndarray
281
+ Linear sum of all the samples in a subcluster. Prevents holding
282
+ all sample data in memory.
283
+
284
+ squared_sum_ : float
285
+ Sum of the squared l2 norms of all samples belonging to a subcluster.
286
+
287
+ centroid_ : ndarray of shape (branching_factor + 1, n_features)
288
+ Centroid of the subcluster. Prevent recomputing of centroids when
289
+ ``CFNode.centroids_`` is called.
290
+
291
+ child_ : _CFNode
292
+ Child Node of the subcluster. Once a given _CFNode is set as the child
293
+ of the _CFNode, it is set to ``self.child_``.
294
+
295
+ sq_norm_ : ndarray of shape (branching_factor + 1,)
296
+ Squared norm of the subcluster. Used to prevent recomputing when
297
+ pairwise minimum distances are computed.
298
+ """
299
+
300
+ def __init__(self, *, linear_sum=None):
301
+ if linear_sum is None:
302
+ self.n_samples_ = 0
303
+ self.squared_sum_ = 0.0
304
+ self.centroid_ = self.linear_sum_ = 0
305
+ else:
306
+ self.n_samples_ = 1
307
+ self.centroid_ = self.linear_sum_ = linear_sum
308
+ self.squared_sum_ = self.sq_norm_ = np.dot(
309
+ self.linear_sum_, self.linear_sum_
310
+ )
311
+ self.child_ = None
312
+
313
+ def update(self, subcluster):
314
+ self.n_samples_ += subcluster.n_samples_
315
+ self.linear_sum_ += subcluster.linear_sum_
316
+ self.squared_sum_ += subcluster.squared_sum_
317
+ self.centroid_ = self.linear_sum_ / self.n_samples_
318
+ self.sq_norm_ = np.dot(self.centroid_, self.centroid_)
319
+
320
+ def merge_subcluster(self, nominee_cluster, threshold):
321
+ """Check if a cluster is worthy enough to be merged. If
322
+ yes then merge.
323
+ """
324
+ new_ss = self.squared_sum_ + nominee_cluster.squared_sum_
325
+ new_ls = self.linear_sum_ + nominee_cluster.linear_sum_
326
+ new_n = self.n_samples_ + nominee_cluster.n_samples_
327
+ new_centroid = (1 / new_n) * new_ls
328
+ new_sq_norm = np.dot(new_centroid, new_centroid)
329
+
330
+ # The squared radius of the cluster is defined:
331
+ # r^2 = sum_i ||x_i - c||^2 / n
332
+ # with x_i the n points assigned to the cluster and c its centroid:
333
+ # c = sum_i x_i / n
334
+ # This can be expanded to:
335
+ # r^2 = sum_i ||x_i||^2 / n - 2 < sum_i x_i / n, c> + n ||c||^2 / n
336
+ # and therefore simplifies to:
337
+ # r^2 = sum_i ||x_i||^2 / n - ||c||^2
338
+ sq_radius = new_ss / new_n - new_sq_norm
339
+
340
+ if sq_radius <= threshold**2:
341
+ (
342
+ self.n_samples_,
343
+ self.linear_sum_,
344
+ self.squared_sum_,
345
+ self.centroid_,
346
+ self.sq_norm_,
347
+ ) = (new_n, new_ls, new_ss, new_centroid, new_sq_norm)
348
+ return True
349
+ return False
350
+
351
+ @property
352
+ def radius(self):
353
+ """Return radius of the subcluster"""
354
+ # Because of numerical issues, this could become negative
355
+ sq_radius = self.squared_sum_ / self.n_samples_ - self.sq_norm_
356
+ return sqrt(max(0, sq_radius))
357
+
358
+
359
+ class Birch(
360
+ ClassNamePrefixFeaturesOutMixin, ClusterMixin, TransformerMixin, BaseEstimator
361
+ ):
362
+ """Implements the BIRCH clustering algorithm.
363
+
364
+ It is a memory-efficient, online-learning algorithm provided as an
365
+ alternative to :class:`MiniBatchKMeans`. It constructs a tree
366
+ data structure with the cluster centroids being read off the leaf.
367
+ These can be either the final cluster centroids or can be provided as input
368
+ to another clustering algorithm such as :class:`AgglomerativeClustering`.
369
+
370
+ Read more in the :ref:`User Guide <birch>`.
371
+
372
+ .. versionadded:: 0.16
373
+
374
+ Parameters
375
+ ----------
376
+ threshold : float, default=0.5
377
+ The radius of the subcluster obtained by merging a new sample and the
378
+ closest subcluster should be lesser than the threshold. Otherwise a new
379
+ subcluster is started. Setting this value to be very low promotes
380
+ splitting and vice-versa.
381
+
382
+ branching_factor : int, default=50
383
+ Maximum number of CF subclusters in each node. If a new samples enters
384
+ such that the number of subclusters exceed the branching_factor then
385
+ that node is split into two nodes with the subclusters redistributed
386
+ in each. The parent subcluster of that node is removed and two new
387
+ subclusters are added as parents of the 2 split nodes.
388
+
389
+ n_clusters : int, instance of sklearn.cluster model or None, default=3
390
+ Number of clusters after the final clustering step, which treats the
391
+ subclusters from the leaves as new samples.
392
+
393
+ - `None` : the final clustering step is not performed and the
394
+ subclusters are returned as they are.
395
+
396
+ - :mod:`sklearn.cluster` Estimator : If a model is provided, the model
397
+ is fit treating the subclusters as new samples and the initial data
398
+ is mapped to the label of the closest subcluster.
399
+
400
+ - `int` : the model fit is :class:`AgglomerativeClustering` with
401
+ `n_clusters` set to be equal to the int.
402
+
403
+ compute_labels : bool, default=True
404
+ Whether or not to compute labels for each fit.
405
+
406
+ Attributes
407
+ ----------
408
+ root_ : _CFNode
409
+ Root of the CFTree.
410
+
411
+ dummy_leaf_ : _CFNode
412
+ Start pointer to all the leaves.
413
+
414
+ subcluster_centers_ : ndarray
415
+ Centroids of all subclusters read directly from the leaves.
416
+
417
+ subcluster_labels_ : ndarray
418
+ Labels assigned to the centroids of the subclusters after
419
+ they are clustered globally.
420
+
421
+ labels_ : ndarray of shape (n_samples,)
422
+ Array of labels assigned to the input data.
423
+ if partial_fit is used instead of fit, they are assigned to the
424
+ last batch of data.
425
+
426
+ n_features_in_ : int
427
+ Number of features seen during :term:`fit`.
428
+
429
+ .. versionadded:: 0.24
430
+
431
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
432
+ Names of features seen during :term:`fit`. Defined only when `X`
433
+ has feature names that are all strings.
434
+
435
+ .. versionadded:: 1.0
436
+
437
+ See Also
438
+ --------
439
+ MiniBatchKMeans : Alternative implementation that does incremental updates
440
+ of the centers' positions using mini-batches.
441
+
442
+ Notes
443
+ -----
444
+ The tree data structure consists of nodes with each node consisting of
445
+ a number of subclusters. The maximum number of subclusters in a node
446
+ is determined by the branching factor. Each subcluster maintains a
447
+ linear sum, squared sum and the number of samples in that subcluster.
448
+ In addition, each subcluster can also have a node as its child, if the
449
+ subcluster is not a member of a leaf node.
450
+
451
+ For a new point entering the root, it is merged with the subcluster closest
452
+ to it and the linear sum, squared sum and the number of samples of that
453
+ subcluster are updated. This is done recursively till the properties of
454
+ the leaf node are updated.
455
+
456
+ See :ref:`sphx_glr_auto_examples_cluster_plot_birch_vs_minibatchkmeans.py` for a
457
+ comparison with :class:`~sklearn.cluster.MiniBatchKMeans`.
458
+
459
+ References
460
+ ----------
461
+ * Tian Zhang, Raghu Ramakrishnan, Maron Livny
462
+ BIRCH: An efficient data clustering method for large databases.
463
+ https://www.cs.sfu.ca/CourseCentral/459/han/papers/zhang96.pdf
464
+
465
+ * Roberto Perdisci
466
+ JBirch - Java implementation of BIRCH clustering algorithm
467
+ https://code.google.com/archive/p/jbirch
468
+
469
+ Examples
470
+ --------
471
+ >>> from sklearn.cluster import Birch
472
+ >>> X = [[0, 1], [0.3, 1], [-0.3, 1], [0, -1], [0.3, -1], [-0.3, -1]]
473
+ >>> brc = Birch(n_clusters=None)
474
+ >>> brc.fit(X)
475
+ Birch(n_clusters=None)
476
+ >>> brc.predict(X)
477
+ array([0, 0, 0, 1, 1, 1])
478
+
479
+ For a comparison of the BIRCH clustering algorithm with other clustering algorithms,
480
+ see :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
481
+ """
482
+
483
+ _parameter_constraints: dict = {
484
+ "threshold": [Interval(Real, 0.0, None, closed="neither")],
485
+ "branching_factor": [Interval(Integral, 1, None, closed="neither")],
486
+ "n_clusters": [None, ClusterMixin, Interval(Integral, 1, None, closed="left")],
487
+ "compute_labels": ["boolean"],
488
+ }
489
+
490
+ def __init__(
491
+ self,
492
+ *,
493
+ threshold=0.5,
494
+ branching_factor=50,
495
+ n_clusters=3,
496
+ compute_labels=True,
497
+ ):
498
+ self.threshold = threshold
499
+ self.branching_factor = branching_factor
500
+ self.n_clusters = n_clusters
501
+ self.compute_labels = compute_labels
502
+
503
+ @_fit_context(prefer_skip_nested_validation=True)
504
+ def fit(self, X, y=None):
505
+ """
506
+ Build a CF Tree for the input data.
507
+
508
+ Parameters
509
+ ----------
510
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
511
+ Input data.
512
+
513
+ y : Ignored
514
+ Not used, present here for API consistency by convention.
515
+
516
+ Returns
517
+ -------
518
+ self
519
+ Fitted estimator.
520
+ """
521
+ return self._fit(X, partial=False)
522
+
523
+ def _fit(self, X, partial):
524
+ has_root = getattr(self, "root_", None)
525
+ first_call = not (partial and has_root)
526
+
527
+ X = validate_data(
528
+ self,
529
+ X,
530
+ accept_sparse="csr",
531
+ reset=first_call,
532
+ dtype=[np.float64, np.float32],
533
+ )
534
+ threshold = self.threshold
535
+ branching_factor = self.branching_factor
536
+
537
+ n_samples, n_features = X.shape
538
+
539
+ # If partial_fit is called for the first time or fit is called, we
540
+ # start a new tree.
541
+ if first_call:
542
+ # The first root is the leaf. Manipulate this object throughout.
543
+ self.root_ = _CFNode(
544
+ threshold=threshold,
545
+ branching_factor=branching_factor,
546
+ is_leaf=True,
547
+ n_features=n_features,
548
+ dtype=X.dtype,
549
+ )
550
+
551
+ # To enable getting back subclusters.
552
+ self.dummy_leaf_ = _CFNode(
553
+ threshold=threshold,
554
+ branching_factor=branching_factor,
555
+ is_leaf=True,
556
+ n_features=n_features,
557
+ dtype=X.dtype,
558
+ )
559
+ self.dummy_leaf_.next_leaf_ = self.root_
560
+ self.root_.prev_leaf_ = self.dummy_leaf_
561
+
562
+ # Cannot vectorize. Enough to convince to use cython.
563
+ if not sparse.issparse(X):
564
+ iter_func = iter
565
+ else:
566
+ iter_func = _iterate_sparse_X
567
+
568
+ for sample in iter_func(X):
569
+ subcluster = _CFSubcluster(linear_sum=sample)
570
+ split = self.root_.insert_cf_subcluster(subcluster)
571
+
572
+ if split:
573
+ new_subcluster1, new_subcluster2 = _split_node(
574
+ self.root_, threshold, branching_factor
575
+ )
576
+ del self.root_
577
+ self.root_ = _CFNode(
578
+ threshold=threshold,
579
+ branching_factor=branching_factor,
580
+ is_leaf=False,
581
+ n_features=n_features,
582
+ dtype=X.dtype,
583
+ )
584
+ self.root_.append_subcluster(new_subcluster1)
585
+ self.root_.append_subcluster(new_subcluster2)
586
+
587
+ centroids = np.concatenate([leaf.centroids_ for leaf in self._get_leaves()])
588
+ self.subcluster_centers_ = centroids
589
+ self._n_features_out = self.subcluster_centers_.shape[0]
590
+
591
+ self._global_clustering(X)
592
+ return self
593
+
594
+ def _get_leaves(self):
595
+ """
596
+ Retrieve the leaves of the CF Node.
597
+
598
+ Returns
599
+ -------
600
+ leaves : list of shape (n_leaves,)
601
+ List of the leaf nodes.
602
+ """
603
+ leaf_ptr = self.dummy_leaf_.next_leaf_
604
+ leaves = []
605
+ while leaf_ptr is not None:
606
+ leaves.append(leaf_ptr)
607
+ leaf_ptr = leaf_ptr.next_leaf_
608
+ return leaves
609
+
610
+ @_fit_context(prefer_skip_nested_validation=True)
611
+ def partial_fit(self, X=None, y=None):
612
+ """
613
+ Online learning. Prevents rebuilding of CFTree from scratch.
614
+
615
+ Parameters
616
+ ----------
617
+ X : {array-like, sparse matrix} of shape (n_samples, n_features), \
618
+ default=None
619
+ Input data. If X is not provided, only the global clustering
620
+ step is done.
621
+
622
+ y : Ignored
623
+ Not used, present here for API consistency by convention.
624
+
625
+ Returns
626
+ -------
627
+ self
628
+ Fitted estimator.
629
+ """
630
+ if X is None:
631
+ # Perform just the final global clustering step.
632
+ self._global_clustering()
633
+ return self
634
+ else:
635
+ return self._fit(X, partial=True)
636
+
637
+ def predict(self, X):
638
+ """
639
+ Predict data using the ``centroids_`` of subclusters.
640
+
641
+ Avoid computation of the row norms of X.
642
+
643
+ Parameters
644
+ ----------
645
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
646
+ Input data.
647
+
648
+ Returns
649
+ -------
650
+ labels : ndarray of shape(n_samples,)
651
+ Labelled data.
652
+ """
653
+ check_is_fitted(self)
654
+ X = validate_data(self, X, accept_sparse="csr", reset=False)
655
+ return self._predict(X)
656
+
657
+ def _predict(self, X):
658
+ """Predict data using the ``centroids_`` of subclusters."""
659
+ kwargs = {"Y_norm_squared": self._subcluster_norms}
660
+
661
+ with config_context(assume_finite=True):
662
+ argmin = pairwise_distances_argmin(
663
+ X, self.subcluster_centers_, metric_kwargs=kwargs
664
+ )
665
+ return self.subcluster_labels_[argmin]
666
+
667
+ def transform(self, X):
668
+ """
669
+ Transform X into subcluster centroids dimension.
670
+
671
+ Each dimension represents the distance from the sample point to each
672
+ cluster centroid.
673
+
674
+ Parameters
675
+ ----------
676
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
677
+ Input data.
678
+
679
+ Returns
680
+ -------
681
+ X_trans : {array-like, sparse matrix} of shape (n_samples, n_clusters)
682
+ Transformed data.
683
+ """
684
+ check_is_fitted(self)
685
+ X = validate_data(self, X, accept_sparse="csr", reset=False)
686
+ with config_context(assume_finite=True):
687
+ return euclidean_distances(X, self.subcluster_centers_)
688
+
689
+ def _global_clustering(self, X=None):
690
+ """
691
+ Global clustering for the subclusters obtained after fitting
692
+ """
693
+ clusterer = self.n_clusters
694
+ centroids = self.subcluster_centers_
695
+ compute_labels = (X is not None) and self.compute_labels
696
+
697
+ # Preprocessing for the global clustering.
698
+ not_enough_centroids = False
699
+ if isinstance(clusterer, Integral):
700
+ clusterer = AgglomerativeClustering(n_clusters=self.n_clusters)
701
+ # There is no need to perform the global clustering step.
702
+ if len(centroids) < self.n_clusters:
703
+ not_enough_centroids = True
704
+
705
+ # To use in predict to avoid recalculation.
706
+ self._subcluster_norms = row_norms(self.subcluster_centers_, squared=True)
707
+
708
+ if clusterer is None or not_enough_centroids:
709
+ self.subcluster_labels_ = np.arange(len(centroids))
710
+ if not_enough_centroids:
711
+ warnings.warn(
712
+ "Number of subclusters found (%d) by BIRCH is less "
713
+ "than (%d). Decrease the threshold."
714
+ % (len(centroids), self.n_clusters),
715
+ ConvergenceWarning,
716
+ )
717
+ else:
718
+ # The global clustering step that clusters the subclusters of
719
+ # the leaves. It assumes the centroids of the subclusters as
720
+ # samples and finds the final centroids.
721
+ self.subcluster_labels_ = clusterer.fit_predict(self.subcluster_centers_)
722
+
723
+ if compute_labels:
724
+ self.labels_ = self._predict(X)
725
+
726
+ def __sklearn_tags__(self):
727
+ tags = super().__sklearn_tags__()
728
+ tags.transformer_tags.preserves_dtype = ["float64", "float32"]
729
+ tags.input_tags.sparse = True
730
+ return tags
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_bisect_k_means.py ADDED
@@ -0,0 +1,543 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Bisecting K-means clustering."""
2
+
3
+ # Authors: The scikit-learn developers
4
+ # SPDX-License-Identifier: BSD-3-Clause
5
+
6
+ import warnings
7
+
8
+ import numpy as np
9
+ import scipy.sparse as sp
10
+
11
+ from sklearn.base import _fit_context
12
+ from sklearn.cluster._k_means_common import _inertia_dense, _inertia_sparse
13
+ from sklearn.cluster._kmeans import (
14
+ _BaseKMeans,
15
+ _kmeans_single_elkan,
16
+ _kmeans_single_lloyd,
17
+ _labels_inertia_threadpool_limit,
18
+ )
19
+ from sklearn.utils._openmp_helpers import _openmp_effective_n_threads
20
+ from sklearn.utils._param_validation import Integral, Interval, StrOptions
21
+ from sklearn.utils.extmath import row_norms
22
+ from sklearn.utils.validation import (
23
+ _check_sample_weight,
24
+ check_is_fitted,
25
+ check_random_state,
26
+ validate_data,
27
+ )
28
+
29
+
30
+ class _BisectingTree:
31
+ """Tree structure representing the hierarchical clusters of BisectingKMeans."""
32
+
33
+ def __init__(self, center, indices, score):
34
+ """Create a new cluster node in the tree.
35
+
36
+ The node holds the center of this cluster and the indices of the data points
37
+ that belong to it.
38
+ """
39
+ self.center = center
40
+ self.indices = indices
41
+ self.score = score
42
+
43
+ self.left = None
44
+ self.right = None
45
+
46
+ def split(self, labels, centers, scores):
47
+ """Split the cluster node into two subclusters."""
48
+ self.left = _BisectingTree(
49
+ indices=self.indices[labels == 0], center=centers[0], score=scores[0]
50
+ )
51
+ self.right = _BisectingTree(
52
+ indices=self.indices[labels == 1], center=centers[1], score=scores[1]
53
+ )
54
+
55
+ # reset the indices attribute to save memory
56
+ self.indices = None
57
+
58
+ def get_cluster_to_bisect(self):
59
+ """Return the cluster node to bisect next.
60
+
61
+ It's based on the score of the cluster, which can be either the number of
62
+ data points assigned to that cluster or the inertia of that cluster
63
+ (see `bisecting_strategy` for details).
64
+ """
65
+ max_score = None
66
+
67
+ for cluster_leaf in self.iter_leaves():
68
+ if max_score is None or cluster_leaf.score > max_score:
69
+ max_score = cluster_leaf.score
70
+ best_cluster_leaf = cluster_leaf
71
+
72
+ return best_cluster_leaf
73
+
74
+ def iter_leaves(self):
75
+ """Iterate over all the cluster leaves in the tree."""
76
+ if self.left is None:
77
+ yield self
78
+ else:
79
+ yield from self.left.iter_leaves()
80
+ yield from self.right.iter_leaves()
81
+
82
+
83
+ class BisectingKMeans(_BaseKMeans):
84
+ """Bisecting K-Means clustering.
85
+
86
+ Read more in the :ref:`User Guide <bisect_k_means>`.
87
+
88
+ .. versionadded:: 1.1
89
+
90
+ Parameters
91
+ ----------
92
+ n_clusters : int, default=8
93
+ The number of clusters to form as well as the number of
94
+ centroids to generate.
95
+
96
+ init : {'k-means++', 'random'} or callable, default='random'
97
+ Method for initialization:
98
+
99
+ 'k-means++' : selects initial cluster centers for k-mean
100
+ clustering in a smart way to speed up convergence. See section
101
+ Notes in k_init for more details.
102
+
103
+ 'random': choose `n_clusters` observations (rows) at random from data
104
+ for the initial centroids.
105
+
106
+ If a callable is passed, it should take arguments X, n_clusters and a
107
+ random state and return an initialization.
108
+
109
+ n_init : int, default=1
110
+ Number of time the inner k-means algorithm will be run with different
111
+ centroid seeds in each bisection.
112
+ That will result producing for each bisection best output of n_init
113
+ consecutive runs in terms of inertia.
114
+
115
+ random_state : int, RandomState instance or None, default=None
116
+ Determines random number generation for centroid initialization
117
+ in inner K-Means. Use an int to make the randomness deterministic.
118
+ See :term:`Glossary <random_state>`.
119
+
120
+ max_iter : int, default=300
121
+ Maximum number of iterations of the inner k-means algorithm at each
122
+ bisection.
123
+
124
+ verbose : int, default=0
125
+ Verbosity mode.
126
+
127
+ tol : float, default=1e-4
128
+ Relative tolerance with regards to Frobenius norm of the difference
129
+ in the cluster centers of two consecutive iterations to declare
130
+ convergence. Used in inner k-means algorithm at each bisection to pick
131
+ best possible clusters.
132
+
133
+ copy_x : bool, default=True
134
+ When pre-computing distances it is more numerically accurate to center
135
+ the data first. If copy_x is True (default), then the original data is
136
+ not modified. If False, the original data is modified, and put back
137
+ before the function returns, but small numerical differences may be
138
+ introduced by subtracting and then adding the data mean. Note that if
139
+ the original data is not C-contiguous, a copy will be made even if
140
+ copy_x is False. If the original data is sparse, but not in CSR format,
141
+ a copy will be made even if copy_x is False.
142
+
143
+ algorithm : {"lloyd", "elkan"}, default="lloyd"
144
+ Inner K-means algorithm used in bisection.
145
+ The classical EM-style algorithm is `"lloyd"`.
146
+ The `"elkan"` variation can be more efficient on some datasets with
147
+ well-defined clusters, by using the triangle inequality. However it's
148
+ more memory intensive due to the allocation of an extra array of shape
149
+ `(n_samples, n_clusters)`.
150
+
151
+ bisecting_strategy : {"biggest_inertia", "largest_cluster"},\
152
+ default="biggest_inertia"
153
+ Defines how bisection should be performed:
154
+
155
+ - "biggest_inertia" means that BisectingKMeans will always check
156
+ all calculated cluster for cluster with biggest SSE
157
+ (Sum of squared errors) and bisect it. This approach concentrates on
158
+ precision, but may be costly in terms of execution time (especially for
159
+ larger amount of data points).
160
+
161
+ - "largest_cluster" - BisectingKMeans will always split cluster with
162
+ largest amount of points assigned to it from all clusters
163
+ previously calculated. That should work faster than picking by SSE
164
+ ('biggest_inertia') and may produce similar results in most cases.
165
+
166
+ Attributes
167
+ ----------
168
+ cluster_centers_ : ndarray of shape (n_clusters, n_features)
169
+ Coordinates of cluster centers. If the algorithm stops before fully
170
+ converging (see ``tol`` and ``max_iter``), these will not be
171
+ consistent with ``labels_``.
172
+
173
+ labels_ : ndarray of shape (n_samples,)
174
+ Labels of each point.
175
+
176
+ inertia_ : float
177
+ Sum of squared distances of samples to their closest cluster center,
178
+ weighted by the sample weights if provided.
179
+
180
+ n_features_in_ : int
181
+ Number of features seen during :term:`fit`.
182
+
183
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
184
+ Names of features seen during :term:`fit`. Defined only when `X`
185
+ has feature names that are all strings.
186
+
187
+ See Also
188
+ --------
189
+ KMeans : Original implementation of K-Means algorithm.
190
+
191
+ Notes
192
+ -----
193
+ It might be inefficient when n_cluster is less than 3, due to unnecessary
194
+ calculations for that case.
195
+
196
+ Examples
197
+ --------
198
+ >>> from sklearn.cluster import BisectingKMeans
199
+ >>> import numpy as np
200
+ >>> X = np.array([[1, 1], [10, 1], [3, 1],
201
+ ... [10, 0], [2, 1], [10, 2],
202
+ ... [10, 8], [10, 9], [10, 10]])
203
+ >>> bisect_means = BisectingKMeans(n_clusters=3, random_state=0).fit(X)
204
+ >>> bisect_means.labels_
205
+ array([0, 2, 0, 2, 0, 2, 1, 1, 1], dtype=int32)
206
+ >>> bisect_means.predict([[0, 0], [12, 3]])
207
+ array([0, 2], dtype=int32)
208
+ >>> bisect_means.cluster_centers_
209
+ array([[ 2., 1.],
210
+ [10., 9.],
211
+ [10., 1.]])
212
+
213
+ For a comparison between BisectingKMeans and K-Means refer to example
214
+ :ref:`sphx_glr_auto_examples_cluster_plot_bisect_kmeans.py`.
215
+ """
216
+
217
+ _parameter_constraints: dict = {
218
+ **_BaseKMeans._parameter_constraints,
219
+ "init": [StrOptions({"k-means++", "random"}), callable],
220
+ "n_init": [Interval(Integral, 1, None, closed="left")],
221
+ "copy_x": ["boolean"],
222
+ "algorithm": [StrOptions({"lloyd", "elkan"})],
223
+ "bisecting_strategy": [StrOptions({"biggest_inertia", "largest_cluster"})],
224
+ }
225
+
226
+ def __init__(
227
+ self,
228
+ n_clusters=8,
229
+ *,
230
+ init="random",
231
+ n_init=1,
232
+ random_state=None,
233
+ max_iter=300,
234
+ verbose=0,
235
+ tol=1e-4,
236
+ copy_x=True,
237
+ algorithm="lloyd",
238
+ bisecting_strategy="biggest_inertia",
239
+ ):
240
+ super().__init__(
241
+ n_clusters=n_clusters,
242
+ init=init,
243
+ max_iter=max_iter,
244
+ verbose=verbose,
245
+ random_state=random_state,
246
+ tol=tol,
247
+ n_init=n_init,
248
+ )
249
+
250
+ self.copy_x = copy_x
251
+ self.algorithm = algorithm
252
+ self.bisecting_strategy = bisecting_strategy
253
+
254
+ def _warn_mkl_vcomp(self, n_active_threads):
255
+ """Warn when vcomp and mkl are both present"""
256
+ warnings.warn(
257
+ "BisectingKMeans is known to have a memory leak on Windows "
258
+ "with MKL, when there are less chunks than available "
259
+ "threads. You can avoid it by setting the environment"
260
+ f" variable OMP_NUM_THREADS={n_active_threads}."
261
+ )
262
+
263
+ def _inertia_per_cluster(self, X, centers, labels, sample_weight):
264
+ """Calculate the sum of squared errors (inertia) per cluster.
265
+
266
+ Parameters
267
+ ----------
268
+ X : {ndarray, csr_matrix} of shape (n_samples, n_features)
269
+ The input samples.
270
+
271
+ centers : ndarray of shape (n_clusters=2, n_features)
272
+ The cluster centers.
273
+
274
+ labels : ndarray of shape (n_samples,)
275
+ Index of the cluster each sample belongs to.
276
+
277
+ sample_weight : ndarray of shape (n_samples,)
278
+ The weights for each observation in X.
279
+
280
+ Returns
281
+ -------
282
+ inertia_per_cluster : ndarray of shape (n_clusters=2,)
283
+ Sum of squared errors (inertia) for each cluster.
284
+ """
285
+ n_clusters = centers.shape[0] # = 2 since centers comes from a bisection
286
+ _inertia = _inertia_sparse if sp.issparse(X) else _inertia_dense
287
+
288
+ inertia_per_cluster = np.empty(n_clusters)
289
+ for label in range(n_clusters):
290
+ inertia_per_cluster[label] = _inertia(
291
+ X, sample_weight, centers, labels, self._n_threads, single_label=label
292
+ )
293
+
294
+ return inertia_per_cluster
295
+
296
+ def _bisect(self, X, x_squared_norms, sample_weight, cluster_to_bisect):
297
+ """Split a cluster into 2 subsclusters.
298
+
299
+ Parameters
300
+ ----------
301
+ X : {ndarray, csr_matrix} of shape (n_samples, n_features)
302
+ Training instances to cluster.
303
+
304
+ x_squared_norms : ndarray of shape (n_samples,)
305
+ Squared euclidean norm of each data point.
306
+
307
+ sample_weight : ndarray of shape (n_samples,)
308
+ The weights for each observation in X.
309
+
310
+ cluster_to_bisect : _BisectingTree node object
311
+ The cluster node to split.
312
+ """
313
+ X = X[cluster_to_bisect.indices]
314
+ x_squared_norms = x_squared_norms[cluster_to_bisect.indices]
315
+ sample_weight = sample_weight[cluster_to_bisect.indices]
316
+
317
+ best_inertia = None
318
+
319
+ # Split samples in X into 2 clusters.
320
+ # Repeating `n_init` times to obtain best clusters
321
+ for _ in range(self.n_init):
322
+ centers_init = self._init_centroids(
323
+ X,
324
+ x_squared_norms=x_squared_norms,
325
+ init=self.init,
326
+ random_state=self._random_state,
327
+ n_centroids=2,
328
+ sample_weight=sample_weight,
329
+ )
330
+
331
+ labels, inertia, centers, _ = self._kmeans_single(
332
+ X,
333
+ sample_weight,
334
+ centers_init,
335
+ max_iter=self.max_iter,
336
+ verbose=self.verbose,
337
+ tol=self.tol,
338
+ n_threads=self._n_threads,
339
+ )
340
+
341
+ # allow small tolerance on the inertia to accommodate for
342
+ # non-deterministic rounding errors due to parallel computation
343
+ if best_inertia is None or inertia < best_inertia * (1 - 1e-6):
344
+ best_labels = labels
345
+ best_centers = centers
346
+ best_inertia = inertia
347
+
348
+ if self.verbose:
349
+ print(f"New centroids from bisection: {best_centers}")
350
+
351
+ if self.bisecting_strategy == "biggest_inertia":
352
+ scores = self._inertia_per_cluster(
353
+ X, best_centers, best_labels, sample_weight
354
+ )
355
+ else: # bisecting_strategy == "largest_cluster"
356
+ # Using minlength to make sure that we have the counts for both labels even
357
+ # if all samples are labelled 0.
358
+ scores = np.bincount(best_labels, minlength=2)
359
+
360
+ cluster_to_bisect.split(best_labels, best_centers, scores)
361
+
362
+ @_fit_context(prefer_skip_nested_validation=True)
363
+ def fit(self, X, y=None, sample_weight=None):
364
+ """Compute bisecting k-means clustering.
365
+
366
+ Parameters
367
+ ----------
368
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
369
+
370
+ Training instances to cluster.
371
+
372
+ .. note:: The data will be converted to C ordering,
373
+ which will cause a memory copy
374
+ if the given data is not C-contiguous.
375
+
376
+ y : Ignored
377
+ Not used, present here for API consistency by convention.
378
+
379
+ sample_weight : array-like of shape (n_samples,), default=None
380
+ The weights for each observation in X. If None, all observations
381
+ are assigned equal weight. `sample_weight` is not used during
382
+ initialization if `init` is a callable.
383
+
384
+ Returns
385
+ -------
386
+ self
387
+ Fitted estimator.
388
+ """
389
+ X = validate_data(
390
+ self,
391
+ X,
392
+ accept_sparse="csr",
393
+ dtype=[np.float64, np.float32],
394
+ order="C",
395
+ copy=self.copy_x,
396
+ accept_large_sparse=False,
397
+ )
398
+
399
+ self._check_params_vs_input(X)
400
+
401
+ self._random_state = check_random_state(self.random_state)
402
+ sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
403
+ self._n_threads = _openmp_effective_n_threads()
404
+
405
+ if self.algorithm == "lloyd" or self.n_clusters == 1:
406
+ self._kmeans_single = _kmeans_single_lloyd
407
+ self._check_mkl_vcomp(X, X.shape[0])
408
+ else:
409
+ self._kmeans_single = _kmeans_single_elkan
410
+
411
+ # Subtract of mean of X for more accurate distance computations
412
+ if not sp.issparse(X):
413
+ self._X_mean = X.mean(axis=0)
414
+ X -= self._X_mean
415
+
416
+ # Initialize the hierarchical clusters tree
417
+ self._bisecting_tree = _BisectingTree(
418
+ indices=np.arange(X.shape[0]),
419
+ center=X.mean(axis=0),
420
+ score=0,
421
+ )
422
+
423
+ x_squared_norms = row_norms(X, squared=True)
424
+
425
+ for _ in range(self.n_clusters - 1):
426
+ # Chose cluster to bisect
427
+ cluster_to_bisect = self._bisecting_tree.get_cluster_to_bisect()
428
+
429
+ # Split this cluster into 2 subclusters
430
+ self._bisect(X, x_squared_norms, sample_weight, cluster_to_bisect)
431
+
432
+ # Aggregate final labels and centers from the bisecting tree
433
+ self.labels_ = np.full(X.shape[0], -1, dtype=np.int32)
434
+ self.cluster_centers_ = np.empty((self.n_clusters, X.shape[1]), dtype=X.dtype)
435
+
436
+ for i, cluster_node in enumerate(self._bisecting_tree.iter_leaves()):
437
+ self.labels_[cluster_node.indices] = i
438
+ self.cluster_centers_[i] = cluster_node.center
439
+ cluster_node.label = i # label final clusters for future prediction
440
+ cluster_node.indices = None # release memory
441
+
442
+ # Restore original data
443
+ if not sp.issparse(X):
444
+ X += self._X_mean
445
+ self.cluster_centers_ += self._X_mean
446
+
447
+ _inertia = _inertia_sparse if sp.issparse(X) else _inertia_dense
448
+ self.inertia_ = _inertia(
449
+ X, sample_weight, self.cluster_centers_, self.labels_, self._n_threads
450
+ )
451
+
452
+ self._n_features_out = self.cluster_centers_.shape[0]
453
+
454
+ return self
455
+
456
+ def predict(self, X):
457
+ """Predict which cluster each sample in X belongs to.
458
+
459
+ Prediction is made by going down the hierarchical tree
460
+ in searching of closest leaf cluster.
461
+
462
+ In the vector quantization literature, `cluster_centers_` is called
463
+ the code book and each value returned by `predict` is the index of
464
+ the closest code in the code book.
465
+
466
+ Parameters
467
+ ----------
468
+ X : {array-like, sparse matrix} of shape (n_samples, n_features)
469
+ New data to predict.
470
+
471
+ Returns
472
+ -------
473
+ labels : ndarray of shape (n_samples,)
474
+ Index of the cluster each sample belongs to.
475
+ """
476
+ check_is_fitted(self)
477
+
478
+ X = self._check_test_data(X)
479
+ x_squared_norms = row_norms(X, squared=True)
480
+
481
+ # sample weights are unused but necessary in cython helpers
482
+ sample_weight = np.ones_like(x_squared_norms)
483
+
484
+ labels = self._predict_recursive(X, sample_weight, self._bisecting_tree)
485
+
486
+ return labels
487
+
488
+ def _predict_recursive(self, X, sample_weight, cluster_node):
489
+ """Predict recursively by going down the hierarchical tree.
490
+
491
+ Parameters
492
+ ----------
493
+ X : {ndarray, csr_matrix} of shape (n_samples, n_features)
494
+ The data points, currently assigned to `cluster_node`, to predict between
495
+ the subclusters of this node.
496
+
497
+ sample_weight : ndarray of shape (n_samples,)
498
+ The weights for each observation in X.
499
+
500
+ cluster_node : _BisectingTree node object
501
+ The cluster node of the hierarchical tree.
502
+
503
+ Returns
504
+ -------
505
+ labels : ndarray of shape (n_samples,)
506
+ Index of the cluster each sample belongs to.
507
+ """
508
+ if cluster_node.left is None:
509
+ # This cluster has no subcluster. Labels are just the label of the cluster.
510
+ return np.full(X.shape[0], cluster_node.label, dtype=np.int32)
511
+
512
+ # Determine if data points belong to the left or right subcluster
513
+ centers = np.vstack((cluster_node.left.center, cluster_node.right.center))
514
+ if hasattr(self, "_X_mean"):
515
+ centers += self._X_mean
516
+
517
+ cluster_labels = _labels_inertia_threadpool_limit(
518
+ X,
519
+ sample_weight,
520
+ centers,
521
+ self._n_threads,
522
+ return_inertia=False,
523
+ )
524
+ mask = cluster_labels == 0
525
+
526
+ # Compute the labels for each subset of the data points.
527
+ labels = np.full(X.shape[0], -1, dtype=np.int32)
528
+
529
+ labels[mask] = self._predict_recursive(
530
+ X[mask], sample_weight[mask], cluster_node.left
531
+ )
532
+
533
+ labels[~mask] = self._predict_recursive(
534
+ X[~mask], sample_weight[~mask], cluster_node.right
535
+ )
536
+
537
+ return labels
538
+
539
+ def __sklearn_tags__(self):
540
+ tags = super().__sklearn_tags__()
541
+ tags.input_tags.sparse = True
542
+ tags.transformer_tags.preserves_dtype = ["float64", "float32"]
543
+ return tags
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan.py ADDED
@@ -0,0 +1,512 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ DBSCAN: Density-Based Spatial Clustering of Applications with Noise
3
+ """
4
+
5
+ # Authors: The scikit-learn developers
6
+ # SPDX-License-Identifier: BSD-3-Clause
7
+
8
+ import warnings
9
+ from numbers import Integral, Real
10
+
11
+ import numpy as np
12
+ from scipy import sparse
13
+
14
+ from sklearn.base import BaseEstimator, ClusterMixin, _fit_context
15
+ from sklearn.cluster._dbscan_inner import dbscan_inner
16
+ from sklearn.metrics.pairwise import _VALID_METRICS
17
+ from sklearn.neighbors import NearestNeighbors
18
+ from sklearn.utils._param_validation import Interval, StrOptions, validate_params
19
+ from sklearn.utils.validation import _check_sample_weight, validate_data
20
+
21
+
22
+ @validate_params(
23
+ {
24
+ "X": ["array-like", "sparse matrix"],
25
+ "sample_weight": ["array-like", None],
26
+ },
27
+ prefer_skip_nested_validation=False,
28
+ )
29
+ def dbscan(
30
+ X,
31
+ eps=0.5,
32
+ *,
33
+ min_samples=5,
34
+ metric="minkowski",
35
+ metric_params=None,
36
+ algorithm="auto",
37
+ leaf_size=30,
38
+ p=2,
39
+ sample_weight=None,
40
+ n_jobs=None,
41
+ ):
42
+ """Perform DBSCAN clustering from vector array or distance matrix.
43
+
44
+ This function is a wrapper around :class:`~cluster.DBSCAN`, suitable for
45
+ quick, standalone clustering tasks. For estimator-based workflows, where
46
+ estimator attributes or pipeline integration is required, prefer
47
+ :class:`~cluster.DBSCAN`.
48
+
49
+ DBSCAN (Density-Based Spatial Clustering of Applications with Noise) is a
50
+ density-based clustering algorithm that groups together points that are
51
+ closely packed while marking points in low-density regions as outliers.
52
+
53
+ Read more in the :ref:`User Guide <dbscan>`.
54
+
55
+ Parameters
56
+ ----------
57
+ X : {array-like, scipy sparse matrix} of shape (n_samples, n_features) or \
58
+ (n_samples, n_samples)
59
+ A feature array, or array of distances between samples if
60
+ ``metric='precomputed'``. When using precomputed distances, X must
61
+ be a square symmetric matrix.
62
+
63
+ eps : float, default=0.5
64
+ The maximum distance between two samples for one to be considered
65
+ as in the neighborhood of the other. This is not a maximum bound
66
+ on the distances of points within a cluster. This is the most
67
+ important DBSCAN parameter to choose appropriately for your data set
68
+ and distance function. Smaller values result in more clusters,
69
+ while larger values result in fewer, larger clusters.
70
+
71
+ min_samples : int, default=5
72
+ The number of samples (or total weight) in a neighborhood for a point
73
+ to be considered as a core point. This includes the point itself.
74
+ Higher values yield fewer, denser clusters, while lower values yield
75
+ more, sparser clusters.
76
+
77
+ metric : str or callable, default='minkowski'
78
+ The metric to use when calculating distance between instances in a
79
+ feature array. If metric is a string or callable, it must be one of
80
+ the options allowed by :func:`sklearn.metrics.pairwise_distances` for
81
+ its metric parameter.
82
+ If metric is "precomputed", X is assumed to be a distance matrix and
83
+ must be square during fit.
84
+ X may be a :term:`sparse graph <sparse graph>`,
85
+ in which case only "nonzero" elements may be considered neighbors.
86
+
87
+ metric_params : dict, default=None
88
+ Additional keyword arguments for the metric function.
89
+
90
+ .. versionadded:: 0.19
91
+
92
+ algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
93
+ The algorithm to be used by the NearestNeighbors module
94
+ to compute pointwise distances and find nearest neighbors.
95
+ 'auto' will attempt to decide the most appropriate algorithm
96
+ based on the values passed to :meth:`fit` method.
97
+ See :class:`~sklearn.neighbors.NearestNeighbors` documentation for
98
+ details.
99
+
100
+ leaf_size : int, default=30
101
+ Leaf size passed to BallTree or cKDTree. This can affect the speed
102
+ of the construction and query, as well as the memory required
103
+ to store the tree. The optimal value depends
104
+ on the nature of the problem. Generally, smaller leaf sizes
105
+ lead to faster queries but slower construction.
106
+
107
+ p : float, default=2
108
+ Power parameter for the Minkowski metric. When p = 1, this is equivalent
109
+ to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2.
110
+ For arbitrary p, minkowski_distance (l_p) is used. This parameter is expected
111
+ to be positive.
112
+
113
+ sample_weight : array-like of shape (n_samples,), default=None
114
+ Weight of each sample, such that a sample with a weight of at least
115
+ ``min_samples`` is by itself a core sample; a sample with negative
116
+ weight may inhibit its eps-neighbor from being core.
117
+ Note that weights are absolute, and default to 1.
118
+
119
+ n_jobs : int, default=None
120
+ The number of parallel jobs to run for neighbors search. ``None`` means
121
+ 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means
122
+ using all processors. See :term:`Glossary <n_jobs>` for more details.
123
+ If precomputed distances are used, parallel execution is not available
124
+ and thus n_jobs will have no effect.
125
+
126
+ Returns
127
+ -------
128
+ core_samples : ndarray of shape (n_core_samples,)
129
+ Indices of core samples.
130
+
131
+ labels : ndarray of shape (n_samples,)
132
+ Cluster labels for each point. Noisy samples are given the label -1.
133
+ Non-negative integers indicate cluster membership.
134
+
135
+ See Also
136
+ --------
137
+ DBSCAN : An estimator interface for this clustering algorithm.
138
+ OPTICS : A similar estimator interface clustering at multiple values of
139
+ eps. Our implementation is optimized for memory usage.
140
+
141
+ Notes
142
+ -----
143
+ For an example, see :ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py`.
144
+
145
+ This implementation bulk-computes all neighborhood queries, which increases
146
+ the memory complexity to O(n.d) where d is the average number of neighbors,
147
+ while original DBSCAN had memory complexity O(n). It may attract a higher
148
+ memory complexity when querying these nearest neighborhoods, depending
149
+ on the ``algorithm``.
150
+
151
+ One way to avoid the query complexity is to pre-compute sparse
152
+ neighborhoods in chunks using
153
+ :func:`NearestNeighbors.radius_neighbors_graph
154
+ <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>` with
155
+ ``mode='distance'``, then using ``metric='precomputed'`` here.
156
+
157
+ Another way to reduce memory and computation time is to remove
158
+ (near-)duplicate points and use ``sample_weight`` instead.
159
+
160
+ :class:`~sklearn.cluster.OPTICS` provides a similar clustering with lower
161
+ memory usage.
162
+
163
+ References
164
+ ----------
165
+ Ester, M., H. P. Kriegel, J. Sander, and X. Xu, `"A Density-Based
166
+ Algorithm for Discovering Clusters in Large Spatial Databases with Noise"
167
+ <https://www.dbs.ifi.lmu.de/Publikationen/Papers/KDD-96.final.frame.pdf>`_.
168
+ In: Proceedings of the 2nd International Conference on Knowledge Discovery
169
+ and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
170
+
171
+ Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017).
172
+ :doi:`"DBSCAN revisited, revisited: why and how you should (still) use DBSCAN."
173
+ <10.1145/3068335>`
174
+ ACM Transactions on Database Systems (TODS), 42(3), 19.
175
+
176
+ Examples
177
+ --------
178
+ >>> from sklearn.cluster import dbscan
179
+ >>> X = [[1, 2], [2, 2], [2, 3], [8, 7], [8, 8], [25, 80]]
180
+ >>> core_samples, labels = dbscan(X, eps=3, min_samples=2)
181
+ >>> core_samples
182
+ array([0, 1, 2, 3, 4])
183
+ >>> labels
184
+ array([ 0, 0, 0, 1, 1, -1])
185
+ """
186
+
187
+ est = DBSCAN(
188
+ eps=eps,
189
+ min_samples=min_samples,
190
+ metric=metric,
191
+ metric_params=metric_params,
192
+ algorithm=algorithm,
193
+ leaf_size=leaf_size,
194
+ p=p,
195
+ n_jobs=n_jobs,
196
+ )
197
+ est.fit(X, sample_weight=sample_weight)
198
+ return est.core_sample_indices_, est.labels_
199
+
200
+
201
+ class DBSCAN(ClusterMixin, BaseEstimator):
202
+ """Perform DBSCAN clustering from vector array or distance matrix.
203
+
204
+ DBSCAN - Density-Based Spatial Clustering of Applications with Noise.
205
+ Finds core samples of high density and expands clusters from them.
206
+ This algorithm is particularly good for data which contains clusters of
207
+ similar density and can find clusters of arbitrary shape.
208
+
209
+ Unlike K-means, DBSCAN does not require specifying the number of clusters
210
+ in advance and can identify outliers as noise points.
211
+
212
+ This implementation has a worst case memory complexity of :math:`O({n}^2)`,
213
+ which can occur when the `eps` param is large and `min_samples` is low,
214
+ while the original DBSCAN only uses linear memory.
215
+ For further details, see the Notes below.
216
+
217
+ Read more in the :ref:`User Guide <dbscan>`.
218
+
219
+ Parameters
220
+ ----------
221
+ eps : float, default=0.5
222
+ The maximum distance between two samples for one to be considered
223
+ as in the neighborhood of the other. This is not a maximum bound
224
+ on the distances of points within a cluster. This is the most
225
+ important DBSCAN parameter to choose appropriately for your data set
226
+ and distance function. Smaller values generally lead to more clusters.
227
+
228
+ min_samples : int, default=5
229
+ The number of samples (or total weight) in a neighborhood for a point to
230
+ be considered as a core point. This includes the point itself. If
231
+ `min_samples` is set to a higher value, DBSCAN will find denser clusters,
232
+ whereas if it is set to a lower value, the found clusters will be more
233
+ sparse.
234
+
235
+ metric : str, or callable, default='euclidean'
236
+ The metric to use when calculating distance between instances in a
237
+ feature array. If metric is a string or callable, it must be one of
238
+ the options allowed by :func:`sklearn.metrics.pairwise_distances` for
239
+ its metric parameter.
240
+ If metric is "precomputed", X is assumed to be a distance matrix and
241
+ must be square. X may be a :term:`sparse graph`, in which
242
+ case only "nonzero" elements may be considered neighbors for DBSCAN.
243
+
244
+ .. versionadded:: 0.17
245
+ metric *precomputed* to accept precomputed sparse matrix.
246
+
247
+ metric_params : dict, default=None
248
+ Additional keyword arguments for the metric function.
249
+
250
+ .. versionadded:: 0.19
251
+
252
+ algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
253
+ The algorithm to be used by the NearestNeighbors module
254
+ to compute pointwise distances and find nearest neighbors.
255
+ 'auto' will attempt to decide the most appropriate algorithm
256
+ based on the values passed to :meth:`fit` method.
257
+ See :class:`~sklearn.neighbors.NearestNeighbors` documentation for
258
+ details.
259
+
260
+ leaf_size : int, default=30
261
+ Leaf size passed to BallTree or cKDTree. This can affect the speed
262
+ of the construction and query, as well as the memory required
263
+ to store the tree. The optimal value depends
264
+ on the nature of the problem.
265
+
266
+ p : float, default=None
267
+ The power of the Minkowski metric to be used to calculate distance
268
+ between points. If None, then ``p=2`` (equivalent to the Euclidean
269
+ distance). When p=1, this is equivalent to Manhattan distance.
270
+
271
+ n_jobs : int, default=None
272
+ The number of parallel jobs to run.
273
+ ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
274
+ ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
275
+ for more details.
276
+
277
+ Attributes
278
+ ----------
279
+ core_sample_indices_ : ndarray of shape (n_core_samples,)
280
+ Indices of core samples.
281
+
282
+ components_ : ndarray of shape (n_core_samples, n_features)
283
+ Copy of each core sample found by training.
284
+
285
+ labels_ : ndarray of shape (n_samples,)
286
+ Cluster labels for each point in the dataset given to fit().
287
+ Noisy samples are given the label -1. Non-negative integers
288
+ indicate cluster membership.
289
+
290
+ n_features_in_ : int
291
+ Number of features seen during :term:`fit`.
292
+
293
+ .. versionadded:: 0.24
294
+
295
+ feature_names_in_ : ndarray of shape (`n_features_in_`,)
296
+ Names of features seen during :term:`fit`. Defined only when `X`
297
+ has feature names that are all strings.
298
+
299
+ .. versionadded:: 1.0
300
+
301
+ See Also
302
+ --------
303
+ OPTICS : A similar clustering at multiple values of eps. Our implementation
304
+ is optimized for memory usage.
305
+
306
+ Notes
307
+ -----
308
+ This implementation bulk-computes all neighborhood queries, which increases
309
+ the memory complexity to O(n.d) where d is the average number of neighbors,
310
+ while original DBSCAN had memory complexity O(n). It may attract a higher
311
+ memory complexity when querying these nearest neighborhoods, depending
312
+ on the ``algorithm``.
313
+
314
+ One way to avoid the query complexity is to pre-compute sparse
315
+ neighborhoods in chunks using
316
+ :func:`NearestNeighbors.radius_neighbors_graph
317
+ <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>` with
318
+ ``mode='distance'``, then using ``metric='precomputed'`` here.
319
+
320
+ Another way to reduce memory and computation time is to remove
321
+ (near-)duplicate points and use ``sample_weight`` instead.
322
+
323
+ :class:`~sklearn.cluster.OPTICS` provides a similar clustering with lower memory
324
+ usage.
325
+
326
+ References
327
+ ----------
328
+ Ester, M., H. P. Kriegel, J. Sander, and X. Xu, `"A Density-Based
329
+ Algorithm for Discovering Clusters in Large Spatial Databases with Noise"
330
+ <https://www.dbs.ifi.lmu.de/Publikationen/Papers/KDD-96.final.frame.pdf>`_.
331
+ In: Proceedings of the 2nd International Conference on Knowledge Discovery
332
+ and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
333
+
334
+ Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017).
335
+ :doi:`"DBSCAN revisited, revisited: why and how you should (still) use DBSCAN."
336
+ <10.1145/3068335>`
337
+ ACM Transactions on Database Systems (TODS), 42(3), 19.
338
+
339
+ Examples
340
+ --------
341
+ >>> from sklearn.cluster import DBSCAN
342
+ >>> import numpy as np
343
+ >>> X = np.array([[1, 2], [2, 2], [2, 3],
344
+ ... [8, 7], [8, 8], [25, 80]])
345
+ >>> clustering = DBSCAN(eps=3, min_samples=2).fit(X)
346
+ >>> clustering.labels_
347
+ array([ 0, 0, 0, 1, 1, -1])
348
+ >>> clustering
349
+ DBSCAN(eps=3, min_samples=2)
350
+
351
+ For an example, see
352
+ :ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py`.
353
+
354
+ For a comparison of DBSCAN with other clustering algorithms, see
355
+ :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
356
+ """
357
+
358
+ _parameter_constraints: dict = {
359
+ "eps": [Interval(Real, 0.0, None, closed="neither")],
360
+ "min_samples": [Interval(Integral, 1, None, closed="left")],
361
+ "metric": [
362
+ StrOptions(set(_VALID_METRICS) | {"precomputed"}),
363
+ callable,
364
+ ],
365
+ "metric_params": [dict, None],
366
+ "algorithm": [StrOptions({"auto", "ball_tree", "kd_tree", "brute"})],
367
+ "leaf_size": [Interval(Integral, 1, None, closed="left")],
368
+ "p": [Interval(Real, 0.0, None, closed="left"), None],
369
+ "n_jobs": [Integral, None],
370
+ }
371
+
372
+ def __init__(
373
+ self,
374
+ eps=0.5,
375
+ *,
376
+ min_samples=5,
377
+ metric="euclidean",
378
+ metric_params=None,
379
+ algorithm="auto",
380
+ leaf_size=30,
381
+ p=None,
382
+ n_jobs=None,
383
+ ):
384
+ self.eps = eps
385
+ self.min_samples = min_samples
386
+ self.metric = metric
387
+ self.metric_params = metric_params
388
+ self.algorithm = algorithm
389
+ self.leaf_size = leaf_size
390
+ self.p = p
391
+ self.n_jobs = n_jobs
392
+
393
+ @_fit_context(
394
+ # DBSCAN.metric is not validated yet
395
+ prefer_skip_nested_validation=False
396
+ )
397
+ def fit(self, X, y=None, sample_weight=None):
398
+ """Perform DBSCAN clustering from features, or distance matrix.
399
+
400
+ Parameters
401
+ ----------
402
+ X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
403
+ (n_samples, n_samples)
404
+ Training instances to cluster, or distances between instances if
405
+ ``metric='precomputed'``. If a sparse matrix is provided, it will
406
+ be converted into a sparse ``csr_matrix``.
407
+
408
+ y : Ignored
409
+ Not used, present here for API consistency by convention.
410
+
411
+ sample_weight : array-like of shape (n_samples,), default=None
412
+ Weight of each sample, such that a sample with a weight of at least
413
+ ``min_samples`` is by itself a core sample; a sample with a
414
+ negative weight may inhibit its eps-neighbor from being core.
415
+ Note that weights are absolute, and default to 1.
416
+
417
+ Returns
418
+ -------
419
+ self : object
420
+ Returns a fitted instance of self.
421
+ """
422
+ X = validate_data(self, X, accept_sparse="csr")
423
+
424
+ if sample_weight is not None:
425
+ sample_weight = _check_sample_weight(sample_weight, X)
426
+
427
+ # Calculate neighborhood for all samples. This leaves the original
428
+ # point in, which needs to be considered later (i.e. point i is in the
429
+ # neighborhood of point i. While True, its useless information)
430
+ if self.metric == "precomputed" and sparse.issparse(X):
431
+ # set the diagonal to explicit values, as a point is its own
432
+ # neighbor
433
+ X = X.copy() # copy to avoid in-place modification
434
+ with warnings.catch_warnings():
435
+ warnings.simplefilter("ignore", sparse.SparseEfficiencyWarning)
436
+ X.setdiag(X.diagonal())
437
+
438
+ neighbors_model = NearestNeighbors(
439
+ radius=self.eps,
440
+ algorithm=self.algorithm,
441
+ leaf_size=self.leaf_size,
442
+ metric=self.metric,
443
+ metric_params=self.metric_params,
444
+ p=self.p,
445
+ n_jobs=self.n_jobs,
446
+ )
447
+ neighbors_model.fit(X)
448
+ # This has worst case O(n^2) memory complexity
449
+ neighborhoods = neighbors_model.radius_neighbors(X, return_distance=False)
450
+
451
+ if sample_weight is None:
452
+ n_neighbors = np.array([len(neighbors) for neighbors in neighborhoods])
453
+ else:
454
+ n_neighbors = np.array(
455
+ [np.sum(sample_weight[neighbors]) for neighbors in neighborhoods]
456
+ )
457
+
458
+ # Initially, all samples are noise.
459
+ labels = np.full(X.shape[0], -1, dtype=np.intp)
460
+
461
+ # A list of all core samples found.
462
+ core_samples = np.asarray(n_neighbors >= self.min_samples, dtype=np.uint8)
463
+ dbscan_inner(core_samples, neighborhoods, labels)
464
+
465
+ self.core_sample_indices_ = np.where(core_samples)[0]
466
+ self.labels_ = labels
467
+
468
+ if len(self.core_sample_indices_):
469
+ # fix for scipy sparse indexing issue
470
+ self.components_ = X[self.core_sample_indices_].copy()
471
+ else:
472
+ # no core samples
473
+ self.components_ = np.empty((0, X.shape[1]))
474
+ return self
475
+
476
+ def fit_predict(self, X, y=None, sample_weight=None):
477
+ """Compute clusters from a data or distance matrix and predict labels.
478
+
479
+ This method fits the model and returns the cluster labels in a single step.
480
+ It is equivalent to calling fit(X).labels_.
481
+
482
+ Parameters
483
+ ----------
484
+ X : {array-like, sparse matrix} of shape (n_samples, n_features), or \
485
+ (n_samples, n_samples)
486
+ Training instances to cluster, or distances between instances if
487
+ ``metric='precomputed'``. If a sparse matrix is provided, it will
488
+ be converted into a sparse ``csr_matrix``.
489
+
490
+ y : Ignored
491
+ Not used, present here for API consistency by convention.
492
+
493
+ sample_weight : array-like of shape (n_samples,), default=None
494
+ Weight of each sample, such that a sample with a weight of at least
495
+ ``min_samples`` is by itself a core sample; a sample with a
496
+ negative weight may inhibit its eps-neighbor from being core.
497
+ Note that weights are absolute, and default to 1.
498
+
499
+ Returns
500
+ -------
501
+ labels : ndarray of shape (n_samples,)
502
+ Cluster labels. Noisy samples are given the label -1.
503
+ Non-negative integers indicate cluster membership.
504
+ """
505
+ self.fit(X, sample_weight=sample_weight)
506
+ return self.labels_
507
+
508
+ def __sklearn_tags__(self):
509
+ tags = super().__sklearn_tags__()
510
+ tags.input_tags.pairwise = self.metric == "precomputed"
511
+ tags.input_tags.sparse = True
512
+ return tags
loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_dbscan_inner.cp312-win_amd64.lib ADDED
Binary file (2.12 kB). View file