diff --git a/.gitattributes b/.gitattributes index 4223ad6f9fb7682665e42fa9b68fdb05adced652..21a1dadaaf19de5a2af55b584f5d26c46896651b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -676,3 +676,5 @@ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_lloyd.cp312- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/_svmlight_format_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/china.jpg filter=lfs diff=lfs merge=lfs -text loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/flower.jpg filter=lfs diff=lfs merge=lfs -text +loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text +loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..ecd8d1b12a547833c2d00ed29be640a12167d082 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:301396b4a42c814b1a15038ddfcbcf5c8590501231747d0dc2a500b84b2fd0df +size 328 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..1d119ce6ec907e4689015911b16bcbfe8552b4e8 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dee83987fffa8ec20e23b3cabc00d42beb7a469af6bd803909998c1687fa634 +size 934 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz new file mode 100644 index 0000000000000000000000000000000000000000..c82d051bccb1b232214b31c73114f4f78749d810 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c5fd93ffec7deb63a940fd698534dd7ebb7db349fc183930041cbf17e60e2cc +size 6471 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a0f72505ffd12e41fdd34455ca1dfb6b224160f6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fcb215595bb8223321943f0df50f5a5013445bf1819d53d3cb5bfa90d9e79c0 +size 595 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..89818ff01633a27976f11fd38a70cb1a652dce77 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38b74e7f02a61ff55bcfac4d87103d5bffc43febb0c019d9aaa162f8f7693068 +size 272 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..c152f7e5d9f72441b2fc6aa9f96af8f9ef9fc690 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8742a74bd5bc120acd9186c8a8737cb420ed9b009fade00b24e7ce5217797f2c +size 722 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz new file mode 100644 index 0000000000000000000000000000000000000000..010258ddd3f64ab3d63665f106946a34b241d68e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f623e777c0a36ae6c82fae10a7c2088cb383298ea244595bf8dc95449c9be4c4 +size 2326 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..2d2b568ee4692ad61c20ab051723efceb318b816 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ccbf138e0663895f9cf511136bc6395c153f6238af2eacb6a367e86e15d1a71 +size 1492 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..9564b2e437ee328b195f6289af99be51032c64d0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0985045a454c8186b4e690ebefb6cea1ef7c13292c98d50abda470a0ff3ad425 +size 312 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a54754b666113a517f58ff509416f461e92636e4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3736e7feb7ad30c68675c2c4e48a9fb262e80308c9083b100ddd0339da1fc282 +size 348 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz new file mode 100644 index 0000000000000000000000000000000000000000..2ad2b8b4fd397ee8d61b44fb77b26076f643335d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8d00c6690576a9ec39e1cb77054e13296be0fdebab0fb35a64a0e8627b6e6f3 +size 4519 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..92ba4281fe86b5273792d24afaabb04eef03199d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d38fdd601b67bb9c6d16152f53ddf166a0cfcfef4fa86438e899bfe449226c +size 1798 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..b2fce3413fd38f4c4f80ef7d6b198b4ac740a90a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:407424fb79cc30b8e9ff90900b3bf29244ac7f3797f278b5be602843f959b4ee +size 425 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..52ae92392967d187709107d1c1bc9709c085b519 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0703b0ae20b9ff75087dc601640ee58f1c2ad6768858ea21a245151da9ba8e4c +size 301 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..6bde2de0c6047726f26476a514d27a0d03c7d4b5 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70d4596ad879547863109da8675c2b789d07df66b526d7ebcbce9616c4c9b94c +size 347 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..911f6823bb1bf0d9de5120e23e902e9a0a39a2bc --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8743b2d93d2c62a82fb47e1fbc002b97e25adcfb5bf1fcb26b58ad0bed15bd48 +size 1074 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz new file mode 100644 index 0000000000000000000000000000000000000000..225208c948bd5270b3911828bead9d2fd3af3fbb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96142b5e00dfec2617b0c22d7192b340ae2c28ec3ffc3a894c5be746b970a59 +size 3303 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a7ff82cef2a309d55bcae99900bdd51b6bbc675e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5c7e79aa41ef580838fb9fc1906280f076c47be1741fddd5004ddb500eb57fe +size 898 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..466d7fab3f54e053ae4abc1044c671ac525accc0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33cbd6ae945ba04969370ab35604e9363c87256393493382b5118a89d59386d6 +size 268 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..76bb2da49d2e31a888153004b5177dc2a0c2f46c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bce20aae7fd903796d96d5b3a3677b7058fbc5f3fe0996ee9d491e4ee23d132 +size 293 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..c628aa1d9076067123d34c4c392a3a215dae524b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9f4b9317997df63ed8d2bb073a3906344c0e0be017fd384eaec36ced8b94bae +size 330 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..285c038aae89afa2eb0c334cdf28a9d0f6e2cb32 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424cd47c12a51c7bb8d8169fac80fb5601f152bd78468b241d4b115bf7d22f20 +size 1121 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz new file mode 100644 index 0000000000000000000000000000000000000000..25bb1bc7760d28c156677d8d257421b3805299c1 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afe4736924606638984e573235191025d419c545d31dc8874c96b72f5ec5db73 +size 2342 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..da14f86aac08c072962c2eecf6fe18cf319c5718 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ef3551ad47d48023c5a1f1cf077047a9a4b95544bb91d4a86097f8b574f8d07 +size 656 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..22da3f227189e339ed4d2b3861866ded65d999a6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:489b177126cb7f335cb220709233b946d3a0ad71d38bba6d48b79187146e585a +size 817 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..c8c1985e0bf13abce1abad45a5d872ccbcd44478 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:278a52a52d569f07d14c6a7877b104762c77daac429fb1fd9817a0378d6ec634 +size 805 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz new file mode 100644 index 0000000000000000000000000000000000000000..b3ce4b7991c2223af6097adb8d1f553088d1ece0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5830c82112f62a400c82ac1f1b5eb61c29c0a7cc72ba56d2aeff0fae8a60f9 +size 1625 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_classification.txt b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_classification.txt new file mode 100644 index 0000000000000000000000000000000000000000..b4b53167c4ccc823f78f880014aa7cd89cf7cccd --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_classification.txt @@ -0,0 +1,9 @@ +# comment +# note: the next line contains a tab +1.0 3:2.5 11:-5.2 16:1.5 # and an inline comment +2.0 6:1.0 13:-3 +# another comment +3.0 21:27 +4.0 2:1.234567890123456e10 # double precision value +1.0 # empty line, all zeros +2.0 3:0 # explicit zeros diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid.txt b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid.txt new file mode 100644 index 0000000000000000000000000000000000000000..1f97ffaabf6f1bc869647273924dac8b2b3662d1 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid.txt @@ -0,0 +1,3 @@ +python 2:2.5 10:-5.2 15:1.5 +2.0 5:1.0 12:-3 +3.0 20:27 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid_order.txt b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid_order.txt new file mode 100644 index 0000000000000000000000000000000000000000..1a98bf580658b4fc8e432819bcf0248d2d04d4af --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid_order.txt @@ -0,0 +1 @@ +-1 5:2.5 2:-5.2 15:1.5 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_multilabel.txt b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_multilabel.txt new file mode 100644 index 0000000000000000000000000000000000000000..9731a8c4e1d750ed1667b10491675207c55a316c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_multilabel.txt @@ -0,0 +1,5 @@ +# multilabel dataset in SVMlight format +1,0 2:2.5 10:-5.2 15:1.5 +2 5:1.0 12:-3 + 2:3.5 11:26 +1,2 20:27 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_20news.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_20news.py new file mode 100644 index 0000000000000000000000000000000000000000..ea56465eada836596719dda790533cc5783fdf3a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_20news.py @@ -0,0 +1,143 @@ +"""Test the 20news downloader, if the data is available, +or if specifically requested via environment variable +(e.g. for CI jobs).""" + +from functools import partial +from unittest.mock import patch + +import numpy as np +import pytest +import scipy.sparse as sp + +from sklearn.datasets.tests.test_common import ( + check_as_frame, + check_pandas_dependency_message, + check_return_X_y, +) +from sklearn.preprocessing import normalize +from sklearn.utils._testing import assert_allclose_dense_sparse + + +def test_20news(fetch_20newsgroups_fxt): + data = fetch_20newsgroups_fxt(subset="all", shuffle=False) + assert data.DESCR.startswith(".. _20newsgroups_dataset:") + + # Extract a reduced dataset + data2cats = fetch_20newsgroups_fxt( + subset="all", categories=data.target_names[-1:-3:-1], shuffle=False + ) + # Check that the ordering of the target_names is the same + # as the ordering in the full dataset + assert data2cats.target_names == data.target_names[-2:] + # Assert that we have only 0 and 1 as labels + assert np.unique(data2cats.target).tolist() == [0, 1] + + # Check that the number of filenames is consistent with data/target + assert len(data2cats.filenames) == len(data2cats.target) + assert len(data2cats.filenames) == len(data2cats.data) + + # Check that the first entry of the reduced dataset corresponds to + # the first entry of the corresponding category in the full dataset + entry1 = data2cats.data[0] + category = data2cats.target_names[data2cats.target[0]] + label = data.target_names.index(category) + entry2 = data.data[np.where(data.target == label)[0][0]] + assert entry1 == entry2 + + # check that return_X_y option + X, y = fetch_20newsgroups_fxt(subset="all", shuffle=False, return_X_y=True) + assert len(X) == len(data.data) + assert y.shape == data.target.shape + + +def test_20news_length_consistency(fetch_20newsgroups_fxt): + """Checks the length consistencies within the bunch + + This is a non-regression test for a bug present in 0.16.1. + """ + # Extract the full dataset + data = fetch_20newsgroups_fxt(subset="all") + assert len(data["data"]) == len(data.data) + assert len(data["target"]) == len(data.target) + assert len(data["filenames"]) == len(data.filenames) + + +def test_20news_vectorized(fetch_20newsgroups_vectorized_fxt): + # test subset = train + bunch = fetch_20newsgroups_vectorized_fxt(subset="train") + assert sp.issparse(bunch.data) and bunch.data.format == "csr" + assert bunch.data.shape == (11314, 130107) + assert bunch.target.shape[0] == 11314 + assert bunch.data.dtype == np.float64 + assert bunch.DESCR.startswith(".. _20newsgroups_dataset:") + + # test subset = test + bunch = fetch_20newsgroups_vectorized_fxt(subset="test") + assert sp.issparse(bunch.data) and bunch.data.format == "csr" + assert bunch.data.shape == (7532, 130107) + assert bunch.target.shape[0] == 7532 + assert bunch.data.dtype == np.float64 + assert bunch.DESCR.startswith(".. _20newsgroups_dataset:") + + # test return_X_y option + fetch_func = partial(fetch_20newsgroups_vectorized_fxt, subset="test") + check_return_X_y(bunch, fetch_func) + + # test subset = all + bunch = fetch_20newsgroups_vectorized_fxt(subset="all") + assert sp.issparse(bunch.data) and bunch.data.format == "csr" + assert bunch.data.shape == (11314 + 7532, 130107) + assert bunch.target.shape[0] == 11314 + 7532 + assert bunch.data.dtype == np.float64 + assert bunch.DESCR.startswith(".. _20newsgroups_dataset:") + + +def test_20news_normalization(fetch_20newsgroups_vectorized_fxt): + X = fetch_20newsgroups_vectorized_fxt(normalize=False) + X_ = fetch_20newsgroups_vectorized_fxt(normalize=True) + X_norm = X_["data"][:100] + X = X["data"][:100] + + assert_allclose_dense_sparse(X_norm, normalize(X)) + assert np.allclose(np.linalg.norm(X_norm.todense(), axis=1), 1) + + +def test_20news_as_frame(fetch_20newsgroups_vectorized_fxt): + pd = pytest.importorskip("pandas") + + bunch = fetch_20newsgroups_vectorized_fxt(as_frame=True) + check_as_frame(bunch, fetch_20newsgroups_vectorized_fxt) + + frame = bunch.frame + assert frame.shape == (11314, 130108) + assert all([isinstance(col, pd.SparseDtype) for col in bunch.data.dtypes]) + + # Check a small subset of features + for expected_feature in [ + "beginner", + "beginners", + "beginning", + "beginnings", + "begins", + "begley", + "begone", + ]: + assert expected_feature in frame.keys() + assert "category_class" in frame.keys() + assert bunch.target.name == "category_class" + + +def test_as_frame_no_pandas(fetch_20newsgroups_vectorized_fxt, hide_available_pandas): + check_pandas_dependency_message(fetch_20newsgroups_vectorized_fxt) + + +def test_outdated_pickle(fetch_20newsgroups_vectorized_fxt): + with patch("os.path.exists") as mock_is_exist: + with patch("joblib.load") as mock_load: + # mock that the dataset was cached + mock_is_exist.return_value = True + # mock that we have an outdated pickle with only X and y returned + mock_load.return_value = ("X", "y") + err_msg = "The cached dataset located in" + with pytest.raises(ValueError, match=err_msg): + fetch_20newsgroups_vectorized_fxt(as_frame=True) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_arff_parser.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_arff_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..a0d7be2c045ae52002d832c6187075519579af65 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_arff_parser.py @@ -0,0 +1,284 @@ +import textwrap +from io import BytesIO + +import pytest + +from sklearn.datasets._arff_parser import ( + _liac_arff_parser, + _pandas_arff_parser, + _post_process_frame, + load_arff_from_gzip_file, +) + + +@pytest.mark.parametrize( + "feature_names, target_names", + [ + ( + [ + "col_int_as_integer", + "col_int_as_numeric", + "col_float_as_real", + "col_float_as_numeric", + ], + ["col_categorical", "col_string"], + ), + ( + [ + "col_int_as_integer", + "col_int_as_numeric", + "col_float_as_real", + "col_float_as_numeric", + ], + ["col_categorical"], + ), + ( + [ + "col_int_as_integer", + "col_int_as_numeric", + "col_float_as_real", + "col_float_as_numeric", + ], + [], + ), + ], +) +def test_post_process_frame(feature_names, target_names): + """Check the behaviour of the post-processing function for splitting a dataframe.""" + pd = pytest.importorskip("pandas") + + X_original = pd.DataFrame( + { + "col_int_as_integer": [1, 2, 3], + "col_int_as_numeric": [1, 2, 3], + "col_float_as_real": [1.0, 2.0, 3.0], + "col_float_as_numeric": [1.0, 2.0, 3.0], + "col_categorical": ["a", "b", "c"], + "col_string": ["a", "b", "c"], + } + ) + + X, y = _post_process_frame(X_original, feature_names, target_names) + assert isinstance(X, pd.DataFrame) + if len(target_names) >= 2: + assert isinstance(y, pd.DataFrame) + elif len(target_names) == 1: + assert isinstance(y, pd.Series) + else: + assert y is None + + +def test_load_arff_from_gzip_file_error_parser(): + """An error will be raised if the parser is not known.""" + # None of the input parameters are required to be accurate since the check + # of the parser will be carried out first. + + err_msg = "Unknown parser: 'xxx'. Should be 'liac-arff' or 'pandas'" + with pytest.raises(ValueError, match=err_msg): + load_arff_from_gzip_file("xxx", "xxx", "xxx", "xxx", "xxx", "xxx") + + +@pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser]) +def test_pandas_arff_parser_strip_single_quotes(parser_func): + """Check that we properly strip single quotes from the data.""" + pd = pytest.importorskip("pandas") + + arff_file = BytesIO( + textwrap.dedent( + """ + @relation 'toy' + @attribute 'cat_single_quote' {'A', 'B', 'C'} + @attribute 'str_single_quote' string + @attribute 'str_nested_quote' string + @attribute 'class' numeric + @data + 'A','some text','\"expect double quotes\"',0 + """ + ).encode("utf-8") + ) + + columns_info = { + "cat_single_quote": { + "data_type": "nominal", + "name": "cat_single_quote", + }, + "str_single_quote": { + "data_type": "string", + "name": "str_single_quote", + }, + "str_nested_quote": { + "data_type": "string", + "name": "str_nested_quote", + }, + "class": { + "data_type": "numeric", + "name": "class", + }, + } + + feature_names = [ + "cat_single_quote", + "str_single_quote", + "str_nested_quote", + ] + target_names = ["class"] + + # We don't strip single quotes for string columns with the pandas parser. + expected_values = { + "cat_single_quote": "A", + "str_single_quote": ( + "some text" if parser_func is _liac_arff_parser else "'some text'" + ), + "str_nested_quote": ( + '"expect double quotes"' + if parser_func is _liac_arff_parser + else "'\"expect double quotes\"'" + ), + "class": 0, + } + + _, _, frame, _ = parser_func( + arff_file, + output_arrays_type="pandas", + openml_columns_info=columns_info, + feature_names_to_select=feature_names, + target_names_to_select=target_names, + ) + + assert frame.columns.tolist() == feature_names + target_names + pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0)) + + +@pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser]) +def test_pandas_arff_parser_strip_double_quotes(parser_func): + """Check that we properly strip double quotes from the data.""" + pd = pytest.importorskip("pandas") + + arff_file = BytesIO( + textwrap.dedent( + """ + @relation 'toy' + @attribute 'cat_double_quote' {"A", "B", "C"} + @attribute 'str_double_quote' string + @attribute 'str_nested_quote' string + @attribute 'class' numeric + @data + "A","some text","\'expect double quotes\'",0 + """ + ).encode("utf-8") + ) + + columns_info = { + "cat_double_quote": { + "data_type": "nominal", + "name": "cat_double_quote", + }, + "str_double_quote": { + "data_type": "string", + "name": "str_double_quote", + }, + "str_nested_quote": { + "data_type": "string", + "name": "str_nested_quote", + }, + "class": { + "data_type": "numeric", + "name": "class", + }, + } + + feature_names = [ + "cat_double_quote", + "str_double_quote", + "str_nested_quote", + ] + target_names = ["class"] + + expected_values = { + "cat_double_quote": "A", + "str_double_quote": "some text", + "str_nested_quote": "'expect double quotes'", + "class": 0, + } + + _, _, frame, _ = parser_func( + arff_file, + output_arrays_type="pandas", + openml_columns_info=columns_info, + feature_names_to_select=feature_names, + target_names_to_select=target_names, + ) + + assert frame.columns.tolist() == feature_names + target_names + pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0)) + + +@pytest.mark.parametrize( + "parser_func", + [ + # internal quotes are not considered to follow the ARFF spec in LIAC ARFF + pytest.param(_liac_arff_parser, marks=pytest.mark.xfail), + _pandas_arff_parser, + ], +) +def test_pandas_arff_parser_strip_no_quotes(parser_func): + """Check that we properly parse with no quotes characters.""" + pd = pytest.importorskip("pandas") + + arff_file = BytesIO( + textwrap.dedent( + """ + @relation 'toy' + @attribute 'cat_without_quote' {A, B, C} + @attribute 'str_without_quote' string + @attribute 'str_internal_quote' string + @attribute 'class' numeric + @data + A,some text,'internal' quote,0 + """ + ).encode("utf-8") + ) + + columns_info = { + "cat_without_quote": { + "data_type": "nominal", + "name": "cat_without_quote", + }, + "str_without_quote": { + "data_type": "string", + "name": "str_without_quote", + }, + "str_internal_quote": { + "data_type": "string", + "name": "str_internal_quote", + }, + "class": { + "data_type": "numeric", + "name": "class", + }, + } + + feature_names = [ + "cat_without_quote", + "str_without_quote", + "str_internal_quote", + ] + target_names = ["class"] + + expected_values = { + "cat_without_quote": "A", + "str_without_quote": "some text", + "str_internal_quote": "'internal' quote", + "class": 0, + } + + _, _, frame, _ = parser_func( + arff_file, + output_arrays_type="pandas", + openml_columns_info=columns_info, + feature_names_to_select=feature_names, + target_names_to_select=target_names, + ) + + assert frame.columns.tolist() == feature_names + target_names + pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0)) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_base.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_base.py new file mode 100644 index 0000000000000000000000000000000000000000..815a495b0a36ccd799e30dc7a0d4d92946a1601f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_base.py @@ -0,0 +1,659 @@ +import hashlib +import io +import os +import re +import shutil +import tempfile +import warnings +from functools import partial +from importlib import resources +from pathlib import Path +from pickle import dumps, loads +from unittest.mock import Mock +from urllib.error import HTTPError +from urllib.parse import urlparse + +import numpy as np +import pytest + +from sklearn.datasets import ( + clear_data_home, + fetch_file, + get_data_home, + load_breast_cancer, + load_diabetes, + load_digits, + load_files, + load_iris, + load_linnerud, + load_sample_image, + load_sample_images, + load_wine, +) +from sklearn.datasets._base import ( + RemoteFileMetadata, + _derive_folder_and_filename_from_url, + _fetch_remote, + load_csv_data, + load_gzip_compressed_csv_data, +) +from sklearn.datasets.tests.test_common import check_as_frame +from sklearn.preprocessing import scale +from sklearn.utils import Bunch + + +class _DummyPath: + """Minimal class that implements the os.PathLike interface.""" + + def __init__(self, path): + self.path = path + + def __fspath__(self): + return self.path + + +def _remove_dir(path): + if os.path.isdir(path): + shutil.rmtree(path) + + +@pytest.fixture(scope="module") +def data_home(tmpdir_factory): + tmp_file = str(tmpdir_factory.mktemp("scikit_learn_data_home_test")) + yield tmp_file + _remove_dir(tmp_file) + + +@pytest.fixture(scope="module") +def load_files_root(tmpdir_factory): + tmp_file = str(tmpdir_factory.mktemp("scikit_learn_load_files_test")) + yield tmp_file + _remove_dir(tmp_file) + + +@pytest.fixture +def test_category_dir_1(load_files_root): + test_category_dir1 = tempfile.mkdtemp(dir=load_files_root) + sample_file = tempfile.NamedTemporaryFile(dir=test_category_dir1, delete=False) + sample_file.write(b"Hello World!\n") + sample_file.close() + yield str(test_category_dir1) + _remove_dir(test_category_dir1) + + +@pytest.fixture +def test_category_dir_2(load_files_root): + test_category_dir2 = tempfile.mkdtemp(dir=load_files_root) + yield str(test_category_dir2) + _remove_dir(test_category_dir2) + + +@pytest.mark.thread_unsafe +@pytest.mark.parametrize("path_container", [None, Path, _DummyPath]) +def test_data_home(path_container, data_home): + # get_data_home will point to a pre-existing folder + if path_container is not None: + data_home = path_container(data_home) + data_home = get_data_home(data_home=data_home) + assert data_home == data_home + assert os.path.exists(data_home) + + # clear_data_home will delete both the content and the folder it-self + if path_container is not None: + data_home = path_container(data_home) + clear_data_home(data_home=data_home) + assert not os.path.exists(data_home) + + # if the folder is missing it will be created again + data_home = get_data_home(data_home=data_home) + assert os.path.exists(data_home) + + +def test_default_empty_load_files(load_files_root): + res = load_files(load_files_root) + assert len(res.filenames) == 0 + assert len(res.target_names) == 0 + assert res.DESCR is None + + +def test_default_load_files(test_category_dir_1, test_category_dir_2, load_files_root): + res = load_files(load_files_root) + assert len(res.filenames) == 1 + assert len(res.target_names) == 2 + assert res.DESCR is None + assert res.data == [b"Hello World!\n"] + + +def test_load_files_w_categories_desc_and_encoding( + test_category_dir_1, test_category_dir_2, load_files_root +): + category = os.path.abspath(test_category_dir_1).split(os.sep).pop() + res = load_files( + load_files_root, description="test", categories=[category], encoding="utf-8" + ) + + assert len(res.filenames) == 1 + assert len(res.target_names) == 1 + assert res.DESCR == "test" + assert res.data == ["Hello World!\n"] + + +def test_load_files_wo_load_content( + test_category_dir_1, test_category_dir_2, load_files_root +): + res = load_files(load_files_root, load_content=False) + assert len(res.filenames) == 1 + assert len(res.target_names) == 2 + assert res.DESCR is None + assert res.get("data") is None + + +@pytest.mark.parametrize("allowed_extensions", ([".txt"], [".txt", ".json"])) +def test_load_files_allowed_extensions(tmp_path, allowed_extensions): + """Check the behaviour of `allowed_extension` in `load_files`.""" + d = tmp_path / "sub" + d.mkdir() + files = ("file1.txt", "file2.json", "file3.json", "file4.md") + paths = [d / f for f in files] + for p in paths: + p.write_bytes(b"hello") + res = load_files(tmp_path, allowed_extensions=allowed_extensions) + assert set([str(p) for p in paths if p.suffix in allowed_extensions]) == set( + res.filenames + ) + + +@pytest.mark.parametrize( + "filename, expected_n_samples, expected_n_features, expected_target_names", + [ + ("wine_data.csv", 178, 13, ["class_0", "class_1", "class_2"]), + ("iris.csv", 150, 4, ["setosa", "versicolor", "virginica"]), + ("breast_cancer.csv", 569, 30, ["malignant", "benign"]), + ], +) +def test_load_csv_data( + filename, expected_n_samples, expected_n_features, expected_target_names +): + actual_data, actual_target, actual_target_names = load_csv_data(filename) + assert actual_data.shape[0] == expected_n_samples + assert actual_data.shape[1] == expected_n_features + assert actual_target.shape[0] == expected_n_samples + np.testing.assert_array_equal(actual_target_names, expected_target_names) + + +def test_load_csv_data_with_descr(): + data_file_name = "iris.csv" + descr_file_name = "iris.rst" + + res_without_descr = load_csv_data(data_file_name=data_file_name) + res_with_descr = load_csv_data( + data_file_name=data_file_name, descr_file_name=descr_file_name + ) + assert len(res_with_descr) == 4 + assert len(res_without_descr) == 3 + + np.testing.assert_array_equal(res_with_descr[0], res_without_descr[0]) + np.testing.assert_array_equal(res_with_descr[1], res_without_descr[1]) + np.testing.assert_array_equal(res_with_descr[2], res_without_descr[2]) + + assert res_with_descr[-1].startswith(".. _iris_dataset:") + + +@pytest.mark.parametrize( + "filename, kwargs, expected_shape", + [ + ("diabetes_data_raw.csv.gz", {}, [442, 10]), + ("diabetes_target.csv.gz", {}, [442]), + ("digits.csv.gz", {"delimiter": ","}, [1797, 65]), + ], +) +def test_load_gzip_compressed_csv_data(filename, kwargs, expected_shape): + actual_data = load_gzip_compressed_csv_data(filename, **kwargs) + assert actual_data.shape == tuple(expected_shape) + + +def test_load_gzip_compressed_csv_data_with_descr(): + data_file_name = "diabetes_target.csv.gz" + descr_file_name = "diabetes.rst" + + expected_data = load_gzip_compressed_csv_data(data_file_name=data_file_name) + actual_data, descr = load_gzip_compressed_csv_data( + data_file_name=data_file_name, + descr_file_name=descr_file_name, + ) + + np.testing.assert_array_equal(actual_data, expected_data) + assert descr.startswith(".. _diabetes_dataset:") + + +def test_load_sample_images(): + try: + res = load_sample_images() + assert len(res.images) == 2 + assert len(res.filenames) == 2 + images = res.images + + # assert is china image + assert np.all(images[0][0, 0, :] == np.array([174, 201, 231], dtype=np.uint8)) + # assert is flower image + assert np.all(images[1][0, 0, :] == np.array([2, 19, 13], dtype=np.uint8)) + assert res.DESCR + except ImportError: + warnings.warn("Could not load sample images, PIL is not available.") + + +def test_load_sample_image(): + try: + china = load_sample_image("china.jpg") + assert china.dtype == "uint8" + assert china.shape == (427, 640, 3) + except ImportError: + warnings.warn("Could not load sample images, PIL is not available.") + + +def test_load_diabetes_raw(): + """Test to check that we load a scaled version by default but that we can + get an unscaled version when setting `scaled=False`.""" + diabetes_raw = load_diabetes(scaled=False) + assert diabetes_raw.data.shape == (442, 10) + assert diabetes_raw.target.size == 442 + assert len(diabetes_raw.feature_names) == 10 + assert diabetes_raw.DESCR + + diabetes_default = load_diabetes() + + np.testing.assert_allclose( + scale(diabetes_raw.data) / (442**0.5), diabetes_default.data, atol=1e-04 + ) + + +@pytest.mark.parametrize( + "loader_func, data_shape, target_shape, n_target, has_descr, filenames", + [ + (load_breast_cancer, (569, 30), (569,), 2, True, ["filename"]), + (load_wine, (178, 13), (178,), 3, True, []), + (load_iris, (150, 4), (150,), 3, True, ["filename"]), + ( + load_linnerud, + (20, 3), + (20, 3), + 3, + True, + ["data_filename", "target_filename"], + ), + (load_diabetes, (442, 10), (442,), None, True, []), + (load_digits, (1797, 64), (1797,), 10, True, []), + (partial(load_digits, n_class=9), (1617, 64), (1617,), 10, True, []), + ], +) +def test_loader(loader_func, data_shape, target_shape, n_target, has_descr, filenames): + bunch = loader_func() + + assert isinstance(bunch, Bunch) + assert bunch.data.shape == data_shape + assert bunch.target.shape == target_shape + if hasattr(bunch, "feature_names"): + assert len(bunch.feature_names) == data_shape[1] + if n_target is not None: + assert len(bunch.target_names) == n_target + if has_descr: + assert bunch.DESCR + if filenames: + assert "data_module" in bunch + assert all( + [ + f in bunch + and (resources.files(bunch["data_module"]) / bunch[f]).is_file() + for f in filenames + ] + ) + + +@pytest.mark.parametrize( + "loader_func, data_dtype, target_dtype", + [ + (load_breast_cancer, np.float64, int), + (load_diabetes, np.float64, np.float64), + (load_digits, np.float64, int), + (load_iris, np.float64, int), + (load_linnerud, np.float64, np.float64), + (load_wine, np.float64, int), + ], +) +def test_toy_dataset_frame_dtype(loader_func, data_dtype, target_dtype): + default_result = loader_func() + check_as_frame( + default_result, + loader_func, + expected_data_dtype=data_dtype, + expected_target_dtype=target_dtype, + ) + + +def test_loads_dumps_bunch(): + bunch = Bunch(x="x") + bunch_from_pkl = loads(dumps(bunch)) + bunch_from_pkl.x = "y" + assert bunch_from_pkl["x"] == bunch_from_pkl.x + + +def test_bunch_pickle_generated_with_0_16_and_read_with_0_17(): + bunch = Bunch(key="original") + # This reproduces a problem when Bunch pickles have been created + # with scikit-learn 0.16 and are read with 0.17. Basically there + # is a surprising behaviour because reading bunch.key uses + # bunch.__dict__ (which is non empty for 0.16 Bunch objects) + # whereas assigning into bunch.key uses bunch.__setattr__. See + # https://github.com/scikit-learn/scikit-learn/issues/6196 for + # more details + bunch.__dict__["key"] = "set from __dict__" + bunch_from_pkl = loads(dumps(bunch)) + # After loading from pickle the __dict__ should have been ignored + assert bunch_from_pkl.key == "original" + assert bunch_from_pkl["key"] == "original" + # Making sure that changing the attr does change the value + # associated with __getitem__ as well + bunch_from_pkl.key = "changed" + assert bunch_from_pkl.key == "changed" + assert bunch_from_pkl["key"] == "changed" + + +def test_bunch_dir(): + # check that dir (important for autocomplete) shows attributes + data = load_iris() + assert "data" in dir(data) + + +def test_load_boston_error(): + """Check that we raise the ethical warning when trying to import `load_boston`.""" + msg = "The Boston housing prices dataset has an ethical problem" + with pytest.raises(ImportError, match=msg): + from sklearn.datasets import load_boston # noqa: F401 + + # other non-existing function should raise the usual import error + msg = "cannot import name 'non_existing_function' from 'sklearn.datasets'" + with pytest.raises(ImportError, match=msg): + from sklearn.datasets import non_existing_function # noqa: F401 + + +def test_fetch_remote_raise_warnings_with_invalid_url(monkeypatch): + """Check retry mechanism in _fetch_remote.""" + + url = "https://scikit-learn.org/this_file_does_not_exist.tar.gz" + invalid_remote_file = RemoteFileMetadata("invalid_file", url, None) + urlretrieve_mock = Mock( + side_effect=HTTPError( + url=url, code=404, msg="Not Found", hdrs=None, fp=io.BytesIO() + ) + ) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + with pytest.warns(UserWarning, match="Retry downloading") as record: + with pytest.raises(HTTPError, match="HTTP Error 404"): + _fetch_remote(invalid_remote_file, n_retries=3, delay=0) + + assert urlretrieve_mock.call_count == 4 + + for r in record: + assert str(r.message) == f"Retry downloading from url: {url}" + assert len(record) == 3 + + +def test_derive_folder_and_filename_from_url(): + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/file.tar.gz" + ) + assert folder == "example.com" + assert filename == "file.tar.gz" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/نمونه نماینده.data" + ) + assert folder == "example.com" + assert filename == "نمونه-نماینده.data" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/path/to-/.file.tar.gz" + ) + assert folder == "example.com/path_to" + assert filename == "file.tar.gz" + + folder, filename = _derive_folder_and_filename_from_url("https://example.com/") + assert folder == "example.com" + assert filename == "downloaded_file" + + folder, filename = _derive_folder_and_filename_from_url("https://example.com") + assert folder == "example.com" + assert filename == "downloaded_file" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/path/@to/data.json?param=value" + ) + assert folder == "example.com/path_to" + assert filename == "data.json" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/path/@@to._/-_.data.json.#anchor" + ) + assert folder == "example.com/path_to" + assert filename == "data.json" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com//some_file.txt" + ) + assert folder == "example.com" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url( + "http://example/../some_file.txt" + ) + assert folder == "example" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/!.'.,/some_file.txt" + ) + assert folder == "example.com" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/a/!.'.,/b/some_file.txt" + ) + assert folder == "example.com/a_b" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url("https://example.com/!.'.,") + assert folder == "example.com" + assert filename == "downloaded_file" + + with pytest.raises(ValueError, match="Invalid URL"): + _derive_folder_and_filename_from_url("https:/../") + + +def _mock_urlretrieve(server_side): + def _urlretrieve_mock(url, local_path): + server_root = Path(server_side) + file_path = urlparse(url).path.strip("/") + if not (server_root / file_path).exists(): + raise HTTPError(url, 404, "Not Found", None, None) + shutil.copy(server_root / file_path, local_path) + + return Mock(side_effect=_urlretrieve_mock) + + +def test_fetch_file_using_data_home(monkeypatch, tmpdir): + tmpdir = Path(tmpdir) + server_side = tmpdir / "server_side" + server_side.mkdir() + data_file = server_side / "data.jsonl" + server_data = '{"a": 1, "b": 2}\n' + data_file.write_text(server_data, encoding="utf-8") + + server_subfolder = server_side / "subfolder" + server_subfolder.mkdir() + other_data_file = server_subfolder / "other_file.txt" + other_data_file.write_text("Some important text data.", encoding="utf-8") + + data_home = tmpdir / "data_home" + data_home.mkdir() + + urlretrieve_mock = _mock_urlretrieve(server_side) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + monkeypatch.setattr( + "sklearn.datasets._base.get_data_home", Mock(return_value=data_home) + ) + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + ) + assert fetched_file_path == data_home / "example.com" / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + + fetched_file_path = fetch_file( + "https://example.com/subfolder/other_file.txt", + ) + assert ( + fetched_file_path == data_home / "example.com" / "subfolder" / "other_file.txt" + ) + assert fetched_file_path.read_text(encoding="utf-8") == other_data_file.read_text( + "utf-8" + ) + + expected_warning_msg = re.escape( + "Retry downloading from url: https://example.com/subfolder/invalid.txt" + ) + with pytest.raises(HTTPError): + with pytest.warns(match=expected_warning_msg): + fetch_file( + "https://example.com/subfolder/invalid.txt", + delay=0, + ) + + local_subfolder = data_home / "example.com" / "subfolder" + assert sorted(local_subfolder.iterdir()) == [local_subfolder / "other_file.txt"] + + +def test_fetch_file_without_sha256(monkeypatch, tmpdir): + server_side = tmpdir.mkdir("server_side") + data_file = Path(server_side / "data.jsonl") + server_data = '{"a": 1, "b": 2}\n' + data_file.write_text(server_data, encoding="utf-8") + + client_side = tmpdir.mkdir("client_side") + + urlretrieve_mock = _mock_urlretrieve(server_side) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + # The first call should trigger a download: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Fetching again the same file to the same folder should do nothing: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Deleting and calling again should re-download + fetched_file_path.unlink() + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 2 + + +def test_fetch_file_with_sha256(monkeypatch, tmpdir): + server_side = tmpdir.mkdir("server_side") + data_file = Path(server_side / "data.jsonl") + server_data = '{"a": 1, "b": 2}\n' + data_file.write_text(server_data, encoding="utf-8") + expected_sha256 = hashlib.sha256(data_file.read_bytes()).hexdigest() + + client_side = tmpdir.mkdir("client_side") + + urlretrieve_mock = _mock_urlretrieve(server_side) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + # The first call should trigger a download. + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Fetching again the same file to the same folder should do nothing when + # the sha256 match: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Corrupting the local data should yield a warning and trigger a new download: + fetched_file_path.write_text("corrupted contents", encoding="utf-8") + expected_msg = ( + r"SHA256 checksum of existing local file data.jsonl " + rf"\(.*\) differs from expected \({expected_sha256}\): " + r"re-downloading from https://example.com/data.jsonl \." + ) + with pytest.warns(match=expected_msg): + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 2 + + # Calling again should do nothing: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 2 + + # Deleting the local file and calling again should redownload without warning: + fetched_file_path.unlink() + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 3 + + # Calling without a sha256 should also work without redownloading: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 3 + + # Calling with a wrong sha256 should raise an informative exception: + non_matching_sha256 = "deadbabecafebeef" + expected_warning_msg = "differs from expected" + expected_error_msg = re.escape( + f"The SHA256 checksum of data.jsonl ({expected_sha256}) differs from " + f"expected ({non_matching_sha256})." + ) + with pytest.raises(OSError, match=expected_error_msg): + with pytest.warns(match=expected_warning_msg): + fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + sha256=non_matching_sha256, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_california_housing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_california_housing.py new file mode 100644 index 0000000000000000000000000000000000000000..524fc32557ddc0b67067e967d76656acc13cde1d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_california_housing.py @@ -0,0 +1,38 @@ +"""Test the california_housing loader, if the data is available, +or if specifically requested via environment variable +(e.g. for CI jobs).""" + +from functools import partial + +import pytest + +from sklearn.datasets.tests.test_common import check_return_X_y + + +def test_fetch(fetch_california_housing_fxt): + data = fetch_california_housing_fxt() + assert (20640, 8) == data.data.shape + assert (20640,) == data.target.shape + assert data.DESCR.startswith(".. _california_housing_dataset:") + + # test return_X_y option + fetch_func = partial(fetch_california_housing_fxt) + check_return_X_y(data, fetch_func) + + +def test_fetch_asframe(fetch_california_housing_fxt): + pd = pytest.importorskip("pandas") + bunch = fetch_california_housing_fxt(as_frame=True) + frame = bunch.frame + assert hasattr(bunch, "frame") is True + assert frame.shape == (20640, 9) + assert isinstance(bunch.data, pd.DataFrame) + assert isinstance(bunch.target, pd.Series) + + +def test_pandas_dependency_message(fetch_california_housing_fxt, hide_available_pandas): + # Check that pandas is imported lazily and that an informative error + # message is raised when pandas is missing: + expected_msg = "fetch_california_housing with as_frame=True requires pandas" + with pytest.raises(ImportError, match=expected_msg): + fetch_california_housing_fxt(as_frame=True) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_common.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_common.py new file mode 100644 index 0000000000000000000000000000000000000000..7c11526c4d5a62b395556d233def43f7088b9bfe --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_common.py @@ -0,0 +1,136 @@ +"""Test loaders for common functionality.""" + +import inspect +import os + +import numpy as np +import pytest + +import sklearn.datasets + + +def is_pillow_installed(): + try: + import PIL # noqa: F401 + + return True + except ImportError: + return False + + +FETCH_PYTEST_MARKERS = { + "return_X_y": { + "fetch_20newsgroups": pytest.mark.xfail( + reason="X is a list and does not have a shape argument" + ), + "fetch_openml": pytest.mark.xfail( + reason="fetch_opeml requires a dataset name or id" + ), + "fetch_lfw_people": pytest.mark.skipif( + not is_pillow_installed(), reason="pillow is not installed" + ), + }, + "as_frame": { + "fetch_openml": pytest.mark.xfail( + reason="fetch_opeml requires a dataset name or id" + ), + }, +} + + +def check_pandas_dependency_message(fetch_func): + try: + import pandas # noqa: F401 + + pytest.skip("This test requires pandas to not be installed") + except ImportError: + # Check that pandas is imported lazily and that an informative error + # message is raised when pandas is missing: + name = fetch_func.__name__ + expected_msg = f"{name} with as_frame=True requires pandas" + with pytest.raises(ImportError, match=expected_msg): + fetch_func(as_frame=True) + + +def check_return_X_y(bunch, dataset_func): + X_y_tuple = dataset_func(return_X_y=True) + assert isinstance(X_y_tuple, tuple) + assert X_y_tuple[0].shape == bunch.data.shape + assert X_y_tuple[1].shape == bunch.target.shape + + +def check_as_frame( + bunch, dataset_func, expected_data_dtype=None, expected_target_dtype=None +): + pd = pytest.importorskip("pandas") + frame_bunch = dataset_func(as_frame=True) + assert hasattr(frame_bunch, "frame") + assert isinstance(frame_bunch.frame, pd.DataFrame) + assert isinstance(frame_bunch.data, pd.DataFrame) + assert frame_bunch.data.shape == bunch.data.shape + if frame_bunch.target.ndim > 1: + assert isinstance(frame_bunch.target, pd.DataFrame) + else: + assert isinstance(frame_bunch.target, pd.Series) + assert frame_bunch.target.shape[0] == bunch.target.shape[0] + if expected_data_dtype is not None: + assert np.all(frame_bunch.data.dtypes == expected_data_dtype) + if expected_target_dtype is not None: + assert np.all(frame_bunch.target.dtypes == expected_target_dtype) + + # Test for return_X_y and as_frame=True + frame_X, frame_y = dataset_func(as_frame=True, return_X_y=True) + assert isinstance(frame_X, pd.DataFrame) + if frame_y.ndim > 1: + assert isinstance(frame_y, pd.DataFrame) + else: + assert isinstance(frame_y, pd.Series) + + +def _skip_network_tests(): + return os.environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "1" + + +def _generate_func_supporting_param(param, dataset_type=("load", "fetch")): + markers_fetch = FETCH_PYTEST_MARKERS.get(param, {}) + for name, obj in inspect.getmembers(sklearn.datasets): + if not inspect.isfunction(obj): + continue + + is_dataset_type = any([name.startswith(t) for t in dataset_type]) + is_support_param = param in inspect.signature(obj).parameters + if is_dataset_type and is_support_param: + # check if we should skip if we don't have network support + marks = [ + pytest.mark.skipif( + condition=name.startswith("fetch") and _skip_network_tests(), + reason="Skip because fetcher requires internet network", + ) + ] + if name in markers_fetch: + marks.append(markers_fetch[name]) + + yield pytest.param(name, obj, marks=marks) + + +@pytest.mark.parametrize( + "name, dataset_func", _generate_func_supporting_param("return_X_y") +) +def test_common_check_return_X_y(name, dataset_func): + bunch = dataset_func() + check_return_X_y(bunch, dataset_func) + + +@pytest.mark.parametrize( + "name, dataset_func", _generate_func_supporting_param("as_frame") +) +def test_common_check_as_frame(name, dataset_func): + bunch = dataset_func() + check_as_frame(bunch, dataset_func) + + +@pytest.mark.parametrize( + "name, dataset_func", _generate_func_supporting_param("as_frame") +) +def test_common_check_pandas_dependency(name, dataset_func): + check_pandas_dependency_message(dataset_func) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_covtype.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_covtype.py new file mode 100644 index 0000000000000000000000000000000000000000..5f4a3624fd010e55be9e417a6c29f3fa2c49e2cf --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_covtype.py @@ -0,0 +1,55 @@ +"""Test the covtype loader, if the data is available, +or if specifically requested via environment variable +(e.g. for CI jobs).""" + +from functools import partial + +import pytest + +from sklearn.datasets.tests.test_common import check_return_X_y + + +def test_fetch(fetch_covtype_fxt, global_random_seed): + data1 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed) + data2 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed + 1) + + X1, X2 = data1["data"], data2["data"] + assert (581012, 54) == X1.shape + assert X1.shape == X2.shape + + assert X1.sum() == X2.sum() + + y1, y2 = data1["target"], data2["target"] + assert (X1.shape[0],) == y1.shape + assert (X1.shape[0],) == y2.shape + + descr_prefix = ".. _covtype_dataset:" + assert data1.DESCR.startswith(descr_prefix) + assert data2.DESCR.startswith(descr_prefix) + + # test return_X_y option + fetch_func = partial(fetch_covtype_fxt) + check_return_X_y(data1, fetch_func) + + +def test_fetch_asframe(fetch_covtype_fxt): + pytest.importorskip("pandas") + + bunch = fetch_covtype_fxt(as_frame=True) + assert hasattr(bunch, "frame") + frame = bunch.frame + assert frame.shape == (581012, 55) + assert bunch.data.shape == (581012, 54) + assert bunch.target.shape == (581012,) + + column_names = set(frame.columns) + + # enumerated names are added correctly + assert set(f"Wilderness_Area_{i}" for i in range(4)) < column_names + assert set(f"Soil_Type_{i}" for i in range(40)) < column_names + + +def test_pandas_dependency_message(fetch_covtype_fxt, hide_available_pandas): + expected_msg = "fetch_covtype with as_frame=True requires pandas" + with pytest.raises(ImportError, match=expected_msg): + fetch_covtype_fxt(as_frame=True) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_kddcup99.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_kddcup99.py new file mode 100644 index 0000000000000000000000000000000000000000..2eaafddcaaf16ccbe576476cf654c366b10a61fa --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_kddcup99.py @@ -0,0 +1,89 @@ +"""Test kddcup99 loader, if the data is available, +or if specifically requested via environment variable +(e.g. for CI jobs). + +Only 'percent10' mode is tested, as the full data +is too big to use in unit-testing. +""" + +from functools import partial + +import pytest + +from sklearn.datasets.tests.test_common import ( + check_as_frame, + check_pandas_dependency_message, + check_return_X_y, +) + + +@pytest.mark.parametrize("as_frame", [True, False]) +@pytest.mark.parametrize( + "subset, n_samples, n_features", + [ + (None, 494021, 41), + ("SA", 100655, 41), + ("SF", 73237, 4), + ("http", 58725, 3), + ("smtp", 9571, 3), + ], +) +def test_fetch_kddcup99_percent10( + fetch_kddcup99_fxt, as_frame, subset, n_samples, n_features +): + data = fetch_kddcup99_fxt(subset=subset, as_frame=as_frame) + assert data.data.shape == (n_samples, n_features) + assert data.target.shape == (n_samples,) + if as_frame: + assert data.frame.shape == (n_samples, n_features + 1) + assert data.DESCR.startswith(".. _kddcup99_dataset:") + + +def test_fetch_kddcup99_return_X_y(fetch_kddcup99_fxt): + fetch_func = partial(fetch_kddcup99_fxt, subset="smtp") + data = fetch_func() + check_return_X_y(data, fetch_func) + + +def test_fetch_kddcup99_as_frame(fetch_kddcup99_fxt): + bunch = fetch_kddcup99_fxt() + check_as_frame(bunch, fetch_kddcup99_fxt) + + +def test_fetch_kddcup99_shuffle(fetch_kddcup99_fxt): + dataset = fetch_kddcup99_fxt( + random_state=0, + subset="SA", + percent10=True, + ) + dataset_shuffled = fetch_kddcup99_fxt( + random_state=0, + subset="SA", + shuffle=True, + percent10=True, + ) + assert set(dataset["target"]) == set(dataset_shuffled["target"]) + assert dataset_shuffled.data.shape == dataset.data.shape + assert dataset_shuffled.target.shape == dataset.target.shape + + +def test_pandas_dependency_message(fetch_kddcup99_fxt, hide_available_pandas): + check_pandas_dependency_message(fetch_kddcup99_fxt) + + +def test_corrupted_file_error_message(fetch_kddcup99_fxt, tmp_path): + """Check that a nice error message is raised when cache is corrupted.""" + kddcup99_dir = tmp_path / "kddcup99_10-py3" + kddcup99_dir.mkdir() + samples_path = kddcup99_dir / "samples" + + with samples_path.open("wb") as f: + f.write(b"THIS IS CORRUPTED") + + msg = ( + "The cache for fetch_kddcup99 is invalid, please " + f"delete {kddcup99_dir} and run the fetch_kddcup99 again" + ) + + with pytest.raises(OSError, match=msg): + fetch_kddcup99_fxt(data_home=str(tmp_path)) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_lfw.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_lfw.py new file mode 100644 index 0000000000000000000000000000000000000000..c4b546058362c8d614f8110e7b34c7b33f0d3b94 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_lfw.py @@ -0,0 +1,229 @@ +"""This test for the LFW require medium-size data downloading and processing + +If the data has not been already downloaded by running the examples, +the tests won't run (skipped). + +If the test are run, the first execution will be long (typically a bit +more than a couple of minutes) but as the dataset loader is leveraging +joblib, successive runs will be fast (less than 200ms). +""" + +import random +from functools import partial + +import numpy as np +import pytest + +from sklearn.datasets import fetch_lfw_pairs, fetch_lfw_people +from sklearn.datasets.tests.test_common import check_return_X_y +from sklearn.utils._testing import assert_array_equal + +FAKE_NAMES = [ + "Abdelatif_Smith", + "Abhati_Kepler", + "Camara_Alvaro", + "Chen_Dupont", + "John_Lee", + "Lin_Bauman", + "Onur_Lopez", +] + + +@pytest.fixture(scope="module") +def mock_empty_data_home(tmp_path_factory): + data_dir = tmp_path_factory.mktemp("scikit_learn_empty_test") + + yield data_dir + + +@pytest.fixture(scope="module") +def mock_data_home(tmp_path_factory): + """Test fixture run once and common to all tests of this module""" + Image = pytest.importorskip("PIL.Image") + + data_dir = tmp_path_factory.mktemp("scikit_learn_lfw_test") + lfw_home = data_dir / "lfw_home" + lfw_home.mkdir(parents=True, exist_ok=True) + + random_state = random.Random(42) + np_rng = np.random.RandomState(42) + + # generate some random jpeg files for each person + counts = {} + for name in FAKE_NAMES: + folder_name = lfw_home / "lfw_funneled" / name + folder_name.mkdir(parents=True, exist_ok=True) + + n_faces = np_rng.randint(1, 5) + counts[name] = n_faces + for i in range(n_faces): + file_path = folder_name / (name + "_%04d.jpg" % i) + uniface = np_rng.randint(0, 255, size=(250, 250, 3)) + img = Image.fromarray(uniface.astype(np.uint8)) + img.save(file_path) + + # add some random file pollution to test robustness + (lfw_home / "lfw_funneled" / ".test.swp").write_bytes( + b"Text file to be ignored by the dataset loader." + ) + + # generate some pairing metadata files using the same format as LFW + with open(lfw_home / "pairsDevTrain.txt", "wb") as f: + f.write(b"10\n") + more_than_two = [name for name, count in counts.items() if count >= 2] + for i in range(5): + name = random_state.choice(more_than_two) + first, second = random_state.sample(range(counts[name]), 2) + f.write(("%s\t%d\t%d\n" % (name, first, second)).encode()) + + for i in range(5): + first_name, second_name = random_state.sample(FAKE_NAMES, 2) + first_index = np_rng.choice(np.arange(counts[first_name])) + second_index = np_rng.choice(np.arange(counts[second_name])) + f.write( + ( + "%s\t%d\t%s\t%d\n" + % (first_name, first_index, second_name, second_index) + ).encode() + ) + + (lfw_home / "pairsDevTest.txt").write_bytes( + b"Fake place holder that won't be tested" + ) + (lfw_home / "pairs.txt").write_bytes(b"Fake place holder that won't be tested") + + yield data_dir + + +def test_load_empty_lfw_people(mock_empty_data_home): + with pytest.raises(OSError): + fetch_lfw_people(data_home=mock_empty_data_home, download_if_missing=False) + + +def test_load_fake_lfw_people(mock_data_home): + lfw_people = fetch_lfw_people( + data_home=mock_data_home, min_faces_per_person=3, download_if_missing=False + ) + + # The data is croped around the center as a rectangular bounding box + # around the face. Colors are converted to gray levels: + assert lfw_people.images.shape == (10, 62, 47) + assert lfw_people.data.shape == (10, 2914) + + # the target is array of person integer ids + assert_array_equal(lfw_people.target, [2, 0, 1, 0, 2, 0, 2, 1, 1, 2]) + + # names of the persons can be found using the target_names array + expected_classes = ["Abdelatif Smith", "Abhati Kepler", "Onur Lopez"] + assert_array_equal(lfw_people.target_names, expected_classes) + + # It is possible to ask for the original data without any croping or color + # conversion and not limit on the number of picture per person + lfw_people = fetch_lfw_people( + data_home=mock_data_home, + resize=None, + slice_=None, + color=True, + download_if_missing=False, + ) + assert lfw_people.images.shape == (17, 250, 250, 3) + assert lfw_people.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:") + + # the ids and class names are the same as previously + assert_array_equal( + lfw_people.target, [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2] + ) + assert_array_equal( + lfw_people.target_names, + [ + "Abdelatif Smith", + "Abhati Kepler", + "Camara Alvaro", + "Chen Dupont", + "John Lee", + "Lin Bauman", + "Onur Lopez", + ], + ) + + # test return_X_y option + fetch_func = partial( + fetch_lfw_people, + data_home=mock_data_home, + resize=None, + slice_=None, + color=True, + download_if_missing=False, + ) + check_return_X_y(lfw_people, fetch_func) + + +def test_load_fake_lfw_people_too_restrictive(mock_data_home): + with pytest.raises(ValueError): + fetch_lfw_people( + data_home=mock_data_home, + min_faces_per_person=100, + download_if_missing=False, + ) + + +def test_load_empty_lfw_pairs(mock_empty_data_home): + with pytest.raises(OSError): + fetch_lfw_pairs(data_home=mock_empty_data_home, download_if_missing=False) + + +def test_load_fake_lfw_pairs(mock_data_home): + lfw_pairs_train = fetch_lfw_pairs( + data_home=mock_data_home, download_if_missing=False + ) + + # The data is croped around the center as a rectangular bounding box + # around the face. Colors are converted to gray levels: + assert lfw_pairs_train.pairs.shape == (10, 2, 62, 47) + + # the target is whether the person is the same or not + assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) + + # names of the persons can be found using the target_names array + expected_classes = ["Different persons", "Same person"] + assert_array_equal(lfw_pairs_train.target_names, expected_classes) + + # It is possible to ask for the original data without any croping or color + # conversion + lfw_pairs_train = fetch_lfw_pairs( + data_home=mock_data_home, + resize=None, + slice_=None, + color=True, + download_if_missing=False, + ) + assert lfw_pairs_train.pairs.shape == (10, 2, 250, 250, 3) + + # the ids and class names are the same as previously + assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) + assert_array_equal(lfw_pairs_train.target_names, expected_classes) + + assert lfw_pairs_train.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:") + + +def test_fetch_lfw_people_internal_cropping(mock_data_home): + """Check that we properly crop the images. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/24942 + """ + # If cropping was not done properly and we don't resize the images, the images would + # have their original size (250x250) and the image would not fit in the NumPy array + # pre-allocated based on `slice_` parameter. + slice_ = (slice(70, 195), slice(78, 172)) + lfw = fetch_lfw_people( + data_home=mock_data_home, + min_faces_per_person=3, + download_if_missing=False, + resize=None, + slice_=slice_, + ) + assert lfw.images[0].shape == ( + slice_[0].stop - slice_[0].start, + slice_[1].stop - slice_[1].start, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_olivetti_faces.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_olivetti_faces.py new file mode 100644 index 0000000000000000000000000000000000000000..83c377dbd3bee37778c10911856c4a3554bf3668 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_olivetti_faces.py @@ -0,0 +1,26 @@ +"""Test Olivetti faces fetcher, if the data is available, +or if specifically requested via environment variable +(e.g. for CI jobs).""" + +import numpy as np + +from sklearn.datasets.tests.test_common import check_return_X_y +from sklearn.utils import Bunch +from sklearn.utils._testing import assert_array_equal + + +def test_olivetti_faces(fetch_olivetti_faces_fxt): + data = fetch_olivetti_faces_fxt(shuffle=True, random_state=0) + + assert isinstance(data, Bunch) + for expected_keys in ("data", "images", "target", "DESCR"): + assert expected_keys in data.keys() + + assert data.data.shape == (400, 4096) + assert data.images.shape == (400, 64, 64) + assert data.target.shape == (400,) + assert_array_equal(np.unique(np.sort(data.target)), np.arange(40)) + assert data.DESCR.startswith(".. _olivetti_faces_dataset:") + + # test the return_X_y option + check_return_X_y(data, fetch_olivetti_faces_fxt) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_openml.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_openml.py new file mode 100644 index 0000000000000000000000000000000000000000..0efa5b43737f6f1e096b6f4f46f07dde9ac54c44 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_openml.py @@ -0,0 +1,1636 @@ +"""Test the openml loader.""" + +import gzip +import json +import os +import re +from functools import partial +from importlib import resources +from io import BytesIO +from urllib.error import HTTPError + +import numpy as np +import pytest +import scipy.sparse + +import sklearn +from sklearn import config_context +from sklearn.datasets import fetch_openml as fetch_openml_orig +from sklearn.datasets._openml import ( + _get_local_path, + _open_openml_url, + _retry_with_clean_cache, +) +from sklearn.utils import Bunch +from sklearn.utils._optional_dependencies import check_pandas_support +from sklearn.utils._testing import ( + SkipTest, + assert_allclose, + assert_array_equal, +) + +OPENML_TEST_DATA_MODULE = "sklearn.datasets.tests.data.openml" +# if True, urlopen will be monkey patched to only use local files +test_offline = True +_MONKEY_PATCH_LOCAL_OPENML_PATH = "data/v1/download/{}" + + +class _MockHTTPResponse: + def __init__(self, data, is_gzip): + self.data = data + self.is_gzip = is_gzip + + def read(self, amt=-1): + return self.data.read(amt) + + def close(self): + self.data.close() + + def info(self): + if self.is_gzip: + return {"Content-Encoding": "gzip"} + return {} + + def __iter__(self): + return iter(self.data) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return False + + +# Disable the disk-based cache when testing `fetch_openml`: +# the mock data in sklearn/datasets/tests/data/openml/ is not always consistent +# with the version on openml.org. If one were to load the dataset outside of +# the tests, it may result in data that does not represent openml.org. +fetch_openml = partial(fetch_openml_orig, data_home=None) + + +def _monkey_patch_webbased_functions(context, data_id, gzip_response): + # monkey patches the urlopen function. Important note: Do NOT use this + # in combination with a regular cache directory, as the files that are + # stored as cache should not be mixed up with real openml datasets + url_prefix_data_description = "https://api.openml.org/api/v1/json/data/" + url_prefix_data_features = "https://api.openml.org/api/v1/json/data/features/" + url_prefix_download_data = "https://www.openml.org/data/v1/download" + url_prefix_data_list = "https://api.openml.org/api/v1/json/data/list/" + + path_suffix = ".gz" + read_fn = gzip.open + + data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}" + + def _file_name(url, suffix): + output = ( + re.sub(r"\W", "-", url[len("https://api.openml.org/") :]) + + suffix + + path_suffix + ) + # Shorten the filenames to have better compatibility with windows 10 + # and filenames > 260 characters + return ( + output.replace("-json-data-list", "-jdl") + .replace("-json-data-features", "-jdf") + .replace("-json-data-qualities", "-jdq") + .replace("-json-data", "-jd") + .replace("-data_name", "-dn") + .replace("-download", "-dl") + .replace("-limit", "-l") + .replace("-data_version", "-dv") + .replace("-status", "-s") + .replace("-deactivated", "-dact") + .replace("-active", "-act") + ) + + def _mock_urlopen_shared(url, has_gzip_header, expected_prefix, suffix): + assert url.startswith(expected_prefix), ( + f"{expected_prefix!r} does not match {url!r}" + ) + + data_file_name = _file_name(url, suffix) + data_file_path = resources.files(data_module) / data_file_name + + with data_file_path.open("rb") as f: + if has_gzip_header and gzip_response: + fp = BytesIO(f.read()) + return _MockHTTPResponse(fp, True) + else: + decompressed_f = read_fn(f, "rb") + fp = BytesIO(decompressed_f.read()) + return _MockHTTPResponse(fp, False) + + def _mock_urlopen_data_description(url, has_gzip_header): + return _mock_urlopen_shared( + url=url, + has_gzip_header=has_gzip_header, + expected_prefix=url_prefix_data_description, + suffix=".json", + ) + + def _mock_urlopen_data_features(url, has_gzip_header): + return _mock_urlopen_shared( + url=url, + has_gzip_header=has_gzip_header, + expected_prefix=url_prefix_data_features, + suffix=".json", + ) + + def _mock_urlopen_download_data(url, has_gzip_header): + # For simplicity the mock filenames don't contain the filename, i.e. + # the last part of the data description url after the last /. + # For example for id_1, data description download url is: + # gunzip -c sklearn/datasets/tests/data/openml/id_1/api-v1-jd-1.json.gz | grep '"url" # noqa: E501 + # "https:\/\/www.openml.org\/data\/v1\/download\/1\/anneal.arff" + # but the mock filename does not contain anneal.arff and is: + # sklearn/datasets/tests/data/openml/id_1/data-v1-dl-1.arff.gz. + # We only keep the part of the url before the last / + url_without_filename = url.rsplit("/", 1)[0] + + return _mock_urlopen_shared( + url=url_without_filename, + has_gzip_header=has_gzip_header, + expected_prefix=url_prefix_download_data, + suffix=".arff", + ) + + def _mock_urlopen_data_list(url, has_gzip_header): + assert url.startswith(url_prefix_data_list), ( + f"{url_prefix_data_list!r} does not match {url!r}" + ) + + data_file_name = _file_name(url, ".json") + data_file_path = resources.files(data_module) / data_file_name + + # load the file itself, to simulate a http error + with data_file_path.open("rb") as f: + decompressed_f = read_fn(f, "rb") + decoded_s = decompressed_f.read().decode("utf-8") + json_data = json.loads(decoded_s) + if "error" in json_data: + raise HTTPError( + url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO() + ) + + with data_file_path.open("rb") as f: + if has_gzip_header: + fp = BytesIO(f.read()) + return _MockHTTPResponse(fp, True) + else: + decompressed_f = read_fn(f, "rb") + fp = BytesIO(decompressed_f.read()) + return _MockHTTPResponse(fp, False) + + def _mock_urlopen(request, *args, **kwargs): + url = request.get_full_url() + has_gzip_header = request.get_header("Accept-encoding") == "gzip" + if url.startswith(url_prefix_data_list): + return _mock_urlopen_data_list(url, has_gzip_header) + elif url.startswith(url_prefix_data_features): + return _mock_urlopen_data_features(url, has_gzip_header) + elif url.startswith(url_prefix_download_data): + return _mock_urlopen_download_data(url, has_gzip_header) + elif url.startswith(url_prefix_data_description): + return _mock_urlopen_data_description(url, has_gzip_header) + else: + raise ValueError("Unknown mocking URL pattern: %s" % url) + + # XXX: Global variable + if test_offline: + context.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen) + + +############################################################################### +# Test the behaviour of `fetch_openml` depending of the input parameters. + + +@pytest.mark.parametrize( + "data_id, dataset_params, n_samples, n_features, n_targets", + [ + # iris + (61, {"data_id": 61}, 150, 4, 1), + (61, {"name": "iris", "version": 1}, 150, 4, 1), + # anneal + (2, {"data_id": 2}, 11, 38, 1), + (2, {"name": "anneal", "version": 1}, 11, 38, 1), + # cpu + (561, {"data_id": 561}, 209, 7, 1), + (561, {"name": "cpu", "version": 1}, 209, 7, 1), + # emotions + (40589, {"data_id": 40589}, 13, 72, 6), + # adult-census + (1119, {"data_id": 1119}, 10, 14, 1), + (1119, {"name": "adult-census"}, 10, 14, 1), + # miceprotein + (40966, {"data_id": 40966}, 7, 77, 1), + (40966, {"name": "MiceProtein"}, 7, 77, 1), + # titanic + (40945, {"data_id": 40945}, 1309, 13, 1), + ], +) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_as_frame_true( + monkeypatch, + data_id, + dataset_params, + n_samples, + n_features, + n_targets, + parser, + gzip_response, +): + """Check the behaviour of `fetch_openml` with `as_frame=True`. + + Fetch by ID and/or name (depending if the file was previously cached). + """ + pd = pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response) + bunch = fetch_openml( + as_frame=True, + cache=False, + parser=parser, + **dataset_params, + ) + + assert int(bunch.details["id"]) == data_id + assert isinstance(bunch, Bunch) + + assert isinstance(bunch.frame, pd.DataFrame) + assert bunch.frame.shape == (n_samples, n_features + n_targets) + + assert isinstance(bunch.data, pd.DataFrame) + assert bunch.data.shape == (n_samples, n_features) + + if n_targets == 1: + assert isinstance(bunch.target, pd.Series) + assert bunch.target.shape == (n_samples,) + else: + assert isinstance(bunch.target, pd.DataFrame) + assert bunch.target.shape == (n_samples, n_targets) + + assert bunch.categories is None + + +@pytest.mark.parametrize( + "data_id, dataset_params, n_samples, n_features, n_targets", + [ + # iris + (61, {"data_id": 61}, 150, 4, 1), + (61, {"name": "iris", "version": 1}, 150, 4, 1), + # anneal + (2, {"data_id": 2}, 11, 38, 1), + (2, {"name": "anneal", "version": 1}, 11, 38, 1), + # cpu + (561, {"data_id": 561}, 209, 7, 1), + (561, {"name": "cpu", "version": 1}, 209, 7, 1), + # emotions + (40589, {"data_id": 40589}, 13, 72, 6), + # adult-census + (1119, {"data_id": 1119}, 10, 14, 1), + (1119, {"name": "adult-census"}, 10, 14, 1), + # miceprotein + (40966, {"data_id": 40966}, 7, 77, 1), + (40966, {"name": "MiceProtein"}, 7, 77, 1), + ], +) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_as_frame_false( + monkeypatch, + data_id, + dataset_params, + n_samples, + n_features, + n_targets, + parser, +): + """Check the behaviour of `fetch_openml` with `as_frame=False`. + + Fetch both by ID and/or name + version. + """ + pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch = fetch_openml( + as_frame=False, + cache=False, + parser=parser, + **dataset_params, + ) + assert int(bunch.details["id"]) == data_id + assert isinstance(bunch, Bunch) + + assert bunch.frame is None + + assert isinstance(bunch.data, np.ndarray) + assert bunch.data.shape == (n_samples, n_features) + + assert isinstance(bunch.target, np.ndarray) + if n_targets == 1: + assert bunch.target.shape == (n_samples,) + else: + assert bunch.target.shape == (n_samples, n_targets) + + assert isinstance(bunch.categories, dict) + + +@pytest.mark.parametrize("data_id", [61, 1119, 40945]) +def test_fetch_openml_consistency_parser(monkeypatch, data_id): + """Check the consistency of the LIAC-ARFF and pandas parsers.""" + pd = pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch_liac = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser="liac-arff", + ) + bunch_pandas = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser="pandas", + ) + + # The data frames for the input features should match up to some numerical + # dtype conversions (e.g. float64 <=> Int64) due to limitations of the + # LIAC-ARFF parser. + data_liac, data_pandas = bunch_liac.data, bunch_pandas.data + + def convert_numerical_dtypes(series): + pandas_series = data_pandas[series.name] + if pd.api.types.is_numeric_dtype(pandas_series): + return series.astype(pandas_series.dtype) + else: + return series + + data_liac_with_fixed_dtypes = data_liac.apply(convert_numerical_dtypes) + pd.testing.assert_frame_equal(data_liac_with_fixed_dtypes, data_pandas) + + # Let's also check that the .frame attributes also match + frame_liac, frame_pandas = bunch_liac.frame, bunch_pandas.frame + + # Note that the .frame attribute is a superset of the .data attribute: + pd.testing.assert_frame_equal(frame_pandas[bunch_pandas.feature_names], data_pandas) + + # However the remaining columns, typically the target(s), are not necessarily + # dtyped similarly by both parsers due to limitations of the LIAC-ARFF parser. + # Therefore, extra dtype conversions are required for those columns: + + def convert_numerical_and_categorical_dtypes(series): + pandas_series = frame_pandas[series.name] + if pd.api.types.is_numeric_dtype(pandas_series): + return series.astype(pandas_series.dtype) + elif isinstance(pandas_series.dtype, pd.CategoricalDtype): + # Compare categorical features by converting categorical liac uses + # strings to denote the categories, we rename the categories to make + # them comparable to the pandas parser. Fixing this behavior in + # LIAC-ARFF would allow to check the consistency in the future but + # we do not plan to maintain the LIAC-ARFF on the long term. + return series.cat.rename_categories(pandas_series.cat.categories) + else: + return series + + frame_liac_with_fixed_dtypes = frame_liac.apply( + convert_numerical_and_categorical_dtypes + ) + pd.testing.assert_frame_equal(frame_liac_with_fixed_dtypes, frame_pandas) + + +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_equivalence_array_dataframe(monkeypatch, parser): + """Check the equivalence of the dataset when using `as_frame=False` and + `as_frame=True`. + """ + pytest.importorskip("pandas") + + data_id = 61 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch_as_frame_true = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) + + bunch_as_frame_false = fetch_openml( + data_id=data_id, + as_frame=False, + cache=False, + parser=parser, + ) + + assert_allclose(bunch_as_frame_false.data, bunch_as_frame_true.data) + assert_array_equal(bunch_as_frame_false.target, bunch_as_frame_true.target) + + +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_iris_pandas(monkeypatch, parser): + """Check fetching on a numerical only dataset with string labels.""" + pd = pytest.importorskip("pandas") + CategoricalDtype = pd.api.types.CategoricalDtype + data_id = 61 + data_shape = (150, 4) + target_shape = (150,) + frame_shape = (150, 5) + + target_dtype = CategoricalDtype( + ["Iris-setosa", "Iris-versicolor", "Iris-virginica"] + ) + data_dtypes = [np.float64] * 4 + data_names = ["sepallength", "sepalwidth", "petallength", "petalwidth"] + target_name = "class" + + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + + bunch = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) + data = bunch.data + target = bunch.target + frame = bunch.frame + + assert isinstance(data, pd.DataFrame) + assert np.all(data.dtypes == data_dtypes) + assert data.shape == data_shape + assert np.all(data.columns == data_names) + assert np.all(bunch.feature_names == data_names) + assert bunch.target_names == [target_name] + + assert isinstance(target, pd.Series) + assert target.dtype == target_dtype + assert target.shape == target_shape + assert target.name == target_name + assert target.index.is_unique + + assert isinstance(frame, pd.DataFrame) + assert frame.shape == frame_shape + assert np.all(frame.dtypes == data_dtypes + [target_dtype]) + assert frame.index.is_unique + + +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +@pytest.mark.parametrize("target_column", ["petalwidth", ["petalwidth", "petallength"]]) +def test_fetch_openml_forcing_targets(monkeypatch, parser, target_column): + """Check that we can force the target to not be the default target.""" + pd = pytest.importorskip("pandas") + + data_id = 61 + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + bunch_forcing_target = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + target_column=target_column, + parser=parser, + ) + bunch_default = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) + + pd.testing.assert_frame_equal(bunch_forcing_target.frame, bunch_default.frame) + if isinstance(target_column, list): + pd.testing.assert_index_equal( + bunch_forcing_target.target.columns, pd.Index(target_column) + ) + assert bunch_forcing_target.data.shape == (150, 3) + else: + assert bunch_forcing_target.target.name == target_column + assert bunch_forcing_target.data.shape == (150, 4) + + +@pytest.mark.parametrize("data_id", [61, 2, 561, 40589, 1119]) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_equivalence_frame_return_X_y(monkeypatch, data_id, parser): + """Check the behaviour of `return_X_y=True` when `as_frame=True`.""" + pd = pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + return_X_y=False, + parser=parser, + ) + X, y = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + return_X_y=True, + parser=parser, + ) + + pd.testing.assert_frame_equal(bunch.data, X) + if isinstance(y, pd.Series): + pd.testing.assert_series_equal(bunch.target, y) + else: + pd.testing.assert_frame_equal(bunch.target, y) + + +@pytest.mark.parametrize("data_id", [61, 561, 40589, 1119]) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_equivalence_array_return_X_y(monkeypatch, data_id, parser): + """Check the behaviour of `return_X_y=True` when `as_frame=False`.""" + pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch = fetch_openml( + data_id=data_id, + as_frame=False, + cache=False, + return_X_y=False, + parser=parser, + ) + X, y = fetch_openml( + data_id=data_id, + as_frame=False, + cache=False, + return_X_y=True, + parser=parser, + ) + + assert_array_equal(bunch.data, X) + assert_array_equal(bunch.target, y) + + +def test_fetch_openml_difference_parsers(monkeypatch): + """Check the difference between liac-arff and pandas parser.""" + pytest.importorskip("pandas") + + data_id = 1119 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + # When `as_frame=False`, the categories will be ordinally encoded with + # liac-arff parser while this is not the case with pandas parser. + as_frame = False + bunch_liac_arff = fetch_openml( + data_id=data_id, + as_frame=as_frame, + cache=False, + parser="liac-arff", + ) + bunch_pandas = fetch_openml( + data_id=data_id, + as_frame=as_frame, + cache=False, + parser="pandas", + ) + + assert bunch_liac_arff.data.dtype.kind == "f" + assert bunch_pandas.data.dtype == "O" + + +############################################################################### +# Test the ARFF parsing on several dataset to check if detect the correct +# types (categories, integers, floats). + + +@pytest.fixture(scope="module") +def datasets_column_names(): + """Returns the columns names for each dataset.""" + return { + 61: ["sepallength", "sepalwidth", "petallength", "petalwidth", "class"], + 2: [ + "family", + "product-type", + "steel", + "carbon", + "hardness", + "temper_rolling", + "condition", + "formability", + "strength", + "non-ageing", + "surface-finish", + "surface-quality", + "enamelability", + "bc", + "bf", + "bt", + "bw%2Fme", + "bl", + "m", + "chrom", + "phos", + "cbond", + "marvi", + "exptl", + "ferro", + "corr", + "blue%2Fbright%2Fvarn%2Fclean", + "lustre", + "jurofm", + "s", + "p", + "shape", + "thick", + "width", + "len", + "oil", + "bore", + "packing", + "class", + ], + 561: ["vendor", "MYCT", "MMIN", "MMAX", "CACH", "CHMIN", "CHMAX", "class"], + 40589: [ + "Mean_Acc1298_Mean_Mem40_Centroid", + "Mean_Acc1298_Mean_Mem40_Rolloff", + "Mean_Acc1298_Mean_Mem40_Flux", + "Mean_Acc1298_Mean_Mem40_MFCC_0", + "Mean_Acc1298_Mean_Mem40_MFCC_1", + "Mean_Acc1298_Mean_Mem40_MFCC_2", + "Mean_Acc1298_Mean_Mem40_MFCC_3", + "Mean_Acc1298_Mean_Mem40_MFCC_4", + "Mean_Acc1298_Mean_Mem40_MFCC_5", + "Mean_Acc1298_Mean_Mem40_MFCC_6", + "Mean_Acc1298_Mean_Mem40_MFCC_7", + "Mean_Acc1298_Mean_Mem40_MFCC_8", + "Mean_Acc1298_Mean_Mem40_MFCC_9", + "Mean_Acc1298_Mean_Mem40_MFCC_10", + "Mean_Acc1298_Mean_Mem40_MFCC_11", + "Mean_Acc1298_Mean_Mem40_MFCC_12", + "Mean_Acc1298_Std_Mem40_Centroid", + "Mean_Acc1298_Std_Mem40_Rolloff", + "Mean_Acc1298_Std_Mem40_Flux", + "Mean_Acc1298_Std_Mem40_MFCC_0", + "Mean_Acc1298_Std_Mem40_MFCC_1", + "Mean_Acc1298_Std_Mem40_MFCC_2", + "Mean_Acc1298_Std_Mem40_MFCC_3", + "Mean_Acc1298_Std_Mem40_MFCC_4", + "Mean_Acc1298_Std_Mem40_MFCC_5", + "Mean_Acc1298_Std_Mem40_MFCC_6", + "Mean_Acc1298_Std_Mem40_MFCC_7", + "Mean_Acc1298_Std_Mem40_MFCC_8", + "Mean_Acc1298_Std_Mem40_MFCC_9", + "Mean_Acc1298_Std_Mem40_MFCC_10", + "Mean_Acc1298_Std_Mem40_MFCC_11", + "Mean_Acc1298_Std_Mem40_MFCC_12", + "Std_Acc1298_Mean_Mem40_Centroid", + "Std_Acc1298_Mean_Mem40_Rolloff", + "Std_Acc1298_Mean_Mem40_Flux", + "Std_Acc1298_Mean_Mem40_MFCC_0", + "Std_Acc1298_Mean_Mem40_MFCC_1", + "Std_Acc1298_Mean_Mem40_MFCC_2", + "Std_Acc1298_Mean_Mem40_MFCC_3", + "Std_Acc1298_Mean_Mem40_MFCC_4", + "Std_Acc1298_Mean_Mem40_MFCC_5", + "Std_Acc1298_Mean_Mem40_MFCC_6", + "Std_Acc1298_Mean_Mem40_MFCC_7", + "Std_Acc1298_Mean_Mem40_MFCC_8", + "Std_Acc1298_Mean_Mem40_MFCC_9", + "Std_Acc1298_Mean_Mem40_MFCC_10", + "Std_Acc1298_Mean_Mem40_MFCC_11", + "Std_Acc1298_Mean_Mem40_MFCC_12", + "Std_Acc1298_Std_Mem40_Centroid", + "Std_Acc1298_Std_Mem40_Rolloff", + "Std_Acc1298_Std_Mem40_Flux", + "Std_Acc1298_Std_Mem40_MFCC_0", + "Std_Acc1298_Std_Mem40_MFCC_1", + "Std_Acc1298_Std_Mem40_MFCC_2", + "Std_Acc1298_Std_Mem40_MFCC_3", + "Std_Acc1298_Std_Mem40_MFCC_4", + "Std_Acc1298_Std_Mem40_MFCC_5", + "Std_Acc1298_Std_Mem40_MFCC_6", + "Std_Acc1298_Std_Mem40_MFCC_7", + "Std_Acc1298_Std_Mem40_MFCC_8", + "Std_Acc1298_Std_Mem40_MFCC_9", + "Std_Acc1298_Std_Mem40_MFCC_10", + "Std_Acc1298_Std_Mem40_MFCC_11", + "Std_Acc1298_Std_Mem40_MFCC_12", + "BH_LowPeakAmp", + "BH_LowPeakBPM", + "BH_HighPeakAmp", + "BH_HighPeakBPM", + "BH_HighLowRatio", + "BHSUM1", + "BHSUM2", + "BHSUM3", + "amazed.suprised", + "happy.pleased", + "relaxing.calm", + "quiet.still", + "sad.lonely", + "angry.aggresive", + ], + 1119: [ + "age", + "workclass", + "fnlwgt:", + "education:", + "education-num:", + "marital-status:", + "occupation:", + "relationship:", + "race:", + "sex:", + "capital-gain:", + "capital-loss:", + "hours-per-week:", + "native-country:", + "class", + ], + 40966: [ + "DYRK1A_N", + "ITSN1_N", + "BDNF_N", + "NR1_N", + "NR2A_N", + "pAKT_N", + "pBRAF_N", + "pCAMKII_N", + "pCREB_N", + "pELK_N", + "pERK_N", + "pJNK_N", + "PKCA_N", + "pMEK_N", + "pNR1_N", + "pNR2A_N", + "pNR2B_N", + "pPKCAB_N", + "pRSK_N", + "AKT_N", + "BRAF_N", + "CAMKII_N", + "CREB_N", + "ELK_N", + "ERK_N", + "GSK3B_N", + "JNK_N", + "MEK_N", + "TRKA_N", + "RSK_N", + "APP_N", + "Bcatenin_N", + "SOD1_N", + "MTOR_N", + "P38_N", + "pMTOR_N", + "DSCR1_N", + "AMPKA_N", + "NR2B_N", + "pNUMB_N", + "RAPTOR_N", + "TIAM1_N", + "pP70S6_N", + "NUMB_N", + "P70S6_N", + "pGSK3B_N", + "pPKCG_N", + "CDK5_N", + "S6_N", + "ADARB1_N", + "AcetylH3K9_N", + "RRP1_N", + "BAX_N", + "ARC_N", + "ERBB4_N", + "nNOS_N", + "Tau_N", + "GFAP_N", + "GluR3_N", + "GluR4_N", + "IL1B_N", + "P3525_N", + "pCASP9_N", + "PSD95_N", + "SNCA_N", + "Ubiquitin_N", + "pGSK3B_Tyr216_N", + "SHH_N", + "BAD_N", + "BCL2_N", + "pS6_N", + "pCFOS_N", + "SYP_N", + "H3AcK18_N", + "EGR1_N", + "H3MeK4_N", + "CaNA_N", + "class", + ], + 40945: [ + "pclass", + "survived", + "name", + "sex", + "age", + "sibsp", + "parch", + "ticket", + "fare", + "cabin", + "embarked", + "boat", + "body", + "home.dest", + ], + } + + +@pytest.fixture(scope="module") +def datasets_missing_values(): + return { + 61: {}, + 2: { + "family": 11, + "temper_rolling": 9, + "condition": 2, + "formability": 4, + "non-ageing": 10, + "surface-finish": 11, + "enamelability": 11, + "bc": 11, + "bf": 10, + "bt": 11, + "bw%2Fme": 8, + "bl": 9, + "m": 11, + "chrom": 11, + "phos": 11, + "cbond": 10, + "marvi": 11, + "exptl": 11, + "ferro": 11, + "corr": 11, + "blue%2Fbright%2Fvarn%2Fclean": 11, + "lustre": 8, + "jurofm": 11, + "s": 11, + "p": 11, + "oil": 10, + "packing": 11, + }, + 561: {}, + 40589: {}, + 1119: {}, + 40966: {"BCL2_N": 7}, + 40945: { + "age": 263, + "fare": 1, + "cabin": 1014, + "embarked": 2, + "boat": 823, + "body": 1188, + "home.dest": 564, + }, + } + + +@pytest.mark.parametrize( + "data_id, parser, expected_n_categories, expected_n_floats, expected_n_ints", + [ + # iris dataset + (61, "liac-arff", 1, 4, 0), + (61, "pandas", 1, 4, 0), + # anneal dataset + (2, "liac-arff", 33, 6, 0), + (2, "pandas", 33, 2, 4), + # cpu dataset + (561, "liac-arff", 1, 7, 0), + (561, "pandas", 1, 0, 7), + # emotions dataset + (40589, "liac-arff", 6, 72, 0), + (40589, "pandas", 6, 69, 3), + # adult-census dataset + (1119, "liac-arff", 9, 6, 0), + (1119, "pandas", 9, 0, 6), + # miceprotein + (40966, "liac-arff", 1, 77, 0), + (40966, "pandas", 1, 77, 0), + # titanic + (40945, "liac-arff", 3, 6, 0), + (40945, "pandas", 3, 3, 3), + ], +) +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_types_inference( + monkeypatch, + data_id, + parser, + expected_n_categories, + expected_n_floats, + expected_n_ints, + gzip_response, + datasets_column_names, + datasets_missing_values, +): + """Check that `fetch_openml` infer the right number of categories, integers, and + floats.""" + pd = pytest.importorskip("pandas") + CategoricalDtype = pd.api.types.CategoricalDtype + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response) + + bunch = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) + frame = bunch.frame + + n_categories = len( + [dtype for dtype in frame.dtypes if isinstance(dtype, CategoricalDtype)] + ) + n_floats = len([dtype for dtype in frame.dtypes if dtype.kind == "f"]) + n_ints = len([dtype for dtype in frame.dtypes if dtype.kind == "i"]) + + assert n_categories == expected_n_categories + assert n_floats == expected_n_floats + assert n_ints == expected_n_ints + + assert frame.columns.tolist() == datasets_column_names[data_id] + + frame_feature_to_n_nan = frame.isna().sum().to_dict() + for name, n_missing in frame_feature_to_n_nan.items(): + expected_missing = datasets_missing_values[data_id].get(name, 0) + assert n_missing == expected_missing + + +############################################################################### +# Test some more specific behaviour + + +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"parser": "unknown"}, + "The 'parser' parameter of fetch_openml must be a str among", + ), + ( + {"as_frame": "unknown"}, + "The 'as_frame' parameter of fetch_openml must be an instance", + ), + ], +) +def test_fetch_openml_validation_parameter(monkeypatch, params, err_msg): + data_id = 1119 + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + with pytest.raises(ValueError, match=err_msg): + fetch_openml(data_id=data_id, **params) + + +@pytest.mark.parametrize( + "params", + [ + {"as_frame": True, "parser": "auto"}, + {"as_frame": "auto", "parser": "auto"}, + {"as_frame": False, "parser": "pandas"}, + {"as_frame": False, "parser": "auto"}, + ], +) +def test_fetch_openml_requires_pandas_error(monkeypatch, params): + """Check that we raise the proper errors when we require pandas.""" + data_id = 1119 + try: + check_pandas_support("test_fetch_openml_requires_pandas") + except ImportError: + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + err_msg = "requires pandas to be installed. Alternatively, explicitly" + with pytest.raises(ImportError, match=err_msg): + fetch_openml(data_id=data_id, **params) + else: + raise SkipTest("This test requires pandas to not be installed.") + + +@pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive") +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"parser": "pandas"}, + "Sparse ARFF datasets cannot be loaded with parser='pandas'", + ), + ( + {"as_frame": True}, + "Sparse ARFF datasets cannot be loaded with as_frame=True.", + ), + ( + {"parser": "pandas", "as_frame": True}, + "Sparse ARFF datasets cannot be loaded with as_frame=True.", + ), + ], +) +def test_fetch_openml_sparse_arff_error(monkeypatch, params, err_msg): + """Check that we raise the expected error for sparse ARFF datasets and + a wrong set of incompatible parameters. + """ + pytest.importorskip("pandas") + data_id = 292 + + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + with pytest.raises(ValueError, match=err_msg): + fetch_openml( + data_id=data_id, + cache=False, + **params, + ) + + +@pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive") +@pytest.mark.parametrize( + "data_id, data_type", + [ + (61, "dataframe"), # iris dataset version 1 + (292, "sparse"), # Australian dataset version 1 + ], +) +def test_fetch_openml_auto_mode(monkeypatch, data_id, data_type): + """Check the auto mode of `fetch_openml`.""" + pd = pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + data = fetch_openml(data_id=data_id, as_frame="auto", cache=False) + klass = pd.DataFrame if data_type == "dataframe" else scipy.sparse.csr_matrix + assert isinstance(data.data, klass) + + +def test_convert_arff_data_dataframe_warning_low_memory_pandas(monkeypatch): + """Check that we raise a warning regarding the working memory when using + LIAC-ARFF parser.""" + pytest.importorskip("pandas") + + data_id = 1119 + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + + msg = "Could not adhere to working_memory config." + with pytest.warns(UserWarning, match=msg): + with config_context(working_memory=1e-6): + fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser="liac-arff", + ) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_iris_warn_multiple_version(monkeypatch, gzip_response): + """Check that a warning is raised when multiple versions exist and no version is + requested.""" + data_id = 61 + data_name = "iris" + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + + msg = re.escape( + "Multiple active versions of the dataset matching the name" + " iris exist. Versions may be fundamentally different, " + "returning version 1. Available versions:\n" + "- version 1, status: active\n" + " url: https://www.openml.org/search?type=data&id=61\n" + "- version 3, status: active\n" + " url: https://www.openml.org/search?type=data&id=969\n" + ) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + name=data_name, + as_frame=False, + cache=False, + parser="liac-arff", + ) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_no_target(monkeypatch, gzip_response): + """Check that we can get a dataset without target.""" + data_id = 61 + target_column = None + expected_observations = 150 + expected_features = 5 + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + data = fetch_openml( + data_id=data_id, + target_column=target_column, + cache=False, + as_frame=False, + parser="liac-arff", + ) + assert data.data.shape == (expected_observations, expected_features) + assert data.target is None + + +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_missing_values_pandas(monkeypatch, gzip_response, parser): + """check that missing values in categories are compatible with pandas + categorical""" + pytest.importorskip("pandas") + + data_id = 42585 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response) + penguins = fetch_openml( + data_id=data_id, + cache=False, + as_frame=True, + parser=parser, + ) + + cat_dtype = penguins.data.dtypes["sex"] + # there are nans in the categorical + assert penguins.data["sex"].isna().any() + assert_array_equal(cat_dtype.categories, ["FEMALE", "MALE", "_"]) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize( + "dataset_params", + [ + {"data_id": 40675}, + {"data_id": None, "name": "glass2", "version": 1}, + ], +) +def test_fetch_openml_inactive(monkeypatch, gzip_response, dataset_params): + """Check that we raise a warning when the dataset is inactive.""" + data_id = 40675 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + msg = "Version 1 of dataset glass2 is inactive," + with pytest.warns(UserWarning, match=msg): + glass2 = fetch_openml( + cache=False, as_frame=False, parser="liac-arff", **dataset_params + ) + assert glass2.data.shape == (163, 9) + assert glass2.details["id"] == "40675" + + +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize( + "data_id, params, err_type, err_msg", + [ + (40675, {"name": "glass2"}, ValueError, "No active dataset glass2 found"), + ( + 61, + {"data_id": 61, "target_column": ["sepalwidth", "class"]}, + ValueError, + "Can only handle homogeneous multi-target datasets", + ), + ( + 40945, + {"data_id": 40945, "as_frame": False}, + ValueError, + ( + "STRING attributes are not supported for array representation. Try" + " as_frame=True" + ), + ), + ( + 2, + {"data_id": 2, "target_column": "family", "as_frame": True}, + ValueError, + "Target column 'family'", + ), + ( + 2, + {"data_id": 2, "target_column": "family", "as_frame": False}, + ValueError, + "Target column 'family'", + ), + ( + 61, + {"data_id": 61, "target_column": "undefined"}, + KeyError, + "Could not find target_column='undefined'", + ), + ( + 61, + {"data_id": 61, "target_column": ["undefined", "class"]}, + KeyError, + "Could not find target_column='undefined'", + ), + ], +) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_error( + monkeypatch, gzip_response, data_id, params, err_type, err_msg, parser +): + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + if params.get("as_frame", True) or parser == "pandas": + pytest.importorskip("pandas") + with pytest.raises(err_type, match=err_msg): + fetch_openml(cache=False, parser=parser, **params) + + +@pytest.mark.parametrize( + "params, err_type, err_msg", + [ + ( + {"data_id": -1, "name": None, "version": "version"}, + ValueError, + "The 'version' parameter of fetch_openml must be an int in the range", + ), + ( + {"data_id": -1, "name": "nAmE"}, + ValueError, + "The 'data_id' parameter of fetch_openml must be an int in the range", + ), + ( + {"data_id": -1, "name": "nAmE", "version": "version"}, + ValueError, + "The 'version' parameter of fetch_openml must be an int", + ), + ( + {}, + ValueError, + "Neither name nor data_id are provided. Please provide name or data_id.", + ), + ], +) +def test_fetch_openml_raises_illegal_argument(params, err_type, err_msg): + with pytest.raises(err_type, match=err_msg): + fetch_openml(**params) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_warn_ignore_attribute(monkeypatch, gzip_response): + data_id = 40966 + expected_row_id_msg = "target_column='{}' has flag is_row_identifier." + expected_ignore_msg = "target_column='{}' has flag is_ignore." + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + # single column test + target_col = "MouseID" + msg = expected_row_id_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=target_col, + cache=False, + as_frame=False, + parser="liac-arff", + ) + target_col = "Genotype" + msg = expected_ignore_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=target_col, + cache=False, + as_frame=False, + parser="liac-arff", + ) + # multi column test + target_col = "MouseID" + msg = expected_row_id_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=[target_col, "class"], + cache=False, + as_frame=False, + parser="liac-arff", + ) + target_col = "Genotype" + msg = expected_ignore_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=[target_col, "class"], + cache=False, + as_frame=False, + parser="liac-arff", + ) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_dataset_with_openml_error(monkeypatch, gzip_response): + data_id = 1 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + msg = "OpenML registered a problem with the dataset. It might be unusable. Error:" + with pytest.warns(UserWarning, match=msg): + fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff") + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_dataset_with_openml_warning(monkeypatch, gzip_response): + data_id = 3 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + msg = "OpenML raised a warning on the dataset. It might be unusable. Warning:" + with pytest.warns(UserWarning, match=msg): + fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff") + + +def test_fetch_openml_overwrite_default_params_read_csv(monkeypatch): + """Check that we can overwrite the default parameters of `read_csv`.""" + pytest.importorskip("pandas") + data_id = 1590 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) + + common_params = { + "data_id": data_id, + "as_frame": True, + "cache": False, + "parser": "pandas", + } + + # By default, the initial spaces are skipped. We checked that setting the parameter + # `skipinitialspace` to False will have an effect. + adult_without_spaces = fetch_openml(**common_params) + adult_with_spaces = fetch_openml( + **common_params, read_csv_kwargs={"skipinitialspace": False} + ) + assert all( + cat.startswith(" ") for cat in adult_with_spaces.frame["class"].cat.categories + ) + assert not any( + cat.startswith(" ") + for cat in adult_without_spaces.frame["class"].cat.categories + ) + + +############################################################################### +# Test cache, retry mechanisms, checksum, etc. + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_open_openml_url_cache(monkeypatch, gzip_response, tmpdir): + data_id = 61 + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff" + url = f"https://www.openml.org/{openml_path}" + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) + # first fill the cache + response1 = _open_openml_url(url, cache_directory) + # assert file exists + location = _get_local_path(openml_path, cache_directory) + assert os.path.isfile(location) + # redownload, to utilize cache + response2 = _open_openml_url(url, cache_directory) + assert response1.read() == response2.read() + + +@pytest.mark.parametrize("write_to_disk", [True, False]) +def test_open_openml_url_unlinks_local_path(monkeypatch, tmpdir, write_to_disk): + data_id = 61 + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff" + url = f"https://www.openml.org/{openml_path}" + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) + location = _get_local_path(openml_path, cache_directory) + + def _mock_urlopen(request, *args, **kwargs): + if write_to_disk: + with open(location, "w") as f: + f.write("") + raise ValueError("Invalid request") + + monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen) + + with pytest.raises(ValueError, match="Invalid request"): + _open_openml_url(url, cache_directory) + + assert not os.path.exists(location) + + +def test_retry_with_clean_cache(tmpdir): + data_id = 61 + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) + location = _get_local_path(openml_path, cache_directory) + os.makedirs(os.path.dirname(location)) + + with open(location, "w") as f: + f.write("") + + @_retry_with_clean_cache(openml_path, cache_directory) + def _load_data(): + # The first call will raise an error since location exists + if os.path.exists(location): + raise Exception("File exist!") + return 1 + + warn_msg = "Invalid cache, redownloading file" + with pytest.warns(RuntimeWarning, match=warn_msg): + result = _load_data() + assert result == 1 + + +def test_retry_with_clean_cache_http_error(tmpdir): + data_id = 61 + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) + + @_retry_with_clean_cache(openml_path, cache_directory) + def _load_data(): + raise HTTPError( + url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO() + ) + + error_msg = "Simulated mock error" + with pytest.raises(HTTPError, match=error_msg): + _load_data() + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_cache(monkeypatch, gzip_response, tmpdir): + def _mock_urlopen_raise(request, *args, **kwargs): + raise ValueError( + "This mechanism intends to test correct cache" + "handling. As such, urlopen should never be " + "accessed. URL: %s" % request.get_full_url() + ) + + data_id = 61 + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + X_fetched, y_fetched = fetch_openml( + data_id=data_id, + cache=True, + data_home=cache_directory, + return_X_y=True, + as_frame=False, + parser="liac-arff", + ) + + monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen_raise) + + X_cached, y_cached = fetch_openml( + data_id=data_id, + cache=True, + data_home=cache_directory, + return_X_y=True, + as_frame=False, + parser="liac-arff", + ) + np.testing.assert_array_equal(X_fetched, X_cached) + np.testing.assert_array_equal(y_fetched, y_cached) + + +@pytest.mark.parametrize( + "as_frame, parser", + [ + (True, "liac-arff"), + (False, "liac-arff"), + (True, "pandas"), + (False, "pandas"), + ], +) +def test_fetch_openml_verify_checksum(monkeypatch, as_frame, tmpdir, parser): + """Check that the checksum is working as expected.""" + if as_frame or parser == "pandas": + pytest.importorskip("pandas") + + data_id = 2 + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + + # create a temporary modified arff file + original_data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}" + original_data_file_name = "data-v1-dl-1666876.arff.gz" + original_data_path = resources.files(original_data_module) / original_data_file_name + corrupt_copy_path = tmpdir / "test_invalid_checksum.arff" + with original_data_path.open("rb") as orig_file: + orig_gzip = gzip.open(orig_file, "rb") + data = bytearray(orig_gzip.read()) + data[len(data) - 1] = 37 + + with gzip.GzipFile(corrupt_copy_path, "wb") as modified_gzip: + modified_gzip.write(data) + + # Requests are already mocked by monkey_patch_webbased_functions. + # We want to reuse that mock for all requests except file download, + # hence creating a thin mock over the original mock + mocked_openml_url = sklearn.datasets._openml.urlopen + + def swap_file_mock(request, *args, **kwargs): + url = request.get_full_url() + if url.endswith("data/v1/download/1666876/anneal.arff"): + with open(corrupt_copy_path, "rb") as f: + corrupted_data = f.read() + return _MockHTTPResponse(BytesIO(corrupted_data), is_gzip=True) + else: + return mocked_openml_url(request) + + monkeypatch.setattr(sklearn.datasets._openml, "urlopen", swap_file_mock) + + # validate failed checksum + with pytest.raises(ValueError) as exc: + sklearn.datasets.fetch_openml( + data_id=data_id, cache=False, as_frame=as_frame, parser=parser + ) + # exception message should have file-path + assert exc.match("1666876") + + +def test_open_openml_url_retry_on_network_error(monkeypatch): + def _mock_urlopen_network_error(request, *args, **kwargs): + raise HTTPError( + url=None, code=404, msg="Simulated network error", hdrs=None, fp=BytesIO() + ) + + monkeypatch.setattr( + sklearn.datasets._openml, "urlopen", _mock_urlopen_network_error + ) + + invalid_openml_url = "https://api.openml.org/invalid-url" + + with pytest.warns( + UserWarning, + match=re.escape( + "A network error occurred while downloading" + f" {invalid_openml_url}. Retrying..." + ), + ) as record: + with pytest.raises(HTTPError, match="Simulated network error") as exc_info: + _open_openml_url(invalid_openml_url, None, delay=0) + assert len(record) == 3 + # Avoid a ResourceWarning on Python 3.14 and later. + exc_info.value.close() + + +############################################################################### +# Non-regressiont tests + + +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize("parser", ("liac-arff", "pandas")) +def test_fetch_openml_with_ignored_feature(monkeypatch, gzip_response, parser): + """Check that we can load the "zoo" dataset. + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/14340 + """ + if parser == "pandas": + pytest.importorskip("pandas") + data_id = 62 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + + dataset = sklearn.datasets.fetch_openml( + data_id=data_id, cache=False, as_frame=False, parser=parser + ) + assert dataset is not None + # The dataset has 17 features, including 1 ignored (animal), + # so we assert that we don't have the ignored feature in the final Bunch + assert dataset["data"].shape == (101, 16) + assert "animal" not in dataset["feature_names"] + + +def test_fetch_openml_strip_quotes(monkeypatch): + """Check that we strip the single quotes when used as a string delimiter. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/23381 + """ + pd = pytest.importorskip("pandas") + data_id = 40966 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) + + common_params = {"as_frame": True, "cache": False, "data_id": data_id} + mice_pandas = fetch_openml(parser="pandas", **common_params) + mice_liac_arff = fetch_openml(parser="liac-arff", **common_params) + pd.testing.assert_series_equal(mice_pandas.target, mice_liac_arff.target) + assert not mice_pandas.target.str.startswith("'").any() + assert not mice_pandas.target.str.endswith("'").any() + + # similar behaviour should be observed when the column is not the target + mice_pandas = fetch_openml(parser="pandas", target_column="NUMB_N", **common_params) + mice_liac_arff = fetch_openml( + parser="liac-arff", target_column="NUMB_N", **common_params + ) + pd.testing.assert_series_equal( + mice_pandas.frame["class"], mice_liac_arff.frame["class"] + ) + assert not mice_pandas.frame["class"].str.startswith("'").any() + assert not mice_pandas.frame["class"].str.endswith("'").any() + + +def test_fetch_openml_leading_whitespace(monkeypatch): + """Check that we can strip leading whitespace in pandas parser. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/25311 + """ + pd = pytest.importorskip("pandas") + data_id = 1590 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) + + common_params = {"as_frame": True, "cache": False, "data_id": data_id} + adult_pandas = fetch_openml(parser="pandas", **common_params) + adult_liac_arff = fetch_openml(parser="liac-arff", **common_params) + pd.testing.assert_series_equal( + adult_pandas.frame["class"], adult_liac_arff.frame["class"] + ) + + +def test_fetch_openml_quotechar_escapechar(monkeypatch): + """Check that we can handle escapechar and single/double quotechar. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/25478 + """ + pd = pytest.importorskip("pandas") + data_id = 42074 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) + + common_params = {"as_frame": True, "cache": False, "data_id": data_id} + adult_pandas = fetch_openml(parser="pandas", **common_params) + adult_liac_arff = fetch_openml(parser="liac-arff", **common_params) + pd.testing.assert_frame_equal(adult_pandas.frame, adult_liac_arff.frame) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_rcv1.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_rcv1.py new file mode 100644 index 0000000000000000000000000000000000000000..c415c9df977c36221c61ed4803dd26f6d0cfbef4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_rcv1.py @@ -0,0 +1,71 @@ +"""Test the rcv1 loader, if the data is available, +or if specifically requested via environment variable +(e.g. for CI jobs).""" + +from functools import partial + +import numpy as np +import scipy.sparse as sp + +from sklearn.datasets.tests.test_common import check_return_X_y +from sklearn.utils._testing import assert_almost_equal, assert_array_equal + + +def test_fetch_rcv1(fetch_rcv1_fxt, global_random_seed): + data1 = fetch_rcv1_fxt(shuffle=False) + X1, Y1 = data1.data, data1.target + cat_list, s1 = data1.target_names.tolist(), data1.sample_id + + # test sparsity + assert sp.issparse(X1) + assert sp.issparse(Y1) + assert 60915113 == X1.data.size + assert 2606875 == Y1.data.size + + # test shapes + assert (804414, 47236) == X1.shape + assert (804414, 103) == Y1.shape + assert (804414,) == s1.shape + assert 103 == len(cat_list) + + # test descr + assert data1.DESCR.startswith(".. _rcv1_dataset:") + + # test ordering of categories + first_categories = ["C11", "C12", "C13", "C14", "C15", "C151"] + assert_array_equal(first_categories, cat_list[:6]) + + # test number of sample for some categories + some_categories = ("GMIL", "E143", "CCAT") + number_non_zero_in_cat = (5, 1206, 381327) + for num, cat in zip(number_non_zero_in_cat, some_categories): + j = cat_list.index(cat) + assert num == Y1[:, j].data.size + + # test shuffling and subset + data2 = fetch_rcv1_fxt( + shuffle=True, subset="train", random_state=global_random_seed + ) + X2, Y2 = data2.data, data2.target + s2 = data2.sample_id + + # test return_X_y option + fetch_func = partial(fetch_rcv1_fxt, shuffle=False, subset="train") + check_return_X_y(data2, fetch_func) + + # The first 23149 samples are the training samples + assert_array_equal(np.sort(s1[:23149]), np.sort(s2)) + + # test some precise values + some_sample_ids = (2286, 3274, 14042) + for sample_id in some_sample_ids: + idx1 = s1.tolist().index(sample_id) + idx2 = s2.tolist().index(sample_id) + + feature_values_1 = X1[idx1, :].toarray() + feature_values_2 = X2[idx2, :].toarray() + assert_almost_equal(feature_values_1, feature_values_2) + + target_values_1 = Y1[idx1, :].toarray() + target_values_2 = Y2[idx2, :].toarray() + assert_almost_equal(target_values_1, target_values_2) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_samples_generator.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_samples_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..f514080bf8a6f141792b1d28e14e8986ee6dd438 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_samples_generator.py @@ -0,0 +1,744 @@ +import re +from collections import defaultdict +from functools import partial + +import numpy as np +import pytest +import scipy.sparse as sp + +from sklearn.datasets import ( + make_biclusters, + make_blobs, + make_checkerboard, + make_circles, + make_classification, + make_friedman1, + make_friedman2, + make_friedman3, + make_hastie_10_2, + make_low_rank_matrix, + make_moons, + make_multilabel_classification, + make_regression, + make_s_curve, + make_sparse_coded_signal, + make_sparse_spd_matrix, + make_sparse_uncorrelated, + make_spd_matrix, + make_swiss_roll, +) +from sklearn.utils._testing import ( + assert_allclose, + assert_allclose_dense_sparse, + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.validation import assert_all_finite + + +def test_make_classification(): + weights = [0.1, 0.25] + X, y = make_classification( + n_samples=100, + n_features=20, + n_informative=5, + n_redundant=1, + n_repeated=1, + n_classes=3, + n_clusters_per_class=1, + hypercube=False, + shift=None, + scale=None, + weights=weights, + random_state=0, + ) + + assert weights == [0.1, 0.25] + assert X.shape == (100, 20), "X shape mismatch" + assert y.shape == (100,), "y shape mismatch" + assert np.unique(y).shape == (3,), "Unexpected number of classes" + assert sum(y == 0) == 10, "Unexpected number of samples in class #0" + assert sum(y == 1) == 25, "Unexpected number of samples in class #1" + assert sum(y == 2) == 65, "Unexpected number of samples in class #2" + + # Test for n_features > 30 + X, y = make_classification( + n_samples=2000, + n_features=31, + n_informative=31, + n_redundant=0, + n_repeated=0, + hypercube=True, + scale=0.5, + random_state=0, + ) + + assert X.shape == (2000, 31), "X shape mismatch" + assert y.shape == (2000,), "y shape mismatch" + assert ( + np.unique(X.view([("", X.dtype)] * X.shape[1])) + .view(X.dtype) + .reshape(-1, X.shape[1]) + .shape[0] + == 2000 + ), "Unexpected number of unique rows" + + +def test_make_classification_informative_features(): + """Test the construction of informative features in make_classification + + Also tests `n_clusters_per_class`, `n_classes`, `hypercube` and + fully-specified `weights`. + """ + # Create very separate clusters; check that vertices are unique and + # correspond to classes + class_sep = 1e6 + make = partial( + make_classification, + class_sep=class_sep, + n_redundant=0, + n_repeated=0, + flip_y=0, + shift=0, + scale=1, + shuffle=False, + ) + + for n_informative, weights, n_clusters_per_class in [ + (2, [1], 1), + (2, [1 / 3] * 3, 1), + (2, [1 / 4] * 4, 1), + (2, [1 / 2] * 2, 2), + (2, [3 / 4, 1 / 4], 2), + (10, [1 / 3] * 3, 10), + (64, [1], 1), + ]: + n_classes = len(weights) + n_clusters = n_classes * n_clusters_per_class + n_samples = n_clusters * 50 + + for hypercube in (False, True): + X, y = make( + n_samples=n_samples, + n_classes=n_classes, + weights=weights, + n_features=n_informative, + n_informative=n_informative, + n_clusters_per_class=n_clusters_per_class, + hypercube=hypercube, + random_state=0, + ) + + assert X.shape == (n_samples, n_informative) + assert y.shape == (n_samples,) + + # Cluster by sign, viewed as strings to allow uniquing + signs = np.sign(X) + signs = signs.view(dtype="|S{0}".format(signs.strides[0])).ravel() + unique_signs, cluster_index = np.unique(signs, return_inverse=True) + + assert len(unique_signs) == n_clusters, ( + "Wrong number of clusters, or not in distinct quadrants" + ) + + clusters_by_class = defaultdict(set) + for cluster, cls in zip(cluster_index, y): + clusters_by_class[cls].add(cluster) + for clusters in clusters_by_class.values(): + assert len(clusters) == n_clusters_per_class, ( + "Wrong number of clusters per class" + ) + assert len(clusters_by_class) == n_classes, "Wrong number of classes" + + assert_array_almost_equal( + np.bincount(y) / len(y) // weights, + [1] * n_classes, + err_msg="Wrong number of samples per class", + ) + + # Ensure on vertices of hypercube + for cluster in range(len(unique_signs)): + centroid = X[cluster_index == cluster].mean(axis=0) + if hypercube: + assert_array_almost_equal( + np.abs(centroid) / class_sep, + np.ones(n_informative), + decimal=5, + err_msg="Clusters are not centered on hypercube vertices", + ) + else: + with pytest.raises(AssertionError): + assert_array_almost_equal( + np.abs(centroid) / class_sep, + np.ones(n_informative), + decimal=5, + err_msg=( + "Clusters should not be centered on hypercube vertices" + ), + ) + + with pytest.raises(ValueError): + make(n_features=2, n_informative=2, n_classes=5, n_clusters_per_class=1) + with pytest.raises(ValueError): + make(n_features=2, n_informative=2, n_classes=3, n_clusters_per_class=2) + + +def test_make_classification_return_x_y(): + """ + Test that make_classification returns a Bunch when return_X_y is False. + + Also that bunch.X is the same as X + """ + + kwargs = { + "n_samples": 100, + "n_features": 20, + "n_informative": 5, + "n_redundant": 1, + "n_repeated": 1, + "n_classes": 3, + "n_clusters_per_class": 2, + "weights": None, + "flip_y": 0.01, + "class_sep": 1.0, + "hypercube": True, + "shift": 0.0, + "scale": 1.0, + "shuffle": True, + "random_state": 42, + "return_X_y": True, + } + + X, y = make_classification(**kwargs) + + kwargs["return_X_y"] = False + bunch = make_classification(**kwargs) + + assert ( + hasattr(bunch, "DESCR") + and hasattr(bunch, "parameters") + and hasattr(bunch, "feature_info") + and hasattr(bunch, "X") + and hasattr(bunch, "y") + ) + + def count(str_): + return bunch.feature_info.count(str_) + + assert np.array_equal(X, bunch.X) + assert np.array_equal(y, bunch.y) + assert bunch.DESCR == make_classification.__doc__ + assert bunch.parameters == kwargs + assert count("informative") == kwargs["n_informative"] + assert count("redundant") == kwargs["n_redundant"] + assert count("repeated") == kwargs["n_repeated"] + + +@pytest.mark.parametrize( + "weights, err_type, err_msg", + [ + ([], ValueError, "Weights specified but incompatible with number of classes."), + ( + [0.25, 0.75, 0.1], + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ( + np.array([]), + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ( + np.array([0.25, 0.75, 0.1]), + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ( + np.random.random(3), + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ], +) +def test_make_classification_weights_type(weights, err_type, err_msg): + with pytest.raises(err_type, match=err_msg): + make_classification(weights=weights) + + +@pytest.mark.parametrize("kwargs", [{}, {"n_classes": 3, "n_informative": 3}]) +def test_make_classification_weights_array_or_list_ok(kwargs): + X1, y1 = make_classification(weights=[0.1, 0.9], random_state=0, **kwargs) + X2, y2 = make_classification(weights=np.array([0.1, 0.9]), random_state=0, **kwargs) + assert_almost_equal(X1, X2) + assert_almost_equal(y1, y2) + + +def test_make_multilabel_classification_return_sequences(): + for allow_unlabeled, min_length in zip((True, False), (0, 1)): + X, Y = make_multilabel_classification( + n_samples=100, + n_features=20, + n_classes=3, + random_state=0, + return_indicator=False, + allow_unlabeled=allow_unlabeled, + ) + assert X.shape == (100, 20), "X shape mismatch" + if not allow_unlabeled: + assert max([max(y) for y in Y]) == 2 + assert min([len(y) for y in Y]) == min_length + assert max([len(y) for y in Y]) <= 3 + + +def test_make_multilabel_classification_return_indicator(): + for allow_unlabeled, min_length in zip((True, False), (0, 1)): + X, Y = make_multilabel_classification( + n_samples=25, + n_features=20, + n_classes=3, + random_state=0, + allow_unlabeled=allow_unlabeled, + ) + assert X.shape == (25, 20), "X shape mismatch" + assert Y.shape == (25, 3), "Y shape mismatch" + assert np.all(np.sum(Y, axis=0) > min_length) + + # Also test return_distributions and return_indicator with True + X2, Y2, p_c, p_w_c = make_multilabel_classification( + n_samples=25, + n_features=20, + n_classes=3, + random_state=0, + allow_unlabeled=allow_unlabeled, + return_distributions=True, + ) + + assert_array_almost_equal(X, X2) + assert_array_equal(Y, Y2) + assert p_c.shape == (3,) + assert_almost_equal(p_c.sum(), 1) + assert p_w_c.shape == (20, 3) + assert_almost_equal(p_w_c.sum(axis=0), [1] * 3) + + +def test_make_multilabel_classification_return_indicator_sparse(): + for allow_unlabeled, min_length in zip((True, False), (0, 1)): + X, Y = make_multilabel_classification( + n_samples=25, + n_features=20, + n_classes=3, + random_state=0, + return_indicator="sparse", + allow_unlabeled=allow_unlabeled, + ) + assert X.shape == (25, 20), "X shape mismatch" + assert Y.shape == (25, 3), "Y shape mismatch" + assert sp.issparse(Y) + + +def test_make_hastie_10_2(): + X, y = make_hastie_10_2(n_samples=100, random_state=0) + assert X.shape == (100, 10), "X shape mismatch" + assert y.shape == (100,), "y shape mismatch" + assert np.unique(y).shape == (2,), "Unexpected number of classes" + + +def test_make_regression(global_random_seed): + X, y, c = make_regression( + n_samples=200, + n_features=10, + n_informative=3, + effective_rank=5, + coef=True, + bias=0.0, + noise=1.0, + random_state=global_random_seed, + ) + + assert X.shape == (200, 10), "X shape mismatch" + assert y.shape == (200,), "y shape mismatch" + assert c.shape == (10,), "coef shape mismatch" + assert sum(c != 0.0) == 3, "Unexpected number of informative features" + + # Test that y ~= np.dot(X, c) + bias + N(0, 1.0). + assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1) + + # Test with small number of features. + X, y = make_regression(n_samples=100, n_features=1) # n_informative=3 + assert X.shape == (100, 1) + + +def test_make_regression_multitarget(global_random_seed): + X, y, c = make_regression( + n_samples=100, + n_features=10, + n_informative=3, + n_targets=3, + coef=True, + noise=1.0, + random_state=global_random_seed, + ) + + assert X.shape == (100, 10), "X shape mismatch" + assert y.shape == (100, 3), "y shape mismatch" + assert c.shape == (10, 3), "coef shape mismatch" + assert_array_equal(sum(c != 0.0), 3, "Unexpected number of informative features") + + # Test that y ~= np.dot(X, c) + bias + N(0, 1.0) + assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1) + + +def test_make_blobs(global_random_seed): + cluster_stds = np.array([0.05, 0.2, 0.4]) + cluster_centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) + X, y = make_blobs( + random_state=global_random_seed, + n_samples=50, + n_features=2, + centers=cluster_centers, + cluster_std=cluster_stds, + ) + + assert X.shape == (50, 2), "X shape mismatch" + assert y.shape == (50,), "y shape mismatch" + assert np.unique(y).shape == (3,), "Unexpected number of blobs" + for i, (ctr, std) in enumerate(zip(cluster_centers, cluster_stds)): + assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std") + + +def test_make_blobs_n_samples_list(): + n_samples = [50, 30, 20] + X, y = make_blobs(n_samples=n_samples, n_features=2, random_state=0) + + assert X.shape == (sum(n_samples), 2), "X shape mismatch" + assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), ( + "Incorrect number of samples per blob" + ) + + +def test_make_blobs_n_samples_list_with_centers(global_random_seed): + n_samples = [20, 20, 20] + centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) + cluster_stds = np.array([0.05, 0.2, 0.4]) + X, y = make_blobs( + n_samples=n_samples, + centers=centers, + cluster_std=cluster_stds, + random_state=global_random_seed, + ) + + assert X.shape == (sum(n_samples), 2), "X shape mismatch" + assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), ( + "Incorrect number of samples per blob" + ) + for i, (ctr, std) in enumerate(zip(centers, cluster_stds)): + assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std") + + +@pytest.mark.parametrize( + "n_samples", [[5, 3, 0], np.array([5, 3, 0]), tuple([5, 3, 0])] +) +def test_make_blobs_n_samples_centers_none(n_samples): + centers = None + X, y = make_blobs(n_samples=n_samples, centers=centers, random_state=0) + + assert X.shape == (sum(n_samples), 2), "X shape mismatch" + assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), ( + "Incorrect number of samples per blob" + ) + + +def test_make_blobs_return_centers(): + n_samples = [10, 20] + n_features = 3 + X, y, centers = make_blobs( + n_samples=n_samples, n_features=n_features, return_centers=True, random_state=0 + ) + + assert centers.shape == (len(n_samples), n_features) + + +def test_make_blobs_error(): + n_samples = [20, 20, 20] + centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) + cluster_stds = np.array([0.05, 0.2, 0.4]) + wrong_centers_msg = re.escape( + "Length of `n_samples` not consistent with number of centers. " + f"Got n_samples = {n_samples} and centers = {centers[:-1]}" + ) + with pytest.raises(ValueError, match=wrong_centers_msg): + make_blobs(n_samples, centers=centers[:-1]) + wrong_std_msg = re.escape( + "Length of `clusters_std` not consistent with number of centers. " + f"Got centers = {centers} and cluster_std = {cluster_stds[:-1]}" + ) + with pytest.raises(ValueError, match=wrong_std_msg): + make_blobs(n_samples, centers=centers, cluster_std=cluster_stds[:-1]) + wrong_type_msg = "Parameter `centers` must be array-like. Got {!r} instead".format( + 3 + ) + with pytest.raises(ValueError, match=wrong_type_msg): + make_blobs(n_samples, centers=3) + + +def test_make_friedman1(global_random_seed): + X, y = make_friedman1( + n_samples=5, n_features=10, noise=0.0, random_state=global_random_seed + ) + + assert X.shape == (5, 10), "X shape mismatch" + assert y.shape == (5,), "y shape mismatch" + + assert_array_almost_equal( + y, + 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + + 20 * (X[:, 2] - 0.5) ** 2 + + 10 * X[:, 3] + + 5 * X[:, 4], + ) + + +def test_make_friedman2(global_random_seed): + X, y = make_friedman2(n_samples=5, noise=0.0, random_state=global_random_seed) + + assert X.shape == (5, 4), "X shape mismatch" + assert y.shape == (5,), "y shape mismatch" + + assert_array_almost_equal( + y, (X[:, 0] ** 2 + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5 + ) + + +def test_make_friedman3(global_random_seed): + X, y = make_friedman3(n_samples=5, noise=0.0, random_state=global_random_seed) + + assert X.shape == (5, 4), "X shape mismatch" + assert y.shape == (5,), "y shape mismatch" + + assert_array_almost_equal( + y, np.arctan((X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) / X[:, 0]) + ) + + +def test_make_low_rank_matrix(): + X = make_low_rank_matrix( + n_samples=50, + n_features=25, + effective_rank=5, + tail_strength=0.01, + random_state=0, + ) + + assert X.shape == (50, 25), "X shape mismatch" + + from numpy.linalg import svd + + u, s, v = svd(X) + assert sum(s) - 5 < 0.1, "X rank is not approximately 5" + + +def test_make_sparse_coded_signal(global_random_seed): + Y, D, X = make_sparse_coded_signal( + n_samples=5, + n_components=8, + n_features=10, + n_nonzero_coefs=3, + random_state=global_random_seed, + ) + assert Y.shape == (5, 10), "Y shape mismatch" + assert D.shape == (8, 10), "D shape mismatch" + assert X.shape == (5, 8), "X shape mismatch" + for row in X: + assert len(np.flatnonzero(row)) == 3, "Non-zero coefs mismatch" + assert_allclose(Y, X @ D) + assert_allclose(np.sqrt((D**2).sum(axis=1)), np.ones(D.shape[0])) + + +def test_make_sparse_uncorrelated(): + X, y = make_sparse_uncorrelated(n_samples=5, n_features=10, random_state=0) + + assert X.shape == (5, 10), "X shape mismatch" + assert y.shape == (5,), "y shape mismatch" + + +def test_make_spd_matrix(global_random_seed): + X = make_spd_matrix(n_dim=5, random_state=global_random_seed) + + assert X.shape == (5, 5), "X shape mismatch" + assert_array_almost_equal(X, X.T) + + from numpy.linalg import eig + + eigenvalues, _ = eig(X) + assert np.all(eigenvalues > 0), "X is not positive-definite" + + +@pytest.mark.parametrize("norm_diag", [True, False]) +@pytest.mark.parametrize( + "sparse_format", [None, "bsr", "coo", "csc", "csr", "dia", "dok", "lil"] +) +def test_make_sparse_spd_matrix(norm_diag, sparse_format, global_random_seed): + n_dim = 5 + X = make_sparse_spd_matrix( + n_dim=n_dim, + norm_diag=norm_diag, + sparse_format=sparse_format, + random_state=global_random_seed, + ) + + assert X.shape == (n_dim, n_dim), "X shape mismatch" + if sparse_format is None: + assert not sp.issparse(X) + assert_allclose(X, X.T) + Xarr = X + else: + assert sp.issparse(X) and X.format == sparse_format + assert_allclose_dense_sparse(X, X.T) + Xarr = X.toarray() + + from numpy.linalg import eig + + # Do not use scipy.sparse.linalg.eigs because it cannot find all eigenvalues + eigenvalues, _ = eig(Xarr) + assert np.all(eigenvalues > 0), "X is not positive-definite" + + if norm_diag: + # Check that leading diagonal elements are 1 + assert_array_almost_equal(Xarr.diagonal(), np.ones(n_dim)) + + +@pytest.mark.parametrize("hole", [False, True]) +def test_make_swiss_roll(global_random_seed, hole): + X, t = make_swiss_roll( + n_samples=5, noise=0.0, random_state=global_random_seed, hole=hole + ) + + assert X.shape == (5, 3) + assert t.shape == (5,) + assert_array_almost_equal(X[:, 0], t * np.cos(t)) + assert_array_almost_equal(X[:, 2], t * np.sin(t)) + + +def test_make_s_curve(global_random_seed): + X, t = make_s_curve(n_samples=5, noise=0.0, random_state=global_random_seed) + + assert X.shape == (5, 3), "X shape mismatch" + assert t.shape == (5,), "t shape mismatch" + assert_array_almost_equal(X[:, 0], np.sin(t)) + assert_array_almost_equal(X[:, 2], np.sign(t) * (np.cos(t) - 1)) + + +def test_make_biclusters(): + X, rows, cols = make_biclusters( + shape=(100, 100), n_clusters=4, shuffle=True, random_state=0 + ) + assert X.shape == (100, 100), "X shape mismatch" + assert rows.shape == (4, 100), "rows shape mismatch" + assert cols.shape == ( + 4, + 100, + ), "columns shape mismatch" + assert_all_finite(X) + assert_all_finite(rows) + assert_all_finite(cols) + + X2, _, _ = make_biclusters( + shape=(100, 100), n_clusters=4, shuffle=True, random_state=0 + ) + assert_array_almost_equal(X, X2) + + +def test_make_checkerboard(): + X, rows, cols = make_checkerboard( + shape=(100, 100), n_clusters=(20, 5), shuffle=True, random_state=0 + ) + assert X.shape == (100, 100), "X shape mismatch" + assert rows.shape == (100, 100), "rows shape mismatch" + assert cols.shape == ( + 100, + 100, + ), "columns shape mismatch" + + X, rows, cols = make_checkerboard( + shape=(100, 100), n_clusters=2, shuffle=True, random_state=0 + ) + assert_all_finite(X) + assert_all_finite(rows) + assert_all_finite(cols) + + X1, _, _ = make_checkerboard( + shape=(100, 100), n_clusters=2, shuffle=True, random_state=0 + ) + X2, _, _ = make_checkerboard( + shape=(100, 100), n_clusters=2, shuffle=True, random_state=0 + ) + assert_array_almost_equal(X1, X2) + + +def test_make_moons(global_random_seed): + X, y = make_moons(3, shuffle=False, random_state=global_random_seed) + for x, label in zip(X, y): + center = [0.0, 0.0] if label == 0 else [1.0, 0.5] + dist_sqr = ((x - center) ** 2).sum() + assert_almost_equal( + dist_sqr, 1.0, err_msg="Point is not on expected unit circle" + ) + + +def test_make_moons_unbalanced(): + X, y = make_moons(n_samples=(7, 5)) + assert np.sum(y == 0) == 7 and np.sum(y == 1) == 5, ( + "Number of samples in a moon is wrong" + ) + assert X.shape == (12, 2), "X shape mismatch" + assert y.shape == (12,), "y shape mismatch" + + with pytest.raises( + ValueError, + match=r"`n_samples` can be either an int or a two-element tuple.", + ): + make_moons(n_samples=(10,)) + + +def test_make_circles(): + factor = 0.3 + + for n_samples, n_outer, n_inner in [(7, 3, 4), (8, 4, 4)]: + # Testing odd and even case, because in the past make_circles always + # created an even number of samples. + X, y = make_circles(n_samples, shuffle=False, noise=None, factor=factor) + assert X.shape == (n_samples, 2), "X shape mismatch" + assert y.shape == (n_samples,), "y shape mismatch" + center = [0.0, 0.0] + for x, label in zip(X, y): + dist_sqr = ((x - center) ** 2).sum() + dist_exp = 1.0 if label == 0 else factor**2 + dist_exp = 1.0 if label == 0 else factor**2 + assert_almost_equal( + dist_sqr, dist_exp, err_msg="Point is not on expected circle" + ) + + assert X[y == 0].shape == ( + n_outer, + 2, + ), "Samples not correctly distributed across circles." + assert X[y == 1].shape == ( + n_inner, + 2, + ), "Samples not correctly distributed across circles." + + +def test_make_circles_unbalanced(): + X, y = make_circles(n_samples=(2, 8)) + + assert np.sum(y == 0) == 2, "Number of samples in inner circle is wrong" + assert np.sum(y == 1) == 8, "Number of samples in outer circle is wrong" + assert X.shape == (10, 2), "X shape mismatch" + assert y.shape == (10,), "y shape mismatch" + + with pytest.raises( + ValueError, + match="When a tuple, n_samples must have exactly two elements.", + ): + make_circles(n_samples=(10,)) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_svmlight_format.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_svmlight_format.py new file mode 100644 index 0000000000000000000000000000000000000000..6dc171a5f452bd75602143f544ca45d97cb3a888 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_svmlight_format.py @@ -0,0 +1,613 @@ +import gzip +import os +import shutil +from bz2 import BZ2File +from importlib import resources +from io import BytesIO +from tempfile import NamedTemporaryFile + +import numpy as np +import pytest +import scipy.sparse as sp + +import sklearn +from sklearn.datasets import dump_svmlight_file, load_svmlight_file, load_svmlight_files +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, + create_memmap_backed_data, +) +from sklearn.utils.fixes import CSR_CONTAINERS + +TEST_DATA_MODULE = "sklearn.datasets.tests.data" +datafile = "svmlight_classification.txt" +multifile = "svmlight_multilabel.txt" +invalidfile = "svmlight_invalid.txt" +invalidfile2 = "svmlight_invalid_order.txt" + + +def _svmlight_local_test_file_path(filename): + return resources.files(TEST_DATA_MODULE) / filename + + +def _load_svmlight_local_test_file(filename, **kwargs): + """ + Helper to load resource `filename` with `importlib.resources` + """ + data_path = _svmlight_local_test_file_path(filename) + with data_path.open("rb") as f: + return load_svmlight_file(f, **kwargs) + + +def test_load_svmlight_file(): + X, y = _load_svmlight_local_test_file(datafile) + + # test X's shape + assert X.indptr.shape[0] == 7 + assert X.shape[0] == 6 + assert X.shape[1] == 21 + assert y.shape[0] == 6 + + # test X's non-zero values + for i, j, val in ( + (0, 2, 2.5), + (0, 10, -5.2), + (0, 15, 1.5), + (1, 5, 1.0), + (1, 12, -3), + (2, 20, 27), + ): + assert X[i, j] == val + + # tests X's zero values + assert X[0, 3] == 0 + assert X[0, 5] == 0 + assert X[1, 8] == 0 + assert X[1, 16] == 0 + assert X[2, 18] == 0 + + # test can change X's values + X[0, 2] *= 2 + assert X[0, 2] == 5 + + # test y + assert_array_equal(y, [1, 2, 3, 4, 1, 2]) + + +def test_load_svmlight_file_fd(): + # test loading from file descriptor + + # GH20081: testing equality between path-based and + # fd-based load_svmlight_file + + data_path = resources.files(TEST_DATA_MODULE) / datafile + data_path = str(data_path) + X1, y1 = load_svmlight_file(data_path) + + fd = os.open(data_path, os.O_RDONLY) + try: + X2, y2 = load_svmlight_file(fd) + assert_array_almost_equal(X1.data, X2.data) + assert_array_almost_equal(y1, y2) + finally: + os.close(fd) + + +def test_load_svmlight_pathlib(): + # test loading from file descriptor + data_path = _svmlight_local_test_file_path(datafile) + X1, y1 = load_svmlight_file(str(data_path)) + X2, y2 = load_svmlight_file(data_path) + + assert_allclose(X1.data, X2.data) + assert_allclose(y1, y2) + + +def test_load_svmlight_file_multilabel(): + X, y = _load_svmlight_local_test_file(multifile, multilabel=True) + assert y == [(0, 1), (2,), (), (1, 2)] + + +def test_load_svmlight_files(): + data_path = _svmlight_local_test_file_path(datafile) + X_train, y_train, X_test, y_test = load_svmlight_files( + [str(data_path)] * 2, dtype=np.float32 + ) + assert_array_equal(X_train.toarray(), X_test.toarray()) + assert_array_almost_equal(y_train, y_test) + assert X_train.dtype == np.float32 + assert X_test.dtype == np.float32 + + X1, y1, X2, y2, X3, y3 = load_svmlight_files([str(data_path)] * 3, dtype=np.float64) + assert X1.dtype == X2.dtype + assert X2.dtype == X3.dtype + assert X3.dtype == np.float64 + + +def test_load_svmlight_file_n_features(): + X, y = _load_svmlight_local_test_file(datafile, n_features=22) + + # test X'shape + assert X.indptr.shape[0] == 7 + assert X.shape[0] == 6 + assert X.shape[1] == 22 + + # test X's non-zero values + for i, j, val in ((0, 2, 2.5), (0, 10, -5.2), (1, 5, 1.0), (1, 12, -3)): + assert X[i, j] == val + + # 21 features in file + with pytest.raises(ValueError): + _load_svmlight_local_test_file(datafile, n_features=20) + + +def test_load_compressed(): + X, y = _load_svmlight_local_test_file(datafile) + + with NamedTemporaryFile(prefix="sklearn-test", suffix=".gz") as tmp: + tmp.close() # necessary under windows + with _svmlight_local_test_file_path(datafile).open("rb") as f: + with gzip.open(tmp.name, "wb") as fh_out: + shutil.copyfileobj(f, fh_out) + Xgz, ygz = load_svmlight_file(tmp.name) + # because we "close" it manually and write to it, + # we need to remove it manually. + os.remove(tmp.name) + assert_array_almost_equal(X.toarray(), Xgz.toarray()) + assert_array_almost_equal(y, ygz) + + with NamedTemporaryFile(prefix="sklearn-test", suffix=".bz2") as tmp: + tmp.close() # necessary under windows + with _svmlight_local_test_file_path(datafile).open("rb") as f: + with BZ2File(tmp.name, "wb") as fh_out: + shutil.copyfileobj(f, fh_out) + Xbz, ybz = load_svmlight_file(tmp.name) + # because we "close" it manually and write to it, + # we need to remove it manually. + os.remove(tmp.name) + assert_array_almost_equal(X.toarray(), Xbz.toarray()) + assert_array_almost_equal(y, ybz) + + +def test_load_invalid_file(): + with pytest.raises(ValueError): + _load_svmlight_local_test_file(invalidfile) + + +def test_load_invalid_order_file(): + with pytest.raises(ValueError): + _load_svmlight_local_test_file(invalidfile2) + + +def test_load_zero_based(): + f = BytesIO(b"-1 4:1.\n1 0:1\n") + with pytest.raises(ValueError): + load_svmlight_file(f, zero_based=False) + + +def test_load_zero_based_auto(): + data1 = b"-1 1:1 2:2 3:3\n" + data2 = b"-1 0:0 1:1\n" + + f1 = BytesIO(data1) + X, y = load_svmlight_file(f1, zero_based="auto") + assert X.shape == (1, 3) + + f1 = BytesIO(data1) + f2 = BytesIO(data2) + X1, y1, X2, y2 = load_svmlight_files([f1, f2], zero_based="auto") + assert X1.shape == (1, 4) + assert X2.shape == (1, 4) + + +def test_load_with_qid(): + # load svmfile with qid attribute + data = b""" + 3 qid:1 1:0.53 2:0.12 + 2 qid:1 1:0.13 2:0.1 + 7 qid:2 1:0.87 2:0.12""" + X, y = load_svmlight_file(BytesIO(data), query_id=False) + assert_array_equal(y, [3, 2, 7]) + assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]]) + res1 = load_svmlight_files([BytesIO(data)], query_id=True) + res2 = load_svmlight_file(BytesIO(data), query_id=True) + for X, y, qid in (res1, res2): + assert_array_equal(y, [3, 2, 7]) + assert_array_equal(qid, [1, 1, 2]) + assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]]) + + +@pytest.mark.skip( + "testing the overflow of 32 bit sparse indexing requires a large amount of memory" +) +def test_load_large_qid(): + """ + load large libsvm / svmlight file with qid attribute. Tests 64-bit query ID + """ + data = b"\n".join( + ( + "3 qid:{0} 1:0.53 2:0.12\n2 qid:{0} 1:0.13 2:0.1".format(i).encode() + for i in range(1, 40 * 1000 * 1000) + ) + ) + X, y, qid = load_svmlight_file(BytesIO(data), query_id=True) + assert_array_equal(y[-4:], [3, 2, 3, 2]) + assert_array_equal(np.unique(qid), np.arange(1, 40 * 1000 * 1000)) + + +def test_load_invalid_file2(): + with pytest.raises(ValueError): + data_path = _svmlight_local_test_file_path(datafile) + invalid_path = _svmlight_local_test_file_path(invalidfile) + load_svmlight_files([str(data_path), str(invalid_path), str(data_path)]) + + +def test_not_a_filename(): + # in python 3 integers are valid file opening arguments (taken as unix + # file descriptors) + with pytest.raises(TypeError): + load_svmlight_file(0.42) + + +def test_invalid_filename(): + with pytest.raises(OSError): + load_svmlight_file("trou pic nic douille") + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_dump(csr_container): + X_sparse, y_dense = _load_svmlight_local_test_file(datafile) + X_dense = X_sparse.toarray() + y_sparse = csr_container(np.atleast_2d(y_dense)) + + # slicing a csr_matrix can unsort its .indices, so test that we sort + # those correctly + X_sliced = X_sparse[np.arange(X_sparse.shape[0])] + y_sliced = y_sparse[np.arange(y_sparse.shape[0])] + + for X in (X_sparse, X_dense, X_sliced): + for y in (y_sparse, y_dense, y_sliced): + for zero_based in (True, False): + for dtype in [np.float32, np.float64, np.int32, np.int64]: + f = BytesIO() + # we need to pass a comment to get the version info in; + # LibSVM doesn't grok comments so they're not put in by + # default anymore. + + if sp.issparse(y) and y.shape[0] == 1: + # make sure y's shape is: (n_samples, n_labels) + # when it is sparse + y = y.T + + # Note: with dtype=np.int32 we are performing unsafe casts, + # where X.astype(dtype) overflows. The result is + # then platform dependent and X_dense.astype(dtype) may be + # different from X_sparse.astype(dtype).asarray(). + X_input = X.astype(dtype) + + dump_svmlight_file( + X_input, y, f, comment="test", zero_based=zero_based + ) + f.seek(0) + + comment = f.readline() + comment = str(comment, "utf-8") + + assert "scikit-learn %s" % sklearn.__version__ in comment + + comment = f.readline() + comment = str(comment, "utf-8") + + assert ["one", "zero"][zero_based] + "-based" in comment + + X2, y2 = load_svmlight_file(f, dtype=dtype, zero_based=zero_based) + assert X2.dtype == dtype + assert_array_equal(X2.sorted_indices().indices, X2.indices) + + X2_dense = X2.toarray() + if sp.issparse(X_input): + X_input_dense = X_input.toarray() + else: + X_input_dense = X_input + + if dtype == np.float32: + # allow a rounding error at the last decimal place + assert_array_almost_equal(X_input_dense, X2_dense, 4) + assert_array_almost_equal( + y_dense.astype(dtype, copy=False), y2, 4 + ) + else: + # allow a rounding error at the last decimal place + assert_array_almost_equal(X_input_dense, X2_dense, 15) + assert_array_almost_equal( + y_dense.astype(dtype, copy=False), y2, 15 + ) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_dump_multilabel(csr_container): + X = [[1, 0, 3, 0, 5], [0, 0, 0, 0, 0], [0, 5, 0, 1, 0]] + y_dense = [[0, 1, 0], [1, 0, 1], [1, 1, 0]] + y_sparse = csr_container(y_dense) + for y in [y_dense, y_sparse]: + f = BytesIO() + dump_svmlight_file(X, y, f, multilabel=True) + f.seek(0) + # make sure it dumps multilabel correctly + assert f.readline() == b"1 0:1 2:3 4:5\n" + assert f.readline() == b"0,2 \n" + assert f.readline() == b"0,1 1:5 3:1\n" + + +def test_dump_concise(): + one = 1 + two = 2.1 + three = 3.01 + exact = 1.000000000000001 + # loses the last decimal place + almost = 1.0000000000000001 + X = [ + [one, two, three, exact, almost], + [1e9, 2e18, 3e27, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ] + y = [one, two, three, exact, almost] + f = BytesIO() + dump_svmlight_file(X, y, f) + f.seek(0) + # make sure it's using the most concise format possible + assert f.readline() == b"1 0:1 1:2.1 2:3.01 3:1.000000000000001 4:1\n" + assert f.readline() == b"2.1 0:1000000000 1:2e+18 2:3e+27\n" + assert f.readline() == b"3.01 \n" + assert f.readline() == b"1.000000000000001 \n" + assert f.readline() == b"1 \n" + f.seek(0) + # make sure it's correct too :) + X2, y2 = load_svmlight_file(f) + assert_array_almost_equal(X, X2.toarray()) + assert_array_almost_equal(y, y2) + + +def test_dump_comment(): + X, y = _load_svmlight_local_test_file(datafile) + X = X.toarray() + + f = BytesIO() + ascii_comment = "This is a comment\nspanning multiple lines." + dump_svmlight_file(X, y, f, comment=ascii_comment, zero_based=False) + f.seek(0) + + X2, y2 = load_svmlight_file(f, zero_based=False) + assert_array_almost_equal(X, X2.toarray()) + assert_array_almost_equal(y, y2) + + # XXX we have to update this to support Python 3.x + utf8_comment = b"It is true that\n\xc2\xbd\xc2\xb2 = \xc2\xbc" + f = BytesIO() + with pytest.raises(UnicodeDecodeError): + dump_svmlight_file(X, y, f, comment=utf8_comment) + + unicode_comment = utf8_comment.decode("utf-8") + f = BytesIO() + dump_svmlight_file(X, y, f, comment=unicode_comment, zero_based=False) + f.seek(0) + + X2, y2 = load_svmlight_file(f, zero_based=False) + assert_array_almost_equal(X, X2.toarray()) + assert_array_almost_equal(y, y2) + + f = BytesIO() + with pytest.raises(ValueError): + dump_svmlight_file(X, y, f, comment="I've got a \0.") + + +def test_dump_invalid(): + X, y = _load_svmlight_local_test_file(datafile) + + f = BytesIO() + y2d = [y] + with pytest.raises(ValueError): + dump_svmlight_file(X, y2d, f) + + f = BytesIO() + with pytest.raises(ValueError): + dump_svmlight_file(X, y[:-1], f) + + +def test_dump_query_id(): + # test dumping a file with query_id + X, y = _load_svmlight_local_test_file(datafile) + X = X.toarray() + query_id = np.arange(X.shape[0]) // 2 + f = BytesIO() + dump_svmlight_file(X, y, f, query_id=query_id, zero_based=True) + + f.seek(0) + X1, y1, query_id1 = load_svmlight_file(f, query_id=True, zero_based=True) + assert_array_almost_equal(X, X1.toarray()) + assert_array_almost_equal(y, y1) + assert_array_almost_equal(query_id, query_id1) + + +def test_load_with_long_qid(): + # load svmfile with longint qid attribute + data = b""" + 1 qid:0 0:1 1:2 2:3 + 0 qid:72048431380967004 0:1440446648 1:72048431380967004 2:236784985 + 0 qid:-9223372036854775807 0:1440446648 1:72048431380967004 2:236784985 + 3 qid:9223372036854775807 0:1440446648 1:72048431380967004 2:236784985""" + X, y, qid = load_svmlight_file(BytesIO(data), query_id=True) + + true_X = [ + [1, 2, 3], + [1440446648, 72048431380967004, 236784985], + [1440446648, 72048431380967004, 236784985], + [1440446648, 72048431380967004, 236784985], + ] + + true_y = [1, 0, 0, 3] + trueQID = [0, 72048431380967004, -9223372036854775807, 9223372036854775807] + assert_array_equal(y, true_y) + assert_array_equal(X.toarray(), true_X) + assert_array_equal(qid, trueQID) + + f = BytesIO() + dump_svmlight_file(X, y, f, query_id=qid, zero_based=True) + f.seek(0) + X, y, qid = load_svmlight_file(f, query_id=True, zero_based=True) + assert_array_equal(y, true_y) + assert_array_equal(X.toarray(), true_X) + assert_array_equal(qid, trueQID) + + f.seek(0) + X, y = load_svmlight_file(f, query_id=False, zero_based=True) + assert_array_equal(y, true_y) + assert_array_equal(X.toarray(), true_X) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_load_zeros(csr_container): + f = BytesIO() + true_X = csr_container(np.zeros(shape=(3, 4))) + true_y = np.array([0, 1, 0]) + dump_svmlight_file(true_X, true_y, f) + + for zero_based in ["auto", True, False]: + f.seek(0) + X, y = load_svmlight_file(f, n_features=4, zero_based=zero_based) + assert_array_almost_equal(y, true_y) + assert_array_almost_equal(X.toarray(), true_X.toarray()) + + +@pytest.mark.parametrize("sparsity", [0, 0.1, 0.5, 0.99, 1]) +@pytest.mark.parametrize("n_samples", [13, 101]) +@pytest.mark.parametrize("n_features", [2, 7, 41]) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_load_with_offsets(sparsity, n_samples, n_features, csr_container): + rng = np.random.RandomState(0) + X = rng.uniform(low=0.0, high=1.0, size=(n_samples, n_features)) + if sparsity: + X[X < sparsity] = 0.0 + X = csr_container(X) + y = rng.randint(low=0, high=2, size=n_samples) + + f = BytesIO() + dump_svmlight_file(X, y, f) + f.seek(0) + + size = len(f.getvalue()) + + # put some marks that are likely to happen anywhere in a row + mark_0 = 0 + mark_1 = size // 3 + length_0 = mark_1 - mark_0 + mark_2 = 4 * size // 5 + length_1 = mark_2 - mark_1 + + # load the original sparse matrix into 3 independent CSR matrices + X_0, y_0 = load_svmlight_file( + f, n_features=n_features, offset=mark_0, length=length_0 + ) + X_1, y_1 = load_svmlight_file( + f, n_features=n_features, offset=mark_1, length=length_1 + ) + X_2, y_2 = load_svmlight_file(f, n_features=n_features, offset=mark_2) + + y_concat = np.concatenate([y_0, y_1, y_2]) + X_concat = sp.vstack([X_0, X_1, X_2]) + assert_array_almost_equal(y, y_concat) + assert_array_almost_equal(X.toarray(), X_concat.toarray()) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_load_offset_exhaustive_splits(csr_container): + rng = np.random.RandomState(0) + X = np.array( + [ + [0, 0, 0, 0, 0, 0], + [1, 2, 3, 4, 0, 6], + [1, 2, 3, 4, 0, 6], + [0, 0, 0, 0, 0, 0], + [1, 0, 3, 0, 0, 0], + [0, 0, 0, 0, 0, 1], + [1, 0, 0, 0, 0, 0], + ] + ) + X = csr_container(X) + n_samples, n_features = X.shape + y = rng.randint(low=0, high=2, size=n_samples) + query_id = np.arange(n_samples) // 2 + + f = BytesIO() + dump_svmlight_file(X, y, f, query_id=query_id) + f.seek(0) + + size = len(f.getvalue()) + + # load the same data in 2 parts with all the possible byte offsets to + # locate the split so has to test for particular boundary cases + for mark in range(size): + f.seek(0) + X_0, y_0, q_0 = load_svmlight_file( + f, n_features=n_features, query_id=True, offset=0, length=mark + ) + X_1, y_1, q_1 = load_svmlight_file( + f, n_features=n_features, query_id=True, offset=mark, length=-1 + ) + q_concat = np.concatenate([q_0, q_1]) + y_concat = np.concatenate([y_0, y_1]) + X_concat = sp.vstack([X_0, X_1]) + assert_array_almost_equal(y, y_concat) + assert_array_equal(query_id, q_concat) + assert_array_almost_equal(X.toarray(), X_concat.toarray()) + + +def test_load_with_offsets_error(): + with pytest.raises(ValueError, match="n_features is required"): + _load_svmlight_local_test_file(datafile, offset=3, length=3) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_multilabel_y_explicit_zeros(tmp_path, csr_container): + """ + Ensure that if y contains explicit zeros (i.e. elements of y.data equal to + 0) then those explicit zeros are not encoded. + """ + save_path = str(tmp_path / "svm_explicit_zero") + rng = np.random.RandomState(42) + X = rng.randn(3, 5).astype(np.float64) + indptr = np.array([0, 2, 3, 6]) + indices = np.array([0, 2, 2, 0, 1, 2]) + # The first and last element are explicit zeros. + data = np.array([0, 1, 1, 1, 1, 0]) + y = csr_container((data, indices, indptr), shape=(3, 3)) + # y as a dense array would look like + # [[0, 0, 1], + # [0, 0, 1], + # [1, 1, 0]] + + dump_svmlight_file(X, y, save_path, multilabel=True) + + _, y_load = load_svmlight_file(save_path, multilabel=True) + y_true = [(2.0,), (2.0,), (0.0, 1.0)] + assert y_load == y_true + + +def test_dump_read_only(tmp_path): + """Ensure that there is no ValueError when dumping a read-only `X`. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/28026 + """ + rng = np.random.RandomState(42) + X = rng.randn(5, 2) + y = rng.randn(5) + + # Convert to memmap-backed which are read-only + X, y = create_memmap_backed_data([X, y]) + + save_path = str(tmp_path / "svm_read_only") + dump_svmlight_file(X, y, save_path) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..18602a7cb7e7760f5637f0a6cac5680f7406412e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/__init__.py @@ -0,0 +1,50 @@ +"""Matrix decomposition algorithms. + +These include PCA, NMF, ICA, and more. Most of the algorithms of this module can be +regarded as dimensionality reduction techniques. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from sklearn.decomposition._dict_learning import ( + DictionaryLearning, + MiniBatchDictionaryLearning, + SparseCoder, + dict_learning, + dict_learning_online, + sparse_encode, +) +from sklearn.decomposition._factor_analysis import FactorAnalysis +from sklearn.decomposition._fastica import FastICA, fastica +from sklearn.decomposition._incremental_pca import IncrementalPCA +from sklearn.decomposition._kernel_pca import KernelPCA +from sklearn.decomposition._lda import LatentDirichletAllocation +from sklearn.decomposition._nmf import NMF, MiniBatchNMF, non_negative_factorization +from sklearn.decomposition._pca import PCA +from sklearn.decomposition._sparse_pca import MiniBatchSparsePCA, SparsePCA +from sklearn.decomposition._truncated_svd import TruncatedSVD +from sklearn.utils.extmath import randomized_svd + +__all__ = [ + "NMF", + "PCA", + "DictionaryLearning", + "FactorAnalysis", + "FastICA", + "IncrementalPCA", + "KernelPCA", + "LatentDirichletAllocation", + "MiniBatchDictionaryLearning", + "MiniBatchNMF", + "MiniBatchSparsePCA", + "SparseCoder", + "SparsePCA", + "TruncatedSVD", + "dict_learning", + "dict_learning_online", + "fastica", + "non_negative_factorization", + "randomized_svd", + "sparse_encode", +] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_base.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..866842387234fe160c97ea7058f94f3dbda6f3a5 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_base.py @@ -0,0 +1,210 @@ +"""Principal Component Analysis Base Classes""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from abc import ABCMeta, abstractmethod + +import numpy as np +from scipy import linalg + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, +) +from sklearn.utils._array_api import _add_to_diagonal, device, get_namespace +from sklearn.utils.validation import check_array, check_is_fitted, validate_data + + +class _BasePCA( + ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator, metaclass=ABCMeta +): + """Base class for PCA methods. + + Warning: This class should not be used directly. + Use derived classes instead. + """ + + def get_covariance(self): + """Compute data covariance with the generative model. + + ``cov = components_.T * S**2 * components_ + sigma2 * eye(n_features)`` + where S**2 contains the explained variances, and sigma2 contains the + noise variances. + + Returns + ------- + cov : array of shape=(n_features, n_features) + Estimated covariance of data. + """ + xp, _ = get_namespace(self.components_) + + components_ = self.components_ + exp_var = self.explained_variance_ + if self.whiten: + components_ = components_ * xp.sqrt(exp_var[:, np.newaxis]) + exp_var_diff = exp_var - self.noise_variance_ + exp_var_diff = xp.where( + exp_var > self.noise_variance_, + exp_var_diff, + xp.asarray(0.0, device=device(exp_var), dtype=exp_var.dtype), + ) + cov = (components_.T * exp_var_diff) @ components_ + _add_to_diagonal(cov, self.noise_variance_, xp) + return cov + + def get_precision(self): + """Compute data precision matrix with the generative model. + + Equals the inverse of the covariance but computed with + the matrix inversion lemma for efficiency. + + Returns + ------- + precision : array, shape=(n_features, n_features) + Estimated precision of data. + """ + xp, is_array_api_compliant = get_namespace(self.components_) + + n_features = self.components_.shape[1] + + # handle corner cases first + if self.n_components_ == 0: + return xp.eye(n_features) / self.noise_variance_ + + if is_array_api_compliant: + linalg_inv = xp.linalg.inv + else: + linalg_inv = linalg.inv + + if self.noise_variance_ == 0.0: + return linalg_inv(self.get_covariance()) + + # Get precision using matrix inversion lemma + components_ = self.components_ + exp_var = self.explained_variance_ + if self.whiten: + components_ = components_ * xp.sqrt(exp_var[:, np.newaxis]) + exp_var_diff = exp_var - self.noise_variance_ + exp_var_diff = xp.where( + exp_var > self.noise_variance_, + exp_var_diff, + xp.asarray(0.0, device=device(exp_var)), + ) + precision = components_ @ components_.T / self.noise_variance_ + _add_to_diagonal(precision, 1.0 / exp_var_diff, xp) + precision = components_.T @ linalg_inv(precision) @ components_ + precision /= -(self.noise_variance_**2) + _add_to_diagonal(precision, 1.0 / self.noise_variance_, xp) + return precision + + @abstractmethod + def fit(self, X, y=None): + """Placeholder for fit. Subclasses should implement this method! + + Fit the model with X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + self : object + Returns the instance itself. + """ + + def transform(self, X): + """Apply dimensionality reduction to X. + + X is projected on the first principal components previously extracted + from a training set. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + New data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + Returns + ------- + X_new : array-like of shape (n_samples, n_components) + Projection of X in the first principal components, where `n_samples` + is the number of samples and `n_components` is the number of the components. + """ + xp, _ = get_namespace(X, self.components_, self.explained_variance_) + + check_is_fitted(self) + + X = validate_data( + self, + X, + dtype=[xp.float64, xp.float32], + accept_sparse=("csr", "csc"), + reset=False, + ) + return self._transform(X, xp=xp, x_is_centered=False) + + def _transform(self, X, xp, x_is_centered=False): + X_transformed = X @ self.components_.T + if not x_is_centered: + # Apply the centering after the projection. + # For dense X this avoids copying or mutating the data passed by + # the caller. + # For sparse X it keeps sparsity and avoids having to wrap X into + # a linear operator. + X_transformed -= xp.reshape(self.mean_, (1, -1)) @ self.components_.T + if self.whiten: + # For some solvers (such as "arpack" and "covariance_eigh"), on + # rank deficient data, some components can have a variance + # arbitrarily close to zero, leading to non-finite results when + # whitening. To avoid this problem we clip the variance below. + scale = xp.sqrt(self.explained_variance_) + min_scale = xp.finfo(scale.dtype).eps + scale[scale < min_scale] = min_scale + X_transformed /= scale + return X_transformed + + def inverse_transform(self, X): + """Transform data back to its original space. + + In other words, return an input `X_original` whose transform would be X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_components) + New data, where `n_samples` is the number of samples + and `n_components` is the number of components. + + Returns + ------- + X_original : array-like of shape (n_samples, n_features) + Original data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + Notes + ----- + If whitening is enabled, inverse_transform will compute the + exact inverse operation, which includes reversing whitening. + """ + xp, _ = get_namespace(X, self.components_, self.explained_variance_) + + check_is_fitted(self) + + X = check_array(X, input_name="X", dtype=[xp.float64, xp.float32]) + + if self.whiten: + scaled_components = ( + xp.sqrt(self.explained_variance_[:, np.newaxis]) * self.components_ + ) + return X @ scaled_components + self.mean_ + else: + return X @ self.components_ + self.mean_ + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..0942c1372aedeccf8c10887c40569b0cd2836cb6 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..f6bf1f89da6d78915e9c418dac01e9f0a2d05f7e Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.pyx new file mode 100644 index 0000000000000000000000000000000000000000..5e12ff8ac084f3e82b30b031a1f6da863a31b443 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_cdnmf_fast.pyx @@ -0,0 +1,38 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from cython cimport floating +from libc.math cimport fabs + + +def _update_cdnmf_fast(floating[:, ::1] W, floating[:, :] HHt, + floating[:, :] XHt, Py_ssize_t[::1] permutation): + cdef: + floating violation = 0 + Py_ssize_t n_components = W.shape[1] + Py_ssize_t n_samples = W.shape[0] # n_features for H update + floating grad, pg, hess + Py_ssize_t i, r, s, t + + with nogil: + for s in range(n_components): + t = permutation[s] + + for i in range(n_samples): + # gradient = GW[t, i] where GW = np.dot(W, HHt) - XHt + grad = -XHt[i, t] + + for r in range(n_components): + grad += HHt[t, r] * W[i, r] + + # projected gradient + pg = min(0., grad) if W[i, t] == 0 else grad + violation += fabs(pg) + + # Hessian + hess = HHt[t, t] + + if hess != 0: + W[i, t] = max(W[i, t] - grad / hess, 0.) + + return violation diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_dict_learning.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_dict_learning.py new file mode 100644 index 0000000000000000000000000000000000000000..c50c28b1c645f83c4f68bd1388c1acb14373fddb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_dict_learning.py @@ -0,0 +1,2341 @@ +"""Dictionary learning.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools +import sys +import time +from numbers import Integral, Real + +import numpy as np +from joblib import effective_n_jobs +from scipy import linalg + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.linear_model import Lars, Lasso, LassoLars, orthogonal_mp_gram +from sklearn.utils import check_array, check_random_state, gen_batches, gen_even_slices +from sklearn.utils._param_validation import Interval, StrOptions, validate_params +from sklearn.utils.extmath import _randomized_svd, row_norms, svd_flip +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.validation import check_is_fitted, validate_data + + +def _check_positive_coding(method, positive): + if positive and method in ["omp", "lars"]: + raise ValueError( + "Positive constraint not supported for '{}' coding method.".format(method) + ) + + +def _sparse_encode_precomputed( + X, + dictionary, + *, + gram=None, + cov=None, + algorithm="lasso_lars", + regularization=None, + copy_cov=True, + init=None, + max_iter=1000, + verbose=0, + positive=False, +): + """Generic sparse coding with precomputed Gram and/or covariance matrices. + + Each row of the result is the solution to a Lasso problem. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + Data matrix. + + dictionary : ndarray of shape (n_components, n_features) + The dictionary matrix against which to solve the sparse coding of + the data. Some of the algorithms assume normalized rows. + + gram : ndarray of shape (n_components, n_components), default=None + Precomputed Gram matrix, `dictionary * dictionary'` + gram can be `None` if method is 'threshold'. + + cov : ndarray of shape (n_components, n_samples), default=None + Precomputed covariance, `dictionary * X'`. + + algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'}, \ + default='lasso_lars' + The algorithm used: + + * `'lars'`: uses the least angle regression method + (`linear_model.lars_path`); + * `'lasso_lars'`: uses Lars to compute the Lasso solution; + * `'lasso_cd'`: uses the coordinate descent method to compute the + Lasso solution (`linear_model.Lasso`). lasso_lars will be faster if + the estimated components are sparse; + * `'omp'`: uses orthogonal matching pursuit to estimate the sparse + solution; + * `'threshold'`: squashes to zero all coefficients less than + regularization from the projection `dictionary * data'`. + + regularization : int or float, default=None + The regularization parameter. It corresponds to alpha when + algorithm is `'lasso_lars'`, `'lasso_cd'` or `'threshold'`. + Otherwise it corresponds to `n_nonzero_coefs`. + + init : ndarray of shape (n_samples, n_components), default=None + Initialization value of the sparse code. Only used if + `algorithm='lasso_cd'`. + + max_iter : int, default=1000 + Maximum number of iterations to perform if `algorithm='lasso_cd'` or + `'lasso_lars'`. + + copy_cov : bool, default=True + Whether to copy the precomputed covariance matrix; if `False`, it may + be overwritten. + + verbose : int, default=0 + Controls the verbosity; the higher, the more messages. + + positive: bool, default=False + Whether to enforce a positivity constraint on the sparse code. + + .. versionadded:: 0.20 + + Returns + ------- + code : ndarray of shape (n_components, n_features) + The sparse codes. + """ + n_samples, n_features = X.shape + n_components = dictionary.shape[0] + + if algorithm == "lasso_lars": + alpha = float(regularization) / n_features # account for scaling + try: + err_mgt = np.seterr(all="ignore") + + # Not passing in verbose=max(0, verbose-1) because Lars.fit already + # corrects the verbosity level. + lasso_lars = LassoLars( + alpha=alpha, + fit_intercept=False, + verbose=verbose, + precompute=gram, + fit_path=False, + positive=positive, + max_iter=max_iter, + ) + lasso_lars.fit(dictionary.T, X.T, Xy=cov) + new_code = lasso_lars.coef_ + finally: + np.seterr(**err_mgt) + + elif algorithm == "lasso_cd": + alpha = float(regularization) / n_features # account for scaling + + # TODO: Make verbosity argument for Lasso? + # sklearn.linear_model.coordinate_descent.enet_path has a verbosity + # argument that we could pass in from Lasso. + clf = Lasso( + alpha=alpha, + fit_intercept=False, + precompute=gram, + tol=1e-8, # TODO: This parameter should be exposed. + max_iter=max_iter, + warm_start=True, + positive=positive, + ) + + if init is not None: + # In some workflows using coordinate descent algorithms: + # - users might provide NumPy arrays with read-only buffers + # - `joblib` might memmap arrays making their buffer read-only + # TODO: move this handling (which is currently too broad) + # closer to the actual private function which need buffers to be writable. + if not init.flags["WRITEABLE"]: + init = np.array(init) + clf.coef_ = init + + clf.fit(dictionary.T, X.T, check_input=False) + new_code = clf.coef_ + + elif algorithm == "lars": + try: + err_mgt = np.seterr(all="ignore") + + # Not passing in verbose=max(0, verbose-1) because Lars.fit already + # corrects the verbosity level. + lars = Lars( + fit_intercept=False, + verbose=verbose, + precompute=gram, + n_nonzero_coefs=int(regularization), + fit_path=False, + ) + lars.fit(dictionary.T, X.T, Xy=cov) + new_code = lars.coef_ + finally: + np.seterr(**err_mgt) + + elif algorithm == "threshold": + new_code = (np.sign(cov) * np.maximum(np.abs(cov) - regularization, 0)).T + if positive: + np.clip(new_code, 0, None, out=new_code) + + elif algorithm == "omp": + new_code = orthogonal_mp_gram( + Gram=gram, + Xy=cov, + n_nonzero_coefs=int(regularization), + tol=None, + norms_squared=row_norms(X, squared=True), + copy_Xy=copy_cov, + ).T + + return new_code.reshape(n_samples, n_components) + + +@validate_params( + { + "X": ["array-like"], + "dictionary": ["array-like"], + "gram": ["array-like", None], + "cov": ["array-like", None], + "algorithm": [ + StrOptions({"lasso_lars", "lasso_cd", "lars", "omp", "threshold"}) + ], + "n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None], + "alpha": [Interval(Real, 0, None, closed="left"), None], + "copy_cov": ["boolean"], + "init": ["array-like", None], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "n_jobs": [Integral, None], + "check_input": ["boolean"], + "verbose": ["verbose"], + "positive": ["boolean"], + }, + prefer_skip_nested_validation=True, +) +# XXX : could be moved to the linear_model module +def sparse_encode( + X, + dictionary, + *, + gram=None, + cov=None, + algorithm="lasso_lars", + n_nonzero_coefs=None, + alpha=None, + copy_cov=True, + init=None, + max_iter=1000, + n_jobs=None, + check_input=True, + verbose=0, + positive=False, +): + """Sparse coding. + + Each row of the result is the solution to a sparse coding problem. + The goal is to find a sparse array `code` such that:: + + X ~= code * dictionary + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Data matrix. + + dictionary : array-like of shape (n_components, n_features) + The dictionary matrix against which to solve the sparse coding of + the data. Some of the algorithms assume normalized rows for meaningful + output. + + gram : array-like of shape (n_components, n_components), default=None + Precomputed Gram matrix, `dictionary * dictionary'`. + + cov : array-like of shape (n_components, n_samples), default=None + Precomputed covariance, `dictionary' * X`. + + algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'}, \ + default='lasso_lars' + The algorithm used: + + * `'lars'`: uses the least angle regression method + (`linear_model.lars_path`); + * `'lasso_lars'`: uses Lars to compute the Lasso solution; + * `'lasso_cd'`: uses the coordinate descent method to compute the + Lasso solution (`linear_model.Lasso`). lasso_lars will be faster if + the estimated components are sparse; + * `'omp'`: uses orthogonal matching pursuit to estimate the sparse + solution; + * `'threshold'`: squashes to zero all coefficients less than + regularization from the projection `dictionary * data'`. + + n_nonzero_coefs : int, default=None + Number of nonzero coefficients to target in each column of the + solution. This is only used by `algorithm='lars'` and `algorithm='omp'` + and is overridden by `alpha` in the `omp` case. If `None`, then + `n_nonzero_coefs=int(n_features / 10)`. + + alpha : float, default=None + If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the + penalty applied to the L1 norm. + If `algorithm='threshold'`, `alpha` is the absolute value of the + threshold below which coefficients will be squashed to zero. + If `algorithm='omp'`, `alpha` is the tolerance parameter: the value of + the reconstruction error targeted. In this case, it overrides + `n_nonzero_coefs`. + If `None`, default to 1. + + copy_cov : bool, default=True + Whether to copy the precomputed covariance matrix; if `False`, it may + be overwritten. + + init : ndarray of shape (n_samples, n_components), default=None + Initialization value of the sparse codes. Only used if + `algorithm='lasso_cd'`. + + max_iter : int, default=1000 + Maximum number of iterations to perform if `algorithm='lasso_cd'` or + `'lasso_lars'`. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + check_input : bool, default=True + If `False`, the input arrays X and dictionary will not be checked. + + verbose : int, default=0 + Controls the verbosity; the higher, the more messages. + + positive : bool, default=False + Whether to enforce positivity when finding the encoding. + + .. versionadded:: 0.20 + + Returns + ------- + code : ndarray of shape (n_samples, n_components) + The sparse codes. + + See Also + -------- + sklearn.linear_model.lars_path : Compute Least Angle Regression or Lasso + path using LARS algorithm. + sklearn.linear_model.orthogonal_mp : Solves Orthogonal Matching Pursuit problems. + sklearn.linear_model.Lasso : Train Linear Model with L1 prior as regularizer. + SparseCoder : Find a sparse representation of data from a fixed precomputed + dictionary. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.decomposition import sparse_encode + >>> X = np.array([[-1, -1, -1], [0, 0, 3]]) + >>> dictionary = np.array( + ... [[0, 1, 0], + ... [-1, -1, 2], + ... [1, 1, 1], + ... [0, 1, 1], + ... [0, 2, 1]], + ... dtype=np.float64 + ... ) + >>> sparse_encode(X, dictionary, alpha=1e-10) + array([[ 0., 0., -1., 0., 0.], + [ 0., 1., 1., 0., 0.]]) + """ + if check_input: + order = "C" if algorithm == "lasso_cd" else None + dictionary = check_array( + dictionary, order=order, dtype=[np.float64, np.float32] + ) + X = check_array(X, order=order, dtype=[np.float64, np.float32]) + + if dictionary.shape[1] != X.shape[1]: + raise ValueError( + "Dictionary and X have different numbers of features:" + "dictionary.shape: {} X.shape{}".format(dictionary.shape, X.shape) + ) + + _check_positive_coding(algorithm, positive) + + return _sparse_encode( + X, + dictionary, + gram=gram, + cov=cov, + algorithm=algorithm, + n_nonzero_coefs=n_nonzero_coefs, + alpha=alpha, + copy_cov=copy_cov, + init=init, + max_iter=max_iter, + n_jobs=n_jobs, + verbose=verbose, + positive=positive, + ) + + +def _sparse_encode( + X, + dictionary, + *, + gram=None, + cov=None, + algorithm="lasso_lars", + n_nonzero_coefs=None, + alpha=None, + copy_cov=True, + init=None, + max_iter=1000, + n_jobs=None, + verbose=0, + positive=False, +): + """Sparse coding without input/parameter validation.""" + + n_samples, n_features = X.shape + n_components = dictionary.shape[0] + + if algorithm in ("lars", "omp"): + regularization = n_nonzero_coefs + if regularization is None: + regularization = min(max(n_features / 10, 1), n_components) + else: + regularization = alpha + if regularization is None: + regularization = 1.0 + + if gram is None and algorithm != "threshold": + gram = np.dot(dictionary, dictionary.T).astype(X.dtype, copy=False) + + if cov is None and algorithm != "lasso_cd": + copy_cov = False + cov = np.dot(dictionary, X.T) + + if effective_n_jobs(n_jobs) == 1 or algorithm == "threshold": + code = _sparse_encode_precomputed( + X, + dictionary, + gram=gram, + cov=cov, + algorithm=algorithm, + regularization=regularization, + copy_cov=copy_cov, + init=init, + max_iter=max_iter, + verbose=verbose, + positive=positive, + ) + return code + + # Enter parallel code block + n_samples = X.shape[0] + n_components = dictionary.shape[0] + code = np.empty((n_samples, n_components)) + slices = list(gen_even_slices(n_samples, effective_n_jobs(n_jobs))) + + code_views = Parallel(n_jobs=n_jobs, verbose=verbose)( + delayed(_sparse_encode_precomputed)( + X[this_slice], + dictionary, + gram=gram, + cov=cov[:, this_slice] if cov is not None else None, + algorithm=algorithm, + regularization=regularization, + copy_cov=copy_cov, + init=init[this_slice] if init is not None else None, + max_iter=max_iter, + verbose=verbose, + positive=positive, + ) + for this_slice in slices + ) + for this_slice, this_view in zip(slices, code_views): + code[this_slice] = this_view + return code + + +def _update_dict( + dictionary, + Y, + code, + A=None, + B=None, + verbose=False, + random_state=None, + positive=False, +): + """Update the dense dictionary factor in place. + + Parameters + ---------- + dictionary : ndarray of shape (n_components, n_features) + Value of the dictionary at the previous iteration. + + Y : ndarray of shape (n_samples, n_features) + Data matrix. + + code : ndarray of shape (n_samples, n_components) + Sparse coding of the data against which to optimize the dictionary. + + A : ndarray of shape (n_components, n_components), default=None + Together with `B`, sufficient stats of the online model to update the + dictionary. + + B : ndarray of shape (n_features, n_components), default=None + Together with `A`, sufficient stats of the online model to update the + dictionary. + + verbose: bool, default=False + Degree of output the procedure will print. + + random_state : int, RandomState instance or None, default=None + Used for randomly initializing the dictionary. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. + + positive : bool, default=False + Whether to enforce positivity when finding the dictionary. + + .. versionadded:: 0.20 + """ + n_samples, n_components = code.shape + random_state = check_random_state(random_state) + + if A is None: + A = code.T @ code + if B is None: + B = Y.T @ code + + n_unused = 0 + + for k in range(n_components): + if A[k, k] > 1e-6: + # 1e-6 is arbitrary but consistent with the spams implementation + dictionary[k] += (B[:, k] - A[k] @ dictionary) / A[k, k] + else: + # kth atom is almost never used -> sample a new one from the data + newd = Y[random_state.choice(n_samples)] + + # add small noise to avoid making the sparse coding ill conditioned + noise_level = 0.01 * (newd.std() or 1) # avoid 0 std + noise = random_state.normal(0, noise_level, size=len(newd)) + + dictionary[k] = newd + noise + code[:, k] = 0 + n_unused += 1 + + if positive: + np.clip(dictionary[k], 0, None, out=dictionary[k]) + + # Projection on the constraint set ||V_k|| <= 1 + dictionary[k] /= max(linalg.norm(dictionary[k]), 1) + + if verbose and n_unused > 0: + print(f"{n_unused} unused atoms resampled.") + + +def _dict_learning( + X, + n_components, + *, + alpha, + max_iter, + tol, + method, + n_jobs, + dict_init, + code_init, + callback, + verbose, + random_state, + return_n_iter, + positive_dict, + positive_code, + method_max_iter, +): + """Main dictionary learning algorithm""" + t0 = time.time() + # Init the code and the dictionary with SVD of Y + if code_init is not None and dict_init is not None: + code = np.array(code_init, order="F") + # Don't copy V, it will happen below + dictionary = dict_init + else: + code, S, dictionary = linalg.svd(X, full_matrices=False) + # flip the initial code's sign to enforce deterministic output + code, dictionary = svd_flip(code, dictionary) + dictionary = S[:, np.newaxis] * dictionary + r = len(dictionary) + if n_components <= r: # True even if n_components=None + code = code[:, :n_components] + dictionary = dictionary[:n_components, :] + else: + code = np.c_[code, np.zeros((len(code), n_components - r))] + dictionary = np.r_[ + dictionary, np.zeros((n_components - r, dictionary.shape[1])) + ] + + # Fortran-order dict better suited for the sparse coding which is the + # bottleneck of this algorithm. + dictionary = np.asfortranarray(dictionary) + + errors = [] + current_cost = np.nan + + if verbose == 1: + print("[dict_learning]", end=" ") + + # If max_iter is 0, number of iterations returned should be zero + ii = -1 + + for ii in range(max_iter): + dt = time.time() - t0 + if verbose == 1: + sys.stdout.write(".") + sys.stdout.flush() + elif verbose: + print( + "Iteration % 3i (elapsed time: % 3is, % 4.1fmn, current cost % 7.3f)" + % (ii, dt, dt / 60, current_cost) + ) + + # Update code + code = sparse_encode( + X, + dictionary, + algorithm=method, + alpha=alpha, + init=code, + n_jobs=n_jobs, + positive=positive_code, + max_iter=method_max_iter, + verbose=verbose, + ) + + # Update dictionary in place + _update_dict( + dictionary, + X, + code, + verbose=verbose, + random_state=random_state, + positive=positive_dict, + ) + + # Cost function + current_cost = 0.5 * np.sum((X - code @ dictionary) ** 2) + alpha * np.sum( + np.abs(code) + ) + errors.append(current_cost) + + if ii > 0: + dE = errors[-2] - errors[-1] + # assert(dE >= -tol * errors[-1]) + if dE < tol * errors[-1]: + if verbose == 1: + # A line return + print("") + elif verbose: + print("--- Convergence reached after %d iterations" % ii) + break + if ii % 5 == 0 and callback is not None: + callback(locals()) + + if return_n_iter: + return code, dictionary, errors, ii + 1 + else: + return code, dictionary, errors + + +@validate_params( + { + "X": ["array-like"], + "return_code": ["boolean"], + "method": [StrOptions({"cd", "lars"})], + "method_max_iter": [Interval(Integral, 0, None, closed="left")], + }, + prefer_skip_nested_validation=False, +) +def dict_learning_online( + X, + n_components=2, + *, + alpha=1, + max_iter=100, + return_code=True, + dict_init=None, + callback=None, + batch_size=256, + verbose=False, + shuffle=True, + n_jobs=None, + method="lars", + random_state=None, + positive_dict=False, + positive_code=False, + method_max_iter=1000, + tol=1e-3, + max_no_improvement=10, +): + """Solve a dictionary learning matrix factorization problem online. + + Finds the best dictionary and the corresponding sparse code for + approximating the data matrix X by solving:: + + (U^*, V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 + (U,V) + with || V_k ||_2 = 1 for all 0 <= k < n_components + + where V is the dictionary and U is the sparse code. ||.||_Fro stands for + the Frobenius norm and ||.||_1,1 stands for the entry-wise matrix norm + which is the sum of the absolute values of all the entries in the matrix. + This is accomplished by repeatedly iterating over mini-batches by slicing + the input data. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Data matrix. + + n_components : int or None, default=2 + Number of dictionary atoms to extract. If None, then ``n_components`` + is set to ``n_features``. + + alpha : float, default=1 + Sparsity controlling parameter. + + max_iter : int, default=100 + Maximum number of iterations over the complete dataset before + stopping independently of any early stopping criterion heuristics. + + .. versionadded:: 1.1 + + return_code : bool, default=True + Whether to also return the code U or just the dictionary `V`. + + dict_init : ndarray of shape (n_components, n_features), default=None + Initial values for the dictionary for warm restart scenarios. + If `None`, the initial values for the dictionary are created + with an SVD decomposition of the data via + :func:`~sklearn.utils.extmath.randomized_svd`. + + callback : callable, default=None + A callable that gets invoked at the end of each iteration. + + batch_size : int, default=256 + The number of samples to take in each batch. + + .. versionchanged:: 1.3 + The default value of `batch_size` changed from 3 to 256 in version 1.3. + + verbose : bool, default=False + To control the verbosity of the procedure. + + shuffle : bool, default=True + Whether to shuffle the data before splitting it in batches. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + method : {'lars', 'cd'}, default='lars' + * `'lars'`: uses the least angle regression method to solve the lasso + problem (`linear_model.lars_path`); + * `'cd'`: uses the coordinate descent method to compute the + Lasso solution (`linear_model.Lasso`). Lars will be faster if + the estimated components are sparse. + + random_state : int, RandomState instance or None, default=None + Used for initializing the dictionary when ``dict_init`` is not + specified, randomly shuffling the data when ``shuffle`` is set to + ``True``, and updating the dictionary. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + positive_dict : bool, default=False + Whether to enforce positivity when finding the dictionary. + + .. versionadded:: 0.20 + + positive_code : bool, default=False + Whether to enforce positivity when finding the code. + + .. versionadded:: 0.20 + + method_max_iter : int, default=1000 + Maximum number of iterations to perform when solving the lasso problem. + + .. versionadded:: 0.22 + + tol : float, default=1e-3 + Control early stopping based on the norm of the differences in the + dictionary between 2 steps. + + To disable early stopping based on changes in the dictionary, set + `tol` to 0.0. + + .. versionadded:: 1.1 + + max_no_improvement : int, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + + To disable convergence detection based on cost function, set + `max_no_improvement` to None. + + .. versionadded:: 1.1 + + Returns + ------- + code : ndarray of shape (n_samples, n_components), + The sparse code (only returned if `return_code=True`). + + dictionary : ndarray of shape (n_components, n_features), + The solutions to the dictionary learning problem. + + n_iter : int + Number of iterations run. Returned only if `return_n_iter` is + set to `True`. + + See Also + -------- + dict_learning : Solve a dictionary learning matrix factorization problem. + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchDictionaryLearning : A faster, less accurate, version of the dictionary + learning algorithm. + SparsePCA : Sparse Principal Components Analysis. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_sparse_coded_signal + >>> from sklearn.decomposition import dict_learning_online + >>> X, _, _ = make_sparse_coded_signal( + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, + ... random_state=42, + ... ) + >>> U, V = dict_learning_online( + ... X, n_components=15, alpha=0.2, max_iter=20, batch_size=3, random_state=42 + ... ) + + We can check the level of sparsity of `U`: + + >>> np.mean(U == 0) + np.float64(0.53) + + We can compare the average squared euclidean norm of the reconstruction + error of the sparse coded signal relative to the squared euclidean norm of + the original signal: + + >>> X_hat = U @ V + >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) + np.float64(0.053) + """ + transform_algorithm = "lasso_" + method + + est = MiniBatchDictionaryLearning( + n_components=n_components, + alpha=alpha, + max_iter=max_iter, + n_jobs=n_jobs, + fit_algorithm=method, + batch_size=batch_size, + shuffle=shuffle, + dict_init=dict_init, + random_state=random_state, + transform_algorithm=transform_algorithm, + transform_alpha=alpha, + positive_code=positive_code, + positive_dict=positive_dict, + transform_max_iter=method_max_iter, + verbose=verbose, + callback=callback, + tol=tol, + max_no_improvement=max_no_improvement, + ).fit(X) + + if not return_code: + return est.components_ + else: + code = est.transform(X) + return code, est.components_ + + +@validate_params( + { + "X": ["array-like"], + "method": [StrOptions({"lars", "cd"})], + "return_n_iter": ["boolean"], + "method_max_iter": [Interval(Integral, 0, None, closed="left")], + }, + prefer_skip_nested_validation=False, +) +def dict_learning( + X, + n_components, + *, + alpha, + max_iter=100, + tol=1e-8, + method="lars", + n_jobs=None, + dict_init=None, + code_init=None, + callback=None, + verbose=False, + random_state=None, + return_n_iter=False, + positive_dict=False, + positive_code=False, + method_max_iter=1000, +): + """Solve a dictionary learning matrix factorization problem. + + Finds the best dictionary and the corresponding sparse code for + approximating the data matrix X by solving:: + + (U^*, V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 + (U,V) + with || V_k ||_2 = 1 for all 0 <= k < n_components + + where V is the dictionary and U is the sparse code. ||.||_Fro stands for + the Frobenius norm and ||.||_1,1 stands for the entry-wise matrix norm + which is the sum of the absolute values of all the entries in the matrix. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Data matrix. + + n_components : int + Number of dictionary atoms to extract. + + alpha : int or float + Sparsity controlling parameter. + + max_iter : int, default=100 + Maximum number of iterations to perform. + + tol : float, default=1e-8 + Tolerance for the stopping condition. + + method : {'lars', 'cd'}, default='lars' + The method used: + + * `'lars'`: uses the least angle regression method to solve the lasso + problem (`linear_model.lars_path`); + * `'cd'`: uses the coordinate descent method to compute the + Lasso solution (`linear_model.Lasso`). Lars will be faster if + the estimated components are sparse. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + dict_init : ndarray of shape (n_components, n_features), default=None + Initial value for the dictionary for warm restart scenarios. Only used + if `code_init` and `dict_init` are not None. + + code_init : ndarray of shape (n_samples, n_components), default=None + Initial value for the sparse code for warm restart scenarios. Only used + if `code_init` and `dict_init` are not None. + + callback : callable, default=None + Callable that gets invoked every five iterations. + + verbose : bool, default=False + To control the verbosity of the procedure. + + random_state : int, RandomState instance or None, default=None + Used for randomly initializing the dictionary. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. + + return_n_iter : bool, default=False + Whether or not to return the number of iterations. + + positive_dict : bool, default=False + Whether to enforce positivity when finding the dictionary. + + .. versionadded:: 0.20 + + positive_code : bool, default=False + Whether to enforce positivity when finding the code. + + .. versionadded:: 0.20 + + method_max_iter : int, default=1000 + Maximum number of iterations to perform. + + .. versionadded:: 0.22 + + Returns + ------- + code : ndarray of shape (n_samples, n_components) + The sparse code factor in the matrix factorization. + + dictionary : ndarray of shape (n_components, n_features), + The dictionary factor in the matrix factorization. + + errors : array + Vector of errors at each iteration. + + n_iter : int + Number of iterations run. Returned only if `return_n_iter` is + set to True. + + See Also + -------- + dict_learning_online : Solve a dictionary learning matrix factorization + problem online. + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchDictionaryLearning : A faster, less accurate version + of the dictionary learning algorithm. + SparsePCA : Sparse Principal Components Analysis. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_sparse_coded_signal + >>> from sklearn.decomposition import dict_learning + >>> X, _, _ = make_sparse_coded_signal( + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, + ... random_state=42, + ... ) + >>> U, V, errors = dict_learning(X, n_components=15, alpha=0.1, random_state=42) + + We can check the level of sparsity of `U`: + + >>> np.mean(U == 0) + np.float64(0.62) + + We can compare the average squared euclidean norm of the reconstruction + error of the sparse coded signal relative to the squared euclidean norm of + the original signal: + + >>> X_hat = U @ V + >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) + np.float64(0.0192) + """ + estimator = DictionaryLearning( + n_components=n_components, + alpha=alpha, + max_iter=max_iter, + tol=tol, + fit_algorithm=method, + n_jobs=n_jobs, + dict_init=dict_init, + callback=callback, + code_init=code_init, + verbose=verbose, + random_state=random_state, + positive_code=positive_code, + positive_dict=positive_dict, + transform_max_iter=method_max_iter, + ).set_output(transform="default") + code = estimator.fit_transform(X) + if return_n_iter: + return ( + code, + estimator.components_, + estimator.error_, + estimator.n_iter_, + ) + return code, estimator.components_, estimator.error_ + + +class _BaseSparseCoding(ClassNamePrefixFeaturesOutMixin, TransformerMixin): + """Base class from SparseCoder and DictionaryLearning algorithms.""" + + def __init__( + self, + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, + ): + self.transform_algorithm = transform_algorithm + self.transform_n_nonzero_coefs = transform_n_nonzero_coefs + self.transform_alpha = transform_alpha + self.transform_max_iter = transform_max_iter + self.split_sign = split_sign + self.n_jobs = n_jobs + self.positive_code = positive_code + + def _transform(self, X, dictionary): + """Private method allowing to accommodate both DictionaryLearning and + SparseCoder.""" + X = validate_data(self, X, reset=False) + + if hasattr(self, "alpha") and self.transform_alpha is None: + transform_alpha = self.alpha + else: + transform_alpha = self.transform_alpha + + code = sparse_encode( + X, + dictionary, + algorithm=self.transform_algorithm, + n_nonzero_coefs=self.transform_n_nonzero_coefs, + alpha=transform_alpha, + max_iter=self.transform_max_iter, + n_jobs=self.n_jobs, + positive=self.positive_code, + ) + + if self.split_sign: + # feature vector is split into a positive and negative side + n_samples, n_features = code.shape + split_code = np.empty((n_samples, 2 * n_features)) + split_code[:, :n_features] = np.maximum(code, 0) + split_code[:, n_features:] = -np.minimum(code, 0) + code = split_code + + return code + + def transform(self, X): + """Encode the data as a sparse combination of the dictionary atoms. + + Coding method is determined by the object parameter + `transform_algorithm`. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + Test data to be transformed, must have the same number of + features as the data used to train the model. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Transformed data. + """ + check_is_fitted(self) + return self._transform(X, self.components_) + + def _inverse_transform(self, code, dictionary): + """Private method allowing to accommodate both DictionaryLearning and + SparseCoder.""" + code = check_array(code) + # compute number of expected features in code + expected_n_components = dictionary.shape[0] + if self.split_sign: + expected_n_components += expected_n_components + if not code.shape[1] == expected_n_components: + raise ValueError( + "The number of components in the code is different from the " + "number of components in the dictionary." + f"Expected {expected_n_components}, got {code.shape[1]}." + ) + if self.split_sign: + n_samples, n_features = code.shape + n_features //= 2 + code = code[:, :n_features] - code[:, n_features:] + + return code @ dictionary + + def inverse_transform(self, X): + """Transform data back to its original space. + + Parameters + ---------- + X : array-like of shape (n_samples, n_components) + Data to be transformed back. Must have the same number of + components as the data used to train the model. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Transformed data. + """ + check_is_fitted(self) + return self._inverse_transform(X, self.components_) + + +class SparseCoder(_BaseSparseCoding, BaseEstimator): + """Sparse coding. + + Finds a sparse representation of data against a fixed, precomputed + dictionary. + + Each row of the result is the solution to a sparse coding problem. + The goal is to find a sparse array `code` such that:: + + X ~= code * dictionary + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + dictionary : ndarray of shape (n_components, n_features) + The dictionary atoms used for sparse coding. Lines are assumed to be + normalized to unit norm. + + transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \ + 'threshold'}, default='omp' + Algorithm used to transform the data: + + - `'lars'`: uses the least angle regression method + (`linear_model.lars_path`); + - `'lasso_lars'`: uses Lars to compute the Lasso solution; + - `'lasso_cd'`: uses the coordinate descent method to compute the + Lasso solution (linear_model.Lasso). `'lasso_lars'` will be faster if + the estimated components are sparse; + - `'omp'`: uses orthogonal matching pursuit to estimate the sparse + solution; + - `'threshold'`: squashes to zero all coefficients less than alpha from + the projection ``dictionary * X'``. + + transform_n_nonzero_coefs : int, default=None + Number of nonzero coefficients to target in each column of the + solution. This is only used by `algorithm='lars'` and `algorithm='omp'` + and is overridden by `alpha` in the `omp` case. If `None`, then + `transform_n_nonzero_coefs=int(n_features / 10)`. + + transform_alpha : float, default=None + If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the + penalty applied to the L1 norm. + If `algorithm='threshold'`, `alpha` is the absolute value of the + threshold below which coefficients will be squashed to zero. + If `algorithm='omp'`, `alpha` is the tolerance parameter: the value of + the reconstruction error targeted. In this case, it overrides + `n_nonzero_coefs`. + If `None`, default to 1. + + split_sign : bool, default=False + Whether to split the sparse feature vector into the concatenation of + its negative part and its positive part. This can improve the + performance of downstream classifiers. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + positive_code : bool, default=False + Whether to enforce positivity when finding the code. + + .. versionadded:: 0.20 + + transform_max_iter : int, default=1000 + Maximum number of iterations to perform if `algorithm='lasso_cd'` or + `lasso_lars`. + + .. versionadded:: 0.22 + + Attributes + ---------- + n_components_ : int + Number of atoms. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchDictionaryLearning : A faster, less accurate, version of the + dictionary learning algorithm. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparsePCA : Sparse Principal Components Analysis. + sparse_encode : Sparse coding where each row of the result is the solution + to a sparse coding problem. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.decomposition import SparseCoder + >>> X = np.array([[-1, -1, -1], [0, 0, 3]]) + >>> dictionary = np.array( + ... [[0, 1, 0], + ... [-1, -1, 2], + ... [1, 1, 1], + ... [0, 1, 1], + ... [0, 2, 1]], + ... dtype=np.float64 + ... ) + >>> coder = SparseCoder( + ... dictionary=dictionary, transform_algorithm='lasso_lars', + ... transform_alpha=1e-10, + ... ) + >>> coder.transform(X) + array([[ 0., 0., -1., 0., 0.], + [ 0., 1., 1., 0., 0.]]) + """ + + _parameter_constraints: dict = { + "dictionary": ["array-like"], + "transform_algorithm": [ + StrOptions({"lasso_lars", "lasso_cd", "lars", "omp", "threshold"}) + ], + "transform_n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None], + "transform_alpha": [Interval(Real, 0, None, closed="left"), None], + "split_sign": ["boolean"], + "n_jobs": [Integral, None], + "positive_code": ["boolean"], + "transform_max_iter": [Interval(Integral, 0, None, closed="left")], + } + + def __init__( + self, + dictionary, + *, + transform_algorithm="omp", + transform_n_nonzero_coefs=None, + transform_alpha=None, + split_sign=False, + n_jobs=None, + positive_code=False, + transform_max_iter=1000, + ): + super().__init__( + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, + ) + self.dictionary = dictionary + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Only validate the parameters of the estimator. + + This method allows to: (i) validate the parameters of the estimator and + (ii) be consistent with the scikit-learn transformer API. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. Only used for input validation. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + X = validate_data(self, X) + self.n_components_ = self.dictionary.shape[0] + if X.shape[1] != self.dictionary.shape[1]: + raise ValueError( + "Dictionary and X have different numbers of features:" + f"dictionary.shape: {self.dictionary.shape} X.shape{X.shape}" + ) + return self + + def transform(self, X, y=None): + """Encode the data as a sparse combination of the dictionary atoms. + + Coding method is determined by the object parameter + `transform_algorithm`. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Transformed data. + """ + return super()._transform(X, self.dictionary) + + def inverse_transform(self, X): + """Transform data back to its original space. + + Parameters + ---------- + X : array-like of shape (n_samples, n_components) + Data to be transformed back. Must have the same number of + components as the data used to train the model. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Transformed data. + """ + return self._inverse_transform(X, self.dictionary) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.requires_fit = False + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.n_components_ + + +class DictionaryLearning(_BaseSparseCoding, BaseEstimator): + """Dictionary learning. + + Finds a dictionary (a set of atoms) that performs well at sparsely + encoding the fitted data. + + Solves the optimization problem:: + + (U^*,V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 + (U,V) + with || V_k ||_2 <= 1 for all 0 <= k < n_components + + ||.||_Fro stands for the Frobenius norm and ||.||_1,1 stands for + the entry-wise matrix norm which is the sum of the absolute values + of all the entries in the matrix. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=None + Number of dictionary elements to extract. If None, then ``n_components`` + is set to ``n_features``. + + alpha : float, default=1.0 + Sparsity controlling parameter. + + max_iter : int, default=1000 + Maximum number of iterations to perform. + + tol : float, default=1e-8 + Tolerance for numerical error. + + fit_algorithm : {'lars', 'cd'}, default='lars' + * `'lars'`: uses the least angle regression method to solve the lasso + problem (:func:`~sklearn.linear_model.lars_path`); + * `'cd'`: uses the coordinate descent method to compute the + Lasso solution (:class:`~sklearn.linear_model.Lasso`). Lars will be + faster if the estimated components are sparse. + + .. versionadded:: 0.17 + *cd* coordinate descent method to improve speed. + + transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \ + 'threshold'}, default='omp' + Algorithm used to transform the data: + + - `'lars'`: uses the least angle regression method + (:func:`~sklearn.linear_model.lars_path`); + - `'lasso_lars'`: uses Lars to compute the Lasso solution. + - `'lasso_cd'`: uses the coordinate descent method to compute the + Lasso solution (:class:`~sklearn.linear_model.Lasso`). `'lasso_lars'` + will be faster if the estimated components are sparse. + - `'omp'`: uses orthogonal matching pursuit to estimate the sparse + solution. + - `'threshold'`: squashes to zero all coefficients less than alpha from + the projection ``dictionary * X'``. + + .. versionadded:: 0.17 + *lasso_cd* coordinate descent method to improve speed. + + transform_n_nonzero_coefs : int, default=None + Number of nonzero coefficients to target in each column of the + solution. This is only used by `algorithm='lars'` and + `algorithm='omp'`. If `None`, then + `transform_n_nonzero_coefs=int(n_features / 10)`. + + transform_alpha : float, default=None + If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the + penalty applied to the L1 norm. + If `algorithm='threshold'`, `alpha` is the absolute value of the + threshold below which coefficients will be squashed to zero. + If `None`, defaults to `alpha`. + + .. versionchanged:: 1.2 + When None, default value changed from 1.0 to `alpha`. + + n_jobs : int or None, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + code_init : ndarray of shape (n_samples, n_components), default=None + Initial value for the code, for warm restart. Only used if `code_init` + and `dict_init` are not None. + + dict_init : ndarray of shape (n_components, n_features), default=None + Initial values for the dictionary, for warm restart. Only used if + `code_init` and `dict_init` are not None. + + callback : callable, default=None + Callable that gets invoked every five iterations. + + .. versionadded:: 1.3 + + verbose : bool, default=False + To control the verbosity of the procedure. + + split_sign : bool, default=False + Whether to split the sparse feature vector into the concatenation of + its negative part and its positive part. This can improve the + performance of downstream classifiers. + + random_state : int, RandomState instance or None, default=None + Used for initializing the dictionary when ``dict_init`` is not + specified, randomly shuffling the data when ``shuffle`` is set to + ``True``, and updating the dictionary. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + positive_code : bool, default=False + Whether to enforce positivity when finding the code. + + .. versionadded:: 0.20 + + positive_dict : bool, default=False + Whether to enforce positivity when finding the dictionary. + + .. versionadded:: 0.20 + + transform_max_iter : int, default=1000 + Maximum number of iterations to perform if `algorithm='lasso_cd'` or + `'lasso_lars'`. + + .. versionadded:: 0.22 + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + dictionary atoms extracted from the data + + error_ : array + vector of errors at each iteration + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_iter_ : int + Number of iterations run. + + See Also + -------- + MiniBatchDictionaryLearning: A faster, less accurate, version of the + dictionary learning algorithm. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparseCoder : Find a sparse representation of data from a fixed, + precomputed dictionary. + SparsePCA : Sparse Principal Components Analysis. + + References + ---------- + + J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning + for sparse coding (https://www.di.ens.fr/~fbach/mairal_icml09.pdf) + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_sparse_coded_signal + >>> from sklearn.decomposition import DictionaryLearning + >>> X, dictionary, code = make_sparse_coded_signal( + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, + ... random_state=42, + ... ) + >>> dict_learner = DictionaryLearning( + ... n_components=15, transform_algorithm='lasso_lars', transform_alpha=0.1, + ... random_state=42, + ... ) + >>> X_transformed = dict_learner.fit(X).transform(X) + + We can check the level of sparsity of `X_transformed`: + + >>> np.mean(X_transformed == 0) + np.float64(0.527) + + We can compare the average squared euclidean norm of the reconstruction + error of the sparse coded signal relative to the squared euclidean norm of + the original signal: + + >>> X_hat = X_transformed @ dict_learner.components_ + >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) + np.float64(0.056) + """ + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left")], + "fit_algorithm": [StrOptions({"lars", "cd"})], + "transform_algorithm": [ + StrOptions({"lasso_lars", "lasso_cd", "lars", "omp", "threshold"}) + ], + "transform_n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None], + "transform_alpha": [Interval(Real, 0, None, closed="left"), None], + "n_jobs": [Integral, None], + "code_init": [np.ndarray, None], + "dict_init": [np.ndarray, None], + "callback": [callable, None], + "verbose": ["verbose"], + "split_sign": ["boolean"], + "random_state": ["random_state"], + "positive_code": ["boolean"], + "positive_dict": ["boolean"], + "transform_max_iter": [Interval(Integral, 0, None, closed="left")], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + max_iter=1000, + tol=1e-8, + fit_algorithm="lars", + transform_algorithm="omp", + transform_n_nonzero_coefs=None, + transform_alpha=None, + n_jobs=None, + code_init=None, + dict_init=None, + callback=None, + verbose=False, + split_sign=False, + random_state=None, + positive_code=False, + positive_dict=False, + transform_max_iter=1000, + ): + super().__init__( + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, + ) + self.n_components = n_components + self.alpha = alpha + self.max_iter = max_iter + self.tol = tol + self.fit_algorithm = fit_algorithm + self.code_init = code_init + self.dict_init = dict_init + self.callback = callback + self.verbose = verbose + self.random_state = random_state + self.positive_dict = positive_dict + + def fit(self, X, y=None): + """Fit the model from data in X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + self.fit_transform(X) + return self + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None): + """Fit the model from data in X and return the transformed data. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + V : ndarray of shape (n_samples, n_components) + Transformed data. + """ + _check_positive_coding(method=self.fit_algorithm, positive=self.positive_code) + + method = "lasso_" + self.fit_algorithm + + random_state = check_random_state(self.random_state) + X = validate_data(self, X) + + if self.n_components is None: + n_components = X.shape[1] + else: + n_components = self.n_components + + V, U, E, self.n_iter_ = _dict_learning( + X, + n_components, + alpha=self.alpha, + tol=self.tol, + max_iter=self.max_iter, + method=method, + method_max_iter=self.transform_max_iter, + n_jobs=self.n_jobs, + code_init=self.code_init, + dict_init=self.dict_init, + callback=self.callback, + verbose=self.verbose, + random_state=random_state, + return_n_iter=True, + positive_dict=self.positive_dict, + positive_code=self.positive_code, + ) + self.components_ = U + self.error_ = E + + return V + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + +class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): + """Mini-batch dictionary learning. + + Finds a dictionary (a set of atoms) that performs well at sparsely + encoding the fitted data. + + Solves the optimization problem:: + + (U^*,V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 + (U,V) + with || V_k ||_2 <= 1 for all 0 <= k < n_components + + ||.||_Fro stands for the Frobenius norm and ||.||_1,1 stands for + the entry-wise matrix norm which is the sum of the absolute values + of all the entries in the matrix. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=None + Number of dictionary elements to extract. + + alpha : float, default=1 + Sparsity controlling parameter. + + max_iter : int, default=1_000 + Maximum number of iterations over the complete dataset before + stopping independently of any early stopping criterion heuristics. + + .. versionadded:: 1.1 + + fit_algorithm : {'lars', 'cd'}, default='lars' + The algorithm used: + + - `'lars'`: uses the least angle regression method to solve the lasso + problem (`linear_model.lars_path`) + - `'cd'`: uses the coordinate descent method to compute the + Lasso solution (`linear_model.Lasso`). Lars will be faster if + the estimated components are sparse. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + batch_size : int, default=256 + Number of samples in each mini-batch. + + .. versionchanged:: 1.3 + The default value of `batch_size` changed from 3 to 256 in version 1.3. + + shuffle : bool, default=True + Whether to shuffle the samples before forming batches. + + dict_init : ndarray of shape (n_components, n_features), default=None + Initial value of the dictionary for warm restart scenarios. + + transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \ + 'threshold'}, default='omp' + Algorithm used to transform the data: + + - `'lars'`: uses the least angle regression method + (`linear_model.lars_path`); + - `'lasso_lars'`: uses Lars to compute the Lasso solution. + - `'lasso_cd'`: uses the coordinate descent method to compute the + Lasso solution (`linear_model.Lasso`). `'lasso_lars'` will be faster + if the estimated components are sparse. + - `'omp'`: uses orthogonal matching pursuit to estimate the sparse + solution. + - `'threshold'`: squashes to zero all coefficients less than alpha from + the projection ``dictionary * X'``. + + transform_n_nonzero_coefs : int, default=None + Number of nonzero coefficients to target in each column of the + solution. This is only used by `algorithm='lars'` and + `algorithm='omp'`. If `None`, then + `transform_n_nonzero_coefs=int(n_features / 10)`. + + transform_alpha : float, default=None + If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the + penalty applied to the L1 norm. + If `algorithm='threshold'`, `alpha` is the absolute value of the + threshold below which coefficients will be squashed to zero. + If `None`, defaults to `alpha`. + + .. versionchanged:: 1.2 + When None, default value changed from 1.0 to `alpha`. + + verbose : bool or int, default=False + To control the verbosity of the procedure. + + split_sign : bool, default=False + Whether to split the sparse feature vector into the concatenation of + its negative part and its positive part. This can improve the + performance of downstream classifiers. + + random_state : int, RandomState instance or None, default=None + Used for initializing the dictionary when ``dict_init`` is not + specified, randomly shuffling the data when ``shuffle`` is set to + ``True``, and updating the dictionary. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + positive_code : bool, default=False + Whether to enforce positivity when finding the code. + + .. versionadded:: 0.20 + + positive_dict : bool, default=False + Whether to enforce positivity when finding the dictionary. + + .. versionadded:: 0.20 + + transform_max_iter : int, default=1000 + Maximum number of iterations to perform if `algorithm='lasso_cd'` or + `'lasso_lars'`. + + .. versionadded:: 0.22 + + callback : callable, default=None + A callable that gets invoked at the end of each iteration. + + .. versionadded:: 1.1 + + tol : float, default=1e-3 + Control early stopping based on the norm of the differences in the + dictionary between 2 steps. + + To disable early stopping based on changes in the dictionary, set + `tol` to 0.0. + + .. versionadded:: 1.1 + + max_no_improvement : int, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + + To disable convergence detection based on cost function, set + `max_no_improvement` to None. + + .. versionadded:: 1.1 + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Components extracted from the data. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_iter_ : int + Number of iterations over the full dataset. + + n_steps_ : int + Number of mini-batches processed. + + .. versionadded:: 1.1 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparseCoder : Find a sparse representation of data from a fixed, + precomputed dictionary. + SparsePCA : Sparse Principal Components Analysis. + + References + ---------- + + J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning + for sparse coding (https://www.di.ens.fr/~fbach/mairal_icml09.pdf) + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_sparse_coded_signal + >>> from sklearn.decomposition import MiniBatchDictionaryLearning + >>> X, dictionary, code = make_sparse_coded_signal( + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, + ... random_state=42) + >>> dict_learner = MiniBatchDictionaryLearning( + ... n_components=15, batch_size=3, transform_algorithm='lasso_lars', + ... transform_alpha=0.1, max_iter=20, random_state=42) + >>> X_transformed = dict_learner.fit_transform(X) + + We can check the level of sparsity of `X_transformed`: + + >>> np.mean(X_transformed == 0) > 0.5 + np.True_ + + We can compare the average squared euclidean norm of the reconstruction + error of the sparse coded signal relative to the squared euclidean norm of + the original signal: + + >>> X_hat = X_transformed @ dict_learner.components_ + >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) + np.float64(0.052) + + For a more detailed example, see + :ref:`sphx_glr_auto_examples_decomposition_plot_image_denoising.py` + """ + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "fit_algorithm": [StrOptions({"cd", "lars"})], + "n_jobs": [None, Integral], + "batch_size": [Interval(Integral, 1, None, closed="left")], + "shuffle": ["boolean"], + "dict_init": [None, np.ndarray], + "transform_algorithm": [ + StrOptions({"lasso_lars", "lasso_cd", "lars", "omp", "threshold"}) + ], + "transform_n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None], + "transform_alpha": [Interval(Real, 0, None, closed="left"), None], + "verbose": ["verbose"], + "split_sign": ["boolean"], + "random_state": ["random_state"], + "positive_code": ["boolean"], + "positive_dict": ["boolean"], + "transform_max_iter": [Interval(Integral, 0, None, closed="left")], + "callback": [None, callable], + "tol": [Interval(Real, 0, None, closed="left")], + "max_no_improvement": [Interval(Integral, 0, None, closed="left"), None], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + max_iter=1_000, + fit_algorithm="lars", + n_jobs=None, + batch_size=256, + shuffle=True, + dict_init=None, + transform_algorithm="omp", + transform_n_nonzero_coefs=None, + transform_alpha=None, + verbose=False, + split_sign=False, + random_state=None, + positive_code=False, + positive_dict=False, + transform_max_iter=1000, + callback=None, + tol=1e-3, + max_no_improvement=10, + ): + super().__init__( + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, + ) + self.n_components = n_components + self.alpha = alpha + self.max_iter = max_iter + self.fit_algorithm = fit_algorithm + self.dict_init = dict_init + self.verbose = verbose + self.shuffle = shuffle + self.batch_size = batch_size + self.split_sign = split_sign + self.random_state = random_state + self.positive_dict = positive_dict + self.callback = callback + self.max_no_improvement = max_no_improvement + self.tol = tol + + def _check_params(self, X): + # n_components + self._n_components = self.n_components + if self._n_components is None: + self._n_components = X.shape[1] + + # fit_algorithm + _check_positive_coding(self.fit_algorithm, self.positive_code) + self._fit_algorithm = "lasso_" + self.fit_algorithm + + # batch_size + self._batch_size = min(self.batch_size, X.shape[0]) + + def _initialize_dict(self, X, random_state): + """Initialization of the dictionary.""" + if self.dict_init is not None: + dictionary = self.dict_init + else: + # Init V with SVD of X + _, S, dictionary = _randomized_svd( + X, self._n_components, random_state=random_state + ) + dictionary = S[:, np.newaxis] * dictionary + + if self._n_components <= len(dictionary): + dictionary = dictionary[: self._n_components, :] + else: + dictionary = np.concatenate( + ( + dictionary, + np.zeros( + (self._n_components - len(dictionary), dictionary.shape[1]), + dtype=dictionary.dtype, + ), + ) + ) + + dictionary = check_array(dictionary, order="F", dtype=X.dtype, copy=False) + dictionary = np.require(dictionary, requirements="W") + + return dictionary + + def _update_inner_stats(self, X, code, batch_size, step): + """Update the inner stats inplace.""" + if step < batch_size - 1: + theta = (step + 1) * batch_size + else: + theta = batch_size**2 + step + 1 - batch_size + beta = (theta + 1 - batch_size) / (theta + 1) + + self._A *= beta + self._A += code.T @ code / batch_size + self._B *= beta + self._B += X.T @ code / batch_size + + def _minibatch_step(self, X, dictionary, random_state, step): + """Perform the update on the dictionary for one minibatch.""" + batch_size = X.shape[0] + + # Compute code for this batch + code = _sparse_encode( + X, + dictionary, + algorithm=self._fit_algorithm, + alpha=self.alpha, + n_jobs=self.n_jobs, + positive=self.positive_code, + max_iter=self.transform_max_iter, + verbose=self.verbose, + ) + + batch_cost = ( + 0.5 * ((X - code @ dictionary) ** 2).sum() + + self.alpha * np.sum(np.abs(code)) + ) / batch_size + + # Update inner stats + self._update_inner_stats(X, code, batch_size, step) + + # Update dictionary + _update_dict( + dictionary, + X, + code, + self._A, + self._B, + verbose=self.verbose, + random_state=random_state, + positive=self.positive_dict, + ) + + return batch_cost + + def _check_convergence( + self, X, batch_cost, new_dict, old_dict, n_samples, step, n_steps + ): + """Helper function to encapsulate the early stopping logic. + + Early stopping is based on two factors: + - A small change of the dictionary between two minibatch updates. This is + controlled by the tol parameter. + - No more improvement on a smoothed estimate of the objective function for a + a certain number of consecutive minibatch updates. This is controlled by + the max_no_improvement parameter. + """ + batch_size = X.shape[0] + + # counts steps starting from 1 for user friendly verbose mode. + step = step + 1 + + # Ignore 100 first steps or 1 epoch to avoid initializing the ewa_cost with a + # too bad value + if step <= min(100, n_samples / batch_size): + if self.verbose: + print(f"Minibatch step {step}/{n_steps}: mean batch cost: {batch_cost}") + return False + + # Compute an Exponentially Weighted Average of the cost function to + # monitor the convergence while discarding minibatch-local stochastic + # variability: https://en.wikipedia.org/wiki/Moving_average + if self._ewa_cost is None: + self._ewa_cost = batch_cost + else: + alpha = batch_size / (n_samples + 1) + alpha = min(alpha, 1) + self._ewa_cost = self._ewa_cost * (1 - alpha) + batch_cost * alpha + + if self.verbose: + print( + f"Minibatch step {step}/{n_steps}: mean batch cost: " + f"{batch_cost}, ewa cost: {self._ewa_cost}" + ) + + # Early stopping based on change of dictionary + dict_diff = linalg.norm(new_dict - old_dict) / self._n_components + if self.tol > 0 and dict_diff <= self.tol: + if self.verbose: + print(f"Converged (small dictionary change) at step {step}/{n_steps}") + return True + + # Early stopping heuristic due to lack of improvement on smoothed + # cost function + if self._ewa_cost_min is None or self._ewa_cost < self._ewa_cost_min: + self._no_improvement = 0 + self._ewa_cost_min = self._ewa_cost + else: + self._no_improvement += 1 + + if ( + self.max_no_improvement is not None + and self._no_improvement >= self.max_no_improvement + ): + if self.verbose: + print( + "Converged (lack of improvement in objective function) " + f"at step {step}/{n_steps}" + ) + return True + + return False + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the model from data in X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + X = validate_data( + self, X, dtype=[np.float64, np.float32], order="C", copy=False + ) + + self._check_params(X) + self._random_state = check_random_state(self.random_state) + + dictionary = self._initialize_dict(X, self._random_state) + old_dict = dictionary.copy() + + if self.shuffle: + X_train = X.copy() + self._random_state.shuffle(X_train) + else: + X_train = X + + n_samples, n_features = X_train.shape + + if self.verbose: + print("[dict_learning]") + + # Inner stats + self._A = np.zeros( + (self._n_components, self._n_components), dtype=X_train.dtype + ) + self._B = np.zeros((n_features, self._n_components), dtype=X_train.dtype) + + # Attributes to monitor the convergence + self._ewa_cost = None + self._ewa_cost_min = None + self._no_improvement = 0 + + batches = gen_batches(n_samples, self._batch_size) + batches = itertools.cycle(batches) + n_steps_per_iter = int(np.ceil(n_samples / self._batch_size)) + n_steps = self.max_iter * n_steps_per_iter + + i = -1 # to allow max_iter = 0 + + for i, batch in zip(range(n_steps), batches): + X_batch = X_train[batch] + + batch_cost = self._minibatch_step( + X_batch, dictionary, self._random_state, i + ) + + if self._check_convergence( + X_batch, batch_cost, dictionary, old_dict, n_samples, i, n_steps + ): + break + + # XXX callback param added for backward compat in #18975 but a common + # unified callback API should be preferred + if self.callback is not None: + self.callback(locals()) + + old_dict[:] = dictionary + + self.n_steps_ = i + 1 + self.n_iter_ = np.ceil(self.n_steps_ / n_steps_per_iter) + self.components_ = dictionary + + return self + + @_fit_context(prefer_skip_nested_validation=True) + def partial_fit(self, X, y=None): + """Update the model using the data in X as a mini-batch. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Return the instance itself. + """ + has_components = hasattr(self, "components_") + + X = validate_data( + self, X, dtype=[np.float64, np.float32], order="C", reset=not has_components + ) + + if not has_components: + # This instance has not been fitted yet (fit or partial_fit) + self._check_params(X) + self._random_state = check_random_state(self.random_state) + + dictionary = self._initialize_dict(X, self._random_state) + + self.n_steps_ = 0 + + self._A = np.zeros((self._n_components, self._n_components), dtype=X.dtype) + self._B = np.zeros((X.shape[1], self._n_components), dtype=X.dtype) + else: + dictionary = self.components_ + + self._minibatch_step(X, dictionary, self._random_state, self.n_steps_) + + self.components_ = dictionary + self.n_steps_ += 1 + + return self + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_factor_analysis.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_factor_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..91f399257537af987bac2f0a00f85fcac018ecee --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_factor_analysis.py @@ -0,0 +1,457 @@ +"""Factor Analysis. + +A latent linear variable model. + +FactorAnalysis is similar to probabilistic PCA implemented by PCA.score +While PCA assumes Gaussian noise with the same variance for each +feature, the FactorAnalysis model assumes different variances for +each of them. + +This implementation is based on David Barber's Book, +Bayesian Reasoning and Machine Learning, +http://www.cs.ucl.ac.uk/staff/d.barber/brml, +Algorithm 21.1 +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings +from math import log, sqrt +from numbers import Integral, Real + +import numpy as np +from scipy import linalg + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils import check_random_state +from sklearn.utils._param_validation import Interval, StrOptions +from sklearn.utils.extmath import _randomized_svd, fast_logdet, squared_norm +from sklearn.utils.validation import check_is_fitted, validate_data + + +class FactorAnalysis(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): + """Factor Analysis (FA). + + A simple linear generative model with Gaussian latent variables. + + The observations are assumed to be caused by a linear transformation of + lower dimensional latent factors and added Gaussian noise. + Without loss of generality the factors are distributed according to a + Gaussian with zero mean and unit covariance. The noise is also zero mean + and has an arbitrary diagonal covariance matrix. + + If we would restrict the model further, by assuming that the Gaussian + noise is even isotropic (all diagonal entries are the same) we would obtain + :class:`PCA`. + + FactorAnalysis performs a maximum likelihood estimate of the so-called + `loading` matrix, the transformation of the latent variables to the + observed ones, using SVD based approach. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.13 + + Parameters + ---------- + n_components : int, default=None + Dimensionality of latent space, the number of components + of ``X`` that are obtained after ``transform``. + If None, n_components is set to the number of features. + + tol : float, default=1e-2 + Stopping tolerance for log-likelihood increase. + + copy : bool, default=True + Whether to make a copy of X. If ``False``, the input X gets overwritten + during fitting. + + max_iter : int, default=1000 + Maximum number of iterations. + + noise_variance_init : array-like of shape (n_features,), default=None + The initial guess of the noise variance for each feature. + If None, it defaults to np.ones(n_features). + + svd_method : {'lapack', 'randomized'}, default='randomized' + Which SVD method to use. If 'lapack' use standard SVD from + scipy.linalg, if 'randomized' use fast ``randomized_svd`` function. + Defaults to 'randomized'. For most applications 'randomized' will + be sufficiently precise while providing significant speed gains. + Accuracy can also be improved by setting higher values for + `iterated_power`. If this is not sufficient, for maximum precision + you should choose 'lapack'. + + iterated_power : int, default=3 + Number of iterations for the power method. 3 by default. Only used + if ``svd_method`` equals 'randomized'. + + rotation : {'varimax', 'quartimax'}, default=None + If not None, apply the indicated rotation. Currently, varimax and + quartimax are implemented. See + `"The varimax criterion for analytic rotation in factor analysis" + `_ + H. F. Kaiser, 1958. + + .. versionadded:: 0.24 + + random_state : int or RandomState instance, default=0 + Only used when ``svd_method`` equals 'randomized'. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Components with maximum variance. + + loglike_ : list of shape (n_iterations,) + The log likelihood at each iteration. + + noise_variance_ : ndarray of shape (n_features,) + The estimated noise variance for each feature. + + n_iter_ : int + Number of iterations run. + + mean_ : ndarray of shape (n_features,) + Per-feature empirical mean, estimated from the training set. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + PCA: Principal component analysis is also a latent linear variable model + which however assumes equal noise variance for each feature. + This extra assumption makes probabilistic PCA faster as it can be + computed in closed form. + FastICA: Independent component analysis, a latent variable model with + non-Gaussian latent variables. + + References + ---------- + - David Barber, Bayesian Reasoning and Machine Learning, + Algorithm 21.1. + + - Christopher M. Bishop: Pattern Recognition and Machine Learning, + Chapter 12.2.4. + + Examples + -------- + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import FactorAnalysis + >>> X, _ = load_digits(return_X_y=True) + >>> transformer = FactorAnalysis(n_components=7, random_state=0) + >>> X_transformed = transformer.fit_transform(X) + >>> X_transformed.shape + (1797, 7) + """ + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 0, None, closed="left"), None], + "tol": [Interval(Real, 0.0, None, closed="left")], + "copy": ["boolean"], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "noise_variance_init": ["array-like", None], + "svd_method": [StrOptions({"randomized", "lapack"})], + "iterated_power": [Interval(Integral, 0, None, closed="left")], + "rotation": [StrOptions({"varimax", "quartimax"}), None], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + tol=1e-2, + copy=True, + max_iter=1000, + noise_variance_init=None, + svd_method="randomized", + iterated_power=3, + rotation=None, + random_state=0, + ): + self.n_components = n_components + self.copy = copy + self.tol = tol + self.max_iter = max_iter + self.svd_method = svd_method + + self.noise_variance_init = noise_variance_init + self.iterated_power = iterated_power + self.random_state = random_state + self.rotation = rotation + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the FactorAnalysis model to X using SVD based approach. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + y : Ignored + Ignored parameter. + + Returns + ------- + self : object + FactorAnalysis class instance. + """ + X = validate_data( + self, X, copy=self.copy, dtype=np.float64, force_writeable=True + ) + + n_samples, n_features = X.shape + n_components = self.n_components + if n_components is None: + n_components = n_features + + self.mean_ = np.mean(X, axis=0) + X -= self.mean_ + + # some constant terms + nsqrt = sqrt(n_samples) + llconst = n_features * log(2.0 * np.pi) + n_components + var = np.var(X, axis=0) + + if self.noise_variance_init is None: + psi = np.ones(n_features, dtype=X.dtype) + else: + if len(self.noise_variance_init) != n_features: + raise ValueError( + "noise_variance_init dimension does not " + "with number of features : %d != %d" + % (len(self.noise_variance_init), n_features) + ) + psi = np.array(self.noise_variance_init) + + loglike = [] + old_ll = -np.inf + SMALL = 1e-12 + + # we'll modify svd outputs to return unexplained variance + # to allow for unified computation of loglikelihood + if self.svd_method == "lapack": + + def my_svd(X): + _, s, Vt = linalg.svd(X, full_matrices=False, check_finite=False) + return ( + s[:n_components], + Vt[:n_components], + squared_norm(s[n_components:]), + ) + + else: # svd_method == "randomized" + random_state = check_random_state(self.random_state) + + def my_svd(X): + _, s, Vt = _randomized_svd( + X, + n_components, + random_state=random_state, + n_iter=self.iterated_power, + ) + return s, Vt, squared_norm(X) - squared_norm(s) + + for i in range(self.max_iter): + # SMALL helps numerics + sqrt_psi = np.sqrt(psi) + SMALL + s, Vt, unexp_var = my_svd(X / (sqrt_psi * nsqrt)) + s **= 2 + # Use 'maximum' here to avoid sqrt problems. + W = np.sqrt(np.maximum(s - 1.0, 0.0))[:, np.newaxis] * Vt + del Vt + W *= sqrt_psi + + # loglikelihood + ll = llconst + np.sum(np.log(s)) + ll += unexp_var + np.sum(np.log(psi)) + ll *= -n_samples / 2.0 + loglike.append(ll) + if (ll - old_ll) < self.tol: + break + old_ll = ll + + psi = np.maximum(var - np.sum(W**2, axis=0), SMALL) + else: + warnings.warn( + "FactorAnalysis did not converge." + " You might want" + " to increase the number of iterations.", + ConvergenceWarning, + ) + + self.components_ = W + if self.rotation is not None: + self.components_ = self._rotate(W) + self.noise_variance_ = psi + self.loglike_ = loglike + self.n_iter_ = i + 1 + return self + + def transform(self, X): + """Apply dimensionality reduction to X using the model. + + Compute the expected mean of the latent variables. + See Barber, 21.2.33 (or Bishop, 12.66). + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + The latent variables of X. + """ + check_is_fitted(self) + + X = validate_data(self, X, reset=False) + Ih = np.eye(len(self.components_)) + + X_transformed = X - self.mean_ + + Wpsi = self.components_ / self.noise_variance_ + cov_z = linalg.inv(Ih + np.dot(Wpsi, self.components_.T)) + tmp = np.dot(X_transformed, Wpsi.T) + X_transformed = np.dot(tmp, cov_z) + + return X_transformed + + def get_covariance(self): + """Compute data covariance with the FactorAnalysis model. + + ``cov = components_.T * components_ + diag(noise_variance)`` + + Returns + ------- + cov : ndarray of shape (n_features, n_features) + Estimated covariance of data. + """ + check_is_fitted(self) + + cov = np.dot(self.components_.T, self.components_) + cov.flat[:: len(cov) + 1] += self.noise_variance_ # modify diag inplace + return cov + + def get_precision(self): + """Compute data precision matrix with the FactorAnalysis model. + + Returns + ------- + precision : ndarray of shape (n_features, n_features) + Estimated precision of data. + """ + check_is_fitted(self) + + n_features = self.components_.shape[1] + + # handle corner cases first + if self.n_components == 0: + return np.diag(1.0 / self.noise_variance_) + if self.n_components == n_features: + return linalg.inv(self.get_covariance()) + + # Get precision using matrix inversion lemma + components_ = self.components_ + precision = np.dot(components_ / self.noise_variance_, components_.T) + precision.flat[:: len(precision) + 1] += 1.0 + precision = np.dot(components_.T, np.dot(linalg.inv(precision), components_)) + precision /= self.noise_variance_[:, np.newaxis] + precision /= -self.noise_variance_[np.newaxis, :] + precision.flat[:: len(precision) + 1] += 1.0 / self.noise_variance_ + return precision + + def score_samples(self, X): + """Compute the log-likelihood of each sample. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + The data. + + Returns + ------- + ll : ndarray of shape (n_samples,) + Log-likelihood of each sample under the current model. + """ + check_is_fitted(self) + X = validate_data(self, X, reset=False) + Xr = X - self.mean_ + precision = self.get_precision() + n_features = X.shape[1] + log_like = -0.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1) + log_like -= 0.5 * (n_features * log(2.0 * np.pi) - fast_logdet(precision)) + return log_like + + def score(self, X, y=None): + """Compute the average log-likelihood of the samples. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + The data. + + y : Ignored + Ignored parameter. + + Returns + ------- + ll : float + Average log-likelihood of the samples under the current model. + """ + return np.mean(self.score_samples(X)) + + def _rotate(self, components, n_components=None, tol=1e-6): + "Rotate the factor analysis solution." + # note that tol is not exposed + return _ortho_rotation(components.T, method=self.rotation, tol=tol)[ + : self.n_components + ] + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + +def _ortho_rotation(components, method="varimax", tol=1e-6, max_iter=100): + """Return rotated components.""" + nrow, ncol = components.shape + rotation_matrix = np.eye(ncol) + var = 0 + + for _ in range(max_iter): + comp_rot = np.dot(components, rotation_matrix) + if method == "varimax": + tmp = comp_rot * np.transpose((comp_rot**2).sum(axis=0) / nrow) + elif method == "quartimax": + tmp = 0 + u, s, v = np.linalg.svd(np.dot(components.T, comp_rot**3 - tmp)) + rotation_matrix = np.dot(u, v) + var_new = np.sum(s) + if var != 0 and var_new < var * (1 + tol): + break + var = var_new + + return np.dot(components, rotation_matrix).T diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_fastica.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_fastica.py new file mode 100644 index 0000000000000000000000000000000000000000..d464562b44747bdf37003bd194e7ba130525eedb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_fastica.py @@ -0,0 +1,809 @@ +""" +Python implementation of the fast ICA algorithms. + +Reference: Tables 8.3 and 8.4 page 196 in the book: +Independent Component Analysis, by Hyvarinen et al. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings +from numbers import Integral, Real + +import numpy as np +from scipy import linalg + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils import as_float_array, check_array, check_random_state +from sklearn.utils._param_validation import ( + Interval, + Options, + StrOptions, + validate_params, +) +from sklearn.utils.validation import check_is_fitted, validate_data + +__all__ = ["FastICA", "fastica"] + + +def _gs_decorrelation(w, W, j): + """ + Orthonormalize w wrt the first j rows of W. + + Parameters + ---------- + w : ndarray of shape (n,) + Array to be orthogonalized + + W : ndarray of shape (p, n) + Null space definition + + j : int < p + The no of (from the first) rows of Null space W wrt which w is + orthogonalized. + + Notes + ----- + Assumes that W is orthogonal + w changed in place + """ + w -= np.linalg.multi_dot([w, W[:j].T, W[:j]]) + return w + + +def _sym_decorrelation(W): + """Symmetric decorrelation + i.e. W <- (W * W.T) ^{-1/2} * W + """ + s, u = linalg.eigh(np.dot(W, W.T)) + # Avoid sqrt of negative values because of rounding errors. Note that + # np.sqrt(tiny) is larger than tiny and therefore this clipping also + # prevents division by zero in the next step. + s = np.clip(s, a_min=np.finfo(W.dtype).tiny, a_max=None) + + # u (resp. s) contains the eigenvectors (resp. square roots of + # the eigenvalues) of W * W.T + return np.linalg.multi_dot([u * (1.0 / np.sqrt(s)), u.T, W]) + + +def _ica_def(X, tol, g, fun_args, max_iter, w_init): + """Deflationary FastICA using fun approx to neg-entropy function + + Used internally by FastICA. + """ + + n_components = w_init.shape[0] + W = np.zeros((n_components, n_components), dtype=X.dtype) + n_iter = [] + + # j is the index of the extracted component + for j in range(n_components): + w = w_init[j, :].copy() + w /= np.sqrt((w**2).sum()) + + for i in range(max_iter): + gwtx, g_wtx = g(np.dot(w.T, X), fun_args) + + w1 = (X * gwtx).mean(axis=1) - g_wtx.mean() * w + + _gs_decorrelation(w1, W, j) + + w1 /= np.sqrt((w1**2).sum()) + + lim = np.abs(np.abs((w1 * w).sum()) - 1) + w = w1 + if lim < tol: + break + + n_iter.append(i + 1) + W[j, :] = w + + return W, max(n_iter) + + +def _ica_par(X, tol, g, fun_args, max_iter, w_init): + """Parallel FastICA. + + Used internally by FastICA --main loop + + """ + W = _sym_decorrelation(w_init) + del w_init + p_ = float(X.shape[1]) + for ii in range(max_iter): + gwtx, g_wtx = g(np.dot(W, X), fun_args) + W1 = _sym_decorrelation(np.dot(gwtx, X.T) / p_ - g_wtx[:, np.newaxis] * W) + del gwtx, g_wtx + # builtin max, abs are faster than numpy counter parts. + # np.einsum allows having the lowest memory footprint. + # It is faster than np.diag(np.dot(W1, W.T)). + lim = max(abs(abs(np.einsum("ij,ij->i", W1, W)) - 1)) + W = W1 + if lim < tol: + break + else: + warnings.warn( + ( + "FastICA did not converge. Consider increasing " + "tolerance or the maximum number of iterations." + ), + ConvergenceWarning, + ) + + return W, ii + 1 + + +# Some standard non-linear functions. +# XXX: these should be optimized, as they can be a bottleneck. +def _logcosh(x, fun_args=None): + alpha = fun_args.get("alpha", 1.0) # comment it out? + + x *= alpha + gx = np.tanh(x, x) # apply the tanh inplace + g_x = np.empty(x.shape[0], dtype=x.dtype) + # XXX compute in chunks to avoid extra allocation + for i, gx_i in enumerate(gx): # please don't vectorize. + g_x[i] = (alpha * (1 - gx_i**2)).mean() + return gx, g_x + + +def _exp(x, fun_args): + exp = np.exp(-(x**2) / 2) + gx = x * exp + g_x = (1 - x**2) * exp + return gx, g_x.mean(axis=-1) + + +def _cube(x, fun_args): + return x**3, (3 * x**2).mean(axis=-1) + + +@validate_params( + { + "X": ["array-like"], + "return_X_mean": ["boolean"], + "compute_sources": ["boolean"], + "return_n_iter": ["boolean"], + }, + prefer_skip_nested_validation=False, +) +def fastica( + X, + n_components=None, + *, + algorithm="parallel", + whiten="unit-variance", + fun="logcosh", + fun_args=None, + max_iter=200, + tol=1e-04, + w_init=None, + whiten_solver="svd", + random_state=None, + return_X_mean=False, + compute_sources=True, + return_n_iter=False, +): + """Perform Fast Independent Component Analysis. + + The implementation is based on [1]_. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples and + `n_features` is the number of features. + + n_components : int, default=None + Number of components to use. If None is passed, all are used. + + algorithm : {'parallel', 'deflation'}, default='parallel' + Specify which algorithm to use for FastICA. + + whiten : str or bool, default='unit-variance' + Specify the whitening strategy to use. + + - If 'arbitrary-variance', a whitening with variance + arbitrary is used. + - If 'unit-variance', the whitening matrix is rescaled to ensure that + each recovered source has unit variance. + - If False, the data is already considered to be whitened, and no + whitening is performed. + + .. versionchanged:: 1.3 + The default value of `whiten` changed to 'unit-variance' in 1.3. + + fun : {'logcosh', 'exp', 'cube'} or callable, default='logcosh' + The functional form of the G function used in the + approximation to neg-entropy. Could be either 'logcosh', 'exp', + or 'cube'. + You can also provide your own function. It should return a tuple + containing the value of the function, and of its derivative, in the + point. The derivative should be averaged along its last dimension. + Example:: + + def my_g(x): + return x ** 3, (3 * x ** 2).mean(axis=-1) + + fun_args : dict, default=None + Arguments to send to the functional form. + If empty or None and if fun='logcosh', fun_args will take value + {'alpha' : 1.0}. + + max_iter : int, default=200 + Maximum number of iterations to perform. + + tol : float, default=1e-4 + A positive scalar giving the tolerance at which the + un-mixing matrix is considered to have converged. + + w_init : ndarray of shape (n_components, n_components), default=None + Initial un-mixing array. If `w_init=None`, then an array of values + drawn from a normal distribution is used. + + whiten_solver : {"eigh", "svd"}, default="svd" + The solver to use for whitening. + + - "svd" is more stable numerically if the problem is degenerate, and + often faster when `n_samples <= n_features`. + + - "eigh" is generally more memory efficient when + `n_samples >= n_features`, and can be faster when + `n_samples >= 50 * n_features`. + + .. versionadded:: 1.2 + + random_state : int, RandomState instance or None, default=None + Used to initialize ``w_init`` when not specified, with a + normal distribution. Pass an int, for reproducible results + across multiple function calls. + See :term:`Glossary `. + + return_X_mean : bool, default=False + If True, X_mean is returned too. + + compute_sources : bool, default=True + If False, sources are not computed, but only the rotation matrix. + This can save memory when working with big data. Defaults to True. + + return_n_iter : bool, default=False + Whether or not to return the number of iterations. + + Returns + ------- + K : ndarray of shape (n_components, n_features) or None + If whiten is 'True', K is the pre-whitening matrix that projects data + onto the first n_components principal components. If whiten is 'False', + K is 'None'. + + W : ndarray of shape (n_components, n_components) + The square matrix that unmixes the data after whitening. + The mixing matrix is the pseudo-inverse of matrix ``W K`` + if K is not None, else it is the inverse of W. + + S : ndarray of shape (n_samples, n_components) or None + Estimated source matrix. + + X_mean : ndarray of shape (n_features,) + The mean over features. Returned only if return_X_mean is True. + + n_iter : int + If the algorithm is "deflation", n_iter is the + maximum number of iterations run across all components. Else + they are just the number of iterations taken to converge. This is + returned only when return_n_iter is set to `True`. + + Notes + ----- + The data matrix X is considered to be a linear combination of + non-Gaussian (independent) components i.e. X = AS where columns of S + contain the independent components and A is a linear mixing + matrix. In short ICA attempts to `un-mix' the data by estimating an + un-mixing matrix W where ``S = W K X.`` + While FastICA was proposed to estimate as many sources + as features, it is possible to estimate less by setting + n_components < n_features. It this case K is not a square matrix + and the estimated A is the pseudo-inverse of ``W K``. + + This implementation was originally made for data of shape + [n_features, n_samples]. Now the input is transposed + before the algorithm is applied. This makes it slightly + faster for Fortran-ordered input. + + References + ---------- + .. [1] A. Hyvarinen and E. Oja, "Fast Independent Component Analysis", + Algorithms and Applications, Neural Networks, 13(4-5), 2000, + pp. 411-430. + + Examples + -------- + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import fastica + >>> X, _ = load_digits(return_X_y=True) + >>> K, W, S = fastica(X, n_components=7, random_state=0, whiten='unit-variance') + >>> K.shape + (7, 64) + >>> W.shape + (7, 7) + >>> S.shape + (1797, 7) + """ + est = FastICA( + n_components=n_components, + algorithm=algorithm, + whiten=whiten, + fun=fun, + fun_args=fun_args, + max_iter=max_iter, + tol=tol, + w_init=w_init, + whiten_solver=whiten_solver, + random_state=random_state, + ) + est._validate_params() + S = est._fit_transform(X, compute_sources=compute_sources) + + if est.whiten in ["unit-variance", "arbitrary-variance"]: + K = est.whitening_ + X_mean = est.mean_ + else: + K = None + X_mean = None + + returned_values = [K, est._unmixing, S] + if return_X_mean: + returned_values.append(X_mean) + if return_n_iter: + returned_values.append(est.n_iter_) + + return returned_values + + +class FastICA(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): + """FastICA: a fast algorithm for Independent Component Analysis. + + The implementation is based on [1]_. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=None + Number of components to use. If None is passed, all are used. + + algorithm : {'parallel', 'deflation'}, default='parallel' + Specify which algorithm to use for FastICA. + + whiten : str or bool, default='unit-variance' + Specify the whitening strategy to use. + + - If 'arbitrary-variance', a whitening with variance + arbitrary is used. + - If 'unit-variance', the whitening matrix is rescaled to ensure that + each recovered source has unit variance. + - If False, the data is already considered to be whitened, and no + whitening is performed. + + .. versionchanged:: 1.3 + The default value of `whiten` changed to 'unit-variance' in 1.3. + + fun : {'logcosh', 'exp', 'cube'} or callable, default='logcosh' + The functional form of the G function used in the + approximation to neg-entropy. Could be either 'logcosh', 'exp', + or 'cube'. + You can also provide your own function. It should return a tuple + containing the value of the function, and of its derivative, in the + point. The derivative should be averaged along its last dimension. + Example:: + + def my_g(x): + return x ** 3, (3 * x ** 2).mean(axis=-1) + + fun_args : dict, default=None + Arguments to send to the functional form. + If empty or None and if fun='logcosh', fun_args will take value + {'alpha' : 1.0}. + + max_iter : int, default=200 + Maximum number of iterations during fit. + + tol : float, default=1e-4 + A positive scalar giving the tolerance at which the + un-mixing matrix is considered to have converged. + + w_init : array-like of shape (n_components, n_components), default=None + Initial un-mixing array. If `w_init=None`, then an array of values + drawn from a normal distribution is used. + + whiten_solver : {"eigh", "svd"}, default="svd" + The solver to use for whitening. + + - "svd" is more stable numerically if the problem is degenerate, and + often faster when `n_samples <= n_features`. + + - "eigh" is generally more memory efficient when + `n_samples >= n_features`, and can be faster when + `n_samples >= 50 * n_features`. + + .. versionadded:: 1.2 + + random_state : int, RandomState instance or None, default=None + Used to initialize ``w_init`` when not specified, with a + normal distribution. Pass an int, for reproducible results + across multiple function calls. + See :term:`Glossary `. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + The linear operator to apply to the data to get the independent + sources. This is equal to the unmixing matrix when ``whiten`` is + False, and equal to ``np.dot(unmixing_matrix, self.whitening_)`` when + ``whiten`` is True. + + mixing_ : ndarray of shape (n_features, n_components) + The pseudo-inverse of ``components_``. It is the linear operator + that maps independent sources to the data. + + mean_ : ndarray of shape(n_features,) + The mean over features. Only set if `self.whiten` is True. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_iter_ : int + If the algorithm is "deflation", n_iter is the + maximum number of iterations run across all components. Else + they are just the number of iterations taken to converge. + + whitening_ : ndarray of shape (n_components, n_features) + Only set if whiten is 'True'. This is the pre-whitening matrix + that projects data onto the first `n_components` principal components. + + See Also + -------- + PCA : Principal component analysis (PCA). + IncrementalPCA : Incremental principal components analysis (IPCA). + KernelPCA : Kernel Principal component analysis (KPCA). + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparsePCA : Sparse Principal Components Analysis (SparsePCA). + + References + ---------- + .. [1] A. Hyvarinen and E. Oja, Independent Component Analysis: + Algorithms and Applications, Neural Networks, 13(4-5), 2000, + pp. 411-430. + + Examples + -------- + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import FastICA + >>> X, _ = load_digits(return_X_y=True) + >>> transformer = FastICA(n_components=7, + ... random_state=0, + ... whiten='unit-variance') + >>> X_transformed = transformer.fit_transform(X) + >>> X_transformed.shape + (1797, 7) + """ + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "algorithm": [StrOptions({"parallel", "deflation"})], + "whiten": [ + StrOptions({"arbitrary-variance", "unit-variance"}), + Options(bool, {False}), + ], + "fun": [StrOptions({"logcosh", "exp", "cube"}), callable], + "fun_args": [dict, None], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0.0, None, closed="left")], + "w_init": ["array-like", None], + "whiten_solver": [StrOptions({"eigh", "svd"})], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + algorithm="parallel", + whiten="unit-variance", + fun="logcosh", + fun_args=None, + max_iter=200, + tol=1e-4, + w_init=None, + whiten_solver="svd", + random_state=None, + ): + super().__init__() + self.n_components = n_components + self.algorithm = algorithm + self.whiten = whiten + self.fun = fun + self.fun_args = fun_args + self.max_iter = max_iter + self.tol = tol + self.w_init = w_init + self.whiten_solver = whiten_solver + self.random_state = random_state + + def _fit_transform(self, X, compute_sources=False): + """Fit the model. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + compute_sources : bool, default=False + If False, sources are not computes but only the rotation matrix. + This can save memory when working with big data. Defaults to False. + + Returns + ------- + S : ndarray of shape (n_samples, n_components) or None + Sources matrix. `None` if `compute_sources` is `False`. + """ + XT = validate_data( + self, + X, + copy=self.whiten, + dtype=[np.float64, np.float32], + ensure_min_samples=2, + ).T + fun_args = {} if self.fun_args is None else self.fun_args + random_state = check_random_state(self.random_state) + + alpha = fun_args.get("alpha", 1.0) + if not 1 <= alpha <= 2: + raise ValueError("alpha must be in [1,2]") + + if self.fun == "logcosh": + g = _logcosh + elif self.fun == "exp": + g = _exp + elif self.fun == "cube": + g = _cube + elif callable(self.fun): + + def g(x, fun_args): + return self.fun(x, **fun_args) + + n_features, n_samples = XT.shape + n_components = self.n_components + if not self.whiten and n_components is not None: + n_components = None + warnings.warn("Ignoring n_components with whiten=False.") + + if n_components is None: + n_components = min(n_samples, n_features) + if n_components > min(n_samples, n_features): + n_components = min(n_samples, n_features) + warnings.warn( + "n_components is too large: it will be set to %s" % n_components + ) + + if self.whiten: + # Centering the features of X + X_mean = XT.mean(axis=-1) + XT -= X_mean[:, np.newaxis] + + # Whitening and preprocessing by PCA + if self.whiten_solver == "eigh": + # Faster when num_samples >> n_features + d, u = linalg.eigh(XT.dot(X)) + sort_indices = np.argsort(d)[::-1] + eps = np.finfo(d.dtype).eps * 10 + degenerate_idx = d < eps + if np.any(degenerate_idx): + warnings.warn( + "There are some small singular values, using " + "whiten_solver = 'svd' might lead to more " + "accurate results." + ) + d[degenerate_idx] = eps # For numerical issues + np.sqrt(d, out=d) + d, u = d[sort_indices], u[:, sort_indices] + elif self.whiten_solver == "svd": + u, d = linalg.svd(XT, full_matrices=False, check_finite=False)[:2] + + # Give consistent eigenvectors for both svd solvers + u *= np.sign(u[0]) + + K = (u / d).T[:n_components] # see (6.33) p.140 + del u, d + X1 = np.dot(K, XT) + # see (13.6) p.267 Here X1 is white and data + # in X has been projected onto a subspace by PCA + X1 *= np.sqrt(n_samples) + else: + # X must be casted to floats to avoid typing issues with numpy + # 2.0 and the line below + X1 = as_float_array(XT, copy=False) # copy has been taken care of + + w_init = self.w_init + if w_init is None: + w_init = np.asarray( + random_state.normal(size=(n_components, n_components)), dtype=X1.dtype + ) + + else: + w_init = np.asarray(w_init) + if w_init.shape != (n_components, n_components): + raise ValueError( + "w_init has invalid shape -- should be %(shape)s" + % {"shape": (n_components, n_components)} + ) + + kwargs = { + "tol": self.tol, + "g": g, + "fun_args": fun_args, + "max_iter": self.max_iter, + "w_init": w_init, + } + + if self.algorithm == "parallel": + W, n_iter = _ica_par(X1, **kwargs) + elif self.algorithm == "deflation": + W, n_iter = _ica_def(X1, **kwargs) + del X1 + + self.n_iter_ = n_iter + + if compute_sources: + if self.whiten: + S = np.linalg.multi_dot([W, K, XT]).T + else: + S = np.dot(W, XT).T + else: + S = None + + if self.whiten: + if self.whiten == "unit-variance": + if not compute_sources: + S = np.linalg.multi_dot([W, K, XT]).T + S_std = np.std(S, axis=0, keepdims=True) + S /= S_std + W /= S_std.T + + self.components_ = np.dot(W, K) + self.mean_ = X_mean + self.whitening_ = K + else: + self.components_ = W + + self.mixing_ = linalg.pinv(self.components_, check_finite=False) + self._unmixing = W + + return S + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None): + """Fit the model and recover the sources from X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Estimated sources obtained by transforming the data with the + estimated unmixing matrix. + """ + return self._fit_transform(X, compute_sources=True) + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the model to X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + self._fit_transform(X, compute_sources=False) + return self + + def transform(self, X, copy=True): + """Recover the sources from X (apply the unmixing matrix). + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Data to transform, where `n_samples` is the number of samples + and `n_features` is the number of features. + + copy : bool, default=True + If False, data passed to fit can be overwritten. Defaults to True. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Estimated sources obtained by transforming the data with the + estimated unmixing matrix. + """ + check_is_fitted(self) + + X = validate_data( + self, + X, + copy=(copy and self.whiten), + dtype=[np.float64, np.float32], + reset=False, + ) + if self.whiten: + X -= self.mean_ + + return np.dot(X, self.components_.T) + + def inverse_transform(self, X, copy=True): + """Transform the sources back to the mixed data (apply mixing matrix). + + Parameters + ---------- + X : array-like of shape (n_samples, n_components) + Sources, where `n_samples` is the number of samples + and `n_components` is the number of components. + copy : bool, default=True + If False, data passed to fit are overwritten. Defaults to True. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Reconstructed data obtained with the mixing matrix. + """ + check_is_fitted(self) + + X = check_array(X, copy=(copy and self.whiten), dtype=[np.float64, np.float32]) + X = np.dot(X, self.mixing_.T) + if self.whiten: + X += self.mean_ + + return X + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_incremental_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_incremental_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..9410d9b162cec125e17376b80820e629acaf84e5 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_incremental_pca.py @@ -0,0 +1,423 @@ +"""Incremental Principal Components Analysis.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral + +import numpy as np +from scipy import linalg, sparse + +from sklearn.base import _fit_context +from sklearn.decomposition._base import _BasePCA +from sklearn.utils import gen_batches, metadata_routing +from sklearn.utils._param_validation import Interval +from sklearn.utils.extmath import _incremental_mean_and_var, svd_flip +from sklearn.utils.validation import validate_data + + +class IncrementalPCA(_BasePCA): + """Incremental principal components analysis (IPCA). + + Linear dimensionality reduction using Singular Value Decomposition of + the data, keeping only the most significant singular vectors to + project the data to a lower dimensional space. The input data is centered + but not scaled for each feature before applying the SVD. + + Depending on the size of the input data, this algorithm can be much more + memory efficient than a PCA, and allows sparse input. + + This algorithm has constant memory complexity, on the order + of ``batch_size * n_features``, enabling use of np.memmap files without + loading the entire file into memory. For sparse matrices, the input + is converted to dense in batches (in order to be able to subtract the + mean) which avoids storing the entire dense matrix at any one time. + + The computational overhead of each SVD is + ``O(batch_size * n_features ** 2)``, but only 2 * batch_size samples + remain in memory at a time. There will be ``n_samples / batch_size`` SVD + computations to get the principal components, versus 1 large SVD of + complexity ``O(n_samples * n_features ** 2)`` for PCA. + + For a usage example, see + :ref:`sphx_glr_auto_examples_decomposition_plot_incremental_pca.py`. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.16 + + Parameters + ---------- + n_components : int, default=None + Number of components to keep. If ``n_components`` is ``None``, + then ``n_components`` is set to ``min(n_samples, n_features)``. + + whiten : bool, default=False + When True (False by default) the ``components_`` vectors are divided + by ``n_samples`` times ``components_`` to ensure uncorrelated outputs + with unit component-wise variances. + + Whitening will remove some information from the transformed signal + (the relative variance scales of the components) but can sometimes + improve the predictive accuracy of the downstream estimators by + making data respect some hard-wired assumptions. + + copy : bool, default=True + If False, X will be overwritten. ``copy=False`` can be used to + save memory but is unsafe for general use. + + batch_size : int, default=None + The number of samples to use for each batch. Only used when calling + ``fit``. If ``batch_size`` is ``None``, then ``batch_size`` + is inferred from the data and set to ``5 * n_features``, to provide a + balance between approximation accuracy and memory consumption. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Principal axes in feature space, representing the directions of + maximum variance in the data. Equivalently, the right singular + vectors of the centered input data, parallel to its eigenvectors. + The components are sorted by decreasing ``explained_variance_``. + + explained_variance_ : ndarray of shape (n_components,) + Variance explained by each of the selected components. + + explained_variance_ratio_ : ndarray of shape (n_components,) + Percentage of variance explained by each of the selected components. + If all components are stored, the sum of explained variances is equal + to 1.0. + + singular_values_ : ndarray of shape (n_components,) + The singular values corresponding to each of the selected components. + The singular values are equal to the 2-norms of the ``n_components`` + variables in the lower-dimensional space. + + mean_ : ndarray of shape (n_features,) + Per-feature empirical mean, aggregate over calls to ``partial_fit``. + + var_ : ndarray of shape (n_features,) + Per-feature empirical variance, aggregate over calls to + ``partial_fit``. + + noise_variance_ : float + The estimated noise covariance following the Probabilistic PCA model + from Tipping and Bishop 1999. See "Pattern Recognition and + Machine Learning" by C. Bishop, 12.2.1 p. 574 or + http://www.miketipping.com/papers/met-mppca.pdf. + + n_components_ : int + The estimated number of components. Relevant when + ``n_components=None``. + + n_samples_seen_ : int + The number of samples processed by the estimator. Will be reset on + new calls to fit, but increments across ``partial_fit`` calls. + + batch_size_ : int + Inferred batch size from ``batch_size``. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + PCA : Principal component analysis (PCA). + KernelPCA : Kernel Principal component analysis (KPCA). + SparsePCA : Sparse Principal Components Analysis (SparsePCA). + TruncatedSVD : Dimensionality reduction using truncated SVD. + + Notes + ----- + Implements the incremental PCA model from Ross et al. (2008) [1]_. + This model is an extension of the Sequential Karhunen-Loeve Transform + from Levy and Lindenbaum (2000) [2]_. + + We have specifically abstained from an optimization used by authors of both + papers, a QR decomposition used in specific situations to reduce the + algorithmic complexity of the SVD. The source for this technique is + *Matrix Computations* (Golub and Van Loan 1997 [3]_). + This technique has been omitted because it is + advantageous only when decomposing a matrix with ``n_samples`` (rows) + >= 5/3 * ``n_features`` (columns), and hurts the readability of the + implemented algorithm. This would be a good opportunity for future + optimization, if it is deemed necessary. + + References + ---------- + .. [1] D. Ross, J. Lim, R. Lin, M. Yang. Incremental Learning for Robust + Visual Tracking, International Journal of Computer Vision, Volume 77, + Issue 1-3, pp. 125-141, May 2008. + https://www.cs.toronto.edu/~dross/ivt/RossLimLinYang_ijcv.pdf + + .. [2] :doi:`A. Levy and M. Lindenbaum, Sequential Karhunen-Loeve + Basis Extraction and its Application to Images, + IEEE Transactions on Image Processing, Volume 9, + Number 8, pp. 1371-1374, August 2000. <10.1109/83.855432>` + + .. [3] G. Golub and C. Van Loan. Matrix Computations, Third Edition, + Chapter 5, Section 5.4.4, pp. 252-253, 1997. + + Examples + -------- + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import IncrementalPCA + >>> from scipy import sparse + >>> X, _ = load_digits(return_X_y=True) + >>> transformer = IncrementalPCA(n_components=7, batch_size=200) + >>> # either partially fit on smaller batches of data + >>> transformer.partial_fit(X[:100, :]) + IncrementalPCA(batch_size=200, n_components=7) + >>> # or let the fit function itself divide the data into batches + >>> X_sparse = sparse.csr_matrix(X) + >>> X_transformed = transformer.fit_transform(X_sparse) + >>> X_transformed.shape + (1797, 7) + """ + + __metadata_request__partial_fit = {"check_input": metadata_routing.UNUSED} + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "whiten": ["boolean"], + "copy": ["boolean"], + "batch_size": [Interval(Integral, 1, None, closed="left"), None], + } + + def __init__(self, n_components=None, *, whiten=False, copy=True, batch_size=None): + self.n_components = n_components + self.whiten = whiten + self.copy = copy + self.batch_size = batch_size + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the model with X, using minibatches of size batch_size. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + self.components_ = None + self.n_samples_seen_ = 0 + self.mean_ = 0.0 + self.var_ = 0.0 + self.singular_values_ = None + self.explained_variance_ = None + self.explained_variance_ratio_ = None + self.noise_variance_ = None + + X = validate_data( + self, + X, + accept_sparse=["csr", "csc", "lil"], + copy=self.copy, + dtype=[np.float64, np.float32], + force_writeable=True, + ) + n_samples, n_features = X.shape + + if self.batch_size is None: + self.batch_size_ = 5 * n_features + else: + self.batch_size_ = self.batch_size + + for batch in gen_batches( + n_samples, self.batch_size_, min_batch_size=self.n_components or 0 + ): + X_batch = X[batch] + if sparse.issparse(X_batch): + X_batch = X_batch.toarray() + self.partial_fit(X_batch, check_input=False) + + return self + + @_fit_context(prefer_skip_nested_validation=True) + def partial_fit(self, X, y=None, check_input=True): + """Incremental fit with X. All of X is processed as a single batch. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + check_input : bool, default=True + Run check_array on X. + + Returns + ------- + self : object + Returns the instance itself. + """ + first_pass = not hasattr(self, "components_") + + if check_input: + if sparse.issparse(X): + raise TypeError( + "IncrementalPCA.partial_fit does not support " + "sparse input. Either convert data to dense " + "or use IncrementalPCA.fit to do so in batches." + ) + X = validate_data( + self, + X, + copy=self.copy, + dtype=[np.float64, np.float32], + force_writeable=True, + reset=first_pass, + ) + n_samples, n_features = X.shape + if first_pass: + self.components_ = None + + if self.n_components is None: + if self.components_ is None: + self.n_components_ = min(n_samples, n_features) + else: + self.n_components_ = self.components_.shape[0] + elif not self.n_components <= n_features: + raise ValueError( + "n_components=%r invalid for n_features=%d, need " + "more rows than columns for IncrementalPCA " + "processing" % (self.n_components, n_features) + ) + elif self.n_components > n_samples and first_pass: + raise ValueError( + f"n_components={self.n_components} must be less or equal to " + f"the batch number of samples {n_samples} for the first " + "partial_fit call." + ) + else: + self.n_components_ = self.n_components + + if (self.components_ is not None) and ( + self.components_.shape[0] != self.n_components_ + ): + raise ValueError( + "Number of input features has changed from %i " + "to %i between calls to partial_fit! Try " + "setting n_components to a fixed value." + % (self.components_.shape[0], self.n_components_) + ) + + # This is the first partial_fit + if not hasattr(self, "n_samples_seen_"): + self.n_samples_seen_ = 0 + self.mean_ = 0.0 + self.var_ = 0.0 + + # Update stats - they are 0 if this is the first step + col_mean, col_var, n_total_samples = _incremental_mean_and_var( + X, + last_mean=self.mean_, + last_variance=self.var_, + last_sample_count=np.repeat(self.n_samples_seen_, X.shape[1]), + ) + n_total_samples = n_total_samples[0] + + # Whitening + if self.n_samples_seen_ == 0: + # If it is the first step, simply whiten X + X -= col_mean + else: + col_batch_mean = np.mean(X, axis=0) + X -= col_batch_mean + # Build matrix of combined previous basis and new data + mean_correction = np.sqrt( + (self.n_samples_seen_ / n_total_samples) * n_samples + ) * (self.mean_ - col_batch_mean) + X = np.vstack( + ( + self.singular_values_.reshape((-1, 1)) * self.components_, + X, + mean_correction, + ) + ) + + U, S, Vt = linalg.svd(X, full_matrices=False, check_finite=False) + U, Vt = svd_flip(U, Vt, u_based_decision=False) + explained_variance = S**2 / (n_total_samples - 1) + explained_variance_ratio = S**2 / np.sum(col_var * n_total_samples) + + self.n_samples_seen_ = n_total_samples + self.components_ = Vt[: self.n_components_] + self.singular_values_ = S[: self.n_components_] + self.mean_ = col_mean + self.var_ = col_var + self.explained_variance_ = explained_variance[: self.n_components_] + self.explained_variance_ratio_ = explained_variance_ratio[: self.n_components_] + # we already checked `self.n_components <= n_samples` above + if self.n_components_ not in (n_samples, n_features): + self.noise_variance_ = explained_variance[self.n_components_ :].mean() + else: + self.noise_variance_ = 0.0 + return self + + def transform(self, X): + """Apply dimensionality reduction to X. + + X is projected on the first principal components previously extracted + from a training set, using minibatches of size batch_size if X is + sparse. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + New data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Projection of X in the first principal components. + + Examples + -------- + + >>> import numpy as np + >>> from sklearn.decomposition import IncrementalPCA + >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], + ... [1, 1], [2, 1], [3, 2]]) + >>> ipca = IncrementalPCA(n_components=2, batch_size=3) + >>> ipca.fit(X) + IncrementalPCA(batch_size=3, n_components=2) + >>> ipca.transform(X) # doctest: +SKIP + """ + if sparse.issparse(X): + n_samples = X.shape[0] + output = [] + for batch in gen_batches( + n_samples, self.batch_size_, min_batch_size=self.n_components or 0 + ): + output.append(super().transform(X[batch].toarray())) + return np.vstack(output) + else: + return super().transform(X) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + # Beware that fit accepts sparse data but partial_fit doesn't + tags.input_tags.sparse = True + return tags diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_kernel_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_kernel_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..8546287f4dc128f30a0f42edf0805bf7f70f42b5 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_kernel_pca.py @@ -0,0 +1,579 @@ +"""Kernel Principal Components Analysis.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real + +import numpy as np +from scipy import linalg +from scipy.linalg import eigh +from scipy.sparse.linalg import eigsh + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.exceptions import NotFittedError +from sklearn.metrics.pairwise import pairwise_kernels +from sklearn.preprocessing import KernelCenterer +from sklearn.utils._arpack import _init_arpack_v0 +from sklearn.utils._param_validation import Interval, StrOptions +from sklearn.utils.extmath import _randomized_eigsh, svd_flip +from sklearn.utils.validation import ( + _check_psd_eigenvalues, + check_is_fitted, + validate_data, +) + + +class KernelPCA(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): + """Kernel Principal component analysis (KPCA). + + Non-linear dimensionality reduction through the use of kernels [1]_, see also + :ref:`metrics`. + + It uses the :func:`scipy.linalg.eigh` LAPACK implementation of the full SVD + or the :func:`scipy.sparse.linalg.eigsh` ARPACK implementation of the + truncated SVD, depending on the shape of the input data and the number of + components to extract. It can also use a randomized truncated SVD by the + method proposed in [3]_, see `eigen_solver`. + + For a usage example and comparison between + Principal Components Analysis (PCA) and its kernelized version (KPCA), see + :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py`. + + For a usage example in denoising images using KPCA, see + :ref:`sphx_glr_auto_examples_applications_plot_digits_denoising.py`. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=None + Number of components. If None, all non-zero components are kept. + + kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'cosine', 'precomputed'} \ + or callable, default='linear' + Kernel used for PCA. + + gamma : float, default=None + Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other + kernels. If ``gamma`` is ``None``, then it is set to ``1/n_features``. + + degree : float, default=3 + Degree for poly kernels. Ignored by other kernels. + + coef0 : float, default=1 + Independent term in poly and sigmoid kernels. + Ignored by other kernels. + + kernel_params : dict, default=None + Parameters (keyword arguments) and + values for kernel passed as callable object. + Ignored by other kernels. + + alpha : float, default=1.0 + Hyperparameter of the ridge regression that learns the + inverse transform (when fit_inverse_transform=True). + + fit_inverse_transform : bool, default=False + Learn the inverse transform for non-precomputed kernels + (i.e. learn to find the pre-image of a point). This method is based + on [2]_. + + eigen_solver : {'auto', 'dense', 'arpack', 'randomized'}, \ + default='auto' + Select eigensolver to use. If `n_components` is much + less than the number of training samples, randomized (or arpack to a + smaller extent) may be more efficient than the dense eigensolver. + Randomized SVD is performed according to the method of Halko et al + [3]_. + + auto : + the solver is selected by a default policy based on n_samples + (the number of training samples) and `n_components`: + if the number of components to extract is less than 10 (strict) and + the number of samples is more than 200 (strict), the 'arpack' + method is enabled. Otherwise the exact full eigenvalue + decomposition is computed and optionally truncated afterwards + ('dense' method). + dense : + run exact full eigenvalue decomposition calling the standard + LAPACK solver via `scipy.linalg.eigh`, and select the components + by postprocessing + arpack : + run SVD truncated to n_components calling ARPACK solver using + `scipy.sparse.linalg.eigsh`. It requires strictly + 0 < n_components < n_samples + randomized : + run randomized SVD by the method of Halko et al. [3]_. The current + implementation selects eigenvalues based on their module; therefore + using this method can lead to unexpected results if the kernel is + not positive semi-definite. See also [4]_. + + .. versionchanged:: 1.0 + `'randomized'` was added. + + tol : float, default=0 + Convergence tolerance for arpack. + If 0, optimal value will be chosen by arpack. + + max_iter : int, default=None + Maximum number of iterations for arpack. + If None, optimal value will be chosen by arpack. + + iterated_power : int >= 0, or 'auto', default='auto' + Number of iterations for the power method computed by + svd_solver == 'randomized'. When 'auto', it is set to 7 when + `n_components < 0.1 * min(X.shape)`, other it is set to 4. + + .. versionadded:: 1.0 + + remove_zero_eig : bool, default=False + If True, then all components with zero eigenvalues are removed, so + that the number of components in the output may be < n_components + (and sometimes even zero due to numerical instability). + When n_components is None, this parameter is ignored and components + with zero eigenvalues are removed regardless. + + random_state : int, RandomState instance or None, default=None + Used when ``eigen_solver`` == 'arpack' or 'randomized'. Pass an int + for reproducible results across multiple function calls. + See :term:`Glossary `. + + .. versionadded:: 0.18 + + copy_X : bool, default=True + If True, input X is copied and stored by the model in the `X_fit_` + attribute. If no further changes will be done to X, setting + `copy_X=False` saves memory by storing a reference. + + .. versionadded:: 0.18 + + n_jobs : int, default=None + The number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + .. versionadded:: 0.18 + + Attributes + ---------- + eigenvalues_ : ndarray of shape (n_components,) + Eigenvalues of the centered kernel matrix in decreasing order. + If `n_components` and `remove_zero_eig` are not set, + then all values are stored. + + eigenvectors_ : ndarray of shape (n_samples, n_components) + Eigenvectors of the centered kernel matrix. If `n_components` and + `remove_zero_eig` are not set, then all components are stored. + + dual_coef_ : ndarray of shape (n_samples, n_features) + Inverse transform matrix. Only available when + ``fit_inverse_transform`` is True. + + X_transformed_fit_ : ndarray of shape (n_samples, n_components) + Projection of the fitted data on the kernel principal components. + Only available when ``fit_inverse_transform`` is True. + + X_fit_ : ndarray of shape (n_samples, n_features) + The data used to fit the model. If `copy_X=False`, then `X_fit_` is + a reference. This attribute is used for the calls to transform. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + gamma_ : float + Kernel coefficient for rbf, poly and sigmoid kernels. When `gamma` + is explicitly provided, this is just the same as `gamma`. When `gamma` + is `None`, this is the actual value of kernel coefficient. + + .. versionadded:: 1.3 + + See Also + -------- + FastICA : A fast algorithm for Independent Component Analysis. + IncrementalPCA : Incremental Principal Component Analysis. + NMF : Non-Negative Matrix Factorization. + PCA : Principal Component Analysis. + SparsePCA : Sparse Principal Component Analysis. + TruncatedSVD : Dimensionality reduction using truncated SVD. + + References + ---------- + .. [1] `Schölkopf, Bernhard, Alexander Smola, and Klaus-Robert Müller. + "Kernel principal component analysis." + International conference on artificial neural networks. + Springer, Berlin, Heidelberg, 1997. + `_ + + .. [2] `Bakır, Gökhan H., Jason Weston, and Bernhard Schölkopf. + "Learning to find pre-images." + Advances in neural information processing systems 16 (2004): 449-456. + `_ + + .. [3] :arxiv:`Halko, Nathan, Per-Gunnar Martinsson, and Joel A. Tropp. + "Finding structure with randomness: Probabilistic algorithms for + constructing approximate matrix decompositions." + SIAM review 53.2 (2011): 217-288. <0909.4061>` + + .. [4] `Martinsson, Per-Gunnar, Vladimir Rokhlin, and Mark Tygert. + "A randomized algorithm for the decomposition of matrices." + Applied and Computational Harmonic Analysis 30.1 (2011): 47-68. + `_ + + Examples + -------- + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import KernelPCA + >>> X, _ = load_digits(return_X_y=True) + >>> transformer = KernelPCA(n_components=7, kernel='linear') + >>> X_transformed = transformer.fit_transform(X) + >>> X_transformed.shape + (1797, 7) + """ + + _parameter_constraints: dict = { + "n_components": [ + Interval(Integral, 1, None, closed="left"), + None, + ], + "kernel": [ + StrOptions({"linear", "poly", "rbf", "sigmoid", "cosine", "precomputed"}), + callable, + ], + "gamma": [ + Interval(Real, 0, None, closed="left"), + None, + ], + "degree": [Interval(Real, 0, None, closed="left")], + "coef0": [Interval(Real, None, None, closed="neither")], + "kernel_params": [dict, None], + "alpha": [Interval(Real, 0, None, closed="left")], + "fit_inverse_transform": ["boolean"], + "eigen_solver": [StrOptions({"auto", "dense", "arpack", "randomized"})], + "tol": [Interval(Real, 0, None, closed="left")], + "max_iter": [ + Interval(Integral, 1, None, closed="left"), + None, + ], + "iterated_power": [ + Interval(Integral, 0, None, closed="left"), + StrOptions({"auto"}), + ], + "remove_zero_eig": ["boolean"], + "random_state": ["random_state"], + "copy_X": ["boolean"], + "n_jobs": [None, Integral], + } + + def __init__( + self, + n_components=None, + *, + kernel="linear", + gamma=None, + degree=3, + coef0=1, + kernel_params=None, + alpha=1.0, + fit_inverse_transform=False, + eigen_solver="auto", + tol=0, + max_iter=None, + iterated_power="auto", + remove_zero_eig=False, + random_state=None, + copy_X=True, + n_jobs=None, + ): + self.n_components = n_components + self.kernel = kernel + self.kernel_params = kernel_params + self.gamma = gamma + self.degree = degree + self.coef0 = coef0 + self.alpha = alpha + self.fit_inverse_transform = fit_inverse_transform + self.eigen_solver = eigen_solver + self.tol = tol + self.max_iter = max_iter + self.iterated_power = iterated_power + self.remove_zero_eig = remove_zero_eig + self.random_state = random_state + self.n_jobs = n_jobs + self.copy_X = copy_X + + def _get_kernel(self, X, Y=None): + if callable(self.kernel): + params = self.kernel_params or {} + else: + params = {"gamma": self.gamma_, "degree": self.degree, "coef0": self.coef0} + return pairwise_kernels( + X, Y, metric=self.kernel, filter_params=True, n_jobs=self.n_jobs, **params + ) + + def _fit_transform_in_place(self, K): + """Fit's using kernel K""" + # center kernel in place + K = self._centerer.fit(K).transform(K, copy=False) + + # adjust n_components according to user inputs + if self.n_components is None: + n_components = K.shape[0] # use all dimensions + else: + n_components = min(K.shape[0], self.n_components) + + # compute eigenvectors + if self.eigen_solver == "auto": + if K.shape[0] > 200 and n_components < 10: + eigen_solver = "arpack" + else: + eigen_solver = "dense" + else: + eigen_solver = self.eigen_solver + + if eigen_solver == "dense": + # Note: subset_by_index specifies the indices of smallest/largest to return + self.eigenvalues_, self.eigenvectors_ = eigh( + K, subset_by_index=(K.shape[0] - n_components, K.shape[0] - 1) + ) + elif eigen_solver == "arpack": + v0 = _init_arpack_v0(K.shape[0], self.random_state) + self.eigenvalues_, self.eigenvectors_ = eigsh( + K, n_components, which="LA", tol=self.tol, maxiter=self.max_iter, v0=v0 + ) + elif eigen_solver == "randomized": + self.eigenvalues_, self.eigenvectors_ = _randomized_eigsh( + K, + n_components=n_components, + n_iter=self.iterated_power, + random_state=self.random_state, + selection="module", + ) + + # make sure that the eigenvalues are ok and fix numerical issues + self.eigenvalues_ = _check_psd_eigenvalues( + self.eigenvalues_, enable_warnings=False + ) + + # flip eigenvectors' sign to enforce deterministic output + self.eigenvectors_, _ = svd_flip(u=self.eigenvectors_, v=None) + + # sort eigenvectors in descending order + indices = self.eigenvalues_.argsort()[::-1] + self.eigenvalues_ = self.eigenvalues_[indices] + self.eigenvectors_ = self.eigenvectors_[:, indices] + + # remove eigenvectors with a zero eigenvalue (null space) if required + if self.remove_zero_eig or self.n_components is None: + self.eigenvectors_ = self.eigenvectors_[:, self.eigenvalues_ > 0] + self.eigenvalues_ = self.eigenvalues_[self.eigenvalues_ > 0] + + # Maintenance note on Eigenvectors normalization + # ---------------------------------------------- + # there is a link between + # the eigenvectors of K=Phi(X)'Phi(X) and the ones of Phi(X)Phi(X)' + # if v is an eigenvector of K + # then Phi(X)v is an eigenvector of Phi(X)Phi(X)' + # if u is an eigenvector of Phi(X)Phi(X)' + # then Phi(X)'u is an eigenvector of Phi(X)'Phi(X) + # + # At this stage our self.eigenvectors_ (the v) have norm 1, we need to scale + # them so that eigenvectors in kernel feature space (the u) have norm=1 + # instead + # + # We COULD scale them here: + # self.eigenvectors_ = self.eigenvectors_ / np.sqrt(self.eigenvalues_) + # + # But choose to perform that LATER when needed, in `fit()` and in + # `transform()`. + + return K + + def _fit_inverse_transform(self, X_transformed, X): + if hasattr(X, "tocsr"): + raise NotImplementedError( + "Inverse transform not implemented for sparse matrices!" + ) + + n_samples = X_transformed.shape[0] + K = self._get_kernel(X_transformed) + K.flat[:: n_samples + 1] += self.alpha + self.dual_coef_ = linalg.solve(K, X, assume_a="pos", overwrite_a=True) + self.X_transformed_fit_ = X_transformed + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the model from data in X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + if self.fit_inverse_transform and self.kernel == "precomputed": + raise ValueError("Cannot fit_inverse_transform with a precomputed kernel.") + X = validate_data(self, X, accept_sparse="csr", copy=self.copy_X) + self.gamma_ = 1 / X.shape[1] if self.gamma is None else self.gamma + self._centerer = KernelCenterer().set_output(transform="default") + K = self._get_kernel(X) + # When kernel="precomputed", K is X but it's safe to perform in place operations + # on K because a copy was made before if requested by copy_X. + self._fit_transform_in_place(K) + + if self.fit_inverse_transform: + # no need to use the kernel to transform X, use shortcut expression + X_transformed = self.eigenvectors_ * np.sqrt(self.eigenvalues_) + + self._fit_inverse_transform(X_transformed, X) + + self.X_fit_ = X + return self + + def fit_transform(self, X, y=None, **params): + """Fit the model from data in X and transform X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + **params : kwargs + Parameters (keyword arguments) and values passed to + the fit_transform instance. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Transformed values. + """ + self.fit(X, **params) + + # no need to use the kernel to transform X, use shortcut expression + X_transformed = self.eigenvectors_ * np.sqrt(self.eigenvalues_) + + if self.fit_inverse_transform: + self._fit_inverse_transform(X_transformed, X) + + return X_transformed + + def transform(self, X): + """Transform X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Projection of X in the first principal components, where `n_samples` + is the number of samples and `n_components` is the number of the components. + """ + check_is_fitted(self) + X = validate_data(self, X, accept_sparse="csr", reset=False) + + # Compute centered gram matrix between X and training data X_fit_ + K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) + + # scale eigenvectors (properly account for null-space for dot product) + non_zeros = np.flatnonzero(self.eigenvalues_) + scaled_alphas = np.zeros_like(self.eigenvectors_) + scaled_alphas[:, non_zeros] = self.eigenvectors_[:, non_zeros] / np.sqrt( + self.eigenvalues_[non_zeros] + ) + + # Project with a scalar product between K and the scaled eigenvectors + return np.dot(K, scaled_alphas) + + def inverse_transform(self, X): + """Transform X back to original space. + + ``inverse_transform`` approximates the inverse transformation using + a learned pre-image. The pre-image is learned by kernel ridge + regression of the original data on their low-dimensional representation + vectors. + + .. note: + :meth:`~sklearn.decomposition.fit` internally uses a centered + kernel. As the centered kernel no longer contains the information + of the mean of kernel features, such information is not taken into + account in reconstruction. + + .. note:: + When users want to compute inverse transformation for 'linear' + kernel, it is recommended that they use + :class:`~sklearn.decomposition.PCA` instead. Unlike + :class:`~sklearn.decomposition.PCA`, + :class:`~sklearn.decomposition.KernelPCA`'s ``inverse_transform`` + does not reconstruct the mean of data when 'linear' kernel is used + due to the use of centered kernel. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_components) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Original data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + References + ---------- + `Bakır, Gökhan H., Jason Weston, and Bernhard Schölkopf. + "Learning to find pre-images." + Advances in neural information processing systems 16 (2004): 449-456. + `_ + """ + if not self.fit_inverse_transform: + raise NotFittedError( + "The fit_inverse_transform parameter was not" + " set to True when instantiating and hence " + "the inverse transform is not available." + ) + + K = self._get_kernel(X, self.X_transformed_fit_) + return np.dot(K, self.dual_coef_) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + tags.input_tags.pairwise = self.kernel == "precomputed" + return tags + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.eigenvalues_.shape[0] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_lda.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_lda.py new file mode 100644 index 0000000000000000000000000000000000000000..af191ced2981a9d26e5403eba9b6a8c75f8fc4b7 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_lda.py @@ -0,0 +1,956 @@ +""" + +============================================================= +Online Latent Dirichlet Allocation with variational inference +============================================================= + +This implementation is modified from Matthew D. Hoffman's onlineldavb code +Link: https://github.com/blei-lab/onlineldavb +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real + +import numpy as np +import scipy.sparse as sp +from joblib import effective_n_jobs +from scipy.special import gammaln, logsumexp + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.decomposition._online_lda_fast import ( + _dirichlet_expectation_1d as cy_dirichlet_expectation_1d, +) +from sklearn.decomposition._online_lda_fast import _dirichlet_expectation_2d +from sklearn.decomposition._online_lda_fast import mean_change as cy_mean_change +from sklearn.utils import check_random_state, gen_batches, gen_even_slices +from sklearn.utils._param_validation import Interval, StrOptions +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.validation import check_is_fitted, check_non_negative, validate_data + +EPS = np.finfo(float).eps + + +def _update_doc_distribution( + X, + exp_topic_word_distr, + doc_topic_prior, + max_doc_update_iter, + mean_change_tol, + cal_sstats, + random_state, +): + """E-step: update document-topic distribution. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + exp_topic_word_distr : ndarray of shape (n_topics, n_features) + Exponential value of expectation of log topic word distribution. + In the literature, this is `exp(E[log(beta)])`. + + doc_topic_prior : float + Prior of document topic distribution `theta`. + + max_doc_update_iter : int + Max number of iterations for updating document topic distribution in + the E-step. + + mean_change_tol : float + Stopping tolerance for updating document topic distribution in E-step. + + cal_sstats : bool + Parameter that indicate to calculate sufficient statistics or not. + Set `cal_sstats` to `True` when we need to run M-step. + + random_state : RandomState instance or None + Parameter that indicate how to initialize document topic distribution. + Set `random_state` to None will initialize document topic distribution + to a constant number. + + Returns + ------- + (doc_topic_distr, suff_stats) : + `doc_topic_distr` is unnormalized topic distribution for each document. + In the literature, this is `gamma`. we can calculate `E[log(theta)]` + from it. + `suff_stats` is expected sufficient statistics for the M-step. + When `cal_sstats == False`, this will be None. + + """ + is_sparse_x = sp.issparse(X) + n_samples, n_features = X.shape + n_topics = exp_topic_word_distr.shape[0] + + if random_state: + doc_topic_distr = random_state.gamma(100.0, 0.01, (n_samples, n_topics)).astype( + X.dtype, copy=False + ) + else: + doc_topic_distr = np.ones((n_samples, n_topics), dtype=X.dtype) + + # In the literature, this is `exp(E[log(theta)])` + exp_doc_topic = np.exp(_dirichlet_expectation_2d(doc_topic_distr)) + + # diff on `component_` (only calculate it when `cal_diff` is True) + suff_stats = ( + np.zeros(exp_topic_word_distr.shape, dtype=X.dtype) if cal_sstats else None + ) + + if is_sparse_x: + X_data = X.data + X_indices = X.indices + X_indptr = X.indptr + + # These cython functions are called in a nested loop on usually very small arrays + # (length=n_topics). In that case, finding the appropriate signature of the + # fused-typed function can be more costly than its execution, hence the dispatch + # is done outside of the loop. + ctype = "float" if X.dtype == np.float32 else "double" + mean_change = cy_mean_change[ctype] + dirichlet_expectation_1d = cy_dirichlet_expectation_1d[ctype] + eps = np.finfo(X.dtype).eps + + for idx_d in range(n_samples): + if is_sparse_x: + ids = X_indices[X_indptr[idx_d] : X_indptr[idx_d + 1]] + cnts = X_data[X_indptr[idx_d] : X_indptr[idx_d + 1]] + else: + ids = np.nonzero(X[idx_d, :])[0] + cnts = X[idx_d, ids] + + doc_topic_d = doc_topic_distr[idx_d, :] + # The next one is a copy, since the inner loop overwrites it. + exp_doc_topic_d = exp_doc_topic[idx_d, :].copy() + exp_topic_word_d = exp_topic_word_distr[:, ids] + + # Iterate between `doc_topic_d` and `norm_phi` until convergence + for _ in range(0, max_doc_update_iter): + last_d = doc_topic_d + + # The optimal phi_{dwk} is proportional to + # exp(E[log(theta_{dk})]) * exp(E[log(beta_{dw})]). + norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + eps + + doc_topic_d = exp_doc_topic_d * np.dot(cnts / norm_phi, exp_topic_word_d.T) + # Note: adds doc_topic_prior to doc_topic_d, in-place. + dirichlet_expectation_1d(doc_topic_d, doc_topic_prior, exp_doc_topic_d) + + if mean_change(last_d, doc_topic_d) < mean_change_tol: + break + doc_topic_distr[idx_d, :] = doc_topic_d + + # Contribution of document d to the expected sufficient + # statistics for the M step. + if cal_sstats: + norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + eps + suff_stats[:, ids] += np.outer(exp_doc_topic_d, cnts / norm_phi) + + return (doc_topic_distr, suff_stats) + + +class LatentDirichletAllocation( + ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator +): + """Latent Dirichlet Allocation with online variational Bayes algorithm. + + The implementation is based on [1]_ and [2]_. + + .. versionadded:: 0.17 + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=10 + Number of topics. + + .. versionchanged:: 0.19 + ``n_topics`` was renamed to ``n_components`` + + doc_topic_prior : float, default=None + Prior of document topic distribution `theta`. If the value is None, + defaults to `1 / n_components`. + In [1]_, this is called `alpha`. + + topic_word_prior : float, default=None + Prior of topic word distribution `beta`. If the value is None, defaults + to `1 / n_components`. + In [1]_, this is called `eta`. + + learning_method : {'batch', 'online'}, default='batch' + Method used to update `_component`. Only used in :meth:`fit` method. + In general, if the data size is large, the online update will be much + faster than the batch update. + + Valid options: + + - 'batch': Batch variational Bayes method. Use all training data in each EM + update. Old `components_` will be overwritten in each iteration. + - 'online': Online variational Bayes method. In each EM update, use mini-batch + of training data to update the ``components_`` variable incrementally. The + learning rate is controlled by the ``learning_decay`` and the + ``learning_offset`` parameters. + + .. versionchanged:: 0.20 + The default learning method is now ``"batch"``. + + learning_decay : float, default=0.7 + It is a parameter that control learning rate in the online learning + method. The value should be set between (0.5, 1.0] to guarantee + asymptotic convergence. When the value is 0.0 and batch_size is + ``n_samples``, the update method is same as batch learning. In the + literature, this is called kappa. + + learning_offset : float, default=10.0 + A (positive) parameter that downweights early iterations in online + learning. It should be greater than 1.0. In the literature, this is + called tau_0. + + max_iter : int, default=10 + The maximum number of passes over the training data (aka epochs). + It only impacts the behavior in the :meth:`fit` method, and not the + :meth:`partial_fit` method. + + batch_size : int, default=128 + Number of documents to use in each EM iteration. Only used in online + learning. + + evaluate_every : int, default=-1 + How often to evaluate perplexity. Only used in `fit` method. + set it to 0 or negative number to not evaluate perplexity in + training at all. Evaluating perplexity can help you check convergence + in training process, but it will also increase total training time. + Evaluating perplexity in every iteration might increase training time + up to two-fold. + + total_samples : int, default=1e6 + Total number of documents. Only used in the :meth:`partial_fit` method. + + perp_tol : float, default=1e-1 + Perplexity tolerance. Only used when ``evaluate_every`` is greater than 0. + + mean_change_tol : float, default=1e-3 + Stopping tolerance for updating document topic distribution in E-step. + + max_doc_update_iter : int, default=100 + Max number of iterations for updating document topic distribution in + the E-step. + + n_jobs : int, default=None + The number of jobs to use in the E-step. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + verbose : int, default=0 + Verbosity level. + + random_state : int, RandomState instance or None, default=None + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Variational parameters for topic word distribution. Since the complete + conditional for topic word distribution is a Dirichlet, + ``components_[i, j]`` can be viewed as pseudocount that represents the + number of times word `j` was assigned to topic `i`. + It can also be viewed as distribution over the words for each topic + after normalization: + ``model.components_ / model.components_.sum(axis=1)[:, np.newaxis]``. + + exp_dirichlet_component_ : ndarray of shape (n_components, n_features) + Exponential value of expectation of log topic word distribution. + In the literature, this is `exp(E[log(beta)])`. + + n_batch_iter_ : int + Number of iterations of the EM step. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_iter_ : int + Number of passes over the dataset. + + bound_ : float + Final perplexity score on training set. + + doc_topic_prior_ : float + Prior of document topic distribution `theta`. If the value is None, + it is `1 / n_components`. + + random_state_ : RandomState instance + RandomState instance that is generated either from a seed, the random + number generator or by `np.random`. + + topic_word_prior_ : float + Prior of topic word distribution `beta`. If the value is None, it is + `1 / n_components`. + + See Also + -------- + sklearn.discriminant_analysis.LinearDiscriminantAnalysis: + A classifier with a linear decision boundary, generated by fitting + class conditional densities to the data and using Bayes' rule. + + References + ---------- + .. [1] "Online Learning for Latent Dirichlet Allocation", Matthew D. + Hoffman, David M. Blei, Francis Bach, 2010. + https://github.com/blei-lab/onlineldavb + + .. [2] "Stochastic Variational Inference", Matthew D. Hoffman, + David M. Blei, Chong Wang, John Paisley, 2013. + https://jmlr.org/papers/volume14/hoffman13a/hoffman13a.pdf + + Examples + -------- + >>> from sklearn.decomposition import LatentDirichletAllocation + >>> from sklearn.datasets import make_multilabel_classification + >>> # This produces a feature matrix of token counts, similar to what + >>> # CountVectorizer would produce on text. + >>> X, _ = make_multilabel_classification(random_state=0) + >>> lda = LatentDirichletAllocation(n_components=5, + ... random_state=0) + >>> lda.fit(X) + LatentDirichletAllocation(...) + >>> # get topics for some given samples: + >>> lda.transform(X[-2:]) + array([[0.00360392, 0.25499205, 0.0036211 , 0.64236448, 0.09541846], + [0.15297572, 0.00362644, 0.44412786, 0.39568399, 0.003586 ]]) + """ + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 0, None, closed="neither")], + "doc_topic_prior": [None, Interval(Real, 0, 1, closed="both")], + "topic_word_prior": [None, Interval(Real, 0, 1, closed="both")], + "learning_method": [StrOptions({"batch", "online"})], + "learning_decay": [Interval(Real, 0, 1, closed="both")], + "learning_offset": [Interval(Real, 1.0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "batch_size": [Interval(Integral, 0, None, closed="neither")], + "evaluate_every": [Interval(Integral, None, None, closed="neither")], + "total_samples": [Interval(Real, 0, None, closed="neither")], + "perp_tol": [Interval(Real, 0, None, closed="left")], + "mean_change_tol": [Interval(Real, 0, None, closed="left")], + "max_doc_update_iter": [Interval(Integral, 0, None, closed="left")], + "n_jobs": [None, Integral], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=10, + *, + doc_topic_prior=None, + topic_word_prior=None, + learning_method="batch", + learning_decay=0.7, + learning_offset=10.0, + max_iter=10, + batch_size=128, + evaluate_every=-1, + total_samples=1e6, + perp_tol=1e-1, + mean_change_tol=1e-3, + max_doc_update_iter=100, + n_jobs=None, + verbose=0, + random_state=None, + ): + self.n_components = n_components + self.doc_topic_prior = doc_topic_prior + self.topic_word_prior = topic_word_prior + self.learning_method = learning_method + self.learning_decay = learning_decay + self.learning_offset = learning_offset + self.max_iter = max_iter + self.batch_size = batch_size + self.evaluate_every = evaluate_every + self.total_samples = total_samples + self.perp_tol = perp_tol + self.mean_change_tol = mean_change_tol + self.max_doc_update_iter = max_doc_update_iter + self.n_jobs = n_jobs + self.verbose = verbose + self.random_state = random_state + + def _init_latent_vars(self, n_features, dtype=np.float64): + """Initialize latent variables.""" + + self.random_state_ = check_random_state(self.random_state) + self.n_batch_iter_ = 1 + self.n_iter_ = 0 + + if self.doc_topic_prior is None: + self.doc_topic_prior_ = 1.0 / self.n_components + else: + self.doc_topic_prior_ = self.doc_topic_prior + + if self.topic_word_prior is None: + self.topic_word_prior_ = 1.0 / self.n_components + else: + self.topic_word_prior_ = self.topic_word_prior + + init_gamma = 100.0 + init_var = 1.0 / init_gamma + # In the literature, this is called `lambda` + self.components_ = self.random_state_.gamma( + init_gamma, init_var, (self.n_components, n_features) + ).astype(dtype, copy=False) + + # In the literature, this is `exp(E[log(beta)])` + self.exp_dirichlet_component_ = np.exp( + _dirichlet_expectation_2d(self.components_) + ) + + def _e_step(self, X, cal_sstats, random_init, parallel=None): + """E-step in EM update. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + cal_sstats : bool + Parameter that indicate whether to calculate sufficient statistics + or not. Set ``cal_sstats`` to True when we need to run M-step. + + random_init : bool + Parameter that indicate whether to initialize document topic + distribution randomly in the E-step. Set it to True in training + steps. + + parallel : joblib.Parallel, default=None + Pre-initialized instance of joblib.Parallel. + + Returns + ------- + (doc_topic_distr, suff_stats) : + `doc_topic_distr` is unnormalized topic distribution for each + document. In the literature, this is called `gamma`. + `suff_stats` is expected sufficient statistics for the M-step. + When `cal_sstats == False`, it will be None. + + """ + + # Run e-step in parallel + random_state = self.random_state_ if random_init else None + + # TODO: make Parallel._effective_n_jobs public instead? + n_jobs = effective_n_jobs(self.n_jobs) + if parallel is None: + parallel = Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) + results = parallel( + delayed(_update_doc_distribution)( + X[idx_slice, :], + self.exp_dirichlet_component_, + self.doc_topic_prior_, + self.max_doc_update_iter, + self.mean_change_tol, + cal_sstats, + random_state, + ) + for idx_slice in gen_even_slices(X.shape[0], n_jobs) + ) + + # merge result + doc_topics, sstats_list = zip(*results) + doc_topic_distr = np.vstack(doc_topics) + + if cal_sstats: + # This step finishes computing the sufficient statistics for the + # M-step. + suff_stats = np.zeros(self.components_.shape, dtype=self.components_.dtype) + for sstats in sstats_list: + suff_stats += sstats + suff_stats *= self.exp_dirichlet_component_ + else: + suff_stats = None + + return (doc_topic_distr, suff_stats) + + def _em_step(self, X, total_samples, batch_update, parallel=None): + """EM update for 1 iteration. + + update `component_` by batch VB or online VB. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + total_samples : int + Total number of documents. It is only used when + batch_update is `False`. + + batch_update : bool + Parameter that controls updating method. + `True` for batch learning, `False` for online learning. + + parallel : joblib.Parallel, default=None + Pre-initialized instance of joblib.Parallel + + Returns + ------- + doc_topic_distr : ndarray of shape (n_samples, n_components) + Unnormalized document topic distribution. + """ + + # E-step + _, suff_stats = self._e_step( + X, cal_sstats=True, random_init=True, parallel=parallel + ) + + # M-step + if batch_update: + self.components_ = self.topic_word_prior_ + suff_stats + else: + # online update + # In the literature, the weight is `rho` + weight = np.power( + self.learning_offset + self.n_batch_iter_, -self.learning_decay + ) + doc_ratio = float(total_samples) / X.shape[0] + self.components_ *= 1 - weight + self.components_ += weight * ( + self.topic_word_prior_ + doc_ratio * suff_stats + ) + + # update `component_` related variables + self.exp_dirichlet_component_ = np.exp( + _dirichlet_expectation_2d(self.components_) + ) + self.n_batch_iter_ += 1 + return + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.positive_only = True + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float32", "float64"] + return tags + + def _check_non_neg_array(self, X, reset_n_features, whom): + """check X format + + check X format and make sure no negative value in X. + + Parameters + ---------- + X : array-like or sparse matrix + + """ + dtype = [np.float64, np.float32] if reset_n_features else self.components_.dtype + + X = validate_data( + self, + X, + reset=reset_n_features, + accept_sparse="csr", + dtype=dtype, + ) + check_non_negative(X, whom) + + return X + + @_fit_context(prefer_skip_nested_validation=True) + def partial_fit(self, X, y=None): + """Online VB with Mini-Batch update. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + y : Ignored + Not used, present here for API consistency by convention. + + Returns + ------- + self + Partially fitted estimator. + """ + first_time = not hasattr(self, "components_") + + X = self._check_non_neg_array( + X, reset_n_features=first_time, whom="LatentDirichletAllocation.partial_fit" + ) + n_samples, n_features = X.shape + batch_size = self.batch_size + + # initialize parameters or check + if first_time: + self._init_latent_vars(n_features, dtype=X.dtype) + + if n_features != self.components_.shape[1]: + raise ValueError( + "The provided data has %d dimensions while " + "the model was trained with feature size %d." + % (n_features, self.components_.shape[1]) + ) + + n_jobs = effective_n_jobs(self.n_jobs) + with Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) as parallel: + for idx_slice in gen_batches(n_samples, batch_size): + self._em_step( + X[idx_slice, :], + total_samples=self.total_samples, + batch_update=False, + parallel=parallel, + ) + + return self + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Learn model for the data X with variational Bayes method. + + When `learning_method` is 'online', use mini-batch update. + Otherwise, use batch update. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + y : Ignored + Not used, present here for API consistency by convention. + + Returns + ------- + self + Fitted estimator. + """ + X = self._check_non_neg_array( + X, reset_n_features=True, whom="LatentDirichletAllocation.fit" + ) + n_samples, n_features = X.shape + max_iter = self.max_iter + evaluate_every = self.evaluate_every + learning_method = self.learning_method + + batch_size = self.batch_size + + # initialize parameters + self._init_latent_vars(n_features, dtype=X.dtype) + # change to perplexity later + last_bound = None + n_jobs = effective_n_jobs(self.n_jobs) + with Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) as parallel: + for i in range(max_iter): + if learning_method == "online": + for idx_slice in gen_batches(n_samples, batch_size): + self._em_step( + X[idx_slice, :], + total_samples=n_samples, + batch_update=False, + parallel=parallel, + ) + else: + # batch update + self._em_step( + X, total_samples=n_samples, batch_update=True, parallel=parallel + ) + + # check perplexity + if evaluate_every > 0 and (i + 1) % evaluate_every == 0: + doc_topics_distr, _ = self._e_step( + X, cal_sstats=False, random_init=False, parallel=parallel + ) + bound = self._perplexity_precomp_distr( + X, doc_topics_distr, sub_sampling=False + ) + if self.verbose: + print( + "iteration: %d of max_iter: %d, perplexity: %.4f" + % (i + 1, max_iter, bound) + ) + + if last_bound and abs(last_bound - bound) < self.perp_tol: + break + last_bound = bound + + elif self.verbose: + print("iteration: %d of max_iter: %d" % (i + 1, max_iter)) + self.n_iter_ += 1 + + # calculate final perplexity value on train set + doc_topics_distr, _ = self._e_step( + X, cal_sstats=False, random_init=False, parallel=parallel + ) + self.bound_ = self._perplexity_precomp_distr( + X, doc_topics_distr, sub_sampling=False + ) + + return self + + def _unnormalized_transform(self, X): + """Transform data X according to fitted model. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + Returns + ------- + doc_topic_distr : ndarray of shape (n_samples, n_components) + Document topic distribution for X. + """ + doc_topic_distr, _ = self._e_step(X, cal_sstats=False, random_init=False) + + return doc_topic_distr + + def transform(self, X, *, normalize=True): + """Transform data X according to the fitted model. + + .. versionchanged:: 0.18 + `doc_topic_distr` is now normalized. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + normalize : bool, default=True + Whether to normalize the document topic distribution. + + Returns + ------- + doc_topic_distr : ndarray of shape (n_samples, n_components) + Document topic distribution for X. + """ + check_is_fitted(self) + X = self._check_non_neg_array( + X, reset_n_features=False, whom="LatentDirichletAllocation.transform" + ) + doc_topic_distr = self._unnormalized_transform(X) + if normalize: + doc_topic_distr /= doc_topic_distr.sum(axis=1)[:, np.newaxis] + return doc_topic_distr + + def fit_transform(self, X, y=None, *, normalize=True): + """ + Fit to data, then transform it. + + Fits transformer to `X` and `y` and returns a transformed version of `X`. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input samples. + + y : array-like of shape (n_samples,) or (n_samples, n_outputs), \ + default=None + Target values (None for unsupervised transformations). + + normalize : bool, default=True + Whether to normalize the document topic distribution in `transform`. + + Returns + ------- + X_new : ndarray array of shape (n_samples, n_components) + Transformed array. + """ + return self.fit(X, y).transform(X, normalize=normalize) + + def _approx_bound(self, X, doc_topic_distr, sub_sampling): + """Estimate the variational bound. + + Estimate the variational bound over "all documents" using only the + documents passed in as X. Since log-likelihood of each word cannot + be computed directly, we use this bound to estimate it. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + doc_topic_distr : ndarray of shape (n_samples, n_components) + Document topic distribution. In the literature, this is called + gamma. + + sub_sampling : bool, default=False + Compensate for subsampling of documents. + It is used in calculate bound in online learning. + + Returns + ------- + score : float + + """ + + def _loglikelihood(prior, distr, dirichlet_distr, size): + # calculate log-likelihood + score = np.sum((prior - distr) * dirichlet_distr) + score += np.sum(gammaln(distr) - gammaln(prior)) + score += np.sum(gammaln(prior * size) - gammaln(np.sum(distr, 1))) + return score + + is_sparse_x = sp.issparse(X) + n_samples, n_components = doc_topic_distr.shape + n_features = self.components_.shape[1] + score = 0 + + dirichlet_doc_topic = _dirichlet_expectation_2d(doc_topic_distr) + dirichlet_component_ = _dirichlet_expectation_2d(self.components_) + doc_topic_prior = self.doc_topic_prior_ + topic_word_prior = self.topic_word_prior_ + + if is_sparse_x: + X_data = X.data + X_indices = X.indices + X_indptr = X.indptr + + # E[log p(docs | theta, beta)] + for idx_d in range(0, n_samples): + if is_sparse_x: + ids = X_indices[X_indptr[idx_d] : X_indptr[idx_d + 1]] + cnts = X_data[X_indptr[idx_d] : X_indptr[idx_d + 1]] + else: + ids = np.nonzero(X[idx_d, :])[0] + cnts = X[idx_d, ids] + temp = ( + dirichlet_doc_topic[idx_d, :, np.newaxis] + dirichlet_component_[:, ids] + ) + norm_phi = logsumexp(temp, axis=0) + score += np.dot(cnts, norm_phi) + + # compute E[log p(theta | alpha) - log q(theta | gamma)] + score += _loglikelihood( + doc_topic_prior, doc_topic_distr, dirichlet_doc_topic, self.n_components + ) + + # Compensate for the subsampling of the population of documents + if sub_sampling: + doc_ratio = float(self.total_samples) / n_samples + score *= doc_ratio + + # E[log p(beta | eta) - log q (beta | lambda)] + score += _loglikelihood( + topic_word_prior, self.components_, dirichlet_component_, n_features + ) + + return score + + def score(self, X, y=None): + """Calculate approximate log-likelihood as score. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + y : Ignored + Not used, present here for API consistency by convention. + + Returns + ------- + score : float + Use approximate bound as score. + """ + check_is_fitted(self) + X = self._check_non_neg_array( + X, reset_n_features=False, whom="LatentDirichletAllocation.score" + ) + + doc_topic_distr = self._unnormalized_transform(X) + score = self._approx_bound(X, doc_topic_distr, sub_sampling=False) + return score + + def _perplexity_precomp_distr(self, X, doc_topic_distr=None, sub_sampling=False): + """Calculate approximate perplexity for data X with ability to accept + precomputed doc_topic_distr + + Perplexity is defined as exp(-1. * log-likelihood per word) + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + doc_topic_distr : ndarray of shape (n_samples, n_components), \ + default=None + Document topic distribution. + If it is None, it will be generated by applying transform on X. + + Returns + ------- + score : float + Perplexity score. + """ + if doc_topic_distr is None: + doc_topic_distr = self._unnormalized_transform(X) + else: + n_samples, n_components = doc_topic_distr.shape + if n_samples != X.shape[0]: + raise ValueError( + "Number of samples in X and doc_topic_distr do not match." + ) + + if n_components != self.n_components: + raise ValueError("Number of topics does not match.") + + current_samples = X.shape[0] + bound = self._approx_bound(X, doc_topic_distr, sub_sampling) + + if sub_sampling: + word_cnt = X.sum() * (float(self.total_samples) / current_samples) + else: + word_cnt = X.sum() + perword_bound = bound / word_cnt + + return np.exp(-1.0 * perword_bound) + + def perplexity(self, X, sub_sampling=False): + """Calculate approximate perplexity for data X. + + Perplexity is defined as exp(-1. * log-likelihood per word) + + .. versionchanged:: 0.19 + *doc_topic_distr* argument has been deprecated and is ignored + because user no longer has access to unnormalized distribution + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document word matrix. + + sub_sampling : bool + Do sub-sampling or not. + + Returns + ------- + score : float + Perplexity score. + """ + check_is_fitted(self) + X = self._check_non_neg_array( + X, reset_n_features=True, whom="LatentDirichletAllocation.perplexity" + ) + return self._perplexity_precomp_distr(X, sub_sampling=sub_sampling) + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_nmf.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_nmf.py new file mode 100644 index 0000000000000000000000000000000000000000..4e4380c9091ce643f906c8195250b25e36cde226 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_nmf.py @@ -0,0 +1,2401 @@ +"""Non-negative matrix factorization.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools +import time +import warnings +from abc import ABC +from math import sqrt +from numbers import Integral, Real + +import numpy as np +import scipy.sparse as sp +from scipy import linalg + +from sklearn._config import config_context +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.decomposition._cdnmf_fast import _update_cdnmf_fast +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils import check_array, check_random_state, gen_batches +from sklearn.utils._param_validation import Interval, StrOptions, validate_params +from sklearn.utils.extmath import _randomized_svd, safe_sparse_dot, squared_norm +from sklearn.utils.validation import check_is_fitted, check_non_negative, validate_data + +EPSILON = np.finfo(np.float32).eps + + +def norm(x): + """Dot product-based Euclidean norm implementation. + + See: http://fa.bianp.net/blog/2011/computing-the-vector-norm/ + + Parameters + ---------- + x : array-like + Vector for which to compute the norm. + """ + return sqrt(squared_norm(x)) + + +def trace_dot(X, Y): + """Trace of np.dot(X, Y.T). + + Parameters + ---------- + X : array-like + First matrix. + Y : array-like + Second matrix. + """ + return np.dot(X.ravel(), Y.ravel()) + + +def _check_init(A, shape, whom): + A = check_array(A) + if shape[0] != "auto" and A.shape[0] != shape[0]: + raise ValueError( + f"Array with wrong first dimension passed to {whom}. Expected {shape[0]}, " + f"but got {A.shape[0]}." + ) + if shape[1] != "auto" and A.shape[1] != shape[1]: + raise ValueError( + f"Array with wrong second dimension passed to {whom}. Expected {shape[1]}, " + f"but got {A.shape[1]}." + ) + check_non_negative(A, whom) + if np.max(A) == 0: + raise ValueError(f"Array passed to {whom} is full of zeros.") + + +def _beta_divergence(X, W, H, beta, square_root=False): + """Compute the beta-divergence of X and dot(W, H). + + Parameters + ---------- + X : float or array-like of shape (n_samples, n_features) + + W : float or array-like of shape (n_samples, n_components) + + H : float or array-like of shape (n_components, n_features) + + beta : float or {'frobenius', 'kullback-leibler', 'itakura-saito'} + Parameter of the beta-divergence. + If beta == 2, this is half the Frobenius *squared* norm. + If beta == 1, this is the generalized Kullback-Leibler divergence. + If beta == 0, this is the Itakura-Saito divergence. + Else, this is the general beta-divergence. + + square_root : bool, default=False + If True, return np.sqrt(2 * res) + For beta == 2, it corresponds to the Frobenius norm. + + Returns + ------- + res : float + Beta divergence of X and np.dot(X, H). + """ + beta = _beta_loss_to_float(beta) + + # The method can be called with scalars + if not sp.issparse(X): + X = np.atleast_2d(X) + W = np.atleast_2d(W) + H = np.atleast_2d(H) + + # Frobenius norm + if beta == 2: + # Avoid the creation of the dense np.dot(W, H) if X is sparse. + if sp.issparse(X): + norm_X = np.dot(X.data, X.data) + norm_WH = trace_dot(np.linalg.multi_dot([W.T, W, H]), H) + cross_prod = trace_dot((X @ H.T), W) + res = (norm_X + norm_WH - 2.0 * cross_prod) / 2.0 + else: + res = squared_norm(X - np.dot(W, H)) / 2.0 + + if square_root: + return np.sqrt(res * 2) + else: + return res + + if sp.issparse(X): + # compute np.dot(W, H) only where X is nonzero + WH_data = _special_sparse_dot(W, H, X).data + X_data = X.data + else: + WH = np.dot(W, H) + WH_data = WH.ravel() + X_data = X.ravel() + + # do not affect the zeros: here 0 ** (-1) = 0 and not infinity + indices = X_data > EPSILON + WH_data = WH_data[indices] + X_data = X_data[indices] + + # used to avoid division by zero + WH_data[WH_data < EPSILON] = EPSILON + + # generalized Kullback-Leibler divergence + if beta == 1: + # fast and memory efficient computation of np.sum(np.dot(W, H)) + sum_WH = np.dot(np.sum(W, axis=0), np.sum(H, axis=1)) + # computes np.sum(X * log(X / WH)) only where X is nonzero + div = X_data / WH_data + res = np.dot(X_data, np.log(div)) + # add full np.sum(np.dot(W, H)) - np.sum(X) + res += sum_WH - X_data.sum() + + # Itakura-Saito divergence + elif beta == 0: + div = X_data / WH_data + res = np.sum(div) - np.prod(X.shape) - np.sum(np.log(div)) + + # beta-divergence, beta not in (0, 1, 2) + else: + if sp.issparse(X): + # slow loop, but memory efficient computation of : + # np.sum(np.dot(W, H) ** beta) + sum_WH_beta = 0 + for i in range(X.shape[1]): + sum_WH_beta += np.sum(np.dot(W, H[:, i]) ** beta) + + else: + sum_WH_beta = np.sum(WH**beta) + + sum_X_WH = np.dot(X_data, WH_data ** (beta - 1)) + res = (X_data**beta).sum() - beta * sum_X_WH + res += sum_WH_beta * (beta - 1) + res /= beta * (beta - 1) + + if square_root: + res = max(res, 0) # avoid negative number due to rounding errors + return np.sqrt(2 * res) + else: + return res + + +def _special_sparse_dot(W, H, X): + """Computes np.dot(W, H), only where X is non zero.""" + if sp.issparse(X): + ii, jj = X.nonzero() + n_vals = ii.shape[0] + dot_vals = np.empty(n_vals) + n_components = W.shape[1] + + batch_size = max(n_components, n_vals // n_components) + for start in range(0, n_vals, batch_size): + batch = slice(start, start + batch_size) + dot_vals[batch] = np.multiply(W[ii[batch], :], H.T[jj[batch], :]).sum( + axis=1 + ) + + WH = sp.coo_matrix((dot_vals, (ii, jj)), shape=X.shape) + return WH.tocsr() + else: + return np.dot(W, H) + + +def _beta_loss_to_float(beta_loss): + """Convert string beta_loss to float.""" + beta_loss_map = {"frobenius": 2, "kullback-leibler": 1, "itakura-saito": 0} + if isinstance(beta_loss, str): + beta_loss = beta_loss_map[beta_loss] + return beta_loss + + +def _initialize_nmf(X, n_components, init=None, eps=1e-6, random_state=None): + """Algorithms for NMF initialization. + + Computes an initial guess for the non-negative + rank k matrix approximation for X: X = WH. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data matrix to be decomposed. + + n_components : int + The number of components desired in the approximation. + + init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar'}, default=None + Method used to initialize the procedure. + Valid options: + + - None: 'nndsvda' if n_components <= min(n_samples, n_features), + otherwise 'random'. + + - 'random': non-negative random matrices, scaled with: + sqrt(X.mean() / n_components) + + - 'nndsvd': Nonnegative Double Singular Value Decomposition (NNDSVD) + initialization (better for sparseness) + + - 'nndsvda': NNDSVD with zeros filled with the average of X + (better when sparsity is not desired) + + - 'nndsvdar': NNDSVD with zeros filled with small random values + (generally faster, less accurate alternative to NNDSVDa + for when sparsity is not desired) + + - 'custom': use custom matrices W and H + + .. versionchanged:: 1.1 + When `init=None` and n_components is less than n_samples and n_features + defaults to `nndsvda` instead of `nndsvd`. + + eps : float, default=1e-6 + Truncate all values less then this in output to zero. + + random_state : int, RandomState instance or None, default=None + Used when ``init`` == 'nndsvdar' or 'random'. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. + + Returns + ------- + W : array-like of shape (n_samples, n_components) + Initial guesses for solving X ~= WH. + + H : array-like of shape (n_components, n_features) + Initial guesses for solving X ~= WH. + + References + ---------- + C. Boutsidis, E. Gallopoulos: SVD based initialization: A head start for + nonnegative matrix factorization - Pattern Recognition, 2008 + http://tinyurl.com/nndsvd + """ + check_non_negative(X, "NMF initialization") + n_samples, n_features = X.shape + + if ( + init is not None + and init != "random" + and n_components > min(n_samples, n_features) + ): + raise ValueError( + "init = '{}' can only be used when " + "n_components <= min(n_samples, n_features)".format(init) + ) + + if init is None: + if n_components <= min(n_samples, n_features): + init = "nndsvda" + else: + init = "random" + + # Random initialization + if init == "random": + avg = np.sqrt(X.mean() / n_components) + rng = check_random_state(random_state) + H = avg * rng.standard_normal(size=(n_components, n_features)).astype( + X.dtype, copy=False + ) + W = avg * rng.standard_normal(size=(n_samples, n_components)).astype( + X.dtype, copy=False + ) + np.abs(H, out=H) + np.abs(W, out=W) + return W, H + + # NNDSVD initialization + U, S, V = _randomized_svd(X, n_components, random_state=random_state) + W = np.zeros_like(U) + H = np.zeros_like(V) + + # The leading singular triplet is non-negative + # so it can be used as is for initialization. + W[:, 0] = np.sqrt(S[0]) * np.abs(U[:, 0]) + H[0, :] = np.sqrt(S[0]) * np.abs(V[0, :]) + + for j in range(1, n_components): + x, y = U[:, j], V[j, :] + + # extract positive and negative parts of column vectors + x_p, y_p = np.maximum(x, 0), np.maximum(y, 0) + x_n, y_n = np.abs(np.minimum(x, 0)), np.abs(np.minimum(y, 0)) + + # and their norms + x_p_nrm, y_p_nrm = norm(x_p), norm(y_p) + x_n_nrm, y_n_nrm = norm(x_n), norm(y_n) + + m_p, m_n = x_p_nrm * y_p_nrm, x_n_nrm * y_n_nrm + + # choose update + if m_p > m_n: + u = x_p / x_p_nrm + v = y_p / y_p_nrm + sigma = m_p + else: + u = x_n / x_n_nrm + v = y_n / y_n_nrm + sigma = m_n + + lbd = np.sqrt(S[j] * sigma) + W[:, j] = lbd * u + H[j, :] = lbd * v + + W[W < eps] = 0 + H[H < eps] = 0 + + if init == "nndsvd": + pass + elif init == "nndsvda": + avg = X.mean() + W[W == 0] = avg + H[H == 0] = avg + elif init == "nndsvdar": + rng = check_random_state(random_state) + avg = X.mean() + W[W == 0] = abs(avg * rng.standard_normal(size=len(W[W == 0])) / 100) + H[H == 0] = abs(avg * rng.standard_normal(size=len(H[H == 0])) / 100) + else: + raise ValueError( + "Invalid init parameter: got %r instead of one of %r" + % (init, (None, "random", "nndsvd", "nndsvda", "nndsvdar")) + ) + + return W, H + + +def _update_coordinate_descent(X, W, Ht, l1_reg, l2_reg, shuffle, random_state): + """Helper function for _fit_coordinate_descent. + + Update W to minimize the objective function, iterating once over all + coordinates. By symmetry, to update H, one can call + _update_coordinate_descent(X.T, Ht, W, ...). + + """ + n_components = Ht.shape[1] + + HHt = np.dot(Ht.T, Ht) + XHt = safe_sparse_dot(X, Ht) + + # L2 regularization corresponds to increase of the diagonal of HHt + if l2_reg != 0.0: + # adds l2_reg only on the diagonal + HHt.flat[:: n_components + 1] += l2_reg + # L1 regularization corresponds to decrease of each element of XHt + if l1_reg != 0.0: + XHt -= l1_reg + + if shuffle: + permutation = random_state.permutation(n_components) + else: + permutation = np.arange(n_components) + # The following seems to be required on 64-bit Windows w/ Python 3.5. + permutation = np.asarray(permutation, dtype=np.intp) + return _update_cdnmf_fast(W, HHt, XHt, permutation) + + +def _fit_coordinate_descent( + X, + W, + H, + tol=1e-4, + max_iter=200, + l1_reg_W=0, + l1_reg_H=0, + l2_reg_W=0, + l2_reg_H=0, + update_H=True, + verbose=0, + shuffle=False, + random_state=None, +): + """Compute Non-negative Matrix Factorization (NMF) with Coordinate Descent + + The objective function is minimized with an alternating minimization of W + and H. Each minimization is done with a cyclic (up to a permutation of the + features) Coordinate Descent. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Constant matrix. + + W : array-like of shape (n_samples, n_components) + Initial guess for the solution. + + H : array-like of shape (n_components, n_features) + Initial guess for the solution. + + tol : float, default=1e-4 + Tolerance of the stopping condition. + + max_iter : int, default=200 + Maximum number of iterations before timing out. + + l1_reg_W : float, default=0. + L1 regularization parameter for W. + + l1_reg_H : float, default=0. + L1 regularization parameter for H. + + l2_reg_W : float, default=0. + L2 regularization parameter for W. + + l2_reg_H : float, default=0. + L2 regularization parameter for H. + + update_H : bool, default=True + Set to True, both W and H will be estimated from initial guesses. + Set to False, only W will be estimated. + + verbose : int, default=0 + The verbosity level. + + shuffle : bool, default=False + If true, randomize the order of coordinates in the CD solver. + + random_state : int, RandomState instance or None, default=None + Used to randomize the coordinates in the CD solver, when + ``shuffle`` is set to ``True``. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Solution to the non-negative least squares problem. + + H : ndarray of shape (n_components, n_features) + Solution to the non-negative least squares problem. + + n_iter : int + The number of iterations done by the algorithm. + + References + ---------- + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. + """ + # so W and Ht are both in C order in memory + Ht = check_array(H.T, order="C") + X = check_array(X, accept_sparse="csr") + + rng = check_random_state(random_state) + + for n_iter in range(1, max_iter + 1): + violation = 0.0 + + # Update W + violation += _update_coordinate_descent( + X, W, Ht, l1_reg_W, l2_reg_W, shuffle, rng + ) + # Update H + if update_H: + violation += _update_coordinate_descent( + X.T, Ht, W, l1_reg_H, l2_reg_H, shuffle, rng + ) + + if n_iter == 1: + violation_init = violation + + if violation_init == 0: + break + + if verbose: + print("violation:", violation / violation_init) + + if violation / violation_init <= tol: + if verbose: + print("Converged at iteration", n_iter + 1) + break + + return W, Ht.T, n_iter + + +def _multiplicative_update_w( + X, + W, + H, + beta_loss, + l1_reg_W, + l2_reg_W, + gamma, + H_sum=None, + HHt=None, + XHt=None, + update_H=True, +): + """Update W in Multiplicative Update NMF.""" + if beta_loss == 2: + # Numerator + if XHt is None: + XHt = safe_sparse_dot(X, H.T) + if update_H: + # avoid a copy of XHt, which will be re-computed (update_H=True) + numerator = XHt + else: + # preserve the XHt, which is not re-computed (update_H=False) + numerator = XHt.copy() + + # Denominator + if HHt is None: + HHt = np.dot(H, H.T) + denominator = np.dot(W, HHt) + + else: + # Numerator + # if X is sparse, compute WH only where X is non zero + WH_safe_X = _special_sparse_dot(W, H, X) + if sp.issparse(X): + WH_safe_X_data = WH_safe_X.data + X_data = X.data + else: + WH_safe_X_data = WH_safe_X + X_data = X + # copy used in the Denominator + WH = WH_safe_X.copy() + if beta_loss - 1.0 < 0: + WH[WH < EPSILON] = EPSILON + + # to avoid taking a negative power of zero + if beta_loss - 2.0 < 0: + WH_safe_X_data[WH_safe_X_data < EPSILON] = EPSILON + + if beta_loss == 1: + np.divide(X_data, WH_safe_X_data, out=WH_safe_X_data) + elif beta_loss == 0: + # speeds up computation time + # refer to /numpy/numpy/issues/9363 + WH_safe_X_data **= -1 + WH_safe_X_data **= 2 + # element-wise multiplication + WH_safe_X_data *= X_data + else: + WH_safe_X_data **= beta_loss - 2 + # element-wise multiplication + WH_safe_X_data *= X_data + + # here numerator = dot(X * (dot(W, H) ** (beta_loss - 2)), H.T) + numerator = safe_sparse_dot(WH_safe_X, H.T) + + # Denominator + if beta_loss == 1: + if H_sum is None: + H_sum = np.sum(H, axis=1) # shape(n_components, ) + denominator = H_sum[np.newaxis, :] + + else: + # computation of WHHt = dot(dot(W, H) ** beta_loss - 1, H.T) + if sp.issparse(X): + # memory efficient computation + # (compute row by row, avoiding the dense matrix WH) + WHHt = np.empty(W.shape) + for i in range(X.shape[0]): + WHi = np.dot(W[i, :], H) + if beta_loss - 1 < 0: + WHi[WHi < EPSILON] = EPSILON + WHi **= beta_loss - 1 + WHHt[i, :] = np.dot(WHi, H.T) + else: + WH **= beta_loss - 1 + WHHt = np.dot(WH, H.T) + denominator = WHHt + + # Add L1 and L2 regularization + if l1_reg_W > 0: + denominator += l1_reg_W + if l2_reg_W > 0: + denominator = denominator + l2_reg_W * W + denominator[denominator == 0] = EPSILON + + numerator /= denominator + delta_W = numerator + + # gamma is in ]0, 1] + if gamma != 1: + delta_W **= gamma + + W *= delta_W + + return W, H_sum, HHt, XHt + + +def _multiplicative_update_h( + X, W, H, beta_loss, l1_reg_H, l2_reg_H, gamma, A=None, B=None, rho=None +): + """update H in Multiplicative Update NMF.""" + if beta_loss == 2: + numerator = safe_sparse_dot(W.T, X) + denominator = np.linalg.multi_dot([W.T, W, H]) + + else: + # Numerator + WH_safe_X = _special_sparse_dot(W, H, X) + if sp.issparse(X): + WH_safe_X_data = WH_safe_X.data + X_data = X.data + else: + WH_safe_X_data = WH_safe_X + X_data = X + # copy used in the Denominator + WH = WH_safe_X.copy() + if beta_loss - 1.0 < 0: + WH[WH < EPSILON] = EPSILON + + # to avoid division by zero + if beta_loss - 2.0 < 0: + WH_safe_X_data[WH_safe_X_data < EPSILON] = EPSILON + + if beta_loss == 1: + np.divide(X_data, WH_safe_X_data, out=WH_safe_X_data) + elif beta_loss == 0: + # speeds up computation time + # refer to /numpy/numpy/issues/9363 + WH_safe_X_data **= -1 + WH_safe_X_data **= 2 + # element-wise multiplication + WH_safe_X_data *= X_data + else: + WH_safe_X_data **= beta_loss - 2 + # element-wise multiplication + WH_safe_X_data *= X_data + + # here numerator = dot(W.T, (dot(W, H) ** (beta_loss - 2)) * X) + numerator = safe_sparse_dot(W.T, WH_safe_X) + + # Denominator + if beta_loss == 1: + W_sum = np.sum(W, axis=0) # shape(n_components, ) + W_sum[W_sum == 0] = 1.0 + denominator = W_sum[:, np.newaxis] + + # beta_loss not in (1, 2) + else: + # computation of WtWH = dot(W.T, dot(W, H) ** beta_loss - 1) + if sp.issparse(X): + # memory efficient computation + # (compute column by column, avoiding the dense matrix WH) + WtWH = np.empty(H.shape) + for i in range(X.shape[1]): + WHi = np.dot(W, H[:, i]) + if beta_loss - 1 < 0: + WHi[WHi < EPSILON] = EPSILON + WHi **= beta_loss - 1 + WtWH[:, i] = np.dot(W.T, WHi) + else: + WH **= beta_loss - 1 + WtWH = np.dot(W.T, WH) + denominator = WtWH + + # Add L1 and L2 regularization + if l1_reg_H > 0: + denominator += l1_reg_H + if l2_reg_H > 0: + denominator = denominator + l2_reg_H * H + denominator[denominator == 0] = EPSILON + + if A is not None and B is not None: + # Updates for the online nmf + if gamma != 1: + H **= 1 / gamma + numerator *= H + A *= rho + B *= rho + A += numerator + B += denominator + H = A / B + + if gamma != 1: + H **= gamma + else: + delta_H = numerator + delta_H /= denominator + if gamma != 1: + delta_H **= gamma + H *= delta_H + + return H + + +def _fit_multiplicative_update( + X, + W, + H, + beta_loss="frobenius", + max_iter=200, + tol=1e-4, + l1_reg_W=0, + l1_reg_H=0, + l2_reg_W=0, + l2_reg_H=0, + update_H=True, + verbose=0, +): + """Compute Non-negative Matrix Factorization with Multiplicative Update. + + The objective function is _beta_divergence(X, WH) and is minimized with an + alternating minimization of W and H. Each minimization is done with a + Multiplicative Update. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Constant input matrix. + + W : array-like of shape (n_samples, n_components) + Initial guess for the solution. + + H : array-like of shape (n_components, n_features) + Initial guess for the solution. + + beta_loss : float or {'frobenius', 'kullback-leibler', \ + 'itakura-saito'}, default='frobenius' + String must be in {'frobenius', 'kullback-leibler', 'itakura-saito'}. + Beta divergence to be minimized, measuring the distance between X + and the dot product WH. Note that values different from 'frobenius' + (or 2) and 'kullback-leibler' (or 1) lead to significantly slower + fits. Note that for beta_loss <= 0 (or 'itakura-saito'), the input + matrix X cannot contain zeros. + + max_iter : int, default=200 + Number of iterations. + + tol : float, default=1e-4 + Tolerance of the stopping condition. + + l1_reg_W : float, default=0. + L1 regularization parameter for W. + + l1_reg_H : float, default=0. + L1 regularization parameter for H. + + l2_reg_W : float, default=0. + L2 regularization parameter for W. + + l2_reg_H : float, default=0. + L2 regularization parameter for H. + + update_H : bool, default=True + Set to True, both W and H will be estimated from initial guesses. + Set to False, only W will be estimated. + + verbose : int, default=0 + The verbosity level. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Solution to the non-negative least squares problem. + + H : ndarray of shape (n_components, n_features) + Solution to the non-negative least squares problem. + + n_iter : int + The number of iterations done by the algorithm. + + References + ---------- + Lee, D. D., & Seung, H., S. (2001). Algorithms for Non-negative Matrix + Factorization. Adv. Neural Inform. Process. Syst.. 13. + Fevotte, C., & Idier, J. (2011). Algorithms for nonnegative matrix + factorization with the beta-divergence. Neural Computation, 23(9). + """ + start_time = time.time() + + beta_loss = _beta_loss_to_float(beta_loss) + + # gamma for Maximization-Minimization (MM) algorithm [Fevotte 2011] + if beta_loss < 1: + gamma = 1.0 / (2.0 - beta_loss) + elif beta_loss > 2: + gamma = 1.0 / (beta_loss - 1.0) + else: + gamma = 1.0 + + # used for the convergence criterion + error_at_init = _beta_divergence(X, W, H, beta_loss, square_root=True) + previous_error = error_at_init + + H_sum, HHt, XHt = None, None, None + for n_iter in range(1, max_iter + 1): + # update W + # H_sum, HHt and XHt are saved and reused if not update_H + W, H_sum, HHt, XHt = _multiplicative_update_w( + X, + W, + H, + beta_loss=beta_loss, + l1_reg_W=l1_reg_W, + l2_reg_W=l2_reg_W, + gamma=gamma, + H_sum=H_sum, + HHt=HHt, + XHt=XHt, + update_H=update_H, + ) + + # necessary for stability with beta_loss < 1 + if beta_loss < 1: + W[W < np.finfo(np.float64).eps] = 0.0 + + # update H (only at fit or fit_transform) + if update_H: + H = _multiplicative_update_h( + X, + W, + H, + beta_loss=beta_loss, + l1_reg_H=l1_reg_H, + l2_reg_H=l2_reg_H, + gamma=gamma, + ) + + # These values will be recomputed since H changed + H_sum, HHt, XHt = None, None, None + + # necessary for stability with beta_loss < 1 + if beta_loss <= 1: + H[H < np.finfo(np.float64).eps] = 0.0 + + # test convergence criterion every 10 iterations + if tol > 0 and n_iter % 10 == 0: + error = _beta_divergence(X, W, H, beta_loss, square_root=True) + + if verbose: + iter_time = time.time() + print( + "Epoch %02d reached after %.3f seconds, error: %f" + % (n_iter, iter_time - start_time, error) + ) + + if (previous_error - error) / error_at_init < tol: + break + previous_error = error + + # do not print if we have already printed in the convergence test + if verbose and (tol == 0 or n_iter % 10 != 0): + end_time = time.time() + print( + "Epoch %02d reached after %.3f seconds." % (n_iter, end_time - start_time) + ) + + return W, H, n_iter + + +@validate_params( + { + "X": ["array-like", "sparse matrix"], + "W": ["array-like", None], + "H": ["array-like", None], + "update_H": ["boolean"], + }, + prefer_skip_nested_validation=False, +) +def non_negative_factorization( + X, + W=None, + H=None, + n_components="auto", + *, + init=None, + update_H=True, + solver="cd", + beta_loss="frobenius", + tol=1e-4, + max_iter=200, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + random_state=None, + verbose=0, + shuffle=False, +): + """Compute Non-negative Matrix Factorization (NMF). + + Find two non-negative matrices (W, H) whose product approximates the non- + negative matrix X. This factorization can be used for example for + dimensionality reduction, source separation or topic extraction. + + The objective function is: + + .. math:: + + L(W, H) &= 0.5 * ||X - WH||_{loss}^2 + + &+ alpha\\_W * l1\\_ratio * n\\_features * ||vec(W)||_1 + + &+ alpha\\_H * l1\\_ratio * n\\_samples * ||vec(H)||_1 + + &+ 0.5 * alpha\\_W * (1 - l1\\_ratio) * n\\_features * ||W||_{Fro}^2 + + &+ 0.5 * alpha\\_H * (1 - l1\\_ratio) * n\\_samples * ||H||_{Fro}^2, + + where :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) and + :math:`||vec(A)||_1 = \\sum_{i,j} abs(A_{ij})` (Elementwise L1 norm) + + The generic norm :math:`||X - WH||_{loss}^2` may represent + the Frobenius norm or another supported beta-divergence loss. + The choice between options is controlled by the `beta_loss` parameter. + + The regularization terms are scaled by `n_features` for `W` and by `n_samples` for + `H` to keep their impact balanced with respect to one another and to the data fit + term as independent as possible of the size `n_samples` of the training set. + + The objective function is minimized with an alternating minimization of W + and H. If H is given and update_H=False, it solves for W only. + + Note that the transformed data is named W and the components matrix is named H. In + the NMF literature, the naming convention is usually the opposite since the data + matrix X is transposed. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Constant matrix. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is initialised as an array of zeros, unless + `solver='mu'`, then it is filled with values calculated by + `np.sqrt(X.mean() / self._n_components)`. + If `None`, uses the initialisation method specified in `init`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is used as a constant, to solve for W only. + If `None`, uses the initialisation method specified in `init`. + + n_components : int or {'auto'} or None, default='auto' + Number of components. If `None`, all features are kept. + If `n_components='auto'`, the number of components is automatically inferred + from `W` or `H` shapes. + + .. versionchanged:: 1.4 + Added `'auto'` value. + + .. versionchanged:: 1.6 + Default value changed from `None` to `'auto'`. + + init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar', 'custom'}, default=None + Method used to initialize the procedure. + + Valid options: + + - None: 'nndsvda' if n_components < n_features, otherwise 'random'. + - 'random': non-negative random matrices, scaled with: + `sqrt(X.mean() / n_components)` + - 'nndsvd': Nonnegative Double Singular Value Decomposition (NNDSVD) + initialization (better for sparseness) + - 'nndsvda': NNDSVD with zeros filled with the average of X + (better when sparsity is not desired) + - 'nndsvdar': NNDSVD with zeros filled with small random values + (generally faster, less accurate alternative to NNDSVDa + for when sparsity is not desired) + - 'custom': If `update_H=True`, use custom matrices W and H which must both + be provided. If `update_H=False`, then only custom matrix H is used. + + .. versionchanged:: 0.23 + The default value of `init` changed from 'random' to None in 0.23. + + .. versionchanged:: 1.1 + When `init=None` and n_components is less than n_samples and n_features + defaults to `nndsvda` instead of `nndsvd`. + + update_H : bool, default=True + Set to True, both W and H will be estimated from initial guesses. + Set to False, only W will be estimated. + + solver : {'cd', 'mu'}, default='cd' + Numerical solver to use: + + - 'cd' is a Coordinate Descent solver that uses Fast Hierarchical + Alternating Least Squares (Fast HALS). + - 'mu' is a Multiplicative Update solver. + + .. versionadded:: 0.17 + Coordinate Descent solver. + + .. versionadded:: 0.19 + Multiplicative Update solver. + + beta_loss : float or {'frobenius', 'kullback-leibler', \ + 'itakura-saito'}, default='frobenius' + Beta divergence to be minimized, measuring the distance between X + and the dot product WH. Note that values different from 'frobenius' + (or 2) and 'kullback-leibler' (or 1) lead to significantly slower + fits. Note that for beta_loss <= 0 (or 'itakura-saito'), the input + matrix X cannot contain zeros. Used only in 'mu' solver. + + .. versionadded:: 0.19 + + tol : float, default=1e-4 + Tolerance of the stopping condition. + + max_iter : int, default=200 + Maximum number of iterations before timing out. + + alpha_W : float, default=0.0 + Constant that multiplies the regularization terms of `W`. Set it to zero + (default) to have no regularization on `W`. + + .. versionadded:: 1.0 + + alpha_H : float or "same", default="same" + Constant that multiplies the regularization terms of `H`. Set it to zero to + have no regularization on `H`. If "same" (default), it takes the same value as + `alpha_W`. + + .. versionadded:: 1.0 + + l1_ratio : float, default=0.0 + The regularization mixing parameter, with 0 <= l1_ratio <= 1. + For l1_ratio = 0 the penalty is an elementwise L2 penalty + (aka Frobenius Norm). + For l1_ratio = 1 it is an elementwise L1 penalty. + For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2. + + random_state : int, RandomState instance or None, default=None + Used for NMF initialisation (when ``init`` == 'nndsvdar' or + 'random'), and in Coordinate Descent. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + verbose : int, default=0 + The verbosity level. + + shuffle : bool, default=False + If true, randomize the order of coordinates in the CD solver. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Solution to the non-negative least squares problem. + + H : ndarray of shape (n_components, n_features) + Solution to the non-negative least squares problem. + + n_iter : int + Actual number of iterations. + + References + ---------- + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. + + .. [2] :doi:`"Algorithms for nonnegative matrix factorization with the + beta-divergence" <10.1162/NECO_a_00168>` + Fevotte, C., & Idier, J. (2011). Neural Computation, 23(9). + + Examples + -------- + >>> import numpy as np + >>> X = np.array([[1,1], [2, 1], [3, 1.2], [4, 1], [5, 0.8], [6, 1]]) + >>> from sklearn.decomposition import non_negative_factorization + >>> W, H, n_iter = non_negative_factorization( + ... X, n_components=2, init='random', random_state=0) + """ + est = NMF( + n_components=n_components, + init=init, + solver=solver, + beta_loss=beta_loss, + tol=tol, + max_iter=max_iter, + random_state=random_state, + alpha_W=alpha_W, + alpha_H=alpha_H, + l1_ratio=l1_ratio, + verbose=verbose, + shuffle=shuffle, + ) + est._validate_params() + + X = check_array(X, accept_sparse=("csr", "csc"), dtype=[np.float64, np.float32]) + + with config_context(assume_finite=True): + W, H, n_iter = est._fit_transform(X, W=W, H=H, update_H=update_H) + + return W, H, n_iter + + +class _BaseNMF(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator, ABC): + """Base class for NMF and MiniBatchNMF.""" + + _parameter_constraints: dict = { + "n_components": [ + Interval(Integral, 1, None, closed="left"), + None, + StrOptions({"auto"}), + ], + "init": [ + StrOptions({"random", "nndsvd", "nndsvda", "nndsvdar", "custom"}), + None, + ], + "beta_loss": [ + StrOptions({"frobenius", "kullback-leibler", "itakura-saito"}), + Real, + ], + "tol": [Interval(Real, 0, None, closed="left")], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "random_state": ["random_state"], + "alpha_W": [Interval(Real, 0, None, closed="left")], + "alpha_H": [Interval(Real, 0, None, closed="left"), StrOptions({"same"})], + "l1_ratio": [Interval(Real, 0, 1, closed="both")], + "verbose": ["verbose"], + } + + def __init__( + self, + n_components="auto", + *, + init=None, + beta_loss="frobenius", + tol=1e-4, + max_iter=200, + random_state=None, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + verbose=0, + ): + self.n_components = n_components + self.init = init + self.beta_loss = beta_loss + self.tol = tol + self.max_iter = max_iter + self.random_state = random_state + self.alpha_W = alpha_W + self.alpha_H = alpha_H + self.l1_ratio = l1_ratio + self.verbose = verbose + + def _check_params(self, X): + # n_components + self._n_components = self.n_components + if self._n_components is None: + self._n_components = X.shape[1] + + # beta_loss + self._beta_loss = _beta_loss_to_float(self.beta_loss) + + def _check_w_h(self, X, W, H, update_H): + """Check W and H, or initialize them.""" + n_samples, n_features = X.shape + + if self.init == "custom" and update_H: + _check_init(H, (self._n_components, n_features), "NMF (input H)") + _check_init(W, (n_samples, self._n_components), "NMF (input W)") + if self._n_components == "auto": + self._n_components = H.shape[0] + + if H.dtype != X.dtype or W.dtype != X.dtype: + raise TypeError( + "H and W should have the same dtype as X. Got " + "H.dtype = {} and W.dtype = {}.".format(H.dtype, W.dtype) + ) + + elif not update_H: + if W is not None: + warnings.warn( + "When update_H=False, the provided initial W is not used.", + RuntimeWarning, + ) + + _check_init(H, (self._n_components, n_features), "NMF (input H)") + if self._n_components == "auto": + self._n_components = H.shape[0] + + if H.dtype != X.dtype: + raise TypeError( + "H should have the same dtype as X. Got H.dtype = {}.".format( + H.dtype + ) + ) + + # 'mu' solver should not be initialized by zeros + if self.solver == "mu": + avg = np.sqrt(X.mean() / self._n_components) + W = np.full((n_samples, self._n_components), avg, dtype=X.dtype) + else: + W = np.zeros((n_samples, self._n_components), dtype=X.dtype) + + else: + if W is not None or H is not None: + warnings.warn( + ( + "When init!='custom', provided W or H are ignored. Set " + " init='custom' to use them as initialization." + ), + RuntimeWarning, + ) + + if self._n_components == "auto": + self._n_components = X.shape[1] + + W, H = _initialize_nmf( + X, self._n_components, init=self.init, random_state=self.random_state + ) + + return W, H + + def _compute_regularization(self, X): + """Compute scaled regularization terms.""" + n_samples, n_features = X.shape + alpha_W = self.alpha_W + alpha_H = self.alpha_W if self.alpha_H == "same" else self.alpha_H + + l1_reg_W = n_features * alpha_W * self.l1_ratio + l1_reg_H = n_samples * alpha_H * self.l1_ratio + l2_reg_W = n_features * alpha_W * (1.0 - self.l1_ratio) + l2_reg_H = n_samples * alpha_H * (1.0 - self.l1_ratio) + + return l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H + + def fit(self, X, y=None, **params): + """Learn a NMF model for the data X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + **params : kwargs + Parameters (keyword arguments) and values passed to + the fit_transform instance. + + Returns + ------- + self : object + Returns the instance itself. + """ + # param validation is done in fit_transform + + self.fit_transform(X, **params) + return self + + def inverse_transform(self, X): + """Transform data back to its original space. + + .. versionadded:: 0.18 + + Parameters + ---------- + X : {ndarray, sparse matrix} of shape (n_samples, n_components) + Transformed data matrix. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Returns a data matrix of the original shape. + """ + + check_is_fitted(self) + return X @ self.components_ + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.positive_only = True + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + +class NMF(_BaseNMF): + """Non-Negative Matrix Factorization (NMF). + + Find two non-negative matrices, i.e. matrices with all non-negative elements, (W, H) + whose product approximates the non-negative matrix X. This factorization can be used + for example for dimensionality reduction, source separation or topic extraction. + + The objective function is: + + .. math:: + + L(W, H) &= 0.5 * ||X - WH||_{loss}^2 + + &+ alpha\\_W * l1\\_ratio * n\\_features * ||vec(W)||_1 + + &+ alpha\\_H * l1\\_ratio * n\\_samples * ||vec(H)||_1 + + &+ 0.5 * alpha\\_W * (1 - l1\\_ratio) * n\\_features * ||W||_{Fro}^2 + + &+ 0.5 * alpha\\_H * (1 - l1\\_ratio) * n\\_samples * ||H||_{Fro}^2, + + where :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) and + :math:`||vec(A)||_1 = \\sum_{i,j} abs(A_{ij})` (Elementwise L1 norm). + + The generic norm :math:`||X - WH||_{loss}` may represent + the Frobenius norm or another supported beta-divergence loss. + The choice between options is controlled by the `beta_loss` parameter. + + The regularization terms are scaled by `n_features` for `W` and by `n_samples` for + `H` to keep their impact balanced with respect to one another and to the data fit + term as independent as possible of the size `n_samples` of the training set. + + The objective function is minimized with an alternating minimization of W + and H. + + Note that the transformed data is named W and the components matrix is named H. In + the NMF literature, the naming convention is usually the opposite since the data + matrix X is transposed. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int or {'auto'} or None, default='auto' + Number of components. If `None`, all features are kept. + If `n_components='auto'`, the number of components is automatically inferred + from W or H shapes. + + .. versionchanged:: 1.4 + Added `'auto'` value. + + .. versionchanged:: 1.6 + Default value changed from `None` to `'auto'`. + + init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar', 'custom'}, default=None + Method used to initialize the procedure. + Valid options: + + - `None`: 'nndsvda' if n_components <= min(n_samples, n_features), + otherwise random. + + - `'random'`: non-negative random matrices, scaled with: + `sqrt(X.mean() / n_components)` + + - `'nndsvd'`: Nonnegative Double Singular Value Decomposition (NNDSVD) + initialization (better for sparseness) + + - `'nndsvda'`: NNDSVD with zeros filled with the average of X + (better when sparsity is not desired) + + - `'nndsvdar'` NNDSVD with zeros filled with small random values + (generally faster, less accurate alternative to NNDSVDa + for when sparsity is not desired) + + - `'custom'`: Use custom matrices `W` and `H` which must both be provided. + + .. versionchanged:: 1.1 + When `init=None` and n_components is less than n_samples and n_features + defaults to `nndsvda` instead of `nndsvd`. + + solver : {'cd', 'mu'}, default='cd' + Numerical solver to use: + + - 'cd' is a Coordinate Descent solver. + - 'mu' is a Multiplicative Update solver. + + .. versionadded:: 0.17 + Coordinate Descent solver. + + .. versionadded:: 0.19 + Multiplicative Update solver. + + beta_loss : float or {'frobenius', 'kullback-leibler', \ + 'itakura-saito'}, default='frobenius' + Beta divergence to be minimized, measuring the distance between X + and the dot product WH. Note that values different from 'frobenius' + (or 2) and 'kullback-leibler' (or 1) lead to significantly slower + fits. Note that for beta_loss <= 0 (or 'itakura-saito'), the input + matrix X cannot contain zeros. Used only in 'mu' solver. + + .. versionadded:: 0.19 + + tol : float, default=1e-4 + Tolerance of the stopping condition. + + max_iter : int, default=200 + Maximum number of iterations before timing out. + + random_state : int, RandomState instance or None, default=None + Used for initialisation (when ``init`` == 'nndsvdar' or + 'random'), and in Coordinate Descent. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + alpha_W : float, default=0.0 + Constant that multiplies the regularization terms of `W`. Set it to zero + (default) to have no regularization on `W`. + + .. versionadded:: 1.0 + + alpha_H : float or "same", default="same" + Constant that multiplies the regularization terms of `H`. Set it to zero to + have no regularization on `H`. If "same" (default), it takes the same value as + `alpha_W`. + + .. versionadded:: 1.0 + + l1_ratio : float, default=0.0 + The regularization mixing parameter, with 0 <= l1_ratio <= 1. + For l1_ratio = 0 the penalty is an elementwise L2 penalty + (aka Frobenius Norm). + For l1_ratio = 1 it is an elementwise L1 penalty. + For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2. + + .. versionadded:: 0.17 + Regularization parameter *l1_ratio* used in the Coordinate Descent + solver. + + verbose : int, default=0 + Whether to be verbose. + + shuffle : bool, default=False + If true, randomize the order of coordinates in the CD solver. + + .. versionadded:: 0.17 + *shuffle* parameter used in the Coordinate Descent solver. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Factorization matrix, sometimes called 'dictionary'. + + n_components_ : int + The number of components. It is same as the `n_components` parameter + if it was given. Otherwise, it will be same as the number of + features. + + reconstruction_err_ : float + Frobenius norm of the matrix difference, or beta-divergence, between + the training data ``X`` and the reconstructed data ``WH`` from + the fitted model. + + n_iter_ : int + Actual number of iterations. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + PCA : Principal component analysis. + SparseCoder : Find a sparse representation of data from a fixed, + precomputed dictionary. + SparsePCA : Sparse Principal Components Analysis. + TruncatedSVD : Dimensionality reduction using truncated SVD. + + References + ---------- + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. + + .. [2] :doi:`"Algorithms for nonnegative matrix factorization with the + beta-divergence" <10.1162/NECO_a_00168>` + Fevotte, C., & Idier, J. (2011). Neural Computation, 23(9). + + Examples + -------- + >>> import numpy as np + >>> X = np.array([[1, 1], [2, 1], [3, 1.2], [4, 1], [5, 0.8], [6, 1]]) + >>> from sklearn.decomposition import NMF + >>> model = NMF(n_components=2, init='random', random_state=0) + >>> W = model.fit_transform(X) + >>> H = model.components_ + """ + + _parameter_constraints: dict = { + **_BaseNMF._parameter_constraints, + "solver": [StrOptions({"mu", "cd"})], + "shuffle": ["boolean"], + } + + def __init__( + self, + n_components="auto", + *, + init=None, + solver="cd", + beta_loss="frobenius", + tol=1e-4, + max_iter=200, + random_state=None, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + verbose=0, + shuffle=False, + ): + super().__init__( + n_components=n_components, + init=init, + beta_loss=beta_loss, + tol=tol, + max_iter=max_iter, + random_state=random_state, + alpha_W=alpha_W, + alpha_H=alpha_H, + l1_ratio=l1_ratio, + verbose=verbose, + ) + + self.solver = solver + self.shuffle = shuffle + + def _check_params(self, X): + super()._check_params(X) + + # solver + if self.solver != "mu" and self.beta_loss not in (2, "frobenius"): + # 'mu' is the only solver that handles other beta losses than 'frobenius' + raise ValueError( + f"Invalid beta_loss parameter: solver {self.solver!r} does not handle " + f"beta_loss = {self.beta_loss!r}" + ) + if self.solver == "mu" and self.init == "nndsvd": + warnings.warn( + ( + "The multiplicative update ('mu') solver cannot update " + "zeros present in the initialization, and so leads to " + "poorer results when used jointly with init='nndsvd'. " + "You may try init='nndsvda' or init='nndsvdar' instead." + ), + UserWarning, + ) + + return self + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None, W=None, H=None): + """Learn a NMF model for the data X and returns the transformed data. + + This is more efficient than calling fit followed by transform. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + """ + X = validate_data( + self, X, accept_sparse=("csr", "csc"), dtype=[np.float64, np.float32] + ) + + with config_context(assume_finite=True): + W, H, n_iter = self._fit_transform(X, W=W, H=H) + + self.reconstruction_err_ = _beta_divergence( + X, W, H, self._beta_loss, square_root=True + ) + + self.n_components_ = H.shape[0] + self.components_ = H + self.n_iter_ = n_iter + + return W + + def _fit_transform(self, X, y=None, W=None, H=None, update_H=True): + """Learn a NMF model for the data X and returns the transformed data. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Data matrix to be decomposed + + y : Ignored + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is initialised as an array of zeros, unless + `solver='mu'`, then it is filled with values calculated by + `np.sqrt(X.mean() / self._n_components)`. + If `None`, uses the initialisation method specified in `init`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is used as a constant, to solve for W only. + If `None`, uses the initialisation method specified in `init`. + + update_H : bool, default=True + If True, both W and H will be estimated from initial guesses, + this corresponds to a call to the 'fit_transform' method. + If False, only W will be estimated, this corresponds to a call + to the 'transform' method. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + + H : ndarray of shape (n_components, n_features) + Factorization matrix, sometimes called 'dictionary'. + + n_iter_ : int + Actual number of iterations. + """ + # check parameters + self._check_params(X) + + if X.min() == 0 and self._beta_loss <= 0: + raise ValueError( + "When beta_loss <= 0 and X contains zeros, " + "the solver may diverge. Please add small values " + "to X, or use a positive beta_loss." + ) + + # initialize or check W and H + W, H = self._check_w_h(X, W, H, update_H) + + # scale the regularization terms + l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H = self._compute_regularization(X) + + if self.solver == "cd": + W, H, n_iter = _fit_coordinate_descent( + X, + W, + H, + self.tol, + self.max_iter, + l1_reg_W, + l1_reg_H, + l2_reg_W, + l2_reg_H, + update_H=update_H, + verbose=self.verbose, + shuffle=self.shuffle, + random_state=self.random_state, + ) + elif self.solver == "mu": + W, H, n_iter, *_ = _fit_multiplicative_update( + X, + W, + H, + self._beta_loss, + self.max_iter, + self.tol, + l1_reg_W, + l1_reg_H, + l2_reg_W, + l2_reg_H, + update_H, + self.verbose, + ) + else: + raise ValueError("Invalid solver parameter '%s'." % self.solver) + + if n_iter == self.max_iter and self.tol > 0: + warnings.warn( + "Maximum number of iterations %d reached. Increase " + "it to improve convergence." % self.max_iter, + ConvergenceWarning, + ) + + return W, H, n_iter + + def transform(self, X): + """Transform the data X according to the fitted NMF model. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + """ + check_is_fitted(self) + X = validate_data( + self, + X, + accept_sparse=("csr", "csc"), + dtype=[np.float64, np.float32], + reset=False, + ensure_non_negative=True, + ) + + with config_context(assume_finite=True): + W, *_ = self._fit_transform(X, H=self.components_, update_H=False) + + return W + + +class MiniBatchNMF(_BaseNMF): + """Mini-Batch Non-Negative Matrix Factorization (NMF). + + .. versionadded:: 1.1 + + Find two non-negative matrices, i.e. matrices with all non-negative elements, + (`W`, `H`) whose product approximates the non-negative matrix `X`. This + factorization can be used for example for dimensionality reduction, source + separation or topic extraction. + + The objective function is: + + .. math:: + + L(W, H) &= 0.5 * ||X - WH||_{loss}^2 + + &+ alpha\\_W * l1\\_ratio * n\\_features * ||vec(W)||_1 + + &+ alpha\\_H * l1\\_ratio * n\\_samples * ||vec(H)||_1 + + &+ 0.5 * alpha\\_W * (1 - l1\\_ratio) * n\\_features * ||W||_{Fro}^2 + + &+ 0.5 * alpha\\_H * (1 - l1\\_ratio) * n\\_samples * ||H||_{Fro}^2, + + where :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) and + :math:`||vec(A)||_1 = \\sum_{i,j} abs(A_{ij})` (Elementwise L1 norm). + + The generic norm :math:`||X - WH||_{loss}^2` may represent + the Frobenius norm or another supported beta-divergence loss. + The choice between options is controlled by the `beta_loss` parameter. + + The objective function is minimized with an alternating minimization of `W` + and `H`. + + Note that the transformed data is named `W` and the components matrix is + named `H`. In the NMF literature, the naming convention is usually the opposite + since the data matrix `X` is transposed. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int or {'auto'} or None, default='auto' + Number of components. If `None`, all features are kept. + If `n_components='auto'`, the number of components is automatically inferred + from W or H shapes. + + .. versionchanged:: 1.4 + Added `'auto'` value. + + .. versionchanged:: 1.6 + Default value changed from `None` to `'auto'`. + + init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar', 'custom'}, default=None + Method used to initialize the procedure. + Valid options: + + - `None`: 'nndsvda' if `n_components <= min(n_samples, n_features)`, + otherwise random. + + - `'random'`: non-negative random matrices, scaled with: + `sqrt(X.mean() / n_components)` + + - `'nndsvd'`: Nonnegative Double Singular Value Decomposition (NNDSVD) + initialization (better for sparseness). + + - `'nndsvda'`: NNDSVD with zeros filled with the average of X + (better when sparsity is not desired). + + - `'nndsvdar'` NNDSVD with zeros filled with small random values + (generally faster, less accurate alternative to NNDSVDa + for when sparsity is not desired). + + - `'custom'`: Use custom matrices `W` and `H` which must both be provided. + + batch_size : int, default=1024 + Number of samples in each mini-batch. Large batch sizes + give better long-term convergence at the cost of a slower start. + + beta_loss : float or {'frobenius', 'kullback-leibler', \ + 'itakura-saito'}, default='frobenius' + Beta divergence to be minimized, measuring the distance between `X` + and the dot product `WH`. Note that values different from 'frobenius' + (or 2) and 'kullback-leibler' (or 1) lead to significantly slower + fits. Note that for `beta_loss <= 0` (or 'itakura-saito'), the input + matrix `X` cannot contain zeros. + + tol : float, default=1e-4 + Control early stopping based on the norm of the differences in `H` + between 2 steps. To disable early stopping based on changes in `H`, set + `tol` to 0.0. + + max_no_improvement : int, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + To disable convergence detection based on cost function, set + `max_no_improvement` to None. + + max_iter : int, default=200 + Maximum number of iterations over the complete dataset before + timing out. + + alpha_W : float, default=0.0 + Constant that multiplies the regularization terms of `W`. Set it to zero + (default) to have no regularization on `W`. + + alpha_H : float or "same", default="same" + Constant that multiplies the regularization terms of `H`. Set it to zero to + have no regularization on `H`. If "same" (default), it takes the same value as + `alpha_W`. + + l1_ratio : float, default=0.0 + The regularization mixing parameter, with 0 <= l1_ratio <= 1. + For l1_ratio = 0 the penalty is an elementwise L2 penalty + (aka Frobenius Norm). + For l1_ratio = 1 it is an elementwise L1 penalty. + For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2. + + forget_factor : float, default=0.7 + Amount of rescaling of past information. Its value could be 1 with + finite datasets. Choosing values < 1 is recommended with online + learning as more recent batches will weight more than past batches. + + fresh_restarts : bool, default=False + Whether to completely solve for W at each step. Doing fresh restarts will likely + lead to a better solution for a same number of iterations but it is much slower. + + fresh_restarts_max_iter : int, default=30 + Maximum number of iterations when solving for W at each step. Only used when + doing fresh restarts. These iterations may be stopped early based on a small + change of W controlled by `tol`. + + transform_max_iter : int, default=None + Maximum number of iterations when solving for W at transform time. + If None, it defaults to `max_iter`. + + random_state : int, RandomState instance or None, default=None + Used for initialisation (when ``init`` == 'nndsvdar' or + 'random'), and in Coordinate Descent. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + verbose : bool, default=False + Whether to be verbose. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Factorization matrix, sometimes called 'dictionary'. + + n_components_ : int + The number of components. It is same as the `n_components` parameter + if it was given. Otherwise, it will be same as the number of + features. + + reconstruction_err_ : float + Frobenius norm of the matrix difference, or beta-divergence, between + the training data `X` and the reconstructed data `WH` from + the fitted model. + + n_iter_ : int + Actual number of started iterations over the whole dataset. + + n_steps_ : int + Number of mini-batches processed. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + See Also + -------- + NMF : Non-negative matrix factorization. + MiniBatchDictionaryLearning : Finds a dictionary that can best be used to represent + data using a sparse code. + + References + ---------- + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. + + .. [2] :doi:`"Algorithms for nonnegative matrix factorization with the + beta-divergence" <10.1162/NECO_a_00168>` + Fevotte, C., & Idier, J. (2011). Neural Computation, 23(9). + + .. [3] :doi:`"Online algorithms for nonnegative matrix factorization with the + Itakura-Saito divergence" <10.1109/ASPAA.2011.6082314>` + Lefevre, A., Bach, F., Fevotte, C. (2011). WASPA. + + Examples + -------- + >>> import numpy as np + >>> X = np.array([[1, 1], [2, 1], [3, 1.2], [4, 1], [5, 0.8], [6, 1]]) + >>> from sklearn.decomposition import MiniBatchNMF + >>> model = MiniBatchNMF(n_components=2, init='random', random_state=0) + >>> W = model.fit_transform(X) + >>> H = model.components_ + """ + + _parameter_constraints: dict = { + **_BaseNMF._parameter_constraints, + "max_no_improvement": [Interval(Integral, 1, None, closed="left"), None], + "batch_size": [Interval(Integral, 1, None, closed="left")], + "forget_factor": [Interval(Real, 0, 1, closed="both")], + "fresh_restarts": ["boolean"], + "fresh_restarts_max_iter": [Interval(Integral, 1, None, closed="left")], + "transform_max_iter": [Interval(Integral, 1, None, closed="left"), None], + } + + def __init__( + self, + n_components="auto", + *, + init=None, + batch_size=1024, + beta_loss="frobenius", + tol=1e-4, + max_no_improvement=10, + max_iter=200, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + forget_factor=0.7, + fresh_restarts=False, + fresh_restarts_max_iter=30, + transform_max_iter=None, + random_state=None, + verbose=0, + ): + super().__init__( + n_components=n_components, + init=init, + beta_loss=beta_loss, + tol=tol, + max_iter=max_iter, + random_state=random_state, + alpha_W=alpha_W, + alpha_H=alpha_H, + l1_ratio=l1_ratio, + verbose=verbose, + ) + + self.max_no_improvement = max_no_improvement + self.batch_size = batch_size + self.forget_factor = forget_factor + self.fresh_restarts = fresh_restarts + self.fresh_restarts_max_iter = fresh_restarts_max_iter + self.transform_max_iter = transform_max_iter + + def _check_params(self, X): + super()._check_params(X) + + # batch_size + self._batch_size = min(self.batch_size, X.shape[0]) + + # forget_factor + self._rho = self.forget_factor ** (self._batch_size / X.shape[0]) + + # gamma for Maximization-Minimization (MM) algorithm [Fevotte 2011] + if self._beta_loss < 1: + self._gamma = 1.0 / (2.0 - self._beta_loss) + elif self._beta_loss > 2: + self._gamma = 1.0 / (self._beta_loss - 1.0) + else: + self._gamma = 1.0 + + # transform_max_iter + self._transform_max_iter = ( + self.max_iter + if self.transform_max_iter is None + else self.transform_max_iter + ) + + return self + + def _solve_W(self, X, H, max_iter): + """Minimize the objective function w.r.t W. + + Update W with H being fixed, until convergence. This is the heart + of `transform` but it's also used during `fit` when doing fresh restarts. + """ + avg = np.sqrt(X.mean() / self._n_components) + W = np.full((X.shape[0], self._n_components), avg, dtype=X.dtype) + W_buffer = W.copy() + + # Get scaled regularization terms. Done for each minibatch to take into account + # variable sizes of minibatches. + l1_reg_W, _, l2_reg_W, _ = self._compute_regularization(X) + + for _ in range(max_iter): + W, *_ = _multiplicative_update_w( + X, W, H, self._beta_loss, l1_reg_W, l2_reg_W, self._gamma + ) + + W_diff = linalg.norm(W - W_buffer) / linalg.norm(W) + if self.tol > 0 and W_diff <= self.tol: + break + + W_buffer[:] = W + + return W + + def _minibatch_step(self, X, W, H, update_H): + """Perform the update of W and H for one minibatch.""" + batch_size = X.shape[0] + + # get scaled regularization terms. Done for each minibatch to take into account + # variable sizes of minibatches. + l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H = self._compute_regularization(X) + + # update W + if self.fresh_restarts or W is None: + W = self._solve_W(X, H, self.fresh_restarts_max_iter) + else: + W, *_ = _multiplicative_update_w( + X, W, H, self._beta_loss, l1_reg_W, l2_reg_W, self._gamma + ) + + # necessary for stability with beta_loss < 1 + if self._beta_loss < 1: + W[W < np.finfo(np.float64).eps] = 0.0 + + batch_cost = ( + _beta_divergence(X, W, H, self._beta_loss) + + l1_reg_W * W.sum() + + l1_reg_H * H.sum() + + l2_reg_W * (W**2).sum() + + l2_reg_H * (H**2).sum() + ) / batch_size + + # update H (only at fit or fit_transform) + if update_H: + H[:] = _multiplicative_update_h( + X, + W, + H, + beta_loss=self._beta_loss, + l1_reg_H=l1_reg_H, + l2_reg_H=l2_reg_H, + gamma=self._gamma, + A=self._components_numerator, + B=self._components_denominator, + rho=self._rho, + ) + + # necessary for stability with beta_loss < 1 + if self._beta_loss <= 1: + H[H < np.finfo(np.float64).eps] = 0.0 + + return batch_cost + + def _minibatch_convergence( + self, X, batch_cost, H, H_buffer, n_samples, step, n_steps + ): + """Helper function to encapsulate the early stopping logic""" + batch_size = X.shape[0] + + # counts steps starting from 1 for user friendly verbose mode. + step = step + 1 + + # Ignore first iteration because H is not updated yet. + if step == 1: + if self.verbose: + print(f"Minibatch step {step}/{n_steps}: mean batch cost: {batch_cost}") + return False + + # Compute an Exponentially Weighted Average of the cost function to + # monitor the convergence while discarding minibatch-local stochastic + # variability: https://en.wikipedia.org/wiki/Moving_average + if self._ewa_cost is None: + self._ewa_cost = batch_cost + else: + alpha = batch_size / (n_samples + 1) + alpha = min(alpha, 1) + self._ewa_cost = self._ewa_cost * (1 - alpha) + batch_cost * alpha + + # Log progress to be able to monitor convergence + if self.verbose: + print( + f"Minibatch step {step}/{n_steps}: mean batch cost: " + f"{batch_cost}, ewa cost: {self._ewa_cost}" + ) + + # Early stopping based on change of H + H_diff = linalg.norm(H - H_buffer) / linalg.norm(H) + if self.tol > 0 and H_diff <= self.tol: + if self.verbose: + print(f"Converged (small H change) at step {step}/{n_steps}") + return True + + # Early stopping heuristic due to lack of improvement on smoothed + # cost function + if self._ewa_cost_min is None or self._ewa_cost < self._ewa_cost_min: + self._no_improvement = 0 + self._ewa_cost_min = self._ewa_cost + else: + self._no_improvement += 1 + + if ( + self.max_no_improvement is not None + and self._no_improvement >= self.max_no_improvement + ): + if self.verbose: + print( + "Converged (lack of improvement in objective function) " + f"at step {step}/{n_steps}" + ) + return True + + return False + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None, W=None, H=None): + """Learn a NMF model for the data X and returns the transformed data. + + This is more efficient than calling fit followed by transform. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Data matrix to be decomposed. + + y : Ignored + Not used, present here for API consistency by convention. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + """ + X = validate_data( + self, X, accept_sparse=("csr", "csc"), dtype=[np.float64, np.float32] + ) + + with config_context(assume_finite=True): + W, H, n_iter, n_steps = self._fit_transform(X, W=W, H=H) + + self.reconstruction_err_ = _beta_divergence( + X, W, H, self._beta_loss, square_root=True + ) + + self.n_components_ = H.shape[0] + self.components_ = H + self.n_iter_ = n_iter + self.n_steps_ = n_steps + + return W + + def _fit_transform(self, X, W=None, H=None, update_H=True): + """Learn a NMF model for the data X and returns the transformed data. + + Parameters + ---------- + X : {ndarray, sparse matrix} of shape (n_samples, n_features) + Data matrix to be decomposed. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is initialised as an array of zeros, unless + `solver='mu'`, then it is filled with values calculated by + `np.sqrt(X.mean() / self._n_components)`. + If `None`, uses the initialisation method specified in `init`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is used as a constant, to solve for W only. + If `None`, uses the initialisation method specified in `init`. + + update_H : bool, default=True + If True, both W and H will be estimated from initial guesses, + this corresponds to a call to the `fit_transform` method. + If False, only W will be estimated, this corresponds to a call + to the `transform` method. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + + H : ndarray of shape (n_components, n_features) + Factorization matrix, sometimes called 'dictionary'. + + n_iter : int + Actual number of started iterations over the whole dataset. + + n_steps : int + Number of mini-batches processed. + """ + check_non_negative(X, "MiniBatchNMF (input X)") + self._check_params(X) + + if X.min() == 0 and self._beta_loss <= 0: + raise ValueError( + "When beta_loss <= 0 and X contains zeros, " + "the solver may diverge. Please add small values " + "to X, or use a positive beta_loss." + ) + + n_samples = X.shape[0] + + # initialize or check W and H + W, H = self._check_w_h(X, W, H, update_H) + H_buffer = H.copy() + + # Initialize auxiliary matrices + self._components_numerator = H.copy() + self._components_denominator = np.ones(H.shape, dtype=H.dtype) + + # Attributes to monitor the convergence + self._ewa_cost = None + self._ewa_cost_min = None + self._no_improvement = 0 + + batches = gen_batches(n_samples, self._batch_size) + batches = itertools.cycle(batches) + n_steps_per_iter = int(np.ceil(n_samples / self._batch_size)) + n_steps = self.max_iter * n_steps_per_iter + + for i, batch in zip(range(n_steps), batches): + batch_cost = self._minibatch_step(X[batch], W[batch], H, update_H) + + if update_H and self._minibatch_convergence( + X[batch], batch_cost, H, H_buffer, n_samples, i, n_steps + ): + break + + H_buffer[:] = H + + if self.fresh_restarts: + W = self._solve_W(X, H, self._transform_max_iter) + + n_steps = i + 1 + n_iter = int(np.ceil(n_steps / n_steps_per_iter)) + + if n_iter == self.max_iter and self.tol > 0: + warnings.warn( + ( + f"Maximum number of iterations {self.max_iter} reached. " + "Increase it to improve convergence." + ), + ConvergenceWarning, + ) + + return W, H, n_iter, n_steps + + def transform(self, X): + """Transform the data X according to the fitted MiniBatchNMF model. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Data matrix to be transformed by the model. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + """ + check_is_fitted(self) + X = validate_data( + self, + X, + accept_sparse=("csr", "csc"), + dtype=[np.float64, np.float32], + reset=False, + ) + + W = self._solve_W(X, self.components_, self._transform_max_iter) + + return W + + @_fit_context(prefer_skip_nested_validation=True) + def partial_fit(self, X, y=None, W=None, H=None): + """Update the model using the data in `X` as a mini-batch. + + This method is expected to be called several times consecutively + on different chunks of a dataset so as to implement out-of-core + or online learning. + + This is especially useful when the whole dataset is too big to fit in + memory at once (see :ref:`scaling_strategies`). + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Data matrix to be decomposed. + + y : Ignored + Not used, present here for API consistency by convention. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + Only used for the first call to `partial_fit`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + Only used for the first call to `partial_fit`. + + Returns + ------- + self + Returns the instance itself. + """ + has_components = hasattr(self, "components_") + + X = validate_data( + self, + X, + accept_sparse=("csr", "csc"), + dtype=[np.float64, np.float32], + reset=not has_components, + ) + + if not has_components: + # This instance has not been fitted yet (fit or partial_fit) + self._check_params(X) + _, H = self._check_w_h(X, W=W, H=H, update_H=True) + + self._components_numerator = H.copy() + self._components_denominator = np.ones(H.shape, dtype=H.dtype) + self.n_steps_ = 0 + else: + H = self.components_ + + self._minibatch_step(X, None, H, update_H=True) + + self.n_components_ = H.shape[0] + self.components_ = H + self.n_steps_ += 1 + + return self diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..50dd82cabff2ce7758cab967957abf16277a5298 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..46b2796b6c3d8ebf7908b6f49f2f33ffc716aa5d Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.pyx new file mode 100644 index 0000000000000000000000000000000000000000..c8fe1ffcd7bf1ebc9c191f98ad84192a296c24b9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_online_lda_fast.pyx @@ -0,0 +1,110 @@ +import numpy as np + + +from cython cimport floating +from libc.math cimport exp, fabs, log + +from sklearn.utils._typedefs cimport float64_t, intp_t + + +def mean_change(const floating[:] arr_1, const floating[:] arr_2): + """Calculate the mean difference between two arrays. + + Equivalent to np.abs(arr_1 - arr2).mean(). + """ + + cdef float64_t total, diff + cdef intp_t i, size + + size = arr_1.shape[0] + total = 0.0 + for i in range(size): + diff = fabs(arr_1[i] - arr_2[i]) + total += diff + + return total / size + + +def _dirichlet_expectation_1d( + floating[:] doc_topic, + floating doc_topic_prior, + floating[:] out +): + """Dirichlet expectation for a single sample: + exp(E[log(theta)]) for theta ~ Dir(doc_topic) + after adding doc_topic_prior to doc_topic, in-place. + + Equivalent to + doc_topic += doc_topic_prior + out[:] = np.exp(psi(doc_topic) - psi(np.sum(doc_topic))) + """ + + cdef floating dt, psi_total, total + cdef intp_t i, size + + size = doc_topic.shape[0] + + total = 0.0 + for i in range(size): + dt = doc_topic[i] + doc_topic_prior + doc_topic[i] = dt + total += dt + psi_total = psi(total) + + for i in range(size): + out[i] = exp(psi(doc_topic[i]) - psi_total) + + +def _dirichlet_expectation_2d(const floating[:, :] arr): + """Dirichlet expectation for multiple samples: + E[log(theta)] for theta ~ Dir(arr). + + Equivalent to psi(arr) - psi(np.sum(arr, axis=1))[:, np.newaxis]. + + Note that unlike _dirichlet_expectation_1d, this function doesn't compute + the exp and doesn't add in the prior. + """ + cdef floating row_total, psi_row_total + cdef floating[:, :] d_exp + cdef intp_t i, j, n_rows, n_cols + + n_rows = arr.shape[0] + n_cols = arr.shape[1] + + d_exp = np.empty_like(arr) + for i in range(n_rows): + row_total = 0 + for j in range(n_cols): + row_total += arr[i, j] + psi_row_total = psi(row_total) + + for j in range(n_cols): + d_exp[i, j] = psi(arr[i, j]) - psi_row_total + + return d_exp.base + + +# Psi function for positive arguments. Optimized for speed, not accuracy. +# +# After: J. Bernardo (1976). Algorithm AS 103: Psi (Digamma) Function. +# https://www.uv.es/~bernardo/1976AppStatist.pdf +cdef floating psi(floating x) noexcept nogil: + cdef double EULER = 0.577215664901532860606512090082402431 + if x <= 1e-6: + # psi(x) = -EULER - 1/x + O(x) + return -EULER - 1. / x + + cdef floating r, result = 0 + + # psi(x + 1) = psi(x) + 1/x + while x < 6: + result -= 1. / x + x += 1 + + # psi(x) = log(x) - 1/(2x) - 1/(12x**2) + 1/(120x**4) - 1/(252x**6) + # + O(1/x**8) + r = 1. / x + result += log(x) - .5 * r + r = r * r + result -= r * ((1./12.) - r * ((1./120.) - r * (1./252.))) + return result diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..fdf39aa399cbb323324d01a72e634e8448ceaf99 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_pca.py @@ -0,0 +1,852 @@ +"""Principal Component Analysis.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from math import lgamma, log, sqrt +from numbers import Integral, Real + +import numpy as np +from scipy import linalg +from scipy.sparse import issparse +from scipy.sparse.linalg import svds + +from sklearn.base import _fit_context +from sklearn.decomposition._base import _BasePCA +from sklearn.utils import check_random_state +from sklearn.utils._arpack import _init_arpack_v0 +from sklearn.utils._array_api import device, get_namespace +from sklearn.utils._param_validation import Interval, RealNotInt, StrOptions +from sklearn.utils.extmath import _randomized_svd, fast_logdet, svd_flip +from sklearn.utils.sparsefuncs import _implicit_column_offset, mean_variance_axis +from sklearn.utils.validation import check_is_fitted, validate_data + + +def _assess_dimension(spectrum, rank, n_samples): + """Compute the log-likelihood of a rank ``rank`` dataset. + + The dataset is assumed to be embedded in gaussian noise of shape(n, + dimf) having spectrum ``spectrum``. This implements the method of + T. P. Minka. + + Parameters + ---------- + spectrum : ndarray of shape (n_features,) + Data spectrum. + rank : int + Tested rank value. It should be strictly lower than n_features, + otherwise the method isn't specified (division by zero in equation + (31) from the paper). + n_samples : int + Number of samples. + + Returns + ------- + ll : float + The log-likelihood. + + References + ---------- + This implements the method of `Thomas P. Minka: + Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604 + `_ + """ + xp, _ = get_namespace(spectrum) + + n_features = spectrum.shape[0] + if not 1 <= rank < n_features: + raise ValueError("the tested rank should be in [1, n_features - 1]") + + eps = 1e-15 + + if spectrum[rank - 1] < eps: + # When the tested rank is associated with a small eigenvalue, there's + # no point in computing the log-likelihood: it's going to be very + # small and won't be the max anyway. Also, it can lead to numerical + # issues below when computing pa, in particular in log((spectrum[i] - + # spectrum[j]) because this will take the log of something very small. + return -xp.inf + + pu = -rank * log(2.0) + for i in range(1, rank + 1): + pu += ( + lgamma((n_features - i + 1) / 2.0) - log(xp.pi) * (n_features - i + 1) / 2.0 + ) + + pl = xp.sum(xp.log(spectrum[:rank])) + pl = -pl * n_samples / 2.0 + + v = max(eps, xp.sum(spectrum[rank:]) / (n_features - rank)) + pv = -log(v) * n_samples * (n_features - rank) / 2.0 + + m = n_features * rank - rank * (rank + 1.0) / 2.0 + pp = log(2.0 * xp.pi) * (m + rank) / 2.0 + + pa = 0.0 + spectrum_ = xp.asarray(spectrum, copy=True) + spectrum_[rank:n_features] = v + for i in range(rank): + for j in range(i + 1, spectrum.shape[0]): + pa += log( + (spectrum[i] - spectrum[j]) * (1.0 / spectrum_[j] - 1.0 / spectrum_[i]) + ) + log(n_samples) + + ll = pu + pl + pv + pp - pa / 2.0 - rank * log(n_samples) / 2.0 + + return ll + + +def _infer_dimension(spectrum, n_samples): + """Infers the dimension of a dataset with a given spectrum. + + The returned value will be in [1, n_features - 1]. + """ + xp, _ = get_namespace(spectrum) + + ll = xp.empty_like(spectrum) + ll[0] = -xp.inf # we don't want to return n_components = 0 + for rank in range(1, spectrum.shape[0]): + ll[rank] = _assess_dimension(spectrum, rank, n_samples) + return xp.argmax(ll) + + +class PCA(_BasePCA): + """Principal component analysis (PCA). + + Linear dimensionality reduction using Singular Value Decomposition of the + data to project it to a lower dimensional space. The input data is centered + but not scaled for each feature before applying the SVD. + + It uses the LAPACK implementation of the full SVD or a randomized truncated + SVD by the method of Halko et al. 2009, depending on the shape of the input + data and the number of components to extract. + + With sparse inputs, the ARPACK implementation of the truncated SVD can be + used (i.e. through :func:`scipy.sparse.linalg.svds`). Alternatively, one + may consider :class:`TruncatedSVD` where the data are not centered. + + Notice that this class only supports sparse inputs for some solvers such as + "arpack" and "covariance_eigh". See :class:`TruncatedSVD` for an + alternative with sparse data. + + For a usage example, see + :ref:`sphx_glr_auto_examples_decomposition_plot_pca_iris.py` + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, float or 'mle', default=None + Number of components to keep. + if n_components is not set all components are kept:: + + n_components == min(n_samples, n_features) + + If ``n_components == 'mle'`` and ``svd_solver == 'full'``, Minka's + MLE is used to guess the dimension. Use of ``n_components == 'mle'`` + will interpret ``svd_solver == 'auto'`` as ``svd_solver == 'full'``. + + If ``0 < n_components < 1`` and ``svd_solver == 'full'``, select the + number of components such that the amount of variance that needs to be + explained is greater than the percentage specified by n_components. + + If ``svd_solver == 'arpack'``, the number of components must be + strictly less than the minimum of n_features and n_samples. + + Hence, the None case results in:: + + n_components == min(n_samples, n_features) - 1 + + copy : bool, default=True + If False, data passed to fit are overwritten and running + fit(X).transform(X) will not yield the expected results, + use fit_transform(X) instead. + + whiten : bool, default=False + When True (False by default) the `components_` vectors are multiplied + by the square root of n_samples and then divided by the singular values + to ensure uncorrelated outputs with unit component-wise variances. + + Whitening will remove some information from the transformed signal + (the relative variance scales of the components) but can sometime + improve the predictive accuracy of the downstream estimators by + making their data respect some hard-wired assumptions. + + svd_solver : {'auto', 'full', 'covariance_eigh', 'arpack', 'randomized'},\ + default='auto' + "auto" : + The solver is selected by a default 'auto' policy is based on `X.shape` and + `n_components`: if the input data has fewer than 1000 features and + more than 10 times as many samples, then the "covariance_eigh" + solver is used. Otherwise, if the input data is larger than 500x500 + and the number of components to extract is lower than 80% of the + smallest dimension of the data, then the more efficient + "randomized" method is selected. Otherwise the exact "full" SVD is + computed and optionally truncated afterwards. + "full" : + Run exact full SVD calling the standard LAPACK solver via + `scipy.linalg.svd` and select the components by postprocessing + "covariance_eigh" : + Precompute the covariance matrix (on centered data), run a + classical eigenvalue decomposition on the covariance matrix + typically using LAPACK and select the components by postprocessing. + This solver is very efficient for n_samples >> n_features and small + n_features. It is, however, not tractable otherwise for large + n_features (large memory footprint required to materialize the + covariance matrix). Also note that compared to the "full" solver, + this solver effectively doubles the condition number and is + therefore less numerical stable (e.g. on input data with a large + range of singular values). + "arpack" : + Run SVD truncated to `n_components` calling ARPACK solver via + `scipy.sparse.linalg.svds`. It requires strictly + `0 < n_components < min(X.shape)` + "randomized" : + Run randomized SVD by the method of Halko et al. + + .. versionadded:: 0.18.0 + + .. versionchanged:: 1.5 + Added the 'covariance_eigh' solver. + + tol : float, default=0.0 + Tolerance for singular values computed by svd_solver == 'arpack'. + Must be of range [0.0, infinity). + + .. versionadded:: 0.18.0 + + iterated_power : int or 'auto', default='auto' + Number of iterations for the power method computed by + svd_solver == 'randomized'. + Must be of range [0, infinity). + + .. versionadded:: 0.18.0 + + n_oversamples : int, default=10 + This parameter is only relevant when `svd_solver="randomized"`. + It corresponds to the additional number of random vectors to sample the + range of `X` so as to ensure proper conditioning. See + :func:`~sklearn.utils.extmath.randomized_svd` for more details. + + .. versionadded:: 1.1 + + power_iteration_normalizer : {'auto', 'QR', 'LU', 'none'}, default='auto' + Power iteration normalizer for randomized SVD solver. + Not used by ARPACK. See :func:`~sklearn.utils.extmath.randomized_svd` + for more details. + + .. versionadded:: 1.1 + + random_state : int, RandomState instance or None, default=None + Used when the 'arpack' or 'randomized' solvers are used. Pass an int + for reproducible results across multiple function calls. + See :term:`Glossary `. + + .. versionadded:: 0.18.0 + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Principal axes in feature space, representing the directions of + maximum variance in the data. Equivalently, the right singular + vectors of the centered input data, parallel to its eigenvectors. + The components are sorted by decreasing ``explained_variance_``. + + explained_variance_ : ndarray of shape (n_components,) + The amount of variance explained by each of the selected components. + The variance estimation uses `n_samples - 1` degrees of freedom. + + Equal to n_components largest eigenvalues + of the covariance matrix of X. + + .. versionadded:: 0.18 + + explained_variance_ratio_ : ndarray of shape (n_components,) + Percentage of variance explained by each of the selected components. + + If ``n_components`` is not set then all components are stored and the + sum of the ratios is equal to 1.0. + + singular_values_ : ndarray of shape (n_components,) + The singular values corresponding to each of the selected components. + The singular values are equal to the 2-norms of the ``n_components`` + variables in the lower-dimensional space. + + .. versionadded:: 0.19 + + mean_ : ndarray of shape (n_features,) + Per-feature empirical mean, estimated from the training set. + + Equal to `X.mean(axis=0)`. + + n_components_ : int + The estimated number of components. When n_components is set + to 'mle' or a number between 0 and 1 (with svd_solver == 'full') this + number is estimated from input data. Otherwise it equals the parameter + n_components, or the lesser value of n_features and n_samples + if n_components is None. + + n_samples_ : int + Number of samples in the training data. + + noise_variance_ : float + The estimated noise covariance following the Probabilistic PCA model + from Tipping and Bishop 1999. See "Pattern Recognition and + Machine Learning" by C. Bishop, 12.2.1 p. 574 or + http://www.miketipping.com/papers/met-mppca.pdf. It is required to + compute the estimated data covariance and score samples. + + Equal to the average of (min(n_features, n_samples) - n_components) + smallest eigenvalues of the covariance matrix of X. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + KernelPCA : Kernel Principal Component Analysis. + SparsePCA : Sparse Principal Component Analysis. + TruncatedSVD : Dimensionality reduction using truncated SVD. + IncrementalPCA : Incremental Principal Component Analysis. + + References + ---------- + For n_components == 'mle', this class uses the method from: + `Minka, T. P.. "Automatic choice of dimensionality for PCA". + In NIPS, pp. 598-604 `_ + + Implements the probabilistic PCA model from: + `Tipping, M. E., and Bishop, C. M. (1999). "Probabilistic principal + component analysis". Journal of the Royal Statistical Society: + Series B (Statistical Methodology), 61(3), 611-622. + `_ + via the score and score_samples methods. + + For svd_solver == 'arpack', refer to `scipy.sparse.linalg.svds`. + + For svd_solver == 'randomized', see: + :doi:`Halko, N., Martinsson, P. G., and Tropp, J. A. (2011). + "Finding structure with randomness: Probabilistic algorithms for + constructing approximate matrix decompositions". + SIAM review, 53(2), 217-288. + <10.1137/090771806>` + and also + :doi:`Martinsson, P. G., Rokhlin, V., and Tygert, M. (2011). + "A randomized algorithm for the decomposition of matrices". + Applied and Computational Harmonic Analysis, 30(1), 47-68. + <10.1016/j.acha.2010.02.003>` + + Examples + -------- + >>> import numpy as np + >>> from sklearn.decomposition import PCA + >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) + >>> pca = PCA(n_components=2) + >>> pca.fit(X) + PCA(n_components=2) + >>> print(pca.explained_variance_ratio_) + [0.9924 0.0075] + >>> print(pca.singular_values_) + [6.30061 0.54980] + + >>> pca = PCA(n_components=2, svd_solver='full') + >>> pca.fit(X) + PCA(n_components=2, svd_solver='full') + >>> print(pca.explained_variance_ratio_) + [0.9924 0.00755] + >>> print(pca.singular_values_) + [6.30061 0.54980] + + >>> pca = PCA(n_components=1, svd_solver='arpack') + >>> pca.fit(X) + PCA(n_components=1, svd_solver='arpack') + >>> print(pca.explained_variance_ratio_) + [0.99244] + >>> print(pca.singular_values_) + [6.30061] + """ + + _parameter_constraints: dict = { + "n_components": [ + Interval(Integral, 0, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="neither"), + StrOptions({"mle"}), + None, + ], + "copy": ["boolean"], + "whiten": ["boolean"], + "svd_solver": [ + StrOptions({"auto", "full", "covariance_eigh", "arpack", "randomized"}) + ], + "tol": [Interval(Real, 0, None, closed="left")], + "iterated_power": [ + StrOptions({"auto"}), + Interval(Integral, 0, None, closed="left"), + ], + "n_oversamples": [Interval(Integral, 1, None, closed="left")], + "power_iteration_normalizer": [StrOptions({"auto", "QR", "LU", "none"})], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + copy=True, + whiten=False, + svd_solver="auto", + tol=0.0, + iterated_power="auto", + n_oversamples=10, + power_iteration_normalizer="auto", + random_state=None, + ): + self.n_components = n_components + self.copy = copy + self.whiten = whiten + self.svd_solver = svd_solver + self.tol = tol + self.iterated_power = iterated_power + self.n_oversamples = n_oversamples + self.power_iteration_normalizer = power_iteration_normalizer + self.random_state = random_state + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the model with X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Ignored. + + Returns + ------- + self : object + Returns the instance itself. + """ + self._fit(X) + return self + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None): + """Fit the model with X and apply the dimensionality reduction on X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Ignored. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Transformed values. + + Notes + ----- + This method returns a Fortran-ordered array. To convert it to a + C-ordered array, use 'np.ascontiguousarray'. + """ + U, S, _, X, x_is_centered, xp = self._fit(X) + if U is not None: + U = U[:, : self.n_components_] + + if self.whiten: + # X_new = X * V / S * sqrt(n_samples) = U * sqrt(n_samples) + U *= sqrt(X.shape[0] - 1) + else: + # X_new = X * V = U * S * Vt * V = U * S + U *= S[: self.n_components_] + + return U + else: # solver="covariance_eigh" does not compute U at fit time. + return self._transform(X, xp, x_is_centered=x_is_centered) + + def _fit(self, X): + """Dispatch to the right submethod depending on the chosen solver.""" + xp, is_array_api_compliant = get_namespace(X) + + # Raise an error for sparse input and unsupported svd_solver + if issparse(X) and self.svd_solver not in ["auto", "arpack", "covariance_eigh"]: + raise TypeError( + 'PCA only support sparse inputs with the "arpack" and' + f' "covariance_eigh" solvers, while "{self.svd_solver}" was passed. See' + " TruncatedSVD for a possible alternative." + ) + if self.svd_solver == "arpack" and is_array_api_compliant: + raise ValueError( + "PCA with svd_solver='arpack' is not supported for Array API inputs." + ) + + # Validate the data, without ever forcing a copy as any solver that + # supports sparse input data and the `covariance_eigh` solver are + # written in a way to avoid the need for any inplace modification of + # the input data contrary to the other solvers. + # The copy will happen + # later, only if needed, once the solver negotiation below is done. + X = validate_data( + self, + X, + dtype=[xp.float64, xp.float32], + force_writeable=True, + accept_sparse=("csr", "csc"), + ensure_2d=True, + copy=False, + ) + self._fit_svd_solver = self.svd_solver + if self._fit_svd_solver == "auto" and issparse(X): + self._fit_svd_solver = "arpack" + + if self.n_components is None: + if self._fit_svd_solver != "arpack": + n_components = min(X.shape) + else: + n_components = min(X.shape) - 1 + else: + n_components = self.n_components + + if self._fit_svd_solver == "auto": + # Tall and skinny problems are best handled by precomputing the + # covariance matrix. + if X.shape[1] <= 1_000 and X.shape[0] >= 10 * X.shape[1]: + self._fit_svd_solver = "covariance_eigh" + # Small problem or n_components == 'mle', just call full PCA + elif max(X.shape) <= 500 or n_components == "mle": + self._fit_svd_solver = "full" + elif 1 <= n_components < 0.8 * min(X.shape): + self._fit_svd_solver = "randomized" + # This is also the case of n_components in (0, 1) + else: + self._fit_svd_solver = "full" + + # Call different fits for either full or truncated SVD + if self._fit_svd_solver in ("full", "covariance_eigh"): + return self._fit_full(X, n_components, xp, is_array_api_compliant) + elif self._fit_svd_solver in ["arpack", "randomized"]: + return self._fit_truncated(X, n_components, xp) + + def _fit_full(self, X, n_components, xp, is_array_api_compliant): + """Fit the model by computing full SVD on X.""" + n_samples, n_features = X.shape + + if n_components == "mle": + if n_samples < n_features: + raise ValueError( + "n_components='mle' is only supported if n_samples >= n_features" + ) + elif not 0 <= n_components <= min(n_samples, n_features): + raise ValueError( + f"n_components={n_components} must be between 0 and " + f"min(n_samples, n_features)={min(n_samples, n_features)} with " + f"svd_solver={self._fit_svd_solver!r}" + ) + + self.mean_ = xp.mean(X, axis=0) + # When X is a scipy sparse matrix, self.mean_ is a numpy matrix, so we need + # to transform it to a 1D array. Note that this is not the case when X + # is a scipy sparse array. + # TODO: remove the following two lines when scikit-learn only depends + # on scipy versions that no longer support scipy.sparse matrices. + self.mean_ = xp.reshape(xp.asarray(self.mean_), (-1,)) + + if self._fit_svd_solver == "full": + X_centered = xp.asarray(X, copy=True) if self.copy else X + X_centered -= self.mean_ + x_is_centered = not self.copy + + if not is_array_api_compliant: + # Use scipy.linalg with NumPy/SciPy inputs for the sake of not + # introducing unanticipated behavior changes. In the long run we + # could instead decide to always use xp.linalg.svd for all inputs, + # but that would make this code rely on numpy's SVD instead of + # scipy's. It's not 100% clear whether they use the same LAPACK + # solver by default though (assuming both are built against the + # same BLAS). + U, S, Vt = linalg.svd(X_centered, full_matrices=False) + else: + U, S, Vt = xp.linalg.svd(X_centered, full_matrices=False) + explained_variance_ = (S**2) / (n_samples - 1) + + else: + assert self._fit_svd_solver == "covariance_eigh" + # In the following, we center the covariance matrix C afterwards + # (without centering the data X first) to avoid an unnecessary copy + # of X. Note that the mean_ attribute is still needed to center + # test data in the transform method. + # + # Note: at the time of writing, `xp.cov` does not exist in the + # Array API standard: + # https://github.com/data-apis/array-api/issues/43 + # + # Besides, using `numpy.cov`, as of numpy 1.26.0, would not be + # memory efficient for our use case when `n_samples >> n_features`: + # `numpy.cov` centers a copy of the data before computing the + # matrix product instead of subtracting a small `(n_features, + # n_features)` square matrix from the gram matrix X.T @ X, as we do + # below. + x_is_centered = False + C = X.T @ X + C -= ( + n_samples + * xp.reshape(self.mean_, (-1, 1)) + * xp.reshape(self.mean_, (1, -1)) + ) + C /= n_samples - 1 + eigenvals, eigenvecs = xp.linalg.eigh(C) + + # When X is a scipy sparse matrix, the following two datastructures + # are returned as instances of the soft-deprecated numpy.matrix + # class. Note that this problem does not occur when X is a scipy + # sparse array (or another other kind of supported array). + # TODO: remove the following two lines when scikit-learn only + # depends on scipy versions that no longer support scipy.sparse + # matrices. + eigenvals = xp.reshape(xp.asarray(eigenvals), (-1,)) + eigenvecs = xp.asarray(eigenvecs) + + eigenvals = xp.flip(eigenvals, axis=0) + eigenvecs = xp.flip(eigenvecs, axis=1) + + # The covariance matrix C is positive semi-definite by + # construction. However, the eigenvalues returned by xp.linalg.eigh + # can be slightly negative due to numerical errors. This would be + # an issue for the subsequent sqrt, hence the manual clipping. + eigenvals[eigenvals < 0.0] = 0.0 + explained_variance_ = eigenvals + + # Re-construct SVD of centered X indirectly and make it consistent + # with the other solvers. + S = xp.sqrt(eigenvals * (n_samples - 1)) + Vt = eigenvecs.T + U = None + + # flip eigenvectors' sign to enforce deterministic output + U, Vt = svd_flip(U, Vt, u_based_decision=False) + + components_ = Vt + + # Get variance explained by singular values + total_var = xp.sum(explained_variance_) + explained_variance_ratio_ = explained_variance_ / total_var + singular_values_ = xp.asarray(S, copy=True) # Store the singular values. + + # Postprocess the number of components required + if n_components == "mle": + n_components = _infer_dimension(explained_variance_, n_samples) + elif 0 < n_components < 1.0: + # number of components for which the cumulated explained + # variance percentage is superior to the desired threshold + # side='right' ensures that number of features selected + # their variance is always greater than n_components float + # passed. More discussion in issue: #15669 + ratio_cumsum = xp.cumulative_sum(explained_variance_ratio_) + n_components = ( + xp.searchsorted( + ratio_cumsum, + xp.asarray(n_components, device=device(ratio_cumsum)), + side="right", + ) + + 1 + ) + + # Compute noise covariance using Probabilistic PCA model + # The sigma2 maximum likelihood (cf. eq. 12.46) + if n_components < min(n_features, n_samples): + self.noise_variance_ = xp.mean(explained_variance_[n_components:]) + else: + self.noise_variance_ = 0.0 + + self.n_samples_ = n_samples + self.n_components_ = n_components + # Assign a copy of the result of the truncation of the components in + # order to: + # - release the memory used by the discarded components, + # - ensure that the kept components are allocated contiguously in + # memory to make the transform method faster by leveraging cache + # locality. + self.components_ = xp.asarray(components_[:n_components, :], copy=True) + + # We do the same for the other arrays for the sake of consistency. + self.explained_variance_ = xp.asarray( + explained_variance_[:n_components], copy=True + ) + self.explained_variance_ratio_ = xp.asarray( + explained_variance_ratio_[:n_components], copy=True + ) + self.singular_values_ = xp.asarray(singular_values_[:n_components], copy=True) + + return U, S, Vt, X, x_is_centered, xp + + def _fit_truncated(self, X, n_components, xp): + """Fit the model by computing truncated SVD (by ARPACK or randomized) + on X. + """ + n_samples, n_features = X.shape + + svd_solver = self._fit_svd_solver + if isinstance(n_components, str): + raise ValueError( + "n_components=%r cannot be a string with svd_solver='%s'" + % (n_components, svd_solver) + ) + elif not 1 <= n_components <= min(n_samples, n_features): + raise ValueError( + "n_components=%r must be between 1 and " + "min(n_samples, n_features)=%r with " + "svd_solver='%s'" + % (n_components, min(n_samples, n_features), svd_solver) + ) + elif svd_solver == "arpack" and n_components == min(n_samples, n_features): + raise ValueError( + "n_components=%r must be strictly less than " + "min(n_samples, n_features)=%r with " + "svd_solver='%s'" + % (n_components, min(n_samples, n_features), svd_solver) + ) + + random_state = check_random_state(self.random_state) + + # Center data + total_var = None + if issparse(X): + self.mean_, var = mean_variance_axis(X, axis=0) + total_var = var.sum() * n_samples / (n_samples - 1) # ddof=1 + X_centered = _implicit_column_offset(X, self.mean_) + x_is_centered = False + else: + self.mean_ = xp.mean(X, axis=0) + X_centered = xp.asarray(X, copy=True) if self.copy else X + X_centered -= self.mean_ + x_is_centered = not self.copy + + if svd_solver == "arpack": + v0 = _init_arpack_v0(min(X.shape), random_state) + U, S, Vt = svds(X_centered, k=n_components, tol=self.tol, v0=v0) + # svds doesn't abide by scipy.linalg.svd/randomized_svd + # conventions, so reverse its outputs. + S = S[::-1] + # flip eigenvectors' sign to enforce deterministic output + U, Vt = svd_flip(U[:, ::-1], Vt[::-1], u_based_decision=False) + + elif svd_solver == "randomized": + # sign flipping is done inside + U, S, Vt = _randomized_svd( + X_centered, + n_components=n_components, + n_oversamples=self.n_oversamples, + n_iter=self.iterated_power, + power_iteration_normalizer=self.power_iteration_normalizer, + flip_sign=False, + random_state=random_state, + ) + U, Vt = svd_flip(U, Vt, u_based_decision=False) + + self.n_samples_ = n_samples + self.components_ = Vt + self.n_components_ = n_components + + # Get variance explained by singular values + self.explained_variance_ = (S**2) / (n_samples - 1) + + # Workaround in-place variance calculation since at the time numpy + # did not have a way to calculate variance in-place. + # + # TODO: update this code to either: + # * Use the array-api variance calculation, unless memory usage suffers + # * Update sklearn.utils.extmath._incremental_mean_and_var to support array-api + # See: https://github.com/scikit-learn/scikit-learn/pull/18689#discussion_r1335540991 + if total_var is None: + N = X.shape[0] - 1 + X_centered **= 2 + total_var = xp.sum(X_centered) / N + + self.explained_variance_ratio_ = self.explained_variance_ / total_var + self.singular_values_ = xp.asarray(S, copy=True) # Store the singular values. + + if self.n_components_ < min(n_features, n_samples): + self.noise_variance_ = total_var - xp.sum(self.explained_variance_) + self.noise_variance_ /= min(n_features, n_samples) - n_components + else: + self.noise_variance_ = 0.0 + + return U, S, Vt, X, x_is_centered, xp + + def score_samples(self, X): + """Return the log-likelihood of each sample. + + See. "Pattern Recognition and Machine Learning" + by C. Bishop, 12.2.1 p. 574 + or http://www.miketipping.com/papers/met-mppca.pdf + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data. + + Returns + ------- + ll : ndarray of shape (n_samples,) + Log-likelihood of each sample under the current model. + """ + check_is_fitted(self) + xp, _ = get_namespace(X) + X = validate_data(self, X, dtype=[xp.float64, xp.float32], reset=False) + Xr = X - self.mean_ + n_features = X.shape[1] + precision = self.get_precision() + log_like = -0.5 * xp.sum(Xr * (Xr @ precision), axis=1) + log_like -= 0.5 * (n_features * log(2.0 * np.pi) - fast_logdet(precision)) + return log_like + + def score(self, X, y=None): + """Return the average log-likelihood of all samples. + + See. "Pattern Recognition and Machine Learning" + by C. Bishop, 12.2.1 p. 574 + or http://www.miketipping.com/papers/met-mppca.pdf + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data. + + y : Ignored + Ignored. + + Returns + ------- + ll : float + Average log-likelihood of the samples under the current model. + """ + xp, _ = get_namespace(X) + return float(xp.mean(self.score_samples(X))) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + solver = getattr(self, "_fit_svd_solver", self.svd_solver) + tags.array_api_support = solver not in ["arpack", "randomized"] or ( + solver == "randomized" and self.power_iteration_normalizer == "QR" + ) + tags.input_tags.sparse = self.svd_solver in ( + "auto", + "arpack", + "covariance_eigh", + ) + return tags diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_sparse_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_sparse_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..31359c5a3a9d77943ff79e29dbf151e49d4a250b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_sparse_pca.py @@ -0,0 +1,551 @@ +"""Matrix factorization with Sparse PCA.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real + +import numpy as np + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.decomposition._dict_learning import ( + MiniBatchDictionaryLearning, + dict_learning, +) +from sklearn.linear_model import ridge_regression +from sklearn.utils import check_random_state +from sklearn.utils._param_validation import Interval, StrOptions +from sklearn.utils.extmath import svd_flip +from sklearn.utils.validation import check_array, check_is_fitted, validate_data + + +class _BaseSparsePCA(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): + """Base class for SparsePCA and MiniBatchSparsePCA""" + + _parameter_constraints: dict = { + "n_components": [None, Interval(Integral, 1, None, closed="left")], + "alpha": [Interval(Real, 0.0, None, closed="left")], + "ridge_alpha": [Interval(Real, 0.0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "tol": [Interval(Real, 0.0, None, closed="left")], + "method": [StrOptions({"lars", "cd"})], + "n_jobs": [Integral, None], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + ridge_alpha=0.01, + max_iter=1000, + tol=1e-8, + method="lars", + n_jobs=None, + verbose=False, + random_state=None, + ): + self.n_components = n_components + self.alpha = alpha + self.ridge_alpha = ridge_alpha + self.max_iter = max_iter + self.tol = tol + self.method = method + self.n_jobs = n_jobs + self.verbose = verbose + self.random_state = random_state + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the model from data in X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present here for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + random_state = check_random_state(self.random_state) + X = validate_data(self, X) + + self.mean_ = X.mean(axis=0) + X = X - self.mean_ + + if self.n_components is None: + n_components = X.shape[1] + else: + n_components = self.n_components + + return self._fit(X, n_components, random_state) + + def transform(self, X): + """Least Squares projection of the data onto the sparse components. + + To avoid instability issues in case the system is under-determined, + regularization can be applied (Ridge regression) via the + `ridge_alpha` parameter. + + Note that Sparse PCA components orthogonality is not enforced as in PCA + hence one cannot use a simple linear projection. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + Test data to be transformed, must have the same number of + features as the data used to train the model. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Transformed data. + """ + check_is_fitted(self) + + X = validate_data(self, X, reset=False) + X = X - self.mean_ + + U = ridge_regression( + self.components_.T, X.T, self.ridge_alpha, solver="cholesky" + ) + + return U + + def inverse_transform(self, X): + """Transform data from the latent space to the original space. + + This inversion is an approximation due to the loss of information + induced by the forward decomposition. + + .. versionadded:: 1.2 + + Parameters + ---------- + X : ndarray of shape (n_samples, n_components) + Data in the latent space. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Reconstructed data in the original space. + """ + check_is_fitted(self) + X = check_array(X) + + return (X @ self.components_) + self.mean_ + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + +class SparsePCA(_BaseSparsePCA): + """Sparse Principal Components Analysis (SparsePCA). + + Finds the set of sparse components that can optimally reconstruct + the data. The amount of sparseness is controllable by the coefficient + of the L1 penalty, given by the parameter alpha. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=None + Number of sparse atoms to extract. If None, then ``n_components`` + is set to ``n_features``. + + alpha : float, default=1 + Sparsity controlling parameter. Higher values lead to sparser + components. + + ridge_alpha : float, default=0.01 + Amount of ridge shrinkage to apply in order to improve + conditioning when calling the transform method. + + max_iter : int, default=1000 + Maximum number of iterations to perform. + + tol : float, default=1e-8 + Tolerance for the stopping condition. + + method : {'lars', 'cd'}, default='lars' + Method to be used for optimization. + lars: uses the least angle regression method to solve the lasso problem + (linear_model.lars_path) + cd: uses the coordinate descent method to compute the + Lasso solution (linear_model.Lasso). Lars will be faster if + the estimated components are sparse. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + U_init : ndarray of shape (n_samples, n_components), default=None + Initial values for the loadings for warm restart scenarios. Only used + if `U_init` and `V_init` are not None. + + V_init : ndarray of shape (n_components, n_features), default=None + Initial values for the components for warm restart scenarios. Only used + if `U_init` and `V_init` are not None. + + verbose : int or bool, default=False + Controls the verbosity; the higher, the more messages. Defaults to 0. + + random_state : int, RandomState instance or None, default=None + Used during dictionary learning. Pass an int for reproducible results + across multiple function calls. + See :term:`Glossary `. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Sparse components extracted from the data. + + error_ : ndarray + Vector of errors at each iteration. + + n_components_ : int + Estimated number of components. + + .. versionadded:: 0.23 + + n_iter_ : int + Number of iterations run. + + mean_ : ndarray of shape (n_features,) + Per-feature empirical mean, estimated from the training set. + Equal to ``X.mean(axis=0)``. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + PCA : Principal Component Analysis implementation. + MiniBatchSparsePCA : Mini batch variant of `SparsePCA` that is faster but less + accurate. + DictionaryLearning : Generic dictionary learning problem using a sparse code. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_friedman1 + >>> from sklearn.decomposition import SparsePCA + >>> X, _ = make_friedman1(n_samples=200, n_features=30, random_state=0) + >>> transformer = SparsePCA(n_components=5, random_state=0) + >>> transformer.fit(X) + SparsePCA(...) + >>> X_transformed = transformer.transform(X) + >>> X_transformed.shape + (200, 5) + >>> # most values in the components_ are zero (sparsity) + >>> np.mean(transformer.components_ == 0) + np.float64(0.9666) + """ + + _parameter_constraints: dict = { + **_BaseSparsePCA._parameter_constraints, + "U_init": [None, np.ndarray], + "V_init": [None, np.ndarray], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + ridge_alpha=0.01, + max_iter=1000, + tol=1e-8, + method="lars", + n_jobs=None, + U_init=None, + V_init=None, + verbose=False, + random_state=None, + ): + super().__init__( + n_components=n_components, + alpha=alpha, + ridge_alpha=ridge_alpha, + max_iter=max_iter, + tol=tol, + method=method, + n_jobs=n_jobs, + verbose=verbose, + random_state=random_state, + ) + self.U_init = U_init + self.V_init = V_init + + def _fit(self, X, n_components, random_state): + """Specialized `fit` for SparsePCA.""" + + code_init = self.V_init.T if self.V_init is not None else None + dict_init = self.U_init.T if self.U_init is not None else None + code, dictionary, E, self.n_iter_ = dict_learning( + X.T, + n_components, + alpha=self.alpha, + tol=self.tol, + max_iter=self.max_iter, + method=self.method, + n_jobs=self.n_jobs, + verbose=self.verbose, + random_state=random_state, + code_init=code_init, + dict_init=dict_init, + return_n_iter=True, + ) + # flip eigenvectors' sign to enforce deterministic output + code, dictionary = svd_flip(code, dictionary, u_based_decision=True) + self.components_ = code.T + components_norm = np.linalg.norm(self.components_, axis=1)[:, np.newaxis] + components_norm[components_norm == 0] = 1 + self.components_ /= components_norm + self.n_components_ = len(self.components_) + + self.error_ = E + return self + + +class MiniBatchSparsePCA(_BaseSparsePCA): + """Mini-batch Sparse Principal Components Analysis. + + Finds the set of sparse components that can optimally reconstruct + the data. The amount of sparseness is controllable by the coefficient + of the L1 penalty, given by the parameter alpha. + + For an example comparing sparse PCA to PCA, see + :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=None + Number of sparse atoms to extract. If None, then ``n_components`` + is set to ``n_features``. + + alpha : int, default=1 + Sparsity controlling parameter. Higher values lead to sparser + components. + + ridge_alpha : float, default=0.01 + Amount of ridge shrinkage to apply in order to improve + conditioning when calling the transform method. + + max_iter : int, default=1_000 + Maximum number of iterations over the complete dataset before + stopping independently of any early stopping criterion heuristics. + + .. versionadded:: 1.2 + + callback : callable, default=None + Callable that gets invoked every five iterations. + + batch_size : int, default=3 + The number of features to take in each mini batch. + + verbose : int or bool, default=False + Controls the verbosity; the higher, the more messages. Defaults to 0. + + shuffle : bool, default=True + Whether to shuffle the data before splitting it in batches. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + method : {'lars', 'cd'}, default='lars' + Method to be used for optimization. + lars: uses the least angle regression method to solve the lasso problem + (linear_model.lars_path) + cd: uses the coordinate descent method to compute the + Lasso solution (linear_model.Lasso). Lars will be faster if + the estimated components are sparse. + + random_state : int, RandomState instance or None, default=None + Used for random shuffling when ``shuffle`` is set to ``True``, + during online dictionary learning. Pass an int for reproducible results + across multiple function calls. + See :term:`Glossary `. + + tol : float, default=1e-3 + Control early stopping based on the norm of the differences in the + dictionary between 2 steps. + + To disable early stopping based on changes in the dictionary, set + `tol` to 0.0. + + .. versionadded:: 1.1 + + max_no_improvement : int or None, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + + To disable convergence detection based on cost function, set + `max_no_improvement` to `None`. + + .. versionadded:: 1.1 + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Sparse components extracted from the data. + + n_components_ : int + Estimated number of components. + + .. versionadded:: 0.23 + + n_iter_ : int + Number of iterations run. + + mean_ : ndarray of shape (n_features,) + Per-feature empirical mean, estimated from the training set. + Equal to ``X.mean(axis=0)``. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + IncrementalPCA : Incremental principal components analysis. + PCA : Principal component analysis. + SparsePCA : Sparse Principal Components Analysis. + TruncatedSVD : Dimensionality reduction using truncated SVD. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_friedman1 + >>> from sklearn.decomposition import MiniBatchSparsePCA + >>> X, _ = make_friedman1(n_samples=200, n_features=30, random_state=0) + >>> transformer = MiniBatchSparsePCA(n_components=5, batch_size=50, + ... max_iter=10, random_state=0) + >>> transformer.fit(X) + MiniBatchSparsePCA(...) + >>> X_transformed = transformer.transform(X) + >>> X_transformed.shape + (200, 5) + >>> # most values in the components_ are zero (sparsity) + >>> np.mean(transformer.components_ == 0) + np.float64(0.9) + """ + + _parameter_constraints: dict = { + **_BaseSparsePCA._parameter_constraints, + "max_iter": [Interval(Integral, 0, None, closed="left")], + "callback": [None, callable], + "batch_size": [Interval(Integral, 1, None, closed="left")], + "shuffle": ["boolean"], + "max_no_improvement": [Interval(Integral, 0, None, closed="left"), None], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + ridge_alpha=0.01, + max_iter=1_000, + callback=None, + batch_size=3, + verbose=False, + shuffle=True, + n_jobs=None, + method="lars", + random_state=None, + tol=1e-3, + max_no_improvement=10, + ): + super().__init__( + n_components=n_components, + alpha=alpha, + ridge_alpha=ridge_alpha, + max_iter=max_iter, + tol=tol, + method=method, + n_jobs=n_jobs, + verbose=verbose, + random_state=random_state, + ) + self.callback = callback + self.batch_size = batch_size + self.shuffle = shuffle + self.max_no_improvement = max_no_improvement + + def _fit(self, X, n_components, random_state): + """Specialized `fit` for MiniBatchSparsePCA.""" + + transform_algorithm = "lasso_" + self.method + est = MiniBatchDictionaryLearning( + n_components=n_components, + alpha=self.alpha, + max_iter=self.max_iter, + dict_init=None, + batch_size=self.batch_size, + shuffle=self.shuffle, + n_jobs=self.n_jobs, + fit_algorithm=self.method, + random_state=random_state, + transform_algorithm=transform_algorithm, + transform_alpha=self.alpha, + verbose=self.verbose, + callback=self.callback, + tol=self.tol, + max_no_improvement=self.max_no_improvement, + ) + est.set_output(transform="default") + est.fit(X.T) + + self.components_, self.n_iter_ = est.transform(X.T).T, est.n_iter_ + + components_norm = np.linalg.norm(self.components_, axis=1)[:, np.newaxis] + components_norm[components_norm == 0] = 1 + self.components_ /= components_norm + self.n_components_ = len(self.components_) + + return self diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_truncated_svd.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_truncated_svd.py new file mode 100644 index 0000000000000000000000000000000000000000..ef82df7d6e4adbab1639c372d2cada9d089497a2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/_truncated_svd.py @@ -0,0 +1,322 @@ +"""Truncated SVD for sparse matrices, aka latent semantic analysis (LSA).""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real + +import numpy as np +import scipy.sparse as sp +from scipy.sparse.linalg import svds + +from sklearn.base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.utils import check_array, check_random_state +from sklearn.utils._arpack import _init_arpack_v0 +from sklearn.utils._param_validation import Interval, StrOptions +from sklearn.utils.extmath import _randomized_svd, safe_sparse_dot, svd_flip +from sklearn.utils.sparsefuncs import mean_variance_axis +from sklearn.utils.validation import check_is_fitted, validate_data + +__all__ = ["TruncatedSVD"] + + +class TruncatedSVD(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): + """Dimensionality reduction using truncated SVD (aka LSA). + + This transformer performs linear dimensionality reduction by means of + truncated singular value decomposition (SVD). Contrary to PCA, this + estimator does not center the data before computing the singular value + decomposition. This means it can work with sparse matrices + efficiently. + + In particular, truncated SVD works on term count/tf-idf matrices as + returned by the vectorizers in :mod:`sklearn.feature_extraction.text`. In + that context, it is known as latent semantic analysis (LSA). + + This estimator supports two algorithms: a fast randomized SVD solver, and + a "naive" algorithm that uses ARPACK as an eigensolver on `X * X.T` or + `X.T * X`, whichever is more efficient. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int, default=2 + Desired dimensionality of output data. + If algorithm='arpack', must be strictly less than the number of features. + If algorithm='randomized', must be less than or equal to the number of features. + The default value is useful for visualisation. For LSA, a value of + 100 is recommended. + + algorithm : {'arpack', 'randomized'}, default='randomized' + SVD solver to use. Either "arpack" for the ARPACK wrapper in SciPy + (scipy.sparse.linalg.svds), or "randomized" for the randomized + algorithm due to Halko (2009). + + n_iter : int, default=5 + Number of iterations for randomized SVD solver. Not used by ARPACK. The + default is larger than the default in + :func:`~sklearn.utils.extmath.randomized_svd` to handle sparse + matrices that may have large slowly decaying spectrum. + + n_oversamples : int, default=10 + Number of oversamples for randomized SVD solver. Not used by ARPACK. + See :func:`~sklearn.utils.extmath.randomized_svd` for a complete + description. + + .. versionadded:: 1.1 + + power_iteration_normalizer : {'auto', 'QR', 'LU', 'none'}, default='auto' + Power iteration normalizer for randomized SVD solver. + Not used by ARPACK. See :func:`~sklearn.utils.extmath.randomized_svd` + for more details. + + .. versionadded:: 1.1 + + random_state : int, RandomState instance or None, default=None + Used during randomized svd. Pass an int for reproducible results across + multiple function calls. + See :term:`Glossary `. + + tol : float, default=0.0 + Tolerance for ARPACK. 0 means machine precision. Ignored by randomized + SVD solver. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + The right singular vectors of the input data. + + explained_variance_ : ndarray of shape (n_components,) + The variance of the training samples transformed by a projection to + each component. + + explained_variance_ratio_ : ndarray of shape (n_components,) + Percentage of variance explained by each of the selected components. + + singular_values_ : ndarray of shape (n_components,) + The singular values corresponding to each of the selected components. + The singular values are equal to the 2-norms of the ``n_components`` + variables in the lower-dimensional space. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + FactorAnalysis : A simple linear generative model with + Gaussian latent variables. + IncrementalPCA : Incremental principal components analysis. + KernelPCA : Kernel Principal component analysis. + NMF : Non-Negative Matrix Factorization. + PCA : Principal component analysis. + + Notes + ----- + SVD suffers from a problem called "sign indeterminacy", which means the + sign of the ``components_`` and the output from transform depend on the + algorithm and random state. To work around this, fit instances of this + class to data once, then keep the instance around to do transformations. + + References + ---------- + :arxiv:`Halko, et al. (2009). "Finding structure with randomness: + Stochastic algorithms for constructing approximate matrix decompositions" + <0909.4061>` + + Examples + -------- + >>> from sklearn.decomposition import TruncatedSVD + >>> from scipy.sparse import csr_matrix + >>> import numpy as np + >>> np.random.seed(0) + >>> X_dense = np.random.rand(100, 100) + >>> X_dense[:, 2 * np.arange(50)] = 0 + >>> X = csr_matrix(X_dense) + >>> svd = TruncatedSVD(n_components=5, n_iter=7, random_state=42) + >>> svd.fit(X) + TruncatedSVD(n_components=5, n_iter=7, random_state=42) + >>> print(svd.explained_variance_ratio_) + [0.0157 0.0512 0.0499 0.0479 0.0453] + >>> print(svd.explained_variance_ratio_.sum()) + 0.2102 + >>> print(svd.singular_values_) + [35.2410 4.5981 4.5420 4.4486 4.3288] + """ + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left")], + "algorithm": [StrOptions({"arpack", "randomized"})], + "n_iter": [Interval(Integral, 0, None, closed="left")], + "n_oversamples": [Interval(Integral, 1, None, closed="left")], + "power_iteration_normalizer": [StrOptions({"auto", "OR", "LU", "none"})], + "random_state": ["random_state"], + "tol": [Interval(Real, 0, None, closed="left")], + } + + def __init__( + self, + n_components=2, + *, + algorithm="randomized", + n_iter=5, + n_oversamples=10, + power_iteration_normalizer="auto", + random_state=None, + tol=0.0, + ): + self.algorithm = algorithm + self.n_components = n_components + self.n_iter = n_iter + self.n_oversamples = n_oversamples + self.power_iteration_normalizer = power_iteration_normalizer + self.random_state = random_state + self.tol = tol + + def fit(self, X, y=None): + """Fit model on training data X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data. + + y : Ignored + Not used, present here for API consistency by convention. + + Returns + ------- + self : object + Returns the transformer object. + """ + self.fit_transform(X) + return self + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None): + """Fit model to X and perform dimensionality reduction on X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data. + + y : Ignored + Not used, present here for API consistency by convention. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Reduced version of X. This will always be a dense array. + """ + X = validate_data(self, X, accept_sparse=["csr", "csc"], ensure_min_features=2) + random_state = check_random_state(self.random_state) + + if self.algorithm == "arpack": + v0 = _init_arpack_v0(min(X.shape), random_state) + U, Sigma, VT = svds(X, k=self.n_components, tol=self.tol, v0=v0) + # svds doesn't abide by scipy.linalg.svd/randomized_svd + # conventions, so reverse its outputs. + Sigma = Sigma[::-1] + # u_based_decision=False is needed to be consistent with PCA. + U, VT = svd_flip(U[:, ::-1], VT[::-1], u_based_decision=False) + + elif self.algorithm == "randomized": + if self.n_components > X.shape[1]: + raise ValueError( + f"n_components({self.n_components}) must be <=" + f" n_features({X.shape[1]})." + ) + U, Sigma, VT = _randomized_svd( + X, + self.n_components, + n_iter=self.n_iter, + n_oversamples=self.n_oversamples, + power_iteration_normalizer=self.power_iteration_normalizer, + random_state=random_state, + flip_sign=False, + ) + U, VT = svd_flip(U, VT, u_based_decision=False) + + self.components_ = VT + + # As a result of the SVD approximation error on X ~ U @ Sigma @ V.T, + # X @ V is not the same as U @ Sigma + if self.algorithm == "randomized" or ( + self.algorithm == "arpack" and self.tol > 0 + ): + X_transformed = safe_sparse_dot(X, self.components_.T) + else: + X_transformed = U * Sigma + + # Calculate explained variance & explained variance ratio + self.explained_variance_ = exp_var = np.var(X_transformed, axis=0) + if sp.issparse(X): + _, full_var = mean_variance_axis(X, axis=0) + full_var = full_var.sum() + else: + full_var = np.var(X, axis=0).sum() + self.explained_variance_ratio_ = exp_var / full_var + self.singular_values_ = Sigma # Store the singular values. + + return X_transformed + + def transform(self, X): + """Perform dimensionality reduction on X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + New data. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Reduced version of X. This will always be a dense array. + """ + check_is_fitted(self) + X = validate_data(self, X, accept_sparse=["csr", "csc"], reset=False) + return safe_sparse_dot(X, self.components_.T) + + def inverse_transform(self, X): + """Transform X back to its original space. + + Returns an array X_original whose transform would be X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_components) + New data. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Note that this is always a dense array. + """ + X = check_array(X) + return np.dot(X, self.components_) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/meson.build b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/meson.build new file mode 100644 index 0000000000000000000000000000000000000000..8437f9b30d3aa3ac91a456d4ed009c28d1bace48 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/meson.build @@ -0,0 +1,14 @@ +py.extension_module( + '_online_lda_fast', + [cython_gen.process('_online_lda_fast.pyx'), utils_cython_tree], + subdir: 'sklearn/decomposition', + install: true +) + +py.extension_module( + '_cdnmf_fast', + cython_gen.process('_cdnmf_fast.pyx'), + dependencies: [np_dep], + subdir: 'sklearn/decomposition', + install: true +) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_dict_learning.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_dict_learning.py new file mode 100644 index 0000000000000000000000000000000000000000..2c85cc4629a41c88fd6d3484b34543b88c25d1ea --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_dict_learning.py @@ -0,0 +1,1015 @@ +import itertools +import warnings +from functools import partial + +import numpy as np +import pytest + +import sklearn +from sklearn.base import clone +from sklearn.decomposition import ( + DictionaryLearning, + MiniBatchDictionaryLearning, + SparseCoder, + dict_learning, + dict_learning_online, + sparse_encode, +) +from sklearn.decomposition._dict_learning import _update_dict +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils import check_array +from sklearn.utils._testing import ( + TempMemmap, + assert_allclose, + assert_array_almost_equal, + assert_array_equal, + ignore_warnings, +) +from sklearn.utils.estimator_checks import ( + check_transformer_data_not_an_array, + check_transformer_general, + check_transformers_unfitted, +) +from sklearn.utils.parallel import Parallel + +rng_global = np.random.RandomState(0) +n_samples, n_features = 10, 8 +X = rng_global.randn(n_samples, n_features) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +def test_sparse_encode_shapes_omp(): + rng = np.random.RandomState(0) + algorithms = ["omp", "lasso_lars", "lasso_cd", "lars", "threshold"] + for n_components, n_samples in itertools.product([1, 5], [1, 9]): + X_ = rng.randn(n_samples, n_features) + dictionary = rng.randn(n_components, n_features) + for algorithm, n_jobs in itertools.product(algorithms, [1, 2]): + code = sparse_encode(X_, dictionary, algorithm=algorithm, n_jobs=n_jobs) + assert code.shape == (n_samples, n_components) + + +def test_dict_learning_shapes(): + n_components = 5 + dico = DictionaryLearning(n_components, random_state=0).fit(X) + assert dico.components_.shape == (n_components, n_features) + + n_components = 1 + dico = DictionaryLearning(n_components, random_state=0).fit(X) + assert dico.components_.shape == (n_components, n_features) + assert dico.transform(X).shape == (X.shape[0], n_components) + + +def test_dict_learning_overcomplete(): + n_components = 12 + dico = DictionaryLearning(n_components, random_state=0).fit(X) + assert dico.components_.shape == (n_components, n_features) + + +def test_max_iter(): + def ricker_function(resolution, center, width): + """Discrete sub-sampled Ricker (Mexican hat) wavelet""" + x = np.linspace(0, resolution - 1, resolution) + x = ( + (2 / (np.sqrt(3 * width) * np.pi**0.25)) + * (1 - (x - center) ** 2 / width**2) + * np.exp(-((x - center) ** 2) / (2 * width**2)) + ) + return x + + def ricker_matrix(width, resolution, n_components): + """Dictionary of Ricker (Mexican hat) wavelets""" + centers = np.linspace(0, resolution - 1, n_components) + D = np.empty((n_components, resolution)) + for i, center in enumerate(centers): + D[i] = ricker_function(resolution, center, width) + D /= np.sqrt(np.sum(D**2, axis=1))[:, np.newaxis] + return D + + transform_algorithm = "lasso_cd" + resolution = 256 + subsampling = 3 # subsampling factor + n_components = resolution // subsampling + + # Compute a wavelet dictionary + D_multi = np.r_[ + tuple( + ricker_matrix( + width=w, resolution=resolution, n_components=n_components // 5 + ) + for w in (10, 50, 100, 500) + ) + ] + + X = np.linspace(0, resolution - 1, resolution) + first_quarter = X < resolution / 4 + X[first_quarter] = 3.0 + X[np.logical_not(first_quarter)] = -1.0 + X = X.reshape(1, -1) + + # check that the underlying model fails to converge + with pytest.warns(ConvergenceWarning): + model = SparseCoder( + D_multi, transform_algorithm=transform_algorithm, transform_max_iter=1 + ) + model.fit_transform(X) + + # check that the underlying model converges w/o warnings + with warnings.catch_warnings(): + warnings.simplefilter("error", ConvergenceWarning) + model = SparseCoder( + D_multi, transform_algorithm=transform_algorithm, transform_max_iter=500 + ) + model.fit_transform(X) + + +def test_dict_learning_lars_positive_parameter(): + n_components = 5 + alpha = 1 + err_msg = "Positive constraint not supported for 'lars' coding method." + with pytest.raises(ValueError, match=err_msg): + dict_learning(X, n_components, alpha=alpha, positive_code=True) + + +@pytest.mark.parametrize( + "transform_algorithm", + [ + "lasso_lars", + "lasso_cd", + "threshold", + ], +) +@pytest.mark.parametrize("positive_code", [False, True]) +@pytest.mark.parametrize("positive_dict", [False, True]) +def test_dict_learning_positivity(transform_algorithm, positive_code, positive_dict): + n_components = 5 + dico = DictionaryLearning( + n_components, + transform_algorithm=transform_algorithm, + random_state=0, + positive_code=positive_code, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) + + code = dico.transform(X) + if positive_dict: + assert (dico.components_ >= 0).all() + else: + assert (dico.components_ < 0).any() + if positive_code: + assert (code >= 0).all() + else: + assert (code < 0).any() + + +@pytest.mark.parametrize("positive_dict", [False, True]) +def test_dict_learning_lars_dict_positivity(positive_dict): + n_components = 5 + dico = DictionaryLearning( + n_components, + transform_algorithm="lars", + random_state=0, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) + + if positive_dict: + assert (dico.components_ >= 0).all() + else: + assert (dico.components_ < 0).any() + + +def test_dict_learning_lars_code_positivity(): + n_components = 5 + dico = DictionaryLearning( + n_components, + transform_algorithm="lars", + random_state=0, + positive_code=True, + fit_algorithm="cd", + ).fit(X) + + err_msg = "Positive constraint not supported for '{}' coding method." + err_msg = err_msg.format("lars") + with pytest.raises(ValueError, match=err_msg): + dico.transform(X) + + +def test_dict_learning_reconstruction(): + n_components = 12 + dico = DictionaryLearning( + n_components, transform_algorithm="omp", transform_alpha=0.001, random_state=0 + ) + code = dico.fit(X).transform(X) + assert_array_almost_equal(np.dot(code, dico.components_), X) + assert_array_almost_equal(dico.inverse_transform(code), X) + + dico.set_params(transform_algorithm="lasso_lars") + code = dico.transform(X) + assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2) + assert_array_almost_equal(dico.inverse_transform(code), X, decimal=2) + + # test error raised for wrong code size + with pytest.raises(ValueError, match="Expected 12, got 11."): + dico.inverse_transform(code[:, :-1]) + + # used to test lars here too, but there's no guarantee the number of + # nonzero atoms is right. + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +def test_dict_learning_reconstruction_parallel(): + # regression test that parallel reconstruction works with n_jobs>1 + n_components = 12 + dico = DictionaryLearning( + n_components, + transform_algorithm="omp", + transform_alpha=0.001, + random_state=0, + n_jobs=4, + ) + code = dico.fit(X).transform(X) + assert_array_almost_equal(np.dot(code, dico.components_), X) + + dico.set_params(transform_algorithm="lasso_lars") + code = dico.transform(X) + assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +def test_dict_learning_lassocd_readonly_data(): + n_components = 12 + with TempMemmap(X) as X_read_only: + dico = DictionaryLearning( + n_components, + transform_algorithm="lasso_cd", + transform_alpha=0.001, + random_state=0, + n_jobs=4, + ) + with ignore_warnings(category=ConvergenceWarning): + code = dico.fit(X_read_only).transform(X_read_only) + assert_array_almost_equal( + np.dot(code, dico.components_), X_read_only, decimal=2 + ) + + +def test_dict_learning_nonzero_coefs(): + n_components = 4 + dico = DictionaryLearning( + n_components, + transform_algorithm="lars", + transform_n_nonzero_coefs=3, + random_state=0, + ) + code = dico.fit(X).transform(X[np.newaxis, 1]) + assert len(np.flatnonzero(code)) == 3 + + dico.set_params(transform_algorithm="omp") + code = dico.transform(X[np.newaxis, 1]) + assert len(np.flatnonzero(code)) == 3 + + +def test_dict_learning_split(): + n_components = 5 + dico = DictionaryLearning( + n_components, transform_algorithm="threshold", random_state=0 + ) + code = dico.fit(X).transform(X) + Xr = dico.inverse_transform(code) + + dico.split_sign = True + split_code = dico.transform(X) + + assert_array_almost_equal( + split_code[:, :n_components] - split_code[:, n_components:], code + ) + + Xr2 = dico.inverse_transform(split_code) + assert_array_almost_equal(Xr, Xr2) + + +def test_dict_learning_online_shapes(): + rng = np.random.RandomState(0) + n_components = 8 + + code, dictionary = dict_learning_online( + X, + n_components=n_components, + batch_size=4, + max_iter=10, + method="cd", + random_state=rng, + return_code=True, + ) + assert code.shape == (n_samples, n_components) + assert dictionary.shape == (n_components, n_features) + assert np.dot(code, dictionary).shape == X.shape + + dictionary = dict_learning_online( + X, + n_components=n_components, + batch_size=4, + max_iter=10, + method="cd", + random_state=rng, + return_code=False, + ) + assert dictionary.shape == (n_components, n_features) + + +def test_dict_learning_online_lars_positive_parameter(): + err_msg = "Positive constraint not supported for 'lars' coding method." + with pytest.raises(ValueError, match=err_msg): + dict_learning_online(X, batch_size=4, max_iter=10, positive_code=True) + + +@pytest.mark.parametrize( + "transform_algorithm", + [ + "lasso_lars", + "lasso_cd", + "threshold", + ], +) +@pytest.mark.parametrize("positive_code", [False, True]) +@pytest.mark.parametrize("positive_dict", [False, True]) +def test_minibatch_dictionary_learning_positivity( + transform_algorithm, positive_code, positive_dict +): + n_components = 8 + dico = MiniBatchDictionaryLearning( + n_components, + batch_size=4, + max_iter=10, + transform_algorithm=transform_algorithm, + random_state=0, + positive_code=positive_code, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) + + code = dico.transform(X) + if positive_dict: + assert (dico.components_ >= 0).all() + else: + assert (dico.components_ < 0).any() + if positive_code: + assert (code >= 0).all() + else: + assert (code < 0).any() + + +@pytest.mark.parametrize("positive_dict", [False, True]) +def test_minibatch_dictionary_learning_lars(positive_dict): + n_components = 8 + + dico = MiniBatchDictionaryLearning( + n_components, + batch_size=4, + max_iter=10, + transform_algorithm="lars", + random_state=0, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) + + if positive_dict: + assert (dico.components_ >= 0).all() + else: + assert (dico.components_ < 0).any() + + +@pytest.mark.parametrize("positive_code", [False, True]) +@pytest.mark.parametrize("positive_dict", [False, True]) +def test_dict_learning_online_positivity(positive_code, positive_dict): + rng = np.random.RandomState(0) + n_components = 8 + + code, dictionary = dict_learning_online( + X, + n_components=n_components, + batch_size=4, + method="cd", + alpha=1, + random_state=rng, + positive_dict=positive_dict, + positive_code=positive_code, + ) + if positive_dict: + assert (dictionary >= 0).all() + else: + assert (dictionary < 0).any() + if positive_code: + assert (code >= 0).all() + else: + assert (code < 0).any() + + +def test_dict_learning_online_verbosity(): + # test verbosity for better coverage + n_components = 5 + import sys + from io import StringIO + + old_stdout = sys.stdout + try: + sys.stdout = StringIO() + + # convergence monitoring verbosity + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, verbose=1, tol=0.1, random_state=0 + ) + dico.fit(X) + dico = MiniBatchDictionaryLearning( + n_components, + batch_size=4, + max_iter=5, + verbose=1, + max_no_improvement=2, + random_state=0, + ) + dico.fit(X) + # higher verbosity level + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, verbose=2, random_state=0 + ) + dico.fit(X) + + # function API verbosity + dict_learning_online( + X, + n_components=n_components, + batch_size=4, + alpha=1, + verbose=1, + random_state=0, + ) + dict_learning_online( + X, + n_components=n_components, + batch_size=4, + alpha=1, + verbose=2, + random_state=0, + ) + finally: + sys.stdout = old_stdout + + assert dico.components_.shape == (n_components, n_features) + + +def test_dict_learning_online_estimator_shapes(): + n_components = 5 + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, random_state=0 + ) + dico.fit(X) + assert dico.components_.shape == (n_components, n_features) + + +def test_dict_learning_online_overcomplete(): + n_components = 12 + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, random_state=0 + ).fit(X) + assert dico.components_.shape == (n_components, n_features) + + +def test_dict_learning_online_initialization(): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=0, dict_init=V, random_state=0 + ).fit(X) + assert_array_equal(dico.components_, V) + + +def test_dict_learning_online_readonly_initialization(): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) + V.setflags(write=False) + MiniBatchDictionaryLearning( + n_components, + batch_size=4, + max_iter=1, + dict_init=V, + random_state=0, + shuffle=False, + ).fit(X) + + +def test_dict_learning_online_partial_fit(): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + dict1 = MiniBatchDictionaryLearning( + n_components, + max_iter=10, + batch_size=1, + alpha=1, + shuffle=False, + dict_init=V, + max_no_improvement=None, + tol=0.0, + random_state=0, + ).fit(X) + dict2 = MiniBatchDictionaryLearning( + n_components, alpha=1, dict_init=V, random_state=0 + ) + for i in range(10): + for sample in X: + dict2.partial_fit(sample[np.newaxis, :]) + + assert not np.all(sparse_encode(X, dict1.components_, alpha=1) == 0) + assert_array_almost_equal(dict1.components_, dict2.components_, decimal=2) + + # partial_fit should ignore max_iter (#17433) + assert dict1.n_steps_ == dict2.n_steps_ == 100 + + +def test_sparse_encode_shapes(): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + for algo in ("lasso_lars", "lasso_cd", "lars", "omp", "threshold"): + code = sparse_encode(X, V, algorithm=algo) + assert code.shape == (n_samples, n_components) + + +@pytest.mark.parametrize("algo", ["lasso_lars", "lasso_cd", "threshold"]) +@pytest.mark.parametrize("positive", [False, True]) +def test_sparse_encode_positivity(algo, positive): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + code = sparse_encode(X, V, algorithm=algo, positive=positive) + if positive: + assert (code >= 0).all() + else: + assert (code < 0).any() + + +@pytest.mark.parametrize("algo", ["lars", "omp"]) +def test_sparse_encode_unavailable_positivity(algo): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + err_msg = "Positive constraint not supported for '{}' coding method." + err_msg = err_msg.format(algo) + with pytest.raises(ValueError, match=err_msg): + sparse_encode(X, V, algorithm=algo, positive=True) + + +def test_sparse_encode_input(): + n_components = 100 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + Xf = check_array(X, order="F") + for algo in ("lasso_lars", "lasso_cd", "lars", "omp", "threshold"): + a = sparse_encode(X, V, algorithm=algo) + b = sparse_encode(Xf, V, algorithm=algo) + assert_array_almost_equal(a, b) + + +def test_sparse_encode_error(): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + code = sparse_encode(X, V, alpha=0.001) + assert not np.all(code == 0) + assert np.sqrt(np.sum((np.dot(code, V) - X) ** 2)) < 0.1 + + +def test_sparse_encode_error_default_sparsity(): + rng = np.random.RandomState(0) + X = rng.randn(100, 64) + D = rng.randn(2, 64) + code = ignore_warnings(sparse_encode)(X, D, algorithm="omp", n_nonzero_coefs=None) + assert code.shape == (100, 2) + + +def test_sparse_coder_estimator(): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.randn(n_components, n_features) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + coder = SparseCoder( + dictionary=V, transform_algorithm="lasso_lars", transform_alpha=0.001 + ) + code = coder.fit_transform(X) + Xr = coder.inverse_transform(code) + assert not np.all(code == 0) + assert np.sqrt(np.sum((np.dot(code, V) - X) ** 2)) < 0.1 + np.testing.assert_allclose(Xr, np.dot(code, V)) + + +def test_sparse_coder_estimator_clone(): + n_components = 12 + rng = np.random.RandomState(0) + V = rng.normal(size=(n_components, n_features)) # random init + V /= np.sum(V**2, axis=1)[:, np.newaxis] + coder = SparseCoder( + dictionary=V, transform_algorithm="lasso_lars", transform_alpha=0.001 + ) + cloned = clone(coder) + assert id(cloned) != id(coder) + np.testing.assert_allclose(cloned.dictionary, coder.dictionary) + assert id(cloned.dictionary) != id(coder.dictionary) + data = np.random.rand(n_samples, n_features).astype(np.float32) + np.testing.assert_allclose(cloned.transform(data), coder.transform(data)) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +def test_sparse_coder_parallel_mmap(): + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/5956 + # Test that SparseCoder does not error by passing reading only + # arrays to child processes + + rng = np.random.RandomState(777) + n_components, n_features = 40, 64 + init_dict = rng.rand(n_components, n_features) + # Ensure that `data` is >2M. Joblib memory maps arrays + # if they are larger than 1MB. The 4 accounts for float32 + # data type + n_samples = int(2e6) // (4 * n_features) + data = np.random.rand(n_samples, n_features).astype(np.float32) + + sc = SparseCoder(init_dict, transform_algorithm="omp", n_jobs=2) + sc.fit_transform(data) + + +def test_sparse_coder_common_transformer(): + rng = np.random.RandomState(777) + n_components, n_features = 40, 3 + init_dict = rng.rand(n_components, n_features) + + sc = SparseCoder(init_dict) + + check_transformer_data_not_an_array(sc.__class__.__name__, sc) + check_transformer_general(sc.__class__.__name__, sc) + check_transformer_general_memmap = partial( + check_transformer_general, readonly_memmap=True + ) + check_transformer_general_memmap(sc.__class__.__name__, sc) + check_transformers_unfitted(sc.__class__.__name__, sc) + + +def test_sparse_coder_n_features_in(): + d = np.array([[1, 2, 3], [1, 2, 3]]) + X = np.array([[1, 2, 3]]) + sc = SparseCoder(d) + sc.fit(X) + assert sc.n_features_in_ == d.shape[1] + + +def test_sparse_encoder_feature_number_error(): + n_components = 10 + rng = np.random.RandomState(0) + D = rng.uniform(size=(n_components, n_features)) + X = rng.uniform(size=(n_samples, n_features + 1)) + coder = SparseCoder(D) + with pytest.raises( + ValueError, match="Dictionary and X have different numbers of features" + ): + coder.fit(X) + + +def test_update_dict(): + # Check the dict update in batch mode vs online mode + # Non-regression test for #4866 + rng = np.random.RandomState(0) + + code = np.array([[0.5, -0.5], [0.1, 0.9]]) + dictionary = np.array([[1.0, 0.0], [0.6, 0.8]]) + + X = np.dot(code, dictionary) + rng.randn(2, 2) + + # full batch update + newd_batch = dictionary.copy() + _update_dict(newd_batch, X, code) + + # online update + A = np.dot(code.T, code) + B = np.dot(X.T, code) + newd_online = dictionary.copy() + _update_dict(newd_online, X, code, A, B) + + assert_allclose(newd_batch, newd_online) + + +@pytest.mark.parametrize( + "algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize("data_type", (np.float32, np.float64)) +# Note: do not check integer input because `lasso_lars` and `lars` fail with +# `ValueError` in `_lars_path_solver` +def test_sparse_encode_dtype_match(data_type, algorithm): + n_components = 6 + rng = np.random.RandomState(0) + dictionary = rng.randn(n_components, n_features) + code = sparse_encode( + X.astype(data_type), dictionary.astype(data_type), algorithm=algorithm + ) + assert code.dtype == data_type + + +@pytest.mark.parametrize( + "algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +def test_sparse_encode_numerical_consistency(algorithm): + # verify numerical consistency among np.float32 and np.float64 + rtol = 1e-4 + n_components = 6 + rng = np.random.RandomState(0) + dictionary = rng.randn(n_components, n_features) + code_32 = sparse_encode( + X.astype(np.float32), dictionary.astype(np.float32), algorithm=algorithm + ) + code_64 = sparse_encode( + X.astype(np.float64), dictionary.astype(np.float64), algorithm=algorithm + ) + assert_allclose(code_32, code_64, rtol=rtol) + + +@pytest.mark.parametrize( + "transform_algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize("data_type", (np.float32, np.float64)) +# Note: do not check integer input because `lasso_lars` and `lars` fail with +# `ValueError` in `_lars_path_solver` +def test_sparse_coder_dtype_match(data_type, transform_algorithm): + # Verify preserving dtype for transform in sparse coder + n_components = 6 + rng = np.random.RandomState(0) + dictionary = rng.randn(n_components, n_features) + coder = SparseCoder( + dictionary.astype(data_type), transform_algorithm=transform_algorithm + ) + code = coder.transform(X.astype(data_type)) + assert code.dtype == data_type + + +@pytest.mark.parametrize("fit_algorithm", ("lars", "cd")) +@pytest.mark.parametrize( + "transform_algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_dictionary_learning_dtype_match( + data_type, + expected_type, + fit_algorithm, + transform_algorithm, +): + # Verify preserving dtype for fit and transform in dictionary learning class + dict_learner = DictionaryLearning( + n_components=8, + fit_algorithm=fit_algorithm, + transform_algorithm=transform_algorithm, + random_state=0, + ) + dict_learner.fit(X.astype(data_type)) + assert dict_learner.components_.dtype == expected_type + assert dict_learner.transform(X.astype(data_type)).dtype == expected_type + + +@pytest.mark.parametrize("fit_algorithm", ("lars", "cd")) +@pytest.mark.parametrize( + "transform_algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_minibatch_dictionary_learning_dtype_match( + data_type, + expected_type, + fit_algorithm, + transform_algorithm, +): + # Verify preserving dtype for fit and transform in minibatch dictionary learning + dict_learner = MiniBatchDictionaryLearning( + n_components=8, + batch_size=10, + fit_algorithm=fit_algorithm, + transform_algorithm=transform_algorithm, + max_iter=100, + tol=1e-1, + random_state=0, + ) + dict_learner.fit(X.astype(data_type)) + + assert dict_learner.components_.dtype == expected_type + assert dict_learner.transform(X.astype(data_type)).dtype == expected_type + assert dict_learner._A.dtype == expected_type + assert dict_learner._B.dtype == expected_type + + +@pytest.mark.parametrize("method", ("lars", "cd")) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_dict_learning_dtype_match(data_type, expected_type, method): + # Verify output matrix dtype + rng = np.random.RandomState(0) + n_components = 8 + code, dictionary, _ = dict_learning( + X.astype(data_type), + n_components=n_components, + alpha=1, + random_state=rng, + method=method, + ) + assert code.dtype == expected_type + assert dictionary.dtype == expected_type + + +@pytest.mark.parametrize("method", ("lars", "cd")) +def test_dict_learning_numerical_consistency(method): + # verify numerically consistent among np.float32 and np.float64 + rtol = 1e-4 + n_components = 4 + alpha = 2 + + U_64, V_64, _ = dict_learning( + X.astype(np.float64), + n_components=n_components, + alpha=alpha, + random_state=0, + method=method, + ) + U_32, V_32, _ = dict_learning( + X.astype(np.float32), + n_components=n_components, + alpha=alpha, + random_state=0, + method=method, + ) + + # Optimal solution (U*, V*) is not unique. + # If (U*, V*) is optimal solution, (-U*,-V*) is also optimal, + # and (column permutated U*, row permutated V*) are also optional + # as long as holding UV. + # So here UV, ||U||_1,1 and sum(||V_k||_2^2) are verified + # instead of comparing directly U and V. + assert_allclose(np.matmul(U_64, V_64), np.matmul(U_32, V_32), rtol=rtol) + assert_allclose(np.sum(np.abs(U_64)), np.sum(np.abs(U_32)), rtol=rtol) + assert_allclose(np.sum(V_64**2), np.sum(V_32**2), rtol=rtol) + # verify an obtained solution is not degenerate + assert np.mean(U_64 != 0.0) > 0.05 + assert np.count_nonzero(U_64 != 0.0) == np.count_nonzero(U_32 != 0.0) + + +@pytest.mark.parametrize("method", ("lars", "cd")) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_dict_learning_online_dtype_match(data_type, expected_type, method): + # Verify output matrix dtype + rng = np.random.RandomState(0) + n_components = 8 + code, dictionary = dict_learning_online( + X.astype(data_type), + n_components=n_components, + alpha=1, + batch_size=10, + random_state=rng, + method=method, + ) + assert code.dtype == expected_type + assert dictionary.dtype == expected_type + + +@pytest.mark.parametrize("method", ("lars", "cd")) +def test_dict_learning_online_numerical_consistency(method): + # verify numerically consistent among np.float32 and np.float64 + rtol = 1e-4 + n_components = 4 + alpha = 1 + + U_64, V_64 = dict_learning_online( + X.astype(np.float64), + n_components=n_components, + max_iter=1_000, + alpha=alpha, + batch_size=10, + random_state=0, + method=method, + tol=0.0, + max_no_improvement=None, + ) + U_32, V_32 = dict_learning_online( + X.astype(np.float32), + n_components=n_components, + max_iter=1_000, + alpha=alpha, + batch_size=10, + random_state=0, + method=method, + tol=0.0, + max_no_improvement=None, + ) + + # Optimal solution (U*, V*) is not unique. + # If (U*, V*) is optimal solution, (-U*,-V*) is also optimal, + # and (column permutated U*, row permutated V*) are also optional + # as long as holding UV. + # So here UV, ||U||_1,1 and sum(||V_k||_2) are verified + # instead of comparing directly U and V. + assert_allclose(np.matmul(U_64, V_64), np.matmul(U_32, V_32), rtol=rtol) + assert_allclose(np.sum(np.abs(U_64)), np.sum(np.abs(U_32)), rtol=rtol) + assert_allclose(np.sum(V_64**2), np.sum(V_32**2), rtol=rtol) + # verify an obtained solution is not degenerate + assert np.mean(U_64 != 0.0) > 0.05 + assert np.count_nonzero(U_64 != 0.0) == np.count_nonzero(U_32 != 0.0) + + +@pytest.mark.parametrize( + "estimator", + [ + SparseCoder(rng_global.uniform(size=(n_features, n_features))), + DictionaryLearning(), + MiniBatchDictionaryLearning(batch_size=4, max_iter=10), + ], + ids=lambda x: x.__class__.__name__, +) +def test_get_feature_names_out(estimator): + """Check feature names for dict learning estimators.""" + estimator.fit(X) + n_components = X.shape[1] + + feature_names_out = estimator.get_feature_names_out() + estimator_name = estimator.__class__.__name__.lower() + assert_array_equal( + feature_names_out, + [f"{estimator_name}{i}" for i in range(n_components)], + ) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +def test_cd_work_on_joblib_memmapped_data(monkeypatch): + monkeypatch.setattr( + sklearn.decomposition._dict_learning, + "Parallel", + partial(Parallel, max_nbytes=100), + ) + + rng = np.random.RandomState(0) + X_train = rng.randn(10, 10) + + dict_learner = DictionaryLearning( + n_components=5, + random_state=0, + n_jobs=2, + fit_algorithm="cd", + max_iter=50, + verbose=True, + ) + + # This must run and complete without error. + dict_learner.fit(X_train) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_factor_analysis.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_factor_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..6f57c5bc1d84494b099719510ace721538570656 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_factor_analysis.py @@ -0,0 +1,109 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from itertools import combinations + +import numpy as np +import pytest + +from sklearn.decomposition import FactorAnalysis +from sklearn.decomposition._factor_analysis import _ortho_rotation +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils._testing import assert_almost_equal, assert_array_almost_equal + + +def test_factor_analysis(global_random_seed): + # Test FactorAnalysis ability to recover the data covariance structure + rng = np.random.RandomState(global_random_seed) + n_samples, n_features, n_components = 20, 5, 3 + + # Some random settings for the generative model + W = rng.randn(n_components, n_features) + # latent variable of dim 3, 20 of it + h = rng.randn(n_samples, n_components) + # using gamma to model different noise variance + # per component + noise = rng.gamma(1, size=n_features) * rng.randn(n_samples, n_features) + + # generate observations + # wlog, mean is 0 + X = np.dot(h, W) + noise + + fas = [] + for method in ["randomized", "lapack"]: + fa = FactorAnalysis(n_components=n_components, svd_method=method) + fa.fit(X) + fas.append(fa) + + X_t = fa.transform(X) + assert X_t.shape == (n_samples, n_components) + + assert_almost_equal(fa.loglike_[-1], fa.score_samples(X).sum()) + assert_almost_equal(fa.score_samples(X).mean(), fa.score(X)) + + diff = np.all(np.diff(fa.loglike_)) + assert diff > 0.0, "Log likelihood dif not increase" + + # Sample Covariance + scov = np.cov(X, rowvar=0.0, bias=1.0) + + # Model Covariance + mcov = fa.get_covariance() + diff = np.sum(np.abs(scov - mcov)) / W.size + assert diff < 0.2, "Mean absolute difference is %f" % diff + fa = FactorAnalysis( + n_components=n_components, noise_variance_init=np.ones(n_features) + ) + with pytest.raises(ValueError): + fa.fit(X[:, :2]) + + def f(x, y): + return np.abs(getattr(x, y)) # sign will not be equal + + fa1, fa2 = fas + for attr in ["loglike_", "components_", "noise_variance_"]: + assert_almost_equal(f(fa1, attr), f(fa2, attr)) + + fa1.max_iter = 1 + fa1.verbose = True + with pytest.warns(ConvergenceWarning): + fa1.fit(X) + + # Test get_covariance and get_precision with n_components == n_features + # with n_components < n_features and with n_components == 0 + for n_components in [0, 2, X.shape[1]]: + fa.n_components = n_components + fa.fit(X) + cov = fa.get_covariance() + precision = fa.get_precision() + assert_array_almost_equal(np.dot(cov, precision), np.eye(X.shape[1]), 12) + + # test rotation + n_components = 2 + + results, projections = {}, {} + for method in (None, "varimax", "quartimax"): + fa_var = FactorAnalysis(n_components=n_components, rotation=method) + results[method] = fa_var.fit_transform(X) + projections[method] = fa_var.get_covariance() + for rot1, rot2 in combinations([None, "varimax", "quartimax"], 2): + assert not np.allclose(results[rot1], results[rot2]) + assert np.allclose(projections[rot1], projections[rot2], atol=3) + + # test against R's psych::principal with rotate="varimax" + # (i.e., the values below stem from rotating the components in R) + # R's factor analysis returns quite different values; therefore, we only + # test the rotation itself + factors = np.array( + [ + [0.89421016, -0.35854928, -0.27770122, 0.03773647], + [-0.45081822, -0.89132754, 0.0932195, -0.01787973], + [0.99500666, -0.02031465, 0.05426497, -0.11539407], + [0.96822861, -0.06299656, 0.24411001, 0.07540887], + ] + ) + r_solution = np.array( + [[0.962, 0.052], [-0.141, 0.989], [0.949, -0.300], [0.937, -0.251]] + ) + rotated = _ortho_rotation(factors[:, :n_components], method="varimax").T + assert_array_almost_equal(np.abs(rotated), np.abs(r_solution), decimal=3) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_fastica.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_fastica.py new file mode 100644 index 0000000000000000000000000000000000000000..cc55e9caa65021c9a6278933e946f55ac7ace027 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_fastica.py @@ -0,0 +1,457 @@ +""" +Test the fastica algorithm. +""" + +import itertools +import os +import warnings + +import numpy as np +import pytest +from scipy import stats + +from sklearn.decomposition import PCA, FastICA, fastica +from sklearn.decomposition._fastica import _gs_decorrelation +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils._testing import assert_allclose, ignore_warnings + + +def center_and_norm(x, axis=-1): + """Centers and norms x **in place** + + Parameters + ----------- + x: ndarray + Array with an axis of observations (statistical units) measured on + random variables. + axis: int, optional + Axis along which the mean and variance are calculated. + """ + x = np.rollaxis(x, axis) + x -= x.mean(axis=0) + x /= x.std(axis=0) + + +def test_gs(global_random_seed): + # Test gram schmidt orthonormalization + # generate a random orthogonal matrix + rng = np.random.RandomState(global_random_seed) + W, _, _ = np.linalg.svd(rng.randn(10, 10)) + w = rng.randn(10) + _gs_decorrelation(w, W, 10) + assert (w**2).sum() < 1.0e-10 + w = rng.randn(10) + u = _gs_decorrelation(w, W, 5) + tmp = np.dot(u, W.T) + assert (tmp[:5] ** 2).sum() < 1.0e-10 + + +def test_fastica_attributes_dtypes(global_dtype): + rng = np.random.RandomState(0) + X = rng.random_sample((100, 10)).astype(global_dtype, copy=False) + fica = FastICA( + n_components=5, max_iter=1000, whiten="unit-variance", random_state=0 + ).fit(X) + assert fica.components_.dtype == global_dtype + assert fica.mixing_.dtype == global_dtype + assert fica.mean_.dtype == global_dtype + assert fica.whitening_.dtype == global_dtype + + +def test_fastica_return_dtypes(global_dtype): + rng = np.random.RandomState(0) + X = rng.random_sample((100, 10)).astype(global_dtype, copy=False) + k_, mixing_, s_ = fastica( + X, max_iter=1000, whiten="unit-variance", random_state=rng + ) + assert k_.dtype == global_dtype + assert mixing_.dtype == global_dtype + assert s_.dtype == global_dtype + + +@pytest.mark.parametrize("add_noise", [True, False]) +def test_fastica_simple(add_noise, global_random_seed, global_dtype): + if ( + global_random_seed == 20 + and global_dtype == np.float32 + and not add_noise + and os.getenv("DISTRIB") == "ubuntu" + ): + pytest.xfail( + "FastICA instability with Ubuntu Atlas build with float32 " + "global_dtype. For more details, see " + "https://github.com/scikit-learn/scikit-learn/issues/24131#issuecomment-1208091119" + ) + + # Test the FastICA algorithm on very simple data. + rng = np.random.RandomState(global_random_seed) + n_samples = 1000 + # Generate two sources: + s1 = (2 * np.sin(np.linspace(0, 100, n_samples)) > 0) - 1 + s2 = stats.t.rvs(1, size=n_samples, random_state=global_random_seed) + s = np.c_[s1, s2].T + center_and_norm(s) + s = s.astype(global_dtype) + s1, s2 = s + + # Mixing angle + phi = 0.6 + mixing = np.array([[np.cos(phi), np.sin(phi)], [np.sin(phi), -np.cos(phi)]]) + mixing = mixing.astype(global_dtype) + m = np.dot(mixing, s) + + if add_noise: + m += 0.1 * rng.randn(2, 1000) + + center_and_norm(m) + + # function as fun arg + def g_test(x): + return x**3, (3 * x**2).mean(axis=-1) + + algos = ["parallel", "deflation"] + nls = ["logcosh", "exp", "cube", g_test] + whitening = ["arbitrary-variance", "unit-variance", False] + for algo, nl, whiten in itertools.product(algos, nls, whitening): + if whiten: + k_, mixing_, s_ = fastica( + m.T, fun=nl, whiten=whiten, algorithm=algo, random_state=rng + ) + with pytest.raises(ValueError): + fastica(m.T, fun=np.tanh, whiten=whiten, algorithm=algo) + else: + pca = PCA(n_components=2, whiten=True, random_state=rng) + X = pca.fit_transform(m.T) + k_, mixing_, s_ = fastica( + X, fun=nl, algorithm=algo, whiten=False, random_state=rng + ) + with pytest.raises(ValueError): + fastica(X, fun=np.tanh, algorithm=algo) + s_ = s_.T + # Check that the mixing model described in the docstring holds: + if whiten: + # XXX: exact reconstruction to standard relative tolerance is not + # possible. This is probably expected when add_noise is True but we + # also need a non-trivial atol in float32 when add_noise is False. + # + # Note that the 2 sources are non-Gaussian in this test. + atol = 1e-5 if global_dtype == np.float32 else 0 + assert_allclose(np.dot(np.dot(mixing_, k_), m), s_, atol=atol) + + center_and_norm(s_) + s1_, s2_ = s_ + # Check to see if the sources have been estimated + # in the wrong order + if abs(np.dot(s1_, s2)) > abs(np.dot(s1_, s1)): + s2_, s1_ = s_ + s1_ *= np.sign(np.dot(s1_, s1)) + s2_ *= np.sign(np.dot(s2_, s2)) + + # Check that we have estimated the original sources + if not add_noise: + assert_allclose(np.dot(s1_, s1) / n_samples, 1, atol=1e-2) + assert_allclose(np.dot(s2_, s2) / n_samples, 1, atol=1e-2) + else: + assert_allclose(np.dot(s1_, s1) / n_samples, 1, atol=1e-1) + assert_allclose(np.dot(s2_, s2) / n_samples, 1, atol=1e-1) + + # Test FastICA class + _, _, sources_fun = fastica( + m.T, fun=nl, algorithm=algo, random_state=global_random_seed + ) + ica = FastICA(fun=nl, algorithm=algo, random_state=global_random_seed) + sources = ica.fit_transform(m.T) + assert ica.components_.shape == (2, 2) + assert sources.shape == (1000, 2) + + assert_allclose(sources_fun, sources) + # Set atol to account for the different magnitudes of the elements in sources + # (from 1e-4 to 1e1). + atol = np.max(np.abs(sources)) * (1e-5 if global_dtype == np.float32 else 1e-7) + assert_allclose(sources, ica.transform(m.T), atol=atol) + + assert ica.mixing_.shape == (2, 2) + + ica = FastICA(fun=np.tanh, algorithm=algo) + with pytest.raises(ValueError): + ica.fit(m.T) + + +def test_fastica_nowhiten(): + m = [[0, 1], [1, 0]] + + # test for issue #697 + ica = FastICA(n_components=1, whiten=False, random_state=0) + warn_msg = "Ignoring n_components with whiten=False." + with pytest.warns(UserWarning, match=warn_msg): + ica.fit(m) + assert hasattr(ica, "mixing_") + + +def test_fastica_convergence_fail(global_random_seed): + # Test the FastICA algorithm on very simple data + # (see test_non_square_fastica). + # Ensure a ConvergenceWarning raised if the tolerance is sufficiently low. + rng = np.random.RandomState(global_random_seed) + + n_samples = 1000 + # Generate two sources: + t = np.linspace(0, 100, n_samples) + s1 = np.sin(t) + s2 = np.ceil(np.sin(np.pi * t)) + s = np.c_[s1, s2].T + center_and_norm(s) + + # Mixing matrix + mixing = rng.randn(6, 2) + m = np.dot(mixing, s) + + # Do fastICA with tolerance 0. to ensure failing convergence + warn_msg = ( + "FastICA did not converge. Consider increasing tolerance " + "or the maximum number of iterations." + ) + with pytest.warns(ConvergenceWarning, match=warn_msg): + ica = FastICA( + algorithm="parallel", n_components=2, random_state=rng, max_iter=2, tol=0.0 + ) + ica.fit(m.T) + + +@pytest.mark.parametrize("add_noise", [True, False]) +def test_non_square_fastica(global_random_seed, add_noise): + # Test the FastICA algorithm on very simple data. + rng = np.random.RandomState(global_random_seed) + + n_samples = 1000 + # Generate two sources: + t = np.linspace(0, 100, n_samples) + s1 = np.sin(t) + s2 = np.ceil(np.sin(np.pi * t)) + s = np.c_[s1, s2].T + center_and_norm(s) + s1, s2 = s + + # Mixing matrix + mixing = rng.randn(6, 2) + m = np.dot(mixing, s) + + if add_noise: + m += 0.1 * rng.randn(6, n_samples) + + center_and_norm(m) + + k_, mixing_, s_ = fastica( + m.T, n_components=2, whiten="unit-variance", random_state=rng + ) + s_ = s_.T + + # Check that the mixing model described in the docstring holds: + assert_allclose(s_, np.dot(np.dot(mixing_, k_), m)) + + center_and_norm(s_) + s1_, s2_ = s_ + # Check to see if the sources have been estimated + # in the wrong order + if abs(np.dot(s1_, s2)) > abs(np.dot(s1_, s1)): + s2_, s1_ = s_ + s1_ *= np.sign(np.dot(s1_, s1)) + s2_ *= np.sign(np.dot(s2_, s2)) + + # Check that we have estimated the original sources + if not add_noise: + assert_allclose(np.dot(s1_, s1) / n_samples, 1, atol=1e-3) + assert_allclose(np.dot(s2_, s2) / n_samples, 1, atol=1e-3) + + +def test_fit_transform(global_random_seed, global_dtype): + """Test unit variance of transformed data using FastICA algorithm. + + Check that `fit_transform` gives the same result as applying + `fit` and then `transform`. + + Bug #13056 + """ + # multivariate uniform data in [0, 1] + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((100, 10)).astype(global_dtype) + max_iter = 300 + for whiten, n_components in [["unit-variance", 5], [False, None]]: + n_components_ = n_components if n_components is not None else X.shape[1] + + ica = FastICA( + n_components=n_components, max_iter=max_iter, whiten=whiten, random_state=0 + ) + with warnings.catch_warnings(): + # make sure that numerical errors do not cause sqrt of negative + # values + warnings.simplefilter("error", RuntimeWarning) + # XXX: for some seeds, the model does not converge. + # However this is not what we test here. + warnings.simplefilter("ignore", ConvergenceWarning) + Xt = ica.fit_transform(X) + assert ica.components_.shape == (n_components_, 10) + assert Xt.shape == (X.shape[0], n_components_) + + ica2 = FastICA( + n_components=n_components, max_iter=max_iter, whiten=whiten, random_state=0 + ) + with warnings.catch_warnings(): + # make sure that numerical errors do not cause sqrt of negative + # values + warnings.simplefilter("error", RuntimeWarning) + warnings.simplefilter("ignore", ConvergenceWarning) + ica2.fit(X) + assert ica2.components_.shape == (n_components_, 10) + Xt2 = ica2.transform(X) + + # XXX: we have to set atol for this test to pass for all seeds when + # fitting with float32 data. Is this revealing a bug? + if global_dtype: + atol = np.abs(Xt2).mean() / 1e6 + else: + atol = 0.0 # the default rtol is enough for float64 data + assert_allclose(Xt, Xt2, atol=atol) + + +@pytest.mark.filterwarnings("ignore:Ignoring n_components with whiten=False.") +@pytest.mark.parametrize( + "whiten, n_components, expected_mixing_shape", + [ + ("arbitrary-variance", 5, (10, 5)), + ("arbitrary-variance", 10, (10, 10)), + ("unit-variance", 5, (10, 5)), + ("unit-variance", 10, (10, 10)), + (False, 5, (10, 10)), + (False, 10, (10, 10)), + ], +) +def test_inverse_transform( + whiten, n_components, expected_mixing_shape, global_random_seed, global_dtype +): + # Test FastICA.inverse_transform + n_samples = 100 + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((n_samples, 10)).astype(global_dtype) + + ica = FastICA(n_components=n_components, random_state=rng, whiten=whiten) + with warnings.catch_warnings(): + # For some dataset (depending on the value of global_dtype) the model + # can fail to converge but this should not impact the definition of + # a valid inverse transform. + warnings.simplefilter("ignore", ConvergenceWarning) + Xt = ica.fit_transform(X) + assert ica.mixing_.shape == expected_mixing_shape + X2 = ica.inverse_transform(Xt) + assert X.shape == X2.shape + + # reversibility test in non-reduction case + if n_components == X.shape[1]: + # XXX: we have to set atol for this test to pass for all seeds when + # fitting with float32 data. Is this revealing a bug? + if global_dtype: + # XXX: dividing by a smaller number makes + # tests fail for some seeds. + atol = np.abs(X2).mean() / 1e5 + else: + atol = 0.0 # the default rtol is enough for float64 data + assert_allclose(X, X2, atol=atol) + + +def test_fastica_errors(): + n_features = 3 + n_samples = 10 + rng = np.random.RandomState(0) + X = rng.random_sample((n_samples, n_features)) + w_init = rng.randn(n_features + 1, n_features + 1) + with pytest.raises(ValueError, match=r"alpha must be in \[1,2\]"): + fastica(X, fun_args={"alpha": 0}) + with pytest.raises( + ValueError, match=r"w_init has invalid shape.+should be \(3L?, 3L?\)" + ): + fastica(X, w_init=w_init) + + +def test_fastica_whiten_unit_variance(global_random_seed): + """Test unit variance of transformed data using FastICA algorithm. + + Bug #13056 + """ + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((100, 10)) + n_components = X.shape[1] + ica = FastICA(n_components=n_components, whiten="unit-variance", random_state=0) + Xt = ica.fit_transform(X) + + assert np.var(Xt) == pytest.approx(1.0) + + +@pytest.mark.parametrize("whiten", ["arbitrary-variance", "unit-variance", False]) +@pytest.mark.parametrize("return_X_mean", [True, False]) +@pytest.mark.parametrize("return_n_iter", [True, False]) +def test_fastica_output_shape(whiten, return_X_mean, return_n_iter): + n_features = 3 + n_samples = 10 + rng = np.random.RandomState(0) + X = rng.random_sample((n_samples, n_features)) + + expected_len = 3 + return_X_mean + return_n_iter + + out = fastica( + X, whiten=whiten, return_n_iter=return_n_iter, return_X_mean=return_X_mean + ) + + assert len(out) == expected_len + if not whiten: + assert out[0] is None + + +@pytest.mark.parametrize("add_noise", [True, False]) +def test_fastica_simple_different_solvers(add_noise, global_random_seed): + """Test FastICA is consistent between whiten_solvers.""" + rng = np.random.RandomState(global_random_seed) + n_samples = 1000 + # Generate two sources: + s1 = (2 * np.sin(np.linspace(0, 100, n_samples)) > 0) - 1 + s2 = stats.t.rvs(1, size=n_samples, random_state=rng) + s = np.c_[s1, s2].T + center_and_norm(s) + s1, s2 = s + + # Mixing angle + phi = rng.rand() * 2 * np.pi + mixing = np.array([[np.cos(phi), np.sin(phi)], [np.sin(phi), -np.cos(phi)]]) + m = np.dot(mixing, s) + + if add_noise: + m += 0.1 * rng.randn(2, 1000) + + center_and_norm(m) + + outs = {} + for solver in ("svd", "eigh"): + ica = FastICA(random_state=0, whiten="unit-variance", whiten_solver=solver) + sources = ica.fit_transform(m.T) + outs[solver] = sources + assert ica.components_.shape == (2, 2) + assert sources.shape == (1000, 2) + + # compared numbers are not all on the same magnitude. Using a small atol to + # make the test less brittle + assert_allclose(outs["eigh"], outs["svd"], atol=1e-12) + + +def test_fastica_eigh_low_rank_warning(global_random_seed): + """Test FastICA eigh solver raises warning for low-rank data.""" + rng = np.random.RandomState(global_random_seed) + A = rng.randn(10, 2) + X = A @ A.T + ica = FastICA(random_state=0, whiten="unit-variance", whiten_solver="eigh") + msg = "There are some small singular values" + + with pytest.warns(UserWarning, match=msg): + with ignore_warnings(category=ConvergenceWarning): + # The FastICA solver may not converge for some data with specific + # random seeds but this happens after the whiten step so this is + # not want we want to test here. + ica.fit(X) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_incremental_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_incremental_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..82146ad5112dae4ee15b9c96cadf59c3f31e70de --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_incremental_pca.py @@ -0,0 +1,487 @@ +"""Tests for Incremental PCA.""" + +import itertools +import warnings + +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +from sklearn import datasets +from sklearn.decomposition import PCA, IncrementalPCA +from sklearn.utils._testing import ( + assert_allclose_dense_sparse, + assert_almost_equal, + assert_array_almost_equal, +) +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS, LIL_CONTAINERS + +iris = datasets.load_iris() + + +def test_incremental_pca(): + # Incremental PCA on dense arrays. + X = iris.data + batch_size = X.shape[0] // 3 + ipca = IncrementalPCA(n_components=2, batch_size=batch_size) + pca = PCA(n_components=2) + pca.fit_transform(X) + + X_transformed = ipca.fit_transform(X) + + assert X_transformed.shape == (X.shape[0], 2) + np.testing.assert_allclose( + ipca.explained_variance_ratio_.sum(), + pca.explained_variance_ratio_.sum(), + rtol=1e-3, + ) + + for n_components in [1, 2, X.shape[1]]: + ipca = IncrementalPCA(n_components, batch_size=batch_size) + ipca.fit(X) + cov = ipca.get_covariance() + precision = ipca.get_precision() + np.testing.assert_allclose( + np.dot(cov, precision), np.eye(X.shape[1]), atol=1e-13 + ) + + +@pytest.mark.parametrize( + "sparse_container", CSC_CONTAINERS + CSR_CONTAINERS + LIL_CONTAINERS +) +def test_incremental_pca_sparse(sparse_container): + # Incremental PCA on sparse arrays. + X = iris.data + pca = PCA(n_components=2) + pca.fit_transform(X) + X_sparse = sparse_container(X) + batch_size = X_sparse.shape[0] // 3 + ipca = IncrementalPCA(n_components=2, batch_size=batch_size) + + X_transformed = ipca.fit_transform(X_sparse) + + assert X_transformed.shape == (X_sparse.shape[0], 2) + np.testing.assert_allclose( + ipca.explained_variance_ratio_.sum(), + pca.explained_variance_ratio_.sum(), + rtol=1e-3, + ) + + for n_components in [1, 2, X.shape[1]]: + ipca = IncrementalPCA(n_components, batch_size=batch_size) + ipca.fit(X_sparse) + cov = ipca.get_covariance() + precision = ipca.get_precision() + np.testing.assert_allclose( + np.dot(cov, precision), np.eye(X_sparse.shape[1]), atol=1e-13 + ) + + with pytest.raises( + TypeError, + match=( + "IncrementalPCA.partial_fit does not support " + "sparse input. Either convert data to dense " + "or use IncrementalPCA.fit to do so in batches." + ), + ): + ipca.partial_fit(X_sparse) + + +def test_incremental_pca_check_projection(global_random_seed): + # Test that the projection of data is correct. + rng = np.random.RandomState(global_random_seed) + n, p = 100, 3 + X = rng.randn(n, p) * 0.1 + X[:10] += np.array([3, 4, 5]) + Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5]) + + # Get the reconstruction of the generated data X + # Note that Xt has the same "components" as X, just separated + # This is what we want to ensure is recreated correctly + Yt = IncrementalPCA(n_components=2).fit(X).transform(Xt) + + # Normalize + Yt /= np.sqrt((Yt**2).sum()) + + # Make sure that the first element of Yt is ~1, this means + # the reconstruction worked as expected + assert_almost_equal(np.abs(Yt[0][0]), 1.0, 1) + + +def test_incremental_pca_inverse(global_random_seed): + # Test that the projection of data can be inverted. + rng = np.random.RandomState(global_random_seed) + n, p = 50, 3 + X = rng.randn(n, p) # spherical data + X[:, 1] *= 0.00001 # make middle component relatively small + X += [5, 4, 3] # make a large mean + + # same check that we can find the original data from the transformed + # signal (since the data is almost of rank n_components) + ipca = IncrementalPCA(n_components=2, batch_size=10).fit(X) + Y = ipca.transform(X) + Y_inverse = ipca.inverse_transform(Y) + assert_almost_equal(X, Y_inverse, decimal=3) + + +def test_incremental_pca_validation(): + # Test that n_components is <= n_features. + X = np.array([[0, 1, 0], [1, 0, 0]]) + n_samples, n_features = X.shape + n_components = 4 + with pytest.raises( + ValueError, + match=( + "n_components={} invalid" + " for n_features={}, need more rows than" + " columns for IncrementalPCA" + " processing".format(n_components, n_features) + ), + ): + IncrementalPCA(n_components, batch_size=10).fit(X) + + # Test that n_components is also <= n_samples in first call to partial fit. + n_components = 3 + with pytest.raises( + ValueError, + match=( + f"n_components={n_components} must be less or equal to the batch " + f"number of samples {n_samples} for the first partial_fit call." + ), + ): + IncrementalPCA(n_components=n_components).partial_fit(X) + + +def test_n_samples_equal_n_components(): + # Ensures no warning is raised when n_samples==n_components + # Non-regression test for gh-19050 + ipca = IncrementalPCA(n_components=5) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + ipca.partial_fit(np.random.randn(5, 7)) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + ipca.fit(np.random.randn(5, 7)) + + +def test_n_components_none(): + # Ensures that n_components == None is handled correctly + rng = np.random.RandomState(1999) + for n_samples, n_features in [(50, 10), (10, 50)]: + X = rng.rand(n_samples, n_features) + ipca = IncrementalPCA(n_components=None) + + # First partial_fit call, ipca.n_components_ is inferred from + # min(X.shape) + ipca.partial_fit(X) + assert ipca.n_components_ == min(X.shape) + + # Second partial_fit call, ipca.n_components_ is inferred from + # ipca.components_ computed from the first partial_fit call + ipca.partial_fit(X) + assert ipca.n_components_ == ipca.components_.shape[0] + + +def test_incremental_pca_set_params(): + # Test that components_ sign is stable over batch sizes. + rng = np.random.RandomState(1999) + n_samples = 100 + n_features = 20 + X = rng.randn(n_samples, n_features) + X2 = rng.randn(n_samples, n_features) + X3 = rng.randn(n_samples, n_features) + ipca = IncrementalPCA(n_components=20) + ipca.fit(X) + # Decreasing number of components + ipca.set_params(n_components=10) + with pytest.raises(ValueError): + ipca.partial_fit(X2) + # Increasing number of components + ipca.set_params(n_components=15) + with pytest.raises(ValueError): + ipca.partial_fit(X3) + # Returning to original setting + ipca.set_params(n_components=20) + ipca.partial_fit(X) + + +def test_incremental_pca_num_features_change(): + # Test that changing n_components will raise an error. + rng = np.random.RandomState(1999) + n_samples = 100 + X = rng.randn(n_samples, 20) + X2 = rng.randn(n_samples, 50) + ipca = IncrementalPCA(n_components=None) + ipca.fit(X) + with pytest.raises(ValueError): + ipca.partial_fit(X2) + + +def test_incremental_pca_batch_signs(global_random_seed): + # Test that components_ sign is stable over batch sizes. + rng = np.random.RandomState(global_random_seed) + n_samples = 100 + n_features = 3 + X = rng.randn(n_samples, n_features) + all_components = [] + batch_sizes = np.arange(10, 20) + for batch_size in batch_sizes: + ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X) + all_components.append(ipca.components_) + + for i, j in itertools.pairwise(all_components): + assert_almost_equal(np.sign(i), np.sign(j), decimal=6) + + +def test_incremental_pca_partial_fit_small_batch(): + # Test that there is no minimum batch size after the first partial_fit + # Non-regression test + rng = np.random.RandomState(1999) + n, p = 50, 3 + X = rng.randn(n, p) # spherical data + X[:, 1] *= 0.00001 # make middle component relatively small + X += [5, 4, 3] # make a large mean + + n_components = p + pipca = IncrementalPCA(n_components=n_components) + pipca.partial_fit(X[:n_components]) + for idx in range(n_components, n): + pipca.partial_fit(X[idx : idx + 1]) + + pca = PCA(n_components=n_components) + pca.fit(X) + + assert_allclose(pca.components_, pipca.components_, atol=1e-3) + + +def test_incremental_pca_batch_values(global_random_seed): + # Test that components_ values are stable over batch sizes. + rng = np.random.RandomState(global_random_seed) + n_samples = 100 + n_features = 3 + X = rng.randn(n_samples, n_features) + all_components = [] + batch_sizes = np.arange(20, 40, 3) + for batch_size in batch_sizes: + ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X) + all_components.append(ipca.components_) + + for i, j in itertools.pairwise(all_components): + assert_almost_equal(i, j, decimal=1) + + +def test_incremental_pca_batch_rank(): + # Test sample size in each batch is always larger or equal to n_components + rng = np.random.RandomState(1999) + n_samples = 100 + n_features = 20 + X = rng.randn(n_samples, n_features) + all_components = [] + batch_sizes = np.arange(20, 90, 3) + for batch_size in batch_sizes: + ipca = IncrementalPCA(n_components=20, batch_size=batch_size).fit(X) + all_components.append(ipca.components_) + + for components_i, components_j in itertools.pairwise(all_components): + assert_allclose_dense_sparse(components_i, components_j) + + +def test_incremental_pca_partial_fit(global_random_seed): + # Test that fit and partial_fit get equivalent results. + rng = np.random.RandomState(global_random_seed) + n, p = 50, 3 + X = rng.randn(n, p) # spherical data + X[:, 1] *= 0.00001 # make middle component relatively small + X += [5, 4, 3] # make a large mean + + # same check that we can find the original data from the transformed + # signal (since the data is almost of rank n_components) + batch_size = 10 + ipca = IncrementalPCA(n_components=2, batch_size=batch_size).fit(X) + pipca = IncrementalPCA(n_components=2, batch_size=batch_size) + # Add one to make sure endpoint is included + batch_itr = np.arange(0, n + 1, batch_size) + for i, j in itertools.pairwise(batch_itr): + pipca.partial_fit(X[i:j, :]) + assert_almost_equal(ipca.components_, pipca.components_, decimal=3) + + +def test_incremental_pca_against_pca_iris(): + # Test that IncrementalPCA and PCA are approximate (to a sign flip). + X = iris.data + + Y_pca = PCA(n_components=2).fit_transform(X) + Y_ipca = IncrementalPCA(n_components=2, batch_size=25).fit_transform(X) + + assert_almost_equal(np.abs(Y_pca), np.abs(Y_ipca), 1) + + +def test_incremental_pca_against_pca_random_data(global_random_seed): + # Test that IncrementalPCA and PCA are approximate (to a sign flip). + rng = np.random.RandomState(global_random_seed) + n_samples = 100 + n_features = 3 + X = rng.randn(n_samples, n_features) + 5 * rng.rand(1, n_features) + + Y_pca = PCA(n_components=3).fit_transform(X) + Y_ipca = IncrementalPCA(n_components=3, batch_size=25).fit_transform(X) + + assert_almost_equal(np.abs(Y_pca), np.abs(Y_ipca), 1) + + +def test_explained_variances(): + # Test that PCA and IncrementalPCA calculations match + X = datasets.make_low_rank_matrix( + 1000, 100, tail_strength=0.0, effective_rank=10, random_state=1999 + ) + prec = 3 + n_samples, n_features = X.shape + for nc in [None, 99]: + pca = PCA(n_components=nc).fit(X) + ipca = IncrementalPCA(n_components=nc, batch_size=100).fit(X) + assert_almost_equal( + pca.explained_variance_, ipca.explained_variance_, decimal=prec + ) + assert_almost_equal( + pca.explained_variance_ratio_, ipca.explained_variance_ratio_, decimal=prec + ) + assert_almost_equal(pca.noise_variance_, ipca.noise_variance_, decimal=prec) + + +def test_singular_values(global_random_seed): + # Check that the IncrementalPCA output has the correct singular values + + rng = np.random.RandomState(global_random_seed) + n_samples = 1000 + n_features = 100 + + X = datasets.make_low_rank_matrix( + n_samples, n_features, tail_strength=0.0, effective_rank=10, random_state=rng + ) + + pca = PCA(n_components=10, svd_solver="full", random_state=rng).fit(X) + ipca = IncrementalPCA(n_components=10, batch_size=150).fit(X) + assert_array_almost_equal(pca.singular_values_, ipca.singular_values_, 2) + + # Compare to the Frobenius norm + X_pca = pca.transform(X) + X_ipca = ipca.transform(X) + assert_array_almost_equal( + np.sum(pca.singular_values_**2.0), np.linalg.norm(X_pca, "fro") ** 2.0, 12 + ) + assert_array_almost_equal( + np.sum(ipca.singular_values_**2.0), np.linalg.norm(X_ipca, "fro") ** 2.0, 2 + ) + + # Compare to the 2-norms of the score vectors + assert_array_almost_equal( + pca.singular_values_, np.sqrt(np.sum(X_pca**2.0, axis=0)), 12 + ) + assert_array_almost_equal( + ipca.singular_values_, np.sqrt(np.sum(X_ipca**2.0, axis=0)), 2 + ) + + # Set the singular values and see what we get back + rng = np.random.RandomState(global_random_seed) + n_samples = 100 + n_features = 110 + + X = datasets.make_low_rank_matrix( + n_samples, n_features, tail_strength=0.0, effective_rank=3, random_state=rng + ) + + pca = PCA(n_components=3, svd_solver="full", random_state=rng) + ipca = IncrementalPCA(n_components=3, batch_size=100) + + X_pca = pca.fit_transform(X) + X_pca /= np.sqrt(np.sum(X_pca**2.0, axis=0)) + X_pca[:, 0] *= 3.142 + X_pca[:, 1] *= 2.718 + + X_hat = np.dot(X_pca, pca.components_) + pca.fit(X_hat) + ipca.fit(X_hat) + assert_array_almost_equal(pca.singular_values_, [3.142, 2.718, 1.0], 14) + assert_array_almost_equal(ipca.singular_values_, [3.142, 2.718, 1.0], 14) + + +def test_whitening(global_random_seed): + # Test that PCA and IncrementalPCA transforms match to sign flip. + X = datasets.make_low_rank_matrix( + 1000, 10, tail_strength=0.0, effective_rank=2, random_state=global_random_seed + ) + atol = 1e-3 + for nc in [None, 9]: + pca = PCA(whiten=True, n_components=nc).fit(X) + ipca = IncrementalPCA(whiten=True, n_components=nc, batch_size=250).fit(X) + + # Since the data is rank deficient, some components are pure noise. We + # should not expect those dimensions to carry any signal and their + # values might be arbitrarily changed by implementation details of the + # internal SVD solver. We therefore filter them out before comparison. + stable_mask = pca.explained_variance_ratio_ > 1e-12 + + Xt_pca = pca.transform(X) + Xt_ipca = ipca.transform(X) + assert_allclose( + np.abs(Xt_pca)[:, stable_mask], + np.abs(Xt_ipca)[:, stable_mask], + atol=atol, + ) + + # The noisy dimensions are in the null space of the inverse transform, + # so they are not influencing the reconstruction. We therefore don't + # need to apply the mask here. + Xinv_ipca = ipca.inverse_transform(Xt_ipca) + Xinv_pca = pca.inverse_transform(Xt_pca) + assert_allclose(X, Xinv_ipca, atol=atol) + assert_allclose(X, Xinv_pca, atol=atol) + assert_allclose(Xinv_pca, Xinv_ipca, atol=atol) + + +def test_incremental_pca_partial_fit_float_division(): + # Test to ensure float division is used in all versions of Python + # (non-regression test for issue #9489) + + rng = np.random.RandomState(0) + A = rng.randn(5, 3) + 2 + B = rng.randn(7, 3) + 5 + + pca = IncrementalPCA(n_components=2) + pca.partial_fit(A) + # Set n_samples_seen_ to be a floating point number instead of an int + pca.n_samples_seen_ = float(pca.n_samples_seen_) + pca.partial_fit(B) + singular_vals_float_samples_seen = pca.singular_values_ + + pca2 = IncrementalPCA(n_components=2) + pca2.partial_fit(A) + pca2.partial_fit(B) + singular_vals_int_samples_seen = pca2.singular_values_ + + np.testing.assert_allclose( + singular_vals_float_samples_seen, singular_vals_int_samples_seen + ) + + +def test_incremental_pca_fit_overflow_error(): + # Test for overflow error on Windows OS + # (non-regression test for issue #17693) + rng = np.random.RandomState(0) + A = rng.rand(500000, 2) + + ipca = IncrementalPCA(n_components=2, batch_size=10000) + ipca.fit(A) + + pca = PCA(n_components=2) + pca.fit(A) + + np.testing.assert_allclose(ipca.singular_values_, pca.singular_values_) + + +def test_incremental_pca_feature_names_out(): + """Check feature names out for IncrementalPCA.""" + ipca = IncrementalPCA(n_components=2).fit(iris.data) + + names = ipca.get_feature_names_out() + assert_array_equal([f"incrementalpca{i}" for i in range(2)], names) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_kernel_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_kernel_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..a271fcda3f21b79132cbb408ea7b25a99f95df66 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_kernel_pca.py @@ -0,0 +1,566 @@ +import warnings + +import numpy as np +import pytest + +import sklearn +from sklearn.datasets import load_iris, make_blobs, make_circles +from sklearn.decomposition import PCA, KernelPCA +from sklearn.exceptions import NotFittedError +from sklearn.linear_model import Perceptron +from sklearn.metrics.pairwise import rbf_kernel +from sklearn.model_selection import GridSearchCV +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.fixes import CSR_CONTAINERS +from sklearn.utils.validation import _check_psd_eigenvalues + + +def test_kernel_pca(global_random_seed): + """Nominal test for all solvers and all known kernels + a custom one + + It tests + - that fit_transform is equivalent to fit+transform + - that the shapes of transforms and inverse transforms are correct + """ + rng = np.random.RandomState(global_random_seed) + X_fit = rng.random_sample((5, 4)) + X_pred = rng.random_sample((2, 4)) + + def histogram(x, y, **kwargs): + # Histogram kernel implemented as a callable. + assert kwargs == {} # no kernel_params that we didn't ask for + return np.minimum(x, y).sum() + + for eigen_solver in ("auto", "dense", "arpack", "randomized"): + for kernel in ("linear", "rbf", "poly", histogram): + # histogram kernel produces singular matrix inside linalg.solve + # XXX use a least-squares approximation? + inv = not callable(kernel) + + # transform fit data + kpca = KernelPCA( + 4, kernel=kernel, eigen_solver=eigen_solver, fit_inverse_transform=inv + ) + X_fit_transformed = kpca.fit_transform(X_fit) + X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit) + assert_array_almost_equal( + np.abs(X_fit_transformed), np.abs(X_fit_transformed2) + ) + + # non-regression test: previously, gamma would be 0 by default, + # forcing all eigenvalues to 0 under the poly kernel + assert X_fit_transformed.size != 0 + + # transform new data + X_pred_transformed = kpca.transform(X_pred) + assert X_pred_transformed.shape[1] == X_fit_transformed.shape[1] + + # inverse transform + if inv: + X_pred2 = kpca.inverse_transform(X_pred_transformed) + assert X_pred2.shape == X_pred.shape + + +def test_kernel_pca_invalid_parameters(): + """Check that kPCA raises an error if the parameters are invalid + + Tests fitting inverse transform with a precomputed kernel raises a + ValueError. + """ + estimator = KernelPCA( + n_components=10, fit_inverse_transform=True, kernel="precomputed" + ) + err_ms = "Cannot fit_inverse_transform with a precomputed kernel" + with pytest.raises(ValueError, match=err_ms): + estimator.fit(np.random.randn(10, 10)) + + +def test_kernel_pca_consistent_transform(global_random_seed): + """Check robustness to mutations in the original training array + + Test that after fitting a kPCA model, it stays independent of any + mutation of the values of the original data object by relying on an + internal copy. + """ + # X_fit_ needs to retain the old, unmodified copy of X + state = np.random.RandomState(global_random_seed) + X = state.rand(10, 10) + kpca = KernelPCA(random_state=state).fit(X) + transformed1 = kpca.transform(X) + + X_copy = X.copy() + X[:, 0] = 666 + transformed2 = kpca.transform(X_copy) + assert_array_almost_equal(transformed1, transformed2) + + +def test_kernel_pca_deterministic_output(global_random_seed): + """Test that Kernel PCA produces deterministic output + + Tests that the same inputs and random state produce the same output. + """ + rng = np.random.RandomState(global_random_seed) + X = rng.rand(10, 10) + eigen_solver = ("arpack", "dense") + + for solver in eigen_solver: + transformed_X = np.zeros((20, 2)) + for i in range(20): + kpca = KernelPCA(n_components=2, eigen_solver=solver, random_state=rng) + transformed_X[i, :] = kpca.fit_transform(X)[0] + assert_allclose(transformed_X, np.tile(transformed_X[0, :], 20).reshape(20, 2)) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_kernel_pca_sparse(csr_container, global_random_seed): + """Test that kPCA works on a sparse data input. + + Same test as ``test_kernel_pca except inverse_transform`` since it's not + implemented for sparse matrices. + """ + rng = np.random.RandomState(global_random_seed) + X_fit = csr_container(rng.random_sample((5, 4))) + X_pred = csr_container(rng.random_sample((2, 4))) + + for eigen_solver in ("auto", "arpack", "randomized"): + for kernel in ("linear", "rbf", "poly"): + # transform fit data + kpca = KernelPCA( + 4, + kernel=kernel, + eigen_solver=eigen_solver, + fit_inverse_transform=False, + random_state=0, + ) + X_fit_transformed = kpca.fit_transform(X_fit) + X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit) + assert_array_almost_equal( + np.abs(X_fit_transformed), np.abs(X_fit_transformed2) + ) + + # transform new data + X_pred_transformed = kpca.transform(X_pred) + assert X_pred_transformed.shape[1] == X_fit_transformed.shape[1] + + # inverse transform: not available for sparse matrices + # XXX: should we raise another exception type here? For instance: + # NotImplementedError. + with pytest.raises(NotFittedError): + kpca.inverse_transform(X_pred_transformed) + + +@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"]) +@pytest.mark.parametrize("n_features", [4, 10]) +def test_kernel_pca_linear_kernel(solver, n_features, global_random_seed): + """Test that kPCA with linear kernel is equivalent to PCA for all solvers. + + KernelPCA with linear kernel should produce the same output as PCA. + """ + rng = np.random.RandomState(global_random_seed) + X_fit = rng.random_sample((5, n_features)) + X_pred = rng.random_sample((2, n_features)) + + # for a linear kernel, kernel PCA should find the same projection as PCA + # modulo the sign (direction) + # fit only the first four components: fifth is near zero eigenvalue, so + # can be trimmed due to roundoff error + n_comps = 3 if solver == "arpack" else 4 + assert_array_almost_equal( + np.abs(KernelPCA(n_comps, eigen_solver=solver).fit(X_fit).transform(X_pred)), + np.abs( + PCA(n_comps, svd_solver=solver if solver != "dense" else "full") + .fit(X_fit) + .transform(X_pred) + ), + ) + + +def test_kernel_pca_n_components(): + """Test that `n_components` is correctly taken into account for projections + + For all solvers this tests that the output has the correct shape depending + on the selected number of components. + """ + rng = np.random.RandomState(0) + X_fit = rng.random_sample((5, 4)) + X_pred = rng.random_sample((2, 4)) + + for eigen_solver in ("dense", "arpack", "randomized"): + for c in [1, 2, 4]: + kpca = KernelPCA(n_components=c, eigen_solver=eigen_solver) + shape = kpca.fit(X_fit).transform(X_pred).shape + + assert shape == (2, c) + + +def test_remove_zero_eig(): + """Check that the ``remove_zero_eig`` parameter works correctly. + + Tests that the null-space (Zero) eigenvalues are removed when + remove_zero_eig=True, whereas they are not by default. + """ + X = np.array([[1 - 1e-30, 1], [1, 1], [1, 1 - 1e-20]]) + + # n_components=None (default) => remove_zero_eig is True + kpca = KernelPCA() + Xt = kpca.fit_transform(X) + assert Xt.shape == (3, 0) + + kpca = KernelPCA(n_components=2) + Xt = kpca.fit_transform(X) + assert Xt.shape == (3, 2) + + kpca = KernelPCA(n_components=2, remove_zero_eig=True) + Xt = kpca.fit_transform(X) + assert Xt.shape == (3, 0) + + +def test_leave_zero_eig(): + """Non-regression test for issue #12141 (PR #12143) + + This test checks that fit().transform() returns the same result as + fit_transform() in case of non-removed zero eigenvalue. + """ + X_fit = np.array([[1, 1], [0, 0]]) + + # Assert that even with all np warnings on, there is no div by zero warning + with warnings.catch_warnings(): + # There might be warnings about the kernel being badly conditioned, + # but there should not be warnings about division by zero. + # (Numpy division by zero warning can have many message variants, but + # at least we know that it is a RuntimeWarning so let's check only this) + warnings.simplefilter("error", RuntimeWarning) + with np.errstate(all="warn"): + k = KernelPCA(n_components=2, remove_zero_eig=False, eigen_solver="dense") + # Fit, then transform + A = k.fit(X_fit).transform(X_fit) + # Do both at once + B = k.fit_transform(X_fit) + # Compare + assert_array_almost_equal(np.abs(A), np.abs(B)) + + +def test_kernel_pca_precomputed(global_random_seed): + """Test that kPCA works with a precomputed kernel, for all solvers""" + rng = np.random.RandomState(global_random_seed) + X_fit = rng.random_sample((5, 4)) + X_pred = rng.random_sample((2, 4)) + + for eigen_solver in ("dense", "arpack", "randomized"): + X_kpca = ( + KernelPCA(4, eigen_solver=eigen_solver, random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + + X_kpca2 = ( + KernelPCA( + 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0 + ) + .fit(np.dot(X_fit, X_fit.T)) + .transform(np.dot(X_pred, X_fit.T)) + ) + + X_kpca_train = KernelPCA( + 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0 + ).fit_transform(np.dot(X_fit, X_fit.T)) + + X_kpca_train2 = ( + KernelPCA( + 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0 + ) + .fit(np.dot(X_fit, X_fit.T)) + .transform(np.dot(X_fit, X_fit.T)) + ) + + assert_array_almost_equal(np.abs(X_kpca), np.abs(X_kpca2)) + + assert_array_almost_equal(np.abs(X_kpca_train), np.abs(X_kpca_train2)) + + +@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"]) +def test_kernel_pca_precomputed_non_symmetric(solver): + """Check that the kernel centerer works. + + Tests that a non symmetric precomputed kernel is actually accepted + because the kernel centerer does its job correctly. + """ + + # a non symmetric gram matrix + K = [[1, 2], [3, 40]] + kpca = KernelPCA( + kernel="precomputed", eigen_solver=solver, n_components=1, random_state=0 + ) + kpca.fit(K) # no error + + # same test with centered kernel + Kc = [[9, -9], [-9, 9]] + kpca_c = KernelPCA( + kernel="precomputed", eigen_solver=solver, n_components=1, random_state=0 + ) + kpca_c.fit(Kc) + + # comparison between the non-centered and centered versions + assert_array_equal(kpca.eigenvectors_, kpca_c.eigenvectors_) + assert_array_equal(kpca.eigenvalues_, kpca_c.eigenvalues_) + + +def test_gridsearch_pipeline(): + """Check that kPCA works as expected in a grid search pipeline + + Test if we can do a grid-search to find parameters to separate + circles with a perceptron model. + """ + X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0) + kpca = KernelPCA(kernel="rbf", n_components=2) + pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron(max_iter=5))]) + param_grid = dict(kernel_pca__gamma=2.0 ** np.arange(-2, 2)) + grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid) + grid_search.fit(X, y) + assert grid_search.best_score_ == 1 + + +def test_gridsearch_pipeline_precomputed(): + """Check that kPCA works as expected in a grid search pipeline (2) + + Test if we can do a grid-search to find parameters to separate + circles with a perceptron model. This test uses a precomputed kernel. + """ + X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0) + kpca = KernelPCA(kernel="precomputed", n_components=2) + pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron(max_iter=5))]) + param_grid = dict(Perceptron__max_iter=np.arange(1, 5)) + grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid) + X_kernel = rbf_kernel(X, gamma=2.0) + grid_search.fit(X_kernel, y) + assert grid_search.best_score_ == 1 + + +def test_nested_circles(): + """Check that kPCA projects in a space where nested circles are separable + + Tests that 2D nested circles become separable with a perceptron when + projected in the first 2 kPCA using an RBF kernel, while raw samples + are not directly separable in the original space. + """ + X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0) + + # 2D nested circles are not linearly separable + train_score = Perceptron(max_iter=5).fit(X, y).score(X, y) + assert train_score < 0.8 + + # Project the circles data into the first 2 components of a RBF Kernel + # PCA model. + # Note that the gamma value is data dependent. If this test breaks + # and the gamma value has to be updated, the Kernel PCA example will + # have to be updated too. + kpca = KernelPCA( + kernel="rbf", n_components=2, fit_inverse_transform=True, gamma=2.0 + ) + X_kpca = kpca.fit_transform(X) + + # The data is perfectly linearly separable in that space + train_score = Perceptron(max_iter=5).fit(X_kpca, y).score(X_kpca, y) + assert train_score == 1.0 + + +def test_kernel_conditioning(): + """Check that ``_check_psd_eigenvalues`` is correctly called in kPCA + + Non-regression test for issue #12140 (PR #12145). + """ + + # create a pathological X leading to small non-zero eigenvalue + X = [[5, 1], [5 + 1e-8, 1e-8], [5 + 1e-8, 0]] + kpca = KernelPCA(kernel="linear", n_components=2, fit_inverse_transform=True) + kpca.fit(X) + + # check that the small non-zero eigenvalue was correctly set to zero + assert kpca.eigenvalues_.min() == 0 + assert np.all(kpca.eigenvalues_ == _check_psd_eigenvalues(kpca.eigenvalues_)) + + +@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"]) +def test_precomputed_kernel_not_psd(solver): + """Check how KernelPCA works with non-PSD kernels depending on n_components + + Tests for all methods what happens with a non PSD gram matrix (this + can happen in an isomap scenario, or with custom kernel functions, or + maybe with ill-posed datasets). + + When ``n_component`` is large enough to capture a negative eigenvalue, an + error should be raised. Otherwise, KernelPCA should run without error + since the negative eigenvalues are not selected. + """ + + # a non PSD kernel with large eigenvalues, already centered + # it was captured from an isomap call and multiplied by 100 for compacity + K = [ + [4.48, -1.0, 8.07, 2.33, 2.33, 2.33, -5.76, -12.78], + [-1.0, -6.48, 4.5, -1.24, -1.24, -1.24, -0.81, 7.49], + [8.07, 4.5, 15.48, 2.09, 2.09, 2.09, -11.1, -23.23], + [2.33, -1.24, 2.09, 4.0, -3.65, -3.65, 1.02, -0.9], + [2.33, -1.24, 2.09, -3.65, 4.0, -3.65, 1.02, -0.9], + [2.33, -1.24, 2.09, -3.65, -3.65, 4.0, 1.02, -0.9], + [-5.76, -0.81, -11.1, 1.02, 1.02, 1.02, 4.86, 9.75], + [-12.78, 7.49, -23.23, -0.9, -0.9, -0.9, 9.75, 21.46], + ] + # this gram matrix has 5 positive eigenvalues and 3 negative ones + # [ 52.72, 7.65, 7.65, 5.02, 0. , -0. , -6.13, -15.11] + + # 1. ask for enough components to get a significant negative one + kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=7) + # make sure that the appropriate error is raised + with pytest.raises(ValueError, match="There are significant negative eigenvalues"): + kpca.fit(K) + + # 2. ask for a small enough n_components to get only positive ones + kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=2) + if solver == "randomized": + # the randomized method is still inconsistent with the others on this + # since it selects the eigenvalues based on the largest 2 modules, not + # on the largest 2 values. + # + # At least we can ensure that we return an error instead of returning + # the wrong eigenvalues + with pytest.raises( + ValueError, match="There are significant negative eigenvalues" + ): + kpca.fit(K) + else: + # general case: make sure that it works + kpca.fit(K) + + +@pytest.mark.parametrize("n_components", [4, 10, 20]) +def test_kernel_pca_solvers_equivalence(n_components): + """Check that 'dense' 'arpack' & 'randomized' solvers give similar results""" + + # Generate random data + n_train, n_test = 1_000, 100 + X, _ = make_circles( + n_samples=(n_train + n_test), factor=0.3, noise=0.05, random_state=0 + ) + X_fit, X_pred = X[:n_train, :], X[n_train:, :] + + # reference (full) + ref_pred = ( + KernelPCA(n_components, eigen_solver="dense", random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + + # arpack + a_pred = ( + KernelPCA(n_components, eigen_solver="arpack", random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + # check that the result is still correct despite the approx + assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred)) + + # randomized + r_pred = ( + KernelPCA(n_components, eigen_solver="randomized", random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + # check that the result is still correct despite the approximation + assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred)) + + +def test_kernel_pca_inverse_transform_reconstruction(): + """Test if the reconstruction is a good approximation. + + Note that in general it is not possible to get an arbitrarily good + reconstruction because of kernel centering that does not + preserve all the information of the original data. + """ + X, *_ = make_blobs(n_samples=100, n_features=4, random_state=0) + + kpca = KernelPCA( + n_components=20, kernel="rbf", fit_inverse_transform=True, alpha=1e-3 + ) + X_trans = kpca.fit_transform(X) + X_reconst = kpca.inverse_transform(X_trans) + assert np.linalg.norm(X - X_reconst) / np.linalg.norm(X) < 1e-1 + + +def test_kernel_pca_raise_not_fitted_error(): + X = np.random.randn(15).reshape(5, 3) + kpca = KernelPCA() + kpca.fit(X) + with pytest.raises(NotFittedError): + kpca.inverse_transform(X) + + +def test_32_64_decomposition_shape(): + """Test that the decomposition is similar for 32 and 64 bits data + + Non regression test for + https://github.com/scikit-learn/scikit-learn/issues/18146 + """ + X, y = make_blobs( + n_samples=30, centers=[[0, 0, 0], [1, 1, 1]], random_state=0, cluster_std=0.1 + ) + X = StandardScaler().fit_transform(X) + X -= X.min() + + # Compare the shapes (corresponds to the number of non-zero eigenvalues) + kpca = KernelPCA() + assert kpca.fit_transform(X).shape == kpca.fit_transform(X.astype(np.float32)).shape + + +def test_kernel_pca_feature_names_out(): + """Check feature names out for KernelPCA.""" + X, *_ = make_blobs(n_samples=100, n_features=4, random_state=0) + kpca = KernelPCA(n_components=2).fit(X) + + names = kpca.get_feature_names_out() + assert_array_equal([f"kernelpca{i}" for i in range(2)], names) + + +def test_kernel_pca_inverse_correct_gamma(global_random_seed): + """Check that gamma is set correctly when not provided. + + Non-regression test for #26280 + """ + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((5, 4)) + + kwargs = { + "n_components": 2, + "random_state": rng, + "fit_inverse_transform": True, + "kernel": "rbf", + } + + expected_gamma = 1 / X.shape[1] + kpca1 = KernelPCA(gamma=None, **kwargs).fit(X) + kpca2 = KernelPCA(gamma=expected_gamma, **kwargs).fit(X) + + assert kpca1.gamma_ == expected_gamma + assert kpca2.gamma_ == expected_gamma + + X1_recon = kpca1.inverse_transform(kpca1.transform(X)) + X2_recon = kpca2.inverse_transform(kpca1.transform(X)) + + assert_allclose(X1_recon, X2_recon) + + +def test_kernel_pca_pandas_output(): + """Check that KernelPCA works with pandas output when the solver is arpack. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27579 + """ + pytest.importorskip("pandas") + X, _ = load_iris(as_frame=True, return_X_y=True) + with sklearn.config_context(transform_output="pandas"): + KernelPCA(n_components=2, eigen_solver="arpack").fit_transform(X) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_nmf.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_nmf.py new file mode 100644 index 0000000000000000000000000000000000000000..4dfc4e1c0aa53a51a88db0a232d44b646ff8516c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_nmf.py @@ -0,0 +1,1010 @@ +import re +import sys +from io import StringIO + +import numpy as np +import pytest +from scipy import linalg + +from sklearn.base import clone +from sklearn.decomposition import NMF, MiniBatchNMF, non_negative_factorization +from sklearn.decomposition import _nmf as nmf # For testing internals +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils._testing import ( + assert_allclose, + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.extmath import squared_norm +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_convergence_warning(Estimator, solver): + convergence_warning = ( + "Maximum number of iterations 1 reached. Increase it to improve convergence." + ) + A = np.ones((2, 2)) + with pytest.warns(ConvergenceWarning, match=convergence_warning): + Estimator(max_iter=1, n_components="auto", **solver).fit(A) + + +def test_initialize_nn_output(): + # Test that initialization does not return negative values + rng = np.random.mtrand.RandomState(42) + data = np.abs(rng.randn(10, 10)) + for init in ("random", "nndsvd", "nndsvda", "nndsvdar"): + W, H = nmf._initialize_nmf(data, 10, init=init, random_state=0) + assert not ((W < 0).any() or (H < 0).any()) + + +@pytest.mark.filterwarnings( + r"ignore:The multiplicative update \('mu'\) solver cannot update zeros present in" + r" the initialization", +) +def test_parameter_checking(): + # Here we only check for invalid parameter values that are not already + # automatically tested in the common tests. + + A = np.ones((2, 2)) + + msg = "Invalid beta_loss parameter: solver 'cd' does not handle beta_loss = 1.0" + with pytest.raises(ValueError, match=msg): + NMF(solver="cd", beta_loss=1.0).fit(A) + msg = "Negative values in data passed to" + with pytest.raises(ValueError, match=msg): + NMF().fit(-A) + clf = NMF(2, tol=0.1).fit(A) + with pytest.raises(ValueError, match=msg): + clf.transform(-A) + with pytest.raises(ValueError, match=msg): + nmf._initialize_nmf(-A, 2, "nndsvd") + + for init in ["nndsvd", "nndsvda", "nndsvdar"]: + msg = re.escape( + "init = '{}' can only be used when " + "n_components <= min(n_samples, n_features)".format(init) + ) + with pytest.raises(ValueError, match=msg): + NMF(3, init=init).fit(A) + with pytest.raises(ValueError, match=msg): + MiniBatchNMF(3, init=init).fit(A) + with pytest.raises(ValueError, match=msg): + nmf._initialize_nmf(A, 3, init) + + +def test_initialize_close(): + # Test NNDSVD error + # Test that _initialize_nmf error is less than the standard deviation of + # the entries in the matrix. + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(10, 10)) + W, H = nmf._initialize_nmf(A, 10, init="nndsvd") + error = linalg.norm(np.dot(W, H) - A) + sdev = linalg.norm(A - A.mean()) + assert error <= sdev + + +def test_initialize_variants(): + # Test NNDSVD variants correctness + # Test that the variants 'nndsvda' and 'nndsvdar' differ from basic + # 'nndsvd' only where the basic version has zeros. + rng = np.random.mtrand.RandomState(42) + data = np.abs(rng.randn(10, 10)) + W0, H0 = nmf._initialize_nmf(data, 10, init="nndsvd") + Wa, Ha = nmf._initialize_nmf(data, 10, init="nndsvda") + War, Har = nmf._initialize_nmf(data, 10, init="nndsvdar", random_state=0) + + for ref, evl in ((W0, Wa), (W0, War), (H0, Ha), (H0, Har)): + assert_almost_equal(evl[ref != 0], ref[ref != 0]) + + +# ignore UserWarning raised when both solver='mu' and init='nndsvd' +@pytest.mark.filterwarnings( + r"ignore:The multiplicative update \('mu'\) solver cannot update zeros present in" + r" the initialization" +) +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +@pytest.mark.parametrize("init", (None, "nndsvd", "nndsvda", "nndsvdar", "random")) +@pytest.mark.parametrize("alpha_W", (0.0, 1.0)) +@pytest.mark.parametrize("alpha_H", (0.0, 1.0, "same")) +def test_nmf_fit_nn_output(Estimator, solver, init, alpha_W, alpha_H): + # Test that the decomposition does not contain negative values + A = np.c_[5.0 - np.arange(1, 6), 5.0 + np.arange(1, 6)] + model = Estimator( + n_components=2, + init=init, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=0, + **solver, + ) + transf = model.fit_transform(A) + assert not ((model.components_ < 0).any() or (transf < 0).any()) + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_fit_close(Estimator, solver): + rng = np.random.mtrand.RandomState(42) + # Test that the fit is not too far away + pnmf = Estimator( + 5, + init="nndsvdar", + random_state=0, + max_iter=600, + **solver, + ) + X = np.abs(rng.randn(6, 5)) + assert pnmf.fit(X).reconstruction_err_ < 0.1 + + +def test_nmf_true_reconstruction(): + # Test that the fit is not too far away from an exact solution + # (by construction) + n_samples = 15 + n_features = 10 + n_components = 5 + beta_loss = 1 + batch_size = 3 + max_iter = 1000 + + rng = np.random.mtrand.RandomState(42) + W_true = np.zeros([n_samples, n_components]) + W_array = np.abs(rng.randn(n_samples)) + for j in range(n_components): + W_true[j % n_samples, j] = W_array[j % n_samples] + H_true = np.zeros([n_components, n_features]) + H_array = np.abs(rng.randn(n_components)) + for j in range(n_features): + H_true[j % n_components, j] = H_array[j % n_components] + X = np.dot(W_true, H_true) + + model = NMF( + n_components=n_components, + solver="mu", + beta_loss=beta_loss, + max_iter=max_iter, + random_state=0, + ) + transf = model.fit_transform(X) + X_calc = np.dot(transf, model.components_) + + assert model.reconstruction_err_ < 0.1 + assert_allclose(X, X_calc) + + mbmodel = MiniBatchNMF( + n_components=n_components, + beta_loss=beta_loss, + batch_size=batch_size, + random_state=0, + max_iter=max_iter, + ) + transf = mbmodel.fit_transform(X) + X_calc = np.dot(transf, mbmodel.components_) + + assert mbmodel.reconstruction_err_ < 0.1 + assert_allclose(X, X_calc, atol=1) + + +@pytest.mark.parametrize("solver", ["cd", "mu"]) +def test_nmf_transform(solver): + # Test that fit_transform is equivalent to fit.transform for NMF + # Test that NMF.transform returns close values + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(6, 5)) + m = NMF( + solver=solver, + n_components=3, + init="random", + random_state=0, + tol=1e-6, + ) + ft = m.fit_transform(A) + t = m.transform(A) + assert_allclose(ft, t, atol=1e-1) + + +def test_minibatch_nmf_transform(): + # Test that fit_transform is equivalent to fit.transform for MiniBatchNMF + # Only guaranteed with fresh restarts + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(6, 5)) + m = MiniBatchNMF( + n_components=3, + random_state=0, + tol=1e-3, + fresh_restarts=True, + ) + ft = m.fit_transform(A) + t = m.transform(A) + assert_allclose(ft, t) + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_transform_custom_init(Estimator, solver): + # Smoke test that checks if NMF.transform works with custom initialization + random_state = np.random.RandomState(0) + A = np.abs(random_state.randn(6, 5)) + n_components = 4 + avg = np.sqrt(A.mean() / n_components) + H_init = np.abs(avg * random_state.randn(n_components, 5)) + W_init = np.abs(avg * random_state.randn(6, n_components)) + + m = Estimator( + n_components=n_components, init="custom", random_state=0, tol=1e-3, **solver + ) + m.fit_transform(A, W=W_init, H=H_init) + m.transform(A) + + +@pytest.mark.parametrize("solver", ("cd", "mu")) +def test_nmf_inverse_transform(solver): + # Test that NMF.inverse_transform returns close values + random_state = np.random.RandomState(0) + A = np.abs(random_state.randn(6, 4)) + m = NMF( + solver=solver, + n_components=4, + init="random", + random_state=0, + max_iter=1000, + ) + ft = m.fit_transform(A) + A_new = m.inverse_transform(ft) + assert_array_almost_equal(A, A_new, decimal=2) + + +def test_mbnmf_inverse_transform(): + # Test that MiniBatchNMF.transform followed by MiniBatchNMF.inverse_transform + # is close to the identity + rng = np.random.RandomState(0) + A = np.abs(rng.randn(6, 4)) + nmf = MiniBatchNMF( + random_state=rng, + max_iter=500, + init="nndsvdar", + fresh_restarts=True, + ) + ft = nmf.fit_transform(A) + A_new = nmf.inverse_transform(ft) + assert_allclose(A, A_new, rtol=1e-3, atol=1e-2) + + +@pytest.mark.parametrize("Estimator", [NMF, MiniBatchNMF]) +def test_n_components_greater_n_features(Estimator): + # Smoke test for the case of more components than features. + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(30, 10)) + Estimator(n_components=15, random_state=0, tol=1e-2).fit(A) + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS) +@pytest.mark.parametrize("alpha_W", (0.0, 1.0)) +@pytest.mark.parametrize("alpha_H", (0.0, 1.0, "same")) +def test_nmf_sparse_input(Estimator, solver, sparse_container, alpha_W, alpha_H): + # Test that sparse matrices are accepted as input + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(10, 10)) + A[:, 2 * np.arange(5)] = 0 + A_sparse = sparse_container(A) + + est1 = Estimator( + n_components=5, + init="random", + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=0, + tol=0, + max_iter=100, + **solver, + ) + est2 = clone(est1) + + W1 = est1.fit_transform(A) + W2 = est2.fit_transform(A_sparse) + H1 = est1.components_ + H2 = est2.components_ + + assert_allclose(W1, W2) + assert_allclose(H1, H2) + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_nmf_sparse_transform(Estimator, solver, csc_container): + # Test that transform works on sparse data. Issue #2124 + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(3, 2)) + A[1, 1] = 0 + A = csc_container(A) + + model = Estimator(random_state=0, n_components=2, max_iter=400, **solver) + A_fit_tr = model.fit_transform(A) + A_tr = model.transform(A) + assert_allclose(A_fit_tr, A_tr, atol=1e-1) + + +@pytest.mark.parametrize("init", ["random", "nndsvd"]) +@pytest.mark.parametrize("solver", ("cd", "mu")) +@pytest.mark.parametrize("alpha_W", (0.0, 1.0)) +@pytest.mark.parametrize("alpha_H", (0.0, 1.0, "same")) +def test_non_negative_factorization_consistency(init, solver, alpha_W, alpha_H): + # Test that the function is called in the same way, either directly + # or through the NMF class + max_iter = 500 + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(10, 10)) + A[:, 2 * np.arange(5)] = 0 + + W_nmf, H, _ = non_negative_factorization( + A, + init=init, + solver=solver, + max_iter=max_iter, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=1, + tol=1e-2, + ) + W_nmf_2, H, _ = non_negative_factorization( + A, + H=H, + update_H=False, + init=init, + solver=solver, + max_iter=max_iter, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=1, + tol=1e-2, + ) + + model_class = NMF( + init=init, + solver=solver, + max_iter=max_iter, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=1, + tol=1e-2, + ) + W_cls = model_class.fit_transform(A) + W_cls_2 = model_class.transform(A) + + assert_allclose(W_nmf, W_cls) + assert_allclose(W_nmf_2, W_cls_2) + + +def test_non_negative_factorization_checking(): + # Note that the validity of parameter types and range of possible values + # for scalar numerical or str parameters is already checked in the common + # tests. Here we only check for problems that cannot be captured by simple + # declarative constraints on the valid parameter values. + + A = np.ones((2, 2)) + # Test parameters checking in public function + nnmf = non_negative_factorization + msg = re.escape("Negative values in data passed to NMF (input H)") + with pytest.raises(ValueError, match=msg): + nnmf(A, A, -A, 2, init="custom") + msg = re.escape("Negative values in data passed to NMF (input W)") + with pytest.raises(ValueError, match=msg): + nnmf(A, -A, A, 2, init="custom") + msg = re.escape("Array passed to NMF (input H) is full of zeros") + with pytest.raises(ValueError, match=msg): + nnmf(A, A, 0 * A, 2, init="custom") + + +def _beta_divergence_dense(X, W, H, beta): + """Compute the beta-divergence of X and W.H for dense array only. + + Used as a reference for testing nmf._beta_divergence. + """ + WH = np.dot(W, H) + + if beta == 2: + return squared_norm(X - WH) / 2 + + WH_Xnonzero = WH[X != 0] + X_nonzero = X[X != 0] + np.maximum(WH_Xnonzero, 1e-9, out=WH_Xnonzero) + + if beta == 1: + res = np.sum(X_nonzero * np.log(X_nonzero / WH_Xnonzero)) + res += WH.sum() - X.sum() + + elif beta == 0: + div = X_nonzero / WH_Xnonzero + res = np.sum(div) - X.size - np.sum(np.log(div)) + else: + res = (X_nonzero**beta).sum() + res += (beta - 1) * (WH**beta).sum() + res -= beta * (X_nonzero * (WH_Xnonzero ** (beta - 1))).sum() + res /= beta * (beta - 1) + + return res + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_beta_divergence(csr_container): + # Compare _beta_divergence with the reference _beta_divergence_dense + n_samples = 20 + n_features = 10 + n_components = 5 + beta_losses = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0] + + # initialization + rng = np.random.mtrand.RandomState(42) + X = rng.randn(n_samples, n_features) + np.clip(X, 0, None, out=X) + X_csr = csr_container(X) + W, H = nmf._initialize_nmf(X, n_components, init="random", random_state=42) + + for beta in beta_losses: + ref = _beta_divergence_dense(X, W, H, beta) + loss = nmf._beta_divergence(X, W, H, beta) + loss_csr = nmf._beta_divergence(X_csr, W, H, beta) + + assert_almost_equal(ref, loss, decimal=7) + assert_almost_equal(ref, loss_csr, decimal=7) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_special_sparse_dot(csr_container): + # Test the function that computes np.dot(W, H), only where X is non zero. + n_samples = 10 + n_features = 5 + n_components = 3 + rng = np.random.mtrand.RandomState(42) + X = rng.randn(n_samples, n_features) + np.clip(X, 0, None, out=X) + X_csr = csr_container(X) + + W = np.abs(rng.randn(n_samples, n_components)) + H = np.abs(rng.randn(n_components, n_features)) + + WH_safe = nmf._special_sparse_dot(W, H, X_csr) + WH = nmf._special_sparse_dot(W, H, X) + + # test that both results have same values, in X_csr nonzero elements + ii, jj = X_csr.nonzero() + WH_safe_data = np.asarray(WH_safe[ii, jj]).ravel() + assert_array_almost_equal(WH_safe_data, WH[ii, jj], decimal=10) + + # test that WH_safe and X_csr have the same sparse structure + assert_array_equal(WH_safe.indices, X_csr.indices) + assert_array_equal(WH_safe.indptr, X_csr.indptr) + assert_array_equal(WH_safe.shape, X_csr.shape) + + +@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning") +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_nmf_multiplicative_update_sparse(csr_container): + # Compare sparse and dense input in multiplicative update NMF + # Also test continuity of the results with respect to beta_loss parameter + n_samples = 20 + n_features = 10 + n_components = 5 + alpha = 0.1 + l1_ratio = 0.5 + n_iter = 20 + + # initialization + rng = np.random.mtrand.RandomState(1337) + X = rng.randn(n_samples, n_features) + X = np.abs(X) + X_csr = csr_container(X) + W0, H0 = nmf._initialize_nmf(X, n_components, init="random", random_state=42) + + for beta_loss in (-1.2, 0, 0.2, 1.0, 2.0, 2.5): + # Reference with dense array X + W, H = W0.copy(), H0.copy() + W1, H1, _ = non_negative_factorization( + X, + W, + H, + n_components, + init="custom", + update_H=True, + solver="mu", + beta_loss=beta_loss, + max_iter=n_iter, + alpha_W=alpha, + l1_ratio=l1_ratio, + random_state=42, + ) + + # Compare with sparse X + W, H = W0.copy(), H0.copy() + W2, H2, _ = non_negative_factorization( + X_csr, + W, + H, + n_components, + init="custom", + update_H=True, + solver="mu", + beta_loss=beta_loss, + max_iter=n_iter, + alpha_W=alpha, + l1_ratio=l1_ratio, + random_state=42, + ) + + assert_allclose(W1, W2, atol=1e-7) + assert_allclose(H1, H2, atol=1e-7) + + # Compare with almost same beta_loss, since some values have a specific + # behavior, but the results should be continuous w.r.t beta_loss + beta_loss -= 1.0e-5 + W, H = W0.copy(), H0.copy() + W3, H3, _ = non_negative_factorization( + X_csr, + W, + H, + n_components, + init="custom", + update_H=True, + solver="mu", + beta_loss=beta_loss, + max_iter=n_iter, + alpha_W=alpha, + l1_ratio=l1_ratio, + random_state=42, + ) + + assert_allclose(W1, W3, atol=1e-4) + assert_allclose(H1, H3, atol=1e-4) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_nmf_negative_beta_loss(csr_container): + # Test that an error is raised if beta_loss < 0 and X contains zeros. + # Test that the output has not NaN values when the input contains zeros. + n_samples = 6 + n_features = 5 + n_components = 3 + + rng = np.random.mtrand.RandomState(42) + X = rng.randn(n_samples, n_features) + np.clip(X, 0, None, out=X) + X_csr = csr_container(X) + + def _assert_nmf_no_nan(X, beta_loss): + W, H, _ = non_negative_factorization( + X, + init="random", + n_components=n_components, + solver="mu", + beta_loss=beta_loss, + random_state=0, + max_iter=1000, + ) + assert not np.any(np.isnan(W)) + assert not np.any(np.isnan(H)) + + msg = "When beta_loss <= 0 and X contains zeros, the solver may diverge." + for beta_loss in (-0.6, 0.0): + with pytest.raises(ValueError, match=msg): + _assert_nmf_no_nan(X, beta_loss) + _assert_nmf_no_nan(X + 1e-9, beta_loss) + + for beta_loss in (0.2, 1.0, 1.2, 2.0, 2.5): + _assert_nmf_no_nan(X, beta_loss) + _assert_nmf_no_nan(X_csr, beta_loss) + + +@pytest.mark.parametrize("beta_loss", [-0.5, 0.0]) +def test_minibatch_nmf_negative_beta_loss(beta_loss): + """Check that an error is raised if beta_loss < 0 and X contains zeros.""" + rng = np.random.RandomState(0) + X = rng.normal(size=(6, 5)) + X[X < 0] = 0 + + nmf = MiniBatchNMF(beta_loss=beta_loss, random_state=0) + + msg = "When beta_loss <= 0 and X contains zeros, the solver may diverge." + with pytest.raises(ValueError, match=msg): + nmf.fit(X) + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_regularization(Estimator, solver): + # Test the effect of L1 and L2 regularizations + n_samples = 6 + n_features = 5 + n_components = 3 + rng = np.random.mtrand.RandomState(42) + X = np.abs(rng.randn(n_samples, n_features)) + + # L1 regularization should increase the number of zeros + l1_ratio = 1.0 + regul = Estimator( + n_components=n_components, + alpha_W=0.5, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + model = Estimator( + n_components=n_components, + alpha_W=0.0, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + + W_regul = regul.fit_transform(X) + W_model = model.fit_transform(X) + + H_regul = regul.components_ + H_model = model.components_ + + eps = np.finfo(np.float64).eps + W_regul_n_zeros = W_regul[W_regul <= eps].size + W_model_n_zeros = W_model[W_model <= eps].size + H_regul_n_zeros = H_regul[H_regul <= eps].size + H_model_n_zeros = H_model[H_model <= eps].size + + assert W_regul_n_zeros > W_model_n_zeros + assert H_regul_n_zeros > H_model_n_zeros + + # L2 regularization should decrease the sum of the squared norm + # of the matrices W and H + l1_ratio = 0.0 + regul = Estimator( + n_components=n_components, + alpha_W=0.5, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + model = Estimator( + n_components=n_components, + alpha_W=0.0, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + + W_regul = regul.fit_transform(X) + W_model = model.fit_transform(X) + + H_regul = regul.components_ + H_model = model.components_ + + assert (linalg.norm(W_model)) ** 2.0 + (linalg.norm(H_model)) ** 2.0 > ( + linalg.norm(W_regul) + ) ** 2.0 + (linalg.norm(H_regul)) ** 2.0 + + +@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning") +@pytest.mark.parametrize("solver", ("cd", "mu")) +def test_nmf_decreasing(solver): + # test that the objective function is decreasing at each iteration + n_samples = 20 + n_features = 15 + n_components = 10 + alpha = 0.1 + l1_ratio = 0.5 + tol = 0.0 + + # initialization + rng = np.random.mtrand.RandomState(42) + X = rng.randn(n_samples, n_features) + np.abs(X, X) + W0, H0 = nmf._initialize_nmf(X, n_components, init="random", random_state=42) + + for beta_loss in (-1.2, 0, 0.2, 1.0, 2.0, 2.5): + if solver != "mu" and beta_loss != 2: + # not implemented + continue + W, H = W0.copy(), H0.copy() + previous_loss = None + for _ in range(30): + # one more iteration starting from the previous results + W, H, _ = non_negative_factorization( + X, + W, + H, + beta_loss=beta_loss, + init="custom", + n_components=n_components, + max_iter=1, + alpha_W=alpha, + solver=solver, + tol=tol, + l1_ratio=l1_ratio, + verbose=0, + random_state=0, + update_H=True, + ) + + loss = ( + nmf._beta_divergence(X, W, H, beta_loss) + + alpha * l1_ratio * n_features * W.sum() + + alpha * l1_ratio * n_samples * H.sum() + + alpha * (1 - l1_ratio) * n_features * (W**2).sum() + + alpha * (1 - l1_ratio) * n_samples * (H**2).sum() + ) + if previous_loss is not None: + assert previous_loss > loss + previous_loss = loss + + +def test_nmf_underflow(): + # Regression test for an underflow issue in _beta_divergence + rng = np.random.RandomState(0) + n_samples, n_features, n_components = 10, 2, 2 + X = np.abs(rng.randn(n_samples, n_features)) * 10 + W = np.abs(rng.randn(n_samples, n_components)) * 10 + H = np.abs(rng.randn(n_components, n_features)) + + X[0, 0] = 0 + ref = nmf._beta_divergence(X, W, H, beta=1.0) + X[0, 0] = 1e-323 + res = nmf._beta_divergence(X, W, H, beta=1.0) + assert_almost_equal(res, ref) + + +@pytest.mark.parametrize( + "dtype_in, dtype_out", + [ + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ], +) +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_dtype_match(Estimator, solver, dtype_in, dtype_out): + # Check that NMF preserves dtype (float32 and float64) + X = np.random.RandomState(0).randn(20, 15).astype(dtype_in, copy=False) + np.abs(X, out=X) + + nmf = Estimator( + alpha_W=1.0, + alpha_H=1.0, + tol=1e-2, + random_state=0, + **solver, + ) + + assert nmf.fit(X).transform(X).dtype == dtype_out + assert nmf.fit_transform(X).dtype == dtype_out + assert nmf.components_.dtype == dtype_out + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_float32_float64_consistency(Estimator, solver): + # Check that the result of NMF is the same between float32 and float64 + X = np.random.RandomState(0).randn(50, 7) + np.abs(X, out=X) + nmf32 = Estimator(random_state=0, tol=1e-3, **solver) + W32 = nmf32.fit_transform(X.astype(np.float32)) + nmf64 = Estimator(random_state=0, tol=1e-3, **solver) + W64 = nmf64.fit_transform(X) + + assert_allclose(W32, W64, atol=1e-5) + + +@pytest.mark.parametrize("Estimator", [NMF, MiniBatchNMF]) +def test_nmf_custom_init_dtype_error(Estimator): + # Check that an error is raise if custom H and/or W don't have the same + # dtype as X. + rng = np.random.RandomState(0) + X = rng.random_sample((20, 15)) + H = rng.random_sample((15, 15)).astype(np.float32) + W = rng.random_sample((20, 15)) + + with pytest.raises(TypeError, match="should have the same dtype as X"): + Estimator(init="custom").fit(X, H=H, W=W) + + with pytest.raises(TypeError, match="should have the same dtype as X"): + non_negative_factorization(X, H=H, update_H=False) + + +@pytest.mark.parametrize("beta_loss", [-0.5, 0, 0.5, 1, 1.5, 2, 2.5]) +def test_nmf_minibatchnmf_equivalence(beta_loss): + # Test that MiniBatchNMF is equivalent to NMF when batch_size = n_samples and + # forget_factor 0.0 (stopping criterion put aside) + rng = np.random.mtrand.RandomState(42) + X = np.abs(rng.randn(48, 5)) + + nmf = NMF( + n_components=5, + beta_loss=beta_loss, + solver="mu", + random_state=0, + tol=0, + ) + mbnmf = MiniBatchNMF( + n_components=5, + beta_loss=beta_loss, + random_state=0, + tol=0, + max_no_improvement=None, + batch_size=X.shape[0], + forget_factor=0.0, + ) + W = nmf.fit_transform(X) + mbW = mbnmf.fit_transform(X) + assert_allclose(W, mbW) + + +def test_minibatch_nmf_partial_fit(): + # Check fit / partial_fit equivalence. Applicable only with fresh restarts. + rng = np.random.mtrand.RandomState(42) + X = np.abs(rng.randn(100, 5)) + + n_components = 5 + batch_size = 10 + max_iter = 2 + + mbnmf1 = MiniBatchNMF( + n_components=n_components, + init="custom", + random_state=0, + max_iter=max_iter, + batch_size=batch_size, + tol=0, + max_no_improvement=None, + fresh_restarts=False, + ) + mbnmf2 = MiniBatchNMF(n_components=n_components, init="custom", random_state=0) + + # Force the same init of H (W is recomputed anyway) to be able to compare results. + W, H = nmf._initialize_nmf( + X, n_components=n_components, init="random", random_state=0 + ) + + mbnmf1.fit(X, W=W, H=H) + for i in range(max_iter): + for j in range(batch_size): + mbnmf2.partial_fit(X[j : j + batch_size], W=W[:batch_size], H=H) + + assert mbnmf1.n_steps_ == mbnmf2.n_steps_ + assert_allclose(mbnmf1.components_, mbnmf2.components_) + + +def test_feature_names_out(): + """Check feature names out for NMF.""" + random_state = np.random.RandomState(0) + X = np.abs(random_state.randn(10, 4)) + nmf = NMF(n_components=3).fit(X) + + names = nmf.get_feature_names_out() + assert_array_equal([f"nmf{i}" for i in range(3)], names) + + +def test_minibatch_nmf_verbose(): + # Check verbose mode of MiniBatchNMF for better coverage. + A = np.random.RandomState(0).random_sample((100, 10)) + nmf = MiniBatchNMF(tol=1e-2, random_state=0, verbose=1) + old_stdout = sys.stdout + sys.stdout = StringIO() + try: + nmf.fit(A) + finally: + sys.stdout = old_stdout + + +@pytest.mark.parametrize("Estimator", [NMF, MiniBatchNMF]) +def test_nmf_n_components_auto(Estimator): + # Check that n_components is correctly inferred + # from the provided custom initialization. + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + W = rng.random_sample((6, 2)) + H = rng.random_sample((2, 5)) + est = Estimator( + n_components="auto", + init="custom", + random_state=0, + tol=1e-6, + ) + est.fit_transform(X, W=W, H=H) + assert est._n_components == H.shape[0] + + +def test_nmf_non_negative_factorization_n_components_auto(): + # Check that n_components is correctly inferred from the provided + # custom initialization. + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + W_init = rng.random_sample((6, 2)) + H_init = rng.random_sample((2, 5)) + W, H, _ = non_negative_factorization( + X, W=W_init, H=H_init, init="custom", n_components="auto" + ) + assert H.shape == H_init.shape + assert W.shape == W_init.shape + + +def test_nmf_n_components_auto_no_h_update(): + # Tests that non_negative_factorization does not fail when setting + # n_components="auto" also tests that the inferred n_component + # value is the right one. + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + H_true = rng.random_sample((2, 5)) + W, H, _ = non_negative_factorization( + X, H=H_true, n_components="auto", update_H=False + ) # should not fail + assert_allclose(H, H_true) + assert W.shape == (X.shape[0], H_true.shape[0]) + + +def test_nmf_w_h_not_used_warning(): + # Check that warnings are raised if user provided W and H are not used + # and initialization overrides value of W or H + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + W_init = rng.random_sample((6, 2)) + H_init = rng.random_sample((2, 5)) + with pytest.warns( + RuntimeWarning, + match="When init!='custom', provided W or H are ignored", + ): + non_negative_factorization(X, H=H_init, update_H=True, n_components="auto") + + with pytest.warns( + RuntimeWarning, + match="When init!='custom', provided W or H are ignored", + ): + non_negative_factorization( + X, W=W_init, H=H_init, update_H=True, n_components="auto" + ) + + with pytest.warns( + RuntimeWarning, match="When update_H=False, the provided initial W is not used." + ): + # When update_H is False, W is ignored regardless of init + # TODO: use the provided W when init="custom". + non_negative_factorization( + X, W=W_init, H=H_init, update_H=False, n_components="auto" + ) + + +def test_nmf_custom_init_shape_error(): + # Check that an informative error is raised when custom initialization does not + # have the right shape + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + H = rng.random_sample((2, 5)) + nmf = NMF(n_components=2, init="custom", random_state=0) + + with pytest.raises(ValueError, match="Array with wrong first dimension passed"): + nmf.fit(X, H=H, W=rng.random_sample((5, 2))) + + with pytest.raises(ValueError, match="Array with wrong second dimension passed"): + nmf.fit(X, H=H, W=rng.random_sample((6, 3))) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_online_lda.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_online_lda.py new file mode 100644 index 0000000000000000000000000000000000000000..420935bf00812852af30a331d6f2f40ac1d22cff --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_online_lda.py @@ -0,0 +1,489 @@ +import sys +from io import StringIO + +import numpy as np +import pytest +from numpy.testing import assert_array_equal +from scipy.linalg import block_diag +from scipy.special import psi + +from sklearn.decomposition import LatentDirichletAllocation +from sklearn.decomposition._online_lda_fast import ( + _dirichlet_expectation_1d, + _dirichlet_expectation_2d, +) +from sklearn.exceptions import NotFittedError +from sklearn.utils._testing import ( + assert_allclose, + assert_almost_equal, + assert_array_almost_equal, + if_safe_multiprocessing_with_blas, +) +from sklearn.utils.fixes import CSR_CONTAINERS + + +def _build_sparse_array(csr_container): + # Create 3 topics and each topic has 3 distinct words. + # (Each word only belongs to a single topic.) + n_components = 3 + block = np.full((3, 3), n_components, dtype=int) + blocks = [block] * n_components + X = block_diag(*blocks) + X = csr_container(X) + return (n_components, X) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_default_prior_params(csr_container): + # default prior parameter should be `1 / topics` + # and verbose params should not affect result + n_components, X = _build_sparse_array(csr_container) + prior = 1.0 / n_components + lda_1 = LatentDirichletAllocation( + n_components=n_components, + doc_topic_prior=prior, + topic_word_prior=prior, + random_state=0, + ) + lda_2 = LatentDirichletAllocation(n_components=n_components, random_state=0) + topic_distr_1 = lda_1.fit_transform(X) + topic_distr_2 = lda_2.fit_transform(X) + assert_almost_equal(topic_distr_1, topic_distr_2) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_fit_batch(csr_container): + # Test LDA batch learning_offset (`fit` method with 'batch' learning) + rng = np.random.RandomState(0) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + evaluate_every=1, + learning_method="batch", + random_state=rng, + ) + lda.fit(X) + + correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] + for component in lda.components_: + # Find top 3 words in each LDA component + top_idx = set(component.argsort()[-3:][::-1]) + assert tuple(sorted(top_idx)) in correct_idx_grps + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_fit_online(csr_container): + # Test LDA online learning (`fit` method with 'online' learning) + rng = np.random.RandomState(0) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + learning_offset=10.0, + evaluate_every=1, + learning_method="online", + random_state=rng, + ) + lda.fit(X) + + correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] + for component in lda.components_: + # Find top 3 words in each LDA component + top_idx = set(component.argsort()[-3:][::-1]) + assert tuple(sorted(top_idx)) in correct_idx_grps + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_partial_fit(csr_container): + # Test LDA online learning (`partial_fit` method) + # (same as test_lda_batch) + rng = np.random.RandomState(0) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + learning_offset=10.0, + total_samples=100, + random_state=rng, + ) + for i in range(3): + lda.partial_fit(X) + + correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] + for c in lda.components_: + top_idx = set(c.argsort()[-3:][::-1]) + assert tuple(sorted(top_idx)) in correct_idx_grps + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_dense_input(csr_container): + # Test LDA with dense input. + rng = np.random.RandomState(0) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, learning_method="batch", random_state=rng + ) + lda.fit(X.toarray()) + + correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] + for component in lda.components_: + # Find top 3 words in each LDA component + top_idx = set(component.argsort()[-3:][::-1]) + assert tuple(sorted(top_idx)) in correct_idx_grps + + +def test_lda_transform(): + # Test LDA transform. + # Transform result cannot be negative and should be normalized by default + rng = np.random.RandomState(0) + X = rng.randint(5, size=(20, 10)) + n_components = 3 + lda = LatentDirichletAllocation(n_components=n_components, random_state=rng) + X_trans = lda.fit_transform(X) + assert (X_trans > 0.0).any() + assert_array_almost_equal(np.sum(X_trans, axis=1), np.ones(X_trans.shape[0])) + + X_trans_unnormalized = lda.transform(X, normalize=False) + assert_array_almost_equal( + X_trans, X_trans_unnormalized / X_trans_unnormalized.sum(axis=1)[:, np.newaxis] + ) + + +@pytest.mark.parametrize("method", ("online", "batch")) +def test_lda_fit_transform(method): + # Test LDA fit_transform & transform + # fit_transform and transform result should be the same + rng = np.random.RandomState(0) + X = rng.randint(10, size=(50, 20)) + lda = LatentDirichletAllocation( + n_components=5, learning_method=method, random_state=rng + ) + X_fit = lda.fit_transform(X) + X_trans = lda.transform(X) + assert_array_almost_equal(X_fit, X_trans, 4) + + +def test_lda_negative_input(): + # test pass dense matrix with sparse negative input. + X = np.full((5, 10), -1.0) + lda = LatentDirichletAllocation() + regex = r"^Negative values in data passed" + with pytest.raises(ValueError, match=regex): + lda.fit(X) + + +def test_lda_no_component_error(): + # test `perplexity` before `fit` + rng = np.random.RandomState(0) + X = rng.randint(4, size=(20, 10)) + lda = LatentDirichletAllocation() + regex = ( + "This LatentDirichletAllocation instance is not fitted yet. " + "Call 'fit' with appropriate arguments before using this " + "estimator." + ) + with pytest.raises(NotFittedError, match=regex): + lda.perplexity(X) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +@if_safe_multiprocessing_with_blas +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +@pytest.mark.parametrize("method", ("online", "batch")) +def test_lda_multi_jobs(method, csr_container): + n_components, X = _build_sparse_array(csr_container) + # Test LDA batch training with multi CPU + rng = np.random.RandomState(0) + lda = LatentDirichletAllocation( + n_components=n_components, + n_jobs=2, + learning_method=method, + evaluate_every=1, + random_state=rng, + ) + lda.fit(X) + + correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] + for c in lda.components_: + top_idx = set(c.argsort()[-3:][::-1]) + assert tuple(sorted(top_idx)) in correct_idx_grps + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +@if_safe_multiprocessing_with_blas +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_partial_fit_multi_jobs(csr_container): + # Test LDA online training with multi CPU + rng = np.random.RandomState(0) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + n_jobs=2, + learning_offset=5.0, + total_samples=30, + random_state=rng, + ) + for i in range(2): + lda.partial_fit(X) + + correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] + for c in lda.components_: + top_idx = set(c.argsort()[-3:][::-1]) + assert tuple(sorted(top_idx)) in correct_idx_grps + + +def test_lda_preplexity_mismatch(): + # test dimension mismatch in `perplexity` method + rng = np.random.RandomState(0) + n_components = rng.randint(3, 6) + n_samples = rng.randint(6, 10) + X = np.random.randint(4, size=(n_samples, 10)) + lda = LatentDirichletAllocation( + n_components=n_components, + learning_offset=5.0, + total_samples=20, + random_state=rng, + ) + lda.fit(X) + # invalid samples + invalid_n_samples = rng.randint(4, size=(n_samples + 1, n_components)) + with pytest.raises(ValueError, match=r"Number of samples"): + lda._perplexity_precomp_distr(X, invalid_n_samples) + # invalid topic number + invalid_n_components = rng.randint(4, size=(n_samples, n_components + 1)) + with pytest.raises(ValueError, match=r"Number of topics"): + lda._perplexity_precomp_distr(X, invalid_n_components) + + +@pytest.mark.parametrize("method", ("online", "batch")) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_perplexity(method, csr_container): + # Test LDA perplexity for batch training + # perplexity should be lower after each iteration + n_components, X = _build_sparse_array(csr_container) + lda_1 = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method=method, + total_samples=100, + random_state=0, + ) + lda_2 = LatentDirichletAllocation( + n_components=n_components, + max_iter=10, + learning_method=method, + total_samples=100, + random_state=0, + ) + lda_1.fit(X) + perp_1 = lda_1.perplexity(X, sub_sampling=False) + + lda_2.fit(X) + perp_2 = lda_2.perplexity(X, sub_sampling=False) + assert perp_1 >= perp_2 + + perp_1_subsampling = lda_1.perplexity(X, sub_sampling=True) + perp_2_subsampling = lda_2.perplexity(X, sub_sampling=True) + assert perp_1_subsampling >= perp_2_subsampling + + +@pytest.mark.parametrize("method", ("online", "batch")) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_score(method, csr_container): + # Test LDA score for batch training + # score should be higher after each iteration + n_components, X = _build_sparse_array(csr_container) + lda_1 = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method=method, + total_samples=100, + random_state=0, + ) + lda_2 = LatentDirichletAllocation( + n_components=n_components, + max_iter=10, + learning_method=method, + total_samples=100, + random_state=0, + ) + lda_1.fit_transform(X) + score_1 = lda_1.score(X) + + lda_2.fit_transform(X) + score_2 = lda_2.score(X) + assert score_2 >= score_1 + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_perplexity_input_format(csr_container): + # Test LDA perplexity for sparse and dense input + # score should be the same for both dense and sparse input + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method="batch", + total_samples=100, + random_state=0, + ) + lda.fit(X) + perp_1 = lda.perplexity(X) + perp_2 = lda.perplexity(X.toarray()) + assert_almost_equal(perp_1, perp_2) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_score_perplexity(csr_container): + # Test the relationship between LDA score and perplexity + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, max_iter=10, random_state=0 + ) + lda.fit(X) + perplexity_1 = lda.perplexity(X, sub_sampling=False) + + score = lda.score(X) + perplexity_2 = np.exp(-1.0 * (score / np.sum(X.data))) + assert_almost_equal(perplexity_1, perplexity_2) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_fit_perplexity(csr_container): + # Test that the perplexity computed during fit is consistent with what is + # returned by the perplexity method + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method="batch", + random_state=0, + evaluate_every=1, + ) + lda.fit(X) + + # Perplexity computed at end of fit method + perplexity1 = lda.bound_ + + # Result of perplexity method on the train set + perplexity2 = lda.perplexity(X) + + assert_almost_equal(perplexity1, perplexity2) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_empty_docs(csr_container): + """Test LDA on empty document (all-zero rows).""" + Z = np.zeros((5, 4)) + for X in [Z, csr_container(Z)]: + lda = LatentDirichletAllocation(max_iter=750).fit(X) + assert_almost_equal( + lda.components_.sum(axis=0), np.ones(lda.components_.shape[1]) + ) + + +def test_dirichlet_expectation(): + """Test Cython version of Dirichlet expectation calculation.""" + x = np.logspace(-100, 10, 10000) + expectation = np.empty_like(x) + _dirichlet_expectation_1d(x, 0, expectation) + assert_allclose(expectation, np.exp(psi(x) - psi(np.sum(x))), atol=1e-19) + + x = x.reshape(100, 100) + assert_allclose( + _dirichlet_expectation_2d(x), + psi(x) - psi(np.sum(x, axis=1)[:, np.newaxis]), + rtol=1e-11, + atol=3e-9, + ) + + +def check_verbosity( + verbose, evaluate_every, expected_lines, expected_perplexities, csr_container +): + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + max_iter=3, + learning_method="batch", + verbose=verbose, + evaluate_every=evaluate_every, + random_state=0, + ) + out = StringIO() + old_out, sys.stdout = sys.stdout, out + try: + lda.fit(X) + finally: + sys.stdout = old_out + + n_lines = out.getvalue().count("\n") + n_perplexity = out.getvalue().count("perplexity") + assert expected_lines == n_lines + assert expected_perplexities == n_perplexity + + +@pytest.mark.parametrize( + "verbose,evaluate_every,expected_lines,expected_perplexities", + [ + (False, 1, 0, 0), + (False, 0, 0, 0), + (True, 0, 3, 0), + (True, 1, 3, 3), + (True, 2, 3, 1), + ], +) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +@pytest.mark.thread_unsafe # manually captured stdout +def test_verbosity( + verbose, evaluate_every, expected_lines, expected_perplexities, csr_container +): + check_verbosity( + verbose, evaluate_every, expected_lines, expected_perplexities, csr_container + ) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_feature_names_out(csr_container): + """Check feature names out for LatentDirichletAllocation.""" + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation(n_components=n_components).fit(X) + + names = lda.get_feature_names_out() + assert_array_equal( + [f"latentdirichletallocation{i}" for i in range(n_components)], names + ) + + +@pytest.mark.parametrize("learning_method", ("batch", "online")) +def test_lda_dtype_match(learning_method, global_dtype): + """Check data type preservation of fitted attributes.""" + rng = np.random.RandomState(0) + X = rng.uniform(size=(20, 10)).astype(global_dtype, copy=False) + + lda = LatentDirichletAllocation( + n_components=5, random_state=0, learning_method=learning_method + ) + lda.fit(X) + assert lda.components_.dtype == global_dtype + assert lda.exp_dirichlet_component_.dtype == global_dtype + + +@pytest.mark.parametrize("learning_method", ("batch", "online")) +def test_lda_numerical_consistency(learning_method, global_random_seed): + """Check numerical consistency between np.float32 and np.float64.""" + rng = np.random.RandomState(global_random_seed) + X64 = rng.uniform(size=(20, 10)) + X32 = X64.astype(np.float32) + + lda_64 = LatentDirichletAllocation( + n_components=5, random_state=global_random_seed, learning_method=learning_method + ).fit(X64) + lda_32 = LatentDirichletAllocation( + n_components=5, random_state=global_random_seed, learning_method=learning_method + ).fit(X32) + + assert_allclose(lda_32.components_, lda_64.components_) + assert_allclose(lda_32.transform(X32), lda_64.transform(X64)) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..c53f5f4d042c173cdcdba5ec41c745eeb31fcb4b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_pca.py @@ -0,0 +1,1155 @@ +import os +import re +import warnings + +import numpy as np +import pytest +import scipy as sp +from numpy.testing import assert_array_equal + +from sklearn import config_context, datasets +from sklearn.base import clone +from sklearn.datasets import load_iris, make_classification, make_low_rank_matrix +from sklearn.decomposition import PCA +from sklearn.decomposition._pca import _assess_dimension, _infer_dimension +from sklearn.utils._array_api import ( + _atol_for_type, + _convert_to_numpy, + _get_namespace_device_dtype_ids, + yield_namespace_device_dtype_combinations, +) +from sklearn.utils._array_api import device as array_device +from sklearn.utils._test_common.instance_generator import _get_check_estimator_ids +from sklearn.utils._testing import _array_api_for_tests, assert_allclose +from sklearn.utils.estimator_checks import ( + check_array_api_input_and_values, +) +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS + +iris = datasets.load_iris() +PCA_SOLVERS = ["full", "covariance_eigh", "arpack", "randomized", "auto"] + +# `SPARSE_M` and `SPARSE_N` could be larger, but be aware: +# * SciPy's generation of random sparse matrix can be costly +# * A (SPARSE_M, SPARSE_N) dense array is allocated to compare against +SPARSE_M, SPARSE_N = 1000, 300 # arbitrary +SPARSE_MAX_COMPONENTS = min(SPARSE_M, SPARSE_N) + + +def _check_fitted_pca_close(pca1, pca2, rtol=1e-7, atol=1e-12): + assert_allclose(pca1.components_, pca2.components_, rtol=rtol, atol=atol) + assert_allclose( + pca1.explained_variance_, pca2.explained_variance_, rtol=rtol, atol=atol + ) + assert_allclose(pca1.singular_values_, pca2.singular_values_, rtol=rtol, atol=atol) + assert_allclose(pca1.mean_, pca2.mean_, rtol=rtol, atol=atol) + assert_allclose(pca1.noise_variance_, pca2.noise_variance_, rtol=rtol, atol=atol) + + assert pca1.n_components_ == pca2.n_components_ + assert pca1.n_samples_ == pca2.n_samples_ + assert pca1.n_features_in_ == pca2.n_features_in_ + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +@pytest.mark.parametrize("n_components", range(1, iris.data.shape[1])) +def test_pca(svd_solver, n_components): + X = iris.data + pca = PCA(n_components=n_components, svd_solver=svd_solver) + + # check the shape of fit.transform + X_r = pca.fit(X).transform(X) + assert X_r.shape[1] == n_components + + # check the equivalence of fit.transform and fit_transform + X_r2 = pca.fit_transform(X) + assert_allclose(X_r, X_r2) + X_r = pca.transform(X) + assert_allclose(X_r, X_r2) + + # Test get_covariance and get_precision + cov = pca.get_covariance() + precision = pca.get_precision() + assert_allclose(np.dot(cov, precision), np.eye(X.shape[1]), atol=1e-12) + + +@pytest.mark.parametrize("density", [0.01, 0.1, 0.30]) +@pytest.mark.parametrize("n_components", [1, 2, 10]) +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +@pytest.mark.parametrize("svd_solver", ["arpack", "covariance_eigh"]) +@pytest.mark.parametrize("scale", [1, 10, 100]) +def test_pca_sparse( + global_random_seed, svd_solver, sparse_container, n_components, density, scale +): + """Check that the results are the same for sparse and dense input.""" + + # Set atol in addition of the default rtol to account for the very wide range of + # result values (1e-8 to 1e0). + atol = 1e-12 + transform_atol = 1e-10 + + random_state = np.random.default_rng(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=density, + ) + ) + # Scale the data + vary the column means + scale_vector = random_state.random(X.shape[1]) * scale + X = X.multiply(scale_vector) + + pca = PCA( + n_components=n_components, + svd_solver=svd_solver, + random_state=global_random_seed, + ) + pca.fit(X) + + Xd = X.toarray() + pcad = PCA( + n_components=n_components, + svd_solver=svd_solver, + random_state=global_random_seed, + ) + pcad.fit(Xd) + + # Fitted attributes equality + _check_fitted_pca_close(pca, pcad, atol=atol) + + # Test transform + X2 = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=density, + ) + ) + X2d = X2.toarray() + + assert_allclose(pca.transform(X2), pca.transform(X2d), atol=transform_atol) + assert_allclose(pca.transform(X2), pcad.transform(X2d), atol=transform_atol) + + +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_pca_sparse_fit_transform(global_random_seed, sparse_container): + random_state = np.random.default_rng(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=0.01, + ) + ) + X2 = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=0.01, + ) + ) + + pca_fit = PCA(n_components=10, svd_solver="arpack", random_state=global_random_seed) + pca_fit_transform = PCA( + n_components=10, svd_solver="arpack", random_state=global_random_seed + ) + + pca_fit.fit(X) + transformed_X = pca_fit_transform.fit_transform(X) + + _check_fitted_pca_close(pca_fit, pca_fit_transform) + assert_allclose(transformed_X, pca_fit_transform.transform(X)) + assert_allclose(transformed_X, pca_fit.transform(X)) + assert_allclose(pca_fit.transform(X2), pca_fit_transform.transform(X2)) + + +@pytest.mark.parametrize("svd_solver", ["randomized", "full"]) +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_sparse_pca_solver_error(global_random_seed, svd_solver, sparse_container): + random_state = np.random.RandomState(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + ) + ) + pca = PCA(n_components=30, svd_solver=svd_solver) + error_msg_pattern = ( + 'PCA only support sparse inputs with the "arpack" and "covariance_eigh"' + f' solvers, while "{svd_solver}" was passed' + ) + with pytest.raises(TypeError, match=error_msg_pattern): + pca.fit(X) + + +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_sparse_pca_auto_arpack_singluar_values_consistency( + global_random_seed, sparse_container +): + """Check that "auto" and "arpack" solvers are equivalent for sparse inputs.""" + random_state = np.random.RandomState(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + ) + ) + pca_arpack = PCA(n_components=10, svd_solver="arpack").fit(X) + pca_auto = PCA(n_components=10, svd_solver="auto").fit(X) + assert_allclose(pca_arpack.singular_values_, pca_auto.singular_values_, rtol=5e-3) + + +def test_no_empty_slice_warning(): + # test if we avoid numpy warnings for computing over empty arrays + n_components = 10 + n_features = n_components + 2 # anything > n_comps triggered it in 0.16 + X = np.random.uniform(-1, 1, size=(n_components, n_features)) + pca = PCA(n_components=n_components) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + pca.fit(X) + + +@pytest.mark.parametrize("copy", [True, False]) +@pytest.mark.parametrize("solver", PCA_SOLVERS) +def test_whitening(solver, copy): + # Check that PCA output has unit-variance + rng = np.random.RandomState(0) + n_samples = 100 + n_features = 80 + n_components = 30 + rank = 50 + + # some low rank data with correlated features + X = np.dot( + rng.randn(n_samples, rank), + np.dot(np.diag(np.linspace(10.0, 1.0, rank)), rng.randn(rank, n_features)), + ) + # the component-wise variance of the first 50 features is 3 times the + # mean component-wise variance of the remaining 30 features + X[:, :50] *= 3 + + assert X.shape == (n_samples, n_features) + + # the component-wise variance is thus highly varying: + assert X.std(axis=0).std() > 43.8 + + # whiten the data while projecting to the lower dim subspace + X_ = X.copy() # make sure we keep an original across iterations. + pca = PCA( + n_components=n_components, + whiten=True, + copy=copy, + svd_solver=solver, + random_state=0, + iterated_power=7, + ) + # test fit_transform + X_whitened = pca.fit_transform(X_.copy()) + assert X_whitened.shape == (n_samples, n_components) + X_whitened2 = pca.transform(X_) + assert_allclose(X_whitened, X_whitened2, rtol=5e-4) + + assert_allclose(X_whitened.std(ddof=1, axis=0), np.ones(n_components)) + assert_allclose(X_whitened.mean(axis=0), np.zeros(n_components), atol=1e-12) + + X_ = X.copy() + pca = PCA( + n_components=n_components, whiten=False, copy=copy, svd_solver=solver + ).fit(X_.copy()) + X_unwhitened = pca.transform(X_) + assert X_unwhitened.shape == (n_samples, n_components) + + # in that case the output components still have varying variances + assert X_unwhitened.std(axis=0).std() == pytest.approx(74.1, rel=1e-1) + # we always center, so no test for non-centering. + + +@pytest.mark.parametrize( + "other_svd_solver", sorted(list(set(PCA_SOLVERS) - {"full", "auto"})) +) +@pytest.mark.parametrize("data_shape", ["tall", "wide"]) +@pytest.mark.parametrize("rank_deficient", [False, True]) +@pytest.mark.parametrize("whiten", [False, True]) +def test_pca_solver_equivalence( + other_svd_solver, + data_shape, + rank_deficient, + whiten, + global_random_seed, + global_dtype, +): + if data_shape == "tall": + n_samples, n_features = 100, 30 + else: + n_samples, n_features = 30, 100 + n_samples_test = 10 + + if rank_deficient: + rng = np.random.default_rng(global_random_seed) + rank = min(n_samples, n_features) // 2 + X = rng.standard_normal( + size=(n_samples + n_samples_test, rank) + ) @ rng.standard_normal(size=(rank, n_features)) + else: + X = make_low_rank_matrix( + n_samples=n_samples + n_samples_test, + n_features=n_features, + tail_strength=0.5, + random_state=global_random_seed, + ) + # With a non-zero tail strength, the data is actually full-rank. + rank = min(n_samples, n_features) + + X = X.astype(global_dtype, copy=False) + X_train, X_test = X[:n_samples], X[n_samples:] + + if global_dtype == np.float32: + tols = dict(atol=3e-2, rtol=1e-5) + variance_threshold = 1e-5 + else: + tols = dict(atol=1e-10, rtol=1e-12) + variance_threshold = 1e-12 + + extra_other_kwargs = {} + if other_svd_solver == "randomized": + # Only check for a truncated result with a large number of iterations + # to make sure that we can recover precise results. + n_components = 10 + extra_other_kwargs = {"iterated_power": 50} + elif other_svd_solver == "arpack": + # Test all components except the last one which cannot be estimated by + # arpack. + n_components = np.minimum(n_samples, n_features) - 1 + else: + # Test all components to high precision. + n_components = None + + pca_full = PCA(n_components=n_components, svd_solver="full", whiten=whiten) + pca_other = PCA( + n_components=n_components, + svd_solver=other_svd_solver, + whiten=whiten, + random_state=global_random_seed, + **extra_other_kwargs, + ) + X_trans_full_train = pca_full.fit_transform(X_train) + assert np.isfinite(X_trans_full_train).all() + assert X_trans_full_train.dtype == global_dtype + X_trans_other_train = pca_other.fit_transform(X_train) + assert np.isfinite(X_trans_other_train).all() + assert X_trans_other_train.dtype == global_dtype + + assert (pca_full.explained_variance_ >= 0).all() + assert_allclose(pca_full.explained_variance_, pca_other.explained_variance_, **tols) + assert_allclose( + pca_full.explained_variance_ratio_, + pca_other.explained_variance_ratio_, + **tols, + ) + reference_components = pca_full.components_ + assert np.isfinite(reference_components).all() + other_components = pca_other.components_ + assert np.isfinite(other_components).all() + + # For some choice of n_components and data distribution, some components + # might be pure noise, let's ignore them in the comparison: + stable = pca_full.explained_variance_ > variance_threshold + assert stable.sum() > 1 + assert_allclose(reference_components[stable], other_components[stable], **tols) + + # As a result the output of fit_transform should be the same: + assert_allclose( + X_trans_other_train[:, stable], X_trans_full_train[:, stable], **tols + ) + + # And similarly for the output of transform on new data (except for the + # last component that can be underdetermined): + X_trans_full_test = pca_full.transform(X_test) + assert np.isfinite(X_trans_full_test).all() + assert X_trans_full_test.dtype == global_dtype + X_trans_other_test = pca_other.transform(X_test) + assert np.isfinite(X_trans_other_test).all() + assert X_trans_other_test.dtype == global_dtype + assert_allclose(X_trans_other_test[:, stable], X_trans_full_test[:, stable], **tols) + + # Check that inverse transform reconstructions for both solvers are + # compatible. + X_recons_full_test = pca_full.inverse_transform(X_trans_full_test) + assert np.isfinite(X_recons_full_test).all() + assert X_recons_full_test.dtype == global_dtype + X_recons_other_test = pca_other.inverse_transform(X_trans_other_test) + assert np.isfinite(X_recons_other_test).all() + assert X_recons_other_test.dtype == global_dtype + + if pca_full.components_.shape[0] == pca_full.components_.shape[1]: + # In this case, the models should have learned the same invertible + # transform. They should therefore both be able to reconstruct the test + # data. + assert_allclose(X_recons_full_test, X_test, **tols) + assert_allclose(X_recons_other_test, X_test, **tols) + elif pca_full.components_.shape[0] < rank: + # In the absence of noisy components, both models should be able to + # reconstruct the same low-rank approximation of the original data. + assert pca_full.explained_variance_.min() > variance_threshold + assert_allclose(X_recons_full_test, X_recons_other_test, **tols) + else: + # When n_features > n_samples and n_components is larger than the rank + # of the training set, the output of the `inverse_transform` function + # is ill-defined. We can only check that we reach the same fixed point + # after another round of transform: + assert_allclose( + pca_full.transform(X_recons_full_test)[:, stable], + pca_other.transform(X_recons_other_test)[:, stable], + **tols, + ) + + +@pytest.mark.parametrize( + "X", + [ + np.random.RandomState(0).randn(100, 80), + datasets.make_classification(100, 80, n_informative=78, random_state=0)[0], + np.random.RandomState(0).randn(10, 100), + ], + ids=["random-tall", "correlated-tall", "random-wide"], +) +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_explained_variance_empirical(X, svd_solver): + pca = PCA(n_components=2, svd_solver=svd_solver, random_state=0) + X_pca = pca.fit_transform(X) + assert_allclose(pca.explained_variance_, np.var(X_pca, ddof=1, axis=0)) + + expected_result = np.linalg.eig(np.cov(X, rowvar=False))[0] + expected_result = sorted(expected_result, reverse=True)[:2] + assert_allclose(pca.explained_variance_, expected_result, rtol=5e-3) + + +@pytest.mark.parametrize("svd_solver", ["arpack", "randomized"]) +def test_pca_singular_values_consistency(svd_solver): + rng = np.random.RandomState(0) + n_samples, n_features = 100, 80 + X = rng.randn(n_samples, n_features) + + pca_full = PCA(n_components=2, svd_solver="full", random_state=rng) + pca_other = PCA(n_components=2, svd_solver=svd_solver, random_state=rng) + + pca_full.fit(X) + pca_other.fit(X) + + assert_allclose(pca_full.singular_values_, pca_other.singular_values_, rtol=5e-3) + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_singular_values(svd_solver): + rng = np.random.RandomState(0) + n_samples, n_features = 100, 80 + X = rng.randn(n_samples, n_features) + + pca = PCA(n_components=2, svd_solver=svd_solver, random_state=rng) + X_trans = pca.fit_transform(X) + + # compare to the Frobenius norm + assert_allclose( + np.sum(pca.singular_values_**2), np.linalg.norm(X_trans, "fro") ** 2 + ) + # Compare to the 2-norms of the score vectors + assert_allclose(pca.singular_values_, np.sqrt(np.sum(X_trans**2, axis=0))) + + # set the singular values and see what er get back + n_samples, n_features = 100, 110 + X = rng.randn(n_samples, n_features) + + pca = PCA(n_components=3, svd_solver=svd_solver, random_state=rng) + X_trans = pca.fit_transform(X) + X_trans /= np.sqrt(np.sum(X_trans**2, axis=0)) + X_trans[:, 0] *= 3.142 + X_trans[:, 1] *= 2.718 + X_hat = np.dot(X_trans, pca.components_) + pca.fit(X_hat) + assert_allclose(pca.singular_values_, [3.142, 2.718, 1.0]) + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_check_projection(svd_solver): + # Test that the projection of data is correct + rng = np.random.RandomState(0) + n, p = 100, 3 + X = rng.randn(n, p) * 0.1 + X[:10] += np.array([3, 4, 5]) + Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5]) + + Yt = PCA(n_components=2, svd_solver=svd_solver).fit(X).transform(Xt) + Yt /= np.sqrt((Yt**2).sum()) + + assert_allclose(np.abs(Yt[0][0]), 1.0, rtol=5e-3) + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_check_projection_list(svd_solver): + # Test that the projection of data is correct + X = [[1.0, 0.0], [0.0, 1.0]] + pca = PCA(n_components=1, svd_solver=svd_solver, random_state=0) + X_trans = pca.fit_transform(X) + assert X_trans.shape, (2, 1) + assert_allclose(X_trans.mean(), 0.00, atol=1e-12) + assert_allclose(X_trans.std(), 0.71, rtol=5e-3) + + +@pytest.mark.parametrize("svd_solver", ["full", "arpack", "randomized"]) +@pytest.mark.parametrize("whiten", [False, True]) +def test_pca_inverse(svd_solver, whiten): + # Test that the projection of data can be inverted + rng = np.random.RandomState(0) + n, p = 50, 3 + X = rng.randn(n, p) # spherical data + X[:, 1] *= 0.00001 # make middle component relatively small + X += [5, 4, 3] # make a large mean + + # same check that we can find the original data from the transformed + # signal (since the data is almost of rank n_components) + pca = PCA(n_components=2, svd_solver=svd_solver, whiten=whiten).fit(X) + Y = pca.transform(X) + Y_inverse = pca.inverse_transform(Y) + assert_allclose(X, Y_inverse, rtol=5e-6) + + +@pytest.mark.parametrize( + "data", [np.array([[0, 1, 0], [1, 0, 0]]), np.array([[0, 1, 0], [1, 0, 0]]).T] +) +@pytest.mark.parametrize( + "svd_solver, n_components, err_msg", + [ + ("arpack", 0, r"must be between 1 and min\(n_samples, n_features\)"), + ("randomized", 0, r"must be between 1 and min\(n_samples, n_features\)"), + ("arpack", 2, r"must be strictly less than min"), + ( + "auto", + 3, + ( + r"n_components=3 must be between 0 and min\(n_samples, " + r"n_features\)=2 with svd_solver='full'" + ), + ), + ], +) +def test_pca_validation(svd_solver, data, n_components, err_msg): + # Ensures that solver-specific extreme inputs for the n_components + # parameter raise errors + smallest_d = 2 # The smallest dimension + pca_fitted = PCA(n_components, svd_solver=svd_solver) + + with pytest.raises(ValueError, match=err_msg): + pca_fitted.fit(data) + + # Additional case for arpack + if svd_solver == "arpack": + n_components = smallest_d + + err_msg = ( + "n_components={}L? must be strictly less than " + r"min\(n_samples, n_features\)={}L? with " + "svd_solver='arpack'".format(n_components, smallest_d) + ) + with pytest.raises(ValueError, match=err_msg): + PCA(n_components, svd_solver=svd_solver).fit(data) + + +@pytest.mark.parametrize( + "solver, n_components_", + [ + ("full", min(iris.data.shape)), + ("arpack", min(iris.data.shape) - 1), + ("randomized", min(iris.data.shape)), + ], +) +@pytest.mark.parametrize("data", [iris.data, iris.data.T]) +def test_n_components_none(data, solver, n_components_): + pca = PCA(svd_solver=solver) + pca.fit(data) + assert pca.n_components_ == n_components_ + + +@pytest.mark.parametrize("svd_solver", ["auto", "full"]) +def test_n_components_mle(svd_solver): + # Ensure that n_components == 'mle' doesn't raise error for auto/full + rng = np.random.RandomState(0) + n_samples, n_features = 600, 10 + X = rng.randn(n_samples, n_features) + pca = PCA(n_components="mle", svd_solver=svd_solver) + pca.fit(X) + assert pca.n_components_ == 1 + + +@pytest.mark.parametrize("svd_solver", ["arpack", "randomized"]) +def test_n_components_mle_error(svd_solver): + # Ensure that n_components == 'mle' will raise an error for unsupported + # solvers + rng = np.random.RandomState(0) + n_samples, n_features = 600, 10 + X = rng.randn(n_samples, n_features) + pca = PCA(n_components="mle", svd_solver=svd_solver) + err_msg = "n_components='mle' cannot be a string with svd_solver='{}'".format( + svd_solver + ) + with pytest.raises(ValueError, match=err_msg): + pca.fit(X) + + +def test_pca_dim(): + # Check automated dimensionality setting + rng = np.random.RandomState(0) + n, p = 100, 5 + X = rng.randn(n, p) * 0.1 + X[:10] += np.array([3, 4, 5, 1, 2]) + pca = PCA(n_components="mle", svd_solver="full").fit(X) + assert pca.n_components == "mle" + assert pca.n_components_ == 1 + + +def test_infer_dim_1(): + # TODO: explain what this is testing + # Or at least use explicit variable names... + n, p = 1000, 5 + rng = np.random.RandomState(0) + X = ( + rng.randn(n, p) * 0.1 + + rng.randn(n, 1) * np.array([3, 4, 5, 1, 2]) + + np.array([1, 0, 7, 4, 6]) + ) + pca = PCA(n_components=p, svd_solver="full") + pca.fit(X) + spect = pca.explained_variance_ + ll = np.array([_assess_dimension(spect, k, n) for k in range(1, p)]) + assert ll[1] > ll.max() - 0.01 * n + + +def test_infer_dim_2(): + # TODO: explain what this is testing + # Or at least use explicit variable names... + n, p = 1000, 5 + rng = np.random.RandomState(0) + X = rng.randn(n, p) * 0.1 + X[:10] += np.array([3, 4, 5, 1, 2]) + X[10:20] += np.array([6, 0, 7, 2, -1]) + pca = PCA(n_components=p, svd_solver="full") + pca.fit(X) + spect = pca.explained_variance_ + assert _infer_dimension(spect, n) > 1 + + +def test_infer_dim_3(): + n, p = 100, 5 + rng = np.random.RandomState(0) + X = rng.randn(n, p) * 0.1 + X[:10] += np.array([3, 4, 5, 1, 2]) + X[10:20] += np.array([6, 0, 7, 2, -1]) + X[30:40] += 2 * np.array([-1, 1, -1, 1, -1]) + pca = PCA(n_components=p, svd_solver="full") + pca.fit(X) + spect = pca.explained_variance_ + assert _infer_dimension(spect, n) > 2 + + +@pytest.mark.parametrize( + "X, n_components, n_components_validated", + [ + (iris.data, 0.95, 2), # row > col + (iris.data, 0.01, 1), # row > col + (np.random.RandomState(0).rand(5, 20), 0.5, 2), + ], # row < col +) +def test_infer_dim_by_explained_variance(X, n_components, n_components_validated): + pca = PCA(n_components=n_components, svd_solver="full") + pca.fit(X) + assert pca.n_components == pytest.approx(n_components) + assert pca.n_components_ == n_components_validated + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_score(svd_solver): + # Test that probabilistic PCA scoring yields a reasonable score + n, p = 1000, 3 + rng = np.random.RandomState(0) + X = rng.randn(n, p) * 0.1 + np.array([3, 4, 5]) + pca = PCA(n_components=2, svd_solver=svd_solver) + pca.fit(X) + + ll1 = pca.score(X) + h = -0.5 * np.log(2 * np.pi * np.exp(1) * 0.1**2) * p + assert_allclose(ll1 / h, 1, rtol=5e-2) + + ll2 = pca.score(rng.randn(n, p) * 0.2 + np.array([3, 4, 5])) + assert ll1 > ll2 + + pca = PCA(n_components=2, whiten=True, svd_solver=svd_solver) + pca.fit(X) + ll2 = pca.score(X) + assert ll1 > ll2 + + +def test_pca_score3(): + # Check that probabilistic PCA selects the right model + n, p = 200, 3 + rng = np.random.RandomState(0) + Xl = rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) + np.array([1, 0, 7]) + Xt = rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) + np.array([1, 0, 7]) + ll = np.zeros(p) + for k in range(p): + pca = PCA(n_components=k, svd_solver="full") + pca.fit(Xl) + ll[k] = pca.score(Xt) + + assert ll.argmax() == 1 + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_sanity_noise_variance(svd_solver): + # Sanity check for the noise_variance_. For more details see + # https://github.com/scikit-learn/scikit-learn/issues/7568 + # https://github.com/scikit-learn/scikit-learn/issues/8541 + # https://github.com/scikit-learn/scikit-learn/issues/8544 + X, _ = datasets.load_digits(return_X_y=True) + pca = PCA(n_components=30, svd_solver=svd_solver, random_state=0) + pca.fit(X) + assert np.all((pca.explained_variance_ - pca.noise_variance_) >= 0) + + +@pytest.mark.parametrize("svd_solver", ["arpack", "randomized"]) +def test_pca_score_consistency_solvers(svd_solver): + # Check the consistency of score between solvers + X, _ = datasets.load_digits(return_X_y=True) + pca_full = PCA(n_components=30, svd_solver="full", random_state=0) + pca_other = PCA(n_components=30, svd_solver=svd_solver, random_state=0) + pca_full.fit(X) + pca_other.fit(X) + assert_allclose(pca_full.score(X), pca_other.score(X), rtol=5e-6) + + +# arpack raises ValueError for n_components == min(n_samples, n_features) +@pytest.mark.parametrize("svd_solver", ["full", "randomized"]) +def test_pca_zero_noise_variance_edge_cases(svd_solver): + # ensure that noise_variance_ is 0 in edge cases + # when n_components == min(n_samples, n_features) + n, p = 100, 3 + rng = np.random.RandomState(0) + X = rng.randn(n, p) * 0.1 + np.array([3, 4, 5]) + + pca = PCA(n_components=p, svd_solver=svd_solver) + pca.fit(X) + assert pca.noise_variance_ == 0 + # Non-regression test for gh-12489 + # ensure no divide-by-zero error for n_components == n_features < n_samples + pca.score(X) + + pca.fit(X.T) + assert pca.noise_variance_ == 0 + # Non-regression test for gh-12489 + # ensure no divide-by-zero error for n_components == n_samples < n_features + pca.score(X.T) + + +@pytest.mark.parametrize( + "n_samples, n_features, n_components, expected_solver", + [ + # case: n_samples < 10 * n_features and max(X.shape) <= 500 => 'full' + (10, 50, 5, "full"), + # case: n_samples > 10 * n_features and n_features < 500 => 'covariance_eigh' + (1000, 50, 50, "covariance_eigh"), + # case: n_components >= .8 * min(X.shape) => 'full' + (1000, 500, 400, "full"), + # n_components >= 1 and n_components < .8*min(X.shape) => 'randomized' + (1000, 500, 10, "randomized"), + # case: n_components in (0,1) => 'full' + (1000, 500, 0.5, "full"), + ], +) +def test_pca_svd_solver_auto(n_samples, n_features, n_components, expected_solver): + data = np.random.RandomState(0).uniform(size=(n_samples, n_features)) + pca_auto = PCA(n_components=n_components, random_state=0) + pca_test = PCA( + n_components=n_components, svd_solver=expected_solver, random_state=0 + ) + pca_auto.fit(data) + assert pca_auto._fit_svd_solver == expected_solver + pca_test.fit(data) + assert_allclose(pca_auto.components_, pca_test.components_) + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_deterministic_output(svd_solver): + rng = np.random.RandomState(0) + X = rng.rand(10, 10) + + transformed_X = np.zeros((20, 2)) + for i in range(20): + pca = PCA(n_components=2, svd_solver=svd_solver, random_state=rng) + transformed_X[i, :] = pca.fit_transform(X)[0] + assert_allclose(transformed_X, np.tile(transformed_X[0, :], 20).reshape(20, 2)) + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_dtype_preservation(svd_solver, global_random_seed): + check_pca_float_dtype_preservation(svd_solver, global_random_seed) + check_pca_int_dtype_upcast_to_double(svd_solver) + + +def check_pca_float_dtype_preservation(svd_solver, seed): + # Ensure that PCA does not upscale the dtype when input is float32 + X = np.random.RandomState(seed).rand(1000, 4) + X_float64 = X.astype(np.float64, copy=False) + X_float32 = X.astype(np.float32) + + pca_64 = PCA(n_components=3, svd_solver=svd_solver, random_state=seed).fit( + X_float64 + ) + pca_32 = PCA(n_components=3, svd_solver=svd_solver, random_state=seed).fit( + X_float32 + ) + + assert pca_64.components_.dtype == np.float64 + assert pca_32.components_.dtype == np.float32 + assert pca_64.transform(X_float64).dtype == np.float64 + assert pca_32.transform(X_float32).dtype == np.float32 + + # The atol and rtol are set such that the test passes for all random seeds + # on all supported platforms on our CI and conda-forge with the default + # random seed. + assert_allclose(pca_64.components_, pca_32.components_, rtol=1e-3, atol=1e-3) + + +def check_pca_int_dtype_upcast_to_double(svd_solver): + # Ensure that all int types will be upcast to float64 + X_i64 = np.random.RandomState(0).randint(0, 1000, (1000, 4)) + X_i64 = X_i64.astype(np.int64, copy=False) + X_i32 = X_i64.astype(np.int32, copy=False) + + pca_64 = PCA(n_components=3, svd_solver=svd_solver, random_state=0).fit(X_i64) + pca_32 = PCA(n_components=3, svd_solver=svd_solver, random_state=0).fit(X_i32) + + assert pca_64.components_.dtype == np.float64 + assert pca_32.components_.dtype == np.float64 + assert pca_64.transform(X_i64).dtype == np.float64 + assert pca_32.transform(X_i32).dtype == np.float64 + + assert_allclose(pca_64.components_, pca_32.components_, rtol=1e-4) + + +def test_pca_n_components_mostly_explained_variance_ratio(): + # when n_components is the second highest cumulative sum of the + # explained_variance_ratio_, then n_components_ should equal the + # number of features in the dataset #15669 + X, y = load_iris(return_X_y=True) + pca1 = PCA().fit(X, y) + + n_components = pca1.explained_variance_ratio_.cumsum()[-2] + pca2 = PCA(n_components=n_components).fit(X, y) + assert pca2.n_components_ == X.shape[1] + + +def test_assess_dimension_bad_rank(): + # Test error when tested rank not in [1, n_features - 1] + spectrum = np.array([1, 1e-30, 1e-30, 1e-30]) + n_samples = 10 + for rank in (0, 5): + with pytest.raises(ValueError, match=r"should be in \[1, n_features - 1\]"): + _assess_dimension(spectrum, rank, n_samples) + + +def test_small_eigenvalues_mle(): + # Test rank associated with tiny eigenvalues are given a log-likelihood of + # -inf. The inferred rank will be 1 + spectrum = np.array([1, 1e-30, 1e-30, 1e-30]) + + assert _assess_dimension(spectrum, rank=1, n_samples=10) > -np.inf + + for rank in (2, 3): + assert _assess_dimension(spectrum, rank, 10) == -np.inf + + assert _infer_dimension(spectrum, 10) == 1 + + +def test_mle_redundant_data(): + # Test 'mle' with pathological X: only one relevant feature should give a + # rank of 1 + X, _ = datasets.make_classification( + n_features=20, + n_informative=1, + n_repeated=18, + n_redundant=1, + n_clusters_per_class=1, + random_state=42, + ) + pca = PCA(n_components="mle").fit(X) + assert pca.n_components_ == 1 + + +def test_fit_mle_too_few_samples(): + # Tests that an error is raised when the number of samples is smaller + # than the number of features during an mle fit + X, _ = datasets.make_classification(n_samples=20, n_features=21, random_state=42) + + pca = PCA(n_components="mle", svd_solver="full") + with pytest.raises( + ValueError, + match="n_components='mle' is only supported if n_samples >= n_features", + ): + pca.fit(X) + + +def test_mle_simple_case(): + # non-regression test for issue + # https://github.com/scikit-learn/scikit-learn/issues/16730 + n_samples, n_dim = 1000, 10 + X = np.random.RandomState(0).randn(n_samples, n_dim) + X[:, -1] = np.mean(X[:, :-1], axis=-1) # true X dim is ndim - 1 + pca_skl = PCA("mle", svd_solver="full") + pca_skl.fit(X) + assert pca_skl.n_components_ == n_dim - 1 + + +def test_assess_dimesion_rank_one(): + # Make sure assess_dimension works properly on a matrix of rank 1 + n_samples, n_features = 9, 6 + X = np.ones((n_samples, n_features)) # rank 1 matrix + _, s, _ = np.linalg.svd(X, full_matrices=True) + # except for rank 1, all eigenvalues are 0 resp. close to 0 (FP) + assert_allclose(s[1:], np.zeros(n_features - 1), atol=1e-12) + + assert np.isfinite(_assess_dimension(s, rank=1, n_samples=n_samples)) + for rank in range(2, n_features): + assert _assess_dimension(s, rank, n_samples) == -np.inf + + +def test_pca_randomized_svd_n_oversamples(): + """Check that exposing and setting `n_oversamples` will provide accurate results + even when `X` as a large number of features. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/20589 + """ + rng = np.random.RandomState(0) + n_features = 100 + X = rng.randn(1_000, n_features) + + # The default value of `n_oversamples` will lead to inaccurate results + # We force it to the number of features. + pca_randomized = PCA( + n_components=1, + svd_solver="randomized", + n_oversamples=n_features, + random_state=0, + ).fit(X) + pca_full = PCA(n_components=1, svd_solver="full").fit(X) + pca_arpack = PCA(n_components=1, svd_solver="arpack", random_state=0).fit(X) + + assert_allclose(np.abs(pca_full.components_), np.abs(pca_arpack.components_)) + assert_allclose(np.abs(pca_randomized.components_), np.abs(pca_arpack.components_)) + + +def test_feature_names_out(): + """Check feature names out for PCA.""" + pca = PCA(n_components=2).fit(iris.data) + + names = pca.get_feature_names_out() + assert_array_equal([f"pca{i}" for i in range(2)], names) + + +@pytest.mark.parametrize("copy", [True, False]) +def test_variance_correctness(copy): + """Check the accuracy of PCA's internal variance calculation""" + rng = np.random.RandomState(0) + X = rng.randn(1000, 200) + pca = PCA().fit(X) + pca_var = pca.explained_variance_ / pca.explained_variance_ratio_ + true_var = np.var(X, ddof=1, axis=0).sum() + np.testing.assert_allclose(pca_var, true_var) + + +def check_array_api_get_precision(name, estimator, array_namespace, device, dtype_name): + xp = _array_api_for_tests(array_namespace, device) + iris_np = iris.data.astype(dtype_name) + iris_xp = xp.asarray(iris_np, device=device) + + estimator.fit(iris_np) + precision_np = estimator.get_precision() + covariance_np = estimator.get_covariance() + + rtol = 2e-4 if iris_np.dtype == "float32" else 2e-7 + with config_context(array_api_dispatch=True): + estimator_xp = clone(estimator).fit(iris_xp) + precision_xp = estimator_xp.get_precision() + assert precision_xp.shape == (4, 4) + assert precision_xp.dtype == iris_xp.dtype + + assert_allclose( + _convert_to_numpy(precision_xp, xp=xp), + precision_np, + rtol=rtol, + atol=_atol_for_type(dtype_name), + ) + covariance_xp = estimator_xp.get_covariance() + assert covariance_xp.shape == (4, 4) + assert covariance_xp.dtype == iris_xp.dtype + + assert_allclose( + _convert_to_numpy(covariance_xp, xp=xp), + covariance_np, + rtol=rtol, + atol=_atol_for_type(dtype_name), + ) + + +@pytest.mark.parametrize( + "array_namespace, device, dtype_name", + yield_namespace_device_dtype_combinations(), + ids=_get_namespace_device_dtype_ids, +) +@pytest.mark.parametrize( + "check", + [check_array_api_input_and_values, check_array_api_get_precision], + ids=_get_check_estimator_ids, +) +@pytest.mark.parametrize( + "estimator", + [ + PCA(n_components=2, svd_solver="full"), + PCA(n_components=2, svd_solver="full", whiten=True), + PCA(n_components=0.1, svd_solver="full", whiten=True), + PCA(n_components=2, svd_solver="covariance_eigh"), + PCA(n_components=2, svd_solver="covariance_eigh", whiten=True), + PCA( + n_components=2, + svd_solver="randomized", + power_iteration_normalizer="QR", + random_state=0, # how to use global_random_seed here? + ), + ], + ids=_get_check_estimator_ids, +) +def test_pca_array_api_compliance( + estimator, check, array_namespace, device, dtype_name +): + name = estimator.__class__.__name__ + estimator = clone(estimator) + check(name, estimator, array_namespace, device=device, dtype_name=dtype_name) + + +@pytest.mark.parametrize( + "array_namespace, device, dtype_name", + yield_namespace_device_dtype_combinations(), + ids=_get_namespace_device_dtype_ids, +) +@pytest.mark.parametrize( + "check", + [check_array_api_get_precision], + ids=_get_check_estimator_ids, +) +@pytest.mark.parametrize( + "estimator", + [ + # PCA with mle cannot use check_array_api_input_and_values because of + # rounding errors in the noisy (low variance) components. Even checking + # the shape of the `components_` is problematic because the number of + # components depends on trimming threshold of the mle algorithm which + # can depend on device-specific rounding errors. + PCA(n_components="mle", svd_solver="full"), + ], + ids=_get_check_estimator_ids, +) +def test_pca_mle_array_api_compliance( + estimator, check, array_namespace, device, dtype_name +): + name = estimator.__class__.__name__ + check(name, estimator, array_namespace, device=device, dtype_name=dtype_name) + + # Simpler variant of the generic check_array_api_input checker tailored for + # the specific case of PCA with mle-trimmed components. + xp = _array_api_for_tests(array_namespace, device) + + X, y = make_classification(random_state=42) + X = X.astype(dtype_name, copy=False) + atol = _atol_for_type(X.dtype) + + est = clone(estimator) + + X_xp = xp.asarray(X, device=device) + y_xp = xp.asarray(y, device=device) + + est.fit(X, y) + + components_np = est.components_ + explained_variance_np = est.explained_variance_ + + est_xp = clone(est) + with config_context(array_api_dispatch=True): + est_xp.fit(X_xp, y_xp) + components_xp = est_xp.components_ + assert array_device(components_xp) == array_device(X_xp) + components_xp_np = _convert_to_numpy(components_xp, xp=xp) + + explained_variance_xp = est_xp.explained_variance_ + assert array_device(explained_variance_xp) == array_device(X_xp) + explained_variance_xp_np = _convert_to_numpy(explained_variance_xp, xp=xp) + + assert components_xp_np.dtype == components_np.dtype + assert components_xp_np.shape[1] == components_np.shape[1] + assert explained_variance_xp_np.dtype == explained_variance_np.dtype + + # Check that the explained variance values match for the + # common components: + min_components = min(components_xp_np.shape[0], components_np.shape[0]) + assert_allclose( + explained_variance_xp_np[:min_components], + explained_variance_np[:min_components], + atol=atol, + ) + + # If the number of components differ, check that the explained variance of + # the trimmed components is very small. + if components_xp_np.shape[0] != components_np.shape[0]: + reference_variance = explained_variance_np[-1] + extra_variance_np = explained_variance_np[min_components:] + extra_variance_xp_np = explained_variance_xp_np[min_components:] + assert all(np.abs(extra_variance_np - reference_variance) < atol) + assert all(np.abs(extra_variance_xp_np - reference_variance) < atol) + + +@pytest.mark.skipif( + os.environ.get("SCIPY_ARRAY_API") != "1", reason="SCIPY_ARRAY_API not set to 1." +) +def test_array_api_error_and_warnings_on_unsupported_params(): + xp = pytest.importorskip("array_api_strict") + iris_xp = xp.asarray(iris.data) + + pca = PCA(n_components=2, svd_solver="arpack", random_state=0) + expected_msg = re.escape( + "PCA with svd_solver='arpack' is not supported for Array API inputs." + ) + with pytest.raises(ValueError, match=expected_msg): + with config_context(array_api_dispatch=True): + pca.fit(iris_xp) + + pca.set_params(svd_solver="randomized", power_iteration_normalizer="LU") + expected_msg = re.escape( + "Array API does not support LU factorization. Set" + " `power_iteration_normalizer='QR'` instead." + ) + with pytest.raises(ValueError, match=expected_msg): + with config_context(array_api_dispatch=True): + pca.fit(iris_xp) + + pca.set_params(svd_solver="randomized", power_iteration_normalizer="auto") + expected_msg = re.escape( + "Array API does not support LU factorization, falling back to QR instead. Set" + " `power_iteration_normalizer='QR'` explicitly to silence this warning." + ) + with pytest.warns(UserWarning, match=expected_msg): + with config_context(array_api_dispatch=True): + pca.fit(iris_xp) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_sparse_pca.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_sparse_pca.py new file mode 100644 index 0000000000000000000000000000000000000000..2a3e243f98bea8d94497d352d83e70be6a1265be --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_sparse_pca.py @@ -0,0 +1,350 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + + +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +from sklearn.datasets import make_low_rank_matrix +from sklearn.decomposition import PCA, MiniBatchSparsePCA, SparsePCA +from sklearn.utils import check_random_state +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + if_safe_multiprocessing_with_blas, +) +from sklearn.utils.extmath import svd_flip + + +def generate_toy_data(n_components, n_samples, image_size, random_state=None): + n_features = image_size[0] * image_size[1] + + rng = check_random_state(random_state) + U = rng.randn(n_samples, n_components) + V = rng.randn(n_components, n_features) + + centers = [(3, 3), (6, 7), (8, 1)] + sz = [1, 2, 1] + for k in range(n_components): + img = np.zeros(image_size) + xmin, xmax = centers[k][0] - sz[k], centers[k][0] + sz[k] + ymin, ymax = centers[k][1] - sz[k], centers[k][1] + sz[k] + img[xmin:xmax][:, ymin:ymax] = 1.0 + V[k, :] = img.ravel() + + # Y is defined by : Y = UV + noise + Y = np.dot(U, V) + Y += 0.1 * rng.randn(Y.shape[0], Y.shape[1]) # Add noise + return Y, U, V + + +# SparsePCA can be a bit slow. To avoid having test times go up, we +# test different aspects of the code in the same test + + +def test_correct_shapes(): + rng = np.random.RandomState(0) + X = rng.randn(12, 10) + spca = SparsePCA(n_components=8, random_state=rng) + U = spca.fit_transform(X) + assert spca.components_.shape == (8, 10) + assert U.shape == (12, 8) + # test overcomplete decomposition + spca = SparsePCA(n_components=13, random_state=rng) + U = spca.fit_transform(X) + assert spca.components_.shape == (13, 10) + assert U.shape == (12, 13) + + +def test_fit_transform(global_random_seed): + alpha = 1 + rng = np.random.RandomState(global_random_seed) + Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) # wide array + spca_lars = SparsePCA( + n_components=3, method="lars", alpha=alpha, random_state=global_random_seed + ) + spca_lars.fit(Y) + + # Test that CD gives similar results + spca_lasso = SparsePCA( + n_components=3, method="cd", random_state=global_random_seed, alpha=alpha + ) + spca_lasso.fit(Y) + assert_allclose(spca_lasso.components_, spca_lars.components_, rtol=5e-4, atol=2e-4) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +@if_safe_multiprocessing_with_blas +def test_fit_transform_parallel(global_random_seed): + alpha = 1 + rng = np.random.RandomState(global_random_seed) + Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) # wide array + spca_lars = SparsePCA( + n_components=3, method="lars", alpha=alpha, random_state=global_random_seed + ) + spca_lars.fit(Y) + U1 = spca_lars.transform(Y) + # Test multiple CPUs + spca = SparsePCA( + n_components=3, + n_jobs=2, + method="lars", + alpha=alpha, + random_state=global_random_seed, + ).fit(Y) + U2 = spca.transform(Y) + assert not np.all(spca_lars.components_ == 0) + assert_array_almost_equal(U1, U2) + + +def test_transform_nan(global_random_seed): + # Test that SparsePCA won't return NaN when there is 0 feature in all + # samples. + rng = np.random.RandomState(global_random_seed) + Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) # wide array + Y[:, 0] = 0 + estimator = SparsePCA(n_components=8, random_state=global_random_seed) + assert not np.any(np.isnan(estimator.fit_transform(Y))) + + +def test_fit_transform_tall(global_random_seed): + rng = np.random.RandomState(global_random_seed) + Y, _, _ = generate_toy_data(3, 65, (8, 8), random_state=rng) # tall array + spca_lars = SparsePCA(n_components=3, method="lars", random_state=rng) + U1 = spca_lars.fit_transform(Y) + spca_lasso = SparsePCA(n_components=3, method="cd", random_state=rng) + U2 = spca_lasso.fit(Y).transform(Y) + assert_allclose(U1, U2, rtol=1e-4, atol=2e-5) + + +def test_initialization(global_random_seed): + rng = np.random.RandomState(global_random_seed) + U_init = rng.randn(5, 3) + V_init = rng.randn(3, 4) + model = SparsePCA( + n_components=3, U_init=U_init, V_init=V_init, max_iter=0, random_state=rng + ) + model.fit(rng.randn(5, 4)) + + expected_components = V_init / np.linalg.norm(V_init, axis=1, keepdims=True) + expected_components = svd_flip(u=expected_components.T, v=None)[0].T + assert_allclose(model.components_, expected_components) + + +def test_mini_batch_correct_shapes(): + rng = np.random.RandomState(0) + X = rng.randn(12, 10) + pca = MiniBatchSparsePCA(n_components=8, max_iter=1, random_state=rng) + U = pca.fit_transform(X) + assert pca.components_.shape == (8, 10) + assert U.shape == (12, 8) + # test overcomplete decomposition + pca = MiniBatchSparsePCA(n_components=13, max_iter=1, random_state=rng) + U = pca.fit_transform(X) + assert pca.components_.shape == (13, 10) + assert U.shape == (12, 13) + + +def test_scaling_fit_transform(global_random_seed): + alpha = 1 + rng = np.random.RandomState(global_random_seed) + Y, _, _ = generate_toy_data(3, 1000, (8, 8), random_state=rng) + spca_lars = SparsePCA(n_components=3, method="lars", alpha=alpha, random_state=rng) + results_train = spca_lars.fit_transform(Y) + results_test = spca_lars.transform(Y[:10]) + assert_allclose(results_train[0], results_test[0]) + + +def test_pca_vs_spca(global_random_seed): + rng = np.random.RandomState(global_random_seed) + Y, _, _ = generate_toy_data(3, 1000, (8, 8), random_state=rng) + Z, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) + spca = SparsePCA(alpha=0, ridge_alpha=0, n_components=2, random_state=rng) + pca = PCA(n_components=2, random_state=rng) + pca.fit(Y) + spca.fit(Y) + results_test_pca = pca.transform(Z) + results_test_spca = spca.transform(Z) + assert_allclose( + np.abs(spca.components_.dot(pca.components_.T)), np.eye(2), atol=1e-4 + ) + results_test_pca *= np.sign(results_test_pca[0, :]) + results_test_spca *= np.sign(results_test_spca[0, :]) + assert_allclose(results_test_pca, results_test_spca, atol=1e-4) + + +@pytest.mark.parametrize("SPCA", [SparsePCA, MiniBatchSparsePCA]) +@pytest.mark.parametrize("n_components", [None, 3]) +def test_spca_n_components_(SPCA, n_components): + rng = np.random.RandomState(0) + n_samples, n_features = 12, 10 + X = rng.randn(n_samples, n_features) + + model = SPCA(n_components=n_components).fit(X) + + if n_components is not None: + assert model.n_components_ == n_components + else: + assert model.n_components_ == n_features + + +@pytest.mark.parametrize("SPCA", (SparsePCA, MiniBatchSparsePCA)) +@pytest.mark.parametrize("method", ("lars", "cd")) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_sparse_pca_dtype_match(SPCA, method, data_type, expected_type): + # Verify output matrix dtype + n_samples, n_features, n_components = 12, 10, 3 + rng = np.random.RandomState(0) + input_array = rng.randn(n_samples, n_features).astype(data_type) + model = SPCA(n_components=n_components, method=method) + transformed = model.fit_transform(input_array) + + assert transformed.dtype == expected_type + assert model.components_.dtype == expected_type + + +@pytest.mark.parametrize("SPCA", (SparsePCA, MiniBatchSparsePCA)) +@pytest.mark.parametrize("method", ("lars", "cd")) +def test_sparse_pca_numerical_consistency(SPCA, method, global_random_seed): + # Verify numericall consistentency among np.float32 and np.float64 + n_samples, n_features, n_components = 20, 20, 5 + input_array = make_low_rank_matrix( + n_samples=n_samples, + n_features=n_features, + effective_rank=n_components, + random_state=global_random_seed, + ) + + model_32 = SPCA( + n_components=n_components, + method=method, + random_state=global_random_seed, + ) + transformed_32 = model_32.fit_transform(input_array.astype(np.float32)) + + model_64 = SPCA( + n_components=n_components, + method=method, + random_state=global_random_seed, + ) + transformed_64 = model_64.fit_transform(input_array.astype(np.float64)) + assert_allclose(transformed_64, transformed_32) + assert_allclose(model_64.components_, model_32.components_) + + +@pytest.mark.parametrize("SPCA", [SparsePCA, MiniBatchSparsePCA]) +def test_spca_feature_names_out(SPCA): + """Check feature names out for *SparsePCA.""" + rng = np.random.RandomState(0) + n_samples, n_features = 12, 10 + X = rng.randn(n_samples, n_features) + + model = SPCA(n_components=4).fit(X) + names = model.get_feature_names_out() + + estimator_name = SPCA.__name__.lower() + assert_array_equal([f"{estimator_name}{i}" for i in range(4)], names) + + +def test_spca_early_stopping(global_random_seed): + """Check that `tol` and `max_no_improvement` act as early stopping.""" + rng = np.random.RandomState(global_random_seed) + n_samples, n_features = 50, 10 + X = rng.randn(n_samples, n_features) + + # vary the tolerance to force the early stopping of one of the model + model_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=0.5, random_state=global_random_seed + ).fit(X) + model_not_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=1e-3, random_state=global_random_seed + ).fit(X) + assert model_early_stopped.n_iter_ < model_not_early_stopped.n_iter_ + + # force the max number of no improvement to a large value to check that + # it does help to early stop + model_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=1e-6, max_no_improvement=2, random_state=global_random_seed + ).fit(X) + model_not_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=1e-6, max_no_improvement=100, random_state=global_random_seed + ).fit(X) + assert model_early_stopped.n_iter_ < model_not_early_stopped.n_iter_ + + +def test_equivalence_components_pca_spca(global_random_seed): + """Check the equivalence of the components found by PCA and SparsePCA. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/23932 + """ + rng = np.random.RandomState(global_random_seed) + X = rng.randn(50, 4) + + n_components = 2 + pca = PCA( + n_components=n_components, + svd_solver="randomized", + random_state=0, + ).fit(X) + spca = SparsePCA( + n_components=n_components, + method="lars", + ridge_alpha=0, + alpha=0, + random_state=0, + ).fit(X) + + assert_allclose(pca.components_, spca.components_) + + +def test_sparse_pca_inverse_transform(global_random_seed): + """Check that `inverse_transform` in `SparsePCA` and `PCA` are similar.""" + rng = np.random.RandomState(global_random_seed) + n_samples, n_features = 10, 5 + X = rng.randn(n_samples, n_features) + + n_components = 2 + spca = SparsePCA( + n_components=n_components, + alpha=1e-12, + ridge_alpha=1e-12, + random_state=global_random_seed, + ) + pca = PCA(n_components=n_components, random_state=global_random_seed) + X_trans_spca = spca.fit_transform(X) + X_trans_pca = pca.fit_transform(X) + assert_allclose( + spca.inverse_transform(X_trans_spca), pca.inverse_transform(X_trans_pca) + ) + + +@pytest.mark.parametrize("SPCA", [SparsePCA, MiniBatchSparsePCA]) +def test_transform_inverse_transform_round_trip(SPCA, global_random_seed): + """Check the `transform` and `inverse_transform` round trip with no loss of + information. + """ + rng = np.random.RandomState(global_random_seed) + n_samples, n_features = 10, 5 + X = rng.randn(n_samples, n_features) + + n_components = n_features + spca = SPCA( + n_components=n_components, + alpha=1e-12, + ridge_alpha=1e-12, + random_state=global_random_seed, + ) + X_trans_spca = spca.fit_transform(X) + assert_allclose(spca.inverse_transform(X_trans_spca), X) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_truncated_svd.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_truncated_svd.py new file mode 100644 index 0000000000000000000000000000000000000000..94d695d7fea0819dbdcce052f678709c72ea28bf --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/decomposition/tests/test_truncated_svd.py @@ -0,0 +1,212 @@ +"""Test truncated SVD transformer.""" + +import numpy as np +import pytest +import scipy.sparse as sp + +from sklearn.decomposition import PCA, TruncatedSVD +from sklearn.utils import check_random_state +from sklearn.utils._testing import assert_allclose, assert_array_less + +SVD_SOLVERS = ["arpack", "randomized"] + + +@pytest.fixture(scope="module") +def X_sparse(): + # Make an X that looks somewhat like a small tf-idf matrix. + rng = check_random_state(42) + X = sp.random(60, 55, density=0.2, format="csr", random_state=rng) + X.data[:] = 1 + np.log(X.data) + return X + + +@pytest.mark.parametrize("solver", ["randomized"]) +@pytest.mark.parametrize("kind", ("dense", "sparse")) +def test_solvers(X_sparse, solver, kind): + X = X_sparse if kind == "sparse" else X_sparse.toarray() + svd_a = TruncatedSVD(30, algorithm="arpack") + svd = TruncatedSVD(30, algorithm=solver, random_state=42, n_oversamples=100) + + Xa = svd_a.fit_transform(X)[:, :6] + Xr = svd.fit_transform(X)[:, :6] + assert_allclose(Xa, Xr, rtol=2e-3) + + comp_a = np.abs(svd_a.components_) + comp = np.abs(svd.components_) + # All elements are equal, but some elements are more equal than others. + assert_allclose(comp_a[:9], comp[:9], rtol=1e-3) + assert_allclose(comp_a[9:], comp[9:], atol=1e-2) + + +@pytest.mark.parametrize("n_components", (10, 25, 41, 55)) +def test_attributes(n_components, X_sparse): + n_features = X_sparse.shape[1] + tsvd = TruncatedSVD(n_components).fit(X_sparse) + assert tsvd.n_components == n_components + assert tsvd.components_.shape == (n_components, n_features) + + +@pytest.mark.parametrize( + "algorithm, n_components", + [ + ("arpack", 55), + ("arpack", 56), + ("randomized", 56), + ], +) +def test_too_many_components(X_sparse, algorithm, n_components): + tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm) + with pytest.raises(ValueError): + tsvd.fit(X_sparse) + + +@pytest.mark.parametrize("fmt", ("array", "csr", "csc", "coo", "lil")) +def test_sparse_formats(fmt, X_sparse): + n_samples = X_sparse.shape[0] + Xfmt = X_sparse.toarray() if fmt == "dense" else getattr(X_sparse, "to" + fmt)() + tsvd = TruncatedSVD(n_components=11) + Xtrans = tsvd.fit_transform(Xfmt) + assert Xtrans.shape == (n_samples, 11) + Xtrans = tsvd.transform(Xfmt) + assert Xtrans.shape == (n_samples, 11) + + +@pytest.mark.parametrize("algo", SVD_SOLVERS) +def test_inverse_transform(algo, X_sparse): + # We need a lot of components for the reconstruction to be "almost + # equal" in all positions. XXX Test means or sums instead? + tsvd = TruncatedSVD(n_components=52, random_state=42, algorithm=algo) + Xt = tsvd.fit_transform(X_sparse) + Xinv = tsvd.inverse_transform(Xt) + assert_allclose(Xinv, X_sparse.toarray(), rtol=1e-1, atol=2e-1) + + +def test_integers(X_sparse): + n_samples = X_sparse.shape[0] + Xint = X_sparse.astype(np.int64) + tsvd = TruncatedSVD(n_components=6) + Xtrans = tsvd.fit_transform(Xint) + assert Xtrans.shape == (n_samples, tsvd.n_components) + + +@pytest.mark.parametrize("kind", ("dense", "sparse")) +@pytest.mark.parametrize("n_components", [10, 20]) +@pytest.mark.parametrize("solver", SVD_SOLVERS) +def test_explained_variance(X_sparse, kind, n_components, solver): + X = X_sparse if kind == "sparse" else X_sparse.toarray() + svd = TruncatedSVD(n_components, algorithm=solver) + X_tr = svd.fit_transform(X) + # Assert that all the values are greater than 0 + assert_array_less(0.0, svd.explained_variance_ratio_) + + # Assert that total explained variance is less than 1 + assert_array_less(svd.explained_variance_ratio_.sum(), 1.0) + + # Test that explained_variance is correct + total_variance = np.var(X_sparse.toarray(), axis=0).sum() + variances = np.var(X_tr, axis=0) + true_explained_variance_ratio = variances / total_variance + + assert_allclose( + svd.explained_variance_ratio_, + true_explained_variance_ratio, + ) + + +@pytest.mark.parametrize("kind", ("dense", "sparse")) +@pytest.mark.parametrize("solver", SVD_SOLVERS) +def test_explained_variance_components_10_20(X_sparse, kind, solver): + X = X_sparse if kind == "sparse" else X_sparse.toarray() + svd_10 = TruncatedSVD(10, algorithm=solver, n_iter=10).fit(X) + svd_20 = TruncatedSVD(20, algorithm=solver, n_iter=10).fit(X) + + # Assert the 1st component is equal + assert_allclose( + svd_10.explained_variance_ratio_, + svd_20.explained_variance_ratio_[:10], + rtol=5e-3, + ) + + # Assert that 20 components has higher explained variance than 10 + assert ( + svd_20.explained_variance_ratio_.sum() > svd_10.explained_variance_ratio_.sum() + ) + + +@pytest.mark.parametrize("solver", SVD_SOLVERS) +def test_singular_values_consistency(solver, global_random_seed): + # Check that the TruncatedSVD output has the correct singular values + rng = np.random.RandomState(global_random_seed) + n_samples, n_features = 100, 80 + X = rng.randn(n_samples, n_features) + + pca = TruncatedSVD(n_components=2, algorithm=solver, random_state=rng).fit(X) + + # Compare to the Frobenius norm + X_pca = pca.transform(X) + assert_allclose( + np.sum(pca.singular_values_**2.0), + np.linalg.norm(X_pca, "fro") ** 2.0, + rtol=1e-2, + ) + + # Compare to the 2-norms of the score vectors + assert_allclose( + pca.singular_values_, np.sqrt(np.sum(X_pca**2.0, axis=0)), rtol=1e-2 + ) + + +@pytest.mark.parametrize("solver", SVD_SOLVERS) +def test_singular_values_expected(solver, global_random_seed): + # Set the singular values and see what we get back + rng = np.random.RandomState(global_random_seed) + n_samples = 100 + n_features = 110 + + X = rng.randn(n_samples, n_features) + + pca = TruncatedSVD(n_components=3, algorithm=solver, random_state=rng) + X_pca = pca.fit_transform(X) + + X_pca /= np.sqrt(np.sum(X_pca**2.0, axis=0)) + X_pca[:, 0] *= 3.142 + X_pca[:, 1] *= 2.718 + + X_hat_pca = np.dot(X_pca, pca.components_) + pca.fit(X_hat_pca) + assert_allclose(pca.singular_values_, [3.142, 2.718, 1.0], rtol=1e-14) + + +def test_truncated_svd_eq_pca(X_sparse): + # TruncatedSVD should be equal to PCA on centered data + + X_dense = X_sparse.toarray() + + X_c = X_dense - X_dense.mean(axis=0) + + params = dict(n_components=10, random_state=42) + + svd = TruncatedSVD(algorithm="arpack", **params) + pca = PCA(svd_solver="arpack", **params) + + Xt_svd = svd.fit_transform(X_c) + Xt_pca = pca.fit_transform(X_c) + + assert_allclose(Xt_svd, Xt_pca, rtol=1e-9) + assert_allclose(pca.mean_, 0, atol=1e-9) + assert_allclose(svd.components_, pca.components_) + + +@pytest.mark.parametrize( + "algorithm, tol", [("randomized", 0.0), ("arpack", 1e-6), ("arpack", 0.0)] +) +@pytest.mark.parametrize("kind", ("dense", "sparse")) +def test_fit_transform(X_sparse, algorithm, tol, kind): + # fit_transform(X) should equal fit(X).transform(X) + X = X_sparse if kind == "sparse" else X_sparse.toarray() + svd = TruncatedSVD( + n_components=5, n_iter=7, random_state=42, algorithm=algorithm, tol=tol + ) + X_transformed_1 = svd.fit_transform(X) + X_transformed_2 = svd.fit(X).transform(X) + assert_allclose(X_transformed_1, X_transformed_2) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/discriminant_analysis.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/discriminant_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..80963d2c1be923ac9255d1e4bebd19090771866e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/discriminant_analysis.py @@ -0,0 +1,1186 @@ +"""Linear and quadratic discriminant analysis.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings +from numbers import Integral, Real + +import numpy as np +import scipy.linalg +from scipy import linalg + +from sklearn.base import ( + BaseEstimator, + ClassifierMixin, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from sklearn.covariance import empirical_covariance, ledoit_wolf, shrunk_covariance +from sklearn.linear_model._base import LinearClassifierMixin +from sklearn.preprocessing import StandardScaler +from sklearn.utils._array_api import _expit, device, get_namespace, size +from sklearn.utils._param_validation import HasMethods, Interval, StrOptions +from sklearn.utils.extmath import softmax +from sklearn.utils.multiclass import check_classification_targets, unique_labels +from sklearn.utils.validation import check_is_fitted, validate_data + +__all__ = ["LinearDiscriminantAnalysis", "QuadraticDiscriminantAnalysis"] + + +def _cov(X, shrinkage=None, covariance_estimator=None): + """Estimate covariance matrix (using optional covariance_estimator). + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input data. + + shrinkage : {'empirical', 'auto'} or float, default=None + Shrinkage parameter, possible values: + - None or 'empirical': no shrinkage (default). + - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. + - float between 0 and 1: fixed shrinkage parameter. + + Shrinkage parameter is ignored if `covariance_estimator` + is not None. + + covariance_estimator : estimator, default=None + If not None, `covariance_estimator` is used to estimate + the covariance matrices instead of relying on the empirical + covariance estimator (with potential shrinkage). + The object should have a fit method and a ``covariance_`` attribute + like the estimators in :mod:`sklearn.covariance``. + If None the shrinkage parameter drives the estimate. + + .. versionadded:: 0.24 + + Returns + ------- + s : ndarray of shape (n_features, n_features) + Estimated covariance matrix. + """ + if covariance_estimator is None: + shrinkage = "empirical" if shrinkage is None else shrinkage + if isinstance(shrinkage, str): + if shrinkage == "auto": + sc = StandardScaler() # standardize features + X = sc.fit_transform(X) + s = ledoit_wolf(X)[0] + # rescale + s = sc.scale_[:, np.newaxis] * s * sc.scale_[np.newaxis, :] + elif shrinkage == "empirical": + s = empirical_covariance(X) + elif isinstance(shrinkage, Real): + s = shrunk_covariance(empirical_covariance(X), shrinkage) + else: + if shrinkage is not None and shrinkage != 0: + raise ValueError( + "covariance_estimator and shrinkage parameters " + "are not None. Only one of the two can be set." + ) + covariance_estimator.fit(X) + if not hasattr(covariance_estimator, "covariance_"): + raise ValueError( + "%s does not have a covariance_ attribute" + % covariance_estimator.__class__.__name__ + ) + s = covariance_estimator.covariance_ + return s + + +def _class_means(X, y): + """Compute class means. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input data. + + y : array-like of shape (n_samples,) or (n_samples, n_targets) + Target values. + + Returns + ------- + means : array-like of shape (n_classes, n_features) + Class means. + """ + xp, is_array_api_compliant = get_namespace(X) + classes, y = xp.unique_inverse(y) + means = xp.zeros((classes.shape[0], X.shape[1]), device=device(X), dtype=X.dtype) + + if is_array_api_compliant: + for i in range(classes.shape[0]): + means[i, :] = xp.mean(X[y == i], axis=0) + else: + # TODO: Explore the choice of using bincount + add.at as it seems sub optimal + # from a performance-wise + cnt = np.bincount(y) + np.add.at(means, y, X) + means /= cnt[:, None] + return means + + +def _class_cov(X, y, priors, shrinkage=None, covariance_estimator=None): + """Compute weighted within-class covariance matrix. + + The per-class covariance are weighted by the class priors. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input data. + + y : array-like of shape (n_samples,) or (n_samples, n_targets) + Target values. + + priors : array-like of shape (n_classes,) + Class priors. + + shrinkage : 'auto' or float, default=None + Shrinkage parameter, possible values: + - None: no shrinkage (default). + - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. + - float between 0 and 1: fixed shrinkage parameter. + + Shrinkage parameter is ignored if `covariance_estimator` is not None. + + covariance_estimator : estimator, default=None + If not None, `covariance_estimator` is used to estimate + the covariance matrices instead of relying the empirical + covariance estimator (with potential shrinkage). + The object should have a fit method and a ``covariance_`` attribute + like the estimators in sklearn.covariance. + If None, the shrinkage parameter drives the estimate. + + .. versionadded:: 0.24 + + Returns + ------- + cov : array-like of shape (n_features, n_features) + Weighted within-class covariance matrix + """ + classes = np.unique(y) + cov = np.zeros(shape=(X.shape[1], X.shape[1])) + for idx, group in enumerate(classes): + Xg = X[y == group, :] + cov += priors[idx] * np.atleast_2d(_cov(Xg, shrinkage, covariance_estimator)) + return cov + + +class DiscriminantAnalysisPredictionMixin: + """Mixin class for QuadraticDiscriminantAnalysis and NearestCentroid.""" + + def decision_function(self, X): + """Apply decision function to an array of samples. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Array of samples (test vectors). + + Returns + ------- + y_scores : ndarray of shape (n_samples,) or (n_samples, n_classes) + Decision function values related to each class, per sample. + In the two-class case, the shape is `(n_samples,)`, giving the + log likelihood ratio of the positive class. + """ + y_scores = self._decision_function(X) + if len(self.classes_) == 2: + return y_scores[:, 1] - y_scores[:, 0] + return y_scores + + def predict(self, X): + """Perform classification on an array of vectors `X`. + + Returns the class label for each sample. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + y_pred : ndarray of shape (n_samples,) + Class label for each sample. + """ + scores = self._decision_function(X) + return self.classes_.take(scores.argmax(axis=1)) + + def predict_proba(self, X): + """Estimate class probabilities. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input data. + + Returns + ------- + y_proba : ndarray of shape (n_samples, n_classes) + Probability estimate of the sample for each class in the + model, where classes are ordered as they are in `self.classes_`. + """ + return np.exp(self.predict_log_proba(X)) + + def predict_log_proba(self, X): + """Estimate log class probabilities. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input data. + + Returns + ------- + y_log_proba : ndarray of shape (n_samples, n_classes) + Estimated log probabilities. + """ + scores = self._decision_function(X) + log_likelihood = scores - scores.max(axis=1)[:, np.newaxis] + return log_likelihood - np.log( + np.exp(log_likelihood).sum(axis=1)[:, np.newaxis] + ) + + +class LinearDiscriminantAnalysis( + ClassNamePrefixFeaturesOutMixin, + LinearClassifierMixin, + TransformerMixin, + BaseEstimator, +): + """Linear Discriminant Analysis. + + A classifier with a linear decision boundary, generated by fitting class + conditional densities to the data and using Bayes' rule. + + The model fits a Gaussian density to each class, assuming that all classes + share the same covariance matrix. + + The fitted model can also be used to reduce the dimensionality of the input + by projecting it to the most discriminative directions, using the + `transform` method. + + .. versionadded:: 0.17 + + For a comparison between + :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis` + and :class:`~sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis`, see + :ref:`sphx_glr_auto_examples_classification_plot_lda_qda.py`. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + solver : {'svd', 'lsqr', 'eigen'}, default='svd' + Solver to use, possible values: + - 'svd': Singular value decomposition (default). + Does not compute the covariance matrix, therefore this solver is + recommended for data with a large number of features. + - 'lsqr': Least squares solution. + Can be combined with shrinkage or custom covariance estimator. + - 'eigen': Eigenvalue decomposition. + Can be combined with shrinkage or custom covariance estimator. + + .. versionchanged:: 1.2 + `solver="svd"` now has experimental Array API support. See the + :ref:`Array API User Guide ` for more details. + + shrinkage : 'auto' or float, default=None + Shrinkage parameter, possible values: + - None: no shrinkage (default). + - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. + - float between 0 and 1: fixed shrinkage parameter. + + This should be left to None if `covariance_estimator` is used. + Note that shrinkage works only with 'lsqr' and 'eigen' solvers. + + For a usage example, see + :ref:`sphx_glr_auto_examples_classification_plot_lda.py`. + + priors : array-like of shape (n_classes,), default=None + The class prior probabilities. By default, the class proportions are + inferred from the training data. + + n_components : int, default=None + Number of components (<= min(n_classes - 1, n_features)) for + dimensionality reduction. If None, will be set to + min(n_classes - 1, n_features). This parameter only affects the + `transform` method. + + For a usage example, see + :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_lda.py`. + + store_covariance : bool, default=False + If True, explicitly compute the weighted within-class covariance + matrix when solver is 'svd'. The matrix is always computed + and stored for the other solvers. + + .. versionadded:: 0.17 + + tol : float, default=1.0e-4 + Absolute threshold for a singular value of X to be considered + significant, used to estimate the rank of X. Dimensions whose + singular values are non-significant are discarded. Only used if + solver is 'svd'. + + .. versionadded:: 0.17 + + covariance_estimator : covariance estimator, default=None + If not None, `covariance_estimator` is used to estimate + the covariance matrices instead of relying on the empirical + covariance estimator (with potential shrinkage). + The object should have a fit method and a ``covariance_`` attribute + like the estimators in :mod:`sklearn.covariance`. + if None the shrinkage parameter drives the estimate. + + This should be left to None if `shrinkage` is used. + Note that `covariance_estimator` works only with 'lsqr' and 'eigen' + solvers. + + .. versionadded:: 0.24 + + Attributes + ---------- + coef_ : ndarray of shape (n_features,) or (n_classes, n_features) + Weight vector(s). + + intercept_ : ndarray of shape (n_classes,) + Intercept term. + + covariance_ : array-like of shape (n_features, n_features) + Weighted within-class covariance matrix. It corresponds to + `sum_k prior_k * C_k` where `C_k` is the covariance matrix of the + samples in class `k`. The `C_k` are estimated using the (potentially + shrunk) biased estimator of covariance. If solver is 'svd', only + exists when `store_covariance` is True. + + explained_variance_ratio_ : ndarray of shape (n_components,) + Percentage of variance explained by each of the selected components. + If ``n_components`` is not set then all components are stored and the + sum of explained variances is equal to 1.0. Only available when eigen + or svd solver is used. + + means_ : array-like of shape (n_classes, n_features) + Class-wise means. + + priors_ : array-like of shape (n_classes,) + Class priors (sum to 1). + + scalings_ : array-like of shape (rank, n_classes - 1) + Scaling of the features in the space spanned by the class centroids. + Only available for 'svd' and 'eigen' solvers. + + xbar_ : array-like of shape (n_features,) + Overall mean. Only present if solver is 'svd'. + + classes_ : array-like of shape (n_classes,) + Unique class labels. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + QuadraticDiscriminantAnalysis : Quadratic Discriminant Analysis. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.discriminant_analysis import LinearDiscriminantAnalysis + >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) + >>> y = np.array([1, 1, 1, 2, 2, 2]) + >>> clf = LinearDiscriminantAnalysis() + >>> clf.fit(X, y) + LinearDiscriminantAnalysis() + >>> print(clf.predict([[-0.8, -1]])) + [1] + """ + + _parameter_constraints: dict = { + "solver": [StrOptions({"svd", "lsqr", "eigen"})], + "shrinkage": [StrOptions({"auto"}), Interval(Real, 0, 1, closed="both"), None], + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "priors": ["array-like", None], + "store_covariance": ["boolean"], + "tol": [Interval(Real, 0, None, closed="left")], + "covariance_estimator": [HasMethods("fit"), None], + } + + def __init__( + self, + solver="svd", + shrinkage=None, + priors=None, + n_components=None, + store_covariance=False, + tol=1e-4, + covariance_estimator=None, + ): + self.solver = solver + self.shrinkage = shrinkage + self.priors = priors + self.n_components = n_components + self.store_covariance = store_covariance # used only in svd solver + self.tol = tol # used only in svd solver + self.covariance_estimator = covariance_estimator + + def _solve_lstsq(self, X, y, shrinkage, covariance_estimator): + """Least squares solver. + + The least squares solver computes a straightforward solution of the + optimal decision rule based directly on the discriminant functions. It + can only be used for classification (with any covariance estimator), + because + estimation of eigenvectors is not performed. Therefore, dimensionality + reduction with the transform is not supported. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + y : array-like of shape (n_samples,) or (n_samples, n_classes) + Target values. + + shrinkage : 'auto', float or None + Shrinkage parameter, possible values: + - None: no shrinkage. + - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. + - float between 0 and 1: fixed shrinkage parameter. + + Shrinkage parameter is ignored if `covariance_estimator` is + not None + + covariance_estimator : estimator, default=None + If not None, `covariance_estimator` is used to estimate + the covariance matrices instead of relying the empirical + covariance estimator (with potential shrinkage). + The object should have a fit method and a ``covariance_`` attribute + like the estimators in sklearn.covariance. + if None the shrinkage parameter drives the estimate. + + .. versionadded:: 0.24 + + Notes + ----- + This solver is based on [1]_, section 2.6.2, pp. 39-41. + + References + ---------- + .. [1] R. O. Duda, P. E. Hart, D. G. Stork. Pattern Classification + (Second Edition). John Wiley & Sons, Inc., New York, 2001. ISBN + 0-471-05669-3. + """ + self.means_ = _class_means(X, y) + self.covariance_ = _class_cov( + X, y, self.priors_, shrinkage, covariance_estimator + ) + self.coef_ = linalg.lstsq(self.covariance_, self.means_.T)[0].T + self.intercept_ = -0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + np.log( + self.priors_ + ) + + def _solve_eigen(self, X, y, shrinkage, covariance_estimator): + """Eigenvalue solver. + + The eigenvalue solver computes the optimal solution of the Rayleigh + coefficient (basically the ratio of between class scatter to within + class scatter). This solver supports both classification and + dimensionality reduction (with any covariance estimator). + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + y : array-like of shape (n_samples,) or (n_samples, n_targets) + Target values. + + shrinkage : 'auto', float or None + Shrinkage parameter, possible values: + - None: no shrinkage. + - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. + - float between 0 and 1: fixed shrinkage constant. + + Shrinkage parameter is ignored if `covariance_estimator` is + not None + + covariance_estimator : estimator, default=None + If not None, `covariance_estimator` is used to estimate + the covariance matrices instead of relying the empirical + covariance estimator (with potential shrinkage). + The object should have a fit method and a ``covariance_`` attribute + like the estimators in sklearn.covariance. + if None the shrinkage parameter drives the estimate. + + .. versionadded:: 0.24 + + Notes + ----- + This solver is based on [1]_, section 3.8.3, pp. 121-124. + + References + ---------- + .. [1] R. O. Duda, P. E. Hart, D. G. Stork. Pattern Classification + (Second Edition). John Wiley & Sons, Inc., New York, 2001. ISBN + 0-471-05669-3. + """ + self.means_ = _class_means(X, y) + self.covariance_ = _class_cov( + X, y, self.priors_, shrinkage, covariance_estimator + ) + + Sw = self.covariance_ # within scatter + St = _cov(X, shrinkage, covariance_estimator) # total scatter + Sb = St - Sw # between scatter + + evals, evecs = linalg.eigh(Sb, Sw) + self.explained_variance_ratio_ = np.sort(evals / np.sum(evals))[::-1][ + : self._max_components + ] + evecs = evecs[:, np.argsort(evals)[::-1]] # sort eigenvectors + + self.scalings_ = evecs + self.coef_ = np.dot(self.means_, evecs).dot(evecs.T) + self.intercept_ = -0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + np.log( + self.priors_ + ) + + def _solve_svd(self, X, y): + """SVD solver. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + y : array-like of shape (n_samples,) or (n_samples, n_targets) + Target values. + """ + xp, is_array_api_compliant = get_namespace(X) + + if is_array_api_compliant: + svd = xp.linalg.svd + else: + svd = scipy.linalg.svd + + n_samples, _ = X.shape + n_classes = self.classes_.shape[0] + + self.means_ = _class_means(X, y) + if self.store_covariance: + self.covariance_ = _class_cov(X, y, self.priors_) + + Xc = [] + for idx, group in enumerate(self.classes_): + Xg = X[y == group] + Xc.append(Xg - self.means_[idx, :]) + + self.xbar_ = self.priors_ @ self.means_ + + Xc = xp.concat(Xc, axis=0) + + # 1) within (univariate) scaling by with classes std-dev + std = xp.std(Xc, axis=0) + # avoid division by zero in normalization + std[std == 0] = 1.0 + fac = xp.asarray(1.0 / (n_samples - n_classes), dtype=X.dtype, device=device(X)) + + # 2) Within variance scaling + X = xp.sqrt(fac) * (Xc / std) + # SVD of centered (within)scaled data + _, S, Vt = svd(X, full_matrices=False) + + rank = xp.sum(xp.astype(S > self.tol, xp.int32)) + # Scaling of within covariance is: V' 1/S + scalings = (Vt[:rank, :] / std).T / S[:rank] + fac = 1.0 if n_classes == 1 else 1.0 / (n_classes - 1) + + # 3) Between variance scaling + # Scale weighted centers + X = ( + (xp.sqrt((n_samples * self.priors_) * fac)) * (self.means_ - self.xbar_).T + ).T @ scalings + # Centers are living in a space with n_classes-1 dim (maximum) + # Use SVD to find projection in the space spanned by the + # (n_classes) centers + _, S, Vt = svd(X, full_matrices=False) + + if self._max_components == 0: + self.explained_variance_ratio_ = xp.empty((0,), dtype=S.dtype) + else: + self.explained_variance_ratio_ = (S**2 / xp.sum(S**2))[ + : self._max_components + ] + + rank = xp.sum(xp.astype(S > self.tol * S[0], xp.int32)) + self.scalings_ = scalings @ Vt.T[:, :rank] + coef = (self.means_ - self.xbar_) @ self.scalings_ + self.intercept_ = -0.5 * xp.sum(coef**2, axis=1) + xp.log(self.priors_) + self.coef_ = coef @ self.scalings_.T + self.intercept_ -= self.xbar_ @ self.coef_.T + + @_fit_context( + # LinearDiscriminantAnalysis.covariance_estimator is not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y): + """Fit the Linear Discriminant Analysis model. + + .. versionchanged:: 0.19 + `store_covariance` and `tol` has been moved to main constructor. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + y : array-like of shape (n_samples,) + Target values. + + Returns + ------- + self : object + Fitted estimator. + """ + xp, _ = get_namespace(X) + + X, y = validate_data( + self, X, y, ensure_min_samples=2, dtype=[xp.float64, xp.float32] + ) + self.classes_ = unique_labels(y) + n_samples, n_features = X.shape + n_classes = self.classes_.shape[0] + + if n_samples == n_classes: + raise ValueError( + "The number of samples must be more than the number of classes." + ) + + if self.priors is None: # estimate priors from sample + _, cnts = xp.unique_counts(y) # non-negative ints + self.priors_ = xp.astype(cnts, X.dtype) / float(n_samples) + else: + self.priors_ = xp.asarray(self.priors, dtype=X.dtype) + + if xp.any(self.priors_ < 0): + raise ValueError("priors must be non-negative") + + if xp.abs(xp.sum(self.priors_) - 1.0) > 1e-5: + warnings.warn("The priors do not sum to 1. Renormalizing", UserWarning) + self.priors_ = self.priors_ / self.priors_.sum() + + # Maximum number of components no matter what n_components is + # specified: + max_components = min(n_classes - 1, n_features) + + if self.n_components is None: + self._max_components = max_components + else: + if self.n_components > max_components: + raise ValueError( + "n_components cannot be larger than min(n_features, n_classes - 1)." + ) + self._max_components = self.n_components + + if self.solver == "svd": + if self.shrinkage is not None: + raise NotImplementedError("shrinkage not supported with 'svd' solver.") + if self.covariance_estimator is not None: + raise ValueError( + "covariance estimator " + "is not supported " + "with svd solver. Try another solver" + ) + self._solve_svd(X, y) + elif self.solver == "lsqr": + self._solve_lstsq( + X, + y, + shrinkage=self.shrinkage, + covariance_estimator=self.covariance_estimator, + ) + elif self.solver == "eigen": + self._solve_eigen( + X, + y, + shrinkage=self.shrinkage, + covariance_estimator=self.covariance_estimator, + ) + if size(self.classes_) == 2: # treat binary case as a special case + coef_ = xp.asarray(self.coef_[1, :] - self.coef_[0, :], dtype=X.dtype) + self.coef_ = xp.reshape(coef_, (1, -1)) + intercept_ = xp.asarray( + self.intercept_[1] - self.intercept_[0], dtype=X.dtype + ) + self.intercept_ = xp.reshape(intercept_, (1,)) + self._n_features_out = self._max_components + return self + + def transform(self, X): + """Project data to maximize class separation. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input data. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) or \ + (n_samples, min(rank, n_components)) + Transformed data. In the case of the 'svd' solver, the shape + is (n_samples, min(rank, n_components)). + """ + if self.solver == "lsqr": + raise NotImplementedError( + "transform not implemented for 'lsqr' solver (use 'svd' or 'eigen')." + ) + check_is_fitted(self) + X = validate_data(self, X, reset=False) + + if self.solver == "svd": + X_new = (X - self.xbar_) @ self.scalings_ + elif self.solver == "eigen": + X_new = X @ self.scalings_ + + return X_new[:, : self._max_components] + + def predict_proba(self, X): + """Estimate probability. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input data. + + Returns + ------- + C : ndarray of shape (n_samples, n_classes) + Estimated probabilities. + """ + check_is_fitted(self) + xp, _ = get_namespace(X) + decision = self.decision_function(X) + if size(self.classes_) == 2: + proba = _expit(decision, xp) + return xp.stack([1 - proba, proba], axis=1) + else: + return softmax(decision) + + def predict_log_proba(self, X): + """Estimate log probability. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input data. + + Returns + ------- + C : ndarray of shape (n_samples, n_classes) + Estimated log probabilities. + """ + xp, _ = get_namespace(X) + prediction = self.predict_proba(X) + + smallest_normal = xp.finfo(prediction.dtype).smallest_normal + prediction[prediction == 0.0] += smallest_normal + return xp.log(prediction) + + def decision_function(self, X): + """Apply decision function to an array of samples. + + The decision function is equal (up to a constant factor) to the + log-posterior of the model, i.e. `log p(y = k | x)`. In a binary + classification setting this instead corresponds to the difference + `log p(y = 1 | x) - log p(y = 0 | x)`. See :ref:`lda_qda_math`. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Array of samples (test vectors). + + Returns + ------- + y_scores : ndarray of shape (n_samples,) or (n_samples, n_classes) + Decision function values related to each class, per sample. + In the two-class case, the shape is `(n_samples,)`, giving the + log likelihood ratio of the positive class. + """ + # Only overrides for the docstring. + return super().decision_function(X) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.array_api_support = True + return tags + + +class QuadraticDiscriminantAnalysis( + DiscriminantAnalysisPredictionMixin, ClassifierMixin, BaseEstimator +): + """Quadratic Discriminant Analysis. + + A classifier with a quadratic decision boundary, generated + by fitting class conditional densities to the data + and using Bayes' rule. + + The model fits a Gaussian density to each class. + + .. versionadded:: 0.17 + + For a comparison between + :class:`~sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis` + and :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis`, see + :ref:`sphx_glr_auto_examples_classification_plot_lda_qda.py`. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + solver : {'svd', 'eigen'}, default='svd' + Solver to use, possible values: + - 'svd': Singular value decomposition (default). + Does not compute the covariance matrix, therefore this solver is + recommended for data with a large number of features. + - 'eigen': Eigenvalue decomposition. + Can be combined with shrinkage or custom covariance estimator. + + shrinkage : 'auto' or float, default=None + Shrinkage parameter, possible values: + - None: no shrinkage (default). + - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. + - float between 0 and 1: fixed shrinkage parameter. + + Enabling shrinkage is expected to improve the model when some + classes have a relatively small number of training data points + compared to the number of features by mitigating overfitting during + the covariance estimation step. + + This should be left to `None` if `covariance_estimator` is used. + Note that shrinkage works only with 'eigen' solver. + + priors : array-like of shape (n_classes,), default=None + Class priors. By default, the class proportions are inferred from the + training data. + + reg_param : float, default=0.0 + Regularizes the per-class covariance estimates by transforming S2 as + ``S2 = (1 - reg_param) * S2 + reg_param * np.eye(n_features)``, + where S2 corresponds to the `scaling_` attribute of a given class. + + store_covariance : bool, default=False + If True, the class covariance matrices are explicitly computed and + stored in the `self.covariance_` attribute. + + .. versionadded:: 0.17 + + tol : float, default=1.0e-4 + Absolute threshold for the covariance matrix to be considered rank + deficient after applying some regularization (see `reg_param`) to each + `Sk` where `Sk` represents covariance matrix for k-th class. This + parameter does not affect the predictions. It controls when a warning + is raised if the covariance matrix is not full rank. + + .. versionadded:: 0.17 + + covariance_estimator : covariance estimator, default=None + If not None, `covariance_estimator` is used to estimate the covariance + matrices instead of relying on the empirical covariance estimator + (with potential shrinkage). The object should have a fit method and + a ``covariance_`` attribute like the estimators in + :mod:`sklearn.covariance`. If None the shrinkage parameter drives the + estimate. + + This should be left to `None` if `shrinkage` is used. + Note that `covariance_estimator` works only with the 'eigen' solver. + + Attributes + ---------- + covariance_ : list of len n_classes of ndarray \ + of shape (n_features, n_features) + For each class, gives the covariance matrix estimated using the + samples of that class. The estimations are unbiased. Only present if + `store_covariance` is True. + + means_ : array-like of shape (n_classes, n_features) + Class-wise means. + + priors_ : array-like of shape (n_classes,) + Class priors (sum to 1). + + rotations_ : list of len n_classes of ndarray of shape (n_features, n_k) + For each class k an array of shape (n_features, n_k), where + ``n_k = min(n_features, number of elements in class k)`` + It is the rotation of the Gaussian distribution, i.e. its + principal axis. It corresponds to `V`, the matrix of eigenvectors + coming from the SVD of `Xk = U S Vt` where `Xk` is the centered + matrix of samples from class k. + + scalings_ : list of len n_classes of ndarray of shape (n_k,) + For each class, contains the scaling of + the Gaussian distributions along its principal axes, i.e. the + variance in the rotated coordinate system. It corresponds to `S^2 / + (n_samples - 1)`, where `S` is the diagonal matrix of singular values + from the SVD of `Xk`, where `Xk` is the centered matrix of samples + from class k. + + classes_ : ndarray of shape (n_classes,) + Unique class labels. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + LinearDiscriminantAnalysis : Linear Discriminant Analysis. + + Examples + -------- + >>> from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis + >>> import numpy as np + >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) + >>> y = np.array([1, 1, 1, 2, 2, 2]) + >>> clf = QuadraticDiscriminantAnalysis() + >>> clf.fit(X, y) + QuadraticDiscriminantAnalysis() + >>> print(clf.predict([[-0.8, -1]])) + [1] + """ + + _parameter_constraints: dict = { + "solver": [StrOptions({"svd", "eigen"})], + "shrinkage": [StrOptions({"auto"}), Interval(Real, 0, 1, closed="both"), None], + "priors": ["array-like", None], + "reg_param": [Interval(Real, 0, 1, closed="both")], + "store_covariance": ["boolean"], + "tol": [Interval(Real, 0, None, closed="left")], + "covariance_estimator": [HasMethods("fit"), None], + } + + def __init__( + self, + *, + solver="svd", + shrinkage=None, + priors=None, + reg_param=0.0, + store_covariance=False, + tol=1.0e-4, + covariance_estimator=None, + ): + self.solver = solver + self.shrinkage = shrinkage + self.priors = priors + self.reg_param = reg_param + self.store_covariance = store_covariance + self.tol = tol + self.covariance_estimator = covariance_estimator + + def _solve_eigen(self, X): + """Eigenvalue solver. + + The eigenvalue solver uses the eigen decomposition of the data + to compute the rotation and scaling matrices used for scoring + new samples. This solver supports use of any covariance estimator. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + """ + n_samples, n_features = X.shape + + cov = _cov(X, self.shrinkage, self.covariance_estimator) + scaling, rotation = linalg.eigh(cov) # scalings are eigenvalues + rotation = rotation[:, np.argsort(scaling)[::-1]] # sort eigenvectors + scaling = scaling[np.argsort(scaling)[::-1]] # sort eigenvalues + return scaling, rotation, cov + + def _solve_svd(self, X): + """SVD solver for Quadratic Discriminant Analysis. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + """ + n_samples, n_features = X.shape + + mean = X.mean(0) + Xc = X - mean + # Xc = U * S * V.T + _, S, Vt = np.linalg.svd(Xc, full_matrices=False) + scaling = (S**2) / (n_samples - 1) # scalings are squared singular values + scaling = ((1 - self.reg_param) * scaling) + self.reg_param + rotation = Vt.T + + cov = None + if self.store_covariance: + # cov = V * (S^2 / (n-1)) * V.T + cov = scaling * Vt.T @ Vt + + return scaling, rotation, cov + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y): + """Fit the model according to the given training data and parameters. + + .. versionchanged:: 0.19 + ``store_covariances`` has been moved to main constructor as + ``store_covariance``. + + .. versionchanged:: 0.19 + ``tol`` has been moved to main constructor. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values (integers). + + Returns + ------- + self : object + Fitted estimator. + """ + X, y = validate_data(self, X, y) + check_classification_targets(y) + self.classes_ = np.unique(y) + n_samples, n_features = X.shape + n_classes = len(self.classes_) + if n_classes < 2: + raise ValueError( + "The number of classes has to be greater than one. Got " + f"{n_classes} class." + ) + if self.priors is None: + _, cnts = np.unique(y, return_counts=True) + self.priors_ = cnts / float(n_samples) + else: + self.priors_ = np.array(self.priors) + + if self.solver == "svd": + if self.shrinkage is not None: + # Support for `shrinkage` could be implemented as in + # https://github.com/scikit-learn/scikit-learn/issues/32590 + raise NotImplementedError("shrinkage not supported with 'svd' solver.") + if self.covariance_estimator is not None: + raise ValueError( + "covariance_estimator is not supported with solver='svd'. " + "Try solver='eigen' instead." + ) + specific_solver = self._solve_svd + elif self.solver == "eigen": + specific_solver = self._solve_eigen + + means = [] + cov = [] + scalings = [] + rotations = [] + for class_idx, class_label in enumerate(self.classes_): + X_class = X[y == class_label, :] + if len(X_class) == 1: + raise ValueError( + "y has only 1 sample in class %s, covariance is ill defined." + % str(self.classes_[class_idx]) + ) + + mean_class = X_class.mean(0) + means.append(mean_class) + + scaling_class, rotation_class, cov_class = specific_solver(X_class) + + rank = np.sum(scaling_class > self.tol) + if rank < n_features: + n_samples_class = X_class.shape[0] + if self.solver == "svd" and n_samples_class <= n_features: + raise linalg.LinAlgError( + f"The covariance matrix of class {class_label} is not full " + f"rank. When using `solver='svd'` the number of samples in " + f"each class should be more than the number of features, but " + f"class {class_label} has {n_samples_class} samples and " + f"{n_features} features. Try using `solver='eigen'` and " + f"setting the parameter `shrinkage` for regularization." + ) + else: + msg_param = "shrinkage" if self.solver == "eigen" else "reg_param" + raise linalg.LinAlgError( + f"The covariance matrix of class {class_label} is not full " + f"rank. Increase the value of `{msg_param}` to reduce the " + f"collinearity.", + ) + + cov.append(cov_class) + scalings.append(scaling_class) + rotations.append(rotation_class) + + if self.store_covariance: + self.covariance_ = cov + self.means_ = np.asarray(means) + self.scalings_ = scalings + self.rotations_ = rotations + return self + + def _decision_function(self, X): + # return log posterior, see eq (4.12) p. 110 of the ESL. + check_is_fitted(self) + + X = validate_data(self, X, reset=False) + norm2 = [] + for i in range(len(self.classes_)): + R = self.rotations_[i] + S = self.scalings_[i] + Xm = X - self.means_[i] + X2 = np.dot(Xm, R * (S ** (-0.5))) + norm2.append(np.sum(X2**2, axis=1)) + norm2 = np.array(norm2).T # shape = [len(X), n_classes] + u = np.asarray([np.sum(np.log(s)) for s in self.scalings_]) + return -0.5 * (norm2 + u) + np.log(self.priors_) + + def decision_function(self, X): + """Apply decision function to an array of samples. + + The decision function is equal (up to a constant factor) to the + log-posterior of the model, i.e. `log p(y = k | x)`. In a binary + classification setting this instead corresponds to the difference + `log p(y = 1 | x) - log p(y = 0 | x)`. See :ref:`lda_qda_math`. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Array of samples (test vectors). + + Returns + ------- + C : ndarray of shape (n_samples,) or (n_samples, n_classes) + Decision function values related to each class, per sample. + In the two-class case, the shape is `(n_samples,)`, giving the + log likelihood ratio of the positive class. + """ + # Only overrides for the docstring. + return super().decision_function(X) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/dummy.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/dummy.py new file mode 100644 index 0000000000000000000000000000000000000000..46e01ae74ef2ffd203295ebbedab68cc36faae26 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/dummy.py @@ -0,0 +1,700 @@ +"""Dummy estimators that implement simple rules of thumb.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings +from numbers import Integral, Real + +import numpy as np +import scipy.sparse as sp + +from sklearn.base import ( + BaseEstimator, + ClassifierMixin, + MultiOutputMixin, + RegressorMixin, + _fit_context, +) +from sklearn.utils import check_random_state +from sklearn.utils._param_validation import Interval, StrOptions +from sklearn.utils.multiclass import class_distribution +from sklearn.utils.random import _random_choice_csc +from sklearn.utils.stats import _weighted_percentile +from sklearn.utils.validation import ( + _check_sample_weight, + _num_samples, + check_array, + check_consistent_length, + check_is_fitted, + validate_data, +) + + +class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): + """DummyClassifier makes predictions that ignore the input features. + + This classifier serves as a simple baseline to compare against other more + complex classifiers. + + The specific behavior of the baseline is selected with the `strategy` + parameter. + + All strategies make predictions that ignore the input feature values passed + as the `X` argument to `fit` and `predict`. The predictions, however, + typically depend on values observed in the `y` parameter passed to `fit`. + + Note that the "stratified" and "uniform" strategies lead to + non-deterministic predictions that can be rendered deterministic by setting + the `random_state` parameter if needed. The other strategies are naturally + deterministic and, once fit, always return the same constant prediction + for any value of `X`. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.13 + + Parameters + ---------- + strategy : {"most_frequent", "prior", "stratified", "uniform", \ + "constant"}, default="prior" + Strategy to use to generate predictions. + + * "most_frequent": the `predict` method always returns the most + frequent class label in the observed `y` argument passed to `fit`. + The `predict_proba` method returns the matching one-hot encoded + vector. + * "prior": the `predict` method always returns the most frequent + class label in the observed `y` argument passed to `fit` (like + "most_frequent"). ``predict_proba`` always returns the empirical + class distribution of `y` also known as the empirical class prior + distribution. + * "stratified": the `predict_proba` method randomly samples one-hot + vectors from a multinomial distribution parametrized by the empirical + class prior probabilities. + The `predict` method returns the class label which got probability + one in the one-hot vector of `predict_proba`. + Each sampled row of both methods is therefore independent and + identically distributed. + * "uniform": generates predictions uniformly at random from the list + of unique classes observed in `y`, i.e. each class has equal + probability. + * "constant": always predicts a constant label that is provided by + the user. This is useful for metrics that evaluate a non-majority + class. + + .. versionchanged:: 0.24 + The default value of `strategy` has changed to "prior" in version + 0.24. + + random_state : int, RandomState instance or None, default=None + Controls the randomness to generate the predictions when + ``strategy='stratified'`` or ``strategy='uniform'``. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + constant : int or str or array-like of shape (n_outputs,), default=None + The explicit constant as predicted by the "constant" strategy. This + parameter is useful only for the "constant" strategy. + + Attributes + ---------- + classes_ : ndarray of shape (n_classes,) or list of such arrays + Unique class labels observed in `y`. For multi-output classification + problems, this attribute is a list of arrays as each output has an + independent set of possible classes. + + n_classes_ : int or list of int + Number of label for each output. + + class_prior_ : ndarray of shape (n_classes,) or list of such arrays + Frequency of each class observed in `y`. For multioutput classification + problems, this is computed independently for each output. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` has + feature names that are all strings. + + n_outputs_ : int + Number of outputs. + + sparse_output_ : bool + True if the array returned from predict is to be in sparse CSC format. + Is automatically set to True if the input `y` is passed in sparse + format. + + See Also + -------- + DummyRegressor : Regressor that makes predictions using simple rules. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.dummy import DummyClassifier + >>> X = np.array([-1, 1, 1, 1]) + >>> y = np.array([0, 1, 1, 1]) + >>> dummy_clf = DummyClassifier(strategy="most_frequent") + >>> dummy_clf.fit(X, y) + DummyClassifier(strategy='most_frequent') + >>> dummy_clf.predict(X) + array([1, 1, 1, 1]) + >>> dummy_clf.score(X, y) + 0.75 + """ + + _parameter_constraints: dict = { + "strategy": [ + StrOptions({"most_frequent", "prior", "stratified", "uniform", "constant"}) + ], + "random_state": ["random_state"], + "constant": [Integral, str, "array-like", None], + } + + def __init__(self, *, strategy="prior", random_state=None, constant=None): + self.strategy = strategy + self.random_state = random_state + self.constant = constant + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y, sample_weight=None): + """Fit the baseline classifier. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + y : array-like of shape (n_samples,) or (n_samples, n_outputs) + Target values. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + Returns + ------- + self : object + Returns the instance itself. + """ + validate_data(self, X, skip_check_array=True) + + self._strategy = self.strategy + + if self._strategy == "uniform" and sp.issparse(y): + y = y.toarray() + warnings.warn( + ( + "A local copy of the target data has been converted " + "to a numpy array. Predicting on sparse target data " + "with the uniform strategy would not save memory " + "and would be slower." + ), + UserWarning, + ) + + self.sparse_output_ = sp.issparse(y) + + if not self.sparse_output_: + y = np.asarray(y) + y = np.atleast_1d(y) + + if y.ndim == 1: + y = np.reshape(y, (-1, 1)) + + self.n_outputs_ = y.shape[1] + + check_consistent_length(X, y) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) + + if self._strategy == "constant": + if self.constant is None: + raise ValueError( + "Constant target value has to be specified " + "when the constant strategy is used." + ) + else: + constant = np.reshape(np.atleast_1d(self.constant), (-1, 1)) + if constant.shape[0] != self.n_outputs_: + raise ValueError( + "Constant target value should have shape (%d, 1)." + % self.n_outputs_ + ) + + (self.classes_, self.n_classes_, self.class_prior_) = class_distribution( + y, sample_weight + ) + + if self._strategy == "constant": + for k in range(self.n_outputs_): + if not any(constant[k][0] == c for c in self.classes_[k]): + # Checking in case of constant strategy if the constant + # provided by the user is in y. + err_msg = ( + "The constant target value must be present in " + "the training data. You provided constant={}. " + "Possible values are: {}.".format( + self.constant, self.classes_[k].tolist() + ) + ) + raise ValueError(err_msg) + + if self.n_outputs_ == 1: + self.n_classes_ = self.n_classes_[0] + self.classes_ = self.classes_[0] + self.class_prior_ = self.class_prior_[0] + + return self + + def predict(self, X): + """Perform classification on test vectors X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Test data. + + Returns + ------- + y : array-like of shape (n_samples,) or (n_samples, n_outputs) + Predicted target values for X. + """ + check_is_fitted(self) + + # numpy random_state expects Python int and not long as size argument + # under Windows + n_samples = _num_samples(X) + rs = check_random_state(self.random_state) + + n_classes_ = self.n_classes_ + classes_ = self.classes_ + class_prior_ = self.class_prior_ + constant = self.constant + if self.n_outputs_ == 1: + # Get same type even for self.n_outputs_ == 1 + n_classes_ = [n_classes_] + classes_ = [classes_] + class_prior_ = [class_prior_] + constant = [constant] + # Compute probability only once + if self._strategy == "stratified": + proba = self.predict_proba(X) + if self.n_outputs_ == 1: + proba = [proba] + + if self.sparse_output_: + class_prob = None + if self._strategy in ("most_frequent", "prior"): + classes_ = [np.array([cp.argmax()]) for cp in class_prior_] + + elif self._strategy == "stratified": + class_prob = class_prior_ + + elif self._strategy == "uniform": + raise ValueError( + "Sparse target prediction is not " + "supported with the uniform strategy" + ) + + elif self._strategy == "constant": + classes_ = [np.array([c]) for c in constant] + + y = _random_choice_csc(n_samples, classes_, class_prob, self.random_state) + else: + if self._strategy in ("most_frequent", "prior"): + y = np.tile( + [ + classes_[k][class_prior_[k].argmax()] + for k in range(self.n_outputs_) + ], + [n_samples, 1], + ) + + elif self._strategy == "stratified": + y = np.vstack( + [ + classes_[k][proba[k].argmax(axis=1)] + for k in range(self.n_outputs_) + ] + ).T + + elif self._strategy == "uniform": + ret = [ + classes_[k][rs.randint(n_classes_[k], size=n_samples)] + for k in range(self.n_outputs_) + ] + y = np.vstack(ret).T + + elif self._strategy == "constant": + y = np.tile(self.constant, (n_samples, 1)) + + if self.n_outputs_ == 1: + y = np.ravel(y) + + return y + + def predict_proba(self, X): + """ + Return probability estimates for the test vectors X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Test data. + + Returns + ------- + P : ndarray of shape (n_samples, n_classes) or list of such arrays + Returns the probability of the sample for each class in + the model, where classes are ordered arithmetically, for each + output. + """ + check_is_fitted(self) + + # numpy random_state expects Python int and not long as size argument + # under Windows + n_samples = _num_samples(X) + rs = check_random_state(self.random_state) + + n_classes_ = self.n_classes_ + classes_ = self.classes_ + class_prior_ = self.class_prior_ + constant = self.constant + if self.n_outputs_ == 1: + # Get same type even for self.n_outputs_ == 1 + n_classes_ = [n_classes_] + classes_ = [classes_] + class_prior_ = [class_prior_] + constant = [constant] + + P = [] + for k in range(self.n_outputs_): + if self._strategy == "most_frequent": + ind = class_prior_[k].argmax() + out = np.zeros((n_samples, n_classes_[k]), dtype=np.float64) + out[:, ind] = 1.0 + elif self._strategy == "prior": + out = np.ones((n_samples, 1)) * class_prior_[k] + + elif self._strategy == "stratified": + out = rs.multinomial(1, class_prior_[k], size=n_samples) + out = out.astype(np.float64) + + elif self._strategy == "uniform": + out = np.ones((n_samples, n_classes_[k]), dtype=np.float64) + out /= n_classes_[k] + + elif self._strategy == "constant": + ind = np.where(classes_[k] == constant[k]) + out = np.zeros((n_samples, n_classes_[k]), dtype=np.float64) + out[:, ind] = 1.0 + + P.append(out) + + if self.n_outputs_ == 1: + P = P[0] + + return P + + def predict_log_proba(self, X): + """ + Return log probability estimates for the test vectors X. + + Parameters + ---------- + X : {array-like, object with finite length or shape} + Training data. + + Returns + ------- + P : ndarray of shape (n_samples, n_classes) or list of such arrays + Returns the log probability of the sample for each class in + the model, where classes are ordered arithmetically for each + output. + """ + proba = self.predict_proba(X) + if self.n_outputs_ == 1: + return np.log(proba) + else: + return [np.log(p) for p in proba] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.classifier_tags.poor_score = True + tags.no_validation = True + return tags + + def score(self, X, y, sample_weight=None): + """Return the mean accuracy on the given test data and labels. + + In multi-label classification, this is the subset accuracy + which is a harsh metric since you require for each sample that + each label set be correctly predicted. + + Parameters + ---------- + X : None or array-like of shape (n_samples, n_features) + Test samples. Passing None as test samples gives the same result + as passing real test samples, since DummyClassifier + operates independently of the sampled observations. + + y : array-like of shape (n_samples,) or (n_samples, n_outputs) + True labels for X. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + Returns + ------- + score : float + Mean accuracy of self.predict(X) w.r.t. y. + """ + if X is None: + X = np.zeros(shape=(len(y), 1)) + return super().score(X, y, sample_weight) + + +class DummyRegressor(MultiOutputMixin, RegressorMixin, BaseEstimator): + """Regressor that makes predictions using simple rules. + + This regressor is useful as a simple baseline to compare with other + (real) regressors. Do not use it for real problems. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.13 + + Parameters + ---------- + strategy : {"mean", "median", "quantile", "constant"}, default="mean" + Strategy to use to generate predictions. + + * "mean": always predicts the mean of the training set + * "median": always predicts the median of the training set + * "quantile": always predicts a specified quantile of the training set, + provided with the quantile parameter. + * "constant": always predicts a constant value that is provided by + the user. + + constant : int or float or array-like of shape (n_outputs,), default=None + The explicit constant as predicted by the "constant" strategy. This + parameter is useful only for the "constant" strategy. + + quantile : float in [0.0, 1.0], default=None + The quantile to predict using the "quantile" strategy. A quantile of + 0.5 corresponds to the median, while 0.0 to the minimum and 1.0 to the + maximum. + + Attributes + ---------- + constant_ : ndarray of shape (1, n_outputs) + Mean or median or quantile of the training targets or constant value + given by the user. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` has + feature names that are all strings. + + n_outputs_ : int + Number of outputs. + + See Also + -------- + DummyClassifier: Classifier that makes predictions using simple rules. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.dummy import DummyRegressor + >>> X = np.array([1.0, 2.0, 3.0, 4.0]) + >>> y = np.array([2.0, 3.0, 5.0, 10.0]) + >>> dummy_regr = DummyRegressor(strategy="mean") + >>> dummy_regr.fit(X, y) + DummyRegressor() + >>> dummy_regr.predict(X) + array([5., 5., 5., 5.]) + >>> dummy_regr.score(X, y) + 0.0 + """ + + _parameter_constraints: dict = { + "strategy": [StrOptions({"mean", "median", "quantile", "constant"})], + "quantile": [Interval(Real, 0.0, 1.0, closed="both"), None], + "constant": [ + Interval(Real, None, None, closed="neither"), + "array-like", + None, + ], + } + + def __init__(self, *, strategy="mean", constant=None, quantile=None): + self.strategy = strategy + self.constant = constant + self.quantile = quantile + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y, sample_weight=None): + """Fit the baseline regressor. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training data. + + y : array-like of shape (n_samples,) or (n_samples, n_outputs) + Target values. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + Returns + ------- + self : object + Fitted estimator. + """ + validate_data(self, X, skip_check_array=True) + + y = check_array(y, ensure_2d=False, input_name="y") + if len(y) == 0: + raise ValueError("y must not be empty.") + + if y.ndim == 1: + y = np.reshape(y, (-1, 1)) + self.n_outputs_ = y.shape[1] + + check_consistent_length(X, y, sample_weight) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) + + if self.strategy == "mean": + self.constant_ = np.average(y, axis=0, weights=sample_weight) + + elif self.strategy == "median": + if sample_weight is None: + self.constant_ = np.median(y, axis=0) + else: + self.constant_ = _weighted_percentile( + y, sample_weight, percentile_rank=50.0 + ) + + elif self.strategy == "quantile": + if self.quantile is None: + raise ValueError( + "When using `strategy='quantile', you have to specify the desired " + "quantile in the range [0, 1]." + ) + percentile_rank = self.quantile * 100.0 + if sample_weight is None: + self.constant_ = np.percentile(y, axis=0, q=percentile_rank) + else: + self.constant_ = _weighted_percentile( + y, sample_weight, percentile_rank=percentile_rank + ) + + elif self.strategy == "constant": + if self.constant is None: + raise TypeError( + "Constant target value has to be specified " + "when the constant strategy is used." + ) + + self.constant_ = check_array( + self.constant, + accept_sparse=["csr", "csc", "coo"], + ensure_2d=False, + ensure_min_samples=0, + ) + + if self.n_outputs_ != 1 and self.constant_.shape[0] != y.shape[1]: + raise ValueError( + "Constant target value should have shape (%d, 1)." % y.shape[1] + ) + + self.constant_ = np.reshape(self.constant_, (1, -1)) + return self + + def predict(self, X, return_std=False): + """Perform classification on test vectors X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Test data. + + return_std : bool, default=False + Whether to return the standard deviation of posterior prediction. + All zeros in this case. + + .. versionadded:: 0.20 + + Returns + ------- + y : array-like of shape (n_samples,) or (n_samples, n_outputs) + Predicted target values for X. + + y_std : array-like of shape (n_samples,) or (n_samples, n_outputs) + Standard deviation of predictive distribution of query points. + """ + check_is_fitted(self) + n_samples = _num_samples(X) + + y = np.full( + (n_samples, self.n_outputs_), + self.constant_, + dtype=np.array(self.constant_).dtype, + ) + y_std = np.zeros((n_samples, self.n_outputs_)) + + if self.n_outputs_ == 1: + y = np.ravel(y) + y_std = np.ravel(y_std) + + return (y, y_std) if return_std else y + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.regressor_tags.poor_score = True + tags.no_validation = True + return tags + + def score(self, X, y, sample_weight=None): + """Return the coefficient of determination R^2 of the prediction. + + The coefficient R^2 is defined as `(1 - u/v)`, where `u` is the + residual sum of squares `((y_true - y_pred) ** 2).sum()` and `v` is the + total sum of squares `((y_true - y_true.mean()) ** 2).sum()`. The best + possible score is 1.0 and it can be negative (because the model can be + arbitrarily worse). A constant model that always predicts the expected + value of y, disregarding the input features, would get a R^2 score of + 0.0. + + Parameters + ---------- + X : None or array-like of shape (n_samples, n_features) + Test samples. Passing None as test samples gives the same result + as passing real test samples, since `DummyRegressor` + operates independently of the sampled observations. + + y : array-like of shape (n_samples,) or (n_samples, n_outputs) + True values for X. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + Returns + ------- + score : float + R^2 of `self.predict(X)` w.r.t. y. + """ + if X is None: + X = np.zeros(shape=(len(y), 1)) + return super().score(X, y, sample_weight) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..706a2572d90118e48e9d1e55c0f80fc4f1c36994 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/__init__.py @@ -0,0 +1,45 @@ +"""Ensemble-based methods for classification, regression and anomaly detection.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from sklearn.ensemble._bagging import BaggingClassifier, BaggingRegressor +from sklearn.ensemble._base import BaseEnsemble +from sklearn.ensemble._forest import ( + ExtraTreesClassifier, + ExtraTreesRegressor, + RandomForestClassifier, + RandomForestRegressor, + RandomTreesEmbedding, +) +from sklearn.ensemble._gb import GradientBoostingClassifier, GradientBoostingRegressor +from sklearn.ensemble._hist_gradient_boosting.gradient_boosting import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.ensemble._iforest import IsolationForest +from sklearn.ensemble._stacking import StackingClassifier, StackingRegressor +from sklearn.ensemble._voting import VotingClassifier, VotingRegressor +from sklearn.ensemble._weight_boosting import AdaBoostClassifier, AdaBoostRegressor + +__all__ = [ + "AdaBoostClassifier", + "AdaBoostRegressor", + "BaggingClassifier", + "BaggingRegressor", + "BaseEnsemble", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", + "StackingClassifier", + "StackingRegressor", + "VotingClassifier", + "VotingRegressor", +] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_bagging.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_bagging.py new file mode 100644 index 0000000000000000000000000000000000000000..c285244604b98872a9a46be61073aca0d0588909 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_bagging.py @@ -0,0 +1,1542 @@ +"""Bagging meta-estimator.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools +import numbers +from abc import ABCMeta, abstractmethod +from functools import partial +from numbers import Integral +from warnings import warn + +import numpy as np + +from sklearn.base import ClassifierMixin, RegressorMixin, _fit_context +from sklearn.ensemble._base import BaseEnsemble, _partition_estimators +from sklearn.metrics import accuracy_score, r2_score +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils import Bunch, _safe_indexing, check_random_state, column_or_1d +from sklearn.utils._mask import indices_to_mask +from sklearn.utils._param_validation import HasMethods, Interval, RealNotInt +from sklearn.utils._tags import get_tags +from sklearn.utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + get_routing_for_object, + process_routing, +) +from sklearn.utils.metaestimators import available_if +from sklearn.utils.multiclass import check_classification_targets +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.random import sample_without_replacement +from sklearn.utils.validation import ( + _check_method_params, + _check_sample_weight, + _estimator_has, + check_is_fitted, + has_fit_parameter, + validate_data, +) + +__all__ = ["BaggingClassifier", "BaggingRegressor"] + +MAX_INT = np.iinfo(np.int32).max + + +def _get_n_samples_bootstrap(n_samples, max_samples, sample_weight): + """ + Get the number of samples in a bootstrap sample. + + Parameters + ---------- + n_samples : int + Number of samples in the dataset. + + max_samples : None, int or float + The maximum number of samples to draw. + + - If None, then draw `n_samples` samples. + - If int, then draw `max_samples` samples. + - If float, then draw `max_samples * n_samples` unweighted samples or + `max_samples * sample_weight.sum()` weighted samples. + + sample_weight : array of shape (n_samples,) or None + Sample weights with frequency semantics when `max_samples` is explicitly + set to a float or integer value. When keeping the `max_samples=None` default + value, the equivalence between fitting with integer weighted data points or + integer repeated data points is no longer guaranteed because the effective + bootstrap size is no longer guaranteed to be equivalent. + + Returns + ------- + n_samples_bootstrap : int + The total number of samples to draw for the bootstrap sample. + """ + if max_samples is None: + return n_samples + elif isinstance(max_samples, Integral): + return max_samples + + if sample_weight is None: + weighted_n_samples = n_samples + weighted_n_samples_msg = f"the number of samples is {weighted_n_samples} " + else: + weighted_n_samples = sample_weight.sum() + weighted_n_samples_msg = ( + f"the total sum of sample weights is {weighted_n_samples} " + ) + + # max_samples Real fractional value relative to weighted_n_samples + n_samples_bootstrap = max(int(max_samples * weighted_n_samples), 1) + # Warn when number of bootstrap samples is suspiciously small + # This heuristic for "suspiciously small" might be adapted if found + # unsuitable in practice + if n_samples_bootstrap < max(10, n_samples ** (1 / 3)): + warn( + f"Using the fractional value {max_samples=} when {weighted_n_samples_msg}" + f"results in a low number ({n_samples_bootstrap}) of bootstrap samples. " + "We recommend passing `max_samples` as an integer instead." + ) + return n_samples_bootstrap + + +def _generate_indices(random_state, bootstrap, n_population, n_samples): + """Draw randomly sampled indices.""" + # Draw sample indices + if bootstrap: + indices = random_state.randint(0, n_population, n_samples) + else: + indices = sample_without_replacement( + n_population, n_samples, random_state=random_state + ) + + return indices + + +def _generate_bagging_indices( + random_state, + bootstrap_features, + bootstrap_samples, + n_features, + n_samples, + max_features, + max_samples, + sample_weight, +): + """Randomly draw feature and sample indices.""" + # Get valid random state + random_state = check_random_state(random_state) + + # Draw indices + feature_indices = _generate_indices( + random_state, bootstrap_features, n_features, max_features + ) + if sample_weight is None: + sample_indices = _generate_indices( + random_state, bootstrap_samples, n_samples, max_samples + ) + else: + normalized_sample_weight = sample_weight / np.sum(sample_weight) + sample_indices = random_state.choice( + n_samples, + max_samples, + replace=bootstrap_samples, + p=normalized_sample_weight, + ) + + return feature_indices, sample_indices + + +def _consumes_sample_weight(estimator): + if _routing_enabled(): + request_or_router = get_routing_for_object(estimator) + consumes_sample_weight = request_or_router.consumes("fit", ("sample_weight",)) + else: + consumes_sample_weight = has_fit_parameter(estimator, "sample_weight") + return consumes_sample_weight + + +def _parallel_build_estimators( + n_estimators, + ensemble, + X, + y, + sample_weight, + seeds, + total_n_estimators, + verbose, + check_input, + fit_params, +): + """Private function used to build a batch of estimators within a job.""" + # Retrieve settings + n_samples, n_features = X.shape + max_features = ensemble._max_features + max_samples = ensemble._max_samples + bootstrap = ensemble.bootstrap + bootstrap_features = ensemble.bootstrap_features + has_check_input = has_fit_parameter(ensemble.estimator_, "check_input") + requires_feature_indexing = bootstrap_features or max_features != n_features + consumes_sample_weight = _consumes_sample_weight(ensemble.estimator_) + + # Build estimators + estimators = [] + estimators_features = [] + + for i in range(n_estimators): + if verbose > 1: + print( + "Building estimator %d of %d for this parallel run (total %d)..." + % (i + 1, n_estimators, total_n_estimators) + ) + + random_state = seeds[i] + estimator = ensemble._make_estimator(append=False, random_state=random_state) + + if has_check_input: + estimator_fit = partial(estimator.fit, check_input=check_input) + else: + estimator_fit = estimator.fit + + # Draw random feature, sample indices (using normalized sample_weight + # as probabilities if provided). + features, indices = _generate_bagging_indices( + random_state, + bootstrap_features, + bootstrap, + n_features, + n_samples, + max_features, + max_samples, + sample_weight, + ) + + fit_params_ = fit_params.copy() + + # Note: Row sampling can be achieved either through setting sample_weight or + # by indexing. The former is more memory efficient. Therefore, use this method + # if possible, otherwise use indexing. + if consumes_sample_weight: + # Row sampling by setting sample_weight + indices_as_sample_weight = np.bincount(indices, minlength=n_samples) + fit_params_["sample_weight"] = indices_as_sample_weight + X_ = X[:, features] if requires_feature_indexing else X + estimator_fit(X_, y, **fit_params_) + else: + # Row sampling by indexing + y_ = _safe_indexing(y, indices) + X_ = _safe_indexing(X, indices) + fit_params_ = _check_method_params(X, params=fit_params_, indices=indices) + if requires_feature_indexing: + X_ = X_[:, features] + estimator_fit(X_, y_, **fit_params_) + + estimators.append(estimator) + estimators_features.append(features) + + return estimators, estimators_features + + +def _parallel_predict_proba( + estimators, + estimators_features, + X, + n_classes, + predict_params=None, + predict_proba_params=None, +): + """Private function used to compute (proba-)predictions within a job.""" + n_samples = X.shape[0] + proba = np.zeros((n_samples, n_classes)) + + for estimator, features in zip(estimators, estimators_features): + if hasattr(estimator, "predict_proba"): + proba_estimator = estimator.predict_proba( + X[:, features], **(predict_params or {}) + ) + + if n_classes == len(estimator.classes_): + proba += proba_estimator + + else: + proba[:, estimator.classes_] += proba_estimator[ + :, range(len(estimator.classes_)) + ] + + else: + # Resort to voting + predictions = estimator.predict( + X[:, features], **(predict_proba_params or {}) + ) + + for i in range(n_samples): + proba[i, predictions[i]] += 1 + + return proba + + +def _parallel_predict_log_proba(estimators, estimators_features, X, n_classes, params): + """Private function used to compute log probabilities within a job.""" + n_samples = X.shape[0] + log_proba = np.empty((n_samples, n_classes)) + log_proba.fill(-np.inf) + all_classes = np.arange(n_classes, dtype=int) + + for estimator, features in zip(estimators, estimators_features): + log_proba_estimator = estimator.predict_log_proba(X[:, features], **params) + + if n_classes == len(estimator.classes_): + log_proba = np.logaddexp(log_proba, log_proba_estimator) + + else: + log_proba[:, estimator.classes_] = np.logaddexp( + log_proba[:, estimator.classes_], + log_proba_estimator[:, range(len(estimator.classes_))], + ) + + missing = np.setdiff1d(all_classes, estimator.classes_) + log_proba[:, missing] = np.logaddexp(log_proba[:, missing], -np.inf) + + return log_proba + + +def _parallel_decision_function(estimators, estimators_features, X, params): + """Private function used to compute decisions within a job.""" + return sum( + estimator.decision_function(X[:, features], **params) + for estimator, features in zip(estimators, estimators_features) + ) + + +def _parallel_predict_regression(estimators, estimators_features, X, params): + """Private function used to compute predictions within a job.""" + return sum( + estimator.predict(X[:, features], **params) + for estimator, features in zip(estimators, estimators_features) + ) + + +class BaseBagging(BaseEnsemble, metaclass=ABCMeta): + """Base class for Bagging meta-estimator. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + _parameter_constraints: dict = { + "estimator": [HasMethods(["fit", "predict"]), None], + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "max_samples": [ + None, + Interval(Integral, 1, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="right"), + ], + "max_features": [ + Interval(Integral, 1, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="right"), + ], + "bootstrap": ["boolean"], + "bootstrap_features": ["boolean"], + "oob_score": ["boolean"], + "warm_start": ["boolean"], + "n_jobs": [None, Integral], + "random_state": ["random_state"], + "verbose": ["verbose"], + } + + @abstractmethod + def __init__( + self, + estimator=None, + n_estimators=10, + *, + max_samples=None, + max_features=1.0, + bootstrap=True, + bootstrap_features=False, + oob_score=False, + warm_start=False, + n_jobs=None, + random_state=None, + verbose=0, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + ) + self.max_samples = max_samples + self.max_features = max_features + self.bootstrap = bootstrap + self.bootstrap_features = bootstrap_features + self.oob_score = oob_score + self.warm_start = warm_start + self.n_jobs = n_jobs + self.random_state = random_state + self.verbose = verbose + + @_fit_context( + # BaseBagging.estimator is not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, sample_weight=None, **fit_params): + """Build a Bagging ensemble of estimators from the training set (X, y). + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only if + they are supported by the base estimator. + + y : array-like of shape (n_samples,) + The target values (class labels in classification, real numbers in + regression). + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, then samples are equally weighted. Used as + probabilities to sample the training set. Note that the expected + frequency semantics for the `sample_weight` parameter are only + fulfilled when sampling with replacement `bootstrap=True` and using + a float or integer `max_samples` (instead of the default + `max_samples=None`). + + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.5 + + Only available if `enable_metadata_routing=True`, + which can be set by using + ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + + Returns + ------- + self : object + Fitted estimator. + """ + _raise_for_params(fit_params, self, "fit") + + # Convert data (X is required to be 2d and indexable) + X, y = validate_data( + self, + X, + y, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + multi_output=True, + ) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, dtype=None) + + if not self.bootstrap: + warn( + f"When fitting {self.__class__.__name__} with sample_weight " + f"it is recommended to use bootstrap=True, got {self.bootstrap}." + ) + + return self._fit( + X, + y, + max_samples=self.max_samples, + sample_weight=sample_weight, + **fit_params, + ) + + def _parallel_args(self): + return {} + + def _fit( + self, + X, + y, + max_samples=None, + max_depth=None, + check_input=True, + sample_weight=None, + **fit_params, + ): + """Build a Bagging ensemble of estimators from the training + set (X, y). + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only if + they are supported by the base estimator. + + y : array-like of shape (n_samples,) + The target values (class labels in classification, real numbers in + regression). + + max_samples : int or float, default=None + Argument to use instead of self.max_samples. + + max_depth : int, default=None + Override value used when constructing base estimator. Only + supported if the base estimator has a max_depth parameter. + + check_input : bool, default=True + Override value used when fitting base estimator. Only supported + if the base estimator has a check_input parameter for fit function. + If the meta-estimator already checks the input, set this value to + False to prevent redundant input validation. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, then samples are equally weighted. + + **fit_params : dict, default=None + Parameters to pass to the :term:`fit` method of the underlying + estimator. + + Returns + ------- + self : object + Fitted estimator. + """ + random_state = check_random_state(self.random_state) + + # Remap output + n_samples = X.shape[0] + self._n_samples = n_samples + y = self._validate_y(y) + + # Check parameters + self._validate_estimator(self._get_estimator()) + + if _routing_enabled(): + routed_params = process_routing(self, "fit", **fit_params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(fit=fit_params) + + if max_depth is not None: + self.estimator_.max_depth = max_depth + + # Validate max_samples + if max_samples is None: + max_samples = self.max_samples + + max_samples = _get_n_samples_bootstrap(X.shape[0], max_samples, sample_weight) + if not self.bootstrap and max_samples > X.shape[0]: + raise ValueError( + f"Effective max_samples={max_samples} must be <= n_samples=" + f"{X.shape[0]} to be able to sample without replacement." + ) + + # Store validated integer row sampling value + self._max_samples = max_samples + + # Validate max_features + if isinstance(self.max_features, numbers.Integral): + max_features = self.max_features + elif isinstance(self.max_features, float): + max_features = int(self.max_features * self.n_features_in_) + + if max_features > self.n_features_in_: + raise ValueError("max_features must be <= n_features") + + max_features = max(1, int(max_features)) + + # Store validated integer feature sampling value + self._max_features = max_features + + # Store sample_weight (needed in _get_estimators_indices). Note that + # we intentionally do not materialize `sample_weight=None` as an array + # of ones to avoid unnecessarily cluttering trained estimator pickles. + self._sample_weight = sample_weight + + # Other checks + if not self.bootstrap and self.oob_score: + raise ValueError("Out of bag estimation only available if bootstrap=True") + + if self.warm_start and self.oob_score: + raise ValueError("Out of bag estimate only available if warm_start=False") + + if hasattr(self, "oob_score_") and self.warm_start: + del self.oob_score_ + + if not self.warm_start or not hasattr(self, "estimators_"): + # Free allocated memory, if any + self.estimators_ = [] + self.estimators_features_ = [] + + n_more_estimators = self.n_estimators - len(self.estimators_) + + if n_more_estimators < 0: + raise ValueError( + "n_estimators=%d must be larger or equal to " + "len(estimators_)=%d when warm_start==True" + % (self.n_estimators, len(self.estimators_)) + ) + + elif n_more_estimators == 0: + warn( + "Warm-start fitting without increasing n_estimators does not " + "fit new trees." + ) + return self + + # Parallel loop + n_jobs, n_estimators, starts = _partition_estimators( + n_more_estimators, self.n_jobs + ) + total_n_estimators = sum(n_estimators) + + # Advance random state to state after training + # the first n_estimators + if self.warm_start and len(self.estimators_) > 0: + random_state.randint(MAX_INT, size=len(self.estimators_)) + + seeds = random_state.randint(MAX_INT, size=n_more_estimators) + self._seeds = seeds + + all_results = Parallel( + n_jobs=n_jobs, verbose=self.verbose, **self._parallel_args() + )( + delayed(_parallel_build_estimators)( + n_estimators[i], + self, + X, + y, + sample_weight, + seeds[starts[i] : starts[i + 1]], + total_n_estimators, + verbose=self.verbose, + check_input=check_input, + fit_params=routed_params.estimator.fit, + ) + for i in range(n_jobs) + ) + + # Reduce + self.estimators_ += list( + itertools.chain.from_iterable(t[0] for t in all_results) + ) + self.estimators_features_ += list( + itertools.chain.from_iterable(t[1] for t in all_results) + ) + + if self.oob_score: + self._set_oob_score(X, y) + + return self + + @abstractmethod + def _set_oob_score(self, X, y): + """Calculate out of bag predictions and score.""" + + def _validate_y(self, y): + if len(y.shape) == 1 or y.shape[1] == 1: + return column_or_1d(y, warn=True) + return y + + def _get_estimators_indices(self): + # Get drawn indices along both sample and feature axes + for seed in self._seeds: + # Operations accessing random_state must be performed identically + # to those in `_parallel_build_estimators()` + feature_indices, sample_indices = _generate_bagging_indices( + seed, + self.bootstrap_features, + self.bootstrap, + self.n_features_in_, + self._n_samples, + self._max_features, + self._max_samples, + self._sample_weight, + ) + + yield feature_indices, sample_indices + + @property + def estimators_samples_(self): + """ + The subset of drawn samples for each base estimator. + + Returns a dynamically generated list of indices identifying + the samples used for fitting each member of the ensemble, i.e., + the in-bag samples. + + Note: the list is re-created at each call to the property in order + to reduce the object memory footprint by not storing the sampling + data. Thus fetching the property may be slower than expected. + """ + return [sample_indices for _, sample_indices in self._get_estimators_indices()] + + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.5 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self) + + method_mapping = MethodMapping() + method_mapping.add(caller="fit", callee="fit").add( + caller="decision_function", callee="decision_function" + ) + + # the router needs to be built depending on whether the sub-estimator has a + # `predict_proba` method (as BaggingClassifier decides dynamically at runtime): + if hasattr(self._get_estimator(), "predict_proba"): + ( + method_mapping.add(caller="predict", callee="predict_proba").add( + caller="predict_proba", callee="predict_proba" + ) + ) + + else: + ( + method_mapping.add(caller="predict", callee="predict").add( + caller="predict_proba", callee="predict" + ) + ) + + # the router needs to be built depending on whether the sub-estimator has a + # `predict_log_proba` method (as BaggingClassifier decides dynamically at + # runtime): + if hasattr(self._get_estimator(), "predict_log_proba"): + method_mapping.add(caller="predict_log_proba", callee="predict_log_proba") + + else: + # if `predict_log_proba` is not available in BaggingClassifier's + # sub-estimator, the routing should go to its `predict_proba` if it is + # available or else to its `predict` method; according to how + # `sample_weight` is passed to the respective methods dynamically at + # runtime: + if hasattr(self._get_estimator(), "predict_proba"): + method_mapping.add(caller="predict_log_proba", callee="predict_proba") + + else: + method_mapping.add(caller="predict_log_proba", callee="predict") + + router.add(estimator=self._get_estimator(), method_mapping=method_mapping) + return router + + @abstractmethod + def _get_estimator(self): + """Resolve which estimator to return.""" + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = get_tags(self._get_estimator()).input_tags.sparse + tags.input_tags.allow_nan = get_tags(self._get_estimator()).input_tags.allow_nan + return tags + + +class BaggingClassifier(ClassifierMixin, BaseBagging): + """A Bagging classifier. + + A Bagging classifier is an ensemble meta-estimator that fits base + classifiers each on random subsets of the original dataset and then + aggregate their individual predictions (either by voting or by averaging) + to form a final prediction. Such a meta-estimator can typically be used as + a way to reduce the variance of a black-box estimator (e.g., a decision + tree), by introducing randomization into its construction procedure and + then making an ensemble out of it. + + This algorithm encompasses several works from the literature. When random + subsets of the dataset are drawn as random subsets of the samples, then + this algorithm is known as Pasting [1]_. If samples are drawn with + replacement, then the method is known as Bagging [2]_. When random subsets + of the dataset are drawn as random subsets of the features, then the method + is known as Random Subspaces [3]_. Finally, when base estimators are built + on subsets of both samples and features, then the method is known as + Random Patches [4]_. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.15 + + Parameters + ---------- + estimator : object, default=None + The base estimator to fit on random subsets of the dataset. + If None, then the base estimator is a + :class:`~sklearn.tree.DecisionTreeClassifier`. + + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + + n_estimators : int, default=10 + The number of base estimators in the ensemble. + + max_samples : int or float, default=None + The number of samples to draw from X to train each base estimator (with + replacement by default, see `bootstrap` for more details). + + - If None, then draw `X.shape[0]` samples irrespective of `sample_weight`. + - If int, then draw `max_samples` samples. + - If float, then draw `max_samples * X.shape[0]` unweighted samples or + `max_samples * sample_weight.sum()` weighted samples. + + max_features : int or float, default=1.0 + The number of features to draw from X to train each base estimator ( + without replacement by default, see `bootstrap_features` for more + details). + + - If int, then draw `max_features` features. + - If float, then draw `max(1, int(max_features * n_features_in_))` features. + + bootstrap : bool, default=True + Whether samples are drawn with replacement. If False, sampling without + replacement is performed. If fitting with `sample_weight`, it is + strongly recommended to choose True, as only drawing with replacement + will ensure the expected frequency semantics of `sample_weight`. + + bootstrap_features : bool, default=False + Whether features are drawn with replacement. + + oob_score : bool, default=False + Whether to use out-of-bag samples to estimate + the generalization error. Only available if bootstrap=True. + + warm_start : bool, default=False + When set to True, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit + a whole new ensemble. See :term:`the Glossary `. + + .. versionadded:: 0.17 + *warm_start* constructor parameter. + + n_jobs : int, default=None + The number of jobs to run in parallel for both :meth:`fit` and + :meth:`predict`. ``None`` means 1 unless in a + :obj:`joblib.parallel_backend` context. ``-1`` means using all + processors. See :term:`Glossary ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls the random resampling of the original dataset + (sample wise and feature wise). + If the base estimator accepts a `random_state` attribute, a different + seed is generated for each instance in the ensemble. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + verbose : int, default=0 + Controls the verbosity when fitting and predicting. + + Attributes + ---------- + estimator_ : estimator + The base estimator from which the ensemble is grown. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + estimators_ : list of estimators + The collection of fitted base estimators. + + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + estimators_features_ : list of arrays + The subset of drawn features for each base estimator. + + classes_ : ndarray of shape (n_classes,) + The classes labels. + + n_classes_ : int or list + The number of classes. + + oob_score_ : float + Score of the training dataset obtained using an out-of-bag estimate. + This attribute exists only when ``oob_score`` is True. + + oob_decision_function_ : ndarray of shape (n_samples, n_classes) + Decision function computed with out-of-bag estimate on the training + set. If n_estimators is small it might be possible that a data point + was never left out during the bootstrap. In this case, + `oob_decision_function_` might contain NaN. This attribute exists + only when ``oob_score`` is True. + + See Also + -------- + BaggingRegressor : A Bagging regressor. + + References + ---------- + + .. [1] L. Breiman, "Pasting small votes for classification in large + databases and on-line", Machine Learning, 36(1), 85-103, 1999. + + .. [2] L. Breiman, "Bagging predictors", Machine Learning, 24(2), 123-140, + 1996. + + .. [3] T. Ho, "The random subspace method for constructing decision + forests", Pattern Analysis and Machine Intelligence, 20(8), 832-844, + 1998. + + .. [4] G. Louppe and P. Geurts, "Ensembles on Random Patches", Machine + Learning and Knowledge Discovery in Databases, 346-361, 2012. + + Examples + -------- + >>> from sklearn.svm import SVC + >>> from sklearn.ensemble import BaggingClassifier + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification(n_samples=100, n_features=4, + ... n_informative=2, n_redundant=0, + ... random_state=0, shuffle=False) + >>> clf = BaggingClassifier(estimator=SVC(), + ... n_estimators=10, random_state=0).fit(X, y) + >>> clf.predict([[0, 0, 0, 0]]) + array([1]) + """ + + def __init__( + self, + estimator=None, + n_estimators=10, + *, + max_samples=None, + max_features=1.0, + bootstrap=True, + bootstrap_features=False, + oob_score=False, + warm_start=False, + n_jobs=None, + random_state=None, + verbose=0, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + max_samples=max_samples, + max_features=max_features, + bootstrap=bootstrap, + bootstrap_features=bootstrap_features, + oob_score=oob_score, + warm_start=warm_start, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + ) + + def _get_estimator(self): + """Resolve which estimator to return (default is DecisionTreeClassifier)""" + if self.estimator is None: + return DecisionTreeClassifier() + return self.estimator + + def _set_oob_score(self, X, y): + n_samples = y.shape[0] + n_classes_ = self.n_classes_ + + predictions = np.zeros((n_samples, n_classes_)) + + for estimator, samples, features in zip( + self.estimators_, self.estimators_samples_, self.estimators_features_ + ): + # Create mask for OOB samples + mask = ~indices_to_mask(samples, n_samples) + + if hasattr(estimator, "predict_proba"): + predictions[mask, :] += estimator.predict_proba( + (X[mask, :])[:, features] + ) + + else: + p = estimator.predict((X[mask, :])[:, features]) + j = 0 + + for i in range(n_samples): + if mask[i]: + predictions[i, p[j]] += 1 + j += 1 + + if (predictions.sum(axis=1) == 0).any(): + warn( + "Some inputs do not have OOB scores. " + "This probably means too few estimators were used " + "to compute any reliable oob estimates." + ) + + oob_decision_function = predictions / predictions.sum(axis=1)[:, np.newaxis] + oob_score = accuracy_score(y, np.argmax(predictions, axis=1)) + + self.oob_decision_function_ = oob_decision_function + self.oob_score_ = oob_score + + def _validate_y(self, y): + y = column_or_1d(y, warn=True) + check_classification_targets(y) + self.classes_, y = np.unique(y, return_inverse=True) + self.n_classes_ = len(self.classes_) + + return y + + def predict(self, X, **params): + """Predict class for X. + + The predicted class of an input sample is computed as the class with + the highest mean predicted probability. If base estimators do not + implement a ``predict_proba`` method, then it resorts to voting. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only if + they are supported by the base estimator. + + **params : dict + Parameters routed to the `predict_proba` (if available) or the `predict` + method (otherwise) of the sub-estimators via the metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + + Returns + ------- + y : ndarray of shape (n_samples,) + The predicted classes. + """ + _raise_for_params(params, self, "predict") + + predicted_probabilitiy = self.predict_proba(X, **params) + return self.classes_.take((np.argmax(predicted_probabilitiy, axis=1)), axis=0) + + def predict_proba(self, X, **params): + """Predict class probabilities for X. + + The predicted class probabilities of an input sample is computed as + the mean predicted class probabilities of the base estimators in the + ensemble. If base estimators do not implement a ``predict_proba`` + method, then it resorts to voting and the predicted class probabilities + of an input sample represents the proportion of estimators predicting + each class. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only if + they are supported by the base estimator. + + **params : dict + Parameters routed to the `predict_proba` (if available) or the `predict` + method (otherwise) of the sub-estimators via the metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes) + The class probabilities of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + """ + _raise_for_params(params, self, "predict_proba") + + check_is_fitted(self) + # Check data + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, + ) + + if _routing_enabled(): + routed_params = process_routing(self, "predict_proba", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(predict_proba=Bunch()) + + # Parallel loop + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) + + all_proba = Parallel( + n_jobs=n_jobs, verbose=self.verbose, **self._parallel_args() + )( + delayed(_parallel_predict_proba)( + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], + X, + self.n_classes_, + predict_params=routed_params.estimator.get("predict", None), + predict_proba_params=routed_params.estimator.get("predict_proba", None), + ) + for i in range(n_jobs) + ) + + # Reduce + proba = sum(all_proba) / self.n_estimators + + return proba + + def predict_log_proba(self, X, **params): + """Predict class log-probabilities for X. + + The predicted class log-probabilities of an input sample is computed as + the log of the mean predicted class probabilities of the base + estimators in the ensemble. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only if + they are supported by the base estimator. + + **params : dict + Parameters routed to the `predict_log_proba`, the `predict_proba` or the + `proba` method of the sub-estimators via the metadata routing API. The + routing is tried in the mentioned order depending on whether this method is + available on the sub-estimator. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes) + The class log-probabilities of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + """ + _raise_for_params(params, self, "predict_log_proba") + + check_is_fitted(self) + + if hasattr(self.estimator_, "predict_log_proba"): + # Check data + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, + ) + + if _routing_enabled(): + routed_params = process_routing(self, "predict_log_proba", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(predict_log_proba=Bunch()) + + # Parallel loop + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) + + all_log_proba = Parallel(n_jobs=n_jobs, verbose=self.verbose)( + delayed(_parallel_predict_log_proba)( + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], + X, + self.n_classes_, + params=routed_params.estimator.predict_log_proba, + ) + for i in range(n_jobs) + ) + + # Reduce + log_proba = all_log_proba[0] + + for j in range(1, len(all_log_proba)): + log_proba = np.logaddexp(log_proba, all_log_proba[j]) + + log_proba -= np.log(self.n_estimators) + + else: + log_proba = np.log(self.predict_proba(X, **params)) + + return log_proba + + @available_if( + _estimator_has("decision_function", delegates=("estimators_", "estimator")) + ) + def decision_function(self, X, **params): + """Average of the decision functions of the base classifiers. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only if + they are supported by the base estimator. + + **params : dict + Parameters routed to the `decision_function` method of the sub-estimators + via the metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + + Returns + ------- + score : ndarray of shape (n_samples, k) + The decision function of the input samples. The columns correspond + to the classes in sorted order, as they appear in the attribute + ``classes_``. Regression and binary classification are special + cases with ``k == 1``, otherwise ``k==n_classes``. + """ + _raise_for_params(params, self, "decision_function") + + check_is_fitted(self) + + # Check data + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, + ) + + if _routing_enabled(): + routed_params = process_routing(self, "decision_function", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(decision_function=Bunch()) + + # Parallel loop + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) + + all_decisions = Parallel(n_jobs=n_jobs, verbose=self.verbose)( + delayed(_parallel_decision_function)( + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], + X, + params=routed_params.estimator.decision_function, + ) + for i in range(n_jobs) + ) + + # Reduce + decisions = sum(all_decisions) / self.n_estimators + + return decisions + + +class BaggingRegressor(RegressorMixin, BaseBagging): + """A Bagging regressor. + + A Bagging regressor is an ensemble meta-estimator that fits base + regressors each on random subsets of the original dataset and then + aggregate their individual predictions (either by voting or by averaging) + to form a final prediction. Such a meta-estimator can typically be used as + a way to reduce the variance of a black-box estimator (e.g., a decision + tree), by introducing randomization into its construction procedure and + then making an ensemble out of it. + + This algorithm encompasses several works from the literature. When random + subsets of the dataset are drawn as random subsets of the samples, then + this algorithm is known as Pasting [1]_. If samples are drawn with + replacement, then the method is known as Bagging [2]_. When random subsets + of the dataset are drawn as random subsets of the features, then the method + is known as Random Subspaces [3]_. Finally, when base estimators are built + on subsets of both samples and features, then the method is known as + Random Patches [4]_. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.15 + + Parameters + ---------- + estimator : object, default=None + The base estimator to fit on random subsets of the dataset. + If None, then the base estimator is a + :class:`~sklearn.tree.DecisionTreeRegressor`. + + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + + n_estimators : int, default=10 + The number of base estimators in the ensemble. + + max_samples : int or float, default=None + The number of samples to draw from X to train each base estimator (with + replacement by default, see `bootstrap` for more details). + + - If None, then draw `X.shape[0]` samples irrespective of `sample_weight`. + - If int, then draw `max_samples` samples. + - If float, then draw `max_samples * X.shape[0]` unweighted samples or + `max_samples * sample_weight.sum()` weighted samples. + + max_features : int or float, default=1.0 + The number of features to draw from X to train each base estimator ( + without replacement by default, see `bootstrap_features` for more + details). + + - If int, then draw `max_features` features. + - If float, then draw `max(1, int(max_features * n_features_in_))` features. + + bootstrap : bool, default=True + Whether samples are drawn with replacement. If False, sampling without + replacement is performed. If fitting with `sample_weight`, it is + strongly recommended to choose True, as only drawing with replacement + will ensure the expected frequency semantics of `sample_weight`. + + bootstrap_features : bool, default=False + Whether features are drawn with replacement. + + oob_score : bool, default=False + Whether to use out-of-bag samples to estimate + the generalization error. Only available if bootstrap=True. + + warm_start : bool, default=False + When set to True, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit + a whole new ensemble. See :term:`the Glossary `. + + n_jobs : int, default=None + The number of jobs to run in parallel for both :meth:`fit` and + :meth:`predict`. ``None`` means 1 unless in a + :obj:`joblib.parallel_backend` context. ``-1`` means using all + processors. See :term:`Glossary ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls the random resampling of the original dataset + (sample wise and feature wise). + If the base estimator accepts a `random_state` attribute, a different + seed is generated for each instance in the ensemble. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + verbose : int, default=0 + Controls the verbosity when fitting and predicting. + + Attributes + ---------- + estimator_ : estimator + The base estimator from which the ensemble is grown. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + estimators_ : list of estimators + The collection of fitted sub-estimators. + + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + estimators_features_ : list of arrays + The subset of drawn features for each base estimator. + + oob_score_ : float + Score of the training dataset obtained using an out-of-bag estimate. + This attribute exists only when ``oob_score`` is True. + + oob_prediction_ : ndarray of shape (n_samples,) + Prediction computed with out-of-bag estimate on the training + set. If n_estimators is small it might be possible that a data point + was never left out during the bootstrap. In this case, + `oob_prediction_` might contain NaN. This attribute exists only + when ``oob_score`` is True. + + See Also + -------- + BaggingClassifier : A Bagging classifier. + + References + ---------- + + .. [1] L. Breiman, "Pasting small votes for classification in large + databases and on-line", Machine Learning, 36(1), 85-103, 1999. + + .. [2] L. Breiman, "Bagging predictors", Machine Learning, 24(2), 123-140, + 1996. + + .. [3] T. Ho, "The random subspace method for constructing decision + forests", Pattern Analysis and Machine Intelligence, 20(8), 832-844, + 1998. + + .. [4] G. Louppe and P. Geurts, "Ensembles on Random Patches", Machine + Learning and Knowledge Discovery in Databases, 346-361, 2012. + + Examples + -------- + >>> from sklearn.svm import SVR + >>> from sklearn.ensemble import BaggingRegressor + >>> from sklearn.datasets import make_regression + >>> X, y = make_regression(n_samples=100, n_features=4, + ... n_informative=2, n_targets=1, + ... random_state=0, shuffle=False) + >>> regr = BaggingRegressor(estimator=SVR(), + ... n_estimators=10, random_state=0).fit(X, y) + >>> regr.predict([[0, 0, 0, 0]]) + array([-2.8720]) + """ + + def __init__( + self, + estimator=None, + n_estimators=10, + *, + max_samples=None, + max_features=1.0, + bootstrap=True, + bootstrap_features=False, + oob_score=False, + warm_start=False, + n_jobs=None, + random_state=None, + verbose=0, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + max_samples=max_samples, + max_features=max_features, + bootstrap=bootstrap, + bootstrap_features=bootstrap_features, + oob_score=oob_score, + warm_start=warm_start, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + ) + + def predict(self, X, **params): + """Predict regression target for X. + + The predicted regression target of an input sample is computed as the + mean predicted regression targets of the estimators in the ensemble. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only if + they are supported by the base estimator. + + **params : dict + Parameters routed to the `predict` method of the sub-estimators via the + metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + + Returns + ------- + y : ndarray of shape (n_samples,) + The predicted values. + """ + _raise_for_params(params, self, "predict") + + check_is_fitted(self) + # Check data + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, + ) + + if _routing_enabled(): + routed_params = process_routing(self, "predict", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(predict=Bunch()) + + # Parallel loop + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) + + all_y_hat = Parallel(n_jobs=n_jobs, verbose=self.verbose)( + delayed(_parallel_predict_regression)( + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], + X, + params=routed_params.estimator.predict, + ) + for i in range(n_jobs) + ) + + # Reduce + y_hat = sum(all_y_hat) / self.n_estimators + + return y_hat + + def _set_oob_score(self, X, y): + n_samples = y.shape[0] + + predictions = np.zeros((n_samples,)) + n_predictions = np.zeros((n_samples,)) + + for estimator, samples, features in zip( + self.estimators_, self.estimators_samples_, self.estimators_features_ + ): + # Create mask for OOB samples + mask = ~indices_to_mask(samples, n_samples) + + predictions[mask] += estimator.predict((X[mask, :])[:, features]) + n_predictions[mask] += 1 + + if (n_predictions == 0).any(): + warn( + "Some inputs do not have OOB scores. " + "This probably means too few estimators were used " + "to compute any reliable oob estimates." + ) + n_predictions[n_predictions == 0] = 1 + + predictions /= n_predictions + + self.oob_prediction_ = predictions + self.oob_score_ = r2_score(y, predictions) + + def _get_estimator(self): + """Resolve which estimator to return (default is DecisionTreeClassifier)""" + if self.estimator is None: + return DecisionTreeRegressor() + return self.estimator diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_base.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..50f8d85789280a453c1e286e7f92ccdeac153af4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_base.py @@ -0,0 +1,313 @@ +"""Base class for ensemble-based estimators.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from abc import ABCMeta, abstractmethod + +import numpy as np +from joblib import effective_n_jobs + +from sklearn.base import ( + BaseEstimator, + MetaEstimatorMixin, + clone, + is_classifier, + is_regressor, +) +from sklearn.utils import Bunch, check_random_state +from sklearn.utils._tags import get_tags +from sklearn.utils._user_interface import _print_elapsed_time +from sklearn.utils.metadata_routing import _routing_enabled +from sklearn.utils.metaestimators import _BaseComposition + + +def _fit_single_estimator( + estimator, X, y, fit_params, message_clsname=None, message=None +): + """Private function used to fit an estimator within a job.""" + # TODO(SLEP6): remove if-condition for unrouted sample_weight when metadata + # routing can't be disabled. + if not _routing_enabled() and "sample_weight" in fit_params: + try: + with _print_elapsed_time(message_clsname, message): + estimator.fit(X, y, sample_weight=fit_params["sample_weight"]) + except TypeError as exc: + if "unexpected keyword argument 'sample_weight'" in str(exc): + raise TypeError( + "Underlying estimator {} does not support sample weights.".format( + estimator.__class__.__name__ + ) + ) from exc + raise + else: + with _print_elapsed_time(message_clsname, message): + estimator.fit(X, y, **fit_params) + return estimator + + +def _set_random_states(estimator, random_state=None): + """Set fixed random_state parameters for an estimator. + + Finds all parameters ending ``random_state`` and sets them to integers + derived from ``random_state``. + + Parameters + ---------- + estimator : estimator supporting get/set_params + Estimator with potential randomness managed by random_state + parameters. + + random_state : int, RandomState instance or None, default=None + Pseudo-random number generator to control the generation of the random + integers. Pass an int for reproducible output across multiple function + calls. + See :term:`Glossary `. + + Notes + ----- + This does not necessarily set *all* ``random_state`` attributes that + control an estimator's randomness, only those accessible through + ``estimator.get_params()``. ``random_state``s not controlled include + those belonging to: + + * cross-validation splitters + * ``scipy.stats`` rvs + """ + random_state = check_random_state(random_state) + to_set = {} + for key in sorted(estimator.get_params(deep=True)): + if key == "random_state" or key.endswith("__random_state"): + to_set[key] = random_state.randint(np.iinfo(np.int32).max) + + if to_set: + estimator.set_params(**to_set) + + +class BaseEnsemble(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta): + """Base class for all ensemble classes. + + Warning: This class should not be used directly. Use derived classes + instead. + + Parameters + ---------- + estimator : object + The base estimator from which the ensemble is built. + + n_estimators : int, default=10 + The number of estimators in the ensemble. + + estimator_params : list of str, default=tuple() + The list of attributes to use as parameters when instantiating a + new base estimator. If none are given, default parameters are used. + + Attributes + ---------- + estimator_ : estimator + The base estimator from which the ensemble is grown. + + estimators_ : list of estimators + The collection of fitted base estimators. + """ + + @abstractmethod + def __init__( + self, + estimator=None, + *, + n_estimators=10, + estimator_params=tuple(), + ): + # Set parameters + self.estimator = estimator + self.n_estimators = n_estimators + self.estimator_params = estimator_params + + # Don't instantiate estimators now! Parameters of estimator might + # still change. Eg., when grid-searching with the nested object syntax. + # self.estimators_ needs to be filled by the derived classes in fit. + + def _validate_estimator(self, default=None): + """Check the base estimator. + + Sets the `estimator_` attributes. + """ + if self.estimator is not None: + self.estimator_ = self.estimator + else: + self.estimator_ = default + + def _make_estimator(self, append=True, random_state=None): + """Make and configure a copy of the `estimator_` attribute. + + Warning: This method should be used to properly instantiate new + sub-estimators. + """ + estimator = clone(self.estimator_) + estimator.set_params(**{p: getattr(self, p) for p in self.estimator_params}) + + if random_state is not None: + _set_random_states(estimator, random_state) + + if append: + self.estimators_.append(estimator) + + return estimator + + def __len__(self): + """Return the number of estimators in the ensemble.""" + return len(self.estimators_) + + def __getitem__(self, index): + """Return the index'th estimator in the ensemble.""" + return self.estimators_[index] + + def __iter__(self): + """Return iterator over estimators in the ensemble.""" + return iter(self.estimators_) + + +def _partition_estimators(n_estimators, n_jobs): + """Private function used to partition estimators between jobs.""" + # Compute the number of jobs + n_jobs = min(effective_n_jobs(n_jobs), n_estimators) + + # Partition estimators between jobs + n_estimators_per_job = np.full(n_jobs, n_estimators // n_jobs, dtype=int) + n_estimators_per_job[: n_estimators % n_jobs] += 1 + starts = np.cumsum(n_estimators_per_job) + + return n_jobs, n_estimators_per_job.tolist(), [0] + starts.tolist() + + +class _BaseHeterogeneousEnsemble( + MetaEstimatorMixin, _BaseComposition, metaclass=ABCMeta +): + """Base class for heterogeneous ensemble of learners. + + Parameters + ---------- + estimators : list of (str, estimator) tuples + The ensemble of estimators to use in the ensemble. Each element of the + list is defined as a tuple of string (i.e. name of the estimator) and + an estimator instance. An estimator can be set to `'drop'` using + `set_params`. + + Attributes + ---------- + estimators_ : list of estimators + The elements of the estimators parameter, having been fitted on the + training data. If an estimator has been set to `'drop'`, it will not + appear in `estimators_`. + """ + + @property + def named_estimators(self): + """Dictionary to access any fitted sub-estimators by name. + + Returns + ------- + :class:`~sklearn.utils.Bunch` + """ + return Bunch(**dict(self.estimators)) + + @abstractmethod + def __init__(self, estimators): + self.estimators = estimators + + def _validate_estimators(self): + if len(self.estimators) == 0 or not all( + isinstance(item, (tuple, list)) and isinstance(item[0], str) + for item in self.estimators + ): + raise ValueError( + "Invalid 'estimators' attribute, 'estimators' should be a " + "non-empty list of (string, estimator) tuples." + ) + names, estimators = zip(*self.estimators) + # defined by MetaEstimatorMixin + self._validate_names(names) + + has_estimator = any(est != "drop" for est in estimators) + if not has_estimator: + raise ValueError( + "All estimators are dropped. At least one is required " + "to be an estimator." + ) + + is_estimator_type = is_classifier if is_classifier(self) else is_regressor + + for est in estimators: + if est != "drop" and not is_estimator_type(est): + raise ValueError( + "The estimator {} should be a {}.".format( + est.__class__.__name__, is_estimator_type.__name__[3:] + ) + ) + + return names, estimators + + def set_params(self, **params): + """ + Set the parameters of an estimator from the ensemble. + + Valid parameter keys can be listed with `get_params()`. Note that you + can directly set the parameters of the estimators contained in + `estimators`. + + Parameters + ---------- + **params : keyword arguments + Specific parameters using e.g. + `set_params(parameter_name=new_value)`. In addition, to setting the + parameters of the estimator, the individual estimator of the + estimators can also be set, or can be removed by setting them to + 'drop'. + + Returns + ------- + self : object + Estimator instance. + """ + super()._set_params("estimators", **params) + return self + + def get_params(self, deep=True): + """ + Get the parameters of an estimator from the ensemble. + + Returns the parameters given in the constructor as well as the + estimators contained within the `estimators` parameter. + + Parameters + ---------- + deep : bool, default=True + Setting it to True gets the various estimators and the parameters + of the estimators as well. + + Returns + ------- + params : dict + Parameter and estimator names mapped to their values or parameter + names mapped to their values. + """ + return super()._get_params("estimators", deep=deep) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + try: + tags.input_tags.allow_nan = all( + get_tags(est[1]).input_tags.allow_nan if est[1] != "drop" else True + for est in self.estimators + ) + tags.input_tags.sparse = all( + get_tags(est[1]).input_tags.sparse if est[1] != "drop" else True + for est in self.estimators + ) + except Exception: + # If `estimators` does not comply with our API (list of tuples) then it will + # fail. In this case, we assume that `allow_nan` and `sparse` are False but + # the parameter validation will raise an error during `fit`. + pass # pragma: no cover + return tags diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_forest.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_forest.py new file mode 100644 index 0000000000000000000000000000000000000000..2231eb12e81f4039837dc82f81cf98567939485c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_forest.py @@ -0,0 +1,3047 @@ +""" +Forest of trees-based ensemble methods. + +Those methods include random forests and extremely randomized trees. + +The module structure is the following: + +- The ``BaseForest`` base class implements a common ``fit`` method for all + the estimators in the module. The ``fit`` method of the base ``Forest`` + class calls the ``fit`` method of each sub-estimator on random samples + (with replacement, a.k.a. bootstrap) of the training set. + + The init of the sub-estimator is further delegated to the + ``BaseEnsemble`` constructor. + +- The ``ForestClassifier`` and ``ForestRegressor`` base classes further + implement the prediction logic by computing an average of the predicted + outcomes of the sub-estimators. + +- The ``RandomForestClassifier`` and ``RandomForestRegressor`` derived + classes provide the user with concrete implementations of + the forest ensemble method using classical, deterministic + ``DecisionTreeClassifier`` and ``DecisionTreeRegressor`` as + sub-estimator implementations. + +- The ``ExtraTreesClassifier`` and ``ExtraTreesRegressor`` derived + classes provide the user with concrete implementations of the + forest ensemble method using the extremely randomized trees + ``ExtraTreeClassifier`` and ``ExtraTreeRegressor`` as + sub-estimator implementations. + +Single and multi-output problems are both handled. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import threading +from abc import ABCMeta, abstractmethod +from numbers import Integral, Real +from warnings import catch_warnings, simplefilter, warn + +import numpy as np +from scipy.sparse import hstack as sparse_hstack +from scipy.sparse import issparse + +from sklearn.base import ( + ClassifierMixin, + MultiOutputMixin, + RegressorMixin, + TransformerMixin, + _fit_context, + is_classifier, +) +from sklearn.ensemble._base import BaseEnsemble, _partition_estimators +from sklearn.exceptions import DataConversionWarning +from sklearn.metrics import accuracy_score, r2_score +from sklearn.preprocessing import OneHotEncoder +from sklearn.tree import ( + BaseDecisionTree, + DecisionTreeClassifier, + DecisionTreeRegressor, + ExtraTreeClassifier, + ExtraTreeRegressor, +) +from sklearn.tree._tree import DOUBLE, DTYPE +from sklearn.utils import check_random_state, compute_sample_weight +from sklearn.utils._param_validation import Interval, RealNotInt, StrOptions +from sklearn.utils._tags import get_tags +from sklearn.utils.multiclass import check_classification_targets, type_of_target +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.validation import ( + _check_feature_names_in, + _check_sample_weight, + _num_samples, + check_is_fitted, + validate_data, +) + +__all__ = [ + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", +] + +MAX_INT = np.iinfo(np.int32).max + + +def _get_n_samples_bootstrap(n_samples, max_samples): + """ + Get the number of samples in a bootstrap sample. + + Parameters + ---------- + n_samples : int + Number of samples in the dataset. + max_samples : int or float + The maximum number of samples to draw from the total available: + - if float, this indicates a fraction of the total and should be + the interval `(0.0, 1.0]`; + - if int, this indicates the exact number of samples; + - if None, this indicates the total number of samples. + + Returns + ------- + n_samples_bootstrap : int + The total number of samples to draw for the bootstrap sample. + """ + if max_samples is None: + return n_samples + + if isinstance(max_samples, Integral): + if max_samples > n_samples: + msg = "`max_samples` must be <= n_samples={} but got value {}" + raise ValueError(msg.format(n_samples, max_samples)) + return max_samples + + if isinstance(max_samples, Real): + return max(round(n_samples * max_samples), 1) + + +def _generate_sample_indices(random_state, n_samples, n_samples_bootstrap): + """ + Private function used to _parallel_build_trees function.""" + + random_instance = check_random_state(random_state) + sample_indices = random_instance.randint( + 0, n_samples, n_samples_bootstrap, dtype=np.int32 + ) + + return sample_indices + + +def _generate_unsampled_indices(random_state, n_samples, n_samples_bootstrap): + """ + Private function used to forest._set_oob_score function.""" + sample_indices = _generate_sample_indices( + random_state, n_samples, n_samples_bootstrap + ) + sample_counts = np.bincount(sample_indices, minlength=n_samples) + unsampled_mask = sample_counts == 0 + indices_range = np.arange(n_samples) + unsampled_indices = indices_range[unsampled_mask] + + return unsampled_indices + + +def _parallel_build_trees( + tree, + bootstrap, + X, + y, + sample_weight, + tree_idx, + n_trees, + verbose=0, + class_weight=None, + n_samples_bootstrap=None, + missing_values_in_feature_mask=None, +): + """ + Private function used to fit a single tree in parallel.""" + if verbose > 1: + print("building tree %d of %d" % (tree_idx + 1, n_trees)) + + if bootstrap: + n_samples = X.shape[0] + if sample_weight is None: + curr_sample_weight = np.ones((n_samples,), dtype=np.float64) + else: + curr_sample_weight = sample_weight.copy() + + indices = _generate_sample_indices( + tree.random_state, n_samples, n_samples_bootstrap + ) + sample_counts = np.bincount(indices, minlength=n_samples) + curr_sample_weight *= sample_counts + + if class_weight == "subsample": + with catch_warnings(): + simplefilter("ignore", DeprecationWarning) + curr_sample_weight *= compute_sample_weight("auto", y, indices=indices) + elif class_weight == "balanced_subsample": + curr_sample_weight *= compute_sample_weight("balanced", y, indices=indices) + + tree._fit( + X, + y, + sample_weight=curr_sample_weight, + check_input=False, + missing_values_in_feature_mask=missing_values_in_feature_mask, + ) + else: + tree._fit( + X, + y, + sample_weight=sample_weight, + check_input=False, + missing_values_in_feature_mask=missing_values_in_feature_mask, + ) + + return tree + + +class BaseForest(MultiOutputMixin, BaseEnsemble, metaclass=ABCMeta): + """ + Base class for forests of trees. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + _parameter_constraints: dict = { + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "bootstrap": ["boolean"], + "oob_score": ["boolean", callable], + "n_jobs": [Integral, None], + "random_state": ["random_state"], + "verbose": ["verbose"], + "warm_start": ["boolean"], + "max_samples": [ + None, + Interval(RealNotInt, 0.0, 1.0, closed="right"), + Interval(Integral, 1, None, closed="left"), + ], + } + + @abstractmethod + def __init__( + self, + estimator, + n_estimators=100, + *, + estimator_params=tuple(), + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + max_samples=None, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + estimator_params=estimator_params, + ) + + self.bootstrap = bootstrap + self.oob_score = oob_score + self.n_jobs = n_jobs + self.random_state = random_state + self.verbose = verbose + self.warm_start = warm_start + self.class_weight = class_weight + self.max_samples = max_samples + + def apply(self, X): + """ + Apply trees in the forest to X, return leaf indices. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + + Returns + ------- + X_leaves : ndarray of shape (n_samples, n_estimators) + For each datapoint x in X and for each tree in the forest, + return the index of the leaf x ends up in. + """ + X = self._validate_X_predict(X) + results = Parallel( + n_jobs=self.n_jobs, + verbose=self.verbose, + prefer="threads", + )(delayed(tree.apply)(X, check_input=False) for tree in self.estimators_) + + return np.array(results).T + + def decision_path(self, X): + """ + Return the decision path in the forest. + + .. versionadded:: 0.18 + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + + Returns + ------- + indicator : sparse matrix of shape (n_samples, n_nodes) + Return a node indicator matrix where non zero elements indicates + that the samples goes through the nodes. The matrix is of CSR + format. + + n_nodes_ptr : ndarray of shape (n_estimators + 1,) + The columns from indicator[n_nodes_ptr[i]:n_nodes_ptr[i+1]] + gives the indicator value for the i-th estimator. + """ + X = self._validate_X_predict(X) + indicators = Parallel( + n_jobs=self.n_jobs, + verbose=self.verbose, + prefer="threads", + )( + delayed(tree.decision_path)(X, check_input=False) + for tree in self.estimators_ + ) + + n_nodes = [0] + n_nodes.extend([i.shape[1] for i in indicators]) + n_nodes_ptr = np.array(n_nodes).cumsum() + + return sparse_hstack(indicators).tocsr(), n_nodes_ptr + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y, sample_weight=None): + """ + Build a forest of trees from the training set (X, y). + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Internally, its dtype will be converted + to ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csc_matrix``. + + y : array-like of shape (n_samples,) or (n_samples, n_outputs) + The target values (class labels in classification, real numbers in + regression). + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, then samples are equally weighted. Splits + that would create child nodes with net zero or negative weight are + ignored while searching for a split in each node. In the case of + classification, splits are also ignored if they would result in any + single class carrying a negative weight in either child node. + + Returns + ------- + self : object + Fitted estimator. + """ + # Validate or convert input data + if issparse(y): + raise ValueError("sparse multilabel-indicator for y is not supported.") + + X, y = validate_data( + self, + X, + y, + multi_output=True, + accept_sparse="csc", + dtype=DTYPE, + ensure_all_finite=False, + ) + # _compute_missing_values_in_feature_mask checks if X has missing values and + # will raise an error if the underlying tree base estimator can't handle missing + # values. Only the criterion is required to determine if the tree supports + # missing values. + estimator = type(self.estimator)(criterion=self.criterion) + missing_values_in_feature_mask = ( + estimator._compute_missing_values_in_feature_mask( + X, estimator_name=self.__class__.__name__ + ) + ) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) + + if issparse(X): + # Pre-sort indices to avoid that each individual tree of the + # ensemble sorts the indices. + X.sort_indices() + + y = np.atleast_1d(y) + if y.ndim == 2 and y.shape[1] == 1: + warn( + ( + "A column-vector y was passed when a 1d array was" + " expected. Please change the shape of y to " + "(n_samples,), for example using ravel()." + ), + DataConversionWarning, + stacklevel=2, + ) + + if y.ndim == 1: + # reshape is necessary to preserve the data contiguity against vs + # [:, np.newaxis] that does not. + y = np.reshape(y, (-1, 1)) + + if self.criterion == "poisson": + if np.any(y < 0): + raise ValueError( + "Some value(s) of y are negative which is " + "not allowed for Poisson regression." + ) + if np.sum(y) <= 0: + raise ValueError( + "Sum of y is not strictly positive which " + "is necessary for Poisson regression." + ) + + self._n_samples, self.n_outputs_ = y.shape + + y, expanded_class_weight = self._validate_y_class_weight(y) + + if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous: + y = np.ascontiguousarray(y, dtype=DOUBLE) + + if expanded_class_weight is not None: + if sample_weight is not None: + sample_weight = sample_weight * expanded_class_weight + else: + sample_weight = expanded_class_weight + + if not self.bootstrap and self.max_samples is not None: + raise ValueError( + "`max_sample` cannot be set if `bootstrap=False`. " + "Either switch to `bootstrap=True` or set " + "`max_sample=None`." + ) + elif self.bootstrap: + n_samples_bootstrap = _get_n_samples_bootstrap( + n_samples=X.shape[0], max_samples=self.max_samples + ) + else: + n_samples_bootstrap = None + + self._n_samples_bootstrap = n_samples_bootstrap + + self._validate_estimator() + + if not self.bootstrap and self.oob_score: + raise ValueError("Out of bag estimation only available if bootstrap=True") + + random_state = check_random_state(self.random_state) + + if not self.warm_start or not hasattr(self, "estimators_"): + # Free allocated memory, if any + self.estimators_ = [] + + n_more_estimators = self.n_estimators - len(self.estimators_) + + if n_more_estimators < 0: + raise ValueError( + "n_estimators=%d must be larger or equal to " + "len(estimators_)=%d when warm_start==True" + % (self.n_estimators, len(self.estimators_)) + ) + + elif n_more_estimators == 0: + warn( + "Warm-start fitting without increasing n_estimators does not " + "fit new trees." + ) + else: + if self.warm_start and len(self.estimators_) > 0: + # We draw from the random state to get the random state we + # would have got if we hadn't used a warm_start. + random_state.randint(MAX_INT, size=len(self.estimators_)) + + trees = [ + self._make_estimator(append=False, random_state=random_state) + for i in range(n_more_estimators) + ] + + # Parallel loop: we prefer the threading backend as the Cython code + # for fitting the trees is internally releasing the Python GIL + # making threading more efficient than multiprocessing in + # that case. However, for joblib 0.12+ we respect any + # parallel_backend contexts set at a higher level, + # since correctness does not rely on using threads. + trees = Parallel( + n_jobs=self.n_jobs, + verbose=self.verbose, + prefer="threads", + )( + delayed(_parallel_build_trees)( + t, + self.bootstrap, + X, + y, + sample_weight, + i, + len(trees), + verbose=self.verbose, + class_weight=self.class_weight, + n_samples_bootstrap=n_samples_bootstrap, + missing_values_in_feature_mask=missing_values_in_feature_mask, + ) + for i, t in enumerate(trees) + ) + + # Collect newly grown trees + self.estimators_.extend(trees) + + if self.oob_score and ( + n_more_estimators > 0 or not hasattr(self, "oob_score_") + ): + y_type = type_of_target(y) + if y_type == "unknown" or ( + is_classifier(self) and y_type == "multiclass-multioutput" + ): + # FIXME: we could consider to support multiclass-multioutput if + # we introduce or reuse a constructor parameter (e.g. + # oob_score) allowing our user to pass a callable defining the + # scoring strategy on OOB sample. + raise ValueError( + "The type of target cannot be used to compute OOB " + f"estimates. Got {y_type} while only the following are " + "supported: continuous, continuous-multioutput, binary, " + "multiclass, multilabel-indicator." + ) + + if callable(self.oob_score): + self._set_oob_score_and_attributes( + X, y, scoring_function=self.oob_score + ) + else: + self._set_oob_score_and_attributes(X, y) + + # Decapsulate classes_ attributes + if hasattr(self, "classes_") and self.n_outputs_ == 1: + self.n_classes_ = self.n_classes_[0] + self.classes_ = self.classes_[0] + + return self + + @abstractmethod + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): + """Compute and set the OOB score and attributes. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data matrix. + y : ndarray of shape (n_samples, n_outputs) + The target matrix. + scoring_function : callable, default=None + Scoring function for OOB score. Default depends on whether + this is a regression (R2 score) or classification problem + (accuracy score). + """ + + def _compute_oob_predictions(self, X, y): + """Compute and set the OOB score. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data matrix. + y : ndarray of shape (n_samples, n_outputs) + The target matrix. + + Returns + ------- + oob_pred : ndarray of shape (n_samples, n_classes, n_outputs) or \ + (n_samples, 1, n_outputs) + The OOB predictions. + """ + # Prediction requires X to be in CSR format + if issparse(X): + X = X.tocsr() + + n_samples = y.shape[0] + n_outputs = self.n_outputs_ + if is_classifier(self) and hasattr(self, "n_classes_"): + # n_classes_ is an ndarray at this stage + # all the supported type of target will have the same number of + # classes in all outputs + oob_pred_shape = (n_samples, self.n_classes_[0], n_outputs) + else: + # for regression, n_classes_ does not exist and we create an empty + # axis to be consistent with the classification case and make + # the array operations compatible with the 2 settings + oob_pred_shape = (n_samples, 1, n_outputs) + + oob_pred = np.zeros(shape=oob_pred_shape, dtype=np.float64) + n_oob_pred = np.zeros((n_samples, n_outputs), dtype=np.int64) + + n_samples_bootstrap = _get_n_samples_bootstrap( + n_samples, + self.max_samples, + ) + for estimator in self.estimators_: + unsampled_indices = _generate_unsampled_indices( + estimator.random_state, + n_samples, + n_samples_bootstrap, + ) + + y_pred = self._get_oob_predictions(estimator, X[unsampled_indices, :]) + oob_pred[unsampled_indices, ...] += y_pred + n_oob_pred[unsampled_indices, :] += 1 + + for k in range(n_outputs): + if (n_oob_pred == 0).any(): + warn( + ( + "Some inputs do not have OOB scores. This probably means " + "too few trees were used to compute any reliable OOB " + "estimates." + ), + UserWarning, + ) + n_oob_pred[n_oob_pred == 0] = 1 + oob_pred[..., k] /= n_oob_pred[..., [k]] + + return oob_pred + + def _validate_y_class_weight(self, y): + # Default implementation + return y, None + + def _validate_X_predict(self, X): + """ + Validate X whenever one tries to predict, apply, predict_proba.""" + check_is_fitted(self) + if self.estimators_[0]._support_missing_values(X): + ensure_all_finite = "allow-nan" + else: + ensure_all_finite = True + + X = validate_data( + self, + X, + dtype=DTYPE, + accept_sparse="csr", + reset=False, + ensure_all_finite=ensure_all_finite, + ) + if issparse(X) and (X.indices.dtype != np.intc or X.indptr.dtype != np.intc): + raise ValueError("No support for np.int64 index based sparse matrices") + return X + + @property + def feature_importances_(self): + """ + The impurity-based feature importances. + + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + Returns + ------- + feature_importances_ : ndarray of shape (n_features,) + The values of this array sum to 1, unless all trees are single node + trees consisting of only the root node, in which case it will be an + array of zeros. + """ + check_is_fitted(self) + + all_importances = Parallel(n_jobs=self.n_jobs, prefer="threads")( + delayed(getattr)(tree, "feature_importances_") + for tree in self.estimators_ + if tree.tree_.node_count > 1 + ) + + if not all_importances: + return np.zeros(self.n_features_in_, dtype=np.float64) + + all_importances = np.mean(all_importances, axis=0, dtype=np.float64) + return all_importances / np.sum(all_importances) + + def _get_estimators_indices(self): + # Get drawn indices along both sample and feature axes + for tree in self.estimators_: + if not self.bootstrap: + yield np.arange(self._n_samples, dtype=np.int32) + else: + # tree.random_state is actually an immutable integer seed rather + # than a mutable RandomState instance, so it's safe to use it + # repeatedly when calling this property. + seed = tree.random_state + # Operations accessing random_state must be performed identically + # to those in `_parallel_build_trees()` + yield _generate_sample_indices( + seed, self._n_samples, self._n_samples_bootstrap + ) + + @property + def estimators_samples_(self): + """The subset of drawn samples for each base estimator. + + Returns a dynamically generated list of indices identifying + the samples used for fitting each member of the ensemble, i.e., + the in-bag samples. + + Note: the list is re-created at each call to the property in order + to reduce the object memory footprint by not storing the sampling + data. Thus fetching the property may be slower than expected. + """ + return [sample_indices for sample_indices in self._get_estimators_indices()] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + # Only the criterion is required to determine if the tree supports + # missing values + estimator = type(self.estimator)(criterion=self.criterion) + tags.input_tags.allow_nan = get_tags(estimator).input_tags.allow_nan + return tags + + +def _accumulate_prediction(predict, X, out, lock): + """ + This is a utility function for joblib's Parallel. + + It can't go locally in ForestClassifier or ForestRegressor, because joblib + complains that it cannot pickle it when placed there. + """ + prediction = predict(X, check_input=False) + with lock: + if len(out) == 1: + out[0] += prediction + else: + for i in range(len(out)): + out[i] += prediction[i] + + +class ForestClassifier(ClassifierMixin, BaseForest, metaclass=ABCMeta): + """ + Base class for forest of trees-based classifiers. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + @abstractmethod + def __init__( + self, + estimator, + n_estimators=100, + *, + estimator_params=tuple(), + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + max_samples=None, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + estimator_params=estimator_params, + bootstrap=bootstrap, + oob_score=oob_score, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + warm_start=warm_start, + class_weight=class_weight, + max_samples=max_samples, + ) + + @staticmethod + def _get_oob_predictions(tree, X): + """Compute the OOB predictions for an individual tree. + + Parameters + ---------- + tree : DecisionTreeClassifier object + A single decision tree classifier. + X : ndarray of shape (n_samples, n_features) + The OOB samples. + + Returns + ------- + y_pred : ndarray of shape (n_samples, n_classes, n_outputs) + The OOB associated predictions. + """ + y_pred = tree.predict_proba(X, check_input=False) + y_pred = np.asarray(y_pred) + if y_pred.ndim == 2: + # binary and multiclass + y_pred = y_pred[..., np.newaxis] + else: + # Roll the first `n_outputs` axis to the last axis. We will reshape + # from a shape of (n_outputs, n_samples, n_classes) to a shape of + # (n_samples, n_classes, n_outputs). + y_pred = np.rollaxis(y_pred, axis=0, start=3) + return y_pred + + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): + """Compute and set the OOB score and attributes. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data matrix. + y : ndarray of shape (n_samples, n_outputs) + The target matrix. + scoring_function : callable, default=None + Scoring function for OOB score. Defaults to `accuracy_score`. + """ + self.oob_decision_function_ = super()._compute_oob_predictions(X, y) + if self.oob_decision_function_.shape[-1] == 1: + # drop the n_outputs axis if there is a single output + self.oob_decision_function_ = self.oob_decision_function_.squeeze(axis=-1) + + if scoring_function is None: + scoring_function = accuracy_score + + self.oob_score_ = scoring_function( + y, np.argmax(self.oob_decision_function_, axis=1) + ) + + def _validate_y_class_weight(self, y): + check_classification_targets(y) + + y = np.copy(y) + expanded_class_weight = None + + if self.class_weight is not None: + y_original = np.copy(y) + + self.classes_ = [] + self.n_classes_ = [] + + y_store_unique_indices = np.zeros(y.shape, dtype=int) + for k in range(self.n_outputs_): + classes_k, y_store_unique_indices[:, k] = np.unique( + y[:, k], return_inverse=True + ) + self.classes_.append(classes_k) + self.n_classes_.append(classes_k.shape[0]) + y = y_store_unique_indices + + if self.class_weight is not None: + valid_presets = ("balanced", "balanced_subsample") + if isinstance(self.class_weight, str): + if self.class_weight not in valid_presets: + raise ValueError( + "Valid presets for class_weight include " + '"balanced" and "balanced_subsample".' + 'Given "%s".' % self.class_weight + ) + if self.warm_start: + warn( + 'class_weight presets "balanced" or ' + '"balanced_subsample" are ' + "not recommended for warm_start if the fitted data " + "differs from the full dataset. In order to use " + '"balanced" weights, use compute_class_weight ' + '("balanced", classes, y). In place of y you can use ' + "a large enough sample of the full training set " + "target to properly estimate the class frequency " + "distributions. Pass the resulting weights as the " + "class_weight parameter." + ) + + if self.class_weight != "balanced_subsample" or not self.bootstrap: + if self.class_weight == "balanced_subsample": + class_weight = "balanced" + else: + class_weight = self.class_weight + expanded_class_weight = compute_sample_weight(class_weight, y_original) + + return y, expanded_class_weight + + def predict(self, X): + """ + Predict class for X. + + The predicted class of an input sample is a vote by the trees in + the forest, weighted by their probability estimates. That is, + the predicted class is the one with highest mean probability + estimate across the trees. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + + Returns + ------- + y : ndarray of shape (n_samples,) or (n_samples, n_outputs) + The predicted classes. + """ + proba = self.predict_proba(X) + + if self.n_outputs_ == 1: + return self.classes_.take(np.argmax(proba, axis=1), axis=0) + + else: + n_samples = proba[0].shape[0] + # all dtypes should be the same, so just take the first + class_type = self.classes_[0].dtype + predictions = np.empty((n_samples, self.n_outputs_), dtype=class_type) + + for k in range(self.n_outputs_): + predictions[:, k] = self.classes_[k].take( + np.argmax(proba[k], axis=1), axis=0 + ) + + return predictions + + def predict_proba(self, X): + """ + Predict class probabilities for X. + + The predicted class probabilities of an input sample are computed as + the mean predicted class probabilities of the trees in the forest. + The class probability of a single tree is the fraction of samples of + the same class in a leaf. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes), or a list of such arrays + The class probabilities of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + """ + check_is_fitted(self) + # Check data + X = self._validate_X_predict(X) + + # Assign chunk of trees to jobs + n_jobs, _, _ = _partition_estimators(self.n_estimators, self.n_jobs) + + # avoid storing the output of every estimator by summing them here + all_proba = [ + np.zeros((X.shape[0], j), dtype=np.float64) + for j in np.atleast_1d(self.n_classes_) + ] + lock = threading.Lock() + Parallel(n_jobs=n_jobs, verbose=self.verbose, require="sharedmem")( + delayed(_accumulate_prediction)(e.predict_proba, X, all_proba, lock) + for e in self.estimators_ + ) + + for proba in all_proba: + proba /= len(self.estimators_) + + if len(all_proba) == 1: + return all_proba[0] + else: + return all_proba + + def predict_log_proba(self, X): + """ + Predict class log-probabilities for X. + + The predicted class log-probabilities of an input sample is computed as + the log of the mean predicted class probabilities of the trees in the + forest. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes), or a list of such arrays + The class probabilities of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + """ + proba = self.predict_proba(X) + + if self.n_outputs_ == 1: + return np.log(proba) + + else: + for k in range(self.n_outputs_): + proba[k] = np.log(proba[k]) + + return proba + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.classifier_tags.multi_label = True + tags.input_tags.sparse = True + return tags + + +class ForestRegressor(RegressorMixin, BaseForest, metaclass=ABCMeta): + """ + Base class for forest of trees-based regressors. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + @abstractmethod + def __init__( + self, + estimator, + n_estimators=100, + *, + estimator_params=tuple(), + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + max_samples=None, + ): + super().__init__( + estimator, + n_estimators=n_estimators, + estimator_params=estimator_params, + bootstrap=bootstrap, + oob_score=oob_score, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + warm_start=warm_start, + max_samples=max_samples, + ) + + def predict(self, X): + """ + Predict regression target for X. + + The predicted regression target of an input sample is computed as the + mean predicted regression targets of the trees in the forest. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + + Returns + ------- + y : ndarray of shape (n_samples,) or (n_samples, n_outputs) + The predicted values. + """ + check_is_fitted(self) + # Check data + X = self._validate_X_predict(X) + + # Assign chunk of trees to jobs + n_jobs, _, _ = _partition_estimators(self.n_estimators, self.n_jobs) + + # avoid storing the output of every estimator by summing them here + if self.n_outputs_ > 1: + y_hat = np.zeros((X.shape[0], self.n_outputs_), dtype=np.float64) + else: + y_hat = np.zeros((X.shape[0]), dtype=np.float64) + + # Parallel loop + lock = threading.Lock() + Parallel(n_jobs=n_jobs, verbose=self.verbose, require="sharedmem")( + delayed(_accumulate_prediction)(e.predict, X, [y_hat], lock) + for e in self.estimators_ + ) + + y_hat /= len(self.estimators_) + + return y_hat + + @staticmethod + def _get_oob_predictions(tree, X): + """Compute the OOB predictions for an individual tree. + + Parameters + ---------- + tree : DecisionTreeRegressor object + A single decision tree regressor. + X : ndarray of shape (n_samples, n_features) + The OOB samples. + + Returns + ------- + y_pred : ndarray of shape (n_samples, 1, n_outputs) + The OOB associated predictions. + """ + y_pred = tree.predict(X, check_input=False) + if y_pred.ndim == 1: + # single output regression + y_pred = y_pred[:, np.newaxis, np.newaxis] + else: + # multioutput regression + y_pred = y_pred[:, np.newaxis, :] + return y_pred + + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): + """Compute and set the OOB score and attributes. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data matrix. + y : ndarray of shape (n_samples, n_outputs) + The target matrix. + scoring_function : callable, default=None + Scoring function for OOB score. Defaults to `r2_score`. + """ + self.oob_prediction_ = super()._compute_oob_predictions(X, y).squeeze(axis=1) + if self.oob_prediction_.shape[-1] == 1: + # drop the n_outputs axis if there is a single output + self.oob_prediction_ = self.oob_prediction_.squeeze(axis=-1) + + if scoring_function is None: + scoring_function = r2_score + + self.oob_score_ = scoring_function(y, self.oob_prediction_) + + def _compute_partial_dependence_recursion(self, grid, target_features): + """Fast partial dependence computation. + + Parameters + ---------- + grid : ndarray of shape (n_samples, n_target_features), dtype=DTYPE + The grid points on which the partial dependence should be + evaluated. + target_features : ndarray of shape (n_target_features), dtype=np.intp + The set of target features for which the partial dependence + should be evaluated. + + Returns + ------- + averaged_predictions : ndarray of shape (n_samples,) + The value of the partial dependence function on each grid point. + """ + grid = np.asarray(grid, dtype=DTYPE, order="C") + target_features = np.asarray(target_features, dtype=np.intp, order="C") + averaged_predictions = np.zeros( + shape=grid.shape[0], dtype=np.float64, order="C" + ) + + for tree in self.estimators_: + # Note: we don't sum in parallel because the GIL isn't released in + # the fast method. + tree.tree_.compute_partial_dependence( + grid, target_features, averaged_predictions + ) + # Average over the forest + averaged_predictions /= len(self.estimators_) + + return averaged_predictions + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags + + +class RandomForestClassifier(ForestClassifier): + """ + A random forest classifier. + + A random forest is a meta estimator that fits a number of decision tree + classifiers on various sub-samples of the dataset and uses averaging to + improve the predictive accuracy and control over-fitting. + Trees in the forest use the best split strategy, i.e. equivalent to passing + `splitter="best"` to the underlying :class:`~sklearn.tree.DecisionTreeClassifier`. + The sub-sample size is controlled with the `max_samples` parameter if + `bootstrap=True` (default), otherwise the whole dataset is used to build + each tree. + + For a comparison between tree-based ensemble models see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py`. + + This estimator has native support for missing values (NaNs). During training, + the tree grower learns at each split point whether samples with missing values + should go to the left or right child, based on the potential gain. When predicting, + samples with missing values are assigned to the left or right child consequently. + If no missing values were encountered for a given feature during training, then + samples with missing values are mapped to whichever child has the most samples. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_estimators : int, default=100 + The number of trees in the forest. + + .. versionchanged:: 0.22 + The default value of ``n_estimators`` changed from 10 to 100 + in 0.22. + + criterion : {"gini", "entropy", "log_loss"}, default="gini" + The function to measure the quality of a split. Supported criteria are + "gini" for the Gini impurity and "log_loss" and "entropy" both for the + Shannon information gain, see :ref:`tree_mathematical_formulation`. + Note: This parameter is tree-specific. + + max_depth : int, default=None + The maximum depth of the tree. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + + min_samples_split : int or float, default=2 + The minimum number of samples required to split an internal node: + + - If int, then consider `min_samples_split` as the minimum number. + - If float, then `min_samples_split` is a fraction and + `ceil(min_samples_split * n_samples)` are the minimum + number of samples for each split. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_samples_leaf : int or float, default=1 + The minimum number of samples required to be at a leaf node. + A split point at any depth will only be considered if it leaves at + least ``min_samples_leaf`` training samples in each of the left and + right branches. This may have the effect of smoothing the model, + especially in regression. + + - If int, then consider `min_samples_leaf` as the minimum number. + - If float, then `min_samples_leaf` is a fraction and + `ceil(min_samples_leaf * n_samples)` are the minimum + number of samples for each node. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_weight_fraction_leaf : float, default=0.0 + The minimum weighted fraction of the sum total of weights (of all + the input samples) required to be at a leaf node. Samples have + equal weight when sample_weight is not provided. + + max_features : {"sqrt", "log2", None}, int or float, default="sqrt" + The number of features to consider when looking for the best split: + + - If int, then consider `max_features` features at each split. + - If float, then `max_features` is a fraction and + `max(1, int(max_features * n_features_in_))` features are considered at each + split. + - If "sqrt", then `max_features=sqrt(n_features)`. + - If "log2", then `max_features=log2(n_features)`. + - If None, then `max_features=n_features`. + + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to `"sqrt"`. + + Note: the search for a split does not stop until at least one + valid partition of the node samples is found, even if it requires to + effectively inspect more than ``max_features`` features. + + max_leaf_nodes : int, default=None + Grow trees with ``max_leaf_nodes`` in best-first fashion. + Best nodes are defined as relative reduction in impurity. + If None then unlimited number of leaf nodes. + + min_impurity_decrease : float, default=0.0 + A node will be split if this split induces a decrease of the impurity + greater than or equal to this value. + + The weighted impurity decrease equation is the following:: + + N_t / N * (impurity - N_t_R / N_t * right_impurity + - N_t_L / N_t * left_impurity) + + where ``N`` is the total number of samples, ``N_t`` is the number of + samples at the current node, ``N_t_L`` is the number of samples in the + left child, and ``N_t_R`` is the number of samples in the right child. + + ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, + if ``sample_weight`` is passed. + + .. versionadded:: 0.19 + + bootstrap : bool, default=True + Whether bootstrap samples are used when building trees. If False, the + whole dataset is used to build each tree. + + oob_score : bool or callable, default=False + Whether to use out-of-bag samples to estimate the generalization score. + By default, :func:`~sklearn.metrics.accuracy_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. + + For an illustration of out-of-bag (OOB) error estimation, see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_ensemble_oob.py`. + + n_jobs : int, default=None + The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, + :meth:`decision_path` and :meth:`apply` are all parallelized over the + trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` + context. ``-1`` means using all processors. See :term:`Glossary + ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls both the randomness of the bootstrapping of the samples used + when building trees (if ``bootstrap=True``) and the sampling of the + features to consider when looking for the best split at each node + (if ``max_features < n_features``). + See :term:`Glossary ` for details. + + verbose : int, default=0 + Controls the verbosity when fitting and predicting. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit a whole + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. + + class_weight : {"balanced", "balanced_subsample"}, dict or list of dicts, \ + default=None + Weights associated with classes in the form ``{class_label: weight}``. + If not given, all classes are supposed to have weight one. For + multi-output problems, a list of dicts can be provided in the same + order as the columns of y. + + Note that for multioutput (including multilabel) weights should be + defined for each class of every column in its own dict. For example, + for four-class multilabel classification weights should be + [{0: 1, 1: 1}, {0: 1, 1: 5}, {0: 1, 1: 1}, {0: 1, 1: 1}] instead of + [{1:1}, {2:5}, {3:1}, {4:1}]. + + The "balanced" mode uses the values of y to automatically adjust + weights inversely proportional to class frequencies in the input data + as ``n_samples / (n_classes * np.bincount(y))`` + + The "balanced_subsample" mode is the same as "balanced" except that + weights are computed based on the bootstrap sample for every tree + grown. + + For multi-output, the weights of each column of y will be multiplied. + + Note that these weights will be multiplied with sample_weight (passed + through the fit method) if sample_weight is specified. + + ccp_alpha : non-negative float, default=0.0 + Complexity parameter used for Minimal Cost-Complexity Pruning. The + subtree with the largest cost complexity that is smaller than + ``ccp_alpha`` will be chosen. By default, no pruning is performed. See + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. + + .. versionadded:: 0.22 + + max_samples : int or float, default=None + If bootstrap is True, the number of samples to draw from X + to train each base estimator. + + - If None (default), then draw `X.shape[0]` samples. + - If int, then draw `max_samples` samples. + - If float, then draw `max(round(n_samples * max_samples), 1)` samples. Thus, + `max_samples` should be in the interval `(0.0, 1.0]`. + + .. versionadded:: 0.22 + + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multiclass classifications (i.e. when `n_classes > 2`), + - multioutput classifications (i.e. when `n_outputs_ > 1`), + - classifications trained on data with missing values. + + The constraints hold over the probability of the positive class. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + + Attributes + ---------- + estimator_ : :class:`~sklearn.tree.DecisionTreeClassifier` + The child estimator template used to create the collection of fitted + sub-estimators. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of DecisionTreeClassifier + The collection of fitted sub-estimators. + + classes_ : ndarray of shape (n_classes,) or a list of such arrays + The classes labels (single output problem), or a list of arrays of + class labels (multi-output problem). + + n_classes_ : int or list + The number of classes (single output problem), or a list containing the + number of classes for each output (multi-output problem). + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_outputs_ : int + The number of outputs when ``fit`` is performed. + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + oob_score_ : float + Score of the training dataset obtained using an out-of-bag estimate. + This attribute exists only when ``oob_score`` is True. + + oob_decision_function_ : ndarray of shape (n_samples, n_classes) or \ + (n_samples, n_classes, n_outputs) + Decision function computed with out-of-bag estimate on the training + set. If n_estimators is small it might be possible that a data point + was never left out during the bootstrap. In this case, + `oob_decision_function_` might contain NaN. This attribute exists + only when ``oob_score`` is True. + + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + + See Also + -------- + sklearn.tree.DecisionTreeClassifier : A decision tree classifier. + sklearn.ensemble.ExtraTreesClassifier : Ensemble of extremely randomized + tree classifiers. + sklearn.ensemble.HistGradientBoostingClassifier : A Histogram-based Gradient + Boosting Classification Tree, very fast for big datasets (n_samples >= + 10_000). + + Notes + ----- + The default values for the parameters controlling the size of the trees + (e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and + unpruned trees which can potentially be very large on some data sets. To + reduce memory consumption, the complexity and size of the trees should be + controlled by setting those parameter values. + + The features are always randomly permuted at each split. Therefore, + the best found split may vary, even with the same training data, + ``max_features=n_features`` and ``bootstrap=False``, if the improvement + of the criterion is identical for several splits enumerated during the + search of the best split. To obtain a deterministic behaviour during + fitting, ``random_state`` has to be fixed. + + References + ---------- + .. [1] :doi:`L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, 2001. + <10.1023/A:1010933404324>` + + Examples + -------- + >>> from sklearn.ensemble import RandomForestClassifier + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification(n_samples=1000, n_features=4, + ... n_informative=2, n_redundant=0, + ... random_state=0, shuffle=False) + >>> clf = RandomForestClassifier(max_depth=2, random_state=0) + >>> clf.fit(X, y) + RandomForestClassifier(...) + >>> print(clf.predict([[0, 0, 0, 0]])) + [1] + """ + + _parameter_constraints: dict = { + **ForestClassifier._parameter_constraints, + **DecisionTreeClassifier._parameter_constraints, + "class_weight": [ + StrOptions({"balanced_subsample", "balanced"}), + dict, + list, + None, + ], + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="gini", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features="sqrt", + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=True, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): + super().__init__( + estimator=DecisionTreeClassifier(), + n_estimators=n_estimators, + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), + bootstrap=bootstrap, + oob_score=oob_score, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + warm_start=warm_start, + class_weight=class_weight, + max_samples=max_samples, + ) + + self.criterion = criterion + self.max_depth = max_depth + self.min_samples_split = min_samples_split + self.min_samples_leaf = min_samples_leaf + self.min_weight_fraction_leaf = min_weight_fraction_leaf + self.max_features = max_features + self.max_leaf_nodes = max_leaf_nodes + self.min_impurity_decrease = min_impurity_decrease + self.monotonic_cst = monotonic_cst + self.ccp_alpha = ccp_alpha + + +class RandomForestRegressor(ForestRegressor): + """ + A random forest regressor. + + A random forest is a meta estimator that fits a number of decision tree + regressors on various sub-samples of the dataset and uses averaging to + improve the predictive accuracy and control over-fitting. + Trees in the forest use the best split strategy, i.e. equivalent to passing + `splitter="best"` to the underlying :class:`~sklearn.tree.DecisionTreeRegressor`. + The sub-sample size is controlled with the `max_samples` parameter if + `bootstrap=True` (default), otherwise the whole dataset is used to build + each tree. + + This estimator has native support for missing values (NaNs). During training, + the tree grower learns at each split point whether samples with missing values + should go to the left or right child, based on the potential gain. When predicting, + samples with missing values are assigned to the left or right child consequently. + If no missing values were encountered for a given feature during training, then + samples with missing values are mapped to whichever child has the most samples. + + For a comparison between tree-based ensemble models see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py`. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_estimators : int, default=100 + The number of trees in the forest. + + .. versionchanged:: 0.22 + The default value of ``n_estimators`` changed from 10 to 100 + in 0.22. + + criterion : {"squared_error", "absolute_error", "friedman_mse", "poisson"}, \ + default="squared_error" + The function to measure the quality of a split. Supported criteria + are "squared_error" for the mean squared error, which is equal to + variance reduction as feature selection criterion and minimizes the L2 + loss using the mean of each terminal node, "friedman_mse", which uses + mean squared error with Friedman's improvement score for potential + splits, "absolute_error" for the mean absolute error, which minimizes + the L1 loss using the median of each terminal node, and "poisson" which + uses reduction in Poisson deviance to find splits. + Training using "absolute_error" is significantly slower + than when using "squared_error". + + .. versionadded:: 0.18 + Mean Absolute Error (MAE) criterion. + + .. versionadded:: 1.0 + Poisson criterion. + + max_depth : int, default=None + The maximum depth of the tree. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + + min_samples_split : int or float, default=2 + The minimum number of samples required to split an internal node: + + - If int, then consider `min_samples_split` as the minimum number. + - If float, then `min_samples_split` is a fraction and + `ceil(min_samples_split * n_samples)` are the minimum + number of samples for each split. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_samples_leaf : int or float, default=1 + The minimum number of samples required to be at a leaf node. + A split point at any depth will only be considered if it leaves at + least ``min_samples_leaf`` training samples in each of the left and + right branches. This may have the effect of smoothing the model, + especially in regression. + + - If int, then consider `min_samples_leaf` as the minimum number. + - If float, then `min_samples_leaf` is a fraction and + `ceil(min_samples_leaf * n_samples)` are the minimum + number of samples for each node. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_weight_fraction_leaf : float, default=0.0 + The minimum weighted fraction of the sum total of weights (of all + the input samples) required to be at a leaf node. Samples have + equal weight when sample_weight is not provided. + + max_features : {"sqrt", "log2", None}, int or float, default=1.0 + The number of features to consider when looking for the best split: + + - If int, then consider `max_features` features at each split. + - If float, then `max_features` is a fraction and + `max(1, int(max_features * n_features_in_))` features are considered at each + split. + - If "sqrt", then `max_features=sqrt(n_features)`. + - If "log2", then `max_features=log2(n_features)`. + - If None or 1.0, then `max_features=n_features`. + + .. note:: + The default of 1.0 is equivalent to bagged trees and more + randomness can be achieved by setting smaller values, e.g. 0.3. + + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to 1.0. + + Note: the search for a split does not stop until at least one + valid partition of the node samples is found, even if it requires to + effectively inspect more than ``max_features`` features. + + max_leaf_nodes : int, default=None + Grow trees with ``max_leaf_nodes`` in best-first fashion. + Best nodes are defined as relative reduction in impurity. + If None then unlimited number of leaf nodes. + + min_impurity_decrease : float, default=0.0 + A node will be split if this split induces a decrease of the impurity + greater than or equal to this value. + + The weighted impurity decrease equation is the following:: + + N_t / N * (impurity - N_t_R / N_t * right_impurity + - N_t_L / N_t * left_impurity) + + where ``N`` is the total number of samples, ``N_t`` is the number of + samples at the current node, ``N_t_L`` is the number of samples in the + left child, and ``N_t_R`` is the number of samples in the right child. + + ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, + if ``sample_weight`` is passed. + + .. versionadded:: 0.19 + + bootstrap : bool, default=True + Whether bootstrap samples are used when building trees. If False, the + whole dataset is used to build each tree. + + oob_score : bool or callable, default=False + Whether to use out-of-bag samples to estimate the generalization score. + By default, :func:`~sklearn.metrics.r2_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. + + For an illustration of out-of-bag (OOB) error estimation, see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_ensemble_oob.py`. + + n_jobs : int, default=None + The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, + :meth:`decision_path` and :meth:`apply` are all parallelized over the + trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` + context. ``-1`` means using all processors. See :term:`Glossary + ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls both the randomness of the bootstrapping of the samples used + when building trees (if ``bootstrap=True``) and the sampling of the + features to consider when looking for the best split at each node + (if ``max_features < n_features``). + See :term:`Glossary ` for details. + + verbose : int, default=0 + Controls the verbosity when fitting and predicting. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit a whole + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. + + ccp_alpha : non-negative float, default=0.0 + Complexity parameter used for Minimal Cost-Complexity Pruning. The + subtree with the largest cost complexity that is smaller than + ``ccp_alpha`` will be chosen. By default, no pruning is performed. See + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. + + .. versionadded:: 0.22 + + max_samples : int or float, default=None + If bootstrap is True, the number of samples to draw from X + to train each base estimator. + + - If None (default), then draw `X.shape[0]` samples. + - If int, then draw `max_samples` samples. + - If float, then draw `max(round(n_samples * max_samples), 1)` samples. Thus, + `max_samples` should be in the interval `(0.0, 1.0]`. + + .. versionadded:: 0.22 + + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonically increasing + - 0: no constraint + - -1: monotonically decreasing + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multioutput regressions (i.e. when `n_outputs_ > 1`), + - regressions trained on data with missing values. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + + Attributes + ---------- + estimator_ : :class:`~sklearn.tree.DecisionTreeRegressor` + The child estimator template used to create the collection of fitted + sub-estimators. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of DecisionTreeRegressor + The collection of fitted sub-estimators. + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_outputs_ : int + The number of outputs when ``fit`` is performed. + + oob_score_ : float + Score of the training dataset obtained using an out-of-bag estimate. + This attribute exists only when ``oob_score`` is True. + + oob_prediction_ : ndarray of shape (n_samples,) or (n_samples, n_outputs) + Prediction computed with out-of-bag estimate on the training set. + This attribute exists only when ``oob_score`` is True. + + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + + See Also + -------- + sklearn.tree.DecisionTreeRegressor : A decision tree regressor. + sklearn.ensemble.ExtraTreesRegressor : Ensemble of extremely randomized + tree regressors. + sklearn.ensemble.HistGradientBoostingRegressor : A Histogram-based Gradient + Boosting Regression Tree, very fast for big datasets (n_samples >= + 10_000). + + Notes + ----- + The default values for the parameters controlling the size of the trees + (e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and + unpruned trees which can potentially be very large on some data sets. To + reduce memory consumption, the complexity and size of the trees should be + controlled by setting those parameter values. + + The features are always randomly permuted at each split. Therefore, + the best found split may vary, even with the same training data, + ``max_features=n_features`` and ``bootstrap=False``, if the improvement + of the criterion is identical for several splits enumerated during the + search of the best split. To obtain a deterministic behaviour during + fitting, ``random_state`` has to be fixed. + + The default value ``max_features=1.0`` uses ``n_features`` + rather than ``n_features / 3``. The latter was originally suggested in + [1]_, whereas the former was more recently justified empirically in [2]_. + + References + ---------- + .. [1] :doi:`L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, 2001. + <10.1023/A:1010933404324>` + + .. [2] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized + trees", Machine Learning, 63(1), 3-42, 2006. + + Examples + -------- + >>> from sklearn.ensemble import RandomForestRegressor + >>> from sklearn.datasets import make_regression + >>> X, y = make_regression(n_features=4, n_informative=2, + ... random_state=0, shuffle=False) + >>> regr = RandomForestRegressor(max_depth=2, random_state=0) + >>> regr.fit(X, y) + RandomForestRegressor(...) + >>> print(regr.predict([[0, 0, 0, 0]])) + [-8.32987858] + """ + + _parameter_constraints: dict = { + **ForestRegressor._parameter_constraints, + **DecisionTreeRegressor._parameter_constraints, + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="squared_error", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features=1.0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=True, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): + super().__init__( + estimator=DecisionTreeRegressor(), + n_estimators=n_estimators, + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), + bootstrap=bootstrap, + oob_score=oob_score, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + warm_start=warm_start, + max_samples=max_samples, + ) + + self.criterion = criterion + self.max_depth = max_depth + self.min_samples_split = min_samples_split + self.min_samples_leaf = min_samples_leaf + self.min_weight_fraction_leaf = min_weight_fraction_leaf + self.max_features = max_features + self.max_leaf_nodes = max_leaf_nodes + self.min_impurity_decrease = min_impurity_decrease + self.ccp_alpha = ccp_alpha + self.monotonic_cst = monotonic_cst + + +class ExtraTreesClassifier(ForestClassifier): + """ + An extra-trees classifier. + + This class implements a meta estimator that fits a number of + randomized decision trees (a.k.a. extra-trees) on various sub-samples + of the dataset and uses averaging to improve the predictive accuracy + and control over-fitting. + + This estimator has native support for missing values (NaNs) for + random splits. During training, a random threshold will be chosen + to split the non-missing values on. Then the non-missing values will be sent + to the left and right child based on the randomly selected threshold, while + the missing values will also be randomly sent to the left or right child. + This is repeated for every feature considered at each split. The best split + among these is chosen. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_estimators : int, default=100 + The number of trees in the forest. + + .. versionchanged:: 0.22 + The default value of ``n_estimators`` changed from 10 to 100 + in 0.22. + + criterion : {"gini", "entropy", "log_loss"}, default="gini" + The function to measure the quality of a split. Supported criteria are + "gini" for the Gini impurity and "log_loss" and "entropy" both for the + Shannon information gain, see :ref:`tree_mathematical_formulation`. + Note: This parameter is tree-specific. + + max_depth : int, default=None + The maximum depth of the tree. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + + min_samples_split : int or float, default=2 + The minimum number of samples required to split an internal node: + + - If int, then consider `min_samples_split` as the minimum number. + - If float, then `min_samples_split` is a fraction and + `ceil(min_samples_split * n_samples)` are the minimum + number of samples for each split. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_samples_leaf : int or float, default=1 + The minimum number of samples required to be at a leaf node. + A split point at any depth will only be considered if it leaves at + least ``min_samples_leaf`` training samples in each of the left and + right branches. This may have the effect of smoothing the model, + especially in regression. + + - If int, then consider `min_samples_leaf` as the minimum number. + - If float, then `min_samples_leaf` is a fraction and + `ceil(min_samples_leaf * n_samples)` are the minimum + number of samples for each node. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_weight_fraction_leaf : float, default=0.0 + The minimum weighted fraction of the sum total of weights (of all + the input samples) required to be at a leaf node. Samples have + equal weight when sample_weight is not provided. + + max_features : {"sqrt", "log2", None}, int or float, default="sqrt" + The number of features to consider when looking for the best split: + + - If int, then consider `max_features` features at each split. + - If float, then `max_features` is a fraction and + `max(1, int(max_features * n_features_in_))` features are considered at each + split. + - If "sqrt", then `max_features=sqrt(n_features)`. + - If "log2", then `max_features=log2(n_features)`. + - If None, then `max_features=n_features`. + + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to `"sqrt"`. + + Note: the search for a split does not stop until at least one + valid partition of the node samples is found, even if it requires to + effectively inspect more than ``max_features`` features. + + max_leaf_nodes : int, default=None + Grow trees with ``max_leaf_nodes`` in best-first fashion. + Best nodes are defined as relative reduction in impurity. + If None then unlimited number of leaf nodes. + + min_impurity_decrease : float, default=0.0 + A node will be split if this split induces a decrease of the impurity + greater than or equal to this value. + + The weighted impurity decrease equation is the following:: + + N_t / N * (impurity - N_t_R / N_t * right_impurity + - N_t_L / N_t * left_impurity) + + where ``N`` is the total number of samples, ``N_t`` is the number of + samples at the current node, ``N_t_L`` is the number of samples in the + left child, and ``N_t_R`` is the number of samples in the right child. + + ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, + if ``sample_weight`` is passed. + + .. versionadded:: 0.19 + + bootstrap : bool, default=False + Whether bootstrap samples are used when building trees. If False, the + whole dataset is used to build each tree. + + oob_score : bool or callable, default=False + Whether to use out-of-bag samples to estimate the generalization score. + By default, :func:`~sklearn.metrics.accuracy_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. + + For an illustration of out-of-bag (OOB) error estimation, see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_ensemble_oob.py`. + + n_jobs : int, default=None + The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, + :meth:`decision_path` and :meth:`apply` are all parallelized over the + trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` + context. ``-1`` means using all processors. See :term:`Glossary + ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls 3 sources of randomness: + + - the bootstrapping of the samples used when building trees + (if ``bootstrap=True``) + - the sampling of the features to consider when looking for the best + split at each node (if ``max_features < n_features``) + - the draw of the splits for each of the `max_features` + + See :term:`Glossary ` for details. + + verbose : int, default=0 + Controls the verbosity when fitting and predicting. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit a whole + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. + + class_weight : {"balanced", "balanced_subsample"}, dict or list of dicts, \ + default=None + Weights associated with classes in the form ``{class_label: weight}``. + If not given, all classes are supposed to have weight one. For + multi-output problems, a list of dicts can be provided in the same + order as the columns of y. + + Note that for multioutput (including multilabel) weights should be + defined for each class of every column in its own dict. For example, + for four-class multilabel classification weights should be + [{0: 1, 1: 1}, {0: 1, 1: 5}, {0: 1, 1: 1}, {0: 1, 1: 1}] instead of + [{1:1}, {2:5}, {3:1}, {4:1}]. + + The "balanced" mode uses the values of y to automatically adjust + weights inversely proportional to class frequencies in the input data + as ``n_samples / (n_classes * np.bincount(y))`` + + The "balanced_subsample" mode is the same as "balanced" except that + weights are computed based on the bootstrap sample for every tree + grown. + + For multi-output, the weights of each column of y will be multiplied. + + Note that these weights will be multiplied with sample_weight (passed + through the fit method) if sample_weight is specified. + + ccp_alpha : non-negative float, default=0.0 + Complexity parameter used for Minimal Cost-Complexity Pruning. The + subtree with the largest cost complexity that is smaller than + ``ccp_alpha`` will be chosen. By default, no pruning is performed. See + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. + + .. versionadded:: 0.22 + + max_samples : int or float, default=None + If bootstrap is True, the number of samples to draw from X + to train each base estimator. + + - If None (default), then draw `X.shape[0]` samples. + - If int, then draw `max_samples` samples. + - If float, then draw `max_samples * X.shape[0]` samples. Thus, + `max_samples` should be in the interval `(0.0, 1.0]`. + + .. versionadded:: 0.22 + + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonically increasing + - 0: no constraint + - -1: monotonically decreasing + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multiclass classifications (i.e. when `n_classes > 2`), + - multioutput classifications (i.e. when `n_outputs_ > 1`), + - classifications trained on data with missing values. + + The constraints hold over the probability of the positive class. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + + Attributes + ---------- + estimator_ : :class:`~sklearn.tree.ExtraTreeClassifier` + The child estimator template used to create the collection of fitted + sub-estimators. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of DecisionTreeClassifier + The collection of fitted sub-estimators. + + classes_ : ndarray of shape (n_classes,) or a list of such arrays + The classes labels (single output problem), or a list of arrays of + class labels (multi-output problem). + + n_classes_ : int or list + The number of classes (single output problem), or a list containing the + number of classes for each output (multi-output problem). + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_outputs_ : int + The number of outputs when ``fit`` is performed. + + oob_score_ : float + Score of the training dataset obtained using an out-of-bag estimate. + This attribute exists only when ``oob_score`` is True. + + oob_decision_function_ : ndarray of shape (n_samples, n_classes) or \ + (n_samples, n_classes, n_outputs) + Decision function computed with out-of-bag estimate on the training + set. If n_estimators is small it might be possible that a data point + was never left out during the bootstrap. In this case, + `oob_decision_function_` might contain NaN. This attribute exists + only when ``oob_score`` is True. + + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + + See Also + -------- + ExtraTreesRegressor : An extra-trees regressor with random splits. + RandomForestClassifier : A random forest classifier with optimal splits. + RandomForestRegressor : Ensemble regressor using trees with optimal splits. + + Notes + ----- + The default values for the parameters controlling the size of the trees + (e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and + unpruned trees which can potentially be very large on some data sets. To + reduce memory consumption, the complexity and size of the trees should be + controlled by setting those parameter values. + + References + ---------- + .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized + trees", Machine Learning, 63(1), 3-42, 2006. + + Examples + -------- + >>> from sklearn.ensemble import ExtraTreesClassifier + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification(n_features=4, random_state=0) + >>> clf = ExtraTreesClassifier(n_estimators=100, random_state=0) + >>> clf.fit(X, y) + ExtraTreesClassifier(random_state=0) + >>> clf.predict([[0, 0, 0, 0]]) + array([1]) + """ + + _parameter_constraints: dict = { + **ForestClassifier._parameter_constraints, + **DecisionTreeClassifier._parameter_constraints, + "class_weight": [ + StrOptions({"balanced_subsample", "balanced"}), + dict, + list, + None, + ], + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="gini", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features="sqrt", + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): + super().__init__( + estimator=ExtraTreeClassifier(), + n_estimators=n_estimators, + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), + bootstrap=bootstrap, + oob_score=oob_score, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + warm_start=warm_start, + class_weight=class_weight, + max_samples=max_samples, + ) + + self.criterion = criterion + self.max_depth = max_depth + self.min_samples_split = min_samples_split + self.min_samples_leaf = min_samples_leaf + self.min_weight_fraction_leaf = min_weight_fraction_leaf + self.max_features = max_features + self.max_leaf_nodes = max_leaf_nodes + self.min_impurity_decrease = min_impurity_decrease + self.ccp_alpha = ccp_alpha + self.monotonic_cst = monotonic_cst + + +class ExtraTreesRegressor(ForestRegressor): + """ + An extra-trees regressor. + + This class implements a meta estimator that fits a number of + randomized decision trees (a.k.a. extra-trees) on various sub-samples + of the dataset and uses averaging to improve the predictive accuracy + and control over-fitting. + + This estimator has native support for missing values (NaNs) for + random splits. During training, a random threshold will be chosen + to split the non-missing values on. Then the non-missing values will be sent + to the left and right child based on the randomly selected threshold, while + the missing values will also be randomly sent to the left or right child. + This is repeated for every feature considered at each split. The best split + among these is chosen. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_estimators : int, default=100 + The number of trees in the forest. + + .. versionchanged:: 0.22 + The default value of ``n_estimators`` changed from 10 to 100 + in 0.22. + + criterion : {"squared_error", "absolute_error", "friedman_mse", "poisson"}, \ + default="squared_error" + The function to measure the quality of a split. Supported criteria + are "squared_error" for the mean squared error, which is equal to + variance reduction as feature selection criterion and minimizes the L2 + loss using the mean of each terminal node, "friedman_mse", which uses + mean squared error with Friedman's improvement score for potential + splits, "absolute_error" for the mean absolute error, which minimizes + the L1 loss using the median of each terminal node, and "poisson" which + uses reduction in Poisson deviance to find splits. + Training using "absolute_error" is significantly slower + than when using "squared_error". + + .. versionadded:: 0.18 + Mean Absolute Error (MAE) criterion. + + max_depth : int, default=None + The maximum depth of the tree. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + + min_samples_split : int or float, default=2 + The minimum number of samples required to split an internal node: + + - If int, then consider `min_samples_split` as the minimum number. + - If float, then `min_samples_split` is a fraction and + `ceil(min_samples_split * n_samples)` are the minimum + number of samples for each split. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_samples_leaf : int or float, default=1 + The minimum number of samples required to be at a leaf node. + A split point at any depth will only be considered if it leaves at + least ``min_samples_leaf`` training samples in each of the left and + right branches. This may have the effect of smoothing the model, + especially in regression. + + - If int, then consider `min_samples_leaf` as the minimum number. + - If float, then `min_samples_leaf` is a fraction and + `ceil(min_samples_leaf * n_samples)` are the minimum + number of samples for each node. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_weight_fraction_leaf : float, default=0.0 + The minimum weighted fraction of the sum total of weights (of all + the input samples) required to be at a leaf node. Samples have + equal weight when sample_weight is not provided. + + max_features : {"sqrt", "log2", None}, int or float, default=1.0 + The number of features to consider when looking for the best split: + + - If int, then consider `max_features` features at each split. + - If float, then `max_features` is a fraction and + `max(1, int(max_features * n_features_in_))` features are considered at each + split. + - If "sqrt", then `max_features=sqrt(n_features)`. + - If "log2", then `max_features=log2(n_features)`. + - If None or 1.0, then `max_features=n_features`. + + .. note:: + The default of 1.0 is equivalent to bagged trees and more + randomness can be achieved by setting smaller values, e.g. 0.3. + + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to 1.0. + + Note: the search for a split does not stop until at least one + valid partition of the node samples is found, even if it requires to + effectively inspect more than ``max_features`` features. + + max_leaf_nodes : int, default=None + Grow trees with ``max_leaf_nodes`` in best-first fashion. + Best nodes are defined as relative reduction in impurity. + If None then unlimited number of leaf nodes. + + min_impurity_decrease : float, default=0.0 + A node will be split if this split induces a decrease of the impurity + greater than or equal to this value. + + The weighted impurity decrease equation is the following:: + + N_t / N * (impurity - N_t_R / N_t * right_impurity + - N_t_L / N_t * left_impurity) + + where ``N`` is the total number of samples, ``N_t`` is the number of + samples at the current node, ``N_t_L`` is the number of samples in the + left child, and ``N_t_R`` is the number of samples in the right child. + + ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, + if ``sample_weight`` is passed. + + .. versionadded:: 0.19 + + bootstrap : bool, default=False + Whether bootstrap samples are used when building trees. If False, the + whole dataset is used to build each tree. + + oob_score : bool or callable, default=False + Whether to use out-of-bag samples to estimate the generalization score. + By default, :func:`~sklearn.metrics.r2_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. + + For an illustration of out-of-bag (OOB) error estimation, see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_ensemble_oob.py`. + + n_jobs : int, default=None + The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, + :meth:`decision_path` and :meth:`apply` are all parallelized over the + trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` + context. ``-1`` means using all processors. See :term:`Glossary + ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls 3 sources of randomness: + + - the bootstrapping of the samples used when building trees + (if ``bootstrap=True``) + - the sampling of the features to consider when looking for the best + split at each node (if ``max_features < n_features``) + - the draw of the splits for each of the `max_features` + + See :term:`Glossary ` for details. + + verbose : int, default=0 + Controls the verbosity when fitting and predicting. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit a whole + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. + + ccp_alpha : non-negative float, default=0.0 + Complexity parameter used for Minimal Cost-Complexity Pruning. The + subtree with the largest cost complexity that is smaller than + ``ccp_alpha`` will be chosen. By default, no pruning is performed. See + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. + + .. versionadded:: 0.22 + + max_samples : int or float, default=None + If bootstrap is True, the number of samples to draw from X + to train each base estimator. + + - If None (default), then draw `X.shape[0]` samples. + - If int, then draw `max_samples` samples. + - If float, then draw `max_samples * X.shape[0]` samples. Thus, + `max_samples` should be in the interval `(0.0, 1.0]`. + + .. versionadded:: 0.22 + + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonically increasing + - 0: no constraint + - -1: monotonically decreasing + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multioutput regressions (i.e. when `n_outputs_ > 1`), + - regressions trained on data with missing values. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + + Attributes + ---------- + estimator_ : :class:`~sklearn.tree.ExtraTreeRegressor` + The child estimator template used to create the collection of fitted + sub-estimators. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of DecisionTreeRegressor + The collection of fitted sub-estimators. + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_outputs_ : int + The number of outputs. + + oob_score_ : float + Score of the training dataset obtained using an out-of-bag estimate. + This attribute exists only when ``oob_score`` is True. + + oob_prediction_ : ndarray of shape (n_samples,) or (n_samples, n_outputs) + Prediction computed with out-of-bag estimate on the training set. + This attribute exists only when ``oob_score`` is True. + + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + + See Also + -------- + ExtraTreesClassifier : An extra-trees classifier with random splits. + RandomForestClassifier : A random forest classifier with optimal splits. + RandomForestRegressor : Ensemble regressor using trees with optimal splits. + + Notes + ----- + The default values for the parameters controlling the size of the trees + (e.g. ``max_depth``, ``min_samples_leaf``, etc.) lead to fully grown and + unpruned trees which can potentially be very large on some data sets. To + reduce memory consumption, the complexity and size of the trees should be + controlled by setting those parameter values. + + References + ---------- + .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees", + Machine Learning, 63(1), 3-42, 2006. + + Examples + -------- + >>> from sklearn.datasets import load_diabetes + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.ensemble import ExtraTreesRegressor + >>> X, y = load_diabetes(return_X_y=True) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> reg = ExtraTreesRegressor(n_estimators=100, random_state=0).fit( + ... X_train, y_train) + >>> reg.score(X_test, y_test) + 0.2727... + """ + + _parameter_constraints: dict = { + **ForestRegressor._parameter_constraints, + **DecisionTreeRegressor._parameter_constraints, + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="squared_error", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features=1.0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): + super().__init__( + estimator=ExtraTreeRegressor(), + n_estimators=n_estimators, + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), + bootstrap=bootstrap, + oob_score=oob_score, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + warm_start=warm_start, + max_samples=max_samples, + ) + + self.criterion = criterion + self.max_depth = max_depth + self.min_samples_split = min_samples_split + self.min_samples_leaf = min_samples_leaf + self.min_weight_fraction_leaf = min_weight_fraction_leaf + self.max_features = max_features + self.max_leaf_nodes = max_leaf_nodes + self.min_impurity_decrease = min_impurity_decrease + self.ccp_alpha = ccp_alpha + self.monotonic_cst = monotonic_cst + + +class RandomTreesEmbedding(TransformerMixin, BaseForest): + """ + An ensemble of totally random trees. + + An unsupervised transformation of a dataset to a high-dimensional + sparse representation. A datapoint is coded according to which leaf of + each tree it is sorted into. Using a one-hot encoding of the leaves, + this leads to a binary coding with as many ones as there are trees in + the forest. + + The dimensionality of the resulting representation is + ``n_out <= n_estimators * max_leaf_nodes``. If ``max_leaf_nodes == None``, + the number of leaf nodes is at most ``n_estimators * 2 ** max_depth``. + + For an example of applying Random Trees Embedding to non-linear + classification, see + :ref:`sphx_glr_auto_examples_ensemble_plot_random_forest_embedding.py`. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_estimators : int, default=100 + Number of trees in the forest. + + .. versionchanged:: 0.22 + The default value of ``n_estimators`` changed from 10 to 100 + in 0.22. + + max_depth : int, default=5 + The maximum depth of each tree. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + + min_samples_split : int or float, default=2 + The minimum number of samples required to split an internal node: + + - If int, then consider `min_samples_split` as the minimum number. + - If float, then `min_samples_split` is a fraction and + `ceil(min_samples_split * n_samples)` is the minimum + number of samples for each split. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_samples_leaf : int or float, default=1 + The minimum number of samples required to be at a leaf node. + A split point at any depth will only be considered if it leaves at + least ``min_samples_leaf`` training samples in each of the left and + right branches. This may have the effect of smoothing the model, + especially in regression. + + - If int, then consider `min_samples_leaf` as the minimum number. + - If float, then `min_samples_leaf` is a fraction and + `ceil(min_samples_leaf * n_samples)` is the minimum + number of samples for each node. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_weight_fraction_leaf : float, default=0.0 + The minimum weighted fraction of the sum total of weights (of all + the input samples) required to be at a leaf node. Samples have + equal weight when sample_weight is not provided. + + max_leaf_nodes : int, default=None + Grow trees with ``max_leaf_nodes`` in best-first fashion. + Best nodes are defined as relative reduction in impurity. + If None then unlimited number of leaf nodes. + + min_impurity_decrease : float, default=0.0 + A node will be split if this split induces a decrease of the impurity + greater than or equal to this value. + + The weighted impurity decrease equation is the following:: + + N_t / N * (impurity - N_t_R / N_t * right_impurity + - N_t_L / N_t * left_impurity) + + where ``N`` is the total number of samples, ``N_t`` is the number of + samples at the current node, ``N_t_L`` is the number of samples in the + left child, and ``N_t_R`` is the number of samples in the right child. + + ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, + if ``sample_weight`` is passed. + + .. versionadded:: 0.19 + + sparse_output : bool, default=True + Whether or not to return a sparse CSR matrix, as default behavior, + or to return a dense array compatible with dense pipeline operators. + + n_jobs : int, default=None + The number of jobs to run in parallel. :meth:`fit`, :meth:`transform`, + :meth:`decision_path` and :meth:`apply` are all parallelized over the + trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` + context. ``-1`` means using all processors. See :term:`Glossary + ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls the generation of the random `y` used to fit the trees + and the draw of the splits for each feature at the trees' nodes. + See :term:`Glossary ` for details. + + verbose : int, default=0 + Controls the verbosity when fitting and predicting. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit a whole + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. + + Attributes + ---------- + estimator_ : :class:`~sklearn.tree.ExtraTreeRegressor` instance + The child estimator template used to create the collection of fitted + sub-estimators. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of :class:`~sklearn.tree.ExtraTreeRegressor` instances + The collection of fitted sub-estimators. + + feature_importances_ : ndarray of shape (n_features,) + The feature importances (the higher, the more important the feature). + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_outputs_ : int + The number of outputs when ``fit`` is performed. + + one_hot_encoder_ : OneHotEncoder instance + One-hot encoder used to create the sparse embedding. + + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + + See Also + -------- + ExtraTreesClassifier : An extra-trees classifier. + ExtraTreesRegressor : An extra-trees regressor. + RandomForestClassifier : A random forest classifier. + RandomForestRegressor : A random forest regressor. + sklearn.tree.ExtraTreeClassifier: An extremely randomized + tree classifier. + sklearn.tree.ExtraTreeRegressor : An extremely randomized + tree regressor. + + References + ---------- + .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees", + Machine Learning, 63(1), 3-42, 2006. + .. [2] Moosmann, F. and Triggs, B. and Jurie, F. "Fast discriminative + visual codebooks using randomized clustering forests" + NIPS 2007. + + Examples + -------- + >>> from sklearn.ensemble import RandomTreesEmbedding + >>> X = [[0,0], [1,0], [0,1], [-1,0], [0,-1]] + >>> random_trees = RandomTreesEmbedding( + ... n_estimators=5, random_state=0, max_depth=1).fit(X) + >>> X_sparse_embedding = random_trees.transform(X) + >>> X_sparse_embedding.toarray() + array([[0., 1., 1., 0., 1., 0., 0., 1., 1., 0.], + [0., 1., 1., 0., 1., 0., 0., 1., 1., 0.], + [0., 1., 0., 1., 0., 1., 0., 1., 0., 1.], + [1., 0., 1., 0., 1., 0., 1., 0., 1., 0.], + [0., 1., 1., 0., 1., 0., 0., 1., 1., 0.]]) + """ + + _parameter_constraints: dict = { + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "n_jobs": [Integral, None], + "verbose": ["verbose"], + "warm_start": ["boolean"], + **BaseDecisionTree._parameter_constraints, + "sparse_output": ["boolean"], + } + for param in ("max_features", "ccp_alpha", "splitter", "monotonic_cst"): + _parameter_constraints.pop(param) + + criterion = "squared_error" + max_features = 1 + + def __init__( + self, + n_estimators=100, + *, + max_depth=5, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + sparse_output=True, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ): + super().__init__( + estimator=ExtraTreeRegressor(), + n_estimators=n_estimators, + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + ), + bootstrap=False, + oob_score=False, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + warm_start=warm_start, + max_samples=None, + ) + + self.max_depth = max_depth + self.min_samples_split = min_samples_split + self.min_samples_leaf = min_samples_leaf + self.min_weight_fraction_leaf = min_weight_fraction_leaf + self.max_leaf_nodes = max_leaf_nodes + self.min_impurity_decrease = min_impurity_decrease + self.sparse_output = sparse_output + + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): + raise NotImplementedError("OOB score not supported by tree embedding") + + def fit(self, X, y=None, sample_weight=None): + """ + Fit estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Use ``dtype=np.float32`` for maximum + efficiency. Sparse matrices are also supported, use sparse + ``csc_matrix`` for maximum efficiency. + + y : Ignored + Not used, present for API consistency by convention. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, then samples are equally weighted. Splits + that would create child nodes with net zero or negative weight are + ignored while searching for a split in each node. In the case of + classification, splits are also ignored if they would result in any + single class carrying a negative weight in either child node. + + Returns + ------- + self : object + Returns the instance itself. + """ + # Parameters are validated in fit_transform + self.fit_transform(X, y, sample_weight=sample_weight) + return self + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None, sample_weight=None): + """ + Fit estimator and transform dataset. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input data used to build forests. Use ``dtype=np.float32`` for + maximum efficiency. + + y : Ignored + Not used, present for API consistency by convention. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, then samples are equally weighted. Splits + that would create child nodes with net zero or negative weight are + ignored while searching for a split in each node. In the case of + classification, splits are also ignored if they would result in any + single class carrying a negative weight in either child node. + + Returns + ------- + X_transformed : sparse matrix of shape (n_samples, n_out) + Transformed dataset. + """ + rnd = check_random_state(self.random_state) + y = rnd.uniform(size=_num_samples(X)) + super().fit(X, y, sample_weight=sample_weight) + + self.one_hot_encoder_ = OneHotEncoder(sparse_output=self.sparse_output) + output = self.one_hot_encoder_.fit_transform(self.apply(X)) + self._n_features_out = output.shape[1] + return output + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Only used to validate feature names with the names seen in :meth:`fit`. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names, in the format of + `randomtreesembedding_{tree}_{leaf}`, where `tree` is the tree used + to generate the leaf and `leaf` is the index of a leaf node + in that tree. Note that the node indexing scheme is used to + index both nodes with children (split nodes) and leaf nodes. + Only the latter can be present as output features. + As a consequence, there are missing indices in the output + feature names. + """ + check_is_fitted(self, "_n_features_out") + _check_feature_names_in( + self, input_features=input_features, generate_names=False + ) + + feature_names = [ + f"randomtreesembedding_{tree}_{leaf}" + for tree in range(self.n_estimators) + for leaf in self.one_hot_encoder_.categories_[tree] + ] + return np.asarray(feature_names, dtype=object) + + def transform(self, X): + """ + Transform dataset. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input data to be transformed. Use ``dtype=np.float32`` for maximum + efficiency. Sparse matrices are also supported, use sparse + ``csr_matrix`` for maximum efficiency. + + Returns + ------- + X_transformed : sparse matrix of shape (n_samples, n_out) + Transformed dataset. + """ + check_is_fitted(self) + return self.one_hot_encoder_.transform(self.apply(X)) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gb.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gb.py new file mode 100644 index 0000000000000000000000000000000000000000..1e5d7c4246eeae19dbc2c26659bf68743f15fa17 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gb.py @@ -0,0 +1,2204 @@ +"""Gradient Boosted Regression Trees. + +This module contains methods for fitting gradient boosted regression trees for +both classification and regression. + +The module structure is the following: + +- The ``BaseGradientBoosting`` base class implements a common ``fit`` method + for all the estimators in the module. Regression and classification + only differ in the concrete ``LossFunction`` used. + +- ``GradientBoostingClassifier`` implements gradient boosting for + classification problems. + +- ``GradientBoostingRegressor`` implements gradient boosting for + regression problems. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import math +import warnings +from abc import ABCMeta, abstractmethod +from numbers import Integral, Real +from time import time + +import numpy as np +from scipy.sparse import csc_matrix, csr_matrix, issparse + +from sklearn._loss.loss import ( + _LOSSES, + AbsoluteError, + ExponentialLoss, + HalfBinomialLoss, + HalfMultinomialLoss, + HalfSquaredError, + HuberLoss, + PinballLoss, +) +from sklearn.base import ClassifierMixin, RegressorMixin, _fit_context, is_classifier +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble._base import BaseEnsemble +from sklearn.ensemble._gradient_boosting import ( + _random_sample_mask, + predict_stage, + predict_stages, +) +from sklearn.exceptions import NotFittedError +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import LabelEncoder +from sklearn.tree import DecisionTreeRegressor +from sklearn.tree._tree import DOUBLE, DTYPE, TREE_LEAF +from sklearn.utils import check_array, check_random_state, column_or_1d +from sklearn.utils._param_validation import HasMethods, Interval, StrOptions +from sklearn.utils.multiclass import check_classification_targets +from sklearn.utils.stats import _weighted_percentile +from sklearn.utils.validation import ( + _check_sample_weight, + check_is_fitted, + validate_data, +) + +_LOSSES = _LOSSES.copy() +_LOSSES.update( + { + "quantile": PinballLoss, + "huber": HuberLoss, + } +) + + +def _safe_divide(numerator, denominator): + """Prevents overflow and division by zero.""" + # This is used for classifiers where the denominator might become zero exactly. + # For instance for log loss, HalfBinomialLoss, if proba=0 or proba=1 exactly, then + # denominator = hessian = 0, and we should set the node value in the line search to + # zero as there is no improvement of the loss possible. + # For numerical safety, we do this already for extremely tiny values. + if abs(denominator) < 1e-150: + return 0.0 + else: + # Cast to Python float to trigger Python errors, e.g. ZeroDivisionError, + # without relying on `np.errstate` that is not supported by Pyodide. + result = float(numerator) / float(denominator) + # Cast to Python float to trigger a ZeroDivisionError without relying + # on `np.errstate` that is not supported by Pyodide. + result = float(numerator) / float(denominator) + if math.isinf(result): + warnings.warn("overflow encountered in _safe_divide", RuntimeWarning) + return result + + +def _init_raw_predictions(X, estimator, loss, use_predict_proba): + """Return the initial raw predictions. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + The data array. + estimator : object + The estimator to use to compute the predictions. + loss : BaseLoss + An instance of a loss function class. + use_predict_proba : bool + Whether estimator.predict_proba is used instead of estimator.predict. + + Returns + ------- + raw_predictions : ndarray of shape (n_samples, K) + The initial raw predictions. K is equal to 1 for binary + classification and regression, and equal to the number of classes + for multiclass classification. ``raw_predictions`` is casted + into float64. + """ + # TODO: Use loss.fit_intercept_only where appropriate instead of + # DummyRegressor which is the default given by the `init` parameter, + # see also _init_state. + if use_predict_proba: + # Our parameter validation, set via _fit_context and _parameter_constraints + # already guarantees that estimator has a predict_proba method. + predictions = estimator.predict_proba(X) + if not loss.is_multiclass: + predictions = predictions[:, 1] # probability of positive class + eps = np.finfo(np.float64).eps + predictions = np.clip(predictions, eps, 1 - eps, dtype=np.float64) + else: + predictions = estimator.predict(X).astype(np.float64) + + if predictions.ndim == 1: + return loss.link.link(predictions).reshape(-1, 1) + else: + return loss.link.link(predictions) + + +def _update_terminal_regions( + loss, + tree, + X, + y, + neg_gradient, + raw_prediction, + sample_weight, + sample_mask, + learning_rate=0.1, + k=0, +): + """Update the leaf values to be predicted by the tree and raw_prediction. + + The current raw predictions of the model (of this stage) are updated. + + Additionally, the terminal regions (=leaves) of the given tree are updated as well. + This corresponds to the line search step in "Greedy Function Approximation" by + Friedman, Algorithm 1 step 5. + + Update equals: + argmin_{x} loss(y_true, raw_prediction_old + x * tree.value) + + For non-trivial cases like the Binomial loss, the update has no closed formula and + is an approximation, again, see the Friedman paper. + + Also note that the update formula for the SquaredError is the identity. Therefore, + in this case, the leaf values don't need an update and only the raw_predictions are + updated (with the learning rate included). + + Parameters + ---------- + loss : BaseLoss + tree : tree.Tree + The tree object. + X : ndarray of shape (n_samples, n_features) + The data array. + y : ndarray of shape (n_samples,) + The target labels. + neg_gradient : ndarray of shape (n_samples,) + The negative gradient. + raw_prediction : ndarray of shape (n_samples, n_trees_per_iteration) + The raw predictions (i.e. values from the tree leaves) of the + tree ensemble at iteration ``i - 1``. + sample_weight : ndarray of shape (n_samples,) + The weight of each sample. + sample_mask : ndarray of shape (n_samples,) + The sample mask to be used. + learning_rate : float, default=0.1 + Learning rate shrinks the contribution of each tree by + ``learning_rate``. + k : int, default=0 + The index of the estimator being updated. + """ + # compute leaf for each sample in ``X``. + terminal_regions = tree.apply(X) + + if not isinstance(loss, HalfSquaredError): + # mask all which are not in sample mask. + masked_terminal_regions = terminal_regions.copy() + masked_terminal_regions[~sample_mask] = -1 + + if isinstance(loss, HalfBinomialLoss): + + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + # Make a single Newton-Raphson step, see "Additive Logistic Regression: + # A Statistical View of Boosting" FHT00 and note that we use a slightly + # different version (factor 2) of "F" with proba=expit(raw_prediction). + # Our node estimate is given by: + # sum(w * (y - prob)) / sum(w * prob * (1 - prob)) + # we take advantage that: y - prob = neg_gradient + neg_g = neg_gradient.take(indices, axis=0) + prob = y_ - neg_g + # numerator = negative gradient = y - prob + numerator = np.average(neg_g, weights=sw) + # denominator = hessian = prob * (1 - prob) + denominator = np.average(prob * (1 - prob), weights=sw) + return _safe_divide(numerator, denominator) + + elif isinstance(loss, HalfMultinomialLoss): + + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + # we take advantage that: y - prob = neg_gradient + neg_g = neg_gradient.take(indices, axis=0) + prob = y_ - neg_g + K = loss.n_classes + # numerator = negative gradient * (k - 1) / k + # Note: The factor (k - 1)/k appears in the original papers "Greedy + # Function Approximation" by Friedman and "Additive Logistic + # Regression" by Friedman, Hastie, Tibshirani. This factor is, however, + # wrong or at least arbitrary as it directly multiplies the + # learning_rate. We keep it for backward compatibility. + numerator = np.average(neg_g, weights=sw) + numerator *= (K - 1) / K + # denominator = (diagonal) hessian = prob * (1 - prob) + denominator = np.average(prob * (1 - prob), weights=sw) + return _safe_divide(numerator, denominator) + + elif isinstance(loss, ExponentialLoss): + + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + neg_g = neg_gradient.take(indices, axis=0) + # numerator = negative gradient = y * exp(-raw) - (1-y) * exp(raw) + numerator = np.average(neg_g, weights=sw) + # denominator = hessian = y * exp(-raw) + (1-y) * exp(raw) + # if y=0: hessian = exp(raw) = -neg_g + # y=1: hessian = exp(-raw) = neg_g + hessian = neg_g.copy() + hessian[y_ == 0] *= -1 + denominator = np.average(hessian, weights=sw) + return _safe_divide(numerator, denominator) + + else: + + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + return loss.fit_intercept_only( + y_true=y_ - raw_prediction[indices, k], + sample_weight=sw, + ) + + # update each leaf (= perform line search) + for leaf in np.nonzero(tree.children_left == TREE_LEAF)[0]: + indices = np.nonzero(masked_terminal_regions == leaf)[ + 0 + ] # of terminal regions + y_ = y.take(indices, axis=0) + sw = None if sample_weight is None else sample_weight[indices] + update = compute_update(y_, indices, neg_gradient, raw_prediction, k) + + # TODO: Multiply here by learning rate instead of everywhere else. + tree.value[leaf, 0, 0] = update + + # update predictions (both in-bag and out-of-bag) + raw_prediction[:, k] += learning_rate * tree.value[:, 0, 0].take( + terminal_regions, axis=0 + ) + + +def set_huber_delta(loss, y_true, raw_prediction, sample_weight=None): + """Calculate and set self.closs.delta based on self.quantile.""" + abserr = np.abs(y_true - raw_prediction.squeeze()) + # sample_weight is always an ndarray, never None. + delta = _weighted_percentile(abserr, sample_weight, 100 * loss.quantile) + loss.closs.delta = float(delta) + + +class VerboseReporter: + """Reports verbose output to stdout. + + Parameters + ---------- + verbose : int + Verbosity level. If ``verbose==1`` output is printed once in a while + (when iteration mod verbose_mod is zero).; if larger than 1 then output + is printed for each update. + """ + + def __init__(self, verbose): + self.verbose = verbose + + def init(self, est, begin_at_stage=0): + """Initialize reporter + + Parameters + ---------- + est : Estimator + The estimator + + begin_at_stage : int, default=0 + stage at which to begin reporting + """ + # header fields and line format str + header_fields = ["Iter", "Train Loss"] + verbose_fmt = ["{iter:>10d}", "{train_score:>16.4f}"] + # do oob? + if est.subsample < 1: + header_fields.append("OOB Improve") + verbose_fmt.append("{oob_impr:>16.4f}") + header_fields.append("Remaining Time") + verbose_fmt.append("{remaining_time:>16s}") + + # print the header line + print(("%10s " + "%16s " * (len(header_fields) - 1)) % tuple(header_fields)) + + self.verbose_fmt = " ".join(verbose_fmt) + # plot verbose info each time i % verbose_mod == 0 + self.verbose_mod = 1 + self.start_time = time() + self.begin_at_stage = begin_at_stage + + def update(self, j, est): + """Update reporter with new iteration. + + Parameters + ---------- + j : int + The new iteration. + est : Estimator + The estimator. + """ + do_oob = est.subsample < 1 + # we need to take into account if we fit additional estimators. + i = j - self.begin_at_stage # iteration relative to the start iter + if (i + 1) % self.verbose_mod == 0: + oob_impr = est.oob_improvement_[j] if do_oob else 0 + remaining_time = ( + (est.n_estimators - (j + 1)) * (time() - self.start_time) / float(i + 1) + ) + if remaining_time > 60: + remaining_time = "{0:.2f}m".format(remaining_time / 60.0) + else: + remaining_time = "{0:.2f}s".format(remaining_time) + print( + self.verbose_fmt.format( + iter=j + 1, + train_score=est.train_score_[j], + oob_impr=oob_impr, + remaining_time=remaining_time, + ) + ) + if self.verbose == 1 and ((i + 1) // (self.verbose_mod * 10) > 0): + # adjust verbose frequency (powers of 10) + self.verbose_mod *= 10 + + +class BaseGradientBoosting(BaseEnsemble, metaclass=ABCMeta): + """Abstract base class for Gradient Boosting.""" + + _parameter_constraints: dict = { + **DecisionTreeRegressor._parameter_constraints, + "learning_rate": [Interval(Real, 0.0, None, closed="left")], + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "criterion": [StrOptions({"friedman_mse", "squared_error"})], + "subsample": [Interval(Real, 0.0, 1.0, closed="right")], + "verbose": ["verbose"], + "warm_start": ["boolean"], + "validation_fraction": [Interval(Real, 0.0, 1.0, closed="neither")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left"), None], + "tol": [Interval(Real, 0.0, None, closed="left")], + } + _parameter_constraints.pop("splitter") + _parameter_constraints.pop("monotonic_cst") + + @abstractmethod + def __init__( + self, + *, + loss, + learning_rate, + n_estimators, + criterion, + min_samples_split, + min_samples_leaf, + min_weight_fraction_leaf, + max_depth, + min_impurity_decrease, + init, + subsample, + max_features, + ccp_alpha, + random_state, + alpha=0.9, + verbose=0, + max_leaf_nodes=None, + warm_start=False, + validation_fraction=0.1, + n_iter_no_change=None, + tol=1e-4, + ): + self.n_estimators = n_estimators + self.learning_rate = learning_rate + self.loss = loss + self.criterion = criterion + self.min_samples_split = min_samples_split + self.min_samples_leaf = min_samples_leaf + self.min_weight_fraction_leaf = min_weight_fraction_leaf + self.subsample = subsample + self.max_features = max_features + self.max_depth = max_depth + self.min_impurity_decrease = min_impurity_decrease + self.ccp_alpha = ccp_alpha + self.init = init + self.random_state = random_state + self.alpha = alpha + self.verbose = verbose + self.max_leaf_nodes = max_leaf_nodes + self.warm_start = warm_start + self.validation_fraction = validation_fraction + self.n_iter_no_change = n_iter_no_change + self.tol = tol + + @abstractmethod + def _encode_y(self, y=None, sample_weight=None): + """Called by fit to validate and encode y.""" + + @abstractmethod + def _get_loss(self, sample_weight): + """Get loss object from sklearn._loss.loss.""" + + def _fit_stage( + self, + i, + X, + y, + raw_predictions, + sample_weight, + sample_mask, + random_state, + X_csc=None, + X_csr=None, + ): + """Fit another stage of ``n_trees_per_iteration_`` trees.""" + original_y = y + + if isinstance(self._loss, HuberLoss): + set_huber_delta( + loss=self._loss, + y_true=y, + raw_prediction=raw_predictions, + sample_weight=sample_weight, + ) + # TODO: Without oob, i.e. with self.subsample = 1.0, we could call + # self._loss.loss_gradient and use it to set train_score_. + # But note that train_score_[i] is the score AFTER fitting the i-th tree. + # Note: We need the negative gradient! + neg_gradient = -self._loss.gradient( + y_true=y, + raw_prediction=raw_predictions, + sample_weight=None, # We pass sample_weights to the tree directly. + ) + # 2-d views of shape (n_samples, n_trees_per_iteration_) or (n_samples, 1) + # on neg_gradient to simplify the loop over n_trees_per_iteration_. + if neg_gradient.ndim == 1: + neg_g_view = neg_gradient.reshape((-1, 1)) + else: + neg_g_view = neg_gradient + + for k in range(self.n_trees_per_iteration_): + if self._loss.is_multiclass: + y = np.array(original_y == k, dtype=np.float64) + + # induce regression tree on the negative gradient + tree = DecisionTreeRegressor( + criterion=self.criterion, + splitter="best", + max_depth=self.max_depth, + min_samples_split=self.min_samples_split, + min_samples_leaf=self.min_samples_leaf, + min_weight_fraction_leaf=self.min_weight_fraction_leaf, + min_impurity_decrease=self.min_impurity_decrease, + max_features=self.max_features, + max_leaf_nodes=self.max_leaf_nodes, + random_state=random_state, + ccp_alpha=self.ccp_alpha, + ) + + if self.subsample < 1.0: + # no inplace multiplication! + sample_weight = sample_weight * sample_mask.astype(np.float64) + + X = X_csc if X_csc is not None else X + tree.fit( + X, neg_g_view[:, k], sample_weight=sample_weight, check_input=False + ) + + # update tree leaves + X_for_tree_update = X_csr if X_csr is not None else X + _update_terminal_regions( + self._loss, + tree.tree_, + X_for_tree_update, + y, + neg_g_view[:, k], + raw_predictions, + sample_weight, + sample_mask, + learning_rate=self.learning_rate, + k=k, + ) + + # add tree to ensemble + self.estimators_[i, k] = tree + + return raw_predictions + + def _set_max_features(self): + """Set self.max_features_.""" + if isinstance(self.max_features, str): + if self.max_features == "auto": + if is_classifier(self): + max_features = max(1, int(np.sqrt(self.n_features_in_))) + else: + max_features = self.n_features_in_ + elif self.max_features == "sqrt": + max_features = max(1, int(np.sqrt(self.n_features_in_))) + else: # self.max_features == "log2" + max_features = max(1, int(np.log2(self.n_features_in_))) + elif self.max_features is None: + max_features = self.n_features_in_ + elif isinstance(self.max_features, Integral): + max_features = self.max_features + else: # float + max_features = max(1, int(self.max_features * self.n_features_in_)) + + self.max_features_ = max_features + + def _init_state(self): + """Initialize model state and allocate model state data structures.""" + + self.init_ = self.init + if self.init_ is None: + if is_classifier(self): + self.init_ = DummyClassifier(strategy="prior") + elif isinstance(self._loss, (AbsoluteError, HuberLoss)): + self.init_ = DummyRegressor(strategy="quantile", quantile=0.5) + elif isinstance(self._loss, PinballLoss): + self.init_ = DummyRegressor(strategy="quantile", quantile=self.alpha) + else: + self.init_ = DummyRegressor(strategy="mean") + + self.estimators_ = np.empty( + (self.n_estimators, self.n_trees_per_iteration_), dtype=object + ) + self.train_score_ = np.zeros((self.n_estimators,), dtype=np.float64) + # do oob? + if self.subsample < 1.0: + self.oob_improvement_ = np.zeros((self.n_estimators), dtype=np.float64) + self.oob_scores_ = np.zeros((self.n_estimators), dtype=np.float64) + self.oob_score_ = np.nan + + def _clear_state(self): + """Clear the state of the gradient boosting model.""" + if hasattr(self, "estimators_"): + self.estimators_ = np.empty((0, 0), dtype=object) + if hasattr(self, "train_score_"): + del self.train_score_ + if hasattr(self, "oob_improvement_"): + del self.oob_improvement_ + if hasattr(self, "oob_scores_"): + del self.oob_scores_ + if hasattr(self, "oob_score_"): + del self.oob_score_ + if hasattr(self, "init_"): + del self.init_ + if hasattr(self, "_rng"): + del self._rng + + def _resize_state(self): + """Add additional ``n_estimators`` entries to all attributes.""" + # self.n_estimators is the number of additional est to fit + total_n_estimators = self.n_estimators + if total_n_estimators < self.estimators_.shape[0]: + raise ValueError( + "resize with smaller n_estimators %d < %d" + % (total_n_estimators, self.estimators_[0]) + ) + + self.estimators_ = np.resize( + self.estimators_, (total_n_estimators, self.n_trees_per_iteration_) + ) + self.train_score_ = np.resize(self.train_score_, total_n_estimators) + if self.subsample < 1 or hasattr(self, "oob_improvement_"): + # if do oob resize arrays or create new if not available + if hasattr(self, "oob_improvement_"): + self.oob_improvement_ = np.resize( + self.oob_improvement_, total_n_estimators + ) + self.oob_scores_ = np.resize(self.oob_scores_, total_n_estimators) + self.oob_score_ = np.nan + else: + self.oob_improvement_ = np.zeros( + (total_n_estimators,), dtype=np.float64 + ) + self.oob_scores_ = np.zeros((total_n_estimators,), dtype=np.float64) + self.oob_score_ = np.nan + + def _is_fitted(self): + return len(getattr(self, "estimators_", [])) > 0 + + def _check_initialized(self): + """Check that the estimator is initialized, raising an error if not.""" + check_is_fitted(self) + + @_fit_context( + # GradientBoosting*.init is not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, sample_weight=None, monitor=None): + """Fit the gradient boosting model. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + y : array-like of shape (n_samples,) + Target values (strings or integers in classification, real numbers + in regression) + For classification, labels must correspond to classes. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, then samples are equally weighted. Splits + that would create child nodes with net zero or negative weight are + ignored while searching for a split in each node. In the case of + classification, splits are also ignored if they would result in any + single class carrying a negative weight in either child node. + + monitor : callable, default=None + The monitor is called after each iteration with the current + iteration, a reference to the estimator and the local variables of + ``_fit_stages`` as keyword arguments ``callable(i, self, + locals())``. If the callable returns ``True`` the fitting procedure + is stopped. The monitor can be used for various things such as + computing held-out estimates, early stopping, model introspect, and + snapshotting. + + Returns + ------- + self : object + Fitted estimator. + """ + if not self.warm_start: + self._clear_state() + + # Check input + # Since check_array converts both X and y to the same dtype, but the + # trees use different types for X and y, checking them separately. + + X, y = validate_data( + self, + X, + y, + accept_sparse=["csr", "csc", "coo"], + dtype=DTYPE, + multi_output=True, + ) + sample_weight_is_none = sample_weight is None + sample_weight = _check_sample_weight(sample_weight, X) + if sample_weight_is_none: + y = self._encode_y(y=y, sample_weight=None) + else: + y = self._encode_y(y=y, sample_weight=sample_weight) + y = column_or_1d(y, warn=True) # TODO: Is this still required? + + self._set_max_features() + + # self.loss is guaranteed to be a string + self._loss = self._get_loss(sample_weight=sample_weight) + + if self.n_iter_no_change is not None: + stratify = y if is_classifier(self) else None + ( + X_train, + X_val, + y_train, + y_val, + sample_weight_train, + sample_weight_val, + ) = train_test_split( + X, + y, + sample_weight, + random_state=self.random_state, + test_size=self.validation_fraction, + stratify=stratify, + ) + if is_classifier(self): + if self.n_classes_ != np.unique(y_train).shape[0]: + # We choose to error here. The problem is that the init + # estimator would be trained on y, which has some missing + # classes now, so its predictions would not have the + # correct shape. + raise ValueError( + "The training data after the early stopping split " + "is missing some classes. Try using another random " + "seed." + ) + else: + X_train, y_train, sample_weight_train = X, y, sample_weight + X_val = y_val = sample_weight_val = None + + n_samples = X_train.shape[0] + + # First time calling fit. + if not self._is_fitted(): + # init state + self._init_state() + + # fit initial model and initialize raw predictions + if self.init_ == "zero": + raw_predictions = np.zeros( + shape=(n_samples, self.n_trees_per_iteration_), + dtype=np.float64, + ) + else: + # XXX clean this once we have a support_sample_weight tag + if sample_weight_is_none: + self.init_.fit(X_train, y_train) + else: + msg = ( + "The initial estimator {} does not support sample " + "weights.".format(self.init_.__class__.__name__) + ) + try: + self.init_.fit( + X_train, y_train, sample_weight=sample_weight_train + ) + except TypeError as e: + if "unexpected keyword argument 'sample_weight'" in str(e): + # regular estimator without SW support + raise ValueError(msg) from e + else: # regular estimator whose input checking failed + raise + except ValueError as e: + if ( + "pass parameters to specific steps of " + "your pipeline using the " + "stepname__parameter" in str(e) + ): # pipeline + raise ValueError(msg) from e + else: # regular estimator whose input checking failed + raise + + raw_predictions = _init_raw_predictions( + X_train, self.init_, self._loss, is_classifier(self) + ) + + begin_at_stage = 0 + + # The rng state must be preserved if warm_start is True + self._rng = check_random_state(self.random_state) + + # warm start: this is not the first time fit was called + else: + # add more estimators to fitted model + # invariant: warm_start = True + if self.n_estimators < self.estimators_.shape[0]: + raise ValueError( + "n_estimators=%d must be larger or equal to " + "estimators_.shape[0]=%d when " + "warm_start==True" % (self.n_estimators, self.estimators_.shape[0]) + ) + begin_at_stage = self.estimators_.shape[0] + # The requirements of _raw_predict + # are more constrained than fit. It accepts only CSR + # matrices. Finite values have already been checked in _validate_data. + X_train = check_array( + X_train, + dtype=DTYPE, + order="C", + accept_sparse="csr", + ensure_all_finite=False, + ) + raw_predictions = self._raw_predict(X_train) + self._resize_state() + + # fit the boosting stages + n_stages = self._fit_stages( + X_train, + y_train, + raw_predictions, + sample_weight_train, + self._rng, + X_val, + y_val, + sample_weight_val, + begin_at_stage, + monitor, + ) + + # change shape of arrays after fit (early-stopping or additional ests) + if n_stages != self.estimators_.shape[0]: + self.estimators_ = self.estimators_[:n_stages] + self.train_score_ = self.train_score_[:n_stages] + if hasattr(self, "oob_improvement_"): + # OOB scores were computed + self.oob_improvement_ = self.oob_improvement_[:n_stages] + self.oob_scores_ = self.oob_scores_[:n_stages] + self.oob_score_ = self.oob_scores_[-1] + self.n_estimators_ = n_stages + return self + + def _fit_stages( + self, + X, + y, + raw_predictions, + sample_weight, + random_state, + X_val, + y_val, + sample_weight_val, + begin_at_stage=0, + monitor=None, + ): + """Iteratively fits the stages. + + For each stage it computes the progress (OOB, train score) + and delegates to ``_fit_stage``. + Returns the number of stages fit; might differ from ``n_estimators`` + due to early stopping. + """ + n_samples = X.shape[0] + do_oob = self.subsample < 1.0 + sample_mask = np.ones((n_samples,), dtype=bool) + n_inbag = max(1, int(self.subsample * n_samples)) + + if self.verbose: + verbose_reporter = VerboseReporter(verbose=self.verbose) + verbose_reporter.init(self, begin_at_stage) + + X_csc = csc_matrix(X) if issparse(X) else None + X_csr = csr_matrix(X) if issparse(X) else None + + if self.n_iter_no_change is not None: + loss_history = np.full(self.n_iter_no_change, np.inf) + # We create a generator to get the predictions for X_val after + # the addition of each successive stage + y_val_pred_iter = self._staged_raw_predict(X_val, check_input=False) + + # Older versions of GBT had its own loss functions. With the new common + # private loss function submodule _loss, we often are a factor of 2 + # away from the old version. Here we keep backward compatibility for + # oob_scores_ and oob_improvement_, even if the old way is quite + # inconsistent (sometimes the gradient is half the gradient, sometimes + # not). + if isinstance( + self._loss, + ( + HalfSquaredError, + HalfBinomialLoss, + ), + ): + factor = 2 + else: + factor = 1 + + # perform boosting iterations + i = begin_at_stage + for i in range(begin_at_stage, self.n_estimators): + # subsampling + if do_oob: + sample_mask = _random_sample_mask(n_samples, n_inbag, random_state) + y_oob_masked = y[~sample_mask] + sample_weight_oob_masked = sample_weight[~sample_mask] + if i == 0: # store the initial loss to compute the OOB score + initial_loss = factor * self._loss( + y_true=y_oob_masked, + raw_prediction=raw_predictions[~sample_mask], + sample_weight=sample_weight_oob_masked, + ) + + # fit next stage of trees + raw_predictions = self._fit_stage( + i, + X, + y, + raw_predictions, + sample_weight, + sample_mask, + random_state, + X_csc=X_csc, + X_csr=X_csr, + ) + + # track loss + if do_oob: + self.train_score_[i] = factor * self._loss( + y_true=y[sample_mask], + raw_prediction=raw_predictions[sample_mask], + sample_weight=sample_weight[sample_mask], + ) + self.oob_scores_[i] = factor * self._loss( + y_true=y_oob_masked, + raw_prediction=raw_predictions[~sample_mask], + sample_weight=sample_weight_oob_masked, + ) + previous_loss = initial_loss if i == 0 else self.oob_scores_[i - 1] + self.oob_improvement_[i] = previous_loss - self.oob_scores_[i] + self.oob_score_ = self.oob_scores_[-1] + else: + # no need to fancy index w/ no subsampling + self.train_score_[i] = factor * self._loss( + y_true=y, + raw_prediction=raw_predictions, + sample_weight=sample_weight, + ) + + if self.verbose > 0: + verbose_reporter.update(i, self) + + if monitor is not None: + early_stopping = monitor(i, self, locals()) + if early_stopping: + break + + # We also provide an early stopping based on the score from + # validation set (X_val, y_val), if n_iter_no_change is set + if self.n_iter_no_change is not None: + # By calling next(y_val_pred_iter), we get the predictions + # for X_val after the addition of the current stage + validation_loss = factor * self._loss( + y_val, next(y_val_pred_iter), sample_weight_val + ) + + # Require validation_score to be better (less) than at least + # one of the last n_iter_no_change evaluations + if np.any(validation_loss + self.tol < loss_history): + loss_history[i % len(loss_history)] = validation_loss + else: + break + + return i + 1 + + def _make_estimator(self, append=True): + # we don't need _make_estimator + raise NotImplementedError() + + def _raw_predict_init(self, X): + """Check input and compute raw predictions of the init estimator.""" + self._check_initialized() + X = self.estimators_[0, 0]._validate_X_predict(X, check_input=True) + if self.init_ == "zero": + raw_predictions = np.zeros( + shape=(X.shape[0], self.n_trees_per_iteration_), dtype=np.float64 + ) + else: + raw_predictions = _init_raw_predictions( + X, self.init_, self._loss, is_classifier(self) + ) + return raw_predictions + + def _raw_predict(self, X): + """Return the sum of the trees raw predictions (+ init estimator).""" + check_is_fitted(self) + raw_predictions = self._raw_predict_init(X) + predict_stages(self.estimators_, X, self.learning_rate, raw_predictions) + return raw_predictions + + def _staged_raw_predict(self, X, check_input=True): + """Compute raw predictions of ``X`` for each iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + check_input : bool, default=True + If False, the input arrays X will not be checked. + + Returns + ------- + raw_predictions : generator of ndarray of shape (n_samples, k) + The raw predictions of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + Regression and binary classification are special cases with + ``k == 1``, otherwise ``k==n_classes``. + """ + if check_input: + X = validate_data( + self, X, dtype=DTYPE, order="C", accept_sparse="csr", reset=False + ) + raw_predictions = self._raw_predict_init(X) + for i in range(self.estimators_.shape[0]): + predict_stage(self.estimators_, i, X, self.learning_rate, raw_predictions) + yield raw_predictions.copy() + + @property + def feature_importances_(self): + """The impurity-based feature importances. + + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + Returns + ------- + feature_importances_ : ndarray of shape (n_features,) + The values of this array sum to 1, unless all trees are single node + trees consisting of only the root node, in which case it will be an + array of zeros. + """ + self._check_initialized() + + relevant_trees = [ + tree + for stage in self.estimators_ + for tree in stage + if tree.tree_.node_count > 1 + ] + if not relevant_trees: + # degenerate case where all trees have only one node + return np.zeros(shape=self.n_features_in_, dtype=np.float64) + + relevant_feature_importances = [ + tree.tree_.compute_feature_importances(normalize=False) + for tree in relevant_trees + ] + avg_feature_importances = np.mean( + relevant_feature_importances, axis=0, dtype=np.float64 + ) + return avg_feature_importances / np.sum(avg_feature_importances) + + def _compute_partial_dependence_recursion(self, grid, target_features): + """Fast partial dependence computation. + + Parameters + ---------- + grid : ndarray of shape (n_samples, n_target_features), dtype=np.float32 + The grid points on which the partial dependence should be + evaluated. + target_features : ndarray of shape (n_target_features,), dtype=np.intp + The set of target features for which the partial dependence + should be evaluated. + + Returns + ------- + averaged_predictions : ndarray of shape \ + (n_trees_per_iteration_, n_samples) + The value of the partial dependence function on each grid point. + """ + if self.init is not None: + warnings.warn( + "Using recursion method with a non-constant init predictor " + "will lead to incorrect partial dependence values. " + "Got init=%s." % self.init, + UserWarning, + ) + grid = np.asarray(grid, dtype=DTYPE, order="C") + n_estimators, n_trees_per_stage = self.estimators_.shape + averaged_predictions = np.zeros( + (n_trees_per_stage, grid.shape[0]), dtype=np.float64, order="C" + ) + target_features = np.asarray(target_features, dtype=np.intp, order="C") + + for stage in range(n_estimators): + for k in range(n_trees_per_stage): + tree = self.estimators_[stage, k].tree_ + tree.compute_partial_dependence( + grid, target_features, averaged_predictions[k] + ) + averaged_predictions *= self.learning_rate + + return averaged_predictions + + def apply(self, X): + """Apply trees in the ensemble to X, return leaf indices. + + .. versionadded:: 0.17 + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will + be converted to a sparse ``csr_matrix``. + + Returns + ------- + X_leaves : array-like of shape (n_samples, n_estimators, n_classes) + For each datapoint x in X and for each tree in the ensemble, + return the index of the leaf x ends up in each estimator. + In the case of binary classification n_classes is 1. + """ + + self._check_initialized() + X = self.estimators_[0, 0]._validate_X_predict(X, check_input=True) + + # n_classes will be equal to 1 in the binary classification or the + # regression case. + n_estimators, n_classes = self.estimators_.shape + leaves = np.zeros((X.shape[0], n_estimators, n_classes)) + + for i in range(n_estimators): + for j in range(n_classes): + estimator = self.estimators_[i, j] + leaves[:, i, j] = estimator.apply(X, check_input=False) + + return leaves + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags + + +class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): + """Gradient Boosting for classification. + + This algorithm builds an additive model in a forward stage-wise fashion; it + allows for the optimization of arbitrary differentiable loss functions. In + each stage ``n_classes_`` regression trees are fit on the negative gradient + of the loss function, e.g. binary or multiclass log loss. Binary + classification is a special case where only a single regression tree is + induced. + + :class:`~sklearn.ensemble.HistGradientBoostingClassifier` is a much faster variant + of this algorithm for intermediate and large datasets (`n_samples >= 10_000`) and + supports monotonic constraints. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + loss : {'log_loss', 'exponential'}, default='log_loss' + The loss function to be optimized. 'log_loss' refers to binomial and + multinomial deviance, the same as used in logistic regression. + It is a good choice for classification with probabilistic outputs. + For loss 'exponential', gradient boosting recovers the AdaBoost algorithm. + + learning_rate : float, default=0.1 + Learning rate shrinks the contribution of each tree by `learning_rate`. + There is a trade-off between learning_rate and n_estimators. + Values must be in the range `[0.0, inf)`. + + For an example of the effects of this parameter and its interaction with + ``subsample``, see + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regularization.py`. + + n_estimators : int, default=100 + The number of boosting stages to perform. Gradient boosting + is fairly robust to over-fitting so a large number usually + results in better performance. + Values must be in the range `[1, inf)`. + + subsample : float, default=1.0 + The fraction of samples to be used for fitting the individual base + learners. If smaller than 1.0 this results in Stochastic Gradient + Boosting. `subsample` interacts with the parameter `n_estimators`. + Choosing `subsample < 1.0` leads to a reduction of variance + and an increase in bias. + Values must be in the range `(0.0, 1.0]`. + + criterion : {'friedman_mse', 'squared_error'}, default='friedman_mse' + The function to measure the quality of a split. Supported criteria are + 'friedman_mse' for the mean squared error with improvement score by + Friedman, 'squared_error' for mean squared error. The default value of + 'friedman_mse' is generally the best as it can provide a better + approximation in some cases. + + .. versionadded:: 0.18 + + min_samples_split : int or float, default=2 + The minimum number of samples required to split an internal node: + + - If int, values must be in the range `[2, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and `min_samples_split` + will be `ceil(min_samples_split * n_samples)`. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_samples_leaf : int or float, default=1 + The minimum number of samples required to be at a leaf node. + A split point at any depth will only be considered if it leaves at + least ``min_samples_leaf`` training samples in each of the left and + right branches. This may have the effect of smoothing the model, + especially in regression. + + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0)` and `min_samples_leaf` + will be `ceil(min_samples_leaf * n_samples)`. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_weight_fraction_leaf : float, default=0.0 + The minimum weighted fraction of the sum total of weights (of all + the input samples) required to be at a leaf node. Samples have + equal weight when sample_weight is not provided. + Values must be in the range `[0.0, 0.5]`. + + max_depth : int or None, default=3 + Maximum depth of the individual regression estimators. The maximum + depth limits the number of nodes in the tree. Tune this parameter + for best performance; the best value depends on the interaction + of the input variables. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + If int, values must be in the range `[1, inf)`. + + min_impurity_decrease : float, default=0.0 + A node will be split if this split induces a decrease of the impurity + greater than or equal to this value. + Values must be in the range `[0.0, inf)`. + + The weighted impurity decrease equation is the following:: + + N_t / N * (impurity - N_t_R / N_t * right_impurity + - N_t_L / N_t * left_impurity) + + where ``N`` is the total number of samples, ``N_t`` is the number of + samples at the current node, ``N_t_L`` is the number of samples in the + left child, and ``N_t_R`` is the number of samples in the right child. + + ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, + if ``sample_weight`` is passed. + + .. versionadded:: 0.19 + + init : estimator or 'zero', default=None + An estimator object that is used to compute the initial predictions. + ``init`` has to provide :term:`fit` and :term:`predict_proba`. If + 'zero', the initial raw predictions are set to zero. By default, a + ``DummyEstimator`` predicting the classes priors is used. + + random_state : int, RandomState instance or None, default=None + Controls the random seed given to each Tree estimator at each + boosting iteration. + In addition, it controls the random permutation of the features at + each split (see Notes for more details). + It also controls the random splitting of the training data to obtain a + validation set if `n_iter_no_change` is not None. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + max_features : {'sqrt', 'log2'}, int or float, default=None + The number of features to consider when looking for the best split: + + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and the features + considered at each split will be `max(1, int(max_features * n_features_in_))`. + - If 'sqrt', then `max_features=sqrt(n_features)`. + - If 'log2', then `max_features=log2(n_features)`. + - If None, then `max_features=n_features`. + + Choosing `max_features < n_features` leads to a reduction of variance + and an increase in bias. + + Note: the search for a split does not stop until at least one + valid partition of the node samples is found, even if it requires to + effectively inspect more than ``max_features`` features. + + verbose : int, default=0 + Enable verbose output. If 1 then it prints progress and performance + once in a while (the more trees the lower the frequency). If greater + than 1 then it prints progress and performance for every tree. + Values must be in the range `[0, inf)`. + + max_leaf_nodes : int, default=None + Grow trees with ``max_leaf_nodes`` in best-first fashion. + Best nodes are defined as relative reduction in impurity. + Values must be in the range `[2, inf)`. + If `None`, then unlimited number of leaf nodes. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just erase the + previous solution. See :term:`the Glossary `. + + validation_fraction : float, default=0.1 + The proportion of training data to set aside as validation set for + early stopping. Values must be in the range `(0.0, 1.0)`. + Only used if ``n_iter_no_change`` is set to an integer. + + .. versionadded:: 0.20 + + n_iter_no_change : int, default=None + ``n_iter_no_change`` is used to decide if early stopping will be used + to terminate training when validation score is not improving. By + default it is set to None to disable early stopping. If set to a + number, it will set aside ``validation_fraction`` size of the training + data as validation and terminate training when validation score is not + improving in all of the previous ``n_iter_no_change`` numbers of + iterations. The split is stratified. + Values must be in the range `[1, inf)`. + See + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_early_stopping.py`. + + .. versionadded:: 0.20 + + tol : float, default=1e-4 + Tolerance for the early stopping. When the loss is not improving + by at least tol for ``n_iter_no_change`` iterations (if set to a + number), the training stops. + Values must be in the range `[0.0, inf)`. + + .. versionadded:: 0.20 + + ccp_alpha : non-negative float, default=0.0 + Complexity parameter used for Minimal Cost-Complexity Pruning. The + subtree with the largest cost complexity that is smaller than + ``ccp_alpha`` will be chosen. By default, no pruning is performed. + Values must be in the range `[0.0, inf)`. + See :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. + + .. versionadded:: 0.22 + + Attributes + ---------- + n_estimators_ : int + The number of estimators as selected by early stopping (if + ``n_iter_no_change`` is specified). Otherwise it is set to + ``n_estimators``. + + .. versionadded:: 0.20 + + n_trees_per_iteration_ : int + The number of trees that are built at each iteration. For binary classifiers, + this is always 1. + + .. versionadded:: 1.4.0 + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + oob_improvement_ : ndarray of shape (n_estimators,) + The improvement in loss on the out-of-bag samples + relative to the previous iteration. + ``oob_improvement_[0]`` is the improvement in + loss of the first stage over the ``init`` estimator. + Only available if ``subsample < 1.0``. + + oob_scores_ : ndarray of shape (n_estimators,) + The full history of the loss values on the out-of-bag + samples. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 + + oob_score_ : float + The last value of the loss on the out-of-bag samples. It is + the same as `oob_scores_[-1]`. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 + + train_score_ : ndarray of shape (n_estimators,) + The i-th score ``train_score_[i]`` is the loss of the + model at iteration ``i`` on the in-bag sample. + If ``subsample == 1`` this is the loss on the training data. + + init_ : estimator + The estimator that provides the initial predictions. Set via the ``init`` + argument. + + estimators_ : ndarray of DecisionTreeRegressor of \ + shape (n_estimators, ``n_trees_per_iteration_``) + The collection of fitted sub-estimators. ``n_trees_per_iteration_`` is 1 for + binary classification, otherwise ``n_classes``. + + classes_ : ndarray of shape (n_classes,) + The classes labels. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + n_classes_ : int + The number of classes. + + max_features_ : int + The inferred value of max_features. + + See Also + -------- + HistGradientBoostingClassifier : Histogram-based Gradient Boosting + Classification Tree. + sklearn.tree.DecisionTreeClassifier : A decision tree classifier. + RandomForestClassifier : A meta-estimator that fits a number of decision + tree classifiers on various sub-samples of the dataset and uses + averaging to improve the predictive accuracy and control over-fitting. + AdaBoostClassifier : A meta-estimator that begins by fitting a classifier + on the original dataset and then fits additional copies of the + classifier on the same dataset where the weights of incorrectly + classified instances are adjusted such that subsequent classifiers + focus more on difficult cases. + + Notes + ----- + The features are always randomly permuted at each split. Therefore, + the best found split may vary, even with the same training data and + ``max_features=n_features``, if the improvement of the criterion is + identical for several splits enumerated during the search of the best + split. To obtain a deterministic behaviour during fitting, + ``random_state`` has to be fixed. + + References + ---------- + J. Friedman, Greedy Function Approximation: A Gradient Boosting + Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. + + J. Friedman, Stochastic Gradient Boosting, 1999 + + T. Hastie, R. Tibshirani and J. Friedman. + Elements of Statistical Learning Ed. 2, Springer, 2009. + + Examples + -------- + The following example shows how to fit a gradient boosting classifier with + 100 decision stumps as weak learners. + + >>> from sklearn.datasets import make_hastie_10_2 + >>> from sklearn.ensemble import GradientBoostingClassifier + + >>> X, y = make_hastie_10_2(random_state=0) + >>> X_train, X_test = X[:2000], X[2000:] + >>> y_train, y_test = y[:2000], y[2000:] + + >>> clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, + ... max_depth=1, random_state=0).fit(X_train, y_train) + >>> clf.score(X_test, y_test) + 0.913 + """ + + _parameter_constraints: dict = { + **BaseGradientBoosting._parameter_constraints, + "loss": [StrOptions({"log_loss", "exponential"})], + "init": [StrOptions({"zero"}), None, HasMethods(["fit", "predict_proba"])], + } + + def __init__( + self, + *, + loss="log_loss", + learning_rate=0.1, + n_estimators=100, + subsample=1.0, + criterion="friedman_mse", + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_depth=3, + min_impurity_decrease=0.0, + init=None, + random_state=None, + max_features=None, + verbose=0, + max_leaf_nodes=None, + warm_start=False, + validation_fraction=0.1, + n_iter_no_change=None, + tol=1e-4, + ccp_alpha=0.0, + ): + super().__init__( + loss=loss, + learning_rate=learning_rate, + n_estimators=n_estimators, + criterion=criterion, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + min_weight_fraction_leaf=min_weight_fraction_leaf, + max_depth=max_depth, + init=init, + subsample=subsample, + max_features=max_features, + random_state=random_state, + verbose=verbose, + max_leaf_nodes=max_leaf_nodes, + min_impurity_decrease=min_impurity_decrease, + warm_start=warm_start, + validation_fraction=validation_fraction, + n_iter_no_change=n_iter_no_change, + tol=tol, + ccp_alpha=ccp_alpha, + ) + + def _encode_y(self, y, sample_weight): + # encode classes into 0 ... n_classes - 1 and sets attributes classes_ + # and n_trees_per_iteration_ + check_classification_targets(y) + + label_encoder = LabelEncoder() + encoded_y_int = label_encoder.fit_transform(y) + self.classes_ = label_encoder.classes_ + n_classes = self.classes_.shape[0] + # only 1 tree for binary classification. For multiclass classification, + # we build 1 tree per class. + self.n_trees_per_iteration_ = 1 if n_classes <= 2 else n_classes + encoded_y = encoded_y_int.astype(float, copy=False) + + # From here on, it is additional to the HGBT case. + # expose n_classes_ attribute + self.n_classes_ = n_classes + if sample_weight is None: + n_trim_classes = n_classes + else: + n_trim_classes = np.count_nonzero(np.bincount(encoded_y_int, sample_weight)) + + if n_trim_classes < 2: + raise ValueError( + "y contains %d class after sample_weight " + "trimmed classes with zero weights, while a " + "minimum of 2 classes are required." % n_trim_classes + ) + return encoded_y + + def _get_loss(self, sample_weight): + if self.loss == "log_loss": + if self.n_classes_ == 2: + return HalfBinomialLoss(sample_weight=sample_weight) + else: + return HalfMultinomialLoss( + sample_weight=sample_weight, n_classes=self.n_classes_ + ) + elif self.loss == "exponential": + if self.n_classes_ > 2: + raise ValueError( + f"loss='{self.loss}' is only suitable for a binary classification " + f"problem, you have n_classes={self.n_classes_}. " + "Please use loss='log_loss' instead." + ) + else: + return ExponentialLoss(sample_weight=sample_weight) + + def decision_function(self, X): + """Compute the decision function of ``X``. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Returns + ------- + score : ndarray of shape (n_samples, n_classes) or (n_samples,) + The decision function of the input samples, which corresponds to + the raw values predicted from the trees of the ensemble . The + order of the classes corresponds to that in the attribute + :term:`classes_`. Regression and binary classification produce an + array of shape (n_samples,). + """ + X = validate_data( + self, X, dtype=DTYPE, order="C", accept_sparse="csr", reset=False + ) + raw_predictions = self._raw_predict(X) + if raw_predictions.shape[1] == 1: + return raw_predictions.ravel() + return raw_predictions + + def staged_decision_function(self, X): + """Compute decision function of ``X`` for each iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Yields + ------ + score : generator of ndarray of shape (n_samples, k) + The decision function of the input samples, which corresponds to + the raw values predicted from the trees of the ensemble . The + classes corresponds to that in the attribute :term:`classes_`. + Regression and binary classification are special cases with + ``k == 1``, otherwise ``k==n_classes``. + """ + yield from self._staged_raw_predict(X) + + def predict(self, X): + """Predict class for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Returns + ------- + y : ndarray of shape (n_samples,) + The predicted values. + """ + raw_predictions = self.decision_function(X) + if raw_predictions.ndim == 1: # decision_function already squeezed it + encoded_classes = (raw_predictions >= 0).astype(int) + else: + encoded_classes = np.argmax(raw_predictions, axis=1) + return self.classes_[encoded_classes] + + def staged_predict(self, X): + """Predict class at each stage for X. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted value of the input samples. + """ + if self.n_classes_ == 2: # n_trees_per_iteration_ = 1 + for raw_predictions in self._staged_raw_predict(X): + encoded_classes = (raw_predictions.squeeze() >= 0).astype(int) + yield self.classes_.take(encoded_classes, axis=0) + else: + for raw_predictions in self._staged_raw_predict(X): + encoded_classes = np.argmax(raw_predictions, axis=1) + yield self.classes_.take(encoded_classes, axis=0) + + def predict_proba(self, X): + """Predict class probabilities for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes) + The class probabilities of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + + Raises + ------ + AttributeError + If the ``loss`` does not support probabilities. + """ + raw_predictions = self.decision_function(X) + return self._loss.predict_proba(raw_predictions) + + def predict_log_proba(self, X): + """Predict class log-probabilities for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes) + The class log-probabilities of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + + Raises + ------ + AttributeError + If the ``loss`` does not support probabilities. + """ + proba = self.predict_proba(X) + return np.log(proba) + + def staged_predict_proba(self, X): + """Predict class probabilities at each stage for X. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted value of the input samples. + """ + try: + for raw_predictions in self._staged_raw_predict(X): + yield self._loss.predict_proba(raw_predictions) + except NotFittedError: + raise + except AttributeError as e: + raise AttributeError( + "loss=%r does not support predict_proba" % self.loss + ) from e + + +class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): + """Gradient Boosting for regression. + + This estimator builds an additive model in a forward stage-wise fashion; it + allows for the optimization of arbitrary differentiable loss functions. In + each stage a regression tree is fit on the negative gradient of the given + loss function. + + :class:`~sklearn.ensemble.HistGradientBoostingRegressor` is a much faster variant + of this algorithm for intermediate and large datasets (`n_samples >= 10_000`) and + supports monotonic constraints. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + loss : {'squared_error', 'absolute_error', 'huber', 'quantile'}, \ + default='squared_error' + Loss function to be optimized. 'squared_error' refers to the squared + error for regression. 'absolute_error' refers to the absolute error of + regression and is a robust loss function. 'huber' is a + combination of the two. 'quantile' allows quantile regression (use + `alpha` to specify the quantile). + See + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py` + for an example that demonstrates quantile regression for creating + prediction intervals with `loss='quantile'`. + + learning_rate : float, default=0.1 + Learning rate shrinks the contribution of each tree by `learning_rate`. + There is a trade-off between learning_rate and n_estimators. + Values must be in the range `[0.0, inf)`. + + n_estimators : int, default=100 + The number of boosting stages to perform. Gradient boosting + is fairly robust to over-fitting so a large number usually + results in better performance. + Values must be in the range `[1, inf)`. + + subsample : float, default=1.0 + The fraction of samples to be used for fitting the individual base + learners. If smaller than 1.0 this results in Stochastic Gradient + Boosting. `subsample` interacts with the parameter `n_estimators`. + Choosing `subsample < 1.0` leads to a reduction of variance + and an increase in bias. + Values must be in the range `(0.0, 1.0]`. + + criterion : {'friedman_mse', 'squared_error'}, default='friedman_mse' + The function to measure the quality of a split. Supported criteria are + "friedman_mse" for the mean squared error with improvement score by + Friedman, "squared_error" for mean squared error. The default value of + "friedman_mse" is generally the best as it can provide a better + approximation in some cases. + + .. versionadded:: 0.18 + + min_samples_split : int or float, default=2 + The minimum number of samples required to split an internal node: + + - If int, values must be in the range `[2, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and `min_samples_split` + will be `ceil(min_samples_split * n_samples)`. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_samples_leaf : int or float, default=1 + The minimum number of samples required to be at a leaf node. + A split point at any depth will only be considered if it leaves at + least ``min_samples_leaf`` training samples in each of the left and + right branches. This may have the effect of smoothing the model, + especially in regression. + + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0)` and `min_samples_leaf` + will be `ceil(min_samples_leaf * n_samples)`. + + .. versionchanged:: 0.18 + Added float values for fractions. + + min_weight_fraction_leaf : float, default=0.0 + The minimum weighted fraction of the sum total of weights (of all + the input samples) required to be at a leaf node. Samples have + equal weight when sample_weight is not provided. + Values must be in the range `[0.0, 0.5]`. + + max_depth : int or None, default=3 + Maximum depth of the individual regression estimators. The maximum + depth limits the number of nodes in the tree. Tune this parameter + for best performance; the best value depends on the interaction + of the input variables. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + If int, values must be in the range `[1, inf)`. + + min_impurity_decrease : float, default=0.0 + A node will be split if this split induces a decrease of the impurity + greater than or equal to this value. + Values must be in the range `[0.0, inf)`. + + The weighted impurity decrease equation is the following:: + + N_t / N * (impurity - N_t_R / N_t * right_impurity + - N_t_L / N_t * left_impurity) + + where ``N`` is the total number of samples, ``N_t`` is the number of + samples at the current node, ``N_t_L`` is the number of samples in the + left child, and ``N_t_R`` is the number of samples in the right child. + + ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, + if ``sample_weight`` is passed. + + .. versionadded:: 0.19 + + init : estimator or 'zero', default=None + An estimator object that is used to compute the initial predictions. + ``init`` has to provide :term:`fit` and :term:`predict`. If 'zero', the + initial raw predictions are set to zero. By default a + ``DummyEstimator`` is used, predicting either the average target value + (for loss='squared_error'), or a quantile for the other losses. + + random_state : int, RandomState instance or None, default=None + Controls the random seed given to each Tree estimator at each + boosting iteration. + In addition, it controls the random permutation of the features at + each split (see Notes for more details). + It also controls the random splitting of the training data to obtain a + validation set if `n_iter_no_change` is not None. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + max_features : {'sqrt', 'log2'}, int or float, default=None + The number of features to consider when looking for the best split: + + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and the features + considered at each split will be `max(1, int(max_features * n_features_in_))`. + - If "sqrt", then `max_features=sqrt(n_features)`. + - If "log2", then `max_features=log2(n_features)`. + - If None, then `max_features=n_features`. + + Choosing `max_features < n_features` leads to a reduction of variance + and an increase in bias. + + Note: the search for a split does not stop until at least one + valid partition of the node samples is found, even if it requires to + effectively inspect more than ``max_features`` features. + + alpha : float, default=0.9 + The alpha-quantile of the huber loss function and the quantile + loss function. Only if ``loss='huber'`` or ``loss='quantile'``. + Values must be in the range `(0.0, 1.0)`. + + verbose : int, default=0 + Enable verbose output. If 1 then it prints progress and performance + once in a while (the more trees the lower the frequency). If greater + than 1 then it prints progress and performance for every tree. + Values must be in the range `[0, inf)`. + + max_leaf_nodes : int, default=None + Grow trees with ``max_leaf_nodes`` in best-first fashion. + Best nodes are defined as relative reduction in impurity. + Values must be in the range `[2, inf)`. + If None, then unlimited number of leaf nodes. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just erase the + previous solution. See :term:`the Glossary `. + + validation_fraction : float, default=0.1 + The proportion of training data to set aside as validation set for + early stopping. Values must be in the range `(0.0, 1.0)`. + Only used if ``n_iter_no_change`` is set to an integer. + + .. versionadded:: 0.20 + + n_iter_no_change : int, default=None + ``n_iter_no_change`` is used to decide if early stopping will be used + to terminate training when validation score is not improving. By + default it is set to None to disable early stopping. If set to a + number, it will set aside ``validation_fraction`` size of the training + data as validation and terminate training when validation score is not + improving in all of the previous ``n_iter_no_change`` numbers of + iterations. + Values must be in the range `[1, inf)`. + See + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_early_stopping.py`. + + .. versionadded:: 0.20 + + tol : float, default=1e-4 + Tolerance for the early stopping. When the loss is not improving + by at least tol for ``n_iter_no_change`` iterations (if set to a + number), the training stops. + Values must be in the range `[0.0, inf)`. + + .. versionadded:: 0.20 + + ccp_alpha : non-negative float, default=0.0 + Complexity parameter used for Minimal Cost-Complexity Pruning. The + subtree with the largest cost complexity that is smaller than + ``ccp_alpha`` will be chosen. By default, no pruning is performed. + Values must be in the range `[0.0, inf)`. + See :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. + + .. versionadded:: 0.22 + + Attributes + ---------- + n_estimators_ : int + The number of estimators as selected by early stopping (if + ``n_iter_no_change`` is specified). Otherwise it is set to + ``n_estimators``. + + n_trees_per_iteration_ : int + The number of trees that are built at each iteration. For regressors, this is + always 1. + + .. versionadded:: 1.4.0 + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + oob_improvement_ : ndarray of shape (n_estimators,) + The improvement in loss on the out-of-bag samples + relative to the previous iteration. + ``oob_improvement_[0]`` is the improvement in + loss of the first stage over the ``init`` estimator. + Only available if ``subsample < 1.0``. + + oob_scores_ : ndarray of shape (n_estimators,) + The full history of the loss values on the out-of-bag + samples. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 + + oob_score_ : float + The last value of the loss on the out-of-bag samples. It is + the same as `oob_scores_[-1]`. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 + + train_score_ : ndarray of shape (n_estimators,) + The i-th score ``train_score_[i]`` is the loss of the + model at iteration ``i`` on the in-bag sample. + If ``subsample == 1`` this is the loss on the training data. + + init_ : estimator + The estimator that provides the initial predictions. Set via the ``init`` + argument. + + estimators_ : ndarray of DecisionTreeRegressor of shape (n_estimators, 1) + The collection of fitted sub-estimators. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + max_features_ : int + The inferred value of max_features. + + See Also + -------- + HistGradientBoostingRegressor : Histogram-based Gradient Boosting + Classification Tree. + sklearn.tree.DecisionTreeRegressor : A decision tree regressor. + sklearn.ensemble.RandomForestRegressor : A random forest regressor. + + Notes + ----- + The features are always randomly permuted at each split. Therefore, + the best found split may vary, even with the same training data and + ``max_features=n_features``, if the improvement of the criterion is + identical for several splits enumerated during the search of the best + split. To obtain a deterministic behaviour during fitting, + ``random_state`` has to be fixed. + + References + ---------- + J. Friedman, Greedy Function Approximation: A Gradient Boosting + Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. + + J. Friedman, Stochastic Gradient Boosting, 1999 + + T. Hastie, R. Tibshirani and J. Friedman. + Elements of Statistical Learning Ed. 2, Springer, 2009. + + Examples + -------- + >>> from sklearn.datasets import make_regression + >>> from sklearn.ensemble import GradientBoostingRegressor + >>> from sklearn.model_selection import train_test_split + >>> X, y = make_regression(random_state=0) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> reg = GradientBoostingRegressor(random_state=0) + >>> reg.fit(X_train, y_train) + GradientBoostingRegressor(random_state=0) + >>> reg.predict(X_test[1:2]) + array([-61.1]) + >>> reg.score(X_test, y_test) + 0.4... + + For a detailed example of utilizing + :class:`~sklearn.ensemble.GradientBoostingRegressor` + to fit an ensemble of weak predictive models, please refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py`. + """ + + _parameter_constraints: dict = { + **BaseGradientBoosting._parameter_constraints, + "loss": [StrOptions({"squared_error", "absolute_error", "huber", "quantile"})], + "init": [StrOptions({"zero"}), None, HasMethods(["fit", "predict"])], + "alpha": [Interval(Real, 0.0, 1.0, closed="neither")], + } + + def __init__( + self, + *, + loss="squared_error", + learning_rate=0.1, + n_estimators=100, + subsample=1.0, + criterion="friedman_mse", + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_depth=3, + min_impurity_decrease=0.0, + init=None, + random_state=None, + max_features=None, + alpha=0.9, + verbose=0, + max_leaf_nodes=None, + warm_start=False, + validation_fraction=0.1, + n_iter_no_change=None, + tol=1e-4, + ccp_alpha=0.0, + ): + super().__init__( + loss=loss, + learning_rate=learning_rate, + n_estimators=n_estimators, + criterion=criterion, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + min_weight_fraction_leaf=min_weight_fraction_leaf, + max_depth=max_depth, + init=init, + subsample=subsample, + max_features=max_features, + min_impurity_decrease=min_impurity_decrease, + random_state=random_state, + alpha=alpha, + verbose=verbose, + max_leaf_nodes=max_leaf_nodes, + warm_start=warm_start, + validation_fraction=validation_fraction, + n_iter_no_change=n_iter_no_change, + tol=tol, + ccp_alpha=ccp_alpha, + ) + + def _encode_y(self, y=None, sample_weight=None): + # Just convert y to the expected dtype + self.n_trees_per_iteration_ = 1 + y = y.astype(DOUBLE, copy=False) + return y + + def _get_loss(self, sample_weight): + if self.loss in ("quantile", "huber"): + return _LOSSES[self.loss](sample_weight=sample_weight, quantile=self.alpha) + else: + return _LOSSES[self.loss](sample_weight=sample_weight) + + def predict(self, X): + """Predict regression target for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Returns + ------- + y : ndarray of shape (n_samples,) + The predicted values. + """ + X = validate_data( + self, X, dtype=DTYPE, order="C", accept_sparse="csr", reset=False + ) + # In regression we can directly return the raw value from the trees. + return self._raw_predict(X).ravel() + + def staged_predict(self, X): + """Predict regression target at each stage for X. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted value of the input samples. + """ + for raw_predictions in self._staged_raw_predict(X): + yield raw_predictions.ravel() + + def apply(self, X): + """Apply trees in the ensemble to X, return leaf indices. + + .. versionadded:: 0.17 + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will + be converted to a sparse ``csr_matrix``. + + Returns + ------- + X_leaves : array-like of shape (n_samples, n_estimators) + For each datapoint x in X and for each tree in the ensemble, + return the index of the leaf x ends up in each estimator. + """ + + leaves = super().apply(X) + leaves = leaves.reshape(X.shape[0], self.estimators_.shape[0]) + return leaves diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..9e3ac36268824b281e127e9a911755121627d5fb Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..094c5b81a2148c1c44a9f6febbbdbd5da368ab34 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.pyx new file mode 100644 index 0000000000000000000000000000000000000000..d75abdb65a8363868622cc6659fb3377bb947841 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_gradient_boosting.pyx @@ -0,0 +1,262 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from libc.stdlib cimport free +from libc.string cimport memset + +import numpy as np +from scipy.sparse import issparse + +from sklearn.utils._typedefs cimport float32_t, float64_t, intp_t, int32_t, uint8_t +# Note: _tree uses cimport numpy, cnp.import_array, so we need to include +# numpy headers in the build configuration of this extension +from sklearn.tree._tree cimport Node +from sklearn.tree._tree cimport Tree +from sklearn.tree._utils cimport safe_realloc + + +# no namespace lookup for numpy dtype and array creation +from numpy import zeros as np_zeros + + +# constant to mark tree leafs +cdef intp_t TREE_LEAF = -1 + +cdef void _predict_regression_tree_inplace_fast_dense( + const float32_t[:, ::1] X, + Node* root_node, + double *value, + double scale, + Py_ssize_t k, + float64_t[:, :] out +) noexcept nogil: + """Predicts output for regression tree and stores it in ``out[i, k]``. + + This function operates directly on the data arrays of the tree + data structures. This is 5x faster than the variant above because + it allows us to avoid buffer validation. + + The function assumes that the ndarray that wraps ``X`` is + c-continuous. + + Parameters + ---------- + X : float32_t 2d memory view + The memory view on the data ndarray of the input ``X``. + Assumes that the array is c-continuous. + root_node : tree Node pointer + Pointer to the main node array of the :class:``sklearn.tree.Tree``. + value : np.float64_t pointer + The pointer to the data array of the ``value`` array attribute + of the :class:``sklearn.tree.Tree``. + scale : double + A constant to scale the predictions. + k : int + The index of the tree output to be predicted. Must satisfy + 0 <= ``k`` < ``K``. + out : memory view on array of type np.float64_t + The data array where the predictions are stored. + ``out`` is assumed to be a two-dimensional array of + shape ``(n_samples, K)``. + """ + cdef intp_t n_samples = X.shape[0] + cdef Py_ssize_t i + cdef Node *node + for i in range(n_samples): + node = root_node + # While node not a leaf + while node.left_child != TREE_LEAF: + if X[i, node.feature] <= node.threshold: + node = root_node + node.left_child + else: + node = root_node + node.right_child + out[i, k] += scale * value[node - root_node] + + +def _predict_regression_tree_stages_sparse( + object[:, :] estimators, + object X, + double scale, + float64_t[:, :] out +): + """Predicts output for regression tree inplace and adds scaled value to ``out[i, k]``. + + The function assumes that the ndarray that wraps ``X`` is csr_matrix. + """ + cdef const float32_t[::1] X_data = X.data + cdef const int32_t[::1] X_indices = X.indices + cdef const int32_t[::1] X_indptr = X.indptr + + cdef intp_t n_samples = X.shape[0] + cdef intp_t n_features = X.shape[1] + cdef intp_t n_stages = estimators.shape[0] + cdef intp_t n_outputs = estimators.shape[1] + + # Indices and temporary variables + cdef intp_t sample_i + cdef intp_t feature_i + cdef intp_t stage_i + cdef intp_t output_i + cdef Node *root_node = NULL + cdef Node *node = NULL + cdef double *value = NULL + + cdef Tree tree + cdef Node** nodes = NULL + cdef double** values = NULL + safe_realloc(&nodes, n_stages * n_outputs) + safe_realloc(&values, n_stages * n_outputs) + for stage_i in range(n_stages): + for output_i in range(n_outputs): + tree = estimators[stage_i, output_i].tree_ + nodes[stage_i * n_outputs + output_i] = tree.nodes + values[stage_i * n_outputs + output_i] = tree.value + + # Initialize auxiliary data-structure + cdef float32_t feature_value = 0. + cdef float32_t* X_sample = NULL + + # feature_to_sample as a data structure records the last seen sample + # for each feature; functionally, it is an efficient way to identify + # which features are nonzero in the present sample. + cdef intp_t* feature_to_sample = NULL + + safe_realloc(&X_sample, n_features) + safe_realloc(&feature_to_sample, n_features) + + memset(feature_to_sample, -1, n_features * sizeof(intp_t)) + + # Cycle through all samples + for sample_i in range(n_samples): + for feature_i in range(X_indptr[sample_i], X_indptr[sample_i + 1]): + feature_to_sample[X_indices[feature_i]] = sample_i + X_sample[X_indices[feature_i]] = X_data[feature_i] + + # Cycle through all stages + for stage_i in range(n_stages): + # Cycle through all trees + for output_i in range(n_outputs): + root_node = nodes[stage_i * n_outputs + output_i] + value = values[stage_i * n_outputs + output_i] + node = root_node + + # While node not a leaf + while node.left_child != TREE_LEAF: + # ... and node.right_child != TREE_LEAF: + if feature_to_sample[node.feature] == sample_i: + feature_value = X_sample[node.feature] + else: + feature_value = 0. + + if feature_value <= node.threshold: + node = root_node + node.left_child + else: + node = root_node + node.right_child + out[sample_i, output_i] += scale * value[node - root_node] + + # Free auxiliary arrays + free(X_sample) + free(feature_to_sample) + free(nodes) + free(values) + + +def predict_stages( + object[:, :] estimators, + object X, + double scale, + float64_t[:, :] out +): + """Add predictions of ``estimators`` to ``out``. + + Each estimator is scaled by ``scale`` before its prediction + is added to ``out``. + """ + cdef Py_ssize_t i + cdef Py_ssize_t k + cdef Py_ssize_t n_estimators = estimators.shape[0] + cdef Py_ssize_t K = estimators.shape[1] + cdef Tree tree + + if issparse(X): + if X.format != 'csr': + raise ValueError("When X is a sparse matrix, a CSR format is" + " expected, got {!r}".format(type(X))) + _predict_regression_tree_stages_sparse( + estimators=estimators, X=X, scale=scale, out=out + ) + else: + if not isinstance(X, np.ndarray) or np.isfortran(X): + raise ValueError(f"X should be C-ordered np.ndarray, got {type(X)}") + + for i in range(n_estimators): + for k in range(K): + tree = estimators[i, k].tree_ + + # avoid buffer validation by casting to ndarray + # and get data pointer + # need brackets because of casting operator priority + _predict_regression_tree_inplace_fast_dense( + X=X, + root_node=tree.nodes, + value=tree.value, + scale=scale, + k=k, + out=out + ) + # out[:, k] += scale * tree.predict(X).ravel() + + +def predict_stage( + object[:, :] estimators, + int stage, + object X, + double scale, + float64_t[:, :] out +): + """Add predictions of ``estimators[stage]`` to ``out``. + + Each estimator in the stage is scaled by ``scale`` before + its prediction is added to ``out``. + """ + return predict_stages( + estimators=estimators[stage:stage + 1], X=X, scale=scale, out=out + ) + + +def _random_sample_mask( + intp_t n_total_samples, + intp_t n_total_in_bag, + random_state +): + """Create a random sample mask where ``n_total_in_bag`` elements are set. + + Parameters + ---------- + n_total_samples : int + The length of the resulting mask. + + n_total_in_bag : int + The number of elements in the sample mask which are set to 1. + + random_state : RandomState + A numpy ``RandomState`` object. + + Returns + ------- + sample_mask : np.ndarray, shape=[n_total_samples] + An ndarray where ``n_total_in_bag`` elements are set to ``True`` + the others are ``False``. + """ + cdef float64_t[::1] rand = random_state.uniform(size=n_total_samples) + cdef uint8_t[::1] sample_mask = np_zeros((n_total_samples,), dtype=bool) + + cdef intp_t n_bagged = 0 + cdef intp_t i = 0 + + for i in range(n_total_samples): + if rand[i] * (n_total_samples - i) < (n_total_in_bag - n_bagged): + sample_mask[i] = 1 + n_bagged += 1 + + return sample_mask.base diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f666c11720de19844b20ffc5f4f6ffccc2787757 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/__init__.py @@ -0,0 +1,8 @@ +"""This module implements histogram-based gradient boosting estimators. + +The implementation is a port from pygbm which is itself strongly inspired +from LightGBM. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..977412a62a01e524ccfe3b4780ff153b257e2dd3 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..18b460f5de5393330cc92a969190a223725c3f0e Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.pyx new file mode 100644 index 0000000000000000000000000000000000000000..e4c972cc4d29b97ba53d62213f32fdf374f7adf4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_binning.pyx @@ -0,0 +1,85 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from cython.parallel import prange +from libc.math cimport isnan + +from sklearn.ensemble._hist_gradient_boosting.common cimport X_DTYPE_C, X_BINNED_DTYPE_C +from sklearn.utils._typedefs cimport uint8_t + + +def _map_to_bins(const X_DTYPE_C [:, :] data, + list binning_thresholds, + const uint8_t[::1] is_categorical, + const uint8_t missing_values_bin_idx, + int n_threads, + X_BINNED_DTYPE_C [::1, :] binned): + """Bin continuous and categorical values to discrete integer-coded levels. + + A given value x is mapped into bin value i iff + thresholds[i - 1] < x <= thresholds[i] + + Parameters + ---------- + data : ndarray, shape (n_samples, n_features) + The data to bin. + binning_thresholds : list of arrays + For each feature, stores the increasing numeric values that are + used to separate the bins. + is_categorical : ndarray of uint8_t of shape (n_features,) + Indicates categorical features. + n_threads : int + Number of OpenMP threads to use. + binned : ndarray, shape (n_samples, n_features) + Output array, must be fortran aligned. + """ + cdef: + int feature_idx + + for feature_idx in range(data.shape[1]): + _map_col_to_bins( + data[:, feature_idx], + binning_thresholds[feature_idx], + is_categorical[feature_idx], + missing_values_bin_idx, + n_threads, + binned[:, feature_idx] + ) + + +cdef void _map_col_to_bins( + const X_DTYPE_C [:] data, + const X_DTYPE_C [:] binning_thresholds, + const uint8_t is_categorical, + const uint8_t missing_values_bin_idx, + int n_threads, + X_BINNED_DTYPE_C [:] binned +): + """Binary search to find the bin index for each value in the data.""" + cdef: + int i + int left + int right + int middle + + for i in prange(data.shape[0], schedule='static', nogil=True, + num_threads=n_threads): + if ( + isnan(data[i]) or + # To follow LightGBM's conventions, negative values for + # categorical features are considered as missing values. + (is_categorical and data[i] < 0) + ): + binned[i] = missing_values_bin_idx + else: + # for known values, use binary search + left, right = 0, binning_thresholds.shape[0] + while left < right: + # equal to (right + left - 1) // 2 but avoids overflow + middle = left + (right - left - 1) // 2 + if data[i] <= binning_thresholds[middle]: + right = middle + else: + left = middle + 1 + + binned[i] = left diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..e5281e42e9a1f6fa8924994422b7f50781dc4808 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..526d97ba4c5dce2edc6b4c7d409b083b9052ab2f Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.pxd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.pxd new file mode 100644 index 0000000000000000000000000000000000000000..dcee372b3cdbc55bd54e073f9734296f1ab76a6e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.pxd @@ -0,0 +1,20 @@ +from sklearn.ensemble._hist_gradient_boosting.common cimport X_BINNED_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport BITSET_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport BITSET_INNER_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport X_DTYPE_C +from sklearn.utils._typedefs cimport uint8_t + + +cdef void init_bitset(BITSET_DTYPE_C bitset) noexcept nogil + +cdef void set_bitset(BITSET_DTYPE_C bitset, X_BINNED_DTYPE_C val) noexcept nogil + +cdef uint8_t in_bitset(BITSET_DTYPE_C bitset, X_BINNED_DTYPE_C val) noexcept nogil + +cpdef uint8_t in_bitset_memoryview(const BITSET_INNER_DTYPE_C[:] bitset, + X_BINNED_DTYPE_C val) noexcept nogil + +cdef uint8_t in_bitset_2d_memoryview( + const BITSET_INNER_DTYPE_C[:, :] bitset, + X_BINNED_DTYPE_C val, + unsigned int row) noexcept nogil diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.pyx new file mode 100644 index 0000000000000000000000000000000000000000..e583d5039f439e960f6e32be33d9461f84e6848a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_bitset.pyx @@ -0,0 +1,65 @@ +from sklearn.ensemble._hist_gradient_boosting.common cimport BITSET_INNER_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport BITSET_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport X_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport X_BINNED_DTYPE_C +from sklearn.utils._typedefs cimport uint8_t + + +# A bitset is a data structure used to represent sets of integers in [0, n]. We +# use them to represent sets of features indices (e.g. features that go to the +# left child, or features that are categorical). For familiarity with bitsets +# and bitwise operations: +# https://en.wikipedia.org/wiki/Bit_array +# https://en.wikipedia.org/wiki/Bitwise_operation + + +cdef inline void init_bitset(BITSET_DTYPE_C bitset) noexcept nogil: # OUT + cdef: + unsigned int i + + for i in range(8): + bitset[i] = 0 + + +cdef inline void set_bitset(BITSET_DTYPE_C bitset, # OUT + X_BINNED_DTYPE_C val) noexcept nogil: + bitset[val // 32] |= (1 << (val % 32)) + + +cdef inline uint8_t in_bitset(BITSET_DTYPE_C bitset, + X_BINNED_DTYPE_C val) noexcept nogil: + return (bitset[val // 32] >> (val % 32)) & 1 + + +cpdef inline uint8_t in_bitset_memoryview(const BITSET_INNER_DTYPE_C[:] bitset, + X_BINNED_DTYPE_C val) noexcept nogil: + return (bitset[val // 32] >> (val % 32)) & 1 + + +cdef inline uint8_t in_bitset_2d_memoryview(const BITSET_INNER_DTYPE_C[:, :] bitset, + X_BINNED_DTYPE_C val, + unsigned int row) noexcept nogil: + # Same as above but works on 2d memory views to avoid the creation of 1d + # memory views. See https://github.com/scikit-learn/scikit-learn/issues/17299 + return (bitset[row, val // 32] >> (val % 32)) & 1 + + +cpdef inline void set_bitset_memoryview(BITSET_INNER_DTYPE_C[:] bitset, # OUT + X_BINNED_DTYPE_C val): + bitset[val // 32] |= (1 << (val % 32)) + + +def set_raw_bitset_from_binned_bitset(BITSET_INNER_DTYPE_C[:] raw_bitset, # OUT + BITSET_INNER_DTYPE_C[:] binned_bitset, + X_DTYPE_C[:] categories): + """Set the raw_bitset from the values of the binned bitset + + categories is a mapping from binned category value to raw category value. + """ + cdef: + int binned_cat_value + X_DTYPE_C raw_cat_value + + for binned_cat_value, raw_cat_value in enumerate(categories): + if in_bitset_memoryview(binned_bitset, binned_cat_value): + set_bitset_memoryview(raw_bitset, raw_cat_value) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..9e3ac36268824b281e127e9a911755121627d5fb Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..9453a775cc8e3bf8632bc9809b54e528cdc67272 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.pyx new file mode 100644 index 0000000000000000000000000000000000000000..9da1861201271ad17ba5e86f504915b96881486d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.pyx @@ -0,0 +1,59 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from cython.parallel import prange +import numpy as np + +from sklearn.ensemble._hist_gradient_boosting.common import Y_DTYPE +from sklearn.ensemble._hist_gradient_boosting.common cimport Y_DTYPE_C + + +def _update_raw_predictions( + Y_DTYPE_C [::1] raw_predictions, # OUT + grower, + n_threads, +): + """Update raw_predictions with the predictions of the newest tree. + + This is equivalent to (and much faster than): + raw_predictions += last_estimator.predict(X_train) + + It's only possible for data X_train that is used to train the trees (it + isn't usable for e.g. X_val). + """ + cdef: + unsigned int [::1] starts # start of each leaf in partition + unsigned int [::1] stops # end of each leaf in partition + Y_DTYPE_C [::1] values # value of each leaf + const unsigned int [::1] partition = grower.splitter.partition + list leaves + + leaves = grower.finalized_leaves + starts = np.array([leaf.partition_start for leaf in leaves], + dtype=np.uint32) + stops = np.array([leaf.partition_stop for leaf in leaves], + dtype=np.uint32) + values = np.array([leaf.value for leaf in leaves], dtype=Y_DTYPE) + + _update_raw_predictions_helper(raw_predictions, starts, stops, partition, + values, n_threads) + + +cdef inline void _update_raw_predictions_helper( + Y_DTYPE_C [::1] raw_predictions, # OUT + const unsigned int [::1] starts, + const unsigned int [::1] stops, + const unsigned int [::1] partition, + const Y_DTYPE_C [::1] values, + int n_threads, +): + + cdef: + unsigned int position + int leaf_idx + int n_leaves = starts.shape[0] + + for leaf_idx in prange(n_leaves, schedule='static', nogil=True, + num_threads=n_threads): + for position in range(starts[leaf_idx], stops[leaf_idx]): + raw_predictions[partition[position]] += values[leaf_idx] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..6a3653c1d31d1d08be7d53abf5c6e7b9f6acfafb Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..de25ca60f218a63f466871327a4063f54e8f5df3 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.pyx new file mode 100644 index 0000000000000000000000000000000000000000..945c1683c90588fde19ecbd6192b90369158bfbd --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/_predictor.pyx @@ -0,0 +1,256 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from cython.parallel import prange +from libc.math cimport isnan +import numpy as np + +from sklearn.utils._typedefs cimport intp_t, uint8_t +from sklearn.ensemble._hist_gradient_boosting.common cimport X_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport Y_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common import Y_DTYPE +from sklearn.ensemble._hist_gradient_boosting.common cimport X_BINNED_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport BITSET_INNER_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport node_struct +from sklearn.ensemble._hist_gradient_boosting._bitset cimport in_bitset_2d_memoryview + + +def _predict_from_raw_data( # raw data = non-binned data + const node_struct [:] nodes, + const X_DTYPE_C [:, :] numeric_data, + const BITSET_INNER_DTYPE_C [:, ::1] raw_left_cat_bitsets, + const BITSET_INNER_DTYPE_C [:, ::1] known_cat_bitsets, + const unsigned int [::1] f_idx_map, + int n_threads, + Y_DTYPE_C [:] out): + + cdef: + int i + + for i in prange(numeric_data.shape[0], schedule='static', nogil=True, + num_threads=n_threads): + out[i] = _predict_one_from_raw_data( + nodes, numeric_data, raw_left_cat_bitsets, + known_cat_bitsets, + f_idx_map, i) + + +cdef inline Y_DTYPE_C _predict_one_from_raw_data( + const node_struct [:] nodes, + const X_DTYPE_C [:, :] numeric_data, + const BITSET_INNER_DTYPE_C [:, ::1] raw_left_cat_bitsets, + const BITSET_INNER_DTYPE_C [:, ::1] known_cat_bitsets, + const unsigned int [::1] f_idx_map, + const int row) noexcept nogil: + # Need to pass the whole array and the row index, else prange won't work. + # See issue Cython #2798 + + cdef: + node_struct node = nodes[0] + unsigned int node_idx = 0 + X_DTYPE_C data_val + + while True: + if node.is_leaf: + return node.value + + data_val = numeric_data[row, node.feature_idx] + + if isnan(data_val): + if node.missing_go_to_left: + node_idx = node.left + else: + node_idx = node.right + elif node.is_categorical: + if data_val < 0: + # data_val is not in the accepted range, so it is treated as missing value + node_idx = node.left if node.missing_go_to_left else node.right + elif in_bitset_2d_memoryview( + raw_left_cat_bitsets, + data_val, + node.bitset_idx): + node_idx = node.left + elif in_bitset_2d_memoryview( + known_cat_bitsets, + data_val, + f_idx_map[node.feature_idx]): + node_idx = node.right + else: + # Treat unknown categories as missing. + node_idx = node.left if node.missing_go_to_left else node.right + else: + if data_val <= node.num_threshold: + node_idx = node.left + else: + node_idx = node.right + node = nodes[node_idx] + + +def _predict_from_binned_data( + node_struct [:] nodes, + const X_BINNED_DTYPE_C [:, :] binned_data, + BITSET_INNER_DTYPE_C [:, :] binned_left_cat_bitsets, + const uint8_t missing_values_bin_idx, + int n_threads, + Y_DTYPE_C [:] out): + + cdef: + int i + + for i in prange(binned_data.shape[0], schedule='static', nogil=True, + num_threads=n_threads): + out[i] = _predict_one_from_binned_data(nodes, + binned_data, + binned_left_cat_bitsets, i, + missing_values_bin_idx) + + +cdef inline Y_DTYPE_C _predict_one_from_binned_data( + node_struct [:] nodes, + const X_BINNED_DTYPE_C [:, :] binned_data, + const BITSET_INNER_DTYPE_C [:, :] binned_left_cat_bitsets, + const int row, + const uint8_t missing_values_bin_idx) noexcept nogil: + # Need to pass the whole array and the row index, else prange won't work. + # See issue Cython #2798 + + cdef: + node_struct node = nodes[0] + unsigned int node_idx = 0 + X_BINNED_DTYPE_C data_val + + while True: + if node.is_leaf: + return node.value + + data_val = binned_data[row, node.feature_idx] + + if data_val == missing_values_bin_idx: + if node.missing_go_to_left: + node_idx = node.left + else: + node_idx = node.right + elif node.is_categorical: + if in_bitset_2d_memoryview( + binned_left_cat_bitsets, + data_val, + node.bitset_idx): + node_idx = node.left + else: + node_idx = node.right + else: + if data_val <= node.bin_threshold: + node_idx = node.left + else: + node_idx = node.right + node = nodes[node_idx] + + +def _compute_partial_dependence( + node_struct [:] nodes, + const X_DTYPE_C [:, ::1] X, + const intp_t [:] target_features, + Y_DTYPE_C [:] out +): + """Partial dependence of the response on the ``target_features`` set. + + For each sample in ``X`` a tree traversal is performed. + Each traversal starts from the root with weight 1.0. + + At each non-leaf node that splits on a target feature, either + the left child or the right child is visited based on the feature + value of the current sample, and the weight is not modified. + At each non-leaf node that splits on a complementary feature, + both children are visited and the weight is multiplied by the fraction + of training samples which went to each child. + + At each leaf, the value of the node is multiplied by the current + weight (weights sum to 1 for all visited terminal nodes). + + Parameters + ---------- + nodes : view on array of PREDICTOR_RECORD_DTYPE, shape (n_nodes) + The array representing the predictor tree. + X : view on 2d ndarray, shape (n_samples, n_target_features) + The grid points on which the partial dependence should be + evaluated. + target_features : view on 1d ndarray of intp_t, shape (n_target_features) + The set of target features for which the partial dependence + should be evaluated. + out : view on 1d ndarray, shape (n_samples) + The value of the partial dependence function on each grid + point. + """ + + cdef: + unsigned int current_node_idx + unsigned int [:] node_idx_stack = np.zeros(shape=nodes.shape[0], + dtype=np.uint32) + Y_DTYPE_C [::1] weight_stack = np.zeros(shape=nodes.shape[0], + dtype=Y_DTYPE) + node_struct * current_node # pointer to avoid copying attributes + + unsigned int sample_idx + intp_t feature_idx + unsigned stack_size + Y_DTYPE_C left_sample_frac + Y_DTYPE_C current_weight + Y_DTYPE_C total_weight # used for sanity check only + bint is_target_feature + + for sample_idx in range(X.shape[0]): + # init stacks for current sample + stack_size = 1 + node_idx_stack[0] = 0 # root node + weight_stack[0] = 1 # all the samples are in the root node + total_weight = 0 + + while stack_size > 0: + + # pop the stack + stack_size -= 1 + current_node_idx = node_idx_stack[stack_size] + current_node = &nodes[current_node_idx] + + if current_node.is_leaf: + out[sample_idx] += (weight_stack[stack_size] * + current_node.value) + total_weight += weight_stack[stack_size] + else: + # determine if the split feature is a target feature + is_target_feature = False + for feature_idx in range(target_features.shape[0]): + if target_features[feature_idx] == current_node.feature_idx: + is_target_feature = True + break + + if is_target_feature: + # In this case, we push left or right child on stack + if X[sample_idx, feature_idx] <= current_node.num_threshold: + node_idx_stack[stack_size] = current_node.left + else: + node_idx_stack[stack_size] = current_node.right + stack_size += 1 + else: + # In this case, we push both children onto the stack, + # and give a weight proportional to the number of + # samples going through each branch. + + # push left child + node_idx_stack[stack_size] = current_node.left + left_sample_frac = ( + nodes[current_node.left].count / + current_node.count) + current_weight = weight_stack[stack_size] + weight_stack[stack_size] = current_weight * left_sample_frac + stack_size += 1 + + # push right child + node_idx_stack[stack_size] = current_node.right + weight_stack[stack_size] = ( + current_weight * (1 - left_sample_frac)) + stack_size += 1 + + # Sanity check. Should never happen. + if not (0.999 < total_weight < 1.001): + raise ValueError("Total weight should be 1.0 but was %.9f" %total_weight) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/binning.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/binning.py new file mode 100644 index 0000000000000000000000000000000000000000..424a401eee2b402ad1b43bd4bfef1029599a72ca --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/binning.py @@ -0,0 +1,338 @@ +""" +This module contains the BinMapper class. + +BinMapper is used for mapping a real-valued dataset into integer-valued bins. +Bin thresholds are computed with the quantiles so that each bin contains +approximately the same number of samples. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np + +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.ensemble._hist_gradient_boosting._binning import _map_to_bins +from sklearn.ensemble._hist_gradient_boosting._bitset import set_bitset_memoryview +from sklearn.ensemble._hist_gradient_boosting.common import ( + ALMOST_INF, + X_BINNED_DTYPE, + X_BITSET_INNER_DTYPE, + X_DTYPE, +) +from sklearn.utils import check_array, check_random_state +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.validation import check_is_fitted + + +def _find_binning_thresholds(col_data, max_bins): + """Extract quantiles from a continuous feature. + + Missing values are ignored for finding the thresholds. + + Parameters + ---------- + col_data : array-like, shape (n_samples,) + The continuous feature to bin. + max_bins: int + The maximum number of bins to use for non-missing values. If for a + given feature the number of unique values is less than ``max_bins``, + then those unique values will be used to compute the bin thresholds, + instead of the quantiles + + Return + ------ + binning_thresholds : ndarray of shape(min(max_bins, n_unique_values) - 1,) + The increasing numeric values that can be used to separate the bins. + A given value x will be mapped into bin value i iff + bining_thresholds[i - 1] < x <= binning_thresholds[i] + """ + # ignore missing values when computing bin thresholds + missing_mask = np.isnan(col_data) + if missing_mask.any(): + col_data = col_data[~missing_mask] + # The data will be sorted anyway in np.unique and again in percentile, so we do it + # here. Sorting also returns a contiguous array. + col_data = np.sort(col_data) + distinct_values = np.unique(col_data).astype(X_DTYPE) + if len(distinct_values) <= max_bins: + midpoints = distinct_values[:-1] + distinct_values[1:] + midpoints *= 0.5 + else: + # We could compute approximate midpoint percentiles using the output of + # np.unique(col_data, return_counts) instead but this is more + # work and the performance benefit will be limited because we + # work on a fixed-size subsample of the full data. + percentiles = np.linspace(0, 100, num=max_bins + 1) + percentiles = percentiles[1:-1] + midpoints = np.percentile(col_data, percentiles, method="midpoint").astype( + X_DTYPE + ) + assert midpoints.shape[0] == max_bins - 1 + + # We avoid having +inf thresholds: +inf thresholds are only allowed in + # a "split on nan" situation. + np.clip(midpoints, a_min=None, a_max=ALMOST_INF, out=midpoints) + return midpoints + + +class _BinMapper(TransformerMixin, BaseEstimator): + """Transformer that maps a dataset into integer-valued bins. + + For continuous features, the bins are created in a feature-wise fashion, + using quantiles so that each bins contains approximately the same number + of samples. For large datasets, quantiles are computed on a subset of the + data to speed-up the binning, but the quantiles should remain stable. + + For categorical features, the raw categorical values are expected to be + in [0, 254] (this is not validated here though) and each category + corresponds to a bin. All categorical values must be known at + initialization: transform() doesn't know how to bin unknown categorical + values. Note that transform() is only used on non-training data in the + case of early stopping. + + Features with a small number of values may be binned into less than + ``n_bins`` bins. The last bin (at index ``n_bins - 1``) is always reserved + for missing values. + + Parameters + ---------- + n_bins : int, default=256 + The maximum number of bins to use (including the bin for missing + values). Should be in [3, 256]. Non-missing values are binned on + ``max_bins = n_bins - 1`` bins. The last bin is always reserved for + missing values. If for a given feature the number of unique values is + less than ``max_bins``, then those unique values will be used to + compute the bin thresholds, instead of the quantiles. For categorical + features indicated by ``is_categorical``, the docstring for + ``is_categorical`` details on this procedure. + subsample : int or None, default=2e5 + If ``n_samples > subsample``, then ``sub_samples`` samples will be + randomly chosen to compute the quantiles. If ``None``, the whole data + is used. + is_categorical : ndarray of bool of shape (n_features,), default=None + Indicates categorical features. By default, all features are + considered continuous. + known_categories : list of {ndarray, None} of shape (n_features,), \ + default=none + For each categorical feature, the array indicates the set of unique + categorical values. These should be the possible values over all the + data, not just the training data. For continuous features, the + corresponding entry should be None. + random_state: int, RandomState instance or None, default=None + Pseudo-random number generator to control the random sub-sampling. + Pass an int for reproducible output across multiple + function calls. + See :term:`Glossary `. + n_threads : int, default=None + Number of OpenMP threads to use. `_openmp_effective_n_threads` is called + to determine the effective number of threads use, which takes cgroups CPU + quotes into account. See the docstring of `_openmp_effective_n_threads` + for details. + + Attributes + ---------- + bin_thresholds_ : list of ndarray + For each feature, each array indicates how to map a feature into a + binned feature. The semantic and size depends on the nature of the + feature: + - for real-valued features, the array corresponds to the real-valued + bin thresholds (the upper bound of each bin). There are ``max_bins + - 1`` thresholds, where ``max_bins = n_bins - 1`` is the number of + bins used for non-missing values. + - for categorical features, the array is a map from a binned category + value to the raw category value. The size of the array is equal to + ``min(max_bins, category_cardinality)`` where we ignore missing + values in the cardinality. + n_bins_non_missing_ : ndarray, dtype=np.uint32 + For each feature, gives the number of bins actually used for + non-missing values. For features with a lot of unique values, this is + equal to ``n_bins - 1``. + is_categorical_ : ndarray of shape (n_features,), dtype=np.uint8 + Indicator for categorical features. + missing_values_bin_idx_ : np.uint8 + The index of the bin where missing values are mapped. This is a + constant across all features. This corresponds to the last bin, and + it is always equal to ``n_bins - 1``. Note that if ``n_bins_non_missing_`` + is less than ``n_bins - 1`` for a given feature, then there are + empty (and unused) bins. + """ + + def __init__( + self, + n_bins=256, + subsample=int(2e5), + is_categorical=None, + known_categories=None, + random_state=None, + n_threads=None, + ): + self.n_bins = n_bins + self.subsample = subsample + self.is_categorical = is_categorical + self.known_categories = known_categories + self.random_state = random_state + self.n_threads = n_threads + + def fit(self, X, y=None): + """Fit data X by computing the binning thresholds. + + The last bin is reserved for missing values, whether missing values + are present in the data or not. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data to bin. + y: None + Ignored. + + Returns + ------- + self : object + """ + if not (3 <= self.n_bins <= 256): + # min is 3: at least 2 distinct bins and a missing values bin + raise ValueError( + "n_bins={} should be no smaller than 3 and no larger than 256.".format( + self.n_bins + ) + ) + + X = check_array(X, dtype=[X_DTYPE], ensure_all_finite=False) + max_bins = self.n_bins - 1 + + rng = check_random_state(self.random_state) + if self.subsample is not None and X.shape[0] > self.subsample: + subset = rng.choice(X.shape[0], self.subsample, replace=False) + X = X.take(subset, axis=0) + + if self.is_categorical is None: + self.is_categorical_ = np.zeros(X.shape[1], dtype=np.uint8) + else: + self.is_categorical_ = np.asarray(self.is_categorical, dtype=np.uint8) + + n_features = X.shape[1] + known_categories = self.known_categories + if known_categories is None: + known_categories = [None] * n_features + + # validate is_categorical and known_categories parameters + for f_idx in range(n_features): + is_categorical = self.is_categorical_[f_idx] + known_cats = known_categories[f_idx] + if is_categorical and known_cats is None: + raise ValueError( + f"Known categories for feature {f_idx} must be provided." + ) + if not is_categorical and known_cats is not None: + raise ValueError( + f"Feature {f_idx} isn't marked as a categorical feature, " + "but categories were passed." + ) + + self.missing_values_bin_idx_ = self.n_bins - 1 + + self.bin_thresholds_ = [None] * n_features + n_bins_non_missing = [None] * n_features + + non_cat_thresholds = Parallel(n_jobs=self.n_threads, backend="threading")( + delayed(_find_binning_thresholds)(X[:, f_idx], max_bins) + for f_idx in range(n_features) + if not self.is_categorical_[f_idx] + ) + + non_cat_idx = 0 + for f_idx in range(n_features): + if self.is_categorical_[f_idx]: + # Since categories are assumed to be encoded in + # [0, n_cats] and since n_cats <= max_bins, + # the thresholds *are* the unique categorical values. This will + # lead to the correct mapping in transform() + thresholds = known_categories[f_idx] + n_bins_non_missing[f_idx] = thresholds.shape[0] + self.bin_thresholds_[f_idx] = thresholds + else: + self.bin_thresholds_[f_idx] = non_cat_thresholds[non_cat_idx] + n_bins_non_missing[f_idx] = self.bin_thresholds_[f_idx].shape[0] + 1 + non_cat_idx += 1 + + self.n_bins_non_missing_ = np.array(n_bins_non_missing, dtype=np.uint32) + return self + + def transform(self, X): + """Bin data X. + + Missing values will be mapped to the last bin. + + For categorical features, the mapping will be incorrect for unknown + categories. Since the BinMapper is given known_categories of the + entire training data (i.e. before the call to train_test_split() in + case of early-stopping), this never happens. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The data to bin. + + Returns + ------- + X_binned : array-like of shape (n_samples, n_features) + The binned data (fortran-aligned). + """ + X = check_array(X, dtype=[X_DTYPE], ensure_all_finite=False) + check_is_fitted(self) + if X.shape[1] != self.n_bins_non_missing_.shape[0]: + raise ValueError( + "This estimator was fitted with {} features but {} got passed " + "to transform()".format(self.n_bins_non_missing_.shape[0], X.shape[1]) + ) + + n_threads = _openmp_effective_n_threads(self.n_threads) + binned = np.zeros_like(X, dtype=X_BINNED_DTYPE, order="F") + _map_to_bins( + X, + self.bin_thresholds_, + self.is_categorical_, + self.missing_values_bin_idx_, + n_threads, + binned, + ) + return binned + + def make_known_categories_bitsets(self): + """Create bitsets of known categories. + + Returns + ------- + - known_cat_bitsets : ndarray of shape (n_categorical_features, 8) + Array of bitsets of known categories, for each categorical feature. + - f_idx_map : ndarray of shape (n_features,) + Map from original feature index to the corresponding index in the + known_cat_bitsets array. + """ + + categorical_features_indices = np.flatnonzero(self.is_categorical_) + + n_features = self.is_categorical_.size + n_categorical_features = categorical_features_indices.size + + f_idx_map = np.zeros(n_features, dtype=np.uint32) + f_idx_map[categorical_features_indices] = np.arange( + n_categorical_features, dtype=np.uint32 + ) + + known_categories = self.bin_thresholds_ + + known_cat_bitsets = np.zeros( + (n_categorical_features, 8), dtype=X_BITSET_INNER_DTYPE + ) + + # TODO: complexity is O(n_categorical_features * 255). Maybe this is + # worth cythonizing + for mapped_f_idx, f_idx in enumerate(categorical_features_indices): + for raw_cat_val in known_categories[f_idx]: + set_bitset_memoryview(known_cat_bitsets[mapped_f_idx], raw_cat_val) + + return known_cat_bitsets, f_idx_map diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..e5979a65bdc740768c4db3add7c004b9fdd21938 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..cd2b899dc20c473d82d85db8b87e09cc87db60ed Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.cp312-win_amd64.pyd differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.pxd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.pxd new file mode 100644 index 0000000000000000000000000000000000000000..c52415d3aec72d6cc65a883b85cd124b9e4bbc76 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.pxd @@ -0,0 +1,43 @@ +from sklearn.utils._typedefs cimport float32_t, float64_t, intp_t, uint8_t, uint32_t + + +ctypedef float64_t X_DTYPE_C +ctypedef uint8_t X_BINNED_DTYPE_C +ctypedef float64_t Y_DTYPE_C +ctypedef float32_t G_H_DTYPE_C +ctypedef uint32_t BITSET_INNER_DTYPE_C +ctypedef BITSET_INNER_DTYPE_C[8] BITSET_DTYPE_C + + +cdef packed struct hist_struct: + # Same as histogram dtype but we need a struct to declare views. It needs + # to be packed since by default numpy dtypes aren't aligned + Y_DTYPE_C sum_gradients + Y_DTYPE_C sum_hessians + unsigned int count + + +cdef packed struct node_struct: + # Equivalent struct to PREDICTOR_RECORD_DTYPE to use in memory views. It + # needs to be packed since by default numpy dtypes aren't aligned + Y_DTYPE_C value + unsigned int count + intp_t feature_idx + X_DTYPE_C num_threshold + uint8_t missing_go_to_left + unsigned int left + unsigned int right + Y_DTYPE_C gain + unsigned int depth + uint8_t is_leaf + X_BINNED_DTYPE_C bin_threshold + uint8_t is_categorical + # The index of the corresponding bitsets in the Predictor's bitset arrays. + # Only used if is_categorical is True + unsigned int bitset_idx + + +cpdef enum MonotonicConstraint: + NO_CST = 0 + POS = 1 + NEG = -1 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.pyx new file mode 100644 index 0000000000000000000000000000000000000000..9bb7317dc50ae1a5290ada52dc5648be81077d64 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/common.pyx @@ -0,0 +1,44 @@ +import numpy as np + +# Y_DYTPE is the dtype to which the targets y are converted to. This is also +# dtype for leaf values, gains, and sums of gradients / hessians. The gradients +# and hessians arrays are stored as floats to avoid using too much memory. +Y_DTYPE = np.float64 +X_DTYPE = np.float64 +X_BINNED_DTYPE = np.uint8 # hence max_bins == 256 +# dtype for gradients and hessians arrays +G_H_DTYPE = np.float32 +X_BITSET_INNER_DTYPE = np.uint32 + +# Note that we use Y_DTYPE=float64 to avoid issues with floating point precision when +# summing gradients and hessians (both float32). Those are difficult to protect via +# tools like (Kahan-) Neumaier summation as in CPython, see +# https://github.com/python/cpython/issues/100425, or pairwise summation as numpy, see +# https://github.com/numpy/numpy/pull/3685, due to the way histograms are summed +# (number of additions per bin is not known in advance). See also comment in +# _subtract_histograms. +HISTOGRAM_DTYPE = np.dtype([ + ('sum_gradients', Y_DTYPE), # sum of sample gradients in bin + ('sum_hessians', Y_DTYPE), # sum of sample hessians in bin + ('count', np.uint32), # number of samples in bin +]) + +PREDICTOR_RECORD_DTYPE = np.dtype([ + ('value', Y_DTYPE), + ('count', np.uint32), + ('feature_idx', np.intp), + ('num_threshold', X_DTYPE), + ('missing_go_to_left', np.uint8), + ('left', np.uint32), + ('right', np.uint32), + ('gain', Y_DTYPE), + ('depth', np.uint32), + ('is_leaf', np.uint8), + ('bin_threshold', X_BINNED_DTYPE), + ('is_categorical', np.uint8), + # The index of the corresponding bitsets in the Predictor's bitset arrays. + # Only used if is_categorical is True + ('bitset_idx', np.uint32) +]) + +ALMOST_INF = 1e300 # see LightGBM AvoidInf() diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py new file mode 100644 index 0000000000000000000000000000000000000000..e90d353acff2712b4787b140407f597c47c3e14e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py @@ -0,0 +1,2373 @@ +"""Fast Gradient Boosting decision trees for classification and regression.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools +from abc import ABC, abstractmethod +from contextlib import contextmanager, nullcontext, suppress +from functools import partial +from numbers import Integral, Real +from time import time + +import numpy as np + +from sklearn._loss.loss import ( + _LOSSES, + BaseLoss, + HalfBinomialLoss, + HalfGammaLoss, + HalfMultinomialLoss, + HalfPoissonLoss, + PinballLoss, +) +from sklearn.base import ( + BaseEstimator, + ClassifierMixin, + RegressorMixin, + _fit_context, + is_classifier, +) +from sklearn.compose import ColumnTransformer +from sklearn.ensemble._hist_gradient_boosting._gradient_boosting import ( + _update_raw_predictions, +) +from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper +from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE, X_DTYPE, Y_DTYPE +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +from sklearn.metrics import check_scoring +from sklearn.metrics._scorer import _SCORERS +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import FunctionTransformer, LabelEncoder, OrdinalEncoder +from sklearn.utils import check_random_state, compute_sample_weight, resample +from sklearn.utils._dataframe import is_pandas_df +from sklearn.utils._missing import is_scalar_nan +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.utils._param_validation import Interval, RealNotInt, StrOptions +from sklearn.utils.multiclass import check_classification_targets +from sklearn.utils.validation import ( + _check_monotonic_cst, + _check_sample_weight, + _check_y, + check_array, + check_consistent_length, + check_is_fitted, + validate_data, +) + +_LOSSES = _LOSSES.copy() +_LOSSES.update( + { + "poisson": HalfPoissonLoss, + "gamma": HalfGammaLoss, + "quantile": PinballLoss, + } +) + + +def _update_leaves_values(loss, grower, y_true, raw_prediction, sample_weight): + """Update the leaf values to be predicted by the tree. + + Update equals: + loss.fit_intercept_only(y_true - raw_prediction) + + This is only applied if loss.differentiable is False. + Note: It only works, if the loss is a function of the residual, as is the + case for AbsoluteError and PinballLoss. Otherwise, one would need to get + the minimum of loss(y_true, raw_prediction + x) in x. A few examples: + - AbsoluteError: median(y_true - raw_prediction). + - PinballLoss: quantile(y_true - raw_prediction). + + More background: + For the standard gradient descent method according to "Greedy Function + Approximation: A Gradient Boosting Machine" by Friedman, all loss functions but the + squared loss need a line search step. BaseHistGradientBoosting, however, implements + a so called Newton boosting where the trees are fitted to a 2nd order + approximations of the loss in terms of gradients and hessians. In this case, the + line search step is only necessary if the loss is not smooth, i.e. not + differentiable, which renders the 2nd order approximation invalid. In fact, + non-smooth losses arbitrarily set hessians to 1 and effectively use the standard + gradient descent method with line search. + """ + # TODO: Ideally this should be computed in parallel over the leaves using something + # similar to _update_raw_predictions(), but this requires a cython version of + # median(). + for leaf in grower.finalized_leaves: + indices = leaf.sample_indices + if sample_weight is None: + sw = None + else: + sw = sample_weight[indices] + update = loss.fit_intercept_only( + y_true=y_true[indices] - raw_prediction[indices], + sample_weight=sw, + ) + leaf.value = grower.shrinkage * update + # Note that the regularization is ignored here + + +@contextmanager +def _patch_raw_predict(estimator, raw_predictions): + """Context manager that patches _raw_predict to return raw_predictions. + + `raw_predictions` is typically a precomputed array to avoid redundant + state-wise computations fitting with early stopping enabled: in this case + `raw_predictions` is incrementally updated whenever we add a tree to the + boosted ensemble. + + Note: this makes fitting HistGradientBoosting* models inherently non thread + safe at fit time. However thread-safety at fit time was never guaranteed nor + enforced for scikit-learn estimators in general. + + Thread-safety at prediction/transform time is another matter as those + operations are typically side-effect free and therefore often thread-safe by + default for most scikit-learn models and would like to keep it that way. + Therefore this context manager should only be used at fit time. + + TODO: in the future, we could explore the possibility to extend the scorer + public API to expose a way to compute vales from raw predictions. That would + probably require also making the scorer aware of the inverse link function + used by the estimator which is typically private API for now, hence the need + for this patching mechanism. + """ + orig_raw_predict = estimator._raw_predict + + def _patched_raw_predicts(*args, **kwargs): + return raw_predictions + + estimator._raw_predict = _patched_raw_predicts + yield estimator + estimator._raw_predict = orig_raw_predict + + +class BaseHistGradientBoosting(BaseEstimator, ABC): + """Base class for histogram-based gradient boosting estimators.""" + + _parameter_constraints: dict = { + "loss": [BaseLoss], + "learning_rate": [Interval(Real, 0, None, closed="neither")], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "max_leaf_nodes": [Interval(Integral, 2, None, closed="left"), None], + "max_depth": [Interval(Integral, 1, None, closed="left"), None], + "min_samples_leaf": [Interval(Integral, 1, None, closed="left")], + "l2_regularization": [Interval(Real, 0, None, closed="left")], + "max_features": [Interval(RealNotInt, 0, 1, closed="right")], + "monotonic_cst": ["array-like", dict, None], + "interaction_cst": [ + list, + tuple, + StrOptions({"pairwise", "no_interactions"}), + None, + ], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], + "validation_fraction": [ + Interval(RealNotInt, 0, 1, closed="neither"), + Interval(Integral, 1, None, closed="left"), + None, + ], + "tol": [Interval(Real, 0, None, closed="left")], + "max_bins": [Interval(Integral, 2, 255, closed="both")], + "categorical_features": ["array-like", StrOptions({"from_dtype"}), None], + "warm_start": ["boolean"], + "early_stopping": [StrOptions({"auto"}), "boolean"], + "scoring": [str, callable, None], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + + @abstractmethod + def __init__( + self, + loss, + *, + learning_rate, + max_iter, + max_leaf_nodes, + max_depth, + min_samples_leaf, + l2_regularization, + max_features, + max_bins, + categorical_features, + monotonic_cst, + interaction_cst, + warm_start, + early_stopping, + scoring, + validation_fraction, + n_iter_no_change, + tol, + verbose, + random_state, + ): + self.loss = loss + self.learning_rate = learning_rate + self.max_iter = max_iter + self.max_leaf_nodes = max_leaf_nodes + self.max_depth = max_depth + self.min_samples_leaf = min_samples_leaf + self.l2_regularization = l2_regularization + self.max_features = max_features + self.max_bins = max_bins + self.monotonic_cst = monotonic_cst + self.interaction_cst = interaction_cst + self.categorical_features = categorical_features + self.warm_start = warm_start + self.early_stopping = early_stopping + self.scoring = scoring + self.validation_fraction = validation_fraction + self.n_iter_no_change = n_iter_no_change + self.tol = tol + self.verbose = verbose + self.random_state = random_state + + def _validate_parameters(self): + """Validate parameters passed to __init__. + + The parameters that are directly passed to the grower are checked in + TreeGrower.""" + if self.monotonic_cst is not None and self.n_trees_per_iteration_ != 1: + raise ValueError( + "monotonic constraints are not supported for multiclass classification." + ) + + def _finalize_sample_weight(self, sample_weight, y): + """Finalize sample weight. + + Used by subclasses to adjust sample_weights. This is useful for implementing + class weights. + """ + return sample_weight + + def _preprocess_X(self, X, *, reset): + """Preprocess and validate X. + + Parameters + ---------- + X : {array-like, pandas DataFrame} of shape (n_samples, n_features) + Input data. + + reset : bool + Whether to reset the `n_features_in_` and `feature_names_in_ attributes. + + Returns + ------- + X : ndarray of shape (n_samples, n_features) + Validated input data. + + known_categories : list of ndarray of shape (n_categories,) + List of known categories for each categorical feature. + """ + # If there is a preprocessor, we let the preprocessor handle the validation. + # Otherwise, we validate the data ourselves. + check_X_kwargs = dict(dtype=[X_DTYPE], ensure_all_finite=False) + if not reset: + if self._preprocessor is None: + return validate_data(self, X, reset=False, **check_X_kwargs) + return self._preprocessor.transform(X) + + # At this point, reset is False, which runs during `fit`. + self.is_categorical_ = self._check_categorical_features(X) + + if self.is_categorical_ is None: + self._preprocessor = None + self._is_categorical_remapped = None + + X = validate_data(self, X, **check_X_kwargs) + return X, None + + n_features = X.shape[1] + ordinal_encoder = OrdinalEncoder( + categories="auto", + handle_unknown="use_encoded_value", + unknown_value=np.nan, + encoded_missing_value=np.nan, + dtype=X_DTYPE, + ) + + check_X = partial(check_array, **check_X_kwargs) + numerical_preprocessor = FunctionTransformer(check_X) + self._preprocessor = ColumnTransformer( + [ + ("encoder", ordinal_encoder, self.is_categorical_), + ("numerical", numerical_preprocessor, ~self.is_categorical_), + ] + ) + self._preprocessor.set_output(transform="default") + X = self._preprocessor.fit_transform(X) + # check categories found by the OrdinalEncoder and get their encoded values + known_categories = self._check_categories() + self.n_features_in_ = self._preprocessor.n_features_in_ + with suppress(AttributeError): + self.feature_names_in_ = self._preprocessor.feature_names_in_ + + # The ColumnTransformer's output places the categorical features at the + # beginning + categorical_remapped = np.zeros(n_features, dtype=bool) + categorical_remapped[self._preprocessor.output_indices_["encoder"]] = True + self._is_categorical_remapped = categorical_remapped + + return X, known_categories + + def _check_categories(self): + """Check categories found by the preprocessor and return their encoded values. + + Returns a list of length ``self.n_features_in_``, with one entry per + input feature. + + For non-categorical features, the corresponding entry is ``None``. + + For categorical features, the corresponding entry is an array + containing the categories as encoded by the preprocessor (an + ``OrdinalEncoder``), excluding missing values. The entry is therefore + ``np.arange(n_categories)`` where ``n_categories`` is the number of + unique values in the considered feature column, after removing missing + values. + + If ``n_categories > self.max_bins`` for any feature, a ``ValueError`` + is raised. + """ + encoder = self._preprocessor.named_transformers_["encoder"] + known_categories = [None] * self._preprocessor.n_features_in_ + categorical_column_indices = np.arange(self._preprocessor.n_features_in_)[ + self._preprocessor.output_indices_["encoder"] + ] + for feature_idx, categories in zip( + categorical_column_indices, encoder.categories_ + ): + # OrdinalEncoder always puts np.nan as the last category if the + # training data has missing values. Here we remove it because it is + # already added by the _BinMapper. + if len(categories) and is_scalar_nan(categories[-1]): + categories = categories[:-1] + if categories.size > self.max_bins: + try: + feature_name = repr(encoder.feature_names_in_[feature_idx]) + except AttributeError: + feature_name = f"at index {feature_idx}" + raise ValueError( + f"Categorical feature {feature_name} is expected to " + f"have a cardinality <= {self.max_bins} but actually " + f"has a cardinality of {categories.size}." + ) + known_categories[feature_idx] = np.arange(len(categories), dtype=X_DTYPE) + return known_categories + + def _check_categorical_features(self, X): + """Check and validate categorical features in X + + Parameters + ---------- + X : {array-like, pandas DataFrame} of shape (n_samples, n_features) + Input data. + + Return + ------ + is_categorical : ndarray of shape (n_features,) or None, dtype=bool + Indicates whether a feature is categorical. If no feature is + categorical, this is None. + """ + # Special code for pandas because of a bug in recent pandas, which is + # fixed in main and maybe included in 2.2.1, see + # https://github.com/pandas-dev/pandas/pull/57173. + # Also pandas versions < 1.5.1 do not support the dataframe interchange + if is_pandas_df(X): + X_is_dataframe = True + categorical_columns_mask = np.asarray(X.dtypes == "category") + elif hasattr(X, "__dataframe__"): + X_is_dataframe = True + categorical_columns_mask = np.asarray( + [ + c.dtype[0].name == "CATEGORICAL" + for c in X.__dataframe__().get_columns() + ] + ) + else: + X_is_dataframe = False + categorical_columns_mask = None + + categorical_features = self.categorical_features + + categorical_by_dtype = ( + isinstance(categorical_features, str) + and categorical_features == "from_dtype" + ) + no_categorical_dtype = categorical_features is None or ( + categorical_by_dtype and not X_is_dataframe + ) + + if no_categorical_dtype: + return None + + use_pandas_categorical = categorical_by_dtype and X_is_dataframe + if use_pandas_categorical: + categorical_features = categorical_columns_mask + else: + categorical_features = np.asarray(categorical_features) + + if categorical_features.size == 0: + return None + + if categorical_features.dtype.kind not in ("i", "b", "U", "O"): + raise ValueError( + "categorical_features must be an array-like of bool, int or " + f"str, got: {categorical_features.dtype.name}." + ) + + if categorical_features.dtype.kind == "O": + types = set(type(f) for f in categorical_features) + if types != {str}: + raise ValueError( + "categorical_features must be an array-like of bool, int or " + f"str, got: {', '.join(sorted(t.__name__ for t in types))}." + ) + + n_features = X.shape[1] + # At this point `validate_data` was not called yet because we use the original + # dtypes to discover the categorical features. Thus `feature_names_in_` + # is not defined yet. + feature_names_in_ = getattr(X, "columns", None) + + if categorical_features.dtype.kind in ("U", "O"): + # check for feature names + if feature_names_in_ is None: + raise ValueError( + "categorical_features should be passed as an array of " + "integers or as a boolean mask when the model is fitted " + "on data without feature names." + ) + is_categorical = np.zeros(n_features, dtype=bool) + feature_names = list(feature_names_in_) + for feature_name in categorical_features: + try: + is_categorical[feature_names.index(feature_name)] = True + except ValueError as e: + raise ValueError( + f"categorical_features has an item value '{feature_name}' " + "which is not a valid feature name of the training " + f"data. Observed feature names: {feature_names}" + ) from e + elif categorical_features.dtype.kind == "i": + # check for categorical features as indices + if ( + np.max(categorical_features) >= n_features + or np.min(categorical_features) < 0 + ): + raise ValueError( + "categorical_features set as integer " + "indices must be in [0, n_features - 1]" + ) + is_categorical = np.zeros(n_features, dtype=bool) + is_categorical[categorical_features] = True + else: + if categorical_features.shape[0] != n_features: + raise ValueError( + "categorical_features set as a boolean mask " + "must have shape (n_features,), got: " + f"{categorical_features.shape}" + ) + is_categorical = categorical_features + + if not np.any(is_categorical): + return None + return is_categorical + + def _check_interaction_cst(self, n_features): + """Check and validation for interaction constraints.""" + if self.interaction_cst is None: + return None + + if self.interaction_cst == "no_interactions": + interaction_cst = [[i] for i in range(n_features)] + elif self.interaction_cst == "pairwise": + interaction_cst = itertools.combinations(range(n_features), 2) + else: + interaction_cst = self.interaction_cst + + try: + constraints = [set(group) for group in interaction_cst] + except TypeError: + raise ValueError( + "Interaction constraints must be a sequence of tuples or lists, got:" + f" {self.interaction_cst!r}." + ) + + for group in constraints: + for x in group: + if not (isinstance(x, Integral) and 0 <= x < n_features): + raise ValueError( + "Interaction constraints must consist of integer indices in" + f" [0, n_features - 1] = [0, {n_features - 1}], specifying the" + " position of features, got invalid indices:" + f" {group!r}" + ) + + # Add all not listed features as own group by default. + rest = set(range(n_features)) - set().union(*constraints) + if len(rest) > 0: + constraints.append(rest) + + return constraints + + @_fit_context(prefer_skip_nested_validation=True) + def fit( + self, + X, + y, + sample_weight=None, + *, + X_val=None, + y_val=None, + sample_weight_val=None, + ): + """Fit the gradient boosting model. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. + + y : array-like of shape (n_samples,) + Target values. + + sample_weight : array-like of shape (n_samples,) default=None + Weights of training data. + + .. versionadded:: 0.23 + + X_val : array-like of shape (n_val, n_features) + Additional sample of features for validation used in early stopping. + In a `Pipeline`, `X_val` can be transformed the same way as `X` with + `Pipeline(..., transform_input=["X_val"])`. + + .. versionadded:: 1.7 + + y_val : array-like of shape (n_samples,) + Additional sample of target values for validation used in early stopping. + + .. versionadded:: 1.7 + + sample_weight_val : array-like of shape (n_samples,) default=None + Additional weights for validation used in early stopping. + + .. versionadded:: 1.7 + + Returns + ------- + self : object + Fitted estimator. + """ + fit_start_time = time() + acc_find_split_time = 0.0 # time spent finding the best splits + acc_apply_split_time = 0.0 # time spent splitting nodes + acc_compute_hist_time = 0.0 # time spent computing histograms + # time spent predicting X for gradient and hessians update + acc_prediction_time = 0.0 + X, known_categories = self._preprocess_X(X, reset=True) + y = _check_y(y, estimator=self) + y = self._encode_y(y) + check_consistent_length(X, y) + # Do not create unit sample weights by default to later skip some + # computation + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, dtype=np.float64) + # TODO: remove when PDP supports sample weights + self._fitted_with_sw = True + + sample_weight = self._finalize_sample_weight(sample_weight, y) + + validation_data_provided = X_val is not None or y_val is not None + if validation_data_provided: + if y_val is None: + raise ValueError("X_val is provided, but y_val was not provided.") + if X_val is None: + raise ValueError("y_val is provided, but X_val was not provided.") + X_val = self._preprocess_X(X_val, reset=False) + y_val = _check_y(y_val, estimator=self) + y_val = self._encode_y_val(y_val) + check_consistent_length(X_val, y_val) + if sample_weight_val is not None: + sample_weight_val = _check_sample_weight( + sample_weight_val, X_val, dtype=np.float64 + ) + if self.early_stopping is False: + raise ValueError( + "X_val and y_val are passed to fit while at the same time " + "early_stopping is False. When passing X_val and y_val to fit," + "early_stopping should be set to either 'auto' or True." + ) + + # Note: At this point, we could delete self._label_encoder if it exists. + # But we don't to keep the code even simpler. + + rng = check_random_state(self.random_state) + + # When warm starting, we want to reuse the same seed that was used + # the first time fit was called (e.g. train/val split). + # For feature subsampling, we want to continue with the rng we started with. + if not self.warm_start or not self._is_fitted(): + self._random_seed = rng.randint(np.iinfo(np.uint32).max, dtype="u8") + feature_subsample_seed = rng.randint(np.iinfo(np.uint32).max, dtype="u8") + self._feature_subsample_rng = np.random.default_rng(feature_subsample_seed) + + self._validate_parameters() + monotonic_cst = _check_monotonic_cst(self, self.monotonic_cst) + # _preprocess_X places the categorical features at the beginning, + # change the order of monotonic_cst accordingly + if self.is_categorical_ is not None: + monotonic_cst_remapped = np.concatenate( + ( + monotonic_cst[self.is_categorical_], + monotonic_cst[~self.is_categorical_], + ) + ) + else: + monotonic_cst_remapped = monotonic_cst + + # used for validation in predict + n_samples, self._n_features = X.shape + + # Encode constraints into a list of sets of features indices (integers). + interaction_cst = self._check_interaction_cst(self._n_features) + + # we need this stateful variable to tell raw_predict() that it was + # called from fit() (this current method), and that the data it has + # received is pre-binned. + # predicting is faster on pre-binned data, so we want early stopping + # predictions to be made on pre-binned data. Unfortunately the _scorer + # can only call predict() or predict_proba(), not raw_predict(), and + # there's no way to tell the scorer that it needs to predict binned + # data. + self._in_fit = True + + # `_openmp_effective_n_threads` is used to take cgroups CPU quotes + # into account when determine the maximum number of threads to use. + n_threads = _openmp_effective_n_threads() + + if isinstance(self.loss, str): + self._loss = self._get_loss(sample_weight=sample_weight) + elif isinstance(self.loss, BaseLoss): + self._loss = self.loss + + if self.early_stopping == "auto": + self.do_early_stopping_ = n_samples > 10_000 + else: + self.do_early_stopping_ = self.early_stopping + + # create validation data if needed + self._use_validation_data = ( + self.validation_fraction is not None or validation_data_provided + ) + if ( + self.do_early_stopping_ + and self._use_validation_data + and not validation_data_provided + ): + # stratify for classification + # instead of checking predict_proba, loss.n_classes >= 2 would also work + stratify = y if hasattr(self._loss, "predict_proba") else None + + # Save the state of the RNG for the training and validation split. + # This is needed in order to have the same split when using + # warm starting. + + if sample_weight is None: + X_train, X_val, y_train, y_val = train_test_split( + X, + y, + test_size=self.validation_fraction, + stratify=stratify, + random_state=self._random_seed, + ) + sample_weight_train = sample_weight_val = None + else: + # TODO: incorporate sample_weight in sampling here, as well as + # stratify + ( + X_train, + X_val, + y_train, + y_val, + sample_weight_train, + sample_weight_val, + ) = train_test_split( + X, + y, + sample_weight, + test_size=self.validation_fraction, + stratify=stratify, + random_state=self._random_seed, + ) + else: + X_train, y_train, sample_weight_train = X, y, sample_weight + if not validation_data_provided: + X_val = y_val = sample_weight_val = None + + # Bin the data + # For ease of use of the API, the user-facing GBDT classes accept the + # parameter max_bins, which doesn't take into account the bin for + # missing values (which is always allocated). However, since max_bins + # isn't the true maximal number of bins, all other private classes + # (binmapper, histbuilder...) accept n_bins instead, which is the + # actual total number of bins. Everywhere in the code, the + # convention is that n_bins == max_bins + 1 + n_bins = self.max_bins + 1 # + 1 for missing values + self._bin_mapper = _BinMapper( + n_bins=n_bins, + is_categorical=self._is_categorical_remapped, + known_categories=known_categories, + random_state=self._random_seed, + n_threads=n_threads, + ) + X_binned_train = self._bin_data(X_train, is_training_data=True) + if X_val is not None: + X_binned_val = self._bin_data(X_val, is_training_data=False) + else: + X_binned_val = None + + # Uses binned data to check for missing values + has_missing_values = ( + (X_binned_train == self._bin_mapper.missing_values_bin_idx_) + .any(axis=0) + .astype(np.uint8) + ) + + if self.verbose: + print("Fitting gradient boosted rounds:") + + n_samples = X_binned_train.shape[0] + scoring_is_predefined_string = self.scoring in _SCORERS + need_raw_predictions_val = X_binned_val is not None and ( + scoring_is_predefined_string or self.scoring == "loss" + ) + # First time calling fit, or no warm start + if not (self._is_fitted() and self.warm_start): + # Clear random state and score attributes + self._clear_state() + + # initialize raw_predictions: those are the accumulated values + # predicted by the trees for the training data. raw_predictions has + # shape (n_samples, n_trees_per_iteration) where + # n_trees_per_iterations is n_classes in multiclass classification, + # else 1. + # self._baseline_prediction has shape (1, n_trees_per_iteration) + self._baseline_prediction = self._loss.fit_intercept_only( + y_true=y_train, sample_weight=sample_weight_train + ).reshape((1, -1)) + raw_predictions = np.zeros( + shape=(n_samples, self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", + ) + raw_predictions += self._baseline_prediction + + # predictors is a matrix (list of lists) of TreePredictor objects + # with shape (n_iter_, n_trees_per_iteration) + self._predictors = predictors = [] + + # Initialize structures and attributes related to early stopping + self._scorer = None # set if scoring != loss + raw_predictions_val = None # set if use val and scoring is a string + self.train_score_ = [] + self.validation_score_ = [] + + if self.do_early_stopping_: + # populate train_score and validation_score with the + # predictions of the initial model (before the first tree) + + # Create raw_predictions_val for storing the raw predictions of + # the validation data. + if need_raw_predictions_val: + raw_predictions_val = np.zeros( + shape=(X_binned_val.shape[0], self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", + ) + + raw_predictions_val += self._baseline_prediction + + if self.scoring == "loss": + # we're going to compute scoring w.r.t the loss. As losses + # take raw predictions as input (unlike the scorers), we + # can optimize a bit and avoid repeating computing the + # predictions of the previous trees. We'll reuse + # raw_predictions (as it's needed for training anyway) for + # evaluating the training loss. + + self._check_early_stopping_loss( + raw_predictions=raw_predictions, + y_train=y_train, + sample_weight_train=sample_weight_train, + raw_predictions_val=raw_predictions_val, + y_val=y_val, + sample_weight_val=sample_weight_val, + n_threads=n_threads, + ) + else: + self._scorer = check_scoring(self, self.scoring) + # _scorer is a callable with signature (est, X, y) and + # calls est.predict() or est.predict_proba() depending on + # its nature. + # Unfortunately, each call to _scorer() will compute + # the predictions of all the trees. So we use a subset of + # the training set to compute train scores. + + # Compute the subsample set + ( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + indices_small_train, + ) = self._get_small_trainset( + X_binned_train, + y_train, + sample_weight_train, + self._random_seed, + ) + + # If the scorer is a predefined string, then we optimize + # the evaluation by reusing the incrementally updated raw + # predictions. + if scoring_is_predefined_string: + raw_predictions_small_train = raw_predictions[ + indices_small_train + ] + else: + raw_predictions_small_train = None + + self._check_early_stopping_scorer( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + X_binned_val, + y_val, + sample_weight_val, + raw_predictions_small_train=raw_predictions_small_train, + raw_predictions_val=raw_predictions_val, + ) + begin_at_stage = 0 + + # warm start: this is not the first time fit was called + else: + # Check that the maximum number of iterations is not smaller + # than the number of iterations from the previous fit + if self.max_iter < self.n_iter_: + raise ValueError( + "max_iter=%d must be larger than or equal to " + "n_iter_=%d when warm_start==True" % (self.max_iter, self.n_iter_) + ) + + # Convert array attributes to lists + self.train_score_ = self.train_score_.tolist() + self.validation_score_ = self.validation_score_.tolist() + + # Compute raw predictions + raw_predictions = self._raw_predict(X_binned_train, n_threads=n_threads) + if self.do_early_stopping_ and need_raw_predictions_val: + raw_predictions_val = self._raw_predict( + X_binned_val, n_threads=n_threads + ) + else: + raw_predictions_val = None + + if self.do_early_stopping_ and self.scoring != "loss": + # Compute the subsample set + ( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + indices_small_train, + ) = self._get_small_trainset( + X_binned_train, y_train, sample_weight_train, self._random_seed + ) + + # Get the predictors from the previous fit + predictors = self._predictors + + begin_at_stage = self.n_iter_ + + # initialize gradients and hessians (empty arrays). + # shape = (n_samples, n_trees_per_iteration). + gradient, hessian = self._loss.init_gradient_and_hessian( + n_samples=n_samples, dtype=G_H_DTYPE, order="F" + ) + + for iteration in range(begin_at_stage, self.max_iter): + if self.verbose >= 2: + iteration_start_time = time() + print( + "[{}/{}] ".format(iteration + 1, self.max_iter), end="", flush=True + ) + + # Update gradients and hessians, inplace + # Note that self._loss expects shape (n_samples,) for + # n_trees_per_iteration = 1 else shape (n_samples, n_trees_per_iteration). + if self._loss.constant_hessian: + self._loss.gradient( + y_true=y_train, + raw_prediction=raw_predictions, + sample_weight=sample_weight_train, + gradient_out=gradient, + n_threads=n_threads, + ) + else: + self._loss.gradient_hessian( + y_true=y_train, + raw_prediction=raw_predictions, + sample_weight=sample_weight_train, + gradient_out=gradient, + hessian_out=hessian, + n_threads=n_threads, + ) + + # Append a list since there may be more than 1 predictor per iter + predictors.append([]) + + # 2-d views of shape (n_samples, n_trees_per_iteration_) or (n_samples, 1) + # on gradient and hessian to simplify the loop over n_trees_per_iteration_. + if gradient.ndim == 1: + g_view = gradient.reshape((-1, 1)) + h_view = hessian.reshape((-1, 1)) + else: + g_view = gradient + h_view = hessian + + # Build `n_trees_per_iteration` trees. + for k in range(self.n_trees_per_iteration_): + grower = TreeGrower( + X_binned=X_binned_train, + gradients=g_view[:, k], + hessians=h_view[:, k], + n_bins=n_bins, + n_bins_non_missing=self._bin_mapper.n_bins_non_missing_, + has_missing_values=has_missing_values, + is_categorical=self._is_categorical_remapped, + monotonic_cst=monotonic_cst_remapped, + interaction_cst=interaction_cst, + max_leaf_nodes=self.max_leaf_nodes, + max_depth=self.max_depth, + min_samples_leaf=self.min_samples_leaf, + l2_regularization=self.l2_regularization, + feature_fraction_per_split=self.max_features, + rng=self._feature_subsample_rng, + shrinkage=self.learning_rate, + n_threads=n_threads, + ) + grower.grow() + + acc_apply_split_time += grower.total_apply_split_time + acc_find_split_time += grower.total_find_split_time + acc_compute_hist_time += grower.total_compute_hist_time + + if not self._loss.differentiable: + _update_leaves_values( + loss=self._loss, + grower=grower, + y_true=y_train, + raw_prediction=raw_predictions[:, k], + sample_weight=sample_weight_train, + ) + + predictor = grower.make_predictor( + binning_thresholds=self._bin_mapper.bin_thresholds_ + ) + predictors[-1].append(predictor) + + # Update raw_predictions with the predictions of the newly + # created tree. + tic_pred = time() + _update_raw_predictions(raw_predictions[:, k], grower, n_threads) + toc_pred = time() + acc_prediction_time += toc_pred - tic_pred + + should_early_stop = False + if self.do_early_stopping_: + # Update raw_predictions_val with the newest tree(s) + if need_raw_predictions_val: + for k, pred in enumerate(self._predictors[-1]): + raw_predictions_val[:, k] += pred.predict_binned( + X_binned_val, + self._bin_mapper.missing_values_bin_idx_, + n_threads, + ) + + if self.scoring == "loss": + should_early_stop = self._check_early_stopping_loss( + raw_predictions=raw_predictions, + y_train=y_train, + sample_weight_train=sample_weight_train, + raw_predictions_val=raw_predictions_val, + y_val=y_val, + sample_weight_val=sample_weight_val, + n_threads=n_threads, + ) + + else: + # If the scorer is a predefined string, then we optimize the + # evaluation by reusing the incrementally computed raw predictions. + if scoring_is_predefined_string: + raw_predictions_small_train = raw_predictions[ + indices_small_train + ] + else: + raw_predictions_small_train = None + + should_early_stop = self._check_early_stopping_scorer( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + X_binned_val, + y_val, + sample_weight_val, + raw_predictions_small_train=raw_predictions_small_train, + raw_predictions_val=raw_predictions_val, + ) + + if self.verbose >= 2: + self._print_iteration_stats(iteration_start_time) + + # maybe we could also early stop if all the trees are stumps? + if should_early_stop: + break + + if self.verbose: + duration = time() - fit_start_time + n_total_leaves = sum( + predictor.get_n_leaf_nodes() + for predictors_at_ith_iteration in self._predictors + for predictor in predictors_at_ith_iteration + ) + n_predictors = sum( + len(predictors_at_ith_iteration) + for predictors_at_ith_iteration in self._predictors + ) + print( + "Fit {} trees in {:.3f} s, ({} total leaves)".format( + n_predictors, duration, n_total_leaves + ) + ) + print( + "{:<32} {:.3f}s".format( + "Time spent computing histograms:", acc_compute_hist_time + ) + ) + print( + "{:<32} {:.3f}s".format( + "Time spent finding best splits:", acc_find_split_time + ) + ) + print( + "{:<32} {:.3f}s".format( + "Time spent applying splits:", acc_apply_split_time + ) + ) + print( + "{:<32} {:.3f}s".format("Time spent predicting:", acc_prediction_time) + ) + + self.train_score_ = np.asarray(self.train_score_) + self.validation_score_ = np.asarray(self.validation_score_) + del self._in_fit # hard delete so we're sure it can't be used anymore + return self + + def _is_fitted(self): + return len(getattr(self, "_predictors", [])) > 0 + + def _clear_state(self): + """Clear the state of the gradient boosting model.""" + for var in ("train_score_", "validation_score_"): + if hasattr(self, var): + delattr(self, var) + + def _get_small_trainset(self, X_binned_train, y_train, sample_weight_train, seed): + """Compute the indices of the subsample set and return this set. + + For efficiency, we need to subsample the training set to compute scores + with scorers. + """ + # TODO: incorporate sample_weights here in `resample` + subsample_size = 10000 + if X_binned_train.shape[0] > subsample_size: + indices = np.arange(X_binned_train.shape[0]) + stratify = y_train if is_classifier(self) else None + indices = resample( + indices, + n_samples=subsample_size, + replace=False, + random_state=seed, + stratify=stratify, + ) + X_binned_small_train = X_binned_train[indices] + y_small_train = y_train[indices] + if sample_weight_train is not None: + sample_weight_small_train = sample_weight_train[indices] + else: + sample_weight_small_train = None + X_binned_small_train = np.ascontiguousarray(X_binned_small_train) + return ( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + indices, + ) + else: + return X_binned_train, y_train, sample_weight_train, slice(None) + + def _check_early_stopping_scorer( + self, + X_binned_small_train, + y_small_train, + sample_weight_small_train, + X_binned_val, + y_val, + sample_weight_val, + raw_predictions_small_train=None, + raw_predictions_val=None, + ): + """Check if fitting should be early-stopped based on scorer. + + Scores are computed on validation data or on training data. + """ + if is_classifier(self): + y_small_train = self.classes_[y_small_train.astype(int)] + + self.train_score_.append( + self._score_with_raw_predictions( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + raw_predictions_small_train, + ) + ) + + if self._use_validation_data: + if is_classifier(self): + y_val = self.classes_[y_val.astype(int)] + self.validation_score_.append( + self._score_with_raw_predictions( + X_binned_val, y_val, sample_weight_val, raw_predictions_val + ) + ) + return self._should_stop(self.validation_score_) + else: + return self._should_stop(self.train_score_) + + def _score_with_raw_predictions(self, X, y, sample_weight, raw_predictions=None): + if raw_predictions is None: + patcher_raw_predict = nullcontext() + else: + patcher_raw_predict = _patch_raw_predict(self, raw_predictions) + + with patcher_raw_predict: + if sample_weight is None: + return self._scorer(self, X, y) + else: + return self._scorer(self, X, y, sample_weight=sample_weight) + + def _check_early_stopping_loss( + self, + raw_predictions, + y_train, + sample_weight_train, + raw_predictions_val, + y_val, + sample_weight_val, + n_threads=1, + ): + """Check if fitting should be early-stopped based on loss. + + Scores are computed on validation data or on training data. + """ + self.train_score_.append( + -self._loss( + y_true=y_train, + raw_prediction=raw_predictions, + sample_weight=sample_weight_train, + n_threads=n_threads, + ) + ) + + if self._use_validation_data: + self.validation_score_.append( + -self._loss( + y_true=y_val, + raw_prediction=raw_predictions_val, + sample_weight=sample_weight_val, + n_threads=n_threads, + ) + ) + return self._should_stop(self.validation_score_) + else: + return self._should_stop(self.train_score_) + + def _should_stop(self, scores): + """ + Return True (do early stopping) if the last n scores aren't better + than the (n-1)th-to-last score, up to some tolerance. + """ + reference_position = self.n_iter_no_change + 1 + if len(scores) < reference_position: + return False + + # A higher score is always better. Higher tol means that it will be + # harder for subsequent iteration to be considered an improvement upon + # the reference score, and therefore it is more likely to early stop + # because of the lack of significant improvement. + reference_score = scores[-reference_position] + self.tol + recent_scores = scores[-reference_position + 1 :] + recent_improvements = [score > reference_score for score in recent_scores] + return not any(recent_improvements) + + def _bin_data(self, X, is_training_data): + """Bin data X. + + If is_training_data, then fit the _bin_mapper attribute. + Else, the binned data is converted to a C-contiguous array. + """ + + description = "training" if is_training_data else "validation" + if self.verbose: + print( + "Binning {:.3f} GB of {} data: ".format(X.nbytes / 1e9, description), + end="", + flush=True, + ) + tic = time() + if is_training_data: + X_binned = self._bin_mapper.fit_transform(X) # F-aligned array + else: + X_binned = self._bin_mapper.transform(X) # F-aligned array + # We convert the array to C-contiguous since predicting is faster + # with this layout (training is faster on F-arrays though) + X_binned = np.ascontiguousarray(X_binned) + toc = time() + if self.verbose: + duration = toc - tic + print("{:.3f} s".format(duration)) + + return X_binned + + def _print_iteration_stats(self, iteration_start_time): + """Print info about the current fitting iteration.""" + log_msg = "" + + predictors_of_ith_iteration = [ + predictors_list + for predictors_list in self._predictors[-1] + if predictors_list + ] + n_trees = len(predictors_of_ith_iteration) + max_depth = max( + predictor.get_max_depth() for predictor in predictors_of_ith_iteration + ) + n_leaves = sum( + predictor.get_n_leaf_nodes() for predictor in predictors_of_ith_iteration + ) + + if n_trees == 1: + log_msg += "{} tree, {} leaves, ".format(n_trees, n_leaves) + else: + log_msg += "{} trees, {} leaves ".format(n_trees, n_leaves) + log_msg += "({} on avg), ".format(int(n_leaves / n_trees)) + + log_msg += "max depth = {}, ".format(max_depth) + + if self.do_early_stopping_: + if self.scoring == "loss": + factor = -1 # score_ arrays contain the negative loss + name = "loss" + else: + factor = 1 + name = "score" + log_msg += "train {}: {:.5f}, ".format(name, factor * self.train_score_[-1]) + if self._use_validation_data: + log_msg += "val {}: {:.5f}, ".format( + name, factor * self.validation_score_[-1] + ) + + iteration_time = time() - iteration_start_time + log_msg += "in {:0.3f}s".format(iteration_time) + + print(log_msg) + + def _raw_predict(self, X, n_threads=None): + """Return the sum of the leaves values over all predictors. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. + n_threads : int, default=None + Number of OpenMP threads to use. `_openmp_effective_n_threads` is called + to determine the effective number of threads use, which takes cgroups CPU + quotes into account. See the docstring of `_openmp_effective_n_threads` + for details. + + Returns + ------- + raw_predictions : array, shape (n_samples, n_trees_per_iteration) + The raw predicted values. + """ + check_is_fitted(self) + is_binned = getattr(self, "_in_fit", False) + if not is_binned: + X = self._preprocess_X(X, reset=False) + + n_samples = X.shape[0] + raw_predictions = np.zeros( + shape=(n_samples, self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", + ) + raw_predictions += self._baseline_prediction + + # We intentionally decouple the number of threads used at prediction + # time from the number of threads used at fit time because the model + # can be deployed on a different machine for prediction purposes. + n_threads = _openmp_effective_n_threads(n_threads) + self._predict_iterations( + X, self._predictors, raw_predictions, is_binned, n_threads + ) + return raw_predictions + + def _predict_iterations(self, X, predictors, raw_predictions, is_binned, n_threads): + """Add the predictions of the predictors to raw_predictions.""" + if not is_binned: + ( + known_cat_bitsets, + f_idx_map, + ) = self._bin_mapper.make_known_categories_bitsets() + + for predictors_of_ith_iteration in predictors: + for k, predictor in enumerate(predictors_of_ith_iteration): + if is_binned: + predict = partial( + predictor.predict_binned, + missing_values_bin_idx=self._bin_mapper.missing_values_bin_idx_, + n_threads=n_threads, + ) + else: + predict = partial( + predictor.predict, + known_cat_bitsets=known_cat_bitsets, + f_idx_map=f_idx_map, + n_threads=n_threads, + ) + raw_predictions[:, k] += predict(X) + + def _staged_raw_predict(self, X): + """Compute raw predictions of ``X`` for each iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. + + Yields + ------ + raw_predictions : generator of ndarray of shape \ + (n_samples, n_trees_per_iteration) + The raw predictions of the input samples. The order of the + classes corresponds to that in the attribute :term:`classes_`. + """ + check_is_fitted(self) + X = self._preprocess_X(X, reset=False) + if X.shape[1] != self._n_features: + raise ValueError( + "X has {} features but this estimator was trained with " + "{} features.".format(X.shape[1], self._n_features) + ) + n_samples = X.shape[0] + raw_predictions = np.zeros( + shape=(n_samples, self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", + ) + raw_predictions += self._baseline_prediction + + # We intentionally decouple the number of threads used at prediction + # time from the number of threads used at fit time because the model + # can be deployed on a different machine for prediction purposes. + n_threads = _openmp_effective_n_threads() + for iteration in range(len(self._predictors)): + self._predict_iterations( + X, + self._predictors[iteration : iteration + 1], + raw_predictions, + is_binned=False, + n_threads=n_threads, + ) + yield raw_predictions.copy() + + def _compute_partial_dependence_recursion(self, grid, target_features): + """Fast partial dependence computation. + + Parameters + ---------- + grid : ndarray, shape (n_samples, n_target_features), dtype=np.float32 + The grid points on which the partial dependence should be + evaluated. + target_features : ndarray, shape (n_target_features), dtype=np.intp + The set of target features for which the partial dependence + should be evaluated. + + Returns + ------- + averaged_predictions : ndarray, shape \ + (n_trees_per_iteration, n_samples) + The value of the partial dependence function on each grid point. + """ + + if getattr(self, "_fitted_with_sw", False): + raise NotImplementedError( + "{} does not support partial dependence " + "plots with the 'recursion' method when " + "sample weights were given during fit " + "time.".format(self.__class__.__name__) + ) + + grid = np.asarray(grid, dtype=X_DTYPE, order="C") + averaged_predictions = np.zeros( + (self.n_trees_per_iteration_, grid.shape[0]), dtype=Y_DTYPE + ) + target_features = np.asarray(target_features, dtype=np.intp, order="C") + + for predictors_of_ith_iteration in self._predictors: + for k, predictor in enumerate(predictors_of_ith_iteration): + predictor.compute_partial_dependence( + grid, target_features, averaged_predictions[k] + ) + # Note that the learning rate is already accounted for in the leaves + # values. + + return averaged_predictions + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.allow_nan = True + return tags + + @abstractmethod + def _get_loss(self, sample_weight): + pass + + @abstractmethod + def _encode_y(self, y=None): + pass # pragma: no cover + + @abstractmethod + def _encode_y_val(self, y=None): + pass # pragma: no cover + + @property + def n_iter_(self): + """Number of iterations of the boosting process.""" + check_is_fitted(self) + return len(self._predictors) + + +class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): + """Histogram-based Gradient Boosting Regression Tree. + + This estimator is much faster than + :class:`GradientBoostingRegressor` + for big datasets (n_samples >= 10 000). + + This estimator has native support for missing values (NaNs). During + training, the tree grower learns at each split point whether samples + with missing values should go to the left or right child, based on the + potential gain. When predicting, samples with missing values are + assigned to the left or right child consequently. If no missing values + were encountered for a given feature during training, then samples with + missing values are mapped to whichever child has the most samples. + See :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` for a + usecase example of this feature. + + This implementation is inspired by + `LightGBM `_. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.21 + + Parameters + ---------- + loss : {'squared_error', 'absolute_error', 'gamma', 'poisson', 'quantile'}, \ + default='squared_error' + The loss function to use in the boosting process. Note that the + "squared error", "gamma" and "poisson" losses actually implement + "half least squares loss", "half gamma deviance" and "half poisson + deviance" to simplify the computation of the gradient. Furthermore, + "gamma" and "poisson" losses internally use a log-link, "gamma" + requires ``y > 0`` and "poisson" requires ``y >= 0``. + "quantile" uses the pinball loss. + + .. versionchanged:: 0.23 + Added option 'poisson'. + + .. versionchanged:: 1.1 + Added option 'quantile'. + + .. versionchanged:: 1.3 + Added option 'gamma'. + + quantile : float, default=None + If loss is "quantile", this parameter specifies which quantile to be estimated + and must be between 0 and 1. + learning_rate : float, default=0.1 + The learning rate, also known as *shrinkage*. This is used as a + multiplicative factor for the leaves values. Use ``1`` for no + shrinkage. + max_iter : int, default=100 + The maximum number of iterations of the boosting process, i.e. the + maximum number of trees. + max_leaf_nodes : int or None, default=31 + The maximum number of leaves for each tree. Must be strictly greater + than 1. If None, there is no maximum limit. + max_depth : int or None, default=None + The maximum depth of each tree. The depth of a tree is the number of + edges to go from the root to the deepest leaf. + Depth isn't constrained by default. + min_samples_leaf : int, default=20 + The minimum number of samples per leaf. For small datasets with less + than a few hundred samples, it is recommended to lower this value + since only very shallow trees would be built. + l2_regularization : float, default=0 + The L2 regularization parameter penalizing leaves with small hessians. + Use ``0`` for no regularization (default). + max_features : float, default=1.0 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + If interaction constraints from `interaction_cst` are present, only allowed + features are taken into account for the subsampling. + + .. versionadded:: 1.4 + + max_bins : int, default=255 + The maximum number of bins to use for non-missing values. Before + training, each feature of the input array `X` is binned into + integer-valued bins, which allows for a much faster training stage. + Features with a small number of unique values may use less than + ``max_bins`` bins. In addition to the ``max_bins`` bins, one more bin + is always reserved for missing values. Must be no larger than 255. + categorical_features : array-like of {bool, int, str} of shape (n_features) \ + or shape (n_categorical_features,), default='from_dtype' + Indicates the categorical features. + + - None : no feature will be considered categorical. + - boolean array-like : boolean mask indicating categorical features. + - integer array-like : integer indices indicating categorical + features. + - str array-like: names of categorical features (assuming the training + data has feature names). + - `"from_dtype"`: dataframe columns with dtype "category" are + considered to be categorical features. The input must be an object + exposing a ``__dataframe__`` method such as pandas or polars + DataFrames to use this feature. + + For each categorical feature, there must be at most `max_bins` unique + categories. Negative values for categorical features encoded as numeric + dtypes are treated as missing values. All categorical values are + converted to floating point numbers. This means that categorical values + of 1.0 and 1 are treated as the same category. + + Read more in the :ref:`User Guide ` and + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_categorical.py`. + + .. versionadded:: 0.24 + + .. versionchanged:: 1.2 + Added support for feature names. + + .. versionchanged:: 1.4 + Added `"from_dtype"` option. + + .. versionchanged:: 1.6 + The default value changed from `None` to `"from_dtype"`. + + monotonic_cst : array-like of int of shape (n_features) or dict, default=None + Monotonic constraint to enforce on each feature are specified using the + following integer values: + + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + If a dict with str keys, map feature to monotonic constraints by name. + If an array, the features are mapped to constraints by position. See + :ref:`monotonic_cst_features_names` for a usage example. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.23 + + .. versionchanged:: 1.2 + Accept dict of constraints with feature names as keys. + + interaction_cst : {"pairwise", "no_interactions"} or sequence of lists/tuples/sets \ + of int, default=None + Specify interaction constraints, the sets of features which can + interact with each other in child node splits. + + Each item specifies the set of feature indices that are allowed + to interact with each other. If there are more features than + specified in these constraints, they are treated as if they were + specified as an additional set. + + The strings "pairwise" and "no_interactions" are shorthands for + allowing only pairwise or no interactions, respectively. + + For instance, with 5 features in total, `interaction_cst=[{0, 1}]` + is equivalent to `interaction_cst=[{0, 1}, {2, 3, 4}]`, + and specifies that each branch of a tree will either only split + on features 0 and 1 or only split on features 2, 3 and 4. + + See :ref:`this example` on how to use `interaction_cst`. + + .. versionadded:: 1.2 + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble. For results to be valid, the + estimator should be re-trained on the same data only. + See :term:`the Glossary `. + early_stopping : 'auto' or bool, default='auto' + If 'auto', early stopping is enabled if the sample size is larger than + 10000 or if `X_val` and `y_val` are passed to `fit`. If True, early stopping + is enabled, otherwise early stopping is disabled. + + .. versionadded:: 0.23 + + scoring : str or callable or None, default='loss' + Scoring method to use for early stopping. Only used if `early_stopping` + is enabled. Options: + + - str: see :ref:`scoring_string_names` for options. + - callable: a scorer callable object (e.g., function) with signature + ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details. + - `None`: the :ref:`coefficient of determination ` + (:math:`R^2`) is used. + - 'loss': early stopping is checked w.r.t the loss value. + + validation_fraction : int or float or None, default=0.1 + Proportion (or absolute size) of training data to set aside as + validation data for early stopping. If None, early stopping is done on + the training data. + The value is ignored if either early stopping is not performed, e.g. + `early_stopping=False`, or if `X_val` and `y_val` are passed to fit. + n_iter_no_change : int, default=10 + Used to determine when to "early stop". The fitting process is + stopped when none of the last ``n_iter_no_change`` scores are better + than the ``n_iter_no_change - 1`` -th-to-last one, up to some + tolerance. Only used if early stopping is performed. + tol : float, default=1e-7 + The absolute tolerance to use when comparing scores during early + stopping. The higher the tolerance, the more likely we are to early + stop: higher tolerance means that it will be harder for subsequent + iterations to be considered an improvement upon the reference score. + verbose : int, default=0 + The verbosity level. If not zero, print some information about the + fitting process. ``1`` prints only summary info, ``2`` prints info per + iteration. + random_state : int, RandomState instance or None, default=None + Pseudo-random number generator to control the subsampling in the + binning process, and the train/validation data split if early stopping + is enabled. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + Attributes + ---------- + do_early_stopping_ : bool + Indicates whether early stopping is used during training. + n_iter_ : int + The number of iterations as selected by early stopping, depending on + the `early_stopping` parameter. Otherwise it corresponds to max_iter. + n_trees_per_iteration_ : int + The number of tree that are built at each iteration. For regressors, + this is always 1. + train_score_ : ndarray, shape (n_iter_+1,) + The scores at each iteration on the training data. The first entry + is the score of the ensemble before the first iteration. Scores are + computed according to the ``scoring`` parameter. If ``scoring`` is + not 'loss', scores are computed on a subset of at most 10 000 + samples. Empty if no early stopping. + validation_score_ : ndarray, shape (n_iter_+1,) + The scores at each iteration on the held-out validation data. The + first entry is the score of the ensemble before the first iteration. + Scores are computed according to the ``scoring`` parameter. Empty if + no early stopping or if ``validation_fraction`` is None. + is_categorical_ : ndarray, shape (n_features, ) or None + Boolean mask for the categorical features. ``None`` if there are no + categorical features. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + GradientBoostingRegressor : Exact gradient boosting method that does not + scale as good on datasets with a large number of samples. + sklearn.tree.DecisionTreeRegressor : A decision tree regressor. + RandomForestRegressor : A meta-estimator that fits a number of decision + tree regressors on various sub-samples of the dataset and uses + averaging to improve the statistical performance and control + over-fitting. + AdaBoostRegressor : A meta-estimator that begins by fitting a regressor + on the original dataset and then fits additional copies of the + regressor on the same dataset but where the weights of instances are + adjusted according to the error of the current prediction. As such, + subsequent regressors focus more on difficult cases. + + Examples + -------- + >>> from sklearn.ensemble import HistGradientBoostingRegressor + >>> from sklearn.datasets import load_diabetes + >>> X, y = load_diabetes(return_X_y=True) + >>> est = HistGradientBoostingRegressor().fit(X, y) + >>> est.score(X, y) + 0.92... + """ + + _parameter_constraints: dict = { + **BaseHistGradientBoosting._parameter_constraints, + "loss": [ + StrOptions( + { + "squared_error", + "absolute_error", + "poisson", + "gamma", + "quantile", + } + ), + BaseLoss, + ], + "quantile": [Interval(Real, 0, 1, closed="both"), None], + } + + def __init__( + self, + loss="squared_error", + *, + quantile=None, + learning_rate=0.1, + max_iter=100, + max_leaf_nodes=31, + max_depth=None, + min_samples_leaf=20, + l2_regularization=0.0, + max_features=1.0, + max_bins=255, + categorical_features="from_dtype", + monotonic_cst=None, + interaction_cst=None, + warm_start=False, + early_stopping="auto", + scoring="loss", + validation_fraction=0.1, + n_iter_no_change=10, + tol=1e-7, + verbose=0, + random_state=None, + ): + super().__init__( + loss=loss, + learning_rate=learning_rate, + max_iter=max_iter, + max_leaf_nodes=max_leaf_nodes, + max_depth=max_depth, + min_samples_leaf=min_samples_leaf, + l2_regularization=l2_regularization, + max_features=max_features, + max_bins=max_bins, + monotonic_cst=monotonic_cst, + interaction_cst=interaction_cst, + categorical_features=categorical_features, + early_stopping=early_stopping, + warm_start=warm_start, + scoring=scoring, + validation_fraction=validation_fraction, + n_iter_no_change=n_iter_no_change, + tol=tol, + verbose=verbose, + random_state=random_state, + ) + self.quantile = quantile + + def predict(self, X): + """Predict values for X. + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + The input samples. + + Returns + ------- + y : ndarray, shape (n_samples,) + The predicted values. + """ + check_is_fitted(self) + # Return inverse link of raw predictions after converting + # shape (n_samples, 1) to (n_samples,) + return self._loss.link.inverse(self._raw_predict(X).ravel()) + + def staged_predict(self, X): + """Predict regression target for each iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + .. versionadded:: 0.24 + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted values of the input samples, for each iteration. + """ + for raw_predictions in self._staged_raw_predict(X): + yield self._loss.link.inverse(raw_predictions.ravel()) + + def _encode_y(self, y): + # Just convert y to the expected dtype + self.n_trees_per_iteration_ = 1 + y = y.astype(Y_DTYPE, copy=False) + if self.loss == "gamma": + # Ensure y > 0 + if not np.all(y > 0): + raise ValueError("loss='gamma' requires strictly positive y.") + elif self.loss == "poisson": + # Ensure y >= 0 and sum(y) > 0 + if not (np.all(y >= 0) and np.sum(y) > 0): + raise ValueError( + "loss='poisson' requires non-negative y and sum(y) > 0." + ) + return y + + def _encode_y_val(self, y=None): + return self._encode_y(y) + + def _get_loss(self, sample_weight): + if self.loss == "quantile": + return _LOSSES[self.loss]( + sample_weight=sample_weight, quantile=self.quantile + ) + else: + return _LOSSES[self.loss](sample_weight=sample_weight) + + +class HistGradientBoostingClassifier(ClassifierMixin, BaseHistGradientBoosting): + """Histogram-based Gradient Boosting Classification Tree. + + This estimator is much faster than + :class:`GradientBoostingClassifier` + for big datasets (n_samples >= 10 000). + + This estimator has native support for missing values (NaNs). During + training, the tree grower learns at each split point whether samples + with missing values should go to the left or right child, based on the + potential gain. When predicting, samples with missing values are + assigned to the left or right child consequently. If no missing values + were encountered for a given feature during training, then samples with + missing values are mapped to whichever child has the most samples. + + This implementation is inspired by + `LightGBM `_. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.21 + + Parameters + ---------- + loss : {'log_loss'}, default='log_loss' + The loss function to use in the boosting process. + + For binary classification problems, 'log_loss' is also known as logistic loss, + binomial deviance or binary crossentropy. Internally, the model fits one tree + per boosting iteration and uses the logistic sigmoid function (expit) as + inverse link function to compute the predicted positive class probability. + + For multiclass classification problems, 'log_loss' is also known as multinomial + deviance or categorical crossentropy. Internally, the model fits one tree per + boosting iteration and per class and uses the softmax function as inverse link + function to compute the predicted probabilities of the classes. + + learning_rate : float, default=0.1 + The learning rate, also known as *shrinkage*. This is used as a + multiplicative factor for the leaves values. Use ``1`` for no + shrinkage. + max_iter : int, default=100 + The maximum number of iterations of the boosting process, i.e. the + maximum number of trees for binary classification. For multiclass + classification, `n_classes` trees per iteration are built. + max_leaf_nodes : int or None, default=31 + The maximum number of leaves for each tree. Must be strictly greater + than 1. If None, there is no maximum limit. + max_depth : int or None, default=None + The maximum depth of each tree. The depth of a tree is the number of + edges to go from the root to the deepest leaf. + Depth isn't constrained by default. + min_samples_leaf : int, default=20 + The minimum number of samples per leaf. For small datasets with less + than a few hundred samples, it is recommended to lower this value + since only very shallow trees would be built. + l2_regularization : float, default=0 + The L2 regularization parameter penalizing leaves with small hessians. + Use ``0`` for no regularization (default). + max_features : float, default=1.0 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + If interaction constraints from `interaction_cst` are present, only allowed + features are taken into account for the subsampling. + + .. versionadded:: 1.4 + + max_bins : int, default=255 + The maximum number of bins to use for non-missing values. Before + training, each feature of the input array `X` is binned into + integer-valued bins, which allows for a much faster training stage. + Features with a small number of unique values may use less than + ``max_bins`` bins. In addition to the ``max_bins`` bins, one more bin + is always reserved for missing values. Must be no larger than 255. + categorical_features : array-like of {bool, int, str} of shape (n_features) \ + or shape (n_categorical_features,), default='from_dtype' + Indicates the categorical features. + + - None : no feature will be considered categorical. + - boolean array-like : boolean mask indicating categorical features. + - integer array-like : integer indices indicating categorical + features. + - str array-like: names of categorical features (assuming the training + data has feature names). + - `"from_dtype"`: dataframe columns with dtype "category" are + considered to be categorical features. The input must be an object + exposing a ``__dataframe__`` method such as pandas or polars + DataFrames to use this feature. + + For each categorical feature, there must be at most `max_bins` unique + categories. Negative values for categorical features encoded as numeric + dtypes are treated as missing values. All categorical values are + converted to floating point numbers. This means that categorical values + of 1.0 and 1 are treated as the same category. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.24 + + .. versionchanged:: 1.2 + Added support for feature names. + + .. versionchanged:: 1.4 + Added `"from_dtype"` option. + + .. versionchanged:: 1.6 + The default value changed from `None` to `"from_dtype"`. + + monotonic_cst : array-like of int of shape (n_features) or dict, default=None + Monotonic constraint to enforce on each feature are specified using the + following integer values: + + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + If a dict with str keys, map feature to monotonic constraints by name. + If an array, the features are mapped to constraints by position. See + :ref:`monotonic_cst_features_names` for a usage example. + + The constraints are only valid for binary classifications and hold + over the probability of the positive class. + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.23 + + .. versionchanged:: 1.2 + Accept dict of constraints with feature names as keys. + + interaction_cst : {"pairwise", "no_interactions"} or sequence of lists/tuples/sets \ + of int, default=None + Specify interaction constraints, the sets of features which can + interact with each other in child node splits. + + Each item specifies the set of feature indices that are allowed + to interact with each other. If there are more features than + specified in these constraints, they are treated as if they were + specified as an additional set. + + The strings "pairwise" and "no_interactions" are shorthands for + allowing only pairwise or no interactions, respectively. + + For instance, with 5 features in total, `interaction_cst=[{0, 1}]` + is equivalent to `interaction_cst=[{0, 1}, {2, 3, 4}]`, + and specifies that each branch of a tree will either only split + on features 0 and 1 or only split on features 2, 3 and 4. + + See :ref:`this example` on how to use `interaction_cst`. + + .. versionadded:: 1.2 + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble. For results to be valid, the + estimator should be re-trained on the same data only. + See :term:`the Glossary `. + early_stopping : 'auto' or bool, default='auto' + If 'auto', early stopping is enabled if the sample size is larger than + 10000 or if `X_val` and `y_val` are passed to `fit`. If True, early stopping + is enabled, otherwise early stopping is disabled. + + .. versionadded:: 0.23 + + scoring : str or callable or None, default='loss' + Scoring method to use for early stopping. Only used if `early_stopping` + is enabled. Options: + + - str: see :ref:`scoring_string_names` for options. + - callable: a scorer callable object (e.g., function) with signature + ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details. + - `None`: :ref:`accuracy ` is used. + - 'loss': early stopping is checked w.r.t the loss value. + + validation_fraction : int or float or None, default=0.1 + Proportion (or absolute size) of training data to set aside as + validation data for early stopping. If None, early stopping is done on + the training data. + The value is ignored if either early stopping is not performed, e.g. + `early_stopping=False`, or if `X_val` and `y_val` are passed to fit. + n_iter_no_change : int, default=10 + Used to determine when to "early stop". The fitting process is + stopped when none of the last ``n_iter_no_change`` scores are better + than the ``n_iter_no_change - 1`` -th-to-last one, up to some + tolerance. Only used if early stopping is performed. + tol : float, default=1e-7 + The absolute tolerance to use when comparing scores. The higher the + tolerance, the more likely we are to early stop: higher tolerance + means that it will be harder for subsequent iterations to be + considered an improvement upon the reference score. + verbose : int, default=0 + The verbosity level. If not zero, print some information about the + fitting process. ``1`` prints only summary info, ``2`` prints info per + iteration. + random_state : int, RandomState instance or None, default=None + Pseudo-random number generator to control the subsampling in the + binning process, and the train/validation data split if early stopping + is enabled. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + class_weight : dict or 'balanced', default=None + Weights associated with classes in the form `{class_label: weight}`. + If not given, all classes are supposed to have weight one. + The "balanced" mode uses the values of y to automatically adjust + weights inversely proportional to class frequencies in the input data + as `n_samples / (n_classes * np.bincount(y))`. + Note that these weights will be multiplied with sample_weight (passed + through the fit method) if `sample_weight` is specified. + + .. versionadded:: 1.2 + + Attributes + ---------- + classes_ : array, shape = (n_classes,) + Class labels. + do_early_stopping_ : bool + Indicates whether early stopping is used during training. + n_iter_ : int + The number of iterations as selected by early stopping, depending on + the `early_stopping` parameter. Otherwise it corresponds to max_iter. + n_trees_per_iteration_ : int + The number of tree that are built at each iteration. This is equal to 1 + for binary classification, and to ``n_classes`` for multiclass + classification. + train_score_ : ndarray, shape (n_iter_+1,) + The scores at each iteration on the training data. The first entry + is the score of the ensemble before the first iteration. Scores are + computed according to the ``scoring`` parameter. If ``scoring`` is + not 'loss', scores are computed on a subset of at most 10 000 + samples. Empty if no early stopping. + validation_score_ : ndarray, shape (n_iter_+1,) + The scores at each iteration on the held-out validation data. The + first entry is the score of the ensemble before the first iteration. + Scores are computed according to the ``scoring`` parameter. Empty if + no early stopping or if ``validation_fraction`` is None. + is_categorical_ : ndarray, shape (n_features, ) or None + Boolean mask for the categorical features. ``None`` if there are no + categorical features. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + GradientBoostingClassifier : Exact gradient boosting method that does not + scale as good on datasets with a large number of samples. + sklearn.tree.DecisionTreeClassifier : A decision tree classifier. + RandomForestClassifier : A meta-estimator that fits a number of decision + tree classifiers on various sub-samples of the dataset and uses + averaging to improve the predictive accuracy and control over-fitting. + AdaBoostClassifier : A meta-estimator that begins by fitting a classifier + on the original dataset and then fits additional copies of the + classifier on the same dataset where the weights of incorrectly + classified instances are adjusted such that subsequent classifiers + focus more on difficult cases. + + Examples + -------- + >>> from sklearn.ensemble import HistGradientBoostingClassifier + >>> from sklearn.datasets import load_iris + >>> X, y = load_iris(return_X_y=True) + >>> clf = HistGradientBoostingClassifier().fit(X, y) + >>> clf.score(X, y) + 1.0 + """ + + _parameter_constraints: dict = { + **BaseHistGradientBoosting._parameter_constraints, + "loss": [StrOptions({"log_loss"}), BaseLoss], + "class_weight": [dict, StrOptions({"balanced"}), None], + } + + def __init__( + self, + loss="log_loss", + *, + learning_rate=0.1, + max_iter=100, + max_leaf_nodes=31, + max_depth=None, + min_samples_leaf=20, + l2_regularization=0.0, + max_features=1.0, + max_bins=255, + categorical_features="from_dtype", + monotonic_cst=None, + interaction_cst=None, + warm_start=False, + early_stopping="auto", + scoring="loss", + validation_fraction=0.1, + n_iter_no_change=10, + tol=1e-7, + verbose=0, + random_state=None, + class_weight=None, + ): + super().__init__( + loss=loss, + learning_rate=learning_rate, + max_iter=max_iter, + max_leaf_nodes=max_leaf_nodes, + max_depth=max_depth, + min_samples_leaf=min_samples_leaf, + l2_regularization=l2_regularization, + max_features=max_features, + max_bins=max_bins, + categorical_features=categorical_features, + monotonic_cst=monotonic_cst, + interaction_cst=interaction_cst, + warm_start=warm_start, + early_stopping=early_stopping, + scoring=scoring, + validation_fraction=validation_fraction, + n_iter_no_change=n_iter_no_change, + tol=tol, + verbose=verbose, + random_state=random_state, + ) + self.class_weight = class_weight + + def _finalize_sample_weight(self, sample_weight, y): + """Adjust sample_weights with class_weights.""" + if self.class_weight is None: + return sample_weight + + expanded_class_weight = compute_sample_weight(self.class_weight, y) + + if sample_weight is not None: + return sample_weight * expanded_class_weight + else: + return expanded_class_weight + + def predict(self, X): + """Predict classes for X. + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + The input samples. + + Returns + ------- + y : ndarray, shape (n_samples,) + The predicted classes. + """ + # TODO: This could be done in parallel + raw_predictions = self._raw_predict(X) + if raw_predictions.shape[1] == 1: + # np.argmax([0.5, 0.5]) is 0, not 1. Therefore "> 0" not ">= 0" to be + # consistent with the multiclass case. + encoded_classes = (raw_predictions.ravel() > 0).astype(int) + else: + encoded_classes = np.argmax(raw_predictions, axis=1) + return self.classes_[encoded_classes] + + def staged_predict(self, X): + """Predict classes at each iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + .. versionadded:: 0.24 + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted classes of the input samples, for each iteration. + """ + for raw_predictions in self._staged_raw_predict(X): + if raw_predictions.shape[1] == 1: + # np.argmax([0, 0]) is 0, not 1, therefore "> 0" not ">= 0" + encoded_classes = (raw_predictions.ravel() > 0).astype(int) + else: + encoded_classes = np.argmax(raw_predictions, axis=1) + yield self.classes_.take(encoded_classes, axis=0) + + def predict_proba(self, X): + """Predict class probabilities for X. + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + The input samples. + + Returns + ------- + p : ndarray, shape (n_samples, n_classes) + The class probabilities of the input samples. + """ + raw_predictions = self._raw_predict(X) + return self._loss.predict_proba(raw_predictions) + + def staged_predict_proba(self, X): + """Predict class probabilities at each iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted class probabilities of the input samples, + for each iteration. + """ + for raw_predictions in self._staged_raw_predict(X): + yield self._loss.predict_proba(raw_predictions) + + def decision_function(self, X): + """Compute the decision function of ``X``. + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + The input samples. + + Returns + ------- + decision : ndarray, shape (n_samples,) or \ + (n_samples, n_trees_per_iteration) + The raw predicted values (i.e. the sum of the trees leaves) for + each sample. n_trees_per_iteration is equal to the number of + classes in multiclass classification. + """ + decision = self._raw_predict(X) + if decision.shape[1] == 1: + decision = decision.ravel() + return decision + + def staged_decision_function(self, X): + """Compute decision function of ``X`` for each iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each stage. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. + + Yields + ------ + decision : generator of ndarray of shape (n_samples,) or \ + (n_samples, n_trees_per_iteration) + The decision function of the input samples, which corresponds to + the raw values predicted from the trees of the ensemble . The + classes corresponds to that in the attribute :term:`classes_`. + """ + for staged_decision in self._staged_raw_predict(X): + if staged_decision.shape[1] == 1: + staged_decision = staged_decision.ravel() + yield staged_decision + + def _encode_y(self, y): + """Create self._label_encoder and encode y correspondingly.""" + # encode classes into 0 ... n_classes - 1 and sets attributes classes_ + # and n_trees_per_iteration_ + check_classification_targets(y) + + # We need to store the label encoder in case y_val needs to be label encoded, + # too. + self._label_encoder = LabelEncoder() + encoded_y = self._label_encoder.fit_transform(y) + self.classes_ = self._label_encoder.classes_ + n_classes = self.classes_.shape[0] + # only 1 tree for binary classification. For multiclass classification, + # we build 1 tree per class. + self.n_trees_per_iteration_ = 1 if n_classes <= 2 else n_classes + encoded_y = encoded_y.astype(Y_DTYPE, copy=False) + return encoded_y + + def _encode_y_val(self, y): + encoded_y = self._label_encoder.transform(y) + return encoded_y.astype(Y_DTYPE, copy=False) + + def _get_loss(self, sample_weight): + # At this point self.loss == "log_loss" + if self.n_trees_per_iteration_ == 1: + return HalfBinomialLoss(sample_weight=sample_weight) + else: + return HalfMultinomialLoss( + sample_weight=sample_weight, n_classes=self.n_trees_per_iteration_ + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/grower.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/grower.py new file mode 100644 index 0000000000000000000000000000000000000000..cff5e1cbf95d21e3e4ffb88a344974901bb5929e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/grower.py @@ -0,0 +1,822 @@ +""" +This module contains the TreeGrower class. + +TreeGrower builds a regression tree fitting a Newton-Raphson step, based on +the gradients and hessians of the training data. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import numbers +from heapq import heappop, heappush +from timeit import default_timer as time + +import numpy as np + +from sklearn.ensemble._hist_gradient_boosting._bitset import ( + set_raw_bitset_from_binned_bitset, +) +from sklearn.ensemble._hist_gradient_boosting.common import ( + PREDICTOR_RECORD_DTYPE, + X_BITSET_INNER_DTYPE, + MonotonicConstraint, +) +from sklearn.ensemble._hist_gradient_boosting.histogram import HistogramBuilder +from sklearn.ensemble._hist_gradient_boosting.predictor import TreePredictor +from sklearn.ensemble._hist_gradient_boosting.splitting import Splitter +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + + +class TreeNode: + """Tree Node class used in TreeGrower. + + This isn't used for prediction purposes, only for training (see + TreePredictor). + + Parameters + ---------- + depth : int + The depth of the node, i.e. its distance from the root. + sample_indices : ndarray of shape (n_samples_at_node,), dtype=np.uint32 + The indices of the samples at the node. + partition_start : int + start position of the node's sample_indices in splitter.partition. + partition_stop : int + stop position of the node's sample_indices in splitter.partition. + sum_gradients : float + The sum of the gradients of the samples at the node. + sum_hessians : float + The sum of the hessians of the samples at the node. + + Attributes + ---------- + depth : int + The depth of the node, i.e. its distance from the root. + sample_indices : ndarray of shape (n_samples_at_node,), dtype=np.uint32 + The indices of the samples at the node. + sum_gradients : float + The sum of the gradients of the samples at the node. + sum_hessians : float + The sum of the hessians of the samples at the node. + split_info : SplitInfo or None + The result of the split evaluation. + is_leaf : bool + True if node is a leaf + left_child : TreeNode or None + The left child of the node. None for leaves. + right_child : TreeNode or None + The right child of the node. None for leaves. + value : float or None + The value of the leaf, as computed in finalize_leaf(). None for + non-leaf nodes. + partition_start : int + start position of the node's sample_indices in splitter.partition. + partition_stop : int + stop position of the node's sample_indices in splitter.partition. + allowed_features : None or ndarray, dtype=int + Indices of features allowed to split for children. + interaction_cst_indices : None or list of ints + Indices of the interaction sets that have to be applied on splits of + child nodes. The fewer sets the stronger the constraint as fewer sets + contain fewer features. + children_lower_bound : float + children_upper_bound : float + """ + + def __init__( + self, + *, + depth, + sample_indices, + partition_start, + partition_stop, + sum_gradients, + sum_hessians, + value=None, + ): + self.depth = depth + self.sample_indices = sample_indices + self.n_samples = sample_indices.shape[0] + self.sum_gradients = sum_gradients + self.sum_hessians = sum_hessians + self.value = value + self.is_leaf = False + self.allowed_features = None + self.interaction_cst_indices = None + self.set_children_bounds(float("-inf"), float("+inf")) + self.split_info = None + self.left_child = None + self.right_child = None + self.histograms = None + # start and stop indices of the node in the splitter.partition + # array. Concretely, + # self.sample_indices = view(self.splitter.partition[start:stop]) + # Please see the comments about splitter.partition and + # splitter.split_indices for more info about this design. + # These 2 attributes are only used in _update_raw_prediction, because we + # need to iterate over the leaves and I don't know how to efficiently + # store the sample_indices views because they're all of different sizes. + self.partition_start = partition_start + self.partition_stop = partition_stop + + def set_children_bounds(self, lower, upper): + """Set children values bounds to respect monotonic constraints.""" + + # These are bounds for the node's *children* values, not the node's + # value. The bounds are used in the splitter when considering potential + # left and right child. + self.children_lower_bound = lower + self.children_upper_bound = upper + + def __lt__(self, other_node): + """Comparison for priority queue. + + Nodes with high gain are higher priority than nodes with low gain. + + heapq.heappush only need the '<' operator. + heapq.heappop take the smallest item first (smaller is higher + priority). + + Parameters + ---------- + other_node : TreeNode + The node to compare with. + """ + return self.split_info.gain > other_node.split_info.gain + + +class TreeGrower: + """Tree grower class used to build a tree. + + The tree is fitted to predict the values of a Newton-Raphson step. The + splits are considered in a best-first fashion, and the quality of a + split is defined in splitting._split_gain. + + Parameters + ---------- + X_binned : ndarray of shape (n_samples, n_features), dtype=np.uint8 + The binned input samples. Must be Fortran-aligned. + gradients : ndarray of shape (n_samples,) + The gradients of each training sample. Those are the gradients of the + loss w.r.t the predictions, evaluated at iteration ``i - 1``. + hessians : ndarray of shape (n_samples,) + The hessians of each training sample. Those are the hessians of the + loss w.r.t the predictions, evaluated at iteration ``i - 1``. + max_leaf_nodes : int, default=None + The maximum number of leaves for each tree. If None, there is no + maximum limit. + max_depth : int, default=None + The maximum depth of each tree. The depth of a tree is the number of + edges to go from the root to the deepest leaf. + Depth isn't constrained by default. + min_samples_leaf : int, default=20 + The minimum number of samples per leaf. + min_gain_to_split : float, default=0. + The minimum gain needed to split a node. Splits with lower gain will + be ignored. + min_hessian_to_split : float, default=1e-3 + The minimum sum of hessians needed in each node. Splits that result in + at least one child having a sum of hessians less than + ``min_hessian_to_split`` are discarded. + n_bins : int, default=256 + The total number of bins, including the bin for missing values. Used + to define the shape of the histograms. + n_bins_non_missing : ndarray, dtype=np.uint32, default=None + For each feature, gives the number of bins actually used for + non-missing values. For features with a lot of unique values, this + is equal to ``n_bins - 1``. If it's an int, all features are + considered to have the same number of bins. If None, all features + are considered to have ``n_bins - 1`` bins. + has_missing_values : bool or ndarray, dtype=bool, default=False + Whether each feature contains missing values (in the training data). + If it's a bool, the same value is used for all features. + is_categorical : ndarray of bool of shape (n_features,), default=None + Indicates categorical features. + monotonic_cst : array-like of int of shape (n_features,), dtype=int, default=None + Indicates the monotonic constraint to enforce on each feature. + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + Read more in the :ref:`User Guide `. + interaction_cst : list of sets of integers, default=None + List of interaction constraints. + l2_regularization : float, default=0. + The L2 regularization parameter penalizing leaves with small hessians. + Use ``0`` for no regularization (default). + feature_fraction_per_split : float, default=1 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + rng : Generator + Numpy random Generator used for feature subsampling. + shrinkage : float, default=1. + The shrinkage parameter to apply to the leaves values, also known as + learning rate. + n_threads : int, default=None + Number of OpenMP threads to use. `_openmp_effective_n_threads` is called + to determine the effective number of threads use, which takes cgroups CPU + quotes into account. See the docstring of `_openmp_effective_n_threads` + for details. + + Attributes + ---------- + histogram_builder : HistogramBuilder + splitter : Splitter + root : TreeNode + finalized_leaves : list of TreeNode + splittable_nodes : list of TreeNode + missing_values_bin_idx : int + Equals n_bins - 1 + n_categorical_splits : int + n_features : int + n_nodes : int + total_find_split_time : float + Time spent finding the best splits + total_compute_hist_time : float + Time spent computing histograms + total_apply_split_time : float + Time spent splitting nodes + with_monotonic_cst : bool + Whether there are monotonic constraints that apply. False iff monotonic_cst is + None. + """ + + def __init__( + self, + X_binned, + gradients, + hessians, + max_leaf_nodes=None, + max_depth=None, + min_samples_leaf=20, + min_gain_to_split=0.0, + min_hessian_to_split=1e-3, + n_bins=256, + n_bins_non_missing=None, + has_missing_values=False, + is_categorical=None, + monotonic_cst=None, + interaction_cst=None, + l2_regularization=0.0, + feature_fraction_per_split=1.0, + rng=np.random.default_rng(), + shrinkage=1.0, + n_threads=None, + ): + self._validate_parameters( + X_binned, + min_gain_to_split, + min_hessian_to_split, + ) + n_threads = _openmp_effective_n_threads(n_threads) + + if n_bins_non_missing is None: + n_bins_non_missing = n_bins - 1 + + if isinstance(n_bins_non_missing, numbers.Integral): + n_bins_non_missing = np.array( + [n_bins_non_missing] * X_binned.shape[1], dtype=np.uint32 + ) + else: + n_bins_non_missing = np.asarray(n_bins_non_missing, dtype=np.uint32) + + if isinstance(has_missing_values, bool): + has_missing_values = [has_missing_values] * X_binned.shape[1] + has_missing_values = np.asarray(has_missing_values, dtype=np.uint8) + + # `monotonic_cst` validation is done in _validate_monotonic_cst + # at the estimator level and therefore the following should not be + # needed when using the public API. + if monotonic_cst is None: + monotonic_cst = np.full( + shape=X_binned.shape[1], + fill_value=MonotonicConstraint.NO_CST, + dtype=np.int8, + ) + else: + monotonic_cst = np.asarray(monotonic_cst, dtype=np.int8) + self.with_monotonic_cst = np.any(monotonic_cst != MonotonicConstraint.NO_CST) + + if is_categorical is None: + is_categorical = np.zeros(shape=X_binned.shape[1], dtype=np.uint8) + else: + is_categorical = np.asarray(is_categorical, dtype=np.uint8) + + if np.any( + np.logical_and( + is_categorical == 1, monotonic_cst != MonotonicConstraint.NO_CST + ) + ): + raise ValueError("Categorical features cannot have monotonic constraints.") + + hessians_are_constant = hessians.shape[0] == 1 + self.histogram_builder = HistogramBuilder( + X_binned, n_bins, gradients, hessians, hessians_are_constant, n_threads + ) + missing_values_bin_idx = n_bins - 1 + self.splitter = Splitter( + X_binned=X_binned, + n_bins_non_missing=n_bins_non_missing, + missing_values_bin_idx=missing_values_bin_idx, + has_missing_values=has_missing_values, + is_categorical=is_categorical, + monotonic_cst=monotonic_cst, + l2_regularization=l2_regularization, + min_hessian_to_split=min_hessian_to_split, + min_samples_leaf=min_samples_leaf, + min_gain_to_split=min_gain_to_split, + hessians_are_constant=hessians_are_constant, + feature_fraction_per_split=feature_fraction_per_split, + rng=rng, + n_threads=n_threads, + ) + self.X_binned = X_binned + self.max_leaf_nodes = max_leaf_nodes + self.max_depth = max_depth + self.min_samples_leaf = min_samples_leaf + self.min_gain_to_split = min_gain_to_split + self.n_bins_non_missing = n_bins_non_missing + self.missing_values_bin_idx = missing_values_bin_idx + self.has_missing_values = has_missing_values + self.is_categorical = is_categorical + self.monotonic_cst = monotonic_cst + self.interaction_cst = interaction_cst + self.l2_regularization = l2_regularization + self.shrinkage = shrinkage + self.n_features = X_binned.shape[1] + self.n_threads = n_threads + self.splittable_nodes = [] + self.finalized_leaves = [] + self.total_find_split_time = 0.0 # time spent finding the best splits + self.total_compute_hist_time = 0.0 # time spent computing histograms + self.total_apply_split_time = 0.0 # time spent splitting nodes + self.n_categorical_splits = 0 + self._initialize_root() + self.n_nodes = 1 + + def _validate_parameters( + self, + X_binned, + min_gain_to_split, + min_hessian_to_split, + ): + """Validate parameters passed to __init__. + + Also validate parameters passed to splitter. + """ + if X_binned.dtype != np.uint8: + raise NotImplementedError("X_binned must be of type uint8.") + if not X_binned.flags.f_contiguous: + raise ValueError( + "X_binned should be passed as Fortran contiguous " + "array for maximum efficiency." + ) + if min_gain_to_split < 0: + raise ValueError( + "min_gain_to_split={} must be positive.".format(min_gain_to_split) + ) + if min_hessian_to_split < 0: + raise ValueError( + "min_hessian_to_split={} must be positive.".format(min_hessian_to_split) + ) + + def grow(self): + """Grow the tree, from root to leaves.""" + while self.splittable_nodes: + self.split_next() + + self._apply_shrinkage() + + def _apply_shrinkage(self): + """Multiply leaves values by shrinkage parameter. + + This must be done at the very end of the growing process. If this were + done during the growing process e.g. in finalize_leaf(), then a leaf + would be shrunk but its sibling would potentially not be (if it's a + non-leaf), which would lead to a wrong computation of the 'middle' + value needed to enforce the monotonic constraints. + """ + for leaf in self.finalized_leaves: + leaf.value *= self.shrinkage + + def _initialize_root(self): + """Initialize root node and finalize it if needed.""" + tic = time() + if self.interaction_cst is not None: + allowed_features = set().union(*self.interaction_cst) + allowed_features = np.fromiter( + allowed_features, dtype=np.uint32, count=len(allowed_features) + ) + arbitrary_feature = allowed_features[0] + else: + allowed_features = None + arbitrary_feature = 0 + + # TreeNode init needs the total sum of gradients and hessians. Therefore, we + # first compute the histograms and then compute the total grad/hess on an + # arbitrary feature histogram. This way we replace a loop over n_samples by a + # loop over n_bins. + histograms = self.histogram_builder.compute_histograms_brute( + self.splitter.partition, # =self.root.sample_indices + allowed_features, + ) + self.total_compute_hist_time += time() - tic + + tic = time() + n_samples = self.X_binned.shape[0] + depth = 0 + histogram_array = np.asarray(histograms[arbitrary_feature]) + sum_gradients = histogram_array["sum_gradients"].sum() + if self.histogram_builder.hessians_are_constant: + sum_hessians = self.histogram_builder.hessians[0] * n_samples + else: + sum_hessians = histogram_array["sum_hessians"].sum() + self.root = TreeNode( + depth=depth, + sample_indices=self.splitter.partition, + partition_start=0, + partition_stop=n_samples, + sum_gradients=sum_gradients, + sum_hessians=sum_hessians, + value=0, + ) + + if self.root.n_samples < 2 * self.min_samples_leaf: + # Do not even bother computing any splitting statistics. + self._finalize_leaf(self.root) + return + if sum_hessians < self.splitter.min_hessian_to_split: + self._finalize_leaf(self.root) + return + + if self.interaction_cst is not None: + self.root.interaction_cst_indices = range(len(self.interaction_cst)) + self.root.allowed_features = allowed_features + + self.root.histograms = histograms + + self._compute_best_split_and_push(self.root) + self.total_find_split_time += time() - tic + + def _compute_best_split_and_push(self, node): + """Compute the best possible split (SplitInfo) of a given node. + + Also push it in the heap of splittable nodes if gain isn't zero. + The gain of a node is 0 if either all the leaves are pure + (best gain = 0), or if no split would satisfy the constraints, + (min_hessians_to_split, min_gain_to_split, min_samples_leaf) + """ + + node.split_info = self.splitter.find_node_split( + n_samples=node.n_samples, + histograms=node.histograms, + sum_gradients=node.sum_gradients, + sum_hessians=node.sum_hessians, + value=node.value, + lower_bound=node.children_lower_bound, + upper_bound=node.children_upper_bound, + allowed_features=node.allowed_features, + ) + + if node.split_info.gain <= 0: # no valid split + self._finalize_leaf(node) + else: + heappush(self.splittable_nodes, node) + + def split_next(self): + """Split the node with highest potential gain. + + Returns + ------- + left : TreeNode + The resulting left child. + right : TreeNode + The resulting right child. + """ + # Consider the node with the highest loss reduction (a.k.a. gain) + node = heappop(self.splittable_nodes) + + tic = time() + ( + sample_indices_left, + sample_indices_right, + right_child_pos, + ) = self.splitter.split_indices(node.split_info, node.sample_indices) + self.total_apply_split_time += time() - tic + + depth = node.depth + 1 + n_leaf_nodes = len(self.finalized_leaves) + len(self.splittable_nodes) + n_leaf_nodes += 2 + + left_child_node = TreeNode( + depth=depth, + sample_indices=sample_indices_left, + partition_start=node.partition_start, + partition_stop=node.partition_start + right_child_pos, + sum_gradients=node.split_info.sum_gradient_left, + sum_hessians=node.split_info.sum_hessian_left, + value=node.split_info.value_left, + ) + right_child_node = TreeNode( + depth=depth, + sample_indices=sample_indices_right, + partition_start=left_child_node.partition_stop, + partition_stop=node.partition_stop, + sum_gradients=node.split_info.sum_gradient_right, + sum_hessians=node.split_info.sum_hessian_right, + value=node.split_info.value_right, + ) + + node.right_child = right_child_node + node.left_child = left_child_node + + # set interaction constraints (the indices of the constraints sets) + if self.interaction_cst is not None: + # Calculate allowed_features and interaction_cst_indices only once. Child + # nodes inherit them before they get split. + ( + left_child_node.allowed_features, + left_child_node.interaction_cst_indices, + ) = self._compute_interactions(node) + right_child_node.interaction_cst_indices = ( + left_child_node.interaction_cst_indices + ) + right_child_node.allowed_features = left_child_node.allowed_features + + if not self.has_missing_values[node.split_info.feature_idx]: + # If no missing values are encountered at fit time, then samples + # with missing values during predict() will go to whichever child + # has the most samples. + node.split_info.missing_go_to_left = ( + left_child_node.n_samples > right_child_node.n_samples + ) + + self.n_nodes += 2 + self.n_categorical_splits += node.split_info.is_categorical + + if self.max_leaf_nodes is not None and n_leaf_nodes == self.max_leaf_nodes: + self._finalize_leaf(left_child_node) + self._finalize_leaf(right_child_node) + self._finalize_splittable_nodes() + return left_child_node, right_child_node + + if self.max_depth is not None and depth == self.max_depth: + self._finalize_leaf(left_child_node) + self._finalize_leaf(right_child_node) + return left_child_node, right_child_node + + if left_child_node.n_samples < self.min_samples_leaf * 2: + self._finalize_leaf(left_child_node) + if right_child_node.n_samples < self.min_samples_leaf * 2: + self._finalize_leaf(right_child_node) + + if self.with_monotonic_cst: + # Set value bounds for respecting monotonic constraints + # See test_nodes_values() for details + if ( + self.monotonic_cst[node.split_info.feature_idx] + == MonotonicConstraint.NO_CST + ): + lower_left = lower_right = node.children_lower_bound + upper_left = upper_right = node.children_upper_bound + else: + mid = (left_child_node.value + right_child_node.value) / 2 + if ( + self.monotonic_cst[node.split_info.feature_idx] + == MonotonicConstraint.POS + ): + lower_left, upper_left = node.children_lower_bound, mid + lower_right, upper_right = mid, node.children_upper_bound + else: # NEG + lower_left, upper_left = mid, node.children_upper_bound + lower_right, upper_right = node.children_lower_bound, mid + left_child_node.set_children_bounds(lower_left, upper_left) + right_child_node.set_children_bounds(lower_right, upper_right) + + # Compute histograms of children, and compute their best possible split + # (if needed) + should_split_left = not left_child_node.is_leaf + should_split_right = not right_child_node.is_leaf + if should_split_left or should_split_right: + # We will compute the histograms of both nodes even if one of them + # is a leaf, since computing the second histogram is very cheap + # (using histogram subtraction). + n_samples_left = left_child_node.sample_indices.shape[0] + n_samples_right = right_child_node.sample_indices.shape[0] + if n_samples_left < n_samples_right: + smallest_child = left_child_node + largest_child = right_child_node + else: + smallest_child = right_child_node + largest_child = left_child_node + + # We use the brute O(n_samples) method on the child that has the + # smallest number of samples, and the subtraction trick O(n_bins) + # on the other one. + # Note that both left and right child have the same allowed_features. + tic = time() + smallest_child.histograms = self.histogram_builder.compute_histograms_brute( + smallest_child.sample_indices, smallest_child.allowed_features + ) + largest_child.histograms = ( + self.histogram_builder.compute_histograms_subtraction( + node.histograms, + smallest_child.histograms, + smallest_child.allowed_features, + ) + ) + # node.histograms is reused in largest_child.histograms. To break cyclic + # memory references and help garbage collection, we set it to None. + node.histograms = None + self.total_compute_hist_time += time() - tic + + tic = time() + if should_split_left: + self._compute_best_split_and_push(left_child_node) + if should_split_right: + self._compute_best_split_and_push(right_child_node) + self.total_find_split_time += time() - tic + + # Release memory used by histograms as they are no longer needed + # for leaf nodes since they won't be split. + for child in (left_child_node, right_child_node): + if child.is_leaf: + del child.histograms + + # Release memory used by histograms as they are no longer needed for + # internal nodes once children histograms have been computed. + del node.histograms + + return left_child_node, right_child_node + + def _compute_interactions(self, node): + r"""Compute features allowed by interactions to be inherited by child nodes. + + Example: Assume constraints [{0, 1}, {1, 2}]. + 1 <- Both constraint groups could be applied from now on + / \ + 1 2 <- Left split still fulfills both constraint groups. + / \ / \ Right split at feature 2 has only group {1, 2} from now on. + + LightGBM uses the same logic for overlapping groups. See + https://github.com/microsoft/LightGBM/issues/4481 for details. + + Parameters: + ---------- + node : TreeNode + A node that might have children. Based on its feature_idx, the interaction + constraints for possible child nodes are computed. + + Returns + ------- + allowed_features : ndarray, dtype=uint32 + Indices of features allowed to split for children. + interaction_cst_indices : list of ints + Indices of the interaction sets that have to be applied on splits of + child nodes. The fewer sets the stronger the constraint as fewer sets + contain fewer features. + """ + # Note: + # - Case of no interactions is already captured before function call. + # - This is for nodes that are already split and have a + # node.split_info.feature_idx. + allowed_features = set() + interaction_cst_indices = [] + for i in node.interaction_cst_indices: + if node.split_info.feature_idx in self.interaction_cst[i]: + interaction_cst_indices.append(i) + allowed_features.update(self.interaction_cst[i]) + return ( + np.fromiter(allowed_features, dtype=np.uint32, count=len(allowed_features)), + interaction_cst_indices, + ) + + def _finalize_leaf(self, node): + """Make node a leaf of the tree being grown.""" + + node.is_leaf = True + self.finalized_leaves.append(node) + + def _finalize_splittable_nodes(self): + """Transform all splittable nodes into leaves. + + Used when some constraint is met e.g. maximum number of leaves or + maximum depth.""" + while len(self.splittable_nodes) > 0: + node = self.splittable_nodes.pop() + self._finalize_leaf(node) + + def make_predictor(self, binning_thresholds): + """Make a TreePredictor object out of the current tree. + + Parameters + ---------- + binning_thresholds : array-like of floats + Corresponds to the bin_thresholds_ attribute of the BinMapper. + For each feature, this stores: + + - the bin frontiers for continuous features + - the unique raw category values for categorical features + + Returns + ------- + A TreePredictor object. + """ + predictor_nodes = np.zeros(self.n_nodes, dtype=PREDICTOR_RECORD_DTYPE) + binned_left_cat_bitsets = np.zeros( + (self.n_categorical_splits, 8), dtype=X_BITSET_INNER_DTYPE + ) + raw_left_cat_bitsets = np.zeros( + (self.n_categorical_splits, 8), dtype=X_BITSET_INNER_DTYPE + ) + _fill_predictor_arrays( + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + self.root, + binning_thresholds, + self.n_bins_non_missing, + ) + return TreePredictor( + predictor_nodes, binned_left_cat_bitsets, raw_left_cat_bitsets + ) + + +def _fill_predictor_arrays( + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + grower_node, + binning_thresholds, + n_bins_non_missing, + next_free_node_idx=0, + next_free_bitset_idx=0, +): + """Helper used in make_predictor to set the TreePredictor fields.""" + node = predictor_nodes[next_free_node_idx] + node["count"] = grower_node.n_samples + node["depth"] = grower_node.depth + if grower_node.split_info is not None: + node["gain"] = grower_node.split_info.gain + else: + node["gain"] = -1 + + node["value"] = grower_node.value + + if grower_node.is_leaf: + # Leaf node + node["is_leaf"] = True + return next_free_node_idx + 1, next_free_bitset_idx + + split_info = grower_node.split_info + feature_idx, bin_idx = split_info.feature_idx, split_info.bin_idx + node["feature_idx"] = feature_idx + node["bin_threshold"] = bin_idx + node["missing_go_to_left"] = split_info.missing_go_to_left + node["is_categorical"] = split_info.is_categorical + + if split_info.bin_idx == n_bins_non_missing[feature_idx] - 1: + # Split is on the last non-missing bin: it's a "split on nans". + # All nans go to the right, the rest go to the left. + # Note: for categorical splits, bin_idx is 0 and we rely on the bitset + node["num_threshold"] = np.inf + elif split_info.is_categorical: + categories = binning_thresholds[feature_idx] + node["bitset_idx"] = next_free_bitset_idx + binned_left_cat_bitsets[next_free_bitset_idx] = split_info.left_cat_bitset + set_raw_bitset_from_binned_bitset( + raw_left_cat_bitsets[next_free_bitset_idx], + split_info.left_cat_bitset, + categories, + ) + next_free_bitset_idx += 1 + else: + node["num_threshold"] = binning_thresholds[feature_idx][bin_idx] + + next_free_node_idx += 1 + + node["left"] = next_free_node_idx + next_free_node_idx, next_free_bitset_idx = _fill_predictor_arrays( + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + grower_node.left_child, + binning_thresholds=binning_thresholds, + n_bins_non_missing=n_bins_non_missing, + next_free_node_idx=next_free_node_idx, + next_free_bitset_idx=next_free_bitset_idx, + ) + + node["right"] = next_free_node_idx + return _fill_predictor_arrays( + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + grower_node.right_child, + binning_thresholds=binning_thresholds, + n_bins_non_missing=n_bins_non_missing, + next_free_node_idx=next_free_node_idx, + next_free_bitset_idx=next_free_bitset_idx, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..246c6682b7a8e1a1e574d4455a043e62aa5ccee9 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..92377885f06fe39b62168aa57daa06a3cad27027 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.pyd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0d49cbb3591cf4f0abf3d7bfc0d8ab3cdd206af49eafbd56c501a59f45fd080 +size 128512 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.pyx new file mode 100644 index 0000000000000000000000000000000000000000..e05fe3d8c84d3849648d11b420fc72b508a4aa69 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.pyx @@ -0,0 +1,520 @@ +"""This module contains routines for building histograms.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +cimport cython +from cython.parallel import prange +from libc.string cimport memset + +import numpy as np + +from sklearn.ensemble._hist_gradient_boosting.common import HISTOGRAM_DTYPE +from sklearn.ensemble._hist_gradient_boosting.common cimport hist_struct +from sklearn.ensemble._hist_gradient_boosting.common cimport X_BINNED_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport G_H_DTYPE_C +from sklearn.utils._typedefs cimport uint8_t + + +# Notes: +# - IN views are read-only, OUT views are write-only +# - In a lot of functions here, we pass feature_idx and the whole 2d +# histograms arrays instead of just histograms[feature_idx]. This is because +# Cython generated C code will have strange Python interactions (likely +# related to the GIL release and the custom histogram dtype) when using 1d +# histogram arrays that come from 2d arrays. +# - The for loops are un-wrapped, for example: +# +# for i in range(n): +# array[i] = i +# +# will become +# +# for i in range(n // 4): +# array[i] = i +# array[i + 1] = i + 1 +# array[i + 2] = i + 2 +# array[i + 3] = i + 3 +# +# This is to hint gcc that it can auto-vectorize these 4 operations and +# perform them all at once. + + +@cython.final +cdef class HistogramBuilder: + """A Histogram builder... used to build histograms. + + A histogram is an array with n_bins entries of type HISTOGRAM_DTYPE. Each + feature has its own histogram. A histogram contains the sum of gradients + and hessians of all the samples belonging to each bin. + + There are different ways to build a histogram: + - by subtraction: hist(child) = hist(parent) - hist(sibling) + - from scratch. In this case we have routines that update the hessians + or not (not useful when hessians are constant for some losses e.g. + least squares). Also, there's a special case for the root which + contains all the samples, leading to some possible optimizations. + Overall all the implementations look the same, and are optimized for + cache hit. + + Parameters + ---------- + X_binned : ndarray of int, shape (n_samples, n_features) + The binned input samples. Must be Fortran-aligned. + n_bins : int + The total number of bins, including the bin for missing values. Used + to define the shape of the histograms. + gradients : ndarray, shape (n_samples,) + The gradients of each training sample. Those are the gradients of the + loss w.r.t the predictions, evaluated at iteration i - 1. + hessians : ndarray, shape (n_samples,) + The hessians of each training sample. Those are the hessians of the + loss w.r.t the predictions, evaluated at iteration i - 1. + hessians_are_constant : bool + Whether hessians are constant. + """ + cdef public: + const X_BINNED_DTYPE_C [::1, :] X_binned + unsigned int n_features + unsigned int n_bins + G_H_DTYPE_C [::1] gradients + G_H_DTYPE_C [::1] hessians + G_H_DTYPE_C [::1] ordered_gradients + G_H_DTYPE_C [::1] ordered_hessians + uint8_t hessians_are_constant + int n_threads + + def __init__(self, const X_BINNED_DTYPE_C [::1, :] X_binned, + unsigned int n_bins, G_H_DTYPE_C [::1] gradients, + G_H_DTYPE_C [::1] hessians, + uint8_t hessians_are_constant, + int n_threads): + + self.X_binned = X_binned + self.n_features = X_binned.shape[1] + # Note: all histograms will have bins, but some of the + # bins may be unused if a feature has a small number of unique values. + self.n_bins = n_bins + self.gradients = gradients + self.hessians = hessians + # for root node, gradients and hessians are already ordered + self.ordered_gradients = gradients.copy() + self.ordered_hessians = hessians.copy() + self.hessians_are_constant = hessians_are_constant + self.n_threads = n_threads + + def compute_histograms_brute( + HistogramBuilder self, + const unsigned int [::1] sample_indices, # IN + const unsigned int [:] allowed_features=None, # IN + ): + """Compute the histograms of the node by scanning through all the data. + + For a given feature, the complexity is O(n_samples) + + Parameters + ---------- + sample_indices : array of int, shape (n_samples_at_node,) + The indices of the samples at the node to split. + + allowed_features : None or ndarray, dtype=np.uint32 + Indices of the features that are allowed by interaction constraints to be + split. + + Returns + ------- + histograms : ndarray of HISTOGRAM_DTYPE, shape (n_features, n_bins) + The computed histograms of the current node. + """ + cdef: + int n_samples + int feature_idx + int f_idx + int i + # need local views to avoid python interactions + uint8_t hessians_are_constant = self.hessians_are_constant + int n_allowed_features = self.n_features + G_H_DTYPE_C [::1] ordered_gradients = self.ordered_gradients + G_H_DTYPE_C [::1] gradients = self.gradients + G_H_DTYPE_C [::1] ordered_hessians = self.ordered_hessians + G_H_DTYPE_C [::1] hessians = self.hessians + # Histograms will be initialized to zero later within a prange + hist_struct [:, ::1] histograms = np.empty( + shape=(self.n_features, self.n_bins), + dtype=HISTOGRAM_DTYPE + ) + bint has_interaction_cst = allowed_features is not None + int n_threads = self.n_threads + + if has_interaction_cst: + n_allowed_features = allowed_features.shape[0] + + with nogil: + n_samples = sample_indices.shape[0] + + # Populate ordered_gradients and ordered_hessians. (Already done + # for root) Ordering the gradients and hessians helps to improve + # cache hit. + if sample_indices.shape[0] != gradients.shape[0]: + if hessians_are_constant: + for i in prange(n_samples, schedule='static', + num_threads=n_threads): + ordered_gradients[i] = gradients[sample_indices[i]] + else: + for i in prange(n_samples, schedule='static', + num_threads=n_threads): + ordered_gradients[i] = gradients[sample_indices[i]] + ordered_hessians[i] = hessians[sample_indices[i]] + + # Compute histogram of each feature + for f_idx in prange( + n_allowed_features, schedule='static', num_threads=n_threads + ): + if has_interaction_cst: + feature_idx = allowed_features[f_idx] + else: + feature_idx = f_idx + + self._compute_histogram_brute_single_feature( + feature_idx, sample_indices, histograms + ) + + return histograms + + cdef void _compute_histogram_brute_single_feature( + HistogramBuilder self, + const int feature_idx, + const unsigned int [::1] sample_indices, # IN + hist_struct [:, ::1] histograms) noexcept nogil: # OUT + """Compute the histogram for a given feature.""" + + cdef: + unsigned int n_samples = sample_indices.shape[0] + const X_BINNED_DTYPE_C [::1] X_binned = \ + self.X_binned[:, feature_idx] + unsigned int root_node = X_binned.shape[0] == n_samples + G_H_DTYPE_C [::1] ordered_gradients = \ + self.ordered_gradients[:n_samples] + G_H_DTYPE_C [::1] ordered_hessians = \ + self.ordered_hessians[:n_samples] + uint8_t hessians_are_constant = \ + self.hessians_are_constant + + # Set histograms to zero. + memset(&histograms[feature_idx, 0], 0, self.n_bins * sizeof(hist_struct)) + + if root_node: + if hessians_are_constant: + _build_histogram_root_no_hessian(feature_idx, X_binned, + ordered_gradients, + histograms) + else: + _build_histogram_root(feature_idx, X_binned, + ordered_gradients, ordered_hessians, + histograms) + else: + if hessians_are_constant: + _build_histogram_no_hessian(feature_idx, + sample_indices, X_binned, + ordered_gradients, histograms) + else: + _build_histogram(feature_idx, sample_indices, + X_binned, ordered_gradients, + ordered_hessians, histograms) + + def compute_histograms_subtraction( + HistogramBuilder self, + hist_struct [:, ::1] parent_histograms, # IN and OUT + hist_struct [:, ::1] sibling_histograms, # IN + const unsigned int [:] allowed_features=None, # IN + ): + """Compute the histograms of the node using the subtraction trick. + + hist(parent) = hist(left_child) + hist(right_child) + + For a given feature, the complexity is O(n_bins). This is much more + efficient than compute_histograms_brute, but it's only possible for one + of the siblings. + + Parameters + ---------- + parent_histograms : ndarray of HISTOGRAM_DTYPE, \ + shape (n_features, n_bins) + The histograms of the parent. + sibling_histograms : ndarray of HISTOGRAM_DTYPE, \ + shape (n_features, n_bins) + The histograms of the sibling. + allowed_features : None or ndarray, dtype=np.uint32 + Indices of the features that are allowed by interaction constraints to be + split. + + Returns + ------- + histograms : ndarray of HISTOGRAM_DTYPE, shape(n_features, n_bins) + The computed histograms of the current node. + We repurpose parent_histograms for this and don't need to allocate new + memory. + """ + + cdef: + int feature_idx + int f_idx + int n_allowed_features = self.n_features + bint has_interaction_cst = allowed_features is not None + int n_threads = self.n_threads + + if has_interaction_cst: + n_allowed_features = allowed_features.shape[0] + + # Compute histogram of each feature + for f_idx in prange(n_allowed_features, schedule='static', nogil=True, + num_threads=n_threads): + if has_interaction_cst: + feature_idx = allowed_features[f_idx] + else: + feature_idx = f_idx + + _subtract_histograms( + feature_idx, + self.n_bins, + parent_histograms, + sibling_histograms, + ) + return parent_histograms + + +cpdef void _build_histogram_naive( + const int feature_idx, + unsigned int [:] sample_indices, # IN + X_BINNED_DTYPE_C [:] binned_feature, # IN + G_H_DTYPE_C [:] ordered_gradients, # IN + G_H_DTYPE_C [:] ordered_hessians, # IN + hist_struct [:, :] out) noexcept nogil: # OUT + """Build histogram in a naive way, without optimizing for cache hit. + + Used in tests to compare with the optimized version.""" + cdef: + unsigned int i + unsigned int n_samples = sample_indices.shape[0] + unsigned int sample_idx + unsigned int bin_idx + + for i in range(n_samples): + sample_idx = sample_indices[i] + bin_idx = binned_feature[sample_idx] + out[feature_idx, bin_idx].sum_gradients += ordered_gradients[i] + out[feature_idx, bin_idx].sum_hessians += ordered_hessians[i] + out[feature_idx, bin_idx].count += 1 + + +cpdef void _subtract_histograms( + const int feature_idx, + unsigned int n_bins, + hist_struct [:, ::1] hist_a, # IN and OUT + hist_struct [:, ::1] hist_b, # IN +) noexcept nogil: # OUT + """compute hist_a = hist_a - hist_b""" + # Note that subtraction of large sums of floating point numbers, as we have here, + # can exhibit catastrophic cancallation. This is in particular true for gradients + # as they can be positive and negative, while hessians are non-negative. + # Remember that gradients and hessians are originally computed in + # G_H_DTYPE_C = float32 precision. Therefore, if sum_gradients and sum_hessians are + # float64, we don't loose precision. But if we also used float32 for summation, we + # would need to take care of floating point errors. + # + # Note that we could protect for negative hessians by setting: + # sum_hessians = max(0, sum_hessians) + # But as we use float64 for summing float32, that's veeeery unlikely. + cdef: + unsigned int i = 0 + for i in range(n_bins): + hist_a[feature_idx, i].sum_gradients -= hist_b[feature_idx, i].sum_gradients + hist_a[feature_idx, i].sum_hessians -= hist_b[feature_idx, i].sum_hessians + hist_a[feature_idx, i].count -= hist_b[feature_idx, i].count + + +cpdef void _build_histogram( + const int feature_idx, + const unsigned int [::1] sample_indices, # IN + const X_BINNED_DTYPE_C [::1] binned_feature, # IN + const G_H_DTYPE_C [::1] ordered_gradients, # IN + const G_H_DTYPE_C [::1] ordered_hessians, # IN + hist_struct [:, ::1] out) noexcept nogil: # OUT + """Return histogram for a given feature.""" + cdef: + unsigned int i = 0 + unsigned int n_node_samples = sample_indices.shape[0] + unsigned int unrolled_upper = (n_node_samples // 4) * 4 + + unsigned int bin_0 + unsigned int bin_1 + unsigned int bin_2 + unsigned int bin_3 + unsigned int bin_idx + + for i in range(0, unrolled_upper, 4): + bin_0 = binned_feature[sample_indices[i]] + bin_1 = binned_feature[sample_indices[i + 1]] + bin_2 = binned_feature[sample_indices[i + 2]] + bin_3 = binned_feature[sample_indices[i + 3]] + + out[feature_idx, bin_0].sum_gradients += ordered_gradients[i] + out[feature_idx, bin_1].sum_gradients += ordered_gradients[i + 1] + out[feature_idx, bin_2].sum_gradients += ordered_gradients[i + 2] + out[feature_idx, bin_3].sum_gradients += ordered_gradients[i + 3] + + out[feature_idx, bin_0].sum_hessians += ordered_hessians[i] + out[feature_idx, bin_1].sum_hessians += ordered_hessians[i + 1] + out[feature_idx, bin_2].sum_hessians += ordered_hessians[i + 2] + out[feature_idx, bin_3].sum_hessians += ordered_hessians[i + 3] + + out[feature_idx, bin_0].count += 1 + out[feature_idx, bin_1].count += 1 + out[feature_idx, bin_2].count += 1 + out[feature_idx, bin_3].count += 1 + + for i in range(unrolled_upper, n_node_samples): + bin_idx = binned_feature[sample_indices[i]] + out[feature_idx, bin_idx].sum_gradients += ordered_gradients[i] + out[feature_idx, bin_idx].sum_hessians += ordered_hessians[i] + out[feature_idx, bin_idx].count += 1 + + +cpdef void _build_histogram_no_hessian( + const int feature_idx, + const unsigned int [::1] sample_indices, # IN + const X_BINNED_DTYPE_C [::1] binned_feature, # IN + const G_H_DTYPE_C [::1] ordered_gradients, # IN + hist_struct [:, ::1] out) noexcept nogil: # OUT + """Return histogram for a given feature, not updating hessians. + + Used when the hessians of the loss are constant (typically LS loss). + """ + + cdef: + unsigned int i = 0 + unsigned int n_node_samples = sample_indices.shape[0] + unsigned int unrolled_upper = (n_node_samples // 4) * 4 + + unsigned int bin_0 + unsigned int bin_1 + unsigned int bin_2 + unsigned int bin_3 + unsigned int bin_idx + + for i in range(0, unrolled_upper, 4): + bin_0 = binned_feature[sample_indices[i]] + bin_1 = binned_feature[sample_indices[i + 1]] + bin_2 = binned_feature[sample_indices[i + 2]] + bin_3 = binned_feature[sample_indices[i + 3]] + + out[feature_idx, bin_0].sum_gradients += ordered_gradients[i] + out[feature_idx, bin_1].sum_gradients += ordered_gradients[i + 1] + out[feature_idx, bin_2].sum_gradients += ordered_gradients[i + 2] + out[feature_idx, bin_3].sum_gradients += ordered_gradients[i + 3] + + out[feature_idx, bin_0].count += 1 + out[feature_idx, bin_1].count += 1 + out[feature_idx, bin_2].count += 1 + out[feature_idx, bin_3].count += 1 + + for i in range(unrolled_upper, n_node_samples): + bin_idx = binned_feature[sample_indices[i]] + out[feature_idx, bin_idx].sum_gradients += ordered_gradients[i] + out[feature_idx, bin_idx].count += 1 + + +cpdef void _build_histogram_root( + const int feature_idx, + const X_BINNED_DTYPE_C [::1] binned_feature, # IN + const G_H_DTYPE_C [::1] all_gradients, # IN + const G_H_DTYPE_C [::1] all_hessians, # IN + hist_struct [:, ::1] out) noexcept nogil: # OUT + """Compute histogram of the root node. + + Unlike other nodes, the root node has to find the split among *all* the + samples from the training set. binned_feature and all_gradients / + all_hessians already have a consistent ordering. + """ + + cdef: + unsigned int i = 0 + unsigned int n_samples = binned_feature.shape[0] + unsigned int unrolled_upper = (n_samples // 4) * 4 + + unsigned int bin_0 + unsigned int bin_1 + unsigned int bin_2 + unsigned int bin_3 + unsigned int bin_idx + + for i in range(0, unrolled_upper, 4): + + bin_0 = binned_feature[i] + bin_1 = binned_feature[i + 1] + bin_2 = binned_feature[i + 2] + bin_3 = binned_feature[i + 3] + + out[feature_idx, bin_0].sum_gradients += all_gradients[i] + out[feature_idx, bin_1].sum_gradients += all_gradients[i + 1] + out[feature_idx, bin_2].sum_gradients += all_gradients[i + 2] + out[feature_idx, bin_3].sum_gradients += all_gradients[i + 3] + + out[feature_idx, bin_0].sum_hessians += all_hessians[i] + out[feature_idx, bin_1].sum_hessians += all_hessians[i + 1] + out[feature_idx, bin_2].sum_hessians += all_hessians[i + 2] + out[feature_idx, bin_3].sum_hessians += all_hessians[i + 3] + + out[feature_idx, bin_0].count += 1 + out[feature_idx, bin_1].count += 1 + out[feature_idx, bin_2].count += 1 + out[feature_idx, bin_3].count += 1 + + for i in range(unrolled_upper, n_samples): + bin_idx = binned_feature[i] + out[feature_idx, bin_idx].sum_gradients += all_gradients[i] + out[feature_idx, bin_idx].sum_hessians += all_hessians[i] + out[feature_idx, bin_idx].count += 1 + + +cpdef void _build_histogram_root_no_hessian( + const int feature_idx, + const X_BINNED_DTYPE_C [::1] binned_feature, # IN + const G_H_DTYPE_C [::1] all_gradients, # IN + hist_struct [:, ::1] out) noexcept nogil: # OUT + """Compute histogram of the root node, not updating hessians. + + Used when the hessians of the loss are constant (typically LS loss). + """ + + cdef: + unsigned int i = 0 + unsigned int n_samples = binned_feature.shape[0] + unsigned int unrolled_upper = (n_samples // 4) * 4 + + unsigned int bin_0 + unsigned int bin_1 + unsigned int bin_2 + unsigned int bin_3 + unsigned int bin_idx + + for i in range(0, unrolled_upper, 4): + bin_0 = binned_feature[i] + bin_1 = binned_feature[i + 1] + bin_2 = binned_feature[i + 2] + bin_3 = binned_feature[i + 3] + + out[feature_idx, bin_0].sum_gradients += all_gradients[i] + out[feature_idx, bin_1].sum_gradients += all_gradients[i + 1] + out[feature_idx, bin_2].sum_gradients += all_gradients[i + 2] + out[feature_idx, bin_3].sum_gradients += all_gradients[i + 3] + + out[feature_idx, bin_0].count += 1 + out[feature_idx, bin_1].count += 1 + out[feature_idx, bin_2].count += 1 + out[feature_idx, bin_3].count += 1 + + for i in range(unrolled_upper, n_samples): + bin_idx = binned_feature[i] + out[feature_idx, bin_idx].sum_gradients += all_gradients[i] + out[feature_idx, bin_idx].count += 1 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/meson.build b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/meson.build new file mode 100644 index 0000000000000000000000000000000000000000..95fbda36774738bd635c2d1ff710e48f0df73688 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/meson.build @@ -0,0 +1,20 @@ +hist_gradient_boosting_extension_metadata = { + '_gradient_boosting': {'sources': [cython_gen.process('_gradient_boosting.pyx')], + 'dependencies': [openmp_dep]}, + 'histogram': {'sources': [cython_gen.process('histogram.pyx')], 'dependencies': [openmp_dep]}, + 'splitting': {'sources': [cython_gen.process('splitting.pyx')], 'dependencies': [openmp_dep]}, + '_binning': {'sources': [cython_gen.process('_binning.pyx')], 'dependencies': [openmp_dep]}, + '_predictor': {'sources': [cython_gen.process('_predictor.pyx')], 'dependencies': [openmp_dep]}, + '_bitset': {'sources': [cython_gen.process('_bitset.pyx')]}, + 'common': {'sources': [cython_gen.process('common.pyx')]}, +} + +foreach ext_name, ext_dict : hist_gradient_boosting_extension_metadata + py.extension_module( + ext_name, + ext_dict.get('sources'), + dependencies: ext_dict.get('dependencies', []), + subdir: 'sklearn/ensemble/_hist_gradient_boosting', + install: true + ) +endforeach diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/predictor.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..3f73122b329bd643be6be20e8b78954202a7e7b9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/predictor.py @@ -0,0 +1,149 @@ +""" +This module contains the TreePredictor class which is used for prediction. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np + +from sklearn.ensemble._hist_gradient_boosting._predictor import ( + _compute_partial_dependence, + _predict_from_binned_data, + _predict_from_raw_data, +) +from sklearn.ensemble._hist_gradient_boosting.common import ( + PREDICTOR_RECORD_DTYPE, + Y_DTYPE, +) + + +class TreePredictor: + """Tree class used for predictions. + + Parameters + ---------- + nodes : ndarray of PREDICTOR_RECORD_DTYPE + The nodes of the tree. + binned_left_cat_bitsets : ndarray of shape (n_categorical_splits, 8), dtype=uint32 + Array of bitsets for binned categories used in predict_binned when a + split is categorical. + raw_left_cat_bitsets : ndarray of shape (n_categorical_splits, 8), dtype=uint32 + Array of bitsets for raw categories used in predict when a split is + categorical. + """ + + def __init__(self, nodes, binned_left_cat_bitsets, raw_left_cat_bitsets): + self.nodes = nodes + self.binned_left_cat_bitsets = binned_left_cat_bitsets + self.raw_left_cat_bitsets = raw_left_cat_bitsets + + def get_n_leaf_nodes(self): + """Return number of leaves.""" + return int(self.nodes["is_leaf"].sum()) + + def get_max_depth(self): + """Return maximum depth among all leaves.""" + return int(self.nodes["depth"].max()) + + def predict(self, X, known_cat_bitsets, f_idx_map, n_threads): + """Predict raw values for non-binned data. + + Parameters + ---------- + X : ndarray, shape (n_samples, n_features) + The input samples. + + known_cat_bitsets : ndarray of shape (n_categorical_features, 8) + Array of bitsets of known categories, for each categorical feature. + + f_idx_map : ndarray of shape (n_features,) + Map from original feature index to the corresponding index in the + known_cat_bitsets array. + + n_threads : int + Number of OpenMP threads to use. + + Returns + ------- + y : ndarray, shape (n_samples,) + The raw predicted values. + """ + out = np.empty(X.shape[0], dtype=Y_DTYPE) + + _predict_from_raw_data( + self.nodes, + X, + self.raw_left_cat_bitsets, + known_cat_bitsets, + f_idx_map, + n_threads, + out, + ) + return out + + def predict_binned(self, X, missing_values_bin_idx, n_threads): + """Predict raw values for binned data. + + Parameters + ---------- + X : ndarray, shape (n_samples, n_features) + The input samples. + missing_values_bin_idx : uint8 + Index of the bin that is used for missing values. This is the + index of the last bin and is always equal to max_bins (as passed + to the GBDT classes), or equivalently to n_bins - 1. + n_threads : int + Number of OpenMP threads to use. + + Returns + ------- + y : ndarray, shape (n_samples,) + The raw predicted values. + """ + out = np.empty(X.shape[0], dtype=Y_DTYPE) + _predict_from_binned_data( + self.nodes, + X, + self.binned_left_cat_bitsets, + missing_values_bin_idx, + n_threads, + out, + ) + return out + + def compute_partial_dependence(self, grid, target_features, out): + """Fast partial dependence computation. + + Parameters + ---------- + grid : ndarray, shape (n_samples, n_target_features) + The grid points on which the partial dependence should be + evaluated. + target_features : ndarray, shape (n_target_features) + The set of target features for which the partial dependence + should be evaluated. + out : ndarray, shape (n_samples) + The value of the partial dependence function on each grid + point. + """ + _compute_partial_dependence(self.nodes, grid, target_features, out) + + def __setstate__(self, state): + try: + super().__setstate__(state) + except AttributeError: + self.__dict__.update(state) + + # The dtype of feature_idx is np.intp which is platform dependent. Here, we + # make sure that saving and loading on different bitness systems works without + # errors. For instance, on a 64 bit Python runtime, np.intp = np.int64, + # while on 32 bit np.intp = np.int32. + # + # TODO: consider always using platform agnostic dtypes for fitted + # estimator attributes. For this particular estimator, this would + # mean replacing the intp field of PREDICTOR_RECORD_DTYPE by an int32 + # field. Ideally this should be done consistently throughout + # scikit-learn along with a common test. + if self.nodes.dtype != PREDICTOR_RECORD_DTYPE: + self.nodes = self.nodes.astype(PREDICTOR_RECORD_DTYPE, casting="same_kind") diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.lib b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.lib new file mode 100644 index 0000000000000000000000000000000000000000..4d9c7ff1dee8a92309d8ad16db173ea09c5ab62c Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.lib differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..c1667311b0352e1782818d0e615cf98ce1389e3b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.pyd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff8b750829b51d9279b8b414c01bc0cde543bcd76558c16340ea43af5a41e882 +size 137728 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx new file mode 100644 index 0000000000000000000000000000000000000000..c8f257ee8d5adab7530dca3a81c5a075363b00f1 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx @@ -0,0 +1,1201 @@ +"""This module contains routines and data structures to: + +- Find the best possible split of a node. For a given node, a split is + characterized by a feature and a bin. +- Apply a split to a node, i.e. split the indices of the samples at the node + into the newly created left and right children. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +cimport cython +from cython.parallel import prange +import numpy as np +from libc.math cimport INFINITY, ceil +from libc.stdlib cimport malloc, free, qsort +from libc.string cimport memcpy + +from sklearn.utils._typedefs cimport uint8_t +from sklearn.ensemble._hist_gradient_boosting.common cimport X_BINNED_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport Y_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport hist_struct +from sklearn.ensemble._hist_gradient_boosting.common cimport BITSET_INNER_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport BITSET_DTYPE_C +from sklearn.ensemble._hist_gradient_boosting.common cimport MonotonicConstraint +from sklearn.ensemble._hist_gradient_boosting._bitset cimport init_bitset +from sklearn.ensemble._hist_gradient_boosting._bitset cimport set_bitset +from sklearn.ensemble._hist_gradient_boosting._bitset cimport in_bitset + + +cdef struct split_info_struct: + # Same as the SplitInfo class, but we need a C struct to use it in the + # nogil sections and to use in arrays. + Y_DTYPE_C gain + int feature_idx + unsigned int bin_idx + uint8_t missing_go_to_left + Y_DTYPE_C sum_gradient_left + Y_DTYPE_C sum_gradient_right + Y_DTYPE_C sum_hessian_left + Y_DTYPE_C sum_hessian_right + unsigned int n_samples_left + unsigned int n_samples_right + Y_DTYPE_C value_left + Y_DTYPE_C value_right + uint8_t is_categorical + BITSET_DTYPE_C left_cat_bitset + + +# used in categorical splits for sorting categories by increasing values of +# sum_gradients / sum_hessians +cdef struct categorical_info: + X_BINNED_DTYPE_C bin_idx + Y_DTYPE_C value + + +class SplitInfo: + """Pure data class to store information about a potential split. + + Parameters + ---------- + gain : float + The gain of the split. + feature_idx : int + The index of the feature to be split. + bin_idx : int + The index of the bin on which the split is made. Should be ignored if + `is_categorical` is True: `left_cat_bitset` will be used to determine + the split. + missing_go_to_left : bool + Whether missing values should go to the left child. This is used + whether the split is categorical or not. + sum_gradient_left : float + The sum of the gradients of all the samples in the left child. + sum_hessian_left : float + The sum of the hessians of all the samples in the left child. + sum_gradient_right : float + The sum of the gradients of all the samples in the right child. + sum_hessian_right : float + The sum of the hessians of all the samples in the right child. + n_samples_left : int, default=0 + The number of samples in the left child. + n_samples_right : int + The number of samples in the right child. + is_categorical : bool + Whether the split is done on a categorical feature. + left_cat_bitset : ndarray of shape=(8,), dtype=uint32 or None + Bitset representing the categories that go to the left. This is used + only when `is_categorical` is True. + Note that missing values are part of that bitset if there are missing + values in the training data. For missing values, we rely on that + bitset for splitting, but at prediction time, we rely on + missing_go_to_left. + """ + def __init__(self, gain, feature_idx, bin_idx, + missing_go_to_left, sum_gradient_left, sum_hessian_left, + sum_gradient_right, sum_hessian_right, n_samples_left, + n_samples_right, value_left, value_right, + is_categorical, left_cat_bitset): + self.gain = gain + self.feature_idx = feature_idx + self.bin_idx = bin_idx + self.missing_go_to_left = missing_go_to_left + self.sum_gradient_left = sum_gradient_left + self.sum_hessian_left = sum_hessian_left + self.sum_gradient_right = sum_gradient_right + self.sum_hessian_right = sum_hessian_right + self.n_samples_left = n_samples_left + self.n_samples_right = n_samples_right + self.value_left = value_left + self.value_right = value_right + self.is_categorical = is_categorical + self.left_cat_bitset = left_cat_bitset + + +@cython.final +cdef class Splitter: + """Splitter used to find the best possible split at each node. + + A split (see SplitInfo) is characterized by a feature and a bin. + + The Splitter is also responsible for partitioning the samples among the + leaves of the tree (see split_indices() and the partition attribute). + + Parameters + ---------- + X_binned : ndarray of int, shape (n_samples, n_features) + The binned input samples. Must be Fortran-aligned. + n_bins_non_missing : ndarray, shape (n_features,) + For each feature, gives the number of bins actually used for + non-missing values. + missing_values_bin_idx : uint8 + Index of the bin that is used for missing values. This is the index of + the last bin and is always equal to max_bins (as passed to the GBDT + classes), or equivalently to n_bins - 1. + has_missing_values : ndarray, shape (n_features,) + Whether missing values were observed in the training data, for each + feature. + is_categorical : ndarray of bool of shape (n_features,) + Indicates categorical features. + monotonic_cst : ndarray of int of shape (n_features,), dtype=int + Indicates the monotonic constraint to enforce on each feature. + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + Read more in the :ref:`User Guide `. + l2_regularization : float + The L2 regularization parameter. + min_hessian_to_split : float, default=1e-3 + The minimum sum of hessians needed in each node. Splits that result in + at least one child having a sum of hessians less than + min_hessian_to_split are discarded. + min_samples_leaf : int, default=20 + The minimum number of samples per leaf. + min_gain_to_split : float, default=0.0 + The minimum gain needed to split a node. Splits with lower gain will + be ignored. + hessians_are_constant: bool, default is False + Whether hessians are constant. + feature_fraction_per_split : float, default=1 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + rng : Generator + n_threads : int, default=1 + Number of OpenMP threads to use. + """ + cdef public: + const X_BINNED_DTYPE_C [::1, :] X_binned + unsigned int n_features + const unsigned int [::1] n_bins_non_missing + uint8_t missing_values_bin_idx + const uint8_t [::1] has_missing_values + const uint8_t [::1] is_categorical + const signed char [::1] monotonic_cst + uint8_t hessians_are_constant + Y_DTYPE_C l2_regularization + Y_DTYPE_C min_hessian_to_split + unsigned int min_samples_leaf + Y_DTYPE_C min_gain_to_split + Y_DTYPE_C feature_fraction_per_split + rng + + unsigned int [::1] partition + unsigned int [::1] left_indices_buffer + unsigned int [::1] right_indices_buffer + int n_threads + + def __init__(self, + const X_BINNED_DTYPE_C [::1, :] X_binned, + const unsigned int [::1] n_bins_non_missing, + const uint8_t missing_values_bin_idx, + const uint8_t [::1] has_missing_values, + const uint8_t [::1] is_categorical, + const signed char [::1] monotonic_cst, + Y_DTYPE_C l2_regularization, + Y_DTYPE_C min_hessian_to_split=1e-3, + unsigned int min_samples_leaf=20, + Y_DTYPE_C min_gain_to_split=0., + uint8_t hessians_are_constant=False, + Y_DTYPE_C feature_fraction_per_split=1.0, + rng=np.random.RandomState(), + unsigned int n_threads=1): + + self.X_binned = X_binned + self.n_features = X_binned.shape[1] + self.n_bins_non_missing = n_bins_non_missing + self.missing_values_bin_idx = missing_values_bin_idx + self.has_missing_values = has_missing_values + self.is_categorical = is_categorical + self.monotonic_cst = monotonic_cst + self.l2_regularization = l2_regularization + self.min_hessian_to_split = min_hessian_to_split + self.min_samples_leaf = min_samples_leaf + self.min_gain_to_split = min_gain_to_split + self.hessians_are_constant = hessians_are_constant + self.feature_fraction_per_split = feature_fraction_per_split + self.rng = rng + self.n_threads = n_threads + + # The partition array maps each sample index into the leaves of the + # tree (a leaf in this context is a node that isn't split yet, not + # necessarily a 'finalized' leaf). Initially, the root contains all + # the indices, e.g.: + # partition = [abcdefghijkl] + # After a call to split_indices, it may look e.g. like this: + # partition = [cef|abdghijkl] + # we have 2 leaves, the left one is at position 0 and the second one at + # position 3. The order of the samples is irrelevant. + self.partition = np.arange(X_binned.shape[0], dtype=np.uint32) + # buffers used in split_indices to support parallel splitting. + self.left_indices_buffer = np.empty_like(self.partition) + self.right_indices_buffer = np.empty_like(self.partition) + + def split_indices(Splitter self, split_info, unsigned int [::1] + sample_indices): + """Split samples into left and right arrays. + + The split is performed according to the best possible split + (split_info). + + Ultimately, this is nothing but a partition of the sample_indices + array with a given pivot, exactly like a quicksort subroutine. + + Parameters + ---------- + split_info : SplitInfo + The SplitInfo of the node to split. + sample_indices : ndarray of unsigned int, shape (n_samples_at_node,) + The indices of the samples at the node to split. This is a view + on self.partition, and it is modified inplace by placing the + indices of the left child at the beginning, and the indices of + the right child at the end. + + Returns + ------- + left_indices : ndarray of int, shape (n_left_samples,) + The indices of the samples in the left child. This is a view on + self.partition. + right_indices : ndarray of int, shape (n_right_samples,) + The indices of the samples in the right child. This is a view on + self.partition. + right_child_position : int + The position of the right child in ``sample_indices``. + """ + # This is a multi-threaded implementation inspired by lightgbm. Here + # is a quick break down. Let's suppose we want to split a node with 24 + # samples named from a to x. self.partition looks like this (the * are + # indices in other leaves that we don't care about): + # partition = [*************abcdefghijklmnopqrstuvwx****************] + # ^ ^ + # node_position node_position + node.n_samples + + # Ultimately, we want to reorder the samples inside the boundaries of + # the leaf (which becomes a node) to now represent the samples in its + # left and right child. For example: + # partition = [*************abefilmnopqrtuxcdghjksvw*****************] + # ^ ^ + # left_child_pos right_child_pos + # Note that left_child_pos always takes the value of node_position, + # and right_child_pos = left_child_pos + left_child.n_samples. The + # order of the samples inside a leaf is irrelevant. + + # 1. sample_indices is a view on this region a..x. We conceptually + # divide it into n_threads regions. Each thread will be responsible + # for its own region. Here is an example with 4 threads: + # sample_indices = [abcdef|ghijkl|mnopqr|stuvwx] + # 2. Each thread processes 6 = 24 // 4 entries and maps them into + # left_indices_buffer or right_indices_buffer. For example, we could + # have the following mapping ('.' denotes an undefined entry): + # - left_indices_buffer = [abef..|il....|mnopqr|tux...] + # - right_indices_buffer = [cd....|ghjk..|......|svw...] + # 3. We keep track of the start positions of the regions (the '|') in + # ``offset_in_buffers`` as well as the size of each region. We also + # keep track of the number of samples put into the left/right child + # by each thread. Concretely: + # - left_counts = [4, 2, 6, 3] + # - right_counts = [2, 4, 0, 3] + # 4. Finally, we put left/right_indices_buffer back into the + # sample_indices, without any undefined entries and the partition + # looks as expected + # partition = [*************abefilmnopqrtuxcdghjksvw***************] + + # Note: We here show left/right_indices_buffer as being the same size + # as sample_indices for simplicity, but in reality they are of the + # same size as partition. + + cdef: + int n_samples = sample_indices.shape[0] + X_BINNED_DTYPE_C bin_idx = split_info.bin_idx + uint8_t missing_go_to_left = split_info.missing_go_to_left + uint8_t missing_values_bin_idx = self.missing_values_bin_idx + int feature_idx = split_info.feature_idx + const X_BINNED_DTYPE_C [::1] X_binned = \ + self.X_binned[:, feature_idx] + unsigned int [::1] left_indices_buffer = self.left_indices_buffer + unsigned int [::1] right_indices_buffer = self.right_indices_buffer + uint8_t is_categorical = split_info.is_categorical + # Cython is unhappy if we set left_cat_bitset to + # split_info.left_cat_bitset directly, so we need a tmp var + BITSET_INNER_DTYPE_C [:] cat_bitset_tmp = split_info.left_cat_bitset + BITSET_DTYPE_C left_cat_bitset + int n_threads = self.n_threads + + int [:] sizes = np.full(n_threads, n_samples // n_threads, + dtype=np.int32) + int [:] offset_in_buffers = np.zeros(n_threads, dtype=np.int32) + int [:] left_counts = np.empty(n_threads, dtype=np.int32) + int [:] right_counts = np.empty(n_threads, dtype=np.int32) + int left_count + int right_count + int start + int stop + int i + int thread_idx + int sample_idx + int right_child_position + uint8_t turn_left + int [:] left_offset = np.zeros(n_threads, dtype=np.int32) + int [:] right_offset = np.zeros(n_threads, dtype=np.int32) + + # only set left_cat_bitset when is_categorical is True + if is_categorical: + left_cat_bitset = &cat_bitset_tmp[0] + + with nogil: + for thread_idx in range(n_samples % n_threads): + sizes[thread_idx] += 1 + + for thread_idx in range(1, n_threads): + offset_in_buffers[thread_idx] = \ + offset_in_buffers[thread_idx - 1] + sizes[thread_idx - 1] + + # map indices from sample_indices to left/right_indices_buffer + for thread_idx in prange(n_threads, schedule='static', + chunksize=1, num_threads=n_threads): + left_count = 0 + right_count = 0 + + start = offset_in_buffers[thread_idx] + stop = start + sizes[thread_idx] + for i in range(start, stop): + sample_idx = sample_indices[i] + turn_left = sample_goes_left( + missing_go_to_left, + missing_values_bin_idx, bin_idx, + X_binned[sample_idx], is_categorical, + left_cat_bitset) + + if turn_left: + left_indices_buffer[start + left_count] = sample_idx + left_count = left_count + 1 + else: + right_indices_buffer[start + right_count] = sample_idx + right_count = right_count + 1 + + left_counts[thread_idx] = left_count + right_counts[thread_idx] = right_count + + # position of right child = just after the left child + right_child_position = 0 + for thread_idx in range(n_threads): + right_child_position += left_counts[thread_idx] + + # offset of each thread in sample_indices for left and right + # child, i.e. where each thread will start to write. + right_offset[0] = right_child_position + for thread_idx in range(1, n_threads): + left_offset[thread_idx] = \ + left_offset[thread_idx - 1] + left_counts[thread_idx - 1] + right_offset[thread_idx] = \ + right_offset[thread_idx - 1] + right_counts[thread_idx - 1] + + # map indices in left/right_indices_buffer back into + # sample_indices. This also updates self.partition since + # sample_indices is a view. + for thread_idx in prange(n_threads, schedule='static', + chunksize=1, num_threads=n_threads): + memcpy( + &sample_indices[left_offset[thread_idx]], + &left_indices_buffer[offset_in_buffers[thread_idx]], + sizeof(unsigned int) * left_counts[thread_idx] + ) + if right_counts[thread_idx] > 0: + # If we're splitting the rightmost node of the tree, i.e. the + # rightmost node in the partition array, and if n_threads >= 2, one + # might have right_counts[-1] = 0 and right_offset[-1] = len(sample_indices) + # leading to evaluating + # + # &sample_indices[right_offset[-1]] = &samples_indices[n_samples_at_node] + # = &partition[n_samples_in_tree] + # + # which is an out-of-bounds read access that can cause a segmentation fault. + # When boundscheck=True, removing this check produces this exception: + # + # IndexError: Out of bounds on buffer access + # + memcpy( + &sample_indices[right_offset[thread_idx]], + &right_indices_buffer[offset_in_buffers[thread_idx]], + sizeof(unsigned int) * right_counts[thread_idx] + ) + + return (sample_indices[:right_child_position], + sample_indices[right_child_position:], + right_child_position) + + def find_node_split( + Splitter self, + unsigned int n_samples, + hist_struct [:, ::1] histograms, # IN + const Y_DTYPE_C sum_gradients, + const Y_DTYPE_C sum_hessians, + const Y_DTYPE_C value, + const Y_DTYPE_C lower_bound=-INFINITY, + const Y_DTYPE_C upper_bound=INFINITY, + const unsigned int [:] allowed_features=None, + ): + """For each feature, find the best bin to split on at a given node. + + Return the best split info among all features. + + Parameters + ---------- + n_samples : int + The number of samples at the node. + histograms : ndarray of HISTOGRAM_DTYPE of \ + shape (n_features, max_bins) + The histograms of the current node. + sum_gradients : float + The sum of the gradients for each sample at the node. + sum_hessians : float + The sum of the hessians for each sample at the node. + value : float + The bounded value of the current node. We directly pass the value + instead of re-computing it from sum_gradients and sum_hessians, + because we need to compute the loss and the gain based on the + *bounded* value: computing the value from + sum_gradients / sum_hessians would give the unbounded value, and + the interaction with min_gain_to_split would not be correct + anymore. Side note: we can't use the lower_bound / upper_bound + parameters either because these refer to the bounds of the + children, not the bounds of the current node. + lower_bound : float + Lower bound for the children values for respecting the monotonic + constraints. + upper_bound : float + Upper bound for the children values for respecting the monotonic + constraints. + allowed_features : None or ndarray, dtype=np.uint32 + Indices of the features that are allowed by interaction constraints to be + split. + + Returns + ------- + best_split_info : SplitInfo + The info about the best possible split among all features. + """ + cdef: + int feature_idx + int split_info_idx + int best_split_info_idx + int n_allowed_features + split_info_struct split_info + split_info_struct * split_infos + const uint8_t [::1] has_missing_values = self.has_missing_values + const uint8_t [::1] is_categorical = self.is_categorical + const signed char [::1] monotonic_cst = self.monotonic_cst + int n_threads = self.n_threads + bint has_interaction_cst = False + Y_DTYPE_C feature_fraction_per_split = self.feature_fraction_per_split + uint8_t [:] subsample_mask # same as npy_bool + int n_subsampled_features + + has_interaction_cst = allowed_features is not None + if has_interaction_cst: + n_allowed_features = allowed_features.shape[0] + else: + n_allowed_features = self.n_features + + if feature_fraction_per_split < 1.0: + # We do all random sampling before the nogil and make sure that we sample + # exactly n_subsampled_features >= 1 features. + n_subsampled_features = max( + 1, + int(ceil(feature_fraction_per_split * n_allowed_features)), + ) + subsample_mask_arr = np.full(n_allowed_features, False) + subsample_mask_arr[:n_subsampled_features] = True + self.rng.shuffle(subsample_mask_arr) + # https://github.com/numpy/numpy/issues/18273 + subsample_mask = subsample_mask_arr + + with nogil: + + split_infos = malloc( + n_allowed_features * sizeof(split_info_struct)) + + # split_info_idx is index of split_infos of size n_allowed_features. + # features_idx is the index of the feature column in X. + for split_info_idx in prange(n_allowed_features, schedule='static', + num_threads=n_threads): + if has_interaction_cst: + feature_idx = allowed_features[split_info_idx] + else: + feature_idx = split_info_idx + + split_infos[split_info_idx].feature_idx = feature_idx + + # For each feature, find best bin to split on + # Start with a gain of -1 if no better split is found, that + # means one of the constraints isn't respected + # (min_samples_leaf, etc.) and the grower will later turn the + # node into a leaf. + split_infos[split_info_idx].gain = -1 + split_infos[split_info_idx].is_categorical = is_categorical[feature_idx] + + # Note that subsample_mask is indexed by split_info_idx and not by + # feature_idx because we only need to exclude the same features again + # and again. We do NOT need to access the features directly by using + # allowed_features. + if feature_fraction_per_split < 1.0 and not subsample_mask[split_info_idx]: + continue + + if is_categorical[feature_idx]: + self._find_best_bin_to_split_category( + feature_idx, has_missing_values[feature_idx], + histograms, n_samples, sum_gradients, sum_hessians, + value, monotonic_cst[feature_idx], lower_bound, + upper_bound, &split_infos[split_info_idx]) + else: + # We will scan bins from left to right (in all cases), and + # if there are any missing values, we will also scan bins + # from right to left. This way, we can consider whichever + # case yields the best gain: either missing values go to + # the right (left to right scan) or to the left (right to + # left case). See algo 3 from the XGBoost paper + # https://arxiv.org/abs/1603.02754 + # Note: for the categorical features above, this isn't + # needed since missing values are considered a native + # category. + self._find_best_bin_to_split_left_to_right( + feature_idx, has_missing_values[feature_idx], + histograms, n_samples, sum_gradients, sum_hessians, + value, monotonic_cst[feature_idx], + lower_bound, upper_bound, &split_infos[split_info_idx]) + + if has_missing_values[feature_idx]: + # We need to explore both directions to check whether + # sending the nans to the left child would lead to a higher + # gain + self._find_best_bin_to_split_right_to_left( + feature_idx, histograms, n_samples, + sum_gradients, sum_hessians, + value, monotonic_cst[feature_idx], + lower_bound, upper_bound, &split_infos[split_info_idx]) + + # then compute best possible split among all features + # split_info is set to the best of split_infos + best_split_info_idx = self._find_best_feature_to_split_helper( + split_infos, n_allowed_features + ) + split_info = split_infos[best_split_info_idx] + + out = SplitInfo( + split_info.gain, + split_info.feature_idx, + split_info.bin_idx, + split_info.missing_go_to_left, + split_info.sum_gradient_left, + split_info.sum_hessian_left, + split_info.sum_gradient_right, + split_info.sum_hessian_right, + split_info.n_samples_left, + split_info.n_samples_right, + split_info.value_left, + split_info.value_right, + split_info.is_categorical, + None, # left_cat_bitset will only be set if the split is categorical + ) + # Only set bitset if the split is categorical + if split_info.is_categorical: + out.left_cat_bitset = np.asarray(split_info.left_cat_bitset, dtype=np.uint32) + + free(split_infos) + return out + + cdef int _find_best_feature_to_split_helper( + self, + split_info_struct * split_infos, # IN + int n_allowed_features, + ) noexcept nogil: + """Return the index of split_infos with the best feature split.""" + cdef: + int split_info_idx + int best_split_info_idx = 0 + + for split_info_idx in range(1, n_allowed_features): + if (split_infos[split_info_idx].gain > split_infos[best_split_info_idx].gain): + best_split_info_idx = split_info_idx + return best_split_info_idx + + cdef void _find_best_bin_to_split_left_to_right( + Splitter self, + unsigned int feature_idx, + uint8_t has_missing_values, + const hist_struct [:, ::1] histograms, # IN + unsigned int n_samples, + Y_DTYPE_C sum_gradients, + Y_DTYPE_C sum_hessians, + Y_DTYPE_C value, + signed char monotonic_cst, + Y_DTYPE_C lower_bound, + Y_DTYPE_C upper_bound, + split_info_struct * split_info) noexcept nogil: # OUT + """Find best bin to split on for a given feature. + + Splits that do not satisfy the splitting constraints + (min_gain_to_split, etc.) are discarded here. + + We scan node from left to right. This version is called whether there + are missing values or not. If any, missing values are assigned to the + right node. + """ + cdef: + unsigned int bin_idx + unsigned int n_samples_left + unsigned int n_samples_right + unsigned int n_samples_ = n_samples + # We set the 'end' variable such that the last non-missing-values + # bin never goes to the left child (which would result in and + # empty right child), unless there are missing values, since these + # would go to the right child. + unsigned int end = \ + self.n_bins_non_missing[feature_idx] - 1 + has_missing_values + Y_DTYPE_C sum_hessian_left + Y_DTYPE_C sum_hessian_right + Y_DTYPE_C sum_gradient_left + Y_DTYPE_C sum_gradient_right + Y_DTYPE_C loss_current_node + Y_DTYPE_C gain + uint8_t found_better_split = False + + Y_DTYPE_C best_sum_hessian_left + Y_DTYPE_C best_sum_gradient_left + unsigned int best_bin_idx + unsigned int best_n_samples_left + Y_DTYPE_C best_gain = -1 + hist_struct hist + + sum_gradient_left, sum_hessian_left = 0., 0. + n_samples_left = 0 + + loss_current_node = _loss_from_value(value, sum_gradients) + + for bin_idx in range(end): + hist = histograms[feature_idx, bin_idx] + n_samples_left += hist.count + n_samples_right = n_samples_ - n_samples_left + + if self.hessians_are_constant: + sum_hessian_left += hist.count + else: + sum_hessian_left += \ + hist.sum_hessians + sum_hessian_right = sum_hessians - sum_hessian_left + + sum_gradient_left += hist.sum_gradients + sum_gradient_right = sum_gradients - sum_gradient_left + + if n_samples_left < self.min_samples_leaf: + continue + if n_samples_right < self.min_samples_leaf: + # won't get any better + break + + if sum_hessian_left < self.min_hessian_to_split: + continue + if sum_hessian_right < self.min_hessian_to_split: + # won't get any better (hessians are > 0 since loss is convex) + break + + gain = _split_gain(sum_gradient_left, sum_hessian_left, + sum_gradient_right, sum_hessian_right, + loss_current_node, + monotonic_cst, + lower_bound, + upper_bound, + self.l2_regularization) + + if gain > best_gain and gain > self.min_gain_to_split: + found_better_split = True + best_gain = gain + best_bin_idx = bin_idx + best_sum_gradient_left = sum_gradient_left + best_sum_hessian_left = sum_hessian_left + best_n_samples_left = n_samples_left + + if found_better_split: + split_info.gain = best_gain + split_info.bin_idx = best_bin_idx + # we scan from left to right so missing values go to the right + split_info.missing_go_to_left = False + split_info.sum_gradient_left = best_sum_gradient_left + split_info.sum_gradient_right = sum_gradients - best_sum_gradient_left + split_info.sum_hessian_left = best_sum_hessian_left + split_info.sum_hessian_right = sum_hessians - best_sum_hessian_left + split_info.n_samples_left = best_n_samples_left + split_info.n_samples_right = n_samples - best_n_samples_left + + # We recompute best values here but it's cheap + split_info.value_left = compute_node_value( + split_info.sum_gradient_left, split_info.sum_hessian_left, + lower_bound, upper_bound, self.l2_regularization) + + split_info.value_right = compute_node_value( + split_info.sum_gradient_right, split_info.sum_hessian_right, + lower_bound, upper_bound, self.l2_regularization) + + cdef void _find_best_bin_to_split_right_to_left( + self, + unsigned int feature_idx, + const hist_struct [:, ::1] histograms, # IN + unsigned int n_samples, + Y_DTYPE_C sum_gradients, + Y_DTYPE_C sum_hessians, + Y_DTYPE_C value, + signed char monotonic_cst, + Y_DTYPE_C lower_bound, + Y_DTYPE_C upper_bound, + split_info_struct * split_info) noexcept nogil: # OUT + """Find best bin to split on for a given feature. + + Splits that do not satisfy the splitting constraints + (min_gain_to_split, etc.) are discarded here. + + We scan node from right to left. This version is only called when + there are missing values. Missing values are assigned to the left + child. + + If no missing value are present in the data this method isn't called + since only calling _find_best_bin_to_split_left_to_right is enough. + """ + + cdef: + unsigned int bin_idx + unsigned int n_samples_left + unsigned int n_samples_right + unsigned int n_samples_ = n_samples + Y_DTYPE_C sum_hessian_left + Y_DTYPE_C sum_hessian_right + Y_DTYPE_C sum_gradient_left + Y_DTYPE_C sum_gradient_right + Y_DTYPE_C loss_current_node + Y_DTYPE_C gain + unsigned int start = self.n_bins_non_missing[feature_idx] - 2 + uint8_t found_better_split = False + + Y_DTYPE_C best_sum_hessian_left + Y_DTYPE_C best_sum_gradient_left + unsigned int best_bin_idx + unsigned int best_n_samples_left + Y_DTYPE_C best_gain = split_info.gain # computed during previous scan + hist_struct hist + + sum_gradient_right, sum_hessian_right = 0., 0. + n_samples_right = 0 + + loss_current_node = _loss_from_value(value, sum_gradients) + + for bin_idx in range(start, -1, -1): + hist = histograms[feature_idx, bin_idx + 1] + n_samples_right += hist.count + n_samples_left = n_samples_ - n_samples_right + + if self.hessians_are_constant: + sum_hessian_right += hist.count + else: + sum_hessian_right += \ + hist.sum_hessians + sum_hessian_left = sum_hessians - sum_hessian_right + + sum_gradient_right += \ + hist.sum_gradients + sum_gradient_left = sum_gradients - sum_gradient_right + + if n_samples_right < self.min_samples_leaf: + continue + if n_samples_left < self.min_samples_leaf: + # won't get any better + break + + if sum_hessian_right < self.min_hessian_to_split: + continue + if sum_hessian_left < self.min_hessian_to_split: + # won't get any better (hessians are > 0 since loss is convex) + break + + gain = _split_gain(sum_gradient_left, sum_hessian_left, + sum_gradient_right, sum_hessian_right, + loss_current_node, + monotonic_cst, + lower_bound, + upper_bound, + self.l2_regularization) + + if gain > best_gain and gain > self.min_gain_to_split: + found_better_split = True + best_gain = gain + best_bin_idx = bin_idx + best_sum_gradient_left = sum_gradient_left + best_sum_hessian_left = sum_hessian_left + best_n_samples_left = n_samples_left + + if found_better_split: + split_info.gain = best_gain + split_info.bin_idx = best_bin_idx + # we scan from right to left so missing values go to the left + split_info.missing_go_to_left = True + split_info.sum_gradient_left = best_sum_gradient_left + split_info.sum_gradient_right = sum_gradients - best_sum_gradient_left + split_info.sum_hessian_left = best_sum_hessian_left + split_info.sum_hessian_right = sum_hessians - best_sum_hessian_left + split_info.n_samples_left = best_n_samples_left + split_info.n_samples_right = n_samples - best_n_samples_left + + # We recompute best values here but it's cheap + split_info.value_left = compute_node_value( + split_info.sum_gradient_left, split_info.sum_hessian_left, + lower_bound, upper_bound, self.l2_regularization) + + split_info.value_right = compute_node_value( + split_info.sum_gradient_right, split_info.sum_hessian_right, + lower_bound, upper_bound, self.l2_regularization) + + cdef void _find_best_bin_to_split_category( + self, + unsigned int feature_idx, + uint8_t has_missing_values, + const hist_struct [:, ::1] histograms, # IN + unsigned int n_samples, + Y_DTYPE_C sum_gradients, + Y_DTYPE_C sum_hessians, + Y_DTYPE_C value, + char monotonic_cst, + Y_DTYPE_C lower_bound, + Y_DTYPE_C upper_bound, + split_info_struct * split_info) noexcept nogil: # OUT + """Find best split for categorical features. + + Categories are first sorted according to their variance, and then + a scan is performed as if categories were ordered quantities. + + Ref: "On Grouping for Maximum Homogeneity", Walter D. Fisher + """ + + cdef: + unsigned int bin_idx + unsigned int n_bins_non_missing = self.n_bins_non_missing[feature_idx] + unsigned int missing_values_bin_idx = self.missing_values_bin_idx + categorical_info * cat_infos + unsigned int sorted_cat_idx + unsigned int n_used_bins = 0 + int [2] scan_direction + int direction = 0 + int best_direction = 0 + unsigned int middle + unsigned int i + const hist_struct[::1] feature_hist = histograms[feature_idx, :] + hist_struct hist + Y_DTYPE_C sum_gradients_bin + Y_DTYPE_C sum_hessians_bin + Y_DTYPE_C loss_current_node + Y_DTYPE_C sum_gradient_left, sum_hessian_left + Y_DTYPE_C sum_gradient_right, sum_hessian_right + unsigned int n_samples_left, n_samples_right + Y_DTYPE_C gain + Y_DTYPE_C best_gain = -1.0 + uint8_t found_better_split = False + Y_DTYPE_C best_sum_hessian_left + Y_DTYPE_C best_sum_gradient_left + unsigned int best_n_samples_left + unsigned int best_cat_infos_thresh + # Reduces the effect of noises in categorical features, + # especially for categories with few data. Called cat_smooth in + # LightGBM. TODO: Make this user adjustable? + Y_DTYPE_C MIN_CAT_SUPPORT = 10. + # this is equal to 1 for losses where hessians are constant + Y_DTYPE_C support_factor = n_samples / sum_hessians + + # Details on the split finding: + # We first order categories by their sum_gradients / sum_hessians + # values, and we exclude categories that don't respect MIN_CAT_SUPPORT + # from this sorted array. Missing values are treated just like any + # other category. The low-support categories will always be mapped to + # the right child. We scan the sorted categories array from left to + # right and from right to left, and we stop at the middle. + + # Considering ordered categories A B C D, with E being a low-support + # category: A B C D + # ^ + # midpoint + # The scans will consider the following split-points: + # * left to right: + # A - B C D E + # A B - C D E + # * right to left: + # D - A B C E + # C D - A B E + + # Note that since we stop at the middle and since low-support + # categories (E) are always mapped to the right, the following splits + # aren't considered: + # A E - B C D + # D E - A B C + # Basically, we're forcing E to always be mapped to the child that has + # *at least half of the categories* (and this child is always the right + # child, by convention). + + # Also note that if we scanned in only one direction (e.g. left to + # right), we would only consider the following splits: + # A - B C D E + # A B - C D E + # A B C - D E + # and thus we would be missing on D - A B C E and on C D - A B E + + cat_infos = malloc( + (n_bins_non_missing + has_missing_values) * sizeof(categorical_info)) + + # fill cat_infos while filtering out categories based on MIN_CAT_SUPPORT + for bin_idx in range(n_bins_non_missing): + hist = feature_hist[bin_idx] + if self.hessians_are_constant: + sum_hessians_bin = hist.count + else: + sum_hessians_bin = hist.sum_hessians + if sum_hessians_bin * support_factor >= MIN_CAT_SUPPORT: + cat_infos[n_used_bins].bin_idx = bin_idx + sum_gradients_bin = hist.sum_gradients + + cat_infos[n_used_bins].value = ( + sum_gradients_bin / (sum_hessians_bin + MIN_CAT_SUPPORT) + ) + n_used_bins += 1 + + # Also add missing values bin so that nans are considered as a category + if has_missing_values: + hist = feature_hist[missing_values_bin_idx] + if self.hessians_are_constant: + sum_hessians_bin = hist.count + else: + sum_hessians_bin = hist.sum_hessians + if sum_hessians_bin * support_factor >= MIN_CAT_SUPPORT: + cat_infos[n_used_bins].bin_idx = missing_values_bin_idx + sum_gradients_bin = ( + hist.sum_gradients + ) + + cat_infos[n_used_bins].value = ( + sum_gradients_bin / (sum_hessians_bin + MIN_CAT_SUPPORT) + ) + n_used_bins += 1 + + # not enough categories to form a split + if n_used_bins <= 1: + free(cat_infos) + return + + qsort(cat_infos, n_used_bins, sizeof(categorical_info), + compare_cat_infos) + + loss_current_node = _loss_from_value(value, sum_gradients) + + scan_direction[0], scan_direction[1] = 1, -1 + for direction in scan_direction: + if direction == 1: + middle = (n_used_bins + 1) // 2 + else: + middle = (n_used_bins + 1) // 2 - 1 + + # The categories we'll consider will go to the left child + sum_gradient_left, sum_hessian_left = 0., 0. + n_samples_left = 0 + + for i in range(middle): + sorted_cat_idx = i if direction == 1 else n_used_bins - 1 - i + bin_idx = cat_infos[sorted_cat_idx].bin_idx + hist = feature_hist[bin_idx] + + n_samples_left += hist.count + n_samples_right = n_samples - n_samples_left + + if self.hessians_are_constant: + sum_hessian_left += hist.count + else: + sum_hessian_left += hist.sum_hessians + sum_hessian_right = sum_hessians - sum_hessian_left + + sum_gradient_left += hist.sum_gradients + sum_gradient_right = sum_gradients - sum_gradient_left + + if ( + n_samples_left < self.min_samples_leaf or + sum_hessian_left < self.min_hessian_to_split + ): + continue + if ( + n_samples_right < self.min_samples_leaf or + sum_hessian_right < self.min_hessian_to_split + ): + break + + gain = _split_gain(sum_gradient_left, sum_hessian_left, + sum_gradient_right, sum_hessian_right, + loss_current_node, monotonic_cst, + lower_bound, upper_bound, + self.l2_regularization) + if gain > best_gain and gain > self.min_gain_to_split: + found_better_split = True + best_gain = gain + best_cat_infos_thresh = sorted_cat_idx + best_sum_gradient_left = sum_gradient_left + best_sum_hessian_left = sum_hessian_left + best_n_samples_left = n_samples_left + best_direction = direction + + if found_better_split: + split_info.gain = best_gain + + # split_info.bin_idx is unused for categorical splits: left_cat_bitset + # is used instead and set below + split_info.bin_idx = 0 + + split_info.sum_gradient_left = best_sum_gradient_left + split_info.sum_gradient_right = sum_gradients - best_sum_gradient_left + split_info.sum_hessian_left = best_sum_hessian_left + split_info.sum_hessian_right = sum_hessians - best_sum_hessian_left + split_info.n_samples_left = best_n_samples_left + split_info.n_samples_right = n_samples - best_n_samples_left + + # We recompute best values here but it's cheap + split_info.value_left = compute_node_value( + split_info.sum_gradient_left, split_info.sum_hessian_left, + lower_bound, upper_bound, self.l2_regularization) + + split_info.value_right = compute_node_value( + split_info.sum_gradient_right, split_info.sum_hessian_right, + lower_bound, upper_bound, self.l2_regularization) + + # create bitset with values from best_cat_infos_thresh + init_bitset(split_info.left_cat_bitset) + if best_direction == 1: + for sorted_cat_idx in range(best_cat_infos_thresh + 1): + bin_idx = cat_infos[sorted_cat_idx].bin_idx + set_bitset(split_info.left_cat_bitset, bin_idx) + else: + for sorted_cat_idx in range(n_used_bins - 1, best_cat_infos_thresh - 1, -1): + bin_idx = cat_infos[sorted_cat_idx].bin_idx + set_bitset(split_info.left_cat_bitset, bin_idx) + + if has_missing_values: + split_info.missing_go_to_left = in_bitset( + split_info.left_cat_bitset, missing_values_bin_idx) + + free(cat_infos) + + +cdef int compare_cat_infos(const void * a, const void * b) noexcept nogil: + return -1 if (a).value < (b).value else 1 + +cdef inline Y_DTYPE_C _split_gain( + Y_DTYPE_C sum_gradient_left, + Y_DTYPE_C sum_hessian_left, + Y_DTYPE_C sum_gradient_right, + Y_DTYPE_C sum_hessian_right, + Y_DTYPE_C loss_current_node, + signed char monotonic_cst, + Y_DTYPE_C lower_bound, + Y_DTYPE_C upper_bound, + Y_DTYPE_C l2_regularization) noexcept nogil: + """Loss reduction + + Compute the reduction in loss after taking a split, compared to keeping + the node a leaf of the tree. + + See Equation 7 of: + :arxiv:`T. Chen, C. Guestrin, (2016) XGBoost: A Scalable Tree Boosting System, + <1603.02754>.` + """ + cdef: + Y_DTYPE_C gain + Y_DTYPE_C value_left + Y_DTYPE_C value_right + + # Compute values of potential left and right children + value_left = compute_node_value(sum_gradient_left, sum_hessian_left, + lower_bound, upper_bound, + l2_regularization) + value_right = compute_node_value(sum_gradient_right, sum_hessian_right, + lower_bound, upper_bound, + l2_regularization) + + if ((monotonic_cst == MonotonicConstraint.POS and value_left > value_right) or + (monotonic_cst == MonotonicConstraint.NEG and value_left < value_right)): + # don't consider this split since it does not respect the monotonic + # constraints. Note that these comparisons need to be done on values + # that have already been clipped to take the monotonic constraints into + # account (if any). + return -1 + + gain = loss_current_node + gain -= _loss_from_value(value_left, sum_gradient_left) + gain -= _loss_from_value(value_right, sum_gradient_right) + # Note that for the gain to be correct (and for min_gain_to_split to work + # as expected), we need all values to be bounded (current node, left child + # and right child). + + return gain + +cdef inline Y_DTYPE_C _loss_from_value( + Y_DTYPE_C value, + Y_DTYPE_C sum_gradient) noexcept nogil: + """Return loss of a node from its (bounded) value + + See Equation 6 of: + :arxiv:`T. Chen, C. Guestrin, (2016) XGBoost: A Scalable Tree Boosting System, + <1603.02754>.` + """ + return sum_gradient * value + +cdef inline uint8_t sample_goes_left( + uint8_t missing_go_to_left, + uint8_t missing_values_bin_idx, + X_BINNED_DTYPE_C split_bin_idx, + X_BINNED_DTYPE_C bin_value, + uint8_t is_categorical, + BITSET_DTYPE_C left_cat_bitset) noexcept nogil: + """Helper to decide whether sample should go to left or right child.""" + + if is_categorical: + # note: if any, missing values are encoded in left_cat_bitset + return in_bitset(left_cat_bitset, bin_value) + else: + return ( + ( + missing_go_to_left and + bin_value == missing_values_bin_idx + ) + or ( + bin_value <= split_bin_idx + )) + + +cpdef inline Y_DTYPE_C compute_node_value( + Y_DTYPE_C sum_gradient, + Y_DTYPE_C sum_hessian, + Y_DTYPE_C lower_bound, + Y_DTYPE_C upper_bound, + Y_DTYPE_C l2_regularization) noexcept nogil: + """Compute a node's value. + + The value is capped in the [lower_bound, upper_bound] interval to respect + monotonic constraints. Shrinkage is ignored. + + See Equation 5 of: + :arxiv:`T. Chen, C. Guestrin, (2016) XGBoost: A Scalable Tree Boosting System, + <1603.02754>.` + """ + + cdef: + Y_DTYPE_C value + + value = -sum_gradient / (sum_hessian + l2_regularization + 1e-15) + + if value < lower_bound: + value = lower_bound + elif value > upper_bound: + value = upper_bound + + return value diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py new file mode 100644 index 0000000000000000000000000000000000000000..1e71771903fdd89063b0bdad3853a7c5ffa85e38 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py @@ -0,0 +1,489 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +from sklearn.ensemble._hist_gradient_boosting.binning import ( + _BinMapper, + _find_binning_thresholds, + _map_to_bins, +) +from sklearn.ensemble._hist_gradient_boosting.common import ( + ALMOST_INF, + X_BINNED_DTYPE, + X_DTYPE, +) +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + +n_threads = _openmp_effective_n_threads() + + +DATA = ( + np.random.RandomState(42) + .normal(loc=[0, 10], scale=[1, 0.01], size=(int(1e6), 2)) + .astype(X_DTYPE) +) + + +def test_find_binning_thresholds_regular_data(): + data = np.linspace(0, 10, 1001) + bin_thresholds = _find_binning_thresholds(data, max_bins=10) + assert_allclose(bin_thresholds, [1, 2, 3, 4, 5, 6, 7, 8, 9]) + + bin_thresholds = _find_binning_thresholds(data, max_bins=5) + assert_allclose(bin_thresholds, [2, 4, 6, 8]) + + +def test_find_binning_thresholds_small_regular_data(): + data = np.linspace(0, 10, 11) + + bin_thresholds = _find_binning_thresholds(data, max_bins=5) + assert_allclose(bin_thresholds, [2, 4, 6, 8]) + + bin_thresholds = _find_binning_thresholds(data, max_bins=10) + assert_allclose(bin_thresholds, [1, 2, 3, 4, 5, 6, 7, 8, 9]) + + bin_thresholds = _find_binning_thresholds(data, max_bins=11) + assert_allclose(bin_thresholds, np.arange(10) + 0.5) + + bin_thresholds = _find_binning_thresholds(data, max_bins=255) + assert_allclose(bin_thresholds, np.arange(10) + 0.5) + + +def test_find_binning_thresholds_random_data(): + bin_thresholds = [ + _find_binning_thresholds(DATA[:, i], max_bins=255) for i in range(2) + ] + for i in range(len(bin_thresholds)): + assert bin_thresholds[i].shape == (254,) # 255 - 1 + assert bin_thresholds[i].dtype == DATA.dtype + + assert_allclose( + bin_thresholds[0][[64, 128, 192]], np.array([-0.7, 0.0, 0.7]), atol=1e-1 + ) + + assert_allclose( + bin_thresholds[1][[64, 128, 192]], np.array([9.99, 10.00, 10.01]), atol=1e-2 + ) + + +def test_find_binning_thresholds_low_n_bins(): + bin_thresholds = [ + _find_binning_thresholds(DATA[:, i], max_bins=128) for i in range(2) + ] + for i in range(len(bin_thresholds)): + assert bin_thresholds[i].shape == (127,) # 128 - 1 + assert bin_thresholds[i].dtype == DATA.dtype + + +@pytest.mark.parametrize("n_bins", (2, 257)) +def test_invalid_n_bins(n_bins): + err_msg = "n_bins={} should be no smaller than 3 and no larger than 256".format( + n_bins + ) + with pytest.raises(ValueError, match=err_msg): + _BinMapper(n_bins=n_bins).fit(DATA) + + +def test_bin_mapper_n_features_transform(): + mapper = _BinMapper(n_bins=42, random_state=42).fit(DATA) + err_msg = "This estimator was fitted with 2 features but 4 got passed" + with pytest.raises(ValueError, match=err_msg): + mapper.transform(np.repeat(DATA, 2, axis=1)) + + +@pytest.mark.parametrize("max_bins", [16, 128, 255]) +def test_map_to_bins(max_bins): + bin_thresholds = [ + _find_binning_thresholds(DATA[:, i], max_bins=max_bins) for i in range(2) + ] + binned = np.zeros_like(DATA, dtype=X_BINNED_DTYPE, order="F") + is_categorical = np.zeros(2, dtype=np.uint8) + last_bin_idx = max_bins + _map_to_bins(DATA, bin_thresholds, is_categorical, last_bin_idx, n_threads, binned) + assert binned.shape == DATA.shape + assert binned.dtype == np.uint8 + assert binned.flags.f_contiguous + + min_indices = DATA.argmin(axis=0) + max_indices = DATA.argmax(axis=0) + + for feature_idx, min_idx in enumerate(min_indices): + assert binned[min_idx, feature_idx] == 0 + for feature_idx, max_idx in enumerate(max_indices): + assert binned[max_idx, feature_idx] == max_bins - 1 + + +@pytest.mark.parametrize("max_bins", [5, 10, 42]) +def test_bin_mapper_random_data(max_bins): + n_samples, n_features = DATA.shape + + expected_count_per_bin = n_samples // max_bins + tol = int(0.05 * expected_count_per_bin) + + # max_bins is the number of bins for non-missing values + n_bins = max_bins + 1 + mapper = _BinMapper(n_bins=n_bins, random_state=42).fit(DATA) + binned = mapper.transform(DATA) + + assert binned.shape == (n_samples, n_features) + assert binned.dtype == np.uint8 + assert_array_equal(binned.min(axis=0), np.array([0, 0])) + assert_array_equal(binned.max(axis=0), np.array([max_bins - 1, max_bins - 1])) + assert len(mapper.bin_thresholds_) == n_features + for bin_thresholds_feature in mapper.bin_thresholds_: + assert bin_thresholds_feature.shape == (max_bins - 1,) + assert bin_thresholds_feature.dtype == DATA.dtype + assert np.all(mapper.n_bins_non_missing_ == max_bins) + + # Check that the binned data is approximately balanced across bins. + for feature_idx in range(n_features): + for bin_idx in range(max_bins): + count = (binned[:, feature_idx] == bin_idx).sum() + assert abs(count - expected_count_per_bin) < tol + + +@pytest.mark.parametrize("n_samples, max_bins", [(5, 5), (5, 10), (5, 11), (42, 255)]) +def test_bin_mapper_small_random_data(n_samples, max_bins): + data = np.random.RandomState(42).normal(size=n_samples).reshape(-1, 1) + assert len(np.unique(data)) == n_samples + + # max_bins is the number of bins for non-missing values + n_bins = max_bins + 1 + mapper = _BinMapper(n_bins=n_bins, random_state=42) + binned = mapper.fit_transform(data) + + assert binned.shape == data.shape + assert binned.dtype == np.uint8 + assert_array_equal(binned.ravel()[np.argsort(data.ravel())], np.arange(n_samples)) + + +@pytest.mark.parametrize( + "max_bins, n_distinct, multiplier", + [ + (5, 5, 1), + (5, 5, 3), + (255, 12, 42), + ], +) +def test_bin_mapper_identity_repeated_values(max_bins, n_distinct, multiplier): + data = np.array(list(range(n_distinct)) * multiplier).reshape(-1, 1) + # max_bins is the number of bins for non-missing values + n_bins = max_bins + 1 + binned = _BinMapper(n_bins=n_bins).fit_transform(data) + assert_array_equal(data, binned) + + +@pytest.mark.parametrize("n_distinct", [2, 7, 42]) +def test_bin_mapper_repeated_values_invariance(n_distinct): + rng = np.random.RandomState(42) + distinct_values = rng.normal(size=n_distinct) + assert len(np.unique(distinct_values)) == n_distinct + + repeated_indices = rng.randint(low=0, high=n_distinct, size=1000) + data = distinct_values[repeated_indices] + rng.shuffle(data) + assert_array_equal(np.unique(data), np.sort(distinct_values)) + + data = data.reshape(-1, 1) + + mapper_1 = _BinMapper(n_bins=n_distinct + 1) + binned_1 = mapper_1.fit_transform(data) + assert_array_equal(np.unique(binned_1[:, 0]), np.arange(n_distinct)) + + # Adding more bins to the mapper yields the same results (same thresholds) + mapper_2 = _BinMapper(n_bins=min(256, n_distinct * 3) + 1) + binned_2 = mapper_2.fit_transform(data) + + assert_allclose(mapper_1.bin_thresholds_[0], mapper_2.bin_thresholds_[0]) + assert_array_equal(binned_1, binned_2) + + +@pytest.mark.parametrize( + "max_bins, scale, offset", + [ + (3, 2, -1), + (42, 1, 0), + (255, 0.3, 42), + ], +) +def test_bin_mapper_identity_small(max_bins, scale, offset): + data = np.arange(max_bins).reshape(-1, 1) * scale + offset + # max_bins is the number of bins for non-missing values + n_bins = max_bins + 1 + binned = _BinMapper(n_bins=n_bins).fit_transform(data) + assert_array_equal(binned, np.arange(max_bins).reshape(-1, 1)) + + +@pytest.mark.parametrize( + "max_bins_small, max_bins_large", + [ + (2, 2), + (3, 3), + (4, 4), + (42, 42), + (255, 255), + (5, 17), + (42, 255), + ], +) +def test_bin_mapper_idempotence(max_bins_small, max_bins_large): + assert max_bins_large >= max_bins_small + data = np.random.RandomState(42).normal(size=30000).reshape(-1, 1) + mapper_small = _BinMapper(n_bins=max_bins_small + 1) + mapper_large = _BinMapper(n_bins=max_bins_small + 1) + binned_small = mapper_small.fit_transform(data) + binned_large = mapper_large.fit_transform(binned_small) + assert_array_equal(binned_small, binned_large) + + +@pytest.mark.parametrize("n_bins", [10, 100, 256]) +@pytest.mark.parametrize("diff", [-5, 0, 5]) +def test_n_bins_non_missing(n_bins, diff): + # Check that n_bins_non_missing is n_unique_values when + # there are not a lot of unique values, else n_bins - 1. + + n_unique_values = n_bins + diff + X = list(range(n_unique_values)) * 2 + X = np.array(X).reshape(-1, 1) + mapper = _BinMapper(n_bins=n_bins).fit(X) + assert np.all(mapper.n_bins_non_missing_ == min(n_bins - 1, n_unique_values)) + + +def test_subsample(): + # Make sure bin thresholds are different when applying subsampling + mapper_no_subsample = _BinMapper(subsample=None, random_state=0).fit(DATA) + mapper_subsample = _BinMapper(subsample=256, random_state=0).fit(DATA) + + for feature in range(DATA.shape[1]): + assert not np.allclose( + mapper_no_subsample.bin_thresholds_[feature], + mapper_subsample.bin_thresholds_[feature], + rtol=1e-4, + ) + + +@pytest.mark.parametrize( + "n_bins, n_bins_non_missing, X_trans_expected", + [ + ( + 256, + [4, 2, 2], + [ + [0, 0, 0], # 255 <=> missing value + [255, 255, 0], + [1, 0, 0], + [255, 1, 1], + [2, 1, 1], + [3, 0, 0], + ], + ), + ( + 3, + [2, 2, 2], + [ + [0, 0, 0], # 2 <=> missing value + [2, 2, 0], + [0, 0, 0], + [2, 1, 1], + [1, 1, 1], + [1, 0, 0], + ], + ), + ], +) +def test_missing_values_support(n_bins, n_bins_non_missing, X_trans_expected): + # check for missing values: make sure nans are mapped to the last bin + # and that the _BinMapper attributes are correct + + X = [ + [1, 1, 0], + [np.nan, np.nan, 0], + [2, 1, 0], + [np.nan, 2, 1], + [3, 2, 1], + [4, 1, 0], + ] + + X = np.array(X) + + mapper = _BinMapper(n_bins=n_bins) + mapper.fit(X) + + assert_array_equal(mapper.n_bins_non_missing_, n_bins_non_missing) + + for feature_idx in range(X.shape[1]): + assert ( + len(mapper.bin_thresholds_[feature_idx]) + == n_bins_non_missing[feature_idx] - 1 + ) + + assert mapper.missing_values_bin_idx_ == n_bins - 1 + + X_trans = mapper.transform(X) + assert_array_equal(X_trans, X_trans_expected) + + +def test_infinite_values(): + # Make sure infinite values are properly handled. + bin_mapper = _BinMapper() + + X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) + + bin_mapper.fit(X) + assert_allclose(bin_mapper.bin_thresholds_[0], [-np.inf, 0.5, ALMOST_INF]) + assert bin_mapper.n_bins_non_missing_ == [4] + + expected_binned_X = np.array([0, 1, 2, 3]).reshape(-1, 1) + assert_array_equal(bin_mapper.transform(X), expected_binned_X) + + +@pytest.mark.parametrize("n_bins", [15, 256]) +def test_categorical_feature(n_bins): + # Basic test for categorical features + # we make sure that categories are mapped into [0, n_categories - 1] and + # that nans are mapped to the last bin + X = np.array( + [[4] * 500 + [1] * 3 + [10] * 4 + [0] * 4 + [13] + [7] * 5 + [np.nan] * 2], + dtype=X_DTYPE, + ).T + known_categories = [np.unique(X[~np.isnan(X)])] + + bin_mapper = _BinMapper( + n_bins=n_bins, + is_categorical=np.array([True]), + known_categories=known_categories, + ).fit(X) + assert bin_mapper.n_bins_non_missing_ == [6] + assert_array_equal(bin_mapper.bin_thresholds_[0], [0, 1, 4, 7, 10, 13]) + + X = np.array([[0, 1, 4, np.nan, 7, 10, 13]], dtype=X_DTYPE).T + expected_trans = np.array([[0, 1, 2, n_bins - 1, 3, 4, 5]]).T + assert_array_equal(bin_mapper.transform(X), expected_trans) + + # Negative categories are mapped to the missing values' bin + # (i.e. the bin of index `missing_values_bin_idx_ == n_bins - 1). + # Unknown positive categories does not happen in practice and tested + # for illustration purpose. + X = np.array([[-4, -1, 100]], dtype=X_DTYPE).T + expected_trans = np.array([[n_bins - 1, n_bins - 1, 6]]).T + assert_array_equal(bin_mapper.transform(X), expected_trans) + + +def test_categorical_feature_negative_missing(): + """Make sure bin mapper treats negative categories as missing values.""" + X = np.array( + [[4] * 500 + [1] * 3 + [5] * 10 + [-1] * 3 + [np.nan] * 4], dtype=X_DTYPE + ).T + bin_mapper = _BinMapper( + n_bins=4, + is_categorical=np.array([True]), + known_categories=[np.array([1, 4, 5], dtype=X_DTYPE)], + ).fit(X) + + assert bin_mapper.n_bins_non_missing_ == [3] + + X = np.array([[-1, 1, 3, 5, np.nan]], dtype=X_DTYPE).T + + # Negative values for categorical features are considered as missing values. + # They are mapped to the bin of index `bin_mapper.missing_values_bin_idx_`, + # which is 3 here. + assert bin_mapper.missing_values_bin_idx_ == 3 + expected_trans = np.array([[3, 0, 1, 2, 3]]).T + assert_array_equal(bin_mapper.transform(X), expected_trans) + + +@pytest.mark.parametrize("n_bins", (128, 256)) +def test_categorical_with_numerical_features(n_bins): + # basic check for binmapper with mixed data + X1 = np.arange(10, 20).reshape(-1, 1) # numerical + X2 = np.arange(10, 15).reshape(-1, 1) # categorical + X2 = np.r_[X2, X2] + X = np.c_[X1, X2] + known_categories = [None, np.unique(X2).astype(X_DTYPE)] + + bin_mapper = _BinMapper( + n_bins=n_bins, + is_categorical=np.array([False, True]), + known_categories=known_categories, + ).fit(X) + + assert_array_equal(bin_mapper.n_bins_non_missing_, [10, 5]) + + bin_thresholds = bin_mapper.bin_thresholds_ + assert len(bin_thresholds) == 2 + assert_array_equal(bin_thresholds[1], np.arange(10, 15)) + + expected_X_trans = [ + [0, 0], + [1, 1], + [2, 2], + [3, 3], + [4, 4], + [5, 0], + [6, 1], + [7, 2], + [8, 3], + [9, 4], + ] + assert_array_equal(bin_mapper.transform(X), expected_X_trans) + + +def test_make_known_categories_bitsets(): + # Check the output of make_known_categories_bitsets + X = np.array( + [[14, 2, 30], [30, 4, 70], [40, 10, 180], [40, 240, 180]], dtype=X_DTYPE + ) + + bin_mapper = _BinMapper( + n_bins=256, + is_categorical=np.array([False, True, True]), + known_categories=[None, X[:, 1], X[:, 2]], + ) + bin_mapper.fit(X) + + known_cat_bitsets, f_idx_map = bin_mapper.make_known_categories_bitsets() + + # Note that for non-categorical features, values are left to 0 + expected_f_idx_map = np.array([0, 0, 1], dtype=np.uint8) + assert_allclose(expected_f_idx_map, f_idx_map) + + expected_cat_bitset = np.zeros((2, 8), dtype=np.uint32) + + # first categorical feature: [2, 4, 10, 240] + f_idx = 1 + mapped_f_idx = f_idx_map[f_idx] + expected_cat_bitset[mapped_f_idx, 0] = 2**2 + 2**4 + 2**10 + # 240 = 32**7 + 16, therefore the 16th bit of the 7th array is 1. + expected_cat_bitset[mapped_f_idx, 7] = 2**16 + + # second categorical feature [30, 70, 180] + f_idx = 2 + mapped_f_idx = f_idx_map[f_idx] + expected_cat_bitset[mapped_f_idx, 0] = 2**30 + expected_cat_bitset[mapped_f_idx, 2] = 2**6 + expected_cat_bitset[mapped_f_idx, 5] = 2**20 + + assert_allclose(expected_cat_bitset, known_cat_bitsets) + + +@pytest.mark.parametrize( + "is_categorical, known_categories, match", + [ + (np.array([True]), [None], "Known categories for feature 0 must be provided"), + ( + np.array([False]), + np.array([1, 2, 3]), + "isn't marked as a categorical feature, but categories were passed", + ), + ], +) +def test_categorical_parameters(is_categorical, known_categories, match): + # test the validation of the is_categorical and known_categories parameters + + X = np.array([[1, 2, 3]], dtype=X_DTYPE) + + bin_mapper = _BinMapper( + is_categorical=is_categorical, known_categories=known_categories + ) + with pytest.raises(ValueError, match=match): + bin_mapper.fit(X) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_bitset.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_bitset.py new file mode 100644 index 0000000000000000000000000000000000000000..0d33a3ffb72755008533eb97de39fe77661b1809 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_bitset.py @@ -0,0 +1,64 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from sklearn.ensemble._hist_gradient_boosting._bitset import ( + in_bitset_memoryview, + set_bitset_memoryview, + set_raw_bitset_from_binned_bitset, +) +from sklearn.ensemble._hist_gradient_boosting.common import X_DTYPE + + +@pytest.mark.parametrize( + "values_to_insert, expected_bitset", + [ + ([0, 4, 33], np.array([2**0 + 2**4, 2**1, 0], dtype=np.uint32)), + ( + [31, 32, 33, 79], + np.array([2**31, 2**0 + 2**1, 2**15], dtype=np.uint32), + ), + ], +) +def test_set_get_bitset(values_to_insert, expected_bitset): + n_32bits_ints = 3 + bitset = np.zeros(n_32bits_ints, dtype=np.uint32) + for value in values_to_insert: + set_bitset_memoryview(bitset, value) + assert_allclose(expected_bitset, bitset) + for value in range(32 * n_32bits_ints): + if value in values_to_insert: + assert in_bitset_memoryview(bitset, value) + else: + assert not in_bitset_memoryview(bitset, value) + + +@pytest.mark.parametrize( + "raw_categories, binned_cat_to_insert, expected_raw_bitset", + [ + ( + [3, 4, 5, 10, 31, 32, 43], + [0, 2, 4, 5, 6], + [2**3 + 2**5 + 2**31, 2**0 + 2**11], + ), + ([3, 33, 50, 52], [1, 3], [0, 2**1 + 2**20]), + ], +) +def test_raw_bitset_from_binned_bitset( + raw_categories, binned_cat_to_insert, expected_raw_bitset +): + binned_bitset = np.zeros(2, dtype=np.uint32) + raw_bitset = np.zeros(2, dtype=np.uint32) + raw_categories = np.asarray(raw_categories, dtype=X_DTYPE) + + for val in binned_cat_to_insert: + set_bitset_memoryview(binned_bitset, val) + + set_raw_bitset_from_binned_bitset(raw_bitset, binned_bitset, raw_categories) + + assert_allclose(expected_raw_bitset, raw_bitset) + for binned_cat_val, raw_cat_val in enumerate(raw_categories): + if binned_cat_val in binned_cat_to_insert: + assert in_bitset_memoryview(raw_bitset, raw_cat_val) + else: + assert not in_bitset_memoryview(raw_bitset, raw_cat_val) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py new file mode 100644 index 0000000000000000000000000000000000000000..2d0fa8d9e5022321ccf57417dca1c57ce66a4a8c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py @@ -0,0 +1,279 @@ +import numpy as np +import pytest + +from sklearn.datasets import make_classification, make_regression +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper +from sklearn.ensemble._hist_gradient_boosting.utils import get_equivalent_estimator +from sklearn.metrics import accuracy_score +from sklearn.model_selection import train_test_split + + +@pytest.mark.parametrize("seed", range(5)) +@pytest.mark.parametrize( + "loss", + [ + "squared_error", + "poisson", + pytest.param( + "gamma", + marks=pytest.mark.skip("LightGBM with gamma loss has larger deviation."), + ), + ], +) +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize( + "n_samples, max_leaf_nodes", + [ + (255, 4096), + (1000, 8), + ], +) +def test_same_predictions_regression( + seed, loss, min_samples_leaf, n_samples, max_leaf_nodes +): + # Make sure sklearn has the same predictions as lightgbm for easy targets. + # + # In particular when the size of the trees are bound and the number of + # samples is large enough, the structure of the prediction trees found by + # LightGBM and sklearn should be exactly identical. + # + # Notes: + # - Several candidate splits may have equal gains when the number of + # samples in a node is low (and because of float errors). Therefore the + # predictions on the test set might differ if the structure of the tree + # is not exactly the same. To avoid this issue we only compare the + # predictions on the test set when the number of samples is large enough + # and max_leaf_nodes is low enough. + # - To ignore discrepancies caused by small differences in the binning + # strategy, data is pre-binned if n_samples > 255. + # - We don't check the absolute_error loss here. This is because + # LightGBM's computation of the median (used for the initial value of + # raw_prediction) is a bit off (they'll e.g. return midpoints when there + # is no need to.). Since these tests only run 1 iteration, the + # discrepancy between the initial values leads to biggish differences in + # the predictions. These differences are much smaller with more + # iterations. + pytest.importorskip("lightgbm") + + rng = np.random.RandomState(seed=seed) + max_iter = 1 + max_bins = 255 + + X, y = make_regression( + n_samples=n_samples, n_features=5, n_informative=5, random_state=0 + ) + + if loss in ("gamma", "poisson"): + # make the target positive + y = np.abs(y) + np.mean(np.abs(y)) + + if n_samples > 255: + # bin data and convert it to float32 so that the estimator doesn't + # treat it as pre-binned + X = _BinMapper(n_bins=max_bins + 1).fit_transform(X).astype(np.float32) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) + + est_sklearn = HistGradientBoostingRegressor( + loss=loss, + max_iter=max_iter, + max_bins=max_bins, + learning_rate=1, + early_stopping=False, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=max_leaf_nodes, + ) + est_lightgbm = get_equivalent_estimator(est_sklearn, lib="lightgbm") + est_lightgbm.set_params(min_sum_hessian_in_leaf=0) + + est_lightgbm.fit(X_train, y_train) + est_sklearn.fit(X_train, y_train) + + # We need X to be treated a numerical data, not pre-binned data. + X_train, X_test = X_train.astype(np.float32), X_test.astype(np.float32) + + pred_lightgbm = est_lightgbm.predict(X_train) + pred_sklearn = est_sklearn.predict(X_train) + if loss in ("gamma", "poisson"): + # More than 65% of the predictions must be close up to the 2nd decimal. + # TODO: We are not entirely satisfied with this lax comparison, but the root + # cause is not clear, maybe algorithmic differences. One such example is the + # poisson_max_delta_step parameter of LightGBM which does not exist in HGBT. + assert ( + np.mean(np.isclose(pred_lightgbm, pred_sklearn, rtol=1e-2, atol=1e-2)) + > 0.65 + ) + else: + # Less than 1% of the predictions may deviate more than 1e-3 in relative terms. + assert np.mean(np.isclose(pred_lightgbm, pred_sklearn, rtol=1e-3)) > 1 - 0.01 + + if max_leaf_nodes < 10 and n_samples >= 1000 and loss in ("squared_error",): + pred_lightgbm = est_lightgbm.predict(X_test) + pred_sklearn = est_sklearn.predict(X_test) + # Less than 1% of the predictions may deviate more than 1e-4 in relative terms. + assert np.mean(np.isclose(pred_lightgbm, pred_sklearn, rtol=1e-4)) > 1 - 0.01 + + +@pytest.mark.parametrize("seed", range(5)) +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize( + "n_samples, max_leaf_nodes", + [ + (255, 4096), + (1000, 8), + ], +) +def test_same_predictions_classification( + seed, min_samples_leaf, n_samples, max_leaf_nodes +): + # Same as test_same_predictions_regression but for classification + pytest.importorskip("lightgbm") + + rng = np.random.RandomState(seed=seed) + max_iter = 1 + n_classes = 2 + max_bins = 255 + + X, y = make_classification( + n_samples=n_samples, + n_classes=n_classes, + n_features=5, + n_informative=5, + n_redundant=0, + random_state=0, + ) + + if n_samples > 255: + # bin data and convert it to float32 so that the estimator doesn't + # treat it as pre-binned + X = _BinMapper(n_bins=max_bins + 1).fit_transform(X).astype(np.float32) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) + + est_sklearn = HistGradientBoostingClassifier( + loss="log_loss", + max_iter=max_iter, + max_bins=max_bins, + learning_rate=1, + early_stopping=False, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=max_leaf_nodes, + ) + est_lightgbm = get_equivalent_estimator( + est_sklearn, lib="lightgbm", n_classes=n_classes + ) + + est_lightgbm.fit(X_train, y_train) + est_sklearn.fit(X_train, y_train) + + # We need X to be treated a numerical data, not pre-binned data. + X_train, X_test = X_train.astype(np.float32), X_test.astype(np.float32) + + pred_lightgbm = est_lightgbm.predict(X_train) + pred_sklearn = est_sklearn.predict(X_train) + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 + + acc_lightgbm = accuracy_score(y_train, pred_lightgbm) + acc_sklearn = accuracy_score(y_train, pred_sklearn) + np.testing.assert_almost_equal(acc_lightgbm, acc_sklearn) + + if max_leaf_nodes < 10 and n_samples >= 1000: + pred_lightgbm = est_lightgbm.predict(X_test) + pred_sklearn = est_sklearn.predict(X_test) + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 + + acc_lightgbm = accuracy_score(y_test, pred_lightgbm) + acc_sklearn = accuracy_score(y_test, pred_sklearn) + np.testing.assert_almost_equal(acc_lightgbm, acc_sklearn, decimal=2) + + +@pytest.mark.parametrize("seed", range(5)) +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize( + "n_samples, max_leaf_nodes", + [ + (255, 4096), + (10000, 8), + ], +) +def test_same_predictions_multiclass_classification( + seed, min_samples_leaf, n_samples, max_leaf_nodes +): + # Same as test_same_predictions_regression but for classification + pytest.importorskip("lightgbm") + + rng = np.random.RandomState(seed=seed) + n_classes = 3 + max_iter = 1 + max_bins = 255 + lr = 1 + + X, y = make_classification( + n_samples=n_samples, + n_classes=n_classes, + n_features=5, + n_informative=5, + n_redundant=0, + n_clusters_per_class=1, + random_state=0, + ) + + if n_samples > 255: + # bin data and convert it to float32 so that the estimator doesn't + # treat it as pre-binned + X = _BinMapper(n_bins=max_bins + 1).fit_transform(X).astype(np.float32) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) + + est_sklearn = HistGradientBoostingClassifier( + loss="log_loss", + max_iter=max_iter, + max_bins=max_bins, + learning_rate=lr, + early_stopping=False, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=max_leaf_nodes, + ) + est_lightgbm = get_equivalent_estimator( + est_sklearn, lib="lightgbm", n_classes=n_classes + ) + + est_lightgbm.fit(X_train, y_train) + est_sklearn.fit(X_train, y_train) + + # We need X to be treated a numerical data, not pre-binned data. + X_train, X_test = X_train.astype(np.float32), X_test.astype(np.float32) + + pred_lightgbm = est_lightgbm.predict(X_train) + pred_sklearn = est_sklearn.predict(X_train) + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 + + proba_lightgbm = est_lightgbm.predict_proba(X_train) + proba_sklearn = est_sklearn.predict_proba(X_train) + # assert more than 75% of the predicted probabilities are the same up to + # the second decimal + assert np.mean(np.abs(proba_lightgbm - proba_sklearn) < 1e-2) > 0.75 + + acc_lightgbm = accuracy_score(y_train, pred_lightgbm) + acc_sklearn = accuracy_score(y_train, pred_sklearn) + + np.testing.assert_allclose(acc_lightgbm, acc_sklearn, rtol=0, atol=5e-2) + + if max_leaf_nodes < 10 and n_samples >= 1000: + pred_lightgbm = est_lightgbm.predict(X_test) + pred_sklearn = est_sklearn.predict(X_test) + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 + + proba_lightgbm = est_lightgbm.predict_proba(X_train) + proba_sklearn = est_sklearn.predict_proba(X_train) + # assert more than 75% of the predicted probabilities are the same up + # to the second decimal + assert np.mean(np.abs(proba_lightgbm - proba_sklearn) < 1e-2) > 0.75 + + acc_lightgbm = accuracy_score(y_test, pred_lightgbm) + acc_sklearn = accuracy_score(y_test, pred_sklearn) + np.testing.assert_almost_equal(acc_lightgbm, acc_sklearn, decimal=2) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py new file mode 100644 index 0000000000000000000000000000000000000000..c6ce16a68a7d76fe6619d771dbbcf21aca2d476b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py @@ -0,0 +1,1760 @@ +import copyreg +import io +import pickle +import re +import warnings +from unittest.mock import Mock + +import joblib +import numpy as np +import pytest +from joblib.numpy_pickle import NumpyPickler +from numpy.testing import assert_allclose, assert_array_equal + +import sklearn +from sklearn._loss.loss import ( + AbsoluteError, + HalfBinomialLoss, + HalfSquaredError, + PinballLoss, +) +from sklearn.base import BaseEstimator, TransformerMixin, clone, is_regressor +from sklearn.compose import make_column_transformer +from sklearn.datasets import make_classification, make_low_rank_matrix, make_regression +from sklearn.dummy import DummyRegressor +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper +from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +from sklearn.ensemble._hist_gradient_boosting.predictor import TreePredictor +from sklearn.exceptions import NotFittedError +from sklearn.metrics import get_scorer, mean_gamma_deviance, mean_poisson_deviance +from sklearn.model_selection import cross_val_score, train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import KBinsDiscretizer, MinMaxScaler, OneHotEncoder +from sklearn.utils import check_random_state, shuffle +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.utils._testing import _convert_container +from sklearn.utils.fixes import _IS_32BIT + +n_threads = _openmp_effective_n_threads() + +X_classification, y_classification = make_classification(random_state=0) +X_regression, y_regression = make_regression(random_state=0) +X_multi_classification, y_multi_classification = make_classification( + n_classes=3, n_informative=3, random_state=0 +) + + +def _make_dumb_dataset(n_samples): + """Make a dumb dataset to test early stopping.""" + rng = np.random.RandomState(42) + X_dumb = rng.randn(n_samples, 1) + y_dumb = (X_dumb[:, 0] > 0).astype("int64") + return X_dumb, y_dumb + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"interaction_cst": [0, 1]}, + "Interaction constraints must be a sequence of tuples or lists", + ), + ( + {"interaction_cst": [{0, 9999}]}, + r"Interaction constraints must consist of integer indices in \[0," + r" n_features - 1\] = \[.*\], specifying the position of features,", + ), + ( + {"interaction_cst": [{-1, 0}]}, + r"Interaction constraints must consist of integer indices in \[0," + r" n_features - 1\] = \[.*\], specifying the position of features,", + ), + ( + {"interaction_cst": [{0.5}]}, + r"Interaction constraints must consist of integer indices in \[0," + r" n_features - 1\] = \[.*\], specifying the position of features,", + ), + ], +) +def test_init_parameters_validation(GradientBoosting, X, y, params, err_msg): + with pytest.raises(ValueError, match=err_msg): + GradientBoosting(**params).fit(X, y) + + +@pytest.mark.parametrize( + "scoring, validation_fraction, early_stopping, n_iter_no_change, tol", + [ + ("neg_mean_squared_error", 0.1, True, 5, 1e-7), # use scorer + ("neg_mean_squared_error", None, True, 5, 1e-1), # use scorer on train + (None, 0.1, True, 5, 1e-7), # same with default scorer + (None, None, True, 5, 1e-1), + ("loss", 0.1, True, 5, 1e-7), # use loss + ("loss", None, True, 5, 1e-1), # use loss on training data + (None, None, False, 5, 0.0), # no early stopping + ], +) +def test_early_stopping_regression( + scoring, validation_fraction, early_stopping, n_iter_no_change, tol +): + max_iter = 200 + + X, y = make_regression(n_samples=50, random_state=0) + + gb = HistGradientBoostingRegressor( + verbose=1, # just for coverage + min_samples_leaf=5, # easier to overfit fast + scoring=scoring, + tol=tol, + early_stopping=early_stopping, + validation_fraction=validation_fraction, + max_iter=max_iter, + n_iter_no_change=n_iter_no_change, + random_state=0, + ) + gb.fit(X, y) + + if early_stopping: + assert n_iter_no_change <= gb.n_iter_ < max_iter + else: + assert gb.n_iter_ == max_iter + + +@pytest.mark.parametrize( + "data", + ( + make_classification(n_samples=30, random_state=0), + make_classification( + n_samples=30, n_classes=3, n_clusters_per_class=1, random_state=0 + ), + ), +) +@pytest.mark.parametrize( + "scoring, validation_fraction, early_stopping, n_iter_no_change, tol", + [ + ("accuracy", 0.1, True, 5, 1e-7), # use scorer + ("accuracy", None, True, 5, 1e-1), # use scorer on training data + (None, 0.1, True, 5, 1e-7), # same with default scorer + (None, None, True, 5, 1e-1), + ("loss", 0.1, True, 5, 1e-7), # use loss + ("loss", None, True, 5, 1e-1), # use loss on training data + (None, None, False, 5, 0.0), # no early stopping + ], +) +def test_early_stopping_classification( + data, scoring, validation_fraction, early_stopping, n_iter_no_change, tol +): + max_iter = 50 + + X, y = data + + gb = HistGradientBoostingClassifier( + verbose=2, # just for coverage + min_samples_leaf=5, # easier to overfit fast + scoring=scoring, + tol=tol, + early_stopping=early_stopping, + validation_fraction=validation_fraction, + max_iter=max_iter, + n_iter_no_change=n_iter_no_change, + random_state=0, + ) + gb.fit(X, y) + + if early_stopping is True: + assert n_iter_no_change <= gb.n_iter_ < max_iter + else: + assert gb.n_iter_ == max_iter + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, *_make_dumb_dataset(10000)), + (HistGradientBoostingClassifier, *_make_dumb_dataset(10001)), + (HistGradientBoostingRegressor, *_make_dumb_dataset(10000)), + (HistGradientBoostingRegressor, *_make_dumb_dataset(10001)), + ], +) +def test_early_stopping_default(GradientBoosting, X, y): + # Test that early stopping is enabled by default if and only if there + # are more than 10000 samples + gb = GradientBoosting(max_iter=10, n_iter_no_change=2, tol=1e-1) + gb.fit(X, y) + if X.shape[0] > 10000: + assert gb.n_iter_ < gb.max_iter + else: + assert gb.n_iter_ == gb.max_iter + + +@pytest.mark.parametrize( + "scores, n_iter_no_change, tol, stopping", + [ + ([], 1, 0.001, False), # not enough iterations + ([1, 1, 1], 5, 0.001, False), # not enough iterations + ([1, 1, 1, 1, 1], 5, 0.001, False), # not enough iterations + ([1, 2, 3, 4, 5, 6], 5, 0.001, False), # significant improvement + ([1, 2, 3, 4, 5, 6], 5, 0.0, False), # significant improvement + ([1, 2, 3, 4, 5, 6], 5, 0.999, False), # significant improvement + ([1, 2, 3, 4, 5, 6], 5, 5 - 1e-5, False), # significant improvement + ([1] * 6, 5, 0.0, True), # no significant improvement + ([1] * 6, 5, 0.001, True), # no significant improvement + ([1] * 6, 5, 5, True), # no significant improvement + ], +) +def test_should_stop(scores, n_iter_no_change, tol, stopping): + gbdt = HistGradientBoostingClassifier(n_iter_no_change=n_iter_no_change, tol=tol) + assert gbdt._should_stop(scores) == stopping + + +def test_absolute_error(): + # For coverage only. + X, y = make_regression(n_samples=500, random_state=0) + gbdt = HistGradientBoostingRegressor(loss="absolute_error", random_state=0) + gbdt.fit(X, y) + assert gbdt.score(X, y) > 0.9 + + +def test_absolute_error_sample_weight(): + # non regression test for issue #19400 + # make sure no error is thrown during fit of + # HistGradientBoostingRegressor with absolute_error loss function + # and passing sample_weight + rng = np.random.RandomState(0) + n_samples = 100 + X = rng.uniform(-1, 1, size=(n_samples, 2)) + y = rng.uniform(-1, 1, size=n_samples) + sample_weight = rng.uniform(0, 1, size=n_samples) + gbdt = HistGradientBoostingRegressor(loss="absolute_error") + gbdt.fit(X, y, sample_weight=sample_weight) + + +@pytest.mark.parametrize("y", [([1.0, -2.0, 0.0]), ([0.0, 1.0, 2.0])]) +def test_gamma_y_positive(y): + # Test that ValueError is raised if any y_i <= 0. + err_msg = r"loss='gamma' requires strictly positive y." + gbdt = HistGradientBoostingRegressor(loss="gamma", random_state=0) + with pytest.raises(ValueError, match=err_msg): + gbdt.fit(np.zeros(shape=(len(y), 1)), y) + + +def test_gamma(): + # For a Gamma distributed target, we expect an HGBT trained with the Gamma deviance + # (loss) to give better results than an HGBT with any other loss function, measured + # in out-of-sample Gamma deviance as metric/score. + # Note that squared error could potentially predict negative values which is + # invalid (np.inf) for the Gamma deviance. A Poisson HGBT (having a log link) + # does not have that defect. + # Important note: It seems that a Poisson HGBT almost always has better + # out-of-sample performance than the Gamma HGBT, measured in Gamma deviance. + # LightGBM shows the same behaviour. Hence, we only compare to a squared error + # HGBT, but not to a Poisson deviance HGBT. + rng = np.random.RandomState(42) + n_train, n_test, n_features = 500, 100, 20 + X = make_low_rank_matrix( + n_samples=n_train + n_test, + n_features=n_features, + random_state=rng, + ) + # We create a log-linear Gamma model. This gives y.min ~ 1e-2, y.max ~ 1e2 + coef = rng.uniform(low=-10, high=20, size=n_features) + # Numpy parametrizes gamma(shape=k, scale=theta) with mean = k * theta and + # variance = k * theta^2. We parametrize it instead with mean = exp(X @ coef) + # and variance = dispersion * mean^2 by setting k = 1 / dispersion, + # theta = dispersion * mean. + dispersion = 0.5 + y = rng.gamma(shape=1 / dispersion, scale=dispersion * np.exp(X @ coef)) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=n_test, random_state=rng + ) + gbdt_gamma = HistGradientBoostingRegressor(loss="gamma", random_state=123) + gbdt_mse = HistGradientBoostingRegressor(loss="squared_error", random_state=123) + dummy = DummyRegressor(strategy="mean") + for model in (gbdt_gamma, gbdt_mse, dummy): + model.fit(X_train, y_train) + + for X, y in [(X_train, y_train), (X_test, y_test)]: + loss_gbdt_gamma = mean_gamma_deviance(y, gbdt_gamma.predict(X)) + # We restrict the squared error HGBT to predict at least the minimum seen y at + # train time to make it strictly positive. + loss_gbdt_mse = mean_gamma_deviance( + y, np.maximum(np.min(y_train), gbdt_mse.predict(X)) + ) + loss_dummy = mean_gamma_deviance(y, dummy.predict(X)) + assert loss_gbdt_gamma < loss_dummy + assert loss_gbdt_gamma < loss_gbdt_mse + + +@pytest.mark.parametrize("quantile", [0.2, 0.5, 0.8]) +def test_quantile_asymmetric_error(quantile): + """Test quantile regression for asymmetric distributed targets.""" + n_samples = 10_000 + rng = np.random.RandomState(42) + # take care that X @ coef + intercept > 0 + X = np.concatenate( + ( + np.abs(rng.randn(n_samples)[:, None]), + -rng.randint(2, size=(n_samples, 1)), + ), + axis=1, + ) + intercept = 1.23 + coef = np.array([0.5, -2]) + # For an exponential distribution with rate lambda, e.g. exp(-lambda * x), + # the quantile at level q is: + # quantile(q) = - log(1 - q) / lambda + # scale = 1/lambda = -quantile(q) / log(1-q) + y = rng.exponential( + scale=-(X @ coef + intercept) / np.log(1 - quantile), size=n_samples + ) + model = HistGradientBoostingRegressor( + loss="quantile", + quantile=quantile, + max_iter=25, + random_state=0, + max_leaf_nodes=10, + ).fit(X, y) + assert_allclose(np.mean(model.predict(X) > y), quantile, rtol=1e-2) + + pinball_loss = PinballLoss(quantile=quantile) + loss_true_quantile = pinball_loss(y, X @ coef + intercept) + loss_pred_quantile = pinball_loss(y, model.predict(X)) + # we are overfitting + assert loss_pred_quantile <= loss_true_quantile + + +@pytest.mark.parametrize("y", [([1.0, -2.0, 0.0]), ([0.0, 0.0, 0.0])]) +def test_poisson_y_positive(y): + # Test that ValueError is raised if either one y_i < 0 or sum(y_i) <= 0. + err_msg = r"loss='poisson' requires non-negative y and sum\(y\) > 0." + gbdt = HistGradientBoostingRegressor(loss="poisson", random_state=0) + with pytest.raises(ValueError, match=err_msg): + gbdt.fit(np.zeros(shape=(len(y), 1)), y) + + +def test_poisson(): + # For Poisson distributed target, Poisson loss should give better results + # than least squares measured in Poisson deviance as metric. + rng = np.random.RandomState(42) + n_train, n_test, n_features = 500, 100, 100 + X = make_low_rank_matrix( + n_samples=n_train + n_test, n_features=n_features, random_state=rng + ) + # We create a log-linear Poisson model and downscale coef as it will get + # exponentiated. + coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0) + y = rng.poisson(lam=np.exp(X @ coef)) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=n_test, random_state=rng + ) + gbdt_pois = HistGradientBoostingRegressor(loss="poisson", random_state=rng) + gbdt_ls = HistGradientBoostingRegressor(loss="squared_error", random_state=rng) + gbdt_pois.fit(X_train, y_train) + gbdt_ls.fit(X_train, y_train) + dummy = DummyRegressor(strategy="mean").fit(X_train, y_train) + + for X, y in [(X_train, y_train), (X_test, y_test)]: + metric_pois = mean_poisson_deviance(y, gbdt_pois.predict(X)) + # squared_error might produce non-positive predictions => clip + metric_ls = mean_poisson_deviance(y, np.clip(gbdt_ls.predict(X), 1e-15, None)) + metric_dummy = mean_poisson_deviance(y, dummy.predict(X)) + assert metric_pois < metric_ls + assert metric_pois < metric_dummy + + +def test_binning_train_validation_are_separated(): + # Make sure training and validation data are binned separately. + # See issue 13926 + + rng = np.random.RandomState(0) + validation_fraction = 0.2 + gb = HistGradientBoostingClassifier( + early_stopping=True, validation_fraction=validation_fraction, random_state=rng + ) + gb.fit(X_classification, y_classification) + mapper_training_data = gb._bin_mapper + + # Note that since the data is small there is no subsampling and the + # random_state doesn't matter + mapper_whole_data = _BinMapper(random_state=0) + mapper_whole_data.fit(X_classification) + + n_samples = X_classification.shape[0] + assert np.all( + mapper_training_data.n_bins_non_missing_ + == int((1 - validation_fraction) * n_samples) + ) + assert np.all( + mapper_training_data.n_bins_non_missing_ + != mapper_whole_data.n_bins_non_missing_ + ) + + +def test_missing_values_trivial(): + # sanity check for missing values support. With only one feature and + # y == isnan(X), the gbdt is supposed to reach perfect accuracy on the + # training set. + + n_samples = 100 + n_features = 1 + rng = np.random.RandomState(0) + + X = rng.normal(size=(n_samples, n_features)) + mask = rng.binomial(1, 0.5, size=X.shape).astype(bool) + X[mask] = np.nan + y = mask.ravel() + gb = HistGradientBoostingClassifier() + gb.fit(X, y) + + assert gb.score(X, y) == pytest.approx(1) + + +@pytest.mark.parametrize("problem", ("classification", "regression")) +@pytest.mark.parametrize( + ( + "missing_proportion, expected_min_score_classification, " + "expected_min_score_regression" + ), + [(0.1, 0.97, 0.89), (0.2, 0.93, 0.81), (0.5, 0.79, 0.52)], +) +def test_missing_values_resilience( + problem, + missing_proportion, + expected_min_score_classification, + expected_min_score_regression, +): + # Make sure the estimators can deal with missing values and still yield + # decent predictions + + rng = np.random.RandomState(0) + n_samples = 1000 + n_features = 2 + if problem == "regression": + X, y = make_regression( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + random_state=rng, + ) + gb = HistGradientBoostingRegressor() + expected_min_score = expected_min_score_regression + else: + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + n_redundant=0, + n_repeated=0, + random_state=rng, + ) + gb = HistGradientBoostingClassifier() + expected_min_score = expected_min_score_classification + + mask = rng.binomial(1, missing_proportion, size=X.shape).astype(bool) + X[mask] = np.nan + + gb.fit(X, y) + + assert gb.score(X, y) > expected_min_score + + +@pytest.mark.parametrize( + "data", + [ + make_classification(random_state=0, n_classes=2), + make_classification(random_state=0, n_classes=3, n_informative=3), + ], + ids=["binary_log_loss", "multiclass_log_loss"], +) +def test_zero_division_hessians(data): + # non regression test for issue #14018 + # make sure we avoid zero division errors when computing the leaves values. + + # If the learning rate is too high, the raw predictions are bad and will + # saturate the softmax (or sigmoid in binary classif). This leads to + # probabilities being exactly 0 or 1, gradients being constant, and + # hessians being zero. + X, y = data + gb = HistGradientBoostingClassifier(learning_rate=100, max_iter=10) + gb.fit(X, y) + + +def test_small_trainset(): + # Make sure that the small trainset is stratified and has the expected + # length (10k samples) + n_samples = 20000 + original_distrib = {0: 0.1, 1: 0.2, 2: 0.3, 3: 0.4} + rng = np.random.RandomState(42) + X = rng.randn(n_samples).reshape(n_samples, 1) + y = [ + [class_] * int(prop * n_samples) for (class_, prop) in original_distrib.items() + ] + y = shuffle(np.concatenate(y)) + gb = HistGradientBoostingClassifier() + + # Compute the small training set + X_small, y_small, *_ = gb._get_small_trainset( + X, y, seed=42, sample_weight_train=None + ) + + # Compute the class distribution in the small training set + unique, counts = np.unique(y_small, return_counts=True) + small_distrib = {class_: count / 10000 for (class_, count) in zip(unique, counts)} + + # Test that the small training set has the expected length + assert X_small.shape[0] == 10000 + assert y_small.shape[0] == 10000 + + # Test that the class distributions in the whole dataset and in the small + # training set are identical + assert small_distrib == pytest.approx(original_distrib) + + +def test_missing_values_minmax_imputation(): + # Compare the buit-in missing value handling of Histogram GBC with an + # a-priori missing value imputation strategy that should yield the same + # results in terms of decision function. + # + # Each feature (containing NaNs) is replaced by 2 features: + # - one where the nans are replaced by min(feature) - 1 + # - one where the nans are replaced by max(feature) + 1 + # A split where nans go to the left has an equivalent split in the + # first (min) feature, and a split where nans go to the right has an + # equivalent split in the second (max) feature. + # + # Assuming the data is such that there is never a tie to select the best + # feature to split on during training, the learned decision trees should be + # strictly equivalent (learn a sequence of splits that encode the same + # decision function). + # + # The MinMaxImputer transformer is meant to be a toy implementation of the + # "Missing In Attributes" (MIA) missing value handling for decision trees + # https://www.sciencedirect.com/science/article/abs/pii/S0167865508000305 + # The implementation of MIA as an imputation transformer was suggested by + # "Remark 3" in :arxiv:'<1902.06931>` + + class MinMaxImputer(TransformerMixin, BaseEstimator): + def fit(self, X, y=None): + mm = MinMaxScaler().fit(X) + self.data_min_ = mm.data_min_ + self.data_max_ = mm.data_max_ + return self + + def transform(self, X): + X_min, X_max = X.copy(), X.copy() + + for feature_idx in range(X.shape[1]): + nan_mask = np.isnan(X[:, feature_idx]) + X_min[nan_mask, feature_idx] = self.data_min_[feature_idx] - 1 + X_max[nan_mask, feature_idx] = self.data_max_[feature_idx] + 1 + + return np.concatenate([X_min, X_max], axis=1) + + def make_missing_value_data(n_samples=int(1e4), seed=0): + rng = np.random.RandomState(seed) + X, y = make_regression(n_samples=n_samples, n_features=4, random_state=rng) + + # Pre-bin the data to ensure a deterministic handling by the 2 + # strategies and also make it easier to insert np.nan in a structured + # way: + X = KBinsDiscretizer( + n_bins=42, encode="ordinal", quantile_method="averaged_inverted_cdf" + ).fit_transform(X) + + # First feature has missing values completely at random: + rnd_mask = rng.rand(X.shape[0]) > 0.9 + X[rnd_mask, 0] = np.nan + + # Second and third features have missing values for extreme values + # (censoring missingness): + low_mask = X[:, 1] == 0 + X[low_mask, 1] = np.nan + + high_mask = X[:, 2] == X[:, 2].max() + X[high_mask, 2] = np.nan + + # Make the last feature nan pattern very informative: + y_max = np.percentile(y, 70) + y_max_mask = y >= y_max + y[y_max_mask] = y_max + X[y_max_mask, 3] = np.nan + + # Check that there is at least one missing value in each feature: + for feature_idx in range(X.shape[1]): + assert any(np.isnan(X[:, feature_idx])) + + # Let's use a test set to check that the learned decision function is + # the same as evaluated on unseen data. Otherwise it could just be the + # case that we find two independent ways to overfit the training set. + return train_test_split(X, y, random_state=rng) + + # n_samples need to be large enough to minimize the likelihood of having + # several candidate splits with the same gain value in a given tree. + X_train, X_test, y_train, y_test = make_missing_value_data( + n_samples=int(1e4), seed=0 + ) + + # Use a small number of leaf nodes and iterations so as to keep + # under-fitting models to minimize the likelihood of ties when training the + # model. + gbm1 = HistGradientBoostingRegressor(max_iter=100, max_leaf_nodes=5, random_state=0) + gbm1.fit(X_train, y_train) + + gbm2 = make_pipeline(MinMaxImputer(), clone(gbm1)) + gbm2.fit(X_train, y_train) + + # Check that the model reach the same score: + assert gbm1.score(X_train, y_train) == pytest.approx(gbm2.score(X_train, y_train)) + + assert gbm1.score(X_test, y_test) == pytest.approx(gbm2.score(X_test, y_test)) + + # Check the individual prediction match as a finer grained + # decision function check. + assert_allclose(gbm1.predict(X_train), gbm2.predict(X_train)) + assert_allclose(gbm1.predict(X_test), gbm2.predict(X_test)) + + +def test_infinite_values(): + # Basic test for infinite values + + X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) + y = np.array([0, 0, 1, 1]) + + gbdt = HistGradientBoostingRegressor(min_samples_leaf=1) + gbdt.fit(X, y) + np.testing.assert_allclose(gbdt.predict(X), y, atol=1e-4) + + +def test_consistent_lengths(): + X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) + y = np.array([0, 0, 1, 1]) + sample_weight = np.array([0.1, 0.3, 0.1]) + gbdt = HistGradientBoostingRegressor() + with pytest.raises(ValueError, match=r"sample_weight.shape == \(3,\), expected"): + gbdt.fit(X, y, sample_weight) + + with pytest.raises( + ValueError, match="Found input variables with inconsistent number" + ): + gbdt.fit(X, y[1:]) + + +def test_infinite_values_missing_values(): + # High level test making sure that inf and nan values are properly handled + # when both are present. This is similar to + # test_split_on_nan_with_infinite_values() in test_grower.py, though we + # cannot check the predictions for binned values here. + + X = np.asarray([-np.inf, 0, 1, np.inf, np.nan]).reshape(-1, 1) + y_isnan = np.isnan(X.ravel()) + y_isinf = X.ravel() == np.inf + + stump_clf = HistGradientBoostingClassifier( + min_samples_leaf=1, max_iter=1, learning_rate=1, max_depth=2 + ) + + assert stump_clf.fit(X, y_isinf).score(X, y_isinf) == 1 + assert stump_clf.fit(X, y_isnan).score(X, y_isnan) == 1 + + +@pytest.mark.parametrize("scoring", [None, "loss"]) +def test_string_target_early_stopping(scoring): + # Regression tests for #14709 where the targets need to be encoded before + # to compute the score + rng = np.random.RandomState(42) + X = rng.randn(100, 10) + y = np.array(["x"] * 50 + ["y"] * 50, dtype=object) + gbrt = HistGradientBoostingClassifier(n_iter_no_change=10, scoring=scoring) + gbrt.fit(X, y) + + +def test_zero_sample_weights_regression(): + # Make sure setting a SW to zero amounts to ignoring the corresponding + # sample + + X = [[1, 0], [1, 0], [1, 0], [0, 1]] + y = [0, 0, 1, 0] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1] + gb = HistGradientBoostingRegressor(min_samples_leaf=1) + gb.fit(X, y, sample_weight=sample_weight) + assert gb.predict([[1, 0]])[0] > 0.5 + + +def test_zero_sample_weights_classification(): + # Make sure setting a SW to zero amounts to ignoring the corresponding + # sample + + X = [[1, 0], [1, 0], [1, 0], [0, 1]] + y = [0, 0, 1, 0] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1] + gb = HistGradientBoostingClassifier(loss="log_loss", min_samples_leaf=1) + gb.fit(X, y, sample_weight=sample_weight) + assert_array_equal(gb.predict([[1, 0]]), [1]) + + X = [[1, 0], [1, 0], [1, 0], [0, 1], [1, 1]] + y = [0, 0, 1, 0, 2] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1, 1] + gb = HistGradientBoostingClassifier(loss="log_loss", min_samples_leaf=1) + gb.fit(X, y, sample_weight=sample_weight) + assert_array_equal(gb.predict([[1, 0]]), [1]) + + +@pytest.mark.parametrize( + "problem", ("regression", "binary_classification", "multiclass_classification") +) +@pytest.mark.parametrize("duplication", ("half", "all")) +def test_sample_weight_effect(problem, duplication): + # High level test to make sure that duplicating a sample is equivalent to + # giving it weight of 2. + + # fails for n_samples > 255 because binning does not take sample weights + # into account. Keeping n_samples <= 255 makes + # sure only unique values are used so SW have no effect on binning. + n_samples = 255 + n_features = 2 + if problem == "regression": + X, y = make_regression( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + random_state=0, + ) + Klass = HistGradientBoostingRegressor + else: + n_classes = 2 if problem == "binary_classification" else 3 + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + n_redundant=0, + n_clusters_per_class=1, + n_classes=n_classes, + random_state=0, + ) + Klass = HistGradientBoostingClassifier + + # This test can't pass if min_samples_leaf > 1 because that would force 2 + # samples to be in the same node in est_sw, while these samples would be + # free to be separate in est_dup: est_dup would just group together the + # duplicated samples. + est = Klass(min_samples_leaf=1) + + # Create dataset with duplicate and corresponding sample weights + if duplication == "half": + lim = n_samples // 2 + else: + lim = n_samples + X_dup = np.r_[X, X[:lim]] + y_dup = np.r_[y, y[:lim]] + sample_weight = np.ones(shape=(n_samples)) + sample_weight[:lim] = 2 + + est_sw = clone(est).fit(X, y, sample_weight=sample_weight) + est_dup = clone(est).fit(X_dup, y_dup) + + # checking raw_predict is stricter than just predict for classification + assert np.allclose(est_sw._raw_predict(X_dup), est_dup._raw_predict(X_dup)) + + +@pytest.mark.parametrize("Loss", (HalfSquaredError, AbsoluteError)) +def test_sum_hessians_are_sample_weight(Loss): + # For losses with constant hessians, the sum_hessians field of the + # histograms must be equal to the sum of the sample weight of samples at + # the corresponding bin. + + rng = np.random.RandomState(0) + n_samples = 1000 + n_features = 2 + X, y = make_regression(n_samples=n_samples, n_features=n_features, random_state=rng) + bin_mapper = _BinMapper() + X_binned = bin_mapper.fit_transform(X) + + # While sample weights are supposed to be positive, this still works. + sample_weight = rng.normal(size=n_samples) + + loss = Loss(sample_weight=sample_weight) + gradients, hessians = loss.init_gradient_and_hessian( + n_samples=n_samples, dtype=G_H_DTYPE + ) + gradients, hessians = gradients.reshape((-1, 1)), hessians.reshape((-1, 1)) + raw_predictions = rng.normal(size=(n_samples, 1)) + loss.gradient_hessian( + y_true=y, + raw_prediction=raw_predictions, + sample_weight=sample_weight, + gradient_out=gradients, + hessian_out=hessians, + n_threads=n_threads, + ) + + # build sum_sample_weight which contains the sum of the sample weights at + # each bin (for each feature). This must be equal to the sum_hessians + # field of the corresponding histogram + sum_sw = np.zeros(shape=(n_features, bin_mapper.n_bins)) + for feature_idx in range(n_features): + for sample_idx in range(n_samples): + sum_sw[feature_idx, X_binned[sample_idx, feature_idx]] += sample_weight[ + sample_idx + ] + + # Build histogram + grower = TreeGrower( + X_binned, gradients[:, 0], hessians[:, 0], n_bins=bin_mapper.n_bins + ) + histograms = grower.histogram_builder.compute_histograms_brute( + grower.root.sample_indices + ) + + for feature_idx in range(n_features): + for bin_idx in range(bin_mapper.n_bins): + assert histograms[feature_idx, bin_idx]["sum_hessians"] == ( + pytest.approx(sum_sw[feature_idx, bin_idx], rel=1e-5) + ) + + +def test_max_depth_max_leaf_nodes(): + # Non regression test for + # https://github.com/scikit-learn/scikit-learn/issues/16179 + # there was a bug when the max_depth and the max_leaf_nodes criteria were + # met at the same time, which would lead to max_leaf_nodes not being + # respected. + X, y = make_classification(random_state=0) + est = HistGradientBoostingClassifier(max_depth=2, max_leaf_nodes=3, max_iter=1).fit( + X, y + ) + tree = est._predictors[0][0] + assert tree.get_max_depth() == 2 + assert tree.get_n_leaf_nodes() == 3 # would be 4 prior to bug fix + + +def test_early_stopping_on_test_set_with_warm_start(): + # Non regression test for #16661 where second fit fails with + # warm_start=True, early_stopping is on, and no validation set + X, y = make_classification(random_state=0) + gb = HistGradientBoostingClassifier( + max_iter=1, + scoring="loss", + warm_start=True, + early_stopping=True, + n_iter_no_change=1, + validation_fraction=None, + ) + + gb.fit(X, y) + # does not raise on second call + gb.set_params(max_iter=2) + gb.fit(X, y) + + +def test_early_stopping_with_sample_weights(monkeypatch): + """Check that sample weights is passed in to the scorer and _raw_predict is not + called.""" + + mock_scorer = Mock(side_effect=get_scorer("neg_median_absolute_error")) + + def mock_check_scoring(estimator, scoring): + assert scoring == "neg_median_absolute_error" + return mock_scorer + + monkeypatch.setattr( + sklearn.ensemble._hist_gradient_boosting.gradient_boosting, + "check_scoring", + mock_check_scoring, + ) + + X, y = make_regression(random_state=0) + sample_weight = np.ones_like(y) + hist = HistGradientBoostingRegressor( + max_iter=2, + early_stopping=True, + random_state=0, + scoring="neg_median_absolute_error", + ) + mock_raw_predict = Mock(side_effect=hist._raw_predict) + hist._raw_predict = mock_raw_predict + hist.fit(X, y, sample_weight=sample_weight) + + # _raw_predict should never be called with scoring as a string + assert mock_raw_predict.call_count == 0 + + # For scorer is called twice (train and val) for the baseline score, and twice + # per iteration (train and val) after that. So 6 times in total for `max_iter=2`. + assert mock_scorer.call_count == 6 + for arg_list in mock_scorer.call_args_list: + assert "sample_weight" in arg_list[1] + + +def test_raw_predict_is_called_with_custom_scorer(): + """Custom scorer will still call _raw_predict.""" + + mock_scorer = Mock(side_effect=get_scorer("neg_median_absolute_error")) + + X, y = make_regression(random_state=0) + hist = HistGradientBoostingRegressor( + max_iter=2, + early_stopping=True, + random_state=0, + scoring=mock_scorer, + ) + mock_raw_predict = Mock(side_effect=hist._raw_predict) + hist._raw_predict = mock_raw_predict + hist.fit(X, y) + + # `_raw_predict` and scorer is called twice (train and val) for the baseline score, + # and twice per iteration (train and val) after that. So 6 times in total for + # `max_iter=2`. + assert mock_raw_predict.call_count == 6 + assert mock_scorer.call_count == 6 + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +def test_single_node_trees(Est): + # Make sure it's still possible to build single-node trees. In that case + # the value of the root is set to 0. That's a correct value: if the tree is + # single-node that's because min_gain_to_split is not respected right from + # the root, so we don't want the tree to have any impact on the + # predictions. + + X, y = make_classification(random_state=0) + y[:] = 1 # constant target will lead to a single root node + + est = Est(max_iter=20) + est.fit(X, y) + + assert all(len(predictor[0].nodes) == 1 for predictor in est._predictors) + assert all(predictor[0].nodes[0]["value"] == 0 for predictor in est._predictors) + # Still gives correct predictions thanks to the baseline prediction + assert_allclose(est.predict(X), y) + + +@pytest.mark.parametrize( + "Est, loss, X, y", + [ + ( + HistGradientBoostingClassifier, + HalfBinomialLoss(sample_weight=None), + X_classification, + y_classification, + ), + ( + HistGradientBoostingRegressor, + HalfSquaredError(sample_weight=None), + X_regression, + y_regression, + ), + ], +) +def test_custom_loss(Est, loss, X, y): + est = Est(loss=loss, max_iter=20) + est.fit(X, y) + + +@pytest.mark.parametrize( + "HistGradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ( + HistGradientBoostingClassifier, + X_multi_classification, + y_multi_classification, + ), + ], +) +def test_staged_predict(HistGradientBoosting, X, y): + # Test whether staged predictor eventually gives + # the same prediction. + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.5, random_state=0 + ) + gb = HistGradientBoosting(max_iter=10) + + # test raise NotFittedError if not fitted + with pytest.raises(NotFittedError): + next(gb.staged_predict(X_test)) + + gb.fit(X_train, y_train) + + # test if the staged predictions of each iteration + # are equal to the corresponding predictions of the same estimator + # trained from scratch. + # this also test limit case when max_iter = 1 + method_names = ( + ["predict"] + if is_regressor(gb) + else ["predict", "predict_proba", "decision_function"] + ) + for method_name in method_names: + staged_method = getattr(gb, "staged_" + method_name) + staged_predictions = list(staged_method(X_test)) + assert len(staged_predictions) == gb.n_iter_ + for n_iter, staged_predictions in enumerate(staged_method(X_test), 1): + aux = HistGradientBoosting(max_iter=n_iter) + aux.fit(X_train, y_train) + pred_aux = getattr(aux, method_name)(X_test) + + assert_allclose(staged_predictions, pred_aux) + assert staged_predictions.shape == pred_aux.shape + + +@pytest.mark.parametrize("insert_missing", [False, True]) +@pytest.mark.parametrize( + "Est", (HistGradientBoostingRegressor, HistGradientBoostingClassifier) +) +@pytest.mark.parametrize("bool_categorical_parameter", [True, False]) +@pytest.mark.parametrize("missing_value", [np.nan, -1]) +def test_unknown_categories_nan( + insert_missing, Est, bool_categorical_parameter, missing_value +): + # Make sure no error is raised at predict if a category wasn't seen during + # fit. We also make sure they're treated as nans. + + rng = np.random.RandomState(0) + n_samples = 1000 + f1 = rng.rand(n_samples) + f2 = rng.randint(4, size=n_samples) + X = np.c_[f1, f2] + y = np.zeros(shape=n_samples) + y[X[:, 1] % 2 == 0] = 1 + + if bool_categorical_parameter: + categorical_features = [False, True] + else: + categorical_features = [1] + + if insert_missing: + mask = rng.binomial(1, 0.01, size=X.shape).astype(bool) + assert mask.sum() > 0 + X[mask] = missing_value + + est = Est(max_iter=20, categorical_features=categorical_features).fit(X, y) + assert_array_equal(est.is_categorical_, [False, True]) + + # Make sure no error is raised on unknown categories and nans + # unknown categories will be treated as nans + X_test = np.zeros((10, X.shape[1]), dtype=float) + X_test[:5, 1] = 30 + X_test[5:, 1] = missing_value + assert len(np.unique(est.predict(X_test))) == 1 + + +def test_categorical_encoding_strategies(): + # Check native categorical handling vs different encoding strategies. We + # make sure that native encoding needs only 1 split to achieve a perfect + # prediction on a simple dataset. In contrast, OneHotEncoded data needs + # more depth / splits, and treating categories as ordered (just using + # OrdinalEncoder) requires even more depth. + + # dataset with one random continuous feature, and one categorical feature + # with values in [0, 5], e.g. from an OrdinalEncoder. + # class == 1 iff categorical value in {0, 2, 4} + rng = np.random.RandomState(0) + n_samples = 10_000 + f1 = rng.rand(n_samples) + f2 = rng.randint(6, size=n_samples) + X = np.c_[f1, f2] + y = np.zeros(shape=n_samples) + y[X[:, 1] % 2 == 0] = 1 + + # make sure dataset is balanced so that the baseline_prediction doesn't + # influence predictions too much with max_iter = 1 + assert 0.49 < y.mean() < 0.51 + + native_cat_specs = [ + [False, True], + [1], + ] + try: + import pandas as pd + + X = pd.DataFrame(X, columns=["f_0", "f_1"]) + native_cat_specs.append(["f_1"]) + except ImportError: + pass + + for native_cat_spec in native_cat_specs: + clf_cat = HistGradientBoostingClassifier( + max_iter=1, max_depth=1, categorical_features=native_cat_spec + ) + clf_cat.fit(X, y) + + # Using native categorical encoding, we get perfect predictions with just + # one split + assert cross_val_score(clf_cat, X, y).mean() == 1 + + # quick sanity check for the bitset: 0, 2, 4 = 2**0 + 2**2 + 2**4 = 21 + expected_left_bitset = [21, 0, 0, 0, 0, 0, 0, 0] + left_bitset = clf_cat.fit(X, y)._predictors[0][0].raw_left_cat_bitsets[0] + assert_array_equal(left_bitset, expected_left_bitset) + + # Treating categories as ordered, we need more depth / more splits to get + # the same predictions + clf_no_cat = HistGradientBoostingClassifier( + max_iter=1, max_depth=4, categorical_features=None + ) + assert cross_val_score(clf_no_cat, X, y).mean() < 0.9 + + clf_no_cat.set_params(max_depth=5) + assert cross_val_score(clf_no_cat, X, y).mean() == 1 + + # Using OHEd data, we need less splits than with pure OEd data, but we + # still need more splits than with the native categorical splits + ct = make_column_transformer( + (OneHotEncoder(sparse_output=False), [1]), remainder="passthrough" + ) + X_ohe = ct.fit_transform(X) + clf_no_cat.set_params(max_depth=2) + assert cross_val_score(clf_no_cat, X_ohe, y).mean() < 0.9 + + clf_no_cat.set_params(max_depth=3) + assert cross_val_score(clf_no_cat, X_ohe, y).mean() == 1 + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +@pytest.mark.parametrize( + "categorical_features, monotonic_cst, expected_msg", + [ + ( + [b"hello", b"world"], + None, + re.escape( + "categorical_features must be an array-like of bool, int or str, " + "got: bytes40." + ), + ), + ( + np.array([b"hello", 1.3], dtype=object), + None, + re.escape( + "categorical_features must be an array-like of bool, int or str, " + "got: bytes, float." + ), + ), + ( + [0, -1], + None, + re.escape( + "categorical_features set as integer indices must be in " + "[0, n_features - 1]" + ), + ), + ( + [True, True, False, False, True], + None, + re.escape( + "categorical_features set as a boolean mask must have shape " + "(n_features,)" + ), + ), + ( + [True, True, False, False], + [0, -1, 0, 1], + "Categorical features cannot have monotonic constraints", + ), + ], +) +def test_categorical_spec_errors( + Est, categorical_features, monotonic_cst, expected_msg +): + # Test errors when categories are specified incorrectly + n_samples = 100 + X, y = make_classification(random_state=0, n_features=4, n_samples=n_samples) + rng = np.random.RandomState(0) + X[:, 0] = rng.randint(0, 10, size=n_samples) + X[:, 1] = rng.randint(0, 10, size=n_samples) + est = Est(categorical_features=categorical_features, monotonic_cst=monotonic_cst) + + with pytest.raises(ValueError, match=expected_msg): + est.fit(X, y) + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +def test_categorical_spec_errors_with_feature_names(Est): + pd = pytest.importorskip("pandas") + n_samples = 10 + X = pd.DataFrame( + { + "f0": range(n_samples), + "f1": range(n_samples), + "f2": [1.0] * n_samples, + } + ) + y = [0, 1] * (n_samples // 2) + + est = Est(categorical_features=["f0", "f1", "f3"]) + expected_msg = re.escape( + "categorical_features has an item value 'f3' which is not a valid " + "feature name of the training data." + ) + with pytest.raises(ValueError, match=expected_msg): + est.fit(X, y) + + est = Est(categorical_features=["f0", "f1"]) + expected_msg = re.escape( + "categorical_features should be passed as an array of integers or " + "as a boolean mask when the model is fitted on data without feature " + "names." + ) + with pytest.raises(ValueError, match=expected_msg): + est.fit(X.to_numpy(), y) + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +@pytest.mark.parametrize("categorical_features", ([False, False], [])) +@pytest.mark.parametrize("as_array", (True, False)) +def test_categorical_spec_no_categories(Est, categorical_features, as_array): + # Make sure we can properly detect that no categorical features are present + # even if the categorical_features parameter is not None + X = np.arange(10).reshape(5, 2) + y = np.arange(5) + if as_array: + categorical_features = np.asarray(categorical_features) + est = Est(categorical_features=categorical_features).fit(X, y) + assert est.is_categorical_ is None + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +@pytest.mark.parametrize( + "use_pandas, feature_name", [(False, "at index 0"), (True, "'f0'")] +) +def test_categorical_bad_encoding_errors(Est, use_pandas, feature_name): + # Test errors when categories are encoded incorrectly + + gb = Est(categorical_features=[True], max_bins=2) + + if use_pandas: + pd = pytest.importorskip("pandas") + X = pd.DataFrame({"f0": [0, 1, 2]}) + else: + X = np.array([[0, 1, 2]]).T + y = np.arange(3) + msg = ( + f"Categorical feature {feature_name} is expected to have a " + "cardinality <= 2 but actually has a cardinality of 3." + ) + with pytest.raises(ValueError, match=msg): + gb.fit(X, y) + + # nans are ignored in the counts + X = np.array([[0, 1, np.nan]]).T + y = np.arange(3) + gb.fit(X, y) + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +def test_uint8_predict(Est): + # Non regression test for + # https://github.com/scikit-learn/scikit-learn/issues/18408 + # Make sure X can be of dtype uint8 (i.e. X_BINNED_DTYPE) in predict. It + # will be converted to X_DTYPE. + + rng = np.random.RandomState(0) + + X = rng.randint(0, 100, size=(10, 2)).astype(np.uint8) + y = rng.randint(0, 2, size=10).astype(np.uint8) + est = Est() + est.fit(X, y) + est.predict(X) + + +@pytest.mark.parametrize( + "interaction_cst, n_features, result", + [ + (None, 931, None), + ([{0, 1}], 2, [{0, 1}]), + ("pairwise", 2, [{0, 1}]), + ("pairwise", 4, [{0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}]), + ("no_interactions", 2, [{0}, {1}]), + ("no_interactions", 4, [{0}, {1}, {2}, {3}]), + ([(1, 0), [5, 1]], 6, [{0, 1}, {1, 5}, {2, 3, 4}]), + ], +) +def test_check_interaction_cst(interaction_cst, n_features, result): + """Check that _check_interaction_cst returns the expected list of sets""" + est = HistGradientBoostingRegressor() + est.set_params(interaction_cst=interaction_cst) + assert est._check_interaction_cst(n_features) == result + + +def test_interaction_cst_numerically(): + """Check that interaction constraints have no forbidden interactions.""" + rng = np.random.RandomState(42) + n_samples = 1000 + X = rng.uniform(size=(n_samples, 2)) + # Construct y with a strong interaction term + # y = x0 + x1 + 5 * x0 * x1 + y = np.hstack((X, 5 * X[:, [0]] * X[:, [1]])).sum(axis=1) + + est = HistGradientBoostingRegressor(random_state=42) + est.fit(X, y) + est_no_interactions = HistGradientBoostingRegressor( + interaction_cst=[{0}, {1}], random_state=42 + ) + est_no_interactions.fit(X, y) + + delta = 0.25 + # Make sure we do not extrapolate out of the training set as tree-based estimators + # are very bad in doing so. + X_test = X[(X[:, 0] < 1 - delta) & (X[:, 1] < 1 - delta)] + X_delta_d_0 = X_test + [delta, 0] + X_delta_0_d = X_test + [0, delta] + X_delta_d_d = X_test + [delta, delta] + + # Note: For the y from above as a function of x0 and x1, we have + # y(x0+d, x1+d) = y(x0, x1) + 5 * d * (2/5 + x0 + x1) + 5 * d**2 + # y(x0+d, x1) = y(x0, x1) + 5 * d * (1/5 + x1) + # y(x0, x1+d) = y(x0, x1) + 5 * d * (1/5 + x0) + # Without interaction constraints, we would expect a result of 5 * d**2 for the + # following expression, but zero with constraints in place. + assert_allclose( + est_no_interactions.predict(X_delta_d_d) + + est_no_interactions.predict(X_test) + - est_no_interactions.predict(X_delta_d_0) + - est_no_interactions.predict(X_delta_0_d), + 0, + atol=1e-12, + ) + + # Correct result of the expressions is 5 * delta**2. But this is hard to achieve by + # a fitted tree-based model. However, with 100 iterations the expression should + # at least be positive! + assert np.all( + est.predict(X_delta_d_d) + + est.predict(X_test) + - est.predict(X_delta_d_0) + - est.predict(X_delta_0_d) + > 0.01 + ) + + +def test_no_user_warning_with_scoring(): + """Check that no UserWarning is raised when scoring is set. + + Non-regression test for #22907. + """ + pd = pytest.importorskip("pandas") + X, y = make_regression(n_samples=50, random_state=0) + X_df = pd.DataFrame(X, columns=[f"col{i}" for i in range(X.shape[1])]) + + est = HistGradientBoostingRegressor( + random_state=0, scoring="neg_mean_absolute_error", early_stopping=True + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + est.fit(X_df, y) + + +def test_class_weights(): + """High level test to check class_weights.""" + n_samples = 255 + n_features = 2 + + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + n_redundant=0, + n_clusters_per_class=1, + n_classes=2, + random_state=0, + ) + y_is_1 = y == 1 + + # class_weight is the same as sample weights with the corresponding class + clf = HistGradientBoostingClassifier( + min_samples_leaf=2, random_state=0, max_depth=2 + ) + sample_weight = np.ones(shape=(n_samples)) + sample_weight[y_is_1] = 3.0 + clf.fit(X, y, sample_weight=sample_weight) + + class_weight = {0: 1.0, 1: 3.0} + clf_class_weighted = clone(clf).set_params(class_weight=class_weight) + clf_class_weighted.fit(X, y) + + assert_allclose(clf.decision_function(X), clf_class_weighted.decision_function(X)) + + # Check that sample_weight and class_weight are multiplicative + clf.fit(X, y, sample_weight=sample_weight**2) + clf_class_weighted.fit(X, y, sample_weight=sample_weight) + assert_allclose(clf.decision_function(X), clf_class_weighted.decision_function(X)) + + # Make imbalanced dataset + X_imb = np.concatenate((X[~y_is_1], X[y_is_1][:10])) + y_imb = np.concatenate((y[~y_is_1], y[y_is_1][:10])) + + # class_weight="balanced" is the same as sample_weights to be + # inversely proportional to n_samples / (n_classes * np.bincount(y)) + clf_balanced = clone(clf).set_params(class_weight="balanced") + clf_balanced.fit(X_imb, y_imb) + + class_weight = y_imb.shape[0] / (2 * np.bincount(y_imb)) + sample_weight = class_weight[y_imb] + clf_sample_weight = clone(clf).set_params(class_weight=None) + clf_sample_weight.fit(X_imb, y_imb, sample_weight=sample_weight) + + assert_allclose( + clf_balanced.decision_function(X_imb), + clf_sample_weight.decision_function(X_imb), + ) + + +def test_unknown_category_that_are_negative(): + """Check that unknown categories that are negative does not error. + + Non-regression test for #24274. + """ + rng = np.random.RandomState(42) + n_samples = 1000 + X = np.c_[rng.rand(n_samples), rng.randint(4, size=n_samples)] + y = np.zeros(shape=n_samples) + y[X[:, 1] % 2 == 0] = 1 + + hist = HistGradientBoostingRegressor( + random_state=0, + categorical_features=[False, True], + max_iter=10, + ).fit(X, y) + + # Check that negative values from the second column are treated like a + # missing category + X_test_neg = np.asarray([[1, -2], [3, -4]]) + X_test_nan = np.asarray([[1, np.nan], [3, np.nan]]) + + assert_allclose(hist.predict(X_test_neg), hist.predict(X_test_nan)) + + +@pytest.mark.parametrize( + ("GradientBoosting", "make_X_y"), + [ + (HistGradientBoostingClassifier, make_classification), + (HistGradientBoostingRegressor, make_regression), + ], +) +@pytest.mark.parametrize("sample_weight", [False, True]) +def test_X_val_in_fit(GradientBoosting, make_X_y, sample_weight, global_random_seed): + """Test that passing X_val, y_val in fit is same as validation fraction.""" + rng = np.random.RandomState(42) + n_samples = 100 + X, y = make_X_y(n_samples=n_samples, random_state=rng) + if sample_weight: + sample_weight = np.abs(rng.normal(size=n_samples)) + data = (X, y, sample_weight) + else: + sample_weight = None + data = (X, y) + rng_seed = global_random_seed + + # Fit with validation fraction and early stopping. + m1 = GradientBoosting( + early_stopping=True, + validation_fraction=0.5, + random_state=rng_seed, + ) + m1.fit(X, y, sample_weight) + + # Do train-test split ourselves. + rng = check_random_state(rng_seed) + # We do the same as in the fit method. + stratify = y if isinstance(m1, HistGradientBoostingClassifier) else None + random_seed = rng.randint(np.iinfo(np.uint32).max, dtype="u8") + X_train, X_val, y_train, y_val, *sw = train_test_split( + *data, + test_size=0.5, + stratify=stratify, + random_state=random_seed, + ) + if sample_weight is not None: + sample_weight_train = sw[0] + sample_weight_val = sw[1] + else: + sample_weight_train = None + sample_weight_val = None + m2 = GradientBoosting( + early_stopping=True, + random_state=rng_seed, + ) + m2.fit( + X_train, + y_train, + sample_weight=sample_weight_train, + X_val=X_val, + y_val=y_val, + sample_weight_val=sample_weight_val, + ) + + assert_allclose(m2.n_iter_, m1.n_iter_) + assert_allclose(m2.predict(X), m1.predict(X)) + + +def test_X_val_raises_missing_y_val(): + """Test that an error is raised if X_val given but y_val None.""" + X, y = make_classification(n_samples=4) + X, X_val = X[:2], X[2:] + y, y_val = y[:2], y[2:] + with pytest.raises( + ValueError, + match="X_val is provided, but y_val was not provided", + ): + HistGradientBoostingClassifier().fit(X, y, X_val=X_val) + with pytest.raises( + ValueError, + match="y_val is provided, but X_val was not provided", + ): + HistGradientBoostingClassifier().fit(X, y, y_val=y_val) + + +def test_X_val_raises_with_early_stopping_false(): + """Test that an error is raised if X_val given but early_stopping is False.""" + X, y = make_regression(n_samples=4) + X, X_val = X[:2], X[2:] + y, y_val = y[:2], y[2:] + with pytest.raises( + ValueError, + match="X_val and y_val are passed to fit while at the same time", + ): + HistGradientBoostingRegressor(early_stopping=False).fit( + X, y, X_val=X_val, y_val=y_val + ) + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +@pytest.mark.parametrize( + "HistGradientBoosting", + [HistGradientBoostingClassifier, HistGradientBoostingRegressor], +) +def test_dataframe_categorical_results_same_as_ndarray( + dataframe_lib, HistGradientBoosting +): + """Check that pandas categorical give the same results as ndarray.""" + pytest.importorskip(dataframe_lib) + + rng = np.random.RandomState(42) + n_samples = 5_000 + n_cardinality = 50 + max_bins = 100 + f_num = rng.rand(n_samples) + f_cat = rng.randint(n_cardinality, size=n_samples) + + # Make f_cat an informative feature + y = (f_cat % 3 == 0) & (f_num > 0.2) + + X = np.c_[f_num, f_cat] + f_cat = [f"cat{c:0>3}" for c in f_cat] + X_df = _convert_container( + np.asarray([f_num, f_cat]).T, + dataframe_lib, + ["f_num", "f_cat"], + categorical_feature_names=["f_cat"], + ) + + X_train, X_test, X_train_df, X_test_df, y_train, y_test = train_test_split( + X, X_df, y, random_state=0 + ) + + hist_kwargs = dict(max_iter=10, max_bins=max_bins, random_state=0) + hist_np = HistGradientBoosting(categorical_features=[False, True], **hist_kwargs) + hist_np.fit(X_train, y_train) + + hist_pd = HistGradientBoosting(categorical_features="from_dtype", **hist_kwargs) + hist_pd.fit(X_train_df, y_train) + + # Check categories are correct and sorted + categories = hist_pd._preprocessor.named_transformers_["encoder"].categories_[0] + assert_array_equal(categories, np.unique(f_cat)) + + assert len(hist_np._predictors) == len(hist_pd._predictors) + for predictor_1, predictor_2 in zip(hist_np._predictors, hist_pd._predictors): + assert len(predictor_1[0].nodes) == len(predictor_2[0].nodes) + + score_np = hist_np.score(X_test, y_test) + score_pd = hist_pd.score(X_test_df, y_test) + assert score_np == pytest.approx(score_pd) + assert_allclose(hist_np.predict(X_test), hist_pd.predict(X_test_df)) + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +@pytest.mark.parametrize( + "HistGradientBoosting", + [HistGradientBoostingClassifier, HistGradientBoostingRegressor], +) +def test_dataframe_categorical_errors(dataframe_lib, HistGradientBoosting): + """Check error cases for pandas categorical feature.""" + pytest.importorskip(dataframe_lib) + msg = "Categorical feature 'f_cat' is expected to have a cardinality <= 16" + hist = HistGradientBoosting(categorical_features="from_dtype", max_bins=16) + + rng = np.random.RandomState(42) + f_cat = rng.randint(0, high=100, size=100).astype(str) + X_df = _convert_container( + f_cat[:, None], dataframe_lib, ["f_cat"], categorical_feature_names=["f_cat"] + ) + y = rng.randint(0, high=2, size=100) + + with pytest.raises(ValueError, match=msg): + hist.fit(X_df, y) + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +def test_categorical_different_order_same_model(dataframe_lib): + """Check that the order of the categorical gives same model.""" + pytest.importorskip(dataframe_lib) + rng = np.random.RandomState(42) + n_samples = 1_000 + f_ints = rng.randint(low=0, high=2, size=n_samples) + + # Construct a target with some noise + y = f_ints.copy() + flipped = rng.choice([True, False], size=n_samples, p=[0.1, 0.9]) + y[flipped] = 1 - y[flipped] + + # Construct categorical where 0 -> A and 1 -> B and 1 -> A and 0 -> B + f_cat_a_b = np.asarray(["A", "B"])[f_ints] + f_cat_b_a = np.asarray(["B", "A"])[f_ints] + df_a_b = _convert_container( + f_cat_a_b[:, None], + dataframe_lib, + ["f_cat"], + categorical_feature_names=["f_cat"], + ) + df_b_a = _convert_container( + f_cat_b_a[:, None], + dataframe_lib, + ["f_cat"], + categorical_feature_names=["f_cat"], + ) + + hist_a_b = HistGradientBoostingClassifier( + categorical_features="from_dtype", random_state=0 + ) + hist_b_a = HistGradientBoostingClassifier( + categorical_features="from_dtype", random_state=0 + ) + + hist_a_b.fit(df_a_b, y) + hist_b_a.fit(df_b_a, y) + + assert len(hist_a_b._predictors) == len(hist_b_a._predictors) + for predictor_1, predictor_2 in zip(hist_a_b._predictors, hist_b_a._predictors): + assert len(predictor_1[0].nodes) == len(predictor_2[0].nodes) + + +def get_different_bitness_node_ndarray(node_ndarray): + new_dtype_for_indexing_fields = np.int64 if _IS_32BIT else np.int32 + + # field names in Node struct with np.intp types (see + # sklearn/ensemble/_hist_gradient_boosting/common.pyx) + indexing_field_names = ["feature_idx"] + + new_dtype_dict = { + name: dtype for name, (dtype, _) in node_ndarray.dtype.fields.items() + } + for name in indexing_field_names: + new_dtype_dict[name] = new_dtype_for_indexing_fields + + new_dtype = np.dtype( + {"names": list(new_dtype_dict.keys()), "formats": list(new_dtype_dict.values())} + ) + return node_ndarray.astype(new_dtype, casting="same_kind") + + +def reduce_predictor_with_different_bitness(predictor): + cls, args, state = predictor.__reduce__() + + new_state = state.copy() + new_state["nodes"] = get_different_bitness_node_ndarray(new_state["nodes"]) + + return (cls, args, new_state) + + +def test_different_bitness_pickle(): + X, y = make_classification(random_state=0) + + clf = HistGradientBoostingClassifier(random_state=0, max_depth=3) + clf.fit(X, y) + score = clf.score(X, y) + + def pickle_dump_with_different_bitness(): + f = io.BytesIO() + p = pickle.Pickler(f) + p.dispatch_table = copyreg.dispatch_table.copy() + p.dispatch_table[TreePredictor] = reduce_predictor_with_different_bitness + + p.dump(clf) + f.seek(0) + return f + + # Simulate loading a pickle of the same model trained on a platform with different + # bitness that than the platform it will be used to make predictions on: + new_clf = pickle.load(pickle_dump_with_different_bitness()) + new_score = new_clf.score(X, y) + assert score == pytest.approx(new_score) + + +def test_different_bitness_joblib_pickle(): + # Make sure that a platform specific pickle generated on a 64 bit + # platform can be converted at pickle load time into an estimator + # with Cython code that works with the host's native integer precision + # to index nodes in the tree data structure when the host is a 32 bit + # platform (and vice versa). + # + # This is in particular useful to be able to train a model on a 64 bit Linux + # server and deploy the model as part of a (32 bit) WASM in-browser + # application using pyodide. + X, y = make_classification(random_state=0) + + clf = HistGradientBoostingClassifier(random_state=0, max_depth=3) + clf.fit(X, y) + score = clf.score(X, y) + + def joblib_dump_with_different_bitness(): + f = io.BytesIO() + p = NumpyPickler(f) + p.dispatch_table = copyreg.dispatch_table.copy() + p.dispatch_table[TreePredictor] = reduce_predictor_with_different_bitness + + p.dump(clf) + f.seek(0) + return f + + new_clf = joblib.load(joblib_dump_with_different_bitness()) + new_score = new_clf.score(X, y) + assert score == pytest.approx(new_score) + + +def test_pandas_nullable_dtype(): + # Non regression test for https://github.com/scikit-learn/scikit-learn/issues/28317 + pd = pytest.importorskip("pandas") + + rng = np.random.default_rng(0) + X = pd.DataFrame({"a": rng.integers(10, size=100)}).astype(pd.Int64Dtype()) + y = rng.integers(2, size=100) + + clf = HistGradientBoostingClassifier() + clf.fit(X, y) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py new file mode 100644 index 0000000000000000000000000000000000000000..bbbdda13cb94347e4e6c91ef42d258df4cca1fc4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py @@ -0,0 +1,650 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal +from pytest import approx + +from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper +from sklearn.ensemble._hist_gradient_boosting.common import ( + G_H_DTYPE, + X_BINNED_DTYPE, + X_BITSET_INNER_DTYPE, + X_DTYPE, + Y_DTYPE, +) +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +from sklearn.preprocessing import OneHotEncoder +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + +n_threads = _openmp_effective_n_threads() + + +def _make_training_data(n_bins=256, constant_hessian=True): + rng = np.random.RandomState(42) + n_samples = 10000 + + # Generate some test data directly binned so as to test the grower code + # independently of the binning logic. + X_binned = rng.randint(0, n_bins - 1, size=(n_samples, 2), dtype=X_BINNED_DTYPE) + X_binned = np.asfortranarray(X_binned) + + def true_decision_function(input_features): + """Ground truth decision function + + This is a very simple yet asymmetric decision tree. Therefore the + grower code should have no trouble recovering the decision function + from 10000 training samples. + """ + if input_features[0] <= n_bins // 2: + return -1 + else: + return -1 if input_features[1] <= n_bins // 3 else 1 + + target = np.array([true_decision_function(x) for x in X_binned], dtype=Y_DTYPE) + + # Assume a square loss applied to an initial model that always predicts 0 + # (hardcoded for this test): + all_gradients = target.astype(G_H_DTYPE) + shape_hessians = 1 if constant_hessian else all_gradients.shape + all_hessians = np.ones(shape=shape_hessians, dtype=G_H_DTYPE) + + return X_binned, all_gradients, all_hessians + + +def _check_children_consistency(parent, left, right): + # Make sure the samples are correctly dispatched from a parent to its + # children + assert parent.left_child is left + assert parent.right_child is right + + # each sample from the parent is propagated to one of the two children + assert len(left.sample_indices) + len(right.sample_indices) == len( + parent.sample_indices + ) + + assert set(left.sample_indices).union(set(right.sample_indices)) == set( + parent.sample_indices + ) + + # samples are sent either to the left or the right node, never to both + assert set(left.sample_indices).intersection(set(right.sample_indices)) == set() + + +@pytest.mark.parametrize( + "n_bins, constant_hessian, stopping_param, shrinkage", + [ + (11, True, "min_gain_to_split", 0.5), + (11, False, "min_gain_to_split", 1.0), + (11, True, "max_leaf_nodes", 1.0), + (11, False, "max_leaf_nodes", 0.1), + (42, True, "max_leaf_nodes", 0.01), + (42, False, "max_leaf_nodes", 1.0), + (256, True, "min_gain_to_split", 1.0), + (256, True, "max_leaf_nodes", 0.1), + ], +) +def test_grow_tree(n_bins, constant_hessian, stopping_param, shrinkage): + X_binned, all_gradients, all_hessians = _make_training_data( + n_bins=n_bins, constant_hessian=constant_hessian + ) + n_samples = X_binned.shape[0] + + if stopping_param == "max_leaf_nodes": + stopping_param = {"max_leaf_nodes": 3} + else: + stopping_param = {"min_gain_to_split": 0.01} + + grower = TreeGrower( + X_binned, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=shrinkage, + min_samples_leaf=1, + **stopping_param, + ) + + # The root node is not yet split, but the best possible split has + # already been evaluated: + assert grower.root.left_child is None + assert grower.root.right_child is None + + root_split = grower.root.split_info + assert root_split.feature_idx == 0 + assert root_split.bin_idx == n_bins // 2 + assert len(grower.splittable_nodes) == 1 + + # Calling split next applies the next split and computes the best split + # for each of the two newly introduced children nodes. + left_node, right_node = grower.split_next() + + # All training samples have ben split in the two nodes, approximately + # 50%/50% + _check_children_consistency(grower.root, left_node, right_node) + assert len(left_node.sample_indices) > 0.4 * n_samples + assert len(left_node.sample_indices) < 0.6 * n_samples + + if grower.min_gain_to_split > 0: + # The left node is too pure: there is no gain to split it further. + assert left_node.split_info.gain < grower.min_gain_to_split + assert left_node in grower.finalized_leaves + + # The right node can still be split further, this time on feature #1 + split_info = right_node.split_info + assert split_info.gain > 1.0 + assert split_info.feature_idx == 1 + assert split_info.bin_idx == n_bins // 3 + assert right_node.left_child is None + assert right_node.right_child is None + + # The right split has not been applied yet. Let's do it now: + assert len(grower.splittable_nodes) == 1 + right_left_node, right_right_node = grower.split_next() + _check_children_consistency(right_node, right_left_node, right_right_node) + assert len(right_left_node.sample_indices) > 0.1 * n_samples + assert len(right_left_node.sample_indices) < 0.2 * n_samples + + assert len(right_right_node.sample_indices) > 0.2 * n_samples + assert len(right_right_node.sample_indices) < 0.4 * n_samples + + # All the leafs are pure, it is not possible to split any further: + assert not grower.splittable_nodes + + grower._apply_shrinkage() + + # Check the values of the leaves: + assert grower.root.left_child.value == approx(shrinkage) + assert grower.root.right_child.left_child.value == approx(shrinkage) + assert grower.root.right_child.right_child.value == approx(-shrinkage, rel=1e-3) + + +def test_predictor_from_grower(): + # Build a tree on the toy 3-leaf dataset to extract the predictor. + n_bins = 256 + X_binned, all_gradients, all_hessians = _make_training_data(n_bins=n_bins) + grower = TreeGrower( + X_binned, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=1.0, + max_leaf_nodes=3, + min_samples_leaf=5, + ) + grower.grow() + assert grower.n_nodes == 5 # (2 decision nodes + 3 leaves) + + # Check that the node structure can be converted into a predictor + # object to perform predictions at scale + # We pass undefined binning_thresholds because we won't use predict anyway + predictor = grower.make_predictor( + binning_thresholds=np.zeros((X_binned.shape[1], n_bins)) + ) + assert predictor.nodes.shape[0] == 5 + assert predictor.nodes["is_leaf"].sum() == 3 + + # Probe some predictions for each leaf of the tree + # each group of 3 samples corresponds to a condition in _make_training_data + input_data = np.array( + [ + [0, 0], + [42, 99], + [128, 254], + [129, 0], + [129, 85], + [254, 85], + [129, 86], + [129, 254], + [242, 100], + ], + dtype=np.uint8, + ) + missing_values_bin_idx = n_bins - 1 + predictions = predictor.predict_binned( + input_data, missing_values_bin_idx, n_threads + ) + expected_targets = [1, 1, 1, 1, 1, 1, -1, -1, -1] + assert np.allclose(predictions, expected_targets) + + # Check that training set can be recovered exactly: + predictions = predictor.predict_binned(X_binned, missing_values_bin_idx, n_threads) + assert np.allclose(predictions, -all_gradients) + + +@pytest.mark.parametrize( + "n_samples, min_samples_leaf, n_bins, constant_hessian, noise", + [ + (11, 10, 7, True, 0), + (13, 10, 42, False, 0), + (56, 10, 255, True, 0.1), + (101, 3, 7, True, 0), + (200, 42, 42, False, 0), + (300, 55, 255, True, 0.1), + (300, 301, 255, True, 0.1), + ], +) +def test_min_samples_leaf(n_samples, min_samples_leaf, n_bins, constant_hessian, noise): + rng = np.random.RandomState(seed=0) + # data = linear target, 3 features, 1 irrelevant. + X = rng.normal(size=(n_samples, 3)) + y = X[:, 0] - X[:, 1] + if noise: + y_scale = y.std() + y += rng.normal(scale=noise, size=n_samples) * y_scale + mapper = _BinMapper(n_bins=n_bins) + X = mapper.fit_transform(X) + + all_gradients = y.astype(G_H_DTYPE) + shape_hessian = 1 if constant_hessian else all_gradients.shape + all_hessians = np.ones(shape=shape_hessian, dtype=G_H_DTYPE) + grower = TreeGrower( + X, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=1.0, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=n_samples, + ) + grower.grow() + predictor = grower.make_predictor(binning_thresholds=mapper.bin_thresholds_) + + if n_samples >= min_samples_leaf: + for node in predictor.nodes: + if node["is_leaf"]: + assert node["count"] >= min_samples_leaf + else: + assert predictor.nodes.shape[0] == 1 + assert predictor.nodes[0]["is_leaf"] + assert predictor.nodes[0]["count"] == n_samples + + +@pytest.mark.parametrize("n_samples, min_samples_leaf", [(99, 50), (100, 50)]) +def test_min_samples_leaf_root(n_samples, min_samples_leaf): + # Make sure root node isn't split if n_samples is not at least twice + # min_samples_leaf + rng = np.random.RandomState(seed=0) + + n_bins = 256 + + # data = linear target, 3 features, 1 irrelevant. + X = rng.normal(size=(n_samples, 3)) + y = X[:, 0] - X[:, 1] + mapper = _BinMapper(n_bins=n_bins) + X = mapper.fit_transform(X) + + all_gradients = y.astype(G_H_DTYPE) + all_hessians = np.ones(shape=1, dtype=G_H_DTYPE) + grower = TreeGrower( + X, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=1.0, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=n_samples, + ) + grower.grow() + if n_samples >= min_samples_leaf * 2: + assert len(grower.finalized_leaves) >= 2 + else: + assert len(grower.finalized_leaves) == 1 + + +def assert_is_stump(grower): + # To assert that stumps are created when max_depth=1 + for leaf in (grower.root.left_child, grower.root.right_child): + assert leaf.left_child is None + assert leaf.right_child is None + + +@pytest.mark.parametrize("max_depth", [1, 2, 3]) +def test_max_depth(max_depth): + # Make sure max_depth parameter works as expected + rng = np.random.RandomState(seed=0) + + n_bins = 256 + n_samples = 1000 + + # data = linear target, 3 features, 1 irrelevant. + X = rng.normal(size=(n_samples, 3)) + y = X[:, 0] - X[:, 1] + mapper = _BinMapper(n_bins=n_bins) + X = mapper.fit_transform(X) + + all_gradients = y.astype(G_H_DTYPE) + all_hessians = np.ones(shape=1, dtype=G_H_DTYPE) + grower = TreeGrower(X, all_gradients, all_hessians, max_depth=max_depth) + grower.grow() + + depth = max(leaf.depth for leaf in grower.finalized_leaves) + assert depth == max_depth + + if max_depth == 1: + assert_is_stump(grower) + + +def test_input_validation(): + X_binned, all_gradients, all_hessians = _make_training_data() + + X_binned_float = X_binned.astype(np.float32) + with pytest.raises(NotImplementedError, match="X_binned must be of type uint8"): + TreeGrower(X_binned_float, all_gradients, all_hessians) + + X_binned_C_array = np.ascontiguousarray(X_binned) + with pytest.raises( + ValueError, match="X_binned should be passed as Fortran contiguous array" + ): + TreeGrower(X_binned_C_array, all_gradients, all_hessians) + + +def test_init_parameters_validation(): + X_binned, all_gradients, all_hessians = _make_training_data() + with pytest.raises(ValueError, match="min_gain_to_split=-1 must be positive"): + TreeGrower(X_binned, all_gradients, all_hessians, min_gain_to_split=-1) + + with pytest.raises(ValueError, match="min_hessian_to_split=-1 must be positive"): + TreeGrower(X_binned, all_gradients, all_hessians, min_hessian_to_split=-1) + + +def test_missing_value_predict_only(): + # Make sure that missing values are supported at predict time even if they + # were not encountered in the training data: the missing values are + # assigned to whichever child has the most samples. + + rng = np.random.RandomState(0) + n_samples = 100 + X_binned = rng.randint(0, 256, size=(n_samples, 1), dtype=np.uint8) + X_binned = np.asfortranarray(X_binned) + + gradients = rng.normal(size=n_samples).astype(G_H_DTYPE) + hessians = np.ones(shape=1, dtype=G_H_DTYPE) + + grower = TreeGrower( + X_binned, gradients, hessians, min_samples_leaf=5, has_missing_values=False + ) + grower.grow() + + # We pass undefined binning_thresholds because we won't use predict anyway + predictor = grower.make_predictor( + binning_thresholds=np.zeros((X_binned.shape[1], X_binned.max() + 1)) + ) + + # go from root to a leaf, always following node with the most samples. + # That's the path nans are supposed to take + node = predictor.nodes[0] + while not node["is_leaf"]: + left = predictor.nodes[node["left"]] + right = predictor.nodes[node["right"]] + node = left if left["count"] > right["count"] else right + + prediction_main_path = node["value"] + + # now build X_test with only nans, and make sure all predictions are equal + # to prediction_main_path + all_nans = np.full(shape=(n_samples, 1), fill_value=np.nan) + known_cat_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) + f_idx_map = np.zeros(0, dtype=np.uint32) + + y_pred = predictor.predict(all_nans, known_cat_bitsets, f_idx_map, n_threads) + assert np.all(y_pred == prediction_main_path) + + +def test_split_on_nan_with_infinite_values(): + # Make sure the split on nan situations are respected even when there are + # samples with +inf values (we set the threshold to +inf when we have a + # split on nan so this test makes sure this does not introduce edge-case + # bugs). We need to use the private API so that we can also test + # predict_binned(). + + X = np.array([0, 1, np.inf, np.nan, np.nan]).reshape(-1, 1) + # the gradient values will force a split on nan situation + gradients = np.array([0, 0, 0, 100, 100], dtype=G_H_DTYPE) + hessians = np.ones(shape=1, dtype=G_H_DTYPE) + + bin_mapper = _BinMapper() + X_binned = bin_mapper.fit_transform(X) + + n_bins_non_missing = 3 + has_missing_values = True + grower = TreeGrower( + X_binned, + gradients, + hessians, + n_bins_non_missing=n_bins_non_missing, + has_missing_values=has_missing_values, + min_samples_leaf=1, + n_threads=n_threads, + ) + + grower.grow() + + predictor = grower.make_predictor(binning_thresholds=bin_mapper.bin_thresholds_) + + # sanity check: this was a split on nan + assert predictor.nodes[0]["num_threshold"] == np.inf + assert predictor.nodes[0]["bin_threshold"] == n_bins_non_missing - 1 + + known_cat_bitsets, f_idx_map = bin_mapper.make_known_categories_bitsets() + + # Make sure in particular that the +inf sample is mapped to the left child + # Note that lightgbm "fails" here and will assign the inf sample to the + # right child, even though it's a "split on nan" situation. + predictions = predictor.predict(X, known_cat_bitsets, f_idx_map, n_threads) + predictions_binned = predictor.predict_binned( + X_binned, + missing_values_bin_idx=bin_mapper.missing_values_bin_idx_, + n_threads=n_threads, + ) + np.testing.assert_allclose(predictions, -gradients) + np.testing.assert_allclose(predictions_binned, -gradients) + + +def test_grow_tree_categories(): + # Check that the grower produces the right predictor tree when a split is + # categorical + X_binned = np.array([[0, 1] * 11 + [1]], dtype=X_BINNED_DTYPE).T + X_binned = np.asfortranarray(X_binned) + + all_gradients = np.array([10, 1] * 11 + [1], dtype=G_H_DTYPE) + all_hessians = np.ones(1, dtype=G_H_DTYPE) + is_categorical = np.ones(1, dtype=np.uint8) + + grower = TreeGrower( + X_binned, + all_gradients, + all_hessians, + n_bins=4, + shrinkage=1.0, + min_samples_leaf=1, + is_categorical=is_categorical, + n_threads=n_threads, + ) + grower.grow() + assert grower.n_nodes == 3 + + categories = [np.array([4, 9], dtype=X_DTYPE)] + predictor = grower.make_predictor(binning_thresholds=categories) + root = predictor.nodes[0] + assert root["count"] == 23 + assert root["depth"] == 0 + assert root["is_categorical"] + + left, right = predictor.nodes[root["left"]], predictor.nodes[root["right"]] + + # arbitrary validation, but this means ones go to the left. + assert left["count"] >= right["count"] + + # check binned category value (1) + expected_binned_cat_bitset = [2**1] + [0] * 7 + binned_cat_bitset = predictor.binned_left_cat_bitsets + assert_array_equal(binned_cat_bitset[0], expected_binned_cat_bitset) + + # check raw category value (9) + expected_raw_cat_bitsets = [2**9] + [0] * 7 + raw_cat_bitsets = predictor.raw_left_cat_bitsets + assert_array_equal(raw_cat_bitsets[0], expected_raw_cat_bitsets) + + # Note that since there was no missing values during training, the missing + # values aren't part of the bitsets. However, we expect the missing values + # to go to the biggest child (i.e. the left one). + # The left child has a value of -1 = negative gradient. + assert root["missing_go_to_left"] + + # make sure binned missing values are mapped to the left child during + # prediction + prediction_binned = predictor.predict_binned( + np.asarray([[6]]).astype(X_BINNED_DTYPE), + missing_values_bin_idx=6, + n_threads=n_threads, + ) + assert_allclose(prediction_binned, [-1]) # negative gradient + + # make sure raw missing values are mapped to the left child during + # prediction + known_cat_bitsets = np.zeros((1, 8), dtype=np.uint32) # ignored anyway + f_idx_map = np.array([0], dtype=np.uint32) + prediction = predictor.predict( + np.array([[np.nan]]), known_cat_bitsets, f_idx_map, n_threads + ) + assert_allclose(prediction, [-1]) + + +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize("n_unique_categories", (2, 10, 100)) +@pytest.mark.parametrize("target", ("binary", "random", "equal")) +def test_ohe_equivalence(min_samples_leaf, n_unique_categories, target): + # Make sure that native categorical splits are equivalent to using a OHE, + # when given enough depth + + rng = np.random.RandomState(0) + n_samples = 10_000 + X_binned = rng.randint(0, n_unique_categories, size=(n_samples, 1), dtype=np.uint8) + + X_ohe = OneHotEncoder(sparse_output=False).fit_transform(X_binned) + X_ohe = np.asfortranarray(X_ohe).astype(np.uint8) + + if target == "equal": + gradients = X_binned.reshape(-1) + elif target == "binary": + gradients = (X_binned % 2).reshape(-1) + else: + gradients = rng.randn(n_samples) + gradients = gradients.astype(G_H_DTYPE) + + hessians = np.ones(shape=1, dtype=G_H_DTYPE) + + grower_params = { + "min_samples_leaf": min_samples_leaf, + "max_depth": None, + "max_leaf_nodes": None, + } + + grower = TreeGrower( + X_binned, gradients, hessians, is_categorical=[True], **grower_params + ) + grower.grow() + # we pass undefined bin_thresholds because we won't use predict() + predictor = grower.make_predictor( + binning_thresholds=np.zeros((1, n_unique_categories)) + ) + preds = predictor.predict_binned( + X_binned, missing_values_bin_idx=255, n_threads=n_threads + ) + + grower_ohe = TreeGrower(X_ohe, gradients, hessians, **grower_params) + grower_ohe.grow() + predictor_ohe = grower_ohe.make_predictor( + binning_thresholds=np.zeros((X_ohe.shape[1], n_unique_categories)) + ) + preds_ohe = predictor_ohe.predict_binned( + X_ohe, missing_values_bin_idx=255, n_threads=n_threads + ) + + assert predictor.get_max_depth() <= predictor_ohe.get_max_depth() + if target == "binary" and n_unique_categories > 2: + # OHE needs more splits to achieve the same predictions + assert predictor.get_max_depth() < predictor_ohe.get_max_depth() + + np.testing.assert_allclose(preds, preds_ohe) + + +def test_grower_interaction_constraints(): + """Check that grower respects interaction constraints.""" + n_features = 6 + interaction_cst = [{0, 1}, {1, 2}, {3, 4, 5}] + n_samples = 10 + n_bins = 6 + root_feature_splits = [] + + def get_all_children(node): + res = [] + if node.is_leaf: + return res + for n in [node.left_child, node.right_child]: + res.append(n) + res.extend(get_all_children(n)) + return res + + for seed in range(20): + rng = np.random.RandomState(seed) + + X_binned = rng.randint( + 0, n_bins - 1, size=(n_samples, n_features), dtype=X_BINNED_DTYPE + ) + X_binned = np.asfortranarray(X_binned) + gradients = rng.normal(size=n_samples).astype(G_H_DTYPE) + hessians = np.ones(shape=1, dtype=G_H_DTYPE) + + grower = TreeGrower( + X_binned, + gradients, + hessians, + n_bins=n_bins, + min_samples_leaf=1, + interaction_cst=interaction_cst, + n_threads=n_threads, + ) + grower.grow() + + root_feature_idx = grower.root.split_info.feature_idx + root_feature_splits.append(root_feature_idx) + + feature_idx_to_constraint_set = { + 0: {0, 1}, + 1: {0, 1, 2}, + 2: {1, 2}, + 3: {3, 4, 5}, + 4: {3, 4, 5}, + 5: {3, 4, 5}, + } + + root_constraint_set = feature_idx_to_constraint_set[root_feature_idx] + for node in (grower.root.left_child, grower.root.right_child): + # Root's children's allowed_features must be the root's constraints set. + assert_array_equal(node.allowed_features, list(root_constraint_set)) + for node in get_all_children(grower.root): + if node.is_leaf: + continue + # Ensure that each node uses a subset of features of its parent node. + parent_interaction_cst_indices = set(node.interaction_cst_indices) + right_interactions_cst_indices = set( + node.right_child.interaction_cst_indices + ) + left_interactions_cst_indices = set(node.left_child.interaction_cst_indices) + + assert right_interactions_cst_indices.issubset( + parent_interaction_cst_indices + ) + assert left_interactions_cst_indices.issubset( + parent_interaction_cst_indices + ) + # The features used for split must have been present in the root's + # constraint set. + assert node.split_info.feature_idx in root_constraint_set + + # Make sure that every feature is used at least once as split for the root node. + assert ( + len(set(root_feature_splits)) + == len(set().union(*interaction_cst)) + == n_features + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_histogram.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_histogram.py new file mode 100644 index 0000000000000000000000000000000000000000..d910aee7938e99142d0aa964c0b8b915164efc20 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_histogram.py @@ -0,0 +1,239 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +from sklearn.ensemble._hist_gradient_boosting.common import ( + G_H_DTYPE, + HISTOGRAM_DTYPE, + X_BINNED_DTYPE, +) +from sklearn.ensemble._hist_gradient_boosting.histogram import ( + _build_histogram, + _build_histogram_naive, + _build_histogram_no_hessian, + _build_histogram_root, + _build_histogram_root_no_hessian, + _subtract_histograms, +) + + +@pytest.mark.parametrize("build_func", [_build_histogram_naive, _build_histogram]) +def test_build_histogram(build_func): + binned_feature = np.array([0, 2, 0, 1, 2, 0, 2, 1], dtype=X_BINNED_DTYPE) + + # Small sample_indices (below unrolling threshold) + ordered_gradients = np.array([0, 1, 3], dtype=G_H_DTYPE) + ordered_hessians = np.array([1, 1, 2], dtype=G_H_DTYPE) + + sample_indices = np.array([0, 2, 3], dtype=np.uint32) + hist = np.zeros((1, 3), dtype=HISTOGRAM_DTYPE) + build_func( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist + ) + hist = hist[0] + assert_array_equal(hist["count"], [2, 1, 0]) + assert_allclose(hist["sum_gradients"], [1, 3, 0]) + assert_allclose(hist["sum_hessians"], [2, 2, 0]) + + # Larger sample_indices (above unrolling threshold) + sample_indices = np.array([0, 2, 3, 6, 7], dtype=np.uint32) + ordered_gradients = np.array([0, 1, 3, 0, 1], dtype=G_H_DTYPE) + ordered_hessians = np.array([1, 1, 2, 1, 0], dtype=G_H_DTYPE) + + hist = np.zeros((1, 3), dtype=HISTOGRAM_DTYPE) + build_func( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist + ) + hist = hist[0] + assert_array_equal(hist["count"], [2, 2, 1]) + assert_allclose(hist["sum_gradients"], [1, 4, 0]) + assert_allclose(hist["sum_hessians"], [2, 2, 1]) + + +def test_histogram_sample_order_independence(): + # Make sure the order of the samples has no impact on the histogram + # computations + rng = np.random.RandomState(42) + n_sub_samples = 100 + n_samples = 1000 + n_bins = 256 + + binned_feature = rng.randint(0, n_bins - 1, size=n_samples, dtype=X_BINNED_DTYPE) + sample_indices = rng.choice( + np.arange(n_samples, dtype=np.uint32), n_sub_samples, replace=False + ) + ordered_gradients = rng.randn(n_sub_samples).astype(G_H_DTYPE) + hist_gc = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + _build_histogram_no_hessian( + 0, sample_indices, binned_feature, ordered_gradients, hist_gc + ) + + ordered_hessians = rng.exponential(size=n_sub_samples).astype(G_H_DTYPE) + hist_ghc = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + _build_histogram( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist_ghc + ) + + permutation = rng.permutation(n_sub_samples) + hist_gc_perm = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + _build_histogram_no_hessian( + 0, + sample_indices[permutation], + binned_feature, + ordered_gradients[permutation], + hist_gc_perm, + ) + + hist_ghc_perm = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + _build_histogram( + 0, + sample_indices[permutation], + binned_feature, + ordered_gradients[permutation], + ordered_hessians[permutation], + hist_ghc_perm, + ) + + hist_gc = hist_gc[0] + hist_ghc = hist_ghc[0] + hist_gc_perm = hist_gc_perm[0] + hist_ghc_perm = hist_ghc_perm[0] + + assert_allclose(hist_gc["sum_gradients"], hist_gc_perm["sum_gradients"]) + assert_array_equal(hist_gc["count"], hist_gc_perm["count"]) + + assert_allclose(hist_ghc["sum_gradients"], hist_ghc_perm["sum_gradients"]) + assert_allclose(hist_ghc["sum_hessians"], hist_ghc_perm["sum_hessians"]) + assert_array_equal(hist_ghc["count"], hist_ghc_perm["count"]) + + +@pytest.mark.parametrize("constant_hessian", [True, False]) +def test_unrolled_equivalent_to_naive(constant_hessian): + # Make sure the different unrolled histogram computations give the same + # results as the naive one. + rng = np.random.RandomState(42) + n_samples = 10 + n_bins = 5 + sample_indices = np.arange(n_samples).astype(np.uint32) + binned_feature = rng.randint(0, n_bins - 1, size=n_samples, dtype=np.uint8) + ordered_gradients = rng.randn(n_samples).astype(G_H_DTYPE) + if constant_hessian: + ordered_hessians = np.ones(n_samples, dtype=G_H_DTYPE) + else: + ordered_hessians = rng.lognormal(size=n_samples).astype(G_H_DTYPE) + + hist_gc_root = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + hist_ghc_root = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + hist_gc = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + hist_ghc = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + hist_naive = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + + _build_histogram_root_no_hessian(0, binned_feature, ordered_gradients, hist_gc_root) + _build_histogram_root( + 0, binned_feature, ordered_gradients, ordered_hessians, hist_ghc_root + ) + _build_histogram_no_hessian( + 0, sample_indices, binned_feature, ordered_gradients, hist_gc + ) + _build_histogram( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist_ghc + ) + _build_histogram_naive( + 0, + sample_indices, + binned_feature, + ordered_gradients, + ordered_hessians, + hist_naive, + ) + + hist_naive = hist_naive[0] + hist_gc_root = hist_gc_root[0] + hist_ghc_root = hist_ghc_root[0] + hist_gc = hist_gc[0] + hist_ghc = hist_ghc[0] + for hist in (hist_gc_root, hist_ghc_root, hist_gc, hist_ghc): + assert_array_equal(hist["count"], hist_naive["count"]) + assert_allclose(hist["sum_gradients"], hist_naive["sum_gradients"]) + for hist in (hist_ghc_root, hist_ghc): + assert_allclose(hist["sum_hessians"], hist_naive["sum_hessians"]) + for hist in (hist_gc_root, hist_gc): + assert_array_equal(hist["sum_hessians"], np.zeros(n_bins)) + + +@pytest.mark.parametrize("constant_hessian", [True, False]) +def test_hist_subtraction(constant_hessian): + # Make sure the histogram subtraction trick gives the same result as the + # classical method. + rng = np.random.RandomState(42) + n_samples = 10 + n_bins = 5 + sample_indices = np.arange(n_samples).astype(np.uint32) + binned_feature = rng.randint(0, n_bins - 1, size=n_samples, dtype=np.uint8) + ordered_gradients = rng.randn(n_samples).astype(G_H_DTYPE) + if constant_hessian: + ordered_hessians = np.ones(n_samples, dtype=G_H_DTYPE) + else: + ordered_hessians = rng.lognormal(size=n_samples).astype(G_H_DTYPE) + + hist_parent = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + if constant_hessian: + _build_histogram_no_hessian( + 0, sample_indices, binned_feature, ordered_gradients, hist_parent + ) + else: + _build_histogram( + 0, + sample_indices, + binned_feature, + ordered_gradients, + ordered_hessians, + hist_parent, + ) + + mask = rng.randint(0, 2, n_samples).astype(bool) + + sample_indices_left = sample_indices[mask] + ordered_gradients_left = ordered_gradients[mask] + ordered_hessians_left = ordered_hessians[mask] + hist_left = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + if constant_hessian: + _build_histogram_no_hessian( + 0, sample_indices_left, binned_feature, ordered_gradients_left, hist_left + ) + else: + _build_histogram( + 0, + sample_indices_left, + binned_feature, + ordered_gradients_left, + ordered_hessians_left, + hist_left, + ) + + sample_indices_right = sample_indices[~mask] + ordered_gradients_right = ordered_gradients[~mask] + ordered_hessians_right = ordered_hessians[~mask] + hist_right = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) + if constant_hessian: + _build_histogram_no_hessian( + 0, sample_indices_right, binned_feature, ordered_gradients_right, hist_right + ) + else: + _build_histogram( + 0, + sample_indices_right, + binned_feature, + ordered_gradients_right, + ordered_hessians_right, + hist_right, + ) + + hist_left_sub = np.copy(hist_parent) + hist_right_sub = np.copy(hist_parent) + _subtract_histograms(0, n_bins, hist_left_sub, hist_right) + _subtract_histograms(0, n_bins, hist_right_sub, hist_left) + + for key in ("count", "sum_hessians", "sum_gradients"): + assert_allclose(hist_left[key], hist_left_sub[key], rtol=1e-6) + assert_allclose(hist_right[key], hist_right_sub[key], rtol=1e-6) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_constraints.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_constraints.py new file mode 100644 index 0000000000000000000000000000000000000000..ef9485c8f6426f249adf999b6c9e317dfb5f4612 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_constraints.py @@ -0,0 +1,446 @@ +import re + +import numpy as np +import pytest + +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.ensemble._hist_gradient_boosting.common import ( + G_H_DTYPE, + X_BINNED_DTYPE, + MonotonicConstraint, +) +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +from sklearn.ensemble._hist_gradient_boosting.histogram import HistogramBuilder +from sklearn.ensemble._hist_gradient_boosting.splitting import ( + Splitter, + compute_node_value, +) +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.utils._testing import _convert_container + +n_threads = _openmp_effective_n_threads() + + +def is_increasing(a): + return (np.diff(a) >= 0.0).all() + + +def is_decreasing(a): + return (np.diff(a) <= 0.0).all() + + +def assert_leaves_values_monotonic(predictor, monotonic_cst): + # make sure leaves values (from left to right) are either all increasing + # or all decreasing (or neither) depending on the monotonic constraint. + nodes = predictor.nodes + + def get_leaves_values(): + """get leaves values from left to right""" + values = [] + + def depth_first_collect_leaf_values(node_idx): + node = nodes[node_idx] + if node["is_leaf"]: + values.append(node["value"]) + return + depth_first_collect_leaf_values(node["left"]) + depth_first_collect_leaf_values(node["right"]) + + depth_first_collect_leaf_values(0) # start at root (0) + return values + + values = get_leaves_values() + + if monotonic_cst == MonotonicConstraint.NO_CST: + # some increasing, some decreasing + assert not is_increasing(values) and not is_decreasing(values) + elif monotonic_cst == MonotonicConstraint.POS: + # all increasing + assert is_increasing(values) + else: # NEG + # all decreasing + assert is_decreasing(values) + + +def assert_children_values_monotonic(predictor, monotonic_cst): + # Make sure siblings values respect the monotonic constraints. Left should + # be lower (resp greater) than right child if constraint is POS (resp. + # NEG). + # Note that this property alone isn't enough to ensure full monotonicity, + # since we also need to guanrantee that all the descendents of the left + # child won't be greater (resp. lower) than the right child, or its + # descendents. That's why we need to bound the predicted values (this is + # tested in assert_children_values_bounded) + nodes = predictor.nodes + left_lower = [] + left_greater = [] + for node in nodes: + if node["is_leaf"]: + continue + + left_idx = node["left"] + right_idx = node["right"] + + if nodes[left_idx]["value"] < nodes[right_idx]["value"]: + left_lower.append(node) + elif nodes[left_idx]["value"] > nodes[right_idx]["value"]: + left_greater.append(node) + + if monotonic_cst == MonotonicConstraint.NO_CST: + assert left_lower and left_greater + elif monotonic_cst == MonotonicConstraint.POS: + assert left_lower and not left_greater + else: # NEG + assert not left_lower and left_greater + + +def assert_children_values_bounded(grower, monotonic_cst): + # Make sure that the values of the children of a node are bounded by the + # middle value between that node and its sibling (if there is a monotonic + # constraint). + # As a bonus, we also check that the siblings values are properly ordered + # which is slightly redundant with assert_children_values_monotonic (but + # this check is done on the grower nodes whereas + # assert_children_values_monotonic is done on the predictor nodes) + + if monotonic_cst == MonotonicConstraint.NO_CST: + return + + def recursively_check_children_node_values(node, right_sibling=None): + if node.is_leaf: + return + if right_sibling is not None: + middle = (node.value + right_sibling.value) / 2 + if monotonic_cst == MonotonicConstraint.POS: + assert node.left_child.value <= node.right_child.value <= middle + if not right_sibling.is_leaf: + assert ( + middle + <= right_sibling.left_child.value + <= right_sibling.right_child.value + ) + else: # NEG + assert node.left_child.value >= node.right_child.value >= middle + if not right_sibling.is_leaf: + assert ( + middle + >= right_sibling.left_child.value + >= right_sibling.right_child.value + ) + + recursively_check_children_node_values( + node.left_child, right_sibling=node.right_child + ) + recursively_check_children_node_values(node.right_child) + + recursively_check_children_node_values(grower.root) + + +@pytest.mark.parametrize("seed", range(3)) +@pytest.mark.parametrize( + "monotonic_cst", + ( + MonotonicConstraint.NO_CST, + MonotonicConstraint.POS, + MonotonicConstraint.NEG, + ), +) +def test_nodes_values(monotonic_cst, seed): + # Build a single tree with only one feature, and make sure the nodes + # values respect the monotonic constraints. + + # Considering the following tree with a monotonic POS constraint, we + # should have: + # + # root + # / \ + # 5 10 # middle = 7.5 + # / \ / \ + # a b c d + # + # a <= b and c <= d (assert_children_values_monotonic) + # a, b <= middle <= c, d (assert_children_values_bounded) + # a <= b <= c <= d (assert_leaves_values_monotonic) + # + # The last one is a consequence of the others, but can't hurt to check + + rng = np.random.RandomState(seed) + n_samples = 1000 + n_features = 1 + X_binned = rng.randint(0, 255, size=(n_samples, n_features), dtype=np.uint8) + X_binned = np.asfortranarray(X_binned) + + gradients = rng.normal(size=n_samples).astype(G_H_DTYPE) + hessians = np.ones(shape=1, dtype=G_H_DTYPE) + + grower = TreeGrower( + X_binned, gradients, hessians, monotonic_cst=[monotonic_cst], shrinkage=0.1 + ) + grower.grow() + + # grow() will shrink the leaves values at the very end. For our comparison + # tests, we need to revert the shrinkage of the leaves, else we would + # compare the value of a leaf (shrunk) with a node (not shrunk) and the + # test would not be correct. + for leave in grower.finalized_leaves: + leave.value /= grower.shrinkage + + # We pass undefined binning_thresholds because we won't use predict anyway + predictor = grower.make_predictor( + binning_thresholds=np.zeros((X_binned.shape[1], X_binned.max() + 1)) + ) + + # The consistency of the bounds can only be checked on the tree grower + # as the node bounds are not copied into the predictor tree. The + # consistency checks on the values of node children and leaves can be + # done either on the grower tree or on the predictor tree. We only + # do those checks on the predictor tree as the latter is derived from + # the former. + assert_children_values_monotonic(predictor, monotonic_cst) + assert_children_values_bounded(grower, monotonic_cst) + assert_leaves_values_monotonic(predictor, monotonic_cst) + + +@pytest.mark.parametrize("use_feature_names", (True, False)) +def test_predictions(global_random_seed, use_feature_names): + # Train a model with a POS constraint on the first non-categorical feature + # and a NEG constraint on the second non-categorical feature, and make sure + # the constraints are respected by checking the predictions. + # test adapted from lightgbm's test_monotone_constraint(), itself inspired + # by https://xgboost.readthedocs.io/en/latest/tutorials/monotonic.html + + rng = np.random.RandomState(global_random_seed) + + n_samples = 1000 + f_0 = rng.rand(n_samples) # positive correlation with y + f_1 = rng.rand(n_samples) # negative correlation with y + + # extra categorical features, no correlation with y, + # to check the correctness of monotonicity constraint remapping, see issue #28898 + f_a = rng.randint(low=0, high=9, size=n_samples) + f_b = rng.randint(low=0, high=9, size=n_samples) + f_c = rng.randint(low=0, high=9, size=n_samples) + + X = np.c_[f_a, f_0, f_b, f_1, f_c] + columns_name = ["f_a", "f_0", "f_b", "f_1", "f_c"] + constructor_name = "dataframe" if use_feature_names else "array" + X = _convert_container(X, constructor_name, columns_name=columns_name) + + noise = rng.normal(loc=0.0, scale=0.01, size=n_samples) + y = 5 * f_0 + np.sin(10 * np.pi * f_0) - 5 * f_1 - np.cos(10 * np.pi * f_1) + noise + + if use_feature_names: + monotonic_cst = {"f_0": +1, "f_1": -1} + categorical_features = ["f_a", "f_b", "f_c"] + else: + monotonic_cst = [0, +1, 0, -1, 0] + categorical_features = [0, 2, 4] + + gbdt = HistGradientBoostingRegressor( + monotonic_cst=monotonic_cst, categorical_features=categorical_features + ) + gbdt.fit(X, y) + + linspace = np.linspace(0, 1, 100) + sin = np.sin(linspace) + constant = np.full_like(linspace, fill_value=0.5) + + # We now assert the predictions properly respect the constraints, on each + # feature. When testing for a feature we need to set the other one to a + # constant, because the monotonic constraints are only a "all else being + # equal" type of constraints: + # a constraint on the first feature only means that + # x0 < x0' => f(x0, x1) < f(x0', x1) + # while x1 stays constant. + # The constraint does not guanrantee that + # x0 < x0' => f(x0, x1) < f(x0', x1') + + # First non-categorical feature (POS) + # assert pred is all increasing when f_0 is all increasing + X = np.c_[constant, linspace, constant, constant, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) + pred = gbdt.predict(X) + assert is_increasing(pred) + # assert pred actually follows the variations of f_0 + X = np.c_[constant, sin, constant, constant, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) + pred = gbdt.predict(X) + assert np.all((np.diff(pred) >= 0) == (np.diff(sin) >= 0)) + + # Second non-categorical feature (NEG) + # assert pred is all decreasing when f_1 is all increasing + X = np.c_[constant, constant, constant, linspace, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) + pred = gbdt.predict(X) + assert is_decreasing(pred) + # assert pred actually follows the inverse variations of f_1 + X = np.c_[constant, constant, constant, sin, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) + pred = gbdt.predict(X) + assert ((np.diff(pred) <= 0) == (np.diff(sin) >= 0)).all() + + +def test_input_error(): + X = [[1, 2], [2, 3], [3, 4]] + y = [0, 1, 2] + + gbdt = HistGradientBoostingRegressor(monotonic_cst=[1, 0, -1]) + with pytest.raises( + ValueError, match=re.escape("monotonic_cst has shape (3,) but the input data") + ): + gbdt.fit(X, y) + + for monotonic_cst in ([1, 3], [1, -3], [0.3, -0.7]): + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape( + "must be an array-like of -1, 0 or 1. Observed values:" + ) + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X, y) + + gbdt = HistGradientBoostingClassifier(monotonic_cst=[0, 1]) + with pytest.raises( + ValueError, + match="monotonic constraints are not supported for multiclass classification", + ): + gbdt.fit(X, y) + + +def test_input_error_related_to_feature_names(): + pd = pytest.importorskip("pandas") + X = pd.DataFrame({"a": [0, 1, 2], "b": [0, 1, 2]}) + y = np.array([0, 1, 0]) + + monotonic_cst = {"d": 1, "a": 1, "c": -1} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape( + "monotonic_cst contains 2 unexpected feature names: ['c', 'd']." + ) + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X, y) + + monotonic_cst = {k: 1 for k in "abcdefghijklmnopqrstuvwxyz"} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape( + "monotonic_cst contains 24 unexpected feature names: " + "['c', 'd', 'e', 'f', 'g', '...']." + ) + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X, y) + + monotonic_cst = {"a": 1} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape( + "HistGradientBoostingRegressor was not fitted on data with feature " + "names. Pass monotonic_cst as an integer array instead." + ) + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X.values, y) + + monotonic_cst = {"b": -1, "a": "+"} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape("monotonic_cst['a'] must be either -1, 0 or 1. Got '+'.") + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X, y) + + +def test_bounded_value_min_gain_to_split(): + # The purpose of this test is to show that when computing the gain at a + # given split, the value of the current node should be properly bounded to + # respect the monotonic constraints, because it strongly interacts with + # min_gain_to_split. We build a simple example where gradients are [1, 1, + # 100, 1, 1] (hessians are all ones). The best split happens on the 3rd + # bin, and depending on whether the value of the node is bounded or not, + # the min_gain_to_split constraint is or isn't satisfied. + l2_regularization = 0 + min_hessian_to_split = 0 + min_samples_leaf = 1 + n_bins = n_samples = 5 + X_binned = np.arange(n_samples).reshape(-1, 1).astype(X_BINNED_DTYPE) + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_hessians = np.ones(n_samples, dtype=G_H_DTYPE) + all_gradients = np.array([1, 1, 100, 1, 1], dtype=G_H_DTYPE) + sum_gradients = all_gradients.sum() + sum_hessians = all_hessians.sum() + hessians_are_constant = False + + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + children_lower_bound, children_upper_bound = -np.inf, np.inf + + min_gain_to_split = 2000 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + histograms = builder.compute_histograms_brute(sample_indices) + + # Since the gradient array is [1, 1, 100, 1, 1] + # the max possible gain happens on the 3rd bin (or equivalently in the 2nd) + # and is equal to about 1307, which less than min_gain_to_split = 2000, so + # the node is considered unsplittable (gain = -1) + current_lower_bound, current_upper_bound = -np.inf, np.inf + value = compute_node_value( + sum_gradients, + sum_hessians, + current_lower_bound, + current_upper_bound, + l2_regularization, + ) + # the unbounded value is equal to -sum_gradients / sum_hessians + assert value == pytest.approx(-104 / 5) + split_info = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + lower_bound=children_lower_bound, + upper_bound=children_upper_bound, + ) + assert split_info.gain == -1 # min_gain_to_split not respected + + # here again the max possible gain is on the 3rd bin but we now cap the + # value of the node into [-10, inf]. + # This means the gain is now about 2430 which is more than the + # min_gain_to_split constraint. + current_lower_bound, current_upper_bound = -10, np.inf + value = compute_node_value( + sum_gradients, + sum_hessians, + current_lower_bound, + current_upper_bound, + l2_regularization, + ) + assert value == -10 + split_info = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + lower_bound=children_lower_bound, + upper_bound=children_upper_bound, + ) + assert split_info.gain > min_gain_to_split diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..f8873bbce06754ad791e1094797e9c3a9d154230 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py @@ -0,0 +1,187 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from sklearn.datasets import make_regression +from sklearn.ensemble._hist_gradient_boosting._bitset import ( + set_bitset_memoryview, + set_raw_bitset_from_binned_bitset, +) +from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper +from sklearn.ensemble._hist_gradient_boosting.common import ( + ALMOST_INF, + G_H_DTYPE, + PREDICTOR_RECORD_DTYPE, + X_BINNED_DTYPE, + X_BITSET_INNER_DTYPE, + X_DTYPE, +) +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +from sklearn.ensemble._hist_gradient_boosting.predictor import TreePredictor +from sklearn.metrics import r2_score +from sklearn.model_selection import train_test_split +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + +n_threads = _openmp_effective_n_threads() + + +@pytest.mark.parametrize("n_bins", [200, 256]) +def test_regression_dataset(n_bins): + X, y = make_regression( + n_samples=500, n_features=10, n_informative=5, random_state=42 + ) + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) + + mapper = _BinMapper(n_bins=n_bins, random_state=42) + X_train_binned = mapper.fit_transform(X_train) + + # Init gradients and hessians to that of least squares loss + gradients = -y_train.astype(G_H_DTYPE) + hessians = np.ones(1, dtype=G_H_DTYPE) + + min_samples_leaf = 10 + max_leaf_nodes = 30 + grower = TreeGrower( + X_train_binned, + gradients, + hessians, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=max_leaf_nodes, + n_bins=n_bins, + n_bins_non_missing=mapper.n_bins_non_missing_, + ) + grower.grow() + + predictor = grower.make_predictor(binning_thresholds=mapper.bin_thresholds_) + + known_cat_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) + f_idx_map = np.zeros(0, dtype=np.uint32) + + y_pred_train = predictor.predict(X_train, known_cat_bitsets, f_idx_map, n_threads) + assert r2_score(y_train, y_pred_train) > 0.82 + + y_pred_test = predictor.predict(X_test, known_cat_bitsets, f_idx_map, n_threads) + assert r2_score(y_test, y_pred_test) > 0.67 + + +@pytest.mark.parametrize( + "num_threshold, expected_predictions", + [ + (-np.inf, [0, 1, 1, 1]), + (10, [0, 0, 1, 1]), + (20, [0, 0, 0, 1]), + (ALMOST_INF, [0, 0, 0, 1]), + (np.inf, [0, 0, 0, 0]), + ], +) +def test_infinite_values_and_thresholds(num_threshold, expected_predictions): + # Make sure infinite values and infinite thresholds are handled properly. + # In particular, if a value is +inf and the threshold is ALMOST_INF the + # sample should go to the right child. If the threshold is inf (split on + # nan), the +inf sample will go to the left child. + + X = np.array([-np.inf, 10, 20, np.inf]).reshape(-1, 1) + nodes = np.zeros(3, dtype=PREDICTOR_RECORD_DTYPE) + + # We just construct a simple tree with 1 root and 2 children + # parent node + nodes[0]["left"] = 1 + nodes[0]["right"] = 2 + nodes[0]["feature_idx"] = 0 + nodes[0]["num_threshold"] = num_threshold + + # left child + nodes[1]["is_leaf"] = True + nodes[1]["value"] = 0 + + # right child + nodes[2]["is_leaf"] = True + nodes[2]["value"] = 1 + + binned_cat_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) + raw_categorical_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) + known_cat_bitset = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) + f_idx_map = np.zeros(0, dtype=np.uint32) + + predictor = TreePredictor(nodes, binned_cat_bitsets, raw_categorical_bitsets) + predictions = predictor.predict(X, known_cat_bitset, f_idx_map, n_threads) + + assert np.all(predictions == expected_predictions) + + +@pytest.mark.parametrize( + "bins_go_left, expected_predictions", + [ + ([0, 3, 4, 6], [1, 0, 0, 1, 1, 0]), + ([0, 1, 2, 6], [1, 1, 1, 0, 0, 0]), + ([3, 5, 6], [0, 0, 0, 1, 0, 1]), + ], +) +def test_categorical_predictor(bins_go_left, expected_predictions): + # Test predictor outputs are correct with categorical features + + X_binned = np.array([[0, 1, 2, 3, 4, 5]], dtype=X_BINNED_DTYPE).T + categories = np.array([2, 5, 6, 8, 10, 15], dtype=X_DTYPE) + + bins_go_left = np.array(bins_go_left, dtype=X_BINNED_DTYPE) + + # We just construct a simple tree with 1 root and 2 children + # parent node + nodes = np.zeros(3, dtype=PREDICTOR_RECORD_DTYPE) + nodes[0]["left"] = 1 + nodes[0]["right"] = 2 + nodes[0]["feature_idx"] = 0 + nodes[0]["is_categorical"] = True + nodes[0]["missing_go_to_left"] = True + + # left child + nodes[1]["is_leaf"] = True + nodes[1]["value"] = 1 + + # right child + nodes[2]["is_leaf"] = True + nodes[2]["value"] = 0 + + binned_cat_bitsets = np.zeros((1, 8), dtype=X_BITSET_INNER_DTYPE) + raw_categorical_bitsets = np.zeros((1, 8), dtype=X_BITSET_INNER_DTYPE) + for go_left in bins_go_left: + set_bitset_memoryview(binned_cat_bitsets[0], go_left) + + set_raw_bitset_from_binned_bitset( + raw_categorical_bitsets[0], binned_cat_bitsets[0], categories + ) + + predictor = TreePredictor(nodes, binned_cat_bitsets, raw_categorical_bitsets) + + # Check binned data gives correct predictions + prediction_binned = predictor.predict_binned( + X_binned, missing_values_bin_idx=6, n_threads=n_threads + ) + assert_allclose(prediction_binned, expected_predictions) + + # manually construct bitset + known_cat_bitsets = np.zeros((1, 8), dtype=np.uint32) + known_cat_bitsets[0, 0] = np.sum(2**categories, dtype=np.uint32) + f_idx_map = np.array([0], dtype=np.uint32) + + # Check with un-binned data + predictions = predictor.predict( + categories.reshape(-1, 1), known_cat_bitsets, f_idx_map, n_threads + ) + assert_allclose(predictions, expected_predictions) + + # Check missing goes left because missing_values_bin_idx=6 + X_binned_missing = np.array([[6]], dtype=X_BINNED_DTYPE).T + predictions = predictor.predict_binned( + X_binned_missing, missing_values_bin_idx=6, n_threads=n_threads + ) + assert_allclose(predictions, [1]) + + # missing and unknown go left + predictions = predictor.predict( + np.array([[np.nan, 17]], dtype=X_DTYPE).T, + known_cat_bitsets, + f_idx_map, + n_threads, + ) + assert_allclose(predictions, [1, 1]) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_splitting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_splitting.py new file mode 100644 index 0000000000000000000000000000000000000000..61d6345fd6182fc2bfa14427e16109309a0965aa --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_splitting.py @@ -0,0 +1,1070 @@ +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +from sklearn.ensemble._hist_gradient_boosting.common import ( + G_H_DTYPE, + HISTOGRAM_DTYPE, + X_BINNED_DTYPE, + MonotonicConstraint, +) +from sklearn.ensemble._hist_gradient_boosting.histogram import HistogramBuilder +from sklearn.ensemble._hist_gradient_boosting.splitting import ( + Splitter, + compute_node_value, +) +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.utils._testing import skip_if_32bit + +n_threads = _openmp_effective_n_threads() + + +@pytest.mark.parametrize("n_bins", [3, 32, 256]) +def test_histogram_split(n_bins): + rng = np.random.RandomState(42) + feature_idx = 0 + l2_regularization = 0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + X_binned = np.asfortranarray( + rng.randint(0, n_bins - 1, size=(int(1e4), 1)), dtype=X_BINNED_DTYPE + ) + binned_feature = X_binned.T[feature_idx] + sample_indices = np.arange(binned_feature.shape[0], dtype=np.uint32) + ordered_hessians = np.ones_like(binned_feature, dtype=G_H_DTYPE) + all_hessians = ordered_hessians + sum_hessians = all_hessians.sum() + hessians_are_constant = False + + for true_bin in range(1, n_bins - 2): + for sign in [-1, 1]: + ordered_gradients = np.full_like(binned_feature, sign, dtype=G_H_DTYPE) + ordered_gradients[binned_feature <= true_bin] *= -1 + all_gradients = ordered_gradients + sum_gradients = all_gradients.sum() + + builder = HistogramBuilder( + X_binned, + n_bins, + all_gradients, + all_hessians, + hessians_are_constant, + n_threads, + ) + n_bins_non_missing = np.array( + [n_bins - 1] * X_binned.shape[1], dtype=np.uint32 + ) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + sample_indices.shape[0], histograms, sum_gradients, sum_hessians, value + ) + + assert split_info.bin_idx == true_bin + assert split_info.gain >= 0 + assert split_info.feature_idx == feature_idx + assert ( + split_info.n_samples_left + split_info.n_samples_right + == sample_indices.shape[0] + ) + # Constant hessian: 1. per sample. + assert split_info.n_samples_left == split_info.sum_hessian_left + + +@skip_if_32bit +@pytest.mark.parametrize("constant_hessian", [True, False]) +def test_gradient_and_hessian_sanity(constant_hessian): + # This test checks that the values of gradients and hessians are + # consistent in different places: + # - in split_info: si.sum_gradient_left + si.sum_gradient_right must be + # equal to the gradient at the node. Same for hessians. + # - in the histograms: summing 'sum_gradients' over the bins must be + # constant across all features, and those sums must be equal to the + # node's gradient. Same for hessians. + + rng = np.random.RandomState(42) + + n_bins = 10 + n_features = 20 + n_samples = 500 + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + + X_binned = rng.randint( + 0, n_bins, size=(n_samples, n_features), dtype=X_BINNED_DTYPE + ) + X_binned = np.asfortranarray(X_binned) + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_gradients = rng.randn(n_samples).astype(G_H_DTYPE) + sum_gradients = all_gradients.sum() + if constant_hessian: + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_hessians = 1 * n_samples + else: + all_hessians = rng.lognormal(size=n_samples).astype(G_H_DTYPE) + sum_hessians = all_hessians.sum() + + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, constant_hessian, n_threads + ) + n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + constant_hessian, + ) + + hists_parent = builder.compute_histograms_brute(sample_indices) + value_parent = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + si_parent = splitter.find_node_split( + n_samples, hists_parent, sum_gradients, sum_hessians, value_parent + ) + sample_indices_left, sample_indices_right, _ = splitter.split_indices( + si_parent, sample_indices + ) + + hists_left = builder.compute_histograms_brute(sample_indices_left) + value_left = compute_node_value( + si_parent.sum_gradient_left, + si_parent.sum_hessian_left, + -np.inf, + np.inf, + l2_regularization, + ) + hists_right = builder.compute_histograms_brute(sample_indices_right) + value_right = compute_node_value( + si_parent.sum_gradient_right, + si_parent.sum_hessian_right, + -np.inf, + np.inf, + l2_regularization, + ) + si_left = splitter.find_node_split( + n_samples, + hists_left, + si_parent.sum_gradient_left, + si_parent.sum_hessian_left, + value_left, + ) + si_right = splitter.find_node_split( + n_samples, + hists_right, + si_parent.sum_gradient_right, + si_parent.sum_hessian_right, + value_right, + ) + + # make sure that si.sum_gradient_left + si.sum_gradient_right have their + # expected value, same for hessians + for si, indices in ( + (si_parent, sample_indices), + (si_left, sample_indices_left), + (si_right, sample_indices_right), + ): + gradient = si.sum_gradient_right + si.sum_gradient_left + expected_gradient = all_gradients[indices].sum() + hessian = si.sum_hessian_right + si.sum_hessian_left + if constant_hessian: + expected_hessian = indices.shape[0] * all_hessians[0] + else: + expected_hessian = all_hessians[indices].sum() + + assert np.isclose(gradient, expected_gradient) + assert np.isclose(hessian, expected_hessian) + + # make sure sum of gradients in histograms are the same for all features, + # and make sure they're equal to their expected value + hists_parent = np.asarray(hists_parent, dtype=HISTOGRAM_DTYPE) + hists_left = np.asarray(hists_left, dtype=HISTOGRAM_DTYPE) + hists_right = np.asarray(hists_right, dtype=HISTOGRAM_DTYPE) + for hists, indices in ( + (hists_parent, sample_indices), + (hists_left, sample_indices_left), + (hists_right, sample_indices_right), + ): + # note: gradients and hessians have shape (n_features,), + # we're comparing them to *scalars*. This has the benefit of also + # making sure that all the entries are equal across features. + gradients = hists["sum_gradients"].sum(axis=1) # shape = (n_features,) + expected_gradient = all_gradients[indices].sum() # scalar + hessians = hists["sum_hessians"].sum(axis=1) + if constant_hessian: + # 0 is not the actual hessian, but it's not computed in this case + expected_hessian = 0.0 + else: + expected_hessian = all_hessians[indices].sum() + + assert np.allclose(gradients, expected_gradient) + assert np.allclose(hessians, expected_hessian) + + +def test_split_indices(): + # Check that split_indices returns the correct splits and that + # splitter.partition is consistent with what is returned. + rng = np.random.RandomState(421) + + n_bins = 5 + n_samples = 10 + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + + # split will happen on feature 1 and on bin 3 + X_binned = [ + [0, 0], + [0, 3], + [0, 4], + [0, 0], + [0, 0], + [0, 0], + [0, 0], + [0, 4], + [0, 0], + [0, 4], + ] + X_binned = np.asfortranarray(X_binned, dtype=X_BINNED_DTYPE) + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_gradients = rng.randn(n_samples).astype(G_H_DTYPE) + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_gradients = all_gradients.sum() + sum_hessians = 1 * n_samples + hessians_are_constant = True + + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + n_bins_non_missing = np.array([n_bins] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + assert np.all(sample_indices == splitter.partition) + + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + si_root = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) + + # sanity checks for best split + assert si_root.feature_idx == 1 + assert si_root.bin_idx == 3 + + samples_left, samples_right, position_right = splitter.split_indices( + si_root, splitter.partition + ) + assert set(samples_left) == set([0, 1, 3, 4, 5, 6, 8]) + assert set(samples_right) == set([2, 7, 9]) + + assert list(samples_left) == list(splitter.partition[:position_right]) + assert list(samples_right) == list(splitter.partition[position_right:]) + + # Check that the resulting split indices sizes are consistent with the + # count statistics anticipated when looking for the best split. + assert samples_left.shape[0] == si_root.n_samples_left + assert samples_right.shape[0] == si_root.n_samples_right + + +def test_min_gain_to_split(): + # Try to split a pure node (all gradients are equal, same for hessians) + # with min_gain_to_split = 0 and make sure that the node is not split (best + # possible gain = -1). Note: before the strict inequality comparison, this + # test would fail because the node would be split with a gain of 0. + rng = np.random.RandomState(42) + l2_regularization = 0 + min_hessian_to_split = 0 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + n_bins = 255 + n_samples = 100 + X_binned = np.asfortranarray( + rng.randint(0, n_bins, size=(n_samples, 1)), dtype=X_BINNED_DTYPE + ) + binned_feature = X_binned[:, 0] + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_hessians = np.ones_like(binned_feature, dtype=G_H_DTYPE) + all_gradients = np.ones_like(binned_feature, dtype=G_H_DTYPE) + sum_gradients = all_gradients.sum() + sum_hessians = all_hessians.sum() + hessians_are_constant = False + + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) + assert split_info.gain == -1 + + +@pytest.mark.parametrize( + ( + "X_binned, all_gradients, has_missing_values, n_bins_non_missing, " + " expected_split_on_nan, expected_bin_idx, expected_go_to_left" + ), + [ + # basic sanity check with no missing values: given the gradient + # values, the split must occur on bin_idx=3 + ( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], # X_binned + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], # gradients + False, # no missing values + 10, # n_bins_non_missing + False, # don't split on nans + 3, # expected_bin_idx + "not_applicable", + ), + # We replace 2 samples by NaNs (bin_idx=8) + # These 2 samples were mapped to the left node before, so they should + # be mapped to left node again + # Notice how the bin_idx threshold changes from 3 to 1. + ( + [8, 0, 1, 8, 2, 3, 4, 5, 6, 7], # 8 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 1, # cut on bin_idx=1 + True, + ), # missing values go to left + # same as above, but with non-consecutive missing_values_bin + ( + [9, 0, 1, 9, 2, 3, 4, 5, 6, 7], # 9 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 1, # cut on bin_idx=1 + True, + ), # missing values go to left + # this time replacing 2 samples that were on the right. + ( + [0, 1, 2, 3, 8, 4, 8, 5, 6, 7], # 8 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 3, # cut on bin_idx=3 (like in first case) + False, + ), # missing values go to right + # same as above, but with non-consecutive missing_values_bin + ( + [0, 1, 2, 3, 9, 4, 9, 5, 6, 7], # 9 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 3, # cut on bin_idx=3 (like in first case) + False, + ), # missing values go to right + # For the following cases, split_on_nans is True (we replace all of + # the samples with nans, instead of just 2). + ( + [0, 1, 2, 3, 4, 4, 4, 4, 4, 4], # 4 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 4, # n_bins_non_missing + True, # split on nans + 3, # cut on bin_idx=3 + False, + ), # missing values go to right + # same as above, but with non-consecutive missing_values_bin + ( + [0, 1, 2, 3, 9, 9, 9, 9, 9, 9], # 9 <=> missing + [1, 1, 1, 1, 1, 1, 5, 5, 5, 5], + True, # missing values + 4, # n_bins_non_missing + True, # split on nans + 3, # cut on bin_idx=3 + False, + ), # missing values go to right + ( + [6, 6, 6, 6, 0, 1, 2, 3, 4, 5], # 6 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 6, # n_bins_non_missing + True, # split on nans + 5, # cut on bin_idx=5 + False, + ), # missing values go to right + # same as above, but with non-consecutive missing_values_bin + ( + [9, 9, 9, 9, 0, 1, 2, 3, 4, 5], # 9 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 6, # n_bins_non_missing + True, # split on nans + 5, # cut on bin_idx=5 + False, + ), # missing values go to right + ], +) +def test_splitting_missing_values( + X_binned, + all_gradients, + has_missing_values, + n_bins_non_missing, + expected_split_on_nan, + expected_bin_idx, + expected_go_to_left, +): + # Make sure missing values are properly supported. + # we build an artificial example with gradients such that the best split + # is on bin_idx=3, when there are no missing values. + # Then we introduce missing values and: + # - make sure the chosen bin is correct (find_best_bin()): it's + # still the same split, even though the index of the bin may change + # - make sure the missing values are mapped to the correct child + # (split_indices()) + + n_bins = max(X_binned) + 1 + n_samples = len(X_binned) + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + + sample_indices = np.arange(n_samples, dtype=np.uint32) + X_binned = np.array(X_binned, dtype=X_BINNED_DTYPE).reshape(-1, 1) + X_binned = np.asfortranarray(X_binned) + all_gradients = np.array(all_gradients, dtype=G_H_DTYPE) + has_missing_values = np.array([has_missing_values], dtype=np.uint8) + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_gradients = all_gradients.sum() + sum_hessians = 1 * n_samples + hessians_are_constant = True + + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + + n_bins_non_missing = np.array([n_bins_non_missing], dtype=np.uint32) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) + + assert split_info.bin_idx == expected_bin_idx + if has_missing_values: + assert split_info.missing_go_to_left == expected_go_to_left + + split_on_nan = split_info.bin_idx == n_bins_non_missing[0] - 1 + assert split_on_nan == expected_split_on_nan + + # Make sure the split is properly computed. + # This also make sure missing values are properly assigned to the correct + # child in split_indices() + samples_left, samples_right, _ = splitter.split_indices( + split_info, splitter.partition + ) + + if not expected_split_on_nan: + # When we don't split on nans, the split should always be the same. + assert set(samples_left) == set([0, 1, 2, 3]) + assert set(samples_right) == set([4, 5, 6, 7, 8, 9]) + else: + # When we split on nans, samples with missing values are always mapped + # to the right child. + missing_samples_indices = np.flatnonzero( + np.array(X_binned) == missing_values_bin_idx + ) + non_missing_samples_indices = np.flatnonzero( + np.array(X_binned) != missing_values_bin_idx + ) + + assert set(samples_right) == set(missing_samples_indices) + assert set(samples_left) == set(non_missing_samples_indices) + + +@pytest.mark.parametrize( + "X_binned, has_missing_values, n_bins_non_missing, ", + [ + # one category + ([0] * 20, False, 1), + # all categories appear less than MIN_CAT_SUPPORT (hardcoded to 10) + ([0] * 9 + [1] * 8, False, 2), + # only one category appears more than MIN_CAT_SUPPORT + ([0] * 12 + [1] * 8, False, 2), + # missing values + category appear less than MIN_CAT_SUPPORT + # 9 is missing + ([0] * 9 + [1] * 8 + [9] * 4, True, 2), + # no non-missing category + ([9] * 11, True, 0), + ], +) +def test_splitting_categorical_cat_smooth( + X_binned, has_missing_values, n_bins_non_missing +): + # Checks categorical splits are correct when the MIN_CAT_SUPPORT constraint + # isn't respected: there are no splits + + n_bins = max(X_binned) + 1 + n_samples = len(X_binned) + X_binned = np.array([X_binned], dtype=X_BINNED_DTYPE).T + X_binned = np.asfortranarray(X_binned) + + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_gradients = np.ones(n_samples, dtype=G_H_DTYPE) + has_missing_values = np.array([has_missing_values], dtype=np.uint8) + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_gradients = all_gradients.sum() + sum_hessians = n_samples + hessians_are_constant = True + + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + + n_bins_non_missing = np.array([n_bins_non_missing], dtype=np.uint32) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.ones_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) + + # no split found + assert split_info.gain == -1 + + +def _assert_categories_equals_bitset(categories, bitset): + # assert that the bitset exactly corresponds to the categories + # bitset is assumed to be an array of 8 uint32 elements + + # form bitset from threshold + expected_bitset = np.zeros(8, dtype=np.uint32) + for cat in categories: + idx = cat // 32 + shift = cat % 32 + expected_bitset[idx] |= 1 << shift + + # check for equality + assert_array_equal(expected_bitset, bitset) + + +@pytest.mark.parametrize( + ( + "X_binned, all_gradients, expected_categories_left, n_bins_non_missing," + "missing_values_bin_idx, has_missing_values, expected_missing_go_to_left" + ), + [ + # 4 categories + ( + [0, 1, 2, 3] * 11, # X_binned + [10, 1, 10, 10] * 11, # all_gradients + [1], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked + # Make sure that the categories that are on the right (second half) of + # the sorted categories array can still go in the left child. In this + # case, the best split was found when scanning from right to left. + ( + [0, 1, 2, 3] * 11, # X_binned + [10, 10, 10, 1] * 11, # all_gradients + [3], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked + # categories that don't respect MIN_CAT_SUPPORT (cat 4) are always + # mapped to the right child + ( + [0, 1, 2, 3] * 11 + [4] * 5, # X_binned + [10, 10, 10, 1] * 11 + [10] * 5, # all_gradients + [3], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked + # categories that don't respect MIN_CAT_SUPPORT are always mapped to + # the right child: in this case a more sensible split could have been + # 3, 4 - 0, 1, 2 + # But the split is still 3 - 0, 1, 2, 4. this is because we only scan + # up to the middle of the sorted category array (0, 1, 2, 3), and + # because we exclude cat 4 in this array. + ( + [0, 1, 2, 3] * 11 + [4] * 5, # X_binned + [10, 10, 10, 1] * 11 + [1] * 5, # all_gradients + [3], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked + # 4 categories with missing values that go to the right + ( + [0, 1, 2] * 11 + [9] * 11, # X_binned + [10, 1, 10] * 11 + [10] * 11, # all_gradients + [1], # expected_categories_left + 3, # n_bins_non_missing + 9, # missing_values_bin_idx + True, # has_missing_values + False, + ), # expected_missing_go_to_left + # 4 categories with missing values that go to the left + ( + [0, 1, 2] * 11 + [9] * 11, # X_binned + [10, 1, 10] * 11 + [1] * 11, # all_gradients + [1, 9], # expected_categories_left + 3, # n_bins_non_missing + 9, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left + # split is on the missing value + ( + [0, 1, 2, 3, 4] * 11 + [255] * 12, # X_binned + [10, 10, 10, 10, 10] * 11 + [1] * 12, # all_gradients + [255], # expected_categories_left + 5, # n_bins_non_missing + 255, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left + # split on even categories + ( + list(range(60)) * 12, # X_binned + [10, 1] * 360, # all_gradients + list(range(1, 60, 2)), # expected_categories_left + 59, # n_bins_non_missing + 59, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left + # split on every 8 categories + ( + list(range(256)) * 12, # X_binned + [10, 10, 10, 10, 10, 10, 10, 1] * 384, # all_gradients + list(range(7, 256, 8)), # expected_categories_left + 255, # n_bins_non_missing + 255, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left + ], +) +def test_splitting_categorical_sanity( + X_binned, + all_gradients, + expected_categories_left, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + expected_missing_go_to_left, +): + # Tests various combinations of categorical splits + + n_samples = len(X_binned) + n_bins = max(X_binned) + 1 + + X_binned = np.array(X_binned, dtype=X_BINNED_DTYPE).reshape(-1, 1) + X_binned = np.asfortranarray(X_binned) + + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_gradients = np.array(all_gradients, dtype=G_H_DTYPE) + all_hessians = np.ones(1, dtype=G_H_DTYPE) + has_missing_values = np.array([has_missing_values], dtype=np.uint8) + sum_gradients = all_gradients.sum() + sum_hessians = n_samples + hessians_are_constant = True + + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + + n_bins_non_missing = np.array([n_bins_non_missing], dtype=np.uint32) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.ones_like(monotonic_cst, dtype=np.uint8) + + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + histograms = builder.compute_histograms_brute(sample_indices) + + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) + + assert split_info.is_categorical + assert split_info.gain > 0 + _assert_categories_equals_bitset( + expected_categories_left, split_info.left_cat_bitset + ) + if has_missing_values: + assert split_info.missing_go_to_left == expected_missing_go_to_left + # If there is no missing value during training, the flag missing_go_to_left + # is set later in the grower. + + # make sure samples are split correctly + samples_left, samples_right, _ = splitter.split_indices( + split_info, splitter.partition + ) + + left_mask = np.isin(X_binned.ravel(), expected_categories_left) + assert_array_equal(sample_indices[left_mask], samples_left) + assert_array_equal(sample_indices[~left_mask], samples_right) + + +def test_split_interaction_constraints(): + """Check that allowed_features are respected.""" + n_features = 4 + # features 1 and 2 are not allowed to be split on + allowed_features = np.array([0, 3], dtype=np.uint32) + n_bins = 5 + n_samples = 10 + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_hessians = n_samples + hessians_are_constant = True + + split_features = [] + + # The loop is to ensure that we split at least once on each allowed feature (0, 3). + # This is tracked by split_features and checked at the end. + for i in range(10): + rng = np.random.RandomState(919 + i) + X_binned = np.asfortranarray( + rng.randint(0, n_bins - 1, size=(n_samples, n_features)), + dtype=X_BINNED_DTYPE, + ) + X_binned = np.asfortranarray(X_binned, dtype=X_BINNED_DTYPE) + + # Make feature 1 very important + all_gradients = (10 * X_binned[:, 1] + rng.randn(n_samples)).astype(G_H_DTYPE) + sum_gradients = all_gradients.sum() + + builder = HistogramBuilder( + X_binned, + n_bins, + all_gradients, + all_hessians, + hessians_are_constant, + n_threads, + ) + n_bins_non_missing = np.array([n_bins] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + assert np.all(sample_indices == splitter.partition) + + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + + # with all features allowed, feature 1 should be split on as it is the most + # important one by construction of the gradients + si_root = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=None, + ) + assert si_root.feature_idx == 1 + + # only features 0 and 3 are allowed to be split on + si_root = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=allowed_features, + ) + split_features.append(si_root.feature_idx) + assert si_root.feature_idx in allowed_features + + # make sure feature 0 and feature 3 are split on in the constraint setting + assert set(allowed_features) == set(split_features) + + +@pytest.mark.parametrize("forbidden_features", [set(), {1, 3}]) +def test_split_feature_fraction_per_split(forbidden_features): + """Check that feature_fraction_per_split is respected. + + Because we set `n_features = 4` and `feature_fraction_per_split = 0.25`, it means + that calling `splitter.find_node_split` will be allowed to select a split for a + single completely random feature at each call. So if we iterate enough, we should + cover all the allowed features, irrespective of the values of the gradients and + Hessians of the objective. + """ + n_features = 4 + allowed_features = np.array( + list(set(range(n_features)) - forbidden_features), dtype=np.uint32 + ) + n_bins = 5 + n_samples = 40 + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + rng = np.random.default_rng(42) + + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_gradients = rng.uniform(low=0.5, high=1, size=n_samples).astype(G_H_DTYPE) + sum_gradients = all_gradients.sum() + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_hessians = n_samples + hessians_are_constant = True + + X_binned = np.asfortranarray( + rng.integers(low=0, high=n_bins - 1, size=(n_samples, n_features)), + dtype=X_BINNED_DTYPE, + ) + X_binned = np.asfortranarray(X_binned, dtype=X_BINNED_DTYPE) + builder = HistogramBuilder( + X_binned, + n_bins, + all_gradients, + all_hessians, + hessians_are_constant, + n_threads, + ) + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + n_bins_non_missing = np.array([n_bins] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + + params = dict( + X_binned=X_binned, + n_bins_non_missing=n_bins_non_missing, + missing_values_bin_idx=missing_values_bin_idx, + has_missing_values=has_missing_values, + is_categorical=is_categorical, + monotonic_cst=monotonic_cst, + l2_regularization=l2_regularization, + min_hessian_to_split=min_hessian_to_split, + min_samples_leaf=min_samples_leaf, + min_gain_to_split=min_gain_to_split, + hessians_are_constant=hessians_are_constant, + rng=rng, + ) + splitter_subsample = Splitter( + feature_fraction_per_split=0.25, # THIS is the important setting here. + **params, + ) + splitter_all_features = Splitter(feature_fraction_per_split=1.0, **params) + + assert np.all(sample_indices == splitter_subsample.partition) + + split_features_subsample = [] + split_features_all = [] + # The loop is to ensure that we split at least once on each feature. + # This is tracked by split_features and checked at the end. + for i in range(20): + si_root = splitter_subsample.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=allowed_features, + ) + split_features_subsample.append(si_root.feature_idx) + + # This second splitter is our "counterfactual". + si_root = splitter_all_features.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=allowed_features, + ) + split_features_all.append(si_root.feature_idx) + + # Make sure all features are split on. + assert set(split_features_subsample) == set(allowed_features) + + # Make sure, our counterfactual always splits on same feature. + assert len(set(split_features_all)) == 1 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd9d48c86287b29efe8b043d23aa13f16896637 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py @@ -0,0 +1,231 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +from sklearn.base import clone +from sklearn.datasets import make_classification, make_regression +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.metrics import check_scoring + +X_classification, y_classification = make_classification(random_state=0) +X_regression, y_regression = make_regression(random_state=0) + + +def _assert_predictor_equal(gb_1, gb_2, X): + """Assert that two HistGBM instances are identical.""" + # Check identical nodes for each tree + for pred_ith_1, pred_ith_2 in zip(gb_1._predictors, gb_2._predictors): + for predictor_1, predictor_2 in zip(pred_ith_1, pred_ith_2): + assert_array_equal(predictor_1.nodes, predictor_2.nodes) + + # Check identical predictions + assert_allclose(gb_1.predict(X), gb_2.predict(X)) + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +def test_max_iter_with_warm_start_validation(GradientBoosting, X, y): + # Check that a ValueError is raised when the maximum number of iterations + # is smaller than the number of iterations from the previous fit when warm + # start is True. + + estimator = GradientBoosting(max_iter=10, early_stopping=False, warm_start=True) + estimator.fit(X, y) + estimator.set_params(max_iter=5) + err_msg = ( + "max_iter=5 must be larger than or equal to n_iter_=10 when warm_start==True" + ) + with pytest.raises(ValueError, match=err_msg): + estimator.fit(X, y) + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +def test_warm_start_yields_identical_results(GradientBoosting, X, y): + # Make sure that fitting 50 iterations and then 25 with warm start is + # equivalent to fitting 75 iterations. + + rng = 42 + gb_warm_start = GradientBoosting( + n_iter_no_change=100, max_iter=50, random_state=rng, warm_start=True + ) + gb_warm_start.fit(X, y).set_params(max_iter=75).fit(X, y) + + gb_no_warm_start = GradientBoosting( + n_iter_no_change=100, max_iter=75, random_state=rng, warm_start=False + ) + gb_no_warm_start.fit(X, y) + + # Check that both predictors are equal + _assert_predictor_equal(gb_warm_start, gb_no_warm_start, X) + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +def test_warm_start_max_depth(GradientBoosting, X, y): + # Test if possible to fit trees of different depth in ensemble. + gb = GradientBoosting( + max_iter=20, + min_samples_leaf=1, + warm_start=True, + max_depth=2, + early_stopping=False, + ) + gb.fit(X, y) + gb.set_params(max_iter=30, max_depth=3, n_iter_no_change=110) + gb.fit(X, y) + + # First 20 trees have max_depth == 2 + for i in range(20): + assert gb._predictors[i][0].get_max_depth() == 2 + # Last 10 trees have max_depth == 3 + for i in range(1, 11): + assert gb._predictors[-i][0].get_max_depth() == 3 + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +@pytest.mark.parametrize("scoring", (None, "loss")) +def test_warm_start_early_stopping(GradientBoosting, X, y, scoring): + # Make sure that early stopping occurs after a small number of iterations + # when fitting a second time with warm starting. + + n_iter_no_change = 5 + gb = GradientBoosting( + n_iter_no_change=n_iter_no_change, + max_iter=10000, + early_stopping=True, + random_state=42, + warm_start=True, + tol=1e-3, + scoring=scoring, + ) + gb.fit(X, y) + n_iter_first_fit = gb.n_iter_ + gb.fit(X, y) + n_iter_second_fit = gb.n_iter_ + assert 0 < n_iter_second_fit - n_iter_first_fit < n_iter_no_change + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +def test_warm_start_equal_n_estimators(GradientBoosting, X, y): + # Test if warm start with equal n_estimators does nothing + gb_1 = GradientBoosting(max_depth=2, early_stopping=False) + gb_1.fit(X, y) + + gb_2 = clone(gb_1) + gb_2.set_params(max_iter=gb_1.max_iter, warm_start=True, n_iter_no_change=5) + gb_2.fit(X, y) + + # Check that both predictors are equal + _assert_predictor_equal(gb_1, gb_2, X) + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +def test_warm_start_clear(GradientBoosting, X, y): + # Test if fit clears state. + gb_1 = GradientBoosting(n_iter_no_change=5, random_state=42) + gb_1.fit(X, y) + + gb_2 = GradientBoosting(n_iter_no_change=5, random_state=42, warm_start=True) + gb_2.fit(X, y) # inits state + gb_2.set_params(warm_start=False) + gb_2.fit(X, y) # clears old state and equals est + + # Check that both predictors have the same train_score_ and + # validation_score_ attributes + assert_allclose(gb_1.train_score_, gb_2.train_score_) + assert_allclose(gb_1.validation_score_, gb_2.validation_score_) + + # Check that both predictors are equal + _assert_predictor_equal(gb_1, gb_2, X) + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +@pytest.mark.parametrize("rng_type", ("none", "int", "instance")) +def test_random_seeds_warm_start(GradientBoosting, X, y, rng_type): + # Make sure the seeds for train/val split and small trainset subsampling + # are correctly set in a warm start context. + def _get_rng(rng_type): + # Helper to avoid consuming rngs + if rng_type == "none": + return None + elif rng_type == "int": + return 42 + else: + return np.random.RandomState(0) + + random_state = _get_rng(rng_type) + gb_1 = GradientBoosting(early_stopping=True, max_iter=2, random_state=random_state) + gb_1.set_params(scoring=check_scoring(gb_1)) + gb_1.fit(X, y) + random_seed_1_1 = gb_1._random_seed + + gb_1.fit(X, y) + random_seed_1_2 = gb_1._random_seed # clear the old state, different seed + + random_state = _get_rng(rng_type) + gb_2 = GradientBoosting( + early_stopping=True, max_iter=2, random_state=random_state, warm_start=True + ) + gb_2.set_params(scoring=check_scoring(gb_2)) + gb_2.fit(X, y) # inits state + random_seed_2_1 = gb_2._random_seed + gb_2.fit(X, y) # clears old state and equals est + random_seed_2_2 = gb_2._random_seed + + # Without warm starting, the seeds should be + # * all different if random state is None + # * all equal if random state is an integer + # * different when refitting and equal with a new estimator (because + # the random state is mutated) + if rng_type == "none": + assert random_seed_1_1 != random_seed_1_2 != random_seed_2_1 + elif rng_type == "int": + assert random_seed_1_1 == random_seed_1_2 == random_seed_2_1 + else: + assert random_seed_1_1 == random_seed_2_1 != random_seed_1_2 + + # With warm starting, the seeds must be equal + assert random_seed_2_1 == random_seed_2_2 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/utils.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..cf01f7ebcd15eef8e67756b8f45eda41720358a0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/utils.py @@ -0,0 +1,149 @@ +"""This module contains utility routines.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from sklearn.base import is_classifier +from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper + + +def get_equivalent_estimator(estimator, lib="lightgbm", n_classes=None): + """Return an unfitted estimator from another lib with matching hyperparams. + + This utility function takes care of renaming the sklearn parameters into + their LightGBM, XGBoost or CatBoost equivalent parameters. + + # unmapped XGB parameters: + # - min_samples_leaf + # - min_data_in_bin + # - min_split_gain (there is min_split_loss though?) + + # unmapped Catboost parameters: + # max_leaves + # min_* + """ + + if lib not in ("lightgbm", "xgboost", "catboost"): + raise ValueError( + "accepted libs are lightgbm, xgboost, and catboost. got {}".format(lib) + ) + + sklearn_params = estimator.get_params() + + if sklearn_params["loss"] == "auto": + raise ValueError( + "auto loss is not accepted. We need to know if " + "the problem is binary or multiclass classification." + ) + if sklearn_params["early_stopping"]: + raise NotImplementedError("Early stopping should be deactivated.") + + lightgbm_loss_mapping = { + "squared_error": "regression_l2", + "absolute_error": "regression_l1", + "log_loss": "binary" if n_classes == 2 else "multiclass", + "gamma": "gamma", + "poisson": "poisson", + } + + lightgbm_params = { + "objective": lightgbm_loss_mapping[sklearn_params["loss"]], + "learning_rate": sklearn_params["learning_rate"], + "n_estimators": sklearn_params["max_iter"], + "num_leaves": sklearn_params["max_leaf_nodes"], + "max_depth": sklearn_params["max_depth"], + "min_data_in_leaf": sklearn_params["min_samples_leaf"], + "reg_lambda": sklearn_params["l2_regularization"], + "max_bin": sklearn_params["max_bins"], + "min_data_in_bin": 1, + "min_sum_hessian_in_leaf": 1e-3, + "min_split_gain": 0, + "verbosity": 10 if sklearn_params["verbose"] else -10, + "boost_from_average": True, + "enable_bundle": False, # also makes feature order consistent + "subsample_for_bin": _BinMapper().subsample, + "poisson_max_delta_step": 1e-12, + "feature_fraction_bynode": sklearn_params["max_features"], + } + + if sklearn_params["loss"] == "log_loss" and n_classes > 2: + # LightGBM multiplies hessians by 2 in multiclass loss. + lightgbm_params["min_sum_hessian_in_leaf"] *= 2 + # LightGBM 3.0 introduced a different scaling of the hessian for the multiclass + # case. + # It is equivalent of scaling the learning rate. + # See https://github.com/microsoft/LightGBM/pull/3256. + if n_classes is not None: + lightgbm_params["learning_rate"] *= n_classes / (n_classes - 1) + + # XGB + xgboost_loss_mapping = { + "squared_error": "reg:linear", + "absolute_error": "LEAST_ABSOLUTE_DEV_NOT_SUPPORTED", + "log_loss": "reg:logistic" if n_classes == 2 else "multi:softmax", + "gamma": "reg:gamma", + "poisson": "count:poisson", + } + + xgboost_params = { + "tree_method": "hist", + "grow_policy": "lossguide", # so that we can set max_leaves + "objective": xgboost_loss_mapping[sklearn_params["loss"]], + "learning_rate": sklearn_params["learning_rate"], + "n_estimators": sklearn_params["max_iter"], + "max_leaves": sklearn_params["max_leaf_nodes"], + "max_depth": sklearn_params["max_depth"] or 0, + "lambda": sklearn_params["l2_regularization"], + "max_bin": sklearn_params["max_bins"], + "min_child_weight": 1e-3, + "verbosity": 2 if sklearn_params["verbose"] else 0, + "silent": sklearn_params["verbose"] == 0, + "n_jobs": -1, + "colsample_bynode": sklearn_params["max_features"], + } + + # Catboost + catboost_loss_mapping = { + "squared_error": "RMSE", + # catboost does not support MAE when leaf_estimation_method is Newton + "absolute_error": "LEAST_ASBOLUTE_DEV_NOT_SUPPORTED", + "log_loss": "Logloss" if n_classes == 2 else "MultiClass", + "gamma": None, + "poisson": "Poisson", + } + + catboost_params = { + "loss_function": catboost_loss_mapping[sklearn_params["loss"]], + "learning_rate": sklearn_params["learning_rate"], + "iterations": sklearn_params["max_iter"], + "depth": sklearn_params["max_depth"], + "reg_lambda": sklearn_params["l2_regularization"], + "max_bin": sklearn_params["max_bins"], + "feature_border_type": "Median", + "leaf_estimation_method": "Newton", + "verbose": bool(sklearn_params["verbose"]), + } + + if lib == "lightgbm": + from lightgbm import LGBMClassifier, LGBMRegressor + + if is_classifier(estimator): + return LGBMClassifier(**lightgbm_params) + else: + return LGBMRegressor(**lightgbm_params) + + elif lib == "xgboost": + from xgboost import XGBClassifier, XGBRegressor + + if is_classifier(estimator): + return XGBClassifier(**xgboost_params) + else: + return XGBRegressor(**xgboost_params) + + else: + from catboost import CatBoostClassifier, CatBoostRegressor + + if is_classifier(estimator): + return CatBoostClassifier(**catboost_params) + else: + return CatBoostRegressor(**catboost_params) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_iforest.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_iforest.py new file mode 100644 index 0000000000000000000000000000000000000000..35d46a2a269184cc2a01bc14e0f630790b0b5d5c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_iforest.py @@ -0,0 +1,681 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import numbers +import threading +from numbers import Integral, Real +from warnings import warn + +import numpy as np +from scipy.sparse import issparse + +from sklearn.base import OutlierMixin, _fit_context +from sklearn.ensemble._bagging import BaseBagging +from sklearn.tree import ExtraTreeRegressor +from sklearn.tree._tree import DTYPE as tree_dtype +from sklearn.utils import check_array, check_random_state, gen_batches +from sklearn.utils._chunking import get_chunk_n_rows +from sklearn.utils._param_validation import Interval, RealNotInt, StrOptions +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.validation import ( + _check_sample_weight, + _num_samples, + check_is_fitted, + validate_data, +) + +__all__ = ["IsolationForest"] + + +def _parallel_compute_tree_depths( + tree, + X, + features, + tree_decision_path_lengths, + tree_avg_path_lengths, + depths, + lock, +): + """Parallel computation of isolation tree depth.""" + if features is None: + X_subset = X + else: + X_subset = X[:, features] + + leaves_index = tree.apply(X_subset, check_input=False) + + with lock: + depths += ( + tree_decision_path_lengths[leaves_index] + + tree_avg_path_lengths[leaves_index] + - 1.0 + ) + + +class IsolationForest(OutlierMixin, BaseBagging): + """ + Isolation Forest Algorithm. + + Return the anomaly score of each sample using the IsolationForest algorithm + + The IsolationForest 'isolates' observations by randomly selecting a feature + and then randomly selecting a split value between the maximum and minimum + values of the selected feature. + + Since recursive partitioning can be represented by a tree structure, the + number of splittings required to isolate a sample is equivalent to the path + length from the root node to the terminating node. + + This path length, averaged over a forest of such random trees, is a + measure of normality and our decision function. + + Random partitioning produces noticeably shorter paths for anomalies. + Hence, when a forest of random trees collectively produce shorter path + lengths for particular samples, they are highly likely to be anomalies. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.18 + + Parameters + ---------- + n_estimators : int, default=100 + The number of base estimators in the ensemble. + + max_samples : "auto", int or float, default="auto" + The number of samples to draw from X to train each base estimator. + + - If int, then draw `max_samples` samples. + - If float, then draw `max_samples * X.shape[0]` samples. + - If "auto", then `max_samples=min(256, n_samples)`. + + If max_samples is larger than the number of samples provided, + all samples will be used for all trees (no sampling). + + contamination : 'auto' or float, default='auto' + The amount of contamination of the data set, i.e. the proportion + of outliers in the data set. Used when fitting to define the threshold + on the scores of the samples. + + - If 'auto', the threshold is determined as in the + original paper. + - If float, the contamination should be in the range (0, 0.5]. + + .. versionchanged:: 0.22 + The default value of ``contamination`` changed from 0.1 + to ``'auto'``. + + max_features : int or float, default=1.0 + The number of features to draw from X to train each base estimator. + + - If int, then draw `max_features` features. + - If float, then draw `max(1, int(max_features * n_features_in_))` features. + + Note: using a float number less than 1.0 or integer less than number of + features will enable feature subsampling and leads to a longer runtime. + + bootstrap : bool, default=False + If True, individual trees are fit on random subsets of the training + data sampled with replacement. If False, sampling without replacement + is performed. + + n_jobs : int, default=None + The number of jobs to run in parallel for :meth:`fit`. ``None`` means 1 + unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using + all processors. See :term:`Glossary ` for more details. + + random_state : int, RandomState instance or None, default=None + Controls the pseudo-randomness of the selection of the feature + and split values for each branching step and each tree in the forest. + + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. + + verbose : int, default=0 + Controls the verbosity of the tree building process. + + warm_start : bool, default=False + When set to ``True``, reuse the solution of the previous call to fit + and add more estimators to the ensemble, otherwise, just fit a whole + new forest. See :term:`the Glossary `. + + .. versionadded:: 0.21 + + Attributes + ---------- + estimator_ : :class:`~sklearn.tree.ExtraTreeRegressor` instance + The child estimator template used to create the collection of + fitted sub-estimators. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of ExtraTreeRegressor instances + The collection of fitted sub-estimators. + + estimators_features_ : list of ndarray + The subset of drawn features for each base estimator. + + estimators_samples_ : list of ndarray + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. + + max_samples_ : int + The actual number of samples. + + offset_ : float + Offset used to define the decision function from the raw scores. We + have the relation: ``decision_function = score_samples - offset_``. + ``offset_`` is defined as follows. When the contamination parameter is + set to "auto", the offset is equal to -0.5 as the scores of inliers are + close to 0 and the scores of outliers are close to -1. When a + contamination parameter different than "auto" is provided, the offset + is defined in such a way we obtain the expected number of outliers + (samples with decision function < 0) in training. + + .. versionadded:: 0.20 + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + sklearn.covariance.EllipticEnvelope : An object for detecting outliers in a + Gaussian distributed dataset. + sklearn.svm.OneClassSVM : Unsupervised Outlier Detection. + Estimate the support of a high-dimensional distribution. + The implementation is based on libsvm. + sklearn.neighbors.LocalOutlierFactor : Unsupervised Outlier Detection + using Local Outlier Factor (LOF). + + Notes + ----- + The implementation is based on an ensemble of ExtraTreeRegressor. The + maximum depth of each tree is set to ``ceil(log_2(n))`` where + :math:`n` is the number of samples used to build the tree + (see [1]_ for more details). + + References + ---------- + .. [1] F. T. Liu, K. M. Ting and Z. -H. Zhou. + :doi:`"Isolation forest." <10.1109/ICDM.2008.17>` + 2008 Eighth IEEE International Conference on Data Mining (ICDM), + 2008, pp. 413-422. + .. [2] F. T. Liu, K. M. Ting and Z. -H. Zhou. + :doi:`"Isolation-based anomaly detection." + <10.1145/2133360.2133363>` ACM Transactions on + Knowledge Discovery from Data (TKDD) 6.1 (2012): 1-39. + + Examples + -------- + >>> from sklearn.ensemble import IsolationForest + >>> X = [[-1.1], [0.3], [0.5], [100]] + >>> clf = IsolationForest(random_state=0).fit(X) + >>> clf.predict([[0.1], [0], [90]]) + array([ 1, 1, -1]) + + For an example of using isolation forest for anomaly detection see + :ref:`sphx_glr_auto_examples_ensemble_plot_isolation_forest.py`. + """ + + _parameter_constraints: dict = { + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "max_samples": [ + StrOptions({"auto"}), + Interval(Integral, 1, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="right"), + ], + "contamination": [ + StrOptions({"auto"}), + Interval(Real, 0, 0.5, closed="right"), + ], + "max_features": [ + Integral, + Interval(Real, 0, 1, closed="right"), + ], + "bootstrap": ["boolean"], + "n_jobs": [Integral, None], + "random_state": ["random_state"], + "verbose": ["verbose"], + "warm_start": ["boolean"], + } + + def __init__( + self, + *, + n_estimators=100, + max_samples="auto", + contamination="auto", + max_features=1.0, + bootstrap=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ): + super().__init__( + estimator=None, + # here above max_features has no links with self.max_features + bootstrap=bootstrap, + bootstrap_features=False, + n_estimators=n_estimators, + max_samples=max_samples, + max_features=max_features, + warm_start=warm_start, + n_jobs=n_jobs, + random_state=random_state, + verbose=verbose, + ) + + self.contamination = contamination + + def _get_estimator(self): + return ExtraTreeRegressor( + # here max_features has no links with self.max_features + max_features=1, + splitter="random", + random_state=self.random_state, + ) + + def _set_oob_score(self, X, y): + raise NotImplementedError("OOB score not supported by iforest") + + def _parallel_args(self): + # ExtraTreeRegressor releases the GIL, so it's more efficient to use + # a thread-based backend rather than a process-based backend so as + # to avoid suffering from communication overhead and extra memory + # copies. This is only used in the fit method. + return {"prefer": "threads"} + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None, sample_weight=None): + """ + Fit estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Use ``dtype=np.float32`` for maximum + efficiency. Sparse matrices are also supported, use sparse + ``csc_matrix`` for maximum efficiency. + + y : Ignored + Not used, present for API consistency by convention. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, then samples are equally weighted. + + Returns + ------- + self : object + Fitted estimator. + """ + X = validate_data( + self, X, accept_sparse=["csc"], dtype=tree_dtype, ensure_all_finite=False + ) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, dtype=None) + + if issparse(X): + # Pre-sort indices to avoid that each individual tree of the + # ensemble sorts the indices. + X.sort_indices() + + rnd = check_random_state(self.random_state) + y = rnd.uniform(size=X.shape[0]) + + # ensure that max_sample is in [1, n_samples]: + n_samples = X.shape[0] + + if isinstance(self.max_samples, str) and self.max_samples == "auto": + max_samples = min(256, n_samples) + + elif isinstance(self.max_samples, numbers.Integral): + if self.max_samples > n_samples: + warn( + "max_samples (%s) is greater than the " + "total number of samples (%s). max_samples " + "will be set to n_samples for estimation." + % (self.max_samples, n_samples) + ) + max_samples = n_samples + else: + max_samples = self.max_samples + else: # max_samples is float + max_samples = int(self.max_samples * X.shape[0]) + + self.max_samples_ = max_samples + max_depth = int(np.ceil(np.log2(max(max_samples, 2)))) + super()._fit( + X, + y, + max_samples=max_samples, + max_depth=max_depth, + sample_weight=sample_weight, + check_input=False, + ) + + self._average_path_length_per_tree, self._decision_path_lengths = zip( + *[ + ( + _average_path_length(tree.tree_.n_node_samples), + tree.tree_.compute_node_depths(), + ) + for tree in self.estimators_ + ] + ) + + if self.contamination == "auto": + # 0.5 plays a special role as described in the original paper. + # we take the opposite as we consider the opposite of their score. + self.offset_ = -0.5 + return self + + # Else, define offset_ wrt contamination parameter + # To avoid performing input validation a second time we call + # _score_samples rather than score_samples. + # _score_samples expects a CSR matrix, so we convert if necessary. + if issparse(X): + X = X.tocsr() + self.offset_ = np.percentile(self._score_samples(X), 100.0 * self.contamination) + + return self + + def predict(self, X): + """ + Predict if a particular sample is an outlier or not. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Returns + ------- + is_inlier : ndarray of shape (n_samples,) + For each observation, tells whether or not (+1 or -1) it should + be considered as an inlier according to the fitted model. + + Notes + ----- + The predict method can be parallelized by setting a joblib context. This + inherently does NOT use the ``n_jobs`` parameter initialized in the class, + which is used during ``fit``. This is because, predict may actually be faster + without parallelization for a small number of samples, + such as for 1000 samples or less. The user can set the + number of jobs in the joblib context to control the number of parallel jobs. + + .. code-block:: python + + from joblib import parallel_backend + + # Note, we use threading here as the predict method is not CPU bound. + with parallel_backend("threading", n_jobs=4): + model.predict(X) + """ + check_is_fitted(self) + decision_func = self.decision_function(X) + is_inlier = np.ones_like(decision_func, dtype=int) + is_inlier[decision_func < 0] = -1 + return is_inlier + + def decision_function(self, X): + """ + Average anomaly score of X of the base classifiers. + + The anomaly score of an input sample is computed as + the mean anomaly score of the trees in the forest. + + The measure of normality of an observation given a tree is the depth + of the leaf containing this observation, which is equivalent to + the number of splittings required to isolate this point. In case of + several observations n_left in the leaf, the average path length of + an n_left samples isolation tree is added. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. + + Returns + ------- + scores : ndarray of shape (n_samples,) + The anomaly score of the input samples. + The lower, the more abnormal. Negative scores represent outliers, + positive scores represent inliers. + + Notes + ----- + The decision_function method can be parallelized by setting a joblib context. + This inherently does NOT use the ``n_jobs`` parameter initialized in the class, + which is used during ``fit``. This is because, calculating the score may + actually be faster without parallelization for a small number of samples, + such as for 1000 samples or less. + The user can set the number of jobs in the joblib context to control the + number of parallel jobs. + + .. code-block:: python + + from joblib import parallel_backend + + # Note, we use threading here as the decision_function method is + # not CPU bound. + with parallel_backend("threading", n_jobs=4): + model.decision_function(X) + """ + # We subtract self.offset_ to make 0 be the threshold value for being + # an outlier: + + return self.score_samples(X) - self.offset_ + + def score_samples(self, X): + """ + Opposite of the anomaly score defined in the original paper. + + The anomaly score of an input sample is computed as + the mean anomaly score of the trees in the forest. + + The measure of normality of an observation given a tree is the depth + of the leaf containing this observation, which is equivalent to + the number of splittings required to isolate this point. In case of + several observations n_left in the leaf, the average path length of + an n_left samples isolation tree is added. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. + + Returns + ------- + scores : ndarray of shape (n_samples,) + The anomaly score of the input samples. + The lower, the more abnormal. + + Notes + ----- + The score function method can be parallelized by setting a joblib context. This + inherently does NOT use the ``n_jobs`` parameter initialized in the class, + which is used during ``fit``. This is because, calculating the score may + actually be faster without parallelization for a small number of samples, + such as for 1000 samples or less. + The user can set the number of jobs in the joblib context to control the + number of parallel jobs. + + .. code-block:: python + + from joblib import parallel_backend + + # Note, we use threading here as the score_samples method is not CPU bound. + with parallel_backend("threading", n_jobs=4): + model.score(X) + """ + # Check data + X = validate_data( + self, + X, + accept_sparse="csr", + dtype=tree_dtype, + reset=False, + ensure_all_finite=False, + ) + + return self._score_samples(X) + + def _score_samples(self, X): + """Private version of score_samples without input validation. + + Input validation would remove feature names, so we disable it. + """ + # Code structure from ForestClassifier/predict_proba + + check_is_fitted(self) + + # Take the opposite of the scores as bigger is better (here less abnormal) + return -self._compute_chunked_score_samples(X) + + def _compute_chunked_score_samples(self, X): + n_samples = _num_samples(X) + + if self._max_features == X.shape[1]: + subsample_features = False + else: + subsample_features = True + + # We get as many rows as possible within our working_memory budget + # (defined by sklearn.get_config()['working_memory']) to store + # self._max_features in each row during computation. + # + # Note: + # - this will get at least 1 row, even if 1 row of score will + # exceed working_memory. + # - this does only account for temporary memory usage while loading + # the data needed to compute the scores -- the returned scores + # themselves are 1D. + + chunk_n_rows = get_chunk_n_rows( + row_bytes=16 * self._max_features, max_n_rows=n_samples + ) + slices = gen_batches(n_samples, chunk_n_rows) + + scores = np.zeros(n_samples, order="f") + + for sl in slices: + # compute score on the slices of test samples: + scores[sl] = self._compute_score_samples(X[sl], subsample_features) + + return scores + + def _compute_score_samples(self, X, subsample_features): + """ + Compute the score of each samples in X going through the extra trees. + + Parameters + ---------- + X : array-like or sparse matrix + Data matrix. + + subsample_features : bool + Whether features should be subsampled. + + Returns + ------- + scores : ndarray of shape (n_samples,) + The score of each sample in X. + """ + n_samples = X.shape[0] + + depths = np.zeros(n_samples, order="f") + + average_path_length_max_samples = _average_path_length([self._max_samples]) + + # Note: we use default n_jobs value, i.e. sequential computation, which + # we expect to be more performant that parallelizing for small number + # of samples, e.g. < 1k samples. Default n_jobs value can be overridden + # by using joblib.parallel_backend context manager around + # ._compute_score_samples. Using a higher n_jobs may speed up the + # computation of the scores, e.g. for > 1k samples. See + # https://github.com/scikit-learn/scikit-learn/pull/28622 for more + # details. + lock = threading.Lock() + Parallel( + verbose=self.verbose, + require="sharedmem", + )( + delayed(_parallel_compute_tree_depths)( + tree, + X, + features if subsample_features else None, + self._decision_path_lengths[tree_idx], + self._average_path_length_per_tree[tree_idx], + depths, + lock, + ) + for tree_idx, (tree, features) in enumerate( + zip(self.estimators_, self.estimators_features_) + ) + ) + + denominator = len(self.estimators_) * average_path_length_max_samples + scores = 2 ** ( + # For a single training sample, denominator and depth are 0. + # Therefore, we set the score manually to 1. + -np.divide( + depths, denominator, out=np.ones_like(depths), where=denominator != 0 + ) + ) + return scores + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.allow_nan = True + return tags + + +def _average_path_length(n_samples_leaf): + """ + The average path length in an n_samples iTree, which is equal to + the average path length of an unsuccessful BST search since the + latter has the same structure as an isolation tree. + Parameters + ---------- + n_samples_leaf : array-like of shape (n_samples,) + The number of training samples in each test sample leaf, for + each estimators. + + Returns + ------- + average_path_length : ndarray of shape (n_samples,) + """ + + n_samples_leaf = check_array(n_samples_leaf, ensure_2d=False) + + n_samples_leaf_shape = n_samples_leaf.shape + n_samples_leaf = n_samples_leaf.reshape((1, -1)) + average_path_length = np.zeros(n_samples_leaf.shape) + + mask_1 = n_samples_leaf <= 1 + mask_2 = n_samples_leaf == 2 + not_mask = ~np.logical_or(mask_1, mask_2) + + average_path_length[mask_1] = 0.0 + average_path_length[mask_2] = 1.0 + average_path_length[not_mask] = ( + 2.0 * (np.log(n_samples_leaf[not_mask] - 1.0) + np.euler_gamma) + - 2.0 * (n_samples_leaf[not_mask] - 1.0) / n_samples_leaf[not_mask] + ) + + return average_path_length.reshape(n_samples_leaf_shape) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_stacking.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_stacking.py new file mode 100644 index 0000000000000000000000000000000000000000..1ab7327656ca3e012f5e3caf8f37b6f54be533b7 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_stacking.py @@ -0,0 +1,1145 @@ +"""Stacking classifier and regressor.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from abc import ABCMeta, abstractmethod +from copy import deepcopy +from numbers import Integral + +import numpy as np +import scipy.sparse as sparse + +from sklearn.base import ( + ClassifierMixin, + RegressorMixin, + TransformerMixin, + _fit_context, + clone, + is_classifier, + is_regressor, +) +from sklearn.ensemble._base import _BaseHeterogeneousEnsemble, _fit_single_estimator +from sklearn.exceptions import NotFittedError +from sklearn.linear_model import LogisticRegression, RidgeCV +from sklearn.model_selection import check_cv, cross_val_predict +from sklearn.preprocessing import LabelEncoder +from sklearn.utils import Bunch +from sklearn.utils._param_validation import HasMethods, StrOptions +from sklearn.utils._repr_html.estimator import _VisualBlock +from sklearn.utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + process_routing, +) +from sklearn.utils.metaestimators import available_if +from sklearn.utils.multiclass import check_classification_targets, type_of_target +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.validation import ( + _check_feature_names_in, + _check_response_method, + _estimator_has, + check_is_fitted, + column_or_1d, +) + + +class _BaseStacking(TransformerMixin, _BaseHeterogeneousEnsemble, metaclass=ABCMeta): + """Base class for stacking method.""" + + _parameter_constraints: dict = { + "estimators": [list], + "final_estimator": [None, HasMethods("fit")], + "cv": ["cv_object", StrOptions({"prefit"})], + "n_jobs": [None, Integral], + "passthrough": ["boolean"], + "verbose": ["verbose"], + } + + @abstractmethod + def __init__( + self, + estimators, + final_estimator=None, + *, + cv=None, + stack_method="auto", + n_jobs=None, + verbose=0, + passthrough=False, + ): + super().__init__(estimators=estimators) + self.final_estimator = final_estimator + self.cv = cv + self.stack_method = stack_method + self.n_jobs = n_jobs + self.verbose = verbose + self.passthrough = passthrough + + def _clone_final_estimator(self, default): + if self.final_estimator is not None: + self.final_estimator_ = clone(self.final_estimator) + else: + self.final_estimator_ = clone(default) + + def _concatenate_predictions(self, X, predictions): + """Concatenate the predictions of each first layer learner and + possibly the input dataset `X`. + + If `X` is sparse and `self.passthrough` is False, the output of + `transform` will be dense (the predictions). If `X` is sparse + and `self.passthrough` is True, the output of `transform` will + be sparse. + + This helper is in charge of ensuring the predictions are 2D arrays and + it will drop one of the probability column when using probabilities + in the binary case. Indeed, the p(y|c=0) = 1 - p(y|c=1) + + When `y` type is `"multilabel-indicator"`` and the method used is + `predict_proba`, `preds` can be either a `ndarray` of shape + `(n_samples, n_class)` or for some estimators a list of `ndarray`. + This function will drop one of the probability column in this situation as well. + """ + X_meta = [] + for est_idx, preds in enumerate(predictions): + if isinstance(preds, list): + # `preds` is here a list of `n_targets` 2D ndarrays of + # `n_classes` columns. The k-th column contains the + # probabilities of the samples belonging the k-th class. + # + # Since those probabilities must sum to one for each sample, + # we can work with probabilities of `n_classes - 1` classes. + # Hence we drop the first column. + for pred in preds: + X_meta.append(pred[:, 1:]) + elif preds.ndim == 1: + # Some estimator return a 1D array for predictions + # which must be 2-dimensional arrays. + X_meta.append(preds.reshape(-1, 1)) + elif ( + self.stack_method_[est_idx] == "predict_proba" + and len(self.classes_) == 2 + ): + # Remove the first column when using probabilities in + # binary classification because both features `preds` are perfectly + # collinear. + X_meta.append(preds[:, 1:]) + else: + X_meta.append(preds) + + self._n_feature_outs = [pred.shape[1] for pred in X_meta] + if self.passthrough: + X_meta.append(X) + if sparse.issparse(X): + return sparse.hstack(X_meta, format=X.format) + + return np.hstack(X_meta) + + @staticmethod + def _method_name(name, estimator, method): + if estimator == "drop": + return None + if method == "auto": + method = ["predict_proba", "decision_function", "predict"] + try: + method_name = _check_response_method(estimator, method).__name__ + except AttributeError as e: + raise ValueError( + f"Underlying estimator {name} does not implement the method {method}." + ) from e + + return method_name + + @_fit_context( + # estimators in Stacking*.estimators are not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, **fit_params): + """Fit the estimators. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values. + + **fit_params : dict + Dict of metadata, potentially containing sample_weight as a + key-value pair. If sample_weight is not present, then samples are + equally weighted. Note that sample_weight is supported only if all + underlying estimators support sample weights. + + .. versionadded:: 1.6 + + Returns + ------- + self : object + """ + # all_estimators contains all estimators, the one to be fitted and the + # 'drop' string. + names, all_estimators = self._validate_estimators() + self._validate_final_estimator() + + stack_method = [self.stack_method] * len(all_estimators) + + if _routing_enabled(): + routed_params = process_routing(self, "fit", **fit_params) + else: + routed_params = Bunch() + for name in names: + routed_params[name] = Bunch(fit={}) + if "sample_weight" in fit_params: + routed_params[name].fit["sample_weight"] = fit_params[ + "sample_weight" + ] + + if self.cv == "prefit": + self.estimators_ = [] + for estimator in all_estimators: + if estimator != "drop": + check_is_fitted(estimator) + self.estimators_.append(estimator) + else: + # Fit the base estimators on the whole training data. Those + # base estimators will be used in transform, predict, and + # predict_proba. They are exposed publicly. + self.estimators_ = Parallel(n_jobs=self.n_jobs)( + delayed(_fit_single_estimator)( + clone(est), X, y, routed_params[name]["fit"] + ) + for name, est in zip(names, all_estimators) + if est != "drop" + ) + + self.named_estimators_ = Bunch() + est_fitted_idx = 0 + for name_est, org_est in zip(names, all_estimators): + if org_est != "drop": + current_estimator = self.estimators_[est_fitted_idx] + self.named_estimators_[name_est] = current_estimator + est_fitted_idx += 1 + if hasattr(current_estimator, "feature_names_in_"): + self.feature_names_in_ = current_estimator.feature_names_in_ + else: + self.named_estimators_[name_est] = "drop" + + self.stack_method_ = [ + self._method_name(name, est, meth) + for name, est, meth in zip(names, all_estimators, stack_method) + ] + + if self.cv == "prefit": + # Generate predictions from prefit models + predictions = [ + getattr(estimator, predict_method)(X) + for estimator, predict_method in zip(all_estimators, self.stack_method_) + if estimator != "drop" + ] + else: + # To train the meta-classifier using the most data as possible, we use + # a cross-validation to obtain the output of the stacked estimators. + # To ensure that the data provided to each estimator are the same, + # we need to set the random state of the cv if there is one and we + # need to take a copy. + cv = check_cv(self.cv, y=y, classifier=is_classifier(self)) + if hasattr(cv, "random_state") and cv.random_state is None: + cv.random_state = np.random.RandomState() + + predictions = Parallel(n_jobs=self.n_jobs)( + delayed(cross_val_predict)( + clone(est), + X, + y, + cv=deepcopy(cv), + method=meth, + n_jobs=self.n_jobs, + params=routed_params[name]["fit"], + verbose=self.verbose, + ) + for name, est, meth in zip(names, all_estimators, self.stack_method_) + if est != "drop" + ) + + # Only not None or not 'drop' estimators will be used in transform. + # Remove the None from the method as well. + self.stack_method_ = [ + meth + for (meth, est) in zip(self.stack_method_, all_estimators) + if est != "drop" + ] + + X_meta = self._concatenate_predictions(X, predictions) + _fit_single_estimator(self.final_estimator_, X_meta, y, fit_params=fit_params) + + return self + + @property + def n_features_in_(self): + """Number of features seen during :term:`fit`.""" + try: + check_is_fitted(self) + except NotFittedError as nfe: + raise AttributeError( + f"{self.__class__.__name__} object has no attribute n_features_in_" + ) from nfe + return self.estimators_[0].n_features_in_ + + def _transform(self, X): + """Concatenate and return the predictions of the estimators.""" + check_is_fitted(self) + predictions = [ + getattr(est, meth)(X) + for est, meth in zip(self.estimators_, self.stack_method_) + if est != "drop" + ] + return self._concatenate_predictions(X, predictions) + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Input features. The input feature names are only used when `passthrough` is + `True`. + + - If `input_features` is `None`, then `feature_names_in_` is + used as feature names in. If `feature_names_in_` is not defined, + then names are generated: `[x0, x1, ..., x(n_features_in_ - 1)]`. + - If `input_features` is an array-like, then `input_features` must + match `feature_names_in_` if `feature_names_in_` is defined. + + If `passthrough` is `False`, then only the names of `estimators` are used + to generate the output feature names. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names. + """ + check_is_fitted(self, "n_features_in_") + input_features = _check_feature_names_in( + self, input_features, generate_names=self.passthrough + ) + + class_name = self.__class__.__name__.lower() + non_dropped_estimators = ( + name for name, est in self.estimators if est != "drop" + ) + meta_names = [] + for est, n_features_out in zip(non_dropped_estimators, self._n_feature_outs): + if n_features_out == 1: + meta_names.append(f"{class_name}_{est}") + else: + meta_names.extend( + f"{class_name}_{est}{i}" for i in range(n_features_out) + ) + + if self.passthrough: + return np.concatenate((meta_names, input_features)) + + return np.asarray(meta_names, dtype=object) + + @available_if( + _estimator_has("predict", delegates=("final_estimator_", "final_estimator")) + ) + def predict(self, X, **predict_params): + """Predict target for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + **predict_params : dict of str -> obj + Parameters to the `predict` called by the `final_estimator`. Note + that this may be used to return uncertainties from some estimators + with `return_std` or `return_cov`. Be aware that it will only + account for uncertainty in the final estimator. + + Returns + ------- + y_pred : ndarray of shape (n_samples,) or (n_samples, n_output) + Predicted targets. + """ + + check_is_fitted(self) + return self.final_estimator_.predict(self.transform(X), **predict_params) + + def _sk_visual_block_with_final_estimator(self, final_estimator): + names, estimators = zip(*self.estimators) + parallel = _VisualBlock("parallel", estimators, names=names, dash_wrapped=False) + + # final estimator is wrapped in a parallel block to show the label: + # 'final_estimator' in the html repr + final_block = _VisualBlock( + "parallel", [final_estimator], names=["final_estimator"], dash_wrapped=False + ) + return _VisualBlock("serial", (parallel, final_block), dash_wrapped=False) + + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.6 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self) + + # `self.estimators` is a list of (name, est) tuples + for name, estimator in self.estimators: + router.add( + **{name: estimator}, + method_mapping=MethodMapping().add(callee="fit", caller="fit"), + ) + + try: + final_estimator_ = self.final_estimator_ + except AttributeError: + final_estimator_ = self.final_estimator + + router.add( + final_estimator_=final_estimator_, + method_mapping=MethodMapping().add(caller="predict", callee="predict"), + ) + + return router + + +class StackingClassifier(ClassifierMixin, _BaseStacking): + """Stack of estimators with a final classifier. + + Stacked generalization consists in stacking the output of individual + estimator and use a classifier to compute the final prediction. Stacking + allows to use the strength of each individual estimator by using their + output as input of a final estimator. + + Note that `estimators_` are fitted on the full `X` while `final_estimator_` + is trained using cross-validated predictions of the base estimators using + `cross_val_predict`. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.22 + + Parameters + ---------- + estimators : list of (str, estimator) + Base estimators which will be stacked together. Each element of the + list is defined as a tuple of string (i.e. name) and an estimator + instance. An estimator can be set to 'drop' using `set_params`. + + The type of estimator is generally expected to be a classifier. + However, one can pass a regressor for some use case (e.g. ordinal + regression). + + final_estimator : estimator, default=None + A classifier which will be used to combine the base estimators. + The default classifier is a + :class:`~sklearn.linear_model.LogisticRegression`. + + cv : int, cross-validation generator, iterable, or "prefit", default=None + Determines the cross-validation splitting strategy used in + `cross_val_predict` to train `final_estimator`. Possible inputs for + cv are: + + * None, to use the default 5-fold cross validation, + * integer, to specify the number of folds in a (Stratified) KFold, + * An object to be used as a cross-validation generator, + * An iterable yielding train, test splits, + * `"prefit"`, to assume the `estimators` are prefit. In this case, the + estimators will not be refitted. + + For integer/None inputs, if the estimator is a classifier and y is + either binary or multiclass, + :class:`~sklearn.model_selection.StratifiedKFold` is used. + In all other cases, :class:`~sklearn.model_selection.KFold` is used. + These splitters are instantiated with `shuffle=False` so the splits + will be the same across calls. + + Refer :ref:`User Guide ` for the various + cross-validation strategies that can be used here. + + If "prefit" is passed, it is assumed that all `estimators` have + been fitted already. The `final_estimator_` is trained on the `estimators` + predictions on the full training set and are **not** cross validated + predictions. Please note that if the models have been trained on the same + data to train the stacking model, there is a very high risk of overfitting. + + .. versionadded:: 1.1 + The 'prefit' option was added in 1.1 + + .. note:: + A larger number of split will provide no benefits if the number + of training samples is large enough. Indeed, the training time + will increase. ``cv`` is not used for model evaluation but for + prediction. + + stack_method : {'auto', 'predict_proba', 'decision_function', 'predict'}, \ + default='auto' + Methods called for each base estimator. It can be: + + * if 'auto', it will try to invoke, for each estimator, + `'predict_proba'`, `'decision_function'` or `'predict'` in that + order. + * otherwise, one of `'predict_proba'`, `'decision_function'` or + `'predict'`. If the method is not implemented by the estimator, it + will raise an error. + + n_jobs : int, default=None + The number of jobs to run in parallel for `fit` of all `estimators`. + `None` means 1 unless in a `joblib.parallel_backend` context. -1 means + using all processors. See :term:`Glossary ` for more details. + + passthrough : bool, default=False + When False, only the predictions of estimators will be used as + training data for `final_estimator`. When True, the + `final_estimator` is trained on the predictions as well as the + original training data. + + verbose : int, default=0 + Verbosity level. + + Attributes + ---------- + classes_ : ndarray of shape (n_classes,) or list of ndarray if `y` \ + is of type `"multilabel-indicator"`. + Class labels. + + estimators_ : list of estimators + The elements of the `estimators` parameter, having been fitted on the + training data. If an estimator has been set to `'drop'`, it + will not appear in `estimators_`. When `cv="prefit"`, `estimators_` + is set to `estimators` and is not fitted again. + + named_estimators_ : :class:`~sklearn.utils.Bunch` + Attribute to access any fitted sub-estimators by name. + + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying estimator exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 + + final_estimator_ : estimator + The classifier fit on the output of `estimators_` and responsible for + final predictions. + + stack_method_ : list of str + The method used by each base estimator. + + See Also + -------- + StackingRegressor : Stack of estimators with a final regressor. + + Notes + ----- + When `predict_proba` is used by each estimator (i.e. most of the time for + `stack_method='auto'` or specifically for `stack_method='predict_proba'`), + the first column predicted by each estimator will be dropped in the case + of a binary classification problem. Indeed, both feature will be perfectly + collinear. + + In some cases (e.g. ordinal regression), one can pass regressors as the + first layer of the :class:`StackingClassifier`. However, note that `y` will + be internally encoded in a numerically increasing order or lexicographic + order. If this ordering is not adequate, one should manually numerically + encode the classes in the desired order. + + References + ---------- + .. [1] Wolpert, David H. "Stacked generalization." Neural networks 5.2 + (1992): 241-259. + + Examples + -------- + >>> from sklearn.datasets import load_iris + >>> from sklearn.ensemble import RandomForestClassifier + >>> from sklearn.svm import LinearSVC + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.preprocessing import StandardScaler + >>> from sklearn.pipeline import make_pipeline + >>> from sklearn.ensemble import StackingClassifier + >>> X, y = load_iris(return_X_y=True) + >>> estimators = [ + ... ('rf', RandomForestClassifier(n_estimators=10, random_state=42)), + ... ('svr', make_pipeline(StandardScaler(), + ... LinearSVC(random_state=42))) + ... ] + >>> clf = StackingClassifier( + ... estimators=estimators, final_estimator=LogisticRegression() + ... ) + >>> from sklearn.model_selection import train_test_split + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, stratify=y, random_state=42 + ... ) + >>> clf.fit(X_train, y_train).score(X_test, y_test) + 0.9... + """ + + _parameter_constraints: dict = { + **_BaseStacking._parameter_constraints, + "stack_method": [ + StrOptions({"auto", "predict_proba", "decision_function", "predict"}) + ], + } + + def __init__( + self, + estimators, + final_estimator=None, + *, + cv=None, + stack_method="auto", + n_jobs=None, + passthrough=False, + verbose=0, + ): + super().__init__( + estimators=estimators, + final_estimator=final_estimator, + cv=cv, + stack_method=stack_method, + n_jobs=n_jobs, + passthrough=passthrough, + verbose=verbose, + ) + + def _validate_final_estimator(self): + self._clone_final_estimator(default=LogisticRegression()) + if not is_classifier(self.final_estimator_): + raise ValueError( + "'final_estimator' parameter should be a classifier. Got {}".format( + self.final_estimator_ + ) + ) + + def _validate_estimators(self): + """Overload the method of `_BaseHeterogeneousEnsemble` to be more + lenient towards the type of `estimators`. + + Regressors can be accepted for some cases such as ordinal regression. + """ + if len(self.estimators) == 0: + raise ValueError( + "Invalid 'estimators' attribute, 'estimators' should be a " + "non-empty list of (string, estimator) tuples." + ) + names, estimators = zip(*self.estimators) + self._validate_names(names) + + has_estimator = any(est != "drop" for est in estimators) + if not has_estimator: + raise ValueError( + "All estimators are dropped. At least one is required " + "to be an estimator." + ) + + return names, estimators + + def fit(self, X, y, **fit_params): + """Fit the estimators. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values. Note that `y` will be internally encoded in + numerically increasing order or lexicographic order. If the order + matter (e.g. for ordinal regression), one should numerically encode + the target `y` before calling :term:`fit`. + + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.6 + + Only available if `enable_metadata_routing=True`, which can be + set by using ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + + Returns + ------- + self : object + Returns a fitted instance of estimator. + """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + + check_classification_targets(y) + if type_of_target(y) == "multilabel-indicator": + self._label_encoder = [LabelEncoder().fit(yk) for yk in y.T] + self.classes_ = [le.classes_ for le in self._label_encoder] + y_encoded = np.array( + [ + self._label_encoder[target_idx].transform(target) + for target_idx, target in enumerate(y.T) + ] + ).T + else: + self._label_encoder = LabelEncoder().fit(y) + self.classes_ = self._label_encoder.classes_ + y_encoded = self._label_encoder.transform(y) + + return super().fit(X, y_encoded, **fit_params) + + @available_if( + _estimator_has("predict", delegates=("final_estimator_", "final_estimator")) + ) + def predict(self, X, **predict_params): + """Predict target for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + **predict_params : dict of str -> obj + Parameters to the `predict` called by the `final_estimator`. Note + that this may be used to return uncertainties from some estimators + with `return_std` or `return_cov`. Be aware that it will only + account for uncertainty in the final estimator. + + - If `enable_metadata_routing=False` (default): + Parameters directly passed to the `predict` method of the + `final_estimator`. + + - If `enable_metadata_routing=True`: Parameters safely routed to + the `predict` method of the `final_estimator`. See :ref:`Metadata + Routing User Guide ` for more details. + + .. versionchanged:: 1.6 + `**predict_params` can be routed via metadata routing API. + + Returns + ------- + y_pred : ndarray of shape (n_samples,) or (n_samples, n_output) + Predicted targets. + """ + if _routing_enabled(): + routed_params = process_routing(self, "predict", **predict_params) + else: + # TODO(SLEP6): remove when metadata routing cannot be disabled. + routed_params = Bunch() + routed_params.final_estimator_ = Bunch(predict={}) + routed_params.final_estimator_.predict = predict_params + + y_pred = super().predict(X, **routed_params.final_estimator_["predict"]) + if isinstance(self._label_encoder, list): + # Handle the multilabel-indicator case + y_pred = np.array( + [ + self._label_encoder[target_idx].inverse_transform(target) + for target_idx, target in enumerate(y_pred.T) + ] + ).T + else: + y_pred = self._label_encoder.inverse_transform(y_pred) + return y_pred + + @available_if( + _estimator_has( + "predict_proba", delegates=("final_estimator_", "final_estimator") + ) + ) + def predict_proba(self, X): + """Predict class probabilities for `X` using the final estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + probabilities : ndarray of shape (n_samples, n_classes) or \ + list of ndarray of shape (n_output,) + The class probabilities of the input samples. + """ + check_is_fitted(self) + y_pred = self.final_estimator_.predict_proba(self.transform(X)) + + if isinstance(self._label_encoder, list): + # Handle the multilabel-indicator cases + y_pred = np.array([preds[:, 0] for preds in y_pred]).T + return y_pred + + @available_if( + _estimator_has( + "decision_function", delegates=("final_estimator_", "final_estimator") + ) + ) + def decision_function(self, X): + """Decision function for samples in `X` using the final estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + decisions : ndarray of shape (n_samples,), (n_samples, n_classes), \ + or (n_samples, n_classes * (n_classes-1) / 2) + The decision function computed the final estimator. + """ + check_is_fitted(self) + return self.final_estimator_.decision_function(self.transform(X)) + + def transform(self, X): + """Return class labels or probabilities for X for each estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + y_preds : ndarray of shape (n_samples, n_estimators) or \ + (n_samples, n_classes * n_estimators) + Prediction outputs for each estimator. + """ + return self._transform(X) + + def _sk_visual_block_(self): + # If final_estimator's default changes then this should be + # updated. + if self.final_estimator is None: + final_estimator = LogisticRegression() + else: + final_estimator = self.final_estimator + return super()._sk_visual_block_with_final_estimator(final_estimator) + + +class StackingRegressor(RegressorMixin, _BaseStacking): + """Stack of estimators with a final regressor. + + Stacked generalization consists in stacking the output of individual + estimator and use a regressor to compute the final prediction. Stacking + allows to use the strength of each individual estimator by using their + output as input of a final estimator. + + Note that `estimators_` are fitted on the full `X` while `final_estimator_` + is trained using cross-validated predictions of the base estimators using + `cross_val_predict`. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.22 + + Parameters + ---------- + estimators : list of (str, estimator) + Base estimators which will be stacked together. Each element of the + list is defined as a tuple of string (i.e. name) and an estimator + instance. An estimator can be set to 'drop' using `set_params`. + + final_estimator : estimator, default=None + A regressor which will be used to combine the base estimators. + The default regressor is a :class:`~sklearn.linear_model.RidgeCV`. + + cv : int, cross-validation generator, iterable, or "prefit", default=None + Determines the cross-validation splitting strategy used in + `cross_val_predict` to train `final_estimator`. Possible inputs for + cv are: + + * None, to use the default 5-fold cross validation, + * integer, to specify the number of folds in a (Stratified) KFold, + * An object to be used as a cross-validation generator, + * An iterable yielding train, test splits, + * `"prefit"`, to assume the `estimators` are prefit. In this case, the + estimators will not be refitted. + + For integer/None inputs, if the estimator is a classifier and y is + either binary or multiclass, + :class:`~sklearn.model_selection.StratifiedKFold` is used. + In all other cases, :class:`~sklearn.model_selection.KFold` is used. + These splitters are instantiated with `shuffle=False` so the splits + will be the same across calls. + + Refer :ref:`User Guide ` for the various + cross-validation strategies that can be used here. + + If "prefit" is passed, it is assumed that all `estimators` have + been fitted already. The `final_estimator_` is trained on the `estimators` + predictions on the full training set and are **not** cross validated + predictions. Please note that if the models have been trained on the same + data to train the stacking model, there is a very high risk of overfitting. + + .. versionadded:: 1.1 + The 'prefit' option was added in 1.1 + + .. note:: + A larger number of split will provide no benefits if the number + of training samples is large enough. Indeed, the training time + will increase. ``cv`` is not used for model evaluation but for + prediction. + + n_jobs : int, default=None + The number of jobs to run in parallel for `fit` of all `estimators`. + `None` means 1 unless in a `joblib.parallel_backend` context. -1 means + using all processors. See :term:`Glossary ` for more details. + + passthrough : bool, default=False + When False, only the predictions of estimators will be used as + training data for `final_estimator`. When True, the + `final_estimator` is trained on the predictions as well as the + original training data. + + verbose : int, default=0 + Verbosity level. + + Attributes + ---------- + estimators_ : list of estimators + The elements of the `estimators` parameter, having been fitted on the + training data. If an estimator has been set to `'drop'`, it + will not appear in `estimators_`. When `cv="prefit"`, `estimators_` + is set to `estimators` and is not fitted again. + + named_estimators_ : :class:`~sklearn.utils.Bunch` + Attribute to access any fitted sub-estimators by name. + + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying estimator exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 + + final_estimator_ : estimator + The regressor fit on the output of `estimators_` and responsible for + final predictions. + + stack_method_ : list of str + The method used by each base estimator. + + See Also + -------- + StackingClassifier : Stack of estimators with a final classifier. + + References + ---------- + .. [1] Wolpert, David H. "Stacked generalization." Neural networks 5.2 + (1992): 241-259. + + Examples + -------- + >>> from sklearn.datasets import load_diabetes + >>> from sklearn.linear_model import RidgeCV + >>> from sklearn.svm import LinearSVR + >>> from sklearn.ensemble import RandomForestRegressor + >>> from sklearn.ensemble import StackingRegressor + >>> X, y = load_diabetes(return_X_y=True) + >>> estimators = [ + ... ('lr', RidgeCV()), + ... ('svr', LinearSVR(random_state=42)) + ... ] + >>> reg = StackingRegressor( + ... estimators=estimators, + ... final_estimator=RandomForestRegressor(n_estimators=10, + ... random_state=42) + ... ) + >>> from sklearn.model_selection import train_test_split + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=42 + ... ) + >>> reg.fit(X_train, y_train).score(X_test, y_test) + 0.3... + """ + + def __init__( + self, + estimators, + final_estimator=None, + *, + cv=None, + n_jobs=None, + passthrough=False, + verbose=0, + ): + super().__init__( + estimators=estimators, + final_estimator=final_estimator, + cv=cv, + stack_method="predict", + n_jobs=n_jobs, + passthrough=passthrough, + verbose=verbose, + ) + + def _validate_final_estimator(self): + self._clone_final_estimator(default=RidgeCV()) + if not is_regressor(self.final_estimator_): + raise ValueError( + "'final_estimator' parameter should be a regressor. Got {}".format( + self.final_estimator_ + ) + ) + + def fit(self, X, y, **fit_params): + """Fit the estimators. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values. + + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.6 + + Only available if `enable_metadata_routing=True`, which can be + set by using ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + + Returns + ------- + self : object + Returns a fitted instance. + """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + + y = column_or_1d(y, warn=True) + + return super().fit(X, y, **fit_params) + + def transform(self, X): + """Return the predictions for X for each estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + y_preds : ndarray of shape (n_samples, n_estimators) + Prediction outputs for each estimator. + """ + return self._transform(X) + + def fit_transform(self, X, y, **fit_params): + """Fit the estimators and return the predictions for X for each estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values. + + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.6 + + Only available if `enable_metadata_routing=True`, which can be + set by using ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + + Returns + ------- + y_preds : ndarray of shape (n_samples, n_estimators) + Prediction outputs for each estimator. + """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + + return super().fit_transform(X, y, **fit_params) + + @available_if( + _estimator_has("predict", delegates=("final_estimator_", "final_estimator")) + ) + def predict(self, X, **predict_params): + """Predict target for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + **predict_params : dict of str -> obj + Parameters to the `predict` called by the `final_estimator`. Note + that this may be used to return uncertainties from some estimators + with `return_std` or `return_cov`. Be aware that it will only + account for uncertainty in the final estimator. + + - If `enable_metadata_routing=False` (default): + Parameters directly passed to the `predict` method of the + `final_estimator`. + + - If `enable_metadata_routing=True`: Parameters safely routed to + the `predict` method of the `final_estimator`. See :ref:`Metadata + Routing User Guide ` for more details. + + .. versionchanged:: 1.6 + `**predict_params` can be routed via metadata routing API. + + Returns + ------- + y_pred : ndarray of shape (n_samples,) or (n_samples, n_output) + Predicted targets. + """ + if _routing_enabled(): + routed_params = process_routing(self, "predict", **predict_params) + else: + # TODO(SLEP6): remove when metadata routing cannot be disabled. + routed_params = Bunch() + routed_params.final_estimator_ = Bunch(predict={}) + routed_params.final_estimator_.predict = predict_params + + y_pred = super().predict(X, **routed_params.final_estimator_["predict"]) + + return y_pred + + def _sk_visual_block_(self): + # If final_estimator's default changes then this should be + # updated. + if self.final_estimator is None: + final_estimator = RidgeCV() + else: + final_estimator = self.final_estimator + return super()._sk_visual_block_with_final_estimator(final_estimator) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_voting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_voting.py new file mode 100644 index 0000000000000000000000000000000000000000..00a59c35d124a82c80092297c0bb097e758cb2d8 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_voting.py @@ -0,0 +1,734 @@ +""" +Soft Voting/Majority Rule classifier and Voting regressor. + +This module contains: + - A Soft Voting/Majority Rule classifier for classification estimators. + - A Voting regressor for regression estimators. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from abc import abstractmethod +from numbers import Integral + +import numpy as np + +from sklearn.base import ( + ClassifierMixin, + RegressorMixin, + TransformerMixin, + _fit_context, + clone, +) +from sklearn.ensemble._base import _BaseHeterogeneousEnsemble, _fit_single_estimator +from sklearn.exceptions import NotFittedError +from sklearn.preprocessing import LabelEncoder +from sklearn.utils import Bunch +from sklearn.utils._param_validation import StrOptions +from sklearn.utils._repr_html.estimator import _VisualBlock +from sklearn.utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + process_routing, +) +from sklearn.utils.metaestimators import available_if +from sklearn.utils.multiclass import type_of_target +from sklearn.utils.parallel import Parallel, delayed +from sklearn.utils.validation import ( + _check_feature_names_in, + check_is_fitted, + column_or_1d, +) + + +class _BaseVoting(TransformerMixin, _BaseHeterogeneousEnsemble): + """Base class for voting. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + _parameter_constraints: dict = { + "estimators": [list], + "weights": ["array-like", None], + "n_jobs": [None, Integral], + "verbose": ["verbose"], + } + + def _log_message(self, name, idx, total): + if not self.verbose: + return None + return f"({idx} of {total}) Processing {name}" + + @property + def _weights_not_none(self): + """Get the weights of not `None` estimators.""" + if self.weights is None: + return None + return [w for est, w in zip(self.estimators, self.weights) if est[1] != "drop"] + + def _predict(self, X): + """Collect results from clf.predict calls.""" + return np.asarray([est.predict(X) for est in self.estimators_]).T + + @abstractmethod + def fit(self, X, y, **fit_params): + """Get common fit operations.""" + names, clfs = self._validate_estimators() + + if self.weights is not None and len(self.weights) != len(self.estimators): + raise ValueError( + "Number of `estimators` and weights must be equal; got" + f" {len(self.weights)} weights, {len(self.estimators)} estimators" + ) + + if _routing_enabled(): + routed_params = process_routing(self, "fit", **fit_params) + else: + routed_params = Bunch() + for name in names: + routed_params[name] = Bunch(fit={}) + if "sample_weight" in fit_params: + routed_params[name].fit["sample_weight"] = fit_params[ + "sample_weight" + ] + + self.estimators_ = Parallel(n_jobs=self.n_jobs)( + delayed(_fit_single_estimator)( + clone(clf), + X, + y, + fit_params=routed_params[name]["fit"], + message_clsname="Voting", + message=self._log_message(name, idx + 1, len(clfs)), + ) + for idx, (name, clf) in enumerate(zip(names, clfs)) + if clf != "drop" + ) + + self.named_estimators_ = Bunch() + + # Uses 'drop' as placeholder for dropped estimators + est_iter = iter(self.estimators_) + for name, est in self.estimators: + current_est = est if est == "drop" else next(est_iter) + self.named_estimators_[name] = current_est + + if hasattr(current_est, "feature_names_in_"): + self.feature_names_in_ = current_est.feature_names_in_ + + return self + + def fit_transform(self, X, y=None, **fit_params): + """Return class labels or probabilities for each estimator. + + Return predictions for X for each estimator. + + Parameters + ---------- + X : {array-like, sparse matrix, dataframe} of shape \ + (n_samples, n_features) + Input samples. + + y : ndarray of shape (n_samples,), default=None + Target values (None for unsupervised transformations). + + **fit_params : dict + Additional fit parameters. + + Returns + ------- + X_new : ndarray array of shape (n_samples, n_features_new) + Transformed array. + """ + return super().fit_transform(X, y, **fit_params) + + @property + def n_features_in_(self): + """Number of features seen during :term:`fit`.""" + # For consistency with other estimators we raise an AttributeError so + # that hasattr() fails if the estimator isn't fitted. + try: + check_is_fitted(self) + except NotFittedError as nfe: + raise AttributeError( + "{} object has no n_features_in_ attribute.".format( + self.__class__.__name__ + ) + ) from nfe + + return self.estimators_[0].n_features_in_ + + def _sk_visual_block_(self): + names, estimators = zip(*self.estimators) + return _VisualBlock("parallel", estimators, names=names) + + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.5 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self) + + # `self.estimators` is a list of (name, est) tuples + for name, estimator in self.estimators: + router.add( + **{name: estimator}, + method_mapping=MethodMapping().add(callee="fit", caller="fit"), + ) + return router + + +class VotingClassifier(ClassifierMixin, _BaseVoting): + """Soft Voting/Majority Rule classifier for unfitted estimators. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.17 + + Parameters + ---------- + estimators : list of (str, estimator) tuples + Invoking the ``fit`` method on the ``VotingClassifier`` will fit clones + of those original estimators that will be stored in the class attribute + ``self.estimators_``. An estimator can be set to ``'drop'`` using + :meth:`set_params`. + + .. versionchanged:: 0.21 + ``'drop'`` is accepted. Using None was deprecated in 0.22 and + support was removed in 0.24. + + voting : {'hard', 'soft'}, default='hard' + If 'hard', uses predicted class labels for majority rule voting. + Else if 'soft', predicts the class label based on the argmax of + the sums of the predicted probabilities, which is recommended for + an ensemble of well-calibrated classifiers. + + weights : array-like of shape (n_classifiers,), default=None + Sequence of weights (`float` or `int`) to weight the occurrences of + predicted class labels (`hard` voting) or class probabilities + before averaging (`soft` voting). Uses uniform weights if `None`. + + n_jobs : int, default=None + The number of jobs to run in parallel for ``fit``. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + .. versionadded:: 0.18 + + flatten_transform : bool, default=True + Affects shape of transform output only when voting='soft' + If voting='soft' and flatten_transform=True, transform method returns + matrix with shape (n_samples, n_classifiers * n_classes). If + flatten_transform=False, it returns + (n_classifiers, n_samples, n_classes). + + verbose : bool, default=False + If True, the time elapsed while fitting will be printed as it + is completed. + + .. versionadded:: 0.23 + + Attributes + ---------- + estimators_ : list of classifiers + The collection of fitted sub-estimators as defined in ``estimators`` + that are not 'drop'. + + named_estimators_ : :class:`~sklearn.utils.Bunch` + Attribute to access any fitted sub-estimators by name. + + .. versionadded:: 0.20 + + le_ : :class:`~sklearn.preprocessing.LabelEncoder` + Transformer used to encode the labels during fit and decode during + prediction. + + classes_ : ndarray of shape (n_classes,) + The classes labels. + + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying classifier exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 + + See Also + -------- + VotingRegressor : Prediction voting regressor. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.naive_bayes import GaussianNB + >>> from sklearn.ensemble import RandomForestClassifier, VotingClassifier + >>> clf1 = LogisticRegression(random_state=1) + >>> clf2 = RandomForestClassifier(n_estimators=50, random_state=1) + >>> clf3 = GaussianNB() + >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) + >>> y = np.array([1, 1, 1, 2, 2, 2]) + >>> eclf1 = VotingClassifier(estimators=[ + ... ('lr', clf1), ('rf', clf2), ('gnb', clf3)], voting='hard') + >>> eclf1 = eclf1.fit(X, y) + >>> print(eclf1.predict(X)) + [1 1 1 2 2 2] + >>> np.array_equal(eclf1.named_estimators_.lr.predict(X), + ... eclf1.named_estimators_['lr'].predict(X)) + True + >>> eclf2 = VotingClassifier(estimators=[ + ... ('lr', clf1), ('rf', clf2), ('gnb', clf3)], + ... voting='soft') + >>> eclf2 = eclf2.fit(X, y) + >>> print(eclf2.predict(X)) + [1 1 1 2 2 2] + + To drop an estimator, :meth:`set_params` can be used to remove it. Here we + dropped one of the estimators, resulting in 2 fitted estimators: + + >>> eclf2 = eclf2.set_params(lr='drop') + >>> eclf2 = eclf2.fit(X, y) + >>> len(eclf2.estimators_) + 2 + + Setting `flatten_transform=True` with `voting='soft'` flattens output shape of + `transform`: + + >>> eclf3 = VotingClassifier(estimators=[ + ... ('lr', clf1), ('rf', clf2), ('gnb', clf3)], + ... voting='soft', weights=[2,1,1], + ... flatten_transform=True) + >>> eclf3 = eclf3.fit(X, y) + >>> print(eclf3.predict(X)) + [1 1 1 2 2 2] + >>> print(eclf3.transform(X).shape) + (6, 6) + """ + + _parameter_constraints: dict = { + **_BaseVoting._parameter_constraints, + "voting": [StrOptions({"hard", "soft"})], + "flatten_transform": ["boolean"], + } + + def __init__( + self, + estimators, + *, + voting="hard", + weights=None, + n_jobs=None, + flatten_transform=True, + verbose=False, + ): + super().__init__(estimators=estimators) + self.voting = voting + self.weights = weights + self.n_jobs = n_jobs + self.flatten_transform = flatten_transform + self.verbose = verbose + + @_fit_context( + # estimators in VotingClassifier.estimators are not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, **fit_params): + """Fit the estimators. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values. + + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.5 + + Only available if `enable_metadata_routing=True`, + which can be set by using + ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + + Returns + ------- + self : object + Returns the instance itself. + """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + + y_type = type_of_target(y, input_name="y") + if y_type in ("unknown", "continuous"): + # raise a specific ValueError for non-classification tasks + raise ValueError( + f"Unknown label type: {y_type}. Maybe you are trying to fit a " + "classifier, which expects discrete classes on a " + "regression target with continuous values." + ) + elif y_type not in ("binary", "multiclass"): + # raise a NotImplementedError for backward compatibility for non-supported + # classification tasks + raise NotImplementedError( + f"{self.__class__.__name__} only supports binary or multiclass " + "classification. Multilabel and multi-output classification are not " + "supported." + ) + + self.le_ = LabelEncoder().fit(y) + self.classes_ = self.le_.classes_ + transformed_y = self.le_.transform(y) + + return super().fit(X, transformed_y, **fit_params) + + def predict(self, X): + """Predict class labels for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. + + Returns + ------- + maj : array-like of shape (n_samples,) + Predicted class labels. + """ + check_is_fitted(self) + if self.voting == "soft": + maj = np.argmax(self.predict_proba(X), axis=1) + + else: # 'hard' voting + predictions = self._predict(X) + maj = np.apply_along_axis( + lambda x: np.argmax(np.bincount(x, weights=self._weights_not_none)), + axis=1, + arr=predictions, + ) + + maj = self.le_.inverse_transform(maj) + + return maj + + def _collect_probas(self, X): + """Collect results from clf.predict calls.""" + return np.asarray([clf.predict_proba(X) for clf in self.estimators_]) + + def _check_voting(self): + if self.voting == "hard": + raise AttributeError( + f"predict_proba is not available when voting={self.voting!r}" + ) + return True + + @available_if(_check_voting) + def predict_proba(self, X): + """Compute probabilities of possible outcomes for samples in X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. + + Returns + ------- + avg : array-like of shape (n_samples, n_classes) + Weighted average probability for each class per sample. + """ + check_is_fitted(self) + avg = np.average( + self._collect_probas(X), axis=0, weights=self._weights_not_none + ) + return avg + + def transform(self, X): + """Return class labels or probabilities for X for each estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + probabilities_or_labels + If `voting='soft'` and `flatten_transform=True`: + returns ndarray of shape (n_samples, n_classifiers * n_classes), + being class probabilities calculated by each classifier. + If `voting='soft' and `flatten_transform=False`: + ndarray of shape (n_classifiers, n_samples, n_classes) + If `voting='hard'`: + ndarray of shape (n_samples, n_classifiers), being + class labels predicted by each classifier. + """ + check_is_fitted(self) + + if self.voting == "soft": + probas = self._collect_probas(X) + if not self.flatten_transform: + return probas + return np.hstack(probas) + + else: + return self._predict(X) + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Not used, present here for API consistency by convention. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names. + """ + check_is_fitted(self, "n_features_in_") + if self.voting == "soft" and not self.flatten_transform: + raise ValueError( + "get_feature_names_out is not supported when `voting='soft'` and " + "`flatten_transform=False`" + ) + + _check_feature_names_in(self, input_features, generate_names=False) + class_name = self.__class__.__name__.lower() + + active_names = [name for name, est in self.estimators if est != "drop"] + + if self.voting == "hard": + return np.asarray( + [f"{class_name}_{name}" for name in active_names], dtype=object + ) + + # voting == "soft" + n_classes = len(self.classes_) + names_out = [ + f"{class_name}_{name}{i}" for name in active_names for i in range(n_classes) + ] + return np.asarray(names_out, dtype=object) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = [] + return tags + + +class VotingRegressor(RegressorMixin, _BaseVoting): + """Prediction voting regressor for unfitted estimators. + + A voting regressor is an ensemble meta-estimator that fits several base + regressors, each on the whole dataset. Then it averages the individual + predictions to form a final prediction. + + For a detailed example, refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_voting_regressor.py`. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.21 + + Parameters + ---------- + estimators : list of (str, estimator) tuples + Invoking the ``fit`` method on the ``VotingRegressor`` will fit clones + of those original estimators that will be stored in the class attribute + ``self.estimators_``. An estimator can be set to ``'drop'`` using + :meth:`set_params`. + + .. versionchanged:: 0.21 + ``'drop'`` is accepted. Using None was deprecated in 0.22 and + support was removed in 0.24. + + weights : array-like of shape (n_regressors,), default=None + Sequence of weights (`float` or `int`) to weight the occurrences of + predicted values before averaging. Uses uniform weights if `None`. + + n_jobs : int, default=None + The number of jobs to run in parallel for ``fit``. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + verbose : bool, default=False + If True, the time elapsed while fitting will be printed as it + is completed. + + .. versionadded:: 0.23 + + Attributes + ---------- + estimators_ : list of regressors + The collection of fitted sub-estimators as defined in ``estimators`` + that are not 'drop'. + + named_estimators_ : :class:`~sklearn.utils.Bunch` + Attribute to access any fitted sub-estimators by name. + + .. versionadded:: 0.20 + + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying regressor exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 + + See Also + -------- + VotingClassifier : Soft Voting/Majority Rule classifier. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.linear_model import LinearRegression + >>> from sklearn.ensemble import RandomForestRegressor + >>> from sklearn.ensemble import VotingRegressor + >>> from sklearn.neighbors import KNeighborsRegressor + >>> r1 = LinearRegression() + >>> r2 = RandomForestRegressor(n_estimators=10, random_state=1) + >>> r3 = KNeighborsRegressor() + >>> X = np.array([[1, 1], [2, 4], [3, 9], [4, 16], [5, 25], [6, 36]]) + >>> y = np.array([2, 6, 12, 20, 30, 42]) + >>> er = VotingRegressor([('lr', r1), ('rf', r2), ('r3', r3)]) + >>> print(er.fit(X, y).predict(X)) + [ 6.8 8.4 12.5 17.8 26 34] + + In the following example, we drop the `'lr'` estimator with + :meth:`~VotingRegressor.set_params` and fit the remaining two estimators: + + >>> er = er.set_params(lr='drop') + >>> er = er.fit(X, y) + >>> len(er.estimators_) + 2 + """ + + def __init__(self, estimators, *, weights=None, n_jobs=None, verbose=False): + super().__init__(estimators=estimators) + self.weights = weights + self.n_jobs = n_jobs + self.verbose = verbose + + @_fit_context( + # estimators in VotingRegressor.estimators are not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, **fit_params): + """Fit the estimators. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values. + + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.5 + + Only available if `enable_metadata_routing=True`, + which can be set by using + ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + + Returns + ------- + self : object + Fitted estimator. + """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + + y = column_or_1d(y, warn=True) + + return super().fit(X, y, **fit_params) + + def predict(self, X): + """Predict regression target for X. + + The predicted regression target of an input sample is computed as the + mean predicted regression targets of the estimators in the ensemble. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. + + Returns + ------- + y : ndarray of shape (n_samples,) + The predicted values. + """ + check_is_fitted(self) + return np.average(self._predict(X), axis=1, weights=self._weights_not_none) + + def transform(self, X): + """Return predictions for X for each estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. + + Returns + ------- + predictions : ndarray of shape (n_samples, n_classifiers) + Values predicted by each regressor. + """ + check_is_fitted(self) + return self._predict(X) + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Not used, present here for API consistency by convention. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names. + """ + check_is_fitted(self, "n_features_in_") + _check_feature_names_in(self, input_features, generate_names=False) + class_name = self.__class__.__name__.lower() + return np.asarray( + [f"{class_name}_{name}" for name, est in self.estimators if est != "drop"], + dtype=object, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_weight_boosting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_weight_boosting.py new file mode 100644 index 0000000000000000000000000000000000000000..21469d9ff9b4e8b3a2728d5e991be17b8cd51e68 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_weight_boosting.py @@ -0,0 +1,1129 @@ +"""Weight Boosting. + +This module contains weight boosting estimators for both classification and +regression. + +The module structure is the following: + +- The `BaseWeightBoosting` base class implements a common ``fit`` method + for all the estimators in the module. Regression and classification + only differ from each other in the loss function that is optimized. + +- :class:`~sklearn.ensemble.AdaBoostClassifier` implements adaptive boosting + (AdaBoost-SAMME) for classification problems. + +- :class:`~sklearn.ensemble.AdaBoostRegressor` implements adaptive boosting + (AdaBoost.R2) for regression problems. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings +from abc import ABCMeta, abstractmethod +from numbers import Integral, Real + +import numpy as np + +from sklearn.base import ( + ClassifierMixin, + RegressorMixin, + _fit_context, + is_classifier, + is_regressor, +) +from sklearn.ensemble._base import BaseEnsemble +from sklearn.metrics import accuracy_score, r2_score +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils import _safe_indexing, check_random_state +from sklearn.utils._param_validation import HasMethods, Interval, StrOptions +from sklearn.utils.extmath import softmax +from sklearn.utils.metadata_routing import ( + _raise_for_unsupported_routing, + _RoutingNotSupportedMixin, +) +from sklearn.utils.validation import ( + _check_sample_weight, + _num_samples, + check_is_fitted, + has_fit_parameter, + validate_data, +) + +__all__ = [ + "AdaBoostClassifier", + "AdaBoostRegressor", +] + + +class BaseWeightBoosting(BaseEnsemble, metaclass=ABCMeta): + """Base class for AdaBoost estimators. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + _parameter_constraints: dict = { + "estimator": [HasMethods(["fit", "predict"]), None], + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "learning_rate": [Interval(Real, 0, None, closed="neither")], + "random_state": ["random_state"], + } + + @abstractmethod + def __init__( + self, + estimator=None, + *, + n_estimators=50, + estimator_params=tuple(), + learning_rate=1.0, + random_state=None, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + estimator_params=estimator_params, + ) + + self.learning_rate = learning_rate + self.random_state = random_state + + def _check_X(self, X): + # Only called to validate X in non-fit methods, therefore reset=False + return validate_data( + self, + X, + accept_sparse=["csr", "csc"], + ensure_2d=True, + allow_nd=True, + dtype=None, + reset=False, + ) + + @_fit_context( + # AdaBoost*.estimator is not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, sample_weight=None): + """Build a boosted classifier/regressor from the training set (X, y). + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + y : array-like of shape (n_samples,) + The target values. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. If None, the sample weights are initialized to + 1 / n_samples. + + Returns + ------- + self : object + Fitted estimator. + """ + _raise_for_unsupported_routing(self, "fit", sample_weight=sample_weight) + X, y = validate_data( + self, + X, + y, + accept_sparse=["csr", "csc"], + ensure_2d=True, + allow_nd=True, + dtype=None, + y_numeric=is_regressor(self), + ) + + sample_weight = _check_sample_weight( + sample_weight, X, dtype=np.float64, copy=True, ensure_non_negative=True + ) + sample_weight /= sample_weight.sum() + + # Check parameters + self._validate_estimator() + + # Clear any previous fit results + self.estimators_ = [] + self.estimator_weights_ = np.zeros(self.n_estimators, dtype=np.float64) + self.estimator_errors_ = np.ones(self.n_estimators, dtype=np.float64) + + # Initialization of the random number instance that will be used to + # generate a seed at each iteration + random_state = check_random_state(self.random_state) + epsilon = np.finfo(sample_weight.dtype).eps + + zero_weight_mask = sample_weight == 0.0 + for iboost in range(self.n_estimators): + # avoid extremely small sample weight, for details see issue #20320 + sample_weight = np.clip(sample_weight, a_min=epsilon, a_max=None) + # do not clip sample weights that were exactly zero originally + sample_weight[zero_weight_mask] = 0.0 + + # Boosting step + sample_weight, estimator_weight, estimator_error = self._boost( + iboost, X, y, sample_weight, random_state + ) + + # Early termination + if sample_weight is None: + break + self.estimator_weights_[iboost] = estimator_weight + self.estimator_errors_[iboost] = estimator_error + + # Stop if error is zero + if estimator_error == 0: + break + + sample_weight_sum = np.sum(sample_weight) + + if not np.isfinite(sample_weight_sum): + warnings.warn( + ( + "Sample weights have reached infinite values," + f" at iteration {iboost}, causing overflow. " + "Iterations stopped. Try lowering the learning rate." + ), + stacklevel=2, + ) + break + + # Stop if the sum of sample weights has become non-positive + if sample_weight_sum <= 0: + break + + if iboost < self.n_estimators - 1: + # Normalize + sample_weight /= sample_weight_sum + + return self + + @abstractmethod + def _boost(self, iboost, X, y, sample_weight, random_state): + """Implement a single boost. + + Warning: This method needs to be overridden by subclasses. + + Parameters + ---------- + iboost : int + The index of the current boost iteration. + + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + y : array-like of shape (n_samples,) + The target values (class labels). + + sample_weight : array-like of shape (n_samples,) + The current sample weights. + + random_state : RandomState + The current random number generator + + Returns + ------- + sample_weight : array-like of shape (n_samples,) or None + The reweighted sample weights. + If None then boosting has terminated early. + + estimator_weight : float + The weight for the current boost. + If None then boosting has terminated early. + + error : float + The classification error for the current boost. + If None then boosting has terminated early. + """ + pass + + def staged_score(self, X, y, sample_weight=None): + """Return staged scores for X, y. + + This generator method yields the ensemble score after each iteration of + boosting and therefore allows monitoring, such as to determine the + score on a test set after each boost. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + y : array-like of shape (n_samples,) + Labels for X. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + Yields + ------ + z : float + """ + X = self._check_X(X) + + for y_pred in self.staged_predict(X): + if is_classifier(self): + yield accuracy_score(y, y_pred, sample_weight=sample_weight) + else: + yield r2_score(y, y_pred, sample_weight=sample_weight) + + @property + def feature_importances_(self): + """The impurity-based feature importances. + + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + Returns + ------- + feature_importances_ : ndarray of shape (n_features,) + The feature importances. + """ + if self.estimators_ is None or len(self.estimators_) == 0: + raise ValueError( + "Estimator not fitted, call `fit` before `feature_importances_`." + ) + + try: + norm = self.estimator_weights_.sum() + return ( + sum( + weight * clf.feature_importances_ + for weight, clf in zip(self.estimator_weights_, self.estimators_) + ) + / norm + ) + + except AttributeError as e: + raise AttributeError( + "Unable to compute feature importances " + "since estimator does not have a " + "feature_importances_ attribute" + ) from e + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags + + +class AdaBoostClassifier( + _RoutingNotSupportedMixin, ClassifierMixin, BaseWeightBoosting +): + """An AdaBoost classifier. + + An AdaBoost [1]_ classifier is a meta-estimator that begins by fitting a + classifier on the original dataset and then fits additional copies of the + classifier on the same dataset but where the weights of incorrectly + classified instances are adjusted such that subsequent classifiers focus + more on difficult cases. + + This class implements the algorithm based on [2]_. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.14 + + Parameters + ---------- + estimator : object, default=None + The base estimator from which the boosted ensemble is built. + Support for sample weighting is required, as well as proper + ``classes_`` and ``n_classes_`` attributes. If ``None``, then + the base estimator is :class:`~sklearn.tree.DecisionTreeClassifier` + initialized with `max_depth=1`. + + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + + n_estimators : int, default=50 + The maximum number of estimators at which boosting is terminated. + In case of perfect fit, the learning procedure is stopped early. + Values must be in the range `[1, inf)`. + + learning_rate : float, default=1.0 + Weight applied to each classifier at each boosting iteration. A higher + learning rate increases the contribution of each classifier. There is + a trade-off between the `learning_rate` and `n_estimators` parameters. + Values must be in the range `(0.0, inf)`. + + random_state : int, RandomState instance or None, default=None + Controls the random seed given at each `estimator` at each + boosting iteration. + Thus, it is only used when `estimator` exposes a `random_state`. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + Attributes + ---------- + estimator_ : estimator + The base estimator from which the ensemble is grown. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of classifiers + The collection of fitted sub-estimators. + + classes_ : ndarray of shape (n_classes,) + The classes labels. + + n_classes_ : int + The number of classes. + + estimator_weights_ : ndarray of floats + Weights for each estimator in the boosted ensemble. + + estimator_errors_ : ndarray of floats + Classification error for each estimator in the boosted + ensemble. + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances if supported by the + ``estimator`` (when based on decision trees). + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + AdaBoostRegressor : An AdaBoost regressor that begins by fitting a + regressor on the original dataset and then fits additional copies of + the regressor on the same dataset but where the weights of instances + are adjusted according to the error of the current prediction. + + GradientBoostingClassifier : GB builds an additive model in a forward + stage-wise fashion. Regression trees are fit on the negative gradient + of the binomial or multinomial deviance loss function. Binary + classification is a special case where only a single regression tree is + induced. + + sklearn.tree.DecisionTreeClassifier : A non-parametric supervised learning + method used for classification. + Creates a model that predicts the value of a target variable by + learning simple decision rules inferred from the data features. + + References + ---------- + .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of + on-Line Learning and an Application to Boosting", 1995. + + .. [2] :doi:`J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class adaboost." + Statistics and its Interface 2.3 (2009): 349-360. + <10.4310/SII.2009.v2.n3.a8>` + + Examples + -------- + >>> from sklearn.ensemble import AdaBoostClassifier + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification(n_samples=1000, n_features=4, + ... n_informative=2, n_redundant=0, + ... random_state=0, shuffle=False) + >>> clf = AdaBoostClassifier(n_estimators=100, random_state=0) + >>> clf.fit(X, y) + AdaBoostClassifier(n_estimators=100, random_state=0) + >>> clf.predict([[0, 0, 0, 0]]) + array([1]) + >>> clf.score(X, y) + 0.96 + + For a detailed example of using AdaBoost to fit a sequence of DecisionTrees + as weaklearners, please refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_multiclass.py`. + + For a detailed example of using AdaBoost to fit a non-linearly separable + classification dataset composed of two Gaussian quantiles clusters, please + refer to :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_twoclass.py`. + """ + + def __init__( + self, + estimator=None, + *, + n_estimators=50, + learning_rate=1.0, + random_state=None, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + learning_rate=learning_rate, + random_state=random_state, + ) + + def _validate_estimator(self): + """Check the estimator and set the estimator_ attribute.""" + super()._validate_estimator(default=DecisionTreeClassifier(max_depth=1)) + + if not has_fit_parameter(self.estimator_, "sample_weight"): + raise ValueError( + f"{self.estimator.__class__.__name__} doesn't support sample_weight." + ) + + def _boost(self, iboost, X, y, sample_weight, random_state): + """Implement a single boost. + + Perform a single boost according to the discrete SAMME algorithm and return the + updated sample weights. + + Parameters + ---------- + iboost : int + The index of the current boost iteration. + + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. + + y : array-like of shape (n_samples,) + The target values (class labels). + + sample_weight : array-like of shape (n_samples,) + The current sample weights. + + random_state : RandomState instance + The RandomState instance used if the base estimator accepts a + `random_state` attribute. + + Returns + ------- + sample_weight : array-like of shape (n_samples,) or None + The reweighted sample weights. + If None then boosting has terminated early. + + estimator_weight : float + The weight for the current boost. + If None then boosting has terminated early. + + estimator_error : float + The classification error for the current boost. + If None then boosting has terminated early. + """ + estimator = self._make_estimator(random_state=random_state) + + estimator.fit(X, y, sample_weight=sample_weight) + + y_predict = estimator.predict(X) + + if iboost == 0: + self.classes_ = getattr(estimator, "classes_", None) + self.n_classes_ = len(self.classes_) + + # Instances incorrectly classified + incorrect = y_predict != y + + # Error fraction + estimator_error = np.mean(np.average(incorrect, weights=sample_weight, axis=0)) + + # Stop if classification is perfect + if estimator_error <= 0: + return sample_weight, 1.0, 0.0 + + n_classes = self.n_classes_ + + # Stop if the error is at least as bad as random guessing + if estimator_error >= 1.0 - (1.0 / n_classes): + self.estimators_.pop(-1) + if len(self.estimators_) == 0: + raise ValueError( + "BaseClassifier in AdaBoostClassifier " + "ensemble is worse than random, ensemble " + "can not be fit." + ) + return None, None, None + + # Boost weight using multi-class AdaBoost SAMME alg + estimator_weight = self.learning_rate * ( + np.log((1.0 - estimator_error) / estimator_error) + np.log(n_classes - 1.0) + ) + + # Only boost the weights if it will fit again + if not iboost == self.n_estimators - 1: + # Only boost positive weights + sample_weight = np.exp( + np.log(sample_weight) + + estimator_weight * incorrect * (sample_weight > 0) + ) + + return sample_weight, estimator_weight, estimator_error + + def predict(self, X): + """Predict classes for X. + + The predicted class of an input sample is computed as the weighted mean + prediction of the classifiers in the ensemble. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Returns + ------- + y : ndarray of shape (n_samples,) + The predicted classes. + """ + pred = self.decision_function(X) + + if self.n_classes_ == 2: + return self.classes_.take(pred > 0, axis=0) + + return self.classes_.take(np.argmax(pred, axis=1), axis=0) + + def staged_predict(self, X): + """Return staged predictions for X. + + The predicted class of an input sample is computed as the weighted mean + prediction of the classifiers in the ensemble. + + This generator method yields the ensemble prediction after each + iteration of boosting and therefore allows monitoring, such as to + determine the prediction on a test set after each boost. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + The input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted classes. + """ + X = self._check_X(X) + + n_classes = self.n_classes_ + classes = self.classes_ + + if n_classes == 2: + for pred in self.staged_decision_function(X): + yield np.array(classes.take(pred > 0, axis=0)) + + else: + for pred in self.staged_decision_function(X): + yield np.array(classes.take(np.argmax(pred, axis=1), axis=0)) + + def decision_function(self, X): + """Compute the decision function of ``X``. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Returns + ------- + score : ndarray of shape of (n_samples, k) + The decision function of the input samples. The order of + outputs is the same as that of the :term:`classes_` attribute. + Binary classification is a special cases with ``k == 1``, + otherwise ``k==n_classes``. For binary classification, + values closer to -1 or 1 mean more like the first or second + class in ``classes_``, respectively. + """ + check_is_fitted(self) + X = self._check_X(X) + + n_classes = self.n_classes_ + classes = self.classes_[:, np.newaxis] + + if n_classes == 1: + return np.zeros_like(X, shape=(X.shape[0], 1)) + + pred = sum( + np.where( + (estimator.predict(X) == classes).T, + w, + -1 / (n_classes - 1) * w, + ) + for estimator, w in zip(self.estimators_, self.estimator_weights_) + ) + + pred /= self.estimator_weights_.sum() + if n_classes == 2: + pred[:, 0] *= -1 + return pred.sum(axis=1) + return pred + + def staged_decision_function(self, X): + """Compute decision function of ``X`` for each boosting iteration. + + This method allows monitoring (i.e. determine error on testing set) + after each boosting iteration. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Yields + ------ + score : generator of ndarray of shape (n_samples, k) + The decision function of the input samples. The order of + outputs is the same of that of the :term:`classes_` attribute. + Binary classification is a special cases with ``k == 1``, + otherwise ``k==n_classes``. For binary classification, + values closer to -1 or 1 mean more like the first or second + class in ``classes_``, respectively. + """ + check_is_fitted(self) + X = self._check_X(X) + + n_classes = self.n_classes_ + classes = self.classes_[:, np.newaxis] + pred = None + norm = 0.0 + + for weight, estimator in zip(self.estimator_weights_, self.estimators_): + norm += weight + + current_pred = np.where( + (estimator.predict(X) == classes).T, + weight, + -1 / (n_classes - 1) * weight, + ) + + if pred is None: + pred = current_pred + else: + pred += current_pred + + if n_classes == 2: + tmp_pred = np.copy(pred) + tmp_pred[:, 0] *= -1 + yield (tmp_pred / norm).sum(axis=1) + else: + yield pred / norm + + @staticmethod + def _compute_proba_from_decision(decision, n_classes): + """Compute probabilities from the decision function. + + This is based eq. (15) of [1] where: + p(y=c|X) = exp((1 / K-1) f_c(X)) / sum_k(exp((1 / K-1) f_k(X))) + = softmax((1 / K-1) * f(X)) + + References + ---------- + .. [1] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", + 2009. + """ + if n_classes == 2: + decision = np.vstack([-decision, decision]).T / 2 + else: + decision /= n_classes - 1 + return softmax(decision, copy=False) + + def predict_proba(self, X): + """Predict class probabilities for X. + + The predicted class probabilities of an input sample is computed as + the weighted mean predicted class probabilities of the classifiers + in the ensemble. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes) + The class probabilities of the input samples. The order of + outputs is the same of that of the :term:`classes_` attribute. + """ + check_is_fitted(self) + n_classes = self.n_classes_ + + if n_classes == 1: + return np.ones((_num_samples(X), 1)) + + decision = self.decision_function(X) + return self._compute_proba_from_decision(decision, n_classes) + + def staged_predict_proba(self, X): + """Predict class probabilities for X. + + The predicted class probabilities of an input sample is computed as + the weighted mean predicted class probabilities of the classifiers + in the ensemble. + + This generator method yields the ensemble predicted class probabilities + after each iteration of boosting and therefore allows monitoring, such + as to determine the predicted class probabilities on a test set after + each boost. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Yields + ------ + p : generator of ndarray of shape (n_samples,) + The class probabilities of the input samples. The order of + outputs is the same of that of the :term:`classes_` attribute. + """ + + n_classes = self.n_classes_ + + for decision in self.staged_decision_function(X): + yield self._compute_proba_from_decision(decision, n_classes) + + def predict_log_proba(self, X): + """Predict class log-probabilities for X. + + The predicted class log-probabilities of an input sample is computed as + the weighted mean predicted class log-probabilities of the classifiers + in the ensemble. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Returns + ------- + p : ndarray of shape (n_samples, n_classes) + The class probabilities of the input samples. The order of + outputs is the same of that of the :term:`classes_` attribute. + """ + return np.log(self.predict_proba(X)) + + +class AdaBoostRegressor(_RoutingNotSupportedMixin, RegressorMixin, BaseWeightBoosting): + """An AdaBoost regressor. + + An AdaBoost [1] regressor is a meta-estimator that begins by fitting a + regressor on the original dataset and then fits additional copies of the + regressor on the same dataset but where the weights of instances are + adjusted according to the error of the current prediction. As such, + subsequent regressors focus more on difficult cases. + + This class implements the algorithm known as AdaBoost.R2 [2]. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 0.14 + + Parameters + ---------- + estimator : object, default=None + The base estimator from which the boosted ensemble is built. + If ``None``, then the base estimator is + :class:`~sklearn.tree.DecisionTreeRegressor` initialized with + `max_depth=3`. + + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + + n_estimators : int, default=50 + The maximum number of estimators at which boosting is terminated. + In case of perfect fit, the learning procedure is stopped early. + Values must be in the range `[1, inf)`. + + learning_rate : float, default=1.0 + Weight applied to each regressor at each boosting iteration. A higher + learning rate increases the contribution of each regressor. There is + a trade-off between the `learning_rate` and `n_estimators` parameters. + Values must be in the range `(0.0, inf)`. + + loss : {'linear', 'square', 'exponential'}, default='linear' + The loss function to use when updating the weights after each + boosting iteration. + + random_state : int, RandomState instance or None, default=None + Controls the random seed given at each `estimator` at each + boosting iteration. + Thus, it is only used when `estimator` exposes a `random_state`. + In addition, it controls the bootstrap of the weights used to train the + `estimator` at each boosting iteration. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + Attributes + ---------- + estimator_ : estimator + The base estimator from which the ensemble is grown. + + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of regressors + The collection of fitted sub-estimators. + + estimator_weights_ : ndarray of floats + Weights for each estimator in the boosted ensemble. + + estimator_errors_ : ndarray of floats + Regression error for each estimator in the boosted ensemble. + + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances if supported by the + ``estimator`` (when based on decision trees). + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + AdaBoostClassifier : An AdaBoost classifier. + GradientBoostingRegressor : Gradient Boosting Classification Tree. + sklearn.tree.DecisionTreeRegressor : A decision tree regressor. + + References + ---------- + .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of + on-Line Learning and an Application to Boosting", 1995. + + .. [2] H. Drucker, "Improving Regressors using Boosting Techniques", 1997. + + Examples + -------- + >>> from sklearn.ensemble import AdaBoostRegressor + >>> from sklearn.datasets import make_regression + >>> X, y = make_regression(n_features=4, n_informative=2, + ... random_state=0, shuffle=False) + >>> regr = AdaBoostRegressor(random_state=0, n_estimators=100) + >>> regr.fit(X, y) + AdaBoostRegressor(n_estimators=100, random_state=0) + >>> regr.predict([[0, 0, 0, 0]]) + array([4.7972]) + >>> regr.score(X, y) + 0.9771 + + For a detailed example of utilizing :class:`~sklearn.ensemble.AdaBoostRegressor` + to fit a sequence of decision trees as weak learners, please refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_regression.py`. + """ + + _parameter_constraints: dict = { + **BaseWeightBoosting._parameter_constraints, + "loss": [StrOptions({"linear", "square", "exponential"})], + } + + def __init__( + self, + estimator=None, + *, + n_estimators=50, + learning_rate=1.0, + loss="linear", + random_state=None, + ): + super().__init__( + estimator=estimator, + n_estimators=n_estimators, + learning_rate=learning_rate, + random_state=random_state, + ) + + self.loss = loss + self.random_state = random_state + + def _validate_estimator(self): + """Check the estimator and set the estimator_ attribute.""" + super()._validate_estimator(default=DecisionTreeRegressor(max_depth=3)) + + def _boost(self, iboost, X, y, sample_weight, random_state): + """Implement a single boost for regression + + Perform a single boost according to the AdaBoost.R2 algorithm and + return the updated sample weights. + + Parameters + ---------- + iboost : int + The index of the current boost iteration. + + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. + + y : array-like of shape (n_samples,) + The target values (class labels in classification, real numbers in + regression). + + sample_weight : array-like of shape (n_samples,) + The current sample weights. + + random_state : RandomState + The RandomState instance used if the base estimator accepts a + `random_state` attribute. + Controls also the bootstrap of the weights used to train the weak + learner. + + Returns + ------- + sample_weight : array-like of shape (n_samples,) or None + The reweighted sample weights. + If None then boosting has terminated early. + + estimator_weight : float + The weight for the current boost. + If None then boosting has terminated early. + + estimator_error : float + The regression error for the current boost. + If None then boosting has terminated early. + """ + estimator = self._make_estimator(random_state=random_state) + + # Weighted sampling of the training set with replacement + bootstrap_idx = random_state.choice( + np.arange(_num_samples(X)), + size=_num_samples(X), + replace=True, + p=sample_weight, + ) + + # Fit on the bootstrapped sample and obtain a prediction + # for all samples in the training set + X_ = _safe_indexing(X, bootstrap_idx) + y_ = _safe_indexing(y, bootstrap_idx) + estimator.fit(X_, y_) + y_predict = estimator.predict(X) + + error_vect = np.abs(y_predict - y) + sample_mask = sample_weight > 0 + masked_sample_weight = sample_weight[sample_mask] + masked_error_vector = error_vect[sample_mask] + + error_max = masked_error_vector.max() + if error_max != 0: + masked_error_vector /= error_max + + if self.loss == "square": + masked_error_vector **= 2 + elif self.loss == "exponential": + masked_error_vector = 1.0 - np.exp(-masked_error_vector) + + # Calculate the average loss + estimator_error = (masked_sample_weight * masked_error_vector).sum() + + if estimator_error <= 0: + # Stop if fit is perfect + return sample_weight, 1.0, 0.0 + + elif estimator_error >= 0.5: + # Discard current estimator only if it isn't the only one + if len(self.estimators_) > 1: + self.estimators_.pop(-1) + return None, None, None + + beta = estimator_error / (1.0 - estimator_error) + + # Boost weight using AdaBoost.R2 alg + estimator_weight = self.learning_rate * np.log(1.0 / beta) + + if not iboost == self.n_estimators - 1: + sample_weight[sample_mask] *= np.power( + beta, (1.0 - masked_error_vector) * self.learning_rate + ) + + return sample_weight, estimator_weight, estimator_error + + def _get_median_predict(self, X, limit): + # Evaluate predictions of all estimators + predictions = np.array([est.predict(X) for est in self.estimators_[:limit]]).T + + # Sort the predictions + sorted_idx = np.argsort(predictions, axis=1) + + # Find index of median prediction for each sample + weight_cdf = np.cumsum(self.estimator_weights_[sorted_idx], axis=1) + median_or_above = weight_cdf >= 0.5 * weight_cdf[:, -1][:, np.newaxis] + median_idx = median_or_above.argmax(axis=1) + + median_estimators = sorted_idx[np.arange(_num_samples(X)), median_idx] + + # Return median predictions + return predictions[np.arange(_num_samples(X)), median_estimators] + + def predict(self, X): + """Predict regression value for X. + + The predicted regression value of an input sample is computed + as the weighted median prediction of the regressors in the ensemble. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. Sparse matrix can be CSC, CSR, COO, + DOK, or LIL. COO, DOK, and LIL are converted to CSR. + + Returns + ------- + y : ndarray of shape (n_samples,) + The predicted regression values. + """ + check_is_fitted(self) + X = self._check_X(X) + + return self._get_median_predict(X, len(self.estimators_)) + + def staged_predict(self, X): + """Return staged predictions for X. + + The predicted regression value of an input sample is computed + as the weighted median prediction of the regressors in the ensemble. + + This generator method yields the ensemble prediction after each + iteration of boosting and therefore allows monitoring, such as to + determine the prediction on a test set after each boost. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The training input samples. + + Yields + ------ + y : generator of ndarray of shape (n_samples,) + The predicted regression values. + """ + check_is_fitted(self) + X = self._check_X(X) + + for i, _ in enumerate(self.estimators_, 1): + yield self._get_median_predict(X, limit=i) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/meson.build b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/meson.build new file mode 100644 index 0000000000000000000000000000000000000000..d3f92f7d68a1dc51221d1193a3b889a8bb5f4e2d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/meson.build @@ -0,0 +1,9 @@ +py.extension_module( + '_gradient_boosting', + [cython_gen.process('_gradient_boosting.pyx')] + utils_cython_tree, + dependencies: [np_dep], + subdir: 'sklearn/ensemble', + install: true +) + +subdir('_hist_gradient_boosting') diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_bagging.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_bagging.py new file mode 100644 index 0000000000000000000000000000000000000000..4c62148dc996490b106f87e59902853b74e86e93 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_bagging.py @@ -0,0 +1,1216 @@ +""" +Testing for the bagging ensemble module (sklearn.ensemble.bagging). +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import re +import warnings +from itertools import cycle, product + +import joblib +import numpy as np +import pytest + +from sklearn import config_context +from sklearn.base import BaseEstimator +from sklearn.datasets import load_diabetes, load_iris, make_hastie_10_2 +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble import ( + AdaBoostClassifier, + AdaBoostRegressor, + BaggingClassifier, + BaggingRegressor, + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, + RandomForestClassifier, + RandomForestRegressor, +) +from sklearn.ensemble._bagging import _get_n_samples_bootstrap +from sklearn.feature_selection import SelectKBest +from sklearn.linear_model import LogisticRegression, Perceptron +from sklearn.model_selection import GridSearchCV, ParameterGrid, train_test_split +from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import FunctionTransformer, scale +from sklearn.random_projection import SparseRandomProjection +from sklearn.svm import SVC, SVR +from sklearn.tests.metadata_routing_common import ( + ConsumingClassifierWithOnlyPredict, + ConsumingClassifierWithoutPredictLogProba, + ConsumingClassifierWithoutPredictProba, + _Registry, + check_recorded_metadata, +) +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils import check_random_state +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS + +rng = check_random_state(0) + +# also load the iris dataset +# and randomly permute it +iris = load_iris() +perm = rng.permutation(iris.target.size) +iris.data = iris.data[perm] +iris.target = iris.target[perm] + +# also load the diabetes dataset +# and randomly permute it +diabetes = load_diabetes() +perm = rng.permutation(diabetes.target.size) +diabetes.data = diabetes.data[perm] +diabetes.target = diabetes.target[perm] + + +def test_classification(): + # Check classification for various parameter settings. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) + grid = ParameterGrid( + { + "max_samples": [0.5, 1.0], + "max_features": [1, 4], + "bootstrap": [True, False], + "bootstrap_features": [True, False], + } + ) + estimators = [ + None, + DummyClassifier(), + Perceptron(max_iter=20), + DecisionTreeClassifier(max_depth=2), + KNeighborsClassifier(), + SVC(), + ] + # Try different parameter settings with different base classifiers without + # doing the full cartesian product to keep the test durations low. + for params, estimator in zip(grid, cycle(estimators)): + BaggingClassifier( + estimator=estimator, + random_state=rng, + n_estimators=2, + **params, + ).fit(X_train, y_train).predict(X_test) + + +@pytest.mark.parametrize( + "sparse_container, params, method", + product( + CSR_CONTAINERS + CSC_CONTAINERS, + [ + { + "max_samples": 0.5, + "max_features": 2, + "bootstrap": True, + "bootstrap_features": True, + }, + { + "max_samples": 1.0, + "max_features": 4, + "bootstrap": True, + "bootstrap_features": True, + }, + {"max_features": 2, "bootstrap": False, "bootstrap_features": True}, + {"max_samples": 0.5, "bootstrap": True, "bootstrap_features": False}, + ], + ["predict", "predict_proba", "predict_log_proba", "decision_function"], + ), +) +def test_sparse_classification(sparse_container, params, method): + # Check classification for various parameter settings on sparse input. + + class CustomSVC(SVC): + """SVC variant that records the nature of the training set""" + + def fit(self, X, y): + super().fit(X, y) + self.data_type_ = type(X) + return self + + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + scale(iris.data), iris.target, random_state=rng + ) + + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) + # Trained on sparse format + sparse_classifier = BaggingClassifier( + estimator=CustomSVC(kernel="linear", decision_function_shape="ovr"), + random_state=1, + **params, + ).fit(X_train_sparse, y_train) + sparse_results = getattr(sparse_classifier, method)(X_test_sparse) + + # Trained on dense format + dense_classifier = BaggingClassifier( + estimator=CustomSVC(kernel="linear", decision_function_shape="ovr"), + random_state=1, + **params, + ).fit(X_train, y_train) + dense_results = getattr(dense_classifier, method)(X_test) + assert_array_almost_equal(sparse_results, dense_results) + + sparse_type = type(X_train_sparse) + types = [i.data_type_ for i in sparse_classifier.estimators_] + + assert all([t == sparse_type for t in types]) + + +def test_regression(): + # Check regression for various parameter settings. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data[:50], diabetes.target[:50], random_state=rng + ) + grid = ParameterGrid( + { + "max_samples": [0.5, 1.0], + "max_features": [0.5, 1.0], + "bootstrap": [True, False], + "bootstrap_features": [True, False], + } + ) + + for estimator in [ + None, + DummyRegressor(), + DecisionTreeRegressor(), + KNeighborsRegressor(), + SVR(), + ]: + for params in grid: + BaggingRegressor(estimator=estimator, random_state=rng, **params).fit( + X_train, y_train + ).predict(X_test) + + +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_sparse_regression(sparse_container): + # Check regression for various parameter settings on sparse input. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data[:50], diabetes.target[:50], random_state=rng + ) + + class CustomSVR(SVR): + """SVC variant that records the nature of the training set""" + + def fit(self, X, y): + super().fit(X, y) + self.data_type_ = type(X) + return self + + parameter_sets = [ + { + "max_samples": 0.5, + "max_features": 2, + "bootstrap": True, + "bootstrap_features": True, + }, + { + "max_samples": 1.0, + "max_features": 4, + "bootstrap": True, + "bootstrap_features": True, + }, + {"max_features": 2, "bootstrap": False, "bootstrap_features": True}, + {"max_samples": 0.5, "bootstrap": True, "bootstrap_features": False}, + ] + + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) + for params in parameter_sets: + # Trained on sparse format + sparse_classifier = BaggingRegressor( + estimator=CustomSVR(), random_state=1, **params + ).fit(X_train_sparse, y_train) + sparse_results = sparse_classifier.predict(X_test_sparse) + + # Trained on dense format + dense_results = ( + BaggingRegressor(estimator=CustomSVR(), random_state=1, **params) + .fit(X_train, y_train) + .predict(X_test) + ) + + sparse_type = type(X_train_sparse) + types = [i.data_type_ for i in sparse_classifier.estimators_] + + assert_array_almost_equal(sparse_results, dense_results) + assert all([t == sparse_type for t in types]) + assert_array_almost_equal(sparse_results, dense_results) + + +class DummySizeEstimator(BaseEstimator): + def fit(self, X, y): + self.training_size_ = X.shape[0] + self.training_hash_ = joblib.hash(X) + + def predict(self, X): + return np.ones(X.shape[0]) + + +def test_bootstrap_samples(): + # Test that bootstrapping samples generate non-perfect base estimators. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) + + estimator = DecisionTreeRegressor().fit(X_train, y_train) + + # without bootstrap, all trees are perfect on the training set + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_samples=1.0, + bootstrap=False, + random_state=rng, + ).fit(X_train, y_train) + + assert estimator.score(X_train, y_train) == ensemble.score(X_train, y_train) + + # with bootstrap, trees are no longer perfect on the training set + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_samples=1.0, + bootstrap=True, + random_state=rng, + ).fit(X_train, y_train) + + assert estimator.score(X_train, y_train) > ensemble.score(X_train, y_train) + + # check that each sampling correspond to a complete bootstrap resample. + # the size of each bootstrap should be the same as the input data but + # the data should be different (checked using the hash of the data). + ensemble = BaggingRegressor(estimator=DummySizeEstimator(), bootstrap=True).fit( + X_train, y_train + ) + training_hash = [] + for estimator in ensemble.estimators_: + assert estimator.training_size_ == X_train.shape[0] + training_hash.append(estimator.training_hash_) + assert len(set(training_hash)) == len(training_hash) + + +def test_bootstrap_features(): + # Test that bootstrapping features may generate duplicate features. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) + + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_features=1.0, + bootstrap_features=False, + random_state=rng, + ).fit(X_train, y_train) + + for features in ensemble.estimators_features_: + assert diabetes.data.shape[1] == np.unique(features).shape[0] + + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_features=1.0, + bootstrap_features=True, + random_state=rng, + ).fit(X_train, y_train) + + for features in ensemble.estimators_features_: + assert diabetes.data.shape[1] > np.unique(features).shape[0] + + +def test_probability(): + # Predict probabilities. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) + + with np.errstate(divide="ignore", invalid="ignore"): + # Normal case + ensemble = BaggingClassifier( + estimator=DecisionTreeClassifier(), random_state=rng + ).fit(X_train, y_train) + + assert_array_almost_equal( + np.sum(ensemble.predict_proba(X_test), axis=1), np.ones(len(X_test)) + ) + + assert_array_almost_equal( + ensemble.predict_proba(X_test), np.exp(ensemble.predict_log_proba(X_test)) + ) + + # Degenerate case, where some classes are missing + ensemble = BaggingClassifier( + estimator=LogisticRegression(), random_state=rng, max_samples=5 + ).fit(X_train, y_train) + + assert_array_almost_equal( + np.sum(ensemble.predict_proba(X_test), axis=1), np.ones(len(X_test)) + ) + + assert_array_almost_equal( + ensemble.predict_proba(X_test), np.exp(ensemble.predict_log_proba(X_test)) + ) + + +def test_oob_score_classification(): + # Check that oob prediction is a good estimation of the generalization + # error. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) + + for estimator in [DecisionTreeClassifier(), SVC()]: + clf = BaggingClassifier( + estimator=estimator, + n_estimators=100, + bootstrap=True, + oob_score=True, + random_state=rng, + ).fit(X_train, y_train) + + test_score = clf.score(X_test, y_test) + + assert abs(test_score - clf.oob_score_) < 0.1 + + # Test with few estimators + warn_msg = ( + "Some inputs do not have OOB scores. This probably means too few " + "estimators were used to compute any reliable oob estimates." + ) + with pytest.warns(UserWarning, match=warn_msg): + clf = BaggingClassifier( + estimator=estimator, + n_estimators=1, + bootstrap=True, + oob_score=True, + random_state=rng, + ) + clf.fit(X_train, y_train) + + +def test_oob_score_regression(): + # Check that oob prediction is a good estimation of the generalization + # error. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) + + clf = BaggingRegressor( + estimator=DecisionTreeRegressor(), + n_estimators=50, + bootstrap=True, + oob_score=True, + random_state=rng, + ).fit(X_train, y_train) + + test_score = clf.score(X_test, y_test) + + assert abs(test_score - clf.oob_score_) < 0.1 + + # Test with few estimators + warn_msg = ( + "Some inputs do not have OOB scores. This probably means too few " + "estimators were used to compute any reliable oob estimates." + ) + with pytest.warns(UserWarning, match=warn_msg): + regr = BaggingRegressor( + estimator=DecisionTreeRegressor(), + n_estimators=1, + bootstrap=True, + oob_score=True, + random_state=rng, + ) + regr.fit(X_train, y_train) + + +def test_single_estimator(): + # Check singleton ensembles. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) + + clf1 = BaggingRegressor( + estimator=KNeighborsRegressor(), + n_estimators=1, + bootstrap=False, + bootstrap_features=False, + random_state=rng, + ).fit(X_train, y_train) + + clf2 = KNeighborsRegressor().fit(X_train, y_train) + + assert_array_almost_equal(clf1.predict(X_test), clf2.predict(X_test)) + + +def test_error(): + # Test support of decision_function + X, y = iris.data, iris.target + base = DecisionTreeClassifier() + assert not hasattr(BaggingClassifier(base).fit(X, y), "decision_function") + + +def test_parallel_classification(): + # Check parallel classification. + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=0 + ) + + ensemble = BaggingClassifier( + DecisionTreeClassifier(), n_jobs=3, random_state=0 + ).fit(X_train, y_train) + + # predict_proba + y1 = ensemble.predict_proba(X_test) + ensemble.set_params(n_jobs=1) + y2 = ensemble.predict_proba(X_test) + assert_array_almost_equal(y1, y2) + + ensemble = BaggingClassifier( + DecisionTreeClassifier(), n_jobs=1, random_state=0 + ).fit(X_train, y_train) + + y3 = ensemble.predict_proba(X_test) + assert_array_almost_equal(y1, y3) + + # decision_function + ensemble = BaggingClassifier( + SVC(decision_function_shape="ovr"), n_jobs=3, random_state=0 + ).fit(X_train, y_train) + + decisions1 = ensemble.decision_function(X_test) + ensemble.set_params(n_jobs=1) + decisions2 = ensemble.decision_function(X_test) + assert_array_almost_equal(decisions1, decisions2) + + ensemble = BaggingClassifier( + SVC(decision_function_shape="ovr"), n_jobs=1, random_state=0 + ).fit(X_train, y_train) + + decisions3 = ensemble.decision_function(X_test) + assert_array_almost_equal(decisions1, decisions3) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +def test_parallel_regression(): + # Check parallel regression. + rng = check_random_state(0) + + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) + + ensemble = BaggingRegressor(DecisionTreeRegressor(), n_jobs=3, random_state=0).fit( + X_train, y_train + ) + + ensemble.set_params(n_jobs=1) + y1 = ensemble.predict(X_test) + ensemble.set_params(n_jobs=2) + y2 = ensemble.predict(X_test) + assert_array_almost_equal(y1, y2) + + ensemble = BaggingRegressor(DecisionTreeRegressor(), n_jobs=1, random_state=0).fit( + X_train, y_train + ) + + y3 = ensemble.predict(X_test) + assert_array_almost_equal(y1, y3) + + +def test_gridsearch(): + # Check that bagging ensembles can be grid-searched. + # Transform iris into a binary classification task + X, y = iris.data, iris.target + y[y == 2] = 1 + + # Grid search with scoring based on decision_function + parameters = {"n_estimators": (1, 2), "estimator__C": (1, 2)} + + GridSearchCV(BaggingClassifier(SVC()), parameters, scoring="roc_auc").fit(X, y) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +def test_estimator(): + # Check estimator and its default values. + rng = check_random_state(0) + + # Classification + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) + + ensemble = BaggingClassifier(None, n_jobs=3, random_state=0).fit(X_train, y_train) + + assert isinstance(ensemble.estimator_, DecisionTreeClassifier) + + ensemble = BaggingClassifier( + DecisionTreeClassifier(), n_jobs=3, random_state=0 + ).fit(X_train, y_train) + + assert isinstance(ensemble.estimator_, DecisionTreeClassifier) + + ensemble = BaggingClassifier(Perceptron(), n_jobs=3, random_state=0).fit( + X_train, y_train + ) + + assert isinstance(ensemble.estimator_, Perceptron) + + # Regression + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) + + ensemble = BaggingRegressor(None, n_jobs=3, random_state=0).fit(X_train, y_train) + + assert isinstance(ensemble.estimator_, DecisionTreeRegressor) + + ensemble = BaggingRegressor(DecisionTreeRegressor(), n_jobs=3, random_state=0).fit( + X_train, y_train + ) + + assert isinstance(ensemble.estimator_, DecisionTreeRegressor) + + ensemble = BaggingRegressor(SVR(), n_jobs=3, random_state=0).fit(X_train, y_train) + assert isinstance(ensemble.estimator_, SVR) + + +def test_bagging_with_pipeline(): + estimator = BaggingClassifier( + make_pipeline(SelectKBest(k=1), DecisionTreeClassifier()), max_features=2 + ) + estimator.fit(iris.data, iris.target) + assert isinstance(estimator[0].steps[-1][1].random_state, int) + + +def test_warm_start(random_state=42): + # Test if fitting incrementally with warm start gives a forest of the + # right size and the same results as a normal fit. + X, y = make_hastie_10_2(n_samples=20, random_state=1) + + clf_ws = None + for n_estimators in [5, 10]: + if clf_ws is None: + clf_ws = BaggingClassifier( + n_estimators=n_estimators, random_state=random_state, warm_start=True + ) + else: + clf_ws.set_params(n_estimators=n_estimators) + clf_ws.fit(X, y) + assert len(clf_ws) == n_estimators + + clf_no_ws = BaggingClassifier( + n_estimators=10, random_state=random_state, warm_start=False + ) + clf_no_ws.fit(X, y) + + assert set([tree.random_state for tree in clf_ws]) == set( + [tree.random_state for tree in clf_no_ws] + ) + + +def test_warm_start_smaller_n_estimators(): + # Test if warm start'ed second fit with smaller n_estimators raises error. + X, y = make_hastie_10_2(n_samples=20, random_state=1) + clf = BaggingClassifier(n_estimators=5, warm_start=True) + clf.fit(X, y) + clf.set_params(n_estimators=4) + with pytest.raises(ValueError): + clf.fit(X, y) + + +def test_warm_start_equal_n_estimators(): + # Test that nothing happens when fitting without increasing n_estimators + X, y = make_hastie_10_2(n_samples=20, random_state=1) + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=43) + + clf = BaggingClassifier(n_estimators=5, warm_start=True, random_state=83) + clf.fit(X_train, y_train) + + y_pred = clf.predict(X_test) + # modify X to nonsense values, this should not change anything + X_train += 1.0 + + warn_msg = "Warm-start fitting without increasing n_estimators does not" + with pytest.warns(UserWarning, match=warn_msg): + clf.fit(X_train, y_train) + assert_array_equal(y_pred, clf.predict(X_test)) + + +def test_warm_start_equivalence(): + # warm started classifier with 5+5 estimators should be equivalent to + # one classifier with 10 estimators + X, y = make_hastie_10_2(n_samples=20, random_state=1) + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=43) + + clf_ws = BaggingClassifier(n_estimators=5, warm_start=True, random_state=3141) + clf_ws.fit(X_train, y_train) + clf_ws.set_params(n_estimators=10) + clf_ws.fit(X_train, y_train) + y1 = clf_ws.predict(X_test) + + clf = BaggingClassifier(n_estimators=10, warm_start=False, random_state=3141) + clf.fit(X_train, y_train) + y2 = clf.predict(X_test) + + assert_array_almost_equal(y1, y2) + + +def test_warm_start_with_oob_score_fails(): + # Check using oob_score and warm_start simultaneously fails + X, y = make_hastie_10_2(n_samples=20, random_state=1) + clf = BaggingClassifier(n_estimators=5, warm_start=True, oob_score=True) + with pytest.raises(ValueError): + clf.fit(X, y) + + +def test_warning_bootstrap_sample_weight(): + X, y = iris.data, iris.target + sample_weight = np.ones_like(y) + clf = BaggingClassifier(bootstrap=False) + warn_msg = ( + "When fitting BaggingClassifier with sample_weight " + "it is recommended to use bootstrap=True" + ) + with pytest.warns(UserWarning, match=warn_msg): + clf.fit(X, y, sample_weight=sample_weight) + + X, y = diabetes.data, diabetes.target + sample_weight = np.ones_like(y) + reg = BaggingRegressor(bootstrap=False) + warn_msg = ( + "When fitting BaggingRegressor with sample_weight " + "it is recommended to use bootstrap=True" + ) + with pytest.warns(UserWarning, match=warn_msg): + reg.fit(X, y, sample_weight=sample_weight) + + +def test_invalid_sample_weight_max_samples_bootstrap_combinations(): + X, y = iris.data, iris.target + + # Case 1: small weights and fractional max_samples lead to a small + # number of bootstrap samples, which raises a UserWarning. + clf = BaggingClassifier(max_samples=1.0) + sample_weight = np.ones_like(y) / (2 * len(y)) + expected_msg = ( + "Using the fractional value max_samples=1.0 when " + r"the total sum of sample weights is 0.5(\d*) " + r"results in a low number \(1\) of bootstrap samples. " + "We recommend passing `max_samples` as an integer." + ) + with pytest.warns(UserWarning, match=expected_msg): + clf.fit(X, y, sample_weight=sample_weight) + + # Case 2: large weights and bootstrap=False would lead to sampling without + # replacement more than the number of samples, which is not allowed. + clf = BaggingClassifier(bootstrap=False, max_samples=1.0) + sample_weight = np.ones_like(y) + sample_weight[-1] = 2 + expected_msg = re.escape( + "max_samples=151 must be <= n_samples=150 to be able to sample without " + "replacement." + ) + with pytest.raises(ValueError, match=expected_msg): + with pytest.warns( + UserWarning, match="When fitting BaggingClassifier with sample_weight" + ): + clf.fit(X, y, sample_weight=sample_weight) + + +class EstimatorAcceptingSampleWeight(BaseEstimator): + """Fake estimator accepting sample_weight""" + + def fit(self, X, y, sample_weight=None): + """Record values passed during fit""" + self.X_ = X + self.y_ = y + self.sample_weight_ = sample_weight + + def predict(self, X): + pass + + +class EstimatorRejectingSampleWeight(BaseEstimator): + """Fake estimator rejecting sample_weight""" + + def fit(self, X, y): + """Record values passed during fit""" + self.X_ = X + self.y_ = y + + def predict(self, X): + pass + + +@pytest.mark.parametrize("bagging_class", [BaggingRegressor, BaggingClassifier]) +@pytest.mark.parametrize("accept_sample_weight", [False, True]) +@pytest.mark.parametrize("metadata_routing", [False, True]) +@pytest.mark.parametrize("max_samples", [10, 0.8]) +def test_draw_indices_using_sample_weight( + bagging_class, accept_sample_weight, metadata_routing, max_samples +): + X = np.arange(100).reshape(-1, 1) + y = np.repeat([0, 1], 50) + # all indices except 4 and 5 have zero weight + sample_weight = np.zeros(100) + sample_weight[4] = 1 + sample_weight[5] = 2 + if accept_sample_weight: + base_estimator = EstimatorAcceptingSampleWeight() + else: + base_estimator = EstimatorRejectingSampleWeight() + + n_samples, n_features = X.shape + + if isinstance(max_samples, float): + # max_samples passed as a fraction of the input data. Since + # sample_weight are provided, the effective number of samples is the + # sum of the sample weights. + expected_integer_max_samples = int(max_samples * sample_weight.sum()) + else: + expected_integer_max_samples = max_samples + + with config_context(enable_metadata_routing=metadata_routing): + # TODO(slep006): remove block when default routing is implemented + if metadata_routing and accept_sample_weight: + base_estimator = base_estimator.set_fit_request(sample_weight=True) + bagging = bagging_class(base_estimator, max_samples=max_samples, n_estimators=4) + bagging.fit(X, y, sample_weight=sample_weight) + for estimator, samples in zip(bagging.estimators_, bagging.estimators_samples_): + counts = np.bincount(samples, minlength=n_samples) + assert sum(counts) == len(samples) == expected_integer_max_samples + # only indices 4 and 5 should appear + assert np.isin(samples, [4, 5]).all() + if accept_sample_weight: + # sampled indices represented through weighting + assert estimator.X_.shape == (n_samples, n_features) + assert estimator.y_.shape == (n_samples,) + assert_allclose(estimator.X_, X) + assert_allclose(estimator.y_, y) + assert_allclose(estimator.sample_weight_, counts) + else: + # sampled indices represented through indexing + assert estimator.X_.shape == (expected_integer_max_samples, n_features) + assert estimator.y_.shape == (expected_integer_max_samples,) + assert_allclose(estimator.X_, X[samples]) + assert_allclose(estimator.y_, y[samples]) + + +def test_get_n_samples_bootstrap(): + n_samples, max_samples, sample_weight = 10, None, "not_used" + assert _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) == n_samples + + n_samples, max_samples, sample_weight = 10, 5, "not_used" + assert ( + _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) == max_samples + ) + + n_samples, max_samples, sample_weight = 10, 1e-5, None + assert _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) == 1 + + n_samples, max_samples, sample_weight = 10, 0.66, None + warning_msg = ".+the number of samples.+low number.+max_samples.+as an integer" + with pytest.warns(UserWarning, match=warning_msg): + assert _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) == int( + max_samples * n_samples + ) + + n_samples, max_samples, sample_weight = 10, 1e-5, None + with pytest.warns(UserWarning, match=warning_msg): + assert _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) == 1 + + warning_msg_with_weights = ( + ".+the total sum of sample weights.+low number.+max_samples.+as an integer" + ) + rng = np.random.default_rng(0) + n_samples, max_samples, sample_weight = 1_000_000, 1e-5, rng.uniform(size=1_000_000) + with pytest.warns(UserWarning, match=warning_msg_with_weights): + assert _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) == int( + max_samples * sample_weight.sum() + ) + + sample_weight = np.ones(3) + with warnings.catch_warnings(): + warnings.simplefilter("error") + + n_samples, max_samples, sample_weight = 100, 30, None + assert ( + _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) + == max_samples + ) + + n_samples, max_samples, sample_weight = 100, 0.5, rng.uniform(size=100) + assert _get_n_samples_bootstrap(n_samples, max_samples, sample_weight) == int( + max_samples * sample_weight.sum() + ) + + +def test_oob_score_removed_on_warm_start(): + X, y = make_hastie_10_2(n_samples=100, random_state=1) + + clf = BaggingClassifier(n_estimators=5, oob_score=True) + clf.fit(X, y) + + clf.set_params(warm_start=True, oob_score=False, n_estimators=10) + clf.fit(X, y) + + with pytest.raises(AttributeError): + getattr(clf, "oob_score_") + + +def test_oob_score_consistency(): + # Make sure OOB scores are identical when random_state, estimator, and + # training data are fixed and fitting is done twice + X, y = make_hastie_10_2(n_samples=200, random_state=1) + bagging = BaggingClassifier( + KNeighborsClassifier(), + max_samples=0.5, + max_features=0.5, + oob_score=True, + random_state=1, + ) + assert bagging.fit(X, y).oob_score_ == bagging.fit(X, y).oob_score_ + + +def test_estimators_samples(): + # Check that format of estimators_samples_ is correct and that results + # generated at fit time can be identically reproduced at a later time + # using data saved in object attributes. + X, y = make_hastie_10_2(n_samples=200, random_state=1) + bagging = BaggingClassifier( + LogisticRegression(), + max_samples=0.5, + max_features=0.5, + random_state=1, + bootstrap=False, + ) + bagging.fit(X, y) + + # Get relevant attributes + estimators_samples = bagging.estimators_samples_ + estimators_features = bagging.estimators_features_ + estimators = bagging.estimators_ + + # Test for correct formatting + assert len(estimators_samples) == len(estimators) + assert len(estimators_samples[0]) == len(X) // 2 + assert estimators_samples[0].dtype.kind == "i" + + # Re-fit single estimator to test for consistent sampling + estimator_index = 0 + estimator_samples = estimators_samples[estimator_index] + estimator_features = estimators_features[estimator_index] + estimator = estimators[estimator_index] + + X_train = (X[estimator_samples])[:, estimator_features] + y_train = y[estimator_samples] + + orig_coefs = estimator.coef_ + estimator.fit(X_train, y_train) + new_coefs = estimator.coef_ + + assert_array_almost_equal(orig_coefs, new_coefs) + + +def test_estimators_samples_deterministic(): + # This test is a regression test to check that with a random step + # (e.g. SparseRandomProjection) and a given random state, the results + # generated at fit time can be identically reproduced at a later time using + # data saved in object attributes. Check issue #9524 for full discussion. + + iris = load_iris() + X, y = iris.data, iris.target + + base_pipeline = make_pipeline( + SparseRandomProjection(n_components=2), LogisticRegression() + ) + clf = BaggingClassifier(estimator=base_pipeline, max_samples=0.5, random_state=0) + clf.fit(X, y) + pipeline_estimator_coef = clf.estimators_[0].steps[-1][1].coef_.copy() + + estimator = clf.estimators_[0] + estimator_sample = clf.estimators_samples_[0] + estimator_feature = clf.estimators_features_[0] + + X_train = (X[estimator_sample])[:, estimator_feature] + y_train = y[estimator_sample] + + estimator.fit(X_train, y_train) + assert_array_equal(estimator.steps[-1][1].coef_, pipeline_estimator_coef) + + +def test_max_samples_consistency(): + # Make sure validated max_samples and original max_samples are identical + # when valid integer max_samples supplied by user + max_samples = 100 + X, y = make_hastie_10_2(n_samples=2 * max_samples, random_state=1) + bagging = BaggingClassifier( + KNeighborsClassifier(), + max_samples=max_samples, + max_features=0.5, + random_state=1, + ) + bagging.fit(X, y) + assert bagging._max_samples == max_samples + + +def test_set_oob_score_label_encoding(): + # Make sure the oob_score doesn't change when the labels change + # See: https://github.com/scikit-learn/scikit-learn/issues/8933 + random_state = 5 + X = [[-1], [0], [1]] * 5 + Y1 = ["A", "B", "C"] * 5 + Y2 = [-1, 0, 1] * 5 + Y3 = [0, 1, 2] * 5 + x1 = ( + BaggingClassifier(oob_score=True, random_state=random_state) + .fit(X, Y1) + .oob_score_ + ) + x2 = ( + BaggingClassifier(oob_score=True, random_state=random_state) + .fit(X, Y2) + .oob_score_ + ) + x3 = ( + BaggingClassifier(oob_score=True, random_state=random_state) + .fit(X, Y3) + .oob_score_ + ) + assert [x1, x2] == [x3, x3] + + +def replace(X): + X = X.astype("float", copy=True) + X[~np.isfinite(X)] = 0 + return X + + +def test_bagging_regressor_with_missing_inputs(): + # Check that BaggingRegressor can accept X with missing/infinite data + X = np.array( + [ + [1, 3, 5], + [2, None, 6], + [2, np.nan, 6], + [2, np.inf, 6], + [2, -np.inf, 6], + ] + ) + y_values = [ + np.array([2, 3, 3, 3, 3]), + np.array( + [ + [2, 1, 9], + [3, 6, 8], + [3, 6, 8], + [3, 6, 8], + [3, 6, 8], + ] + ), + ] + for y in y_values: + regressor = DecisionTreeRegressor() + pipeline = make_pipeline(FunctionTransformer(replace), regressor) + pipeline.fit(X, y).predict(X) + bagging_regressor = BaggingRegressor(pipeline) + y_hat = bagging_regressor.fit(X, y).predict(X) + assert y.shape == y_hat.shape + + # Verify that exceptions can be raised by wrapper regressor + regressor = DecisionTreeRegressor() + pipeline = make_pipeline(regressor) + with pytest.raises(ValueError): + pipeline.fit(X, y) + bagging_regressor = BaggingRegressor(pipeline) + with pytest.raises(ValueError): + bagging_regressor.fit(X, y) + + +def test_bagging_classifier_with_missing_inputs(): + # Check that BaggingClassifier can accept X with missing/infinite data + X = np.array( + [ + [1, 3, 5], + [2, None, 6], + [2, np.nan, 6], + [2, np.inf, 6], + [2, -np.inf, 6], + ] + ) + y = np.array([3, 6, 6, 6, 6]) + classifier = DecisionTreeClassifier() + pipeline = make_pipeline(FunctionTransformer(replace), classifier) + pipeline.fit(X, y).predict(X) + bagging_classifier = BaggingClassifier(pipeline) + bagging_classifier.fit(X, y) + y_hat = bagging_classifier.predict(X) + assert y.shape == y_hat.shape + bagging_classifier.predict_log_proba(X) + bagging_classifier.predict_proba(X) + + # Verify that exceptions can be raised by wrapper classifier + classifier = DecisionTreeClassifier() + pipeline = make_pipeline(classifier) + with pytest.raises(ValueError): + pipeline.fit(X, y) + bagging_classifier = BaggingClassifier(pipeline) + with pytest.raises(ValueError): + bagging_classifier.fit(X, y) + + +def test_bagging_small_max_features(): + # Check that Bagging estimator can accept low fractional max_features + + X = np.array([[1, 2], [3, 4]]) + y = np.array([1, 0]) + + bagging = BaggingClassifier(LogisticRegression(), max_features=0.3, random_state=1) + bagging.fit(X, y) + + +def test_bagging_get_estimators_indices(global_random_seed): + # Check that Bagging estimator can generate sample indices properly + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/16436 + + rng = np.random.RandomState(global_random_seed) + X = rng.randn(13, 4) + y = np.arange(13) + + class MyEstimator(DecisionTreeRegressor): + """An estimator which stores y indices information at fit.""" + + def fit(self, X, y): + self._sample_indices = y + + clf = BaggingRegressor(estimator=MyEstimator(), n_estimators=1, random_state=0) + clf.fit(X, y) + + assert_array_equal(clf.estimators_[0]._sample_indices, clf.estimators_samples_[0]) + + +@pytest.mark.parametrize( + "bagging, expected_allow_nan", + [ + (BaggingClassifier(HistGradientBoostingClassifier(max_iter=1)), True), + (BaggingRegressor(HistGradientBoostingRegressor(max_iter=1)), True), + (BaggingClassifier(LogisticRegression()), False), + (BaggingRegressor(SVR()), False), + ], +) +def test_bagging_allow_nan_tag(bagging, expected_allow_nan): + """Check that bagging inherits allow_nan tag.""" + assert bagging.__sklearn_tags__().input_tags.allow_nan == expected_allow_nan + + +# Metadata Routing Tests +# ====================== + + +@config_context(enable_metadata_routing=True) +@pytest.mark.parametrize( + "model", + [ + BaggingClassifier( + estimator=RandomForestClassifier(n_estimators=1), n_estimators=1 + ), + BaggingRegressor( + estimator=RandomForestRegressor(n_estimators=1), n_estimators=1 + ), + ], +) +def test_bagging_with_metadata_routing(model): + """Make sure that metadata routing works with non-default estimator.""" + model.fit(iris.data, iris.target) + + +@pytest.mark.parametrize( + "sub_estimator, caller, callee", + [ + (ConsumingClassifierWithoutPredictProba, "predict", "predict"), + ( + ConsumingClassifierWithoutPredictLogProba, + "predict_log_proba", + "predict_proba", + ), + (ConsumingClassifierWithOnlyPredict, "predict_log_proba", "predict"), + ], +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_with_dynamic_method_selection(sub_estimator, caller, callee): + """Test that metadata routing works in `BaggingClassifier` with dynamic selection of + the sub-estimator's methods. Here we test only specific test cases, where + sub-estimator methods are not present and are not tested with `ConsumingClassifier` + (which possesses all the methods) in + sklearn/tests/test_metaestimators_metadata_routing.py: `BaggingClassifier.predict()` + dynamically routes to `predict` if the sub-estimator doesn't have `predict_proba` + and `BaggingClassifier.predict_log_proba()` dynamically routes to `predict_proba` if + the sub-estimator doesn't have `predict_log_proba`, or to `predict`, if it doesn't + have it. + """ + X = np.array([[0, 2], [1, 4], [2, 6]]) + y = [1, 2, 3] + sample_weight, metadata = [1], "a" + registry = _Registry() + estimator = sub_estimator(registry=registry) + set_callee_request = "set_" + callee + "_request" + getattr(estimator, set_callee_request)(sample_weight=True, metadata=True) + + bagging = BaggingClassifier(estimator=estimator) + bagging.fit(X, y) + getattr(bagging, caller)( + X=np.array([[1, 1], [1, 3], [0, 2]]), + sample_weight=sample_weight, + metadata=metadata, + ) + + assert len(registry) + for estimator in registry: + check_recorded_metadata( + obj=estimator, + method=callee, + parent=caller, + sample_weight=sample_weight, + metadata=metadata, + ) + + +# End of Metadata Routing Tests +# ============================= + + +@pytest.mark.parametrize( + "model", + [ + BaggingClassifier( + estimator=AdaBoostClassifier(n_estimators=1), + n_estimators=1, + ), + BaggingRegressor(estimator=AdaBoostRegressor(n_estimators=1), n_estimators=1), + ], +) +def test_bagging_without_support_metadata_routing(model): + """Make sure that we still can use an estimator that does not implement the + metadata routing.""" + model.fit(iris.data, iris.target) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_base.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_base.py new file mode 100644 index 0000000000000000000000000000000000000000..0c71b85336131e456335fadd1ed16dfeee740edb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_base.py @@ -0,0 +1,109 @@ +""" +Testing for the base module (sklearn.ensemble.base). +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from collections import OrderedDict + +import numpy as np + +from sklearn.datasets import load_iris +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis +from sklearn.ensemble import BaggingClassifier +from sklearn.ensemble._base import _set_random_states +from sklearn.feature_selection import SelectFromModel +from sklearn.linear_model import Perceptron +from sklearn.pipeline import Pipeline + + +def test_base(): + # Check BaseEnsemble methods. + ensemble = BaggingClassifier( + estimator=Perceptron(random_state=None), n_estimators=3 + ) + + iris = load_iris() + ensemble.fit(iris.data, iris.target) + ensemble.estimators_ = [] # empty the list and create estimators manually + + ensemble._make_estimator() + random_state = np.random.RandomState(3) + ensemble._make_estimator(random_state=random_state) + ensemble._make_estimator(random_state=random_state) + ensemble._make_estimator(append=False) + + assert 3 == len(ensemble) + assert 3 == len(ensemble.estimators_) + + assert isinstance(ensemble[0], Perceptron) + assert ensemble[0].random_state is None + assert isinstance(ensemble[1].random_state, int) + assert isinstance(ensemble[2].random_state, int) + assert ensemble[1].random_state != ensemble[2].random_state + + np_int_ensemble = BaggingClassifier( + estimator=Perceptron(), n_estimators=np.int32(3) + ) + np_int_ensemble.fit(iris.data, iris.target) + + +def test_set_random_states(): + # Linear Discriminant Analysis doesn't have random state: smoke test + _set_random_states(LinearDiscriminantAnalysis(), random_state=17) + + clf1 = Perceptron(random_state=None) + assert clf1.random_state is None + # check random_state is None still sets + _set_random_states(clf1, None) + assert isinstance(clf1.random_state, int) + + # check random_state fixes results in consistent initialisation + _set_random_states(clf1, 3) + assert isinstance(clf1.random_state, int) + clf2 = Perceptron(random_state=None) + _set_random_states(clf2, 3) + assert clf1.random_state == clf2.random_state + + # nested random_state + + def make_steps(): + return [ + ("sel", SelectFromModel(Perceptron(random_state=None))), + ("clf", Perceptron(random_state=None)), + ] + + est1 = Pipeline(make_steps()) + _set_random_states(est1, 3) + assert isinstance(est1.steps[0][1].estimator.random_state, int) + assert isinstance(est1.steps[1][1].random_state, int) + assert ( + est1.get_params()["sel__estimator__random_state"] + != est1.get_params()["clf__random_state"] + ) + + # ensure multiple random_state parameters are invariant to get_params() + # iteration order + + class AlphaParamPipeline(Pipeline): + def get_params(self, *args, **kwargs): + params = Pipeline.get_params(self, *args, **kwargs).items() + return OrderedDict(sorted(params)) + + class RevParamPipeline(Pipeline): + def get_params(self, *args, **kwargs): + params = Pipeline.get_params(self, *args, **kwargs).items() + return OrderedDict(sorted(params, reverse=True)) + + for cls in [AlphaParamPipeline, RevParamPipeline]: + est2 = cls(make_steps()) + _set_random_states(est2, 3) + assert ( + est1.get_params()["sel__estimator__random_state"] + == est2.get_params()["sel__estimator__random_state"] + ) + assert ( + est1.get_params()["clf__random_state"] + == est2.get_params()["clf__random_state"] + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_common.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_common.py new file mode 100644 index 0000000000000000000000000000000000000000..9075ff55d71fc8845c91b9acde4a5f6686813960 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_common.py @@ -0,0 +1,263 @@ +import numpy as np +import pytest + +from sklearn.base import ClassifierMixin, clone, is_classifier +from sklearn.datasets import ( + load_diabetes, + load_iris, + make_classification, + make_regression, +) +from sklearn.ensemble import ( + RandomForestClassifier, + RandomForestRegressor, + StackingClassifier, + StackingRegressor, + VotingClassifier, + VotingRegressor, +) +from sklearn.impute import SimpleImputer +from sklearn.linear_model import LinearRegression, LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.svm import SVC, SVR, LinearSVC + +X, y = load_iris(return_X_y=True) + +X_r, y_r = load_diabetes(return_X_y=True) + + +@pytest.mark.parametrize( + "X, y, estimator", + [ + ( + *make_classification(n_samples=10), + StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC()), + ("rf", RandomForestClassifier(n_estimators=5, max_depth=3)), + ], + cv=2, + ), + ), + ( + *make_classification(n_samples=10), + VotingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC()), + ("rf", RandomForestClassifier(n_estimators=5, max_depth=3)), + ] + ), + ), + ( + *make_regression(n_samples=10), + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", SVR(kernel="linear")), + ("rf", RandomForestRegressor(n_estimators=5, max_depth=3)), + ], + cv=2, + ), + ), + ( + *make_regression(n_samples=10), + VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", SVR(kernel="linear")), + ("rf", RandomForestRegressor(n_estimators=5, max_depth=3)), + ] + ), + ), + ], + ids=[ + "stacking-classifier", + "voting-classifier", + "stacking-regressor", + "voting-regressor", + ], +) +def test_ensemble_heterogeneous_estimators_behavior(X, y, estimator): + # check that the behavior of `estimators`, `estimators_`, + # `named_estimators`, `named_estimators_` is consistent across all + # ensemble classes and when using `set_params()`. + estimator = clone(estimator) # Avoid side effects from shared instances + + # before fit + assert "svm" in estimator.named_estimators + assert estimator.named_estimators.svm is estimator.estimators[1][1] + assert estimator.named_estimators.svm is estimator.named_estimators["svm"] + + # check fitted attributes + estimator.fit(X, y) + assert len(estimator.named_estimators) == 3 + assert len(estimator.named_estimators_) == 3 + assert sorted(list(estimator.named_estimators_.keys())) == sorted( + ["lr", "svm", "rf"] + ) + + # check that set_params() does not add a new attribute + estimator_new_params = clone(estimator) + svm_estimator = SVC() if is_classifier(estimator) else SVR() + estimator_new_params.set_params(svm=svm_estimator).fit(X, y) + assert not hasattr(estimator_new_params, "svm") + assert ( + estimator_new_params.named_estimators.lr.get_params() + == estimator.named_estimators.lr.get_params() + ) + assert ( + estimator_new_params.named_estimators.rf.get_params() + == estimator.named_estimators.rf.get_params() + ) + + # check the behavior when setting and dropping an estimator + estimator_dropped = clone(estimator) + estimator_dropped.set_params(svm="drop") + estimator_dropped.fit(X, y) + assert len(estimator_dropped.named_estimators) == 3 + assert estimator_dropped.named_estimators.svm == "drop" + assert len(estimator_dropped.named_estimators_) == 3 + assert sorted(list(estimator_dropped.named_estimators_.keys())) == sorted( + ["lr", "svm", "rf"] + ) + for sub_est in estimator_dropped.named_estimators_: + # check that the correspondence is correct + assert not isinstance(sub_est, type(estimator.named_estimators.svm)) + + # check that we can set the parameters of the underlying classifier + estimator.set_params(svm__C=10.0) + estimator.set_params(rf__max_depth=5) + assert ( + estimator.get_params()["svm__C"] + == estimator.get_params()["svm"].get_params()["C"] + ) + assert ( + estimator.get_params()["rf__max_depth"] + == estimator.get_params()["rf"].get_params()["max_depth"] + ) + + +@pytest.mark.parametrize( + "Ensemble", + [VotingClassifier, StackingRegressor, VotingRegressor], +) +def test_ensemble_heterogeneous_estimators_type(Ensemble): + # check that ensemble will fail during validation if the underlying + # estimators are not of the same type (i.e. classifier or regressor) + # StackingClassifier can have an underlying regresor so it's not checked + if issubclass(Ensemble, ClassifierMixin): + X, y = make_classification(n_samples=10) + estimators = [("lr", LinearRegression())] + ensemble_type = "classifier" + else: + X, y = make_regression(n_samples=10) + estimators = [("lr", LogisticRegression())] + ensemble_type = "regressor" + ensemble = Ensemble(estimators=estimators) + + err_msg = "should be a {}".format(ensemble_type) + with pytest.raises(ValueError, match=err_msg): + ensemble.fit(X, y) + + +@pytest.mark.parametrize( + "X, y, Ensemble", + [ + (*make_classification(n_samples=10), StackingClassifier), + (*make_classification(n_samples=10), VotingClassifier), + (*make_regression(n_samples=10), StackingRegressor), + (*make_regression(n_samples=10), VotingRegressor), + ], +) +def test_ensemble_heterogeneous_estimators_name_validation(X, y, Ensemble): + # raise an error when the name contains dunder + if issubclass(Ensemble, ClassifierMixin): + estimators = [("lr__", LogisticRegression())] + else: + estimators = [("lr__", LinearRegression())] + ensemble = Ensemble(estimators=estimators) + + err_msg = r"Estimator names must not contain __: got \['lr__'\]" + with pytest.raises(ValueError, match=err_msg): + ensemble.fit(X, y) + + # raise an error when the name is not unique + if issubclass(Ensemble, ClassifierMixin): + estimators = [("lr", LogisticRegression()), ("lr", LogisticRegression())] + else: + estimators = [("lr", LinearRegression()), ("lr", LinearRegression())] + ensemble = Ensemble(estimators=estimators) + + err_msg = r"Names provided are not unique: \['lr', 'lr'\]" + with pytest.raises(ValueError, match=err_msg): + ensemble.fit(X, y) + + # raise an error when the name conflicts with the parameters + if issubclass(Ensemble, ClassifierMixin): + estimators = [("estimators", LogisticRegression())] + else: + estimators = [("estimators", LinearRegression())] + ensemble = Ensemble(estimators=estimators) + + err_msg = "Estimator names conflict with constructor arguments" + with pytest.raises(ValueError, match=err_msg): + ensemble.fit(X, y) + + +@pytest.mark.parametrize( + "X, y, estimator", + [ + ( + *make_classification(n_samples=10), + StackingClassifier(estimators=[("lr", LogisticRegression())]), + ), + ( + *make_classification(n_samples=10), + VotingClassifier(estimators=[("lr", LogisticRegression())]), + ), + ( + *make_regression(n_samples=10), + StackingRegressor(estimators=[("lr", LinearRegression())]), + ), + ( + *make_regression(n_samples=10), + VotingRegressor(estimators=[("lr", LinearRegression())]), + ), + ], + ids=[ + "stacking-classifier", + "voting-classifier", + "stacking-regressor", + "voting-regressor", + ], +) +def test_ensemble_heterogeneous_estimators_all_dropped(X, y, estimator): + # check that we raise a consistent error when all estimators are + # dropped + estimator.set_params(lr="drop") + with pytest.raises(ValueError, match="All estimators are dropped."): + estimator.fit(X, y) + + +@pytest.mark.parametrize( + "Ensemble, Estimator, X, y", + [ + (StackingClassifier, LogisticRegression, X, y), + (StackingRegressor, LinearRegression, X_r, y_r), + (VotingClassifier, LogisticRegression, X, y), + (VotingRegressor, LinearRegression, X_r, y_r), + ], +) +# FIXME: we should move this test in `estimator_checks` once we are able +# to construct meta-estimator instances +def test_heterogeneous_ensemble_support_missing_values(Ensemble, Estimator, X, y): + # check that Voting and Stacking predictor delegate the missing values + # validation to the underlying estimator. + X = X.copy() + mask = np.random.choice([1, 0], X.shape, p=[0.1, 0.9]).astype(bool) + X[mask] = np.nan + pipe = make_pipeline(SimpleImputer(), Estimator()) + ensemble = Ensemble(estimators=[("pipe1", pipe), ("pipe2", pipe)]) + ensemble.fit(X, y).score(X, y) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_forest.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_forest.py new file mode 100644 index 0000000000000000000000000000000000000000..07152d7dffaab9ad86d5858691e4585c0c45e4e6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_forest.py @@ -0,0 +1,1868 @@ +""" +Testing for the forest module (sklearn.ensemble.forest). +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools +import math +import pickle +from collections import defaultdict +from functools import partial +from itertools import combinations, product +from typing import Any, Dict +from unittest.mock import patch + +import joblib +import numpy as np +import pytest +from scipy.special import comb + +import sklearn +from sklearn import clone, datasets +from sklearn.datasets import make_classification, make_hastie_10_2 +from sklearn.decomposition import TruncatedSVD +from sklearn.dummy import DummyRegressor +from sklearn.ensemble import ( + ExtraTreesClassifier, + ExtraTreesRegressor, + RandomForestClassifier, + RandomForestRegressor, + RandomTreesEmbedding, +) +from sklearn.ensemble._forest import ( + _generate_unsampled_indices, + _get_n_samples_bootstrap, +) +from sklearn.exceptions import NotFittedError +from sklearn.metrics import ( + explained_variance_score, + f1_score, + mean_poisson_deviance, + mean_squared_error, +) +from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split +from sklearn.svm import LinearSVC +from sklearn.tree._classes import SPARSE_SPLITTERS +from sklearn.utils._testing import ( + _convert_container, + assert_allclose, + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, + ignore_warnings, + skip_if_no_parallel, +) +from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS +from sklearn.utils.multiclass import type_of_target +from sklearn.utils.parallel import Parallel +from sklearn.utils.validation import check_random_state + +# toy sample +X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] +y = [-1, -1, -1, 1, 1, 1] +T = [[-1, -1], [2, 2], [3, 2]] +true_result = [-1, 1, 1] + +# Larger classification sample used for testing feature importances +X_large, y_large = datasets.make_classification( + n_samples=500, + n_features=10, + n_informative=3, + n_redundant=0, + n_repeated=0, + shuffle=False, + random_state=0, +) + +# also load the iris dataset +# and randomly permute it +iris = datasets.load_iris() +rng = check_random_state(0) +perm = rng.permutation(iris.target.size) +iris.data = iris.data[perm] +iris.target = iris.target[perm] + +# Make regression dataset +X_reg, y_reg = datasets.make_regression(n_samples=500, n_features=10, random_state=1) + +# also make a hastie_10_2 dataset +hastie_X, hastie_y = datasets.make_hastie_10_2(n_samples=20, random_state=1) +hastie_X = hastie_X.astype(np.float32) + +# Get the default backend in joblib to test parallelism and interaction with +# different backends +DEFAULT_JOBLIB_BACKEND = joblib.parallel.get_active_backend()[0].__class__ + +FOREST_CLASSIFIERS = { + "ExtraTreesClassifier": ExtraTreesClassifier, + "RandomForestClassifier": RandomForestClassifier, +} + +FOREST_REGRESSORS = { + "ExtraTreesRegressor": ExtraTreesRegressor, + "RandomForestRegressor": RandomForestRegressor, +} + +FOREST_TRANSFORMERS = { + "RandomTreesEmbedding": RandomTreesEmbedding, +} + +FOREST_ESTIMATORS: Dict[str, Any] = dict() +FOREST_ESTIMATORS.update(FOREST_CLASSIFIERS) +FOREST_ESTIMATORS.update(FOREST_REGRESSORS) +FOREST_ESTIMATORS.update(FOREST_TRANSFORMERS) + +FOREST_CLASSIFIERS_REGRESSORS: Dict[str, Any] = FOREST_CLASSIFIERS.copy() +FOREST_CLASSIFIERS_REGRESSORS.update(FOREST_REGRESSORS) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_classification_toy(name): + """Check classification on a toy dataset.""" + ForestClassifier = FOREST_CLASSIFIERS[name] + + clf = ForestClassifier(n_estimators=10, random_state=1) + clf.fit(X, y) + assert_array_equal(clf.predict(T), true_result) + assert 10 == len(clf) + + clf = ForestClassifier(n_estimators=10, max_features=1, random_state=1) + clf.fit(X, y) + assert_array_equal(clf.predict(T), true_result) + assert 10 == len(clf) + + # also test apply + leaf_indices = clf.apply(X) + assert leaf_indices.shape == (len(X), clf.n_estimators) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +@pytest.mark.parametrize("criterion", ("gini", "log_loss")) +def test_iris_criterion(name, criterion): + # Check consistency on dataset iris. + ForestClassifier = FOREST_CLASSIFIERS[name] + + clf = ForestClassifier(n_estimators=10, criterion=criterion, random_state=1) + clf.fit(iris.data, iris.target) + score = clf.score(iris.data, iris.target) + assert score > 0.9, "Failed with criterion %s and score = %f" % (criterion, score) + + clf = ForestClassifier( + n_estimators=10, criterion=criterion, max_features=2, random_state=1 + ) + clf.fit(iris.data, iris.target) + score = clf.score(iris.data, iris.target) + assert score > 0.5, "Failed with criterion %s and score = %f" % (criterion, score) + + +@pytest.mark.parametrize("name", FOREST_REGRESSORS) +@pytest.mark.parametrize( + "criterion", ("squared_error", "absolute_error", "friedman_mse") +) +def test_regression_criterion(name, criterion): + # Check consistency on regression dataset. + ForestRegressor = FOREST_REGRESSORS[name] + + reg = ForestRegressor(n_estimators=5, criterion=criterion, random_state=1) + reg.fit(X_reg, y_reg) + score = reg.score(X_reg, y_reg) + assert score > 0.93, ( + "Failed with max_features=None, criterion %s and score = %f" + % ( + criterion, + score, + ) + ) + + reg = ForestRegressor( + n_estimators=5, criterion=criterion, max_features=6, random_state=1 + ) + reg.fit(X_reg, y_reg) + score = reg.score(X_reg, y_reg) + assert score > 0.92, "Failed with max_features=6, criterion %s and score = %f" % ( + criterion, + score, + ) + + +def test_poisson_vs_mse(): + """Test that random forest with poisson criterion performs better than + mse for a poisson target. + + There is a similar test for DecisionTreeRegressor. + """ + rng = np.random.RandomState(42) + n_train, n_test, n_features = 500, 500, 10 + X = datasets.make_low_rank_matrix( + n_samples=n_train + n_test, n_features=n_features, random_state=rng + ) + # We create a log-linear Poisson model and downscale coef as it will get + # exponentiated. + coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0) + y = rng.poisson(lam=np.exp(X @ coef)) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=n_test, random_state=rng + ) + # We prevent some overfitting by setting min_samples_split=10. + forest_poi = RandomForestRegressor( + criterion="poisson", min_samples_leaf=10, max_features="sqrt", random_state=rng + ) + forest_mse = RandomForestRegressor( + criterion="squared_error", + min_samples_leaf=10, + max_features="sqrt", + random_state=rng, + ) + + forest_poi.fit(X_train, y_train) + forest_mse.fit(X_train, y_train) + dummy = DummyRegressor(strategy="mean").fit(X_train, y_train) + + for X, y, data_name in [(X_train, y_train, "train"), (X_test, y_test, "test")]: + metric_poi = mean_poisson_deviance(y, forest_poi.predict(X)) + # squared_error forest might produce non-positive predictions => clip + # If y = 0 for those, the poisson deviance gets too good. + # If we drew more samples, we would eventually get y > 0 and the + # poisson deviance would explode, i.e. be undefined. Therefore, we do + # not clip to a tiny value like 1e-15, but to 1e-6. This acts like a + # small penalty to the non-positive predictions. + metric_mse = mean_poisson_deviance( + y, np.clip(forest_mse.predict(X), 1e-6, None) + ) + metric_dummy = mean_poisson_deviance(y, dummy.predict(X)) + # As squared_error might correctly predict 0 in train set, its train + # score can be better than Poisson. This is no longer the case for the + # test set. But keep the above comment for clipping in mind. + if data_name == "test": + assert metric_poi < metric_mse + assert metric_poi < 0.8 * metric_dummy + + +@pytest.mark.parametrize("criterion", ("poisson", "squared_error")) +def test_balance_property_random_forest(criterion): + """ "Test that sum(y_pred)==sum(y_true) on the training set.""" + rng = np.random.RandomState(42) + n_train, n_test, n_features = 500, 500, 10 + X = datasets.make_low_rank_matrix( + n_samples=n_train + n_test, n_features=n_features, random_state=rng + ) + + coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0) + y = rng.poisson(lam=np.exp(X @ coef)) + + reg = RandomForestRegressor( + criterion=criterion, n_estimators=10, bootstrap=False, random_state=rng + ) + reg.fit(X, y) + + assert np.sum(reg.predict(X)) == pytest.approx(np.sum(y)) + + +@pytest.mark.parametrize("name", FOREST_REGRESSORS) +def test_regressor_attributes(name): + # Regression models should not have a classes_ attribute. + r = FOREST_REGRESSORS[name](random_state=0) + assert not hasattr(r, "classes_") + assert not hasattr(r, "n_classes_") + + r.fit([[1, 2, 3], [4, 5, 6]], [1, 2]) + assert not hasattr(r, "classes_") + assert not hasattr(r, "n_classes_") + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_probability(name): + # Predict probabilities. + ForestClassifier = FOREST_CLASSIFIERS[name] + with np.errstate(divide="ignore"): + clf = ForestClassifier( + n_estimators=10, random_state=1, max_features=1, max_depth=1 + ) + clf.fit(iris.data, iris.target) + assert_array_almost_equal( + np.sum(clf.predict_proba(iris.data), axis=1), np.ones(iris.data.shape[0]) + ) + assert_array_almost_equal( + clf.predict_proba(iris.data), np.exp(clf.predict_log_proba(iris.data)) + ) + + +@pytest.mark.parametrize("dtype", (np.float64, np.float32)) +@pytest.mark.parametrize( + "name, criterion", + itertools.chain( + product(FOREST_CLASSIFIERS, ["gini", "log_loss"]), + product(FOREST_REGRESSORS, ["squared_error", "friedman_mse", "absolute_error"]), + ), +) +def test_importances(dtype, name, criterion): + tolerance = 0.01 + if name in FOREST_REGRESSORS and criterion == "absolute_error": + tolerance = 0.05 + + # cast as dtype + X = X_large.astype(dtype, copy=False) + y = y_large.astype(dtype, copy=False) + + ForestEstimator = FOREST_ESTIMATORS[name] + + est = ForestEstimator(n_estimators=10, criterion=criterion, random_state=0) + est.fit(X, y) + importances = est.feature_importances_ + + # The forest estimator can detect that only the first 3 features of the + # dataset are informative: + n_important = np.sum(importances > 0.1) + assert importances.shape[0] == 10 + assert n_important == 3 + assert np.all(importances[:3] > 0.1) + + # Check with parallel + importances = est.feature_importances_ + est.set_params(n_jobs=2) + importances_parallel = est.feature_importances_ + assert_array_almost_equal(importances, importances_parallel) + + # Check with sample weights + sample_weight = check_random_state(0).randint(1, 10, len(X)) + est = ForestEstimator(n_estimators=10, random_state=0, criterion=criterion) + est.fit(X, y, sample_weight=sample_weight) + importances = est.feature_importances_ + assert np.all(importances >= 0.0) + + for scale in [0.5, 100]: + est = ForestEstimator(n_estimators=10, random_state=0, criterion=criterion) + est.fit(X, y, sample_weight=scale * sample_weight) + importances_bis = est.feature_importances_ + assert np.abs(importances - importances_bis).mean() < tolerance + + +def test_importances_asymptotic(): + # Check whether variable importances of totally randomized trees + # converge towards their theoretical values (See Louppe et al, + # Understanding variable importances in forests of randomized trees, 2013). + + def binomial(k, n): + return 0 if k < 0 or k > n else comb(int(n), int(k), exact=True) + + def entropy(samples): + n_samples = len(samples) + entropy = 0.0 + + for count in np.bincount(samples): + p = 1.0 * count / n_samples + if p > 0: + entropy -= p * np.log2(p) + + return entropy + + def mdi_importance(X_m, X, y): + n_samples, n_features = X.shape + + features = list(range(n_features)) + features.pop(X_m) + values = [np.unique(X[:, i]) for i in range(n_features)] + + imp = 0.0 + + for k in range(n_features): + # Weight of each B of size k + coef = 1.0 / (binomial(k, n_features) * (n_features - k)) + + # For all B of size k + for B in combinations(features, k): + # For all values B=b + for b in product(*[values[B[j]] for j in range(k)]): + mask_b = np.ones(n_samples, dtype=bool) + + for j in range(k): + mask_b &= X[:, B[j]] == b[j] + + X_, y_ = X[mask_b, :], y[mask_b] + n_samples_b = len(X_) + + if n_samples_b > 0: + children = [] + + for xi in values[X_m]: + mask_xi = X_[:, X_m] == xi + children.append(y_[mask_xi]) + + imp += ( + coef + * (1.0 * n_samples_b / n_samples) # P(B=b) + * ( + entropy(y_) + - sum( + [ + entropy(c) * len(c) / n_samples_b + for c in children + ] + ) + ) + ) + + return imp + + data = np.array( + [ + [0, 0, 1, 0, 0, 1, 0, 1], + [1, 0, 1, 1, 1, 0, 1, 2], + [1, 0, 1, 1, 0, 1, 1, 3], + [0, 1, 1, 1, 0, 1, 0, 4], + [1, 1, 0, 1, 0, 1, 1, 5], + [1, 1, 0, 1, 1, 1, 1, 6], + [1, 0, 1, 0, 0, 1, 0, 7], + [1, 1, 1, 1, 1, 1, 1, 8], + [1, 1, 1, 1, 0, 1, 1, 9], + [1, 1, 1, 0, 1, 1, 1, 0], + ] + ) + + X, y = np.array(data[:, :7], dtype=bool), data[:, 7] + n_features = X.shape[1] + + # Compute true importances + true_importances = np.zeros(n_features) + + for i in range(n_features): + true_importances[i] = mdi_importance(i, X, y) + + # Estimate importances with totally randomized trees + clf = ExtraTreesClassifier( + n_estimators=500, max_features=1, criterion="log_loss", random_state=0 + ).fit(X, y) + + importances = ( + sum( + tree.tree_.compute_feature_importances(normalize=False) + for tree in clf.estimators_ + ) + / clf.n_estimators + ) + + # Check correctness + assert_almost_equal(entropy(y), sum(importances)) + assert np.abs(true_importances - importances).mean() < 0.01 + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_unfitted_feature_importances(name): + err_msg = ( + "This {} instance is not fitted yet. Call 'fit' with " + "appropriate arguments before using this estimator.".format(name) + ) + with pytest.raises(NotFittedError, match=err_msg): + getattr(FOREST_ESTIMATORS[name](), "feature_importances_") + + +@pytest.mark.parametrize("ForestClassifier", FOREST_CLASSIFIERS.values()) +@pytest.mark.parametrize("X_type", ["array", "sparse_csr", "sparse_csc"]) +@pytest.mark.parametrize( + "X, y, lower_bound_accuracy", + [ + ( + *datasets.make_classification(n_samples=300, n_classes=2, random_state=0), + 0.9, + ), + ( + *datasets.make_classification( + n_samples=1000, n_classes=3, n_informative=6, random_state=0 + ), + 0.65, + ), + ( + iris.data, + iris.target * 2 + 1, + 0.65, + ), + ( + *datasets.make_multilabel_classification(n_samples=300, random_state=0), + 0.18, + ), + ], +) +@pytest.mark.parametrize("oob_score", [True, partial(f1_score, average="micro")]) +def test_forest_classifier_oob( + ForestClassifier, X, y, X_type, lower_bound_accuracy, oob_score +): + """Check that OOB score is close to score on a test set.""" + X = _convert_container(X, constructor_name=X_type) + X_train, X_test, y_train, y_test = train_test_split( + X, + y, + test_size=0.5, + random_state=0, + ) + classifier = ForestClassifier( + n_estimators=40, + bootstrap=True, + oob_score=oob_score, + random_state=0, + ) + + assert not hasattr(classifier, "oob_score_") + assert not hasattr(classifier, "oob_decision_function_") + + classifier.fit(X_train, y_train) + if callable(oob_score): + test_score = oob_score(y_test, classifier.predict(X_test)) + else: + test_score = classifier.score(X_test, y_test) + assert classifier.oob_score_ >= lower_bound_accuracy + + abs_diff = abs(test_score - classifier.oob_score_) + assert abs_diff <= 0.11, f"{abs_diff=} is greater than 0.11" + + assert hasattr(classifier, "oob_score_") + assert not hasattr(classifier, "oob_prediction_") + assert hasattr(classifier, "oob_decision_function_") + + if y.ndim == 1: + expected_shape = (X_train.shape[0], len(set(y))) + else: + expected_shape = (X_train.shape[0], len(set(y[:, 0])), y.shape[1]) + assert classifier.oob_decision_function_.shape == expected_shape + + +@pytest.mark.parametrize("ForestRegressor", FOREST_REGRESSORS.values()) +@pytest.mark.parametrize("X_type", ["array", "sparse_csr", "sparse_csc"]) +@pytest.mark.parametrize( + "X, y, lower_bound_r2", + [ + ( + *datasets.make_regression( + n_samples=500, n_features=10, n_targets=1, random_state=0 + ), + 0.7, + ), + ( + *datasets.make_regression( + n_samples=500, n_features=10, n_targets=2, random_state=0 + ), + 0.55, + ), + ], +) +@pytest.mark.parametrize("oob_score", [True, explained_variance_score]) +def test_forest_regressor_oob(ForestRegressor, X, y, X_type, lower_bound_r2, oob_score): + """Check that forest-based regressor provide an OOB score close to the + score on a test set.""" + X = _convert_container(X, constructor_name=X_type) + X_train, X_test, y_train, y_test = train_test_split( + X, + y, + test_size=0.5, + random_state=0, + ) + regressor = ForestRegressor( + n_estimators=50, + bootstrap=True, + oob_score=oob_score, + random_state=0, + ) + + assert not hasattr(regressor, "oob_score_") + assert not hasattr(regressor, "oob_prediction_") + + regressor.fit(X_train, y_train) + if callable(oob_score): + test_score = oob_score(y_test, regressor.predict(X_test)) + else: + test_score = regressor.score(X_test, y_test) + assert regressor.oob_score_ >= lower_bound_r2 + + assert abs(test_score - regressor.oob_score_) <= 0.1 + + assert hasattr(regressor, "oob_score_") + assert hasattr(regressor, "oob_prediction_") + assert not hasattr(regressor, "oob_decision_function_") + + if y.ndim == 1: + expected_shape = (X_train.shape[0],) + else: + expected_shape = (X_train.shape[0], y.ndim) + assert regressor.oob_prediction_.shape == expected_shape + + +@pytest.mark.parametrize("ForestEstimator", FOREST_CLASSIFIERS_REGRESSORS.values()) +def test_forest_oob_warning(ForestEstimator): + """Check that a warning is raised when not enough estimator and the OOB + estimates will be inaccurate.""" + estimator = ForestEstimator( + n_estimators=1, + oob_score=True, + bootstrap=True, + random_state=0, + ) + with pytest.warns(UserWarning, match="Some inputs do not have OOB scores"): + estimator.fit(iris.data, iris.target) + + +@pytest.mark.parametrize("ForestEstimator", FOREST_CLASSIFIERS_REGRESSORS.values()) +def test_forest_oob_score_requires_bootstrap(ForestEstimator): + """Check that we raise an error if OOB score is requested without + activating bootstrapping. + """ + X = iris.data + y = iris.target + err_msg = "Out of bag estimation only available if bootstrap=True" + estimator = ForestEstimator(oob_score=True, bootstrap=False) + with pytest.raises(ValueError, match=err_msg): + estimator.fit(X, y) + + +@pytest.mark.parametrize("ForestClassifier", FOREST_CLASSIFIERS.values()) +def test_classifier_error_oob_score_multiclass_multioutput(ForestClassifier): + """Check that we raise an error with when requesting OOB score with + multiclass-multioutput classification target. + """ + rng = np.random.RandomState(42) + X = iris.data + y = rng.randint(low=0, high=5, size=(iris.data.shape[0], 2)) + y_type = type_of_target(y) + assert y_type == "multiclass-multioutput" + estimator = ForestClassifier(oob_score=True, bootstrap=True) + err_msg = "The type of target cannot be used to compute OOB estimates" + with pytest.raises(ValueError, match=err_msg): + estimator.fit(X, y) + + +@pytest.mark.parametrize("ForestRegressor", FOREST_REGRESSORS.values()) +def test_forest_multioutput_integral_regression_target(ForestRegressor): + """Check that multioutput regression with integral values is not interpreted + as a multiclass-multioutput target and OOB score can be computed. + """ + rng = np.random.RandomState(42) + X = iris.data + y = rng.randint(low=0, high=10, size=(iris.data.shape[0], 2)) + estimator = ForestRegressor( + n_estimators=30, oob_score=True, bootstrap=True, random_state=0 + ) + estimator.fit(X, y) + + n_samples_bootstrap = _get_n_samples_bootstrap(len(X), estimator.max_samples) + n_samples_test = X.shape[0] // 4 + oob_pred = np.zeros([n_samples_test, 2]) + for sample_idx, sample in enumerate(X[:n_samples_test]): + n_samples_oob = 0 + oob_pred_sample = np.zeros(2) + for tree in estimator.estimators_: + oob_unsampled_indices = _generate_unsampled_indices( + tree.random_state, len(X), n_samples_bootstrap + ) + if sample_idx in oob_unsampled_indices: + n_samples_oob += 1 + oob_pred_sample += tree.predict(sample.reshape(1, -1)).squeeze() + oob_pred[sample_idx] = oob_pred_sample / n_samples_oob + assert_allclose(oob_pred, estimator.oob_prediction_[:n_samples_test]) + + +@pytest.mark.parametrize("oob_score", [True, False]) +def test_random_trees_embedding_raise_error_oob(oob_score): + with pytest.raises(TypeError, match="got an unexpected keyword argument"): + RandomTreesEmbedding(oob_score=oob_score) + with pytest.raises(NotImplementedError, match="OOB score not supported"): + RandomTreesEmbedding()._set_oob_score_and_attributes(X, y) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_gridsearch(name): + # Check that base trees can be grid-searched. + forest = FOREST_CLASSIFIERS[name]() + clf = GridSearchCV(forest, {"n_estimators": (1, 2), "max_depth": (1, 2)}) + clf.fit(iris.data, iris.target) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_parallel(name): + """Check parallel computations in classification""" + if name in FOREST_CLASSIFIERS: + X = iris.data + y = iris.target + elif name in FOREST_REGRESSORS: + X = X_reg + y = y_reg + + ForestEstimator = FOREST_ESTIMATORS[name] + forest = ForestEstimator(n_estimators=10, n_jobs=3, random_state=0) + + forest.fit(X, y) + assert len(forest) == 10 + + forest.set_params(n_jobs=1) + y1 = forest.predict(X) + forest.set_params(n_jobs=2) + y2 = forest.predict(X) + assert_array_almost_equal(y1, y2, 3) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_pickle(name): + # Check pickability. + if name in FOREST_CLASSIFIERS: + X = iris.data[::2] + y = iris.target[::2] + elif name in FOREST_REGRESSORS: + X = X_reg[::2] + y = y_reg[::2] + + ForestEstimator = FOREST_ESTIMATORS[name] + obj = ForestEstimator(random_state=0) + obj.fit(X, y) + score = obj.score(X, y) + pickle_object = pickle.dumps(obj) + + obj2 = pickle.loads(pickle_object) + assert type(obj2) == obj.__class__ + score2 = obj2.score(X, y) + assert score == score2 + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_multioutput(name): + # Check estimators on multi-output problems. + + X_train = [ + [-2, -1], + [-1, -1], + [-1, -2], + [1, 1], + [1, 2], + [2, 1], + [-2, 1], + [-1, 1], + [-1, 2], + [2, -1], + [1, -1], + [1, -2], + ] + y_train = [ + [-1, 0], + [-1, 0], + [-1, 0], + [1, 1], + [1, 1], + [1, 1], + [-1, 2], + [-1, 2], + [-1, 2], + [1, 3], + [1, 3], + [1, 3], + ] + X_test = [[-1, -1], [1, 1], [-1, 1], [1, -1]] + y_test = [[-1, 0], [1, 1], [-1, 2], [1, 3]] + + est = FOREST_ESTIMATORS[name](random_state=0, bootstrap=False) + y_pred = est.fit(X_train, y_train).predict(X_test) + assert_array_almost_equal(y_pred, y_test) + + if name in FOREST_CLASSIFIERS: + with np.errstate(divide="ignore"): + proba = est.predict_proba(X_test) + assert len(proba) == 2 + assert proba[0].shape == (4, 2) + assert proba[1].shape == (4, 4) + + log_proba = est.predict_log_proba(X_test) + assert len(log_proba) == 2 + assert log_proba[0].shape == (4, 2) + assert log_proba[1].shape == (4, 4) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_multioutput_string(name): + # Check estimators on multi-output problems with string outputs. + + X_train = [ + [-2, -1], + [-1, -1], + [-1, -2], + [1, 1], + [1, 2], + [2, 1], + [-2, 1], + [-1, 1], + [-1, 2], + [2, -1], + [1, -1], + [1, -2], + ] + y_train = [ + ["red", "blue"], + ["red", "blue"], + ["red", "blue"], + ["green", "green"], + ["green", "green"], + ["green", "green"], + ["red", "purple"], + ["red", "purple"], + ["red", "purple"], + ["green", "yellow"], + ["green", "yellow"], + ["green", "yellow"], + ] + X_test = [[-1, -1], [1, 1], [-1, 1], [1, -1]] + y_test = [ + ["red", "blue"], + ["green", "green"], + ["red", "purple"], + ["green", "yellow"], + ] + + est = FOREST_ESTIMATORS[name](random_state=0, bootstrap=False) + y_pred = est.fit(X_train, y_train).predict(X_test) + assert_array_equal(y_pred, y_test) + + with np.errstate(divide="ignore"): + proba = est.predict_proba(X_test) + assert len(proba) == 2 + assert proba[0].shape == (4, 2) + assert proba[1].shape == (4, 4) + + log_proba = est.predict_log_proba(X_test) + assert len(log_proba) == 2 + assert log_proba[0].shape == (4, 2) + assert log_proba[1].shape == (4, 4) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_classes_shape(name): + # Test that n_classes_ and classes_ have proper shape. + ForestClassifier = FOREST_CLASSIFIERS[name] + + # Classification, single output + clf = ForestClassifier(random_state=0).fit(X, y) + + assert clf.n_classes_ == 2 + assert_array_equal(clf.classes_, [-1, 1]) + + # Classification, multi-output + _y = np.vstack((y, np.array(y) * 2)).T + clf = ForestClassifier(random_state=0).fit(X, _y) + + assert_array_equal(clf.n_classes_, [2, 2]) + assert_array_equal(clf.classes_, [[-1, 1], [-2, 2]]) + + +def test_random_trees_dense_type(): + # Test that the `sparse_output` parameter of RandomTreesEmbedding + # works by returning a dense array. + + # Create the RTE with sparse=False + hasher = RandomTreesEmbedding(n_estimators=10, sparse_output=False) + X, y = datasets.make_circles(factor=0.5) + X_transformed = hasher.fit_transform(X) + + # Assert that type is ndarray, not scipy.sparse.csr_matrix + assert isinstance(X_transformed, np.ndarray) + + +def test_random_trees_dense_equal(): + # Test that the `sparse_output` parameter of RandomTreesEmbedding + # works by returning the same array for both argument values. + + # Create the RTEs + hasher_dense = RandomTreesEmbedding( + n_estimators=10, sparse_output=False, random_state=0 + ) + hasher_sparse = RandomTreesEmbedding( + n_estimators=10, sparse_output=True, random_state=0 + ) + X, y = datasets.make_circles(factor=0.5) + X_transformed_dense = hasher_dense.fit_transform(X) + X_transformed_sparse = hasher_sparse.fit_transform(X) + + # Assert that dense and sparse hashers have same array. + assert_array_equal(X_transformed_sparse.toarray(), X_transformed_dense) + + +def test_random_hasher(): + # test random forest hashing on circles dataset + # make sure that it is linearly separable. + # even after projected to two SVD dimensions + # Note: Not all random_states produce perfect results. + hasher = RandomTreesEmbedding(n_estimators=30, random_state=1) + X, y = datasets.make_circles(factor=0.5) + X_transformed = hasher.fit_transform(X) + + # test fit and transform: + hasher = RandomTreesEmbedding(n_estimators=30, random_state=1) + assert_array_equal(hasher.fit(X).transform(X).toarray(), X_transformed.toarray()) + + # one leaf active per data point per forest + assert X_transformed.shape[0] == X.shape[0] + assert_array_equal(X_transformed.sum(axis=1), hasher.n_estimators) + svd = TruncatedSVD(n_components=2) + X_reduced = svd.fit_transform(X_transformed) + linear_clf = LinearSVC() + linear_clf.fit(X_reduced, y) + assert linear_clf.score(X_reduced, y) == 1.0 + + +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_random_hasher_sparse_data(csc_container): + X, y = datasets.make_multilabel_classification(random_state=0) + hasher = RandomTreesEmbedding(n_estimators=30, random_state=1) + X_transformed = hasher.fit_transform(X) + X_transformed_sparse = hasher.fit_transform(csc_container(X)) + assert_array_equal(X_transformed_sparse.toarray(), X_transformed.toarray()) + + +def test_parallel_train(): + rng = check_random_state(12321) + n_samples, n_features = 80, 30 + X_train = rng.randn(n_samples, n_features) + y_train = rng.randint(0, 2, n_samples) + + clfs = [ + RandomForestClassifier(n_estimators=20, n_jobs=n_jobs, random_state=12345).fit( + X_train, y_train + ) + for n_jobs in [1, 2, 3, 8, 16, 32] + ] + + X_test = rng.randn(n_samples, n_features) + probas = [clf.predict_proba(X_test) for clf in clfs] + for proba1, proba2 in itertools.pairwise(probas): + assert_array_almost_equal(proba1, proba2) + + +def test_distribution(): + rng = check_random_state(12321) + + # Single variable with 4 values + X = rng.randint(0, 4, size=(1000, 1)) + y = rng.rand(1000) + n_trees = 500 + + reg = ExtraTreesRegressor(n_estimators=n_trees, random_state=42).fit(X, y) + + uniques = defaultdict(int) + for tree in reg.estimators_: + tree = "".join( + ("%d,%d/" % (f, int(t)) if f >= 0 else "-") + for f, t in zip(tree.tree_.feature, tree.tree_.threshold) + ) + + uniques[tree] += 1 + + uniques = sorted([(1.0 * count / n_trees, tree) for tree, count in uniques.items()]) + + # On a single variable problem where X_0 has 4 equiprobable values, there + # are 5 ways to build a random tree. The more compact (0,1/0,0/--0,2/--) of + # them has probability 1/3 while the 4 others have probability 1/6. + + assert len(uniques) == 5 + assert 0.20 > uniques[0][0] # Rough approximation of 1/6. + assert 0.20 > uniques[1][0] + assert 0.20 > uniques[2][0] + assert 0.20 > uniques[3][0] + assert uniques[4][0] > 0.3 + assert uniques[4][1] == "0,1/0,0/--0,2/--" + + # Two variables, one with 2 values, one with 3 values + X = np.empty((1000, 2)) + X[:, 0] = np.random.randint(0, 2, 1000) + X[:, 1] = np.random.randint(0, 3, 1000) + y = rng.rand(1000) + + reg = ExtraTreesRegressor(max_features=1, random_state=1).fit(X, y) + + uniques = defaultdict(int) + for tree in reg.estimators_: + tree = "".join( + ("%d,%d/" % (f, int(t)) if f >= 0 else "-") + for f, t in zip(tree.tree_.feature, tree.tree_.threshold) + ) + + uniques[tree] += 1 + + uniques = [(count, tree) for tree, count in uniques.items()] + assert len(uniques) == 8 + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_max_leaf_nodes_max_depth(name): + X, y = hastie_X, hastie_y + + # Test precedence of max_leaf_nodes over max_depth. + ForestEstimator = FOREST_ESTIMATORS[name] + est = ForestEstimator( + max_depth=1, max_leaf_nodes=4, n_estimators=1, random_state=0 + ).fit(X, y) + assert est.estimators_[0].get_depth() == 1 + + est = ForestEstimator(max_depth=1, n_estimators=1, random_state=0).fit(X, y) + assert est.estimators_[0].get_depth() == 1 + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_min_samples_split(name): + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + + est = ForestEstimator(min_samples_split=10, n_estimators=1, random_state=0) + est.fit(X, y) + node_idx = est.estimators_[0].tree_.children_left != -1 + node_samples = est.estimators_[0].tree_.n_node_samples[node_idx] + + assert np.min(node_samples) > len(X) * 0.5 - 1, "Failed with {0}".format(name) + + est = ForestEstimator(min_samples_split=0.5, n_estimators=1, random_state=0) + est.fit(X, y) + node_idx = est.estimators_[0].tree_.children_left != -1 + node_samples = est.estimators_[0].tree_.n_node_samples[node_idx] + + assert np.min(node_samples) > len(X) * 0.5 - 1, "Failed with {0}".format(name) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_min_samples_leaf(name): + X, y = hastie_X, hastie_y + + # Test if leaves contain more than leaf_count training examples + ForestEstimator = FOREST_ESTIMATORS[name] + + est = ForestEstimator(min_samples_leaf=5, n_estimators=1, random_state=0) + est.fit(X, y) + out = est.estimators_[0].tree_.apply(X) + node_counts = np.bincount(out) + # drop inner nodes + leaf_count = node_counts[node_counts != 0] + assert np.min(leaf_count) > 4, "Failed with {0}".format(name) + + est = ForestEstimator(min_samples_leaf=0.25, n_estimators=1, random_state=0) + est.fit(X, y) + out = est.estimators_[0].tree_.apply(X) + node_counts = np.bincount(out) + # drop inner nodes + leaf_count = node_counts[node_counts != 0] + assert np.min(leaf_count) > len(X) * 0.25 - 1, "Failed with {0}".format(name) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_min_weight_fraction_leaf(name): + X, y = hastie_X, hastie_y + + # Test if leaves contain at least min_weight_fraction_leaf of the + # training set + ForestEstimator = FOREST_ESTIMATORS[name] + rng = np.random.RandomState(0) + weights = rng.rand(X.shape[0]) + total_weight = np.sum(weights) + + # test both DepthFirstTreeBuilder and BestFirstTreeBuilder + # by setting max_leaf_nodes + for frac in np.linspace(0, 0.5, 6): + est = ForestEstimator( + min_weight_fraction_leaf=frac, n_estimators=1, random_state=0 + ) + if "RandomForest" in name: + est.bootstrap = False + + est.fit(X, y, sample_weight=weights) + out = est.estimators_[0].tree_.apply(X) + node_weights = np.bincount(out, weights=weights) + # drop inner nodes + leaf_weights = node_weights[node_weights != 0] + assert np.min(leaf_weights) >= total_weight * est.min_weight_fraction_leaf, ( + "Failed with {0} min_weight_fraction_leaf={1}".format( + name, est.min_weight_fraction_leaf + ) + ) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_sparse_input(name, sparse_container): + X, y = datasets.make_multilabel_classification(random_state=0, n_samples=50) + + ForestEstimator = FOREST_ESTIMATORS[name] + + dense = ForestEstimator(random_state=0, max_depth=2).fit(X, y) + sparse = ForestEstimator(random_state=0, max_depth=2).fit(sparse_container(X), y) + + assert_array_almost_equal(sparse.apply(X), dense.apply(X)) + + if name in FOREST_CLASSIFIERS or name in FOREST_REGRESSORS: + assert_array_almost_equal(sparse.predict(X), dense.predict(X)) + assert_array_almost_equal( + sparse.feature_importances_, dense.feature_importances_ + ) + + if name in FOREST_CLASSIFIERS: + assert_array_almost_equal(sparse.predict_proba(X), dense.predict_proba(X)) + assert_array_almost_equal( + sparse.predict_log_proba(X), dense.predict_log_proba(X) + ) + + if name in FOREST_TRANSFORMERS: + assert_array_almost_equal( + sparse.transform(X).toarray(), dense.transform(X).toarray() + ) + assert_array_almost_equal( + sparse.fit_transform(X).toarray(), dense.fit_transform(X).toarray() + ) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +@pytest.mark.parametrize("dtype", (np.float64, np.float32)) +def test_memory_layout(name, dtype): + # Test that it works no matter the memory layout + est = FOREST_ESTIMATORS[name](random_state=0, bootstrap=False) + + # Dense + for container, kwargs in ( + (np.asarray, {}), # Nothing + (np.asarray, {"order": "C"}), # C-order + (np.asarray, {"order": "F"}), # F-order + (np.ascontiguousarray, {}), # Contiguous + ): + X = container(iris.data, dtype=dtype, **kwargs) + y = iris.target + assert_array_almost_equal(est.fit(X, y).predict(X), y) + + # Sparse (if applicable) + if est.estimator.splitter in SPARSE_SPLITTERS: + for sparse_container in COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS: + X = sparse_container(iris.data, dtype=dtype) + y = iris.target + assert_array_almost_equal(est.fit(X, y).predict(X), y) + + # Strided + X = np.asarray(iris.data[::3], dtype=dtype) + y = iris.target[::3] + assert_array_almost_equal(est.fit(X, y).predict(X), y) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_1d_input(name): + X = iris.data[:, 0] + X_2d = iris.data[:, 0].reshape((-1, 1)) + y = iris.target + + with ignore_warnings(): + ForestEstimator = FOREST_ESTIMATORS[name] + with pytest.raises(ValueError): + ForestEstimator(n_estimators=1, random_state=0).fit(X, y) + + est = ForestEstimator(random_state=0) + est.fit(X_2d, y) + + if name in FOREST_CLASSIFIERS or name in FOREST_REGRESSORS: + with pytest.raises(ValueError): + est.predict(X) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_class_weights(name): + # Check class_weights resemble sample_weights behavior. + ForestClassifier = FOREST_CLASSIFIERS[name] + + # Iris is balanced, so no effect expected for using 'balanced' weights + clf1 = ForestClassifier(random_state=0) + clf1.fit(iris.data, iris.target) + clf2 = ForestClassifier(class_weight="balanced", random_state=0) + clf2.fit(iris.data, iris.target) + assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_) + + # Make a multi-output problem with three copies of Iris + iris_multi = np.vstack((iris.target, iris.target, iris.target)).T + # Create user-defined weights that should balance over the outputs + clf3 = ForestClassifier( + class_weight=[ + {0: 2.0, 1: 2.0, 2: 1.0}, + {0: 2.0, 1: 1.0, 2: 2.0}, + {0: 1.0, 1: 2.0, 2: 2.0}, + ], + random_state=0, + ) + clf3.fit(iris.data, iris_multi) + assert_almost_equal(clf2.feature_importances_, clf3.feature_importances_) + # Check against multi-output "balanced" which should also have no effect + clf4 = ForestClassifier(class_weight="balanced", random_state=0) + clf4.fit(iris.data, iris_multi) + assert_almost_equal(clf3.feature_importances_, clf4.feature_importances_) + + # Inflate importance of class 1, check against user-defined weights + sample_weight = np.ones(iris.target.shape) + sample_weight[iris.target == 1] *= 100 + class_weight = {0: 1.0, 1: 100.0, 2: 1.0} + clf1 = ForestClassifier(random_state=0) + clf1.fit(iris.data, iris.target, sample_weight) + clf2 = ForestClassifier(class_weight=class_weight, random_state=0) + clf2.fit(iris.data, iris.target) + assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_) + + # Check that sample_weight and class_weight are multiplicative + clf1 = ForestClassifier(random_state=0) + clf1.fit(iris.data, iris.target, sample_weight**2) + clf2 = ForestClassifier(class_weight=class_weight, random_state=0) + clf2.fit(iris.data, iris.target, sample_weight) + assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_class_weight_balanced_and_bootstrap_multi_output(name): + # Test class_weight works for multi-output""" + ForestClassifier = FOREST_CLASSIFIERS[name] + _y = np.vstack((y, np.array(y) * 2)).T + clf = ForestClassifier(class_weight="balanced", random_state=0) + clf.fit(X, _y) + clf = ForestClassifier( + class_weight=[{-1: 0.5, 1: 1.0}, {-2: 1.0, 2: 1.0}], random_state=0 + ) + clf.fit(X, _y) + # smoke test for balanced subsample + clf = ForestClassifier(class_weight="balanced_subsample", random_state=0) + clf.fit(X, _y) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_class_weight_errors(name): + # Test if class_weight raises errors and warnings when expected. + ForestClassifier = FOREST_CLASSIFIERS[name] + _y = np.vstack((y, np.array(y) * 2)).T + + # Warning warm_start with preset + clf = ForestClassifier(class_weight="balanced", warm_start=True, random_state=0) + clf.fit(X, y) + + warn_msg = ( + "Warm-start fitting without increasing n_estimators does not fit new trees." + ) + with pytest.warns(UserWarning, match=warn_msg): + clf.fit(X, _y) + + # Incorrect length list for multi-output + clf = ForestClassifier(class_weight=[{-1: 0.5, 1: 1.0}], random_state=0) + with pytest.raises(ValueError): + clf.fit(X, _y) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start(name): + # Test if fitting incrementally with warm start gives a forest of the + # right size and the same results as a normal fit. + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + est_ws = None + for n_estimators in [5, 10]: + if est_ws is None: + est_ws = ForestEstimator( + n_estimators=n_estimators, random_state=42, warm_start=True + ) + else: + est_ws.set_params(n_estimators=n_estimators) + est_ws.fit(X, y) + assert len(est_ws) == n_estimators + + est_no_ws = ForestEstimator(n_estimators=10, random_state=42, warm_start=False) + est_no_ws.fit(X, y) + + assert set([tree.random_state for tree in est_ws]) == set( + [tree.random_state for tree in est_no_ws] + ) + + assert_array_equal( + est_ws.apply(X), est_no_ws.apply(X), err_msg="Failed with {0}".format(name) + ) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start_clear(name): + # Test if fit clears state and grows a new forest when warm_start==False. + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False, random_state=1) + est.fit(X, y) + + est_2 = ForestEstimator( + n_estimators=5, max_depth=1, warm_start=True, random_state=2 + ) + est_2.fit(X, y) # inits state + est_2.set_params(warm_start=False, random_state=1) + est_2.fit(X, y) # clears old state and equals est + + assert_array_almost_equal(est_2.apply(X), est.apply(X)) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start_smaller_n_estimators(name): + # Test if warm start second fit with smaller n_estimators raises error. + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=True) + est.fit(X, y) + est.set_params(n_estimators=4) + with pytest.raises(ValueError): + est.fit(X, y) + + +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start_equal_n_estimators(name): + # Test if warm start with equal n_estimators does nothing and returns the + # same forest and raises a warning. + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + est = ForestEstimator(n_estimators=5, max_depth=3, warm_start=True, random_state=1) + est.fit(X, y) + + est_2 = ForestEstimator( + n_estimators=5, max_depth=3, warm_start=True, random_state=1 + ) + est_2.fit(X, y) + # Now est_2 equals est. + + est_2.set_params(random_state=2) + warn_msg = ( + "Warm-start fitting without increasing n_estimators does not fit new trees." + ) + with pytest.warns(UserWarning, match=warn_msg): + est_2.fit(X, y) + # If we had fit the trees again we would have got a different forest as we + # changed the random state. + assert_array_equal(est.apply(X), est_2.apply(X)) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_warm_start_oob(name): + # Test that the warm start computes oob score when asked. + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + # Use 15 estimators to avoid 'some inputs do not have OOB scores' warning. + est = ForestEstimator( + n_estimators=15, + max_depth=3, + warm_start=False, + random_state=1, + bootstrap=True, + oob_score=True, + ) + est.fit(X, y) + + est_2 = ForestEstimator( + n_estimators=5, + max_depth=3, + warm_start=False, + random_state=1, + bootstrap=True, + oob_score=False, + ) + est_2.fit(X, y) + + est_2.set_params(warm_start=True, oob_score=True, n_estimators=15) + est_2.fit(X, y) + + assert hasattr(est_2, "oob_score_") + assert est.oob_score_ == est_2.oob_score_ + + # Test that oob_score is computed even if we don't need to train + # additional trees. + est_3 = ForestEstimator( + n_estimators=15, + max_depth=3, + warm_start=True, + random_state=1, + bootstrap=True, + oob_score=False, + ) + est_3.fit(X, y) + assert not hasattr(est_3, "oob_score_") + + est_3.set_params(oob_score=True) + ignore_warnings(est_3.fit)(X, y) + + assert est.oob_score_ == est_3.oob_score_ + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_oob_not_computed_twice(name): + # Check that oob_score is not computed twice when warm_start=True. + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + + est = ForestEstimator( + n_estimators=10, warm_start=True, bootstrap=True, oob_score=True + ) + + with patch.object( + est, "_set_oob_score_and_attributes", wraps=est._set_oob_score_and_attributes + ) as mock_set_oob_score_and_attributes: + est.fit(X, y) + + with pytest.warns(UserWarning, match="Warm-start fitting without increasing"): + est.fit(X, y) + + mock_set_oob_score_and_attributes.assert_called_once() + + +def test_dtype_convert(n_classes=15): + classifier = RandomForestClassifier(random_state=0, bootstrap=False) + + X = np.eye(n_classes) + y = [ch for ch in "ABCDEFGHIJKLMNOPQRSTU"[:n_classes]] + + result = classifier.fit(X, y).predict(X) + assert_array_equal(classifier.classes_, y) + assert_array_equal(result, y) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_decision_path(name): + X, y = hastie_X, hastie_y + n_samples = X.shape[0] + ForestEstimator = FOREST_ESTIMATORS[name] + est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False, random_state=1) + est.fit(X, y) + indicator, n_nodes_ptr = est.decision_path(X) + + assert indicator.shape[1] == n_nodes_ptr[-1] + assert indicator.shape[0] == n_samples + assert_array_equal( + np.diff(n_nodes_ptr), [e.tree_.node_count for e in est.estimators_] + ) + + # Assert that leaves index are correct + leaves = est.apply(X) + for est_id in range(leaves.shape[1]): + leave_indicator = [ + indicator[i, n_nodes_ptr[est_id] + j] + for i, j in enumerate(leaves[:, est_id]) + ] + assert_array_almost_equal(leave_indicator, np.ones(shape=n_samples)) + + +def test_min_impurity_decrease(): + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + all_estimators = [ + RandomForestClassifier, + RandomForestRegressor, + ExtraTreesClassifier, + ExtraTreesRegressor, + ] + + for Estimator in all_estimators: + est = Estimator(min_impurity_decrease=0.1) + est.fit(X, y) + for tree in est.estimators_: + # Simply check if the parameter is passed on correctly. Tree tests + # will suffice for the actual working of this param + assert tree.min_impurity_decrease == 0.1 + + +def test_poisson_y_positive_check(): + est = RandomForestRegressor(criterion="poisson") + X = np.zeros((3, 3)) + + y = [-1, 1, 3] + err_msg = ( + r"Some value\(s\) of y are negative which is " + r"not allowed for Poisson regression." + ) + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) + + y = [0, 0, 0] + err_msg = ( + r"Sum of y is not strictly positive which " + r"is necessary for Poisson regression." + ) + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) + + +# mypy error: Variable "DEFAULT_JOBLIB_BACKEND" is not valid type +class MyBackend(DEFAULT_JOBLIB_BACKEND): # type: ignore[valid-type,misc] + def __init__(self, *args, **kwargs): + self.count = 0 + super().__init__(*args, **kwargs) + + def start_call(self): + self.count += 1 + return super().start_call() + + +joblib.register_parallel_backend("testing", MyBackend) + + +# TODO: remove mark once loky bug is fixed: +# https://github.com/joblib/loky/issues/458 +@pytest.mark.thread_unsafe +@skip_if_no_parallel +def test_backend_respected(): + clf = RandomForestClassifier(n_estimators=10, n_jobs=2) + + with joblib.parallel_backend("testing") as (ba, n_jobs): + clf.fit(X, y) + + assert ba.count > 0 + + # predict_proba requires shared memory. Ensure that's honored. + with joblib.parallel_backend("testing") as (ba, _): + clf.predict_proba(X) + + assert ba.count == 0 + + +def test_forest_feature_importances_sum(): + X, y = make_classification( + n_samples=15, n_informative=3, random_state=1, n_classes=3 + ) + clf = RandomForestClassifier( + min_samples_leaf=5, random_state=42, n_estimators=200 + ).fit(X, y) + assert math.isclose(1, clf.feature_importances_.sum(), abs_tol=1e-7) + + +def test_forest_degenerate_feature_importances(): + # build a forest of single node trees. See #13636 + X = np.zeros((10, 10)) + y = np.ones((10,)) + gbr = RandomForestRegressor(n_estimators=10).fit(X, y) + assert_array_equal(gbr.feature_importances_, np.zeros(10, dtype=np.float64)) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_max_samples_bootstrap(name): + # Check invalid `max_samples` values + est = FOREST_CLASSIFIERS_REGRESSORS[name](bootstrap=False, max_samples=0.5) + err_msg = ( + r"`max_sample` cannot be set if `bootstrap=False`. " + r"Either switch to `bootstrap=True` or set " + r"`max_sample=None`." + ) + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_large_max_samples_exception(name): + # Check invalid `max_samples` + est = FOREST_CLASSIFIERS_REGRESSORS[name](bootstrap=True, max_samples=int(1e9)) + match = "`max_samples` must be <= n_samples=6 but got value 1000000000" + with pytest.raises(ValueError, match=match): + est.fit(X, y) + + +@pytest.mark.parametrize("name", FOREST_REGRESSORS) +def test_max_samples_boundary_regressors(name): + X_train, X_test, y_train, y_test = train_test_split( + X_reg, y_reg, train_size=0.7, test_size=0.3, random_state=0 + ) + + ms_1_model = FOREST_REGRESSORS[name]( + bootstrap=True, max_samples=1.0, random_state=0 + ) + ms_1_predict = ms_1_model.fit(X_train, y_train).predict(X_test) + + ms_None_model = FOREST_REGRESSORS[name]( + bootstrap=True, max_samples=None, random_state=0 + ) + ms_None_predict = ms_None_model.fit(X_train, y_train).predict(X_test) + + ms_1_ms = mean_squared_error(ms_1_predict, y_test) + ms_None_ms = mean_squared_error(ms_None_predict, y_test) + + assert ms_1_ms == pytest.approx(ms_None_ms) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_max_samples_boundary_classifiers(name): + X_train, X_test, y_train, _ = train_test_split( + X_large, y_large, random_state=0, stratify=y_large + ) + + ms_1_model = FOREST_CLASSIFIERS[name]( + bootstrap=True, max_samples=1.0, random_state=0 + ) + ms_1_proba = ms_1_model.fit(X_train, y_train).predict_proba(X_test) + + ms_None_model = FOREST_CLASSIFIERS[name]( + bootstrap=True, max_samples=None, random_state=0 + ) + ms_None_proba = ms_None_model.fit(X_train, y_train).predict_proba(X_test) + + np.testing.assert_allclose(ms_1_proba, ms_None_proba) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_forest_y_sparse(csr_container): + X = [[1, 2, 3]] + y = csr_container([[4, 5, 6]]) + est = RandomForestClassifier() + msg = "sparse multilabel-indicator for y is not supported." + with pytest.raises(ValueError, match=msg): + est.fit(X, y) + + +@pytest.mark.parametrize("ForestClass", [RandomForestClassifier, RandomForestRegressor]) +def test_little_tree_with_small_max_samples(ForestClass): + rng = np.random.RandomState(1) + + X = rng.randn(10000, 2) + y = rng.randn(10000) > 0 + + # First fit with no restriction on max samples + est1 = ForestClass( + n_estimators=1, + random_state=rng, + max_samples=None, + ) + + # Second fit with max samples restricted to just 2 + est2 = ForestClass( + n_estimators=1, + random_state=rng, + max_samples=2, + ) + + est1.fit(X, y) + est2.fit(X, y) + + tree1 = est1.estimators_[0].tree_ + tree2 = est2.estimators_[0].tree_ + + msg = "Tree without `max_samples` restriction should have more nodes" + assert tree1.node_count > tree2.node_count, msg + + +@pytest.mark.parametrize("Forest", FOREST_REGRESSORS) +def test_mse_criterion_object_segfault_smoke_test(Forest): + # This is a smoke test to ensure that passing a mutable criterion + # does not cause a segfault when fitting with concurrent threads. + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/12623 + from sklearn.tree._criterion import MSE + + y = y_reg.reshape(-1, 1) + n_samples, n_outputs = y.shape + mse_criterion = MSE(n_outputs, n_samples) + est = FOREST_REGRESSORS[Forest](n_estimators=2, n_jobs=2, criterion=mse_criterion) + + est.fit(X_reg, y) + + +def test_random_trees_embedding_feature_names_out(): + """Check feature names out for Random Trees Embedding.""" + random_state = np.random.RandomState(0) + X = np.abs(random_state.randn(100, 4)) + hasher = RandomTreesEmbedding( + n_estimators=2, max_depth=2, sparse_output=False, random_state=0 + ).fit(X) + names = hasher.get_feature_names_out() + expected_names = [ + f"randomtreesembedding_{tree}_{leaf}" + # Note: nodes with indices 0, 1 and 4 are internal split nodes and + # therefore do not appear in the expected output feature names. + for tree, leaf in [ + (0, 2), + (0, 3), + (0, 5), + (0, 6), + (1, 2), + (1, 3), + (1, 5), + (1, 6), + ] + ] + assert_array_equal(expected_names, names) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_read_only_buffer(csr_container, monkeypatch): + """RandomForestClassifier must work on readonly sparse data. + + Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/25333 + """ + monkeypatch.setattr( + sklearn.ensemble._forest, + "Parallel", + partial(Parallel, max_nbytes=100), + ) + rng = np.random.RandomState(seed=0) + + X, y = make_classification(n_samples=100, n_features=200, random_state=rng) + X = csr_container(X, copy=True) + + clf = RandomForestClassifier(n_jobs=2, random_state=rng) + cross_val_score(clf, X, y, cv=2) + + +@pytest.mark.parametrize("class_weight", ["balanced_subsample", None]) +def test_round_samples_to_one_when_samples_too_low(class_weight): + """Check low max_samples works and is rounded to one. + + Non-regression test for gh-24037. + """ + X, y = datasets.load_wine(return_X_y=True) + forest = RandomForestClassifier( + n_estimators=10, max_samples=1e-4, class_weight=class_weight, random_state=0 + ) + forest.fit(X, y) + + +@pytest.mark.parametrize("seed", [None, 1]) +@pytest.mark.parametrize("bootstrap", [True, False]) +@pytest.mark.parametrize("ForestClass", FOREST_CLASSIFIERS_REGRESSORS.values()) +def test_estimators_samples(ForestClass, bootstrap, seed): + """Estimators_samples_ property should be consistent. + + Tests consistency across fits and whether or not the seed for the random generator + is set. + """ + X, y = make_hastie_10_2(n_samples=200, random_state=1) + + if bootstrap: + max_samples = 0.5 + else: + max_samples = None + est = ForestClass( + n_estimators=10, + max_samples=max_samples, + max_features=0.5, + random_state=seed, + bootstrap=bootstrap, + ) + est.fit(X, y) + + estimators_samples = est.estimators_samples_.copy() + + # Test repeated calls result in same set of indices + assert_array_equal(estimators_samples, est.estimators_samples_) + estimators = est.estimators_ + + assert isinstance(estimators_samples, list) + assert len(estimators_samples) == len(estimators) + assert estimators_samples[0].dtype == np.int32 + + for i in range(len(estimators)): + if bootstrap: + assert len(estimators_samples[i]) == len(X) // 2 + + # the bootstrap should be a resampling with replacement + assert len(np.unique(estimators_samples[i])) < len(estimators_samples[i]) + else: + assert len(set(estimators_samples[i])) == len(X) + + estimator_index = 0 + estimator_samples = estimators_samples[estimator_index] + estimator = estimators[estimator_index] + + X_train = X[estimator_samples] + y_train = y[estimator_samples] + + orig_tree_values = estimator.tree_.value + estimator = clone(estimator) + estimator.fit(X_train, y_train) + new_tree_values = estimator.tree_.value + assert_allclose(orig_tree_values, new_tree_values) + + +@pytest.mark.parametrize( + "make_data, Forest", + [ + (datasets.make_regression, RandomForestRegressor), + (datasets.make_classification, RandomForestClassifier), + (datasets.make_regression, ExtraTreesRegressor), + (datasets.make_classification, ExtraTreesClassifier), + ], +) +def test_missing_values_is_resilient(make_data, Forest): + """Check that forest can deal with missing values and has decent performance.""" + + rng = np.random.RandomState(0) + n_samples, n_features = 1000, 10 + X, y = make_data(n_samples=n_samples, n_features=n_features, random_state=rng) + + # Create dataset with missing values + X_missing = X.copy() + X_missing[rng.choice([False, True], size=X.shape, p=[0.95, 0.05])] = np.nan + assert np.isnan(X_missing).any() + + X_missing_train, X_missing_test, y_train, y_test = train_test_split( + X_missing, y, random_state=0 + ) + + # Train forest with missing values + forest_with_missing = Forest(random_state=rng, n_estimators=50) + forest_with_missing.fit(X_missing_train, y_train) + score_with_missing = forest_with_missing.score(X_missing_test, y_test) + + # Train forest without missing values + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + forest = Forest(random_state=rng, n_estimators=50) + forest.fit(X_train, y_train) + score_without_missing = forest.score(X_test, y_test) + + # Score is still 80 percent of the forest's score that had no missing values + assert score_with_missing >= 0.80 * score_without_missing + + +@pytest.mark.parametrize( + "Forest", + [ + RandomForestClassifier, + RandomForestRegressor, + ExtraTreesRegressor, + ExtraTreesClassifier, + ], +) +def test_missing_value_is_predictive(Forest): + """Check that the forest learns when missing values are only present for + a predictive feature.""" + rng = np.random.RandomState(0) + n_samples = 300 + expected_score = 0.75 + + X_non_predictive = rng.standard_normal(size=(n_samples, 10)) + y = rng.randint(0, high=2, size=n_samples) + + # Create a predictive feature using `y` and with some noise + X_random_mask = rng.choice([False, True], size=n_samples, p=[0.95, 0.05]) + y_mask = y.astype(bool) + y_mask[X_random_mask] = ~y_mask[X_random_mask] + + predictive_feature = rng.standard_normal(size=n_samples) + predictive_feature[y_mask] = np.nan + assert np.isnan(predictive_feature).any() + + X_predictive = X_non_predictive.copy() + X_predictive[:, 5] = predictive_feature + + ( + X_predictive_train, + X_predictive_test, + X_non_predictive_train, + X_non_predictive_test, + y_train, + y_test, + ) = train_test_split(X_predictive, X_non_predictive, y, random_state=0) + forest_predictive = Forest(random_state=0).fit(X_predictive_train, y_train) + forest_non_predictive = Forest(random_state=0).fit(X_non_predictive_train, y_train) + + predictive_test_score = forest_predictive.score(X_predictive_test, y_test) + + assert predictive_test_score >= expected_score + assert predictive_test_score >= forest_non_predictive.score( + X_non_predictive_test, y_test + ) + + +@pytest.mark.parametrize("Forest", FOREST_REGRESSORS.values()) +def test_non_supported_criterion_raises_error_with_missing_values(Forest): + """Raise error for unsupported criterion when there are missing values.""" + X = np.array([[0, 1, 2], [np.nan, 0, 2.0]]) + y = [0.5, 1.0] + + forest = Forest(criterion="absolute_error") + + msg = ".*does not accept missing values" + with pytest.raises(ValueError, match=msg): + forest.fit(X, y) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_gradient_boosting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_gradient_boosting.py new file mode 100644 index 0000000000000000000000000000000000000000..4880ccb9e44cb48a25d8a013eac7262c18da3b27 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_gradient_boosting.py @@ -0,0 +1,1717 @@ +""" +Testing for the gradient boosting module (sklearn.ensemble.gradient_boosting). +""" + +import re +import warnings + +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from sklearn import datasets +from sklearn.base import clone +from sklearn.datasets import make_classification, make_regression +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor +from sklearn.ensemble._gb import _safe_divide +from sklearn.ensemble._gradient_boosting import predict_stages +from sklearn.exceptions import DataConversionWarning, NotFittedError +from sklearn.linear_model import LinearRegression +from sklearn.metrics import mean_squared_error +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import scale +from sklearn.svm import NuSVR +from sklearn.utils import check_random_state +from sklearn.utils._mocking import NoSampleWeightWrapper +from sklearn.utils._param_validation import InvalidParameterError +from sklearn.utils._testing import ( + assert_array_almost_equal, + assert_array_equal, + skip_if_32bit, +) +from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS + +GRADIENT_BOOSTING_ESTIMATORS = [GradientBoostingClassifier, GradientBoostingRegressor] + +# toy sample +X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] +y = [-1, -1, -1, 1, 1, 1] +T = [[-1, -1], [2, 2], [3, 2]] +true_result = [-1, 1, 1] + +# also make regression dataset +X_reg, y_reg = make_regression( + n_samples=100, n_features=4, n_informative=8, noise=10, random_state=7 +) +y_reg = scale(y_reg) + +rng = np.random.RandomState(0) +# also load the iris dataset +# and randomly permute it +iris = datasets.load_iris() +perm = rng.permutation(iris.target.size) +iris.data = iris.data[perm] +iris.target = iris.target[perm] + + +def test_exponential_n_classes_gt_2(): + """Test exponential loss raises for n_classes > 2.""" + clf = GradientBoostingClassifier(loss="exponential") + msg = "loss='exponential' is only suitable for a binary classification" + with pytest.raises(ValueError, match=msg): + clf.fit(iris.data, iris.target) + + +def test_raise_if_init_has_no_predict_proba(): + """Test raise if init_ has no predict_proba method.""" + clf = GradientBoostingClassifier(init=GradientBoostingRegressor) + msg = ( + "The 'init' parameter of GradientBoostingClassifier must be a str among " + "{'zero'}, None or an object implementing 'fit' and 'predict_proba'." + ) + with pytest.raises(ValueError, match=msg): + clf.fit(X, y) + + +@pytest.mark.parametrize("loss", ("log_loss", "exponential")) +def test_classification_toy(loss, global_random_seed): + # Check classification on a toy dataset. + clf = GradientBoostingClassifier( + loss=loss, n_estimators=10, random_state=global_random_seed + ) + + with pytest.raises(ValueError): + clf.predict(T) + + clf.fit(X, y) + assert_array_equal(clf.predict(T), true_result) + assert 10 == len(clf.estimators_) + + log_loss_decrease = clf.train_score_[:-1] - clf.train_score_[1:] + assert np.any(log_loss_decrease >= 0.0) + + leaves = clf.apply(X) + assert leaves.shape == (6, 10, 1) + + +@pytest.mark.parametrize("loss", ("log_loss", "exponential")) +def test_classification_synthetic(loss, global_random_seed): + # Test GradientBoostingClassifier on synthetic dataset used by + # Hastie et al. in ESLII - Figure 10.9 + # Note that Figure 10.9 reuses the dataset generated for figure 10.2 + # and should have 2_000 train data points and 10_000 test data points. + # Here we intentionally use a smaller variant to make the test run faster, + # but the conclusions are still the same, despite the smaller datasets. + X, y = datasets.make_hastie_10_2(n_samples=2000, random_state=global_random_seed) + + split_idx = 500 + X_train, X_test = X[:split_idx], X[split_idx:] + y_train, y_test = y[:split_idx], y[split_idx:] + + # Increasing the number of trees should decrease the test error + common_params = { + "max_depth": 1, + "learning_rate": 1.0, + "loss": loss, + "random_state": global_random_seed, + } + gbrt_10_stumps = GradientBoostingClassifier(n_estimators=10, **common_params) + gbrt_10_stumps.fit(X_train, y_train) + + gbrt_50_stumps = GradientBoostingClassifier(n_estimators=50, **common_params) + gbrt_50_stumps.fit(X_train, y_train) + + assert gbrt_10_stumps.score(X_test, y_test) < gbrt_50_stumps.score(X_test, y_test) + + # Decision stumps are better suited for this dataset with a large number of + # estimators. + common_params = { + "n_estimators": 200, + "learning_rate": 1.0, + "loss": loss, + "random_state": global_random_seed, + } + gbrt_stumps = GradientBoostingClassifier(max_depth=1, **common_params) + gbrt_stumps.fit(X_train, y_train) + + gbrt_10_nodes = GradientBoostingClassifier(max_leaf_nodes=10, **common_params) + gbrt_10_nodes.fit(X_train, y_train) + + assert gbrt_stumps.score(X_test, y_test) > gbrt_10_nodes.score(X_test, y_test) + + +@pytest.mark.parametrize("loss", ("squared_error", "absolute_error", "huber")) +@pytest.mark.parametrize("subsample", (1.0, 0.5)) +def test_regression_dataset(loss, subsample, global_random_seed): + # Check consistency on regression dataset with least squares + # and least absolute deviation. + ones = np.ones(len(y_reg)) + last_y_pred = None + for sample_weight in [None, ones, 2 * ones]: + # learning_rate, max_depth and n_estimators were adjusted to get a mode + # that is accurate enough to reach a low MSE on the training set while + # keeping the resource used to execute this test low enough. + reg = GradientBoostingRegressor( + n_estimators=30, + loss=loss, + max_depth=4, + subsample=subsample, + min_samples_split=2, + random_state=global_random_seed, + learning_rate=0.5, + ) + + reg.fit(X_reg, y_reg, sample_weight=sample_weight) + leaves = reg.apply(X_reg) + assert leaves.shape == (100, 30) + + y_pred = reg.predict(X_reg) + mse = mean_squared_error(y_reg, y_pred) + assert mse < 0.05 + + if last_y_pred is not None: + # FIXME: We temporarily bypass this test. This is due to the fact + # that GBRT with and without `sample_weight` do not use the same + # implementation of the median during the initialization with the + # `DummyRegressor`. In the future, we should make sure that both + # implementations should be the same. See PR #17377 for more. + # assert_allclose(last_y_pred, y_pred) + pass + + last_y_pred = y_pred + + +@pytest.mark.parametrize("subsample", (1.0, 0.5)) +@pytest.mark.parametrize("sample_weight", (None, 1)) +def test_iris(subsample, sample_weight, global_random_seed): + if sample_weight == 1: + sample_weight = np.ones(len(iris.target)) + # Check consistency on dataset iris. + clf = GradientBoostingClassifier( + n_estimators=100, + loss="log_loss", + random_state=global_random_seed, + subsample=subsample, + ) + clf.fit(iris.data, iris.target, sample_weight=sample_weight) + score = clf.score(iris.data, iris.target) + assert score > 0.9 + + leaves = clf.apply(iris.data) + assert leaves.shape == (150, 100, 3) + + +def test_regression_synthetic(global_random_seed): + # Test on synthetic regression datasets used in Leo Breiman, + # `Bagging Predictors?. Machine Learning 24(2): 123-140 (1996). + random_state = check_random_state(global_random_seed) + regression_params = { + "n_estimators": 100, + "max_depth": 4, + "min_samples_split": 2, + "learning_rate": 0.1, + "loss": "squared_error", + "random_state": global_random_seed, + } + + # Friedman1 + X, y = datasets.make_friedman1(n_samples=1200, random_state=random_state, noise=1.0) + X_train, y_train = X[:200], y[:200] + X_test, y_test = X[200:], y[200:] + + clf = GradientBoostingRegressor(**regression_params) + clf.fit(X_train, y_train) + mse = mean_squared_error(y_test, clf.predict(X_test)) + assert mse < 6.5 + + # Friedman2 + X, y = datasets.make_friedman2(n_samples=1200, random_state=random_state) + X_train, y_train = X[:200], y[:200] + X_test, y_test = X[200:], y[200:] + + clf = GradientBoostingRegressor(**regression_params) + clf.fit(X_train, y_train) + mse = mean_squared_error(y_test, clf.predict(X_test)) + assert mse < 2500.0 + + # Friedman3 + X, y = datasets.make_friedman3(n_samples=1200, random_state=random_state) + X_train, y_train = X[:200], y[:200] + X_test, y_test = X[200:], y[200:] + + clf = GradientBoostingRegressor(**regression_params) + clf.fit(X_train, y_train) + mse = mean_squared_error(y_test, clf.predict(X_test)) + assert mse < 0.025 + + +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (GradientBoostingRegressor, X_reg, y_reg), + (GradientBoostingClassifier, iris.data, iris.target), + ], +) +def test_feature_importances(GradientBoosting, X, y): + # smoke test to check that the gradient boosting expose an attribute + # feature_importances_ + gbdt = GradientBoosting() + assert not hasattr(gbdt, "feature_importances_") + gbdt.fit(X, y) + assert hasattr(gbdt, "feature_importances_") + + +def test_probability_log(global_random_seed): + # Predict probabilities. + clf = GradientBoostingClassifier(n_estimators=100, random_state=global_random_seed) + + with pytest.raises(ValueError): + clf.predict_proba(T) + + clf.fit(X, y) + assert_array_equal(clf.predict(T), true_result) + + # check if probabilities are in [0, 1]. + y_proba = clf.predict_proba(T) + assert np.all(y_proba >= 0.0) + assert np.all(y_proba <= 1.0) + + # derive predictions from probabilities + y_pred = clf.classes_.take(y_proba.argmax(axis=1), axis=0) + assert_array_equal(y_pred, true_result) + + +def test_single_class_with_sample_weight(): + sample_weight = [0, 0, 0, 1, 1, 1] + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + msg = ( + "y contains 1 class after sample_weight trimmed classes with " + "zero weights, while a minimum of 2 classes are required." + ) + with pytest.raises(ValueError, match=msg): + clf.fit(X, y, sample_weight=sample_weight) + + +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_check_inputs_predict_stages(csc_container): + # check that predict_stages through an error if the type of X is not + # supported + x, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + x_sparse_csc = csc_container(x) + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + clf.fit(x, y) + score = np.zeros((y.shape)).reshape(-1, 1) + err_msg = "When X is a sparse matrix, a CSR format is expected" + with pytest.raises(ValueError, match=err_msg): + predict_stages(clf.estimators_, x_sparse_csc, clf.learning_rate, score) + x_fortran = np.asfortranarray(x) + with pytest.raises(ValueError, match="X should be C-ordered np.ndarray"): + predict_stages(clf.estimators_, x_fortran, clf.learning_rate, score) + + +def test_max_feature_regression(global_random_seed): + # Test to make sure random state is set properly. + X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=global_random_seed) + + X_train, X_test = X[:2000], X[2000:] + y_train, y_test = y[:2000], y[2000:] + + gbrt = GradientBoostingClassifier( + n_estimators=100, + min_samples_split=5, + max_depth=2, + learning_rate=0.1, + max_features=2, + random_state=global_random_seed, + ) + gbrt.fit(X_train, y_train) + log_loss = gbrt._loss(y_test, gbrt.decision_function(X_test)) + assert log_loss < 0.5, "GB failed with deviance %.4f" % log_loss + + +def test_feature_importance_regression( + fetch_california_housing_fxt, global_random_seed +): + """Test that Gini importance is calculated correctly. + + This test follows the example from [1]_ (pg. 373). + + .. [1] Friedman, J., Hastie, T., & Tibshirani, R. (2001). The elements + of statistical learning. New York: Springer series in statistics. + """ + california = fetch_california_housing_fxt() + X, y = california.data, california.target + X_train, X_test, y_train, y_test = train_test_split( + X, y, random_state=global_random_seed + ) + + reg = GradientBoostingRegressor( + loss="huber", + learning_rate=0.1, + max_leaf_nodes=6, + n_estimators=100, + random_state=global_random_seed, + ) + reg.fit(X_train, y_train) + sorted_idx = np.argsort(reg.feature_importances_)[::-1] + sorted_features = [california.feature_names[s] for s in sorted_idx] + + # The most important feature is the median income by far. + assert sorted_features[0] == "MedInc" + + # The three subsequent features are the following. Their relative ordering + # might change a bit depending on the randomness of the trees and the + # train / test split. + assert set(sorted_features[1:4]) == {"Longitude", "AveOccup", "Latitude"} + + +def test_max_features(): + # Test if max features is set properly for floats and str. + X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1) + _, n_features = X.shape + + X_train = X[:2000] + y_train = y[:2000] + + gbrt = GradientBoostingClassifier(n_estimators=1, max_features=None) + gbrt.fit(X_train, y_train) + assert gbrt.max_features_ == n_features + + gbrt = GradientBoostingRegressor(n_estimators=1, max_features=None) + gbrt.fit(X_train, y_train) + assert gbrt.max_features_ == n_features + + gbrt = GradientBoostingRegressor(n_estimators=1, max_features=0.3) + gbrt.fit(X_train, y_train) + assert gbrt.max_features_ == int(n_features * 0.3) + + gbrt = GradientBoostingRegressor(n_estimators=1, max_features="sqrt") + gbrt.fit(X_train, y_train) + assert gbrt.max_features_ == int(np.sqrt(n_features)) + + gbrt = GradientBoostingRegressor(n_estimators=1, max_features="log2") + gbrt.fit(X_train, y_train) + assert gbrt.max_features_ == int(np.log2(n_features)) + + gbrt = GradientBoostingRegressor(n_estimators=1, max_features=0.01 / X.shape[1]) + gbrt.fit(X_train, y_train) + assert gbrt.max_features_ == 1 + + +def test_staged_predict(): + # Test whether staged decision function eventually gives + # the same prediction. + X, y = datasets.make_friedman1(n_samples=1200, random_state=1, noise=1.0) + X_train, y_train = X[:200], y[:200] + X_test = X[200:] + clf = GradientBoostingRegressor() + # test raise ValueError if not fitted + with pytest.raises(ValueError): + np.fromiter(clf.staged_predict(X_test), dtype=np.float64) + + clf.fit(X_train, y_train) + y_pred = clf.predict(X_test) + + # test if prediction for last stage equals ``predict`` + for y in clf.staged_predict(X_test): + assert y.shape == y_pred.shape + + assert_array_almost_equal(y_pred, y) + + +def test_staged_predict_proba(): + # Test whether staged predict proba eventually gives + # the same prediction. + X, y = datasets.make_hastie_10_2(n_samples=1200, random_state=1) + X_train, y_train = X[:200], y[:200] + X_test, y_test = X[200:], y[200:] + clf = GradientBoostingClassifier(n_estimators=20) + # test raise NotFittedError if not + with pytest.raises(NotFittedError): + np.fromiter(clf.staged_predict_proba(X_test), dtype=np.float64) + + clf.fit(X_train, y_train) + + # test if prediction for last stage equals ``predict`` + for y_pred in clf.staged_predict(X_test): + assert y_test.shape == y_pred.shape + + assert_array_equal(clf.predict(X_test), y_pred) + + # test if prediction for last stage equals ``predict_proba`` + for staged_proba in clf.staged_predict_proba(X_test): + assert y_test.shape[0] == staged_proba.shape[0] + assert 2 == staged_proba.shape[1] + + assert_array_almost_equal(clf.predict_proba(X_test), staged_proba) + + +@pytest.mark.parametrize("Estimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_staged_functions_defensive(Estimator, global_random_seed): + # test that staged_functions make defensive copies + rng = np.random.RandomState(global_random_seed) + X = rng.uniform(size=(10, 3)) + y = (4 * X[:, 0]).astype(int) + 1 # don't predict zeros + estimator = Estimator() + estimator.fit(X, y) + for func in ["predict", "decision_function", "predict_proba"]: + staged_func = getattr(estimator, "staged_" + func, None) + if staged_func is None: + # regressor has no staged_predict_proba + continue + with warnings.catch_warnings(record=True): + staged_result = list(staged_func(X)) + staged_result[1][:] = 0 + assert np.all(staged_result[0] != 0) + + +def test_serialization(): + # Check model serialization. + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + + clf.fit(X, y) + assert_array_equal(clf.predict(T), true_result) + assert 100 == len(clf.estimators_) + + try: + import cPickle as pickle + except ImportError: + import pickle + + serialized_clf = pickle.dumps(clf, protocol=pickle.HIGHEST_PROTOCOL) + clf = None + clf = pickle.loads(serialized_clf) + assert_array_equal(clf.predict(T), true_result) + assert 100 == len(clf.estimators_) + + +def test_degenerate_targets(): + # Check if we can fit even though all targets are equal. + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + + # classifier should raise exception + with pytest.raises(ValueError): + clf.fit(X, np.ones(len(X))) + + clf = GradientBoostingRegressor(n_estimators=100, random_state=1) + clf.fit(X, np.ones(len(X))) + clf.predict([rng.rand(2)]) + assert_array_equal(np.ones((1,), dtype=np.float64), clf.predict([rng.rand(2)])) + + +def test_quantile_loss(global_random_seed): + # Check if quantile loss with alpha=0.5 equals absolute_error. + clf_quantile = GradientBoostingRegressor( + n_estimators=100, + loss="quantile", + max_depth=4, + alpha=0.5, + random_state=global_random_seed, + ) + + clf_quantile.fit(X_reg, y_reg) + y_quantile = clf_quantile.predict(X_reg) + + clf_ae = GradientBoostingRegressor( + n_estimators=100, + loss="absolute_error", + max_depth=4, + random_state=global_random_seed, + ) + + clf_ae.fit(X_reg, y_reg) + y_ae = clf_ae.predict(X_reg) + assert_allclose(y_quantile, y_ae) + + +def test_symbol_labels(): + # Test with non-integer class labels. + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + + symbol_y = list(map(str, y)) + + clf.fit(X, symbol_y) + assert_array_equal(clf.predict(T), list(map(str, true_result))) + assert 100 == len(clf.estimators_) + + +def test_float_class_labels(): + # Test with float class labels. + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + + float_y = np.asarray(y, dtype=np.float32) + + clf.fit(X, float_y) + assert_array_equal(clf.predict(T), np.asarray(true_result, dtype=np.float32)) + assert 100 == len(clf.estimators_) + + +def test_shape_y(): + # Test with float class labels. + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + + y_ = np.asarray(y, dtype=np.int32) + y_ = y_[:, np.newaxis] + + # This will raise a DataConversionWarning that we want to + # "always" raise, elsewhere the warnings gets ignored in the + # later tests, and the tests that check for this warning fail + warn_msg = ( + "A column-vector y was passed when a 1d array was expected. " + "Please change the shape of y to \\(n_samples, \\), for " + "example using ravel()." + ) + with pytest.warns(DataConversionWarning, match=warn_msg): + clf.fit(X, y_) + assert_array_equal(clf.predict(T), true_result) + assert 100 == len(clf.estimators_) + + +def test_mem_layout(): + # Test with different memory layouts of X and y + X_ = np.asfortranarray(X) + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + clf.fit(X_, y) + assert_array_equal(clf.predict(T), true_result) + assert 100 == len(clf.estimators_) + + X_ = np.ascontiguousarray(X) + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + clf.fit(X_, y) + assert_array_equal(clf.predict(T), true_result) + assert 100 == len(clf.estimators_) + + y_ = np.asarray(y, dtype=np.int32) + y_ = np.ascontiguousarray(y_) + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + clf.fit(X, y_) + assert_array_equal(clf.predict(T), true_result) + assert 100 == len(clf.estimators_) + + y_ = np.asarray(y, dtype=np.int32) + y_ = np.asfortranarray(y_) + clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + clf.fit(X, y_) + assert_array_equal(clf.predict(T), true_result) + assert 100 == len(clf.estimators_) + + +@pytest.mark.parametrize("GradientBoostingEstimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_oob_improvement(GradientBoostingEstimator): + # Test if oob improvement has correct shape and regression test. + estimator = GradientBoostingEstimator( + n_estimators=100, random_state=1, subsample=0.5 + ) + estimator.fit(X, y) + assert estimator.oob_improvement_.shape[0] == 100 + # hard-coded regression test - change if modification in OOB computation + assert_array_almost_equal( + estimator.oob_improvement_[:5], + np.array([0.19, 0.15, 0.12, -0.11, 0.11]), + decimal=2, + ) + + +@pytest.mark.parametrize("GradientBoostingEstimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_oob_scores(GradientBoostingEstimator): + # Test if oob scores has correct shape and regression test. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + estimator = GradientBoostingEstimator( + n_estimators=100, random_state=1, subsample=0.5 + ) + estimator.fit(X, y) + assert estimator.oob_scores_.shape[0] == 100 + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + + estimator = GradientBoostingEstimator( + n_estimators=100, + random_state=1, + subsample=0.5, + n_iter_no_change=5, + ) + estimator.fit(X, y) + assert estimator.oob_scores_.shape[0] < 100 + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + + +@pytest.mark.parametrize( + "GradientBoostingEstimator, oob_attribute", + [ + (GradientBoostingClassifier, "oob_improvement_"), + (GradientBoostingClassifier, "oob_scores_"), + (GradientBoostingClassifier, "oob_score_"), + (GradientBoostingRegressor, "oob_improvement_"), + (GradientBoostingRegressor, "oob_scores_"), + (GradientBoostingRegressor, "oob_score_"), + ], +) +def test_oob_attributes_error(GradientBoostingEstimator, oob_attribute): + """ + Check that we raise an AttributeError when the OOB statistics were not computed. + """ + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + estimator = GradientBoostingEstimator( + n_estimators=100, + random_state=1, + subsample=1.0, + ) + estimator.fit(X, y) + with pytest.raises(AttributeError): + estimator.oob_attribute + + +def test_oob_multilcass_iris(): + # Check OOB improvement on multi-class dataset. + estimator = GradientBoostingClassifier( + n_estimators=100, loss="log_loss", random_state=1, subsample=0.5 + ) + estimator.fit(iris.data, iris.target) + score = estimator.score(iris.data, iris.target) + assert score > 0.9 + assert estimator.oob_improvement_.shape[0] == estimator.n_estimators + assert estimator.oob_scores_.shape[0] == estimator.n_estimators + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + + estimator = GradientBoostingClassifier( + n_estimators=100, + loss="log_loss", + random_state=1, + subsample=0.5, + n_iter_no_change=5, + ) + estimator.fit(iris.data, iris.target) + score = estimator.score(iris.data, iris.target) + assert estimator.oob_improvement_.shape[0] < estimator.n_estimators + assert estimator.oob_scores_.shape[0] < estimator.n_estimators + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + + # hard-coded regression test - change if modification in OOB computation + # FIXME: the following snippet does not yield the same results on 32 bits + # assert_array_almost_equal(estimator.oob_improvement_[:5], + # np.array([12.68, 10.45, 8.18, 6.43, 5.13]), + # decimal=2) + + +@pytest.mark.thread_unsafe # manually captured stdout +def test_verbose_output(): + # Check verbose=1 does not cause error. + import sys + from io import StringIO + + old_stdout = sys.stdout + sys.stdout = StringIO() + clf = GradientBoostingClassifier( + n_estimators=100, random_state=1, verbose=1, subsample=0.8 + ) + clf.fit(X, y) + verbose_output = sys.stdout + sys.stdout = old_stdout + + # check output + verbose_output.seek(0) + header = verbose_output.readline().rstrip() + # with OOB + true_header = " ".join(["%10s"] + ["%16s"] * 3) % ( + "Iter", + "Train Loss", + "OOB Improve", + "Remaining Time", + ) + assert true_header == header + + n_lines = sum(1 for l in verbose_output.readlines()) + # one for 1-10 and then 9 for 20-100 + assert 10 + 9 == n_lines + + +@pytest.mark.thread_unsafe # manually captured stdout +def test_more_verbose_output(): + # Check verbose=2 does not cause error. + import sys + from io import StringIO + + old_stdout = sys.stdout + sys.stdout = StringIO() + clf = GradientBoostingClassifier(n_estimators=100, random_state=1, verbose=2) + clf.fit(X, y) + verbose_output = sys.stdout + sys.stdout = old_stdout + + # check output + verbose_output.seek(0) + header = verbose_output.readline().rstrip() + # no OOB + true_header = " ".join(["%10s"] + ["%16s"] * 2) % ( + "Iter", + "Train Loss", + "Remaining Time", + ) + assert true_header == header + + n_lines = sum(1 for l in verbose_output.readlines()) + # 100 lines for n_estimators==100 + assert 100 == n_lines + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start(Cls, global_random_seed): + # Test if warm start equals fit. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) + est = Cls(n_estimators=200, max_depth=1, random_state=global_random_seed) + est.fit(X, y) + + est_ws = Cls( + n_estimators=100, max_depth=1, warm_start=True, random_state=global_random_seed + ) + est_ws.fit(X, y) + est_ws.set_params(n_estimators=200) + est_ws.fit(X, y) + + if Cls is GradientBoostingRegressor: + assert_allclose(est_ws.predict(X), est.predict(X)) + else: + # Random state is preserved and hence predict_proba must also be + # same + assert_array_equal(est_ws.predict(X), est.predict(X)) + assert_allclose(est_ws.predict_proba(X), est.predict_proba(X)) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_n_estimators(Cls, global_random_seed): + # Test if warm start equals fit - set n_estimators. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) + est = Cls(n_estimators=300, max_depth=1, random_state=global_random_seed) + est.fit(X, y) + + est_ws = Cls( + n_estimators=100, max_depth=1, warm_start=True, random_state=global_random_seed + ) + est_ws.fit(X, y) + est_ws.set_params(n_estimators=300) + est_ws.fit(X, y) + + assert_allclose(est_ws.predict(X), est.predict(X)) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_max_depth(Cls): + # Test if possible to fit trees of different depth in ensemble. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + est = Cls(n_estimators=100, max_depth=1, warm_start=True) + est.fit(X, y) + est.set_params(n_estimators=110, max_depth=2) + est.fit(X, y) + + # last 10 trees have different depth + assert est.estimators_[0, 0].max_depth == 1 + for i in range(1, 11): + assert est.estimators_[-i, 0].max_depth == 2 + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_clear(Cls): + # Test if fit clears state. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + est = Cls(n_estimators=100, max_depth=1) + est.fit(X, y) + + est_2 = Cls(n_estimators=100, max_depth=1, warm_start=True) + est_2.fit(X, y) # inits state + est_2.set_params(warm_start=False) + est_2.fit(X, y) # clears old state and equals est + + assert_array_almost_equal(est_2.predict(X), est.predict(X)) + + +@pytest.mark.parametrize("GradientBoosting", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_state_oob_scores(GradientBoosting): + """ + Check that the states of the OOB scores are cleared when used with `warm_start`. + """ + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + n_estimators = 100 + estimator = GradientBoosting( + n_estimators=n_estimators, + max_depth=1, + subsample=0.5, + warm_start=True, + random_state=1, + ) + estimator.fit(X, y) + oob_scores, oob_score = estimator.oob_scores_, estimator.oob_score_ + assert len(oob_scores) == n_estimators + assert oob_scores[-1] == pytest.approx(oob_score) + + n_more_estimators = 200 + estimator.set_params(n_estimators=n_more_estimators).fit(X, y) + assert len(estimator.oob_scores_) == n_more_estimators + assert_allclose(estimator.oob_scores_[:n_estimators], oob_scores) + + estimator.set_params(n_estimators=n_estimators, warm_start=False).fit(X, y) + assert estimator.oob_scores_ is not oob_scores + assert estimator.oob_score_ is not oob_score + assert_allclose(estimator.oob_scores_, oob_scores) + assert estimator.oob_score_ == pytest.approx(oob_score) + assert oob_scores[-1] == pytest.approx(oob_score) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_smaller_n_estimators(Cls): + # Test if warm start with smaller n_estimators raises error + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + est = Cls(n_estimators=100, max_depth=1, warm_start=True) + est.fit(X, y) + est.set_params(n_estimators=99) + with pytest.raises(ValueError): + est.fit(X, y) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_equal_n_estimators(Cls): + # Test if warm start with equal n_estimators does nothing + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + est = Cls(n_estimators=100, max_depth=1) + est.fit(X, y) + + est2 = clone(est) + est2.set_params(n_estimators=est.n_estimators, warm_start=True) + est2.fit(X, y) + + assert_array_almost_equal(est2.predict(X), est.predict(X)) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_oob_switch(Cls): + # Test if oob can be turned on during warm start. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + est = Cls(n_estimators=100, max_depth=1, warm_start=True) + est.fit(X, y) + est.set_params(n_estimators=110, subsample=0.5) + est.fit(X, y) + + assert_array_equal(est.oob_improvement_[:100], np.zeros(100)) + assert_array_equal(est.oob_scores_[:100], np.zeros(100)) + + # the last 10 are not zeros + assert (est.oob_improvement_[-10:] != 0.0).all() + assert (est.oob_scores_[-10:] != 0.0).all() + + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_oob(Cls): + # Test if warm start OOB equals fit. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + est = Cls(n_estimators=200, max_depth=1, subsample=0.5, random_state=1) + est.fit(X, y) + + est_ws = Cls( + n_estimators=100, max_depth=1, subsample=0.5, random_state=1, warm_start=True + ) + est_ws.fit(X, y) + est_ws.set_params(n_estimators=200) + est_ws.fit(X, y) + + assert_array_almost_equal(est_ws.oob_improvement_[:100], est.oob_improvement_[:100]) + assert_array_almost_equal(est_ws.oob_scores_[:100], est.oob_scores_[:100]) + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) + assert est_ws.oob_scores_[-1] == pytest.approx(est_ws.oob_score_) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_warm_start_sparse(Cls, sparse_container): + # Test that all sparse matrix types are supported + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + est_dense = Cls( + n_estimators=100, max_depth=1, subsample=0.5, random_state=1, warm_start=True + ) + est_dense.fit(X, y) + est_dense.predict(X) + est_dense.set_params(n_estimators=200) + est_dense.fit(X, y) + y_pred_dense = est_dense.predict(X) + + X_sparse = sparse_container(X) + + est_sparse = Cls( + n_estimators=100, + max_depth=1, + subsample=0.5, + random_state=1, + warm_start=True, + ) + est_sparse.fit(X_sparse, y) + est_sparse.predict(X) + est_sparse.set_params(n_estimators=200) + est_sparse.fit(X_sparse, y) + y_pred_sparse = est_sparse.predict(X) + + assert_array_almost_equal( + est_dense.oob_improvement_[:100], est_sparse.oob_improvement_[:100] + ) + assert est_dense.oob_scores_[-1] == pytest.approx(est_dense.oob_score_) + assert_array_almost_equal(est_dense.oob_scores_[:100], est_sparse.oob_scores_[:100]) + assert est_sparse.oob_scores_[-1] == pytest.approx(est_sparse.oob_score_) + assert_array_almost_equal(y_pred_dense, y_pred_sparse) + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_fortran(Cls, global_random_seed): + # Test that feeding an X in Fortran-ordered is giving the same results as + # in C-ordered + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) + est_c = Cls(n_estimators=1, random_state=global_random_seed, warm_start=True) + est_fortran = Cls(n_estimators=1, random_state=global_random_seed, warm_start=True) + + est_c.fit(X, y) + est_c.set_params(n_estimators=11) + est_c.fit(X, y) + + X_fortran = np.asfortranarray(X) + est_fortran.fit(X_fortran, y) + est_fortran.set_params(n_estimators=11) + est_fortran.fit(X_fortran, y) + + assert_allclose(est_c.predict(X), est_fortran.predict(X)) + + +def early_stopping_monitor(i, est, locals): + """Returns True on the 10th iteration.""" + if i == 9: + return True + else: + return False + + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_monitor_early_stopping(Cls): + # Test if monitor return value works. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + + est = Cls(n_estimators=20, max_depth=1, random_state=1, subsample=0.5) + est.fit(X, y, monitor=early_stopping_monitor) + assert est.n_estimators == 20 # this is not altered + assert est.estimators_.shape[0] == 10 + assert est.train_score_.shape[0] == 10 + assert est.oob_improvement_.shape[0] == 10 + assert est.oob_scores_.shape[0] == 10 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) + + # try refit + est.set_params(n_estimators=30) + est.fit(X, y) + assert est.n_estimators == 30 + assert est.estimators_.shape[0] == 30 + assert est.train_score_.shape[0] == 30 + assert est.oob_improvement_.shape[0] == 30 + assert est.oob_scores_.shape[0] == 30 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) + + est = Cls( + n_estimators=20, max_depth=1, random_state=1, subsample=0.5, warm_start=True + ) + est.fit(X, y, monitor=early_stopping_monitor) + assert est.n_estimators == 20 + assert est.estimators_.shape[0] == 10 + assert est.train_score_.shape[0] == 10 + assert est.oob_improvement_.shape[0] == 10 + assert est.oob_scores_.shape[0] == 10 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) + + # try refit + est.set_params(n_estimators=30, warm_start=False) + est.fit(X, y) + assert est.n_estimators == 30 + assert est.train_score_.shape[0] == 30 + assert est.estimators_.shape[0] == 30 + assert est.oob_improvement_.shape[0] == 30 + assert est.oob_scores_.shape[0] == 30 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) + + +def test_complete_classification(): + # Test greedy trees with max_depth + 1 leafs. + from sklearn.tree._tree import TREE_LEAF + + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + k = 4 + + est = GradientBoostingClassifier( + n_estimators=20, max_depth=None, random_state=1, max_leaf_nodes=k + 1 + ) + est.fit(X, y) + + tree = est.estimators_[0, 0].tree_ + assert tree.max_depth == k + assert tree.children_left[tree.children_left == TREE_LEAF].shape[0] == k + 1 + + +def test_complete_regression(): + # Test greedy trees with max_depth + 1 leafs. + from sklearn.tree._tree import TREE_LEAF + + k = 4 + + est = GradientBoostingRegressor( + n_estimators=20, max_depth=None, random_state=1, max_leaf_nodes=k + 1 + ) + est.fit(X_reg, y_reg) + + tree = est.estimators_[-1, 0].tree_ + assert tree.children_left[tree.children_left == TREE_LEAF].shape[0] == k + 1 + + +def test_zero_estimator_reg(global_random_seed): + # Test if init='zero' works for regression by checking that it is better + # than a simple baseline. + + baseline = DummyRegressor(strategy="mean").fit(X_reg, y_reg) + mse_baseline = mean_squared_error(baseline.predict(X_reg), y_reg) + est = GradientBoostingRegressor( + n_estimators=5, + max_depth=1, + random_state=global_random_seed, + init="zero", + learning_rate=0.5, + ) + est.fit(X_reg, y_reg) + y_pred = est.predict(X_reg) + mse_gbdt = mean_squared_error(y_reg, y_pred) + assert mse_gbdt < mse_baseline + + +def test_zero_estimator_clf(global_random_seed): + # Test if init='zero' works for classification. + X = iris.data + y = np.array(iris.target) + + est = GradientBoostingClassifier( + n_estimators=20, max_depth=1, random_state=global_random_seed, init="zero" + ) + est.fit(X, y) + + assert est.score(X, y) > 0.96 + + # binary clf + mask = y != 0 + y[mask] = 1 + y[~mask] = 0 + est = GradientBoostingClassifier( + n_estimators=20, max_depth=1, random_state=global_random_seed, init="zero" + ) + est.fit(X, y) + assert est.score(X, y) > 0.96 + + +@pytest.mark.parametrize("GBEstimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_max_leaf_nodes_max_depth(GBEstimator): + # Test precedence of max_leaf_nodes over max_depth. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + + k = 4 + + est = GBEstimator(max_depth=1, max_leaf_nodes=k).fit(X, y) + tree = est.estimators_[0, 0].tree_ + assert tree.max_depth == 1 + + est = GBEstimator(max_depth=1).fit(X, y) + tree = est.estimators_[0, 0].tree_ + assert tree.max_depth == 1 + + +@pytest.mark.parametrize("GBEstimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_min_impurity_decrease(GBEstimator): + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + + est = GBEstimator(min_impurity_decrease=0.1) + est.fit(X, y) + for tree in est.estimators_.flat: + # Simply check if the parameter is passed on correctly. Tree tests + # will suffice for the actual working of this param + assert tree.min_impurity_decrease == 0.1 + + +def test_warm_start_wo_nestimators_change(): + # Test if warm_start does nothing if n_estimators is not changed. + # Regression test for #3513. + clf = GradientBoostingClassifier(n_estimators=10, warm_start=True) + clf.fit([[0, 1], [2, 3]], [0, 1]) + assert clf.estimators_.shape[0] == 10 + clf.fit([[0, 1], [2, 3]], [0, 1]) + assert clf.estimators_.shape[0] == 10 + + +@pytest.mark.parametrize( + ("loss", "value"), + [ + ("squared_error", 0.5), + ("absolute_error", 0.0), + ("huber", 0.5), + ("quantile", 0.5), + ], +) +def test_non_uniform_weights_toy_edge_case_reg(loss, value): + X = [[1, 0], [1, 0], [1, 0], [0, 1]] + y = [0, 0, 1, 0] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1] + gb = GradientBoostingRegressor(learning_rate=1.0, n_estimators=2, loss=loss) + gb.fit(X, y, sample_weight=sample_weight) + assert gb.predict([[1, 0]])[0] >= value + + +def test_non_uniform_weights_toy_edge_case_clf(): + X = [[1, 0], [1, 0], [1, 0], [0, 1]] + y = [0, 0, 1, 0] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1] + for loss in ("log_loss", "exponential"): + gb = GradientBoostingClassifier(n_estimators=5, loss=loss) + gb.fit(X, y, sample_weight=sample_weight) + assert_array_equal(gb.predict([[1, 0]]), [1]) + + +@skip_if_32bit +@pytest.mark.parametrize( + "EstimatorClass", (GradientBoostingClassifier, GradientBoostingRegressor) +) +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_sparse_input(EstimatorClass, sparse_container): + y, X = datasets.make_multilabel_classification( + random_state=0, n_samples=50, n_features=1, n_classes=20 + ) + y = y[:, 0] + X_sparse = sparse_container(X) + + dense = EstimatorClass( + n_estimators=10, random_state=0, max_depth=2, min_impurity_decrease=1e-7 + ).fit(X, y) + sparse = EstimatorClass( + n_estimators=10, random_state=0, max_depth=2, min_impurity_decrease=1e-7 + ).fit(X_sparse, y) + + assert_array_almost_equal(sparse.apply(X), dense.apply(X)) + assert_array_almost_equal(sparse.predict(X), dense.predict(X)) + assert_array_almost_equal(sparse.feature_importances_, dense.feature_importances_) + + assert_array_almost_equal(sparse.predict(X_sparse), dense.predict(X)) + assert_array_almost_equal(dense.predict(X_sparse), sparse.predict(X)) + + if issubclass(EstimatorClass, GradientBoostingClassifier): + assert_array_almost_equal(sparse.predict_proba(X), dense.predict_proba(X)) + assert_array_almost_equal( + sparse.predict_log_proba(X), dense.predict_log_proba(X) + ) + + assert_array_almost_equal( + sparse.decision_function(X_sparse), sparse.decision_function(X) + ) + assert_array_almost_equal( + dense.decision_function(X_sparse), sparse.decision_function(X) + ) + for res_sparse, res in zip( + sparse.staged_decision_function(X_sparse), + sparse.staged_decision_function(X), + ): + assert_array_almost_equal(res_sparse, res) + + +@pytest.mark.parametrize( + "GradientBoostingEstimator", [GradientBoostingClassifier, GradientBoostingRegressor] +) +def test_gradient_boosting_early_stopping(GradientBoostingEstimator): + # Check if early stopping works as expected, that is empirically check that the + # number of trained estimators is increasing when the tolerance decreases. + + X, y = make_classification(n_samples=1000, random_state=0) + n_estimators = 1000 + + gb_large_tol = GradientBoostingEstimator( + n_estimators=n_estimators, + n_iter_no_change=10, + learning_rate=0.1, + max_depth=3, + random_state=42, + tol=1e-1, + ) + + gb_small_tol = GradientBoostingEstimator( + n_estimators=n_estimators, + n_iter_no_change=10, + learning_rate=0.1, + max_depth=3, + random_state=42, + tol=1e-3, + ) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) + gb_large_tol.fit(X_train, y_train) + gb_small_tol.fit(X_train, y_train) + + assert gb_large_tol.n_estimators_ < gb_small_tol.n_estimators_ < n_estimators + + assert gb_large_tol.score(X_test, y_test) > 0.7 + assert gb_small_tol.score(X_test, y_test) > 0.7 + + +def test_gradient_boosting_without_early_stopping(): + # When early stopping is not used, the number of trained estimators + # must be the one specified. + X, y = make_classification(n_samples=1000, random_state=0) + + gbc = GradientBoostingClassifier( + n_estimators=50, learning_rate=0.1, max_depth=3, random_state=42 + ) + gbc.fit(X, y) + gbr = GradientBoostingRegressor( + n_estimators=30, learning_rate=0.1, max_depth=3, random_state=42 + ) + gbr.fit(X, y) + + # The number of trained estimators must be the one specified. + assert gbc.n_estimators_ == 50 + assert gbr.n_estimators_ == 30 + + +def test_gradient_boosting_validation_fraction(): + X, y = make_classification(n_samples=1000, random_state=0) + + gbc = GradientBoostingClassifier( + n_estimators=100, + n_iter_no_change=10, + validation_fraction=0.1, + learning_rate=0.1, + max_depth=3, + random_state=42, + ) + gbc2 = clone(gbc).set_params(validation_fraction=0.3) + gbc3 = clone(gbc).set_params(n_iter_no_change=20) + + gbr = GradientBoostingRegressor( + n_estimators=100, + n_iter_no_change=10, + learning_rate=0.1, + max_depth=3, + validation_fraction=0.1, + random_state=42, + ) + gbr2 = clone(gbr).set_params(validation_fraction=0.3) + gbr3 = clone(gbr).set_params(n_iter_no_change=20) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) + # Check if validation_fraction has an effect + gbc.fit(X_train, y_train) + gbc2.fit(X_train, y_train) + assert gbc.n_estimators_ != gbc2.n_estimators_ + + gbr.fit(X_train, y_train) + gbr2.fit(X_train, y_train) + assert gbr.n_estimators_ != gbr2.n_estimators_ + + # Check if n_estimators_ increase monotonically with n_iter_no_change + # Set validation + gbc3.fit(X_train, y_train) + gbr3.fit(X_train, y_train) + assert gbr.n_estimators_ < gbr3.n_estimators_ + assert gbc.n_estimators_ < gbc3.n_estimators_ + + +def test_early_stopping_stratified(): + # Make sure data splitting for early stopping is stratified + X = [[1, 2], [2, 3], [3, 4], [4, 5]] + y = [0, 0, 0, 1] + + gbc = GradientBoostingClassifier(n_iter_no_change=5) + with pytest.raises( + ValueError, + match=( + r"The least populated classes in y have only 1 member.*Classes with " + r"too few members are: \[1.0\]" + ), + ): + gbc.fit(X, y) + + +def _make_multiclass(): + return make_classification(n_classes=3, n_clusters_per_class=1) + + +@pytest.mark.parametrize( + "gb, dataset_maker, init_estimator", + [ + (GradientBoostingClassifier, make_classification, DummyClassifier), + (GradientBoostingClassifier, _make_multiclass, DummyClassifier), + (GradientBoostingRegressor, make_regression, DummyRegressor), + ], + ids=["binary classification", "multiclass classification", "regression"], +) +def test_gradient_boosting_with_init( + gb, dataset_maker, init_estimator, global_random_seed +): + # Check that GradientBoostingRegressor works when init is a sklearn + # estimator. + # Check that an error is raised if trying to fit with sample weight but + # initial estimator does not support sample weight + + X, y = dataset_maker() + sample_weight = np.random.RandomState(global_random_seed).rand(100) + + # init supports sample weights + init_est = init_estimator() + gb(init=init_est).fit(X, y, sample_weight=sample_weight) + + # init does not support sample weights + init_est = NoSampleWeightWrapper(init_estimator()) + gb(init=init_est).fit(X, y) # ok no sample weights + with pytest.raises(ValueError, match="estimator.*does not support sample weights"): + gb(init=init_est).fit(X, y, sample_weight=sample_weight) + + +def test_gradient_boosting_with_init_pipeline(): + # Check that the init estimator can be a pipeline (see issue #13466) + + X, y = make_regression(random_state=0) + init = make_pipeline(LinearRegression()) + gb = GradientBoostingRegressor(init=init) + gb.fit(X, y) # pipeline without sample_weight works fine + + with pytest.raises( + ValueError, + match="The initial estimator Pipeline does not support sample weights", + ): + gb.fit(X, y, sample_weight=np.ones(X.shape[0])) + + # Passing sample_weight to a pipeline raises a ValueError. This test makes + # sure we make the distinction between ValueError raised by a pipeline that + # was passed sample_weight, and a InvalidParameterError raised by a regular + # estimator whose input checking failed. + invalid_nu = 1.5 + err_msg = ( + "The 'nu' parameter of NuSVR must be a float in the" + f" range (0.0, 1.0]. Got {invalid_nu} instead." + ) + with pytest.raises(InvalidParameterError, match=re.escape(err_msg)): + # Note that NuSVR properly supports sample_weight + init = NuSVR(gamma="auto", nu=invalid_nu) + gb = GradientBoostingRegressor(init=init) + gb.fit(X, y, sample_weight=np.ones(X.shape[0])) + + +def test_early_stopping_n_classes(): + # when doing early stopping (_, , y_train, _ = train_test_split(X, y)) + # there might be classes in y that are missing in y_train. As the init + # estimator will be trained on y_train, we need to raise an error if this + # happens. + + X = [[1]] * 10 + y = [0, 0] + [1] * 8 # only 2 negative class over 10 samples + gb = GradientBoostingClassifier( + n_iter_no_change=5, random_state=0, validation_fraction=0.8 + ) + with pytest.raises( + ValueError, match="The training data after the early stopping split" + ): + gb.fit(X, y) + + # No error if we let training data be big enough + gb = GradientBoostingClassifier( + n_iter_no_change=5, random_state=0, validation_fraction=0.4 + ) + + +def test_gbr_degenerate_feature_importances(): + # growing an ensemble of single node trees. See #13620 + X = np.zeros((10, 10)) + y = np.ones((10,)) + gbr = GradientBoostingRegressor().fit(X, y) + assert_array_equal(gbr.feature_importances_, np.zeros(10, dtype=np.float64)) + + +def test_huber_vs_mean_and_median(): + """Check that huber lies between absolute and squared error.""" + n_rep = 100 + n_samples = 10 + y = np.tile(np.arange(n_samples), n_rep) + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + + rng = np.random.RandomState(42) + # We want an asymmetric distribution. + y = y + rng.exponential(scale=1, size=y.shape) + + gbt_absolute_error = GradientBoostingRegressor(loss="absolute_error").fit(X, y) + gbt_huber = GradientBoostingRegressor(loss="huber").fit(X, y) + gbt_squared_error = GradientBoostingRegressor().fit(X, y) + + gbt_huber_predictions = gbt_huber.predict(X) + assert np.all(gbt_absolute_error.predict(X) <= gbt_huber_predictions) + assert np.all(gbt_huber_predictions <= gbt_squared_error.predict(X)) + + +def test_safe_divide(): + """Test that _safe_divide handles division by zero.""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert _safe_divide(np.float64(1e300), 0) == 0 + assert _safe_divide(np.float64(0.0), np.float64(0.0)) == 0 + with pytest.warns(RuntimeWarning, match="overflow"): + # np.finfo(float).max = 1.7976931348623157e+308 + _safe_divide(np.float64(1e300), 1e-10) + + +def test_squared_error_exact_backward_compat(): + """Test squared error GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingRegressor(loss="squared_error", n_estimators=100).fit(X, y) + + pred_result = np.array( + [ + 1.39245726e-04, + 1.00010468e00, + 2.00007043e00, + 3.00004051e00, + 4.00000802e00, + 4.99998972e00, + 5.99996312e00, + 6.99993395e00, + 7.99989372e00, + 8.99985660e00, + ] + ) + assert_allclose(gbt.predict(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 4.87246390e-08, + 3.95590036e-08, + 3.21267865e-08, + 2.60970300e-08, + 2.11820178e-08, + 1.71995782e-08, + 1.39695549e-08, + 1.13391770e-08, + 9.19931587e-09, + 7.47000575e-09, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + + # Same but with sample_weights + sample_weights = np.tile([1, 10], n_samples // 2) + gbt = GradientBoostingRegressor(loss="squared_error", n_estimators=100).fit( + X, y, sample_weight=sample_weights + ) + + pred_result = np.array( + [ + 1.52391462e-04, + 1.00011168e00, + 2.00007724e00, + 3.00004638e00, + 4.00001302e00, + 4.99999873e00, + 5.99997093e00, + 6.99994329e00, + 7.99991290e00, + 8.99988727e00, + ] + ) + assert_allclose(gbt.predict(X), pred_result, rtol=1e-6, atol=1e-5) + + train_score = np.array( + [ + 4.12445296e-08, + 3.34418322e-08, + 2.71151383e-08, + 2.19782469e-08, + 1.78173649e-08, + 1.44461976e-08, + 1.17120123e-08, + 9.49485678e-09, + 7.69772505e-09, + 6.24155316e-09, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-3, atol=1e-11) + + +@skip_if_32bit +def test_huber_exact_backward_compat(): + """Test huber GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingRegressor(loss="huber", n_estimators=100, alpha=0.8).fit(X, y) + + assert_allclose(gbt._loss.closs.delta, 0.0001655688041282133) + + pred_result = np.array( + [ + 1.48120765e-04, + 9.99949174e-01, + 2.00116957e00, + 2.99986716e00, + 4.00012064e00, + 5.00002462e00, + 5.99998898e00, + 6.99692549e00, + 8.00006356e00, + 8.99985099e00, + ] + ) + assert_allclose(gbt.predict(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 2.59484709e-07, + 2.19165900e-07, + 1.89644782e-07, + 1.64556454e-07, + 1.38705110e-07, + 1.20373736e-07, + 1.04746082e-07, + 9.13835687e-08, + 8.20245756e-08, + 7.17122188e-08, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + + +def test_binomial_error_exact_backward_compat(): + """Test binary log_loss GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) % 2 + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingClassifier(loss="log_loss", n_estimators=100).fit(X, y) + + pred_result = np.array( + [ + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + ] + ) + assert_allclose(gbt.predict_proba(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 1.07742210e-04, + 9.74889078e-05, + 8.82113863e-05, + 7.98167784e-05, + 7.22210566e-05, + 6.53481907e-05, + 5.91293869e-05, + 5.35023988e-05, + 4.84109045e-05, + 4.38039423e-05, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + + +def test_multinomial_error_exact_backward_compat(): + """Test multiclass log_loss GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) % 4 + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingClassifier(loss="log_loss", n_estimators=100).fit(X, y) + + pred_result = np.array( + [ + [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], + [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], + [1.19417637e-07, 1.19417637e-07, 9.99999675e-01, 8.60526098e-08], + [1.19417637e-07, 1.19417637e-07, 8.60526088e-08, 9.99999675e-01], + [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], + [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], + [1.19417637e-07, 1.19417637e-07, 9.99999675e-01, 8.60526098e-08], + [1.19417637e-07, 1.19417637e-07, 8.60526088e-08, 9.99999675e-01], + [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], + [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], + ] + ) + assert_allclose(gbt.predict_proba(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 1.13300150e-06, + 9.75183397e-07, + 8.39348103e-07, + 7.22433588e-07, + 6.21804338e-07, + 5.35191943e-07, + 4.60643966e-07, + 3.96479930e-07, + 3.41253434e-07, + 2.93719550e-07, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + + +def test_gb_denominator_zero(global_random_seed): + """Test _update_terminal_regions denominator is not zero. + + For instance for log loss based binary classification, the line search step might + become nan/inf as denominator = hessian = prob * (1 - prob) and prob = 0 or 1 can + happen. + Here, we create a situation were this happens (at least with roughly 80%) based + on the random seed. + """ + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=20) + + params = { + "learning_rate": 1.0, + "subsample": 0.5, + "n_estimators": 100, + "max_leaf_nodes": 4, + "max_depth": None, + "random_state": global_random_seed, + "min_samples_leaf": 2, + } + + clf = GradientBoostingClassifier(**params) + # _safe_devide would raise a RuntimeWarning + with warnings.catch_warnings(): + warnings.simplefilter("error") + clf.fit(X, y) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_iforest.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_iforest.py new file mode 100644 index 0000000000000000000000000000000000000000..eb7cfba2ef87f5e2b0c39829fc5b74965fedfc26 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_iforest.py @@ -0,0 +1,395 @@ +""" +Testing for Isolation Forest algorithm (sklearn.ensemble.iforest). +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings +from unittest.mock import Mock, patch + +import numpy as np +import pytest +from joblib import parallel_backend + +from sklearn.datasets import load_diabetes, load_iris, make_classification +from sklearn.ensemble import IsolationForest +from sklearn.ensemble._iforest import _average_path_length +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import ParameterGrid, train_test_split +from sklearn.utils import check_random_state +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, + ignore_warnings, +) +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS + +# load iris & diabetes dataset +iris = load_iris() +diabetes = load_diabetes() + + +def test_iforest(global_random_seed): + """Check Isolation Forest for various parameter settings.""" + X_train = np.array([[0, 1], [1, 2]]) + X_test = np.array([[2, 1], [1, 1]]) + + grid = ParameterGrid( + {"n_estimators": [3], "max_samples": [0.5, 1.0, 3], "bootstrap": [True, False]} + ) + + with ignore_warnings(): + for params in grid: + IsolationForest(random_state=global_random_seed, **params).fit( + X_train + ).predict(X_test) + + +@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS) +def test_iforest_sparse(global_random_seed, sparse_container): + """Check IForest for various parameter settings on sparse input.""" + rng = check_random_state(global_random_seed) + X_train, X_test = train_test_split(diabetes.data[:50], random_state=rng) + grid = ParameterGrid({"max_samples": [0.5, 1.0], "bootstrap": [True, False]}) + + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) + + for params in grid: + # Trained on sparse format + sparse_classifier = IsolationForest( + n_estimators=10, random_state=global_random_seed, **params + ).fit(X_train_sparse) + sparse_results = sparse_classifier.predict(X_test_sparse) + + # Trained on dense format + dense_classifier = IsolationForest( + n_estimators=10, random_state=global_random_seed, **params + ).fit(X_train) + dense_results = dense_classifier.predict(X_test) + + assert_array_equal(sparse_results, dense_results) + + +def test_iforest_error(): + """Test that it gives proper exception on deficient input.""" + X = iris.data + + # The dataset has less than 256 samples, explicitly setting + # max_samples > n_samples should result in a warning. If not set + # explicitly there should be no warning + warn_msg = "max_samples will be set to n_samples for estimation" + with pytest.warns(UserWarning, match=warn_msg): + IsolationForest(max_samples=1000).fit(X) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + IsolationForest(max_samples="auto").fit(X) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + IsolationForest(max_samples=np.int64(2)).fit(X) + + # test X_test n_features match X_train one: + with pytest.raises(ValueError): + IsolationForest().fit(X).predict(X[:, 1:]) + + +def test_recalculate_max_depth(): + """Check max_depth recalculation when max_samples is reset to n_samples""" + X = iris.data + clf = IsolationForest().fit(X) + for est in clf.estimators_: + assert est.max_depth == int(np.ceil(np.log2(X.shape[0]))) + + +def test_max_samples_attribute(): + X = iris.data + clf = IsolationForest().fit(X) + assert clf.max_samples_ == X.shape[0] + + clf = IsolationForest(max_samples=500) + warn_msg = "max_samples will be set to n_samples for estimation" + with pytest.warns(UserWarning, match=warn_msg): + clf.fit(X) + assert clf.max_samples_ == X.shape[0] + + clf = IsolationForest(max_samples=0.4).fit(X) + assert clf.max_samples_ == 0.4 * X.shape[0] + + +def test_iforest_parallel_regression(global_random_seed): + """Check parallel regression.""" + rng = check_random_state(global_random_seed) + + X_train, X_test = train_test_split(diabetes.data, random_state=rng) + + ensemble = IsolationForest(n_jobs=3, random_state=global_random_seed).fit(X_train) + + ensemble.set_params(n_jobs=1) + y1 = ensemble.predict(X_test) + ensemble.set_params(n_jobs=2) + y2 = ensemble.predict(X_test) + assert_array_almost_equal(y1, y2) + + ensemble = IsolationForest(n_jobs=1, random_state=global_random_seed).fit(X_train) + + y3 = ensemble.predict(X_test) + assert_array_almost_equal(y1, y3) + + +def test_iforest_performance(global_random_seed): + """Test Isolation Forest performs well""" + + # Generate train/test data + rng = check_random_state(global_random_seed) + X = 0.3 * rng.randn(600, 2) + X = rng.permutation(np.vstack((X + 2, X - 2))) + X_train = X[:1000] + + # Generate some abnormal novel observations + X_outliers = rng.uniform(low=-1, high=1, size=(200, 2)) + X_test = np.vstack((X[1000:], X_outliers)) + y_test = np.array([0] * 200 + [1] * 200) + + # fit the model + clf = IsolationForest(max_samples=100, random_state=rng).fit(X_train) + + # predict scores (the lower, the more normal) + y_pred = -clf.decision_function(X_test) + + # check that there is at most 6 errors (false positive or false negative) + assert roc_auc_score(y_test, y_pred) > 0.98 + + +@pytest.mark.parametrize("contamination", [0.25, "auto"]) +def test_iforest_works(contamination, global_random_seed): + # toy sample (the last two samples are outliers) + X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [7, 4], [-5, 9]] + + # Test IsolationForest + clf = IsolationForest(random_state=global_random_seed, contamination=contamination) + clf.fit(X) + decision_func = -clf.decision_function(X) + pred = clf.predict(X) + # assert detect outliers: + assert np.min(decision_func[-2:]) > np.max(decision_func[:-2]) + assert_array_equal(pred, 6 * [1] + 2 * [-1]) + + +def test_max_samples_consistency(): + # Make sure validated max_samples in iforest and BaseBagging are identical + X = iris.data + clf = IsolationForest().fit(X) + assert clf.max_samples_ == clf._max_samples + + +def test_iforest_subsampled_features(): + # It tests non-regression for #5732 which failed at predict. + rng = check_random_state(0) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data[:50], diabetes.target[:50], random_state=rng + ) + clf = IsolationForest(max_features=0.8) + clf.fit(X_train, y_train) + clf.predict(X_test) + + +def test_iforest_average_path_length(): + # It tests non-regression for #8549 which used the wrong formula + # for average path length, strictly for the integer case + # Updated to check average path length when input is <= 2 (issue #11839) + result_one = 2.0 * (np.log(4.0) + np.euler_gamma) - 2.0 * 4.0 / 5.0 + result_two = 2.0 * (np.log(998.0) + np.euler_gamma) - 2.0 * 998.0 / 999.0 + assert_allclose(_average_path_length([0]), [0.0]) + assert_allclose(_average_path_length([1]), [0.0]) + assert_allclose(_average_path_length([2]), [1.0]) + assert_allclose(_average_path_length([5]), [result_one]) + assert_allclose(_average_path_length([999]), [result_two]) + assert_allclose( + _average_path_length(np.array([1, 2, 5, 999])), + [0.0, 1.0, result_one, result_two], + ) + # _average_path_length is increasing + avg_path_length = _average_path_length(np.arange(5)) + assert_array_equal(avg_path_length, np.sort(avg_path_length)) + + +def test_score_samples(): + X_train = [[1, 1], [1, 2], [2, 1]] + clf1 = IsolationForest(contamination=0.1).fit(X_train) + clf2 = IsolationForest().fit(X_train) + assert_array_equal( + clf1.score_samples([[2.0, 2.0]]), + clf1.decision_function([[2.0, 2.0]]) + clf1.offset_, + ) + assert_array_equal( + clf2.score_samples([[2.0, 2.0]]), + clf2.decision_function([[2.0, 2.0]]) + clf2.offset_, + ) + assert_array_equal( + clf1.score_samples([[2.0, 2.0]]), clf2.score_samples([[2.0, 2.0]]) + ) + + +def test_iforest_warm_start(): + """Test iterative addition of iTrees to an iForest""" + + rng = check_random_state(0) + X = rng.randn(20, 2) + + # fit first 10 trees + clf = IsolationForest( + n_estimators=10, max_samples=20, random_state=rng, warm_start=True + ) + clf.fit(X) + # remember the 1st tree + tree_1 = clf.estimators_[0] + # fit another 10 trees + clf.set_params(n_estimators=20) + clf.fit(X) + # expecting 20 fitted trees and no overwritten trees + assert len(clf.estimators_) == 20 + assert clf.estimators_[0] is tree_1 + + +# mock get_chunk_n_rows to actually test more than one chunk (here one +# chunk has 3 rows): +@patch( + "sklearn.ensemble._iforest.get_chunk_n_rows", + side_effect=Mock(**{"return_value": 3}), +) +@pytest.mark.parametrize("contamination, n_predict_calls", [(0.25, 3), ("auto", 2)]) +@pytest.mark.thread_unsafe # monkeypatched code +def test_iforest_chunks_works1( + mocked_get_chunk, contamination, n_predict_calls, global_random_seed +): + test_iforest_works(contamination, global_random_seed) + assert mocked_get_chunk.call_count == n_predict_calls + + +# idem with chunk_size = 10 rows +@patch( + "sklearn.ensemble._iforest.get_chunk_n_rows", + side_effect=Mock(**{"return_value": 10}), +) +@pytest.mark.parametrize("contamination, n_predict_calls", [(0.25, 3), ("auto", 2)]) +@pytest.mark.thread_unsafe # monkeypatched code +def test_iforest_chunks_works2( + mocked_get_chunk, contamination, n_predict_calls, global_random_seed +): + test_iforest_works(contamination, global_random_seed) + assert mocked_get_chunk.call_count == n_predict_calls + + +def test_iforest_with_uniform_data(): + """Test whether iforest predicts inliers when using uniform data""" + + # 2-d array of all 1s + X = np.ones((100, 10)) + iforest = IsolationForest() + iforest.fit(X) + + rng = np.random.RandomState(0) + + assert all(iforest.predict(X) == 1) + assert all(iforest.predict(rng.randn(100, 10)) == 1) + assert all(iforest.predict(X + 1) == 1) + assert all(iforest.predict(X - 1) == 1) + + # 2-d array where columns contain the same value across rows + X = np.repeat(rng.randn(1, 10), 100, 0) + iforest = IsolationForest() + iforest.fit(X) + + assert all(iforest.predict(X) == 1) + assert all(iforest.predict(rng.randn(100, 10)) == 1) + assert all(iforest.predict(np.ones((100, 10))) == 1) + + # Single row + X = rng.randn(1, 10) + iforest = IsolationForest() + iforest.fit(X) + + assert all(iforest.predict(X) == 1) + assert all(iforest.predict(rng.randn(100, 10)) == 1) + assert all(iforest.predict(np.ones((100, 10))) == 1) + + +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_iforest_with_n_jobs_does_not_segfault(csc_container): + """Check that Isolation Forest does not segfault with n_jobs=2 + + Non-regression test for #23252 + """ + X, _ = make_classification(n_samples=85_000, n_features=100, random_state=0) + X = csc_container(X) + IsolationForest(n_estimators=10, max_samples=256, n_jobs=2).fit(X) + + +def test_iforest_preserve_feature_names(): + """Check that feature names are preserved when contamination is not "auto". + + Feature names are required for consistency checks during scoring. + + Non-regression test for Issue #25844 + """ + pd = pytest.importorskip("pandas") + rng = np.random.RandomState(0) + + X = pd.DataFrame(data=rng.randn(4), columns=["a"]) + model = IsolationForest(random_state=0, contamination=0.05) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + model.fit(X) + + +@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS) +def test_iforest_sparse_input_float_contamination(sparse_container): + """Check that `IsolationForest` accepts sparse matrix input and float value for + contamination. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27626 + """ + X, _ = make_classification(n_samples=50, n_features=4, random_state=0) + X = sparse_container(X) + X.sort_indices() + contamination = 0.1 + iforest = IsolationForest( + n_estimators=5, contamination=contamination, random_state=0 + ).fit(X) + + X_decision = iforest.decision_function(X) + assert (X_decision < 0).sum() / X.shape[0] == pytest.approx(contamination) + + +@pytest.mark.parametrize("n_jobs", [1, 2]) +@pytest.mark.parametrize("contamination", [0.25, "auto"]) +def test_iforest_predict_parallel(global_random_seed, contamination, n_jobs): + """Check that `IsolationForest.predict` is parallelized.""" + # toy sample (the last two samples are outliers) + X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [7, 4], [-5, 9]] + + # Test IsolationForest + clf = IsolationForest( + random_state=global_random_seed, contamination=contamination, n_jobs=None + ) + clf.fit(X) + decision_func = -clf.decision_function(X) + pred = clf.predict(X) + + # assert detect outliers: + assert np.min(decision_func[-2:]) > np.max(decision_func[:-2]) + assert_array_equal(pred, 6 * [1] + 2 * [-1]) + + clf_parallel = IsolationForest( + random_state=global_random_seed, contamination=contamination, n_jobs=-1 + ) + clf_parallel.fit(X) + with parallel_backend("threading", n_jobs=n_jobs): + pred_paralell = clf_parallel.predict(X) + + # assert the same results as non-parallel + assert_array_equal(pred, pred_paralell) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_stacking.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_stacking.py new file mode 100644 index 0000000000000000000000000000000000000000..b7e7aa9d3f88fa922513f23e8b4c999f7fc65bdd --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_stacking.py @@ -0,0 +1,1020 @@ +"""Test the stacking classifier and regressor.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import re +from unittest.mock import Mock + +import numpy as np +import pytest +from numpy.testing import assert_array_equal +from scipy import sparse + +from sklearn import config_context +from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin, clone +from sklearn.datasets import ( + load_breast_cancer, + load_diabetes, + load_iris, + make_classification, + make_multilabel_classification, + make_regression, +) +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble import ( + RandomForestClassifier, + RandomForestRegressor, + StackingClassifier, + StackingRegressor, +) +from sklearn.exceptions import ConvergenceWarning, NotFittedError +from sklearn.linear_model import ( + LinearRegression, + LogisticRegression, + Ridge, + RidgeClassifier, +) +from sklearn.model_selection import KFold, StratifiedKFold, train_test_split +from sklearn.neighbors import KNeighborsClassifier +from sklearn.neural_network import MLPClassifier +from sklearn.preprocessing import scale +from sklearn.svm import SVC, LinearSVC, LinearSVR +from sklearn.tests.metadata_routing_common import ( + ConsumingClassifier, + ConsumingRegressor, + _Registry, + check_recorded_metadata, +) +from sklearn.utils._mocking import CheckingClassifier +from sklearn.utils._testing import ( + assert_allclose, + assert_allclose_dense_sparse, + ignore_warnings, +) +from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS + +diabetes = load_diabetes() +X_diabetes, y_diabetes = diabetes.data, diabetes.target +iris = load_iris() +X_iris, y_iris = iris.data, iris.target +X_multilabel, y_multilabel = make_multilabel_classification( + n_classes=3, random_state=42 +) +X_binary, y_binary = make_classification(n_classes=2, random_state=42) + + +@pytest.mark.parametrize( + "cv", [3, StratifiedKFold(n_splits=3, shuffle=True, random_state=42)] +) +@pytest.mark.parametrize( + "final_estimator", [None, RandomForestClassifier(random_state=42)] +) +@pytest.mark.parametrize("passthrough", [False, True]) +def test_stacking_classifier_iris(cv, final_estimator, passthrough): + # prescale the data to avoid convergence warning without using a pipeline + # for later assert + X_train, X_test, y_train, y_test = train_test_split( + scale(X_iris), y_iris, stratify=y_iris, random_state=42 + ) + estimators = [("lr", LogisticRegression()), ("svc", LinearSVC())] + clf = StackingClassifier( + estimators=estimators, + final_estimator=final_estimator, + cv=cv, + passthrough=passthrough, + ) + clf.fit(X_train, y_train) + clf.predict(X_test) + clf.predict_proba(X_test) + assert clf.score(X_test, y_test) > 0.8 + + X_trans = clf.transform(X_test) + expected_column_count = 10 if passthrough else 6 + assert X_trans.shape[1] == expected_column_count + if passthrough: + assert_allclose(X_test, X_trans[:, -4:]) + + clf.set_params(lr="drop") + clf.fit(X_train, y_train) + clf.predict(X_test) + clf.predict_proba(X_test) + if final_estimator is None: + # LogisticRegression has decision_function method + clf.decision_function(X_test) + + X_trans = clf.transform(X_test) + expected_column_count_drop = 7 if passthrough else 3 + assert X_trans.shape[1] == expected_column_count_drop + if passthrough: + assert_allclose(X_test, X_trans[:, -4:]) + + +def test_stacking_classifier_drop_column_binary_classification(): + # check that a column is dropped in binary classification + X, y = load_breast_cancer(return_X_y=True) + X_train, X_test, y_train, _ = train_test_split( + scale(X), y, stratify=y, random_state=42 + ) + + # both classifiers implement 'predict_proba' and will both drop one column + estimators = [ + ("lr", LogisticRegression()), + ("rf", RandomForestClassifier(random_state=42)), + ] + clf = StackingClassifier(estimators=estimators, cv=3) + + clf.fit(X_train, y_train) + X_trans = clf.transform(X_test) + assert X_trans.shape[1] == 2 + + # LinearSVC does not implement 'predict_proba' and will not drop one column + estimators = [("lr", LogisticRegression()), ("svc", LinearSVC())] + clf.set_params(estimators=estimators) + + clf.fit(X_train, y_train) + X_trans = clf.transform(X_test) + assert X_trans.shape[1] == 2 + + +def test_stacking_classifier_drop_estimator(): + # prescale the data to avoid convergence warning without using a pipeline + # for later assert + X_train, X_test, y_train, _ = train_test_split( + scale(X_iris), y_iris, stratify=y_iris, random_state=42 + ) + estimators = [("lr", "drop"), ("svc", LinearSVC(random_state=0))] + rf = RandomForestClassifier(n_estimators=10, random_state=42) + clf = StackingClassifier( + estimators=[("svc", LinearSVC(random_state=0))], + final_estimator=rf, + cv=5, + ) + clf_drop = StackingClassifier(estimators=estimators, final_estimator=rf, cv=5) + + clf.fit(X_train, y_train) + clf_drop.fit(X_train, y_train) + assert_allclose(clf.predict(X_test), clf_drop.predict(X_test)) + assert_allclose(clf.predict_proba(X_test), clf_drop.predict_proba(X_test)) + assert_allclose(clf.transform(X_test), clf_drop.transform(X_test)) + + +def test_stacking_regressor_drop_estimator(): + # prescale the data to avoid convergence warning without using a pipeline + # for later assert + X_train, X_test, y_train, _ = train_test_split( + scale(X_diabetes), y_diabetes, random_state=42 + ) + estimators = [("lr", "drop"), ("ridge", Ridge(alpha=1.0))] + rf = RandomForestRegressor(n_estimators=10, random_state=42) + reg = StackingRegressor( + estimators=[("ridge", Ridge(alpha=1.0))], + final_estimator=rf, + cv=5, + ) + reg_drop = StackingRegressor(estimators=estimators, final_estimator=rf, cv=5) + + reg.fit(X_train, y_train) + reg_drop.fit(X_train, y_train) + assert_allclose(reg.predict(X_test), reg_drop.predict(X_test)) + assert_allclose(reg.transform(X_test), reg_drop.transform(X_test)) + + +@pytest.mark.parametrize("cv", [3, KFold(n_splits=3, shuffle=True, random_state=42)]) +@pytest.mark.parametrize( + "final_estimator, predict_params", + [ + (None, {}), + (RandomForestRegressor(random_state=42), {}), + (DummyRegressor(), {"return_std": True}), + ], +) +@pytest.mark.parametrize("passthrough", [False, True]) +def test_stacking_regressor_diabetes(cv, final_estimator, predict_params, passthrough): + # prescale the data to avoid convergence warning without using a pipeline + # for later assert + X_train, X_test, y_train, _ = train_test_split( + scale(X_diabetes), y_diabetes, random_state=42 + ) + estimators = [("lr", LinearRegression()), ("svr", LinearSVR())] + reg = StackingRegressor( + estimators=estimators, + final_estimator=final_estimator, + cv=cv, + passthrough=passthrough, + ) + reg.fit(X_train, y_train) + result = reg.predict(X_test, **predict_params) + expected_result_length = 2 if predict_params else 1 + if predict_params: + assert len(result) == expected_result_length + + X_trans = reg.transform(X_test) + expected_column_count = 12 if passthrough else 2 + assert X_trans.shape[1] == expected_column_count + if passthrough: + assert_allclose(X_test, X_trans[:, -10:]) + + reg.set_params(lr="drop") + reg.fit(X_train, y_train) + reg.predict(X_test) + + X_trans = reg.transform(X_test) + expected_column_count_drop = 11 if passthrough else 1 + assert X_trans.shape[1] == expected_column_count_drop + if passthrough: + assert_allclose(X_test, X_trans[:, -10:]) + + +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_stacking_regressor_sparse_passthrough(sparse_container): + # Check passthrough behavior on a sparse X matrix + X_train, X_test, y_train, _ = train_test_split( + sparse_container(scale(X_diabetes)), y_diabetes, random_state=42 + ) + estimators = [("lr", LinearRegression()), ("svr", LinearSVR())] + rf = RandomForestRegressor(n_estimators=10, random_state=42) + clf = StackingRegressor( + estimators=estimators, final_estimator=rf, cv=5, passthrough=True + ) + clf.fit(X_train, y_train) + X_trans = clf.transform(X_test) + assert_allclose_dense_sparse(X_test, X_trans[:, -10:]) + assert sparse.issparse(X_trans) + assert X_test.format == X_trans.format + + +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_stacking_classifier_sparse_passthrough(sparse_container): + # Check passthrough behavior on a sparse X matrix + X_train, X_test, y_train, _ = train_test_split( + sparse_container(scale(X_iris)), y_iris, random_state=42 + ) + estimators = [("lr", LogisticRegression()), ("svc", LinearSVC())] + rf = RandomForestClassifier(n_estimators=10, random_state=42) + clf = StackingClassifier( + estimators=estimators, final_estimator=rf, cv=5, passthrough=True + ) + clf.fit(X_train, y_train) + X_trans = clf.transform(X_test) + assert_allclose_dense_sparse(X_test, X_trans[:, -4:]) + assert sparse.issparse(X_trans) + assert X_test.format == X_trans.format + + +def test_stacking_classifier_drop_binary_prob(): + # check that classifier will drop one of the probability column for + # binary classification problem + + # Select only the 2 first classes + X_, y_ = scale(X_iris[:100]), y_iris[:100] + + estimators = [("lr", LogisticRegression()), ("rf", RandomForestClassifier())] + clf = StackingClassifier(estimators=estimators) + clf.fit(X_, y_) + X_meta = clf.transform(X_) + assert X_meta.shape[1] == 2 + + +class NoWeightRegressor(RegressorMixin, BaseEstimator): + def fit(self, X, y): + self.reg = DummyRegressor() + return self.reg.fit(X, y) + + def predict(self, X): + return np.ones(X.shape[0]) + + +class NoWeightClassifier(ClassifierMixin, BaseEstimator): + def fit(self, X, y): + self.clf = DummyClassifier(strategy="stratified") + return self.clf.fit(X, y) + + +@pytest.mark.parametrize( + "y, params, type_err, msg_err", + [ + (y_iris, {"estimators": []}, ValueError, "Invalid 'estimators' attribute,"), + ( + y_iris, + { + "estimators": [ + ("lr", LogisticRegression()), + ("svm", SVC(max_iter=50_000)), + ], + "stack_method": "predict_proba", + }, + ValueError, + "does not implement the method predict_proba", + ), + ( + y_iris, + { + "estimators": [ + ("lr", LogisticRegression()), + ("cor", NoWeightClassifier()), + ] + }, + TypeError, + "does not support sample weight", + ), + ( + y_iris, + { + "estimators": [ + ("lr", LogisticRegression()), + ("cor", LinearSVC(max_iter=50_000)), + ], + "final_estimator": NoWeightClassifier(), + }, + TypeError, + "does not support sample weight", + ), + ], +) +def test_stacking_classifier_error(y, params, type_err, msg_err): + with pytest.raises(type_err, match=msg_err): + clf = StackingClassifier(**params, cv=3) + clf.fit(scale(X_iris), y, sample_weight=np.ones(X_iris.shape[0])) + + +@pytest.mark.parametrize( + "y, params, type_err, msg_err", + [ + (y_diabetes, {"estimators": []}, ValueError, "Invalid 'estimators' attribute,"), + ( + y_diabetes, + {"estimators": [("lr", LinearRegression()), ("cor", NoWeightRegressor())]}, + TypeError, + "does not support sample weight", + ), + ( + y_diabetes, + { + "estimators": [ + ("lr", LinearRegression()), + ("cor", LinearSVR()), + ], + "final_estimator": NoWeightRegressor(), + }, + TypeError, + "does not support sample weight", + ), + ], +) +def test_stacking_regressor_error(y, params, type_err, msg_err): + with pytest.raises(type_err, match=msg_err): + reg = StackingRegressor(**params, cv=3) + reg.fit(scale(X_diabetes), y, sample_weight=np.ones(X_diabetes.shape[0])) + + +@pytest.mark.parametrize( + "estimator, X, y", + [ + ( + StackingClassifier( + estimators=[ + ("first", LogisticRegression(random_state=0)), + ("second", LinearSVC(random_state=0)), + ] + ), + X_iris[:100], + y_iris[:100], + ), # keep only classes 0 and 1 + ( + StackingRegressor( + estimators=[ + ("first", Ridge(alpha=1.0)), + ("second", Ridge(alpha=1e-6)), + ] + ), + X_diabetes, + y_diabetes, + ), + ], + ids=["StackingClassifier", "StackingRegressor"], +) +def test_stacking_randomness(estimator, X, y): + # checking that fixing the random state of the CV will lead to the same + # results + estimator_full = clone(estimator) + estimator_full.set_params( + cv=KFold(shuffle=True, random_state=np.random.RandomState(0)) + ) + + estimator_drop = clone(estimator) + estimator_drop.set_params(first="drop") + estimator_drop.set_params( + cv=KFold(shuffle=True, random_state=np.random.RandomState(0)) + ) + + assert_allclose( + estimator_full.fit(X, y).transform(X)[:, 1:], + estimator_drop.fit(X, y).transform(X), + ) + + +def test_stacking_classifier_stratify_default(): + # check that we stratify the classes for the default CV + clf = StackingClassifier( + estimators=[ + ("lr", LogisticRegression(max_iter=10_000)), + ("svm", LinearSVC(max_iter=10_000)), + ] + ) + # since iris is not shuffled, a simple k-fold would not contain the + # 3 classes during training + clf.fit(X_iris, y_iris) + + +@pytest.mark.parametrize( + "stacker, X, y", + [ + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC(random_state=42)), + ], + final_estimator=LogisticRegression(), + cv=KFold(shuffle=True, random_state=42), + ), + *load_breast_cancer(return_X_y=True), + ), + ( + StackingRegressor( + estimators=[ + ("first", Ridge(alpha=1.0)), + ("second", Ridge(alpha=1e-6)), + ], + final_estimator=LinearRegression(), + cv=KFold(shuffle=True, random_state=42), + ), + X_diabetes, + y_diabetes, + ), + ], + ids=["StackingClassifier", "StackingRegressor"], +) +def test_stacking_with_sample_weight(stacker, X, y): + # check that sample weights has an influence on the fitting + # note: ConvergenceWarning are catch since we are not worrying about the + # convergence here + n_half_samples = len(y) // 2 + total_sample_weight = np.array( + [0.1] * n_half_samples + [0.9] * (len(y) - n_half_samples) + ) + X_train, X_test, y_train, _, sample_weight_train, _ = train_test_split( + X, y, total_sample_weight, random_state=42 + ) + + stacker = clone(stacker) + with ignore_warnings(category=ConvergenceWarning): + stacker.fit(X_train, y_train) + y_pred_no_weight = stacker.predict(X_test) + + with ignore_warnings(category=ConvergenceWarning): + stacker.fit(X_train, y_train, sample_weight=np.ones(y_train.shape)) + y_pred_unit_weight = stacker.predict(X_test) + + assert_allclose(y_pred_no_weight, y_pred_unit_weight) + + with ignore_warnings(category=ConvergenceWarning): + stacker.fit(X_train, y_train, sample_weight=sample_weight_train) + y_pred_biased = stacker.predict(X_test) + + assert np.abs(y_pred_no_weight - y_pred_biased).sum() > 0 + + +def test_stacking_classifier_sample_weight_fit_param(): + # check sample_weight is passed to all invocations of fit + stacker = StackingClassifier( + estimators=[("lr", CheckingClassifier(expected_sample_weight=True))], + final_estimator=CheckingClassifier(expected_sample_weight=True), + ) + stacker.fit(X_iris, y_iris, sample_weight=np.ones(X_iris.shape[0])) + + +@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning") +@pytest.mark.parametrize( + "stacker, X, y", + [ + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC(random_state=42)), + ], + final_estimator=LogisticRegression(), + ), + *load_breast_cancer(return_X_y=True), + ), + ( + StackingRegressor( + estimators=[ + ("ridge1", Ridge(alpha=1.0)), + ("ridge2", Ridge(alpha=1e-6)), + ], + final_estimator=LinearRegression(), + ), + X_diabetes, + y_diabetes, + ), + ], + ids=["StackingClassifier", "StackingRegressor"], +) +def test_stacking_cv_influence(stacker, X, y): + # check that the stacking affects the fit of the final estimator but not + # the fit of the base estimators + # note: ConvergenceWarning are caught since we are not worrying about the + # convergence here + stacker_cv_3 = clone(stacker) + stacker_cv_5 = clone(stacker) + + stacker_cv_3.set_params(cv=3) + stacker_cv_5.set_params(cv=5) + + stacker_cv_3.fit(X, y) + stacker_cv_5.fit(X, y) + + # the base estimators should be identical + for est_cv_3, est_cv_5 in zip(stacker_cv_3.estimators_, stacker_cv_5.estimators_): + assert_allclose(est_cv_3.coef_, est_cv_5.coef_) + + # the final estimator should be different + with pytest.raises(AssertionError, match="Not equal"): + assert_allclose( + stacker_cv_3.final_estimator_.coef_, stacker_cv_5.final_estimator_.coef_ + ) + + +@pytest.mark.parametrize( + "Stacker, Estimator, stack_method, final_estimator, X, y", + [ + ( + StackingClassifier, + DummyClassifier, + "predict_proba", + LogisticRegression(random_state=42), + X_iris, + y_iris, + ), + ( + StackingRegressor, + DummyRegressor, + "predict", + LinearRegression(), + X_diabetes, + y_diabetes, + ), + ], +) +def test_stacking_prefit(Stacker, Estimator, stack_method, final_estimator, X, y): + """Check the behaviour of stacking when `cv='prefit'`""" + X_train1, X_train2, y_train1, y_train2 = train_test_split( + X, y, random_state=42, test_size=0.5 + ) + estimators = [ + ("d0", Estimator().fit(X_train1, y_train1)), + ("d1", Estimator().fit(X_train1, y_train1)), + ] + + # mock out fit and stack_method to be asserted later + for _, estimator in estimators: + estimator.fit = Mock(name="fit") + stack_func = getattr(estimator, stack_method) + predict_method_mocked = Mock(side_effect=stack_func) + # Mocking a method will not provide a `__name__` while Python methods + # do and we are using it in `_get_response_method`. + predict_method_mocked.__name__ = stack_method + setattr(estimator, stack_method, predict_method_mocked) + + stacker = Stacker( + estimators=estimators, cv="prefit", final_estimator=final_estimator + ) + stacker.fit(X_train2, y_train2) + + assert stacker.estimators_ == [estimator for _, estimator in estimators] + # fit was not called again + assert all(estimator.fit.call_count == 0 for estimator in stacker.estimators_) + + # stack method is called with the proper inputs + for estimator in stacker.estimators_: + stack_func_mock = getattr(estimator, stack_method) + stack_func_mock.assert_called_with(X_train2) + + +@pytest.mark.parametrize( + "stacker, X, y", + [ + ( + StackingClassifier( + estimators=[("lr", LogisticRegression()), ("svm", SVC())], + cv="prefit", + ), + X_iris, + y_iris, + ), + ( + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR()), + ], + cv="prefit", + ), + X_diabetes, + y_diabetes, + ), + ], +) +def test_stacking_prefit_error(stacker, X, y): + # check that NotFittedError is raised + # if base estimators are not fitted when cv="prefit" + with pytest.raises(NotFittedError): + stacker.fit(X, y) + + +@pytest.mark.parametrize( + "make_dataset, Stacking, Estimator", + [ + (make_classification, StackingClassifier, LogisticRegression), + (make_regression, StackingRegressor, LinearRegression), + ], +) +def test_stacking_without_n_features_in(make_dataset, Stacking, Estimator): + # Stacking supports estimators without `n_features_in_`. Regression test + # for #17353 + + class MyEstimator(Estimator): + """Estimator without n_features_in_""" + + def fit(self, X, y): + super().fit(X, y) + del self.n_features_in_ + + X, y = make_dataset(random_state=0, n_samples=100) + stacker = Stacking(estimators=[("lr", MyEstimator())]) + + msg = f"{Stacking.__name__} object has no attribute n_features_in_" + with pytest.raises(AttributeError, match=msg): + stacker.n_features_in_ + + # Does not raise + stacker.fit(X, y) + + msg = "'MyEstimator' object has no attribute 'n_features_in_'" + with pytest.raises(AttributeError, match=msg): + stacker.n_features_in_ + + +@pytest.mark.parametrize( + "estimator", + [ + # output a 2D array of the probability of the positive class for each output + MLPClassifier(random_state=42), + # output a list of 2D array containing the probability of each class + # for each output + RandomForestClassifier(random_state=42), + ], + ids=["MLPClassifier", "RandomForestClassifier"], +) +def test_stacking_classifier_multilabel_predict_proba(estimator): + """Check the behaviour for the multilabel classification case and the + `predict_proba` stacking method. + + Estimators are not consistent with the output arrays and we need to ensure that + we handle all cases. + """ + X_train, X_test, y_train, y_test = train_test_split( + X_multilabel, y_multilabel, stratify=y_multilabel, random_state=42 + ) + n_outputs = 3 + + estimators = [("est", estimator)] + stacker = StackingClassifier( + estimators=estimators, + final_estimator=KNeighborsClassifier(), + stack_method="predict_proba", + ).fit(X_train, y_train) + + X_trans = stacker.transform(X_test) + assert X_trans.shape == (X_test.shape[0], n_outputs) + # we should not have any collinear classes and thus nothing should sum to 1 + assert not any(np.isclose(X_trans.sum(axis=1), 1.0)) + + y_pred = stacker.predict(X_test) + assert y_pred.shape == y_test.shape + + +def test_stacking_classifier_multilabel_decision_function(): + """Check the behaviour for the multilabel classification case and the + `decision_function` stacking method. Only `RidgeClassifier` supports this + case. + """ + X_train, X_test, y_train, y_test = train_test_split( + X_multilabel, y_multilabel, stratify=y_multilabel, random_state=42 + ) + n_outputs = 3 + + estimators = [("est", RidgeClassifier())] + stacker = StackingClassifier( + estimators=estimators, + final_estimator=KNeighborsClassifier(), + stack_method="decision_function", + ).fit(X_train, y_train) + + X_trans = stacker.transform(X_test) + assert X_trans.shape == (X_test.shape[0], n_outputs) + + y_pred = stacker.predict(X_test) + assert y_pred.shape == y_test.shape + + +@pytest.mark.parametrize("stack_method", ["auto", "predict"]) +@pytest.mark.parametrize("passthrough", [False, True]) +def test_stacking_classifier_multilabel_auto_predict(stack_method, passthrough): + """Check the behaviour for the multilabel classification case for stack methods + supported for all estimators or automatically picked up. + """ + X_train, X_test, y_train, y_test = train_test_split( + X_multilabel, y_multilabel, stratify=y_multilabel, random_state=42 + ) + y_train_before_fit = y_train.copy() + n_outputs = 3 + + estimators = [ + ("mlp", MLPClassifier(random_state=42)), + ("rf", RandomForestClassifier(random_state=42)), + ("ridge", RidgeClassifier()), + ] + final_estimator = KNeighborsClassifier() + + clf = StackingClassifier( + estimators=estimators, + final_estimator=final_estimator, + passthrough=passthrough, + stack_method=stack_method, + ).fit(X_train, y_train) + + # make sure we don't change `y_train` inplace + assert_array_equal(y_train_before_fit, y_train) + + y_pred = clf.predict(X_test) + assert y_pred.shape == y_test.shape + + if stack_method == "auto": + expected_stack_methods = ["predict_proba", "predict_proba", "decision_function"] + else: + expected_stack_methods = ["predict"] * len(estimators) + assert clf.stack_method_ == expected_stack_methods + + n_features_X_trans = n_outputs * len(estimators) + if passthrough: + n_features_X_trans += X_train.shape[1] + X_trans = clf.transform(X_test) + assert X_trans.shape == (X_test.shape[0], n_features_X_trans) + + assert_array_equal(clf.classes_, [np.array([0, 1])] * n_outputs) + + +@pytest.mark.parametrize( + "stacker, feature_names, X, y, expected_names", + [ + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("svm", LinearSVC(random_state=0)), + ] + ), + iris.feature_names, + X_iris, + y_iris, + [ + "stackingclassifier_lr0", + "stackingclassifier_lr1", + "stackingclassifier_lr2", + "stackingclassifier_svm0", + "stackingclassifier_svm1", + "stackingclassifier_svm2", + ], + ), + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("other", "drop"), + ("svm", LinearSVC(random_state=0)), + ] + ), + iris.feature_names, + X_iris[:100], + y_iris[:100], # keep only classes 0 and 1 + [ + "stackingclassifier_lr", + "stackingclassifier_svm", + ], + ), + ( + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR(random_state=0)), + ] + ), + diabetes.feature_names, + X_diabetes, + y_diabetes, + [ + "stackingregressor_lr", + "stackingregressor_svm", + ], + ), + ], + ids=[ + "StackingClassifier_multiclass", + "StackingClassifier_binary", + "StackingRegressor", + ], +) +@pytest.mark.parametrize("passthrough", [True, False]) +def test_get_feature_names_out( + stacker, feature_names, X, y, expected_names, passthrough +): + """Check get_feature_names_out works for stacking.""" + stacker = clone(stacker) + stacker.set_params(passthrough=passthrough) + stacker.fit(scale(X), y) + + if passthrough: + expected_names = np.concatenate((expected_names, feature_names)) + + names_out = stacker.get_feature_names_out(feature_names) + assert_array_equal(names_out, expected_names) + + +def test_stacking_classifier_base_regressor(): + """Check that a regressor can be used as the first layer in `StackingClassifier`.""" + X_train, X_test, y_train, y_test = train_test_split( + scale(X_iris), y_iris, stratify=y_iris, random_state=42 + ) + clf = StackingClassifier(estimators=[("ridge", Ridge())]) + clf.fit(X_train, y_train) + clf.predict(X_test) + clf.predict_proba(X_test) + assert clf.score(X_test, y_test) > 0.8 + + +def test_stacking_final_estimator_attribute_error(): + """Check that we raise the proper AttributeError when the final estimator + does not implement the `decision_function` method, which is decorated with + `available_if`. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/28108 + """ + X, y = make_classification(random_state=42) + + estimators = [ + ("lr", LogisticRegression()), + ("rf", RandomForestClassifier(n_estimators=2, random_state=42)), + ] + # RandomForestClassifier does not implement 'decision_function' and should raise + # an AttributeError + final_estimator = RandomForestClassifier(n_estimators=2, random_state=42) + clf = StackingClassifier( + estimators=estimators, final_estimator=final_estimator, cv=3 + ) + + outer_msg = "This 'StackingClassifier' has no attribute 'decision_function'" + inner_msg = "'RandomForestClassifier' object has no attribute 'decision_function'" + with pytest.raises(AttributeError, match=outer_msg) as exec_info: + clf.fit(X, y).decision_function(X) + assert isinstance(exec_info.value.__cause__, AttributeError) + assert inner_msg in str(exec_info.value.__cause__) + + +# Metadata Routing Tests +# ====================== + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +def test_routing_passed_metadata_not_supported(Estimator, Child): + """Test that the right error message is raised when metadata is passed while + not supported when `enable_metadata_routing=False`.""" + + with pytest.raises( + ValueError, match="is only supported if enable_metadata_routing=True" + ): + Estimator(["clf", Child()]).fit( + X_iris, y_iris, sample_weight=[1, 1, 1, 1, 1], metadata="a" + ) + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +@config_context(enable_metadata_routing=True) +def test_get_metadata_routing_without_fit(Estimator, Child): + # Test that metadata_routing() doesn't raise when called before fit. + est = Estimator([("sub_est", Child())]) + est.get_metadata_routing() + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +@pytest.mark.parametrize( + "prop, prop_value", [("sample_weight", np.ones(X_iris.shape[0])), ("metadata", "a")] +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_for_stacking_estimators(Estimator, Child, prop, prop_value): + """Test that metadata is routed correctly for Stacking*.""" + + est = Estimator( + [ + ( + "sub_est1", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ( + "sub_est2", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ], + final_estimator=Child(registry=_Registry()).set_predict_request(**{prop: True}), + ) + + est.fit(X_iris, y_iris, **{prop: prop_value}) + est.fit_transform(X_iris, y_iris, **{prop: prop_value}) + + est.predict(X_iris, **{prop: prop_value}) + + for estimator in est.estimators: + # access sub-estimator in (name, est) with estimator[1]: + registry = estimator[1].registry + assert len(registry) + for sub_est in registry: + check_recorded_metadata( + obj=sub_est, + method="fit", + parent="fit", + split_params=(prop), + **{prop: prop_value}, + ) + # access final_estimator: + registry = est.final_estimator_.registry + assert len(registry) + check_recorded_metadata( + obj=registry[-1], + method="predict", + parent="predict", + split_params=(prop), + **{prop: prop_value}, + ) + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_error_for_stacking_estimators(Estimator, Child): + """Test that the right error is raised when metadata is not requested.""" + sample_weight, metadata = np.ones(X_iris.shape[0]), "a" + + est = Estimator([("sub_est", Child())]) + + error_message = ( + "[sample_weight, metadata] are passed but are not explicitly set as requested" + f" or not requested for {Child.__name__}.fit" + ) + + with pytest.raises(ValueError, match=re.escape(error_message)): + est.fit(X_iris, y_iris, sample_weight=sample_weight, metadata=metadata) + + +# End of Metadata Routing Tests +# ============================= diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_voting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_voting.py new file mode 100644 index 0000000000000000000000000000000000000000..36cdde002aed5d228f16d3cc252d011809c085c8 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_voting.py @@ -0,0 +1,795 @@ +"""Testing for the VotingClassifier and VotingRegressor""" + +import re + +import numpy as np +import pytest + +from sklearn import config_context, datasets +from sklearn.base import BaseEstimator, ClassifierMixin, clone +from sklearn.calibration import CalibratedClassifierCV +from sklearn.datasets import make_multilabel_classification +from sklearn.dummy import DummyRegressor +from sklearn.ensemble import ( + RandomForestClassifier, + RandomForestRegressor, + VotingClassifier, + VotingRegressor, +) +from sklearn.exceptions import NotFittedError +from sklearn.linear_model import LinearRegression, LogisticRegression +from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split +from sklearn.multiclass import OneVsRestClassifier +from sklearn.naive_bayes import GaussianNB +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import StandardScaler +from sklearn.svm import SVC +from sklearn.tests.metadata_routing_common import ( + ConsumingClassifier, + ConsumingRegressor, + _Registry, + check_recorded_metadata, +) +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils._testing import ( + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) + +# Load datasets +iris = datasets.load_iris() +X, y = iris.data[:, 1:3], iris.target +# Scaled to solve ConvergenceWarning throw by Logistic Regression +X_scaled = StandardScaler().fit_transform(X) + +X_r, y_r = datasets.load_diabetes(return_X_y=True) + + +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"estimators": []}, + "Invalid 'estimators' attribute, 'estimators' should be a non-empty list", + ), + ( + {"estimators": [LogisticRegression()]}, + "Invalid 'estimators' attribute, 'estimators' should be a non-empty list", + ), + ( + {"estimators": [(213, LogisticRegression())]}, + "Invalid 'estimators' attribute, 'estimators' should be a non-empty list", + ), + ( + {"estimators": [("lr", LogisticRegression())], "weights": [1, 2]}, + "Number of `estimators` and weights must be equal", + ), + ], +) +def test_voting_classifier_estimator_init(params, err_msg): + ensemble = VotingClassifier(**params) + with pytest.raises(ValueError, match=err_msg): + ensemble.fit(X, y) + + +def test_predictproba_hardvoting(): + eclf = VotingClassifier( + estimators=[("lr1", LogisticRegression()), ("lr2", LogisticRegression())], + voting="hard", + ) + + inner_msg = "predict_proba is not available when voting='hard'" + outer_msg = "'VotingClassifier' has no attribute 'predict_proba'" + with pytest.raises(AttributeError, match=outer_msg) as exec_info: + eclf.predict_proba + assert isinstance(exec_info.value.__cause__, AttributeError) + assert inner_msg in str(exec_info.value.__cause__) + + assert not hasattr(eclf, "predict_proba") + eclf.fit(X_scaled, y) + assert not hasattr(eclf, "predict_proba") + + +def test_notfitted(): + eclf = VotingClassifier( + estimators=[("lr1", LogisticRegression()), ("lr2", LogisticRegression())], + voting="soft", + ) + ereg = VotingRegressor([("dr", DummyRegressor())]) + msg = ( + "This %s instance is not fitted yet. Call 'fit'" + " with appropriate arguments before using this estimator." + ) + with pytest.raises(NotFittedError, match=msg % "VotingClassifier"): + eclf.predict(X) + with pytest.raises(NotFittedError, match=msg % "VotingClassifier"): + eclf.predict_proba(X) + with pytest.raises(NotFittedError, match=msg % "VotingClassifier"): + eclf.transform(X) + with pytest.raises(NotFittedError, match=msg % "VotingRegressor"): + ereg.predict(X_r) + with pytest.raises(NotFittedError, match=msg % "VotingRegressor"): + ereg.transform(X_r) + + +def test_majority_label_iris(global_random_seed): + """Check classification by majority label on dataset iris.""" + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + clf3 = GaussianNB() + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="hard" + ) + scores = cross_val_score(eclf, X, y, scoring="accuracy") + + assert scores.mean() >= 0.9 + + +def test_tie_situation(): + """Check voting classifier selects smaller class label in tie situation.""" + clf1 = LogisticRegression(random_state=123) + clf2 = RandomForestClassifier(random_state=123) + eclf = VotingClassifier(estimators=[("lr", clf1), ("rf", clf2)], voting="hard") + assert clf1.fit(X, y).predict(X)[52] == 2 + assert clf2.fit(X, y).predict(X)[52] == 1 + assert eclf.fit(X, y).predict(X)[52] == 1 + + +def test_weights_iris(global_random_seed): + """Check classification by average probabilities on dataset iris.""" + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + clf3 = GaussianNB() + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + weights=[1, 2, 10], + ) + scores = cross_val_score(eclf, X_scaled, y, scoring="accuracy") + assert scores.mean() >= 0.9 + + +def test_weights_regressor(): + """Check weighted average regression prediction on diabetes dataset.""" + reg1 = DummyRegressor(strategy="mean") + reg2 = DummyRegressor(strategy="median") + reg3 = DummyRegressor(strategy="quantile", quantile=0.2) + ereg = VotingRegressor( + [("mean", reg1), ("median", reg2), ("quantile", reg3)], weights=[1, 2, 10] + ) + + X_r_train, X_r_test, y_r_train, y_r_test = train_test_split( + X_r, y_r, test_size=0.25 + ) + + reg1_pred = reg1.fit(X_r_train, y_r_train).predict(X_r_test) + reg2_pred = reg2.fit(X_r_train, y_r_train).predict(X_r_test) + reg3_pred = reg3.fit(X_r_train, y_r_train).predict(X_r_test) + ereg_pred = ereg.fit(X_r_train, y_r_train).predict(X_r_test) + + avg = np.average( + np.asarray([reg1_pred, reg2_pred, reg3_pred]), axis=0, weights=[1, 2, 10] + ) + assert_almost_equal(ereg_pred, avg, decimal=2) + + ereg_weights_none = VotingRegressor( + [("mean", reg1), ("median", reg2), ("quantile", reg3)], weights=None + ) + ereg_weights_equal = VotingRegressor( + [("mean", reg1), ("median", reg2), ("quantile", reg3)], weights=[1, 1, 1] + ) + ereg_weights_none.fit(X_r_train, y_r_train) + ereg_weights_equal.fit(X_r_train, y_r_train) + ereg_none_pred = ereg_weights_none.predict(X_r_test) + ereg_equal_pred = ereg_weights_equal.predict(X_r_test) + assert_almost_equal(ereg_none_pred, ereg_equal_pred, decimal=2) + + +def test_predict_on_toy_problem(global_random_seed): + """Manually check predicted class labels for toy dataset.""" + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + clf3 = GaussianNB() + + X = np.array( + [[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2], [2.1, 1.4], [3.1, 2.3]] + ) + + y = np.array([1, 1, 1, 2, 2, 2]) + + assert_array_equal(clf1.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) + assert_array_equal(clf2.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) + assert_array_equal(clf3.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) + + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="hard", + weights=[1, 1, 1], + ) + assert_array_equal(eclf.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) + + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + weights=[1, 1, 1], + ) + assert_array_equal(eclf.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) + + +def test_predict_proba_on_toy_problem(): + """Calculate predicted probabilities on toy dataset.""" + clf1 = LogisticRegression(random_state=123) + clf2 = RandomForestClassifier(random_state=123) + clf3 = GaussianNB() + X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) + y = np.array([1, 1, 2, 2]) + + clf1_res = np.array( + [ + [0.59790391, 0.40209609], + [0.57622162, 0.42377838], + [0.50728456, 0.49271544], + [0.40241774, 0.59758226], + ] + ) + + clf2_res = np.array([[0.8, 0.2], [0.8, 0.2], [0.2, 0.8], [0.3, 0.7]]) + + clf3_res = np.array( + [[0.9985082, 0.0014918], [0.99845843, 0.00154157], [0.0, 1.0], [0.0, 1.0]] + ) + + t00 = (2 * clf1_res[0][0] + clf2_res[0][0] + clf3_res[0][0]) / 4 + t11 = (2 * clf1_res[1][1] + clf2_res[1][1] + clf3_res[1][1]) / 4 + t21 = (2 * clf1_res[2][1] + clf2_res[2][1] + clf3_res[2][1]) / 4 + t31 = (2 * clf1_res[3][1] + clf2_res[3][1] + clf3_res[3][1]) / 4 + + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + weights=[2, 1, 1], + ) + eclf_res = eclf.fit(X, y).predict_proba(X) + + assert_almost_equal(t00, eclf_res[0][0], decimal=1) + assert_almost_equal(t11, eclf_res[1][1], decimal=1) + assert_almost_equal(t21, eclf_res[2][1], decimal=1) + assert_almost_equal(t31, eclf_res[3][1], decimal=1) + + inner_msg = "predict_proba is not available when voting='hard'" + outer_msg = "'VotingClassifier' has no attribute 'predict_proba'" + with pytest.raises(AttributeError, match=outer_msg) as exec_info: + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="hard" + ) + eclf.fit(X, y).predict_proba(X) + + assert isinstance(exec_info.value.__cause__, AttributeError) + assert inner_msg in str(exec_info.value.__cause__) + + +def test_multilabel(): + """Check if error is raised for multilabel classification.""" + X, y = make_multilabel_classification( + n_classes=2, n_labels=1, allow_unlabeled=False, random_state=123 + ) + clf = OneVsRestClassifier(SVC(kernel="linear")) + + eclf = VotingClassifier(estimators=[("ovr", clf)], voting="hard") + + try: + eclf.fit(X, y) + except NotImplementedError: + return + + +def test_gridsearch(): + """Check GridSearch support.""" + clf1 = LogisticRegression(random_state=1) + clf2 = RandomForestClassifier(random_state=1, n_estimators=3) + clf3 = GaussianNB() + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft" + ) + + params = { + "lr__C": [1.0, 100.0], + "voting": ["soft", "hard"], + "weights": [[0.5, 0.5, 0.5], [1.0, 0.5, 0.5]], + } + + grid = GridSearchCV(estimator=eclf, param_grid=params, cv=2) + grid.fit(X_scaled, y) + + +def test_parallel_fit(global_random_seed): + """Check parallel backend of VotingClassifier on toy dataset.""" + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + clf3 = GaussianNB() + X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) + y = np.array([1, 1, 2, 2]) + + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft", n_jobs=1 + ).fit(X, y) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft", n_jobs=2 + ).fit(X, y) + + assert_array_equal(eclf1.predict(X), eclf2.predict(X)) + assert_array_almost_equal(eclf1.predict_proba(X), eclf2.predict_proba(X)) + + +def test_sample_weight(global_random_seed): + """Tests sample_weight parameter of VotingClassifier""" + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + clf3 = CalibratedClassifierCV(SVC(random_state=global_random_seed), ensemble=False) + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("svc", clf3)], voting="soft" + ).fit(X_scaled, y, sample_weight=np.ones((len(y),))) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("svc", clf3)], voting="soft" + ).fit(X_scaled, y) + assert_array_equal(eclf1.predict(X_scaled), eclf2.predict(X_scaled)) + assert_array_almost_equal( + eclf1.predict_proba(X_scaled), eclf2.predict_proba(X_scaled) + ) + sample_weight = np.random.RandomState(global_random_seed).uniform(size=(len(y),)) + eclf3 = VotingClassifier(estimators=[("lr", clf1)], voting="soft") + eclf3.fit(X_scaled, y, sample_weight=sample_weight) + clf1.fit(X_scaled, y, sample_weight) + assert_array_equal(eclf3.predict(X_scaled), clf1.predict(X_scaled)) + assert_array_almost_equal( + eclf3.predict_proba(X_scaled), clf1.predict_proba(X_scaled) + ) + + # check that an error is raised and indicative if sample_weight is not + # supported. + clf4 = KNeighborsClassifier() + eclf3 = VotingClassifier( + estimators=[("lr", clf1), ("svc", clf3), ("knn", clf4)], voting="soft" + ) + msg = "Underlying estimator KNeighborsClassifier does not support sample weights." + with pytest.raises(TypeError, match=msg): + eclf3.fit(X_scaled, y, sample_weight=sample_weight) + + # check that _fit_single_estimator will raise the right error + # it should raise the original error if this is not linked to sample_weight + class ClassifierErrorFit(ClassifierMixin, BaseEstimator): + def fit(self, X_scaled, y, sample_weight): + raise TypeError("Error unrelated to sample_weight.") + + clf = ClassifierErrorFit() + with pytest.raises(TypeError, match="Error unrelated to sample_weight"): + clf.fit(X_scaled, y, sample_weight=sample_weight) + + +def test_sample_weight_kwargs(): + """Check that VotingClassifier passes sample_weight as kwargs""" + + class MockClassifier(ClassifierMixin, BaseEstimator): + """Mock Classifier to check that sample_weight is received as kwargs""" + + def fit(self, X, y, *args, **sample_weight): + assert "sample_weight" in sample_weight + + clf = MockClassifier() + eclf = VotingClassifier(estimators=[("mock", clf)], voting="soft") + + # Should not raise an error. + eclf.fit(X, y, sample_weight=np.ones((len(y),))) + + +def test_voting_classifier_set_params(global_random_seed): + # check equivalence in the output when setting underlying estimators + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier( + n_estimators=10, random_state=global_random_seed, max_depth=None + ) + clf3 = GaussianNB() + + eclf1 = VotingClassifier( + [("lr", clf1), ("rf", clf2)], voting="soft", weights=[1, 2] + ).fit(X_scaled, y) + eclf2 = VotingClassifier( + [("lr", clf1), ("nb", clf3)], voting="soft", weights=[1, 2] + ) + eclf2.set_params(nb=clf2).fit(X_scaled, y) + + assert_array_equal(eclf1.predict(X_scaled), eclf2.predict(X_scaled)) + assert_array_almost_equal( + eclf1.predict_proba(X_scaled), eclf2.predict_proba(X_scaled) + ) + assert eclf2.estimators[0][1].get_params() == clf1.get_params() + assert eclf2.estimators[1][1].get_params() == clf2.get_params() + + +def test_set_estimator_drop(): + # VotingClassifier set_params should be able to set estimators as drop + # Test predict + clf1 = LogisticRegression(random_state=123) + clf2 = RandomForestClassifier(n_estimators=10, random_state=123) + clf3 = GaussianNB() + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("nb", clf3)], + voting="hard", + weights=[1, 0, 0.5], + ).fit(X, y) + + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("nb", clf3)], + voting="hard", + weights=[1, 1, 0.5], + ) + eclf2.set_params(rf="drop").fit(X, y) + + assert_array_equal(eclf1.predict(X), eclf2.predict(X)) + + assert dict(eclf2.estimators)["rf"] == "drop" + assert len(eclf2.estimators_) == 2 + assert all( + isinstance(est, (LogisticRegression, GaussianNB)) for est in eclf2.estimators_ + ) + assert eclf2.get_params()["rf"] == "drop" + + eclf1.set_params(voting="soft").fit(X, y) + eclf2.set_params(voting="soft").fit(X, y) + + assert_array_equal(eclf1.predict(X), eclf2.predict(X)) + assert_array_almost_equal(eclf1.predict_proba(X), eclf2.predict_proba(X)) + msg = "All estimators are dropped. At least one is required" + with pytest.raises(ValueError, match=msg): + eclf2.set_params(lr="drop", rf="drop", nb="drop").fit(X, y) + + # Test soft voting transform + X1 = np.array([[1], [2]]) + y1 = np.array([1, 2]) + eclf1 = VotingClassifier( + estimators=[("rf", clf2), ("nb", clf3)], + voting="soft", + weights=[0, 0.5], + flatten_transform=False, + ).fit(X1, y1) + + eclf2 = VotingClassifier( + estimators=[("rf", clf2), ("nb", clf3)], + voting="soft", + weights=[1, 0.5], + flatten_transform=False, + ) + eclf2.set_params(rf="drop").fit(X1, y1) + assert_array_almost_equal( + eclf1.transform(X1), + np.array([[[0.7, 0.3], [0.3, 0.7]], [[1.0, 0.0], [0.0, 1.0]]]), + ) + assert_array_almost_equal(eclf2.transform(X1), np.array([[[1.0, 0.0], [0.0, 1.0]]])) + eclf1.set_params(voting="hard") + eclf2.set_params(voting="hard") + assert_array_equal(eclf1.transform(X1), np.array([[0, 0], [1, 1]])) + assert_array_equal(eclf2.transform(X1), np.array([[0], [1]])) + + +def test_estimator_weights_format(global_random_seed): + # Test estimator weights inputs as list and array + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2)], weights=[1, 2], voting="soft" + ) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2)], weights=np.array((1, 2)), voting="soft" + ) + eclf1.fit(X_scaled, y) + eclf2.fit(X_scaled, y) + assert_array_almost_equal( + eclf1.predict_proba(X_scaled), eclf2.predict_proba(X_scaled) + ) + + +def test_transform(global_random_seed): + """Check transform method of VotingClassifier on toy dataset.""" + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + clf3 = GaussianNB() + X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) + y = np.array([1, 1, 2, 2]) + + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft" + ).fit(X, y) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + flatten_transform=True, + ).fit(X, y) + eclf3 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + flatten_transform=False, + ).fit(X, y) + + assert_array_equal(eclf1.transform(X).shape, (4, 6)) + assert_array_equal(eclf2.transform(X).shape, (4, 6)) + assert_array_equal(eclf3.transform(X).shape, (3, 4, 2)) + assert_array_almost_equal(eclf1.transform(X), eclf2.transform(X)) + assert_array_almost_equal( + eclf3.transform(X).swapaxes(0, 1).reshape((4, 6)), eclf2.transform(X) + ) + + +@pytest.mark.parametrize( + "X, y, voter", + [ + ( + X, + y, + VotingClassifier( + [ + ("lr", LogisticRegression()), + ("rf", RandomForestClassifier(n_estimators=5)), + ] + ), + ), + ( + X_r, + y_r, + VotingRegressor( + [ + ("lr", LinearRegression()), + ("rf", RandomForestRegressor(n_estimators=5)), + ] + ), + ), + ], +) +def test_none_estimator_with_weights(X, y, voter): + # check that an estimator can be set to 'drop' and passing some weight + # regression test for + # https://github.com/scikit-learn/scikit-learn/issues/13777 + voter = clone(voter) + # Scaled to solve ConvergenceWarning throw by Logistic Regression + X_scaled = StandardScaler().fit_transform(X) + voter.fit(X_scaled, y, sample_weight=np.ones(y.shape)) + voter.set_params(lr="drop") + voter.fit(X_scaled, y, sample_weight=np.ones(y.shape)) + y_pred = voter.predict(X_scaled) + assert y_pred.shape == y.shape + + +@pytest.mark.parametrize( + "est", + [ + VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("tree", DecisionTreeRegressor(random_state=0)), + ] + ), + VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("tree", DecisionTreeClassifier(random_state=0)), + ] + ), + ], + ids=["VotingRegressor", "VotingClassifier"], +) +def test_n_features_in(est): + est = clone(est) + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + assert not hasattr(est, "n_features_in_") + est.fit(X, y) + assert est.n_features_in_ == 2 + + +@pytest.mark.parametrize( + "estimator", + [ + VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("rf", RandomForestRegressor(random_state=123)), + ], + verbose=True, + ), + VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=123)), + ("rf", RandomForestClassifier(random_state=123)), + ], + verbose=True, + ), + ], +) +def test_voting_verbose(estimator, capsys): + X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) + y = np.array([1, 1, 2, 2]) + + pattern = ( + r"\[Voting\].*\(1 of 2\) Processing lr, total=.*\n" + r"\[Voting\].*\(2 of 2\) Processing rf, total=.*\n$" + ) + clone(estimator).fit(X, y) + assert re.match(pattern, capsys.readouterr()[0]) + + +def test_get_features_names_out_regressor(): + """Check get_feature_names_out output for regressor.""" + + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + voting = VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("tree", DecisionTreeRegressor(random_state=0)), + ("ignore", "drop"), + ] + ) + voting.fit(X, y) + + names_out = voting.get_feature_names_out() + expected_names = ["votingregressor_lr", "votingregressor_tree"] + assert_array_equal(names_out, expected_names) + + +@pytest.mark.parametrize( + "kwargs, expected_names", + [ + ( + {"voting": "soft", "flatten_transform": True}, + [ + "votingclassifier_lr0", + "votingclassifier_lr1", + "votingclassifier_lr2", + "votingclassifier_tree0", + "votingclassifier_tree1", + "votingclassifier_tree2", + ], + ), + ({"voting": "hard"}, ["votingclassifier_lr", "votingclassifier_tree"]), + ], +) +def test_get_features_names_out_classifier(kwargs, expected_names): + """Check get_feature_names_out for classifier for different settings.""" + X = [[1, 2], [3, 4], [5, 6], [1, 1.2]] + y = [0, 1, 2, 0] + + voting = VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("tree", DecisionTreeClassifier(random_state=0)), + ], + **kwargs, + ) + voting.fit(X, y) + X_trans = voting.transform(X) + names_out = voting.get_feature_names_out() + + assert X_trans.shape[1] == len(expected_names) + assert_array_equal(names_out, expected_names) + + +def test_get_features_names_out_classifier_error(): + """Check that error is raised when voting="soft" and flatten_transform=False.""" + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + voting = VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("tree", DecisionTreeClassifier(random_state=0)), + ], + voting="soft", + flatten_transform=False, + ) + voting.fit(X, y) + + msg = ( + "get_feature_names_out is not supported when `voting='soft'` and " + "`flatten_transform=False`" + ) + with pytest.raises(ValueError, match=msg): + voting.get_feature_names_out() + + +# Metadata Routing Tests +# ====================== + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +def test_routing_passed_metadata_not_supported(Estimator, Child): + """Test that the right error message is raised when metadata is passed while + not supported when `enable_metadata_routing=False`.""" + + X = np.array([[0, 1], [2, 2], [4, 6]]) + y = [1, 2, 3] + + with pytest.raises( + ValueError, match="is only supported if enable_metadata_routing=True" + ): + Estimator(["clf", Child()]).fit(X, y, sample_weight=[1, 1, 1], metadata="a") + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +@config_context(enable_metadata_routing=True) +def test_get_metadata_routing_without_fit(Estimator, Child): + # Test that metadata_routing() doesn't raise when called before fit. + est = Estimator([("sub_est", Child())]) + est.get_metadata_routing() + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +@pytest.mark.parametrize("prop", ["sample_weight", "metadata"]) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_for_voting_estimators(Estimator, Child, prop): + """Test that metadata is routed correctly for Voting*.""" + X = np.array([[0, 1], [2, 2], [4, 6]]) + y = [1, 2, 3] + sample_weight, metadata = [1, 1, 1], "a" + + est = Estimator( + [ + ( + "sub_est1", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ( + "sub_est2", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ] + ) + + est.fit(X, y, **{prop: sample_weight if prop == "sample_weight" else metadata}) + + for estimator in est.estimators: + if prop == "sample_weight": + kwargs = {prop: sample_weight} + else: + kwargs = {prop: metadata} + # access sub-estimator in (name, est) with estimator[1] + registry = estimator[1].registry + assert len(registry) + for sub_est in registry: + check_recorded_metadata(obj=sub_est, method="fit", parent="fit", **kwargs) + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_error_for_voting_estimators(Estimator, Child): + """Test that the right error is raised when metadata is not requested.""" + X = np.array([[0, 1], [2, 2], [4, 6]]) + y = [1, 2, 3] + sample_weight, metadata = [1, 1, 1], "a" + + est = Estimator([("sub_est", Child())]) + + error_message = ( + "[sample_weight, metadata] are passed but are not explicitly set as requested" + f" or not requested for {Child.__name__}.fit" + ) + + with pytest.raises(ValueError, match=re.escape(error_message)): + est.fit(X, y, sample_weight=sample_weight, metadata=metadata) + + +# End of Metadata Routing Tests +# ============================= diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_weight_boosting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_weight_boosting.py new file mode 100644 index 0000000000000000000000000000000000000000..6faa8f78b32afc609181841e84998fb7fb3d0a95 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/tests/test_weight_boosting.py @@ -0,0 +1,602 @@ +"""Testing for the boost module (sklearn.ensemble.boost).""" + +import re + +import numpy as np +import pytest + +from sklearn import datasets +from sklearn.base import BaseEstimator, clone +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble import AdaBoostClassifier, AdaBoostRegressor +from sklearn.linear_model import LinearRegression +from sklearn.model_selection import GridSearchCV, train_test_split +from sklearn.svm import SVC, SVR +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils import shuffle +from sklearn.utils._mocking import NoSampleWeightWrapper +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.fixes import ( + COO_CONTAINERS, + CSC_CONTAINERS, + CSR_CONTAINERS, + DOK_CONTAINERS, + LIL_CONTAINERS, +) + +# Common random state +rng = np.random.RandomState(0) + +# Toy sample +X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] +y_class = ["foo", "foo", "foo", 1, 1, 1] # test string class labels +y_regr = [-1, -1, -1, 1, 1, 1] +T = [[-1, -1], [2, 2], [3, 2]] +y_t_class = ["foo", 1, 1] +y_t_regr = [-1, 1, 1] + +# Load the iris dataset and randomly permute it +iris = datasets.load_iris() +perm = rng.permutation(iris.target.size) +iris.data, iris.target = shuffle(iris.data, iris.target, random_state=rng) + +# Load the diabetes dataset and randomly permute it +diabetes = datasets.load_diabetes() +diabetes.data, diabetes.target = shuffle( + diabetes.data, diabetes.target, random_state=rng +) + + +def test_oneclass_adaboost_proba(): + # Test predict_proba robustness for one class label input. + # In response to issue #7501 + # https://github.com/scikit-learn/scikit-learn/issues/7501 + y_t = np.ones(len(X)) + clf = AdaBoostClassifier().fit(X, y_t) + assert_array_almost_equal(clf.predict_proba(X), np.ones((len(X), 1))) + + +def test_classification_toy(): + # Check classification on a toy dataset. + clf = AdaBoostClassifier(random_state=0) + clf.fit(X, y_class) + assert_array_equal(clf.predict(T), y_t_class) + assert_array_equal(np.unique(np.asarray(y_t_class)), clf.classes_) + assert clf.predict_proba(T).shape == (len(T), 2) + assert clf.decision_function(T).shape == (len(T),) + + +def test_regression_toy(): + # Check classification on a toy dataset. + clf = AdaBoostRegressor(random_state=0) + clf.fit(X, y_regr) + assert_array_equal(clf.predict(T), y_t_regr) + + +def test_iris(): + # Check consistency on dataset iris. + classes = np.unique(iris.target) + + clf = AdaBoostClassifier() + clf.fit(iris.data, iris.target) + + assert_array_equal(classes, clf.classes_) + proba = clf.predict_proba(iris.data) + + assert proba.shape[1] == len(classes) + assert clf.decision_function(iris.data).shape[1] == len(classes) + + score = clf.score(iris.data, iris.target) + assert score > 0.9, f"Failed with {score = }" + + # Check we used multiple estimators + assert len(clf.estimators_) > 1 + # Check for distinct random states (see issue #7408) + assert len(set(est.random_state for est in clf.estimators_)) == len(clf.estimators_) + + +@pytest.mark.parametrize("loss", ["linear", "square", "exponential"]) +def test_diabetes(loss): + # Check consistency on dataset diabetes. + reg = AdaBoostRegressor(loss=loss, random_state=0) + reg.fit(diabetes.data, diabetes.target) + score = reg.score(diabetes.data, diabetes.target) + assert score > 0.55 + + # Check we used multiple estimators + assert len(reg.estimators_) > 1 + # Check for distinct random states (see issue #7408) + assert len(set(est.random_state for est in reg.estimators_)) == len(reg.estimators_) + + +def test_staged_predict(): + # Check staged predictions. + rng = np.random.RandomState(0) + iris_weights = rng.randint(10, size=iris.target.shape) + diabetes_weights = rng.randint(10, size=diabetes.target.shape) + + clf = AdaBoostClassifier(n_estimators=10) + clf.fit(iris.data, iris.target, sample_weight=iris_weights) + + predictions = clf.predict(iris.data) + staged_predictions = [p for p in clf.staged_predict(iris.data)] + proba = clf.predict_proba(iris.data) + staged_probas = [p for p in clf.staged_predict_proba(iris.data)] + score = clf.score(iris.data, iris.target, sample_weight=iris_weights) + staged_scores = [ + s for s in clf.staged_score(iris.data, iris.target, sample_weight=iris_weights) + ] + + assert len(staged_predictions) == 10 + assert_array_almost_equal(predictions, staged_predictions[-1]) + assert len(staged_probas) == 10 + assert_array_almost_equal(proba, staged_probas[-1]) + assert len(staged_scores) == 10 + assert_array_almost_equal(score, staged_scores[-1]) + + # AdaBoost regression + clf = AdaBoostRegressor(n_estimators=10, random_state=0) + clf.fit(diabetes.data, diabetes.target, sample_weight=diabetes_weights) + + predictions = clf.predict(diabetes.data) + staged_predictions = [p for p in clf.staged_predict(diabetes.data)] + score = clf.score(diabetes.data, diabetes.target, sample_weight=diabetes_weights) + staged_scores = [ + s + for s in clf.staged_score( + diabetes.data, diabetes.target, sample_weight=diabetes_weights + ) + ] + + assert len(staged_predictions) == 10 + assert_array_almost_equal(predictions, staged_predictions[-1]) + assert len(staged_scores) == 10 + assert_array_almost_equal(score, staged_scores[-1]) + + +def test_gridsearch(): + # Check that base trees can be grid-searched. + # AdaBoost classification + boost = AdaBoostClassifier(estimator=DecisionTreeClassifier()) + parameters = { + "n_estimators": (1, 2), + "estimator__max_depth": (1, 2), + } + clf = GridSearchCV(boost, parameters) + clf.fit(iris.data, iris.target) + + # AdaBoost regression + boost = AdaBoostRegressor(estimator=DecisionTreeRegressor(), random_state=0) + parameters = {"n_estimators": (1, 2), "estimator__max_depth": (1, 2)} + clf = GridSearchCV(boost, parameters) + clf.fit(diabetes.data, diabetes.target) + + +def test_pickle(): + # Check pickability. + import pickle + + # Adaboost classifier + obj = AdaBoostClassifier() + obj.fit(iris.data, iris.target) + score = obj.score(iris.data, iris.target) + s = pickle.dumps(obj) + + obj2 = pickle.loads(s) + assert type(obj2) == obj.__class__ + score2 = obj2.score(iris.data, iris.target) + assert score == score2 + + # Adaboost regressor + obj = AdaBoostRegressor(random_state=0) + obj.fit(diabetes.data, diabetes.target) + score = obj.score(diabetes.data, diabetes.target) + s = pickle.dumps(obj) + + obj2 = pickle.loads(s) + assert type(obj2) == obj.__class__ + score2 = obj2.score(diabetes.data, diabetes.target) + assert score == score2 + + +def test_importances(): + # Check variable importances. + X, y = datasets.make_classification( + n_samples=2000, + n_features=10, + n_informative=3, + n_redundant=0, + n_repeated=0, + shuffle=False, + random_state=1, + ) + + clf = AdaBoostClassifier() + + clf.fit(X, y) + importances = clf.feature_importances_ + + assert importances.shape[0] == 10 + assert (importances[:3, np.newaxis] >= importances[3:]).all() + + +def test_adaboost_classifier_sample_weight_error(): + # Test that it gives proper exception on incorrect sample weight. + clf = AdaBoostClassifier() + msg = re.escape("sample_weight.shape == (1,), expected (6,)") + with pytest.raises(ValueError, match=msg): + clf.fit(X, y_class, sample_weight=np.asarray([-1])) + + +def test_estimator(): + # Test different estimators. + from sklearn.ensemble import RandomForestClassifier + + # XXX doesn't work with y_class because RF doesn't support classes_ + # Shouldn't AdaBoost run a LabelBinarizer? + clf = AdaBoostClassifier(RandomForestClassifier()) + clf.fit(X, y_regr) + + clf = AdaBoostClassifier(SVC()) + clf.fit(X, y_class) + + from sklearn.ensemble import RandomForestRegressor + + clf = AdaBoostRegressor(RandomForestRegressor(), random_state=0) + clf.fit(X, y_regr) + + clf = AdaBoostRegressor(SVR(), random_state=0) + clf.fit(X, y_regr) + + # Check that an empty discrete ensemble fails in fit, not predict. + X_fail = [[1, 1], [1, 1], [1, 1], [1, 1]] + y_fail = ["foo", "bar", 1, 2] + clf = AdaBoostClassifier(SVC()) + with pytest.raises(ValueError, match="worse than random"): + clf.fit(X_fail, y_fail) + + +def test_sample_weights_infinite(): + msg = "Sample weights have reached infinite values" + clf = AdaBoostClassifier(n_estimators=30, learning_rate=23.0) + with pytest.warns(UserWarning, match=msg): + clf.fit(iris.data, iris.target) + + +@pytest.mark.parametrize( + "sparse_container, expected_internal_type", + zip( + [ + *CSC_CONTAINERS, + *CSR_CONTAINERS, + *LIL_CONTAINERS, + *COO_CONTAINERS, + *DOK_CONTAINERS, + ], + CSC_CONTAINERS + 4 * CSR_CONTAINERS, + ), +) +def test_sparse_classification(sparse_container, expected_internal_type): + # Check classification with sparse input. + + class CustomSVC(SVC): + """SVC variant that records the nature of the training set.""" + + def fit(self, X, y, sample_weight=None): + """Modification on fit caries data type for later verification.""" + super().fit(X, y, sample_weight=sample_weight) + self.data_type_ = type(X) + return self + + X, y = datasets.make_multilabel_classification( + n_classes=1, n_samples=15, n_features=5, random_state=42 + ) + # Flatten y to a 1d array + y = np.ravel(y) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) + + # Trained on sparse format + sparse_classifier = AdaBoostClassifier( + estimator=CustomSVC(probability=True), + random_state=1, + ).fit(X_train_sparse, y_train) + + # Trained on dense format + dense_classifier = AdaBoostClassifier( + estimator=CustomSVC(probability=True), + random_state=1, + ).fit(X_train, y_train) + + # predict + sparse_clf_results = sparse_classifier.predict(X_test_sparse) + dense_clf_results = dense_classifier.predict(X_test) + assert_array_equal(sparse_clf_results, dense_clf_results) + + # decision_function + sparse_clf_results = sparse_classifier.decision_function(X_test_sparse) + dense_clf_results = dense_classifier.decision_function(X_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # predict_log_proba + sparse_clf_results = sparse_classifier.predict_log_proba(X_test_sparse) + dense_clf_results = dense_classifier.predict_log_proba(X_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # predict_proba + sparse_clf_results = sparse_classifier.predict_proba(X_test_sparse) + dense_clf_results = dense_classifier.predict_proba(X_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # score + sparse_clf_results = sparse_classifier.score(X_test_sparse, y_test) + dense_clf_results = dense_classifier.score(X_test, y_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # staged_decision_function + sparse_clf_results = sparse_classifier.staged_decision_function(X_test_sparse) + dense_clf_results = dense_classifier.staged_decision_function(X_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_almost_equal(sparse_clf_res, dense_clf_res) + + # staged_predict + sparse_clf_results = sparse_classifier.staged_predict(X_test_sparse) + dense_clf_results = dense_classifier.staged_predict(X_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_equal(sparse_clf_res, dense_clf_res) + + # staged_predict_proba + sparse_clf_results = sparse_classifier.staged_predict_proba(X_test_sparse) + dense_clf_results = dense_classifier.staged_predict_proba(X_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_almost_equal(sparse_clf_res, dense_clf_res) + + # staged_score + sparse_clf_results = sparse_classifier.staged_score(X_test_sparse, y_test) + dense_clf_results = dense_classifier.staged_score(X_test, y_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_equal(sparse_clf_res, dense_clf_res) + + # Verify sparsity of data is maintained during training + types = [i.data_type_ for i in sparse_classifier.estimators_] + + assert all([t == expected_internal_type for t in types]) + + +@pytest.mark.parametrize( + "sparse_container, expected_internal_type", + zip( + [ + *CSC_CONTAINERS, + *CSR_CONTAINERS, + *LIL_CONTAINERS, + *COO_CONTAINERS, + *DOK_CONTAINERS, + ], + CSC_CONTAINERS + 4 * CSR_CONTAINERS, + ), +) +def test_sparse_regression(sparse_container, expected_internal_type): + # Check regression with sparse input. + + class CustomSVR(SVR): + """SVR variant that records the nature of the training set.""" + + def fit(self, X, y, sample_weight=None): + """Modification on fit caries data type for later verification.""" + super().fit(X, y, sample_weight=sample_weight) + self.data_type_ = type(X) + return self + + X, y = datasets.make_regression( + n_samples=15, n_features=50, n_targets=1, random_state=42 + ) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) + + # Trained on sparse format + sparse_regressor = AdaBoostRegressor(estimator=CustomSVR(), random_state=1).fit( + X_train_sparse, y_train + ) + + # Trained on dense format + dense_regressor = AdaBoostRegressor(estimator=CustomSVR(), random_state=1).fit( + X_train, y_train + ) + + # predict + sparse_regr_results = sparse_regressor.predict(X_test_sparse) + dense_regr_results = dense_regressor.predict(X_test) + assert_array_almost_equal(sparse_regr_results, dense_regr_results) + + # staged_predict + sparse_regr_results = sparse_regressor.staged_predict(X_test_sparse) + dense_regr_results = dense_regressor.staged_predict(X_test) + for sparse_regr_res, dense_regr_res in zip(sparse_regr_results, dense_regr_results): + assert_array_almost_equal(sparse_regr_res, dense_regr_res) + + types = [i.data_type_ for i in sparse_regressor.estimators_] + + assert all([t == expected_internal_type for t in types]) + + +def test_sample_weight_adaboost_regressor(): + """ + AdaBoostRegressor should work without sample_weights in the base estimator + The random weighted sampling is done internally in the _boost method in + AdaBoostRegressor. + """ + + class DummyEstimator(BaseEstimator): + def fit(self, X, y): + pass + + def predict(self, X): + return np.zeros(X.shape[0]) + + boost = AdaBoostRegressor(DummyEstimator(), n_estimators=3) + boost.fit(X, y_regr) + assert len(boost.estimator_weights_) == len(boost.estimator_errors_) + + +def test_multidimensional_X(): + """ + Check that the AdaBoost estimators can work with n-dimensional + data matrix + """ + rng = np.random.RandomState(0) + + X = rng.randn(51, 3, 3) + yc = rng.choice([0, 1], 51) + yr = rng.randn(51) + + boost = AdaBoostClassifier(DummyClassifier(strategy="most_frequent")) + boost.fit(X, yc) + boost.predict(X) + boost.predict_proba(X) + + boost = AdaBoostRegressor(DummyRegressor()) + boost.fit(X, yr) + boost.predict(X) + + +def test_adaboostclassifier_without_sample_weight(): + X, y = iris.data, iris.target + estimator = NoSampleWeightWrapper(DummyClassifier()) + clf = AdaBoostClassifier(estimator=estimator) + err_msg = "{} doesn't support sample_weight".format(estimator.__class__.__name__) + with pytest.raises(ValueError, match=err_msg): + clf.fit(X, y) + + +def test_adaboostregressor_sample_weight(): + # check that giving weight will have an influence on the error computed + # for a weak learner + rng = np.random.RandomState(42) + X = np.linspace(0, 100, num=1000) + y = (0.8 * X + 0.2) + (rng.rand(X.shape[0]) * 0.0001) + X = X.reshape(-1, 1) + + # add an arbitrary outlier + X[-1] *= 10 + y[-1] = 10000 + + # random_state=0 ensure that the underlying bootstrap will use the outlier + regr_no_outlier = AdaBoostRegressor( + estimator=LinearRegression(), n_estimators=1, random_state=0 + ) + regr_with_weight = clone(regr_no_outlier) + regr_with_outlier = clone(regr_no_outlier) + + # fit 3 models: + # - a model containing the outlier + # - a model without the outlier + # - a model containing the outlier but with a null sample-weight + regr_with_outlier.fit(X, y) + regr_no_outlier.fit(X[:-1], y[:-1]) + sample_weight = np.ones_like(y) + sample_weight[-1] = 0 + regr_with_weight.fit(X, y, sample_weight=sample_weight) + + score_with_outlier = regr_with_outlier.score(X[:-1], y[:-1]) + score_no_outlier = regr_no_outlier.score(X[:-1], y[:-1]) + score_with_weight = regr_with_weight.score(X[:-1], y[:-1]) + + assert score_with_outlier < score_no_outlier + assert score_with_outlier < score_with_weight + assert score_no_outlier == pytest.approx(score_with_weight) + + +def test_adaboost_consistent_predict(): + # check that predict_proba and predict give consistent results + # regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/14084 + X_train, X_test, y_train, y_test = train_test_split( + *datasets.load_digits(return_X_y=True), random_state=42 + ) + model = AdaBoostClassifier(random_state=42) + model.fit(X_train, y_train) + + assert_array_equal( + np.argmax(model.predict_proba(X_test), axis=1), model.predict(X_test) + ) + + +@pytest.mark.parametrize( + "model, X, y", + [ + (AdaBoostClassifier(), iris.data, iris.target), + (AdaBoostRegressor(), diabetes.data, diabetes.target), + ], +) +def test_adaboost_negative_weight_error(model, X, y): + sample_weight = np.ones_like(y) + sample_weight[-1] = -10 + + err_msg = "Negative values in data passed to `sample_weight`" + with pytest.raises(ValueError, match=err_msg): + model.fit(X, y, sample_weight=sample_weight) + + +def test_adaboost_numerically_stable_feature_importance_with_small_weights(): + """Check that we don't create NaN feature importance with numerically + instable inputs. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/20320 + """ + rng = np.random.RandomState(42) + X = rng.normal(size=(1000, 10)) + y = rng.choice([0, 1], size=1000) + sample_weight = np.ones_like(y) * 1e-263 + tree = DecisionTreeClassifier(max_depth=10, random_state=12) + ada_model = AdaBoostClassifier(estimator=tree, n_estimators=20, random_state=12) + ada_model.fit(X, y, sample_weight=sample_weight) + assert np.isnan(ada_model.feature_importances_).sum() == 0 + + +def test_adaboost_decision_function(global_random_seed): + """Check that the decision function respects the symmetric constraint for weak + learners. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/26520 + """ + n_classes = 3 + X, y = datasets.make_classification( + n_classes=n_classes, n_clusters_per_class=1, random_state=global_random_seed + ) + clf = AdaBoostClassifier(n_estimators=1, random_state=global_random_seed).fit(X, y) + + y_score = clf.decision_function(X) + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) + + # With a single learner, we expect to have a decision function in + # {1, - 1 / (n_classes - 1)}. + assert set(np.unique(y_score)) == {1, -1 / (n_classes - 1)} + + # We can assert the same for staged_decision_function since we have a single learner + for y_score in clf.staged_decision_function(X): + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) + + # With a single learner, we expect to have a decision function in + # {1, - 1 / (n_classes - 1)}. + assert set(np.unique(y_score)) == {1, -1 / (n_classes - 1)} + + clf.set_params(n_estimators=5).fit(X, y) + + y_score = clf.decision_function(X) + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) + + for y_score in clf.staged_decision_function(X): + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/exceptions.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..71c484b30aeead793c9201261f5f9d0b57570c0d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/exceptions.py @@ -0,0 +1,249 @@ +"""Custom warnings and errors used across scikit-learn.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "ConvergenceWarning", + "DataConversionWarning", + "DataDimensionalityWarning", + "EfficiencyWarning", + "EstimatorCheckFailedWarning", + "FitFailedWarning", + "NotFittedError", + "PositiveSpectrumWarning", + "SkipTestWarning", + "UndefinedMetricWarning", + "UnsetMetadataPassedError", +] + + +class UnsetMetadataPassedError(ValueError): + """Exception class to raise if a metadata is passed which is not explicitly \ + requested (metadata=True) or not requested (metadata=False). + + .. versionadded:: 1.3 + + Parameters + ---------- + message : str + The message + + unrequested_params : dict + A dictionary of parameters and their values which are provided but not + requested. + + routed_params : dict + A dictionary of routed parameters. + """ + + def __init__(self, *, message, unrequested_params, routed_params): + super().__init__(message) + self.unrequested_params = unrequested_params + self.routed_params = routed_params + + +class NotFittedError(ValueError, AttributeError): + """Exception class to raise if estimator is used before fitting. + + This class inherits from both ValueError and AttributeError to help with + exception handling and backward compatibility. + + Examples + -------- + >>> from sklearn.svm import LinearSVC + >>> from sklearn.exceptions import NotFittedError + >>> try: + ... LinearSVC().predict([[1, 2], [2, 3], [3, 4]]) + ... except NotFittedError as e: + ... print(repr(e)) + NotFittedError("This LinearSVC instance is not fitted yet. Call 'fit' with + appropriate arguments before using this estimator."...) + + .. versionchanged:: 0.18 + Moved from sklearn.utils.validation. + """ + + +class ConvergenceWarning(UserWarning): + """Custom warning to capture convergence problems + + .. versionchanged:: 0.18 + Moved from sklearn.utils. + """ + + +class DataConversionWarning(UserWarning): + """Warning used to notify implicit data conversions happening in the code. + + This warning occurs when some input data needs to be converted or + interpreted in a way that may not match the user's expectations. + + For example, this warning may occur when the user + - passes an integer array to a function which expects float input and + will convert the input + - requests a non-copying operation, but a copy is required to meet the + implementation's data-type expectations; + - passes an input whose shape can be interpreted ambiguously. + + .. versionchanged:: 0.18 + Moved from sklearn.utils.validation. + """ + + +class DataDimensionalityWarning(UserWarning): + """Custom warning to notify potential issues with data dimensionality. + + For example, in random projection, this warning is raised when the + number of components, which quantifies the dimensionality of the target + projection space, is higher than the number of features, which quantifies + the dimensionality of the original source space, to imply that the + dimensionality of the problem will not be reduced. + + .. versionchanged:: 0.18 + Moved from sklearn.utils. + """ + + +class EfficiencyWarning(UserWarning): + """Warning used to notify the user of inefficient computation. + + This warning notifies the user that the efficiency may not be optimal due + to some reason which may be included as a part of the warning message. + This may be subclassed into a more specific Warning class. + + .. versionadded:: 0.18 + """ + + +class FitFailedWarning(RuntimeWarning): + """Warning class used if there is an error while fitting the estimator. + + This Warning is used in meta estimators GridSearchCV and RandomizedSearchCV + and the cross-validation helper function cross_val_score to warn when there + is an error while fitting the estimator. + + .. versionchanged:: 0.18 + Moved from sklearn.cross_validation. + """ + + +class SkipTestWarning(UserWarning): + """Warning class used to notify the user of a test that was skipped. + + For example, one of the estimator checks requires a pandas import. + If the pandas package cannot be imported, the test will be skipped rather + than register as a failure. + """ + + +class UndefinedMetricWarning(UserWarning): + """Warning used when the metric is invalid + + .. versionchanged:: 0.18 + Moved from sklearn.base. + """ + + +class PositiveSpectrumWarning(UserWarning): + """Warning raised when the eigenvalues of a PSD matrix have issues + + This warning is typically raised by ``_check_psd_eigenvalues`` when the + eigenvalues of a positive semidefinite (PSD) matrix such as a gram matrix + (kernel) present significant negative eigenvalues, or bad conditioning i.e. + very small non-zero eigenvalues compared to the largest eigenvalue. + + .. versionadded:: 0.22 + """ + + +class InconsistentVersionWarning(UserWarning): + """Warning raised when an estimator is unpickled with an inconsistent version. + + Parameters + ---------- + estimator_name : str + Estimator name. + + current_sklearn_version : str + Current scikit-learn version. + + original_sklearn_version : str + Original scikit-learn version. + """ + + def __init__( + self, *, estimator_name, current_sklearn_version, original_sklearn_version + ): + self.estimator_name = estimator_name + self.current_sklearn_version = current_sklearn_version + self.original_sklearn_version = original_sklearn_version + + def __str__(self): + return ( + f"Trying to unpickle estimator {self.estimator_name} from version" + f" {self.original_sklearn_version} when " + f"using version {self.current_sklearn_version}. This might lead to breaking" + " code or " + "invalid results. Use at your own risk. " + "For more info please refer to:\n" + "https://scikit-learn.org/stable/model_persistence.html" + "#security-maintainability-limitations" + ) + + +class EstimatorCheckFailedWarning(UserWarning): + """Warning raised when an estimator check from the common tests fails. + + Parameters + ---------- + estimator : estimator object + Estimator instance for which the test failed. + + check_name : str + Name of the check that failed. + + exception : Exception + Exception raised by the failed check. + + status : str + Status of the check. + + expected_to_fail : bool + Whether the check was expected to fail. + + expected_to_fail_reason : str + Reason for the expected failure. + """ + + def __init__( + self, + *, + estimator, + check_name: str, + exception: Exception, + status: str, + expected_to_fail: bool, + expected_to_fail_reason: str, + ): + self.estimator = estimator + self.check_name = check_name + self.exception = exception + self.status = status + self.expected_to_fail = expected_to_fail + self.expected_to_fail_reason = expected_to_fail_reason + + def __repr__(self): + expected_to_fail_str = ( + f"Expected to fail: {self.expected_to_fail_reason}" + if self.expected_to_fail + else "Not expected to fail" + ) + return ( + f"Test {self.check_name} failed for estimator {self.estimator!r}.\n" + f"Expected to fail reason: {expected_to_fail_str}\n" + f"Exception: {self.exception}" + ) + + def __str__(self): + return self.__repr__() diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..69cb5e960e7c3f0ac456ba6048d3445e0b696ec9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/__init__.py @@ -0,0 +1,10 @@ +"""Importable modules that enable the use of experimental features or estimators. + +.. warning:: + + The features and estimators that are experimental aren't subject to + deprecation cycles. Use them at your own risks! +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_halving_search_cv.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_halving_search_cv.py new file mode 100644 index 0000000000000000000000000000000000000000..9a40c157860741502adeeca0eaeff253e04f816a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_halving_search_cv.py @@ -0,0 +1,35 @@ +"""Enables Successive Halving search-estimators + +The API and results of these estimators might change without any deprecation +cycle. + +Importing this file dynamically sets the +:class:`~sklearn.model_selection.HalvingRandomSearchCV` and +:class:`~sklearn.model_selection.HalvingGridSearchCV` as attributes of the +`model_selection` module:: + + >>> # explicitly require this experimental feature + >>> from sklearn.experimental import enable_halving_search_cv # noqa + >>> # now you can import normally from model_selection + >>> from sklearn.model_selection import HalvingRandomSearchCV + >>> from sklearn.model_selection import HalvingGridSearchCV + + +The ``# noqa`` comment comment can be removed: it just tells linters like +flake8 to ignore the import, which appears as unused. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from sklearn import model_selection +from sklearn.model_selection._search_successive_halving import ( + HalvingGridSearchCV, + HalvingRandomSearchCV, +) + +# use settattr to avoid mypy errors when monkeypatching +setattr(model_selection, "HalvingRandomSearchCV", HalvingRandomSearchCV) +setattr(model_selection, "HalvingGridSearchCV", HalvingGridSearchCV) + +model_selection.__all__ += ["HalvingRandomSearchCV", "HalvingGridSearchCV"] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_hist_gradient_boosting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_hist_gradient_boosting.py new file mode 100644 index 0000000000000000000000000000000000000000..f8eaeb6433a0fdb1a7b5475047d311ec3816a8b1 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_hist_gradient_boosting.py @@ -0,0 +1,23 @@ +"""This is now a no-op and can be safely removed from your code. + +It used to enable the use of +:class:`~sklearn.ensemble.HistGradientBoostingClassifier` and +:class:`~sklearn.ensemble.HistGradientBoostingRegressor` when they were still +:term:`experimental`, but these estimators are now stable and can be imported +normally from `sklearn.ensemble`. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# Don't remove this file, we don't want to break users code just because the +# feature isn't experimental anymore. + +import warnings + +warnings.warn( + "Since version 1.0, " + "it is not needed to import enable_hist_gradient_boosting anymore. " + "HistGradientBoostingClassifier and HistGradientBoostingRegressor are now " + "stable and can be normally imported from sklearn.ensemble." +) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_iterative_imputer.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_iterative_imputer.py new file mode 100644 index 0000000000000000000000000000000000000000..04f53251706bf3844627d52fd6ad0167e2a58f4c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/enable_iterative_imputer.py @@ -0,0 +1,23 @@ +"""Enables IterativeImputer + +The API and results of this estimator might change without any deprecation +cycle. + +Importing this file dynamically sets :class:`~sklearn.impute.IterativeImputer` +as an attribute of the impute module:: + + >>> # explicitly require this experimental feature + >>> from sklearn.experimental import enable_iterative_imputer # noqa + >>> # now you can import normally from impute + >>> from sklearn.impute import IterativeImputer +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from sklearn import impute +from sklearn.impute._iterative import IterativeImputer + +# use settattr to avoid mypy errors when monkeypatching +setattr(impute, "IterativeImputer", IterativeImputer) +impute.__all__ += ["IterativeImputer"] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py new file mode 100644 index 0000000000000000000000000000000000000000..aefc7636cd3bed357e7f5fed8e27693e0da6769e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py @@ -0,0 +1,19 @@ +"""Tests for making sure experimental imports work as expected.""" + +import textwrap + +import pytest + +from sklearn.utils._testing import assert_run_python_script_without_output +from sklearn.utils.fixes import _IS_WASM + + +@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess") +def test_import_raises_warning(): + code = """ + import pytest + with pytest.warns(UserWarning, match="it is not needed to import"): + from sklearn.experimental import enable_hist_gradient_boosting # noqa + """ + pattern = "it is not needed to import enable_hist_gradient_boosting anymore" + assert_run_python_script_without_output(textwrap.dedent(code), pattern=pattern) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_iterative_imputer.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_iterative_imputer.py new file mode 100644 index 0000000000000000000000000000000000000000..d4bf087ade69b31cda287168cf125ca0f19bad6e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_iterative_imputer.py @@ -0,0 +1,51 @@ +"""Tests for making sure experimental imports work as expected.""" + +import textwrap + +import pytest + +from sklearn.utils._testing import assert_run_python_script_without_output +from sklearn.utils.fixes import _IS_WASM + + +@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess") +def test_imports_strategies(): + # Make sure different import strategies work or fail as expected. + + # Since Python caches the imported modules, we need to run a child process + # for every test case. Else, the tests would not be independent + # (manually removing the imports from the cache (sys.modules) is not + # recommended and can lead to many complications). + pattern = "IterativeImputer is experimental" + good_import = """ + from sklearn.experimental import enable_iterative_imputer + from sklearn.impute import IterativeImputer + """ + assert_run_python_script_without_output( + textwrap.dedent(good_import), pattern=pattern + ) + + good_import_with_ensemble_first = """ + import sklearn.ensemble + from sklearn.experimental import enable_iterative_imputer + from sklearn.impute import IterativeImputer + """ + assert_run_python_script_without_output( + textwrap.dedent(good_import_with_ensemble_first), + pattern=pattern, + ) + + bad_imports = f""" + import pytest + + with pytest.raises(ImportError, match={pattern!r}): + from sklearn.impute import IterativeImputer + + import sklearn.experimental + with pytest.raises(ImportError, match={pattern!r}): + from sklearn.impute import IterativeImputer + """ + assert_run_python_script_without_output( + textwrap.dedent(bad_imports), + pattern=pattern, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_successive_halving.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_successive_halving.py new file mode 100644 index 0000000000000000000000000000000000000000..1d67951ed0fd375936cd4210127cd46aeee81798 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/experimental/tests/test_enable_successive_halving.py @@ -0,0 +1,53 @@ +"""Tests for making sure experimental imports work as expected.""" + +import textwrap + +import pytest + +from sklearn.utils._testing import assert_run_python_script_without_output +from sklearn.utils.fixes import _IS_WASM + + +@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess") +def test_imports_strategies(): + # Make sure different import strategies work or fail as expected. + + # Since Python caches the imported modules, we need to run a child process + # for every test case. Else, the tests would not be independent + # (manually removing the imports from the cache (sys.modules) is not + # recommended and can lead to many complications). + pattern = "Halving(Grid|Random)SearchCV is experimental" + good_import = """ + from sklearn.experimental import enable_halving_search_cv + from sklearn.model_selection import HalvingGridSearchCV + from sklearn.model_selection import HalvingRandomSearchCV + """ + assert_run_python_script_without_output( + textwrap.dedent(good_import), pattern=pattern + ) + + good_import_with_model_selection_first = """ + import sklearn.model_selection + from sklearn.experimental import enable_halving_search_cv + from sklearn.model_selection import HalvingGridSearchCV + from sklearn.model_selection import HalvingRandomSearchCV + """ + assert_run_python_script_without_output( + textwrap.dedent(good_import_with_model_selection_first), + pattern=pattern, + ) + + bad_imports = f""" + import pytest + + with pytest.raises(ImportError, match={pattern!r}): + from sklearn.model_selection import HalvingGridSearchCV + + import sklearn.experimental + with pytest.raises(ImportError, match={pattern!r}): + from sklearn.model_selection import HalvingRandomSearchCV + """ + assert_run_python_script_without_output( + textwrap.dedent(bad_imports), + pattern=pattern, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35fe943346b23f56c52ffa618a4740deffd986ef --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__init__.py @@ -0,0 +1,5 @@ + +""" +External, bundled dependencies. + +""" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..530e004091618ce9cc4784a59926589167e631ef Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/__init__.cpython-312.pyc differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/_arff.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/_arff.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..526f667705bb3e5c5445a9aa272080c37da40a0b Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/_arff.cpython-312.pyc differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/_array_api_compat_vendor.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/_array_api_compat_vendor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2394497fc5a057264bc32b84968037763f2e6d8 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/__pycache__/_array_api_compat_vendor.cpython-312.pyc differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_arff.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_arff.py new file mode 100644 index 0000000000000000000000000000000000000000..981507667843c59ee3aa673a48792cd5ce2613dc --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_arff.py @@ -0,0 +1,1107 @@ +# ============================================================================= +# Federal University of Rio Grande do Sul (UFRGS) +# Connectionist Artificial Intelligence Laboratory (LIAC) +# Renato de Pontes Pereira - rppereira@inf.ufrgs.br +# ============================================================================= +# Copyright (c) 2011 Renato de Pontes Pereira, renato.ppontes at gmail dot com +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ============================================================================= + +''' +The liac-arff module implements functions to read and write ARFF files in +Python. It was created in the Connectionist Artificial Intelligence Laboratory +(LIAC), which takes place at the Federal University of Rio Grande do Sul +(UFRGS), in Brazil. + +ARFF (Attribute-Relation File Format) is an file format specially created for +describe datasets which are commonly used for machine learning experiments and +software. This file format was created to be used in Weka, the best +representative software for machine learning automated experiments. + +An ARFF file can be divided into two sections: header and data. The Header +describes the metadata of the dataset, including a general description of the +dataset, its name and its attributes. The source below is an example of a +header section in a XOR dataset:: + + % + % XOR Dataset + % + % Created by Renato Pereira + % rppereira@inf.ufrgs.br + % http://inf.ufrgs.br/~rppereira + % + % + @RELATION XOR + + @ATTRIBUTE input1 REAL + @ATTRIBUTE input2 REAL + @ATTRIBUTE y REAL + +The Data section of an ARFF file describes the observations of the dataset, in +the case of XOR dataset:: + + @DATA + 0.0,0.0,0.0 + 0.0,1.0,1.0 + 1.0,0.0,1.0 + 1.0,1.0,0.0 + % + % + % + +Notice that several lines are starting with an ``%`` symbol, denoting a +comment, thus, lines with ``%`` at the beginning will be ignored, except by the +description part at the beginning of the file. The declarations ``@RELATION``, +``@ATTRIBUTE``, and ``@DATA`` are all case insensitive and obligatory. + +For more information and details about the ARFF file description, consult +http://www.cs.waikato.ac.nz/~ml/weka/arff.html + + +ARFF Files in Python +~~~~~~~~~~~~~~~~~~~~ + +This module uses built-ins python objects to represent a deserialized ARFF +file. A dictionary is used as the container of the data and metadata of ARFF, +and have the following keys: + +- **description**: (OPTIONAL) a string with the description of the dataset. +- **relation**: (OBLIGATORY) a string with the name of the dataset. +- **attributes**: (OBLIGATORY) a list of attributes with the following + template:: + + (attribute_name, attribute_type) + + the attribute_name is a string, and attribute_type must be an string + or a list of strings. +- **data**: (OBLIGATORY) a list of data instances. Each data instance must be + a list with values, depending on the attributes. + +The above keys must follow the case which were described, i.e., the keys are +case sensitive. The attribute type ``attribute_type`` must be one of these +strings (they are not case sensitive): ``NUMERIC``, ``INTEGER``, ``REAL`` or +``STRING``. For nominal attributes, the ``atribute_type`` must be a list of +strings. + +In this format, the XOR dataset presented above can be represented as a python +object as:: + + xor_dataset = { + 'description': 'XOR Dataset', + 'relation': 'XOR', + 'attributes': [ + ('input1', 'REAL'), + ('input2', 'REAL'), + ('y', 'REAL'), + ], + 'data': [ + [0.0, 0.0, 0.0], + [0.0, 1.0, 1.0], + [1.0, 0.0, 1.0], + [1.0, 1.0, 0.0] + ] + } + + +Features +~~~~~~~~ + +This module provides several features, including: + +- Read and write ARFF files using python built-in structures, such dictionaries + and lists; +- Supports `scipy.sparse.coo `_ + and lists of dictionaries as used by SVMLight +- Supports the following attribute types: NUMERIC, REAL, INTEGER, STRING, and + NOMINAL; +- Has an interface similar to other built-in modules such as ``json``, or + ``zipfile``; +- Supports read and write the descriptions of files; +- Supports missing values and names with spaces; +- Supports unicode values and names; +- Fully compatible with Python 2.7+, Python 3.5+, pypy and pypy3; +- Under `MIT License `_ + +''' +__author__ = 'Renato de Pontes Pereira, Matthias Feurer, Joel Nothman' +__author_email__ = ('renato.ppontes@gmail.com, ' + 'feurerm@informatik.uni-freiburg.de, ' + 'joel.nothman@gmail.com') +__version__ = '2.4.0' + +import re +import csv +from typing import TYPE_CHECKING +from typing import Optional, List, Dict, Any, Iterator, Union, Tuple + +# CONSTANTS =================================================================== +_SIMPLE_TYPES = ['NUMERIC', 'REAL', 'INTEGER', 'STRING'] + +_TK_DESCRIPTION = '%' +_TK_COMMENT = '%' +_TK_RELATION = '@RELATION' +_TK_ATTRIBUTE = '@ATTRIBUTE' +_TK_DATA = '@DATA' + +_RE_RELATION = re.compile(r'^([^\{\}%,\s]*|\".*\"|\'.*\')$', re.UNICODE) +_RE_ATTRIBUTE = re.compile(r'^(\".*\"|\'.*\'|[^\{\}%,\s]*)\s+(.+)$', re.UNICODE) +_RE_QUOTE_CHARS = re.compile(r'["\'\\\s%,\000-\031]', re.UNICODE) +_RE_ESCAPE_CHARS = re.compile(r'(?=["\'\\%])|[\n\r\t\000-\031]') +_RE_SPARSE_LINE = re.compile(r'^\s*\{.*\}\s*$', re.UNICODE) +_RE_NONTRIVIAL_DATA = re.compile('["\'{}\\s]', re.UNICODE) + +ArffDenseDataType = Iterator[List] +ArffSparseDataType = Tuple[List, ...] + + +if TYPE_CHECKING: + # typing_extensions is available when mypy is installed + from typing_extensions import TypedDict + + class ArffContainerType(TypedDict): + description: str + relation: str + attributes: List + data: Union[ArffDenseDataType, ArffSparseDataType] + +else: + ArffContainerType = Dict[str, Any] + + +def _build_re_values(): + quoted_re = r''' + " # open quote followed by zero or more of: + (?: + (?= len(conversors): + raise BadDataFormat(row) + # XXX: int 0 is used for implicit values, not '0' + values = [values[i] if i in values else 0 for i in + range(len(conversors))] + else: + if len(values) != len(conversors): + raise BadDataFormat(row) + + yield self._decode_values(values, conversors) + + @staticmethod + def _decode_values(values, conversors): + try: + values = [None if value is None else conversor(value) + for conversor, value + in zip(conversors, values)] + except ValueError as exc: + if 'float: ' in str(exc): + raise BadNumericalValue() + return values + + def encode_data(self, data, attributes): + '''(INTERNAL) Encodes a line of data. + + Data instances follow the csv format, i.e, attribute values are + delimited by commas. After converted from csv. + + :param data: a list of values. + :param attributes: a list of attributes. Used to check if data is valid. + :return: a string with the encoded data line. + ''' + current_row = 0 + + for inst in data: + if len(inst) != len(attributes): + raise BadObject( + 'Instance %d has %d attributes, expected %d' % + (current_row, len(inst), len(attributes)) + ) + + new_data = [] + for value in inst: + if value is None or value == '' or value != value: + s = '?' + else: + s = encode_string(str(value)) + new_data.append(s) + + current_row += 1 + yield ','.join(new_data) + + +class _DataListMixin: + """Mixin to return a list from decode_rows instead of a generator""" + def decode_rows(self, stream, conversors): + return list(super().decode_rows(stream, conversors)) + + +class Data(_DataListMixin, DenseGeneratorData): + pass + + +class COOData: + def decode_rows(self, stream, conversors): + data, rows, cols = [], [], [] + for i, row in enumerate(stream): + values = _parse_values(row) + if not isinstance(values, dict): + raise BadLayout() + if not values: + continue + row_cols, values = zip(*sorted(values.items())) + try: + values = [value if value is None else conversors[key](value) + for key, value in zip(row_cols, values)] + except ValueError as exc: + if 'float: ' in str(exc): + raise BadNumericalValue() + raise + except IndexError: + # conversor out of range + raise BadDataFormat(row) + + data.extend(values) + rows.extend([i] * len(values)) + cols.extend(row_cols) + + return data, rows, cols + + def encode_data(self, data, attributes): + num_attributes = len(attributes) + new_data = [] + current_row = 0 + + row = data.row + col = data.col + data = data.data + + # Check if the rows are sorted + if not all(row[i] <= row[i + 1] for i in range(len(row) - 1)): + raise ValueError("liac-arff can only output COO matrices with " + "sorted rows.") + + for v, col, row in zip(data, col, row): + if row > current_row: + # Add empty rows if necessary + while current_row < row: + yield " ".join(["{", ','.join(new_data), "}"]) + new_data = [] + current_row += 1 + + if col >= num_attributes: + raise BadObject( + 'Instance %d has at least %d attributes, expected %d' % + (current_row, col + 1, num_attributes) + ) + + if v is None or v == '' or v != v: + s = '?' + else: + s = encode_string(str(v)) + new_data.append("%d %s" % (col, s)) + + yield " ".join(["{", ','.join(new_data), "}"]) + +class LODGeneratorData: + def decode_rows(self, stream, conversors): + for row in stream: + values = _parse_values(row) + + if not isinstance(values, dict): + raise BadLayout() + try: + yield {key: None if value is None else conversors[key](value) + for key, value in values.items()} + except ValueError as exc: + if 'float: ' in str(exc): + raise BadNumericalValue() + raise + except IndexError: + # conversor out of range + raise BadDataFormat(row) + + def encode_data(self, data, attributes): + current_row = 0 + + num_attributes = len(attributes) + for row in data: + new_data = [] + + if len(row) > 0 and max(row) >= num_attributes: + raise BadObject( + 'Instance %d has %d attributes, expected %d' % + (current_row, max(row) + 1, num_attributes) + ) + + for col in sorted(row): + v = row[col] + if v is None or v == '' or v != v: + s = '?' + else: + s = encode_string(str(v)) + new_data.append("%d %s" % (col, s)) + + current_row += 1 + yield " ".join(["{", ','.join(new_data), "}"]) + +class LODData(_DataListMixin, LODGeneratorData): + pass + + +def _get_data_object_for_decoding(matrix_type): + if matrix_type == DENSE: + return Data() + elif matrix_type == COO: + return COOData() + elif matrix_type == LOD: + return LODData() + elif matrix_type == DENSE_GEN: + return DenseGeneratorData() + elif matrix_type == LOD_GEN: + return LODGeneratorData() + else: + raise ValueError("Matrix type %s not supported." % str(matrix_type)) + +def _get_data_object_for_encoding(matrix): + # Probably a scipy.sparse + if hasattr(matrix, 'format'): + if matrix.format == 'coo': + return COOData() + else: + raise ValueError('Cannot guess matrix format!') + elif isinstance(matrix[0], dict): + return LODData() + else: + return Data() + +# ============================================================================= + +# ADVANCED INTERFACE ========================================================== +class ArffDecoder: + '''An ARFF decoder.''' + + def __init__(self): + '''Constructor.''' + self._conversors = [] + self._current_line = 0 + + def _decode_comment(self, s): + '''(INTERNAL) Decodes a comment line. + + Comments are single line strings starting, obligatorily, with the ``%`` + character, and can have any symbol, including whitespaces or special + characters. + + This method must receive a normalized string, i.e., a string without + padding, including the "\r\n" characters. + + :param s: a normalized string. + :return: a string with the decoded comment. + ''' + res = re.sub(r'^\%( )?', '', s) + return res + + def _decode_relation(self, s): + '''(INTERNAL) Decodes a relation line. + + The relation declaration is a line with the format ``@RELATION + ``, where ``relation-name`` is a string. The string must + start with alphabetic character and must be quoted if the name includes + spaces, otherwise this method will raise a `BadRelationFormat` exception. + + This method must receive a normalized string, i.e., a string without + padding, including the "\r\n" characters. + + :param s: a normalized string. + :return: a string with the decoded relation name. + ''' + _, v = s.split(' ', 1) + v = v.strip() + + if not _RE_RELATION.match(v): + raise BadRelationFormat() + + res = str(v.strip('"\'')) + return res + + def _decode_attribute(self, s): + '''(INTERNAL) Decodes an attribute line. + + The attribute is the most complex declaration in an arff file. All + attributes must follow the template:: + + @attribute + + where ``attribute-name`` is a string, quoted if the name contains any + whitespace, and ``datatype`` can be: + + - Numerical attributes as ``NUMERIC``, ``INTEGER`` or ``REAL``. + - Strings as ``STRING``. + - Dates (NOT IMPLEMENTED). + - Nominal attributes with format: + + {, , , ...} + + The nominal names follow the rules for the attribute names, i.e., they + must be quoted if the name contains whitespaces. + + This method must receive a normalized string, i.e., a string without + padding, including the "\r\n" characters. + + :param s: a normalized string. + :return: a tuple (ATTRIBUTE_NAME, TYPE_OR_VALUES). + ''' + _, v = s.split(' ', 1) + v = v.strip() + + # Verify the general structure of declaration + m = _RE_ATTRIBUTE.match(v) + if not m: + raise BadAttributeFormat() + + # Extracts the raw name and type + name, type_ = m.groups() + + # Extracts the final name + name = str(name.strip('"\'')) + + # Extracts the final type + if type_[:1] == "{" and type_[-1:] == "}": + try: + type_ = _parse_values(type_.strip('{} ')) + except Exception: + raise BadAttributeType() + if isinstance(type_, dict): + raise BadAttributeType() + + else: + # If not nominal, verify the type name + type_ = str(type_).upper() + if type_ not in ['NUMERIC', 'REAL', 'INTEGER', 'STRING']: + raise BadAttributeType() + + return (name, type_) + + def _decode(self, s, encode_nominal=False, matrix_type=DENSE): + '''Do the job the ``encode``.''' + + # Make sure this method is idempotent + self._current_line = 0 + + # If string, convert to a list of lines + if isinstance(s, str): + s = s.strip('\r\n ').replace('\r\n', '\n').split('\n') + + # Create the return object + obj: ArffContainerType = { + 'description': '', + 'relation': '', + 'attributes': [], + 'data': [] + } + attribute_names = {} + + # Create the data helper object + data = _get_data_object_for_decoding(matrix_type) + + # Read all lines + STATE = _TK_DESCRIPTION + s = iter(s) + for row in s: + self._current_line += 1 + # Ignore empty lines + row = row.strip(' \r\n') + if not row: continue + + u_row = row.upper() + + # DESCRIPTION ----------------------------------------------------- + if u_row.startswith(_TK_DESCRIPTION) and STATE == _TK_DESCRIPTION: + obj['description'] += self._decode_comment(row) + '\n' + # ----------------------------------------------------------------- + + # RELATION -------------------------------------------------------- + elif u_row.startswith(_TK_RELATION): + if STATE != _TK_DESCRIPTION: + raise BadLayout() + + STATE = _TK_RELATION + obj['relation'] = self._decode_relation(row) + # ----------------------------------------------------------------- + + # ATTRIBUTE ------------------------------------------------------- + elif u_row.startswith(_TK_ATTRIBUTE): + if STATE != _TK_RELATION and STATE != _TK_ATTRIBUTE: + raise BadLayout() + + STATE = _TK_ATTRIBUTE + + attr = self._decode_attribute(row) + if attr[0] in attribute_names: + raise BadAttributeName(attr[0], attribute_names[attr[0]]) + else: + attribute_names[attr[0]] = self._current_line + obj['attributes'].append(attr) + + if isinstance(attr[1], (list, tuple)): + if encode_nominal: + conversor = EncodedNominalConversor(attr[1]) + else: + conversor = NominalConversor(attr[1]) + else: + CONVERSOR_MAP = {'STRING': str, + 'INTEGER': lambda x: int(float(x)), + 'NUMERIC': float, + 'REAL': float} + conversor = CONVERSOR_MAP[attr[1]] + + self._conversors.append(conversor) + # ----------------------------------------------------------------- + + # DATA ------------------------------------------------------------ + elif u_row.startswith(_TK_DATA): + if STATE != _TK_ATTRIBUTE: + raise BadLayout() + + break + # ----------------------------------------------------------------- + + # COMMENT --------------------------------------------------------- + elif u_row.startswith(_TK_COMMENT): + pass + # ----------------------------------------------------------------- + else: + # Never found @DATA + raise BadLayout() + + def stream(): + for row in s: + self._current_line += 1 + row = row.strip() + # Ignore empty lines and comment lines. + if row and not row.startswith(_TK_COMMENT): + yield row + + # Alter the data object + obj['data'] = data.decode_rows(stream(), self._conversors) + if obj['description'].endswith('\n'): + obj['description'] = obj['description'][:-1] + + return obj + + def decode(self, s, encode_nominal=False, return_type=DENSE): + '''Returns the Python representation of a given ARFF file. + + When a file object is passed as an argument, this method reads lines + iteratively, avoiding to load unnecessary information to the memory. + + :param s: a string or file object with the ARFF file. + :param encode_nominal: boolean, if True perform a label encoding + while reading the .arff file. + :param return_type: determines the data structure used to store the + dataset. Can be one of `arff.DENSE`, `arff.COO`, `arff.LOD`, + `arff.DENSE_GEN` or `arff.LOD_GEN`. + Consult the sections on `working with sparse data`_ and `loading + progressively`_. + ''' + try: + return self._decode(s, encode_nominal=encode_nominal, + matrix_type=return_type) + except ArffException as e: + e.line = self._current_line + raise e + + +class ArffEncoder: + '''An ARFF encoder.''' + + def _encode_comment(self, s=''): + '''(INTERNAL) Encodes a comment line. + + Comments are single line strings starting, obligatorily, with the ``%`` + character, and can have any symbol, including whitespaces or special + characters. + + If ``s`` is None, this method will simply return an empty comment. + + :param s: (OPTIONAL) string. + :return: a string with the encoded comment line. + ''' + if s: + return '%s %s'%(_TK_COMMENT, s) + else: + return '%s' % _TK_COMMENT + + def _encode_relation(self, name): + '''(INTERNAL) Decodes a relation line. + + The relation declaration is a line with the format ``@RELATION + ``, where ``relation-name`` is a string. + + :param name: a string. + :return: a string with the encoded relation declaration. + ''' + for char in ' %{},': + if char in name: + name = '"%s"'%name + break + + return '%s %s'%(_TK_RELATION, name) + + def _encode_attribute(self, name, type_): + '''(INTERNAL) Encodes an attribute line. + + The attribute follow the template:: + + @attribute + + where ``attribute-name`` is a string, and ``datatype`` can be: + + - Numerical attributes as ``NUMERIC``, ``INTEGER`` or ``REAL``. + - Strings as ``STRING``. + - Dates (NOT IMPLEMENTED). + - Nominal attributes with format: + + {, , , ...} + + This method must receive a the name of the attribute and its type, if + the attribute type is nominal, ``type`` must be a list of values. + + :param name: a string. + :param type_: a string or a list of string. + :return: a string with the encoded attribute declaration. + ''' + for char in ' %{},': + if char in name: + name = '"%s"'%name + break + + if isinstance(type_, (tuple, list)): + type_tmp = ['%s' % encode_string(type_k) for type_k in type_] + type_ = '{%s}'%(', '.join(type_tmp)) + + return '%s %s %s'%(_TK_ATTRIBUTE, name, type_) + + def encode(self, obj): + '''Encodes a given object to an ARFF file. + + :param obj: the object containing the ARFF information. + :return: the ARFF file as an string. + ''' + data = [row for row in self.iter_encode(obj)] + + return '\n'.join(data) + + def iter_encode(self, obj): + '''The iterative version of `arff.ArffEncoder.encode`. + + This encodes iteratively a given object and return, one-by-one, the + lines of the ARFF file. + + :param obj: the object containing the ARFF information. + :return: (yields) the ARFF file as strings. + ''' + # DESCRIPTION + if obj.get('description', None): + for row in obj['description'].split('\n'): + yield self._encode_comment(row) + + # RELATION + if not obj.get('relation'): + raise BadObject('Relation name not found or with invalid value.') + + yield self._encode_relation(obj['relation']) + yield '' + + # ATTRIBUTES + if not obj.get('attributes'): + raise BadObject('Attributes not found.') + + attribute_names = set() + for attr in obj['attributes']: + # Verify for bad object format + if not isinstance(attr, (tuple, list)) or \ + len(attr) != 2 or \ + not isinstance(attr[0], str): + raise BadObject('Invalid attribute declaration "%s"'%str(attr)) + + if isinstance(attr[1], str): + # Verify for invalid types + if attr[1] not in _SIMPLE_TYPES: + raise BadObject('Invalid attribute type "%s"'%str(attr)) + + # Verify for bad object format + elif not isinstance(attr[1], (tuple, list)): + raise BadObject('Invalid attribute type "%s"'%str(attr)) + + # Verify attribute name is not used twice + if attr[0] in attribute_names: + raise BadObject('Trying to use attribute name "%s" for the ' + 'second time.' % str(attr[0])) + else: + attribute_names.add(attr[0]) + + yield self._encode_attribute(attr[0], attr[1]) + yield '' + attributes = obj['attributes'] + + # DATA + yield _TK_DATA + if 'data' in obj: + data = _get_data_object_for_encoding(obj.get('data')) + yield from data.encode_data(obj.get('data'), attributes) + + yield '' + +# ============================================================================= + +# BASIC INTERFACE ============================================================= +def load(fp, encode_nominal=False, return_type=DENSE): + '''Load a file-like object containing the ARFF document and convert it into + a Python object. + + :param fp: a file-like object. + :param encode_nominal: boolean, if True perform a label encoding + while reading the .arff file. + :param return_type: determines the data structure used to store the + dataset. Can be one of `arff.DENSE`, `arff.COO`, `arff.LOD`, + `arff.DENSE_GEN` or `arff.LOD_GEN`. + Consult the sections on `working with sparse data`_ and `loading + progressively`_. + :return: a dictionary. + ''' + decoder = ArffDecoder() + return decoder.decode(fp, encode_nominal=encode_nominal, + return_type=return_type) + +def loads(s, encode_nominal=False, return_type=DENSE): + '''Convert a string instance containing the ARFF document into a Python + object. + + :param s: a string object. + :param encode_nominal: boolean, if True perform a label encoding + while reading the .arff file. + :param return_type: determines the data structure used to store the + dataset. Can be one of `arff.DENSE`, `arff.COO`, `arff.LOD`, + `arff.DENSE_GEN` or `arff.LOD_GEN`. + Consult the sections on `working with sparse data`_ and `loading + progressively`_. + :return: a dictionary. + ''' + decoder = ArffDecoder() + return decoder.decode(s, encode_nominal=encode_nominal, + return_type=return_type) + +def dump(obj, fp): + '''Serialize an object representing the ARFF document to a given file-like + object. + + :param obj: a dictionary. + :param fp: a file-like object. + ''' + encoder = ArffEncoder() + generator = encoder.iter_encode(obj) + + last_row = next(generator) + for row in generator: + fp.write(last_row + '\n') + last_row = row + fp.write(last_row) + + return fp + +def dumps(obj): + '''Serialize an object representing the ARFF document, returning a string. + + :param obj: a dictionary. + :return: a string with the ARFF document. + ''' + encoder = ArffEncoder() + return encoder.encode(obj) +# ============================================================================= diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_array_api_compat_vendor.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_array_api_compat_vendor.py new file mode 100644 index 0000000000000000000000000000000000000000..ecd527fbd2c8f06f356788480c08b65bc12a3358 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_array_api_compat_vendor.py @@ -0,0 +1,5 @@ +# DO NOT RENAME THIS FILE +# This is a hook for array_api_extra/_lib/_compat.py +# to co-vendor array_api_compat and potentially override its functions. + +from .array_api_compat import * # noqa: F403 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_numpydoc/__pycache__/docscrape.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_numpydoc/__pycache__/docscrape.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c68debd9647e66b784789e9bd4692bb1f16ea62d Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_numpydoc/__pycache__/docscrape.cpython-312.pyc differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_numpydoc/docscrape.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_numpydoc/docscrape.py new file mode 100644 index 0000000000000000000000000000000000000000..2c30bd4c953734b9b4cd1b28af6a27990ded1599 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_numpydoc/docscrape.py @@ -0,0 +1,759 @@ +"""Extract reference documentation from the NumPy source tree.""" + +import copy +import inspect +import pydoc +import re +import sys +import textwrap +from collections import namedtuple +from collections.abc import Callable, Mapping +from functools import cached_property +from warnings import warn + + +def strip_blank_lines(l): + "Remove leading and trailing blank lines from a list of lines" + while l and not l[0].strip(): + del l[0] + while l and not l[-1].strip(): + del l[-1] + return l + + +class Reader: + """A line-based string reader.""" + + def __init__(self, data): + """ + Parameters + ---------- + data : str + String with lines separated by '\\n'. + + """ + if isinstance(data, list): + self._str = data + else: + self._str = data.split("\n") # store string as list of lines + + self.reset() + + def __getitem__(self, n): + return self._str[n] + + def reset(self): + self._l = 0 # current line nr + + def read(self): + if not self.eof(): + out = self[self._l] + self._l += 1 + return out + else: + return "" + + def seek_next_non_empty_line(self): + for l in self[self._l :]: + if l.strip(): + break + else: + self._l += 1 + + def eof(self): + return self._l >= len(self._str) + + def read_to_condition(self, condition_func): + start = self._l + for line in self[start:]: + if condition_func(line): + return self[start : self._l] + self._l += 1 + if self.eof(): + return self[start : self._l + 1] + return [] + + def read_to_next_empty_line(self): + self.seek_next_non_empty_line() + + def is_empty(line): + return not line.strip() + + return self.read_to_condition(is_empty) + + def read_to_next_unindented_line(self): + def is_unindented(line): + return line.strip() and (len(line.lstrip()) == len(line)) + + return self.read_to_condition(is_unindented) + + def peek(self, n=0): + if self._l + n < len(self._str): + return self[self._l + n] + else: + return "" + + def is_empty(self): + return not "".join(self._str).strip() + + +class ParseError(Exception): + def __str__(self): + message = self.args[0] + if hasattr(self, "docstring"): + message = f"{message} in {self.docstring!r}" + return message + + +Parameter = namedtuple("Parameter", ["name", "type", "desc"]) + + +class NumpyDocString(Mapping): + """Parses a numpydoc string to an abstract representation + + Instances define a mapping from section title to structured data. + + """ + + sections = { + "Signature": "", + "Summary": [""], + "Extended Summary": [], + "Parameters": [], + "Attributes": [], + "Methods": [], + "Returns": [], + "Yields": [], + "Receives": [], + "Other Parameters": [], + "Raises": [], + "Warns": [], + "Warnings": [], + "See Also": [], + "Notes": [], + "References": "", + "Examples": "", + "index": {}, + } + + def __init__(self, docstring, config=None): + orig_docstring = docstring + docstring = textwrap.dedent(docstring).split("\n") + + self._doc = Reader(docstring) + self._parsed_data = copy.deepcopy(self.sections) + + try: + self._parse() + except ParseError as e: + e.docstring = orig_docstring + raise + + def __getitem__(self, key): + return self._parsed_data[key] + + def __setitem__(self, key, val): + if key not in self._parsed_data: + self._error_location(f"Unknown section {key}", error=False) + else: + self._parsed_data[key] = val + + def __iter__(self): + return iter(self._parsed_data) + + def __len__(self): + return len(self._parsed_data) + + def _is_at_section(self): + self._doc.seek_next_non_empty_line() + + if self._doc.eof(): + return False + + l1 = self._doc.peek().strip() # e.g. Parameters + + if l1.startswith(".. index::"): + return True + + l2 = self._doc.peek(1).strip() # ---------- or ========== + if len(l2) >= 3 and (set(l2) in ({"-"}, {"="})) and len(l2) != len(l1): + snip = "\n".join(self._doc._str[:2]) + "..." + self._error_location( + f"potentially wrong underline length... \n{l1} \n{l2} in \n{snip}", + error=False, + ) + return l2.startswith("-" * len(l1)) or l2.startswith("=" * len(l1)) + + def _strip(self, doc): + i = 0 + j = 0 + for i, line in enumerate(doc): + if line.strip(): + break + + for j, line in enumerate(doc[::-1]): + if line.strip(): + break + + return doc[i : len(doc) - j] + + def _read_to_next_section(self): + section = self._doc.read_to_next_empty_line() + + while not self._is_at_section() and not self._doc.eof(): + if not self._doc.peek(-1).strip(): # previous line was empty + section += [""] + + section += self._doc.read_to_next_empty_line() + + return section + + def _read_sections(self): + while not self._doc.eof(): + data = self._read_to_next_section() + name = data[0].strip() + + if name.startswith(".."): # index section + yield name, data[1:] + elif len(data) < 2: + yield StopIteration + else: + yield name, self._strip(data[2:]) + + def _parse_param_list(self, content, single_element_is_type=False): + content = dedent_lines(content) + r = Reader(content) + params = [] + while not r.eof(): + header = r.read().strip() + if " : " in header: + arg_name, arg_type = header.split(" : ", maxsplit=1) + else: + # NOTE: param line with single element should never have a + # a " :" before the description line, so this should probably + # warn. + header = header.removesuffix(" :") + if single_element_is_type: + arg_name, arg_type = "", header + else: + arg_name, arg_type = header, "" + + desc = r.read_to_next_unindented_line() + desc = dedent_lines(desc) + desc = strip_blank_lines(desc) + + params.append(Parameter(arg_name, arg_type, desc)) + + return params + + # See also supports the following formats. + # + # + # SPACE* COLON SPACE+ SPACE* + # ( COMMA SPACE+ )+ (COMMA | PERIOD)? SPACE* + # ( COMMA SPACE+ )* SPACE* COLON SPACE+ SPACE* + + # is one of + # + # COLON COLON BACKTICK BACKTICK + # where + # is a legal function name, and + # is any nonempty sequence of word characters. + # Examples: func_f1 :meth:`func_h1` :obj:`~baz.obj_r` :class:`class_j` + # is a string describing the function. + + _role = r":(?P(py:)?\w+):" + _funcbacktick = r"`(?P(?:~\w+\.)?[a-zA-Z0-9_\.-]+)`" + _funcplain = r"(?P[a-zA-Z0-9_\.-]+)" + _funcname = r"(" + _role + _funcbacktick + r"|" + _funcplain + r")" + _funcnamenext = _funcname.replace("role", "rolenext") + _funcnamenext = _funcnamenext.replace("name", "namenext") + _description = r"(?P\s*:(\s+(?P\S+.*))?)?\s*$" + _func_rgx = re.compile(r"^\s*" + _funcname + r"\s*") + _line_rgx = re.compile( + r"^\s*" + + r"(?P" + + _funcname # group for all function names + + r"(?P([,]\s+" + + _funcnamenext + + r")*)" + + r")" + + r"(?P[,\.])?" # end of "allfuncs" + + _description # Some function lists have a trailing comma (or period) '\s*' + ) + + # Empty elements are replaced with '..' + empty_description = ".." + + def _parse_see_also(self, content): + """ + func_name : Descriptive text + continued text + another_func_name : Descriptive text + func_name1, func_name2, :meth:`func_name`, func_name3 + + """ + + content = dedent_lines(content) + + items = [] + + def parse_item_name(text): + """Match ':role:`name`' or 'name'.""" + m = self._func_rgx.match(text) + if not m: + self._error_location(f"Error parsing See Also entry {line!r}") + role = m.group("role") + name = m.group("name") if role else m.group("name2") + return name, role, m.end() + + rest = [] + for line in content: + if not line.strip(): + continue + + line_match = self._line_rgx.match(line) + description = None + if line_match: + description = line_match.group("desc") + if line_match.group("trailing") and description: + self._error_location( + "Unexpected comma or period after function list at index %d of " + 'line "%s"' % (line_match.end("trailing"), line), + error=False, + ) + if not description and line.startswith(" "): + rest.append(line.strip()) + elif line_match: + funcs = [] + text = line_match.group("allfuncs") + while True: + if not text.strip(): + break + name, role, match_end = parse_item_name(text) + funcs.append((name, role)) + text = text[match_end:].strip() + if text and text[0] == ",": + text = text[1:].strip() + rest = list(filter(None, [description])) + items.append((funcs, rest)) + else: + self._error_location(f"Error parsing See Also entry {line!r}") + return items + + def _parse_index(self, section, content): + """ + .. index:: default + :refguide: something, else, and more + + """ + + def strip_each_in(lst): + return [s.strip() for s in lst] + + out = {} + section = section.split("::") + if len(section) > 1: + out["default"] = strip_each_in(section[1].split(","))[0] + for line in content: + line = line.split(":") + if len(line) > 2: + out[line[1]] = strip_each_in(line[2].split(",")) + return out + + def _parse_summary(self): + """Grab signature (if given) and summary""" + if self._is_at_section(): + return + + # If several signatures present, take the last one + while True: + summary = self._doc.read_to_next_empty_line() + summary_str = " ".join([s.strip() for s in summary]).strip() + compiled = re.compile(r"^([\w., ]+=)?\s*[\w\.]+\(.*\)$") + if compiled.match(summary_str): + self["Signature"] = summary_str + if not self._is_at_section(): + continue + break + + if summary is not None: + self["Summary"] = summary + + if not self._is_at_section(): + self["Extended Summary"] = self._read_to_next_section() + + def _parse(self): + self._doc.reset() + self._parse_summary() + + sections = list(self._read_sections()) + section_names = {section for section, content in sections} + + has_yields = "Yields" in section_names + # We could do more tests, but we are not. Arbitrarily. + if not has_yields and "Receives" in section_names: + msg = "Docstring contains a Receives section but not Yields." + raise ValueError(msg) + + for section, content in sections: + if not section.startswith(".."): + section = (s.capitalize() for s in section.split(" ")) + section = " ".join(section) + if self.get(section): + self._error_location( + "The section %s appears twice in %s" + % (section, "\n".join(self._doc._str)) + ) + + if section in ("Parameters", "Other Parameters", "Attributes", "Methods"): + self[section] = self._parse_param_list(content) + elif section in ("Returns", "Yields", "Raises", "Warns", "Receives"): + self[section] = self._parse_param_list( + content, single_element_is_type=True + ) + elif section.startswith(".. index::"): + self["index"] = self._parse_index(section, content) + elif section == "See Also": + self["See Also"] = self._parse_see_also(content) + else: + self[section] = content + + @property + def _obj(self): + if hasattr(self, "_cls"): + return self._cls + elif hasattr(self, "_f"): + return self._f + return None + + def _error_location(self, msg, error=True): + if self._obj is not None: + # we know where the docs came from: + try: + filename = inspect.getsourcefile(self._obj) + except TypeError: + filename = None + # Make UserWarning more descriptive via object introspection. + # Skip if introspection fails + name = getattr(self._obj, "__name__", None) + if name is None: + name = getattr(getattr(self._obj, "__class__", None), "__name__", None) + if name is not None: + msg += f" in the docstring of {name}" + msg += f" in {filename}." if filename else "" + if error: + raise ValueError(msg) + else: + warn(msg, stacklevel=3) + + # string conversion routines + + def _str_header(self, name, symbol="-"): + return [name, len(name) * symbol] + + def _str_indent(self, doc, indent=4): + return [" " * indent + line for line in doc] + + def _str_signature(self): + if self["Signature"]: + return [self["Signature"].replace("*", r"\*")] + [""] + return [""] + + def _str_summary(self): + if self["Summary"]: + return self["Summary"] + [""] + return [] + + def _str_extended_summary(self): + if self["Extended Summary"]: + return self["Extended Summary"] + [""] + return [] + + def _str_param_list(self, name): + out = [] + if self[name]: + out += self._str_header(name) + for param in self[name]: + parts = [] + if param.name: + parts.append(param.name) + if param.type: + parts.append(param.type) + out += [" : ".join(parts)] + if param.desc and "".join(param.desc).strip(): + out += self._str_indent(param.desc) + out += [""] + return out + + def _str_section(self, name): + out = [] + if self[name]: + out += self._str_header(name) + out += self[name] + out += [""] + return out + + def _str_see_also(self, func_role): + if not self["See Also"]: + return [] + out = [] + out += self._str_header("See Also") + out += [""] + last_had_desc = True + for funcs, desc in self["See Also"]: + assert isinstance(funcs, list) + links = [] + for func, role in funcs: + if role: + link = f":{role}:`{func}`" + elif func_role: + link = f":{func_role}:`{func}`" + else: + link = f"`{func}`_" + links.append(link) + link = ", ".join(links) + out += [link] + if desc: + out += self._str_indent([" ".join(desc)]) + last_had_desc = True + else: + last_had_desc = False + out += self._str_indent([self.empty_description]) + + if last_had_desc: + out += [""] + out += [""] + return out + + def _str_index(self): + idx = self["index"] + out = [] + output_index = False + default_index = idx.get("default", "") + if default_index: + output_index = True + out += [f".. index:: {default_index}"] + for section, references in idx.items(): + if section == "default": + continue + output_index = True + out += [f" :{section}: {', '.join(references)}"] + if output_index: + return out + return "" + + def __str__(self, func_role=""): + out = [] + out += self._str_signature() + out += self._str_summary() + out += self._str_extended_summary() + out += self._str_param_list("Parameters") + for param_list in ("Attributes", "Methods"): + out += self._str_param_list(param_list) + for param_list in ( + "Returns", + "Yields", + "Receives", + "Other Parameters", + "Raises", + "Warns", + ): + out += self._str_param_list(param_list) + out += self._str_section("Warnings") + out += self._str_see_also(func_role) + for s in ("Notes", "References", "Examples"): + out += self._str_section(s) + out += self._str_index() + return "\n".join(out) + + +def dedent_lines(lines): + """Deindent a list of lines maximally""" + return textwrap.dedent("\n".join(lines)).split("\n") + + +class FunctionDoc(NumpyDocString): + def __init__(self, func, role="func", doc=None, config=None): + self._f = func + self._role = role # e.g. "func" or "meth" + + if doc is None: + if func is None: + raise ValueError("No function or docstring given") + doc = inspect.getdoc(func) or "" + if config is None: + config = {} + NumpyDocString.__init__(self, doc, config) + + def get_func(self): + func_name = getattr(self._f, "__name__", self.__class__.__name__) + if inspect.isclass(self._f): + func = getattr(self._f, "__call__", self._f.__init__) + else: + func = self._f + return func, func_name + + def __str__(self): + out = "" + + func, func_name = self.get_func() + + roles = {"func": "function", "meth": "method"} + + if self._role: + if self._role not in roles: + print(f"Warning: invalid role {self._role}") + out += f".. {roles.get(self._role, '')}:: {func_name}\n \n\n" + + out += super().__str__(func_role=self._role) + return out + + +class ObjDoc(NumpyDocString): + def __init__(self, obj, doc=None, config=None): + self._f = obj + if config is None: + config = {} + NumpyDocString.__init__(self, doc, config=config) + + +class ClassDoc(NumpyDocString): + extra_public_methods = ["__call__"] + + def __init__(self, cls, doc=None, modulename="", func_doc=FunctionDoc, config=None): + if not inspect.isclass(cls) and cls is not None: + raise ValueError(f"Expected a class or None, but got {cls!r}") + self._cls = cls + + if "sphinx" in sys.modules: + from sphinx.ext.autodoc import ALL + else: + ALL = object() + + if config is None: + config = {} + self.show_inherited_members = config.get("show_inherited_class_members", True) + + if modulename and not modulename.endswith("."): + modulename += "." + self._mod = modulename + + if doc is None: + if cls is None: + raise ValueError("No class or documentation string given") + doc = pydoc.getdoc(cls) + + NumpyDocString.__init__(self, doc) + + _members = config.get("members", []) + if _members is ALL: + _members = None + _exclude = config.get("exclude-members", []) + + if config.get("show_class_members", True) and _exclude is not ALL: + + def splitlines_x(s): + if not s: + return [] + else: + return s.splitlines() + + for field, items in [ + ("Methods", self.methods), + ("Attributes", self.properties), + ]: + if not self[field]: + doc_list = [] + for name in sorted(items): + if name in _exclude or (_members and name not in _members): + continue + try: + doc_item = pydoc.getdoc(getattr(self._cls, name)) + doc_list.append(Parameter(name, "", splitlines_x(doc_item))) + except AttributeError: + pass # method doesn't exist + self[field] = doc_list + + @property + def methods(self): + if self._cls is None: + return [] + return [ + name + for name, func in inspect.getmembers(self._cls) + if ( + (not name.startswith("_") or name in self.extra_public_methods) + and isinstance(func, Callable) + and self._is_show_member(name) + ) + ] + + @property + def properties(self): + if self._cls is None: + return [] + return [ + name + for name, func in inspect.getmembers(self._cls) + if ( + not name.startswith("_") + and not self._should_skip_member(name, self._cls) + and ( + func is None + or isinstance(func, (property, cached_property)) + or inspect.isdatadescriptor(func) + ) + and self._is_show_member(name) + ) + ] + + @staticmethod + def _should_skip_member(name, klass): + return ( + # Namedtuples should skip everything in their ._fields as the + # docstrings for each of the members is: "Alias for field number X" + issubclass(klass, tuple) + and hasattr(klass, "_asdict") + and hasattr(klass, "_fields") + and name in klass._fields + ) + + def _is_show_member(self, name): + return ( + # show all class members + self.show_inherited_members + # or class member is not inherited + or name in self._cls.__dict__ + ) + + +def get_doc_object( + obj, + what=None, + doc=None, + config=None, + class_doc=ClassDoc, + func_doc=FunctionDoc, + obj_doc=ObjDoc, +): + if what is None: + if inspect.isclass(obj): + what = "class" + elif inspect.ismodule(obj): + what = "module" + elif isinstance(obj, Callable): + what = "function" + else: + what = "object" + if config is None: + config = {} + + if what == "class": + return class_doc(obj, func_doc=func_doc, doc=doc, config=config) + elif what in ("function", "method"): + return func_doc(obj, doc=doc, config=config) + else: + if doc is None: + doc = pydoc.getdoc(obj) + return obj_doc(obj, doc, config=config) \ No newline at end of file diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91058466308bf2abef516d0758326105207bdbe2 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/__init__.cpython-312.pyc differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/_structures.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/_structures.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5137aee4f955105f617c56bb846067d98023dbb3 Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/_structures.cpython-312.pyc differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/version.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab1ae4cab6178a2926614bbfada4b84ba5c9447a Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/__pycache__/version.cpython-312.pyc differ diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/_structures.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/_structures.py new file mode 100644 index 0000000000000000000000000000000000000000..a46378830247d1178d4856253465befce3c93273 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/_structures.py @@ -0,0 +1,90 @@ +"""Vendoered from +https://github.com/pypa/packaging/blob/main/packaging/_structures.py +""" +# Copyright (c) Donald Stufft and individual contributors. +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class InfinityType: + def __repr__(self) -> str: + return "Infinity" + + def __hash__(self) -> int: + return hash(repr(self)) + + def __lt__(self, other: object) -> bool: + return False + + def __le__(self, other: object) -> bool: + return False + + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) + + def __ne__(self, other: object) -> bool: + return not isinstance(other, self.__class__) + + def __gt__(self, other: object) -> bool: + return True + + def __ge__(self, other: object) -> bool: + return True + + def __neg__(self: object) -> "NegativeInfinityType": + return NegativeInfinity + + +Infinity = InfinityType() + + +class NegativeInfinityType: + def __repr__(self) -> str: + return "-Infinity" + + def __hash__(self) -> int: + return hash(repr(self)) + + def __lt__(self, other: object) -> bool: + return True + + def __le__(self, other: object) -> bool: + return True + + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) + + def __ne__(self, other: object) -> bool: + return not isinstance(other, self.__class__) + + def __gt__(self, other: object) -> bool: + return False + + def __ge__(self, other: object) -> bool: + return False + + def __neg__(self: object) -> InfinityType: + return Infinity + + +NegativeInfinity = NegativeInfinityType() diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/version.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/version.py new file mode 100644 index 0000000000000000000000000000000000000000..2c26b4354e719768ac1b1997aad5a98b4237febd --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_packaging/version.py @@ -0,0 +1,535 @@ +"""Vendored from +https://github.com/pypa/packaging/blob/main/packaging/version.py +""" +# Copyright (c) Donald Stufft and individual contributors. +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import collections +import itertools +import re +import warnings +from typing import Callable, Iterator, List, Optional, SupportsInt, Tuple, Union + +from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType + +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] + +InfiniteTypes = Union[InfinityType, NegativeInfinityType] +PrePostDevType = Union[InfiniteTypes, Tuple[str, int]] +SubLocalType = Union[InfiniteTypes, int, str] +LocalType = Union[ + NegativeInfinityType, + Tuple[ + Union[ + SubLocalType, + Tuple[SubLocalType, str], + Tuple[NegativeInfinityType, SubLocalType], + ], + ..., + ], +] +CmpKey = Tuple[ + int, Tuple[int, ...], PrePostDevType, PrePostDevType, PrePostDevType, LocalType +] +LegacyCmpKey = Tuple[int, Tuple[str, ...]] +VersionComparisonMethod = Callable[ + [Union[CmpKey, LegacyCmpKey], Union[CmpKey, LegacyCmpKey]], bool +] + +_Version = collections.namedtuple( + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] +) + + +def parse(version: str) -> Union["LegacyVersion", "Version"]: + """Parse the given version from a string to an appropriate class. + + Parameters + ---------- + version : str + Version in a string format, eg. "0.9.1" or "1.2.dev0". + + Returns + ------- + version : :class:`Version` object or a :class:`LegacyVersion` object + Returned class depends on the given version: if is a valid + PEP 440 version or a legacy version. + """ + try: + return Version(version) + except InvalidVersion: + return LegacyVersion(version) + + +class InvalidVersion(ValueError): + """ + An invalid version was found, users should refer to PEP 440. + """ + + +class _BaseVersion: + _key: Union[CmpKey, LegacyCmpKey] + + def __hash__(self) -> int: + return hash(self._key) + + # Please keep the duplicated `isinstance` check + # in the six comparisons hereunder + # unless you find a way to avoid adding overhead function calls. + def __lt__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key < other._key + + def __le__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key <= other._key + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key == other._key + + def __ge__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key >= other._key + + def __gt__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key > other._key + + def __ne__(self, other: object) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key != other._key + + +class LegacyVersion(_BaseVersion): + def __init__(self, version: str) -> None: + self._version = str(version) + self._key = _legacy_cmpkey(self._version) + + warnings.warn( + "Creating a LegacyVersion has been deprecated and will be " + "removed in the next major release", + DeprecationWarning, + ) + + def __str__(self) -> str: + return self._version + + def __repr__(self) -> str: + return f"" + + @property + def public(self) -> str: + return self._version + + @property + def base_version(self) -> str: + return self._version + + @property + def epoch(self) -> int: + return -1 + + @property + def release(self) -> None: + return None + + @property + def pre(self) -> None: + return None + + @property + def post(self) -> None: + return None + + @property + def dev(self) -> None: + return None + + @property + def local(self) -> None: + return None + + @property + def is_prerelease(self) -> bool: + return False + + @property + def is_postrelease(self) -> bool: + return False + + @property + def is_devrelease(self) -> bool: + return False + + +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) + +_legacy_version_replacement_map = { + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", +} + + +def _parse_version_parts(s: str) -> Iterator[str]: + for part in _legacy_version_component_re.split(s): + part = _legacy_version_replacement_map.get(part, part) + + if not part or part == ".": + continue + + if part[:1] in "0123456789": + # pad for numeric comparison + yield part.zfill(8) + else: + yield "*" + part + + # ensure that alpha/beta/candidate are before final + yield "*final" + + +def _legacy_cmpkey(version: str) -> LegacyCmpKey: + + # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch + # greater than or equal to 0. This will effectively put the LegacyVersion, + # which uses the defacto standard originally implemented by setuptools, + # as before all PEP 440 versions. + epoch = -1 + + # This scheme is taken from pkg_resources.parse_version setuptools prior to + # it's adoption of the packaging library. + parts: List[str] = [] + for part in _parse_version_parts(version.lower()): + if part.startswith("*"): + # remove "-" before a prerelease tag + if part < "*final": + while parts and parts[-1] == "*final-": + parts.pop() + + # remove trailing zeros from each series of numeric parts + while parts and parts[-1] == "00000000": + parts.pop() + + parts.append(part) + + return epoch, tuple(parts) + + +# Deliberately not anchored to the start and end of the string, to make it +# easier for 3rd party code to reuse +VERSION_PATTERN = r""" + v? + (?: + (?:(?P[0-9]+)!)? # epoch + (?P[0-9]+(?:\.[0-9]+)*) # release segment + (?P
                                          # pre-release
+            [-_\.]?
+            (?P(a|b|c|rc|alpha|beta|pre|preview))
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+        (?P                                         # post release
+            (?:-(?P[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?Ppost|rev|r)
+                [-_\.]?
+                (?P[0-9]+)?
+            )
+        )?
+        (?P                                          # dev release
+            [-_\.]?
+            (?Pdev)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+    )
+    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
+
+
+class Version(_BaseVersion):
+
+    _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
+    def __init__(self, version: str) -> None:
+
+        # Validate the version and parse it into pieces
+        match = self._regex.search(version)
+        if not match:
+            raise InvalidVersion(f"Invalid version: '{version}'")
+
+        # Store the parsed out pieces of the version
+        self._version = _Version(
+            epoch=int(match.group("epoch")) if match.group("epoch") else 0,
+            release=tuple(int(i) for i in match.group("release").split(".")),
+            pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")),
+            post=_parse_letter_version(
+                match.group("post_l"), match.group("post_n1") or match.group("post_n2")
+            ),
+            dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")),
+            local=_parse_local_version(match.group("local")),
+        )
+
+        # Generate a key which will be used for sorting
+        self._key = _cmpkey(
+            self._version.epoch,
+            self._version.release,
+            self._version.pre,
+            self._version.post,
+            self._version.dev,
+            self._version.local,
+        )
+
+    def __repr__(self) -> str:
+        return f""
+
+    def __str__(self) -> str:
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append(f"{self.epoch}!")
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        # Pre-release
+        if self.pre is not None:
+            parts.append("".join(str(x) for x in self.pre))
+
+        # Post-release
+        if self.post is not None:
+            parts.append(f".post{self.post}")
+
+        # Development release
+        if self.dev is not None:
+            parts.append(f".dev{self.dev}")
+
+        # Local version segment
+        if self.local is not None:
+            parts.append(f"+{self.local}")
+
+        return "".join(parts)
+
+    @property
+    def epoch(self) -> int:
+        _epoch: int = self._version.epoch
+        return _epoch
+
+    @property
+    def release(self) -> Tuple[int, ...]:
+        _release: Tuple[int, ...] = self._version.release
+        return _release
+
+    @property
+    def pre(self) -> Optional[Tuple[str, int]]:
+        _pre: Optional[Tuple[str, int]] = self._version.pre
+        return _pre
+
+    @property
+    def post(self) -> Optional[int]:
+        return self._version.post[1] if self._version.post else None
+
+    @property
+    def dev(self) -> Optional[int]:
+        return self._version.dev[1] if self._version.dev else None
+
+    @property
+    def local(self) -> Optional[str]:
+        if self._version.local:
+            return ".".join(str(x) for x in self._version.local)
+        else:
+            return None
+
+    @property
+    def public(self) -> str:
+        return str(self).split("+", 1)[0]
+
+    @property
+    def base_version(self) -> str:
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append(f"{self.epoch}!")
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        return "".join(parts)
+
+    @property
+    def is_prerelease(self) -> bool:
+        return self.dev is not None or self.pre is not None
+
+    @property
+    def is_postrelease(self) -> bool:
+        return self.post is not None
+
+    @property
+    def is_devrelease(self) -> bool:
+        return self.dev is not None
+
+    @property
+    def major(self) -> int:
+        return self.release[0] if len(self.release) >= 1 else 0
+
+    @property
+    def minor(self) -> int:
+        return self.release[1] if len(self.release) >= 2 else 0
+
+    @property
+    def micro(self) -> int:
+        return self.release[2] if len(self.release) >= 3 else 0
+
+
+def _parse_letter_version(
+    letter: str, number: Union[str, bytes, SupportsInt]
+) -> Optional[Tuple[str, int]]:
+
+    if letter:
+        # We consider there to be an implicit 0 in a pre-release if there is
+        # not a numeral associated with it.
+        if number is None:
+            number = 0
+
+        # We normalize any letters to their lower case form
+        letter = letter.lower()
+
+        # We consider some words to be alternate spellings of other words and
+        # in those cases we want to normalize the spellings to our preferred
+        # spelling.
+        if letter == "alpha":
+            letter = "a"
+        elif letter == "beta":
+            letter = "b"
+        elif letter in ["c", "pre", "preview"]:
+            letter = "rc"
+        elif letter in ["rev", "r"]:
+            letter = "post"
+
+        return letter, int(number)
+    if not letter and number:
+        # We assume if we are given a number, but we are not given a letter
+        # then this is using the implicit post release syntax (e.g. 1.0-1)
+        letter = "post"
+
+        return letter, int(number)
+
+    return None
+
+
+_local_version_separators = re.compile(r"[\._-]")
+
+
+def _parse_local_version(local: str) -> Optional[LocalType]:
+    """
+    Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
+    """
+    if local is not None:
+        return tuple(
+            part.lower() if not part.isdigit() else int(part)
+            for part in _local_version_separators.split(local)
+        )
+    return None
+
+
+def _cmpkey(
+    epoch: int,
+    release: Tuple[int, ...],
+    pre: Optional[Tuple[str, int]],
+    post: Optional[Tuple[str, int]],
+    dev: Optional[Tuple[str, int]],
+    local: Optional[Tuple[SubLocalType]],
+) -> CmpKey:
+
+    # When we compare a release version, we want to compare it with all of the
+    # trailing zeros removed. So we'll use a reverse the list, drop all the now
+    # leading zeros until we come to something non zero, then take the rest
+    # re-reverse it back into the correct order and make it a tuple and use
+    # that for our sorting key.
+    _release = tuple(
+        reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
+    )
+
+    # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0.
+    # We'll do this by abusing the pre segment, but we _only_ want to do this
+    # if there is not a pre or a post segment. If we have one of those then
+    # the normal sorting rules will handle this case correctly.
+    if pre is None and post is None and dev is not None:
+        _pre: PrePostDevType = NegativeInfinity
+    # Versions without a pre-release (except as noted above) should sort after
+    # those with one.
+    elif pre is None:
+        _pre = Infinity
+    else:
+        _pre = pre
+
+    # Versions without a post segment should sort before those with one.
+    if post is None:
+        _post: PrePostDevType = NegativeInfinity
+
+    else:
+        _post = post
+
+    # Versions without a development segment should sort after those with one.
+    if dev is None:
+        _dev: PrePostDevType = Infinity
+
+    else:
+        _dev = dev
+
+    if local is None:
+        # Versions without a local segment should sort before those with one.
+        _local: LocalType = NegativeInfinity
+    else:
+        # Versions with a local segment need that segment parsed to implement
+        # the sorting rules in PEP440.
+        # - Alpha numeric segments sort before numeric segments
+        # - Alpha numeric segments sort lexicographically
+        # - Numeric segments sort numerically
+        # - Shorter versions sort before longer versions when the prefixes
+        #   match exactly
+        _local = tuple(
+            (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local
+        )
+
+    return epoch, _release, _pre, _post, _dev, _local
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/csgraph/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/csgraph/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..be5a602206f109424d5c92e37dca40f04d0f7547
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/csgraph/__init__.py
@@ -0,0 +1 @@
+from ._laplacian import laplacian
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/csgraph/_laplacian.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/csgraph/_laplacian.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6f93209c1c834ad76e186371d2613d4c36619af
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/_scipy/sparse/csgraph/_laplacian.py
@@ -0,0 +1,557 @@
+"""
+This file is a copy of the scipy.sparse.csgraph._laplacian module from SciPy 1.12
+
+scipy.sparse.csgraph.laplacian supports sparse arrays only starting from Scipy 1.12,
+see https://github.com/scipy/scipy/pull/19156. This vendored file can be removed as
+soon as Scipy 1.12 becomes the minimum supported version.
+
+Laplacian of a compressed-sparse graph
+"""
+
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numpy as np
+from scipy.sparse import issparse
+from scipy.sparse.linalg import LinearOperator
+
+
+###############################################################################
+# Graph laplacian
+def laplacian(
+    csgraph,
+    normed=False,
+    return_diag=False,
+    use_out_degree=False,
+    *,
+    copy=True,
+    form="array",
+    dtype=None,
+    symmetrized=False,
+):
+    """
+    Return the Laplacian of a directed graph.
+
+    Parameters
+    ----------
+    csgraph : array_like or sparse matrix, 2 dimensions
+        Compressed-sparse graph, with shape (N, N).
+    normed : bool, optional
+        If True, then compute symmetrically normalized Laplacian.
+        Default: False.
+    return_diag : bool, optional
+        If True, then also return an array related to vertex degrees.
+        Default: False.
+    use_out_degree : bool, optional
+        If True, then use out-degree instead of in-degree.
+        This distinction matters only if the graph is asymmetric.
+        Default: False.
+    copy : bool, optional
+        If False, then change `csgraph` in place if possible,
+        avoiding doubling the memory use.
+        Default: True, for backward compatibility.
+    form : 'array', or 'function', or 'lo'
+        Determines the format of the output Laplacian:
+
+        * 'array' is a numpy array;
+        * 'function' is a pointer to evaluating the Laplacian-vector
+          or Laplacian-matrix product;
+        * 'lo' results in the format of the `LinearOperator`.
+
+        Choosing 'function' or 'lo' always avoids doubling
+        the memory use, ignoring `copy` value.
+        Default: 'array', for backward compatibility.
+    dtype : None or one of numeric numpy dtypes, optional
+        The dtype of the output. If ``dtype=None``, the dtype of the
+        output matches the dtype of the input csgraph, except for
+        the case ``normed=True`` and integer-like csgraph, where
+        the output dtype is 'float' allowing accurate normalization,
+        but dramatically increasing the memory use.
+        Default: None, for backward compatibility.
+    symmetrized : bool, optional
+        If True, then the output Laplacian is symmetric/Hermitian.
+        The symmetrization is done by ``csgraph + csgraph.T.conj``
+        without dividing by 2 to preserve integer dtypes if possible
+        prior to the construction of the Laplacian.
+        The symmetrization will increase the memory footprint of
+        sparse matrices unless the sparsity pattern is symmetric or
+        `form` is 'function' or 'lo'.
+        Default: False, for backward compatibility.
+
+    Returns
+    -------
+    lap : ndarray, or sparse matrix, or `LinearOperator`
+        The N x N Laplacian of csgraph. It will be a NumPy array (dense)
+        if the input was dense, or a sparse matrix otherwise, or
+        the format of a function or `LinearOperator` if
+        `form` equals 'function' or 'lo', respectively.
+    diag : ndarray, optional
+        The length-N main diagonal of the Laplacian matrix.
+        For the normalized Laplacian, this is the array of square roots
+        of vertex degrees or 1 if the degree is zero.
+
+    Notes
+    -----
+    The Laplacian matrix of a graph is sometimes referred to as the
+    "Kirchhoff matrix" or just the "Laplacian", and is useful in many
+    parts of spectral graph theory.
+    In particular, the eigen-decomposition of the Laplacian can give
+    insight into many properties of the graph, e.g.,
+    is commonly used for spectral data embedding and clustering.
+
+    The constructed Laplacian doubles the memory use if ``copy=True`` and
+    ``form="array"`` which is the default.
+    Choosing ``copy=False`` has no effect unless ``form="array"``
+    or the matrix is sparse in the ``coo`` format, or dense array, except
+    for the integer input with ``normed=True`` that forces the float output.
+
+    Sparse input is reformatted into ``coo`` if ``form="array"``,
+    which is the default.
+
+    If the input adjacency matrix is not symmetric, the Laplacian is
+    also non-symmetric unless ``symmetrized=True`` is used.
+
+    Diagonal entries of the input adjacency matrix are ignored and
+    replaced with zeros for the purpose of normalization where ``normed=True``.
+    The normalization uses the inverse square roots of row-sums of the input
+    adjacency matrix, and thus may fail if the row-sums contain
+    negative or complex with a non-zero imaginary part values.
+
+    The normalization is symmetric, making the normalized Laplacian also
+    symmetric if the input csgraph was symmetric.
+
+    References
+    ----------
+    .. [1] Laplacian matrix. https://en.wikipedia.org/wiki/Laplacian_matrix
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from scipy.sparse import csgraph
+
+    Our first illustration is the symmetric graph
+
+    >>> G = np.arange(4) * np.arange(4)[:, np.newaxis]
+    >>> G
+    array([[0, 0, 0, 0],
+           [0, 1, 2, 3],
+           [0, 2, 4, 6],
+           [0, 3, 6, 9]])
+
+    and its symmetric Laplacian matrix
+
+    >>> csgraph.laplacian(G)
+    array([[ 0,  0,  0,  0],
+           [ 0,  5, -2, -3],
+           [ 0, -2,  8, -6],
+           [ 0, -3, -6,  9]])
+
+    The non-symmetric graph
+
+    >>> G = np.arange(9).reshape(3, 3)
+    >>> G
+    array([[0, 1, 2],
+           [3, 4, 5],
+           [6, 7, 8]])
+
+    has different row- and column sums, resulting in two varieties
+    of the Laplacian matrix, using an in-degree, which is the default
+
+    >>> L_in_degree = csgraph.laplacian(G)
+    >>> L_in_degree
+    array([[ 9, -1, -2],
+           [-3,  8, -5],
+           [-6, -7,  7]])
+
+    or alternatively an out-degree
+
+    >>> L_out_degree = csgraph.laplacian(G, use_out_degree=True)
+    >>> L_out_degree
+    array([[ 3, -1, -2],
+           [-3,  8, -5],
+           [-6, -7, 13]])
+
+    Constructing a symmetric Laplacian matrix, one can add the two as
+
+    >>> L_in_degree + L_out_degree.T
+    array([[ 12,  -4,  -8],
+            [ -4,  16, -12],
+            [ -8, -12,  20]])
+
+    or use the ``symmetrized=True`` option
+
+    >>> csgraph.laplacian(G, symmetrized=True)
+    array([[ 12,  -4,  -8],
+           [ -4,  16, -12],
+           [ -8, -12,  20]])
+
+    that is equivalent to symmetrizing the original graph
+
+    >>> csgraph.laplacian(G + G.T)
+    array([[ 12,  -4,  -8],
+           [ -4,  16, -12],
+           [ -8, -12,  20]])
+
+    The goal of normalization is to make the non-zero diagonal entries
+    of the Laplacian matrix to be all unit, also scaling off-diagonal
+    entries correspondingly. The normalization can be done manually, e.g.,
+
+    >>> G = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0]])
+    >>> L, d = csgraph.laplacian(G, return_diag=True)
+    >>> L
+    array([[ 2, -1, -1],
+           [-1,  2, -1],
+           [-1, -1,  2]])
+    >>> d
+    array([2, 2, 2])
+    >>> scaling = np.sqrt(d)
+    >>> scaling
+    array([1.41421356, 1.41421356, 1.41421356])
+    >>> (1/scaling)*L*(1/scaling)
+    array([[ 1. , -0.5, -0.5],
+           [-0.5,  1. , -0.5],
+           [-0.5, -0.5,  1. ]])
+
+    Or using ``normed=True`` option
+
+    >>> L, d = csgraph.laplacian(G, return_diag=True, normed=True)
+    >>> L
+    array([[ 1. , -0.5, -0.5],
+           [-0.5,  1. , -0.5],
+           [-0.5, -0.5,  1. ]])
+
+    which now instead of the diagonal returns the scaling coefficients
+
+    >>> d
+    array([1.41421356, 1.41421356, 1.41421356])
+
+    Zero scaling coefficients are substituted with 1s, where scaling
+    has thus no effect, e.g.,
+
+    >>> G = np.array([[0, 0, 0], [0, 0, 1], [0, 1, 0]])
+    >>> G
+    array([[0, 0, 0],
+           [0, 0, 1],
+           [0, 1, 0]])
+    >>> L, d = csgraph.laplacian(G, return_diag=True, normed=True)
+    >>> L
+    array([[ 0., -0., -0.],
+           [-0.,  1., -1.],
+           [-0., -1.,  1.]])
+    >>> d
+    array([1., 1., 1.])
+
+    Only the symmetric normalization is implemented, resulting
+    in a symmetric Laplacian matrix if and only if its graph is symmetric
+    and has all non-negative degrees, like in the examples above.
+
+    The output Laplacian matrix is by default a dense array or a sparse matrix
+    inferring its shape, format, and dtype from the input graph matrix:
+
+    >>> G = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0]]).astype(np.float32)
+    >>> G
+    array([[0., 1., 1.],
+           [1., 0., 1.],
+           [1., 1., 0.]], dtype=float32)
+    >>> csgraph.laplacian(G)
+    array([[ 2., -1., -1.],
+           [-1.,  2., -1.],
+           [-1., -1.,  2.]], dtype=float32)
+
+    but can alternatively be generated matrix-free as a LinearOperator:
+
+    >>> L = csgraph.laplacian(G, form="lo")
+    >>> L
+    <3x3 _CustomLinearOperator with dtype=float32>
+    >>> L(np.eye(3))
+    array([[ 2., -1., -1.],
+           [-1.,  2., -1.],
+           [-1., -1.,  2.]])
+
+    or as a lambda-function:
+
+    >>> L = csgraph.laplacian(G, form="function")
+    >>> L
+    . at 0x0000012AE6F5A598>
+    >>> L(np.eye(3))
+    array([[ 2., -1., -1.],
+           [-1.,  2., -1.],
+           [-1., -1.,  2.]])
+
+    The Laplacian matrix is used for
+    spectral data clustering and embedding
+    as well as for spectral graph partitioning.
+    Our final example illustrates the latter
+    for a noisy directed linear graph.
+
+    >>> from scipy.sparse import diags, random
+    >>> from scipy.sparse.linalg import lobpcg
+
+    Create a directed linear graph with ``N=35`` vertices
+    using a sparse adjacency matrix ``G``:
+
+    >>> N = 35
+    >>> G = diags(np.ones(N-1), 1, format="csr")
+
+    Fix a random seed ``rng`` and add a random sparse noise to the graph ``G``:
+
+    >>> rng = np.random.default_rng()
+    >>> G += 1e-2 * random(N, N, density=0.1, random_state=rng)
+
+    Set initial approximations for eigenvectors:
+
+    >>> X = rng.random((N, 2))
+
+    The constant vector of ones is always a trivial eigenvector
+    of the non-normalized Laplacian to be filtered out:
+
+    >>> Y = np.ones((N, 1))
+
+    Alternating (1) the sign of the graph weights allows determining
+    labels for spectral max- and min- cuts in a single loop.
+    Since the graph is undirected, the option ``symmetrized=True``
+    must be used in the construction of the Laplacian.
+    The option ``normed=True`` cannot be used in (2) for the negative weights
+    here as the symmetric normalization evaluates square roots.
+    The option ``form="lo"`` in (2) is matrix-free, i.e., guarantees
+    a fixed memory footprint and read-only access to the graph.
+    Calling the eigenvalue solver ``lobpcg`` (3) computes the Fiedler vector
+    that determines the labels as the signs of its components in (5).
+    Since the sign in an eigenvector is not deterministic and can flip,
+    we fix the sign of the first component to be always +1 in (4).
+
+    >>> for cut in ["max", "min"]:
+    ...     G = -G  # 1.
+    ...     L = csgraph.laplacian(G, symmetrized=True, form="lo")  # 2.
+    ...     _, eves = lobpcg(L, X, Y=Y, largest=False, tol=1e-3)  # 3.
+    ...     eves *= np.sign(eves[0, 0])  # 4.
+    ...     print(cut + "-cut labels:\\n", 1 * (eves[:, 0]>0))  # 5.
+    max-cut labels:
+    [1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1]
+    min-cut labels:
+    [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+
+    As anticipated for a (slightly noisy) linear graph,
+    the max-cut strips all the edges of the graph coloring all
+    odd vertices into one color and all even vertices into another one,
+    while the balanced min-cut partitions the graph
+    in the middle by deleting a single edge.
+    Both determined partitions are optimal.
+    """
+    if csgraph.ndim != 2 or csgraph.shape[0] != csgraph.shape[1]:
+        raise ValueError("csgraph must be a square matrix or array")
+
+    if normed and (
+        np.issubdtype(csgraph.dtype, np.signedinteger)
+        or np.issubdtype(csgraph.dtype, np.uint)
+    ):
+        csgraph = csgraph.astype(np.float64)
+
+    if form == "array":
+        create_lap = _laplacian_sparse if issparse(csgraph) else _laplacian_dense
+    else:
+        create_lap = (
+            _laplacian_sparse_flo if issparse(csgraph) else _laplacian_dense_flo
+        )
+
+    degree_axis = 1 if use_out_degree else 0
+
+    lap, d = create_lap(
+        csgraph,
+        normed=normed,
+        axis=degree_axis,
+        copy=copy,
+        form=form,
+        dtype=dtype,
+        symmetrized=symmetrized,
+    )
+    if return_diag:
+        return lap, d
+    return lap
+
+
+def _setdiag_dense(m, d):
+    step = len(d) + 1
+    m.flat[::step] = d
+
+
+def _laplace(m, d):
+    return lambda v: v * d[:, np.newaxis] - m @ v
+
+
+def _laplace_normed(m, d, nd):
+    laplace = _laplace(m, d)
+    return lambda v: nd[:, np.newaxis] * laplace(v * nd[:, np.newaxis])
+
+
+def _laplace_sym(m, d):
+    return (
+        lambda v: v * d[:, np.newaxis]
+        - m @ v
+        - np.transpose(np.conjugate(np.transpose(np.conjugate(v)) @ m))
+    )
+
+
+def _laplace_normed_sym(m, d, nd):
+    laplace_sym = _laplace_sym(m, d)
+    return lambda v: nd[:, np.newaxis] * laplace_sym(v * nd[:, np.newaxis])
+
+
+def _linearoperator(mv, shape, dtype):
+    return LinearOperator(matvec=mv, matmat=mv, shape=shape, dtype=dtype)
+
+
+def _laplacian_sparse_flo(graph, normed, axis, copy, form, dtype, symmetrized):
+    # The keyword argument `copy` is unused and has no effect here.
+    del copy
+
+    if dtype is None:
+        dtype = graph.dtype
+
+    graph_sum = np.asarray(graph.sum(axis=axis)).ravel()
+    graph_diagonal = graph.diagonal()
+    diag = graph_sum - graph_diagonal
+    if symmetrized:
+        graph_sum += np.asarray(graph.sum(axis=1 - axis)).ravel()
+        diag = graph_sum - graph_diagonal - graph_diagonal
+
+    if normed:
+        isolated_node_mask = diag == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(diag))
+        if symmetrized:
+            md = _laplace_normed_sym(graph, graph_sum, 1.0 / w)
+        else:
+            md = _laplace_normed(graph, graph_sum, 1.0 / w)
+        if form == "function":
+            return md, w.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, w.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+    else:
+        if symmetrized:
+            md = _laplace_sym(graph, graph_sum)
+        else:
+            md = _laplace(graph, graph_sum)
+        if form == "function":
+            return md, diag.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, diag.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+
+
+def _laplacian_sparse(graph, normed, axis, copy, form, dtype, symmetrized):
+    # The keyword argument `form` is unused and has no effect here.
+    del form
+
+    if dtype is None:
+        dtype = graph.dtype
+
+    needs_copy = False
+    if graph.format in ("lil", "dok"):
+        m = graph.tocoo()
+    else:
+        m = graph
+        if copy:
+            needs_copy = True
+
+    if symmetrized:
+        m += m.T.conj()
+
+    w = np.asarray(m.sum(axis=axis)).ravel() - m.diagonal()
+    if normed:
+        m = m.tocoo(copy=needs_copy)
+        isolated_node_mask = w == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(w))
+        m.data /= w[m.row]
+        m.data /= w[m.col]
+        m.data *= -1
+        m.setdiag(1 - isolated_node_mask)
+    else:
+        if m.format == "dia":
+            m = m.copy()
+        else:
+            m = m.tocoo(copy=needs_copy)
+        m.data *= -1
+        m.setdiag(w)
+
+    return m.astype(dtype, copy=False), w.astype(dtype)
+
+
+def _laplacian_dense_flo(graph, normed, axis, copy, form, dtype, symmetrized):
+    if copy:
+        m = np.array(graph)
+    else:
+        m = np.asarray(graph)
+
+    if dtype is None:
+        dtype = m.dtype
+
+    graph_sum = m.sum(axis=axis)
+    graph_diagonal = m.diagonal()
+    diag = graph_sum - graph_diagonal
+    if symmetrized:
+        graph_sum += m.sum(axis=1 - axis)
+        diag = graph_sum - graph_diagonal - graph_diagonal
+
+    if normed:
+        isolated_node_mask = diag == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(diag))
+        if symmetrized:
+            md = _laplace_normed_sym(m, graph_sum, 1.0 / w)
+        else:
+            md = _laplace_normed(m, graph_sum, 1.0 / w)
+        if form == "function":
+            return md, w.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, w.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+    else:
+        if symmetrized:
+            md = _laplace_sym(m, graph_sum)
+        else:
+            md = _laplace(m, graph_sum)
+        if form == "function":
+            return md, diag.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, diag.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+
+
+def _laplacian_dense(graph, normed, axis, copy, form, dtype, symmetrized):
+    if form != "array":
+        raise ValueError(f'{form!r} must be "array"')
+
+    if dtype is None:
+        dtype = graph.dtype
+
+    if copy:
+        m = np.array(graph)
+    else:
+        m = np.asarray(graph)
+
+    if dtype is None:
+        dtype = m.dtype
+
+    if symmetrized:
+        m += m.T.conj()
+    np.fill_diagonal(m, 0)
+    w = m.sum(axis=axis)
+    if normed:
+        isolated_node_mask = w == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(w))
+        m /= w
+        m /= w[:, np.newaxis]
+        m *= -1
+        _setdiag_dense(m, 1 - isolated_node_mask)
+    else:
+        m *= -1
+        _setdiag_dense(m, w)
+
+    return m.astype(dtype, copy=False), w.astype(dtype, copy=False)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/LICENSE b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..769aee58ec5f51743567c67d84de5205606f813b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Consortium for Python Data API Standards
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/README.md b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f163bd4dbdbac8f93d62588840d7cb3bd207c212
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/README.md
@@ -0,0 +1 @@
+Update this directory using maint_tools/vendor_array_api_compat.sh
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..62506d5f3a5872acdd6f8ec9b8540f393babf983
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__init__.py
@@ -0,0 +1,22 @@
+"""
+NumPy Array API compatibility library
+
+This is a small wrapper around NumPy, CuPy, JAX, sparse and others that are
+compatible with the Array API standard https://data-apis.org/array-api/latest/.
+See also NEP 47 https://numpy.org/neps/nep-0047-array-api-standard.html.
+
+Unlike array_api_strict, this is not a strict minimal implementation of the
+Array API, but rather just an extension of the main NumPy namespace with
+changes needed to be compliant with the Array API. See
+https://numpy.org/doc/stable/reference/array_api.html for a full list of
+changes. In particular, unlike array_api_strict, this package does not use a
+separate Array object, but rather just uses numpy.ndarray directly.
+
+Library authors using the Array API may wish to test against array_api_strict
+to ensure they are not using functionality outside of the standard, but prefer
+this implementation for the default when working with NumPy arrays.
+
+"""
+__version__ = '1.12.0'
+
+from .common import *  # noqa: F401, F403
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d74bda52410ca6af6ecd0ad0b47f5524b6f7ec5e
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__pycache__/__init__.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__pycache__/_internal.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__pycache__/_internal.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..916ff3654ea0d0827cc286830665d82d99ffdb2e
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/__pycache__/_internal.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/_internal.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/_internal.py
new file mode 100644
index 0000000000000000000000000000000000000000..640280291c155150d9e9f5db829098ecf1e93c06
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/_internal.py
@@ -0,0 +1,59 @@
+"""
+Internal helpers
+"""
+
+from collections.abc import Callable
+from functools import wraps
+from inspect import signature
+from types import ModuleType
+from typing import TypeVar
+
+_T = TypeVar("_T")
+
+
+def get_xp(xp: ModuleType) -> Callable[[Callable[..., _T]], Callable[..., _T]]:
+    """
+    Decorator to automatically replace xp with the corresponding array module.
+
+    Use like
+
+    import numpy as np
+
+    @get_xp(np)
+    def func(x, /, xp, kwarg=None):
+        return xp.func(x, kwarg=kwarg)
+
+    Note that xp must be a keyword argument and come after all non-keyword
+    arguments.
+
+    """
+
+    def inner(f: Callable[..., _T], /) -> Callable[..., _T]:
+        @wraps(f)
+        def wrapped_f(*args: object, **kwargs: object) -> object:
+            return f(*args, xp=xp, **kwargs)
+
+        sig = signature(f)
+        new_sig = sig.replace(
+            parameters=[par for i, par in sig.parameters.items() if i != "xp"]
+        )
+
+        if wrapped_f.__doc__ is None:
+            wrapped_f.__doc__ = f"""\
+Array API compatibility wrapper for {f.__name__}.
+
+See the corresponding documentation in NumPy/CuPy and/or the array API
+specification for more details.
+
+"""
+        wrapped_f.__signature__ = new_sig  # pyright: ignore[reportAttributeAccessIssue]
+        return wrapped_f  # pyright: ignore[reportReturnType]
+
+    return inner
+
+
+__all__ = ["get_xp"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..abb3bc480b2ab6725632868f8323761f8c90279f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__init__.py
@@ -0,0 +1 @@
+from ._helpers import *  # noqa: F403
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..07c41bbed267e14a715f598ead25a32aa24c3cc6
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/__init__.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_aliases.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_aliases.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..20d2f9e7f7d4d3e4409738796abe0b25d727c7bd
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_aliases.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_fft.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_fft.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..963be6a11020822521a651c11dfcee59b9e5d4fc
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_fft.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_helpers.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_helpers.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..582fd5cf1d7527ec7385559eca5b563c4ab686ba
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_helpers.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_linalg.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_linalg.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2f469bcb54725d5bab06f58ea9ea6c0cd35a0097
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_linalg.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_typing.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_typing.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3abeef8b4b475578c933344635f2c03f2925356f
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/__pycache__/_typing.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_aliases.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_aliases.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2f168183570c53bb5e398ca4e64fd22356a6e4f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_aliases.py
@@ -0,0 +1,727 @@
+"""
+These are functions that are just aliases of existing functions in NumPy.
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Sequence, cast
+
+from ._helpers import _check_device, array_namespace
+from ._helpers import device as _get_device
+from ._helpers import is_cupy_namespace as _is_cupy_namespace
+from ._typing import Array, Device, DType, Namespace
+
+if TYPE_CHECKING:
+    # TODO: import from typing (requires Python >=3.13)
+    from typing_extensions import TypeIs
+
+# These functions are modified from the NumPy versions.
+
+# Creation functions add the device keyword (which does nothing for NumPy and Dask)
+
+
+def arange(
+    start: float,
+    /,
+    stop: float | None = None,
+    step: float = 1,
+    *,
+    xp: Namespace,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.arange(start, stop=stop, step=step, dtype=dtype, **kwargs)
+
+
+def empty(
+    shape: int | tuple[int, ...],
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.empty(shape, dtype=dtype, **kwargs)
+
+
+def empty_like(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.empty_like(x, dtype=dtype, **kwargs)
+
+
+def eye(
+    n_rows: int,
+    n_cols: int | None = None,
+    /,
+    *,
+    xp: Namespace,
+    k: int = 0,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.eye(n_rows, M=n_cols, k=k, dtype=dtype, **kwargs)
+
+
+def full(
+    shape: int | tuple[int, ...],
+    fill_value: complex,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.full(shape, fill_value, dtype=dtype, **kwargs)
+
+
+def full_like(
+    x: Array,
+    /,
+    fill_value: complex,
+    *,
+    xp: Namespace,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.full_like(x, fill_value, dtype=dtype, **kwargs)
+
+
+def linspace(
+    start: float,
+    stop: float,
+    /,
+    num: int,
+    *,
+    xp: Namespace,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    endpoint: bool = True,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.linspace(start, stop, num, dtype=dtype, endpoint=endpoint, **kwargs)
+
+
+def ones(
+    shape: int | tuple[int, ...],
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.ones(shape, dtype=dtype, **kwargs)
+
+
+def ones_like(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.ones_like(x, dtype=dtype, **kwargs)
+
+
+def zeros(
+    shape: int | tuple[int, ...],
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.zeros(shape, dtype=dtype, **kwargs)
+
+
+def zeros_like(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.zeros_like(x, dtype=dtype, **kwargs)
+
+
+# np.unique() is split into four functions in the array API:
+# unique_all, unique_counts, unique_inverse, and unique_values (this is done
+# to remove polymorphic return types).
+
+# The functions here return namedtuples (np.unique() returns a normal
+# tuple).
+
+
+# Note that these named tuples aren't actually part of the standard namespace,
+# but I don't see any issue with exporting the names here regardless.
+class UniqueAllResult(NamedTuple):
+    values: Array
+    indices: Array
+    inverse_indices: Array
+    counts: Array
+
+
+class UniqueCountsResult(NamedTuple):
+    values: Array
+    counts: Array
+
+
+class UniqueInverseResult(NamedTuple):
+    values: Array
+    inverse_indices: Array
+
+
+def _unique_kwargs(xp: Namespace) -> dict[str, bool]:
+    # Older versions of NumPy and CuPy do not have equal_nan. Rather than
+    # trying to parse version numbers, just check if equal_nan is in the
+    # signature.
+    s = inspect.signature(xp.unique)
+    if "equal_nan" in s.parameters:
+        return {"equal_nan": False}
+    return {}
+
+
+def unique_all(x: Array, /, xp: Namespace) -> UniqueAllResult:
+    kwargs = _unique_kwargs(xp)
+    values, indices, inverse_indices, counts = xp.unique(
+        x,
+        return_counts=True,
+        return_index=True,
+        return_inverse=True,
+        **kwargs,
+    )
+    # np.unique() flattens inverse indices, but they need to share x's shape
+    # See https://github.com/numpy/numpy/issues/20638
+    inverse_indices = inverse_indices.reshape(x.shape)
+    return UniqueAllResult(
+        values,
+        indices,
+        inverse_indices,
+        counts,
+    )
+
+
+def unique_counts(x: Array, /, xp: Namespace) -> UniqueCountsResult:
+    kwargs = _unique_kwargs(xp)
+    res = xp.unique(
+        x, return_counts=True, return_index=False, return_inverse=False, **kwargs
+    )
+
+    return UniqueCountsResult(*res)
+
+
+def unique_inverse(x: Array, /, xp: Namespace) -> UniqueInverseResult:
+    kwargs = _unique_kwargs(xp)
+    values, inverse_indices = xp.unique(
+        x,
+        return_counts=False,
+        return_index=False,
+        return_inverse=True,
+        **kwargs,
+    )
+    # xp.unique() flattens inverse indices, but they need to share x's shape
+    # See https://github.com/numpy/numpy/issues/20638
+    inverse_indices = inverse_indices.reshape(x.shape)
+    return UniqueInverseResult(values, inverse_indices)
+
+
+def unique_values(x: Array, /, xp: Namespace) -> Array:
+    kwargs = _unique_kwargs(xp)
+    return xp.unique(
+        x,
+        return_counts=False,
+        return_index=False,
+        return_inverse=False,
+        **kwargs,
+    )
+
+
+# These functions have different keyword argument names
+
+
+def std(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    correction: float = 0.0,  # correction instead of ddof
+    keepdims: bool = False,
+    **kwargs: object,
+) -> Array:
+    return xp.std(x, axis=axis, ddof=correction, keepdims=keepdims, **kwargs)
+
+
+def var(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    correction: float = 0.0,  # correction instead of ddof
+    keepdims: bool = False,
+    **kwargs: object,
+) -> Array:
+    return xp.var(x, axis=axis, ddof=correction, keepdims=keepdims, **kwargs)
+
+
+# cumulative_sum is renamed from cumsum, and adds the include_initial keyword
+# argument
+
+
+def cumulative_sum(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | None = None,
+    dtype: DType | None = None,
+    include_initial: bool = False,
+    **kwargs: object,
+) -> Array:
+    wrapped_xp = array_namespace(x)
+
+    # TODO: The standard is not clear about what should happen when x.ndim == 0.
+    if axis is None:
+        if x.ndim > 1:
+            raise ValueError(
+                "axis must be specified in cumulative_sum for more than one dimension"
+            )
+        axis = 0
+
+    res = xp.cumsum(x, axis=axis, dtype=dtype, **kwargs)
+
+    # np.cumsum does not support include_initial
+    if include_initial:
+        initial_shape = list(x.shape)
+        initial_shape[axis] = 1
+        res = xp.concatenate(
+            [
+                wrapped_xp.zeros(
+                    shape=initial_shape, dtype=res.dtype, device=_get_device(res)
+                ),
+                res,
+            ],
+            axis=axis,
+        )
+    return res
+
+
+def cumulative_prod(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | None = None,
+    dtype: DType | None = None,
+    include_initial: bool = False,
+    **kwargs: object,
+) -> Array:
+    wrapped_xp = array_namespace(x)
+
+    if axis is None:
+        if x.ndim > 1:
+            raise ValueError(
+                "axis must be specified in cumulative_prod for more than one dimension"
+            )
+        axis = 0
+
+    res = xp.cumprod(x, axis=axis, dtype=dtype, **kwargs)
+
+    # np.cumprod does not support include_initial
+    if include_initial:
+        initial_shape = list(x.shape)
+        initial_shape[axis] = 1
+        res = xp.concatenate(
+            [
+                wrapped_xp.ones(
+                    shape=initial_shape, dtype=res.dtype, device=_get_device(res)
+                ),
+                res,
+            ],
+            axis=axis,
+        )
+    return res
+
+
+# The min and max argument names in clip are different and not optional in numpy, and type
+# promotion behavior is different.
+def clip(
+    x: Array,
+    /,
+    min: float | Array | None = None,
+    max: float | Array | None = None,
+    *,
+    xp: Namespace,
+    # TODO: np.clip has other ufunc kwargs
+    out: Array | None = None,
+) -> Array:
+    def _isscalar(a: object) -> TypeIs[int | float | None]:
+        return isinstance(a, (int, float, type(None)))
+
+    min_shape = () if _isscalar(min) else min.shape
+    max_shape = () if _isscalar(max) else max.shape
+
+    wrapped_xp = array_namespace(x)
+
+    result_shape = xp.broadcast_shapes(x.shape, min_shape, max_shape)
+
+    # np.clip does type promotion but the array API clip requires that the
+    # output have the same dtype as x. We do this instead of just downcasting
+    # the result of xp.clip() to handle some corner cases better (e.g.,
+    # avoiding uint64 -> float64 promotion).
+
+    # Note: cases where min or max overflow (integer) or round (float) in the
+    # wrong direction when downcasting to x.dtype are unspecified. This code
+    # just does whatever NumPy does when it downcasts in the assignment, but
+    # other behavior could be preferred, especially for integers. For example,
+    # this code produces:
+
+    # >>> clip(asarray(0, dtype=int8), asarray(128, dtype=int16), None)
+    # -128
+
+    # but an answer of 0 might be preferred. See
+    # https://github.com/numpy/numpy/issues/24976 for more discussion on this issue.
+
+    # At least handle the case of Python integers correctly (see
+    # https://github.com/numpy/numpy/pull/26892).
+    if wrapped_xp.isdtype(x.dtype, "integral"):
+        if type(min) is int and min <= wrapped_xp.iinfo(x.dtype).min:
+            min = None
+        if type(max) is int and max >= wrapped_xp.iinfo(x.dtype).max:
+            max = None
+
+    dev = _get_device(x)
+    if out is None:
+        out = wrapped_xp.empty(result_shape, dtype=x.dtype, device=dev)
+    assert out is not None  # workaround for a type-narrowing issue in pyright
+    out[()] = x
+
+    if min is not None:
+        a = wrapped_xp.asarray(min, dtype=x.dtype, device=dev)
+        a = xp.broadcast_to(a, result_shape)
+        ia = (out < a) | xp.isnan(a)
+        out[ia] = a[ia]
+
+    if max is not None:
+        b = wrapped_xp.asarray(max, dtype=x.dtype, device=dev)
+        b = xp.broadcast_to(b, result_shape)
+        ib = (out > b) | xp.isnan(b)
+        out[ib] = b[ib]
+
+    # Return a scalar for 0-D
+    return out[()]
+
+
+# Unlike transpose(), the axes argument to permute_dims() is required.
+def permute_dims(x: Array, /, axes: tuple[int, ...], xp: Namespace) -> Array:
+    return xp.transpose(x, axes)
+
+
+# np.reshape calls the keyword argument 'newshape' instead of 'shape'
+def reshape(
+    x: Array,
+    /,
+    shape: tuple[int, ...],
+    xp: Namespace,
+    *,
+    copy: Optional[bool] = None,
+    **kwargs: object,
+) -> Array:
+    if copy is True:
+        x = x.copy()
+    elif copy is False:
+        y = x.view()
+        y.shape = shape
+        return y
+    return xp.reshape(x, shape, **kwargs)
+
+
+# The descending keyword is new in sort and argsort, and 'kind' replaced with
+# 'stable'
+def argsort(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int = -1,
+    descending: bool = False,
+    stable: bool = True,
+    **kwargs: object,
+) -> Array:
+    # Note: this keyword argument is different, and the default is different.
+    # We set it in kwargs like this because numpy.sort uses kind='quicksort'
+    # as the default whereas cupy.sort uses kind=None.
+    if stable:
+        kwargs["kind"] = "stable"
+    if not descending:
+        res = xp.argsort(x, axis=axis, **kwargs)
+    else:
+        # As NumPy has no native descending sort, we imitate it here. Note that
+        # simply flipping the results of xp.argsort(x, ...) would not
+        # respect the relative order like it would in native descending sorts.
+        res = xp.flip(
+            xp.argsort(xp.flip(x, axis=axis), axis=axis, **kwargs),
+            axis=axis,
+        )
+        # Rely on flip()/argsort() to validate axis
+        normalised_axis = axis if axis >= 0 else x.ndim + axis
+        max_i = x.shape[normalised_axis] - 1
+        res = max_i - res
+    return res
+
+
+def sort(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int = -1,
+    descending: bool = False,
+    stable: bool = True,
+    **kwargs: object,
+) -> Array:
+    # Note: this keyword argument is different, and the default is different.
+    # We set it in kwargs like this because numpy.sort uses kind='quicksort'
+    # as the default whereas cupy.sort uses kind=None.
+    if stable:
+        kwargs["kind"] = "stable"
+    res = xp.sort(x, axis=axis, **kwargs)
+    if descending:
+        res = xp.flip(res, axis=axis)
+    return res
+
+
+# nonzero should error for zero-dimensional arrays
+def nonzero(x: Array, /, xp: Namespace, **kwargs: object) -> tuple[Array, ...]:
+    if x.ndim == 0:
+        raise ValueError("nonzero() does not support zero-dimensional arrays")
+    return xp.nonzero(x, **kwargs)
+
+
+# ceil, floor, and trunc return integers for integer inputs
+
+
+def ceil(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if xp.issubdtype(x.dtype, xp.integer):
+        return x
+    return xp.ceil(x, **kwargs)
+
+
+def floor(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if xp.issubdtype(x.dtype, xp.integer):
+        return x
+    return xp.floor(x, **kwargs)
+
+
+def trunc(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if xp.issubdtype(x.dtype, xp.integer):
+        return x
+    return xp.trunc(x, **kwargs)
+
+
+# linear algebra functions
+
+
+def matmul(x1: Array, x2: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    return xp.matmul(x1, x2, **kwargs)
+
+
+# Unlike transpose, matrix_transpose only transposes the last two axes.
+def matrix_transpose(x: Array, /, xp: Namespace) -> Array:
+    if x.ndim < 2:
+        raise ValueError("x must be at least 2-dimensional for matrix_transpose")
+    return xp.swapaxes(x, -1, -2)
+
+
+def tensordot(
+    x1: Array,
+    x2: Array,
+    /,
+    xp: Namespace,
+    *,
+    axes: int | tuple[Sequence[int], Sequence[int]] = 2,
+    **kwargs: object,
+) -> Array:
+    return xp.tensordot(x1, x2, axes=axes, **kwargs)
+
+
+def vecdot(x1: Array, x2: Array, /, xp: Namespace, *, axis: int = -1) -> Array:
+    if x1.shape[axis] != x2.shape[axis]:
+        raise ValueError("x1 and x2 must have the same size along the given axis")
+
+    if hasattr(xp, "broadcast_tensors"):
+        _broadcast = xp.broadcast_tensors
+    else:
+        _broadcast = xp.broadcast_arrays
+
+    x1_ = xp.moveaxis(x1, axis, -1)
+    x2_ = xp.moveaxis(x2, axis, -1)
+    x1_, x2_ = _broadcast(x1_, x2_)
+
+    res = xp.conj(x1_[..., None, :]) @ x2_[..., None]
+    return res[..., 0, 0]
+
+
+# isdtype is a new function in the 2022.12 array API specification.
+
+
+def isdtype(
+    dtype: DType,
+    kind: DType | str | tuple[DType | str, ...],
+    xp: Namespace,
+    *,
+    _tuple: bool = True,  # Disallow nested tuples
+) -> bool:
+    """
+    Returns a boolean indicating whether a provided dtype is of a specified data type ``kind``.
+
+    Note that outside of this function, this compat library does not yet fully
+    support complex numbers.
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/generated/array_api.isdtype.html
+    for more details
+    """
+    if isinstance(kind, tuple) and _tuple:
+        return any(
+            isdtype(dtype, k, xp, _tuple=False)
+            for k in cast("tuple[DType | str, ...]", kind)
+        )
+    elif isinstance(kind, str):
+        if kind == "bool":
+            return dtype == xp.bool_
+        elif kind == "signed integer":
+            return xp.issubdtype(dtype, xp.signedinteger)
+        elif kind == "unsigned integer":
+            return xp.issubdtype(dtype, xp.unsignedinteger)
+        elif kind == "integral":
+            return xp.issubdtype(dtype, xp.integer)
+        elif kind == "real floating":
+            return xp.issubdtype(dtype, xp.floating)
+        elif kind == "complex floating":
+            return xp.issubdtype(dtype, xp.complexfloating)
+        elif kind == "numeric":
+            return xp.issubdtype(dtype, xp.number)
+        else:
+            raise ValueError(f"Unrecognized data type kind: {kind!r}")
+    else:
+        # This will allow things that aren't required by the spec, like
+        # isdtype(np.float64, float) or isdtype(np.int64, 'l'). Should we be
+        # more strict here to match the type annotation? Note that the
+        # array_api_strict implementation will be very strict.
+        return dtype == kind
+
+
+# unstack is a new function in the 2023.12 array API standard
+def unstack(x: Array, /, xp: Namespace, *, axis: int = 0) -> tuple[Array, ...]:
+    if x.ndim == 0:
+        raise ValueError("Input array must be at least 1-d.")
+    return tuple(xp.moveaxis(x, axis, 0))
+
+
+# numpy 1.26 does not use the standard definition for sign on complex numbers
+
+
+def sign(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if isdtype(x.dtype, "complex floating", xp=xp):
+        out = (x / xp.abs(x, **kwargs))[...]
+        # sign(0) = 0 but the above formula would give nan
+        out[x == 0j] = 0j
+    else:
+        out = xp.sign(x, **kwargs)
+    # CuPy sign() does not propagate nans. See
+    # https://github.com/data-apis/array-api-compat/issues/136
+    if _is_cupy_namespace(xp) and isdtype(x.dtype, "real floating", xp=xp):
+        out[xp.isnan(x)] = xp.nan
+    return out[()]
+
+
+def finfo(type_: DType | Array, /, xp: Namespace) -> Any:
+    # It is surprisingly difficult to recognize a dtype apart from an array.
+    # np.int64 is not the same as np.asarray(1).dtype!
+    try:
+        return xp.finfo(type_)
+    except (ValueError, TypeError):
+        return xp.finfo(type_.dtype)
+
+
+def iinfo(type_: DType | Array, /, xp: Namespace) -> Any:
+    try:
+        return xp.iinfo(type_)
+    except (ValueError, TypeError):
+        return xp.iinfo(type_.dtype)
+
+
+__all__ = [
+    "arange",
+    "empty",
+    "empty_like",
+    "eye",
+    "full",
+    "full_like",
+    "linspace",
+    "ones",
+    "ones_like",
+    "zeros",
+    "zeros_like",
+    "UniqueAllResult",
+    "UniqueCountsResult",
+    "UniqueInverseResult",
+    "unique_all",
+    "unique_counts",
+    "unique_inverse",
+    "unique_values",
+    "std",
+    "var",
+    "cumulative_sum",
+    "cumulative_prod",
+    "clip",
+    "permute_dims",
+    "reshape",
+    "argsort",
+    "sort",
+    "nonzero",
+    "ceil",
+    "floor",
+    "trunc",
+    "matmul",
+    "matrix_transpose",
+    "tensordot",
+    "vecdot",
+    "isdtype",
+    "unstack",
+    "sign",
+    "finfo",
+    "iinfo",
+]
+_all_ignore = ["inspect", "array_namespace", "NamedTuple"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_fft.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_fft.py
new file mode 100644
index 0000000000000000000000000000000000000000..8792f53cded6e4414c26548cf519f50efde9ca88
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_fft.py
@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Literal, TypeAlias
+
+from ._typing import Array, Device, DType, Namespace
+
+_Norm: TypeAlias = Literal["backward", "ortho", "forward"]
+
+# Note: NumPy fft functions improperly upcast float32 and complex64 to
+# complex128, which is why we require wrapping them all here.
+
+def fft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.fft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def ifft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.ifft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def fftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.fftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def ifftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.ifftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def rfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.rfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype == xp.float32:
+        return res.astype(xp.complex64)
+    return res
+
+def irfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.irfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype == xp.complex64:
+        return res.astype(xp.float32)
+    return res
+
+def rfftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.rfftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype == xp.float32:
+        return res.astype(xp.complex64)
+    return res
+
+def irfftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.irfftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype == xp.complex64:
+        return res.astype(xp.float32)
+    return res
+
+def hfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.hfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.float32)
+    return res
+
+def ihfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.ihfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def fftfreq(
+    n: int,
+    /,
+    xp: Namespace,
+    *,
+    d: float = 1.0,
+    dtype: DType | None = None,
+    device: Device | None = None,
+) -> Array:
+    if device not in ["cpu", None]:
+        raise ValueError(f"Unsupported device {device!r}")
+    res = xp.fft.fftfreq(n, d=d)
+    if dtype is not None:
+        return res.astype(dtype)
+    return res
+
+def rfftfreq(
+    n: int,
+    /,
+    xp: Namespace,
+    *,
+    d: float = 1.0,
+    dtype: DType | None = None,
+    device: Device | None = None,
+) -> Array:
+    if device not in ["cpu", None]:
+        raise ValueError(f"Unsupported device {device!r}")
+    res = xp.fft.rfftfreq(n, d=d)
+    if dtype is not None:
+        return res.astype(dtype)
+    return res
+
+def fftshift(
+    x: Array, /, xp: Namespace, *, axes: int | Sequence[int] | None = None
+) -> Array:
+    return xp.fft.fftshift(x, axes=axes)
+
+def ifftshift(
+    x: Array, /, xp: Namespace, *, axes: int | Sequence[int] | None = None
+) -> Array:
+    return xp.fft.ifftshift(x, axes=axes)
+
+__all__ = [
+    "fft",
+    "ifft",
+    "fftn",
+    "ifftn",
+    "rfft",
+    "irfft",
+    "rfftn",
+    "irfftn",
+    "hfft",
+    "ihfft",
+    "fftfreq",
+    "rfftfreq",
+    "fftshift",
+    "ifftshift",
+]
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_helpers.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..c07335ed7739f1a25aba5697bdf0599f1cb6dcaa
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_helpers.py
@@ -0,0 +1,1058 @@
+"""
+Various helper functions which are not part of the spec.
+
+Functions which start with an underscore are for internal use only but helpers
+that are in __all__ are intended as additional helper functions for use by end
+users of the compat library.
+"""
+
+from __future__ import annotations
+
+import inspect
+import math
+import sys
+import warnings
+from collections.abc import Collection, Hashable
+from functools import lru_cache
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    Literal,
+    SupportsIndex,
+    TypeAlias,
+    TypeGuard,
+    TypeVar,
+    cast,
+    overload,
+)
+
+from ._typing import Array, Device, HasShape, Namespace, SupportsArrayNamespace
+
+if TYPE_CHECKING:
+
+    import dask.array as da
+    import jax
+    import ndonnx as ndx
+    import numpy as np
+    import numpy.typing as npt
+    import sparse  # pyright: ignore[reportMissingTypeStubs]
+    import torch
+
+    # TODO: import from typing (requires Python >=3.13)
+    from typing_extensions import TypeIs, TypeVar
+
+    _SizeT = TypeVar("_SizeT", bound = int | None)
+
+    _ZeroGradientArray: TypeAlias = npt.NDArray[np.void]
+    _CupyArray: TypeAlias = Any  # cupy has no py.typed
+
+    _ArrayApiObj: TypeAlias = (
+        npt.NDArray[Any]
+        | da.Array
+        | jax.Array
+        | ndx.Array
+        | sparse.SparseArray
+        | torch.Tensor
+        | SupportsArrayNamespace[Any]
+        | _CupyArray
+    )
+
+_API_VERSIONS_OLD: Final = frozenset({"2021.12", "2022.12", "2023.12"})
+_API_VERSIONS: Final = _API_VERSIONS_OLD | frozenset({"2024.12"})
+
+
+@lru_cache(100)
+def _issubclass_fast(cls: type, modname: str, clsname: str) -> bool:
+    try:
+        mod = sys.modules[modname]
+    except KeyError:
+        return False
+    parent_cls = getattr(mod, clsname)
+    return issubclass(cls, parent_cls)
+
+
+def _is_jax_zero_gradient_array(x: object) -> TypeGuard[_ZeroGradientArray]:
+    """Return True if `x` is a zero-gradient array.
+
+    These arrays are a design quirk of Jax that may one day be removed.
+    See https://github.com/google/jax/issues/20620.
+    """
+    # Fast exit
+    try:
+        dtype = x.dtype  # type: ignore[attr-defined]
+    except AttributeError:
+        return False
+    cls = cast(Hashable, type(dtype))
+    if not _issubclass_fast(cls, "numpy.dtypes", "VoidDType"):
+        return False
+
+    if "jax" not in sys.modules:
+        return False
+
+    import jax
+    # jax.float0 is a np.dtype([('float0', 'V')])
+    return dtype == jax.float0
+
+
+def is_numpy_array(x: object) -> TypeGuard[npt.NDArray[Any]]:
+    """
+    Return True if `x` is a NumPy array.
+
+    This function does not import NumPy if it has not already been imported
+    and is therefore cheap to use.
+
+    This also returns True for `ndarray` subclasses and NumPy scalar objects.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    # TODO: Should we reject ndarray subclasses?
+    cls = cast(Hashable, type(x))
+    return (
+        _issubclass_fast(cls, "numpy", "ndarray") 
+        or _issubclass_fast(cls, "numpy", "generic")
+    ) and not _is_jax_zero_gradient_array(x)
+
+
+def is_cupy_array(x: object) -> bool:
+    """
+    Return True if `x` is a CuPy array.
+
+    This function does not import CuPy if it has not already been imported
+    and is therefore cheap to use.
+
+    This also returns True for `cupy.ndarray` subclasses and CuPy scalar objects.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "cupy", "ndarray")
+
+
+def is_torch_array(x: object) -> TypeIs[torch.Tensor]:
+    """
+    Return True if `x` is a PyTorch tensor.
+
+    This function does not import PyTorch if it has not already been imported
+    and is therefore cheap to use.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "torch", "Tensor")
+
+
+def is_ndonnx_array(x: object) -> TypeIs[ndx.Array]:
+    """
+    Return True if `x` is a ndonnx Array.
+
+    This function does not import ndonnx if it has not already been imported
+    and is therefore cheap to use.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "ndonnx", "Array")
+
+
+def is_dask_array(x: object) -> TypeIs[da.Array]:
+    """
+    Return True if `x` is a dask.array Array.
+
+    This function does not import dask if it has not already been imported
+    and is therefore cheap to use.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "dask.array", "Array")
+
+
+def is_jax_array(x: object) -> TypeIs[jax.Array]:
+    """
+    Return True if `x` is a JAX array.
+
+    This function does not import JAX if it has not already been imported
+    and is therefore cheap to use.
+
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "jax", "Array") or _is_jax_zero_gradient_array(x)
+
+
+def is_pydata_sparse_array(x: object) -> TypeIs[sparse.SparseArray]:
+    """
+    Return True if `x` is an array from the `sparse` package.
+
+    This function does not import `sparse` if it has not already been imported
+    and is therefore cheap to use.
+
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    """
+    # TODO: Account for other backends.
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "sparse", "SparseArray")
+
+
+def is_array_api_obj(x: object) -> TypeIs[_ArrayApiObj]:  # pyright: ignore[reportUnknownParameterType]
+    """
+    Return True if `x` is an array API compatible array object.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    """
+    return (
+        hasattr(x, '__array_namespace__') 
+        or _is_array_api_cls(cast(Hashable, type(x)))
+    )
+
+
+@lru_cache(100)
+def _is_array_api_cls(cls: type) -> bool:
+    return (
+        # TODO: drop support for numpy<2 which didn't have __array_namespace__
+        _issubclass_fast(cls, "numpy", "ndarray")
+        or _issubclass_fast(cls, "numpy", "generic")
+        or _issubclass_fast(cls, "cupy", "ndarray")
+        or _issubclass_fast(cls, "torch", "Tensor")
+        or _issubclass_fast(cls, "dask.array", "Array")
+        or _issubclass_fast(cls, "sparse", "SparseArray")
+        # TODO: drop support for jax<0.4.32 which didn't have __array_namespace__
+        or _issubclass_fast(cls, "jax", "Array")
+    )
+
+
+def _compat_module_name() -> str:
+    assert __name__.endswith(".common._helpers")
+    return __name__.removesuffix(".common._helpers")
+
+
+@lru_cache(100)
+def is_numpy_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a NumPy namespace.
+
+    This includes both NumPy itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"numpy", _compat_module_name() + ".numpy"}
+
+
+@lru_cache(100)
+def is_cupy_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a CuPy namespace.
+
+    This includes both CuPy itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"cupy", _compat_module_name() + ".cupy"}
+
+
+@lru_cache(100)
+def is_torch_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a PyTorch namespace.
+
+    This includes both PyTorch itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"torch", _compat_module_name() + ".torch"}
+
+
+def is_ndonnx_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is an NDONNX namespace.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ == "ndonnx"
+
+
+@lru_cache(100)
+def is_dask_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a Dask namespace.
+
+    This includes both ``dask.array`` itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"dask.array", _compat_module_name() + ".dask.array"}
+
+
+def is_jax_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a JAX namespace.
+
+    This includes ``jax.numpy`` and ``jax.experimental.array_api`` which existed in
+    older versions of JAX.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"jax.numpy", "jax.experimental.array_api"}
+
+
+def is_pydata_sparse_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a pydata/sparse namespace.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ == "sparse"
+
+
+def is_array_api_strict_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is an array-api-strict namespace.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    """
+    return xp.__name__ == "array_api_strict"
+
+
+def _check_api_version(api_version: str | None) -> None:
+    if api_version in _API_VERSIONS_OLD:
+        warnings.warn(
+            f"The {api_version} version of the array API specification was requested but the returned namespace is actually version 2024.12"
+        )
+    elif api_version is not None and api_version not in _API_VERSIONS:
+        raise ValueError(
+            "Only the 2024.12 version of the array API specification is currently supported"
+        )
+
+
+def array_namespace(
+    *xs: Array | complex | None,
+    api_version: str | None = None,
+    use_compat: bool | None = None,
+) -> Namespace:
+    """
+    Get the array API compatible namespace for the arrays `xs`.
+
+    Parameters
+    ----------
+    xs: arrays
+        one or more arrays. xs can also be Python scalars (bool, int, float,
+        complex, or None), which are ignored.
+
+    api_version: str
+        The newest version of the spec that you need support for (currently
+        the compat library wrapped APIs support v2024.12).
+
+    use_compat: bool or None
+        If None (the default), the native namespace will be returned if it is
+        already array API compatible, otherwise a compat wrapper is used. If
+        True, the compat library wrapped library will be returned. If False,
+        the native library namespace is returned.
+
+    Returns
+    -------
+
+    out: namespace
+        The array API compatible namespace corresponding to the arrays in `xs`.
+
+    Raises
+    ------
+    TypeError
+        If `xs` contains arrays from different array libraries or contains a
+        non-array.
+
+
+    Typical usage is to pass the arguments of a function to
+    `array_namespace()` at the top of a function to get the corresponding
+    array API namespace:
+
+    .. code:: python
+
+       def your_function(x, y):
+           xp = array_api_compat.array_namespace(x, y)
+           # Now use xp as the array library namespace
+           return xp.mean(x, axis=0) + 2*xp.std(y, axis=0)
+
+
+    Wrapped array namespaces can also be imported directly. For example,
+    `array_namespace(np.array(...))` will return `array_api_compat.numpy`.
+    This function will also work for any array library not wrapped by
+    array-api-compat if it explicitly defines `__array_namespace__
+    `__
+    (the wrapped namespace is always preferred if it exists).
+
+    See Also
+    --------
+
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+
+    """
+    if use_compat not in [None, True, False]:
+        raise ValueError("use_compat must be None, True, or False")
+
+    _use_compat = use_compat in [None, True]
+
+    namespaces: set[Namespace] = set()
+    for x in xs:
+        if is_numpy_array(x):
+            import numpy as np
+
+            from .. import numpy as numpy_namespace
+
+            if use_compat is True:
+                _check_api_version(api_version)
+                namespaces.add(numpy_namespace)
+            elif use_compat is False:
+                namespaces.add(np)
+            else:
+                # numpy 2.0+ have __array_namespace__, however, they are not yet fully array API
+                # compatible.
+                namespaces.add(numpy_namespace)
+        elif is_cupy_array(x):
+            if _use_compat:
+                _check_api_version(api_version)
+                from .. import cupy as cupy_namespace
+
+                namespaces.add(cupy_namespace)
+            else:
+                import cupy as cp  # pyright: ignore[reportMissingTypeStubs]
+
+                namespaces.add(cp)
+        elif is_torch_array(x):
+            if _use_compat:
+                _check_api_version(api_version)
+                from .. import torch as torch_namespace
+
+                namespaces.add(torch_namespace)
+            else:
+                import torch
+
+                namespaces.add(torch)
+        elif is_dask_array(x):
+            if _use_compat:
+                _check_api_version(api_version)
+                from ..dask import array as dask_namespace
+
+                namespaces.add(dask_namespace)
+            else:
+                import dask.array as da
+
+                namespaces.add(da)
+        elif is_jax_array(x):
+            if use_compat is True:
+                _check_api_version(api_version)
+                raise ValueError("JAX does not have an array-api-compat wrapper")
+            elif use_compat is False:
+                import jax.numpy as jnp
+            else:
+                # JAX v0.4.32 and newer implements the array API directly in jax.numpy.
+                # For older JAX versions, it is available via jax.experimental.array_api.
+                import jax.numpy
+
+                if hasattr(jax.numpy, "__array_api_version__"):
+                    jnp = jax.numpy
+                else:
+                    import jax.experimental.array_api as jnp  # pyright: ignore[reportMissingImports]
+            namespaces.add(jnp)
+        elif is_pydata_sparse_array(x):
+            if use_compat is True:
+                _check_api_version(api_version)
+                raise ValueError("`sparse` does not have an array-api-compat wrapper")
+            else:
+                import sparse  # pyright: ignore[reportMissingTypeStubs]
+            # `sparse` is already an array namespace. We do not have a wrapper
+            # submodule for it.
+            namespaces.add(sparse)
+        elif hasattr(x, "__array_namespace__"):
+            if use_compat is True:
+                raise ValueError(
+                    "The given array does not have an array-api-compat wrapper"
+                )
+            x = cast("SupportsArrayNamespace[Any]", x)
+            namespaces.add(x.__array_namespace__(api_version=api_version))
+        elif isinstance(x, (bool, int, float, complex, type(None))):
+            continue
+        else:
+            # TODO: Support Python scalars?
+            raise TypeError(f"{type(x).__name__} is not a supported array type")
+
+    if not namespaces:
+        raise TypeError("Unrecognized array input")
+
+    if len(namespaces) != 1:
+        raise TypeError(f"Multiple namespaces for array inputs: {namespaces}")
+
+    (xp,) = namespaces
+
+    return xp
+
+
+# backwards compatibility alias
+get_namespace = array_namespace
+
+
+def _check_device(bare_xp: Namespace, device: Device) -> None:  # pyright: ignore[reportUnusedFunction]
+    """
+    Validate dummy device on device-less array backends.
+
+    Notes
+    -----
+    This function is also invoked by CuPy, which does have multiple devices
+    if there are multiple GPUs available.
+    However, CuPy multi-device support is currently impossible
+    without using the global device or a context manager:
+
+    https://github.com/data-apis/array-api-compat/pull/293
+    """
+    if bare_xp is sys.modules.get("numpy"):
+        if device not in ("cpu", None):
+            raise ValueError(f"Unsupported device for NumPy: {device!r}")
+
+    elif bare_xp is sys.modules.get("dask.array"):
+        if device not in ("cpu", _DASK_DEVICE, None):
+            raise ValueError(f"Unsupported device for Dask: {device!r}")
+
+
+# Placeholder object to represent the dask device
+# when the array backend is not the CPU.
+# (since it is not easy to tell which device a dask array is on)
+class _dask_device:
+    def __repr__(self) -> Literal["DASK_DEVICE"]:
+        return "DASK_DEVICE"
+
+
+_DASK_DEVICE = _dask_device()
+
+
+# device() is not on numpy.ndarray or dask.array and to_device() is not on numpy.ndarray
+# or cupy.ndarray. They are not included in array objects of this library
+# because this library just reuses the respective ndarray classes without
+# wrapping or subclassing them. These helper functions can be used instead of
+# the wrapper functions for libraries that need to support both NumPy/CuPy and
+# other libraries that use devices.
+def device(x: _ArrayApiObj, /) -> Device:
+    """
+    Hardware device the array data resides on.
+
+    This is equivalent to `x.device` according to the `standard
+    `__.
+    This helper is included because some array libraries either do not have
+    the `device` attribute or include it with an incompatible API.
+
+    Parameters
+    ----------
+    x: array
+        array instance from an array API compatible library.
+
+    Returns
+    -------
+    out: device
+        a ``device`` object (see the `Device Support `__
+        section of the array API specification).
+
+    Notes
+    -----
+
+    For NumPy the device is always `"cpu"`. For Dask, the device is always a
+    special `DASK_DEVICE` object.
+
+    See Also
+    --------
+
+    to_device : Move array data to a different device.
+
+    """
+    if is_numpy_array(x):
+        return "cpu"
+    elif is_dask_array(x):
+        # Peek at the metadata of the Dask array to determine type
+        if is_numpy_array(x._meta):  # pyright: ignore
+            # Must be on CPU since backed by numpy
+            return "cpu"
+        return _DASK_DEVICE
+    elif is_jax_array(x):
+        # FIXME Jitted JAX arrays do not have a device attribute
+        #       https://github.com/jax-ml/jax/issues/26000
+        #       Return None in this case. Note that this workaround breaks
+        #       the standard and will result in new arrays being created on the
+        #       default device instead of the same device as the input array(s).
+        x_device = getattr(x, "device", None)
+        # Older JAX releases had .device() as a method, which has been replaced
+        # with a property in accordance with the standard.
+        if inspect.ismethod(x_device):
+            return x_device()
+        else:
+            return x_device
+    elif is_pydata_sparse_array(x):
+        # `sparse` will gain `.device`, so check for this first.
+        x_device = getattr(x, "device", None)
+        if x_device is not None:
+            return x_device
+        # Everything but DOK has this attr.
+        try:
+            inner = x.data  # pyright: ignore
+        except AttributeError:
+            return "cpu"
+        # Return the device of the constituent array
+        return device(inner)  # pyright: ignore
+    return x.device  # pyright: ignore
+
+
+# Prevent shadowing, used below
+_device = device
+
+
+# Based on cupy.array_api.Array.to_device
+def _cupy_to_device(
+    x: _CupyArray,
+    device: Device,
+    /,
+    stream: int | Any | None = None,
+) -> _CupyArray:
+    import cupy as cp
+
+    if device == "cpu":
+        # allowing us to use `to_device(x, "cpu")`
+        # is useful for portable test swapping between
+        # host and device backends
+        return x.get()
+    if not isinstance(device, cp.cuda.Device):
+        raise TypeError(f"Unsupported device type {device!r}")
+
+    if stream is None:
+        with device:
+            return cp.asarray(x)
+
+    # stream can be an int as specified in __dlpack__, or a CuPy stream
+    if isinstance(stream, int):
+        stream = cp.cuda.ExternalStream(stream)
+    elif not isinstance(stream, cp.cuda.Stream):
+        raise TypeError(f"Unsupported stream type {stream!r}")
+
+    with device, stream:
+        return cp.asarray(x)
+
+
+def _torch_to_device(
+    x: torch.Tensor,
+    device: torch.device | str | int,
+    /,
+    stream: None = None,
+) -> torch.Tensor:
+    if stream is not None:
+        raise NotImplementedError
+    return x.to(device)
+
+
+def to_device(x: Array, device: Device, /, *, stream: int | Any | None = None) -> Array:
+    """
+    Copy the array from the device on which it currently resides to the specified ``device``.
+
+    This is equivalent to `x.to_device(device, stream=stream)` according to
+    the `standard
+    `__.
+    This helper is included because some array libraries do not have the
+    `to_device` method.
+
+    Parameters
+    ----------
+
+    x: array
+        array instance from an array API compatible library.
+
+    device: device
+        a ``device`` object (see the `Device Support `__
+        section of the array API specification).
+
+    stream: int | Any | None
+        stream object to use during copy. In addition to the types supported
+        in ``array.__dlpack__``, implementations may choose to support any
+        library-specific stream object with the caveat that any code using
+        such an object would not be portable.
+
+    Returns
+    -------
+
+    out: array
+        an array with the same data and data type as ``x`` and located on the
+        specified ``device``.
+
+    Notes
+    -----
+
+    For NumPy, this function effectively does nothing since the only supported
+    device is the CPU. For CuPy, this method supports CuPy CUDA
+    :external+cupy:class:`Device ` and
+    :external+cupy:class:`Stream ` objects. For PyTorch,
+    this is the same as :external+torch:meth:`x.to(device) `
+    (the ``stream`` argument is not supported in PyTorch).
+
+    See Also
+    --------
+
+    device : Hardware device the array data resides on.
+
+    """
+    if is_numpy_array(x):
+        if stream is not None:
+            raise ValueError("The stream argument to to_device() is not supported")
+        if device == "cpu":
+            return x
+        raise ValueError(f"Unsupported device {device!r}")
+    elif is_cupy_array(x):
+        # cupy does not yet have to_device
+        return _cupy_to_device(x, device, stream=stream)
+    elif is_torch_array(x):
+        return _torch_to_device(x, device, stream=stream)  # pyright: ignore[reportArgumentType]
+    elif is_dask_array(x):
+        if stream is not None:
+            raise ValueError("The stream argument to to_device() is not supported")
+        # TODO: What if our array is on the GPU already?
+        if device == "cpu":
+            return x
+        raise ValueError(f"Unsupported device {device!r}")
+    elif is_jax_array(x):
+        if not hasattr(x, "__array_namespace__"):
+            # In JAX v0.4.31 and older, this import adds to_device method to x...
+            import jax.experimental.array_api  # noqa: F401  # pyright: ignore
+
+            # ... but only on eager JAX. It won't work inside jax.jit.
+            if not hasattr(x, "to_device"):
+                return x
+        return x.to_device(device, stream=stream)
+    elif is_pydata_sparse_array(x) and device == _device(x):
+        # Perform trivial check to return the same array if
+        # device is same instead of err-ing.
+        return x
+    return x.to_device(device, stream=stream)  # pyright: ignore
+
+
+@overload
+def size(x: HasShape[Collection[SupportsIndex]]) -> int: ...
+@overload
+def size(x: HasShape[Collection[None]]) -> None: ...
+@overload
+def size(x: HasShape[Collection[SupportsIndex | None]]) -> int | None: ...
+def size(x: HasShape[Collection[SupportsIndex | None]]) -> int | None:
+    """
+    Return the total number of elements of x.
+
+    This is equivalent to `x.size` according to the `standard
+    `__.
+
+    This helper is included because PyTorch defines `size` in an
+    :external+torch:meth:`incompatible way `.
+    It also fixes dask.array's behaviour which returns nan for unknown sizes, whereas
+    the standard requires None.
+    """
+    # Lazy API compliant arrays, such as ndonnx, can contain None in their shape
+    if None in x.shape:
+        return None
+    out = math.prod(cast("Collection[SupportsIndex]", x.shape))
+    # dask.array.Array.shape can contain NaN
+    return None if math.isnan(out) else out
+
+
+@lru_cache(100)
+def _is_writeable_cls(cls: type) -> bool | None:
+    if (
+        _issubclass_fast(cls, "numpy", "generic")
+        or _issubclass_fast(cls, "jax", "Array")
+        or _issubclass_fast(cls, "sparse", "SparseArray")
+    ):
+        return False
+    if _is_array_api_cls(cls):
+        return True
+    return None
+
+
+def is_writeable_array(x: object) -> bool:
+    """
+    Return False if ``x.__setitem__`` is expected to raise; True otherwise.
+    Return False if `x` is not an array API compatible object.
+
+    Warning
+    -------
+    As there is no standard way to check if an array is writeable without actually
+    writing to it, this function blindly returns True for all unknown array types.
+    """
+    cls = cast(Hashable, type(x))
+    if _issubclass_fast(cls, "numpy", "ndarray"):
+        return cast("npt.NDArray", x).flags.writeable
+    res = _is_writeable_cls(cls)
+    if res is not None:
+        return res
+    return hasattr(x, '__array_namespace__')
+
+
+@lru_cache(100)
+def _is_lazy_cls(cls: type) -> bool | None:
+    if (
+        _issubclass_fast(cls, "numpy", "ndarray")
+        or _issubclass_fast(cls, "numpy", "generic")
+        or _issubclass_fast(cls, "cupy", "ndarray")
+        or _issubclass_fast(cls, "torch", "Tensor")
+        or _issubclass_fast(cls, "sparse", "SparseArray")
+    ):
+        return False
+    if (
+        _issubclass_fast(cls, "jax", "Array")
+        or _issubclass_fast(cls, "dask.array", "Array")
+        or _issubclass_fast(cls, "ndonnx", "Array")
+    ):
+        return True
+    return  None
+
+
+def is_lazy_array(x: object) -> bool:
+    """Return True if x is potentially a future or it may be otherwise impossible or
+    expensive to eagerly read its contents, regardless of their size, e.g. by
+    calling ``bool(x)`` or ``float(x)``.
+
+    Return False otherwise; e.g. ``bool(x)`` etc. is guaranteed to succeed and to be
+    cheap as long as the array has the right dtype and size.
+
+    Note
+    ----
+    This function errs on the side of caution for array types that may or may not be
+    lazy, e.g. JAX arrays, by always returning True for them.
+    """
+    # **JAX note:** while it is possible to determine if you're inside or outside
+    # jax.jit by testing the subclass of a jax.Array object, as well as testing bool()
+    # as we do below for unknown arrays, this is not recommended by JAX best practices.
+
+    # **Dask note:** Dask eagerly computes the graph on __bool__, __float__, and so on.
+    # This behaviour, while impossible to change without breaking backwards
+    # compatibility, is highly detrimental to performance as the whole graph will end
+    # up being computed multiple times.
+
+    # Note: skipping reclassification of JAX zero gradient arrays, as one will
+    # exclusively get them once they leave a jax.grad JIT context.
+    cls = cast(Hashable, type(x))
+    res = _is_lazy_cls(cls)
+    if res is not None:
+        return res
+
+    if not hasattr(x, "__array_namespace__"):
+        return False
+
+    # Unknown Array API compatible object. Note that this test may have dire consequences
+    # in terms of performance, e.g. for a lazy object that eagerly computes the graph
+    # on __bool__ (dask is one such example, which however is special-cased above).
+
+    # Select a single point of the array
+    s = size(cast("HasShape[Collection[SupportsIndex | None]]", x))
+    if s is None:
+        return True
+    xp = array_namespace(x)
+    if s > 1:
+        x = xp.reshape(x, (-1,))[0]
+    # Cast to dtype=bool and deal with size 0 arrays
+    x = xp.any(x)
+
+    try:
+        bool(x)
+        return False
+    # The Array API standard dictactes that __bool__ should raise TypeError if the
+    # output cannot be defined.
+    # Here we allow for it to raise arbitrary exceptions, e.g. like Dask does.
+    except Exception:
+        return True
+
+
+__all__ = [
+    "array_namespace",
+    "device",
+    "get_namespace",
+    "is_array_api_obj",
+    "is_array_api_strict_namespace",
+    "is_cupy_array",
+    "is_cupy_namespace",
+    "is_dask_array",
+    "is_dask_namespace",
+    "is_jax_array",
+    "is_jax_namespace",
+    "is_numpy_array",
+    "is_numpy_namespace",
+    "is_torch_array",
+    "is_torch_namespace",
+    "is_ndonnx_array",
+    "is_ndonnx_namespace",
+    "is_pydata_sparse_array",
+    "is_pydata_sparse_namespace",
+    "is_writeable_array",
+    "is_lazy_array",
+    "size",
+    "to_device",
+]
+
+_all_ignore = ['lru_cache', 'sys', 'math', 'inspect', 'warnings']
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_linalg.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_linalg.py
new file mode 100644
index 0000000000000000000000000000000000000000..071e612fdae84c1494df4ea3783af6e040e3db06
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_linalg.py
@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import math
+from typing import Literal, NamedTuple, cast
+
+import numpy as np
+
+if np.__version__[0] == "2":
+    from numpy.lib.array_utils import normalize_axis_tuple
+else:
+    from numpy.core.numeric import normalize_axis_tuple
+
+from .._internal import get_xp
+from ._aliases import isdtype, matmul, matrix_transpose, tensordot, vecdot
+from ._typing import Array, DType, JustFloat, JustInt, Namespace
+
+
+# These are in the main NumPy namespace but not in numpy.linalg
+def cross(
+    x1: Array,
+    x2: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int = -1,
+    **kwargs: object,
+) -> Array:
+    return xp.cross(x1, x2, axis=axis, **kwargs)
+
+def outer(x1: Array, x2: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    return xp.outer(x1, x2, **kwargs)
+
+class EighResult(NamedTuple):
+    eigenvalues: Array
+    eigenvectors: Array
+
+class QRResult(NamedTuple):
+    Q: Array
+    R: Array
+
+class SlogdetResult(NamedTuple):
+    sign: Array
+    logabsdet: Array
+
+class SVDResult(NamedTuple):
+    U: Array
+    S: Array
+    Vh: Array
+
+# These functions are the same as their NumPy counterparts except they return
+# a namedtuple.
+def eigh(x: Array, /, xp: Namespace, **kwargs: object) -> EighResult:
+    return EighResult(*xp.linalg.eigh(x, **kwargs))
+
+def qr(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    mode: Literal["reduced", "complete"] = "reduced",
+    **kwargs: object,
+) -> QRResult:
+    return QRResult(*xp.linalg.qr(x, mode=mode, **kwargs))
+
+def slogdet(x: Array, /, xp: Namespace, **kwargs: object) -> SlogdetResult:
+    return SlogdetResult(*xp.linalg.slogdet(x, **kwargs))
+
+def svd(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    full_matrices: bool = True,
+    **kwargs: object,
+) -> SVDResult:
+    return SVDResult(*xp.linalg.svd(x, full_matrices=full_matrices, **kwargs))
+
+# These functions have additional keyword arguments
+
+# The upper keyword argument is new from NumPy
+def cholesky(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    upper: bool = False,
+    **kwargs: object,
+) -> Array:
+    L = xp.linalg.cholesky(x, **kwargs)
+    if upper:
+        U = get_xp(xp)(matrix_transpose)(L)
+        if get_xp(xp)(isdtype)(U.dtype, 'complex floating'):
+            U = xp.conj(U)  # pyright: ignore[reportConstantRedefinition]
+        return U
+    return L
+
+# The rtol keyword argument of matrix_rank() and pinv() is new from NumPy.
+# Note that it has a different semantic meaning from tol and rcond.
+def matrix_rank(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    rtol: float | Array | None = None,
+    **kwargs: object,
+) -> Array:
+    # this is different from xp.linalg.matrix_rank, which supports 1
+    # dimensional arrays.
+    if x.ndim < 2:
+        raise xp.linalg.LinAlgError("1-dimensional array given. Array must be at least two-dimensional")
+    S: Array = get_xp(xp)(svdvals)(x, **kwargs)
+    if rtol is None:
+        tol = S.max(axis=-1, keepdims=True) * max(x.shape[-2:]) * xp.finfo(S.dtype).eps
+    else:
+        # this is different from xp.linalg.matrix_rank, which does not
+        # multiply the tolerance by the largest singular value.
+        tol = S.max(axis=-1, keepdims=True)*xp.asarray(rtol)[..., xp.newaxis]
+    return xp.count_nonzero(S > tol, axis=-1)
+
+def pinv(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    rtol: float | Array | None = None,
+    **kwargs: object,
+) -> Array:
+    # this is different from xp.linalg.pinv, which does not multiply the
+    # default tolerance by max(M, N).
+    if rtol is None:
+        rtol = max(x.shape[-2:]) * xp.finfo(x.dtype).eps
+    return xp.linalg.pinv(x, rcond=rtol, **kwargs)
+
+# These functions are new in the array API spec
+
+def matrix_norm(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    keepdims: bool = False,
+    ord: Literal[1, 2, -1, -2] | JustFloat | Literal["fro", "nuc"] | None = "fro",
+) -> Array:
+    return xp.linalg.norm(x, axis=(-2, -1), keepdims=keepdims, ord=ord)
+
+# svdvals is not in NumPy (but it is in SciPy). It is equivalent to
+# xp.linalg.svd(compute_uv=False).
+def svdvals(x: Array, /, xp: Namespace) -> Array | tuple[Array, ...]:
+    return xp.linalg.svd(x, compute_uv=False)
+
+def vector_norm(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    keepdims: bool = False,
+    ord: JustInt | JustFloat = 2,
+) -> Array:
+    # xp.linalg.norm tries to do a matrix norm whenever axis is a 2-tuple or
+    # when axis=None and the input is 2-D, so to force a vector norm, we make
+    # it so the input is 1-D (for axis=None), or reshape so that norm is done
+    # on a single dimension.
+    if axis is None:
+        # Note: xp.linalg.norm() doesn't handle 0-D arrays
+        _x = x.ravel()
+        _axis = 0
+    elif isinstance(axis, tuple):
+        # Note: The axis argument supports any number of axes, whereas
+        # xp.linalg.norm() only supports a single axis for vector norm.
+        normalized_axis = cast(
+            "tuple[int, ...]",
+            normalize_axis_tuple(axis, x.ndim),  # pyright: ignore[reportCallIssue]
+        )
+        rest = tuple(i for i in range(x.ndim) if i not in normalized_axis)
+        newshape = axis + rest
+        _x = xp.transpose(x, newshape).reshape(
+            (math.prod([x.shape[i] for i in axis]), *[x.shape[i] for i in rest]))
+        _axis = 0
+    else:
+        _x = x
+        _axis = axis
+
+    res = xp.linalg.norm(_x, axis=_axis, ord=ord)
+
+    if keepdims:
+        # We can't reuse xp.linalg.norm(keepdims) because of the reshape hacks
+        # above to avoid matrix norm logic.
+        shape = list(x.shape)
+        _axis = cast(
+            "tuple[int, ...]",
+            normalize_axis_tuple(  # pyright: ignore[reportCallIssue]
+                range(x.ndim) if axis is None else axis,
+                x.ndim,
+            ),
+        )
+        for i in _axis:
+            shape[i] = 1
+        res = xp.reshape(res, tuple(shape))
+
+    return res
+
+# xp.diagonal and xp.trace operate on the first two axes whereas these
+# operates on the last two
+
+def diagonal(x: Array, /, xp: Namespace, *, offset: int = 0, **kwargs: object) -> Array:
+    return xp.diagonal(x, offset=offset, axis1=-2, axis2=-1, **kwargs)
+
+def trace(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    offset: int = 0,
+    dtype: DType | None = None,
+    **kwargs: object,
+) -> Array:
+    return xp.asarray(
+        xp.trace(x, offset=offset, dtype=dtype, axis1=-2, axis2=-1, **kwargs)
+    )
+
+__all__ = ['cross', 'matmul', 'outer', 'tensordot', 'EighResult',
+           'QRResult', 'SlogdetResult', 'SVDResult', 'eigh', 'qr', 'slogdet',
+           'svd', 'cholesky', 'matrix_rank', 'pinv', 'matrix_norm',
+           'matrix_transpose', 'svdvals', 'vecdot', 'vector_norm', 'diagonal',
+           'trace']
+
+_all_ignore = ['math', 'normalize_axis_tuple', 'get_xp', 'np', 'isdtype']
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_typing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc402656747130098f7807fb9a9d5476d759578f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/common/_typing.py
@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from types import ModuleType as Namespace
+from typing import (
+    TYPE_CHECKING,
+    Literal,
+    Protocol,
+    TypeAlias,
+    TypedDict,
+    TypeVar,
+    final,
+)
+
+if TYPE_CHECKING:
+    from _typeshed import Incomplete
+
+    SupportsBufferProtocol: TypeAlias = Incomplete
+    Array: TypeAlias = Incomplete
+    Device: TypeAlias = Incomplete
+    DType: TypeAlias = Incomplete
+else:
+    SupportsBufferProtocol = object
+    Array = object
+    Device = object
+    DType = object
+
+
+_T_co = TypeVar("_T_co", covariant=True)
+
+
+# These "Just" types are equivalent to the `Just` type from the `optype` library,
+# apart from them not being `@runtime_checkable`.
+# - docs: https://github.com/jorenham/optype/blob/master/README.md#just
+# - code: https://github.com/jorenham/optype/blob/master/optype/_core/_just.py
+@final
+class JustInt(Protocol):
+    @property
+    def __class__(self, /) -> type[int]: ...
+    @__class__.setter
+    def __class__(self, value: type[int], /) -> None: ...  # pyright: ignore[reportIncompatibleMethodOverride]
+
+
+@final
+class JustFloat(Protocol):
+    @property
+    def __class__(self, /) -> type[float]: ...
+    @__class__.setter
+    def __class__(self, value: type[float], /) -> None: ...  # pyright: ignore[reportIncompatibleMethodOverride]
+
+
+@final
+class JustComplex(Protocol):
+    @property
+    def __class__(self, /) -> type[complex]: ...
+    @__class__.setter
+    def __class__(self, value: type[complex], /) -> None: ...  # pyright: ignore[reportIncompatibleMethodOverride]
+
+
+#
+
+
+class NestedSequence(Protocol[_T_co]):
+    def __getitem__(self, key: int, /) -> _T_co | NestedSequence[_T_co]: ...
+    def __len__(self, /) -> int: ...
+
+
+class SupportsArrayNamespace(Protocol[_T_co]):
+    def __array_namespace__(self, /, *, api_version: str | None) -> _T_co: ...
+
+
+class HasShape(Protocol[_T_co]):
+    @property
+    def shape(self, /) -> _T_co: ...
+
+
+# Return type of `__array_namespace_info__.default_dtypes`
+Capabilities = TypedDict(
+    "Capabilities",
+    {
+        "boolean indexing": bool,
+        "data-dependent shapes": bool,
+        "max dimensions": int,
+    },
+)
+
+# Return type of `__array_namespace_info__.default_dtypes`
+DefaultDTypes = TypedDict(
+    "DefaultDTypes",
+    {
+        "real floating": DType,
+        "complex floating": DType,
+        "integral": DType,
+        "indexing": DType,
+    },
+)
+
+
+_DTypeKind: TypeAlias = Literal[
+    "bool",
+    "signed integer",
+    "unsigned integer",
+    "integral",
+    "real floating",
+    "complex floating",
+    "numeric",
+]
+# Type of the `kind` parameter in `__array_namespace_info__.dtypes`
+DTypeKind: TypeAlias = _DTypeKind | tuple[_DTypeKind, ...]
+
+
+# `__array_namespace_info__.dtypes(kind="bool")`
+class DTypesBool(TypedDict):
+    bool: DType
+
+
+# `__array_namespace_info__.dtypes(kind="signed integer")`
+class DTypesSigned(TypedDict):
+    int8: DType
+    int16: DType
+    int32: DType
+    int64: DType
+
+
+# `__array_namespace_info__.dtypes(kind="unsigned integer")`
+class DTypesUnsigned(TypedDict):
+    uint8: DType
+    uint16: DType
+    uint32: DType
+    uint64: DType
+
+
+# `__array_namespace_info__.dtypes(kind="integral")`
+class DTypesIntegral(DTypesSigned, DTypesUnsigned):
+    pass
+
+
+# `__array_namespace_info__.dtypes(kind="real floating")`
+class DTypesReal(TypedDict):
+    float32: DType
+    float64: DType
+
+
+# `__array_namespace_info__.dtypes(kind="complex floating")`
+class DTypesComplex(TypedDict):
+    complex64: DType
+    complex128: DType
+
+
+# `__array_namespace_info__.dtypes(kind="numeric")`
+class DTypesNumeric(DTypesIntegral, DTypesReal, DTypesComplex):
+    pass
+
+
+# `__array_namespace_info__.dtypes(kind=None)` (default)
+class DTypesAll(DTypesBool, DTypesNumeric):
+    pass
+
+
+# `__array_namespace_info__.dtypes(kind=?)` (fallback)
+DTypesAny: TypeAlias = Mapping[str, DType]
+
+
+__all__ = [
+    "Array",
+    "Capabilities",
+    "DType",
+    "DTypeKind",
+    "DTypesAny",
+    "DTypesAll",
+    "DTypesBool",
+    "DTypesNumeric",
+    "DTypesIntegral",
+    "DTypesSigned",
+    "DTypesUnsigned",
+    "DTypesReal",
+    "DTypesComplex",
+    "DefaultDTypes",
+    "Device",
+    "HasShape",
+    "Namespace",
+    "JustInt",
+    "JustFloat",
+    "JustComplex",
+    "NestedSequence",
+    "SupportsArrayNamespace",
+    "SupportsBufferProtocol",
+]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..aa0c39642cbcc2e493419dd9630cba8296c26c34
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/__init__.py
@@ -0,0 +1,13 @@
+from cupy import * # noqa: F403
+
+# from cupy import * doesn't overwrite these builtin names
+from cupy import abs, max, min, round # noqa: F401
+
+# These imports may overwrite names from the import * above.
+from ._aliases import * # noqa: F403
+
+# See the comment in the numpy __init__.py
+__import__(__package__ + '.linalg')
+__import__(__package__ + '.fft')
+
+__array_api_version__ = '2024.12'
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_aliases.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_aliases.py
new file mode 100644
index 0000000000000000000000000000000000000000..ece08cd14a032ec92a95af20bef6261171834146
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_aliases.py
@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+from typing import Optional
+
+import cupy as cp
+
+from ..common import _aliases, _helpers
+from ..common._typing import NestedSequence, SupportsBufferProtocol
+from .._internal import get_xp
+from ._info import __array_namespace_info__
+from ._typing import Array, Device, DType
+
+bool = cp.bool_
+
+# Basic renames
+acos = cp.arccos
+acosh = cp.arccosh
+asin = cp.arcsin
+asinh = cp.arcsinh
+atan = cp.arctan
+atan2 = cp.arctan2
+atanh = cp.arctanh
+bitwise_left_shift = cp.left_shift
+bitwise_invert = cp.invert
+bitwise_right_shift = cp.right_shift
+concat = cp.concatenate
+pow = cp.power
+
+arange = get_xp(cp)(_aliases.arange)
+empty = get_xp(cp)(_aliases.empty)
+empty_like = get_xp(cp)(_aliases.empty_like)
+eye = get_xp(cp)(_aliases.eye)
+full = get_xp(cp)(_aliases.full)
+full_like = get_xp(cp)(_aliases.full_like)
+linspace = get_xp(cp)(_aliases.linspace)
+ones = get_xp(cp)(_aliases.ones)
+ones_like = get_xp(cp)(_aliases.ones_like)
+zeros = get_xp(cp)(_aliases.zeros)
+zeros_like = get_xp(cp)(_aliases.zeros_like)
+UniqueAllResult = get_xp(cp)(_aliases.UniqueAllResult)
+UniqueCountsResult = get_xp(cp)(_aliases.UniqueCountsResult)
+UniqueInverseResult = get_xp(cp)(_aliases.UniqueInverseResult)
+unique_all = get_xp(cp)(_aliases.unique_all)
+unique_counts = get_xp(cp)(_aliases.unique_counts)
+unique_inverse = get_xp(cp)(_aliases.unique_inverse)
+unique_values = get_xp(cp)(_aliases.unique_values)
+std = get_xp(cp)(_aliases.std)
+var = get_xp(cp)(_aliases.var)
+cumulative_sum = get_xp(cp)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(cp)(_aliases.cumulative_prod)
+clip = get_xp(cp)(_aliases.clip)
+permute_dims = get_xp(cp)(_aliases.permute_dims)
+reshape = get_xp(cp)(_aliases.reshape)
+argsort = get_xp(cp)(_aliases.argsort)
+sort = get_xp(cp)(_aliases.sort)
+nonzero = get_xp(cp)(_aliases.nonzero)
+ceil = get_xp(cp)(_aliases.ceil)
+floor = get_xp(cp)(_aliases.floor)
+trunc = get_xp(cp)(_aliases.trunc)
+matmul = get_xp(cp)(_aliases.matmul)
+matrix_transpose = get_xp(cp)(_aliases.matrix_transpose)
+tensordot = get_xp(cp)(_aliases.tensordot)
+sign = get_xp(cp)(_aliases.sign)
+finfo = get_xp(cp)(_aliases.finfo)
+iinfo = get_xp(cp)(_aliases.iinfo)
+
+
+# asarray also adds the copy keyword, which is not present in numpy 1.0.
+def asarray(
+    obj: (
+        Array 
+        | bool | int | float | complex 
+        | NestedSequence[bool | int | float | complex] 
+        | SupportsBufferProtocol
+    ),
+    /,
+    *,
+    dtype: Optional[DType] = None,
+    device: Optional[Device] = None,
+    copy: Optional[bool] = None,
+    **kwargs,
+) -> Array:
+    """
+    Array API compatibility wrapper for asarray().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    with cp.cuda.Device(device):
+        if copy is None:
+            return cp.asarray(obj, dtype=dtype, **kwargs)
+        else:
+            res = cp.array(obj, dtype=dtype, copy=copy, **kwargs)
+            if not copy and res is not obj:
+                raise ValueError("Unable to avoid copy while creating an array as requested")
+            return res
+
+
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: bool = True,
+    device: Optional[Device] = None,
+) -> Array:
+    if device is None:
+        return x.astype(dtype=dtype, copy=copy)
+    out = _helpers.to_device(x.astype(dtype=dtype, copy=False), device)
+    return out.copy() if copy and out is x else out
+
+
+# cupy.count_nonzero does not have keepdims
+def count_nonzero(
+    x: Array,
+    axis=None,
+    keepdims=False
+) -> Array:
+   result = cp.count_nonzero(x, axis)
+   if keepdims:
+       if axis is None:
+            return cp.reshape(result, [1]*x.ndim)
+       return cp.expand_dims(result, axis)
+   return result
+
+
+# take_along_axis: axis defaults to -1 but in cupy (and numpy) axis is a required arg
+def take_along_axis(x: Array, indices: Array, /, *, axis: int = -1):
+    return cp.take_along_axis(x, indices, axis=axis)
+
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(cp, 'vecdot'):
+    vecdot = cp.vecdot
+else:
+    vecdot = get_xp(cp)(_aliases.vecdot)
+
+if hasattr(cp, 'isdtype'):
+    isdtype = cp.isdtype
+else:
+    isdtype = get_xp(cp)(_aliases.isdtype)
+
+if hasattr(cp, 'unstack'):
+    unstack = cp.unstack
+else:
+    unstack = get_xp(cp)(_aliases.unstack)
+
+__all__ = _aliases.__all__ + ['__array_namespace_info__', 'asarray', 'astype',
+                              'acos', 'acosh', 'asin', 'asinh', 'atan',
+                              'atan2', 'atanh', 'bitwise_left_shift',
+                              'bitwise_invert', 'bitwise_right_shift',
+                              'bool', 'concat', 'count_nonzero', 'pow', 'sign',
+                              'take_along_axis']
+
+_all_ignore = ['cp', 'get_xp']
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_info.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..ecfc3b446845f6c20cb1cd300410ca4c633b01c1
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_info.py
@@ -0,0 +1,336 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+from cupy import (
+    dtype,
+    cuda,
+    bool_ as bool,
+    intp,
+    int8,
+    int16,
+    int32,
+    int64,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    float32,
+    float64,
+    complex64,
+    complex128,
+)
+
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for CuPy.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for CuPy.
+
+    Examples
+    --------
+    >>> info = xp.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': cupy.float64,
+     'complex floating': cupy.complex128,
+     'integral': cupy.int64,
+     'indexing': cupy.int64}
+
+    """
+
+    __module__ = 'cupy'
+
+    def capabilities(self):
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing. Always ``True`` for CuPy.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes. Always ``True`` for
+          CuPy.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self):
+        """
+        The default device used for new CuPy arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new CuPy arrays.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_device()
+        Device(0)
+
+        Notes
+        -----
+        This method returns the static default device when CuPy is initialized.
+        However, the *current* device used by creation functions (``empty`` etc.)
+        can be changed globally or with a context manager.
+
+        See Also
+        --------
+        https://github.com/data-apis/array-api/issues/835
+        """
+        return cuda.Device(0)
+
+    def default_dtypes(self, *, device=None):
+        """
+        The default data types used for new CuPy arrays.
+
+        For CuPy, this always returns the following dictionary:
+
+        - **"real floating"**: ``cupy.float64``
+        - **"complex floating"**: ``cupy.complex128``
+        - **"integral"**: ``cupy.intp``
+        - **"indexing"**: ``cupy.intp``
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the default data types for.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new CuPy
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': cupy.float64,
+         'complex floating': cupy.complex128,
+         'integral': cupy.int64,
+         'indexing': cupy.int64}
+
+        """
+        # TODO: Does this depend on device?
+        return {
+            "real floating": dtype(float64),
+            "complex floating": dtype(complex128),
+            "integral": dtype(intp),
+            "indexing": dtype(intp),
+        }
+
+    def dtypes(self, *, device=None, kind=None):
+        """
+        The array API data types supported by CuPy.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the data types for.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            CuPy data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': cupy.int8,
+         'int16': cupy.int16,
+         'int32': cupy.int32,
+         'int64': cupy.int64}
+
+        """
+        # TODO: Does this depend on device?
+        if kind is None:
+            return {
+                "bool": dtype(bool),
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "bool":
+            return {"bool": bool}
+        if kind == "signed integer":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "integral":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "real floating":
+            return {
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "numeric":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if isinstance(kind, tuple):
+            res = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    def devices(self):
+        """
+        The devices supported by CuPy.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by CuPy.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        """
+        return [cuda.Device(i) for i in range(cuda.runtime.getDeviceCount())]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_typing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a615bc71b50bf0e2f789abbe425ed7428fb1d57
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/_typing.py
@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+__all__ = ["Array", "DType", "Device"]
+_all_ignore = ["cp"]
+
+from typing import TYPE_CHECKING
+
+import cupy as cp
+from cupy import ndarray as Array
+from cupy.cuda.device import Device
+
+if TYPE_CHECKING:
+    # NumPy 1.x on Python 3.10 fails to parse np.dtype[]
+    DType = cp.dtype[
+        cp.intp
+        | cp.int8
+        | cp.int16
+        | cp.int32
+        | cp.int64
+        | cp.uint8
+        | cp.uint16
+        | cp.uint32
+        | cp.uint64
+        | cp.float32
+        | cp.float64
+        | cp.complex64
+        | cp.complex128
+        | cp.bool_
+    ]
+else:
+    DType = cp.dtype
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/fft.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/fft.py
new file mode 100644
index 0000000000000000000000000000000000000000..324fd9a8c680ea26c435c88bbea0133bb5ab1d7d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/fft.py
@@ -0,0 +1,36 @@
+from cupy.fft import * # noqa: F403
+# cupy.fft doesn't have __all__. If it is added, replace this with
+#
+# from cupy.fft import __all__ as linalg_all
+_n = {}
+exec('from cupy.fft import *', _n)
+del _n['__builtins__']
+fft_all = list(_n)
+del _n
+
+from ..common import _fft
+from .._internal import get_xp
+
+import cupy as cp
+
+fft = get_xp(cp)(_fft.fft)
+ifft = get_xp(cp)(_fft.ifft)
+fftn = get_xp(cp)(_fft.fftn)
+ifftn = get_xp(cp)(_fft.ifftn)
+rfft = get_xp(cp)(_fft.rfft)
+irfft = get_xp(cp)(_fft.irfft)
+rfftn = get_xp(cp)(_fft.rfftn)
+irfftn = get_xp(cp)(_fft.irfftn)
+hfft = get_xp(cp)(_fft.hfft)
+ihfft = get_xp(cp)(_fft.ihfft)
+fftfreq = get_xp(cp)(_fft.fftfreq)
+rfftfreq = get_xp(cp)(_fft.rfftfreq)
+fftshift = get_xp(cp)(_fft.fftshift)
+ifftshift = get_xp(cp)(_fft.ifftshift)
+
+__all__ = fft_all + _fft.__all__
+
+del get_xp
+del cp
+del fft_all
+del _fft
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/linalg.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/linalg.py
new file mode 100644
index 0000000000000000000000000000000000000000..49bda5c42d605aea6abbaf072a6cc688e04ee8a3
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/cupy/linalg.py
@@ -0,0 +1,49 @@
+from cupy.linalg import * # noqa: F403
+# cupy.linalg doesn't have __all__. If it is added, replace this with
+#
+# from cupy.linalg import __all__ as linalg_all
+_n = {}
+exec('from cupy.linalg import *', _n)
+del _n['__builtins__']
+linalg_all = list(_n)
+del _n
+
+from ..common import _linalg
+from .._internal import get_xp
+
+import cupy as cp
+
+# These functions are in both the main and linalg namespaces
+from ._aliases import matmul, matrix_transpose, tensordot, vecdot # noqa: F401
+
+cross = get_xp(cp)(_linalg.cross)
+outer = get_xp(cp)(_linalg.outer)
+EighResult = _linalg.EighResult
+QRResult = _linalg.QRResult
+SlogdetResult = _linalg.SlogdetResult
+SVDResult = _linalg.SVDResult
+eigh = get_xp(cp)(_linalg.eigh)
+qr = get_xp(cp)(_linalg.qr)
+slogdet = get_xp(cp)(_linalg.slogdet)
+svd = get_xp(cp)(_linalg.svd)
+cholesky = get_xp(cp)(_linalg.cholesky)
+matrix_rank = get_xp(cp)(_linalg.matrix_rank)
+pinv = get_xp(cp)(_linalg.pinv)
+matrix_norm = get_xp(cp)(_linalg.matrix_norm)
+svdvals = get_xp(cp)(_linalg.svdvals)
+diagonal = get_xp(cp)(_linalg.diagonal)
+trace = get_xp(cp)(_linalg.trace)
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(cp.linalg, 'vector_norm'):
+    vector_norm = cp.linalg.vector_norm
+else:
+    vector_norm = get_xp(cp)(_linalg.vector_norm)
+
+__all__ = linalg_all + _linalg.__all__
+
+del get_xp
+del cp
+del linalg_all
+del _linalg
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e95aa8aa7a3f4f6a01d401d06f241311e38b84b4
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/__init__.py
@@ -0,0 +1,12 @@
+from typing import Final
+
+from dask.array import *  # noqa: F403
+
+# These imports may overwrite names from the import * above.
+from ._aliases import *  # noqa: F403
+
+__array_api_version__: Final = "2024.12"
+
+# See the comment in the numpy __init__.py
+__import__(__package__ + '.linalg')
+__import__(__package__ + '.fft')
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/_aliases.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/_aliases.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb58d1cd389be20713c11cd1df86c6f4357b2caa
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/_aliases.py
@@ -0,0 +1,376 @@
+# pyright: reportPrivateUsage=false
+# pyright: reportUnknownArgumentType=false
+# pyright: reportUnknownMemberType=false
+# pyright: reportUnknownVariableType=false
+
+from __future__ import annotations
+
+from builtins import bool as py_bool
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from typing_extensions import TypeIs
+
+import dask.array as da
+import numpy as np
+from numpy import bool_ as bool
+from numpy import (
+    can_cast,
+    complex64,
+    complex128,
+    float32,
+    float64,
+    int8,
+    int16,
+    int32,
+    int64,
+    result_type,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+
+from ..._internal import get_xp
+from ...common import _aliases, _helpers, array_namespace
+from ...common._typing import (
+    Array,
+    Device,
+    DType,
+    NestedSequence,
+    SupportsBufferProtocol,
+)
+from ._info import __array_namespace_info__
+
+isdtype = get_xp(np)(_aliases.isdtype)
+unstack = get_xp(da)(_aliases.unstack)
+
+
+# da.astype doesn't respect copy=True
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: py_bool = True,
+    device: Device | None = None,
+) -> Array:
+    """
+    Array API compatibility wrapper for astype().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    # TODO: respect device keyword?
+    _helpers._check_device(da, device)
+
+    if not copy and dtype == x.dtype:
+        return x
+    x = x.astype(dtype)
+    return x.copy() if copy else x
+
+
+# Common aliases
+
+
+# This arange func is modified from the common one to
+# not pass stop/step as keyword arguments, which will cause
+# an error with dask
+def arange(
+    start: float,
+    /,
+    stop: float | None = None,
+    step: float = 1,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    """
+    Array API compatibility wrapper for arange().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    # TODO: respect device keyword?
+    _helpers._check_device(da, device)
+
+    args: list[Any] = [start]
+    if stop is not None:
+        args.append(stop)
+    else:
+        # stop is None, so start is actually stop
+        # prepend the default value for start which is 0
+        args.insert(0, 0)
+    args.append(step)
+
+    return da.arange(*args, dtype=dtype, **kwargs)
+
+
+eye = get_xp(da)(_aliases.eye)
+linspace = get_xp(da)(_aliases.linspace)
+UniqueAllResult = get_xp(da)(_aliases.UniqueAllResult)
+UniqueCountsResult = get_xp(da)(_aliases.UniqueCountsResult)
+UniqueInverseResult = get_xp(da)(_aliases.UniqueInverseResult)
+unique_all = get_xp(da)(_aliases.unique_all)
+unique_counts = get_xp(da)(_aliases.unique_counts)
+unique_inverse = get_xp(da)(_aliases.unique_inverse)
+unique_values = get_xp(da)(_aliases.unique_values)
+permute_dims = get_xp(da)(_aliases.permute_dims)
+std = get_xp(da)(_aliases.std)
+var = get_xp(da)(_aliases.var)
+cumulative_sum = get_xp(da)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(da)(_aliases.cumulative_prod)
+empty = get_xp(da)(_aliases.empty)
+empty_like = get_xp(da)(_aliases.empty_like)
+full = get_xp(da)(_aliases.full)
+full_like = get_xp(da)(_aliases.full_like)
+ones = get_xp(da)(_aliases.ones)
+ones_like = get_xp(da)(_aliases.ones_like)
+zeros = get_xp(da)(_aliases.zeros)
+zeros_like = get_xp(da)(_aliases.zeros_like)
+reshape = get_xp(da)(_aliases.reshape)
+matrix_transpose = get_xp(da)(_aliases.matrix_transpose)
+vecdot = get_xp(da)(_aliases.vecdot)
+nonzero = get_xp(da)(_aliases.nonzero)
+ceil = get_xp(np)(_aliases.ceil)
+floor = get_xp(np)(_aliases.floor)
+trunc = get_xp(np)(_aliases.trunc)
+matmul = get_xp(np)(_aliases.matmul)
+tensordot = get_xp(np)(_aliases.tensordot)
+sign = get_xp(np)(_aliases.sign)
+finfo = get_xp(np)(_aliases.finfo)
+iinfo = get_xp(np)(_aliases.iinfo)
+
+
+# asarray also adds the copy keyword, which is not present in numpy 1.0.
+def asarray(
+    obj: complex | NestedSequence[complex] | Array | SupportsBufferProtocol,
+    /,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    copy: py_bool | None = None,
+    **kwargs: object,
+) -> Array:
+    """
+    Array API compatibility wrapper for asarray().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    # TODO: respect device keyword?
+    _helpers._check_device(da, device)
+
+    if isinstance(obj, da.Array):
+        if dtype is not None and dtype != obj.dtype:
+            if copy is False:
+                raise ValueError("Unable to avoid copy when changing dtype")
+            obj = obj.astype(dtype)
+        return obj.copy() if copy else obj  # pyright: ignore[reportAttributeAccessIssue]
+
+    if copy is False:
+        raise ValueError(
+            "Unable to avoid copy when converting a non-dask object to dask"
+        )
+
+    # copy=None to be uniform across dask < 2024.12 and >= 2024.12
+    # see https://github.com/dask/dask/pull/11524/
+    obj = np.array(obj, dtype=dtype, copy=True)
+    return da.from_array(obj)
+
+
+# Element wise aliases
+from dask.array import arccos as acos
+from dask.array import arccosh as acosh
+from dask.array import arcsin as asin
+from dask.array import arcsinh as asinh
+from dask.array import arctan as atan
+from dask.array import arctan2 as atan2
+from dask.array import arctanh as atanh
+
+# Other
+from dask.array import concatenate as concat
+from dask.array import invert as bitwise_invert
+from dask.array import left_shift as bitwise_left_shift
+from dask.array import power as pow
+from dask.array import right_shift as bitwise_right_shift
+
+
+# dask.array.clip does not work unless all three arguments are provided.
+# Furthermore, the masking workaround in common._aliases.clip cannot work with
+# dask (meaning uint64 promoting to float64 is going to just be unfixed for
+# now).
+def clip(
+    x: Array,
+    /,
+    min: float | Array | None = None,
+    max: float | Array | None = None,
+) -> Array:
+    """
+    Array API compatibility wrapper for clip().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+
+    def _isscalar(a: float | Array | None, /) -> TypeIs[float | None]:
+        return a is None or isinstance(a, (int, float))
+
+    min_shape = () if _isscalar(min) else min.shape
+    max_shape = () if _isscalar(max) else max.shape
+
+    # TODO: This won't handle dask unknown shapes
+    result_shape = np.broadcast_shapes(x.shape, min_shape, max_shape)
+
+    if min is not None:
+        min = da.broadcast_to(da.asarray(min), result_shape)
+    if max is not None:
+        max = da.broadcast_to(da.asarray(max), result_shape)
+
+    if min is None and max is None:
+        return da.positive(x)
+
+    if min is None:
+        return astype(da.minimum(x, max), x.dtype)
+    if max is None:
+        return astype(da.maximum(x, min), x.dtype)
+
+    return astype(da.minimum(da.maximum(x, min), max), x.dtype)
+
+
+def _ensure_single_chunk(x: Array, axis: int) -> tuple[Array, Callable[[Array], Array]]:
+    """
+    Make sure that Array is not broken into multiple chunks along axis.
+
+    Returns
+    -------
+    x : Array
+        The input Array with a single chunk along axis.
+    restore : Callable[Array, Array]
+        function to apply to the output to rechunk it back into reasonable chunks
+    """
+    if axis < 0:
+        axis += x.ndim
+    if x.numblocks[axis] < 2:
+        return x, lambda x: x
+
+    # Break chunks on other axes in an attempt to keep chunk size low
+    x = x.rechunk({i: -1 if i == axis else "auto" for i in range(x.ndim)})
+
+    # Rather than reconstructing the original chunks, which can be a
+    # very expensive affair, just break down oversized chunks without
+    # incurring in any transfers over the network.
+    # This has the downside of a risk of overchunking if the array is
+    # then used in operations against other arrays that match the
+    # original chunking pattern.
+    return x, lambda x: x.rechunk()
+
+
+def sort(
+    x: Array,
+    /,
+    *,
+    axis: int = -1,
+    descending: py_bool = False,
+    stable: py_bool = True,
+) -> Array:
+    """
+    Array API compatibility layer around the lack of sort() in Dask.
+
+    Warnings
+    --------
+    This function temporarily rechunks the array along `axis` to a single chunk.
+    This can be extremely inefficient and can lead to out-of-memory errors.
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    x, restore = _ensure_single_chunk(x, axis)
+
+    meta_xp = array_namespace(x._meta)
+    x = da.map_blocks(
+        meta_xp.sort,
+        x,
+        axis=axis,
+        meta=x._meta,
+        dtype=x.dtype,
+        descending=descending,
+        stable=stable,
+    )
+
+    return restore(x)
+
+
+def argsort(
+    x: Array,
+    /,
+    *,
+    axis: int = -1,
+    descending: py_bool = False,
+    stable: py_bool = True,
+) -> Array:
+    """
+    Array API compatibility layer around the lack of argsort() in Dask.
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+
+    Warnings
+    --------
+    This function temporarily rechunks the array along `axis` into a single chunk.
+    This can be extremely inefficient and can lead to out-of-memory errors.
+    """
+    x, restore = _ensure_single_chunk(x, axis)
+
+    meta_xp = array_namespace(x._meta)
+    dtype = meta_xp.argsort(x._meta).dtype
+    meta = meta_xp.astype(x._meta, dtype)
+    x = da.map_blocks(
+        meta_xp.argsort,
+        x,
+        axis=axis,
+        meta=meta,
+        dtype=dtype,
+        descending=descending,
+        stable=stable,
+    )
+
+    return restore(x)
+
+
+# dask.array.count_nonzero does not have keepdims
+def count_nonzero(
+    x: Array,
+    axis: int | None = None,
+    keepdims: py_bool = False,
+) -> Array:
+    result = da.count_nonzero(x, axis)
+    if keepdims:
+        if axis is None:
+            return da.reshape(result, [1] * x.ndim)
+        return da.expand_dims(result, axis)
+    return result
+
+
+__all__ = [
+    "__array_namespace_info__",
+    "count_nonzero",
+    "bool",
+    "int8", "int16", "int32", "int64",
+    "uint8", "uint16", "uint32", "uint64",
+    "float32", "float64",
+    "complex64", "complex128",
+    "asarray", "astype", "can_cast", "result_type",
+    "pow",
+    "concat",
+    "acos", "acosh", "asin", "asinh", "atan", "atan2", "atanh",
+    "bitwise_left_shift", "bitwise_right_shift", "bitwise_invert",
+]  # fmt: skip
+__all__ += _aliases.__all__
+_all_ignore = ["array_namespace", "get_xp", "da", "np"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/_info.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..dddb3ab41a539fa464dda8a06d8bacffbf5629f1
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/_info.py
@@ -0,0 +1,416 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from typing import Literal as L
+from typing import TypeAlias, overload
+
+from numpy import bool_ as bool
+from numpy import (
+    complex64,
+    complex128,
+    dtype,
+    float32,
+    float64,
+    int8,
+    int16,
+    int32,
+    int64,
+    intp,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+
+from ...common._helpers import _DASK_DEVICE, _dask_device
+from ...common._typing import (
+    Capabilities,
+    DefaultDTypes,
+    DType,
+    DTypeKind,
+    DTypesAll,
+    DTypesAny,
+    DTypesBool,
+    DTypesComplex,
+    DTypesIntegral,
+    DTypesNumeric,
+    DTypesReal,
+    DTypesSigned,
+    DTypesUnsigned,
+)
+
+_Device: TypeAlias = L["cpu"] | _dask_device
+
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for Dask.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for Dask.
+
+    Examples
+    --------
+    >>> info = xp.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': dask.float64,
+     'complex floating': dask.complex128,
+     'integral': dask.int64,
+     'indexing': dask.int64}
+
+    """
+
+    __module__ = "dask.array"
+
+    def capabilities(self) -> Capabilities:
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing.
+
+          Dask support boolean indexing as long as both the index
+          and the indexed arrays have known shapes.
+          Note however that the output .shape and .size properties
+          will contain a non-compliant math.nan instead of None.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes.
+
+          Dask implements unique_values et.al.
+          Note however that the output .shape and .size properties
+          will contain a non-compliant math.nan instead of None.
+
+        - **"max dimensions"**: integer indicating the maximum number of
+          dimensions supported by the array library.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self) -> L["cpu"]:
+        """
+        The default device used for new Dask arrays.
+
+        For Dask, this always returns ``'cpu'``.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new Dask arrays.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_device()
+        'cpu'
+
+        """
+        return "cpu"
+
+    def default_dtypes(self, /, *, device: _Device | None = None) -> DefaultDTypes:
+        """
+        The default data types used for new Dask arrays.
+
+        For Dask, this always returns the following dictionary:
+
+        - **"real floating"**: ``numpy.float64``
+        - **"complex floating"**: ``numpy.complex128``
+        - **"integral"**: ``numpy.intp``
+        - **"indexing"**: ``numpy.intp``
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the default data types for.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new Dask
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': dask.float64,
+         'complex floating': dask.complex128,
+         'integral': dask.int64,
+         'indexing': dask.int64}
+
+        """
+        if device not in ["cpu", _DASK_DEVICE, None]:
+            raise ValueError(
+                f'Device not understood. Only "cpu" or _DASK_DEVICE is allowed, '
+                f"but received: {device!r}"
+            )
+        return {
+            "real floating": dtype(float64),
+            "complex floating": dtype(complex128),
+            "integral": dtype(intp),
+            "indexing": dtype(intp),
+        }
+
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: None = None
+    ) -> DTypesAll: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["bool"]
+    ) -> DTypesBool: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["signed integer"]
+    ) -> DTypesSigned: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["unsigned integer"]
+    ) -> DTypesUnsigned: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["integral"]
+    ) -> DTypesIntegral: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["real floating"]
+    ) -> DTypesReal: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["complex floating"]
+    ) -> DTypesComplex: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["numeric"]
+    ) -> DTypesNumeric: ...
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: DTypeKind | None = None
+    ) -> DTypesAny:
+        """
+        The array API data types supported by Dask.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the data types for.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            Dask data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': dask.int8,
+         'int16': dask.int16,
+         'int32': dask.int32,
+         'int64': dask.int64}
+
+        """
+        if device not in ["cpu", _DASK_DEVICE, None]:
+            raise ValueError(
+                'Device not understood. Only "cpu" or _DASK_DEVICE is allowed, but received:'
+                f" {device}"
+            )
+        if kind is None:
+            return {
+                "bool": dtype(bool),
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "bool":
+            return {"bool": bool}
+        if kind == "signed integer":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "integral":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "real floating":
+            return {
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "numeric":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if isinstance(kind, tuple):  # type: ignore[reportUnnecessaryIsinstanceCall]
+            res: dict[str, DType] = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    def devices(self) -> list[_Device]:
+        """
+        The devices supported by Dask.
+
+        For Dask, this always returns ``['cpu', DASK_DEVICE]``.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by Dask.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.devices()
+        ['cpu', DASK_DEVICE]
+
+        """
+        return ["cpu", _DASK_DEVICE]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/fft.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/fft.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee96b7949e181ac548b49a17897b19017fc3ba3a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/fft.py
@@ -0,0 +1,21 @@
+from dask.array.fft import * # noqa: F403
+# dask.array.fft doesn't have __all__. If it is added, replace this with
+#
+# from dask.array.fft import __all__ as linalg_all
+_n = {}
+exec('from dask.array.fft import *', _n)
+for k in ("__builtins__", "Sequence", "annotations", "warnings"):
+    _n.pop(k, None)
+fft_all = list(_n)
+del _n, k
+
+from ...common import _fft
+from ..._internal import get_xp
+
+import dask.array as da
+
+fftfreq = get_xp(da)(_fft.fftfreq)
+rfftfreq = get_xp(da)(_fft.rfftfreq)
+
+__all__ = fft_all + ["fftfreq", "rfftfreq"]
+_all_ignore = ["da", "fft_all", "get_xp", "warnings"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/linalg.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/linalg.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1a6cd4a36edd8c5646822f130f9c80faf3b6cee
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/dask/array/linalg.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from typing import Literal
+
+import dask.array as da
+
+# The `matmul` and `tensordot` functions are in both the main and linalg namespaces
+from dask.array import matmul, outer, tensordot
+
+# Exports
+from dask.array.linalg import *  # noqa: F403
+
+from ..._internal import get_xp
+from ...common import _linalg
+from ...common._typing import Array as _Array
+from ._aliases import matrix_transpose, vecdot
+
+# dask.array.linalg doesn't have __all__. If it is added, replace this with
+#
+# from dask.array.linalg import __all__ as linalg_all
+_n = {}
+exec('from dask.array.linalg import *', _n)
+for k in ('__builtins__', 'annotations', 'operator', 'warnings', 'Array'):
+    _n.pop(k, None)
+linalg_all = list(_n)
+del _n, k
+
+EighResult = _linalg.EighResult
+QRResult = _linalg.QRResult
+SlogdetResult = _linalg.SlogdetResult
+SVDResult = _linalg.SVDResult
+# TODO: use the QR wrapper once dask
+# supports the mode keyword on QR
+# https://github.com/dask/dask/issues/10388
+#qr = get_xp(da)(_linalg.qr)
+def qr(
+    x: _Array,
+    mode: Literal["reduced", "complete"] = "reduced",
+    **kwargs: object,
+) -> QRResult:
+    if mode != "reduced":
+        raise ValueError("dask arrays only support using mode='reduced'")
+    return QRResult(*da.linalg.qr(x, **kwargs))
+trace = get_xp(da)(_linalg.trace)
+cholesky = get_xp(da)(_linalg.cholesky)
+matrix_rank = get_xp(da)(_linalg.matrix_rank)
+matrix_norm = get_xp(da)(_linalg.matrix_norm)
+
+
+# Wrap the svd functions to not pass full_matrices to dask
+# when full_matrices=False (as that is the default behavior for dask),
+# and dask doesn't have the full_matrices keyword
+def svd(x: _Array, full_matrices: bool = True, **kwargs) -> SVDResult:
+    if full_matrices:
+        raise ValueError("full_matrics=True is not supported by dask.")
+    return da.linalg.svd(x, coerce_signs=False, **kwargs)
+
+def svdvals(x: _Array) -> _Array:
+    # TODO: can't avoid computing U or V for dask
+    _, s, _ =  svd(x)
+    return s
+
+vector_norm = get_xp(da)(_linalg.vector_norm)
+diagonal = get_xp(da)(_linalg.diagonal)
+
+__all__ = linalg_all + ["trace", "outer", "matmul", "tensordot",
+                        "matrix_transpose", "vecdot", "EighResult",
+                        "QRResult", "SlogdetResult", "SVDResult", "qr",
+                        "cholesky", "matrix_rank", "matrix_norm", "svdvals",
+                        "vector_norm", "diagonal"]
+
+_all_ignore = ['get_xp', 'da', 'linalg_all', 'warnings']
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..bba663abe2420d042ad23bef768a2d5d3645f665
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__init__.py
@@ -0,0 +1,28 @@
+# ruff: noqa: PLC0414
+from typing import Final
+
+from numpy import *  # noqa: F403  # pyright: ignore[reportWildcardImportFromLibrary]
+
+# from numpy import * doesn't overwrite these builtin names
+from numpy import abs as abs
+from numpy import max as max
+from numpy import min as min
+from numpy import round as round
+
+# These imports may overwrite names from the import * above.
+from ._aliases import *  # noqa: F403
+
+# Don't know why, but we have to do an absolute import to import linalg. If we
+# instead do
+#
+# from . import linalg
+#
+# It doesn't overwrite np.linalg from above. The import is generated
+# dynamically so that the library can be vendored.
+__import__(__package__ + ".linalg")
+
+__import__(__package__ + ".fft")
+
+from .linalg import matrix_transpose, vecdot  # type: ignore[no-redef]  # noqa: F401
+
+__array_api_version__: Final = "2024.12"
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8186b5a65927f3bd9b59b7c695adb75eabefe0fe
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/__init__.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_aliases.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_aliases.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..db502df2c820730ab54faaf4d95ef452f3668249
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_aliases.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_info.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_info.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ea44f86cb3df3a0e2eeadf52502e97616ea6c75c
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_info.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_typing.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_typing.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..54743f5a731e666f33114476d4bbc574c528ae49
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/_typing.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/fft.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/fft.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..260b10dbd775e1639378544fcfaf6b854cebde66
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/fft.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/linalg.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/linalg.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ae08fb7b56c1072a6caea4845782b4a73cf351c3
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/__pycache__/linalg.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_aliases.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_aliases.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae43c075ae792cca29cfac99c9d8830dd59ae8b3
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_aliases.py
@@ -0,0 +1,190 @@
+# pyright: reportPrivateUsage=false
+from __future__ import annotations
+
+from builtins import bool as py_bool
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast
+
+import numpy as np
+
+from .._internal import get_xp
+from ..common import _aliases, _helpers
+from ..common._typing import NestedSequence, SupportsBufferProtocol
+from ._info import __array_namespace_info__
+from ._typing import Array, Device, DType
+
+if TYPE_CHECKING:
+    from typing_extensions import Buffer, TypeIs
+
+# The values of the `_CopyMode` enum can be either `False`, `True`, or `2`:
+# https://github.com/numpy/numpy/blob/5a8a6a79d9c2fff8f07dcab5d41e14f8508d673f/numpy/_globals.pyi#L7-L10
+_Copy: TypeAlias = py_bool | Literal[2] | np._CopyMode
+
+bool = np.bool_
+
+# Basic renames
+acos = np.arccos
+acosh = np.arccosh
+asin = np.arcsin
+asinh = np.arcsinh
+atan = np.arctan
+atan2 = np.arctan2
+atanh = np.arctanh
+bitwise_left_shift = np.left_shift
+bitwise_invert = np.invert
+bitwise_right_shift = np.right_shift
+concat = np.concatenate
+pow = np.power
+
+arange = get_xp(np)(_aliases.arange)
+empty = get_xp(np)(_aliases.empty)
+empty_like = get_xp(np)(_aliases.empty_like)
+eye = get_xp(np)(_aliases.eye)
+full = get_xp(np)(_aliases.full)
+full_like = get_xp(np)(_aliases.full_like)
+linspace = get_xp(np)(_aliases.linspace)
+ones = get_xp(np)(_aliases.ones)
+ones_like = get_xp(np)(_aliases.ones_like)
+zeros = get_xp(np)(_aliases.zeros)
+zeros_like = get_xp(np)(_aliases.zeros_like)
+UniqueAllResult = get_xp(np)(_aliases.UniqueAllResult)
+UniqueCountsResult = get_xp(np)(_aliases.UniqueCountsResult)
+UniqueInverseResult = get_xp(np)(_aliases.UniqueInverseResult)
+unique_all = get_xp(np)(_aliases.unique_all)
+unique_counts = get_xp(np)(_aliases.unique_counts)
+unique_inverse = get_xp(np)(_aliases.unique_inverse)
+unique_values = get_xp(np)(_aliases.unique_values)
+std = get_xp(np)(_aliases.std)
+var = get_xp(np)(_aliases.var)
+cumulative_sum = get_xp(np)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(np)(_aliases.cumulative_prod)
+clip = get_xp(np)(_aliases.clip)
+permute_dims = get_xp(np)(_aliases.permute_dims)
+reshape = get_xp(np)(_aliases.reshape)
+argsort = get_xp(np)(_aliases.argsort)
+sort = get_xp(np)(_aliases.sort)
+nonzero = get_xp(np)(_aliases.nonzero)
+ceil = get_xp(np)(_aliases.ceil)
+floor = get_xp(np)(_aliases.floor)
+trunc = get_xp(np)(_aliases.trunc)
+matmul = get_xp(np)(_aliases.matmul)
+matrix_transpose = get_xp(np)(_aliases.matrix_transpose)
+tensordot = get_xp(np)(_aliases.tensordot)
+sign = get_xp(np)(_aliases.sign)
+finfo = get_xp(np)(_aliases.finfo)
+iinfo = get_xp(np)(_aliases.iinfo)
+
+
+def _supports_buffer_protocol(obj: object) -> TypeIs[Buffer]:  # pyright: ignore[reportUnusedFunction]
+    try:
+        memoryview(obj)  # pyright: ignore[reportArgumentType]
+    except TypeError:
+        return False
+    return True
+
+
+# asarray also adds the copy keyword, which is not present in numpy 1.0.
+# asarray() is different enough between numpy, cupy, and dask, the logic
+# complicated enough that it's easier to define it separately for each module
+# rather than trying to combine everything into one function in common/
+def asarray(
+    obj: Array | complex | NestedSequence[complex] | SupportsBufferProtocol,
+    /,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    copy: _Copy | None = None,
+    **kwargs: Any,
+) -> Array:
+    """
+    Array API compatibility wrapper for asarray().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    _helpers._check_device(np, device)
+
+    if copy is None:
+        copy = np._CopyMode.IF_NEEDED
+    elif copy is False:
+        copy = np._CopyMode.NEVER
+    elif copy is True:
+        copy = np._CopyMode.ALWAYS
+
+    return np.array(obj, copy=copy, dtype=dtype, **kwargs)  # pyright: ignore
+
+
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: py_bool = True,
+    device: Device | None = None,
+) -> Array:
+    _helpers._check_device(np, device)
+    return x.astype(dtype=dtype, copy=copy)
+
+
+# count_nonzero returns a python int for axis=None and keepdims=False
+# https://github.com/numpy/numpy/issues/17562
+def count_nonzero(
+    x: Array,
+    axis: int | tuple[int, ...] | None = None,
+    keepdims: py_bool = False,
+) -> Array:
+    # NOTE: this is currently incorrectly typed in numpy, but will be fixed in
+    # numpy 2.2.5 and 2.3.0: https://github.com/numpy/numpy/pull/28750
+    result = cast("Any", np.count_nonzero(x, axis=axis, keepdims=keepdims))  # pyright: ignore[reportArgumentType, reportCallIssue]
+    if axis is None and not keepdims:
+        return np.asarray(result)
+    return result
+
+
+# take_along_axis: axis defaults to -1 but in numpy axis is a required arg
+def take_along_axis(x: Array, indices: Array, /, *, axis: int = -1):
+    return np.take_along_axis(x, indices, axis=axis)
+
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(np, "vecdot"):
+    vecdot = np.vecdot
+else:
+    vecdot = get_xp(np)(_aliases.vecdot)
+
+if hasattr(np, "isdtype"):
+    isdtype = np.isdtype
+else:
+    isdtype = get_xp(np)(_aliases.isdtype)
+
+if hasattr(np, "unstack"):
+    unstack = np.unstack
+else:
+    unstack = get_xp(np)(_aliases.unstack)
+
+__all__ = [
+    "__array_namespace_info__",
+    "asarray",
+    "astype",
+    "acos",
+    "acosh",
+    "asin",
+    "asinh",
+    "atan",
+    "atan2",
+    "atanh",
+    "bitwise_left_shift",
+    "bitwise_invert",
+    "bitwise_right_shift",
+    "bool",
+    "concat",
+    "count_nonzero",
+    "pow",
+    "take_along_axis"
+]
+__all__ += _aliases.__all__
+_all_ignore = ["np", "get_xp"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_info.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..dfa70a391554e9dec2a451d3d30949a3b1f56a0c
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_info.py
@@ -0,0 +1,366 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+from __future__ import annotations
+
+from numpy import bool_ as bool
+from numpy import (
+    complex64,
+    complex128,
+    dtype,
+    float32,
+    float64,
+    int8,
+    int16,
+    int32,
+    int64,
+    intp,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+
+from ._typing import Device, DType
+
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for NumPy.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for NumPy.
+
+    Examples
+    --------
+    >>> info = np.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': numpy.float64,
+     'complex floating': numpy.complex128,
+     'integral': numpy.int64,
+     'indexing': numpy.int64}
+
+    """
+
+    __module__ = 'numpy'
+
+    def capabilities(self):
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing. Always ``True`` for NumPy.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes. Always ``True`` for
+          NumPy.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self):
+        """
+        The default device used for new NumPy arrays.
+
+        For NumPy, this always returns ``'cpu'``.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new NumPy arrays.
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.default_device()
+        'cpu'
+
+        """
+        return "cpu"
+
+    def default_dtypes(
+        self,
+        *,
+        device: Device | None = None,
+    ) -> dict[str, dtype[intp | float64 | complex128]]:
+        """
+        The default data types used for new NumPy arrays.
+
+        For NumPy, this always returns the following dictionary:
+
+        - **"real floating"**: ``numpy.float64``
+        - **"complex floating"**: ``numpy.complex128``
+        - **"integral"**: ``numpy.intp``
+        - **"indexing"**: ``numpy.intp``
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the default data types for. For NumPy, only
+            ``'cpu'`` is allowed.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new NumPy
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': numpy.float64,
+         'complex floating': numpy.complex128,
+         'integral': numpy.int64,
+         'indexing': numpy.int64}
+
+        """
+        if device not in ["cpu", None]:
+            raise ValueError(
+                'Device not understood. Only "cpu" is allowed, but received:'
+                f' {device}'
+            )
+        return {
+            "real floating": dtype(float64),
+            "complex floating": dtype(complex128),
+            "integral": dtype(intp),
+            "indexing": dtype(intp),
+        }
+
+    def dtypes(
+        self,
+        *,
+        device: Device | None = None,
+        kind: str | tuple[str, ...] | None = None,
+    ) -> dict[str, DType]:
+        """
+        The array API data types supported by NumPy.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the data types for. For NumPy, only ``'cpu'`` is
+            allowed.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            NumPy data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': numpy.int8,
+         'int16': numpy.int16,
+         'int32': numpy.int32,
+         'int64': numpy.int64}
+
+        """
+        if device not in ["cpu", None]:
+            raise ValueError(
+                'Device not understood. Only "cpu" is allowed, but received:'
+                f' {device}'
+            )
+        if kind is None:
+            return {
+                "bool": dtype(bool),
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "bool":
+            return {"bool": dtype(bool)}
+        if kind == "signed integer":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "integral":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "real floating":
+            return {
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "numeric":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if isinstance(kind, tuple):
+            res: dict[str, DType] = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    def devices(self) -> list[Device]:
+        """
+        The devices supported by NumPy.
+
+        For NumPy, this always returns ``['cpu']``.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by NumPy.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.devices()
+        ['cpu']
+
+        """
+        return ["cpu"]
+
+
+__all__ = ["__array_namespace_info__"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_typing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..fb40adb278945d0e4366c24763998205e9a7583a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/_typing.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias
+
+import numpy as np
+
+Device: TypeAlias = Literal["cpu"]
+
+if TYPE_CHECKING:
+
+    # NumPy 1.x on Python 3.10 fails to parse np.dtype[]
+    DType: TypeAlias = np.dtype[
+        np.bool_
+        | np.integer[Any]
+        | np.float32
+        | np.float64
+        | np.complex64
+        | np.complex128
+    ]
+    Array: TypeAlias = np.ndarray[Any, DType]
+else:
+    DType: TypeAlias = np.dtype
+    Array: TypeAlias = np.ndarray
+
+__all__ = ["Array", "DType", "Device"]
+_all_ignore = ["np"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/fft.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/fft.py
new file mode 100644
index 0000000000000000000000000000000000000000..b2d07f237f925ddd9e1168eb51596bda40bc9fa4
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/fft.py
@@ -0,0 +1,35 @@
+import numpy as np
+from numpy.fft import __all__ as fft_all
+from numpy.fft import fft2, ifft2, irfft2, rfft2
+
+from .._internal import get_xp
+from ..common import _fft
+
+fft = get_xp(np)(_fft.fft)
+ifft = get_xp(np)(_fft.ifft)
+fftn = get_xp(np)(_fft.fftn)
+ifftn = get_xp(np)(_fft.ifftn)
+rfft = get_xp(np)(_fft.rfft)
+irfft = get_xp(np)(_fft.irfft)
+rfftn = get_xp(np)(_fft.rfftn)
+irfftn = get_xp(np)(_fft.irfftn)
+hfft = get_xp(np)(_fft.hfft)
+ihfft = get_xp(np)(_fft.ihfft)
+fftfreq = get_xp(np)(_fft.fftfreq)
+rfftfreq = get_xp(np)(_fft.rfftfreq)
+fftshift = get_xp(np)(_fft.fftshift)
+ifftshift = get_xp(np)(_fft.ifftshift)
+
+
+__all__ = ["rfft2", "irfft2", "fft2", "ifft2"]
+__all__ += _fft.__all__
+
+
+def __dir__() -> list[str]:
+    return __all__
+
+
+del get_xp
+del np
+del fft_all
+del _fft
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/linalg.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/linalg.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bb59ec3f550b78b6dfe91119a4649e96b45ea02
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/numpy/linalg.py
@@ -0,0 +1,143 @@
+# pyright: reportAttributeAccessIssue=false
+# pyright: reportUnknownArgumentType=false
+# pyright: reportUnknownMemberType=false
+# pyright: reportUnknownVariableType=false
+
+from __future__ import annotations
+
+import numpy as np
+
+# intersection of `np.linalg.__all__` on numpy 1.22 and 2.2, minus `_linalg.__all__`
+from numpy.linalg import (
+    LinAlgError,
+    cond,
+    det,
+    eig,
+    eigvals,
+    eigvalsh,
+    inv,
+    lstsq,
+    matrix_power,
+    multi_dot,
+    norm,
+    tensorinv,
+    tensorsolve,
+)
+
+from .._internal import get_xp
+from ..common import _linalg
+
+# These functions are in both the main and linalg namespaces
+from ._aliases import matmul, matrix_transpose, tensordot, vecdot  # noqa: F401
+from ._typing import Array
+
+cross = get_xp(np)(_linalg.cross)
+outer = get_xp(np)(_linalg.outer)
+EighResult = _linalg.EighResult
+QRResult = _linalg.QRResult
+SlogdetResult = _linalg.SlogdetResult
+SVDResult = _linalg.SVDResult
+eigh = get_xp(np)(_linalg.eigh)
+qr = get_xp(np)(_linalg.qr)
+slogdet = get_xp(np)(_linalg.slogdet)
+svd = get_xp(np)(_linalg.svd)
+cholesky = get_xp(np)(_linalg.cholesky)
+matrix_rank = get_xp(np)(_linalg.matrix_rank)
+pinv = get_xp(np)(_linalg.pinv)
+matrix_norm = get_xp(np)(_linalg.matrix_norm)
+svdvals = get_xp(np)(_linalg.svdvals)
+diagonal = get_xp(np)(_linalg.diagonal)
+trace = get_xp(np)(_linalg.trace)
+
+# Note: unlike np.linalg.solve, the array API solve() only accepts x2 as a
+# vector when it is exactly 1-dimensional. All other cases treat x2 as a stack
+# of matrices. The np.linalg.solve behavior of allowing stacks of both
+# matrices and vectors is ambiguous c.f.
+# https://github.com/numpy/numpy/issues/15349 and
+# https://github.com/data-apis/array-api/issues/285.
+
+# To workaround this, the below is the code from np.linalg.solve except
+# only calling solve1 in the exactly 1D case.
+
+
+# This code is here instead of in common because it is numpy specific. Also
+# note that CuPy's solve() does not currently support broadcasting (see
+# https://github.com/cupy/cupy/blob/main/cupy/cublas.py#L43).
+def solve(x1: Array, x2: Array, /) -> Array:
+    try:
+        from numpy.linalg._linalg import (
+            _assert_stacked_2d,
+            _assert_stacked_square,
+            _commonType,
+            _makearray,
+            _raise_linalgerror_singular,
+            isComplexType,
+        )
+    except ImportError:
+        from numpy.linalg.linalg import (
+            _assert_stacked_2d,
+            _assert_stacked_square,
+            _commonType,
+            _makearray,
+            _raise_linalgerror_singular,
+            isComplexType,
+        )
+    from numpy.linalg import _umath_linalg
+
+    x1, _ = _makearray(x1)
+    _assert_stacked_2d(x1)
+    _assert_stacked_square(x1)
+    x2, wrap = _makearray(x2)
+    t, result_t = _commonType(x1, x2)
+
+    # This part is different from np.linalg.solve
+    gufunc: np.ufunc
+    if x2.ndim == 1:
+        gufunc = _umath_linalg.solve1
+    else:
+        gufunc = _umath_linalg.solve
+
+    # This does nothing currently but is left in because it will be relevant
+    # when complex dtype support is added to the spec in 2022.
+    signature = "DD->D" if isComplexType(t) else "dd->d"
+    with np.errstate(
+        call=_raise_linalgerror_singular,
+        invalid="call",
+        over="ignore",
+        divide="ignore",
+        under="ignore",
+    ):
+        r: Array = gufunc(x1, x2, signature=signature)
+
+    return wrap(r.astype(result_t, copy=False))
+
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(np.linalg, "vector_norm"):
+    vector_norm = np.linalg.vector_norm
+else:
+    vector_norm = get_xp(np)(_linalg.vector_norm)
+
+
+__all__ = [
+    "LinAlgError",
+    "cond",
+    "det",
+    "eig",
+    "eigvals",
+    "eigvalsh",
+    "inv",
+    "lstsq",
+    "matrix_power",
+    "multi_dot",
+    "norm",
+    "tensorinv",
+    "tensorsolve",
+]
+__all__ += _linalg.__all__
+__all__ += ["solve", "vector_norm"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/py.typed b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b454005ee94951d135a212d55734c69c35bd29e3
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/__init__.py
@@ -0,0 +1,22 @@
+from torch import * # noqa: F403
+
+# Several names are not included in the above import *
+import torch
+for n in dir(torch):
+    if (n.startswith('_')
+        or n.endswith('_')
+        or 'cuda' in n
+        or 'cpu' in n
+        or 'backward' in n):
+        continue
+    exec(f"{n} = torch.{n}")
+del n
+
+# These imports may overwrite names from the import * above.
+from ._aliases import * # noqa: F403
+
+# See the comment in the numpy __init__.py
+__import__(__package__ + '.linalg')
+__import__(__package__ + '.fft')
+
+__array_api_version__ = '2024.12'
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_aliases.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_aliases.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c039633bbcdb3e34a45d939190d8aa8d00540bb
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_aliases.py
@@ -0,0 +1,855 @@
+from __future__ import annotations
+
+from functools import reduce as _reduce, wraps as _wraps
+from builtins import all as _builtin_all, any as _builtin_any
+from typing import Any, List, Optional, Sequence, Tuple, Union, Literal
+
+import torch
+
+from .._internal import get_xp
+from ..common import _aliases
+from ..common._typing import NestedSequence, SupportsBufferProtocol
+from ._info import __array_namespace_info__
+from ._typing import Array, Device, DType
+
+_int_dtypes = {
+    torch.uint8,
+    torch.int8,
+    torch.int16,
+    torch.int32,
+    torch.int64,
+}
+try:
+    # torch >=2.3
+    _int_dtypes |= {torch.uint16, torch.uint32, torch.uint64}
+except AttributeError:
+    pass
+
+
+_array_api_dtypes = {
+    torch.bool,
+    *_int_dtypes,
+    torch.float32,
+    torch.float64,
+    torch.complex64,
+    torch.complex128,
+}
+
+_promotion_table = {
+    # ints
+    (torch.int8, torch.int16): torch.int16,
+    (torch.int8, torch.int32): torch.int32,
+    (torch.int8, torch.int64): torch.int64,
+    (torch.int16, torch.int32): torch.int32,
+    (torch.int16, torch.int64): torch.int64,
+    (torch.int32, torch.int64): torch.int64,
+    # ints and uints (mixed sign)
+    (torch.uint8, torch.int8): torch.int16,
+    (torch.uint8, torch.int16): torch.int16,
+    (torch.uint8, torch.int32): torch.int32,
+    (torch.uint8, torch.int64): torch.int64,
+    # floats
+    (torch.float32, torch.float64): torch.float64,
+    # complexes
+    (torch.complex64, torch.complex128): torch.complex128,
+    # Mixed float and complex
+    (torch.float32, torch.complex64): torch.complex64,
+    (torch.float32, torch.complex128): torch.complex128,
+    (torch.float64, torch.complex64): torch.complex128,
+    (torch.float64, torch.complex128): torch.complex128,
+}
+
+_promotion_table.update({(b, a): c for (a, b), c in _promotion_table.items()})
+_promotion_table.update({(a, a): a for a in _array_api_dtypes})
+
+
+def _two_arg(f):
+    @_wraps(f)
+    def _f(x1, x2, /, **kwargs):
+        x1, x2 = _fix_promotion(x1, x2)
+        return f(x1, x2, **kwargs)
+    if _f.__doc__ is None:
+        _f.__doc__ = f"""\
+Array API compatibility wrapper for torch.{f.__name__}.
+
+See the corresponding PyTorch documentation and/or the array API specification
+for more details.
+
+"""
+    return _f
+
+def _fix_promotion(x1, x2, only_scalar=True):
+    if not isinstance(x1, torch.Tensor) or not isinstance(x2, torch.Tensor):
+        return x1, x2
+    if x1.dtype not in _array_api_dtypes or x2.dtype not in _array_api_dtypes:
+        return x1, x2
+    # If an argument is 0-D pytorch downcasts the other argument
+    if not only_scalar or x1.shape == ():
+        dtype = result_type(x1, x2)
+        x2 = x2.to(dtype)
+    if not only_scalar or x2.shape == ():
+        dtype = result_type(x1, x2)
+        x1 = x1.to(dtype)
+    return x1, x2
+
+
+_py_scalars = (bool, int, float, complex)
+
+
+def result_type(
+    *arrays_and_dtypes: Array | DType | bool | int | float | complex
+) -> DType:
+    num = len(arrays_and_dtypes)
+
+    if num == 0:
+        raise ValueError("At least one array or dtype must be provided")
+
+    elif num == 1:
+        x = arrays_and_dtypes[0]
+        if isinstance(x, torch.dtype):
+            return x
+        return x.dtype
+
+    if num == 2:
+        x, y = arrays_and_dtypes
+        return _result_type(x, y)
+
+    else:
+        # sort scalars so that they are treated last
+        scalars, others = [], []
+        for x in arrays_and_dtypes:
+            if isinstance(x, _py_scalars):
+                scalars.append(x)
+            else:
+                others.append(x)
+        if not others:
+            raise ValueError("At least one array or dtype must be provided")
+
+        # combine left-to-right
+        return _reduce(_result_type, others + scalars)
+
+
+def _result_type(
+    x: Array | DType | bool | int | float | complex,
+    y: Array | DType | bool | int | float | complex,
+) -> DType:
+    if not (isinstance(x, _py_scalars) or isinstance(y, _py_scalars)):
+        xdt = x if isinstance(x, torch.dtype) else x.dtype
+        ydt = y if isinstance(y, torch.dtype) else y.dtype
+
+        try:
+            return _promotion_table[xdt, ydt]
+        except KeyError:
+            pass
+
+    # This doesn't result_type(dtype, dtype) for non-array API dtypes
+    # because torch.result_type only accepts tensors. This does however, allow
+    # cross-kind promotion.
+    x = torch.tensor([], dtype=x) if isinstance(x, torch.dtype) else x
+    y = torch.tensor([], dtype=y) if isinstance(y, torch.dtype) else y
+    return torch.result_type(x, y)
+
+
+def can_cast(from_: Union[DType, Array], to: DType, /) -> bool:
+    if not isinstance(from_, torch.dtype):
+        from_ = from_.dtype
+    return torch.can_cast(from_, to)
+
+# Basic renames
+bitwise_invert = torch.bitwise_not
+newaxis = None
+# torch.conj sets the conjugation bit, which breaks conversion to other
+# libraries. See https://github.com/data-apis/array-api-compat/issues/173
+conj = torch.conj_physical
+
+# Two-arg elementwise functions
+# These require a wrapper to do the correct type promotion on 0-D tensors
+add = _two_arg(torch.add)
+atan2 = _two_arg(torch.atan2)
+bitwise_and = _two_arg(torch.bitwise_and)
+bitwise_left_shift = _two_arg(torch.bitwise_left_shift)
+bitwise_or = _two_arg(torch.bitwise_or)
+bitwise_right_shift = _two_arg(torch.bitwise_right_shift)
+bitwise_xor = _two_arg(torch.bitwise_xor)
+copysign = _two_arg(torch.copysign)
+divide = _two_arg(torch.divide)
+# Also a rename. torch.equal does not broadcast
+equal = _two_arg(torch.eq)
+floor_divide = _two_arg(torch.floor_divide)
+greater = _two_arg(torch.greater)
+greater_equal = _two_arg(torch.greater_equal)
+hypot = _two_arg(torch.hypot)
+less = _two_arg(torch.less)
+less_equal = _two_arg(torch.less_equal)
+logaddexp = _two_arg(torch.logaddexp)
+# logical functions are not included here because they only accept bool in the
+# spec, so type promotion is irrelevant.
+maximum = _two_arg(torch.maximum)
+minimum = _two_arg(torch.minimum)
+multiply = _two_arg(torch.multiply)
+not_equal = _two_arg(torch.not_equal)
+pow = _two_arg(torch.pow)
+remainder = _two_arg(torch.remainder)
+subtract = _two_arg(torch.subtract)
+
+
+def asarray(
+    obj: (
+    Array 
+        | bool | int | float | complex 
+        | NestedSequence[bool | int | float | complex] 
+        | SupportsBufferProtocol
+    ),
+    /,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    copy: bool | None = None,
+    **kwargs: Any,
+) -> Array:
+    # torch.asarray does not respect input->output device propagation
+    # https://github.com/pytorch/pytorch/issues/150199
+    if device is None and isinstance(obj, torch.Tensor):
+        device = obj.device
+    return torch.asarray(obj, dtype=dtype, device=device, copy=copy, **kwargs)
+
+
+# These wrappers are mostly based on the fact that pytorch uses 'dim' instead
+# of 'axis'.
+
+# torch.min and torch.max return a tuple and don't support multiple axes https://github.com/pytorch/pytorch/issues/58745
+def max(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None, keepdims: bool = False) -> Array:
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.clone(x)
+    return torch.amax(x, axis, keepdims=keepdims)
+
+def min(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None, keepdims: bool = False) -> Array:
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.clone(x)
+    return torch.amin(x, axis, keepdims=keepdims)
+
+clip = get_xp(torch)(_aliases.clip)
+unstack = get_xp(torch)(_aliases.unstack)
+cumulative_sum = get_xp(torch)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(torch)(_aliases.cumulative_prod)
+finfo = get_xp(torch)(_aliases.finfo)
+iinfo = get_xp(torch)(_aliases.iinfo)
+
+
+# torch.sort also returns a tuple
+# https://github.com/pytorch/pytorch/issues/70921
+def sort(x: Array, /, *, axis: int = -1, descending: bool = False, stable: bool = True, **kwargs) -> Array:
+    return torch.sort(x, dim=axis, descending=descending, stable=stable, **kwargs).values
+
+def _normalize_axes(axis, ndim):
+    axes = []
+    if ndim == 0 and axis:
+        # Better error message in this case
+        raise IndexError(f"Dimension out of range: {axis[0]}")
+    lower, upper = -ndim, ndim - 1
+    for a in axis:
+        if a < lower or a > upper:
+            # Match torch error message (e.g., from sum())
+            raise IndexError(f"Dimension out of range (expected to be in range of [{lower}, {upper}], but got {a}")
+        if a < 0:
+            a = a + ndim
+        if a in axes:
+            # Use IndexError instead of RuntimeError, and "axis" instead of "dim"
+            raise IndexError(f"Axis {a} appears multiple times in the list of axes")
+        axes.append(a)
+    return sorted(axes)
+
+def _axis_none_keepdims(x, ndim, keepdims):
+    # Apply keepdims when axis=None
+    # (https://github.com/pytorch/pytorch/issues/71209)
+    # Note that this is only valid for the axis=None case.
+    if keepdims:
+        for i in range(ndim):
+            x = torch.unsqueeze(x, 0)
+    return x
+
+def _reduce_multiple_axes(f, x, axis, keepdims=False, **kwargs):
+    # Some reductions don't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    axes = _normalize_axes(axis, x.ndim)
+    for a in reversed(axes):
+        x = torch.movedim(x, a, -1)
+    x = torch.flatten(x, -len(axes))
+
+    out = f(x, -1, **kwargs)
+
+    if keepdims:
+        for a in axes:
+            out = torch.unsqueeze(out, a)
+    return out
+
+
+def _sum_prod_no_axis(x: Array, dtype: DType | None) -> Array:
+    """
+    Implements `sum(..., axis=())` and `prod(..., axis=())`.
+    
+    Works around https://github.com/pytorch/pytorch/issues/29137
+    """
+    if dtype is not None:
+        return x.clone() if dtype == x.dtype else x.to(dtype)
+
+    # We can't upcast uint8 according to the spec because there is no
+    # torch.uint64, so at least upcast to int64 which is what prod does
+    # when axis=None.
+    if x.dtype in (torch.uint8, torch.int8, torch.int16, torch.int32):
+        return x.to(torch.int64)
+
+    return x.clone()
+
+
+def prod(x: Array,
+         /,
+         *,
+         axis: Optional[Union[int, Tuple[int, ...]]] = None,
+         dtype: Optional[DType] = None,
+         keepdims: bool = False,
+         **kwargs) -> Array:
+
+    if axis == ():
+        return _sum_prod_no_axis(x, dtype)
+    # torch.prod doesn't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    if isinstance(axis, tuple):
+        return _reduce_multiple_axes(torch.prod, x, axis, keepdims=keepdims, dtype=dtype, **kwargs)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.prod(x, dtype=dtype, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+
+    return torch.prod(x, axis, dtype=dtype, keepdims=keepdims, **kwargs)
+
+
+def sum(x: Array,
+         /,
+         *,
+         axis: Optional[Union[int, Tuple[int, ...]]] = None,
+         dtype: Optional[DType] = None,
+         keepdims: bool = False,
+         **kwargs) -> Array:
+
+    if axis == ():
+        return _sum_prod_no_axis(x, dtype)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.sum(x, dtype=dtype, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+
+    return torch.sum(x, axis, dtype=dtype, keepdims=keepdims, **kwargs)
+
+def any(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+
+    if axis == ():
+        return x.to(torch.bool)
+    # torch.any doesn't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    if isinstance(axis, tuple):
+        res = _reduce_multiple_axes(torch.any, x, axis, keepdims=keepdims, **kwargs)
+        return res.to(torch.bool)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.any(x, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res.to(torch.bool)
+
+    # torch.any doesn't return bool for uint8
+    return torch.any(x, axis, keepdims=keepdims).to(torch.bool)
+
+def all(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+
+    if axis == ():
+        return x.to(torch.bool)
+    # torch.all doesn't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    if isinstance(axis, tuple):
+        res = _reduce_multiple_axes(torch.all, x, axis, keepdims=keepdims, **kwargs)
+        return res.to(torch.bool)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.all(x, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res.to(torch.bool)
+
+    # torch.all doesn't return bool for uint8
+    return torch.all(x, axis, keepdims=keepdims).to(torch.bool)
+
+def mean(x: Array,
+         /,
+         *,
+         axis: Optional[Union[int, Tuple[int, ...]]] = None,
+         keepdims: bool = False,
+         **kwargs) -> Array:
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.clone(x)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.mean(x, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+    return torch.mean(x, axis, keepdims=keepdims, **kwargs)
+
+def std(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        correction: Union[int, float] = 0.0,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+    # Note, float correction is not supported
+    # https://github.com/pytorch/pytorch/issues/61492. We don't try to
+    # implement it here for now.
+
+    if isinstance(correction, float):
+        _correction = int(correction)
+        if correction != _correction:
+            raise NotImplementedError("float correction in torch std() is not yet supported")
+    else:
+        _correction = correction
+
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.zeros_like(x)
+    if isinstance(axis, int):
+        axis = (axis,)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.std(x, tuple(range(x.ndim)), correction=_correction, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+    return torch.std(x, axis, correction=_correction, keepdims=keepdims, **kwargs)
+
+def var(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        correction: Union[int, float] = 0.0,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+    # Note, float correction is not supported
+    # https://github.com/pytorch/pytorch/issues/61492. We don't try to
+    # implement it here for now.
+
+    # if isinstance(correction, float):
+    #     correction = int(correction)
+
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.zeros_like(x)
+    if isinstance(axis, int):
+        axis = (axis,)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.var(x, tuple(range(x.ndim)), correction=correction, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+    return torch.var(x, axis, correction=correction, keepdims=keepdims, **kwargs)
+
+# torch.concat doesn't support dim=None
+# https://github.com/pytorch/pytorch/issues/70925
+def concat(arrays: Union[Tuple[Array, ...], List[Array]],
+           /,
+           *,
+           axis: Optional[int] = 0,
+           **kwargs) -> Array:
+    if axis is None:
+        arrays = tuple(ar.flatten() for ar in arrays)
+        axis = 0
+    return torch.concat(arrays, axis, **kwargs)
+
+# torch.squeeze only accepts int dim and doesn't require it
+# https://github.com/pytorch/pytorch/issues/70924. Support for tuple dim was
+# added at https://github.com/pytorch/pytorch/pull/89017.
+def squeeze(x: Array, /, axis: Union[int, Tuple[int, ...]]) -> Array:
+    if isinstance(axis, int):
+        axis = (axis,)
+    for a in axis:
+        if x.shape[a] != 1:
+            raise ValueError("squeezed dimensions must be equal to 1")
+    axes = _normalize_axes(axis, x.ndim)
+    # Remove this once pytorch 1.14 is released with the above PR #89017.
+    sequence = [a - i for i, a in enumerate(axes)]
+    for a in sequence:
+        x = torch.squeeze(x, a)
+    return x
+
+# torch.broadcast_to uses size instead of shape
+def broadcast_to(x: Array, /, shape: Tuple[int, ...], **kwargs) -> Array:
+    return torch.broadcast_to(x, shape, **kwargs)
+
+# torch.permute uses dims instead of axes
+def permute_dims(x: Array, /, axes: Tuple[int, ...]) -> Array:
+    return torch.permute(x, axes)
+
+# The axis parameter doesn't work for flip() and roll()
+# https://github.com/pytorch/pytorch/issues/71210. Also torch.flip() doesn't
+# accept axis=None
+def flip(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None, **kwargs) -> Array:
+    if axis is None:
+        axis = tuple(range(x.ndim))
+    # torch.flip doesn't accept dim as an int but the method does
+    # https://github.com/pytorch/pytorch/issues/18095
+    return x.flip(axis, **kwargs)
+
+def roll(x: Array, /, shift: Union[int, Tuple[int, ...]], *, axis: Optional[Union[int, Tuple[int, ...]]] = None, **kwargs) -> Array:
+    return torch.roll(x, shift, axis, **kwargs)
+
+def nonzero(x: Array, /, **kwargs) -> Tuple[Array, ...]:
+    if x.ndim == 0:
+        raise ValueError("nonzero() does not support zero-dimensional arrays")
+    return torch.nonzero(x, as_tuple=True, **kwargs)
+
+
+# torch uses `dim` instead of `axis`
+def diff(
+    x: Array,
+    /,
+    *,
+    axis: int = -1,
+    n: int = 1,
+    prepend: Optional[Array] = None,
+    append: Optional[Array] = None,
+) -> Array:
+    return torch.diff(x, dim=axis, n=n, prepend=prepend, append=append)
+
+
+# torch uses `dim` instead of `axis`, does not have keepdims
+def count_nonzero(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+) -> Array:
+    result = torch.count_nonzero(x, dim=axis)
+    if keepdims:
+        if isinstance(axis, int):
+            return result.unsqueeze(axis)
+        elif isinstance(axis, tuple):
+            n_axis = [x.ndim + ax if ax < 0 else ax for ax in axis]
+            sh = [1 if i in n_axis else x.shape[i] for i in range(x.ndim)]
+            return torch.reshape(result, sh)
+        return _axis_none_keepdims(result, x.ndim, keepdims)
+    else:
+        return result
+
+
+# "repeat" is torch.repeat_interleave;  also the dim argument
+def repeat(x: Array, repeats: int | Array, /, *, axis: int | None = None) -> Array:
+    return torch.repeat_interleave(x, repeats, axis)
+
+
+def where(
+    condition: Array, 
+    x1: Array | bool | int | float | complex, 
+    x2: Array | bool | int | float | complex,
+    /,
+) -> Array:
+    x1, x2 = _fix_promotion(x1, x2)
+    return torch.where(condition, x1, x2)
+
+
+# torch.reshape doesn't have the copy keyword
+def reshape(x: Array,
+            /,
+            shape: Tuple[int, ...],
+            *,
+            copy: Optional[bool] = None,
+            **kwargs) -> Array:
+    if copy is not None:
+        raise NotImplementedError("torch.reshape doesn't yet support the copy keyword")
+    return torch.reshape(x, shape, **kwargs)
+
+# torch.arange doesn't support returning empty arrays
+# (https://github.com/pytorch/pytorch/issues/70915), and doesn't support some
+# keyword argument combinations
+# (https://github.com/pytorch/pytorch/issues/70914)
+def arange(start: Union[int, float],
+           /,
+           stop: Optional[Union[int, float]] = None,
+           step: Union[int, float] = 1,
+           *,
+           dtype: Optional[DType] = None,
+           device: Optional[Device] = None,
+           **kwargs) -> Array:
+    if stop is None:
+        start, stop = 0, start
+    if step > 0 and stop <= start or step < 0 and stop >= start:
+        if dtype is None:
+            if _builtin_all(isinstance(i, int) for i in [start, stop, step]):
+                dtype = torch.int64
+            else:
+                dtype = torch.float32
+        return torch.empty(0, dtype=dtype, device=device, **kwargs)
+    return torch.arange(start, stop, step, dtype=dtype, device=device, **kwargs)
+
+# torch.eye does not accept None as a default for the second argument and
+# doesn't support off-diagonals (https://github.com/pytorch/pytorch/issues/70910)
+def eye(n_rows: int,
+        n_cols: Optional[int] = None,
+        /,
+        *,
+        k: int = 0,
+        dtype: Optional[DType] = None,
+        device: Optional[Device] = None,
+        **kwargs) -> Array:
+    if n_cols is None:
+        n_cols = n_rows
+    z = torch.zeros(n_rows, n_cols, dtype=dtype, device=device, **kwargs)
+    if abs(k) <= n_rows + n_cols:
+        z.diagonal(k).fill_(1)
+    return z
+
+# torch.linspace doesn't have the endpoint parameter
+def linspace(start: Union[int, float],
+             stop: Union[int, float],
+             /,
+             num: int,
+             *,
+             dtype: Optional[DType] = None,
+             device: Optional[Device] = None,
+             endpoint: bool = True,
+             **kwargs) -> Array:
+    if not endpoint:
+        return torch.linspace(start, stop, num+1, dtype=dtype, device=device, **kwargs)[:-1]
+    return torch.linspace(start, stop, num, dtype=dtype, device=device, **kwargs)
+
+# torch.full does not accept an int size
+# https://github.com/pytorch/pytorch/issues/70906
+def full(shape: Union[int, Tuple[int, ...]],
+         fill_value: bool | int | float | complex,
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    if isinstance(shape, int):
+        shape = (shape,)
+
+    return torch.full(shape, fill_value, dtype=dtype, device=device, **kwargs)
+
+# ones, zeros, and empty do not accept shape as a keyword argument
+def ones(shape: Union[int, Tuple[int, ...]],
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    return torch.ones(shape, dtype=dtype, device=device, **kwargs)
+
+def zeros(shape: Union[int, Tuple[int, ...]],
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    return torch.zeros(shape, dtype=dtype, device=device, **kwargs)
+
+def empty(shape: Union[int, Tuple[int, ...]],
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    return torch.empty(shape, dtype=dtype, device=device, **kwargs)
+
+# tril and triu do not call the keyword argument k
+
+def tril(x: Array, /, *, k: int = 0) -> Array:
+    return torch.tril(x, k)
+
+def triu(x: Array, /, *, k: int = 0) -> Array:
+    return torch.triu(x, k)
+
+# Functions that aren't in torch https://github.com/pytorch/pytorch/issues/58742
+def expand_dims(x: Array, /, *, axis: int = 0) -> Array:
+    return torch.unsqueeze(x, axis)
+
+
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: bool = True,
+    device: Optional[Device] = None,
+) -> Array:
+    if device is not None:
+        return x.to(device, dtype=dtype, copy=copy)
+    return x.to(dtype=dtype, copy=copy)
+
+
+def broadcast_arrays(*arrays: Array) -> List[Array]:
+    shape = torch.broadcast_shapes(*[a.shape for a in arrays])
+    return [torch.broadcast_to(a, shape) for a in arrays]
+
+# Note that these named tuples aren't actually part of the standard namespace,
+# but I don't see any issue with exporting the names here regardless.
+from ..common._aliases import (UniqueAllResult, UniqueCountsResult,
+                               UniqueInverseResult)
+
+# https://github.com/pytorch/pytorch/issues/70920
+def unique_all(x: Array) -> UniqueAllResult:
+    # torch.unique doesn't support returning indices.
+    # https://github.com/pytorch/pytorch/issues/36748. The workaround
+    # suggested in that issue doesn't actually function correctly (it relies
+    # on non-deterministic behavior of scatter()).
+    raise NotImplementedError("unique_all() not yet implemented for pytorch (see https://github.com/pytorch/pytorch/issues/36748)")
+
+    # values, inverse_indices, counts = torch.unique(x, return_counts=True, return_inverse=True)
+    # # torch.unique incorrectly gives a 0 count for nan values.
+    # # https://github.com/pytorch/pytorch/issues/94106
+    # counts[torch.isnan(values)] = 1
+    # return UniqueAllResult(values, indices, inverse_indices, counts)
+
+def unique_counts(x: Array) -> UniqueCountsResult:
+    values, counts = torch.unique(x, return_counts=True)
+
+    # torch.unique incorrectly gives a 0 count for nan values.
+    # https://github.com/pytorch/pytorch/issues/94106
+    counts[torch.isnan(values)] = 1
+    return UniqueCountsResult(values, counts)
+
+def unique_inverse(x: Array) -> UniqueInverseResult:
+    values, inverse = torch.unique(x, return_inverse=True)
+    return UniqueInverseResult(values, inverse)
+
+def unique_values(x: Array) -> Array:
+    return torch.unique(x)
+
+def matmul(x1: Array, x2: Array, /, **kwargs) -> Array:
+    # torch.matmul doesn't type promote (but differently from _fix_promotion)
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    return torch.matmul(x1, x2, **kwargs)
+
+matrix_transpose = get_xp(torch)(_aliases.matrix_transpose)
+_vecdot = get_xp(torch)(_aliases.vecdot)
+
+def vecdot(x1: Array, x2: Array, /, *, axis: int = -1) -> Array:
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    return _vecdot(x1, x2, axis=axis)
+
+# torch.tensordot uses dims instead of axes
+def tensordot(
+    x1: Array,
+    x2: Array,
+    /,
+    *, 
+    axes: Union[int, Tuple[Sequence[int], Sequence[int]]] = 2, 
+    **kwargs,
+) -> Array:
+    # Note: torch.tensordot fails with integer dtypes when there is only 1
+    # element in the axis (https://github.com/pytorch/pytorch/issues/84530).
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    return torch.tensordot(x1, x2, dims=axes, **kwargs)
+
+
+def isdtype(
+    dtype: DType, kind: Union[DType, str, Tuple[Union[DType, str], ...]],
+    *, _tuple=True, # Disallow nested tuples
+) -> bool:
+    """
+    Returns a boolean indicating whether a provided dtype is of a specified data type ``kind``.
+
+    Note that outside of this function, this compat library does not yet fully
+    support complex numbers.
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/generated/array_api.isdtype.html
+    for more details
+    """
+    if isinstance(kind, tuple) and _tuple:
+        return _builtin_any(isdtype(dtype, k, _tuple=False) for k in kind)
+    elif isinstance(kind, str):
+        if kind == 'bool':
+            return dtype == torch.bool
+        elif kind == 'signed integer':
+            return dtype in _int_dtypes and dtype.is_signed
+        elif kind == 'unsigned integer':
+            return dtype in _int_dtypes and not dtype.is_signed
+        elif kind == 'integral':
+            return dtype in _int_dtypes
+        elif kind == 'real floating':
+            return dtype.is_floating_point
+        elif kind == 'complex floating':
+            return dtype.is_complex
+        elif kind == 'numeric':
+            return isdtype(dtype, ('integral', 'real floating', 'complex floating'))
+        else:
+            raise ValueError(f"Unrecognized data type kind: {kind!r}")
+    else:
+        return dtype == kind
+
+def take(x: Array, indices: Array, /, *, axis: Optional[int] = None, **kwargs) -> Array:
+    if axis is None:
+        if x.ndim != 1:
+            raise ValueError("axis must be specified when ndim > 1")
+        axis = 0
+    return torch.index_select(x, axis, indices, **kwargs)
+
+
+def take_along_axis(x: Array, indices: Array, /, *, axis: int = -1) -> Array:
+    return torch.take_along_dim(x, indices, dim=axis)
+
+
+def sign(x: Array, /) -> Array:
+    # torch sign() does not support complex numbers and does not propagate
+    # nans. See https://github.com/data-apis/array-api-compat/issues/136
+    if x.dtype.is_complex:
+        out = x/torch.abs(x)
+        # sign(0) = 0 but the above formula would give nan
+        out[x == 0+0j] = 0+0j
+        return out
+    else:
+        out = torch.sign(x)
+        if x.dtype.is_floating_point:
+            out[torch.isnan(x)] = torch.nan
+        return out
+
+
+def meshgrid(*arrays: Array, indexing: Literal['xy', 'ij'] = 'xy') -> List[Array]:
+    # enforce the default of 'xy'
+    # TODO: is the return type a list or a tuple
+    return list(torch.meshgrid(*arrays, indexing='xy'))
+
+
+__all__ = ['__array_namespace_info__', 'asarray', 'result_type', 'can_cast',
+           'permute_dims', 'bitwise_invert', 'newaxis', 'conj', 'add',
+           'atan2', 'bitwise_and', 'bitwise_left_shift', 'bitwise_or',
+           'bitwise_right_shift', 'bitwise_xor', 'copysign', 'count_nonzero',
+           'diff', 'divide',
+           'equal', 'floor_divide', 'greater', 'greater_equal', 'hypot',
+           'less', 'less_equal', 'logaddexp', 'maximum', 'minimum',
+           'multiply', 'not_equal', 'pow', 'remainder', 'subtract', 'max',
+           'min', 'clip', 'unstack', 'cumulative_sum', 'cumulative_prod', 'sort', 'prod', 'sum',
+           'any', 'all', 'mean', 'std', 'var', 'concat', 'squeeze',
+           'broadcast_to', 'flip', 'roll', 'nonzero', 'where', 'reshape',
+           'arange', 'eye', 'linspace', 'full', 'ones', 'zeros', 'empty',
+           'tril', 'triu', 'expand_dims', 'astype', 'broadcast_arrays',
+           'UniqueAllResult', 'UniqueCountsResult', 'UniqueInverseResult',
+           'unique_all', 'unique_counts', 'unique_inverse', 'unique_values',
+           'matmul', 'matrix_transpose', 'vecdot', 'tensordot', 'isdtype',
+           'take', 'take_along_axis', 'sign', 'finfo', 'iinfo', 'repeat', 'meshgrid']
+
+_all_ignore = ['torch', 'get_xp']
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_info.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..167dcf477adece2b49446061b2a67e0bdb6cd67c
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_info.py
@@ -0,0 +1,369 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+import torch
+
+from functools import cache
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for PyTorch.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for PyTorch.
+
+    Examples
+    --------
+    >>> info = xp.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': numpy.float64,
+     'complex floating': numpy.complex128,
+     'integral': numpy.int64,
+     'indexing': numpy.int64}
+
+    """
+
+    __module__ = 'torch'
+
+    def capabilities(self):
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing. Always ``True`` for PyTorch.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes. Always ``True`` for
+          PyTorch.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self):
+        """
+        The default device used for new PyTorch arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new PyTorch arrays.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_device()
+        device(type='cpu')
+
+        Notes
+        -----
+        This method returns the static default device when PyTorch is initialized.
+        However, the *current* device used by creation functions (``empty`` etc.)
+        can be changed at runtime.
+
+        See Also
+        --------
+        https://github.com/data-apis/array-api/issues/835
+        """
+        return torch.device("cpu")
+
+    def default_dtypes(self, *, device=None):
+        """
+        The default data types used for new PyTorch arrays.
+
+        Parameters
+        ----------
+        device : Device, optional
+            The device to get the default data types for.
+            Unused for PyTorch, as all devices use the same default dtypes.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new PyTorch
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': torch.float32,
+         'complex floating': torch.complex64,
+         'integral': torch.int64,
+         'indexing': torch.int64}
+
+        """
+        # Note: if the default is set to float64, the devices like MPS that
+        # don't support float64 will error. We still return the default_dtype
+        # value here because this error doesn't represent a different default
+        # per-device.
+        default_floating = torch.get_default_dtype()
+        default_complex = torch.complex64 if default_floating == torch.float32 else torch.complex128
+        default_integral = torch.int64
+        return {
+            "real floating": default_floating,
+            "complex floating": default_complex,
+            "integral": default_integral,
+            "indexing": default_integral,
+        }
+
+
+    def _dtypes(self, kind):
+        bool = torch.bool
+        int8 = torch.int8
+        int16 = torch.int16
+        int32 = torch.int32
+        int64 = torch.int64
+        uint8 = torch.uint8
+        # uint16, uint32, and uint64 are present in newer versions of pytorch,
+        # but they aren't generally supported by the array API functions, so
+        # we omit them from this function.
+        float32 = torch.float32
+        float64 = torch.float64
+        complex64 = torch.complex64
+        complex128 = torch.complex128
+
+        if kind is None:
+            return {
+                "bool": bool,
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+                "uint8": uint8,
+                "float32": float32,
+                "float64": float64,
+                "complex64": complex64,
+                "complex128": complex128,
+            }
+        if kind == "bool":
+            return {"bool": bool}
+        if kind == "signed integer":
+            return {
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": uint8,
+            }
+        if kind == "integral":
+            return {
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+                "uint8": uint8,
+            }
+        if kind == "real floating":
+            return {
+                "float32": float32,
+                "float64": float64,
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": complex64,
+                "complex128": complex128,
+            }
+        if kind == "numeric":
+            return {
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+                "uint8": uint8,
+                "float32": float32,
+                "float64": float64,
+                "complex64": complex64,
+                "complex128": complex128,
+            }
+        if isinstance(kind, tuple):
+            res = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    @cache
+    def dtypes(self, *, device=None, kind=None):
+        """
+        The array API data types supported by PyTorch.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : Device, optional
+            The device to get the data types for.
+            Unused for PyTorch, as all devices use the same dtypes.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            PyTorch data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': numpy.int8,
+         'int16': numpy.int16,
+         'int32': numpy.int32,
+         'int64': numpy.int64}
+
+        """
+        res = self._dtypes(kind)
+        for k, v in res.copy().items():
+            try:
+                torch.empty((0,), dtype=v, device=device)
+            except:
+                del res[k]
+        return res
+
+    @cache
+    def devices(self):
+        """
+        The devices supported by PyTorch.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by PyTorch.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.devices()
+        [device(type='cpu'), device(type='mps', index=0), device(type='meta')]
+
+        """
+        # Torch doesn't have a straightforward way to get the list of all
+        # currently supported devices. To do this, we first parse the error
+        # message of torch.device to get the list of all possible types of
+        # device:
+        try:
+            torch.device('notadevice')
+            raise AssertionError("unreachable")  # pragma: nocover
+        except RuntimeError as e:
+            # The error message is something like:
+            # "Expected one of cpu, cuda, ipu, xpu, mkldnn, opengl, opencl, ideep, hip, ve, fpga, ort, xla, lazy, vulkan, mps, meta, hpu, mtia, privateuseone device type at start of device string: notadevice"
+            devices_names = e.args[0].split('Expected one of ')[1].split(' device type')[0].split(', ')
+
+        # Next we need to check for different indices for different devices.
+        # device(device_name, index=index) doesn't actually check if the
+        # device name or index is valid. We have to try to create a tensor
+        # with it (which is why this function is cached).
+        devices = []
+        for device_name in devices_names:
+            i = 0
+            while True:
+                try:
+                    a = torch.empty((0,), device=torch.device(device_name, index=i))
+                    if a.device in devices:
+                        break
+                    devices.append(a.device)
+                except:
+                    break
+                i += 1
+
+        return devices
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_typing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2a7a1b11180dd087b662356b90ad7e7fa7a10b6
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/_typing.py
@@ -0,0 +1,3 @@
+__all__ = ["Array", "Device", "DType"]
+
+from torch import device as Device, dtype as DType, Tensor as Array
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/fft.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/fft.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0a6323c35909d2e94ec6dec3c9daf12661e8f8a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/fft.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Union, Sequence, Literal
+
+import torch
+import torch.fft
+from torch.fft import * # noqa: F403
+
+from ._typing import Array
+
+# Several torch fft functions do not map axes to dim
+
+def fftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.fftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def ifftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.ifftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def rfftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.rfftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def irfftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.irfftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def fftshift(
+    x: Array,
+    /,
+    *,
+    axes: Union[int, Sequence[int]] = None,
+    **kwargs,
+) -> Array:
+    return torch.fft.fftshift(x, dim=axes, **kwargs)
+
+def ifftshift(
+    x: Array,
+    /,
+    *,
+    axes: Union[int, Sequence[int]] = None,
+    **kwargs,
+) -> Array:
+    return torch.fft.ifftshift(x, dim=axes, **kwargs)
+
+
+__all__ = torch.fft.__all__ + [
+    "fftn",
+    "ifftn",
+    "rfftn",
+    "irfftn",
+    "fftshift",
+    "ifftshift",
+]
+
+_all_ignore = ['torch']
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/linalg.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/linalg.py
new file mode 100644
index 0000000000000000000000000000000000000000..b023a9f897e20a50d6761f166d1c9c1a22c7a95a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_compat/torch/linalg.py
@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import torch
+from typing import Optional, Union, Tuple
+
+from torch.linalg import * # noqa: F403
+
+# torch.linalg doesn't define __all__
+# from torch.linalg import __all__ as linalg_all
+from torch import linalg as torch_linalg
+linalg_all = [i for i in dir(torch_linalg) if not i.startswith('_')]
+
+# outer is implemented in torch but aren't in the linalg namespace
+from torch import outer
+from ._aliases import _fix_promotion, sum
+# These functions are in both the main and linalg namespaces
+from ._aliases import matmul, matrix_transpose, tensordot
+from ._typing import Array, DType
+from ..common._typing import JustInt, JustFloat
+
+# Note: torch.linalg.cross does not default to axis=-1 (it defaults to the
+# first axis with size 3), see https://github.com/pytorch/pytorch/issues/58743
+
+# torch.cross also does not support broadcasting when it would add new
+# dimensions https://github.com/pytorch/pytorch/issues/39656
+def cross(x1: Array, x2: Array, /, *, axis: int = -1) -> Array:
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    if not (-min(x1.ndim, x2.ndim) <= axis < max(x1.ndim, x2.ndim)):
+        raise ValueError(f"axis {axis} out of bounds for cross product of arrays with shapes {x1.shape} and {x2.shape}")
+    if not (x1.shape[axis] == x2.shape[axis] == 3):
+        raise ValueError(f"cross product axis must have size 3, got {x1.shape[axis]} and {x2.shape[axis]}")
+    x1, x2 = torch.broadcast_tensors(x1, x2)
+    return torch_linalg.cross(x1, x2, dim=axis)
+
+def vecdot(x1: Array, x2: Array, /, *, axis: int = -1, **kwargs) -> Array:
+    from ._aliases import isdtype
+
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+
+    # torch.linalg.vecdot incorrectly allows broadcasting along the contracted dimension
+    if x1.shape[axis] != x2.shape[axis]:
+        raise ValueError("x1 and x2 must have the same size along the given axis")
+
+    # torch.linalg.vecdot doesn't support integer dtypes
+    if isdtype(x1.dtype, 'integral') or isdtype(x2.dtype, 'integral'):
+        if kwargs:
+            raise RuntimeError("vecdot kwargs not supported for integral dtypes")
+
+        x1_ = torch.moveaxis(x1, axis, -1)
+        x2_ = torch.moveaxis(x2, axis, -1)
+        x1_, x2_ = torch.broadcast_tensors(x1_, x2_)
+
+        res = x1_[..., None, :] @ x2_[..., None]
+        return res[..., 0, 0]
+    return torch.linalg.vecdot(x1, x2, dim=axis, **kwargs)
+
+def solve(x1: Array, x2: Array, /, **kwargs) -> Array:
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    # Torch tries to emulate NumPy 1 solve behavior by using batched 1-D solve
+    # whenever
+    # 1. x1.ndim - 1 == x2.ndim
+    # 2. x1.shape[:-1] == x2.shape
+    #
+    # See linalg_solve_is_vector_rhs in
+    # aten/src/ATen/native/LinearAlgebraUtils.h and
+    # TORCH_META_FUNC(_linalg_solve_ex) in
+    # aten/src/ATen/native/BatchLinearAlgebra.cpp in the PyTorch source code.
+    #
+    # The easiest way to work around this is to prepend a size 1 dimension to
+    # x2, since x2 is already one dimension less than x1.
+    #
+    # See https://github.com/pytorch/pytorch/issues/52915
+    if x2.ndim != 1 and x1.ndim - 1 == x2.ndim and x1.shape[:-1] == x2.shape:
+        x2 = x2[None]
+    return torch.linalg.solve(x1, x2, **kwargs)
+
+# torch.trace doesn't support the offset argument and doesn't support stacking
+def trace(x: Array, /, *, offset: int = 0, dtype: Optional[DType] = None) -> Array:
+    # Use our wrapped sum to make sure it does upcasting correctly
+    return sum(torch.diagonal(x, offset=offset, dim1=-2, dim2=-1), axis=-1, dtype=dtype)
+
+def vector_norm(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+    # JustFloat stands for inf | -inf, which are not valid for Literal
+    ord: JustInt | JustFloat = 2,
+    **kwargs,
+) -> Array:
+    # torch.vector_norm incorrectly treats axis=() the same as axis=None
+    if axis == ():
+        out = kwargs.get('out')
+        if out is None:
+            dtype = None
+            if x.dtype == torch.complex64:
+                dtype = torch.float32
+            elif x.dtype == torch.complex128:
+                dtype = torch.float64
+
+            out = torch.zeros_like(x, dtype=dtype)
+
+        # The norm of a single scalar works out to abs(x) in every case except
+        # for ord=0, which is x != 0.
+        if ord == 0:
+            out[:] = (x != 0)
+        else:
+            out[:] = torch.abs(x)
+        return out
+    return torch.linalg.vector_norm(x, ord=ord, axis=axis, keepdim=keepdims, **kwargs)
+
+__all__ = linalg_all + ['outer', 'matmul', 'matrix_transpose', 'tensordot',
+                        'cross', 'vecdot', 'solve', 'trace', 'vector_norm']
+
+_all_ignore = ['torch_linalg', 'sum']
+
+del linalg_all
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/LICENSE b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..16c2698cc772f3bfd428c72bd4554e0463d28498
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Consortium for Python Data API Standards
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/README.md b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..b7b2915a04bcf7ce64b72a39cf0d124b1dcbf85d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/README.md
@@ -0,0 +1 @@
+Update this directory using maint_tools/vendor_array_api_extra.sh
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c100c0828cdffbcc50de77039113bd1c52a592b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__init__.py
@@ -0,0 +1,42 @@
+"""Extra array functions built on top of the array API standard."""
+
+from ._delegation import isclose, nan_to_num, one_hot, pad
+from ._lib._at import at
+from ._lib._funcs import (
+    apply_where,
+    atleast_nd,
+    broadcast_shapes,
+    cov,
+    create_diagonal,
+    default_dtype,
+    expand_dims,
+    kron,
+    nunique,
+    setdiff1d,
+    sinc,
+)
+from ._lib._lazy import lazy_apply
+
+__version__ = "0.8.2"
+
+# pylint: disable=duplicate-code
+__all__ = [
+    "__version__",
+    "apply_where",
+    "at",
+    "atleast_nd",
+    "broadcast_shapes",
+    "cov",
+    "create_diagonal",
+    "default_dtype",
+    "expand_dims",
+    "isclose",
+    "kron",
+    "lazy_apply",
+    "nan_to_num",
+    "nunique",
+    "one_hot",
+    "pad",
+    "setdiff1d",
+    "sinc",
+]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cf5fa5a790f4cc8ca6d242f5d14d84328fd1f208
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__pycache__/__init__.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__pycache__/_delegation.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__pycache__/_delegation.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0fe3f9ee8d9da6d20f2ca9181c0f3ccaeb189c0a
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/__pycache__/_delegation.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_delegation.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_delegation.py
new file mode 100644
index 0000000000000000000000000000000000000000..389002a395255ae04271cb44d0596a8a1c150ead
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_delegation.py
@@ -0,0 +1,328 @@
+"""Delegation to existing implementations for Public API Functions."""
+
+from collections.abc import Sequence
+from types import ModuleType
+from typing import Literal
+
+from ._lib import _funcs
+from ._lib._utils._compat import (
+    array_namespace,
+    is_cupy_namespace,
+    is_dask_namespace,
+    is_jax_namespace,
+    is_numpy_namespace,
+    is_pydata_sparse_namespace,
+    is_torch_namespace,
+)
+from ._lib._utils._compat import device as get_device
+from ._lib._utils._helpers import asarrays
+from ._lib._utils._typing import Array, DType
+
+__all__ = ["isclose", "nan_to_num", "one_hot", "pad"]
+
+
+def isclose(
+    a: Array | complex,
+    b: Array | complex,
+    *,
+    rtol: float = 1e-05,
+    atol: float = 1e-08,
+    equal_nan: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Return a boolean array where two arrays are element-wise equal within a tolerance.
+
+    The tolerance values are positive, typically very small numbers. The relative
+    difference ``(rtol * abs(b))`` and the absolute difference `atol` are added together
+    to compare against the absolute difference between `a` and `b`.
+
+    NaNs are treated as equal if they are in the same place and if ``equal_nan=True``.
+    Infs are treated as equal if they are in the same place and of the same sign in both
+    arrays.
+
+    Parameters
+    ----------
+    a, b : Array | int | float | complex | bool
+        Input objects to compare. At least one must be an array.
+    rtol : array_like, optional
+        The relative tolerance parameter (see Notes).
+    atol : array_like, optional
+        The absolute tolerance parameter (see Notes).
+    equal_nan : bool, optional
+        Whether to compare NaN's as equal. If True, NaN's in `a` will be considered
+        equal to NaN's in `b` in the output array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `a` and `b`. Default: infer.
+
+    Returns
+    -------
+    Array
+        A boolean array of shape broadcasted from `a` and `b`, containing ``True`` where
+        `a` is close to `b`, and ``False`` otherwise.
+
+    Warnings
+    --------
+    The default `atol` is not appropriate for comparing numbers with magnitudes much
+    smaller than one (see notes).
+
+    See Also
+    --------
+    math.isclose : Similar function in stdlib for Python scalars.
+
+    Notes
+    -----
+    For finite values, `isclose` uses the following equation to test whether two
+    floating point values are equivalent::
+
+        absolute(a - b) <= (atol + rtol * absolute(b))
+
+    Unlike the built-in `math.isclose`,
+    the above equation is not symmetric in `a` and `b`,
+    so that ``isclose(a, b)`` might be different from ``isclose(b, a)`` in some rare
+    cases.
+
+    The default value of `atol` is not appropriate when the reference value `b` has
+    magnitude smaller than one. For example, it is unlikely that ``a = 1e-9`` and
+    ``b = 2e-9`` should be considered "close", yet ``isclose(1e-9, 2e-9)`` is ``True``
+    with default settings. Be sure to select `atol` for the use case at hand, especially
+    for defining the threshold below which a non-zero value in `a` will be considered
+    "close" to a very small or zero value in `b`.
+
+    The comparison of `a` and `b` uses standard broadcasting, which means that `a` and
+    `b` need not have the same shape in order for ``isclose(a, b)`` to evaluate to
+    ``True``.
+
+    `isclose` is not defined for non-numeric data types.
+    ``bool`` is considered a numeric data-type for this purpose.
+    """
+    xp = array_namespace(a, b) if xp is None else xp
+
+    if (
+        is_numpy_namespace(xp)
+        or is_cupy_namespace(xp)
+        or is_dask_namespace(xp)
+        or is_jax_namespace(xp)
+    ):
+        return xp.isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan)
+
+    if is_torch_namespace(xp):
+        a, b = asarrays(a, b, xp=xp)  # Array API 2024.12 support
+        return xp.isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan)
+
+    return _funcs.isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan, xp=xp)
+
+
+def nan_to_num(
+    x: Array | float | complex,
+    /,
+    *,
+    fill_value: int | float = 0.0,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Replace NaN with zero and infinity with large finite numbers (default behaviour).
+
+    If `x` is inexact, NaN is replaced by zero or by the user defined value in the
+    `fill_value` keyword, infinity is replaced by the largest finite floating
+    point value representable by ``x.dtype``, and -infinity is replaced by the
+    most negative finite floating point value representable by ``x.dtype``.
+
+    For complex dtypes, the above is applied to each of the real and
+    imaginary components of `x` separately.
+
+    Parameters
+    ----------
+    x : array | float | complex
+        Input data.
+    fill_value : int | float, optional
+        Value to be used to fill NaN values. If no value is passed
+        then NaN values will be replaced with 0.0.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        `x`, with the non-finite values replaced.
+
+    See Also
+    --------
+    array_api.isnan : Shows which elements are Not a Number (NaN).
+
+    Examples
+    --------
+    >>> import array_api_extra as xpx
+    >>> import array_api_strict as xp
+    >>> xpx.nan_to_num(xp.inf)
+    1.7976931348623157e+308
+    >>> xpx.nan_to_num(-xp.inf)
+    -1.7976931348623157e+308
+    >>> xpx.nan_to_num(xp.nan)
+    0.0
+    >>> x = xp.asarray([xp.inf, -xp.inf, xp.nan, -128, 128])
+    >>> xpx.nan_to_num(x)
+    array([ 1.79769313e+308, -1.79769313e+308,  0.00000000e+000, # may vary
+           -1.28000000e+002,  1.28000000e+002])
+    >>> y = xp.asarray([complex(xp.inf, xp.nan), xp.nan, complex(xp.nan, xp.inf)])
+    array([  1.79769313e+308,  -1.79769313e+308,   0.00000000e+000, # may vary
+         -1.28000000e+002,   1.28000000e+002])
+    >>> xpx.nan_to_num(y)
+    array([  1.79769313e+308 +0.00000000e+000j, # may vary
+             0.00000000e+000 +0.00000000e+000j,
+             0.00000000e+000 +1.79769313e+308j])
+    """
+    if isinstance(fill_value, complex):
+        msg = "Complex fill values are not supported."
+        raise TypeError(msg)
+
+    xp = array_namespace(x) if xp is None else xp
+
+    # for scalars we want to output an array
+    y = xp.asarray(x)
+
+    if (
+        is_cupy_namespace(xp)
+        or is_jax_namespace(xp)
+        or is_numpy_namespace(xp)
+        or is_torch_namespace(xp)
+    ):
+        return xp.nan_to_num(y, nan=fill_value)
+
+    return _funcs.nan_to_num(y, fill_value=fill_value, xp=xp)
+
+
+def one_hot(
+    x: Array,
+    /,
+    num_classes: int,
+    *,
+    dtype: DType | None = None,
+    axis: int = -1,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    One-hot encode the given indices.
+
+    Each index in the input `x` is encoded as a vector of zeros of length `num_classes`
+    with the element at the given index set to one.
+
+    Parameters
+    ----------
+    x : array
+        An array with integral dtype whose values are between `0` and `num_classes - 1`.
+    num_classes : int
+        Number of classes in the one-hot dimension.
+    dtype : DType, optional
+        The dtype of the return value.  Defaults to the default float dtype (usually
+        float64).
+    axis : int, optional
+        Position in the expanded axes where the new axis is placed. Default: -1.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        An array having the same shape as `x` except for a new axis at the position
+        given by `axis` having size `num_classes`.  If `axis` is unspecified, it
+        defaults to -1, which appends a new axis.
+
+        If ``x < 0`` or ``x >= num_classes``, then the result is undefined, may raise
+        an exception, or may even cause a bad state.  `x` is not checked.
+
+    Examples
+    --------
+    >>> import array_api_extra as xpx
+    >>> import array_api_strict as xp
+    >>> xpx.one_hot(xp.asarray([1, 2, 0]), 3)
+    Array([[0., 1., 0.],
+          [0., 0., 1.],
+          [1., 0., 0.]], dtype=array_api_strict.float64)
+    """
+    # Validate inputs.
+    if xp is None:
+        xp = array_namespace(x)
+    if not xp.isdtype(x.dtype, "integral"):
+        msg = "x must have an integral dtype."
+        raise TypeError(msg)
+    if dtype is None:
+        dtype = _funcs.default_dtype(xp, device=get_device(x))
+    # Delegate where possible.
+    if is_jax_namespace(xp):
+        from jax.nn import one_hot as jax_one_hot
+
+        return jax_one_hot(x, num_classes, dtype=dtype, axis=axis)
+    if is_torch_namespace(xp):
+        from torch.nn.functional import one_hot as torch_one_hot
+
+        x = xp.astype(x, xp.int64)  # PyTorch only supports int64 here.
+        try:
+            out = torch_one_hot(x, num_classes)
+        except RuntimeError as e:
+            raise IndexError from e
+    else:
+        out = _funcs.one_hot(x, num_classes, xp=xp)
+    out = xp.astype(out, dtype, copy=False)
+    if axis != -1:
+        out = xp.moveaxis(out, -1, axis)
+    return out
+
+
+def pad(
+    x: Array,
+    pad_width: int | tuple[int, int] | Sequence[tuple[int, int]],
+    mode: Literal["constant"] = "constant",
+    *,
+    constant_values: complex = 0,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Pad the input array.
+
+    Parameters
+    ----------
+    x : array
+        Input array.
+    pad_width : int or tuple of ints or sequence of pairs of ints
+        Pad the input array with this many elements from each side.
+        If a sequence of tuples, ``[(before_0, after_0), ... (before_N, after_N)]``,
+        each pair applies to the corresponding axis of ``x``.
+        A single tuple, ``(before, after)``, is equivalent to a list of ``x.ndim``
+        copies of this tuple.
+    mode : str, optional
+        Only "constant" mode is currently supported, which pads with
+        the value passed to `constant_values`.
+    constant_values : python scalar, optional
+        Use this value to pad the input. Default is zero.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        The input array,
+        padded with ``pad_width`` elements equal to ``constant_values``.
+    """
+    xp = array_namespace(x) if xp is None else xp
+
+    if mode != "constant":
+        msg = "Only `'constant'` mode is currently supported"
+        raise NotImplementedError(msg)
+
+    if (
+        is_numpy_namespace(xp)
+        or is_cupy_namespace(xp)
+        or is_jax_namespace(xp)
+        or is_pydata_sparse_namespace(xp)
+    ):
+        return xp.pad(x, pad_width, mode, constant_values=constant_values)
+
+    # https://github.com/pytorch/pytorch/blob/cf76c05b4dc629ac989d1fb8e789d4fac04a095a/torch/_numpy/_funcs_impl.py#L2045-L2056
+    if is_torch_namespace(xp):
+        pad_width = xp.asarray(pad_width)
+        pad_width = xp.broadcast_to(pad_width, (x.ndim, 2))
+        pad_width = xp.flip(pad_width, axis=(0,)).flatten()
+        return xp.nn.functional.pad(x, tuple(pad_width), value=constant_values)  # type: ignore[arg-type]  # pyright: ignore[reportArgumentType]
+
+    return _funcs.pad(x, pad_width, constant_values=constant_values, xp=xp)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c517948b8a47b9c2dad04a00211cff52bfdd4ea
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__init__.py
@@ -0,0 +1 @@
+"""Internals of array-api-extra."""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fb26621522c179488ee975a36321a704a46bd5c4
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/__init__.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_at.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_at.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1b93c92a6b20ba6ba76a300916e3d3f8f4e15201
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_at.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_funcs.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_funcs.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3f93cccdb07ede198ca6875f2a4339a7b154483e
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_funcs.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_lazy.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_lazy.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7e0ecff406231411f66b7e74ab86ad56729db63e
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/__pycache__/_lazy.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_at.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_at.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b1d4356982a05f4754e1c677b09909f47536902
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_at.py
@@ -0,0 +1,463 @@
+"""Update operations for read-only arrays."""
+
+from __future__ import annotations
+
+import operator
+from collections.abc import Callable
+from enum import Enum
+from types import ModuleType
+from typing import TYPE_CHECKING, ClassVar, cast
+
+from ._utils import _compat
+from ._utils._compat import (
+    array_namespace,
+    is_dask_array,
+    is_jax_array,
+    is_torch_array,
+    is_writeable_array,
+)
+from ._utils._helpers import meta_namespace
+from ._utils._typing import Array, SetIndex
+
+if TYPE_CHECKING:  # pragma: no cover
+    # TODO import from typing (requires Python >=3.11)
+    from typing_extensions import Self
+
+
+class _AtOp(Enum):
+    """Operations for use in `xpx.at`."""
+
+    SET = "set"
+    ADD = "add"
+    SUBTRACT = "subtract"
+    MULTIPLY = "multiply"
+    DIVIDE = "divide"
+    POWER = "power"
+    MIN = "min"
+    MAX = "max"
+
+    # @override from Python 3.12
+    def __str__(self) -> str:  # pyright: ignore[reportImplicitOverride]
+        """
+        Return string representation (useful for pytest logs).
+
+        Returns
+        -------
+        str
+            The operation's name.
+        """
+        return self.value
+
+
+class Undef(Enum):
+    """Sentinel for undefined values."""
+
+    UNDEF = 0
+
+
+_undef = Undef.UNDEF
+
+
+class at:  # pylint: disable=invalid-name  # numpydoc ignore=PR02
+    """
+    Update operations for read-only arrays.
+
+    This implements ``jax.numpy.ndarray.at`` for all writeable
+    backends (those that support ``__setitem__``) and routes
+    to the ``.at[]`` method for JAX arrays.
+
+    Parameters
+    ----------
+    x : array
+        Input array.
+    idx : index, optional
+        Only `array API standard compliant indices
+        `_
+        are supported.
+
+        You may use two alternate syntaxes::
+
+          >>> import array_api_extra as xpx
+          >>> xpx.at(x, idx).set(value)  # or add(value), etc.
+          >>> xpx.at(x)[idx].set(value)
+
+    copy : bool, optional
+        None (default)
+            The array parameter *may* be modified in place if it is
+            possible and beneficial for performance.
+            You should not reuse it after calling this function.
+        True
+            Ensure that the inputs are not modified.
+        False
+            Ensure that the update operation writes back to the input.
+            Raise ``ValueError`` if a copy cannot be avoided.
+
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    Updated input array.
+
+    Warnings
+    --------
+    (a) When you omit the ``copy`` parameter, you should never reuse the parameter
+    array later on; ideally, you should reassign it immediately::
+
+        >>> import array_api_extra as xpx
+        >>> x = xpx.at(x, 0).set(2)
+
+    The above best practice pattern ensures that the behaviour won't change depending
+    on whether ``x`` is writeable or not, as the original ``x`` object is dereferenced
+    as soon as ``xpx.at`` returns; this way there is no risk to accidentally update it
+    twice.
+
+    On the reverse, the anti-pattern below must be avoided, as it will result in
+    different behaviour on read-only versus writeable arrays::
+
+        >>> x = xp.asarray([0, 0, 0])
+        >>> y = xpx.at(x, 0).set(2)
+        >>> z = xpx.at(x, 1).set(3)
+
+    In the above example, both calls to ``xpx.at`` update ``x`` in place *if possible*.
+    This causes the behaviour to diverge depending on whether ``x`` is writeable or not:
+
+    - If ``x`` is writeable, then after the snippet above you'll have
+      ``x == y == z == [2, 3, 0]``
+    - If ``x`` is read-only, then you'll end up with
+      ``x == [0, 0, 0]``, ``y == [2, 0, 0]`` and ``z == [0, 3, 0]``.
+
+    The correct pattern to use if you want diverging outputs from the same input is
+    to enforce copies::
+
+        >>> x = xp.asarray([0, 0, 0])
+        >>> y = xpx.at(x, 0).set(2, copy=True)  # Never updates x
+        >>> z = xpx.at(x, 1).set(3)  # May or may not update x in place
+        >>> del x  # avoid accidental reuse of x as we don't know its state anymore
+
+    (b) The array API standard does not support integer array indices.
+    The behaviour of update methods when the index is an array of integers is
+    undefined and will vary between backends; this is particularly true when the
+    index contains multiple occurrences of the same index, e.g.::
+
+        >>> import numpy as np
+        >>> import jax.numpy as jnp
+        >>> import array_api_extra as xpx
+        >>> xpx.at(np.asarray([123]), np.asarray([0, 0])).add(1)
+        array([124])
+        >>> xpx.at(jnp.asarray([123]), jnp.asarray([0, 0])).add(1)
+        Array([125], dtype=int32)
+
+    See Also
+    --------
+    jax.numpy.ndarray.at : Equivalent array method in JAX.
+
+    Notes
+    -----
+    `sparse `_, as well as read-only arrays from libraries
+    not explicitly covered by ``array-api-compat``, are not supported by update
+    methods.
+
+    Boolean masks are supported on Dask and jitted JAX arrays exclusively
+    when `idx` has the same shape as `x` and `y` is 0-dimensional.
+    Note that this support is not available in JAX's native
+    ``x.at[mask].set(y)``.
+
+    This pattern::
+
+        >>> mask = m(x)
+        >>> x[mask] = f(x[mask])
+
+    Can't be replaced by `at`, as it won't work on Dask and JAX inside jax.jit::
+
+        >>> mask = m(x)
+        >>> x = xpx.at(x, mask).set(f(x[mask])  # Crash on Dask and jax.jit
+
+    You should instead use::
+
+        >>> x = xp.where(m(x), f(x), x)
+
+    Examples
+    --------
+    Given either of these equivalent expressions::
+
+      >>> import array_api_extra as xpx
+      >>> x = xpx.at(x)[1].add(2)
+      >>> x = xpx.at(x, 1).add(2)
+
+    If x is a JAX array, they are the same as::
+
+      >>> x = x.at[1].add(2)
+
+    If x is a read-only NumPy array, they are the same as::
+
+      >>> x = x.copy()
+      >>> x[1] += 2
+
+    For other known backends, they are the same as::
+
+      >>> x[1] += 2
+    """
+
+    _x: Array
+    _idx: SetIndex | Undef
+    __slots__: ClassVar[tuple[str, ...]] = ("_idx", "_x")
+
+    def __init__(
+        self, x: Array, idx: SetIndex | Undef = _undef, /
+    ) -> None:  # numpydoc ignore=GL08
+        self._x = x
+        self._idx = idx
+
+    def __getitem__(self, idx: SetIndex, /) -> Self:  # numpydoc ignore=PR01,RT01
+        """
+        Allow for the alternate syntax ``at(x)[start:stop:step]``.
+
+        It looks prettier than ``at(x, slice(start, stop, step))``
+        and feels more intuitive coming from the JAX documentation.
+        """
+        if self._idx is not _undef:
+            msg = "Index has already been set"
+            raise ValueError(msg)
+        return type(self)(self._x, idx)
+
+    def _op(
+        self,
+        at_op: _AtOp,
+        in_place_op: Callable[[Array, Array | complex], Array] | None,
+        out_of_place_op: Callable[[Array, Array], Array] | None,
+        y: Array | complex,
+        /,
+        copy: bool | None,
+        xp: ModuleType | None,
+    ) -> Array:
+        """
+        Implement all update operations.
+
+        Parameters
+        ----------
+        at_op : _AtOp
+            Method of JAX's Array.at[].
+        in_place_op : Callable[[Array, Array | complex], Array] | None
+            In-place operation to apply on mutable backends::
+
+                x[idx] = in_place_op(x[idx], y)
+
+            If None::
+
+                x[idx] = y
+
+        out_of_place_op : Callable[[Array, Array], Array] | None
+            Out-of-place operation to apply when idx is a boolean mask and the backend
+            doesn't support in-place updates::
+
+                x = xp.where(idx, out_of_place_op(x, y), x)
+
+            If None::
+
+                x = xp.where(idx, y, x)
+
+        y : array or complex
+            Right-hand side of the operation.
+        copy : bool or None
+            Whether to copy the input array. See the class docstring for details.
+        xp : array_namespace, optional
+            The array namespace for the input array. Default: infer.
+
+        Returns
+        -------
+        Array
+            Updated `x`.
+        """
+        from ._funcs import apply_where  # pylint: disable=cyclic-import
+
+        x, idx = self._x, self._idx
+        xp = array_namespace(x, y) if xp is None else xp
+
+        if isinstance(idx, Undef):
+            msg = (
+                "Index has not been set.\n"
+                "Usage: either\n"
+                "    at(x, idx).set(value)\n"
+                "or\n"
+                "    at(x)[idx].set(value)\n"
+                "(same for all other methods)."
+            )
+            raise ValueError(msg)
+
+        if copy not in (True, False, None):
+            msg = f"copy must be True, False, or None; got {copy!r}"
+            raise ValueError(msg)
+
+        writeable = None if copy else is_writeable_array(x)
+
+        # JAX inside jax.jit doesn't support in-place updates with boolean
+        # masks; Dask exclusively supports __setitem__ but not iops.
+        # We can handle the common special case of 0-dimensional y
+        # with where(idx, y, x) instead.
+        if (
+            (is_dask_array(idx) or is_jax_array(idx))
+            and idx.dtype == xp.bool
+            and idx.shape == x.shape
+        ):
+            y_xp = xp.asarray(y, dtype=x.dtype, device=_compat.device(x))
+            if y_xp.ndim == 0:
+                if out_of_place_op:  # add(), subtract(), ...
+                    # suppress inf warnings on Dask
+                    out = apply_where(
+                        idx, (x, y_xp), out_of_place_op, fill_value=x, xp=xp
+                    )
+                    # Undo int->float promotion on JAX after _AtOp.DIVIDE
+                    out = xp.astype(out, x.dtype, copy=False)
+                else:  # set()
+                    out = xp.where(idx, y_xp, x)
+
+                if copy is False:
+                    x[()] = out
+                    return x
+                return out
+
+            # else: this will work on eager JAX and crash on jax.jit and Dask
+
+        if copy or (copy is None and not writeable):
+            if is_jax_array(x):
+                # Use JAX's at[]
+                func = cast(
+                    Callable[[Array | complex], Array],
+                    getattr(x.at[idx], at_op.value),  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue,reportUnknownArgumentType]
+                )
+                out = func(y)
+                # Undo int->float promotion on JAX after _AtOp.DIVIDE
+                return xp.astype(out, x.dtype, copy=False)
+
+            # Emulate at[] behaviour for non-JAX arrays
+            # with a copy followed by an update
+
+            x = xp.asarray(x, copy=True)
+            # A copy of a read-only numpy array is writeable
+            # Note: this assumes that a copy of a writeable array is writeable
+            assert not writeable
+            writeable = None
+
+        if writeable is None:
+            writeable = is_writeable_array(x)
+        if not writeable:
+            # sparse crashes here
+            msg = f"Can't update read-only array {x}"
+            raise ValueError(msg)
+
+        # Work around bug in PyTorch where __setitem__ doesn't
+        # always support mismatched dtypes
+        # https://github.com/pytorch/pytorch/issues/150017
+        if is_torch_array(y):
+            y = xp.astype(y, x.dtype, copy=False)
+
+        # Backends without boolean indexing (other than JAX) crash here
+        if in_place_op:  # add(), subtract(), ...
+            x[idx] = in_place_op(x[idx], y)
+        else:  # set()
+            x[idx] = y
+        return x
+
+    def set(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] = y`` and return the update array."""
+        return self._op(_AtOp.SET, None, None, y, copy=copy, xp=xp)
+
+    def add(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] += y`` and return the updated array."""
+
+        # Note for this and all other methods based on _iop:
+        # operator.iadd and operator.add subtly differ in behaviour, as
+        # only iadd will trigger exceptions when y has an incompatible dtype.
+        return self._op(_AtOp.ADD, operator.iadd, operator.add, y, copy=copy, xp=xp)
+
+    def subtract(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] -= y`` and return the updated array."""
+        return self._op(
+            _AtOp.SUBTRACT, operator.isub, operator.sub, y, copy=copy, xp=xp
+        )
+
+    def multiply(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] *= y`` and return the updated array."""
+        return self._op(
+            _AtOp.MULTIPLY, operator.imul, operator.mul, y, copy=copy, xp=xp
+        )
+
+    def divide(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] /= y`` and return the updated array."""
+        return self._op(
+            _AtOp.DIVIDE, operator.itruediv, operator.truediv, y, copy=copy, xp=xp
+        )
+
+    def power(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] **= y`` and return the updated array."""
+        return self._op(_AtOp.POWER, operator.ipow, operator.pow, y, copy=copy, xp=xp)
+
+    def min(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] = minimum(x[idx], y)`` and return the updated array."""
+        # On Dask, this function runs on the chunks, so we need to determine the
+        # namespace that Dask is wrapping.
+        # Note that da.minimum _incidentally_ works on NumPy, CuPy, and sparse
+        # thanks to all these meta-namespaces implementing the __array_ufunc__
+        # interface, but there's no guarantee that it will work for other
+        # wrapped libraries in the future.
+        xp = array_namespace(self._x) if xp is None else xp
+        mxp = meta_namespace(self._x, xp=xp)
+        y = xp.asarray(y)
+        return self._op(_AtOp.MIN, mxp.minimum, mxp.minimum, y, copy=copy, xp=xp)
+
+    def max(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] = maximum(x[idx], y)`` and return the updated array."""
+        # See note on min()
+        xp = array_namespace(self._x) if xp is None else xp
+        mxp = meta_namespace(self._x, xp=xp)
+        y = xp.asarray(y)
+        return self._op(_AtOp.MAX, mxp.maximum, mxp.maximum, y, copy=copy, xp=xp)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_backends.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_backends.py
new file mode 100644
index 0000000000000000000000000000000000000000..716bad72e166acbf7f50a4cb25c50c85154d90c0
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_backends.py
@@ -0,0 +1,72 @@
+"""Backends against which array-api-extra runs its tests."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+import numpy as np
+import pytest
+
+__all__ = ["NUMPY_VERSION", "Backend"]
+
+NUMPY_VERSION = tuple(int(v) for v in np.__version__.split(".")[:3])  # pyright: ignore[reportUnknownArgumentType]
+
+
+class Backend(Enum):  # numpydoc ignore=PR02
+    """
+    All array library backends explicitly tested by array-api-extra.
+
+    Parameters
+    ----------
+    value : str
+        Tag of the backend's module, in the format ``[:]``.
+    """
+
+    # Use : to prevent Enum from deduplicating items with the same value
+    ARRAY_API_STRICT = "array_api_strict"
+    ARRAY_API_STRICTEST = "array_api_strict:strictest"
+    NUMPY = "numpy"
+    NUMPY_READONLY = "numpy:readonly"
+    CUPY = "cupy"
+    TORCH = "torch"
+    TORCH_GPU = "torch:gpu"
+    DASK = "dask.array"
+    SPARSE = "sparse"
+    JAX = "jax.numpy"
+    JAX_GPU = "jax.numpy:gpu"
+
+    @property
+    def modname(self) -> str:  # numpydoc ignore=RT01
+        """Module name to be imported."""
+        return self.value.split(":")[0]
+
+    def like(self, *others: Backend) -> bool:  # numpydoc ignore=PR01,RT01
+        """Check if this backend uses the same module as others."""
+        return any(self.modname == other.modname for other in others)
+
+    def pytest_param(self) -> Any:
+        """
+        Backend as a pytest parameter
+
+        Returns
+        -------
+        pytest.mark.ParameterSet
+        """
+        id_ = (
+            self.name.lower().replace("_gpu", ":gpu").replace("_readonly", ":readonly")
+        )
+
+        marks = []
+        if self.like(Backend.ARRAY_API_STRICT):
+            marks.append(
+                pytest.mark.skipif(
+                    NUMPY_VERSION < (1, 26),
+                    reason="array_api_strict is untested on NumPy <1.26",
+                )
+            )
+        if self.like(Backend.DASK, Backend.JAX):
+            # Monkey-patched by lazy_xp_function
+            marks.append(pytest.mark.thread_unsafe)
+
+        return pytest.param(self, id=id_, marks=marks)  # pyright: ignore[reportUnknownArgumentType]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_funcs.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_funcs.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c1bb957251c830476ed84c5321f5864d28d061a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_funcs.py
@@ -0,0 +1,1031 @@
+"""Array-agnostic implementations for the public API."""
+
+import math
+import warnings
+from collections.abc import Callable, Sequence
+from types import ModuleType, NoneType
+from typing import Literal, cast, overload
+
+from ._at import at
+from ._utils import _compat, _helpers
+from ._utils._compat import array_namespace, is_dask_namespace, is_jax_array
+from ._utils._helpers import (
+    asarrays,
+    capabilities,
+    eager_shape,
+    meta_namespace,
+    ndindex,
+)
+from ._utils._typing import Array, Device, DType
+
+__all__ = [
+    "apply_where",
+    "atleast_nd",
+    "broadcast_shapes",
+    "cov",
+    "create_diagonal",
+    "expand_dims",
+    "kron",
+    "nunique",
+    "pad",
+    "setdiff1d",
+    "sinc",
+]
+
+
+@overload
+def apply_where(  # numpydoc ignore=GL08
+    cond: Array,
+    args: Array | tuple[Array, ...],
+    f1: Callable[..., Array],
+    f2: Callable[..., Array],
+    /,
+    *,
+    xp: ModuleType | None = None,
+) -> Array: ...
+
+
+@overload
+def apply_where(  # numpydoc ignore=GL08
+    cond: Array,
+    args: Array | tuple[Array, ...],
+    f1: Callable[..., Array],
+    /,
+    *,
+    fill_value: Array | complex,
+    xp: ModuleType | None = None,
+) -> Array: ...
+
+
+def apply_where(  # numpydoc ignore=PR01,PR02
+    cond: Array,
+    args: Array | tuple[Array, ...],
+    f1: Callable[..., Array],
+    f2: Callable[..., Array] | None = None,
+    /,
+    *,
+    fill_value: Array | complex | None = None,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Run one of two elementwise functions depending on a condition.
+
+    Equivalent to ``f1(*args) if cond else fill_value`` performed elementwise
+    when `fill_value` is defined, otherwise to ``f1(*args) if cond else f2(*args)``.
+
+    Parameters
+    ----------
+    cond : array
+        The condition, expressed as a boolean array.
+    args : Array or tuple of Arrays
+        Argument(s) to `f1` (and `f2`). Must be broadcastable with `cond`.
+    f1 : callable
+        Elementwise function of `args`, returning a single array.
+        Where `cond` is True, output will be ``f1(arg0[cond], arg1[cond], ...)``.
+    f2 : callable, optional
+        Elementwise function of `args`, returning a single array.
+        Where `cond` is False, output will be ``f2(arg0[cond], arg1[cond], ...)``.
+        Mutually exclusive with `fill_value`.
+    fill_value : Array or scalar, optional
+        If provided, value with which to fill output array where `cond` is False.
+        It does not need to be scalar; it needs however to be broadcastable with
+        `cond` and `args`.
+        Mutually exclusive with `f2`. You must provide one or the other.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `cond` and `args`. Default: infer.
+
+    Returns
+    -------
+    Array
+        An array with elements from the output of `f1` where `cond` is True and either
+        the output of `f2` or `fill_value` where `cond` is False. The returned array has
+        data type determined by type promotion rules between the output of `f1` and
+        either `fill_value` or the output of `f2`.
+
+    Notes
+    -----
+    ``xp.where(cond, f1(*args), f2(*args))`` requires explicitly evaluating `f1` even
+    when `cond` is False, and `f2` when cond is True. This function evaluates each
+    function only for their matching condition, if the backend allows for it.
+
+    On Dask, `f1` and `f2` are applied to the individual chunks and should use functions
+    from the namespace of the chunks.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> a = xp.asarray([5, 4, 3])
+    >>> b = xp.asarray([0, 2, 2])
+    >>> def f(a, b):
+    ...     return a // b
+    >>> xpx.apply_where(b != 0, (a, b), f, fill_value=xp.nan)
+    array([ nan,  2., 1.])
+    """
+    # Parse and normalize arguments
+    if (f2 is None) == (fill_value is None):
+        msg = "Exactly one of `fill_value` or `f2` must be given."
+        raise TypeError(msg)
+    args_ = list(args) if isinstance(args, tuple) else [args]
+    del args
+
+    xp = array_namespace(cond, fill_value, *args_) if xp is None else xp
+
+    if isinstance(fill_value, int | float | complex | NoneType):
+        cond, *args_ = xp.broadcast_arrays(cond, *args_)
+    else:
+        cond, fill_value, *args_ = xp.broadcast_arrays(cond, fill_value, *args_)
+
+    if is_dask_namespace(xp):
+        meta_xp = meta_namespace(cond, fill_value, *args_, xp=xp)
+        # map_blocks doesn't descend into tuples of Arrays
+        return xp.map_blocks(_apply_where, cond, f1, f2, fill_value, *args_, xp=meta_xp)
+    return _apply_where(cond, f1, f2, fill_value, *args_, xp=xp)
+
+
+def _apply_where(  # numpydoc ignore=PR01,RT01
+    cond: Array,
+    f1: Callable[..., Array],
+    f2: Callable[..., Array] | None,
+    fill_value: Array | int | float | complex | bool | None,
+    *args: Array,
+    xp: ModuleType,
+) -> Array:
+    """Helper of `apply_where`. On Dask, this runs on a single chunk."""
+
+    if not capabilities(xp, device=_compat.device(cond))["boolean indexing"]:
+        # jax.jit does not support assignment by boolean mask
+        return xp.where(cond, f1(*args), f2(*args) if f2 is not None else fill_value)
+
+    temp1 = f1(*(arr[cond] for arr in args))
+
+    if f2 is None:
+        dtype = xp.result_type(temp1, fill_value)
+        if isinstance(fill_value, int | float | complex):
+            out = xp.full_like(cond, dtype=dtype, fill_value=fill_value)
+        else:
+            out = xp.astype(fill_value, dtype, copy=True)
+    else:
+        ncond = ~cond
+        temp2 = f2(*(arr[ncond] for arr in args))
+        dtype = xp.result_type(temp1, temp2)
+        out = xp.empty_like(cond, dtype=dtype)
+        out = at(out, ncond).set(temp2)
+
+    return at(out, cond).set(temp1)
+
+
+def atleast_nd(x: Array, /, *, ndim: int, xp: ModuleType | None = None) -> Array:
+    """
+    Recursively expand the dimension of an array to at least `ndim`.
+
+    Parameters
+    ----------
+    x : array
+        Input array.
+    ndim : int
+        The minimum number of dimensions for the result.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        An array with ``res.ndim`` >= `ndim`.
+        If ``x.ndim`` >= `ndim`, `x` is returned.
+        If ``x.ndim`` < `ndim`, `x` is expanded by prepending new axes
+        until ``res.ndim`` equals `ndim`.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.asarray([1])
+    >>> xpx.atleast_nd(x, ndim=3, xp=xp)
+    Array([[[1]]], dtype=array_api_strict.int64)
+
+    >>> x = xp.asarray([[[1, 2],
+    ...                  [3, 4]]])
+    >>> xpx.atleast_nd(x, ndim=1, xp=xp) is x
+    True
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if x.ndim < ndim:
+        x = xp.expand_dims(x, axis=0)
+        x = atleast_nd(x, ndim=ndim, xp=xp)
+    return x
+
+
+# `float` in signature to accept `math.nan` for Dask.
+# `int`s are still accepted as `float` is a superclass of `int` in typing
+def broadcast_shapes(*shapes: tuple[float | None, ...]) -> tuple[int | None, ...]:
+    """
+    Compute the shape of the broadcasted arrays.
+
+    Duplicates :func:`numpy.broadcast_shapes`, with additional support for
+    None and NaN sizes.
+
+    This is equivalent to ``xp.broadcast_arrays(arr1, arr2, ...)[0].shape``
+    without needing to worry about the backend potentially deep copying
+    the arrays.
+
+    Parameters
+    ----------
+    *shapes : tuple[int | None, ...]
+        Shapes of the arrays to broadcast.
+
+    Returns
+    -------
+    tuple[int | None, ...]
+        The shape of the broadcasted arrays.
+
+    See Also
+    --------
+    numpy.broadcast_shapes : Equivalent NumPy function.
+    array_api.broadcast_arrays : Function to broadcast actual arrays.
+
+    Notes
+    -----
+    This function accepts the Array API's ``None`` for unknown sizes,
+    as well as Dask's non-standard ``math.nan``.
+    Regardless of input, the output always contains ``None`` for unknown sizes.
+
+    Examples
+    --------
+    >>> import array_api_extra as xpx
+    >>> xpx.broadcast_shapes((2, 3), (2, 1))
+    (2, 3)
+    >>> xpx.broadcast_shapes((4, 2, 3), (2, 1), (1, 3))
+    (4, 2, 3)
+    """
+    if not shapes:
+        return ()  # Match NumPy output
+
+    ndim = max(len(shape) for shape in shapes)
+    out: list[int | None] = []
+    for axis in range(-ndim, 0):
+        sizes = {shape[axis] for shape in shapes if axis >= -len(shape)}
+        # Dask uses NaN for unknown shape, which predates the Array API spec for None
+        none_size = None in sizes or math.nan in sizes  # noqa: PLW0177
+        sizes -= {1, None, math.nan}
+        if len(sizes) > 1:
+            msg = (
+                "shape mismatch: objects cannot be broadcast to a single shape: "
+                f"{shapes}."
+            )
+            raise ValueError(msg)
+        out.append(None if none_size else cast(int, sizes.pop()) if sizes else 1)
+
+    return tuple(out)
+
+
+def cov(m: Array, /, *, xp: ModuleType | None = None) -> Array:
+    """
+    Estimate a covariance matrix.
+
+    Covariance indicates the level to which two variables vary together.
+    If we examine N-dimensional samples, :math:`X = [x_1, x_2, ... x_N]^T`,
+    then the covariance matrix element :math:`C_{ij}` is the covariance of
+    :math:`x_i` and :math:`x_j`. The element :math:`C_{ii}` is the variance
+    of :math:`x_i`.
+
+    This provides a subset of the functionality of ``numpy.cov``.
+
+    Parameters
+    ----------
+    m : array
+        A 1-D or 2-D array containing multiple variables and observations.
+        Each row of `m` represents a variable, and each column a single
+        observation of all those variables.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `m`. Default: infer.
+
+    Returns
+    -------
+    array
+        The covariance matrix of the variables.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+
+    Consider two variables, :math:`x_0` and :math:`x_1`, which
+    correlate perfectly, but in opposite directions:
+
+    >>> x = xp.asarray([[0, 2], [1, 1], [2, 0]]).T
+    >>> x
+    Array([[0, 1, 2],
+           [2, 1, 0]], dtype=array_api_strict.int64)
+
+    Note how :math:`x_0` increases while :math:`x_1` decreases. The covariance
+    matrix shows this clearly:
+
+    >>> xpx.cov(x, xp=xp)
+    Array([[ 1., -1.],
+           [-1.,  1.]], dtype=array_api_strict.float64)
+
+    Note that element :math:`C_{0,1}`, which shows the correlation between
+    :math:`x_0` and :math:`x_1`, is negative.
+
+    Further, note how `x` and `y` are combined:
+
+    >>> x = xp.asarray([-2.1, -1,  4.3])
+    >>> y = xp.asarray([3,  1.1,  0.12])
+    >>> X = xp.stack((x, y), axis=0)
+    >>> xpx.cov(X, xp=xp)
+    Array([[11.71      , -4.286     ],
+           [-4.286     ,  2.14413333]], dtype=array_api_strict.float64)
+
+    >>> xpx.cov(x, xp=xp)
+    Array(11.71, dtype=array_api_strict.float64)
+
+    >>> xpx.cov(y, xp=xp)
+    Array(2.14413333, dtype=array_api_strict.float64)
+    """
+    if xp is None:
+        xp = array_namespace(m)
+
+    m = xp.asarray(m, copy=True)
+    dtype = (
+        xp.float64 if xp.isdtype(m.dtype, "integral") else xp.result_type(m, xp.float64)
+    )
+
+    m = atleast_nd(m, ndim=2, xp=xp)
+    m = xp.astype(m, dtype)
+
+    avg = _helpers.mean(m, axis=1, xp=xp)
+
+    m_shape = eager_shape(m)
+    fact = m_shape[1] - 1
+
+    if fact <= 0:
+        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning, stacklevel=2)
+        fact = 0
+
+    m -= avg[:, None]
+    m_transpose = m.T
+    if xp.isdtype(m_transpose.dtype, "complex floating"):
+        m_transpose = xp.conj(m_transpose)
+    c = m @ m_transpose
+    c /= fact
+    axes = tuple(axis for axis, length in enumerate(c.shape) if length == 1)
+    return xp.squeeze(c, axis=axes)
+
+
+def one_hot(
+    x: Array,
+    /,
+    num_classes: int,
+    *,
+    xp: ModuleType,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """See docstring in `array_api_extra._delegation.py`."""
+    # TODO: Benchmark whether this is faster on the NumPy backend:
+    # if is_numpy_array(x):
+    #     out = xp.zeros((x.size, num_classes), dtype=dtype)
+    #     out[xp.arange(x.size), xp.reshape(x, (-1,))] = 1
+    #     return xp.reshape(out, (*x.shape, num_classes))
+    range_num_classes = xp.arange(num_classes, dtype=x.dtype, device=_compat.device(x))
+    return x[..., xp.newaxis] == range_num_classes
+
+
+def create_diagonal(
+    x: Array, /, *, offset: int = 0, xp: ModuleType | None = None
+) -> Array:
+    """
+    Construct a diagonal array.
+
+    Parameters
+    ----------
+    x : array
+        An array having shape ``(*batch_dims, k)``.
+    offset : int, optional
+        Offset from the leading diagonal (default is ``0``).
+        Use positive ints for diagonals above the leading diagonal,
+        and negative ints for diagonals below the leading diagonal.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        An array having shape ``(*batch_dims, k+abs(offset), k+abs(offset))`` with `x`
+        on the diagonal (offset by `offset`).
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.asarray([2, 4, 8])
+
+    >>> xpx.create_diagonal(x, xp=xp)
+    Array([[2, 0, 0],
+           [0, 4, 0],
+           [0, 0, 8]], dtype=array_api_strict.int64)
+
+    >>> xpx.create_diagonal(x, offset=-2, xp=xp)
+    Array([[0, 0, 0, 0, 0],
+           [0, 0, 0, 0, 0],
+           [2, 0, 0, 0, 0],
+           [0, 4, 0, 0, 0],
+           [0, 0, 8, 0, 0]], dtype=array_api_strict.int64)
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if x.ndim == 0:
+        err_msg = "`x` must be at least 1-dimensional."
+        raise ValueError(err_msg)
+
+    x_shape = eager_shape(x)
+    batch_dims = x_shape[:-1]
+    n = x_shape[-1] + abs(offset)
+    diag = xp.zeros((*batch_dims, n**2), dtype=x.dtype, device=_compat.device(x))
+
+    target_slice = slice(
+        offset if offset >= 0 else abs(offset) * n,
+        min(n * (n - offset), diag.shape[-1]),
+        n + 1,
+    )
+    for index in ndindex(*batch_dims):
+        diag = at(diag)[(*index, target_slice)].set(x[(*index, slice(None))])
+    return xp.reshape(diag, (*batch_dims, n, n))
+
+
+def default_dtype(
+    xp: ModuleType,
+    kind: Literal[
+        "real floating", "complex floating", "integral", "indexing"
+    ] = "real floating",
+    *,
+    device: Device | None = None,
+) -> DType:
+    """
+    Return the default dtype for the given namespace and device.
+
+    This is a convenience shorthand for
+    ``xp.__array_namespace_info__().default_dtypes(device=device)[kind]``.
+
+    Parameters
+    ----------
+    xp : array_namespace
+        The standard-compatible namespace for which to get the default dtype.
+    kind : {'real floating', 'complex floating', 'integral', 'indexing'}, optional
+        The kind of dtype to return. Default is 'real floating'.
+    device : Device, optional
+        The device for which to get the default dtype. Default: current device.
+
+    Returns
+    -------
+    dtype
+        The default dtype for the given namespace, kind, and device.
+    """
+    dtypes = xp.__array_namespace_info__().default_dtypes(device=device)
+    try:
+        return dtypes[kind]
+    except KeyError as e:
+        domain = ("real floating", "complex floating", "integral", "indexing")
+        assert set(dtypes) == set(domain), f"Non-compliant namespace: {dtypes}"
+        msg = f"Unknown kind '{kind}'. Expected one of {domain}."
+        raise ValueError(msg) from e
+
+
+def expand_dims(
+    a: Array, /, *, axis: int | tuple[int, ...] = (0,), xp: ModuleType | None = None
+) -> Array:
+    """
+    Expand the shape of an array.
+
+    Insert (a) new axis/axes that will appear at the position(s) specified by
+    `axis` in the expanded array shape.
+
+    This is ``xp.expand_dims`` for `axis` an int *or a tuple of ints*.
+    Roughly equivalent to ``numpy.expand_dims`` for NumPy arrays.
+
+    Parameters
+    ----------
+    a : array
+        Array to have its shape expanded.
+    axis : int or tuple of ints, optional
+        Position(s) in the expanded axes where the new axis (or axes) is/are placed.
+        If multiple positions are provided, they should be unique (note that a position
+        given by a positive index could also be referred to by a negative index -
+        that will also result in an error).
+        Default: ``(0,)``.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `a`. Default: infer.
+
+    Returns
+    -------
+    array
+        `a` with an expanded shape.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.asarray([1, 2])
+    >>> x.shape
+    (2,)
+
+    The following is equivalent to ``x[xp.newaxis, :]`` or ``x[xp.newaxis]``:
+
+    >>> y = xpx.expand_dims(x, axis=0, xp=xp)
+    >>> y
+    Array([[1, 2]], dtype=array_api_strict.int64)
+    >>> y.shape
+    (1, 2)
+
+    The following is equivalent to ``x[:, xp.newaxis]``:
+
+    >>> y = xpx.expand_dims(x, axis=1, xp=xp)
+    >>> y
+    Array([[1],
+           [2]], dtype=array_api_strict.int64)
+    >>> y.shape
+    (2, 1)
+
+    ``axis`` may also be a tuple:
+
+    >>> y = xpx.expand_dims(x, axis=(0, 1), xp=xp)
+    >>> y
+    Array([[[1, 2]]], dtype=array_api_strict.int64)
+
+    >>> y = xpx.expand_dims(x, axis=(2, 0), xp=xp)
+    >>> y
+    Array([[[1],
+            [2]]], dtype=array_api_strict.int64)
+    """
+    if xp is None:
+        xp = array_namespace(a)
+
+    if not isinstance(axis, tuple):
+        axis = (axis,)
+    ndim = a.ndim + len(axis)
+    if axis != () and (min(axis) < -ndim or max(axis) >= ndim):
+        err_msg = (
+            f"a provided axis position is out of bounds for array of dimension {a.ndim}"
+        )
+        raise IndexError(err_msg)
+    axis = tuple(dim % ndim for dim in axis)
+    if len(set(axis)) != len(axis):
+        err_msg = "Duplicate dimensions specified in `axis`."
+        raise ValueError(err_msg)
+    for i in sorted(axis):
+        a = xp.expand_dims(a, axis=i)
+    return a
+
+
+def isclose(
+    a: Array | complex,
+    b: Array | complex,
+    *,
+    rtol: float = 1e-05,
+    atol: float = 1e-08,
+    equal_nan: bool = False,
+    xp: ModuleType,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """See docstring in array_api_extra._delegation."""
+    a, b = asarrays(a, b, xp=xp)
+
+    a_inexact = xp.isdtype(a.dtype, ("real floating", "complex floating"))
+    b_inexact = xp.isdtype(b.dtype, ("real floating", "complex floating"))
+    if a_inexact or b_inexact:
+        # prevent warnings on NumPy and Dask on inf - inf
+        mxp = meta_namespace(a, b, xp=xp)
+        out = apply_where(
+            xp.isinf(a) | xp.isinf(b),
+            (a, b),
+            lambda a, b: mxp.isinf(a) & mxp.isinf(b) & (mxp.sign(a) == mxp.sign(b)),  # pyright: ignore[reportUnknownArgumentType]
+            # Note: inf <= inf is True!
+            lambda a, b: mxp.abs(a - b) <= (atol + rtol * mxp.abs(b)),  # pyright: ignore[reportUnknownArgumentType]
+            xp=xp,
+        )
+        if equal_nan:
+            out = xp.where(xp.isnan(a) & xp.isnan(b), True, out)
+        return out
+
+    if xp.isdtype(a.dtype, "bool") or xp.isdtype(b.dtype, "bool"):
+        if atol >= 1 or rtol >= 1:
+            return xp.ones_like(a == b)
+        return a == b
+
+    # integer types
+    atol = int(atol)
+    if rtol == 0:
+        return xp.abs(a - b) <= atol
+
+    # Don't rely on OverflowError, as it is not guaranteed by the Array API.
+    nrtol = int(1.0 / rtol)
+    if nrtol > xp.iinfo(b.dtype).max:
+        # rtol * max_int < 1, so it's inconsequential
+        return xp.abs(a - b) <= atol
+    return xp.abs(a - b) <= (atol + xp.abs(b) // nrtol)
+
+
+def kron(
+    a: Array | complex,
+    b: Array | complex,
+    /,
+    *,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Kronecker product of two arrays.
+
+    Computes the Kronecker product, a composite array made of blocks of the
+    second array scaled by the first.
+
+    Equivalent to ``numpy.kron`` for NumPy arrays.
+
+    Parameters
+    ----------
+    a, b : Array | int | float | complex
+        Input arrays or scalars. At least one must be an array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `a` and `b`. Default: infer.
+
+    Returns
+    -------
+    array
+        The Kronecker product of `a` and `b`.
+
+    Notes
+    -----
+    The function assumes that the number of dimensions of `a` and `b`
+    are the same, if necessary prepending the smallest with ones.
+    If ``a.shape = (r0,r1,..,rN)`` and ``b.shape = (s0,s1,...,sN)``,
+    the Kronecker product has shape ``(r0*s0, r1*s1, ..., rN*SN)``.
+    The elements are products of elements from `a` and `b`, organized
+    explicitly by::
+
+        kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]
+
+    where::
+
+        kt = it * st + jt,  t = 0,...,N
+
+    In the common 2-D case (N=1), the block structure can be visualized::
+
+        [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],
+         [  ...                              ...   ],
+         [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> xpx.kron(xp.asarray([1, 10, 100]), xp.asarray([5, 6, 7]), xp=xp)
+    Array([  5,   6,   7,  50,  60,  70, 500,
+           600, 700], dtype=array_api_strict.int64)
+
+    >>> xpx.kron(xp.asarray([5, 6, 7]), xp.asarray([1, 10, 100]), xp=xp)
+    Array([  5,  50, 500,   6,  60, 600,   7,
+            70, 700], dtype=array_api_strict.int64)
+
+    >>> xpx.kron(xp.eye(2), xp.ones((2, 2)), xp=xp)
+    Array([[1., 1., 0., 0.],
+           [1., 1., 0., 0.],
+           [0., 0., 1., 1.],
+           [0., 0., 1., 1.]], dtype=array_api_strict.float64)
+
+    >>> a = xp.reshape(xp.arange(100), (2, 5, 2, 5))
+    >>> b = xp.reshape(xp.arange(24), (2, 3, 4))
+    >>> c = xpx.kron(a, b, xp=xp)
+    >>> c.shape
+    (2, 10, 6, 20)
+    >>> I = (1, 3, 0, 2)
+    >>> J = (0, 2, 1)
+    >>> J1 = (0,) + J             # extend to ndim=4
+    >>> S1 = (1,) + b.shape
+    >>> K = tuple(xp.asarray(I) * xp.asarray(S1) + xp.asarray(J1))
+    >>> c[K] == a[I]*b[J]
+    Array(True, dtype=array_api_strict.bool)
+    """
+    if xp is None:
+        xp = array_namespace(a, b)
+    a, b = asarrays(a, b, xp=xp)
+
+    singletons = (1,) * (b.ndim - a.ndim)
+    a = cast(Array, xp.broadcast_to(a, singletons + a.shape))
+
+    nd_b, nd_a = b.ndim, a.ndim
+    nd_max = max(nd_b, nd_a)
+    if nd_a == 0 or nd_b == 0:
+        return xp.multiply(a, b)
+
+    a_shape = eager_shape(a)
+    b_shape = eager_shape(b)
+
+    # Equalise the shapes by prepending smaller one with 1s
+    a_shape = (1,) * max(0, nd_b - nd_a) + a_shape
+    b_shape = (1,) * max(0, nd_a - nd_b) + b_shape
+
+    # Insert empty dimensions
+    a_arr = expand_dims(a, axis=tuple(range(nd_b - nd_a)), xp=xp)
+    b_arr = expand_dims(b, axis=tuple(range(nd_a - nd_b)), xp=xp)
+
+    # Compute the product
+    a_arr = expand_dims(a_arr, axis=tuple(range(1, nd_max * 2, 2)), xp=xp)
+    b_arr = expand_dims(b_arr, axis=tuple(range(0, nd_max * 2, 2)), xp=xp)
+    result = xp.multiply(a_arr, b_arr)
+
+    # Reshape back and return
+    res_shape = tuple(a_s * b_s for a_s, b_s in zip(a_shape, b_shape, strict=True))
+    return xp.reshape(result, res_shape)
+
+
+def nan_to_num(  # numpydoc ignore=PR01,RT01
+    x: Array,
+    /,
+    fill_value: int | float = 0.0,
+    *,
+    xp: ModuleType,
+) -> Array:
+    """See docstring in `array_api_extra._delegation.py`."""
+
+    def perform_replacements(  # numpydoc ignore=PR01,RT01
+        x: Array,
+        fill_value: int | float,
+        xp: ModuleType,
+    ) -> Array:
+        """Internal function to perform the replacements."""
+        x = xp.where(xp.isnan(x), fill_value, x)
+
+        # convert infinities to finite values
+        finfo = xp.finfo(x.dtype)
+        idx_posinf = xp.isinf(x) & ~xp.signbit(x)
+        idx_neginf = xp.isinf(x) & xp.signbit(x)
+        x = xp.where(idx_posinf, finfo.max, x)
+        return xp.where(idx_neginf, finfo.min, x)
+
+    if xp.isdtype(x.dtype, "complex floating"):
+        return perform_replacements(
+            xp.real(x),
+            fill_value,
+            xp,
+        ) + 1j * perform_replacements(
+            xp.imag(x),
+            fill_value,
+            xp,
+        )
+
+    if xp.isdtype(x.dtype, "numeric"):
+        return perform_replacements(x, fill_value, xp)
+
+    return x
+
+
+def nunique(x: Array, /, *, xp: ModuleType | None = None) -> Array:
+    """
+    Count the number of unique elements in an array.
+
+    Compatible with JAX and Dask, whose laziness would be otherwise
+    problematic.
+
+    Parameters
+    ----------
+    x : Array
+        Input array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array: 0-dimensional integer array
+        The number of unique elements in `x`. It can be lazy.
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if is_jax_array(x):
+        # size= is JAX-specific
+        # https://github.com/data-apis/array-api/issues/883
+        _, counts = xp.unique_counts(x, size=_compat.size(x))
+        return (counts > 0).sum()
+
+    # There are 3 general use cases:
+    # 1. backend has unique_counts and it returns an array with known shape
+    # 2. backend has unique_counts and it returns a None-sized array;
+    #    e.g. Dask, ndonnx
+    # 3. backend does not have unique_counts; e.g. wrapped JAX
+    if capabilities(xp, device=_compat.device(x))["data-dependent shapes"]:
+        # xp has unique_counts; O(n) complexity
+        _, counts = xp.unique_counts(x)
+        n = _compat.size(counts)
+        if n is None:
+            return xp.sum(xp.ones_like(counts))
+        return xp.asarray(n, device=_compat.device(x))
+
+    # xp does not have unique_counts; O(n*logn) complexity
+    x = xp.reshape(x, (-1,))
+    x = xp.sort(x)
+    mask = x != xp.roll(x, -1)
+    default_int = default_dtype(xp, "integral", device=_compat.device(x))
+    return xp.maximum(
+        # Special cases:
+        # - array is size 0
+        # - array has all elements equal to each other
+        xp.astype(xp.any(~mask), default_int),
+        xp.sum(xp.astype(mask, default_int)),
+    )
+
+
+def pad(
+    x: Array,
+    pad_width: int | tuple[int, int] | Sequence[tuple[int, int]],
+    *,
+    constant_values: complex = 0,
+    xp: ModuleType,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """See docstring in `array_api_extra._delegation.py`."""
+    # make pad_width a list of length-2 tuples of ints
+    if isinstance(pad_width, int):
+        pad_width_seq = [(pad_width, pad_width)] * x.ndim
+    elif (
+        isinstance(pad_width, tuple)
+        and len(pad_width) == 2
+        and all(isinstance(i, int) for i in pad_width)
+    ):
+        pad_width_seq = [cast(tuple[int, int], pad_width)] * x.ndim
+    else:
+        pad_width_seq = cast(list[tuple[int, int]], list(pad_width))
+
+    slices: list[slice] = []
+    newshape: list[int] = []
+    for ax, w_tpl in enumerate(pad_width_seq):
+        if len(w_tpl) != 2:
+            msg = f"expect a 2-tuple (before, after), got {w_tpl}."
+            raise ValueError(msg)
+
+        sh = eager_shape(x)[ax]
+
+        if w_tpl[0] == 0 and w_tpl[1] == 0:
+            sl = slice(None, None, None)
+        else:
+            stop: int | None
+            start, stop = w_tpl
+            stop = None if stop == 0 else -stop
+
+            sl = slice(start, stop, None)
+            sh += w_tpl[0] + w_tpl[1]
+
+        newshape.append(sh)
+        slices.append(sl)
+
+    padded = xp.full(
+        tuple(newshape),
+        fill_value=constant_values,
+        dtype=x.dtype,
+        device=_compat.device(x),
+    )
+    return at(padded, tuple(slices)).set(x)
+
+
+def setdiff1d(
+    x1: Array | complex,
+    x2: Array | complex,
+    /,
+    *,
+    assume_unique: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Find the set difference of two arrays.
+
+    Return the unique values in `x1` that are not in `x2`.
+
+    Parameters
+    ----------
+    x1 : array | int | float | complex | bool
+        Input array.
+    x2 : array
+        Input comparison array.
+    assume_unique : bool
+        If ``True``, the input arrays are both assumed to be unique, which
+        can speed up the calculation. Default is ``False``.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x1` and `x2`. Default: infer.
+
+    Returns
+    -------
+    array
+        1D array of values in `x1` that are not in `x2`. The result
+        is sorted when `assume_unique` is ``False``, but otherwise only sorted
+        if the input is sorted.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+
+    >>> x1 = xp.asarray([1, 2, 3, 2, 4, 1])
+    >>> x2 = xp.asarray([3, 4, 5, 6])
+    >>> xpx.setdiff1d(x1, x2, xp=xp)
+    Array([1, 2], dtype=array_api_strict.int64)
+    """
+    if xp is None:
+        xp = array_namespace(x1, x2)
+    # https://github.com/microsoft/pyright/issues/10103
+    x1_, x2_ = asarrays(x1, x2, xp=xp)
+
+    if assume_unique:
+        x1_ = xp.reshape(x1_, (-1,))
+        x2_ = xp.reshape(x2_, (-1,))
+    else:
+        x1_ = xp.unique_values(x1_)
+        x2_ = xp.unique_values(x2_)
+
+    return x1_[_helpers.in1d(x1_, x2_, assume_unique=True, invert=True, xp=xp)]
+
+
+def sinc(x: Array, /, *, xp: ModuleType | None = None) -> Array:
+    r"""
+    Return the normalized sinc function.
+
+    The sinc function is equal to :math:`\sin(\pi x)/(\pi x)` for any argument
+    :math:`x\ne 0`. ``sinc(0)`` takes the limit value 1, making ``sinc`` not
+    only everywhere continuous but also infinitely differentiable.
+
+    .. note::
+
+        Note the normalization factor of ``pi`` used in the definition.
+        This is the most commonly used definition in signal processing.
+        Use ``sinc(x / xp.pi)`` to obtain the unnormalized sinc function
+        :math:`\sin(x)/x` that is more common in mathematics.
+
+    Parameters
+    ----------
+    x : array
+        Array (possibly multi-dimensional) of values for which to calculate
+        ``sinc(x)``. Must have a real floating point dtype.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        ``sinc(x)`` calculated elementwise, which has the same shape as the input.
+
+    Notes
+    -----
+    The name sinc is short for "sine cardinal" or "sinus cardinalis".
+
+    The sinc function is used in various signal processing applications,
+    including in anti-aliasing, in the construction of a Lanczos resampling
+    filter, and in interpolation.
+
+    For bandlimited interpolation of discrete-time signals, the ideal
+    interpolation kernel is proportional to the sinc function.
+
+    References
+    ----------
+    #. Weisstein, Eric W. "Sinc Function." From MathWorld--A Wolfram Web
+       Resource. https://mathworld.wolfram.com/SincFunction.html
+    #. Wikipedia, "Sinc function",
+       https://en.wikipedia.org/wiki/Sinc_function
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.linspace(-4, 4, 41)
+    >>> xpx.sinc(x, xp=xp)
+    Array([-3.89817183e-17, -4.92362781e-02,
+           -8.40918587e-02, -8.90384387e-02,
+           -5.84680802e-02,  3.89817183e-17,
+            6.68206631e-02,  1.16434881e-01,
+            1.26137788e-01,  8.50444803e-02,
+           -3.89817183e-17, -1.03943254e-01,
+           -1.89206682e-01, -2.16236208e-01,
+           -1.55914881e-01,  3.89817183e-17,
+            2.33872321e-01,  5.04551152e-01,
+            7.56826729e-01,  9.35489284e-01,
+            1.00000000e+00,  9.35489284e-01,
+            7.56826729e-01,  5.04551152e-01,
+            2.33872321e-01,  3.89817183e-17,
+           -1.55914881e-01, -2.16236208e-01,
+           -1.89206682e-01, -1.03943254e-01,
+           -3.89817183e-17,  8.50444803e-02,
+            1.26137788e-01,  1.16434881e-01,
+            6.68206631e-02,  3.89817183e-17,
+           -5.84680802e-02, -8.90384387e-02,
+           -8.40918587e-02, -4.92362781e-02,
+           -3.89817183e-17], dtype=array_api_strict.float64)
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if not xp.isdtype(x.dtype, "real floating"):
+        err_msg = "`x` must have a real floating data type."
+        raise ValueError(err_msg)
+    # no scalars in `where` - array-api#807
+    y = xp.pi * xp.where(
+        xp.astype(x, xp.bool),
+        x,
+        xp.asarray(xp.finfo(x.dtype).eps, dtype=x.dtype, device=_compat.device(x)),
+    )
+    return xp.sin(y) / y
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_lazy.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_lazy.py
new file mode 100644
index 0000000000000000000000000000000000000000..5f40d776de64ed336ddd2b6e81b5608c00b005b2
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_lazy.py
@@ -0,0 +1,357 @@
+"""Public API Functions."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Sequence
+from functools import partial, wraps
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeAlias, cast, overload
+
+from ._funcs import broadcast_shapes
+from ._utils import _compat
+from ._utils._compat import (
+    array_namespace,
+    is_dask_namespace,
+    is_jax_namespace,
+)
+from ._utils._helpers import is_python_scalar
+from ._utils._typing import Array, DType
+
+if TYPE_CHECKING:  # pragma: no cover
+    import numpy as np
+    from numpy.typing import ArrayLike
+
+    NumPyObject: TypeAlias = np.ndarray[Any, Any] | np.generic
+else:
+    # Sphinx hack
+    NumPyObject = Any
+
+P = ParamSpec("P")
+
+
+@overload
+def lazy_apply(  # type: ignore[valid-type]
+    func: Callable[P, Array | ArrayLike],
+    *args: Array | complex | None,
+    shape: tuple[int | None, ...] | None = None,
+    dtype: DType | None = None,
+    as_numpy: bool = False,
+    xp: ModuleType | None = None,
+    **kwargs: P.kwargs,  # pyright: ignore[reportGeneralTypeIssues]
+) -> Array: ...  # numpydoc ignore=GL08
+
+
+@overload
+def lazy_apply(  # type: ignore[valid-type]
+    func: Callable[P, Sequence[Array | ArrayLike]],
+    *args: Array | complex | None,
+    shape: Sequence[tuple[int | None, ...]],
+    dtype: Sequence[DType] | None = None,
+    as_numpy: bool = False,
+    xp: ModuleType | None = None,
+    **kwargs: P.kwargs,  # pyright: ignore[reportGeneralTypeIssues]
+) -> tuple[Array, ...]: ...  # numpydoc ignore=GL08
+
+
+def lazy_apply(  # type: ignore[valid-type]  # numpydoc ignore=GL07,SA04
+    func: Callable[P, Array | ArrayLike | Sequence[Array | ArrayLike]],
+    *args: Array | complex | None,
+    shape: tuple[int | None, ...] | Sequence[tuple[int | None, ...]] | None = None,
+    dtype: DType | Sequence[DType] | None = None,
+    as_numpy: bool = False,
+    xp: ModuleType | None = None,
+    **kwargs: P.kwargs,  # pyright: ignore[reportGeneralTypeIssues]
+) -> Array | tuple[Array, ...]:
+    """
+    Lazily apply an eager function.
+
+    If the backend of the input arrays is lazy, e.g. Dask or jitted JAX, the execution
+    of the function is delayed until the graph is materialized; if it's eager, the
+    function is executed immediately.
+
+    Parameters
+    ----------
+    func : callable
+        The function to apply.
+
+        It must accept one or more array API compliant arrays as positional arguments.
+        If `as_numpy=True`, inputs are converted to NumPy before they are passed to
+        `func`.
+        It must return either a single array-like or a sequence of array-likes.
+
+        `func` must be a pure function, i.e. without side effects, as depending on the
+        backend it may be executed more than once or never.
+    *args : Array | int | float | complex | bool | None
+        One or more Array API compliant arrays, Python scalars, or None's.
+
+        If `as_numpy=True`, you need to be able to apply :func:`numpy.asarray` to
+        non-None args to convert them to NumPy; read notes below about specific
+        backends.
+    shape : tuple[int | None, ...] | Sequence[tuple[int | None, ...]], optional
+        Output shape or sequence of output shapes, one for each output of `func`.
+        Default: assume single output and broadcast shapes of the input arrays.
+    dtype : DType | Sequence[DType], optional
+        Output dtype or sequence of output dtypes, one for each output of `func`.
+        dtype(s) must belong to the same array namespace as the input arrays.
+        Default: infer the result type(s) from the input arrays.
+    as_numpy : bool, optional
+        If True, convert the input arrays to NumPy before passing them to `func`.
+        This is particularly useful to make NumPy-only functions, e.g. written in Cython
+       or Numba, work transparently with array API-compliant arrays.
+        Default: False.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `args`. Default: infer.
+    **kwargs : Any, optional
+        Additional keyword arguments to pass verbatim to `func`.
+        They cannot contain Array objects.
+
+    Returns
+    -------
+    Array | tuple[Array, ...]
+        The result(s) of `func` applied to the input arrays, wrapped in the same
+        array namespace as the inputs.
+        If shape is omitted or a single `tuple[int | None, ...]`, return a single array.
+        Otherwise, return a tuple of arrays.
+
+    Notes
+    -----
+    JAX
+        This allows applying eager functions to jitted JAX arrays, which are lazy.
+        The function won't be applied until the JAX array is materialized.
+        When running inside ``jax.jit``, `shape` must be fully known, i.e. it cannot
+        contain any `None` elements.
+
+        .. warning::
+
+            `func` must never raise inside ``jax.jit``, as the resulting behavior is
+            undefined.
+
+        Using this with `as_numpy=False` is particularly useful to apply non-jittable
+        JAX functions to arrays on GPU devices.
+        If ``as_numpy=True``, the :doc:`jax:transfer_guard` may prevent arrays on a GPU
+        device from being transferred back to CPU. This is treated as an implicit
+        transfer.
+
+    PyTorch, CuPy
+        If ``as_numpy=True``, these backends raise by default if you attempt to convert
+        arrays on a GPU device to NumPy.
+
+    Sparse
+        If ``as_numpy=True``, by default sparse prevents implicit densification through
+        :func:`numpy.asarray`. `This safety mechanism can be disabled
+        `_.
+
+    Dask
+        This allows applying eager functions to Dask arrays.
+        The Dask graph won't be computed until the user calls ``compute()`` or
+        ``persist()`` down the line.
+
+        The function name will be prominently visible on the user-facing Dask
+        dashboard and on Prometheus metrics, so it is recommended for it to be
+        meaningful.
+
+        `lazy_apply` doesn't know if `func` reduces along any axes; also, shape
+        changes are non-trivial in chunked Dask arrays. For these reasons, all inputs
+        will be rechunked into a single chunk.
+
+        .. warning::
+
+           The whole operation needs to fit in memory all at once on a single worker.
+
+        The outputs will also be returned as a single chunk and you should consider
+        rechunking them into smaller chunks afterwards.
+
+        If you want to distribute the calculation across multiple workers, you
+        should use :func:`dask.array.map_blocks`, :func:`dask.array.map_overlap`,
+        :func:`dask.array.blockwise`, or a native Dask wrapper instead of
+        `lazy_apply`.
+
+    Dask wrapping around other backends
+        If ``as_numpy=False``, `func` will receive in input eager arrays of the meta
+        namespace, as defined by the ``._meta`` attribute of the input Dask arrays.
+        The outputs of `func` will be wrapped by the meta namespace, and then wrapped
+        again by Dask.
+
+    Raises
+    ------
+    ValueError
+        When ``xp=jax.numpy``, the output `shape` is unknown (it contains ``None`` on
+        one or more axes) and this function was called inside ``jax.jit``.
+    RuntimeError
+        When ``xp=sparse`` and auto-densification is disabled.
+    Exception (backend-specific)
+        When the backend disallows implicit device to host transfers and the input
+        arrays are on a non-CPU device, e.g. on GPU.
+
+    See Also
+    --------
+    jax.transfer_guard
+    jax.pure_callback
+    dask.array.map_blocks
+    dask.array.map_overlap
+    dask.array.blockwise
+    """
+    args_not_none = [arg for arg in args if arg is not None]
+    array_args = [arg for arg in args_not_none if not is_python_scalar(arg)]
+    if not array_args:
+        msg = "Must have at least one argument array"
+        raise ValueError(msg)
+    if xp is None:
+        xp = array_namespace(*args)
+
+    # Normalize and validate shape and dtype
+    shapes: list[tuple[int | None, ...]]
+    dtypes: list[DType]
+    multi_output = False
+
+    if shape is None:
+        shapes = [broadcast_shapes(*(arg.shape for arg in array_args))]
+    elif all(isinstance(s, int | None) for s in shape):
+        # Do not test for shape to be a tuple
+        # https://github.com/data-apis/array-api/issues/891#issuecomment-2637430522
+        shapes = [cast(tuple[int | None, ...], shape)]
+    else:
+        shapes = list(shape)  # type: ignore[arg-type]  # pyright: ignore[reportAssignmentType]
+        multi_output = True
+
+    if dtype is None:
+        dtypes = [xp.result_type(*args_not_none)] * len(shapes)
+    elif multi_output:
+        if not isinstance(dtype, Sequence):
+            msg = "Got multiple shapes but only one dtype"
+            raise ValueError(msg)
+        dtypes = list(dtype)  # pyright: ignore[reportUnknownArgumentType]
+    else:
+        if isinstance(dtype, Sequence):
+            msg = "Got single shape but multiple dtypes"
+            raise ValueError(msg)
+
+        dtypes = [dtype]
+
+    if len(shapes) != len(dtypes):
+        msg = f"Got {len(shapes)} shapes and {len(dtypes)} dtypes"
+        raise ValueError(msg)
+    del shape
+    del dtype
+    # End of shape and dtype parsing
+
+    # Backend-specific branches
+    if is_dask_namespace(xp):
+        import dask
+
+        metas: list[Array] = [arg._meta for arg in array_args]  # pylint: disable=protected-access    # pyright: ignore[reportAttributeAccessIssue]
+        meta_xp = array_namespace(*metas)
+
+        wrapped = dask.delayed(  # type: ignore[attr-defined]  # pyright: ignore[reportPrivateImportUsage]
+            _lazy_apply_wrapper(func, as_numpy, multi_output, meta_xp),
+            pure=True,
+        )
+        # This finalizes each arg, which is the same as arg.rechunk(-1).
+        # Please read docstring above for why we're not using
+        # dask.array.map_blocks or dask.array.blockwise!
+        delayed_out = wrapped(*args, **kwargs)
+
+        out = tuple(
+            xp.from_delayed(
+                delayed_out[i],  # pyright: ignore[reportIndexIssue]
+                # Dask's unknown shapes diverge from the Array API specification
+                shape=tuple(math.nan if s is None else s for s in shape),
+                dtype=dtype,
+                meta=metas[0],
+            )
+            for i, (shape, dtype) in enumerate(zip(shapes, dtypes, strict=True))
+        )
+
+    elif is_jax_namespace(xp) and _is_jax_jit_enabled(xp):
+        # Delay calling func with jax.pure_callback, which will forward to func eager
+        # JAX arrays. Do not use jax.pure_callback when running outside of the JIT,
+        # as it does not support raising exceptions:
+        # https://github.com/jax-ml/jax/issues/26102
+        import jax
+
+        if any(None in shape for shape in shapes):
+            msg = "Output shape must be fully known when running inside jax.jit"
+            raise ValueError(msg)
+
+        # Shield kwargs from being coerced into JAX arrays.
+        # jax.pure_callback calls jax.jit under the hood, but without the chance of
+        # passing static_argnames / static_argnums.
+        wrapped = _lazy_apply_wrapper(
+            partial(func, **kwargs), as_numpy, multi_output, xp
+        )
+
+        # suppress unused-ignore to run mypy in -e lint as well as -e dev
+        out = cast(  # type: ignore[bad-cast,unused-ignore]
+            tuple[Array, ...],
+            jax.pure_callback(
+                wrapped,
+                tuple(
+                    jax.ShapeDtypeStruct(shape, dtype)  # pyright: ignore[reportUnknownArgumentType]
+                    for shape, dtype in zip(shapes, dtypes, strict=True)
+                ),
+                *args,
+            ),
+        )
+
+    else:
+        # Eager backends, including non-jitted JAX
+        wrapped = _lazy_apply_wrapper(func, as_numpy, multi_output, xp)
+        out = wrapped(*args, **kwargs)
+
+    return out if multi_output else out[0]
+
+
+def _is_jax_jit_enabled(xp: ModuleType) -> bool:  # numpydoc ignore=PR01,RT01
+    """Return True if this function is being called inside ``jax.jit``."""
+    import jax  # pylint: disable=import-outside-toplevel
+
+    x = xp.asarray(False)
+    try:
+        return bool(x)
+    except jax.errors.TracerBoolConversionError:
+        return True
+
+
+def _lazy_apply_wrapper(  # numpydoc ignore=PR01,RT01
+    func: Callable[..., Array | ArrayLike | Sequence[Array | ArrayLike]],
+    as_numpy: bool,
+    multi_output: bool,
+    xp: ModuleType,
+) -> Callable[..., tuple[Array, ...]]:
+    """
+    Helper of `lazy_apply`.
+
+    Given a function that accepts one or more arrays as positional arguments and returns
+    a single array-like or a sequence of array-likes, return a function that accepts the
+    same number of Array API arrays and always returns a tuple of Array API array.
+
+    Any keyword arguments are passed through verbatim to the wrapped function.
+    """
+
+    # On Dask, @wraps causes the graph key to contain the wrapped function's name
+    @wraps(func)
+    def wrapper(
+        *args: Array | complex | None, **kwargs: Any
+    ) -> tuple[Array, ...]:  # numpydoc ignore=GL08
+        args_list = []
+        device = None
+        for arg in args:
+            if arg is not None and not is_python_scalar(arg):
+                if device is None:
+                    device = _compat.device(arg)
+                if as_numpy:
+                    import numpy as np
+
+                    arg = cast(Array, np.asarray(arg))  # pyright: ignore[reportInvalidCast] # noqa: PLW2901
+            args_list.append(arg)
+        assert device is not None
+
+        out = func(*args_list, **kwargs)
+
+        if multi_output:
+            assert isinstance(out, Sequence)
+            return tuple(xp.asarray(o, device=device) for o in out)
+        return (xp.asarray(out, device=device),)
+
+    return wrapper
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_testing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_testing.py
new file mode 100644
index 0000000000000000000000000000000000000000..34780bd604029386fe9ced649b229895f699780b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_testing.py
@@ -0,0 +1,304 @@
+"""
+Testing utilities.
+
+Note that this is private API; don't expect it to be stable.
+See also ..testing for public testing utilities.
+"""
+
+from __future__ import annotations
+
+import math
+from types import ModuleType
+from typing import Any, cast
+
+import numpy as np
+import pytest
+
+from ._utils._compat import (
+    array_namespace,
+    is_array_api_strict_namespace,
+    is_cupy_namespace,
+    is_dask_namespace,
+    is_jax_namespace,
+    is_numpy_namespace,
+    is_pydata_sparse_namespace,
+    is_torch_array,
+    is_torch_namespace,
+    to_device,
+)
+from ._utils._typing import Array, Device
+
+__all__ = ["as_numpy_array", "xp_assert_close", "xp_assert_equal", "xp_assert_less"]
+
+
+def _check_ns_shape_dtype(
+    actual: Array,
+    desired: Array,
+    check_dtype: bool,
+    check_shape: bool,
+    check_scalar: bool,
+) -> ModuleType:  # numpydoc ignore=RT03
+    """
+    Assert that namespace, shape and dtype of the two arrays match.
+
+    Parameters
+    ----------
+    actual : Array
+        The array produced by the tested function.
+    desired : Array
+        The expected array (typically hardcoded).
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
+
+    Returns
+    -------
+    Arrays namespace.
+    """
+    actual_xp = array_namespace(actual)  # Raises on scalars and lists
+    desired_xp = array_namespace(desired)
+
+    msg = f"namespaces do not match: {actual_xp} != f{desired_xp}"
+    assert actual_xp == desired_xp, msg
+
+    # Dask uses nan instead of None for unknown shapes
+    actual_shape = cast(tuple[float, ...], actual.shape)
+    desired_shape = cast(tuple[float, ...], desired.shape)
+    assert None not in actual_shape  # Requires explicit support
+    assert None not in desired_shape
+    if is_dask_namespace(desired_xp):
+        if any(math.isnan(i) for i in actual_shape):
+            actual_shape = actual.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+        if any(math.isnan(i) for i in desired_shape):
+            desired_shape = desired.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+    if check_shape:
+        msg = f"shapes do not match: {actual_shape} != f{desired_shape}"
+        assert actual_shape == desired_shape, msg
+    else:
+        # Ignore shape, but check flattened size. This is normally done by
+        # np.testing.assert_array_equal etc even when strict=False, but not for
+        # non-materializable arrays.
+        actual_size = math.prod(actual_shape)  # pyright: ignore[reportUnknownArgumentType]
+        desired_size = math.prod(desired_shape)  # pyright: ignore[reportUnknownArgumentType]
+        msg = f"sizes do not match: {actual_size} != f{desired_size}"
+        assert actual_size == desired_size, msg
+
+    if check_dtype:
+        msg = f"dtypes do not match: {actual.dtype} != {desired.dtype}"
+        assert actual.dtype == desired.dtype, msg
+
+    if is_numpy_namespace(actual_xp) and check_scalar:
+        # only NumPy distinguishes between scalars and arrays; we do if check_scalar.
+        _msg = (
+            "array-ness does not match:\n Actual: "
+            f"{type(actual)}\n Desired: {type(desired)}"
+        )
+        assert np.isscalar(actual) == np.isscalar(desired), _msg
+
+    return desired_xp
+
+
+def _is_materializable(x: Array) -> bool:
+    """
+    Return True if you can call `as_numpy_array(x)`; False otherwise.
+    """
+    # Important: here we assume that we're not tracing -
+    # e.g. we're not inside `jax.jit`` nor `cupy.cuda.Stream.begin_capture`.
+    return not is_torch_array(x) or x.device.type != "meta"  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+
+def as_numpy_array(array: Array, *, xp: ModuleType) -> np.typing.NDArray[Any]:
+    """
+    Convert array to NumPy, bypassing GPU-CPU transfer guards and densification guards.
+    """
+    if is_cupy_namespace(xp):
+        return xp.asnumpy(array)
+    if is_pydata_sparse_namespace(xp):
+        return array.todense()  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+    if is_torch_namespace(xp):
+        array = to_device(array, "cpu")
+    if is_array_api_strict_namespace(xp):
+        cpu: Device = xp.Device("CPU_DEVICE")
+        array = to_device(array, cpu)
+    if is_jax_namespace(xp):
+        import jax
+
+        # Note: only needed if the transfer guard is enabled
+        cpu = cast(Device, jax.devices("cpu")[0])
+        array = to_device(array, cpu)
+
+    return np.asarray(array)
+
+
+def xp_assert_equal(
+    actual: Array,
+    desired: Array,
+    *,
+    err_msg: str = "",
+    check_dtype: bool = True,
+    check_shape: bool = True,
+    check_scalar: bool = False,
+) -> None:
+    """
+    Array-API compatible version of `np.testing.assert_array_equal`.
+
+    Parameters
+    ----------
+    actual : Array
+        The array produced by the tested function.
+    desired : Array
+        The expected array (typically hardcoded).
+    err_msg : str, optional
+        Error message to display on failure.
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
+
+    See Also
+    --------
+    xp_assert_close : Similar function for inexact equality checks.
+    numpy.testing.assert_array_equal : Similar function for NumPy arrays.
+    """
+    xp = _check_ns_shape_dtype(actual, desired, check_dtype, check_shape, check_scalar)
+    if not _is_materializable(actual):
+        return
+    actual_np = as_numpy_array(actual, xp=xp)
+    desired_np = as_numpy_array(desired, xp=xp)
+    np.testing.assert_array_equal(actual_np, desired_np, err_msg=err_msg)
+
+
+def xp_assert_less(
+    x: Array,
+    y: Array,
+    *,
+    err_msg: str = "",
+    check_dtype: bool = True,
+    check_shape: bool = True,
+    check_scalar: bool = False,
+) -> None:
+    """
+    Array-API compatible version of `np.testing.assert_array_less`.
+
+    Parameters
+    ----------
+    x, y : Array
+        The arrays to compare according to ``x < y`` (elementwise).
+    err_msg : str, optional
+        Error message to display on failure.
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
+
+    See Also
+    --------
+    xp_assert_close : Similar function for inexact equality checks.
+    numpy.testing.assert_array_equal : Similar function for NumPy arrays.
+    """
+    xp = _check_ns_shape_dtype(x, y, check_dtype, check_shape, check_scalar)
+    if not _is_materializable(x):
+        return
+    x_np = as_numpy_array(x, xp=xp)
+    y_np = as_numpy_array(y, xp=xp)
+    np.testing.assert_array_less(x_np, y_np, err_msg=err_msg)
+
+
+def xp_assert_close(
+    actual: Array,
+    desired: Array,
+    *,
+    rtol: float | None = None,
+    atol: float = 0,
+    err_msg: str = "",
+    check_dtype: bool = True,
+    check_shape: bool = True,
+    check_scalar: bool = False,
+) -> None:
+    """
+    Array-API compatible version of `np.testing.assert_allclose`.
+
+    Parameters
+    ----------
+    actual : Array
+        The array produced by the tested function.
+    desired : Array
+        The expected array (typically hardcoded).
+    rtol : float, optional
+        Relative tolerance. Default: dtype-dependent.
+    atol : float, optional
+        Absolute tolerance. Default: 0.
+    err_msg : str, optional
+        Error message to display on failure.
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
+
+    See Also
+    --------
+    xp_assert_equal : Similar function for exact equality checks.
+    isclose : Public function for checking closeness.
+    numpy.testing.assert_allclose : Similar function for NumPy arrays.
+
+    Notes
+    -----
+    The default `atol` and `rtol` differ from `xp.all(xpx.isclose(a, b))`.
+    """
+    xp = _check_ns_shape_dtype(actual, desired, check_dtype, check_shape, check_scalar)
+    if not _is_materializable(actual):
+        return
+
+    if rtol is None:
+        if xp.isdtype(actual.dtype, ("real floating", "complex floating")):
+            # multiplier of 4 is used as for `np.float64` this puts the default `rtol`
+            # roughly half way between sqrt(eps) and the default for
+            # `numpy.testing.assert_allclose`, 1e-7
+            rtol = xp.finfo(actual.dtype).eps ** 0.5 * 4
+        else:
+            rtol = 1e-7
+
+    actual_np = as_numpy_array(actual, xp=xp)
+    desired_np = as_numpy_array(desired, xp=xp)
+    np.testing.assert_allclose(  # pyright: ignore[reportCallIssue]
+        actual_np,
+        desired_np,
+        rtol=rtol,  # pyright: ignore[reportArgumentType]
+        atol=atol,
+        err_msg=err_msg,
+    )
+
+
+def xfail(
+    request: pytest.FixtureRequest, *, reason: str, strict: bool | None = None
+) -> None:
+    """
+    XFAIL the currently running test.
+
+    Unlike ``pytest.xfail``, allow rest of test to execute instead of immediately
+    halting it, so that it may result in a XPASS.
+    xref https://github.com/pandas-dev/pandas/issues/38902
+
+    Parameters
+    ----------
+    request : pytest.FixtureRequest
+        ``request`` argument of the test function.
+    reason : str
+        Reason for the expected failure.
+    strict: bool, optional
+        If True, the test will be marked as failed if it passes.
+        If False, the test will be marked as passed if it fails.
+        Default: ``xfail_strict`` value in ``pyproject.toml``, or False if absent.
+    """
+    if strict is not None:
+        marker = pytest.mark.xfail(reason=reason, strict=strict)
+    else:
+        marker = pytest.mark.xfail(reason=reason)
+    request.node.add_marker(marker)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1b1f7b6de9c8e305d92a25270f7b965b2411713
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__init__.py
@@ -0,0 +1 @@
+"""Modules housing private utility functions."""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/__init__.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ca4996cea4c5864a2e02181b702cb1c9f2454874
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/__init__.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_compat.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_compat.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..75140303f1ed50b0ced5ab72bce914a651988a7d
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_compat.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_helpers.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_helpers.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..150f0947e99f15563c4a086ceb8a36266847319c
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_helpers.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_typing.cpython-312.pyc b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_typing.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b8b77186cbf0874571fdf10a989d7366db4ccf1f
Binary files /dev/null and b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/__pycache__/_typing.cpython-312.pyc differ
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_compat.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f8af00debd92d1f32b64f11e0807445dd13f2a6
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_compat.py
@@ -0,0 +1,74 @@
+"""Acquire helpers from array-api-compat."""
+# Allow packages that vendor both `array-api-extra` and
+# `array-api-compat` to override the import location
+
+# pylint: disable=duplicate-code
+try:
+    from ...._array_api_compat_vendor import (
+        array_namespace,
+        device,
+        is_array_api_obj,
+        is_array_api_strict_namespace,
+        is_cupy_array,
+        is_cupy_namespace,
+        is_dask_array,
+        is_dask_namespace,
+        is_jax_array,
+        is_jax_namespace,
+        is_lazy_array,
+        is_numpy_array,
+        is_numpy_namespace,
+        is_pydata_sparse_array,
+        is_pydata_sparse_namespace,
+        is_torch_array,
+        is_torch_namespace,
+        is_writeable_array,
+        size,
+        to_device,
+    )
+except ImportError:
+    from array_api_compat import (
+        array_namespace,
+        device,
+        is_array_api_obj,
+        is_array_api_strict_namespace,
+        is_cupy_array,
+        is_cupy_namespace,
+        is_dask_array,
+        is_dask_namespace,
+        is_jax_array,
+        is_jax_namespace,
+        is_lazy_array,
+        is_numpy_array,
+        is_numpy_namespace,
+        is_pydata_sparse_array,
+        is_pydata_sparse_namespace,
+        is_torch_array,
+        is_torch_namespace,
+        is_writeable_array,
+        size,
+        to_device,
+    )
+
+__all__ = [
+    "array_namespace",
+    "device",
+    "is_array_api_obj",
+    "is_array_api_strict_namespace",
+    "is_cupy_array",
+    "is_cupy_namespace",
+    "is_dask_array",
+    "is_dask_namespace",
+    "is_jax_array",
+    "is_jax_namespace",
+    "is_lazy_array",
+    "is_numpy_array",
+    "is_numpy_namespace",
+    "is_pydata_sparse_array",
+    "is_pydata_sparse_namespace",
+    "is_torch_array",
+    "is_torch_namespace",
+    "is_writeable_array",
+    "size",
+    "to_device",
+]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_compat.pyi b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_compat.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..3cd63e016056e399601cdb38158c44ec0c5fb86b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_compat.pyi
@@ -0,0 +1,45 @@
+"""Static type stubs for `_compat.py`."""
+
+# https://github.com/scikit-learn/scikit-learn/pull/27910#issuecomment-2568023972
+from __future__ import annotations
+
+from types import ModuleType
+from typing import Any, TypeGuard
+
+# TODO import from typing (requires Python >=3.13)
+from typing_extensions import TypeIs
+
+from ._typing import Array, Device
+
+# pylint: disable=missing-class-docstring,unused-argument
+
+def array_namespace(
+    *xs: Array | complex | None,
+    api_version: str | None = None,
+    use_compat: bool | None = None,
+) -> ModuleType: ...
+def device(x: Array, /) -> Device: ...
+def is_array_api_obj(x: object, /) -> TypeIs[Array]: ...
+def is_array_api_strict_namespace(xp: ModuleType, /) -> bool: ...
+def is_cupy_namespace(xp: ModuleType, /) -> bool: ...
+def is_dask_namespace(xp: ModuleType, /) -> bool: ...
+def is_jax_namespace(xp: ModuleType, /) -> bool: ...
+def is_numpy_namespace(xp: ModuleType, /) -> bool: ...
+def is_pydata_sparse_namespace(xp: ModuleType, /) -> bool: ...
+def is_torch_namespace(xp: ModuleType, /) -> bool: ...
+def is_cupy_array(x: object, /) -> TypeGuard[Array]: ...
+def is_dask_array(x: object, /) -> TypeGuard[Array]: ...
+def is_jax_array(x: object, /) -> TypeGuard[Array]: ...
+def is_numpy_array(x: object, /) -> TypeGuard[Array]: ...
+def is_pydata_sparse_array(x: object, /) -> TypeGuard[Array]: ...
+def is_torch_array(x: object, /) -> TypeGuard[Array]: ...
+def is_lazy_array(x: object, /) -> TypeGuard[Array]: ...
+def is_writeable_array(x: object, /) -> TypeGuard[Array]: ...
+def size(x: Array, /) -> int | None: ...
+def to_device(
+    x: Array,
+    device: Device,  # pylint: disable=redefined-outer-name
+    /,
+    *,
+    stream: int | Any | None = None,
+) -> Array: ...
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_helpers.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2f56102598a92c7a0a8b45e6b13d25134ad2065
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_helpers.py
@@ -0,0 +1,598 @@
+"""Helper functions used by `array_api_extra/_funcs.py`."""
+
+from __future__ import annotations
+
+import io
+import math
+import pickle
+import types
+from collections.abc import Callable, Generator, Iterable
+from functools import wraps
+from types import ModuleType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    ClassVar,
+    Generic,
+    Literal,
+    ParamSpec,
+    TypeAlias,
+    TypeVar,
+    cast,
+)
+
+from . import _compat
+from ._compat import (
+    array_namespace,
+    is_array_api_obj,
+    is_dask_namespace,
+    is_jax_namespace,
+    is_numpy_array,
+    is_pydata_sparse_namespace,
+    is_torch_namespace,
+)
+from ._typing import Array, Device
+
+if TYPE_CHECKING:  # pragma: no cover
+    # TODO import from typing (requires Python >=3.12 and >=3.13)
+    from typing_extensions import TypeIs, override
+else:
+
+    def override(func):
+        return func
+
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+__all__ = [
+    "asarrays",
+    "capabilities",
+    "eager_shape",
+    "in1d",
+    "is_python_scalar",
+    "jax_autojit",
+    "mean",
+    "meta_namespace",
+    "pickle_flatten",
+    "pickle_unflatten",
+]
+
+
+def in1d(
+    x1: Array,
+    x2: Array,
+    /,
+    *,
+    assume_unique: bool = False,
+    invert: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """
+    Check whether each element of an array is also present in a second array.
+
+    Returns a boolean array the same length as `x1` that is True
+    where an element of `x1` is in `x2` and False otherwise.
+
+    This function has been adapted using the original implementation
+    present in numpy:
+    https://github.com/numpy/numpy/blob/v1.26.0/numpy/lib/arraysetops.py#L524-L758
+    """
+    if xp is None:
+        xp = array_namespace(x1, x2)
+
+    x1_shape = eager_shape(x1)
+    x2_shape = eager_shape(x2)
+
+    # This code is run to make the code significantly faster
+    if x2_shape[0] < 10 * x1_shape[0] ** 0.145 and isinstance(x2, Iterable):
+        if invert:
+            mask = xp.ones(x1_shape[0], dtype=xp.bool, device=_compat.device(x1))
+            for a in x2:
+                mask &= x1 != a
+        else:
+            mask = xp.zeros(x1_shape[0], dtype=xp.bool, device=_compat.device(x1))
+            for a in x2:
+                mask |= x1 == a
+        return mask
+
+    rev_idx = xp.empty(0)  # placeholder
+    if not assume_unique:
+        x1, rev_idx = xp.unique_inverse(x1)
+        x2 = xp.unique_values(x2)
+
+    ar = xp.concat((x1, x2))
+    device_ = _compat.device(ar)
+    # We need this to be a stable sort.
+    order = xp.argsort(ar, stable=True)
+    reverse_order = xp.argsort(order, stable=True)
+    sar = xp.take(ar, order, axis=0)
+    ar_size = _compat.size(sar)
+    assert ar_size is not None, "xp.unique*() on lazy backends raises"
+    if ar_size >= 1:
+        bool_ar = sar[1:] != sar[:-1] if invert else sar[1:] == sar[:-1]
+    else:
+        bool_ar = xp.asarray([False]) if invert else xp.asarray([True])
+    flag = xp.concat((bool_ar, xp.asarray([invert], device=device_)))
+    ret = xp.take(flag, reverse_order, axis=0)
+
+    if assume_unique:
+        return ret[: x1.shape[0]]
+    return xp.take(ret, rev_idx, axis=0)
+
+
+def mean(
+    x: Array,
+    /,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    keepdims: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """
+    Complex mean, https://github.com/data-apis/array-api/issues/846.
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if xp.isdtype(x.dtype, "complex floating"):
+        x_real = xp.real(x)
+        x_imag = xp.imag(x)
+        mean_real = xp.mean(x_real, axis=axis, keepdims=keepdims)
+        mean_imag = xp.mean(x_imag, axis=axis, keepdims=keepdims)
+        return mean_real + (mean_imag * xp.asarray(1j))
+    return xp.mean(x, axis=axis, keepdims=keepdims)
+
+
+def is_python_scalar(x: object) -> TypeIs[complex]:  # numpydoc ignore=PR01,RT01
+    """Return True if `x` is a Python scalar, False otherwise."""
+    # isinstance(x, float) returns True for np.float64
+    # isinstance(x, complex) returns True for np.complex128
+    # bool is a subclass of int
+    return isinstance(x, int | float | complex) and not is_numpy_array(x)
+
+
+def asarrays(
+    a: Array | complex,
+    b: Array | complex,
+    xp: ModuleType,
+) -> tuple[Array, Array]:
+    """
+    Ensure both `a` and `b` are arrays.
+
+    If `b` is a python scalar, it is converted to the same dtype as `a`, and vice versa.
+
+    Behavior is not specified when mixing a Python ``float`` and an array with an
+    integer data type; this may give ``float32``, ``float64``, or raise an exception.
+    Behavior is implementation-specific.
+
+    Similarly, behavior is not specified when mixing a Python ``complex`` and an array
+    with a real-valued data type; this may give ``complex64``, ``complex128``, or raise
+    an exception. Behavior is implementation-specific.
+
+    Parameters
+    ----------
+    a, b : Array | int | float | complex | bool
+        Input arrays or scalars. At least one must be an array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    Array, Array
+        The input arrays, possibly converted to arrays if they were scalars.
+
+    See Also
+    --------
+    mixing-arrays-with-python-scalars : Array API specification for the behavior.
+    """
+    a_scalar = is_python_scalar(a)
+    b_scalar = is_python_scalar(b)
+    if not a_scalar and not b_scalar:
+        # This includes misc. malformed input e.g. str
+        return a, b  # type: ignore[return-value]
+
+    swap = False
+    if a_scalar:
+        swap = True
+        b, a = a, b
+
+    if is_array_api_obj(a):
+        # a is an Array API object
+        # b is a int | float | complex | bool
+        xa = a
+
+        # https://data-apis.org/array-api/draft/API_specification/type_promotion.html#mixing-arrays-with-python-scalars
+        same_dtype = {
+            bool: "bool",
+            int: ("integral", "real floating", "complex floating"),
+            float: ("real floating", "complex floating"),
+            complex: "complex floating",
+        }
+        kind = same_dtype[type(cast(complex, b))]
+        if xp.isdtype(a.dtype, kind):
+            xb = xp.asarray(b, dtype=a.dtype)
+        else:
+            # Undefined behaviour. Let the function deal with it, if it can.
+            xb = xp.asarray(b)
+
+    else:
+        # Neither a nor b are Array API objects.
+        # Note: we can only reach this point when one explicitly passes
+        # xp=xp to the calling function; otherwise we fail earlier on
+        # array_namespace(a, b).
+        xa, xb = xp.asarray(a), xp.asarray(b)
+
+    return (xb, xa) if swap else (xa, xb)
+
+
+def ndindex(*x: int) -> Generator[tuple[int, ...]]:
+    """
+    Generate all N-dimensional indices for a given array shape.
+
+    Given the shape of an array, an ndindex instance iterates over the N-dimensional
+    index of the array. At each iteration a tuple of indices is returned, the last
+    dimension is iterated over first.
+
+    This has an identical API to numpy.ndindex.
+
+    Parameters
+    ----------
+    *x : int
+        The shape of the array.
+    """
+    if not x:
+        yield ()
+        return
+    for i in ndindex(*x[:-1]):
+        for j in range(x[-1]):
+            yield *i, j
+
+
+def eager_shape(x: Array, /) -> tuple[int, ...]:
+    """
+    Return shape of an array. Raise if shape is not fully defined.
+
+    Parameters
+    ----------
+    x : Array
+        Input array.
+
+    Returns
+    -------
+    tuple[int, ...]
+        Shape of the array.
+    """
+    shape = x.shape
+    # Dask arrays uses non-standard NaN instead of None
+    if any(s is None or math.isnan(s) for s in shape):
+        msg = "Unsupported lazy shape"
+        raise TypeError(msg)
+    return cast(tuple[int, ...], shape)
+
+
+def meta_namespace(
+    *arrays: Array | complex | None, xp: ModuleType | None = None
+) -> ModuleType:
+    """
+    Get the namespace of Dask chunks.
+
+    On all other backends, just return the namespace of the arrays.
+
+    Parameters
+    ----------
+    *arrays : Array | int | float | complex | bool | None
+        Input arrays.
+    xp : array_namespace, optional
+        The standard-compatible namespace for the input arrays. Default: infer.
+
+    Returns
+    -------
+    array_namespace
+        If xp is Dask, the namespace of the Dask chunks;
+        otherwise, the namespace of the arrays.
+    """
+    xp = array_namespace(*arrays) if xp is None else xp
+    if not is_dask_namespace(xp):
+        return xp
+    # Quietly skip scalars and None's
+    metas = [cast(Array | None, getattr(a, "_meta", None)) for a in arrays]
+    return array_namespace(*metas)
+
+
+def capabilities(
+    xp: ModuleType, *, device: Device | None = None
+) -> dict[str, int | None]:
+    """
+    Return patched ``xp.__array_namespace_info__().capabilities()``.
+
+    TODO this helper should be eventually removed once all the special cases
+    it handles are fixed in the respective backends.
+
+    Parameters
+    ----------
+    xp : array_namespace
+        The standard-compatible namespace.
+    device : Device, optional
+        The device to use.
+
+    Returns
+    -------
+    dict
+        Capabilities of the namespace.
+    """
+    out = xp.__array_namespace_info__().capabilities()
+    if is_pydata_sparse_namespace(xp):
+        if out["boolean indexing"]:
+            # FIXME https://github.com/pydata/sparse/issues/876
+            # boolean indexing is supported, but not when the index is a sparse array.
+            # boolean indexing by list or numpy array is not part of the Array API.
+            out = out.copy()
+            out["boolean indexing"] = False
+    elif is_jax_namespace(xp):
+        if out["boolean indexing"]:  # pragma: no cover
+            # Backwards compatibility with jax <0.6.0
+            # https://github.com/jax-ml/jax/issues/27418
+            out = out.copy()
+            out["boolean indexing"] = False
+    elif is_torch_namespace(xp):
+        # FIXME https://github.com/data-apis/array-api/issues/945
+        device = xp.get_default_device() if device is None else xp.device(device)
+        if device.type == "meta":  # type: ignore[union-attr]  # pyright: ignore[reportAttributeAccessIssue,reportOptionalMemberAccess]
+            out = out.copy()
+            out["boolean indexing"] = False
+            out["data-dependent shapes"] = False
+
+    return out
+
+
+_BASIC_PICKLED_TYPES = frozenset((
+    bool, int, float, complex, str, bytes, bytearray,
+    list, tuple, dict, set, frozenset, range, slice,
+    types.NoneType, types.EllipsisType,
+))  # fmt: skip
+_BASIC_REST_TYPES = frozenset((
+    type, types.BuiltinFunctionType, types.FunctionType, types.ModuleType
+))  # fmt: skip
+
+FlattenRest: TypeAlias = tuple[object, ...]
+
+
+def pickle_flatten(
+    obj: object, cls: type[T] | tuple[type[T], ...]
+) -> tuple[list[T], FlattenRest]:
+    """
+    Use the pickle machinery to extract objects out of an arbitrary container.
+
+    Unlike regular ``pickle.dumps``, this function always succeeds.
+
+    Parameters
+    ----------
+    obj : object
+        The object to pickle.
+    cls : type | tuple[type, ...]
+        One or multiple classes to extract from the object.
+        The instances of these classes inside ``obj`` will not be pickled.
+
+    Returns
+    -------
+    instances : list[cls]
+        All instances of ``cls`` found inside ``obj`` (not pickled).
+    rest
+        Opaque object containing the pickled bytes plus all other objects where
+        ``__reduce__`` / ``__reduce_ex__`` is either not implemented or raised.
+        These are unpickleable objects, types, modules, and functions.
+
+        This object is *typically* hashable save for fairly exotic objects
+        that are neither pickleable nor hashable.
+
+        This object is pickleable if everything except ``instances`` was pickleable
+        in the input object.
+
+    See Also
+    --------
+    pickle_unflatten : Reverse function.
+
+    Examples
+    --------
+    >>> class A:
+    ...     def __repr__(self):
+    ...         return ""
+    >>> class NS:
+    ...     def __repr__(self):
+    ...         return ""
+    ...     def __reduce__(self):
+    ...         assert False, "not serializable"
+    >>> obj = {1: A(), 2: [A(), NS(), A()]}
+    >>> instances, rest = pickle_flatten(obj, A)
+    >>> instances
+    [, , ]
+    >>> pickle_unflatten(instances, rest)
+    {1: , 2: [, , ]}
+
+    This can be also used to swap inner objects; the only constraint is that
+    the number of objects in and out must be the same:
+
+    >>> pickle_unflatten(["foo", "bar", "baz"], rest)
+    {1: "foo", 2: ["bar", , "baz"]}
+    """
+    instances: list[T] = []
+    rest: list[object] = []
+
+    class Pickler(pickle.Pickler):  # numpydoc ignore=GL08
+        """
+        Use the `pickle.Pickler.persistent_id` hook to extract objects.
+        """
+
+        @override
+        def persistent_id(
+            self, obj: object
+        ) -> Literal[0, 1, None]:  # numpydoc ignore=GL08
+            if isinstance(obj, cls):
+                instances.append(obj)  # type: ignore[arg-type]
+                return 0
+
+            typ_ = type(obj)
+            if typ_ in _BASIC_PICKLED_TYPES:  # No subclasses!
+                # If obj is a collection, recursively descend inside it
+                return None
+            if typ_ in _BASIC_REST_TYPES:
+                rest.append(obj)
+                return 1
+
+            try:
+                # Note: a class that defines __slots__ without defining __getstate__
+                # cannot be pickled with __reduce__(), but can with __reduce_ex__(5)
+                _ = obj.__reduce_ex__(pickle.HIGHEST_PROTOCOL)
+            except Exception:  # pylint: disable=broad-exception-caught
+                rest.append(obj)
+                return 1
+
+            # Object can be pickled. Let the Pickler recursively descend inside it.
+            return None
+
+    f = io.BytesIO()
+    p = Pickler(f, protocol=pickle.HIGHEST_PROTOCOL)
+    p.dump(obj)
+    return instances, (f.getvalue(), *rest)
+
+
+def pickle_unflatten(instances: Iterable[object], rest: FlattenRest) -> Any:
+    """
+    Reverse of ``pickle_flatten``.
+
+    Parameters
+    ----------
+    instances : Iterable
+        Inner objects to be reinserted into the flattened container.
+    rest : FlattenRest
+        Extra bits, as returned by ``pickle_flatten``.
+
+    Returns
+    -------
+    object
+        The outer object originally passed to ``pickle_flatten`` after a
+        pickle->unpickle round-trip.
+
+    See Also
+    --------
+    pickle_flatten : Serializing function.
+    pickle.loads : Standard unpickle function.
+
+    Notes
+    -----
+    The `instances` iterable must yield at least the same number of elements as the ones
+    returned by ``pickle_flatten``, but the elements do not need to be the same objects
+    or even the same types of objects. Excess elements, if any, will be left untouched.
+    """
+    iters = iter(instances), iter(rest)
+    pik = cast(bytes, next(iters[1]))
+
+    class Unpickler(pickle.Unpickler):  # numpydoc ignore=GL08
+        """Mirror of the overridden Pickler in pickle_flatten."""
+
+        @override
+        def persistent_load(self, pid: Literal[0, 1]) -> object:  # numpydoc ignore=GL08
+            try:
+                return next(iters[pid])
+            except StopIteration as e:
+                msg = "Not enough objects to unpickle"
+                raise ValueError(msg) from e
+
+    f = io.BytesIO(pik)
+    return Unpickler(f).load()
+
+
+class _AutoJITWrapper(Generic[T]):  # numpydoc ignore=PR01
+    """
+    Helper of :func:`jax_autojit`.
+
+    Wrap arbitrary inputs and outputs of the jitted function and
+    convert them to/from PyTrees.
+    """
+
+    obj: T
+    _registered: ClassVar[bool] = False
+    __slots__: tuple[str, ...] = ("obj",)
+
+    def __init__(self, obj: T) -> None:  # numpydoc ignore=GL08
+        self._register()
+        self.obj = obj
+
+    @classmethod
+    def _register(cls) -> None:  # numpydoc ignore=SS06
+        """
+        Register upon first use instead of at import time, to avoid
+        globally importing JAX.
+        """
+        if not cls._registered:
+            import jax
+
+            jax.tree_util.register_pytree_node(
+                cls,
+                lambda obj: pickle_flatten(obj, jax.Array),  # pyright: ignore[reportUnknownArgumentType]
+                lambda aux_data, children: pickle_unflatten(children, aux_data),  # pyright: ignore[reportUnknownArgumentType]
+            )
+            cls._registered = True
+
+
+def jax_autojit(
+    func: Callable[P, T],
+) -> Callable[P, T]:  # numpydoc ignore=PR01,RT01,SS03
+    """
+    Wrap `func` with ``jax.jit``, with the following differences:
+
+    - Python scalar arguments and return values are not automatically converted to
+      ``jax.Array`` objects.
+    - All non-array arguments are automatically treated as static.
+      Unlike ``jax.jit``, static arguments must be either hashable or serializable with
+      ``pickle``.
+    - Unlike ``jax.jit``, non-array arguments and return values are not limited to
+      tuple/list/dict, but can be any object serializable with ``pickle``.
+    - Automatically descend into non-array arguments and find ``jax.Array`` objects
+      inside them, then rebuild the arguments when entering `func`, swapping the JAX
+      concrete arrays with tracer objects.
+    - Automatically descend into non-array return values and find ``jax.Array`` objects
+      inside them, then rebuild them downstream of exiting the JIT, swapping the JAX
+      tracer objects with concrete arrays.
+
+    See Also
+    --------
+    jax.jit : JAX JIT compilation function.
+
+    Notes
+    -----
+    These are useful choices *for testing purposes only*, which is how this function is
+    intended to be used. The output of ``jax.jit`` is a C++ level callable, that
+    directly dispatches to the compiled kernel after the initial call. In comparison,
+    ``jax_autojit`` incurs a much higher dispatch time.
+
+    Additionally, consider::
+
+        def f(x: Array, y: float, plus: bool) -> Array:
+            return x + y if plus else x - y
+
+        j1 = jax.jit(f, static_argnames="plus")
+        j2 = jax_autojit(f)
+
+    In the above example, ``j2`` requires a lot less setup to be tested effectively than
+    ``j1``, but on the flip side it means that it will be re-traced for every different
+    value of ``y``, which likely makes it not fit for purpose in production.
+    """
+    import jax
+
+    @jax.jit  # type: ignore[misc]  # pyright: ignore[reportUntypedFunctionDecorator]
+    def inner(  # numpydoc ignore=GL08
+        wargs: _AutoJITWrapper[Any],
+    ) -> _AutoJITWrapper[T]:
+        args, kwargs = wargs.obj
+        res = func(*args, **kwargs)  # pyright: ignore[reportCallIssue]
+        return _AutoJITWrapper(res)
+
+    @wraps(func)
+    def outer(*args: P.args, **kwargs: P.kwargs) -> T:  # numpydoc ignore=GL08
+        wargs = _AutoJITWrapper((args, kwargs))
+        return inner(wargs).obj
+
+    return outer
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_typing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0d4c445b41b2e8a4cfe4006a6892fa5ddd8ec10
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_typing.py
@@ -0,0 +1,10 @@
+# numpydoc ignore=GL08
+# pylint: disable=missing-module-docstring,duplicate-code
+
+Array = object
+DType = object
+Device = object
+GetIndex = object
+SetIndex = object
+
+__all__ = ["Array", "DType", "Device", "GetIndex", "SetIndex"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_typing.pyi b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_typing.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..dd864c3e702c2316f3ff52e12af19f25479b4935
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/_lib/_utils/_typing.pyi
@@ -0,0 +1,105 @@
+"""Static typing helpers."""
+
+from __future__ import annotations
+
+from types import EllipsisType
+from typing import Protocol, TypeAlias
+
+# TODO import from typing (requires Python >=3.12)
+from typing_extensions import override
+
+# TODO: use array-api-typing once it is available
+
+class Array(Protocol):  # pylint: disable=missing-class-docstring
+    # Unary operations
+    def __abs__(self) -> Array: ...
+    def __pos__(self) -> Array: ...
+    def __neg__(self) -> Array: ...
+    def __invert__(self) -> Array: ...
+    # Binary operations
+    def __add__(self, other: Array | complex, /) -> Array: ...
+    def __sub__(self, other: Array | complex, /) -> Array: ...
+    def __mul__(self, other: Array | complex, /) -> Array: ...
+    def __truediv__(self, other: Array | complex, /) -> Array: ...
+    def __floordiv__(self, other: Array | complex, /) -> Array: ...
+    def __mod__(self, other: Array | complex, /) -> Array: ...
+    def __pow__(self, other: Array | complex, /) -> Array: ...
+    def __matmul__(self, other: Array, /) -> Array: ...
+    def __and__(self, other: Array | int, /) -> Array: ...
+    def __or__(self, other: Array | int, /) -> Array: ...
+    def __xor__(self, other: Array | int, /) -> Array: ...
+    def __lshift__(self, other: Array | int, /) -> Array: ...
+    def __rshift__(self, other: Array | int, /) -> Array: ...
+    def __lt__(self, other: Array | complex, /) -> Array: ...
+    def __le__(self, other: Array | complex, /) -> Array: ...
+    def __gt__(self, other: Array | complex, /) -> Array: ...
+    def __ge__(self, other: Array | complex, /) -> Array: ...
+    @override
+    def __eq__(self, other: Array | complex, /) -> Array: ...  # type: ignore[override]  # pyright: ignore[reportIncompatibleMethodOverride]
+    @override
+    def __ne__(self, other: Array | complex, /) -> Array: ...  # type: ignore[override]  # pyright: ignore[reportIncompatibleMethodOverride]
+    # Reflected operations
+    def __radd__(self, other: Array | complex, /) -> Array: ...
+    def __rsub__(self, other: Array | complex, /) -> Array: ...
+    def __rmul__(self, other: Array | complex, /) -> Array: ...
+    def __rtruediv__(self, other: Array | complex, /) -> Array: ...
+    def __rfloordiv__(self, other: Array | complex, /) -> Array: ...
+    def __rmod__(self, other: Array | complex, /) -> Array: ...
+    def __rpow__(self, other: Array | complex, /) -> Array: ...
+    def __rmatmul__(self, other: Array, /) -> Array: ...
+    def __rand__(self, other: Array | int, /) -> Array: ...
+    def __ror__(self, other: Array | int, /) -> Array: ...
+    def __rxor__(self, other: Array | int, /) -> Array: ...
+    def __rlshift__(self, other: Array | int, /) -> Array: ...
+    def __rrshift__(self, other: Array | int, /) -> Array: ...
+    # Attributes
+    @property
+    def dtype(self) -> DType: ...
+    @property
+    def device(self) -> Device: ...
+    @property
+    def mT(self) -> Array: ...  # pylint: disable=invalid-name
+    @property
+    def ndim(self) -> int: ...
+    @property
+    def shape(self) -> tuple[int | None, ...]: ...
+    @property
+    def size(self) -> int | None: ...
+    @property
+    def T(self) -> Array: ...  # pylint: disable=invalid-name
+    # Collection operations (note: an Array does not have to be Sized or Iterable)
+    def __getitem__(self, key: GetIndex, /) -> Array: ...
+    def __setitem__(self, key: SetIndex, value: Array | complex, /) -> None: ...
+    # Materialization methods (may raise on lazy arrays)
+    def __bool__(self) -> bool: ...
+    def __complex__(self) -> complex: ...
+    def __float__(self) -> float: ...
+    def __index__(self) -> int: ...
+    def __int__(self) -> int: ...
+
+    # Misc methods (frequently not implemented in Arrays wrapped by array-api-compat)
+    # def __array_namespace__(*, api_version: str | None) -> ModuleType: ...
+    # def __dlpack__(
+    #     *,
+    #     stream: int | Any | None = None,
+    #     max_version: tuple[int, int] | None = None,
+    #     dl_device: tuple[int, int] | None = None,  # tuple[Enum, int]
+    #     copy: bool | None = None,
+    # ) -> Any: ...
+    # def __dlpack_device__() -> tuple[int, int]: ...  # tuple[Enum, int]
+    # def to_device(device: Device, /, *, stream: int | Any | None = None) -> Array: ...
+
+class DType(Protocol):  # pylint: disable=missing-class-docstring
+    pass
+
+class Device(Protocol):  # pylint: disable=missing-class-docstring
+    pass
+
+SetIndex: TypeAlias = (
+    int | slice | EllipsisType | Array | tuple[int | slice | EllipsisType | Array, ...]
+)
+GetIndex: TypeAlias = (
+    SetIndex | None | tuple[int | slice | EllipsisType | None | Array, ...]
+)
+
+__all__ = ["Array", "DType", "Device", "GetIndex", "SetIndex"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/py.typed b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/testing.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/testing.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c1be71ea428b418ce4a6dd43c5467dae6225d37
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/array_api_extra/testing.py
@@ -0,0 +1,415 @@
+"""
+Public testing utilities.
+
+See also _lib._testing for additional private testing utilities.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import enum
+import warnings
+from collections.abc import Callable, Generator, Iterator, Sequence
+from functools import wraps
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast
+
+from ._lib._utils._compat import is_dask_namespace, is_jax_namespace
+from ._lib._utils._helpers import jax_autojit, pickle_flatten, pickle_unflatten
+
+__all__ = ["lazy_xp_function", "patch_lazy_xp_functions"]
+
+if TYPE_CHECKING:  # pragma: no cover
+    # TODO import override from typing (requires Python >=3.12)
+    import pytest
+    from dask.typing import Graph, Key, SchedulerGetCallable
+    from typing_extensions import override
+
+else:
+    # Sphinx hacks
+    SchedulerGetCallable = object
+
+    def override(func):
+        return func
+
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+_ufuncs_tags: dict[object, dict[str, Any]] = {}
+
+
+class Deprecated(enum.Enum):
+    """Unique type for deprecated parameters."""
+
+    DEPRECATED = 1
+
+
+DEPRECATED = Deprecated.DEPRECATED
+
+
+def lazy_xp_function(
+    func: Callable[..., Any],
+    *,
+    allow_dask_compute: bool | int = False,
+    jax_jit: bool = True,
+    static_argnums: Deprecated = DEPRECATED,
+    static_argnames: Deprecated = DEPRECATED,
+) -> None:  # numpydoc ignore=GL07
+    """
+    Tag a function to be tested on lazy backends.
+
+    Tag a function so that when any tests are executed with ``xp=jax.numpy`` the
+    function is replaced with a jitted version of itself, and when it is executed with
+    ``xp=dask.array`` the function will raise if it attempts to materialize the graph.
+    This will be later expanded to provide test coverage for other lazy backends.
+
+    In order for the tag to be effective, the test or a fixture must call
+    :func:`patch_lazy_xp_functions`.
+
+    Parameters
+    ----------
+    func : callable
+        Function to be tested.
+    allow_dask_compute : bool | int, optional
+        Whether `func` is allowed to internally materialize the Dask graph, or maximum
+        number of times it is allowed to do so. This is typically triggered by
+        ``bool()``, ``float()``, or ``np.asarray()``.
+
+        Set to 1 if you are aware that `func` converts the input parameters to NumPy and
+        want to let it do so at least for the time being, knowing that it is going to be
+        extremely detrimental for performance.
+
+        If a test needs values higher than 1 to pass, it is a canary that the conversion
+        to NumPy/bool/float is happening multiple times, which translates to multiple
+        computations of the whole graph. Short of making the function fully lazy, you
+        should at least add explicit calls to ``np.asarray()`` early in the function.
+        *Note:* the counter of `allow_dask_compute` resets after each call to `func`, so
+        a test function that invokes `func` multiple times should still work with this
+        parameter set to 1.
+
+        Set to True to allow `func` to materialize the graph an unlimited number
+        of times.
+
+        Default: False, meaning that `func` must be fully lazy and never materialize the
+        graph.
+    jax_jit : bool, optional
+        Set to True to replace `func` with a smart variant of ``jax.jit(func)`` after
+        calling the :func:`patch_lazy_xp_functions` test helper with ``xp=jax.numpy``.
+        This is the default behaviour.
+        Set to False if `func` is only compatible with eager (non-jitted) JAX.
+
+        Unlike with vanilla ``jax.jit``, all arguments and return types that are not JAX
+        arrays are treated as static; the function can accept and return arbitrary
+        wrappers around JAX arrays. This difference is because, in real life, most users
+        won't wrap the function directly with ``jax.jit`` but rather they will use it
+        within their own code, which is itself then wrapped by ``jax.jit``, and
+        internally consume the function's outputs.
+
+        In other words, the pattern that is being tested is::
+
+            >>> @jax.jit
+            ... def user_func(x):
+            ...     y = user_prepares_inputs(x)
+            ...     z = func(y, some_static_arg=True)
+            ...     return user_consumes(z)
+
+        Default: True.
+    static_argnums :
+        Deprecated; ignored
+    static_argnames :
+        Deprecated; ignored
+
+    See Also
+    --------
+    patch_lazy_xp_functions : Companion function to call from the test or fixture.
+    jax.jit : JAX function to compile a function for performance.
+
+    Examples
+    --------
+    In ``test_mymodule.py``::
+
+      from array_api_extra.testing import lazy_xp_function from mymodule import myfunc
+
+      lazy_xp_function(myfunc)
+
+      def test_myfunc(xp):
+          a = xp.asarray([1, 2])
+          # When xp=jax.numpy, this is similar to `b = jax.jit(myfunc)(a)`
+          # When xp=dask.array, crash on compute() or persist()
+          b = myfunc(a)
+
+    Notes
+    -----
+    In order for this tag to be effective, the test function must be imported into the
+    test module globals without its namespace; alternatively its namespace must be
+    declared in a ``lazy_xp_modules`` list in the test module globals.
+
+    Example 1::
+
+      from mymodule import myfunc
+
+      lazy_xp_function(myfunc)
+
+      def test_myfunc(xp):
+          x = myfunc(xp.asarray([1, 2]))
+
+    Example 2::
+
+      import mymodule
+
+      lazy_xp_modules = [mymodule]
+      lazy_xp_function(mymodule.myfunc)
+
+      def test_myfunc(xp):
+          x = mymodule.myfunc(xp.asarray([1, 2]))
+
+    A test function can circumvent this monkey-patching system by using a namespace
+    outside of the two above patterns. You need to sanitize your code to make sure this
+    only happens intentionally.
+
+    Example 1::
+
+      import mymodule
+      from mymodule import myfunc
+
+      lazy_xp_function(myfunc)
+
+      def test_myfunc(xp):
+          a = xp.asarray([1, 2])
+          b = myfunc(a)  # This is wrapped when xp=jax.numpy or xp=dask.array
+          c = mymodule.myfunc(a)  # This is not
+
+    Example 2::
+
+      import mymodule
+
+      class naked:
+          myfunc = mymodule.myfunc
+
+      lazy_xp_modules = [mymodule]
+      lazy_xp_function(mymodule.myfunc)
+
+      def test_myfunc(xp):
+          a = xp.asarray([1, 2])
+          b = mymodule.myfunc(a)  # This is wrapped when xp=jax.numpy or xp=dask.array
+          c = naked.myfunc(a)  # This is not
+    """
+    if static_argnums is not DEPRECATED or static_argnames is not DEPRECATED:
+        warnings.warn(
+            (
+                "The `static_argnums` and `static_argnames` parameters are deprecated "
+                "and ignored. They will be removed in a future version."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+    tags = {
+        "allow_dask_compute": allow_dask_compute,
+        "jax_jit": jax_jit,
+    }
+
+    try:
+        func._lazy_xp_function = tags  # type: ignore[attr-defined]  # pylint: disable=protected-access  # pyright: ignore[reportFunctionMemberAccess]
+    except AttributeError:  # @cython.vectorize
+        _ufuncs_tags[func] = tags
+
+
+def patch_lazy_xp_functions(
+    request: pytest.FixtureRequest,
+    monkeypatch: pytest.MonkeyPatch | None = None,
+    *,
+    xp: ModuleType,
+) -> contextlib.AbstractContextManager[None]:
+    """
+    Test lazy execution of functions tagged with :func:`lazy_xp_function`.
+
+    If ``xp==jax.numpy``, search for all functions which have been tagged with
+    :func:`lazy_xp_function` in the globals of the module that defines the current test,
+    as well as in the ``lazy_xp_modules`` list in the globals of the same module,
+    and wrap them with :func:`jax.jit`. Unwrap them at the end of the test.
+
+    If ``xp==dask.array``, wrap the functions with a decorator that disables
+    ``compute()`` and ``persist()`` and ensures that exceptions and warnings are raised
+    eagerly.
+
+    This function should be typically called by your library's `xp` fixture that runs
+    tests on multiple backends::
+
+        @pytest.fixture(params=[
+            numpy,
+            array_api_strict,
+            pytest.param(jax.numpy, marks=pytest.mark.thread_unsafe),
+            pytest.param(dask.array, marks=pytest.mark.thread_unsafe),
+        ])
+        def xp(request):
+            with patch_lazy_xp_functions(request, xp=request.param):
+                yield request.param
+
+    but it can be otherwise be called by the test itself too.
+
+    Parameters
+    ----------
+    request : pytest.FixtureRequest
+        Pytest fixture, as acquired by the test itself or by one of its fixtures.
+    monkeypatch : pytest.MonkeyPatch
+        Deprecated
+    xp : array_namespace
+        Array namespace to be tested.
+
+    See Also
+    --------
+    lazy_xp_function : Tag a function to be tested on lazy backends.
+    pytest.FixtureRequest : `request` test function parameter.
+
+    Notes
+    -----
+    This context manager monkey-patches modules and as such is thread unsafe
+    on Dask and JAX. If you run your test suite with
+    `pytest-run-parallel `_,
+    you should mark these backends with ``@pytest.mark.thread_unsafe``, as shown in
+    the example above.
+    """
+    mod = cast(ModuleType, request.module)
+    mods = [mod, *cast(list[ModuleType], getattr(mod, "lazy_xp_modules", []))]
+
+    to_revert: list[tuple[ModuleType, str, object]] = []
+
+    def temp_setattr(mod: ModuleType, name: str, func: object) -> None:
+        """
+        Variant of monkeypatch.setattr, which allows monkey-patching only selected
+        parameters of a test so that pytest-run-parallel can run on the remainder.
+        """
+        assert hasattr(mod, name)
+        to_revert.append((mod, name, getattr(mod, name)))
+        setattr(mod, name, func)
+
+    if monkeypatch is not None:
+        warnings.warn(
+            (
+                "The `monkeypatch` parameter is deprecated and will be removed in a "
+                "future version. "
+                "Use `patch_lazy_xp_function` as a context manager instead."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        # Enable using patch_lazy_xp_function not as a context manager
+        temp_setattr = monkeypatch.setattr  # type: ignore[assignment]  # pyright: ignore[reportAssignmentType]
+
+    def iter_tagged() -> Iterator[
+        tuple[ModuleType, str, Callable[..., Any], dict[str, Any]]
+    ]:
+        for mod in mods:
+            for name, func in mod.__dict__.items():
+                tags: dict[str, Any] | None = None
+                with contextlib.suppress(AttributeError):
+                    tags = func._lazy_xp_function  # pylint: disable=protected-access
+                if tags is None:
+                    with contextlib.suppress(KeyError, TypeError):
+                        tags = _ufuncs_tags[func]
+                if tags is not None:
+                    yield mod, name, func, tags
+
+    if is_dask_namespace(xp):
+        for mod, name, func, tags in iter_tagged():
+            n = tags["allow_dask_compute"]
+            if n is True:
+                n = 1_000_000
+            elif n is False:
+                n = 0
+            wrapped = _dask_wrap(func, n)
+            temp_setattr(mod, name, wrapped)
+
+    elif is_jax_namespace(xp):
+        for mod, name, func, tags in iter_tagged():
+            if tags["jax_jit"]:
+                wrapped = jax_autojit(func)
+                temp_setattr(mod, name, wrapped)
+
+    # We can't just decorate patch_lazy_xp_functions with
+    # @contextlib.contextmanager because it would not work with the
+    # deprecated monkeypatch when not used as a context manager.
+    @contextlib.contextmanager
+    def revert_on_exit() -> Generator[None]:
+        try:
+            yield
+        finally:
+            for mod, name, orig_func in to_revert:
+                setattr(mod, name, orig_func)
+
+    return revert_on_exit()
+
+
+class CountingDaskScheduler(SchedulerGetCallable):
+    """
+    Dask scheduler that counts how many times `dask.compute` is called.
+
+    If the number of times exceeds 'max_count', it raises an error.
+    This is a wrapper around Dask's own 'synchronous' scheduler.
+
+    Parameters
+    ----------
+    max_count : int
+        Maximum number of allowed calls to `dask.compute`.
+    msg : str
+        Assertion to raise when the count exceeds `max_count`.
+    """
+
+    count: int
+    max_count: int
+    msg: str
+
+    def __init__(self, max_count: int, msg: str):  # numpydoc ignore=GL08
+        self.count = 0
+        self.max_count = max_count
+        self.msg = msg
+
+    @override
+    def __call__(
+        self, dsk: Graph, keys: Sequence[Key] | Key, **kwargs: Any
+    ) -> Any:  # numpydoc ignore=GL08
+        import dask
+
+        self.count += 1
+        # This should yield a nice traceback to the
+        # offending line in the user's code
+        assert self.count <= self.max_count, self.msg
+
+        return dask.get(dsk, keys, **kwargs)  # type: ignore[attr-defined]  # pyright: ignore[reportPrivateImportUsage]
+
+
+def _dask_wrap(
+    func: Callable[P, T], n: int
+) -> Callable[P, T]:  # numpydoc ignore=PR01,RT01
+    """
+    Wrap `func` to raise if it attempts to call `dask.compute` more than `n` times.
+
+    After the function returns, materialize the graph in order to re-raise exceptions.
+    """
+    import dask
+    import dask.array as da
+
+    func_name = getattr(func, "__name__", str(func))
+    n_str = f"only up to {n}" if n else "no"
+    msg = (
+        f"Called `dask.compute()` or `dask.persist()` {n + 1} times, "
+        f"but {n_str} calls are allowed. Set "
+        f"`lazy_xp_function({func_name}, allow_dask_compute={n + 1})` "
+        "to allow for more (but note that this will harm performance). "
+    )
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:  # numpydoc ignore=GL08
+        scheduler = CountingDaskScheduler(n, msg)
+        with dask.config.set({"scheduler": scheduler}):  # pyright: ignore[reportPrivateImportUsage]
+            out = func(*args, **kwargs)
+
+        # Block until the graph materializes and reraise exceptions. This allows
+        # `pytest.raises` and `pytest.warns` to work as expected. Note that this would
+        # not work on scheduler='distributed', as it would not block.
+        arrays, rest = pickle_flatten(out, da.Array)
+        arrays = dask.persist(arrays, scheduler="threads")[0]  # type: ignore[attr-defined,no-untyped-call]  # pyright: ignore[reportPrivateImportUsage]
+        return pickle_unflatten(arrays, rest)  # pyright: ignore[reportUnknownArgumentType]
+
+    return wrapper
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/conftest.py b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..835a1c9695d8f8af07d42ef388a330330d69326d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/sklearn/externals/conftest.py
@@ -0,0 +1,6 @@
+# Do not collect any tests in externals. This is more robust than using
+# --ignore because --ignore needs a path and it is not convenient to pass in
+# the externals path (very long install-dependent path in site-packages) when
+# using --pyargs
+def pytest_ignore_collect(collection_path, config):
+    return True