techprotrade commited on
Commit
6043940
·
verified ·
1 Parent(s): c91c831

Add loan-advisor-agent (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz +3 -0
  3. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz +3 -0
  4. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz +3 -0
  5. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/__init__.py +0 -0
  6. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz +3 -0
  7. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz +3 -0
  8. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz +3 -0
  9. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz +3 -0
  10. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/__init__.py +0 -0
  11. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz +3 -0
  12. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz +3 -0
  13. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz +3 -0
  14. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz +3 -0
  15. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/__init__.py +0 -0
  16. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz +3 -0
  17. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz +3 -0
  18. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz +3 -0
  19. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz +3 -0
  20. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz +3 -0
  21. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz +3 -0
  22. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/__init__.py +0 -0
  23. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz +3 -0
  24. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz +3 -0
  25. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz +3 -0
  26. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz +3 -0
  27. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz +3 -0
  28. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz +3 -0
  29. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/__init__.py +0 -0
  30. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz +3 -0
  31. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz +3 -0
  32. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz +3 -0
  33. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz +3 -0
  34. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_classification.txt +9 -0
  35. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid.txt +3 -0
  36. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid_order.txt +1 -0
  37. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_multilabel.txt +5 -0
  38. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_20news.py +143 -0
  39. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_arff_parser.py +284 -0
  40. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_base.py +659 -0
  41. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_california_housing.py +38 -0
  42. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_common.py +136 -0
  43. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_covtype.py +55 -0
  44. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_kddcup99.py +89 -0
  45. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_lfw.py +229 -0
  46. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_olivetti_faces.py +26 -0
  47. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_openml.py +1636 -0
  48. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_rcv1.py +71 -0
  49. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_samples_generator.py +744 -0
  50. loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_svmlight_format.py +613 -0
.gitattributes CHANGED
@@ -676,3 +676,5 @@ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_lloyd.cp312-
676
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/_svmlight_format_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
677
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/china.jpg filter=lfs diff=lfs merge=lfs -text
678
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/flower.jpg filter=lfs diff=lfs merge=lfs -text
 
 
 
676
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/_svmlight_format_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
677
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/china.jpg filter=lfs diff=lfs merge=lfs -text
678
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/flower.jpg filter=lfs diff=lfs merge=lfs -text
679
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
680
+ loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:301396b4a42c814b1a15038ddfcbcf5c8590501231747d0dc2a500b84b2fd0df
3
+ size 328
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dee83987fffa8ec20e23b3cabc00d42beb7a469af6bd803909998c1687fa634
3
+ size 934
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c5fd93ffec7deb63a940fd698534dd7ebb7db349fc183930041cbf17e60e2cc
3
+ size 6471
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/__init__.py ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4fcb215595bb8223321943f0df50f5a5013445bf1819d53d3cb5bfa90d9e79c0
3
+ size 595
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38b74e7f02a61ff55bcfac4d87103d5bffc43febb0c019d9aaa162f8f7693068
3
+ size 272
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8742a74bd5bc120acd9186c8a8737cb420ed9b009fade00b24e7ce5217797f2c
3
+ size 722
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f623e777c0a36ae6c82fae10a7c2088cb383298ea244595bf8dc95449c9be4c4
3
+ size 2326
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/__init__.py ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ccbf138e0663895f9cf511136bc6395c153f6238af2eacb6a367e86e15d1a71
3
+ size 1492
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0985045a454c8186b4e690ebefb6cea1ef7c13292c98d50abda470a0ff3ad425
3
+ size 312
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3736e7feb7ad30c68675c2c4e48a9fb262e80308c9083b100ddd0339da1fc282
3
+ size 348
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8d00c6690576a9ec39e1cb77054e13296be0fdebab0fb35a64a0e8627b6e6f3
3
+ size 4519
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/__init__.py ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a1d38fdd601b67bb9c6d16152f53ddf166a0cfcfef4fa86438e899bfe449226c
3
+ size 1798
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:407424fb79cc30b8e9ff90900b3bf29244ac7f3797f278b5be602843f959b4ee
3
+ size 425
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0703b0ae20b9ff75087dc601640ee58f1c2ad6768858ea21a245151da9ba8e4c
3
+ size 301
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70d4596ad879547863109da8675c2b789d07df66b526d7ebcbce9616c4c9b94c
3
+ size 347
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8743b2d93d2c62a82fb47e1fbc002b97e25adcfb5bf1fcb26b58ad0bed15bd48
3
+ size 1074
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e96142b5e00dfec2617b0c22d7192b340ae2c28ec3ffc3a894c5be746b970a59
3
+ size 3303
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/__init__.py ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5c7e79aa41ef580838fb9fc1906280f076c47be1741fddd5004ddb500eb57fe
3
+ size 898
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33cbd6ae945ba04969370ab35604e9363c87256393493382b5118a89d59386d6
3
+ size 268
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0bce20aae7fd903796d96d5b3a3677b7058fbc5f3fe0996ee9d491e4ee23d132
3
+ size 293
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9f4b9317997df63ed8d2bb073a3906344c0e0be017fd384eaec36ced8b94bae
3
+ size 330
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:424cd47c12a51c7bb8d8169fac80fb5601f152bd78468b241d4b115bf7d22f20
3
+ size 1121
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afe4736924606638984e573235191025d419c545d31dc8874c96b72f5ec5db73
3
+ size 2342
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/__init__.py ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ef3551ad47d48023c5a1f1cf077047a9a4b95544bb91d4a86097f8b574f8d07
3
+ size 656
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:489b177126cb7f335cb220709233b946d3a0ad71d38bba6d48b79187146e585a
3
+ size 817
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:278a52a52d569f07d14c6a7877b104762c77daac429fb1fd9817a0378d6ec634
3
+ size 805
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb5830c82112f62a400c82ac1f1b5eb61c29c0a7cc72ba56d2aeff0fae8a60f9
3
+ size 1625
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_classification.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ # comment
2
+ # note: the next line contains a tab
3
+ 1.0 3:2.5 11:-5.2 16:1.5 # and an inline comment
4
+ 2.0 6:1.0 13:-3
5
+ # another comment
6
+ 3.0 21:27
7
+ 4.0 2:1.234567890123456e10 # double precision value
8
+ 1.0 # empty line, all zeros
9
+ 2.0 3:0 # explicit zeros
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ python 2:2.5 10:-5.2 15:1.5
2
+ 2.0 5:1.0 12:-3
3
+ 3.0 20:27
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid_order.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ -1 5:2.5 2:-5.2 15:1.5
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_multilabel.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # multilabel dataset in SVMlight format
2
+ 1,0 2:2.5 10:-5.2 15:1.5
3
+ 2 5:1.0 12:-3
4
+ 2:3.5 11:26
5
+ 1,2 20:27
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_20news.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test the 20news downloader, if the data is available,
2
+ or if specifically requested via environment variable
3
+ (e.g. for CI jobs)."""
4
+
5
+ from functools import partial
6
+ from unittest.mock import patch
7
+
8
+ import numpy as np
9
+ import pytest
10
+ import scipy.sparse as sp
11
+
12
+ from sklearn.datasets.tests.test_common import (
13
+ check_as_frame,
14
+ check_pandas_dependency_message,
15
+ check_return_X_y,
16
+ )
17
+ from sklearn.preprocessing import normalize
18
+ from sklearn.utils._testing import assert_allclose_dense_sparse
19
+
20
+
21
+ def test_20news(fetch_20newsgroups_fxt):
22
+ data = fetch_20newsgroups_fxt(subset="all", shuffle=False)
23
+ assert data.DESCR.startswith(".. _20newsgroups_dataset:")
24
+
25
+ # Extract a reduced dataset
26
+ data2cats = fetch_20newsgroups_fxt(
27
+ subset="all", categories=data.target_names[-1:-3:-1], shuffle=False
28
+ )
29
+ # Check that the ordering of the target_names is the same
30
+ # as the ordering in the full dataset
31
+ assert data2cats.target_names == data.target_names[-2:]
32
+ # Assert that we have only 0 and 1 as labels
33
+ assert np.unique(data2cats.target).tolist() == [0, 1]
34
+
35
+ # Check that the number of filenames is consistent with data/target
36
+ assert len(data2cats.filenames) == len(data2cats.target)
37
+ assert len(data2cats.filenames) == len(data2cats.data)
38
+
39
+ # Check that the first entry of the reduced dataset corresponds to
40
+ # the first entry of the corresponding category in the full dataset
41
+ entry1 = data2cats.data[0]
42
+ category = data2cats.target_names[data2cats.target[0]]
43
+ label = data.target_names.index(category)
44
+ entry2 = data.data[np.where(data.target == label)[0][0]]
45
+ assert entry1 == entry2
46
+
47
+ # check that return_X_y option
48
+ X, y = fetch_20newsgroups_fxt(subset="all", shuffle=False, return_X_y=True)
49
+ assert len(X) == len(data.data)
50
+ assert y.shape == data.target.shape
51
+
52
+
53
+ def test_20news_length_consistency(fetch_20newsgroups_fxt):
54
+ """Checks the length consistencies within the bunch
55
+
56
+ This is a non-regression test for a bug present in 0.16.1.
57
+ """
58
+ # Extract the full dataset
59
+ data = fetch_20newsgroups_fxt(subset="all")
60
+ assert len(data["data"]) == len(data.data)
61
+ assert len(data["target"]) == len(data.target)
62
+ assert len(data["filenames"]) == len(data.filenames)
63
+
64
+
65
+ def test_20news_vectorized(fetch_20newsgroups_vectorized_fxt):
66
+ # test subset = train
67
+ bunch = fetch_20newsgroups_vectorized_fxt(subset="train")
68
+ assert sp.issparse(bunch.data) and bunch.data.format == "csr"
69
+ assert bunch.data.shape == (11314, 130107)
70
+ assert bunch.target.shape[0] == 11314
71
+ assert bunch.data.dtype == np.float64
72
+ assert bunch.DESCR.startswith(".. _20newsgroups_dataset:")
73
+
74
+ # test subset = test
75
+ bunch = fetch_20newsgroups_vectorized_fxt(subset="test")
76
+ assert sp.issparse(bunch.data) and bunch.data.format == "csr"
77
+ assert bunch.data.shape == (7532, 130107)
78
+ assert bunch.target.shape[0] == 7532
79
+ assert bunch.data.dtype == np.float64
80
+ assert bunch.DESCR.startswith(".. _20newsgroups_dataset:")
81
+
82
+ # test return_X_y option
83
+ fetch_func = partial(fetch_20newsgroups_vectorized_fxt, subset="test")
84
+ check_return_X_y(bunch, fetch_func)
85
+
86
+ # test subset = all
87
+ bunch = fetch_20newsgroups_vectorized_fxt(subset="all")
88
+ assert sp.issparse(bunch.data) and bunch.data.format == "csr"
89
+ assert bunch.data.shape == (11314 + 7532, 130107)
90
+ assert bunch.target.shape[0] == 11314 + 7532
91
+ assert bunch.data.dtype == np.float64
92
+ assert bunch.DESCR.startswith(".. _20newsgroups_dataset:")
93
+
94
+
95
+ def test_20news_normalization(fetch_20newsgroups_vectorized_fxt):
96
+ X = fetch_20newsgroups_vectorized_fxt(normalize=False)
97
+ X_ = fetch_20newsgroups_vectorized_fxt(normalize=True)
98
+ X_norm = X_["data"][:100]
99
+ X = X["data"][:100]
100
+
101
+ assert_allclose_dense_sparse(X_norm, normalize(X))
102
+ assert np.allclose(np.linalg.norm(X_norm.todense(), axis=1), 1)
103
+
104
+
105
+ def test_20news_as_frame(fetch_20newsgroups_vectorized_fxt):
106
+ pd = pytest.importorskip("pandas")
107
+
108
+ bunch = fetch_20newsgroups_vectorized_fxt(as_frame=True)
109
+ check_as_frame(bunch, fetch_20newsgroups_vectorized_fxt)
110
+
111
+ frame = bunch.frame
112
+ assert frame.shape == (11314, 130108)
113
+ assert all([isinstance(col, pd.SparseDtype) for col in bunch.data.dtypes])
114
+
115
+ # Check a small subset of features
116
+ for expected_feature in [
117
+ "beginner",
118
+ "beginners",
119
+ "beginning",
120
+ "beginnings",
121
+ "begins",
122
+ "begley",
123
+ "begone",
124
+ ]:
125
+ assert expected_feature in frame.keys()
126
+ assert "category_class" in frame.keys()
127
+ assert bunch.target.name == "category_class"
128
+
129
+
130
+ def test_as_frame_no_pandas(fetch_20newsgroups_vectorized_fxt, hide_available_pandas):
131
+ check_pandas_dependency_message(fetch_20newsgroups_vectorized_fxt)
132
+
133
+
134
+ def test_outdated_pickle(fetch_20newsgroups_vectorized_fxt):
135
+ with patch("os.path.exists") as mock_is_exist:
136
+ with patch("joblib.load") as mock_load:
137
+ # mock that the dataset was cached
138
+ mock_is_exist.return_value = True
139
+ # mock that we have an outdated pickle with only X and y returned
140
+ mock_load.return_value = ("X", "y")
141
+ err_msg = "The cached dataset located in"
142
+ with pytest.raises(ValueError, match=err_msg):
143
+ fetch_20newsgroups_vectorized_fxt(as_frame=True)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_arff_parser.py ADDED
@@ -0,0 +1,284 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import textwrap
2
+ from io import BytesIO
3
+
4
+ import pytest
5
+
6
+ from sklearn.datasets._arff_parser import (
7
+ _liac_arff_parser,
8
+ _pandas_arff_parser,
9
+ _post_process_frame,
10
+ load_arff_from_gzip_file,
11
+ )
12
+
13
+
14
+ @pytest.mark.parametrize(
15
+ "feature_names, target_names",
16
+ [
17
+ (
18
+ [
19
+ "col_int_as_integer",
20
+ "col_int_as_numeric",
21
+ "col_float_as_real",
22
+ "col_float_as_numeric",
23
+ ],
24
+ ["col_categorical", "col_string"],
25
+ ),
26
+ (
27
+ [
28
+ "col_int_as_integer",
29
+ "col_int_as_numeric",
30
+ "col_float_as_real",
31
+ "col_float_as_numeric",
32
+ ],
33
+ ["col_categorical"],
34
+ ),
35
+ (
36
+ [
37
+ "col_int_as_integer",
38
+ "col_int_as_numeric",
39
+ "col_float_as_real",
40
+ "col_float_as_numeric",
41
+ ],
42
+ [],
43
+ ),
44
+ ],
45
+ )
46
+ def test_post_process_frame(feature_names, target_names):
47
+ """Check the behaviour of the post-processing function for splitting a dataframe."""
48
+ pd = pytest.importorskip("pandas")
49
+
50
+ X_original = pd.DataFrame(
51
+ {
52
+ "col_int_as_integer": [1, 2, 3],
53
+ "col_int_as_numeric": [1, 2, 3],
54
+ "col_float_as_real": [1.0, 2.0, 3.0],
55
+ "col_float_as_numeric": [1.0, 2.0, 3.0],
56
+ "col_categorical": ["a", "b", "c"],
57
+ "col_string": ["a", "b", "c"],
58
+ }
59
+ )
60
+
61
+ X, y = _post_process_frame(X_original, feature_names, target_names)
62
+ assert isinstance(X, pd.DataFrame)
63
+ if len(target_names) >= 2:
64
+ assert isinstance(y, pd.DataFrame)
65
+ elif len(target_names) == 1:
66
+ assert isinstance(y, pd.Series)
67
+ else:
68
+ assert y is None
69
+
70
+
71
+ def test_load_arff_from_gzip_file_error_parser():
72
+ """An error will be raised if the parser is not known."""
73
+ # None of the input parameters are required to be accurate since the check
74
+ # of the parser will be carried out first.
75
+
76
+ err_msg = "Unknown parser: 'xxx'. Should be 'liac-arff' or 'pandas'"
77
+ with pytest.raises(ValueError, match=err_msg):
78
+ load_arff_from_gzip_file("xxx", "xxx", "xxx", "xxx", "xxx", "xxx")
79
+
80
+
81
+ @pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser])
82
+ def test_pandas_arff_parser_strip_single_quotes(parser_func):
83
+ """Check that we properly strip single quotes from the data."""
84
+ pd = pytest.importorskip("pandas")
85
+
86
+ arff_file = BytesIO(
87
+ textwrap.dedent(
88
+ """
89
+ @relation 'toy'
90
+ @attribute 'cat_single_quote' {'A', 'B', 'C'}
91
+ @attribute 'str_single_quote' string
92
+ @attribute 'str_nested_quote' string
93
+ @attribute 'class' numeric
94
+ @data
95
+ 'A','some text','\"expect double quotes\"',0
96
+ """
97
+ ).encode("utf-8")
98
+ )
99
+
100
+ columns_info = {
101
+ "cat_single_quote": {
102
+ "data_type": "nominal",
103
+ "name": "cat_single_quote",
104
+ },
105
+ "str_single_quote": {
106
+ "data_type": "string",
107
+ "name": "str_single_quote",
108
+ },
109
+ "str_nested_quote": {
110
+ "data_type": "string",
111
+ "name": "str_nested_quote",
112
+ },
113
+ "class": {
114
+ "data_type": "numeric",
115
+ "name": "class",
116
+ },
117
+ }
118
+
119
+ feature_names = [
120
+ "cat_single_quote",
121
+ "str_single_quote",
122
+ "str_nested_quote",
123
+ ]
124
+ target_names = ["class"]
125
+
126
+ # We don't strip single quotes for string columns with the pandas parser.
127
+ expected_values = {
128
+ "cat_single_quote": "A",
129
+ "str_single_quote": (
130
+ "some text" if parser_func is _liac_arff_parser else "'some text'"
131
+ ),
132
+ "str_nested_quote": (
133
+ '"expect double quotes"'
134
+ if parser_func is _liac_arff_parser
135
+ else "'\"expect double quotes\"'"
136
+ ),
137
+ "class": 0,
138
+ }
139
+
140
+ _, _, frame, _ = parser_func(
141
+ arff_file,
142
+ output_arrays_type="pandas",
143
+ openml_columns_info=columns_info,
144
+ feature_names_to_select=feature_names,
145
+ target_names_to_select=target_names,
146
+ )
147
+
148
+ assert frame.columns.tolist() == feature_names + target_names
149
+ pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0))
150
+
151
+
152
+ @pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser])
153
+ def test_pandas_arff_parser_strip_double_quotes(parser_func):
154
+ """Check that we properly strip double quotes from the data."""
155
+ pd = pytest.importorskip("pandas")
156
+
157
+ arff_file = BytesIO(
158
+ textwrap.dedent(
159
+ """
160
+ @relation 'toy'
161
+ @attribute 'cat_double_quote' {"A", "B", "C"}
162
+ @attribute 'str_double_quote' string
163
+ @attribute 'str_nested_quote' string
164
+ @attribute 'class' numeric
165
+ @data
166
+ "A","some text","\'expect double quotes\'",0
167
+ """
168
+ ).encode("utf-8")
169
+ )
170
+
171
+ columns_info = {
172
+ "cat_double_quote": {
173
+ "data_type": "nominal",
174
+ "name": "cat_double_quote",
175
+ },
176
+ "str_double_quote": {
177
+ "data_type": "string",
178
+ "name": "str_double_quote",
179
+ },
180
+ "str_nested_quote": {
181
+ "data_type": "string",
182
+ "name": "str_nested_quote",
183
+ },
184
+ "class": {
185
+ "data_type": "numeric",
186
+ "name": "class",
187
+ },
188
+ }
189
+
190
+ feature_names = [
191
+ "cat_double_quote",
192
+ "str_double_quote",
193
+ "str_nested_quote",
194
+ ]
195
+ target_names = ["class"]
196
+
197
+ expected_values = {
198
+ "cat_double_quote": "A",
199
+ "str_double_quote": "some text",
200
+ "str_nested_quote": "'expect double quotes'",
201
+ "class": 0,
202
+ }
203
+
204
+ _, _, frame, _ = parser_func(
205
+ arff_file,
206
+ output_arrays_type="pandas",
207
+ openml_columns_info=columns_info,
208
+ feature_names_to_select=feature_names,
209
+ target_names_to_select=target_names,
210
+ )
211
+
212
+ assert frame.columns.tolist() == feature_names + target_names
213
+ pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0))
214
+
215
+
216
+ @pytest.mark.parametrize(
217
+ "parser_func",
218
+ [
219
+ # internal quotes are not considered to follow the ARFF spec in LIAC ARFF
220
+ pytest.param(_liac_arff_parser, marks=pytest.mark.xfail),
221
+ _pandas_arff_parser,
222
+ ],
223
+ )
224
+ def test_pandas_arff_parser_strip_no_quotes(parser_func):
225
+ """Check that we properly parse with no quotes characters."""
226
+ pd = pytest.importorskip("pandas")
227
+
228
+ arff_file = BytesIO(
229
+ textwrap.dedent(
230
+ """
231
+ @relation 'toy'
232
+ @attribute 'cat_without_quote' {A, B, C}
233
+ @attribute 'str_without_quote' string
234
+ @attribute 'str_internal_quote' string
235
+ @attribute 'class' numeric
236
+ @data
237
+ A,some text,'internal' quote,0
238
+ """
239
+ ).encode("utf-8")
240
+ )
241
+
242
+ columns_info = {
243
+ "cat_without_quote": {
244
+ "data_type": "nominal",
245
+ "name": "cat_without_quote",
246
+ },
247
+ "str_without_quote": {
248
+ "data_type": "string",
249
+ "name": "str_without_quote",
250
+ },
251
+ "str_internal_quote": {
252
+ "data_type": "string",
253
+ "name": "str_internal_quote",
254
+ },
255
+ "class": {
256
+ "data_type": "numeric",
257
+ "name": "class",
258
+ },
259
+ }
260
+
261
+ feature_names = [
262
+ "cat_without_quote",
263
+ "str_without_quote",
264
+ "str_internal_quote",
265
+ ]
266
+ target_names = ["class"]
267
+
268
+ expected_values = {
269
+ "cat_without_quote": "A",
270
+ "str_without_quote": "some text",
271
+ "str_internal_quote": "'internal' quote",
272
+ "class": 0,
273
+ }
274
+
275
+ _, _, frame, _ = parser_func(
276
+ arff_file,
277
+ output_arrays_type="pandas",
278
+ openml_columns_info=columns_info,
279
+ feature_names_to_select=feature_names,
280
+ target_names_to_select=target_names,
281
+ )
282
+
283
+ assert frame.columns.tolist() == feature_names + target_names
284
+ pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0))
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_base.py ADDED
@@ -0,0 +1,659 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import io
3
+ import os
4
+ import re
5
+ import shutil
6
+ import tempfile
7
+ import warnings
8
+ from functools import partial
9
+ from importlib import resources
10
+ from pathlib import Path
11
+ from pickle import dumps, loads
12
+ from unittest.mock import Mock
13
+ from urllib.error import HTTPError
14
+ from urllib.parse import urlparse
15
+
16
+ import numpy as np
17
+ import pytest
18
+
19
+ from sklearn.datasets import (
20
+ clear_data_home,
21
+ fetch_file,
22
+ get_data_home,
23
+ load_breast_cancer,
24
+ load_diabetes,
25
+ load_digits,
26
+ load_files,
27
+ load_iris,
28
+ load_linnerud,
29
+ load_sample_image,
30
+ load_sample_images,
31
+ load_wine,
32
+ )
33
+ from sklearn.datasets._base import (
34
+ RemoteFileMetadata,
35
+ _derive_folder_and_filename_from_url,
36
+ _fetch_remote,
37
+ load_csv_data,
38
+ load_gzip_compressed_csv_data,
39
+ )
40
+ from sklearn.datasets.tests.test_common import check_as_frame
41
+ from sklearn.preprocessing import scale
42
+ from sklearn.utils import Bunch
43
+
44
+
45
+ class _DummyPath:
46
+ """Minimal class that implements the os.PathLike interface."""
47
+
48
+ def __init__(self, path):
49
+ self.path = path
50
+
51
+ def __fspath__(self):
52
+ return self.path
53
+
54
+
55
+ def _remove_dir(path):
56
+ if os.path.isdir(path):
57
+ shutil.rmtree(path)
58
+
59
+
60
+ @pytest.fixture(scope="module")
61
+ def data_home(tmpdir_factory):
62
+ tmp_file = str(tmpdir_factory.mktemp("scikit_learn_data_home_test"))
63
+ yield tmp_file
64
+ _remove_dir(tmp_file)
65
+
66
+
67
+ @pytest.fixture(scope="module")
68
+ def load_files_root(tmpdir_factory):
69
+ tmp_file = str(tmpdir_factory.mktemp("scikit_learn_load_files_test"))
70
+ yield tmp_file
71
+ _remove_dir(tmp_file)
72
+
73
+
74
+ @pytest.fixture
75
+ def test_category_dir_1(load_files_root):
76
+ test_category_dir1 = tempfile.mkdtemp(dir=load_files_root)
77
+ sample_file = tempfile.NamedTemporaryFile(dir=test_category_dir1, delete=False)
78
+ sample_file.write(b"Hello World!\n")
79
+ sample_file.close()
80
+ yield str(test_category_dir1)
81
+ _remove_dir(test_category_dir1)
82
+
83
+
84
+ @pytest.fixture
85
+ def test_category_dir_2(load_files_root):
86
+ test_category_dir2 = tempfile.mkdtemp(dir=load_files_root)
87
+ yield str(test_category_dir2)
88
+ _remove_dir(test_category_dir2)
89
+
90
+
91
+ @pytest.mark.thread_unsafe
92
+ @pytest.mark.parametrize("path_container", [None, Path, _DummyPath])
93
+ def test_data_home(path_container, data_home):
94
+ # get_data_home will point to a pre-existing folder
95
+ if path_container is not None:
96
+ data_home = path_container(data_home)
97
+ data_home = get_data_home(data_home=data_home)
98
+ assert data_home == data_home
99
+ assert os.path.exists(data_home)
100
+
101
+ # clear_data_home will delete both the content and the folder it-self
102
+ if path_container is not None:
103
+ data_home = path_container(data_home)
104
+ clear_data_home(data_home=data_home)
105
+ assert not os.path.exists(data_home)
106
+
107
+ # if the folder is missing it will be created again
108
+ data_home = get_data_home(data_home=data_home)
109
+ assert os.path.exists(data_home)
110
+
111
+
112
+ def test_default_empty_load_files(load_files_root):
113
+ res = load_files(load_files_root)
114
+ assert len(res.filenames) == 0
115
+ assert len(res.target_names) == 0
116
+ assert res.DESCR is None
117
+
118
+
119
+ def test_default_load_files(test_category_dir_1, test_category_dir_2, load_files_root):
120
+ res = load_files(load_files_root)
121
+ assert len(res.filenames) == 1
122
+ assert len(res.target_names) == 2
123
+ assert res.DESCR is None
124
+ assert res.data == [b"Hello World!\n"]
125
+
126
+
127
+ def test_load_files_w_categories_desc_and_encoding(
128
+ test_category_dir_1, test_category_dir_2, load_files_root
129
+ ):
130
+ category = os.path.abspath(test_category_dir_1).split(os.sep).pop()
131
+ res = load_files(
132
+ load_files_root, description="test", categories=[category], encoding="utf-8"
133
+ )
134
+
135
+ assert len(res.filenames) == 1
136
+ assert len(res.target_names) == 1
137
+ assert res.DESCR == "test"
138
+ assert res.data == ["Hello World!\n"]
139
+
140
+
141
+ def test_load_files_wo_load_content(
142
+ test_category_dir_1, test_category_dir_2, load_files_root
143
+ ):
144
+ res = load_files(load_files_root, load_content=False)
145
+ assert len(res.filenames) == 1
146
+ assert len(res.target_names) == 2
147
+ assert res.DESCR is None
148
+ assert res.get("data") is None
149
+
150
+
151
+ @pytest.mark.parametrize("allowed_extensions", ([".txt"], [".txt", ".json"]))
152
+ def test_load_files_allowed_extensions(tmp_path, allowed_extensions):
153
+ """Check the behaviour of `allowed_extension` in `load_files`."""
154
+ d = tmp_path / "sub"
155
+ d.mkdir()
156
+ files = ("file1.txt", "file2.json", "file3.json", "file4.md")
157
+ paths = [d / f for f in files]
158
+ for p in paths:
159
+ p.write_bytes(b"hello")
160
+ res = load_files(tmp_path, allowed_extensions=allowed_extensions)
161
+ assert set([str(p) for p in paths if p.suffix in allowed_extensions]) == set(
162
+ res.filenames
163
+ )
164
+
165
+
166
+ @pytest.mark.parametrize(
167
+ "filename, expected_n_samples, expected_n_features, expected_target_names",
168
+ [
169
+ ("wine_data.csv", 178, 13, ["class_0", "class_1", "class_2"]),
170
+ ("iris.csv", 150, 4, ["setosa", "versicolor", "virginica"]),
171
+ ("breast_cancer.csv", 569, 30, ["malignant", "benign"]),
172
+ ],
173
+ )
174
+ def test_load_csv_data(
175
+ filename, expected_n_samples, expected_n_features, expected_target_names
176
+ ):
177
+ actual_data, actual_target, actual_target_names = load_csv_data(filename)
178
+ assert actual_data.shape[0] == expected_n_samples
179
+ assert actual_data.shape[1] == expected_n_features
180
+ assert actual_target.shape[0] == expected_n_samples
181
+ np.testing.assert_array_equal(actual_target_names, expected_target_names)
182
+
183
+
184
+ def test_load_csv_data_with_descr():
185
+ data_file_name = "iris.csv"
186
+ descr_file_name = "iris.rst"
187
+
188
+ res_without_descr = load_csv_data(data_file_name=data_file_name)
189
+ res_with_descr = load_csv_data(
190
+ data_file_name=data_file_name, descr_file_name=descr_file_name
191
+ )
192
+ assert len(res_with_descr) == 4
193
+ assert len(res_without_descr) == 3
194
+
195
+ np.testing.assert_array_equal(res_with_descr[0], res_without_descr[0])
196
+ np.testing.assert_array_equal(res_with_descr[1], res_without_descr[1])
197
+ np.testing.assert_array_equal(res_with_descr[2], res_without_descr[2])
198
+
199
+ assert res_with_descr[-1].startswith(".. _iris_dataset:")
200
+
201
+
202
+ @pytest.mark.parametrize(
203
+ "filename, kwargs, expected_shape",
204
+ [
205
+ ("diabetes_data_raw.csv.gz", {}, [442, 10]),
206
+ ("diabetes_target.csv.gz", {}, [442]),
207
+ ("digits.csv.gz", {"delimiter": ","}, [1797, 65]),
208
+ ],
209
+ )
210
+ def test_load_gzip_compressed_csv_data(filename, kwargs, expected_shape):
211
+ actual_data = load_gzip_compressed_csv_data(filename, **kwargs)
212
+ assert actual_data.shape == tuple(expected_shape)
213
+
214
+
215
+ def test_load_gzip_compressed_csv_data_with_descr():
216
+ data_file_name = "diabetes_target.csv.gz"
217
+ descr_file_name = "diabetes.rst"
218
+
219
+ expected_data = load_gzip_compressed_csv_data(data_file_name=data_file_name)
220
+ actual_data, descr = load_gzip_compressed_csv_data(
221
+ data_file_name=data_file_name,
222
+ descr_file_name=descr_file_name,
223
+ )
224
+
225
+ np.testing.assert_array_equal(actual_data, expected_data)
226
+ assert descr.startswith(".. _diabetes_dataset:")
227
+
228
+
229
+ def test_load_sample_images():
230
+ try:
231
+ res = load_sample_images()
232
+ assert len(res.images) == 2
233
+ assert len(res.filenames) == 2
234
+ images = res.images
235
+
236
+ # assert is china image
237
+ assert np.all(images[0][0, 0, :] == np.array([174, 201, 231], dtype=np.uint8))
238
+ # assert is flower image
239
+ assert np.all(images[1][0, 0, :] == np.array([2, 19, 13], dtype=np.uint8))
240
+ assert res.DESCR
241
+ except ImportError:
242
+ warnings.warn("Could not load sample images, PIL is not available.")
243
+
244
+
245
+ def test_load_sample_image():
246
+ try:
247
+ china = load_sample_image("china.jpg")
248
+ assert china.dtype == "uint8"
249
+ assert china.shape == (427, 640, 3)
250
+ except ImportError:
251
+ warnings.warn("Could not load sample images, PIL is not available.")
252
+
253
+
254
+ def test_load_diabetes_raw():
255
+ """Test to check that we load a scaled version by default but that we can
256
+ get an unscaled version when setting `scaled=False`."""
257
+ diabetes_raw = load_diabetes(scaled=False)
258
+ assert diabetes_raw.data.shape == (442, 10)
259
+ assert diabetes_raw.target.size == 442
260
+ assert len(diabetes_raw.feature_names) == 10
261
+ assert diabetes_raw.DESCR
262
+
263
+ diabetes_default = load_diabetes()
264
+
265
+ np.testing.assert_allclose(
266
+ scale(diabetes_raw.data) / (442**0.5), diabetes_default.data, atol=1e-04
267
+ )
268
+
269
+
270
+ @pytest.mark.parametrize(
271
+ "loader_func, data_shape, target_shape, n_target, has_descr, filenames",
272
+ [
273
+ (load_breast_cancer, (569, 30), (569,), 2, True, ["filename"]),
274
+ (load_wine, (178, 13), (178,), 3, True, []),
275
+ (load_iris, (150, 4), (150,), 3, True, ["filename"]),
276
+ (
277
+ load_linnerud,
278
+ (20, 3),
279
+ (20, 3),
280
+ 3,
281
+ True,
282
+ ["data_filename", "target_filename"],
283
+ ),
284
+ (load_diabetes, (442, 10), (442,), None, True, []),
285
+ (load_digits, (1797, 64), (1797,), 10, True, []),
286
+ (partial(load_digits, n_class=9), (1617, 64), (1617,), 10, True, []),
287
+ ],
288
+ )
289
+ def test_loader(loader_func, data_shape, target_shape, n_target, has_descr, filenames):
290
+ bunch = loader_func()
291
+
292
+ assert isinstance(bunch, Bunch)
293
+ assert bunch.data.shape == data_shape
294
+ assert bunch.target.shape == target_shape
295
+ if hasattr(bunch, "feature_names"):
296
+ assert len(bunch.feature_names) == data_shape[1]
297
+ if n_target is not None:
298
+ assert len(bunch.target_names) == n_target
299
+ if has_descr:
300
+ assert bunch.DESCR
301
+ if filenames:
302
+ assert "data_module" in bunch
303
+ assert all(
304
+ [
305
+ f in bunch
306
+ and (resources.files(bunch["data_module"]) / bunch[f]).is_file()
307
+ for f in filenames
308
+ ]
309
+ )
310
+
311
+
312
+ @pytest.mark.parametrize(
313
+ "loader_func, data_dtype, target_dtype",
314
+ [
315
+ (load_breast_cancer, np.float64, int),
316
+ (load_diabetes, np.float64, np.float64),
317
+ (load_digits, np.float64, int),
318
+ (load_iris, np.float64, int),
319
+ (load_linnerud, np.float64, np.float64),
320
+ (load_wine, np.float64, int),
321
+ ],
322
+ )
323
+ def test_toy_dataset_frame_dtype(loader_func, data_dtype, target_dtype):
324
+ default_result = loader_func()
325
+ check_as_frame(
326
+ default_result,
327
+ loader_func,
328
+ expected_data_dtype=data_dtype,
329
+ expected_target_dtype=target_dtype,
330
+ )
331
+
332
+
333
+ def test_loads_dumps_bunch():
334
+ bunch = Bunch(x="x")
335
+ bunch_from_pkl = loads(dumps(bunch))
336
+ bunch_from_pkl.x = "y"
337
+ assert bunch_from_pkl["x"] == bunch_from_pkl.x
338
+
339
+
340
+ def test_bunch_pickle_generated_with_0_16_and_read_with_0_17():
341
+ bunch = Bunch(key="original")
342
+ # This reproduces a problem when Bunch pickles have been created
343
+ # with scikit-learn 0.16 and are read with 0.17. Basically there
344
+ # is a surprising behaviour because reading bunch.key uses
345
+ # bunch.__dict__ (which is non empty for 0.16 Bunch objects)
346
+ # whereas assigning into bunch.key uses bunch.__setattr__. See
347
+ # https://github.com/scikit-learn/scikit-learn/issues/6196 for
348
+ # more details
349
+ bunch.__dict__["key"] = "set from __dict__"
350
+ bunch_from_pkl = loads(dumps(bunch))
351
+ # After loading from pickle the __dict__ should have been ignored
352
+ assert bunch_from_pkl.key == "original"
353
+ assert bunch_from_pkl["key"] == "original"
354
+ # Making sure that changing the attr does change the value
355
+ # associated with __getitem__ as well
356
+ bunch_from_pkl.key = "changed"
357
+ assert bunch_from_pkl.key == "changed"
358
+ assert bunch_from_pkl["key"] == "changed"
359
+
360
+
361
+ def test_bunch_dir():
362
+ # check that dir (important for autocomplete) shows attributes
363
+ data = load_iris()
364
+ assert "data" in dir(data)
365
+
366
+
367
+ def test_load_boston_error():
368
+ """Check that we raise the ethical warning when trying to import `load_boston`."""
369
+ msg = "The Boston housing prices dataset has an ethical problem"
370
+ with pytest.raises(ImportError, match=msg):
371
+ from sklearn.datasets import load_boston # noqa: F401
372
+
373
+ # other non-existing function should raise the usual import error
374
+ msg = "cannot import name 'non_existing_function' from 'sklearn.datasets'"
375
+ with pytest.raises(ImportError, match=msg):
376
+ from sklearn.datasets import non_existing_function # noqa: F401
377
+
378
+
379
+ def test_fetch_remote_raise_warnings_with_invalid_url(monkeypatch):
380
+ """Check retry mechanism in _fetch_remote."""
381
+
382
+ url = "https://scikit-learn.org/this_file_does_not_exist.tar.gz"
383
+ invalid_remote_file = RemoteFileMetadata("invalid_file", url, None)
384
+ urlretrieve_mock = Mock(
385
+ side_effect=HTTPError(
386
+ url=url, code=404, msg="Not Found", hdrs=None, fp=io.BytesIO()
387
+ )
388
+ )
389
+ monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
390
+
391
+ with pytest.warns(UserWarning, match="Retry downloading") as record:
392
+ with pytest.raises(HTTPError, match="HTTP Error 404"):
393
+ _fetch_remote(invalid_remote_file, n_retries=3, delay=0)
394
+
395
+ assert urlretrieve_mock.call_count == 4
396
+
397
+ for r in record:
398
+ assert str(r.message) == f"Retry downloading from url: {url}"
399
+ assert len(record) == 3
400
+
401
+
402
+ def test_derive_folder_and_filename_from_url():
403
+ folder, filename = _derive_folder_and_filename_from_url(
404
+ "https://example.com/file.tar.gz"
405
+ )
406
+ assert folder == "example.com"
407
+ assert filename == "file.tar.gz"
408
+
409
+ folder, filename = _derive_folder_and_filename_from_url(
410
+ "https://example.com/نمونه نماینده.data"
411
+ )
412
+ assert folder == "example.com"
413
+ assert filename == "نمونه-نماینده.data"
414
+
415
+ folder, filename = _derive_folder_and_filename_from_url(
416
+ "https://example.com/path/to-/.file.tar.gz"
417
+ )
418
+ assert folder == "example.com/path_to"
419
+ assert filename == "file.tar.gz"
420
+
421
+ folder, filename = _derive_folder_and_filename_from_url("https://example.com/")
422
+ assert folder == "example.com"
423
+ assert filename == "downloaded_file"
424
+
425
+ folder, filename = _derive_folder_and_filename_from_url("https://example.com")
426
+ assert folder == "example.com"
427
+ assert filename == "downloaded_file"
428
+
429
+ folder, filename = _derive_folder_and_filename_from_url(
430
+ "https://example.com/path/@to/data.json?param=value"
431
+ )
432
+ assert folder == "example.com/path_to"
433
+ assert filename == "data.json"
434
+
435
+ folder, filename = _derive_folder_and_filename_from_url(
436
+ "https://example.com/path/@@to._/-_.data.json.#anchor"
437
+ )
438
+ assert folder == "example.com/path_to"
439
+ assert filename == "data.json"
440
+
441
+ folder, filename = _derive_folder_and_filename_from_url(
442
+ "https://example.com//some_file.txt"
443
+ )
444
+ assert folder == "example.com"
445
+ assert filename == "some_file.txt"
446
+
447
+ folder, filename = _derive_folder_and_filename_from_url(
448
+ "http://example/../some_file.txt"
449
+ )
450
+ assert folder == "example"
451
+ assert filename == "some_file.txt"
452
+
453
+ folder, filename = _derive_folder_and_filename_from_url(
454
+ "https://example.com/!.'.,/some_file.txt"
455
+ )
456
+ assert folder == "example.com"
457
+ assert filename == "some_file.txt"
458
+
459
+ folder, filename = _derive_folder_and_filename_from_url(
460
+ "https://example.com/a/!.'.,/b/some_file.txt"
461
+ )
462
+ assert folder == "example.com/a_b"
463
+ assert filename == "some_file.txt"
464
+
465
+ folder, filename = _derive_folder_and_filename_from_url("https://example.com/!.'.,")
466
+ assert folder == "example.com"
467
+ assert filename == "downloaded_file"
468
+
469
+ with pytest.raises(ValueError, match="Invalid URL"):
470
+ _derive_folder_and_filename_from_url("https:/../")
471
+
472
+
473
+ def _mock_urlretrieve(server_side):
474
+ def _urlretrieve_mock(url, local_path):
475
+ server_root = Path(server_side)
476
+ file_path = urlparse(url).path.strip("/")
477
+ if not (server_root / file_path).exists():
478
+ raise HTTPError(url, 404, "Not Found", None, None)
479
+ shutil.copy(server_root / file_path, local_path)
480
+
481
+ return Mock(side_effect=_urlretrieve_mock)
482
+
483
+
484
+ def test_fetch_file_using_data_home(monkeypatch, tmpdir):
485
+ tmpdir = Path(tmpdir)
486
+ server_side = tmpdir / "server_side"
487
+ server_side.mkdir()
488
+ data_file = server_side / "data.jsonl"
489
+ server_data = '{"a": 1, "b": 2}\n'
490
+ data_file.write_text(server_data, encoding="utf-8")
491
+
492
+ server_subfolder = server_side / "subfolder"
493
+ server_subfolder.mkdir()
494
+ other_data_file = server_subfolder / "other_file.txt"
495
+ other_data_file.write_text("Some important text data.", encoding="utf-8")
496
+
497
+ data_home = tmpdir / "data_home"
498
+ data_home.mkdir()
499
+
500
+ urlretrieve_mock = _mock_urlretrieve(server_side)
501
+ monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
502
+
503
+ monkeypatch.setattr(
504
+ "sklearn.datasets._base.get_data_home", Mock(return_value=data_home)
505
+ )
506
+ fetched_file_path = fetch_file(
507
+ "https://example.com/data.jsonl",
508
+ )
509
+ assert fetched_file_path == data_home / "example.com" / "data.jsonl"
510
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
511
+
512
+ fetched_file_path = fetch_file(
513
+ "https://example.com/subfolder/other_file.txt",
514
+ )
515
+ assert (
516
+ fetched_file_path == data_home / "example.com" / "subfolder" / "other_file.txt"
517
+ )
518
+ assert fetched_file_path.read_text(encoding="utf-8") == other_data_file.read_text(
519
+ "utf-8"
520
+ )
521
+
522
+ expected_warning_msg = re.escape(
523
+ "Retry downloading from url: https://example.com/subfolder/invalid.txt"
524
+ )
525
+ with pytest.raises(HTTPError):
526
+ with pytest.warns(match=expected_warning_msg):
527
+ fetch_file(
528
+ "https://example.com/subfolder/invalid.txt",
529
+ delay=0,
530
+ )
531
+
532
+ local_subfolder = data_home / "example.com" / "subfolder"
533
+ assert sorted(local_subfolder.iterdir()) == [local_subfolder / "other_file.txt"]
534
+
535
+
536
+ def test_fetch_file_without_sha256(monkeypatch, tmpdir):
537
+ server_side = tmpdir.mkdir("server_side")
538
+ data_file = Path(server_side / "data.jsonl")
539
+ server_data = '{"a": 1, "b": 2}\n'
540
+ data_file.write_text(server_data, encoding="utf-8")
541
+
542
+ client_side = tmpdir.mkdir("client_side")
543
+
544
+ urlretrieve_mock = _mock_urlretrieve(server_side)
545
+ monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
546
+
547
+ # The first call should trigger a download:
548
+ fetched_file_path = fetch_file(
549
+ "https://example.com/data.jsonl",
550
+ folder=client_side,
551
+ )
552
+ assert fetched_file_path == client_side / "data.jsonl"
553
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
554
+ assert urlretrieve_mock.call_count == 1
555
+
556
+ # Fetching again the same file to the same folder should do nothing:
557
+ fetched_file_path = fetch_file(
558
+ "https://example.com/data.jsonl",
559
+ folder=client_side,
560
+ )
561
+ assert fetched_file_path == client_side / "data.jsonl"
562
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
563
+ assert urlretrieve_mock.call_count == 1
564
+
565
+ # Deleting and calling again should re-download
566
+ fetched_file_path.unlink()
567
+ fetched_file_path = fetch_file(
568
+ "https://example.com/data.jsonl",
569
+ folder=client_side,
570
+ )
571
+ assert fetched_file_path == client_side / "data.jsonl"
572
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
573
+ assert urlretrieve_mock.call_count == 2
574
+
575
+
576
+ def test_fetch_file_with_sha256(monkeypatch, tmpdir):
577
+ server_side = tmpdir.mkdir("server_side")
578
+ data_file = Path(server_side / "data.jsonl")
579
+ server_data = '{"a": 1, "b": 2}\n'
580
+ data_file.write_text(server_data, encoding="utf-8")
581
+ expected_sha256 = hashlib.sha256(data_file.read_bytes()).hexdigest()
582
+
583
+ client_side = tmpdir.mkdir("client_side")
584
+
585
+ urlretrieve_mock = _mock_urlretrieve(server_side)
586
+ monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
587
+
588
+ # The first call should trigger a download.
589
+ fetched_file_path = fetch_file(
590
+ "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
591
+ )
592
+ assert fetched_file_path == client_side / "data.jsonl"
593
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
594
+ assert urlretrieve_mock.call_count == 1
595
+
596
+ # Fetching again the same file to the same folder should do nothing when
597
+ # the sha256 match:
598
+ fetched_file_path = fetch_file(
599
+ "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
600
+ )
601
+ assert fetched_file_path == client_side / "data.jsonl"
602
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
603
+ assert urlretrieve_mock.call_count == 1
604
+
605
+ # Corrupting the local data should yield a warning and trigger a new download:
606
+ fetched_file_path.write_text("corrupted contents", encoding="utf-8")
607
+ expected_msg = (
608
+ r"SHA256 checksum of existing local file data.jsonl "
609
+ rf"\(.*\) differs from expected \({expected_sha256}\): "
610
+ r"re-downloading from https://example.com/data.jsonl \."
611
+ )
612
+ with pytest.warns(match=expected_msg):
613
+ fetched_file_path = fetch_file(
614
+ "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
615
+ )
616
+ assert fetched_file_path == client_side / "data.jsonl"
617
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
618
+ assert urlretrieve_mock.call_count == 2
619
+
620
+ # Calling again should do nothing:
621
+ fetched_file_path = fetch_file(
622
+ "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
623
+ )
624
+ assert fetched_file_path == client_side / "data.jsonl"
625
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
626
+ assert urlretrieve_mock.call_count == 2
627
+
628
+ # Deleting the local file and calling again should redownload without warning:
629
+ fetched_file_path.unlink()
630
+ fetched_file_path = fetch_file(
631
+ "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
632
+ )
633
+ assert fetched_file_path == client_side / "data.jsonl"
634
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
635
+ assert urlretrieve_mock.call_count == 3
636
+
637
+ # Calling without a sha256 should also work without redownloading:
638
+ fetched_file_path = fetch_file(
639
+ "https://example.com/data.jsonl",
640
+ folder=client_side,
641
+ )
642
+ assert fetched_file_path == client_side / "data.jsonl"
643
+ assert fetched_file_path.read_text(encoding="utf-8") == server_data
644
+ assert urlretrieve_mock.call_count == 3
645
+
646
+ # Calling with a wrong sha256 should raise an informative exception:
647
+ non_matching_sha256 = "deadbabecafebeef"
648
+ expected_warning_msg = "differs from expected"
649
+ expected_error_msg = re.escape(
650
+ f"The SHA256 checksum of data.jsonl ({expected_sha256}) differs from "
651
+ f"expected ({non_matching_sha256})."
652
+ )
653
+ with pytest.raises(OSError, match=expected_error_msg):
654
+ with pytest.warns(match=expected_warning_msg):
655
+ fetch_file(
656
+ "https://example.com/data.jsonl",
657
+ folder=client_side,
658
+ sha256=non_matching_sha256,
659
+ )
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_california_housing.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test the california_housing loader, if the data is available,
2
+ or if specifically requested via environment variable
3
+ (e.g. for CI jobs)."""
4
+
5
+ from functools import partial
6
+
7
+ import pytest
8
+
9
+ from sklearn.datasets.tests.test_common import check_return_X_y
10
+
11
+
12
+ def test_fetch(fetch_california_housing_fxt):
13
+ data = fetch_california_housing_fxt()
14
+ assert (20640, 8) == data.data.shape
15
+ assert (20640,) == data.target.shape
16
+ assert data.DESCR.startswith(".. _california_housing_dataset:")
17
+
18
+ # test return_X_y option
19
+ fetch_func = partial(fetch_california_housing_fxt)
20
+ check_return_X_y(data, fetch_func)
21
+
22
+
23
+ def test_fetch_asframe(fetch_california_housing_fxt):
24
+ pd = pytest.importorskip("pandas")
25
+ bunch = fetch_california_housing_fxt(as_frame=True)
26
+ frame = bunch.frame
27
+ assert hasattr(bunch, "frame") is True
28
+ assert frame.shape == (20640, 9)
29
+ assert isinstance(bunch.data, pd.DataFrame)
30
+ assert isinstance(bunch.target, pd.Series)
31
+
32
+
33
+ def test_pandas_dependency_message(fetch_california_housing_fxt, hide_available_pandas):
34
+ # Check that pandas is imported lazily and that an informative error
35
+ # message is raised when pandas is missing:
36
+ expected_msg = "fetch_california_housing with as_frame=True requires pandas"
37
+ with pytest.raises(ImportError, match=expected_msg):
38
+ fetch_california_housing_fxt(as_frame=True)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_common.py ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test loaders for common functionality."""
2
+
3
+ import inspect
4
+ import os
5
+
6
+ import numpy as np
7
+ import pytest
8
+
9
+ import sklearn.datasets
10
+
11
+
12
+ def is_pillow_installed():
13
+ try:
14
+ import PIL # noqa: F401
15
+
16
+ return True
17
+ except ImportError:
18
+ return False
19
+
20
+
21
+ FETCH_PYTEST_MARKERS = {
22
+ "return_X_y": {
23
+ "fetch_20newsgroups": pytest.mark.xfail(
24
+ reason="X is a list and does not have a shape argument"
25
+ ),
26
+ "fetch_openml": pytest.mark.xfail(
27
+ reason="fetch_opeml requires a dataset name or id"
28
+ ),
29
+ "fetch_lfw_people": pytest.mark.skipif(
30
+ not is_pillow_installed(), reason="pillow is not installed"
31
+ ),
32
+ },
33
+ "as_frame": {
34
+ "fetch_openml": pytest.mark.xfail(
35
+ reason="fetch_opeml requires a dataset name or id"
36
+ ),
37
+ },
38
+ }
39
+
40
+
41
+ def check_pandas_dependency_message(fetch_func):
42
+ try:
43
+ import pandas # noqa: F401
44
+
45
+ pytest.skip("This test requires pandas to not be installed")
46
+ except ImportError:
47
+ # Check that pandas is imported lazily and that an informative error
48
+ # message is raised when pandas is missing:
49
+ name = fetch_func.__name__
50
+ expected_msg = f"{name} with as_frame=True requires pandas"
51
+ with pytest.raises(ImportError, match=expected_msg):
52
+ fetch_func(as_frame=True)
53
+
54
+
55
+ def check_return_X_y(bunch, dataset_func):
56
+ X_y_tuple = dataset_func(return_X_y=True)
57
+ assert isinstance(X_y_tuple, tuple)
58
+ assert X_y_tuple[0].shape == bunch.data.shape
59
+ assert X_y_tuple[1].shape == bunch.target.shape
60
+
61
+
62
+ def check_as_frame(
63
+ bunch, dataset_func, expected_data_dtype=None, expected_target_dtype=None
64
+ ):
65
+ pd = pytest.importorskip("pandas")
66
+ frame_bunch = dataset_func(as_frame=True)
67
+ assert hasattr(frame_bunch, "frame")
68
+ assert isinstance(frame_bunch.frame, pd.DataFrame)
69
+ assert isinstance(frame_bunch.data, pd.DataFrame)
70
+ assert frame_bunch.data.shape == bunch.data.shape
71
+ if frame_bunch.target.ndim > 1:
72
+ assert isinstance(frame_bunch.target, pd.DataFrame)
73
+ else:
74
+ assert isinstance(frame_bunch.target, pd.Series)
75
+ assert frame_bunch.target.shape[0] == bunch.target.shape[0]
76
+ if expected_data_dtype is not None:
77
+ assert np.all(frame_bunch.data.dtypes == expected_data_dtype)
78
+ if expected_target_dtype is not None:
79
+ assert np.all(frame_bunch.target.dtypes == expected_target_dtype)
80
+
81
+ # Test for return_X_y and as_frame=True
82
+ frame_X, frame_y = dataset_func(as_frame=True, return_X_y=True)
83
+ assert isinstance(frame_X, pd.DataFrame)
84
+ if frame_y.ndim > 1:
85
+ assert isinstance(frame_y, pd.DataFrame)
86
+ else:
87
+ assert isinstance(frame_y, pd.Series)
88
+
89
+
90
+ def _skip_network_tests():
91
+ return os.environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "1"
92
+
93
+
94
+ def _generate_func_supporting_param(param, dataset_type=("load", "fetch")):
95
+ markers_fetch = FETCH_PYTEST_MARKERS.get(param, {})
96
+ for name, obj in inspect.getmembers(sklearn.datasets):
97
+ if not inspect.isfunction(obj):
98
+ continue
99
+
100
+ is_dataset_type = any([name.startswith(t) for t in dataset_type])
101
+ is_support_param = param in inspect.signature(obj).parameters
102
+ if is_dataset_type and is_support_param:
103
+ # check if we should skip if we don't have network support
104
+ marks = [
105
+ pytest.mark.skipif(
106
+ condition=name.startswith("fetch") and _skip_network_tests(),
107
+ reason="Skip because fetcher requires internet network",
108
+ )
109
+ ]
110
+ if name in markers_fetch:
111
+ marks.append(markers_fetch[name])
112
+
113
+ yield pytest.param(name, obj, marks=marks)
114
+
115
+
116
+ @pytest.mark.parametrize(
117
+ "name, dataset_func", _generate_func_supporting_param("return_X_y")
118
+ )
119
+ def test_common_check_return_X_y(name, dataset_func):
120
+ bunch = dataset_func()
121
+ check_return_X_y(bunch, dataset_func)
122
+
123
+
124
+ @pytest.mark.parametrize(
125
+ "name, dataset_func", _generate_func_supporting_param("as_frame")
126
+ )
127
+ def test_common_check_as_frame(name, dataset_func):
128
+ bunch = dataset_func()
129
+ check_as_frame(bunch, dataset_func)
130
+
131
+
132
+ @pytest.mark.parametrize(
133
+ "name, dataset_func", _generate_func_supporting_param("as_frame")
134
+ )
135
+ def test_common_check_pandas_dependency(name, dataset_func):
136
+ check_pandas_dependency_message(dataset_func)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_covtype.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test the covtype loader, if the data is available,
2
+ or if specifically requested via environment variable
3
+ (e.g. for CI jobs)."""
4
+
5
+ from functools import partial
6
+
7
+ import pytest
8
+
9
+ from sklearn.datasets.tests.test_common import check_return_X_y
10
+
11
+
12
+ def test_fetch(fetch_covtype_fxt, global_random_seed):
13
+ data1 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed)
14
+ data2 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed + 1)
15
+
16
+ X1, X2 = data1["data"], data2["data"]
17
+ assert (581012, 54) == X1.shape
18
+ assert X1.shape == X2.shape
19
+
20
+ assert X1.sum() == X2.sum()
21
+
22
+ y1, y2 = data1["target"], data2["target"]
23
+ assert (X1.shape[0],) == y1.shape
24
+ assert (X1.shape[0],) == y2.shape
25
+
26
+ descr_prefix = ".. _covtype_dataset:"
27
+ assert data1.DESCR.startswith(descr_prefix)
28
+ assert data2.DESCR.startswith(descr_prefix)
29
+
30
+ # test return_X_y option
31
+ fetch_func = partial(fetch_covtype_fxt)
32
+ check_return_X_y(data1, fetch_func)
33
+
34
+
35
+ def test_fetch_asframe(fetch_covtype_fxt):
36
+ pytest.importorskip("pandas")
37
+
38
+ bunch = fetch_covtype_fxt(as_frame=True)
39
+ assert hasattr(bunch, "frame")
40
+ frame = bunch.frame
41
+ assert frame.shape == (581012, 55)
42
+ assert bunch.data.shape == (581012, 54)
43
+ assert bunch.target.shape == (581012,)
44
+
45
+ column_names = set(frame.columns)
46
+
47
+ # enumerated names are added correctly
48
+ assert set(f"Wilderness_Area_{i}" for i in range(4)) < column_names
49
+ assert set(f"Soil_Type_{i}" for i in range(40)) < column_names
50
+
51
+
52
+ def test_pandas_dependency_message(fetch_covtype_fxt, hide_available_pandas):
53
+ expected_msg = "fetch_covtype with as_frame=True requires pandas"
54
+ with pytest.raises(ImportError, match=expected_msg):
55
+ fetch_covtype_fxt(as_frame=True)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_kddcup99.py ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test kddcup99 loader, if the data is available,
2
+ or if specifically requested via environment variable
3
+ (e.g. for CI jobs).
4
+
5
+ Only 'percent10' mode is tested, as the full data
6
+ is too big to use in unit-testing.
7
+ """
8
+
9
+ from functools import partial
10
+
11
+ import pytest
12
+
13
+ from sklearn.datasets.tests.test_common import (
14
+ check_as_frame,
15
+ check_pandas_dependency_message,
16
+ check_return_X_y,
17
+ )
18
+
19
+
20
+ @pytest.mark.parametrize("as_frame", [True, False])
21
+ @pytest.mark.parametrize(
22
+ "subset, n_samples, n_features",
23
+ [
24
+ (None, 494021, 41),
25
+ ("SA", 100655, 41),
26
+ ("SF", 73237, 4),
27
+ ("http", 58725, 3),
28
+ ("smtp", 9571, 3),
29
+ ],
30
+ )
31
+ def test_fetch_kddcup99_percent10(
32
+ fetch_kddcup99_fxt, as_frame, subset, n_samples, n_features
33
+ ):
34
+ data = fetch_kddcup99_fxt(subset=subset, as_frame=as_frame)
35
+ assert data.data.shape == (n_samples, n_features)
36
+ assert data.target.shape == (n_samples,)
37
+ if as_frame:
38
+ assert data.frame.shape == (n_samples, n_features + 1)
39
+ assert data.DESCR.startswith(".. _kddcup99_dataset:")
40
+
41
+
42
+ def test_fetch_kddcup99_return_X_y(fetch_kddcup99_fxt):
43
+ fetch_func = partial(fetch_kddcup99_fxt, subset="smtp")
44
+ data = fetch_func()
45
+ check_return_X_y(data, fetch_func)
46
+
47
+
48
+ def test_fetch_kddcup99_as_frame(fetch_kddcup99_fxt):
49
+ bunch = fetch_kddcup99_fxt()
50
+ check_as_frame(bunch, fetch_kddcup99_fxt)
51
+
52
+
53
+ def test_fetch_kddcup99_shuffle(fetch_kddcup99_fxt):
54
+ dataset = fetch_kddcup99_fxt(
55
+ random_state=0,
56
+ subset="SA",
57
+ percent10=True,
58
+ )
59
+ dataset_shuffled = fetch_kddcup99_fxt(
60
+ random_state=0,
61
+ subset="SA",
62
+ shuffle=True,
63
+ percent10=True,
64
+ )
65
+ assert set(dataset["target"]) == set(dataset_shuffled["target"])
66
+ assert dataset_shuffled.data.shape == dataset.data.shape
67
+ assert dataset_shuffled.target.shape == dataset.target.shape
68
+
69
+
70
+ def test_pandas_dependency_message(fetch_kddcup99_fxt, hide_available_pandas):
71
+ check_pandas_dependency_message(fetch_kddcup99_fxt)
72
+
73
+
74
+ def test_corrupted_file_error_message(fetch_kddcup99_fxt, tmp_path):
75
+ """Check that a nice error message is raised when cache is corrupted."""
76
+ kddcup99_dir = tmp_path / "kddcup99_10-py3"
77
+ kddcup99_dir.mkdir()
78
+ samples_path = kddcup99_dir / "samples"
79
+
80
+ with samples_path.open("wb") as f:
81
+ f.write(b"THIS IS CORRUPTED")
82
+
83
+ msg = (
84
+ "The cache for fetch_kddcup99 is invalid, please "
85
+ f"delete {kddcup99_dir} and run the fetch_kddcup99 again"
86
+ )
87
+
88
+ with pytest.raises(OSError, match=msg):
89
+ fetch_kddcup99_fxt(data_home=str(tmp_path))
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_lfw.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """This test for the LFW require medium-size data downloading and processing
2
+
3
+ If the data has not been already downloaded by running the examples,
4
+ the tests won't run (skipped).
5
+
6
+ If the test are run, the first execution will be long (typically a bit
7
+ more than a couple of minutes) but as the dataset loader is leveraging
8
+ joblib, successive runs will be fast (less than 200ms).
9
+ """
10
+
11
+ import random
12
+ from functools import partial
13
+
14
+ import numpy as np
15
+ import pytest
16
+
17
+ from sklearn.datasets import fetch_lfw_pairs, fetch_lfw_people
18
+ from sklearn.datasets.tests.test_common import check_return_X_y
19
+ from sklearn.utils._testing import assert_array_equal
20
+
21
+ FAKE_NAMES = [
22
+ "Abdelatif_Smith",
23
+ "Abhati_Kepler",
24
+ "Camara_Alvaro",
25
+ "Chen_Dupont",
26
+ "John_Lee",
27
+ "Lin_Bauman",
28
+ "Onur_Lopez",
29
+ ]
30
+
31
+
32
+ @pytest.fixture(scope="module")
33
+ def mock_empty_data_home(tmp_path_factory):
34
+ data_dir = tmp_path_factory.mktemp("scikit_learn_empty_test")
35
+
36
+ yield data_dir
37
+
38
+
39
+ @pytest.fixture(scope="module")
40
+ def mock_data_home(tmp_path_factory):
41
+ """Test fixture run once and common to all tests of this module"""
42
+ Image = pytest.importorskip("PIL.Image")
43
+
44
+ data_dir = tmp_path_factory.mktemp("scikit_learn_lfw_test")
45
+ lfw_home = data_dir / "lfw_home"
46
+ lfw_home.mkdir(parents=True, exist_ok=True)
47
+
48
+ random_state = random.Random(42)
49
+ np_rng = np.random.RandomState(42)
50
+
51
+ # generate some random jpeg files for each person
52
+ counts = {}
53
+ for name in FAKE_NAMES:
54
+ folder_name = lfw_home / "lfw_funneled" / name
55
+ folder_name.mkdir(parents=True, exist_ok=True)
56
+
57
+ n_faces = np_rng.randint(1, 5)
58
+ counts[name] = n_faces
59
+ for i in range(n_faces):
60
+ file_path = folder_name / (name + "_%04d.jpg" % i)
61
+ uniface = np_rng.randint(0, 255, size=(250, 250, 3))
62
+ img = Image.fromarray(uniface.astype(np.uint8))
63
+ img.save(file_path)
64
+
65
+ # add some random file pollution to test robustness
66
+ (lfw_home / "lfw_funneled" / ".test.swp").write_bytes(
67
+ b"Text file to be ignored by the dataset loader."
68
+ )
69
+
70
+ # generate some pairing metadata files using the same format as LFW
71
+ with open(lfw_home / "pairsDevTrain.txt", "wb") as f:
72
+ f.write(b"10\n")
73
+ more_than_two = [name for name, count in counts.items() if count >= 2]
74
+ for i in range(5):
75
+ name = random_state.choice(more_than_two)
76
+ first, second = random_state.sample(range(counts[name]), 2)
77
+ f.write(("%s\t%d\t%d\n" % (name, first, second)).encode())
78
+
79
+ for i in range(5):
80
+ first_name, second_name = random_state.sample(FAKE_NAMES, 2)
81
+ first_index = np_rng.choice(np.arange(counts[first_name]))
82
+ second_index = np_rng.choice(np.arange(counts[second_name]))
83
+ f.write(
84
+ (
85
+ "%s\t%d\t%s\t%d\n"
86
+ % (first_name, first_index, second_name, second_index)
87
+ ).encode()
88
+ )
89
+
90
+ (lfw_home / "pairsDevTest.txt").write_bytes(
91
+ b"Fake place holder that won't be tested"
92
+ )
93
+ (lfw_home / "pairs.txt").write_bytes(b"Fake place holder that won't be tested")
94
+
95
+ yield data_dir
96
+
97
+
98
+ def test_load_empty_lfw_people(mock_empty_data_home):
99
+ with pytest.raises(OSError):
100
+ fetch_lfw_people(data_home=mock_empty_data_home, download_if_missing=False)
101
+
102
+
103
+ def test_load_fake_lfw_people(mock_data_home):
104
+ lfw_people = fetch_lfw_people(
105
+ data_home=mock_data_home, min_faces_per_person=3, download_if_missing=False
106
+ )
107
+
108
+ # The data is croped around the center as a rectangular bounding box
109
+ # around the face. Colors are converted to gray levels:
110
+ assert lfw_people.images.shape == (10, 62, 47)
111
+ assert lfw_people.data.shape == (10, 2914)
112
+
113
+ # the target is array of person integer ids
114
+ assert_array_equal(lfw_people.target, [2, 0, 1, 0, 2, 0, 2, 1, 1, 2])
115
+
116
+ # names of the persons can be found using the target_names array
117
+ expected_classes = ["Abdelatif Smith", "Abhati Kepler", "Onur Lopez"]
118
+ assert_array_equal(lfw_people.target_names, expected_classes)
119
+
120
+ # It is possible to ask for the original data without any croping or color
121
+ # conversion and not limit on the number of picture per person
122
+ lfw_people = fetch_lfw_people(
123
+ data_home=mock_data_home,
124
+ resize=None,
125
+ slice_=None,
126
+ color=True,
127
+ download_if_missing=False,
128
+ )
129
+ assert lfw_people.images.shape == (17, 250, 250, 3)
130
+ assert lfw_people.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:")
131
+
132
+ # the ids and class names are the same as previously
133
+ assert_array_equal(
134
+ lfw_people.target, [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2]
135
+ )
136
+ assert_array_equal(
137
+ lfw_people.target_names,
138
+ [
139
+ "Abdelatif Smith",
140
+ "Abhati Kepler",
141
+ "Camara Alvaro",
142
+ "Chen Dupont",
143
+ "John Lee",
144
+ "Lin Bauman",
145
+ "Onur Lopez",
146
+ ],
147
+ )
148
+
149
+ # test return_X_y option
150
+ fetch_func = partial(
151
+ fetch_lfw_people,
152
+ data_home=mock_data_home,
153
+ resize=None,
154
+ slice_=None,
155
+ color=True,
156
+ download_if_missing=False,
157
+ )
158
+ check_return_X_y(lfw_people, fetch_func)
159
+
160
+
161
+ def test_load_fake_lfw_people_too_restrictive(mock_data_home):
162
+ with pytest.raises(ValueError):
163
+ fetch_lfw_people(
164
+ data_home=mock_data_home,
165
+ min_faces_per_person=100,
166
+ download_if_missing=False,
167
+ )
168
+
169
+
170
+ def test_load_empty_lfw_pairs(mock_empty_data_home):
171
+ with pytest.raises(OSError):
172
+ fetch_lfw_pairs(data_home=mock_empty_data_home, download_if_missing=False)
173
+
174
+
175
+ def test_load_fake_lfw_pairs(mock_data_home):
176
+ lfw_pairs_train = fetch_lfw_pairs(
177
+ data_home=mock_data_home, download_if_missing=False
178
+ )
179
+
180
+ # The data is croped around the center as a rectangular bounding box
181
+ # around the face. Colors are converted to gray levels:
182
+ assert lfw_pairs_train.pairs.shape == (10, 2, 62, 47)
183
+
184
+ # the target is whether the person is the same or not
185
+ assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
186
+
187
+ # names of the persons can be found using the target_names array
188
+ expected_classes = ["Different persons", "Same person"]
189
+ assert_array_equal(lfw_pairs_train.target_names, expected_classes)
190
+
191
+ # It is possible to ask for the original data without any croping or color
192
+ # conversion
193
+ lfw_pairs_train = fetch_lfw_pairs(
194
+ data_home=mock_data_home,
195
+ resize=None,
196
+ slice_=None,
197
+ color=True,
198
+ download_if_missing=False,
199
+ )
200
+ assert lfw_pairs_train.pairs.shape == (10, 2, 250, 250, 3)
201
+
202
+ # the ids and class names are the same as previously
203
+ assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
204
+ assert_array_equal(lfw_pairs_train.target_names, expected_classes)
205
+
206
+ assert lfw_pairs_train.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:")
207
+
208
+
209
+ def test_fetch_lfw_people_internal_cropping(mock_data_home):
210
+ """Check that we properly crop the images.
211
+
212
+ Non-regression test for:
213
+ https://github.com/scikit-learn/scikit-learn/issues/24942
214
+ """
215
+ # If cropping was not done properly and we don't resize the images, the images would
216
+ # have their original size (250x250) and the image would not fit in the NumPy array
217
+ # pre-allocated based on `slice_` parameter.
218
+ slice_ = (slice(70, 195), slice(78, 172))
219
+ lfw = fetch_lfw_people(
220
+ data_home=mock_data_home,
221
+ min_faces_per_person=3,
222
+ download_if_missing=False,
223
+ resize=None,
224
+ slice_=slice_,
225
+ )
226
+ assert lfw.images[0].shape == (
227
+ slice_[0].stop - slice_[0].start,
228
+ slice_[1].stop - slice_[1].start,
229
+ )
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_olivetti_faces.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test Olivetti faces fetcher, if the data is available,
2
+ or if specifically requested via environment variable
3
+ (e.g. for CI jobs)."""
4
+
5
+ import numpy as np
6
+
7
+ from sklearn.datasets.tests.test_common import check_return_X_y
8
+ from sklearn.utils import Bunch
9
+ from sklearn.utils._testing import assert_array_equal
10
+
11
+
12
+ def test_olivetti_faces(fetch_olivetti_faces_fxt):
13
+ data = fetch_olivetti_faces_fxt(shuffle=True, random_state=0)
14
+
15
+ assert isinstance(data, Bunch)
16
+ for expected_keys in ("data", "images", "target", "DESCR"):
17
+ assert expected_keys in data.keys()
18
+
19
+ assert data.data.shape == (400, 4096)
20
+ assert data.images.shape == (400, 64, 64)
21
+ assert data.target.shape == (400,)
22
+ assert_array_equal(np.unique(np.sort(data.target)), np.arange(40))
23
+ assert data.DESCR.startswith(".. _olivetti_faces_dataset:")
24
+
25
+ # test the return_X_y option
26
+ check_return_X_y(data, fetch_olivetti_faces_fxt)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_openml.py ADDED
@@ -0,0 +1,1636 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test the openml loader."""
2
+
3
+ import gzip
4
+ import json
5
+ import os
6
+ import re
7
+ from functools import partial
8
+ from importlib import resources
9
+ from io import BytesIO
10
+ from urllib.error import HTTPError
11
+
12
+ import numpy as np
13
+ import pytest
14
+ import scipy.sparse
15
+
16
+ import sklearn
17
+ from sklearn import config_context
18
+ from sklearn.datasets import fetch_openml as fetch_openml_orig
19
+ from sklearn.datasets._openml import (
20
+ _get_local_path,
21
+ _open_openml_url,
22
+ _retry_with_clean_cache,
23
+ )
24
+ from sklearn.utils import Bunch
25
+ from sklearn.utils._optional_dependencies import check_pandas_support
26
+ from sklearn.utils._testing import (
27
+ SkipTest,
28
+ assert_allclose,
29
+ assert_array_equal,
30
+ )
31
+
32
+ OPENML_TEST_DATA_MODULE = "sklearn.datasets.tests.data.openml"
33
+ # if True, urlopen will be monkey patched to only use local files
34
+ test_offline = True
35
+ _MONKEY_PATCH_LOCAL_OPENML_PATH = "data/v1/download/{}"
36
+
37
+
38
+ class _MockHTTPResponse:
39
+ def __init__(self, data, is_gzip):
40
+ self.data = data
41
+ self.is_gzip = is_gzip
42
+
43
+ def read(self, amt=-1):
44
+ return self.data.read(amt)
45
+
46
+ def close(self):
47
+ self.data.close()
48
+
49
+ def info(self):
50
+ if self.is_gzip:
51
+ return {"Content-Encoding": "gzip"}
52
+ return {}
53
+
54
+ def __iter__(self):
55
+ return iter(self.data)
56
+
57
+ def __enter__(self):
58
+ return self
59
+
60
+ def __exit__(self, exc_type, exc_val, exc_tb):
61
+ return False
62
+
63
+
64
+ # Disable the disk-based cache when testing `fetch_openml`:
65
+ # the mock data in sklearn/datasets/tests/data/openml/ is not always consistent
66
+ # with the version on openml.org. If one were to load the dataset outside of
67
+ # the tests, it may result in data that does not represent openml.org.
68
+ fetch_openml = partial(fetch_openml_orig, data_home=None)
69
+
70
+
71
+ def _monkey_patch_webbased_functions(context, data_id, gzip_response):
72
+ # monkey patches the urlopen function. Important note: Do NOT use this
73
+ # in combination with a regular cache directory, as the files that are
74
+ # stored as cache should not be mixed up with real openml datasets
75
+ url_prefix_data_description = "https://api.openml.org/api/v1/json/data/"
76
+ url_prefix_data_features = "https://api.openml.org/api/v1/json/data/features/"
77
+ url_prefix_download_data = "https://www.openml.org/data/v1/download"
78
+ url_prefix_data_list = "https://api.openml.org/api/v1/json/data/list/"
79
+
80
+ path_suffix = ".gz"
81
+ read_fn = gzip.open
82
+
83
+ data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}"
84
+
85
+ def _file_name(url, suffix):
86
+ output = (
87
+ re.sub(r"\W", "-", url[len("https://api.openml.org/") :])
88
+ + suffix
89
+ + path_suffix
90
+ )
91
+ # Shorten the filenames to have better compatibility with windows 10
92
+ # and filenames > 260 characters
93
+ return (
94
+ output.replace("-json-data-list", "-jdl")
95
+ .replace("-json-data-features", "-jdf")
96
+ .replace("-json-data-qualities", "-jdq")
97
+ .replace("-json-data", "-jd")
98
+ .replace("-data_name", "-dn")
99
+ .replace("-download", "-dl")
100
+ .replace("-limit", "-l")
101
+ .replace("-data_version", "-dv")
102
+ .replace("-status", "-s")
103
+ .replace("-deactivated", "-dact")
104
+ .replace("-active", "-act")
105
+ )
106
+
107
+ def _mock_urlopen_shared(url, has_gzip_header, expected_prefix, suffix):
108
+ assert url.startswith(expected_prefix), (
109
+ f"{expected_prefix!r} does not match {url!r}"
110
+ )
111
+
112
+ data_file_name = _file_name(url, suffix)
113
+ data_file_path = resources.files(data_module) / data_file_name
114
+
115
+ with data_file_path.open("rb") as f:
116
+ if has_gzip_header and gzip_response:
117
+ fp = BytesIO(f.read())
118
+ return _MockHTTPResponse(fp, True)
119
+ else:
120
+ decompressed_f = read_fn(f, "rb")
121
+ fp = BytesIO(decompressed_f.read())
122
+ return _MockHTTPResponse(fp, False)
123
+
124
+ def _mock_urlopen_data_description(url, has_gzip_header):
125
+ return _mock_urlopen_shared(
126
+ url=url,
127
+ has_gzip_header=has_gzip_header,
128
+ expected_prefix=url_prefix_data_description,
129
+ suffix=".json",
130
+ )
131
+
132
+ def _mock_urlopen_data_features(url, has_gzip_header):
133
+ return _mock_urlopen_shared(
134
+ url=url,
135
+ has_gzip_header=has_gzip_header,
136
+ expected_prefix=url_prefix_data_features,
137
+ suffix=".json",
138
+ )
139
+
140
+ def _mock_urlopen_download_data(url, has_gzip_header):
141
+ # For simplicity the mock filenames don't contain the filename, i.e.
142
+ # the last part of the data description url after the last /.
143
+ # For example for id_1, data description download url is:
144
+ # gunzip -c sklearn/datasets/tests/data/openml/id_1/api-v1-jd-1.json.gz | grep '"url" # noqa: E501
145
+ # "https:\/\/www.openml.org\/data\/v1\/download\/1\/anneal.arff"
146
+ # but the mock filename does not contain anneal.arff and is:
147
+ # sklearn/datasets/tests/data/openml/id_1/data-v1-dl-1.arff.gz.
148
+ # We only keep the part of the url before the last /
149
+ url_without_filename = url.rsplit("/", 1)[0]
150
+
151
+ return _mock_urlopen_shared(
152
+ url=url_without_filename,
153
+ has_gzip_header=has_gzip_header,
154
+ expected_prefix=url_prefix_download_data,
155
+ suffix=".arff",
156
+ )
157
+
158
+ def _mock_urlopen_data_list(url, has_gzip_header):
159
+ assert url.startswith(url_prefix_data_list), (
160
+ f"{url_prefix_data_list!r} does not match {url!r}"
161
+ )
162
+
163
+ data_file_name = _file_name(url, ".json")
164
+ data_file_path = resources.files(data_module) / data_file_name
165
+
166
+ # load the file itself, to simulate a http error
167
+ with data_file_path.open("rb") as f:
168
+ decompressed_f = read_fn(f, "rb")
169
+ decoded_s = decompressed_f.read().decode("utf-8")
170
+ json_data = json.loads(decoded_s)
171
+ if "error" in json_data:
172
+ raise HTTPError(
173
+ url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO()
174
+ )
175
+
176
+ with data_file_path.open("rb") as f:
177
+ if has_gzip_header:
178
+ fp = BytesIO(f.read())
179
+ return _MockHTTPResponse(fp, True)
180
+ else:
181
+ decompressed_f = read_fn(f, "rb")
182
+ fp = BytesIO(decompressed_f.read())
183
+ return _MockHTTPResponse(fp, False)
184
+
185
+ def _mock_urlopen(request, *args, **kwargs):
186
+ url = request.get_full_url()
187
+ has_gzip_header = request.get_header("Accept-encoding") == "gzip"
188
+ if url.startswith(url_prefix_data_list):
189
+ return _mock_urlopen_data_list(url, has_gzip_header)
190
+ elif url.startswith(url_prefix_data_features):
191
+ return _mock_urlopen_data_features(url, has_gzip_header)
192
+ elif url.startswith(url_prefix_download_data):
193
+ return _mock_urlopen_download_data(url, has_gzip_header)
194
+ elif url.startswith(url_prefix_data_description):
195
+ return _mock_urlopen_data_description(url, has_gzip_header)
196
+ else:
197
+ raise ValueError("Unknown mocking URL pattern: %s" % url)
198
+
199
+ # XXX: Global variable
200
+ if test_offline:
201
+ context.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen)
202
+
203
+
204
+ ###############################################################################
205
+ # Test the behaviour of `fetch_openml` depending of the input parameters.
206
+
207
+
208
+ @pytest.mark.parametrize(
209
+ "data_id, dataset_params, n_samples, n_features, n_targets",
210
+ [
211
+ # iris
212
+ (61, {"data_id": 61}, 150, 4, 1),
213
+ (61, {"name": "iris", "version": 1}, 150, 4, 1),
214
+ # anneal
215
+ (2, {"data_id": 2}, 11, 38, 1),
216
+ (2, {"name": "anneal", "version": 1}, 11, 38, 1),
217
+ # cpu
218
+ (561, {"data_id": 561}, 209, 7, 1),
219
+ (561, {"name": "cpu", "version": 1}, 209, 7, 1),
220
+ # emotions
221
+ (40589, {"data_id": 40589}, 13, 72, 6),
222
+ # adult-census
223
+ (1119, {"data_id": 1119}, 10, 14, 1),
224
+ (1119, {"name": "adult-census"}, 10, 14, 1),
225
+ # miceprotein
226
+ (40966, {"data_id": 40966}, 7, 77, 1),
227
+ (40966, {"name": "MiceProtein"}, 7, 77, 1),
228
+ # titanic
229
+ (40945, {"data_id": 40945}, 1309, 13, 1),
230
+ ],
231
+ )
232
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
233
+ @pytest.mark.parametrize("gzip_response", [True, False])
234
+ def test_fetch_openml_as_frame_true(
235
+ monkeypatch,
236
+ data_id,
237
+ dataset_params,
238
+ n_samples,
239
+ n_features,
240
+ n_targets,
241
+ parser,
242
+ gzip_response,
243
+ ):
244
+ """Check the behaviour of `fetch_openml` with `as_frame=True`.
245
+
246
+ Fetch by ID and/or name (depending if the file was previously cached).
247
+ """
248
+ pd = pytest.importorskip("pandas")
249
+
250
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response)
251
+ bunch = fetch_openml(
252
+ as_frame=True,
253
+ cache=False,
254
+ parser=parser,
255
+ **dataset_params,
256
+ )
257
+
258
+ assert int(bunch.details["id"]) == data_id
259
+ assert isinstance(bunch, Bunch)
260
+
261
+ assert isinstance(bunch.frame, pd.DataFrame)
262
+ assert bunch.frame.shape == (n_samples, n_features + n_targets)
263
+
264
+ assert isinstance(bunch.data, pd.DataFrame)
265
+ assert bunch.data.shape == (n_samples, n_features)
266
+
267
+ if n_targets == 1:
268
+ assert isinstance(bunch.target, pd.Series)
269
+ assert bunch.target.shape == (n_samples,)
270
+ else:
271
+ assert isinstance(bunch.target, pd.DataFrame)
272
+ assert bunch.target.shape == (n_samples, n_targets)
273
+
274
+ assert bunch.categories is None
275
+
276
+
277
+ @pytest.mark.parametrize(
278
+ "data_id, dataset_params, n_samples, n_features, n_targets",
279
+ [
280
+ # iris
281
+ (61, {"data_id": 61}, 150, 4, 1),
282
+ (61, {"name": "iris", "version": 1}, 150, 4, 1),
283
+ # anneal
284
+ (2, {"data_id": 2}, 11, 38, 1),
285
+ (2, {"name": "anneal", "version": 1}, 11, 38, 1),
286
+ # cpu
287
+ (561, {"data_id": 561}, 209, 7, 1),
288
+ (561, {"name": "cpu", "version": 1}, 209, 7, 1),
289
+ # emotions
290
+ (40589, {"data_id": 40589}, 13, 72, 6),
291
+ # adult-census
292
+ (1119, {"data_id": 1119}, 10, 14, 1),
293
+ (1119, {"name": "adult-census"}, 10, 14, 1),
294
+ # miceprotein
295
+ (40966, {"data_id": 40966}, 7, 77, 1),
296
+ (40966, {"name": "MiceProtein"}, 7, 77, 1),
297
+ ],
298
+ )
299
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
300
+ def test_fetch_openml_as_frame_false(
301
+ monkeypatch,
302
+ data_id,
303
+ dataset_params,
304
+ n_samples,
305
+ n_features,
306
+ n_targets,
307
+ parser,
308
+ ):
309
+ """Check the behaviour of `fetch_openml` with `as_frame=False`.
310
+
311
+ Fetch both by ID and/or name + version.
312
+ """
313
+ pytest.importorskip("pandas")
314
+
315
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
316
+ bunch = fetch_openml(
317
+ as_frame=False,
318
+ cache=False,
319
+ parser=parser,
320
+ **dataset_params,
321
+ )
322
+ assert int(bunch.details["id"]) == data_id
323
+ assert isinstance(bunch, Bunch)
324
+
325
+ assert bunch.frame is None
326
+
327
+ assert isinstance(bunch.data, np.ndarray)
328
+ assert bunch.data.shape == (n_samples, n_features)
329
+
330
+ assert isinstance(bunch.target, np.ndarray)
331
+ if n_targets == 1:
332
+ assert bunch.target.shape == (n_samples,)
333
+ else:
334
+ assert bunch.target.shape == (n_samples, n_targets)
335
+
336
+ assert isinstance(bunch.categories, dict)
337
+
338
+
339
+ @pytest.mark.parametrize("data_id", [61, 1119, 40945])
340
+ def test_fetch_openml_consistency_parser(monkeypatch, data_id):
341
+ """Check the consistency of the LIAC-ARFF and pandas parsers."""
342
+ pd = pytest.importorskip("pandas")
343
+
344
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
345
+ bunch_liac = fetch_openml(
346
+ data_id=data_id,
347
+ as_frame=True,
348
+ cache=False,
349
+ parser="liac-arff",
350
+ )
351
+ bunch_pandas = fetch_openml(
352
+ data_id=data_id,
353
+ as_frame=True,
354
+ cache=False,
355
+ parser="pandas",
356
+ )
357
+
358
+ # The data frames for the input features should match up to some numerical
359
+ # dtype conversions (e.g. float64 <=> Int64) due to limitations of the
360
+ # LIAC-ARFF parser.
361
+ data_liac, data_pandas = bunch_liac.data, bunch_pandas.data
362
+
363
+ def convert_numerical_dtypes(series):
364
+ pandas_series = data_pandas[series.name]
365
+ if pd.api.types.is_numeric_dtype(pandas_series):
366
+ return series.astype(pandas_series.dtype)
367
+ else:
368
+ return series
369
+
370
+ data_liac_with_fixed_dtypes = data_liac.apply(convert_numerical_dtypes)
371
+ pd.testing.assert_frame_equal(data_liac_with_fixed_dtypes, data_pandas)
372
+
373
+ # Let's also check that the .frame attributes also match
374
+ frame_liac, frame_pandas = bunch_liac.frame, bunch_pandas.frame
375
+
376
+ # Note that the .frame attribute is a superset of the .data attribute:
377
+ pd.testing.assert_frame_equal(frame_pandas[bunch_pandas.feature_names], data_pandas)
378
+
379
+ # However the remaining columns, typically the target(s), are not necessarily
380
+ # dtyped similarly by both parsers due to limitations of the LIAC-ARFF parser.
381
+ # Therefore, extra dtype conversions are required for those columns:
382
+
383
+ def convert_numerical_and_categorical_dtypes(series):
384
+ pandas_series = frame_pandas[series.name]
385
+ if pd.api.types.is_numeric_dtype(pandas_series):
386
+ return series.astype(pandas_series.dtype)
387
+ elif isinstance(pandas_series.dtype, pd.CategoricalDtype):
388
+ # Compare categorical features by converting categorical liac uses
389
+ # strings to denote the categories, we rename the categories to make
390
+ # them comparable to the pandas parser. Fixing this behavior in
391
+ # LIAC-ARFF would allow to check the consistency in the future but
392
+ # we do not plan to maintain the LIAC-ARFF on the long term.
393
+ return series.cat.rename_categories(pandas_series.cat.categories)
394
+ else:
395
+ return series
396
+
397
+ frame_liac_with_fixed_dtypes = frame_liac.apply(
398
+ convert_numerical_and_categorical_dtypes
399
+ )
400
+ pd.testing.assert_frame_equal(frame_liac_with_fixed_dtypes, frame_pandas)
401
+
402
+
403
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
404
+ def test_fetch_openml_equivalence_array_dataframe(monkeypatch, parser):
405
+ """Check the equivalence of the dataset when using `as_frame=False` and
406
+ `as_frame=True`.
407
+ """
408
+ pytest.importorskip("pandas")
409
+
410
+ data_id = 61
411
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
412
+ bunch_as_frame_true = fetch_openml(
413
+ data_id=data_id,
414
+ as_frame=True,
415
+ cache=False,
416
+ parser=parser,
417
+ )
418
+
419
+ bunch_as_frame_false = fetch_openml(
420
+ data_id=data_id,
421
+ as_frame=False,
422
+ cache=False,
423
+ parser=parser,
424
+ )
425
+
426
+ assert_allclose(bunch_as_frame_false.data, bunch_as_frame_true.data)
427
+ assert_array_equal(bunch_as_frame_false.target, bunch_as_frame_true.target)
428
+
429
+
430
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
431
+ def test_fetch_openml_iris_pandas(monkeypatch, parser):
432
+ """Check fetching on a numerical only dataset with string labels."""
433
+ pd = pytest.importorskip("pandas")
434
+ CategoricalDtype = pd.api.types.CategoricalDtype
435
+ data_id = 61
436
+ data_shape = (150, 4)
437
+ target_shape = (150,)
438
+ frame_shape = (150, 5)
439
+
440
+ target_dtype = CategoricalDtype(
441
+ ["Iris-setosa", "Iris-versicolor", "Iris-virginica"]
442
+ )
443
+ data_dtypes = [np.float64] * 4
444
+ data_names = ["sepallength", "sepalwidth", "petallength", "petalwidth"]
445
+ target_name = "class"
446
+
447
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
448
+
449
+ bunch = fetch_openml(
450
+ data_id=data_id,
451
+ as_frame=True,
452
+ cache=False,
453
+ parser=parser,
454
+ )
455
+ data = bunch.data
456
+ target = bunch.target
457
+ frame = bunch.frame
458
+
459
+ assert isinstance(data, pd.DataFrame)
460
+ assert np.all(data.dtypes == data_dtypes)
461
+ assert data.shape == data_shape
462
+ assert np.all(data.columns == data_names)
463
+ assert np.all(bunch.feature_names == data_names)
464
+ assert bunch.target_names == [target_name]
465
+
466
+ assert isinstance(target, pd.Series)
467
+ assert target.dtype == target_dtype
468
+ assert target.shape == target_shape
469
+ assert target.name == target_name
470
+ assert target.index.is_unique
471
+
472
+ assert isinstance(frame, pd.DataFrame)
473
+ assert frame.shape == frame_shape
474
+ assert np.all(frame.dtypes == data_dtypes + [target_dtype])
475
+ assert frame.index.is_unique
476
+
477
+
478
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
479
+ @pytest.mark.parametrize("target_column", ["petalwidth", ["petalwidth", "petallength"]])
480
+ def test_fetch_openml_forcing_targets(monkeypatch, parser, target_column):
481
+ """Check that we can force the target to not be the default target."""
482
+ pd = pytest.importorskip("pandas")
483
+
484
+ data_id = 61
485
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
486
+ bunch_forcing_target = fetch_openml(
487
+ data_id=data_id,
488
+ as_frame=True,
489
+ cache=False,
490
+ target_column=target_column,
491
+ parser=parser,
492
+ )
493
+ bunch_default = fetch_openml(
494
+ data_id=data_id,
495
+ as_frame=True,
496
+ cache=False,
497
+ parser=parser,
498
+ )
499
+
500
+ pd.testing.assert_frame_equal(bunch_forcing_target.frame, bunch_default.frame)
501
+ if isinstance(target_column, list):
502
+ pd.testing.assert_index_equal(
503
+ bunch_forcing_target.target.columns, pd.Index(target_column)
504
+ )
505
+ assert bunch_forcing_target.data.shape == (150, 3)
506
+ else:
507
+ assert bunch_forcing_target.target.name == target_column
508
+ assert bunch_forcing_target.data.shape == (150, 4)
509
+
510
+
511
+ @pytest.mark.parametrize("data_id", [61, 2, 561, 40589, 1119])
512
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
513
+ def test_fetch_openml_equivalence_frame_return_X_y(monkeypatch, data_id, parser):
514
+ """Check the behaviour of `return_X_y=True` when `as_frame=True`."""
515
+ pd = pytest.importorskip("pandas")
516
+
517
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
518
+ bunch = fetch_openml(
519
+ data_id=data_id,
520
+ as_frame=True,
521
+ cache=False,
522
+ return_X_y=False,
523
+ parser=parser,
524
+ )
525
+ X, y = fetch_openml(
526
+ data_id=data_id,
527
+ as_frame=True,
528
+ cache=False,
529
+ return_X_y=True,
530
+ parser=parser,
531
+ )
532
+
533
+ pd.testing.assert_frame_equal(bunch.data, X)
534
+ if isinstance(y, pd.Series):
535
+ pd.testing.assert_series_equal(bunch.target, y)
536
+ else:
537
+ pd.testing.assert_frame_equal(bunch.target, y)
538
+
539
+
540
+ @pytest.mark.parametrize("data_id", [61, 561, 40589, 1119])
541
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
542
+ def test_fetch_openml_equivalence_array_return_X_y(monkeypatch, data_id, parser):
543
+ """Check the behaviour of `return_X_y=True` when `as_frame=False`."""
544
+ pytest.importorskip("pandas")
545
+
546
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
547
+ bunch = fetch_openml(
548
+ data_id=data_id,
549
+ as_frame=False,
550
+ cache=False,
551
+ return_X_y=False,
552
+ parser=parser,
553
+ )
554
+ X, y = fetch_openml(
555
+ data_id=data_id,
556
+ as_frame=False,
557
+ cache=False,
558
+ return_X_y=True,
559
+ parser=parser,
560
+ )
561
+
562
+ assert_array_equal(bunch.data, X)
563
+ assert_array_equal(bunch.target, y)
564
+
565
+
566
+ def test_fetch_openml_difference_parsers(monkeypatch):
567
+ """Check the difference between liac-arff and pandas parser."""
568
+ pytest.importorskip("pandas")
569
+
570
+ data_id = 1119
571
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
572
+ # When `as_frame=False`, the categories will be ordinally encoded with
573
+ # liac-arff parser while this is not the case with pandas parser.
574
+ as_frame = False
575
+ bunch_liac_arff = fetch_openml(
576
+ data_id=data_id,
577
+ as_frame=as_frame,
578
+ cache=False,
579
+ parser="liac-arff",
580
+ )
581
+ bunch_pandas = fetch_openml(
582
+ data_id=data_id,
583
+ as_frame=as_frame,
584
+ cache=False,
585
+ parser="pandas",
586
+ )
587
+
588
+ assert bunch_liac_arff.data.dtype.kind == "f"
589
+ assert bunch_pandas.data.dtype == "O"
590
+
591
+
592
+ ###############################################################################
593
+ # Test the ARFF parsing on several dataset to check if detect the correct
594
+ # types (categories, integers, floats).
595
+
596
+
597
+ @pytest.fixture(scope="module")
598
+ def datasets_column_names():
599
+ """Returns the columns names for each dataset."""
600
+ return {
601
+ 61: ["sepallength", "sepalwidth", "petallength", "petalwidth", "class"],
602
+ 2: [
603
+ "family",
604
+ "product-type",
605
+ "steel",
606
+ "carbon",
607
+ "hardness",
608
+ "temper_rolling",
609
+ "condition",
610
+ "formability",
611
+ "strength",
612
+ "non-ageing",
613
+ "surface-finish",
614
+ "surface-quality",
615
+ "enamelability",
616
+ "bc",
617
+ "bf",
618
+ "bt",
619
+ "bw%2Fme",
620
+ "bl",
621
+ "m",
622
+ "chrom",
623
+ "phos",
624
+ "cbond",
625
+ "marvi",
626
+ "exptl",
627
+ "ferro",
628
+ "corr",
629
+ "blue%2Fbright%2Fvarn%2Fclean",
630
+ "lustre",
631
+ "jurofm",
632
+ "s",
633
+ "p",
634
+ "shape",
635
+ "thick",
636
+ "width",
637
+ "len",
638
+ "oil",
639
+ "bore",
640
+ "packing",
641
+ "class",
642
+ ],
643
+ 561: ["vendor", "MYCT", "MMIN", "MMAX", "CACH", "CHMIN", "CHMAX", "class"],
644
+ 40589: [
645
+ "Mean_Acc1298_Mean_Mem40_Centroid",
646
+ "Mean_Acc1298_Mean_Mem40_Rolloff",
647
+ "Mean_Acc1298_Mean_Mem40_Flux",
648
+ "Mean_Acc1298_Mean_Mem40_MFCC_0",
649
+ "Mean_Acc1298_Mean_Mem40_MFCC_1",
650
+ "Mean_Acc1298_Mean_Mem40_MFCC_2",
651
+ "Mean_Acc1298_Mean_Mem40_MFCC_3",
652
+ "Mean_Acc1298_Mean_Mem40_MFCC_4",
653
+ "Mean_Acc1298_Mean_Mem40_MFCC_5",
654
+ "Mean_Acc1298_Mean_Mem40_MFCC_6",
655
+ "Mean_Acc1298_Mean_Mem40_MFCC_7",
656
+ "Mean_Acc1298_Mean_Mem40_MFCC_8",
657
+ "Mean_Acc1298_Mean_Mem40_MFCC_9",
658
+ "Mean_Acc1298_Mean_Mem40_MFCC_10",
659
+ "Mean_Acc1298_Mean_Mem40_MFCC_11",
660
+ "Mean_Acc1298_Mean_Mem40_MFCC_12",
661
+ "Mean_Acc1298_Std_Mem40_Centroid",
662
+ "Mean_Acc1298_Std_Mem40_Rolloff",
663
+ "Mean_Acc1298_Std_Mem40_Flux",
664
+ "Mean_Acc1298_Std_Mem40_MFCC_0",
665
+ "Mean_Acc1298_Std_Mem40_MFCC_1",
666
+ "Mean_Acc1298_Std_Mem40_MFCC_2",
667
+ "Mean_Acc1298_Std_Mem40_MFCC_3",
668
+ "Mean_Acc1298_Std_Mem40_MFCC_4",
669
+ "Mean_Acc1298_Std_Mem40_MFCC_5",
670
+ "Mean_Acc1298_Std_Mem40_MFCC_6",
671
+ "Mean_Acc1298_Std_Mem40_MFCC_7",
672
+ "Mean_Acc1298_Std_Mem40_MFCC_8",
673
+ "Mean_Acc1298_Std_Mem40_MFCC_9",
674
+ "Mean_Acc1298_Std_Mem40_MFCC_10",
675
+ "Mean_Acc1298_Std_Mem40_MFCC_11",
676
+ "Mean_Acc1298_Std_Mem40_MFCC_12",
677
+ "Std_Acc1298_Mean_Mem40_Centroid",
678
+ "Std_Acc1298_Mean_Mem40_Rolloff",
679
+ "Std_Acc1298_Mean_Mem40_Flux",
680
+ "Std_Acc1298_Mean_Mem40_MFCC_0",
681
+ "Std_Acc1298_Mean_Mem40_MFCC_1",
682
+ "Std_Acc1298_Mean_Mem40_MFCC_2",
683
+ "Std_Acc1298_Mean_Mem40_MFCC_3",
684
+ "Std_Acc1298_Mean_Mem40_MFCC_4",
685
+ "Std_Acc1298_Mean_Mem40_MFCC_5",
686
+ "Std_Acc1298_Mean_Mem40_MFCC_6",
687
+ "Std_Acc1298_Mean_Mem40_MFCC_7",
688
+ "Std_Acc1298_Mean_Mem40_MFCC_8",
689
+ "Std_Acc1298_Mean_Mem40_MFCC_9",
690
+ "Std_Acc1298_Mean_Mem40_MFCC_10",
691
+ "Std_Acc1298_Mean_Mem40_MFCC_11",
692
+ "Std_Acc1298_Mean_Mem40_MFCC_12",
693
+ "Std_Acc1298_Std_Mem40_Centroid",
694
+ "Std_Acc1298_Std_Mem40_Rolloff",
695
+ "Std_Acc1298_Std_Mem40_Flux",
696
+ "Std_Acc1298_Std_Mem40_MFCC_0",
697
+ "Std_Acc1298_Std_Mem40_MFCC_1",
698
+ "Std_Acc1298_Std_Mem40_MFCC_2",
699
+ "Std_Acc1298_Std_Mem40_MFCC_3",
700
+ "Std_Acc1298_Std_Mem40_MFCC_4",
701
+ "Std_Acc1298_Std_Mem40_MFCC_5",
702
+ "Std_Acc1298_Std_Mem40_MFCC_6",
703
+ "Std_Acc1298_Std_Mem40_MFCC_7",
704
+ "Std_Acc1298_Std_Mem40_MFCC_8",
705
+ "Std_Acc1298_Std_Mem40_MFCC_9",
706
+ "Std_Acc1298_Std_Mem40_MFCC_10",
707
+ "Std_Acc1298_Std_Mem40_MFCC_11",
708
+ "Std_Acc1298_Std_Mem40_MFCC_12",
709
+ "BH_LowPeakAmp",
710
+ "BH_LowPeakBPM",
711
+ "BH_HighPeakAmp",
712
+ "BH_HighPeakBPM",
713
+ "BH_HighLowRatio",
714
+ "BHSUM1",
715
+ "BHSUM2",
716
+ "BHSUM3",
717
+ "amazed.suprised",
718
+ "happy.pleased",
719
+ "relaxing.calm",
720
+ "quiet.still",
721
+ "sad.lonely",
722
+ "angry.aggresive",
723
+ ],
724
+ 1119: [
725
+ "age",
726
+ "workclass",
727
+ "fnlwgt:",
728
+ "education:",
729
+ "education-num:",
730
+ "marital-status:",
731
+ "occupation:",
732
+ "relationship:",
733
+ "race:",
734
+ "sex:",
735
+ "capital-gain:",
736
+ "capital-loss:",
737
+ "hours-per-week:",
738
+ "native-country:",
739
+ "class",
740
+ ],
741
+ 40966: [
742
+ "DYRK1A_N",
743
+ "ITSN1_N",
744
+ "BDNF_N",
745
+ "NR1_N",
746
+ "NR2A_N",
747
+ "pAKT_N",
748
+ "pBRAF_N",
749
+ "pCAMKII_N",
750
+ "pCREB_N",
751
+ "pELK_N",
752
+ "pERK_N",
753
+ "pJNK_N",
754
+ "PKCA_N",
755
+ "pMEK_N",
756
+ "pNR1_N",
757
+ "pNR2A_N",
758
+ "pNR2B_N",
759
+ "pPKCAB_N",
760
+ "pRSK_N",
761
+ "AKT_N",
762
+ "BRAF_N",
763
+ "CAMKII_N",
764
+ "CREB_N",
765
+ "ELK_N",
766
+ "ERK_N",
767
+ "GSK3B_N",
768
+ "JNK_N",
769
+ "MEK_N",
770
+ "TRKA_N",
771
+ "RSK_N",
772
+ "APP_N",
773
+ "Bcatenin_N",
774
+ "SOD1_N",
775
+ "MTOR_N",
776
+ "P38_N",
777
+ "pMTOR_N",
778
+ "DSCR1_N",
779
+ "AMPKA_N",
780
+ "NR2B_N",
781
+ "pNUMB_N",
782
+ "RAPTOR_N",
783
+ "TIAM1_N",
784
+ "pP70S6_N",
785
+ "NUMB_N",
786
+ "P70S6_N",
787
+ "pGSK3B_N",
788
+ "pPKCG_N",
789
+ "CDK5_N",
790
+ "S6_N",
791
+ "ADARB1_N",
792
+ "AcetylH3K9_N",
793
+ "RRP1_N",
794
+ "BAX_N",
795
+ "ARC_N",
796
+ "ERBB4_N",
797
+ "nNOS_N",
798
+ "Tau_N",
799
+ "GFAP_N",
800
+ "GluR3_N",
801
+ "GluR4_N",
802
+ "IL1B_N",
803
+ "P3525_N",
804
+ "pCASP9_N",
805
+ "PSD95_N",
806
+ "SNCA_N",
807
+ "Ubiquitin_N",
808
+ "pGSK3B_Tyr216_N",
809
+ "SHH_N",
810
+ "BAD_N",
811
+ "BCL2_N",
812
+ "pS6_N",
813
+ "pCFOS_N",
814
+ "SYP_N",
815
+ "H3AcK18_N",
816
+ "EGR1_N",
817
+ "H3MeK4_N",
818
+ "CaNA_N",
819
+ "class",
820
+ ],
821
+ 40945: [
822
+ "pclass",
823
+ "survived",
824
+ "name",
825
+ "sex",
826
+ "age",
827
+ "sibsp",
828
+ "parch",
829
+ "ticket",
830
+ "fare",
831
+ "cabin",
832
+ "embarked",
833
+ "boat",
834
+ "body",
835
+ "home.dest",
836
+ ],
837
+ }
838
+
839
+
840
+ @pytest.fixture(scope="module")
841
+ def datasets_missing_values():
842
+ return {
843
+ 61: {},
844
+ 2: {
845
+ "family": 11,
846
+ "temper_rolling": 9,
847
+ "condition": 2,
848
+ "formability": 4,
849
+ "non-ageing": 10,
850
+ "surface-finish": 11,
851
+ "enamelability": 11,
852
+ "bc": 11,
853
+ "bf": 10,
854
+ "bt": 11,
855
+ "bw%2Fme": 8,
856
+ "bl": 9,
857
+ "m": 11,
858
+ "chrom": 11,
859
+ "phos": 11,
860
+ "cbond": 10,
861
+ "marvi": 11,
862
+ "exptl": 11,
863
+ "ferro": 11,
864
+ "corr": 11,
865
+ "blue%2Fbright%2Fvarn%2Fclean": 11,
866
+ "lustre": 8,
867
+ "jurofm": 11,
868
+ "s": 11,
869
+ "p": 11,
870
+ "oil": 10,
871
+ "packing": 11,
872
+ },
873
+ 561: {},
874
+ 40589: {},
875
+ 1119: {},
876
+ 40966: {"BCL2_N": 7},
877
+ 40945: {
878
+ "age": 263,
879
+ "fare": 1,
880
+ "cabin": 1014,
881
+ "embarked": 2,
882
+ "boat": 823,
883
+ "body": 1188,
884
+ "home.dest": 564,
885
+ },
886
+ }
887
+
888
+
889
+ @pytest.mark.parametrize(
890
+ "data_id, parser, expected_n_categories, expected_n_floats, expected_n_ints",
891
+ [
892
+ # iris dataset
893
+ (61, "liac-arff", 1, 4, 0),
894
+ (61, "pandas", 1, 4, 0),
895
+ # anneal dataset
896
+ (2, "liac-arff", 33, 6, 0),
897
+ (2, "pandas", 33, 2, 4),
898
+ # cpu dataset
899
+ (561, "liac-arff", 1, 7, 0),
900
+ (561, "pandas", 1, 0, 7),
901
+ # emotions dataset
902
+ (40589, "liac-arff", 6, 72, 0),
903
+ (40589, "pandas", 6, 69, 3),
904
+ # adult-census dataset
905
+ (1119, "liac-arff", 9, 6, 0),
906
+ (1119, "pandas", 9, 0, 6),
907
+ # miceprotein
908
+ (40966, "liac-arff", 1, 77, 0),
909
+ (40966, "pandas", 1, 77, 0),
910
+ # titanic
911
+ (40945, "liac-arff", 3, 6, 0),
912
+ (40945, "pandas", 3, 3, 3),
913
+ ],
914
+ )
915
+ @pytest.mark.parametrize("gzip_response", [True, False])
916
+ def test_fetch_openml_types_inference(
917
+ monkeypatch,
918
+ data_id,
919
+ parser,
920
+ expected_n_categories,
921
+ expected_n_floats,
922
+ expected_n_ints,
923
+ gzip_response,
924
+ datasets_column_names,
925
+ datasets_missing_values,
926
+ ):
927
+ """Check that `fetch_openml` infer the right number of categories, integers, and
928
+ floats."""
929
+ pd = pytest.importorskip("pandas")
930
+ CategoricalDtype = pd.api.types.CategoricalDtype
931
+
932
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response)
933
+
934
+ bunch = fetch_openml(
935
+ data_id=data_id,
936
+ as_frame=True,
937
+ cache=False,
938
+ parser=parser,
939
+ )
940
+ frame = bunch.frame
941
+
942
+ n_categories = len(
943
+ [dtype for dtype in frame.dtypes if isinstance(dtype, CategoricalDtype)]
944
+ )
945
+ n_floats = len([dtype for dtype in frame.dtypes if dtype.kind == "f"])
946
+ n_ints = len([dtype for dtype in frame.dtypes if dtype.kind == "i"])
947
+
948
+ assert n_categories == expected_n_categories
949
+ assert n_floats == expected_n_floats
950
+ assert n_ints == expected_n_ints
951
+
952
+ assert frame.columns.tolist() == datasets_column_names[data_id]
953
+
954
+ frame_feature_to_n_nan = frame.isna().sum().to_dict()
955
+ for name, n_missing in frame_feature_to_n_nan.items():
956
+ expected_missing = datasets_missing_values[data_id].get(name, 0)
957
+ assert n_missing == expected_missing
958
+
959
+
960
+ ###############################################################################
961
+ # Test some more specific behaviour
962
+
963
+
964
+ @pytest.mark.parametrize(
965
+ "params, err_msg",
966
+ [
967
+ (
968
+ {"parser": "unknown"},
969
+ "The 'parser' parameter of fetch_openml must be a str among",
970
+ ),
971
+ (
972
+ {"as_frame": "unknown"},
973
+ "The 'as_frame' parameter of fetch_openml must be an instance",
974
+ ),
975
+ ],
976
+ )
977
+ def test_fetch_openml_validation_parameter(monkeypatch, params, err_msg):
978
+ data_id = 1119
979
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
980
+ with pytest.raises(ValueError, match=err_msg):
981
+ fetch_openml(data_id=data_id, **params)
982
+
983
+
984
+ @pytest.mark.parametrize(
985
+ "params",
986
+ [
987
+ {"as_frame": True, "parser": "auto"},
988
+ {"as_frame": "auto", "parser": "auto"},
989
+ {"as_frame": False, "parser": "pandas"},
990
+ {"as_frame": False, "parser": "auto"},
991
+ ],
992
+ )
993
+ def test_fetch_openml_requires_pandas_error(monkeypatch, params):
994
+ """Check that we raise the proper errors when we require pandas."""
995
+ data_id = 1119
996
+ try:
997
+ check_pandas_support("test_fetch_openml_requires_pandas")
998
+ except ImportError:
999
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
1000
+ err_msg = "requires pandas to be installed. Alternatively, explicitly"
1001
+ with pytest.raises(ImportError, match=err_msg):
1002
+ fetch_openml(data_id=data_id, **params)
1003
+ else:
1004
+ raise SkipTest("This test requires pandas to not be installed.")
1005
+
1006
+
1007
+ @pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive")
1008
+ @pytest.mark.parametrize(
1009
+ "params, err_msg",
1010
+ [
1011
+ (
1012
+ {"parser": "pandas"},
1013
+ "Sparse ARFF datasets cannot be loaded with parser='pandas'",
1014
+ ),
1015
+ (
1016
+ {"as_frame": True},
1017
+ "Sparse ARFF datasets cannot be loaded with as_frame=True.",
1018
+ ),
1019
+ (
1020
+ {"parser": "pandas", "as_frame": True},
1021
+ "Sparse ARFF datasets cannot be loaded with as_frame=True.",
1022
+ ),
1023
+ ],
1024
+ )
1025
+ def test_fetch_openml_sparse_arff_error(monkeypatch, params, err_msg):
1026
+ """Check that we raise the expected error for sparse ARFF datasets and
1027
+ a wrong set of incompatible parameters.
1028
+ """
1029
+ pytest.importorskip("pandas")
1030
+ data_id = 292
1031
+
1032
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
1033
+ with pytest.raises(ValueError, match=err_msg):
1034
+ fetch_openml(
1035
+ data_id=data_id,
1036
+ cache=False,
1037
+ **params,
1038
+ )
1039
+
1040
+
1041
+ @pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive")
1042
+ @pytest.mark.parametrize(
1043
+ "data_id, data_type",
1044
+ [
1045
+ (61, "dataframe"), # iris dataset version 1
1046
+ (292, "sparse"), # Australian dataset version 1
1047
+ ],
1048
+ )
1049
+ def test_fetch_openml_auto_mode(monkeypatch, data_id, data_type):
1050
+ """Check the auto mode of `fetch_openml`."""
1051
+ pd = pytest.importorskip("pandas")
1052
+
1053
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
1054
+ data = fetch_openml(data_id=data_id, as_frame="auto", cache=False)
1055
+ klass = pd.DataFrame if data_type == "dataframe" else scipy.sparse.csr_matrix
1056
+ assert isinstance(data.data, klass)
1057
+
1058
+
1059
+ def test_convert_arff_data_dataframe_warning_low_memory_pandas(monkeypatch):
1060
+ """Check that we raise a warning regarding the working memory when using
1061
+ LIAC-ARFF parser."""
1062
+ pytest.importorskip("pandas")
1063
+
1064
+ data_id = 1119
1065
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
1066
+
1067
+ msg = "Could not adhere to working_memory config."
1068
+ with pytest.warns(UserWarning, match=msg):
1069
+ with config_context(working_memory=1e-6):
1070
+ fetch_openml(
1071
+ data_id=data_id,
1072
+ as_frame=True,
1073
+ cache=False,
1074
+ parser="liac-arff",
1075
+ )
1076
+
1077
+
1078
+ @pytest.mark.parametrize("gzip_response", [True, False])
1079
+ def test_fetch_openml_iris_warn_multiple_version(monkeypatch, gzip_response):
1080
+ """Check that a warning is raised when multiple versions exist and no version is
1081
+ requested."""
1082
+ data_id = 61
1083
+ data_name = "iris"
1084
+
1085
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1086
+
1087
+ msg = re.escape(
1088
+ "Multiple active versions of the dataset matching the name"
1089
+ " iris exist. Versions may be fundamentally different, "
1090
+ "returning version 1. Available versions:\n"
1091
+ "- version 1, status: active\n"
1092
+ " url: https://www.openml.org/search?type=data&id=61\n"
1093
+ "- version 3, status: active\n"
1094
+ " url: https://www.openml.org/search?type=data&id=969\n"
1095
+ )
1096
+ with pytest.warns(UserWarning, match=msg):
1097
+ fetch_openml(
1098
+ name=data_name,
1099
+ as_frame=False,
1100
+ cache=False,
1101
+ parser="liac-arff",
1102
+ )
1103
+
1104
+
1105
+ @pytest.mark.parametrize("gzip_response", [True, False])
1106
+ def test_fetch_openml_no_target(monkeypatch, gzip_response):
1107
+ """Check that we can get a dataset without target."""
1108
+ data_id = 61
1109
+ target_column = None
1110
+ expected_observations = 150
1111
+ expected_features = 5
1112
+
1113
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1114
+ data = fetch_openml(
1115
+ data_id=data_id,
1116
+ target_column=target_column,
1117
+ cache=False,
1118
+ as_frame=False,
1119
+ parser="liac-arff",
1120
+ )
1121
+ assert data.data.shape == (expected_observations, expected_features)
1122
+ assert data.target is None
1123
+
1124
+
1125
+ @pytest.mark.parametrize("gzip_response", [True, False])
1126
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
1127
+ def test_missing_values_pandas(monkeypatch, gzip_response, parser):
1128
+ """check that missing values in categories are compatible with pandas
1129
+ categorical"""
1130
+ pytest.importorskip("pandas")
1131
+
1132
+ data_id = 42585
1133
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response)
1134
+ penguins = fetch_openml(
1135
+ data_id=data_id,
1136
+ cache=False,
1137
+ as_frame=True,
1138
+ parser=parser,
1139
+ )
1140
+
1141
+ cat_dtype = penguins.data.dtypes["sex"]
1142
+ # there are nans in the categorical
1143
+ assert penguins.data["sex"].isna().any()
1144
+ assert_array_equal(cat_dtype.categories, ["FEMALE", "MALE", "_"])
1145
+
1146
+
1147
+ @pytest.mark.parametrize("gzip_response", [True, False])
1148
+ @pytest.mark.parametrize(
1149
+ "dataset_params",
1150
+ [
1151
+ {"data_id": 40675},
1152
+ {"data_id": None, "name": "glass2", "version": 1},
1153
+ ],
1154
+ )
1155
+ def test_fetch_openml_inactive(monkeypatch, gzip_response, dataset_params):
1156
+ """Check that we raise a warning when the dataset is inactive."""
1157
+ data_id = 40675
1158
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1159
+ msg = "Version 1 of dataset glass2 is inactive,"
1160
+ with pytest.warns(UserWarning, match=msg):
1161
+ glass2 = fetch_openml(
1162
+ cache=False, as_frame=False, parser="liac-arff", **dataset_params
1163
+ )
1164
+ assert glass2.data.shape == (163, 9)
1165
+ assert glass2.details["id"] == "40675"
1166
+
1167
+
1168
+ @pytest.mark.parametrize("gzip_response", [True, False])
1169
+ @pytest.mark.parametrize(
1170
+ "data_id, params, err_type, err_msg",
1171
+ [
1172
+ (40675, {"name": "glass2"}, ValueError, "No active dataset glass2 found"),
1173
+ (
1174
+ 61,
1175
+ {"data_id": 61, "target_column": ["sepalwidth", "class"]},
1176
+ ValueError,
1177
+ "Can only handle homogeneous multi-target datasets",
1178
+ ),
1179
+ (
1180
+ 40945,
1181
+ {"data_id": 40945, "as_frame": False},
1182
+ ValueError,
1183
+ (
1184
+ "STRING attributes are not supported for array representation. Try"
1185
+ " as_frame=True"
1186
+ ),
1187
+ ),
1188
+ (
1189
+ 2,
1190
+ {"data_id": 2, "target_column": "family", "as_frame": True},
1191
+ ValueError,
1192
+ "Target column 'family'",
1193
+ ),
1194
+ (
1195
+ 2,
1196
+ {"data_id": 2, "target_column": "family", "as_frame": False},
1197
+ ValueError,
1198
+ "Target column 'family'",
1199
+ ),
1200
+ (
1201
+ 61,
1202
+ {"data_id": 61, "target_column": "undefined"},
1203
+ KeyError,
1204
+ "Could not find target_column='undefined'",
1205
+ ),
1206
+ (
1207
+ 61,
1208
+ {"data_id": 61, "target_column": ["undefined", "class"]},
1209
+ KeyError,
1210
+ "Could not find target_column='undefined'",
1211
+ ),
1212
+ ],
1213
+ )
1214
+ @pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
1215
+ def test_fetch_openml_error(
1216
+ monkeypatch, gzip_response, data_id, params, err_type, err_msg, parser
1217
+ ):
1218
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1219
+ if params.get("as_frame", True) or parser == "pandas":
1220
+ pytest.importorskip("pandas")
1221
+ with pytest.raises(err_type, match=err_msg):
1222
+ fetch_openml(cache=False, parser=parser, **params)
1223
+
1224
+
1225
+ @pytest.mark.parametrize(
1226
+ "params, err_type, err_msg",
1227
+ [
1228
+ (
1229
+ {"data_id": -1, "name": None, "version": "version"},
1230
+ ValueError,
1231
+ "The 'version' parameter of fetch_openml must be an int in the range",
1232
+ ),
1233
+ (
1234
+ {"data_id": -1, "name": "nAmE"},
1235
+ ValueError,
1236
+ "The 'data_id' parameter of fetch_openml must be an int in the range",
1237
+ ),
1238
+ (
1239
+ {"data_id": -1, "name": "nAmE", "version": "version"},
1240
+ ValueError,
1241
+ "The 'version' parameter of fetch_openml must be an int",
1242
+ ),
1243
+ (
1244
+ {},
1245
+ ValueError,
1246
+ "Neither name nor data_id are provided. Please provide name or data_id.",
1247
+ ),
1248
+ ],
1249
+ )
1250
+ def test_fetch_openml_raises_illegal_argument(params, err_type, err_msg):
1251
+ with pytest.raises(err_type, match=err_msg):
1252
+ fetch_openml(**params)
1253
+
1254
+
1255
+ @pytest.mark.parametrize("gzip_response", [True, False])
1256
+ def test_warn_ignore_attribute(monkeypatch, gzip_response):
1257
+ data_id = 40966
1258
+ expected_row_id_msg = "target_column='{}' has flag is_row_identifier."
1259
+ expected_ignore_msg = "target_column='{}' has flag is_ignore."
1260
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1261
+ # single column test
1262
+ target_col = "MouseID"
1263
+ msg = expected_row_id_msg.format(target_col)
1264
+ with pytest.warns(UserWarning, match=msg):
1265
+ fetch_openml(
1266
+ data_id=data_id,
1267
+ target_column=target_col,
1268
+ cache=False,
1269
+ as_frame=False,
1270
+ parser="liac-arff",
1271
+ )
1272
+ target_col = "Genotype"
1273
+ msg = expected_ignore_msg.format(target_col)
1274
+ with pytest.warns(UserWarning, match=msg):
1275
+ fetch_openml(
1276
+ data_id=data_id,
1277
+ target_column=target_col,
1278
+ cache=False,
1279
+ as_frame=False,
1280
+ parser="liac-arff",
1281
+ )
1282
+ # multi column test
1283
+ target_col = "MouseID"
1284
+ msg = expected_row_id_msg.format(target_col)
1285
+ with pytest.warns(UserWarning, match=msg):
1286
+ fetch_openml(
1287
+ data_id=data_id,
1288
+ target_column=[target_col, "class"],
1289
+ cache=False,
1290
+ as_frame=False,
1291
+ parser="liac-arff",
1292
+ )
1293
+ target_col = "Genotype"
1294
+ msg = expected_ignore_msg.format(target_col)
1295
+ with pytest.warns(UserWarning, match=msg):
1296
+ fetch_openml(
1297
+ data_id=data_id,
1298
+ target_column=[target_col, "class"],
1299
+ cache=False,
1300
+ as_frame=False,
1301
+ parser="liac-arff",
1302
+ )
1303
+
1304
+
1305
+ @pytest.mark.parametrize("gzip_response", [True, False])
1306
+ def test_dataset_with_openml_error(monkeypatch, gzip_response):
1307
+ data_id = 1
1308
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1309
+ msg = "OpenML registered a problem with the dataset. It might be unusable. Error:"
1310
+ with pytest.warns(UserWarning, match=msg):
1311
+ fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff")
1312
+
1313
+
1314
+ @pytest.mark.parametrize("gzip_response", [True, False])
1315
+ def test_dataset_with_openml_warning(monkeypatch, gzip_response):
1316
+ data_id = 3
1317
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1318
+ msg = "OpenML raised a warning on the dataset. It might be unusable. Warning:"
1319
+ with pytest.warns(UserWarning, match=msg):
1320
+ fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff")
1321
+
1322
+
1323
+ def test_fetch_openml_overwrite_default_params_read_csv(monkeypatch):
1324
+ """Check that we can overwrite the default parameters of `read_csv`."""
1325
+ pytest.importorskip("pandas")
1326
+ data_id = 1590
1327
+ _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
1328
+
1329
+ common_params = {
1330
+ "data_id": data_id,
1331
+ "as_frame": True,
1332
+ "cache": False,
1333
+ "parser": "pandas",
1334
+ }
1335
+
1336
+ # By default, the initial spaces are skipped. We checked that setting the parameter
1337
+ # `skipinitialspace` to False will have an effect.
1338
+ adult_without_spaces = fetch_openml(**common_params)
1339
+ adult_with_spaces = fetch_openml(
1340
+ **common_params, read_csv_kwargs={"skipinitialspace": False}
1341
+ )
1342
+ assert all(
1343
+ cat.startswith(" ") for cat in adult_with_spaces.frame["class"].cat.categories
1344
+ )
1345
+ assert not any(
1346
+ cat.startswith(" ")
1347
+ for cat in adult_without_spaces.frame["class"].cat.categories
1348
+ )
1349
+
1350
+
1351
+ ###############################################################################
1352
+ # Test cache, retry mechanisms, checksum, etc.
1353
+
1354
+
1355
+ @pytest.mark.parametrize("gzip_response", [True, False])
1356
+ def test_open_openml_url_cache(monkeypatch, gzip_response, tmpdir):
1357
+ data_id = 61
1358
+
1359
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1360
+ openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff"
1361
+ url = f"https://www.openml.org/{openml_path}"
1362
+ cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
1363
+ # first fill the cache
1364
+ response1 = _open_openml_url(url, cache_directory)
1365
+ # assert file exists
1366
+ location = _get_local_path(openml_path, cache_directory)
1367
+ assert os.path.isfile(location)
1368
+ # redownload, to utilize cache
1369
+ response2 = _open_openml_url(url, cache_directory)
1370
+ assert response1.read() == response2.read()
1371
+
1372
+
1373
+ @pytest.mark.parametrize("write_to_disk", [True, False])
1374
+ def test_open_openml_url_unlinks_local_path(monkeypatch, tmpdir, write_to_disk):
1375
+ data_id = 61
1376
+ openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff"
1377
+ url = f"https://www.openml.org/{openml_path}"
1378
+ cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
1379
+ location = _get_local_path(openml_path, cache_directory)
1380
+
1381
+ def _mock_urlopen(request, *args, **kwargs):
1382
+ if write_to_disk:
1383
+ with open(location, "w") as f:
1384
+ f.write("")
1385
+ raise ValueError("Invalid request")
1386
+
1387
+ monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen)
1388
+
1389
+ with pytest.raises(ValueError, match="Invalid request"):
1390
+ _open_openml_url(url, cache_directory)
1391
+
1392
+ assert not os.path.exists(location)
1393
+
1394
+
1395
+ def test_retry_with_clean_cache(tmpdir):
1396
+ data_id = 61
1397
+ openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id)
1398
+ cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
1399
+ location = _get_local_path(openml_path, cache_directory)
1400
+ os.makedirs(os.path.dirname(location))
1401
+
1402
+ with open(location, "w") as f:
1403
+ f.write("")
1404
+
1405
+ @_retry_with_clean_cache(openml_path, cache_directory)
1406
+ def _load_data():
1407
+ # The first call will raise an error since location exists
1408
+ if os.path.exists(location):
1409
+ raise Exception("File exist!")
1410
+ return 1
1411
+
1412
+ warn_msg = "Invalid cache, redownloading file"
1413
+ with pytest.warns(RuntimeWarning, match=warn_msg):
1414
+ result = _load_data()
1415
+ assert result == 1
1416
+
1417
+
1418
+ def test_retry_with_clean_cache_http_error(tmpdir):
1419
+ data_id = 61
1420
+ openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id)
1421
+ cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
1422
+
1423
+ @_retry_with_clean_cache(openml_path, cache_directory)
1424
+ def _load_data():
1425
+ raise HTTPError(
1426
+ url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO()
1427
+ )
1428
+
1429
+ error_msg = "Simulated mock error"
1430
+ with pytest.raises(HTTPError, match=error_msg):
1431
+ _load_data()
1432
+
1433
+
1434
+ @pytest.mark.parametrize("gzip_response", [True, False])
1435
+ def test_fetch_openml_cache(monkeypatch, gzip_response, tmpdir):
1436
+ def _mock_urlopen_raise(request, *args, **kwargs):
1437
+ raise ValueError(
1438
+ "This mechanism intends to test correct cache"
1439
+ "handling. As such, urlopen should never be "
1440
+ "accessed. URL: %s" % request.get_full_url()
1441
+ )
1442
+
1443
+ data_id = 61
1444
+ cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
1445
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1446
+ X_fetched, y_fetched = fetch_openml(
1447
+ data_id=data_id,
1448
+ cache=True,
1449
+ data_home=cache_directory,
1450
+ return_X_y=True,
1451
+ as_frame=False,
1452
+ parser="liac-arff",
1453
+ )
1454
+
1455
+ monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen_raise)
1456
+
1457
+ X_cached, y_cached = fetch_openml(
1458
+ data_id=data_id,
1459
+ cache=True,
1460
+ data_home=cache_directory,
1461
+ return_X_y=True,
1462
+ as_frame=False,
1463
+ parser="liac-arff",
1464
+ )
1465
+ np.testing.assert_array_equal(X_fetched, X_cached)
1466
+ np.testing.assert_array_equal(y_fetched, y_cached)
1467
+
1468
+
1469
+ @pytest.mark.parametrize(
1470
+ "as_frame, parser",
1471
+ [
1472
+ (True, "liac-arff"),
1473
+ (False, "liac-arff"),
1474
+ (True, "pandas"),
1475
+ (False, "pandas"),
1476
+ ],
1477
+ )
1478
+ def test_fetch_openml_verify_checksum(monkeypatch, as_frame, tmpdir, parser):
1479
+ """Check that the checksum is working as expected."""
1480
+ if as_frame or parser == "pandas":
1481
+ pytest.importorskip("pandas")
1482
+
1483
+ data_id = 2
1484
+ _monkey_patch_webbased_functions(monkeypatch, data_id, True)
1485
+
1486
+ # create a temporary modified arff file
1487
+ original_data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}"
1488
+ original_data_file_name = "data-v1-dl-1666876.arff.gz"
1489
+ original_data_path = resources.files(original_data_module) / original_data_file_name
1490
+ corrupt_copy_path = tmpdir / "test_invalid_checksum.arff"
1491
+ with original_data_path.open("rb") as orig_file:
1492
+ orig_gzip = gzip.open(orig_file, "rb")
1493
+ data = bytearray(orig_gzip.read())
1494
+ data[len(data) - 1] = 37
1495
+
1496
+ with gzip.GzipFile(corrupt_copy_path, "wb") as modified_gzip:
1497
+ modified_gzip.write(data)
1498
+
1499
+ # Requests are already mocked by monkey_patch_webbased_functions.
1500
+ # We want to reuse that mock for all requests except file download,
1501
+ # hence creating a thin mock over the original mock
1502
+ mocked_openml_url = sklearn.datasets._openml.urlopen
1503
+
1504
+ def swap_file_mock(request, *args, **kwargs):
1505
+ url = request.get_full_url()
1506
+ if url.endswith("data/v1/download/1666876/anneal.arff"):
1507
+ with open(corrupt_copy_path, "rb") as f:
1508
+ corrupted_data = f.read()
1509
+ return _MockHTTPResponse(BytesIO(corrupted_data), is_gzip=True)
1510
+ else:
1511
+ return mocked_openml_url(request)
1512
+
1513
+ monkeypatch.setattr(sklearn.datasets._openml, "urlopen", swap_file_mock)
1514
+
1515
+ # validate failed checksum
1516
+ with pytest.raises(ValueError) as exc:
1517
+ sklearn.datasets.fetch_openml(
1518
+ data_id=data_id, cache=False, as_frame=as_frame, parser=parser
1519
+ )
1520
+ # exception message should have file-path
1521
+ assert exc.match("1666876")
1522
+
1523
+
1524
+ def test_open_openml_url_retry_on_network_error(monkeypatch):
1525
+ def _mock_urlopen_network_error(request, *args, **kwargs):
1526
+ raise HTTPError(
1527
+ url=None, code=404, msg="Simulated network error", hdrs=None, fp=BytesIO()
1528
+ )
1529
+
1530
+ monkeypatch.setattr(
1531
+ sklearn.datasets._openml, "urlopen", _mock_urlopen_network_error
1532
+ )
1533
+
1534
+ invalid_openml_url = "https://api.openml.org/invalid-url"
1535
+
1536
+ with pytest.warns(
1537
+ UserWarning,
1538
+ match=re.escape(
1539
+ "A network error occurred while downloading"
1540
+ f" {invalid_openml_url}. Retrying..."
1541
+ ),
1542
+ ) as record:
1543
+ with pytest.raises(HTTPError, match="Simulated network error") as exc_info:
1544
+ _open_openml_url(invalid_openml_url, None, delay=0)
1545
+ assert len(record) == 3
1546
+ # Avoid a ResourceWarning on Python 3.14 and later.
1547
+ exc_info.value.close()
1548
+
1549
+
1550
+ ###############################################################################
1551
+ # Non-regressiont tests
1552
+
1553
+
1554
+ @pytest.mark.parametrize("gzip_response", [True, False])
1555
+ @pytest.mark.parametrize("parser", ("liac-arff", "pandas"))
1556
+ def test_fetch_openml_with_ignored_feature(monkeypatch, gzip_response, parser):
1557
+ """Check that we can load the "zoo" dataset.
1558
+ Non-regression test for:
1559
+ https://github.com/scikit-learn/scikit-learn/issues/14340
1560
+ """
1561
+ if parser == "pandas":
1562
+ pytest.importorskip("pandas")
1563
+ data_id = 62
1564
+ _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
1565
+
1566
+ dataset = sklearn.datasets.fetch_openml(
1567
+ data_id=data_id, cache=False, as_frame=False, parser=parser
1568
+ )
1569
+ assert dataset is not None
1570
+ # The dataset has 17 features, including 1 ignored (animal),
1571
+ # so we assert that we don't have the ignored feature in the final Bunch
1572
+ assert dataset["data"].shape == (101, 16)
1573
+ assert "animal" not in dataset["feature_names"]
1574
+
1575
+
1576
+ def test_fetch_openml_strip_quotes(monkeypatch):
1577
+ """Check that we strip the single quotes when used as a string delimiter.
1578
+
1579
+ Non-regression test for:
1580
+ https://github.com/scikit-learn/scikit-learn/issues/23381
1581
+ """
1582
+ pd = pytest.importorskip("pandas")
1583
+ data_id = 40966
1584
+ _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
1585
+
1586
+ common_params = {"as_frame": True, "cache": False, "data_id": data_id}
1587
+ mice_pandas = fetch_openml(parser="pandas", **common_params)
1588
+ mice_liac_arff = fetch_openml(parser="liac-arff", **common_params)
1589
+ pd.testing.assert_series_equal(mice_pandas.target, mice_liac_arff.target)
1590
+ assert not mice_pandas.target.str.startswith("'").any()
1591
+ assert not mice_pandas.target.str.endswith("'").any()
1592
+
1593
+ # similar behaviour should be observed when the column is not the target
1594
+ mice_pandas = fetch_openml(parser="pandas", target_column="NUMB_N", **common_params)
1595
+ mice_liac_arff = fetch_openml(
1596
+ parser="liac-arff", target_column="NUMB_N", **common_params
1597
+ )
1598
+ pd.testing.assert_series_equal(
1599
+ mice_pandas.frame["class"], mice_liac_arff.frame["class"]
1600
+ )
1601
+ assert not mice_pandas.frame["class"].str.startswith("'").any()
1602
+ assert not mice_pandas.frame["class"].str.endswith("'").any()
1603
+
1604
+
1605
+ def test_fetch_openml_leading_whitespace(monkeypatch):
1606
+ """Check that we can strip leading whitespace in pandas parser.
1607
+
1608
+ Non-regression test for:
1609
+ https://github.com/scikit-learn/scikit-learn/issues/25311
1610
+ """
1611
+ pd = pytest.importorskip("pandas")
1612
+ data_id = 1590
1613
+ _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
1614
+
1615
+ common_params = {"as_frame": True, "cache": False, "data_id": data_id}
1616
+ adult_pandas = fetch_openml(parser="pandas", **common_params)
1617
+ adult_liac_arff = fetch_openml(parser="liac-arff", **common_params)
1618
+ pd.testing.assert_series_equal(
1619
+ adult_pandas.frame["class"], adult_liac_arff.frame["class"]
1620
+ )
1621
+
1622
+
1623
+ def test_fetch_openml_quotechar_escapechar(monkeypatch):
1624
+ """Check that we can handle escapechar and single/double quotechar.
1625
+
1626
+ Non-regression test for:
1627
+ https://github.com/scikit-learn/scikit-learn/issues/25478
1628
+ """
1629
+ pd = pytest.importorskip("pandas")
1630
+ data_id = 42074
1631
+ _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
1632
+
1633
+ common_params = {"as_frame": True, "cache": False, "data_id": data_id}
1634
+ adult_pandas = fetch_openml(parser="pandas", **common_params)
1635
+ adult_liac_arff = fetch_openml(parser="liac-arff", **common_params)
1636
+ pd.testing.assert_frame_equal(adult_pandas.frame, adult_liac_arff.frame)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_rcv1.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test the rcv1 loader, if the data is available,
2
+ or if specifically requested via environment variable
3
+ (e.g. for CI jobs)."""
4
+
5
+ from functools import partial
6
+
7
+ import numpy as np
8
+ import scipy.sparse as sp
9
+
10
+ from sklearn.datasets.tests.test_common import check_return_X_y
11
+ from sklearn.utils._testing import assert_almost_equal, assert_array_equal
12
+
13
+
14
+ def test_fetch_rcv1(fetch_rcv1_fxt, global_random_seed):
15
+ data1 = fetch_rcv1_fxt(shuffle=False)
16
+ X1, Y1 = data1.data, data1.target
17
+ cat_list, s1 = data1.target_names.tolist(), data1.sample_id
18
+
19
+ # test sparsity
20
+ assert sp.issparse(X1)
21
+ assert sp.issparse(Y1)
22
+ assert 60915113 == X1.data.size
23
+ assert 2606875 == Y1.data.size
24
+
25
+ # test shapes
26
+ assert (804414, 47236) == X1.shape
27
+ assert (804414, 103) == Y1.shape
28
+ assert (804414,) == s1.shape
29
+ assert 103 == len(cat_list)
30
+
31
+ # test descr
32
+ assert data1.DESCR.startswith(".. _rcv1_dataset:")
33
+
34
+ # test ordering of categories
35
+ first_categories = ["C11", "C12", "C13", "C14", "C15", "C151"]
36
+ assert_array_equal(first_categories, cat_list[:6])
37
+
38
+ # test number of sample for some categories
39
+ some_categories = ("GMIL", "E143", "CCAT")
40
+ number_non_zero_in_cat = (5, 1206, 381327)
41
+ for num, cat in zip(number_non_zero_in_cat, some_categories):
42
+ j = cat_list.index(cat)
43
+ assert num == Y1[:, j].data.size
44
+
45
+ # test shuffling and subset
46
+ data2 = fetch_rcv1_fxt(
47
+ shuffle=True, subset="train", random_state=global_random_seed
48
+ )
49
+ X2, Y2 = data2.data, data2.target
50
+ s2 = data2.sample_id
51
+
52
+ # test return_X_y option
53
+ fetch_func = partial(fetch_rcv1_fxt, shuffle=False, subset="train")
54
+ check_return_X_y(data2, fetch_func)
55
+
56
+ # The first 23149 samples are the training samples
57
+ assert_array_equal(np.sort(s1[:23149]), np.sort(s2))
58
+
59
+ # test some precise values
60
+ some_sample_ids = (2286, 3274, 14042)
61
+ for sample_id in some_sample_ids:
62
+ idx1 = s1.tolist().index(sample_id)
63
+ idx2 = s2.tolist().index(sample_id)
64
+
65
+ feature_values_1 = X1[idx1, :].toarray()
66
+ feature_values_2 = X2[idx2, :].toarray()
67
+ assert_almost_equal(feature_values_1, feature_values_2)
68
+
69
+ target_values_1 = Y1[idx1, :].toarray()
70
+ target_values_2 = Y2[idx2, :].toarray()
71
+ assert_almost_equal(target_values_1, target_values_2)
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_samples_generator.py ADDED
@@ -0,0 +1,744 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from collections import defaultdict
3
+ from functools import partial
4
+
5
+ import numpy as np
6
+ import pytest
7
+ import scipy.sparse as sp
8
+
9
+ from sklearn.datasets import (
10
+ make_biclusters,
11
+ make_blobs,
12
+ make_checkerboard,
13
+ make_circles,
14
+ make_classification,
15
+ make_friedman1,
16
+ make_friedman2,
17
+ make_friedman3,
18
+ make_hastie_10_2,
19
+ make_low_rank_matrix,
20
+ make_moons,
21
+ make_multilabel_classification,
22
+ make_regression,
23
+ make_s_curve,
24
+ make_sparse_coded_signal,
25
+ make_sparse_spd_matrix,
26
+ make_sparse_uncorrelated,
27
+ make_spd_matrix,
28
+ make_swiss_roll,
29
+ )
30
+ from sklearn.utils._testing import (
31
+ assert_allclose,
32
+ assert_allclose_dense_sparse,
33
+ assert_almost_equal,
34
+ assert_array_almost_equal,
35
+ assert_array_equal,
36
+ )
37
+ from sklearn.utils.validation import assert_all_finite
38
+
39
+
40
+ def test_make_classification():
41
+ weights = [0.1, 0.25]
42
+ X, y = make_classification(
43
+ n_samples=100,
44
+ n_features=20,
45
+ n_informative=5,
46
+ n_redundant=1,
47
+ n_repeated=1,
48
+ n_classes=3,
49
+ n_clusters_per_class=1,
50
+ hypercube=False,
51
+ shift=None,
52
+ scale=None,
53
+ weights=weights,
54
+ random_state=0,
55
+ )
56
+
57
+ assert weights == [0.1, 0.25]
58
+ assert X.shape == (100, 20), "X shape mismatch"
59
+ assert y.shape == (100,), "y shape mismatch"
60
+ assert np.unique(y).shape == (3,), "Unexpected number of classes"
61
+ assert sum(y == 0) == 10, "Unexpected number of samples in class #0"
62
+ assert sum(y == 1) == 25, "Unexpected number of samples in class #1"
63
+ assert sum(y == 2) == 65, "Unexpected number of samples in class #2"
64
+
65
+ # Test for n_features > 30
66
+ X, y = make_classification(
67
+ n_samples=2000,
68
+ n_features=31,
69
+ n_informative=31,
70
+ n_redundant=0,
71
+ n_repeated=0,
72
+ hypercube=True,
73
+ scale=0.5,
74
+ random_state=0,
75
+ )
76
+
77
+ assert X.shape == (2000, 31), "X shape mismatch"
78
+ assert y.shape == (2000,), "y shape mismatch"
79
+ assert (
80
+ np.unique(X.view([("", X.dtype)] * X.shape[1]))
81
+ .view(X.dtype)
82
+ .reshape(-1, X.shape[1])
83
+ .shape[0]
84
+ == 2000
85
+ ), "Unexpected number of unique rows"
86
+
87
+
88
+ def test_make_classification_informative_features():
89
+ """Test the construction of informative features in make_classification
90
+
91
+ Also tests `n_clusters_per_class`, `n_classes`, `hypercube` and
92
+ fully-specified `weights`.
93
+ """
94
+ # Create very separate clusters; check that vertices are unique and
95
+ # correspond to classes
96
+ class_sep = 1e6
97
+ make = partial(
98
+ make_classification,
99
+ class_sep=class_sep,
100
+ n_redundant=0,
101
+ n_repeated=0,
102
+ flip_y=0,
103
+ shift=0,
104
+ scale=1,
105
+ shuffle=False,
106
+ )
107
+
108
+ for n_informative, weights, n_clusters_per_class in [
109
+ (2, [1], 1),
110
+ (2, [1 / 3] * 3, 1),
111
+ (2, [1 / 4] * 4, 1),
112
+ (2, [1 / 2] * 2, 2),
113
+ (2, [3 / 4, 1 / 4], 2),
114
+ (10, [1 / 3] * 3, 10),
115
+ (64, [1], 1),
116
+ ]:
117
+ n_classes = len(weights)
118
+ n_clusters = n_classes * n_clusters_per_class
119
+ n_samples = n_clusters * 50
120
+
121
+ for hypercube in (False, True):
122
+ X, y = make(
123
+ n_samples=n_samples,
124
+ n_classes=n_classes,
125
+ weights=weights,
126
+ n_features=n_informative,
127
+ n_informative=n_informative,
128
+ n_clusters_per_class=n_clusters_per_class,
129
+ hypercube=hypercube,
130
+ random_state=0,
131
+ )
132
+
133
+ assert X.shape == (n_samples, n_informative)
134
+ assert y.shape == (n_samples,)
135
+
136
+ # Cluster by sign, viewed as strings to allow uniquing
137
+ signs = np.sign(X)
138
+ signs = signs.view(dtype="|S{0}".format(signs.strides[0])).ravel()
139
+ unique_signs, cluster_index = np.unique(signs, return_inverse=True)
140
+
141
+ assert len(unique_signs) == n_clusters, (
142
+ "Wrong number of clusters, or not in distinct quadrants"
143
+ )
144
+
145
+ clusters_by_class = defaultdict(set)
146
+ for cluster, cls in zip(cluster_index, y):
147
+ clusters_by_class[cls].add(cluster)
148
+ for clusters in clusters_by_class.values():
149
+ assert len(clusters) == n_clusters_per_class, (
150
+ "Wrong number of clusters per class"
151
+ )
152
+ assert len(clusters_by_class) == n_classes, "Wrong number of classes"
153
+
154
+ assert_array_almost_equal(
155
+ np.bincount(y) / len(y) // weights,
156
+ [1] * n_classes,
157
+ err_msg="Wrong number of samples per class",
158
+ )
159
+
160
+ # Ensure on vertices of hypercube
161
+ for cluster in range(len(unique_signs)):
162
+ centroid = X[cluster_index == cluster].mean(axis=0)
163
+ if hypercube:
164
+ assert_array_almost_equal(
165
+ np.abs(centroid) / class_sep,
166
+ np.ones(n_informative),
167
+ decimal=5,
168
+ err_msg="Clusters are not centered on hypercube vertices",
169
+ )
170
+ else:
171
+ with pytest.raises(AssertionError):
172
+ assert_array_almost_equal(
173
+ np.abs(centroid) / class_sep,
174
+ np.ones(n_informative),
175
+ decimal=5,
176
+ err_msg=(
177
+ "Clusters should not be centered on hypercube vertices"
178
+ ),
179
+ )
180
+
181
+ with pytest.raises(ValueError):
182
+ make(n_features=2, n_informative=2, n_classes=5, n_clusters_per_class=1)
183
+ with pytest.raises(ValueError):
184
+ make(n_features=2, n_informative=2, n_classes=3, n_clusters_per_class=2)
185
+
186
+
187
+ def test_make_classification_return_x_y():
188
+ """
189
+ Test that make_classification returns a Bunch when return_X_y is False.
190
+
191
+ Also that bunch.X is the same as X
192
+ """
193
+
194
+ kwargs = {
195
+ "n_samples": 100,
196
+ "n_features": 20,
197
+ "n_informative": 5,
198
+ "n_redundant": 1,
199
+ "n_repeated": 1,
200
+ "n_classes": 3,
201
+ "n_clusters_per_class": 2,
202
+ "weights": None,
203
+ "flip_y": 0.01,
204
+ "class_sep": 1.0,
205
+ "hypercube": True,
206
+ "shift": 0.0,
207
+ "scale": 1.0,
208
+ "shuffle": True,
209
+ "random_state": 42,
210
+ "return_X_y": True,
211
+ }
212
+
213
+ X, y = make_classification(**kwargs)
214
+
215
+ kwargs["return_X_y"] = False
216
+ bunch = make_classification(**kwargs)
217
+
218
+ assert (
219
+ hasattr(bunch, "DESCR")
220
+ and hasattr(bunch, "parameters")
221
+ and hasattr(bunch, "feature_info")
222
+ and hasattr(bunch, "X")
223
+ and hasattr(bunch, "y")
224
+ )
225
+
226
+ def count(str_):
227
+ return bunch.feature_info.count(str_)
228
+
229
+ assert np.array_equal(X, bunch.X)
230
+ assert np.array_equal(y, bunch.y)
231
+ assert bunch.DESCR == make_classification.__doc__
232
+ assert bunch.parameters == kwargs
233
+ assert count("informative") == kwargs["n_informative"]
234
+ assert count("redundant") == kwargs["n_redundant"]
235
+ assert count("repeated") == kwargs["n_repeated"]
236
+
237
+
238
+ @pytest.mark.parametrize(
239
+ "weights, err_type, err_msg",
240
+ [
241
+ ([], ValueError, "Weights specified but incompatible with number of classes."),
242
+ (
243
+ [0.25, 0.75, 0.1],
244
+ ValueError,
245
+ "Weights specified but incompatible with number of classes.",
246
+ ),
247
+ (
248
+ np.array([]),
249
+ ValueError,
250
+ "Weights specified but incompatible with number of classes.",
251
+ ),
252
+ (
253
+ np.array([0.25, 0.75, 0.1]),
254
+ ValueError,
255
+ "Weights specified but incompatible with number of classes.",
256
+ ),
257
+ (
258
+ np.random.random(3),
259
+ ValueError,
260
+ "Weights specified but incompatible with number of classes.",
261
+ ),
262
+ ],
263
+ )
264
+ def test_make_classification_weights_type(weights, err_type, err_msg):
265
+ with pytest.raises(err_type, match=err_msg):
266
+ make_classification(weights=weights)
267
+
268
+
269
+ @pytest.mark.parametrize("kwargs", [{}, {"n_classes": 3, "n_informative": 3}])
270
+ def test_make_classification_weights_array_or_list_ok(kwargs):
271
+ X1, y1 = make_classification(weights=[0.1, 0.9], random_state=0, **kwargs)
272
+ X2, y2 = make_classification(weights=np.array([0.1, 0.9]), random_state=0, **kwargs)
273
+ assert_almost_equal(X1, X2)
274
+ assert_almost_equal(y1, y2)
275
+
276
+
277
+ def test_make_multilabel_classification_return_sequences():
278
+ for allow_unlabeled, min_length in zip((True, False), (0, 1)):
279
+ X, Y = make_multilabel_classification(
280
+ n_samples=100,
281
+ n_features=20,
282
+ n_classes=3,
283
+ random_state=0,
284
+ return_indicator=False,
285
+ allow_unlabeled=allow_unlabeled,
286
+ )
287
+ assert X.shape == (100, 20), "X shape mismatch"
288
+ if not allow_unlabeled:
289
+ assert max([max(y) for y in Y]) == 2
290
+ assert min([len(y) for y in Y]) == min_length
291
+ assert max([len(y) for y in Y]) <= 3
292
+
293
+
294
+ def test_make_multilabel_classification_return_indicator():
295
+ for allow_unlabeled, min_length in zip((True, False), (0, 1)):
296
+ X, Y = make_multilabel_classification(
297
+ n_samples=25,
298
+ n_features=20,
299
+ n_classes=3,
300
+ random_state=0,
301
+ allow_unlabeled=allow_unlabeled,
302
+ )
303
+ assert X.shape == (25, 20), "X shape mismatch"
304
+ assert Y.shape == (25, 3), "Y shape mismatch"
305
+ assert np.all(np.sum(Y, axis=0) > min_length)
306
+
307
+ # Also test return_distributions and return_indicator with True
308
+ X2, Y2, p_c, p_w_c = make_multilabel_classification(
309
+ n_samples=25,
310
+ n_features=20,
311
+ n_classes=3,
312
+ random_state=0,
313
+ allow_unlabeled=allow_unlabeled,
314
+ return_distributions=True,
315
+ )
316
+
317
+ assert_array_almost_equal(X, X2)
318
+ assert_array_equal(Y, Y2)
319
+ assert p_c.shape == (3,)
320
+ assert_almost_equal(p_c.sum(), 1)
321
+ assert p_w_c.shape == (20, 3)
322
+ assert_almost_equal(p_w_c.sum(axis=0), [1] * 3)
323
+
324
+
325
+ def test_make_multilabel_classification_return_indicator_sparse():
326
+ for allow_unlabeled, min_length in zip((True, False), (0, 1)):
327
+ X, Y = make_multilabel_classification(
328
+ n_samples=25,
329
+ n_features=20,
330
+ n_classes=3,
331
+ random_state=0,
332
+ return_indicator="sparse",
333
+ allow_unlabeled=allow_unlabeled,
334
+ )
335
+ assert X.shape == (25, 20), "X shape mismatch"
336
+ assert Y.shape == (25, 3), "Y shape mismatch"
337
+ assert sp.issparse(Y)
338
+
339
+
340
+ def test_make_hastie_10_2():
341
+ X, y = make_hastie_10_2(n_samples=100, random_state=0)
342
+ assert X.shape == (100, 10), "X shape mismatch"
343
+ assert y.shape == (100,), "y shape mismatch"
344
+ assert np.unique(y).shape == (2,), "Unexpected number of classes"
345
+
346
+
347
+ def test_make_regression(global_random_seed):
348
+ X, y, c = make_regression(
349
+ n_samples=200,
350
+ n_features=10,
351
+ n_informative=3,
352
+ effective_rank=5,
353
+ coef=True,
354
+ bias=0.0,
355
+ noise=1.0,
356
+ random_state=global_random_seed,
357
+ )
358
+
359
+ assert X.shape == (200, 10), "X shape mismatch"
360
+ assert y.shape == (200,), "y shape mismatch"
361
+ assert c.shape == (10,), "coef shape mismatch"
362
+ assert sum(c != 0.0) == 3, "Unexpected number of informative features"
363
+
364
+ # Test that y ~= np.dot(X, c) + bias + N(0, 1.0).
365
+ assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1)
366
+
367
+ # Test with small number of features.
368
+ X, y = make_regression(n_samples=100, n_features=1) # n_informative=3
369
+ assert X.shape == (100, 1)
370
+
371
+
372
+ def test_make_regression_multitarget(global_random_seed):
373
+ X, y, c = make_regression(
374
+ n_samples=100,
375
+ n_features=10,
376
+ n_informative=3,
377
+ n_targets=3,
378
+ coef=True,
379
+ noise=1.0,
380
+ random_state=global_random_seed,
381
+ )
382
+
383
+ assert X.shape == (100, 10), "X shape mismatch"
384
+ assert y.shape == (100, 3), "y shape mismatch"
385
+ assert c.shape == (10, 3), "coef shape mismatch"
386
+ assert_array_equal(sum(c != 0.0), 3, "Unexpected number of informative features")
387
+
388
+ # Test that y ~= np.dot(X, c) + bias + N(0, 1.0)
389
+ assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1)
390
+
391
+
392
+ def test_make_blobs(global_random_seed):
393
+ cluster_stds = np.array([0.05, 0.2, 0.4])
394
+ cluster_centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
395
+ X, y = make_blobs(
396
+ random_state=global_random_seed,
397
+ n_samples=50,
398
+ n_features=2,
399
+ centers=cluster_centers,
400
+ cluster_std=cluster_stds,
401
+ )
402
+
403
+ assert X.shape == (50, 2), "X shape mismatch"
404
+ assert y.shape == (50,), "y shape mismatch"
405
+ assert np.unique(y).shape == (3,), "Unexpected number of blobs"
406
+ for i, (ctr, std) in enumerate(zip(cluster_centers, cluster_stds)):
407
+ assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std")
408
+
409
+
410
+ def test_make_blobs_n_samples_list():
411
+ n_samples = [50, 30, 20]
412
+ X, y = make_blobs(n_samples=n_samples, n_features=2, random_state=0)
413
+
414
+ assert X.shape == (sum(n_samples), 2), "X shape mismatch"
415
+ assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), (
416
+ "Incorrect number of samples per blob"
417
+ )
418
+
419
+
420
+ def test_make_blobs_n_samples_list_with_centers(global_random_seed):
421
+ n_samples = [20, 20, 20]
422
+ centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
423
+ cluster_stds = np.array([0.05, 0.2, 0.4])
424
+ X, y = make_blobs(
425
+ n_samples=n_samples,
426
+ centers=centers,
427
+ cluster_std=cluster_stds,
428
+ random_state=global_random_seed,
429
+ )
430
+
431
+ assert X.shape == (sum(n_samples), 2), "X shape mismatch"
432
+ assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), (
433
+ "Incorrect number of samples per blob"
434
+ )
435
+ for i, (ctr, std) in enumerate(zip(centers, cluster_stds)):
436
+ assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std")
437
+
438
+
439
+ @pytest.mark.parametrize(
440
+ "n_samples", [[5, 3, 0], np.array([5, 3, 0]), tuple([5, 3, 0])]
441
+ )
442
+ def test_make_blobs_n_samples_centers_none(n_samples):
443
+ centers = None
444
+ X, y = make_blobs(n_samples=n_samples, centers=centers, random_state=0)
445
+
446
+ assert X.shape == (sum(n_samples), 2), "X shape mismatch"
447
+ assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), (
448
+ "Incorrect number of samples per blob"
449
+ )
450
+
451
+
452
+ def test_make_blobs_return_centers():
453
+ n_samples = [10, 20]
454
+ n_features = 3
455
+ X, y, centers = make_blobs(
456
+ n_samples=n_samples, n_features=n_features, return_centers=True, random_state=0
457
+ )
458
+
459
+ assert centers.shape == (len(n_samples), n_features)
460
+
461
+
462
+ def test_make_blobs_error():
463
+ n_samples = [20, 20, 20]
464
+ centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
465
+ cluster_stds = np.array([0.05, 0.2, 0.4])
466
+ wrong_centers_msg = re.escape(
467
+ "Length of `n_samples` not consistent with number of centers. "
468
+ f"Got n_samples = {n_samples} and centers = {centers[:-1]}"
469
+ )
470
+ with pytest.raises(ValueError, match=wrong_centers_msg):
471
+ make_blobs(n_samples, centers=centers[:-1])
472
+ wrong_std_msg = re.escape(
473
+ "Length of `clusters_std` not consistent with number of centers. "
474
+ f"Got centers = {centers} and cluster_std = {cluster_stds[:-1]}"
475
+ )
476
+ with pytest.raises(ValueError, match=wrong_std_msg):
477
+ make_blobs(n_samples, centers=centers, cluster_std=cluster_stds[:-1])
478
+ wrong_type_msg = "Parameter `centers` must be array-like. Got {!r} instead".format(
479
+ 3
480
+ )
481
+ with pytest.raises(ValueError, match=wrong_type_msg):
482
+ make_blobs(n_samples, centers=3)
483
+
484
+
485
+ def test_make_friedman1(global_random_seed):
486
+ X, y = make_friedman1(
487
+ n_samples=5, n_features=10, noise=0.0, random_state=global_random_seed
488
+ )
489
+
490
+ assert X.shape == (5, 10), "X shape mismatch"
491
+ assert y.shape == (5,), "y shape mismatch"
492
+
493
+ assert_array_almost_equal(
494
+ y,
495
+ 10 * np.sin(np.pi * X[:, 0] * X[:, 1])
496
+ + 20 * (X[:, 2] - 0.5) ** 2
497
+ + 10 * X[:, 3]
498
+ + 5 * X[:, 4],
499
+ )
500
+
501
+
502
+ def test_make_friedman2(global_random_seed):
503
+ X, y = make_friedman2(n_samples=5, noise=0.0, random_state=global_random_seed)
504
+
505
+ assert X.shape == (5, 4), "X shape mismatch"
506
+ assert y.shape == (5,), "y shape mismatch"
507
+
508
+ assert_array_almost_equal(
509
+ y, (X[:, 0] ** 2 + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5
510
+ )
511
+
512
+
513
+ def test_make_friedman3(global_random_seed):
514
+ X, y = make_friedman3(n_samples=5, noise=0.0, random_state=global_random_seed)
515
+
516
+ assert X.shape == (5, 4), "X shape mismatch"
517
+ assert y.shape == (5,), "y shape mismatch"
518
+
519
+ assert_array_almost_equal(
520
+ y, np.arctan((X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) / X[:, 0])
521
+ )
522
+
523
+
524
+ def test_make_low_rank_matrix():
525
+ X = make_low_rank_matrix(
526
+ n_samples=50,
527
+ n_features=25,
528
+ effective_rank=5,
529
+ tail_strength=0.01,
530
+ random_state=0,
531
+ )
532
+
533
+ assert X.shape == (50, 25), "X shape mismatch"
534
+
535
+ from numpy.linalg import svd
536
+
537
+ u, s, v = svd(X)
538
+ assert sum(s) - 5 < 0.1, "X rank is not approximately 5"
539
+
540
+
541
+ def test_make_sparse_coded_signal(global_random_seed):
542
+ Y, D, X = make_sparse_coded_signal(
543
+ n_samples=5,
544
+ n_components=8,
545
+ n_features=10,
546
+ n_nonzero_coefs=3,
547
+ random_state=global_random_seed,
548
+ )
549
+ assert Y.shape == (5, 10), "Y shape mismatch"
550
+ assert D.shape == (8, 10), "D shape mismatch"
551
+ assert X.shape == (5, 8), "X shape mismatch"
552
+ for row in X:
553
+ assert len(np.flatnonzero(row)) == 3, "Non-zero coefs mismatch"
554
+ assert_allclose(Y, X @ D)
555
+ assert_allclose(np.sqrt((D**2).sum(axis=1)), np.ones(D.shape[0]))
556
+
557
+
558
+ def test_make_sparse_uncorrelated():
559
+ X, y = make_sparse_uncorrelated(n_samples=5, n_features=10, random_state=0)
560
+
561
+ assert X.shape == (5, 10), "X shape mismatch"
562
+ assert y.shape == (5,), "y shape mismatch"
563
+
564
+
565
+ def test_make_spd_matrix(global_random_seed):
566
+ X = make_spd_matrix(n_dim=5, random_state=global_random_seed)
567
+
568
+ assert X.shape == (5, 5), "X shape mismatch"
569
+ assert_array_almost_equal(X, X.T)
570
+
571
+ from numpy.linalg import eig
572
+
573
+ eigenvalues, _ = eig(X)
574
+ assert np.all(eigenvalues > 0), "X is not positive-definite"
575
+
576
+
577
+ @pytest.mark.parametrize("norm_diag", [True, False])
578
+ @pytest.mark.parametrize(
579
+ "sparse_format", [None, "bsr", "coo", "csc", "csr", "dia", "dok", "lil"]
580
+ )
581
+ def test_make_sparse_spd_matrix(norm_diag, sparse_format, global_random_seed):
582
+ n_dim = 5
583
+ X = make_sparse_spd_matrix(
584
+ n_dim=n_dim,
585
+ norm_diag=norm_diag,
586
+ sparse_format=sparse_format,
587
+ random_state=global_random_seed,
588
+ )
589
+
590
+ assert X.shape == (n_dim, n_dim), "X shape mismatch"
591
+ if sparse_format is None:
592
+ assert not sp.issparse(X)
593
+ assert_allclose(X, X.T)
594
+ Xarr = X
595
+ else:
596
+ assert sp.issparse(X) and X.format == sparse_format
597
+ assert_allclose_dense_sparse(X, X.T)
598
+ Xarr = X.toarray()
599
+
600
+ from numpy.linalg import eig
601
+
602
+ # Do not use scipy.sparse.linalg.eigs because it cannot find all eigenvalues
603
+ eigenvalues, _ = eig(Xarr)
604
+ assert np.all(eigenvalues > 0), "X is not positive-definite"
605
+
606
+ if norm_diag:
607
+ # Check that leading diagonal elements are 1
608
+ assert_array_almost_equal(Xarr.diagonal(), np.ones(n_dim))
609
+
610
+
611
+ @pytest.mark.parametrize("hole", [False, True])
612
+ def test_make_swiss_roll(global_random_seed, hole):
613
+ X, t = make_swiss_roll(
614
+ n_samples=5, noise=0.0, random_state=global_random_seed, hole=hole
615
+ )
616
+
617
+ assert X.shape == (5, 3)
618
+ assert t.shape == (5,)
619
+ assert_array_almost_equal(X[:, 0], t * np.cos(t))
620
+ assert_array_almost_equal(X[:, 2], t * np.sin(t))
621
+
622
+
623
+ def test_make_s_curve(global_random_seed):
624
+ X, t = make_s_curve(n_samples=5, noise=0.0, random_state=global_random_seed)
625
+
626
+ assert X.shape == (5, 3), "X shape mismatch"
627
+ assert t.shape == (5,), "t shape mismatch"
628
+ assert_array_almost_equal(X[:, 0], np.sin(t))
629
+ assert_array_almost_equal(X[:, 2], np.sign(t) * (np.cos(t) - 1))
630
+
631
+
632
+ def test_make_biclusters():
633
+ X, rows, cols = make_biclusters(
634
+ shape=(100, 100), n_clusters=4, shuffle=True, random_state=0
635
+ )
636
+ assert X.shape == (100, 100), "X shape mismatch"
637
+ assert rows.shape == (4, 100), "rows shape mismatch"
638
+ assert cols.shape == (
639
+ 4,
640
+ 100,
641
+ ), "columns shape mismatch"
642
+ assert_all_finite(X)
643
+ assert_all_finite(rows)
644
+ assert_all_finite(cols)
645
+
646
+ X2, _, _ = make_biclusters(
647
+ shape=(100, 100), n_clusters=4, shuffle=True, random_state=0
648
+ )
649
+ assert_array_almost_equal(X, X2)
650
+
651
+
652
+ def test_make_checkerboard():
653
+ X, rows, cols = make_checkerboard(
654
+ shape=(100, 100), n_clusters=(20, 5), shuffle=True, random_state=0
655
+ )
656
+ assert X.shape == (100, 100), "X shape mismatch"
657
+ assert rows.shape == (100, 100), "rows shape mismatch"
658
+ assert cols.shape == (
659
+ 100,
660
+ 100,
661
+ ), "columns shape mismatch"
662
+
663
+ X, rows, cols = make_checkerboard(
664
+ shape=(100, 100), n_clusters=2, shuffle=True, random_state=0
665
+ )
666
+ assert_all_finite(X)
667
+ assert_all_finite(rows)
668
+ assert_all_finite(cols)
669
+
670
+ X1, _, _ = make_checkerboard(
671
+ shape=(100, 100), n_clusters=2, shuffle=True, random_state=0
672
+ )
673
+ X2, _, _ = make_checkerboard(
674
+ shape=(100, 100), n_clusters=2, shuffle=True, random_state=0
675
+ )
676
+ assert_array_almost_equal(X1, X2)
677
+
678
+
679
+ def test_make_moons(global_random_seed):
680
+ X, y = make_moons(3, shuffle=False, random_state=global_random_seed)
681
+ for x, label in zip(X, y):
682
+ center = [0.0, 0.0] if label == 0 else [1.0, 0.5]
683
+ dist_sqr = ((x - center) ** 2).sum()
684
+ assert_almost_equal(
685
+ dist_sqr, 1.0, err_msg="Point is not on expected unit circle"
686
+ )
687
+
688
+
689
+ def test_make_moons_unbalanced():
690
+ X, y = make_moons(n_samples=(7, 5))
691
+ assert np.sum(y == 0) == 7 and np.sum(y == 1) == 5, (
692
+ "Number of samples in a moon is wrong"
693
+ )
694
+ assert X.shape == (12, 2), "X shape mismatch"
695
+ assert y.shape == (12,), "y shape mismatch"
696
+
697
+ with pytest.raises(
698
+ ValueError,
699
+ match=r"`n_samples` can be either an int or a two-element tuple.",
700
+ ):
701
+ make_moons(n_samples=(10,))
702
+
703
+
704
+ def test_make_circles():
705
+ factor = 0.3
706
+
707
+ for n_samples, n_outer, n_inner in [(7, 3, 4), (8, 4, 4)]:
708
+ # Testing odd and even case, because in the past make_circles always
709
+ # created an even number of samples.
710
+ X, y = make_circles(n_samples, shuffle=False, noise=None, factor=factor)
711
+ assert X.shape == (n_samples, 2), "X shape mismatch"
712
+ assert y.shape == (n_samples,), "y shape mismatch"
713
+ center = [0.0, 0.0]
714
+ for x, label in zip(X, y):
715
+ dist_sqr = ((x - center) ** 2).sum()
716
+ dist_exp = 1.0 if label == 0 else factor**2
717
+ dist_exp = 1.0 if label == 0 else factor**2
718
+ assert_almost_equal(
719
+ dist_sqr, dist_exp, err_msg="Point is not on expected circle"
720
+ )
721
+
722
+ assert X[y == 0].shape == (
723
+ n_outer,
724
+ 2,
725
+ ), "Samples not correctly distributed across circles."
726
+ assert X[y == 1].shape == (
727
+ n_inner,
728
+ 2,
729
+ ), "Samples not correctly distributed across circles."
730
+
731
+
732
+ def test_make_circles_unbalanced():
733
+ X, y = make_circles(n_samples=(2, 8))
734
+
735
+ assert np.sum(y == 0) == 2, "Number of samples in inner circle is wrong"
736
+ assert np.sum(y == 1) == 8, "Number of samples in outer circle is wrong"
737
+ assert X.shape == (10, 2), "X shape mismatch"
738
+ assert y.shape == (10,), "y shape mismatch"
739
+
740
+ with pytest.raises(
741
+ ValueError,
742
+ match="When a tuple, n_samples must have exactly two elements.",
743
+ ):
744
+ make_circles(n_samples=(10,))
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_svmlight_format.py ADDED
@@ -0,0 +1,613 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gzip
2
+ import os
3
+ import shutil
4
+ from bz2 import BZ2File
5
+ from importlib import resources
6
+ from io import BytesIO
7
+ from tempfile import NamedTemporaryFile
8
+
9
+ import numpy as np
10
+ import pytest
11
+ import scipy.sparse as sp
12
+
13
+ import sklearn
14
+ from sklearn.datasets import dump_svmlight_file, load_svmlight_file, load_svmlight_files
15
+ from sklearn.utils._testing import (
16
+ assert_allclose,
17
+ assert_array_almost_equal,
18
+ assert_array_equal,
19
+ create_memmap_backed_data,
20
+ )
21
+ from sklearn.utils.fixes import CSR_CONTAINERS
22
+
23
+ TEST_DATA_MODULE = "sklearn.datasets.tests.data"
24
+ datafile = "svmlight_classification.txt"
25
+ multifile = "svmlight_multilabel.txt"
26
+ invalidfile = "svmlight_invalid.txt"
27
+ invalidfile2 = "svmlight_invalid_order.txt"
28
+
29
+
30
+ def _svmlight_local_test_file_path(filename):
31
+ return resources.files(TEST_DATA_MODULE) / filename
32
+
33
+
34
+ def _load_svmlight_local_test_file(filename, **kwargs):
35
+ """
36
+ Helper to load resource `filename` with `importlib.resources`
37
+ """
38
+ data_path = _svmlight_local_test_file_path(filename)
39
+ with data_path.open("rb") as f:
40
+ return load_svmlight_file(f, **kwargs)
41
+
42
+
43
+ def test_load_svmlight_file():
44
+ X, y = _load_svmlight_local_test_file(datafile)
45
+
46
+ # test X's shape
47
+ assert X.indptr.shape[0] == 7
48
+ assert X.shape[0] == 6
49
+ assert X.shape[1] == 21
50
+ assert y.shape[0] == 6
51
+
52
+ # test X's non-zero values
53
+ for i, j, val in (
54
+ (0, 2, 2.5),
55
+ (0, 10, -5.2),
56
+ (0, 15, 1.5),
57
+ (1, 5, 1.0),
58
+ (1, 12, -3),
59
+ (2, 20, 27),
60
+ ):
61
+ assert X[i, j] == val
62
+
63
+ # tests X's zero values
64
+ assert X[0, 3] == 0
65
+ assert X[0, 5] == 0
66
+ assert X[1, 8] == 0
67
+ assert X[1, 16] == 0
68
+ assert X[2, 18] == 0
69
+
70
+ # test can change X's values
71
+ X[0, 2] *= 2
72
+ assert X[0, 2] == 5
73
+
74
+ # test y
75
+ assert_array_equal(y, [1, 2, 3, 4, 1, 2])
76
+
77
+
78
+ def test_load_svmlight_file_fd():
79
+ # test loading from file descriptor
80
+
81
+ # GH20081: testing equality between path-based and
82
+ # fd-based load_svmlight_file
83
+
84
+ data_path = resources.files(TEST_DATA_MODULE) / datafile
85
+ data_path = str(data_path)
86
+ X1, y1 = load_svmlight_file(data_path)
87
+
88
+ fd = os.open(data_path, os.O_RDONLY)
89
+ try:
90
+ X2, y2 = load_svmlight_file(fd)
91
+ assert_array_almost_equal(X1.data, X2.data)
92
+ assert_array_almost_equal(y1, y2)
93
+ finally:
94
+ os.close(fd)
95
+
96
+
97
+ def test_load_svmlight_pathlib():
98
+ # test loading from file descriptor
99
+ data_path = _svmlight_local_test_file_path(datafile)
100
+ X1, y1 = load_svmlight_file(str(data_path))
101
+ X2, y2 = load_svmlight_file(data_path)
102
+
103
+ assert_allclose(X1.data, X2.data)
104
+ assert_allclose(y1, y2)
105
+
106
+
107
+ def test_load_svmlight_file_multilabel():
108
+ X, y = _load_svmlight_local_test_file(multifile, multilabel=True)
109
+ assert y == [(0, 1), (2,), (), (1, 2)]
110
+
111
+
112
+ def test_load_svmlight_files():
113
+ data_path = _svmlight_local_test_file_path(datafile)
114
+ X_train, y_train, X_test, y_test = load_svmlight_files(
115
+ [str(data_path)] * 2, dtype=np.float32
116
+ )
117
+ assert_array_equal(X_train.toarray(), X_test.toarray())
118
+ assert_array_almost_equal(y_train, y_test)
119
+ assert X_train.dtype == np.float32
120
+ assert X_test.dtype == np.float32
121
+
122
+ X1, y1, X2, y2, X3, y3 = load_svmlight_files([str(data_path)] * 3, dtype=np.float64)
123
+ assert X1.dtype == X2.dtype
124
+ assert X2.dtype == X3.dtype
125
+ assert X3.dtype == np.float64
126
+
127
+
128
+ def test_load_svmlight_file_n_features():
129
+ X, y = _load_svmlight_local_test_file(datafile, n_features=22)
130
+
131
+ # test X'shape
132
+ assert X.indptr.shape[0] == 7
133
+ assert X.shape[0] == 6
134
+ assert X.shape[1] == 22
135
+
136
+ # test X's non-zero values
137
+ for i, j, val in ((0, 2, 2.5), (0, 10, -5.2), (1, 5, 1.0), (1, 12, -3)):
138
+ assert X[i, j] == val
139
+
140
+ # 21 features in file
141
+ with pytest.raises(ValueError):
142
+ _load_svmlight_local_test_file(datafile, n_features=20)
143
+
144
+
145
+ def test_load_compressed():
146
+ X, y = _load_svmlight_local_test_file(datafile)
147
+
148
+ with NamedTemporaryFile(prefix="sklearn-test", suffix=".gz") as tmp:
149
+ tmp.close() # necessary under windows
150
+ with _svmlight_local_test_file_path(datafile).open("rb") as f:
151
+ with gzip.open(tmp.name, "wb") as fh_out:
152
+ shutil.copyfileobj(f, fh_out)
153
+ Xgz, ygz = load_svmlight_file(tmp.name)
154
+ # because we "close" it manually and write to it,
155
+ # we need to remove it manually.
156
+ os.remove(tmp.name)
157
+ assert_array_almost_equal(X.toarray(), Xgz.toarray())
158
+ assert_array_almost_equal(y, ygz)
159
+
160
+ with NamedTemporaryFile(prefix="sklearn-test", suffix=".bz2") as tmp:
161
+ tmp.close() # necessary under windows
162
+ with _svmlight_local_test_file_path(datafile).open("rb") as f:
163
+ with BZ2File(tmp.name, "wb") as fh_out:
164
+ shutil.copyfileobj(f, fh_out)
165
+ Xbz, ybz = load_svmlight_file(tmp.name)
166
+ # because we "close" it manually and write to it,
167
+ # we need to remove it manually.
168
+ os.remove(tmp.name)
169
+ assert_array_almost_equal(X.toarray(), Xbz.toarray())
170
+ assert_array_almost_equal(y, ybz)
171
+
172
+
173
+ def test_load_invalid_file():
174
+ with pytest.raises(ValueError):
175
+ _load_svmlight_local_test_file(invalidfile)
176
+
177
+
178
+ def test_load_invalid_order_file():
179
+ with pytest.raises(ValueError):
180
+ _load_svmlight_local_test_file(invalidfile2)
181
+
182
+
183
+ def test_load_zero_based():
184
+ f = BytesIO(b"-1 4:1.\n1 0:1\n")
185
+ with pytest.raises(ValueError):
186
+ load_svmlight_file(f, zero_based=False)
187
+
188
+
189
+ def test_load_zero_based_auto():
190
+ data1 = b"-1 1:1 2:2 3:3\n"
191
+ data2 = b"-1 0:0 1:1\n"
192
+
193
+ f1 = BytesIO(data1)
194
+ X, y = load_svmlight_file(f1, zero_based="auto")
195
+ assert X.shape == (1, 3)
196
+
197
+ f1 = BytesIO(data1)
198
+ f2 = BytesIO(data2)
199
+ X1, y1, X2, y2 = load_svmlight_files([f1, f2], zero_based="auto")
200
+ assert X1.shape == (1, 4)
201
+ assert X2.shape == (1, 4)
202
+
203
+
204
+ def test_load_with_qid():
205
+ # load svmfile with qid attribute
206
+ data = b"""
207
+ 3 qid:1 1:0.53 2:0.12
208
+ 2 qid:1 1:0.13 2:0.1
209
+ 7 qid:2 1:0.87 2:0.12"""
210
+ X, y = load_svmlight_file(BytesIO(data), query_id=False)
211
+ assert_array_equal(y, [3, 2, 7])
212
+ assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]])
213
+ res1 = load_svmlight_files([BytesIO(data)], query_id=True)
214
+ res2 = load_svmlight_file(BytesIO(data), query_id=True)
215
+ for X, y, qid in (res1, res2):
216
+ assert_array_equal(y, [3, 2, 7])
217
+ assert_array_equal(qid, [1, 1, 2])
218
+ assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]])
219
+
220
+
221
+ @pytest.mark.skip(
222
+ "testing the overflow of 32 bit sparse indexing requires a large amount of memory"
223
+ )
224
+ def test_load_large_qid():
225
+ """
226
+ load large libsvm / svmlight file with qid attribute. Tests 64-bit query ID
227
+ """
228
+ data = b"\n".join(
229
+ (
230
+ "3 qid:{0} 1:0.53 2:0.12\n2 qid:{0} 1:0.13 2:0.1".format(i).encode()
231
+ for i in range(1, 40 * 1000 * 1000)
232
+ )
233
+ )
234
+ X, y, qid = load_svmlight_file(BytesIO(data), query_id=True)
235
+ assert_array_equal(y[-4:], [3, 2, 3, 2])
236
+ assert_array_equal(np.unique(qid), np.arange(1, 40 * 1000 * 1000))
237
+
238
+
239
+ def test_load_invalid_file2():
240
+ with pytest.raises(ValueError):
241
+ data_path = _svmlight_local_test_file_path(datafile)
242
+ invalid_path = _svmlight_local_test_file_path(invalidfile)
243
+ load_svmlight_files([str(data_path), str(invalid_path), str(data_path)])
244
+
245
+
246
+ def test_not_a_filename():
247
+ # in python 3 integers are valid file opening arguments (taken as unix
248
+ # file descriptors)
249
+ with pytest.raises(TypeError):
250
+ load_svmlight_file(0.42)
251
+
252
+
253
+ def test_invalid_filename():
254
+ with pytest.raises(OSError):
255
+ load_svmlight_file("trou pic nic douille")
256
+
257
+
258
+ @pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
259
+ def test_dump(csr_container):
260
+ X_sparse, y_dense = _load_svmlight_local_test_file(datafile)
261
+ X_dense = X_sparse.toarray()
262
+ y_sparse = csr_container(np.atleast_2d(y_dense))
263
+
264
+ # slicing a csr_matrix can unsort its .indices, so test that we sort
265
+ # those correctly
266
+ X_sliced = X_sparse[np.arange(X_sparse.shape[0])]
267
+ y_sliced = y_sparse[np.arange(y_sparse.shape[0])]
268
+
269
+ for X in (X_sparse, X_dense, X_sliced):
270
+ for y in (y_sparse, y_dense, y_sliced):
271
+ for zero_based in (True, False):
272
+ for dtype in [np.float32, np.float64, np.int32, np.int64]:
273
+ f = BytesIO()
274
+ # we need to pass a comment to get the version info in;
275
+ # LibSVM doesn't grok comments so they're not put in by
276
+ # default anymore.
277
+
278
+ if sp.issparse(y) and y.shape[0] == 1:
279
+ # make sure y's shape is: (n_samples, n_labels)
280
+ # when it is sparse
281
+ y = y.T
282
+
283
+ # Note: with dtype=np.int32 we are performing unsafe casts,
284
+ # where X.astype(dtype) overflows. The result is
285
+ # then platform dependent and X_dense.astype(dtype) may be
286
+ # different from X_sparse.astype(dtype).asarray().
287
+ X_input = X.astype(dtype)
288
+
289
+ dump_svmlight_file(
290
+ X_input, y, f, comment="test", zero_based=zero_based
291
+ )
292
+ f.seek(0)
293
+
294
+ comment = f.readline()
295
+ comment = str(comment, "utf-8")
296
+
297
+ assert "scikit-learn %s" % sklearn.__version__ in comment
298
+
299
+ comment = f.readline()
300
+ comment = str(comment, "utf-8")
301
+
302
+ assert ["one", "zero"][zero_based] + "-based" in comment
303
+
304
+ X2, y2 = load_svmlight_file(f, dtype=dtype, zero_based=zero_based)
305
+ assert X2.dtype == dtype
306
+ assert_array_equal(X2.sorted_indices().indices, X2.indices)
307
+
308
+ X2_dense = X2.toarray()
309
+ if sp.issparse(X_input):
310
+ X_input_dense = X_input.toarray()
311
+ else:
312
+ X_input_dense = X_input
313
+
314
+ if dtype == np.float32:
315
+ # allow a rounding error at the last decimal place
316
+ assert_array_almost_equal(X_input_dense, X2_dense, 4)
317
+ assert_array_almost_equal(
318
+ y_dense.astype(dtype, copy=False), y2, 4
319
+ )
320
+ else:
321
+ # allow a rounding error at the last decimal place
322
+ assert_array_almost_equal(X_input_dense, X2_dense, 15)
323
+ assert_array_almost_equal(
324
+ y_dense.astype(dtype, copy=False), y2, 15
325
+ )
326
+
327
+
328
+ @pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
329
+ def test_dump_multilabel(csr_container):
330
+ X = [[1, 0, 3, 0, 5], [0, 0, 0, 0, 0], [0, 5, 0, 1, 0]]
331
+ y_dense = [[0, 1, 0], [1, 0, 1], [1, 1, 0]]
332
+ y_sparse = csr_container(y_dense)
333
+ for y in [y_dense, y_sparse]:
334
+ f = BytesIO()
335
+ dump_svmlight_file(X, y, f, multilabel=True)
336
+ f.seek(0)
337
+ # make sure it dumps multilabel correctly
338
+ assert f.readline() == b"1 0:1 2:3 4:5\n"
339
+ assert f.readline() == b"0,2 \n"
340
+ assert f.readline() == b"0,1 1:5 3:1\n"
341
+
342
+
343
+ def test_dump_concise():
344
+ one = 1
345
+ two = 2.1
346
+ three = 3.01
347
+ exact = 1.000000000000001
348
+ # loses the last decimal place
349
+ almost = 1.0000000000000001
350
+ X = [
351
+ [one, two, three, exact, almost],
352
+ [1e9, 2e18, 3e27, 0, 0],
353
+ [0, 0, 0, 0, 0],
354
+ [0, 0, 0, 0, 0],
355
+ [0, 0, 0, 0, 0],
356
+ ]
357
+ y = [one, two, three, exact, almost]
358
+ f = BytesIO()
359
+ dump_svmlight_file(X, y, f)
360
+ f.seek(0)
361
+ # make sure it's using the most concise format possible
362
+ assert f.readline() == b"1 0:1 1:2.1 2:3.01 3:1.000000000000001 4:1\n"
363
+ assert f.readline() == b"2.1 0:1000000000 1:2e+18 2:3e+27\n"
364
+ assert f.readline() == b"3.01 \n"
365
+ assert f.readline() == b"1.000000000000001 \n"
366
+ assert f.readline() == b"1 \n"
367
+ f.seek(0)
368
+ # make sure it's correct too :)
369
+ X2, y2 = load_svmlight_file(f)
370
+ assert_array_almost_equal(X, X2.toarray())
371
+ assert_array_almost_equal(y, y2)
372
+
373
+
374
+ def test_dump_comment():
375
+ X, y = _load_svmlight_local_test_file(datafile)
376
+ X = X.toarray()
377
+
378
+ f = BytesIO()
379
+ ascii_comment = "This is a comment\nspanning multiple lines."
380
+ dump_svmlight_file(X, y, f, comment=ascii_comment, zero_based=False)
381
+ f.seek(0)
382
+
383
+ X2, y2 = load_svmlight_file(f, zero_based=False)
384
+ assert_array_almost_equal(X, X2.toarray())
385
+ assert_array_almost_equal(y, y2)
386
+
387
+ # XXX we have to update this to support Python 3.x
388
+ utf8_comment = b"It is true that\n\xc2\xbd\xc2\xb2 = \xc2\xbc"
389
+ f = BytesIO()
390
+ with pytest.raises(UnicodeDecodeError):
391
+ dump_svmlight_file(X, y, f, comment=utf8_comment)
392
+
393
+ unicode_comment = utf8_comment.decode("utf-8")
394
+ f = BytesIO()
395
+ dump_svmlight_file(X, y, f, comment=unicode_comment, zero_based=False)
396
+ f.seek(0)
397
+
398
+ X2, y2 = load_svmlight_file(f, zero_based=False)
399
+ assert_array_almost_equal(X, X2.toarray())
400
+ assert_array_almost_equal(y, y2)
401
+
402
+ f = BytesIO()
403
+ with pytest.raises(ValueError):
404
+ dump_svmlight_file(X, y, f, comment="I've got a \0.")
405
+
406
+
407
+ def test_dump_invalid():
408
+ X, y = _load_svmlight_local_test_file(datafile)
409
+
410
+ f = BytesIO()
411
+ y2d = [y]
412
+ with pytest.raises(ValueError):
413
+ dump_svmlight_file(X, y2d, f)
414
+
415
+ f = BytesIO()
416
+ with pytest.raises(ValueError):
417
+ dump_svmlight_file(X, y[:-1], f)
418
+
419
+
420
+ def test_dump_query_id():
421
+ # test dumping a file with query_id
422
+ X, y = _load_svmlight_local_test_file(datafile)
423
+ X = X.toarray()
424
+ query_id = np.arange(X.shape[0]) // 2
425
+ f = BytesIO()
426
+ dump_svmlight_file(X, y, f, query_id=query_id, zero_based=True)
427
+
428
+ f.seek(0)
429
+ X1, y1, query_id1 = load_svmlight_file(f, query_id=True, zero_based=True)
430
+ assert_array_almost_equal(X, X1.toarray())
431
+ assert_array_almost_equal(y, y1)
432
+ assert_array_almost_equal(query_id, query_id1)
433
+
434
+
435
+ def test_load_with_long_qid():
436
+ # load svmfile with longint qid attribute
437
+ data = b"""
438
+ 1 qid:0 0:1 1:2 2:3
439
+ 0 qid:72048431380967004 0:1440446648 1:72048431380967004 2:236784985
440
+ 0 qid:-9223372036854775807 0:1440446648 1:72048431380967004 2:236784985
441
+ 3 qid:9223372036854775807 0:1440446648 1:72048431380967004 2:236784985"""
442
+ X, y, qid = load_svmlight_file(BytesIO(data), query_id=True)
443
+
444
+ true_X = [
445
+ [1, 2, 3],
446
+ [1440446648, 72048431380967004, 236784985],
447
+ [1440446648, 72048431380967004, 236784985],
448
+ [1440446648, 72048431380967004, 236784985],
449
+ ]
450
+
451
+ true_y = [1, 0, 0, 3]
452
+ trueQID = [0, 72048431380967004, -9223372036854775807, 9223372036854775807]
453
+ assert_array_equal(y, true_y)
454
+ assert_array_equal(X.toarray(), true_X)
455
+ assert_array_equal(qid, trueQID)
456
+
457
+ f = BytesIO()
458
+ dump_svmlight_file(X, y, f, query_id=qid, zero_based=True)
459
+ f.seek(0)
460
+ X, y, qid = load_svmlight_file(f, query_id=True, zero_based=True)
461
+ assert_array_equal(y, true_y)
462
+ assert_array_equal(X.toarray(), true_X)
463
+ assert_array_equal(qid, trueQID)
464
+
465
+ f.seek(0)
466
+ X, y = load_svmlight_file(f, query_id=False, zero_based=True)
467
+ assert_array_equal(y, true_y)
468
+ assert_array_equal(X.toarray(), true_X)
469
+
470
+
471
+ @pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
472
+ def test_load_zeros(csr_container):
473
+ f = BytesIO()
474
+ true_X = csr_container(np.zeros(shape=(3, 4)))
475
+ true_y = np.array([0, 1, 0])
476
+ dump_svmlight_file(true_X, true_y, f)
477
+
478
+ for zero_based in ["auto", True, False]:
479
+ f.seek(0)
480
+ X, y = load_svmlight_file(f, n_features=4, zero_based=zero_based)
481
+ assert_array_almost_equal(y, true_y)
482
+ assert_array_almost_equal(X.toarray(), true_X.toarray())
483
+
484
+
485
+ @pytest.mark.parametrize("sparsity", [0, 0.1, 0.5, 0.99, 1])
486
+ @pytest.mark.parametrize("n_samples", [13, 101])
487
+ @pytest.mark.parametrize("n_features", [2, 7, 41])
488
+ @pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
489
+ def test_load_with_offsets(sparsity, n_samples, n_features, csr_container):
490
+ rng = np.random.RandomState(0)
491
+ X = rng.uniform(low=0.0, high=1.0, size=(n_samples, n_features))
492
+ if sparsity:
493
+ X[X < sparsity] = 0.0
494
+ X = csr_container(X)
495
+ y = rng.randint(low=0, high=2, size=n_samples)
496
+
497
+ f = BytesIO()
498
+ dump_svmlight_file(X, y, f)
499
+ f.seek(0)
500
+
501
+ size = len(f.getvalue())
502
+
503
+ # put some marks that are likely to happen anywhere in a row
504
+ mark_0 = 0
505
+ mark_1 = size // 3
506
+ length_0 = mark_1 - mark_0
507
+ mark_2 = 4 * size // 5
508
+ length_1 = mark_2 - mark_1
509
+
510
+ # load the original sparse matrix into 3 independent CSR matrices
511
+ X_0, y_0 = load_svmlight_file(
512
+ f, n_features=n_features, offset=mark_0, length=length_0
513
+ )
514
+ X_1, y_1 = load_svmlight_file(
515
+ f, n_features=n_features, offset=mark_1, length=length_1
516
+ )
517
+ X_2, y_2 = load_svmlight_file(f, n_features=n_features, offset=mark_2)
518
+
519
+ y_concat = np.concatenate([y_0, y_1, y_2])
520
+ X_concat = sp.vstack([X_0, X_1, X_2])
521
+ assert_array_almost_equal(y, y_concat)
522
+ assert_array_almost_equal(X.toarray(), X_concat.toarray())
523
+
524
+
525
+ @pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
526
+ def test_load_offset_exhaustive_splits(csr_container):
527
+ rng = np.random.RandomState(0)
528
+ X = np.array(
529
+ [
530
+ [0, 0, 0, 0, 0, 0],
531
+ [1, 2, 3, 4, 0, 6],
532
+ [1, 2, 3, 4, 0, 6],
533
+ [0, 0, 0, 0, 0, 0],
534
+ [1, 0, 3, 0, 0, 0],
535
+ [0, 0, 0, 0, 0, 1],
536
+ [1, 0, 0, 0, 0, 0],
537
+ ]
538
+ )
539
+ X = csr_container(X)
540
+ n_samples, n_features = X.shape
541
+ y = rng.randint(low=0, high=2, size=n_samples)
542
+ query_id = np.arange(n_samples) // 2
543
+
544
+ f = BytesIO()
545
+ dump_svmlight_file(X, y, f, query_id=query_id)
546
+ f.seek(0)
547
+
548
+ size = len(f.getvalue())
549
+
550
+ # load the same data in 2 parts with all the possible byte offsets to
551
+ # locate the split so has to test for particular boundary cases
552
+ for mark in range(size):
553
+ f.seek(0)
554
+ X_0, y_0, q_0 = load_svmlight_file(
555
+ f, n_features=n_features, query_id=True, offset=0, length=mark
556
+ )
557
+ X_1, y_1, q_1 = load_svmlight_file(
558
+ f, n_features=n_features, query_id=True, offset=mark, length=-1
559
+ )
560
+ q_concat = np.concatenate([q_0, q_1])
561
+ y_concat = np.concatenate([y_0, y_1])
562
+ X_concat = sp.vstack([X_0, X_1])
563
+ assert_array_almost_equal(y, y_concat)
564
+ assert_array_equal(query_id, q_concat)
565
+ assert_array_almost_equal(X.toarray(), X_concat.toarray())
566
+
567
+
568
+ def test_load_with_offsets_error():
569
+ with pytest.raises(ValueError, match="n_features is required"):
570
+ _load_svmlight_local_test_file(datafile, offset=3, length=3)
571
+
572
+
573
+ @pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
574
+ def test_multilabel_y_explicit_zeros(tmp_path, csr_container):
575
+ """
576
+ Ensure that if y contains explicit zeros (i.e. elements of y.data equal to
577
+ 0) then those explicit zeros are not encoded.
578
+ """
579
+ save_path = str(tmp_path / "svm_explicit_zero")
580
+ rng = np.random.RandomState(42)
581
+ X = rng.randn(3, 5).astype(np.float64)
582
+ indptr = np.array([0, 2, 3, 6])
583
+ indices = np.array([0, 2, 2, 0, 1, 2])
584
+ # The first and last element are explicit zeros.
585
+ data = np.array([0, 1, 1, 1, 1, 0])
586
+ y = csr_container((data, indices, indptr), shape=(3, 3))
587
+ # y as a dense array would look like
588
+ # [[0, 0, 1],
589
+ # [0, 0, 1],
590
+ # [1, 1, 0]]
591
+
592
+ dump_svmlight_file(X, y, save_path, multilabel=True)
593
+
594
+ _, y_load = load_svmlight_file(save_path, multilabel=True)
595
+ y_true = [(2.0,), (2.0,), (0.0, 1.0)]
596
+ assert y_load == y_true
597
+
598
+
599
+ def test_dump_read_only(tmp_path):
600
+ """Ensure that there is no ValueError when dumping a read-only `X`.
601
+
602
+ Non-regression test for:
603
+ https://github.com/scikit-learn/scikit-learn/issues/28026
604
+ """
605
+ rng = np.random.RandomState(42)
606
+ X = rng.randn(5, 2)
607
+ y = rng.randn(5)
608
+
609
+ # Convert to memmap-backed which are read-only
610
+ X, y = create_memmap_backed_data([X, y])
611
+
612
+ save_path = str(tmp_path / "svm_read_only")
613
+ dump_svmlight_file(X, y, save_path)