Add loan-advisor-agent (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_classification.txt +9 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid.txt +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid_order.txt +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_multilabel.txt +5 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_20news.py +143 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_arff_parser.py +284 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_base.py +659 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_california_housing.py +38 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_common.py +136 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_covtype.py +55 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_kddcup99.py +89 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_lfw.py +229 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_olivetti_faces.py +26 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_openml.py +1636 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_rcv1.py +71 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_samples_generator.py +744 -0
- loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_svmlight_format.py +613 -0
.gitattributes
CHANGED
|
@@ -676,3 +676,5 @@ loan-advisor-agent/.venv/Lib/site-packages/sklearn/cluster/_k_means_lloyd.cp312-
|
|
| 676 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/_svmlight_format_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 677 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/china.jpg filter=lfs diff=lfs merge=lfs -text
|
| 678 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/flower.jpg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 676 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/_svmlight_format_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 677 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/china.jpg filter=lfs diff=lfs merge=lfs -text
|
| 678 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/images/flower.jpg filter=lfs diff=lfs merge=lfs -text
|
| 679 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/histogram.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 680 |
+
loan-advisor-agent/.venv/Lib/site-packages/sklearn/ensemble/_hist_gradient_boosting/splitting.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:301396b4a42c814b1a15038ddfcbcf5c8590501231747d0dc2a500b84b2fd0df
|
| 3 |
+
size 328
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3dee83987fffa8ec20e23b3cabc00d42beb7a469af6bd803909998c1687fa634
|
| 3 |
+
size 934
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1c5fd93ffec7deb63a940fd698534dd7ebb7db349fc183930041cbf17e60e2cc
|
| 3 |
+
size 6471
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4fcb215595bb8223321943f0df50f5a5013445bf1819d53d3cb5bfa90d9e79c0
|
| 3 |
+
size 595
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:38b74e7f02a61ff55bcfac4d87103d5bffc43febb0c019d9aaa162f8f7693068
|
| 3 |
+
size 272
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8742a74bd5bc120acd9186c8a8737cb420ed9b009fade00b24e7ce5217797f2c
|
| 3 |
+
size 722
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f623e777c0a36ae6c82fae10a7c2088cb383298ea244595bf8dc95449c9be4c4
|
| 3 |
+
size 2326
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7ccbf138e0663895f9cf511136bc6395c153f6238af2eacb6a367e86e15d1a71
|
| 3 |
+
size 1492
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0985045a454c8186b4e690ebefb6cea1ef7c13292c98d50abda470a0ff3ad425
|
| 3 |
+
size 312
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3736e7feb7ad30c68675c2c4e48a9fb262e80308c9083b100ddd0339da1fc282
|
| 3 |
+
size 348
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c8d00c6690576a9ec39e1cb77054e13296be0fdebab0fb35a64a0e8627b6e6f3
|
| 3 |
+
size 4519
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a1d38fdd601b67bb9c6d16152f53ddf166a0cfcfef4fa86438e899bfe449226c
|
| 3 |
+
size 1798
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:407424fb79cc30b8e9ff90900b3bf29244ac7f3797f278b5be602843f959b4ee
|
| 3 |
+
size 425
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0703b0ae20b9ff75087dc601640ee58f1c2ad6768858ea21a245151da9ba8e4c
|
| 3 |
+
size 301
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:70d4596ad879547863109da8675c2b789d07df66b526d7ebcbce9616c4c9b94c
|
| 3 |
+
size 347
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8743b2d93d2c62a82fb47e1fbc002b97e25adcfb5bf1fcb26b58ad0bed15bd48
|
| 3 |
+
size 1074
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e96142b5e00dfec2617b0c22d7192b340ae2c28ec3ffc3a894c5be746b970a59
|
| 3 |
+
size 3303
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a5c7e79aa41ef580838fb9fc1906280f076c47be1741fddd5004ddb500eb57fe
|
| 3 |
+
size 898
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:33cbd6ae945ba04969370ab35604e9363c87256393493382b5118a89d59386d6
|
| 3 |
+
size 268
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0bce20aae7fd903796d96d5b3a3677b7058fbc5f3fe0996ee9d491e4ee23d132
|
| 3 |
+
size 293
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a9f4b9317997df63ed8d2bb073a3906344c0e0be017fd384eaec36ced8b94bae
|
| 3 |
+
size 330
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:424cd47c12a51c7bb8d8169fac80fb5601f152bd78468b241d4b115bf7d22f20
|
| 3 |
+
size 1121
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:afe4736924606638984e573235191025d419c545d31dc8874c96b72f5ec5db73
|
| 3 |
+
size 2342
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7ef3551ad47d48023c5a1f1cf077047a9a4b95544bb91d4a86097f8b574f8d07
|
| 3 |
+
size 656
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:489b177126cb7f335cb220709233b946d3a0ad71d38bba6d48b79187146e585a
|
| 3 |
+
size 817
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:278a52a52d569f07d14c6a7877b104762c77daac429fb1fd9817a0378d6ec634
|
| 3 |
+
size 805
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fb5830c82112f62a400c82ac1f1b5eb61c29c0a7cc72ba56d2aeff0fae8a60f9
|
| 3 |
+
size 1625
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_classification.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# comment
|
| 2 |
+
# note: the next line contains a tab
|
| 3 |
+
1.0 3:2.5 11:-5.2 16:1.5 # and an inline comment
|
| 4 |
+
2.0 6:1.0 13:-3
|
| 5 |
+
# another comment
|
| 6 |
+
3.0 21:27
|
| 7 |
+
4.0 2:1.234567890123456e10 # double precision value
|
| 8 |
+
1.0 # empty line, all zeros
|
| 9 |
+
2.0 3:0 # explicit zeros
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
python 2:2.5 10:-5.2 15:1.5
|
| 2 |
+
2.0 5:1.0 12:-3
|
| 3 |
+
3.0 20:27
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_invalid_order.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
-1 5:2.5 2:-5.2 15:1.5
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/data/svmlight_multilabel.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# multilabel dataset in SVMlight format
|
| 2 |
+
1,0 2:2.5 10:-5.2 15:1.5
|
| 3 |
+
2 5:1.0 12:-3
|
| 4 |
+
2:3.5 11:26
|
| 5 |
+
1,2 20:27
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_20news.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test the 20news downloader, if the data is available,
|
| 2 |
+
or if specifically requested via environment variable
|
| 3 |
+
(e.g. for CI jobs)."""
|
| 4 |
+
|
| 5 |
+
from functools import partial
|
| 6 |
+
from unittest.mock import patch
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import pytest
|
| 10 |
+
import scipy.sparse as sp
|
| 11 |
+
|
| 12 |
+
from sklearn.datasets.tests.test_common import (
|
| 13 |
+
check_as_frame,
|
| 14 |
+
check_pandas_dependency_message,
|
| 15 |
+
check_return_X_y,
|
| 16 |
+
)
|
| 17 |
+
from sklearn.preprocessing import normalize
|
| 18 |
+
from sklearn.utils._testing import assert_allclose_dense_sparse
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_20news(fetch_20newsgroups_fxt):
|
| 22 |
+
data = fetch_20newsgroups_fxt(subset="all", shuffle=False)
|
| 23 |
+
assert data.DESCR.startswith(".. _20newsgroups_dataset:")
|
| 24 |
+
|
| 25 |
+
# Extract a reduced dataset
|
| 26 |
+
data2cats = fetch_20newsgroups_fxt(
|
| 27 |
+
subset="all", categories=data.target_names[-1:-3:-1], shuffle=False
|
| 28 |
+
)
|
| 29 |
+
# Check that the ordering of the target_names is the same
|
| 30 |
+
# as the ordering in the full dataset
|
| 31 |
+
assert data2cats.target_names == data.target_names[-2:]
|
| 32 |
+
# Assert that we have only 0 and 1 as labels
|
| 33 |
+
assert np.unique(data2cats.target).tolist() == [0, 1]
|
| 34 |
+
|
| 35 |
+
# Check that the number of filenames is consistent with data/target
|
| 36 |
+
assert len(data2cats.filenames) == len(data2cats.target)
|
| 37 |
+
assert len(data2cats.filenames) == len(data2cats.data)
|
| 38 |
+
|
| 39 |
+
# Check that the first entry of the reduced dataset corresponds to
|
| 40 |
+
# the first entry of the corresponding category in the full dataset
|
| 41 |
+
entry1 = data2cats.data[0]
|
| 42 |
+
category = data2cats.target_names[data2cats.target[0]]
|
| 43 |
+
label = data.target_names.index(category)
|
| 44 |
+
entry2 = data.data[np.where(data.target == label)[0][0]]
|
| 45 |
+
assert entry1 == entry2
|
| 46 |
+
|
| 47 |
+
# check that return_X_y option
|
| 48 |
+
X, y = fetch_20newsgroups_fxt(subset="all", shuffle=False, return_X_y=True)
|
| 49 |
+
assert len(X) == len(data.data)
|
| 50 |
+
assert y.shape == data.target.shape
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def test_20news_length_consistency(fetch_20newsgroups_fxt):
|
| 54 |
+
"""Checks the length consistencies within the bunch
|
| 55 |
+
|
| 56 |
+
This is a non-regression test for a bug present in 0.16.1.
|
| 57 |
+
"""
|
| 58 |
+
# Extract the full dataset
|
| 59 |
+
data = fetch_20newsgroups_fxt(subset="all")
|
| 60 |
+
assert len(data["data"]) == len(data.data)
|
| 61 |
+
assert len(data["target"]) == len(data.target)
|
| 62 |
+
assert len(data["filenames"]) == len(data.filenames)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def test_20news_vectorized(fetch_20newsgroups_vectorized_fxt):
|
| 66 |
+
# test subset = train
|
| 67 |
+
bunch = fetch_20newsgroups_vectorized_fxt(subset="train")
|
| 68 |
+
assert sp.issparse(bunch.data) and bunch.data.format == "csr"
|
| 69 |
+
assert bunch.data.shape == (11314, 130107)
|
| 70 |
+
assert bunch.target.shape[0] == 11314
|
| 71 |
+
assert bunch.data.dtype == np.float64
|
| 72 |
+
assert bunch.DESCR.startswith(".. _20newsgroups_dataset:")
|
| 73 |
+
|
| 74 |
+
# test subset = test
|
| 75 |
+
bunch = fetch_20newsgroups_vectorized_fxt(subset="test")
|
| 76 |
+
assert sp.issparse(bunch.data) and bunch.data.format == "csr"
|
| 77 |
+
assert bunch.data.shape == (7532, 130107)
|
| 78 |
+
assert bunch.target.shape[0] == 7532
|
| 79 |
+
assert bunch.data.dtype == np.float64
|
| 80 |
+
assert bunch.DESCR.startswith(".. _20newsgroups_dataset:")
|
| 81 |
+
|
| 82 |
+
# test return_X_y option
|
| 83 |
+
fetch_func = partial(fetch_20newsgroups_vectorized_fxt, subset="test")
|
| 84 |
+
check_return_X_y(bunch, fetch_func)
|
| 85 |
+
|
| 86 |
+
# test subset = all
|
| 87 |
+
bunch = fetch_20newsgroups_vectorized_fxt(subset="all")
|
| 88 |
+
assert sp.issparse(bunch.data) and bunch.data.format == "csr"
|
| 89 |
+
assert bunch.data.shape == (11314 + 7532, 130107)
|
| 90 |
+
assert bunch.target.shape[0] == 11314 + 7532
|
| 91 |
+
assert bunch.data.dtype == np.float64
|
| 92 |
+
assert bunch.DESCR.startswith(".. _20newsgroups_dataset:")
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def test_20news_normalization(fetch_20newsgroups_vectorized_fxt):
|
| 96 |
+
X = fetch_20newsgroups_vectorized_fxt(normalize=False)
|
| 97 |
+
X_ = fetch_20newsgroups_vectorized_fxt(normalize=True)
|
| 98 |
+
X_norm = X_["data"][:100]
|
| 99 |
+
X = X["data"][:100]
|
| 100 |
+
|
| 101 |
+
assert_allclose_dense_sparse(X_norm, normalize(X))
|
| 102 |
+
assert np.allclose(np.linalg.norm(X_norm.todense(), axis=1), 1)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def test_20news_as_frame(fetch_20newsgroups_vectorized_fxt):
|
| 106 |
+
pd = pytest.importorskip("pandas")
|
| 107 |
+
|
| 108 |
+
bunch = fetch_20newsgroups_vectorized_fxt(as_frame=True)
|
| 109 |
+
check_as_frame(bunch, fetch_20newsgroups_vectorized_fxt)
|
| 110 |
+
|
| 111 |
+
frame = bunch.frame
|
| 112 |
+
assert frame.shape == (11314, 130108)
|
| 113 |
+
assert all([isinstance(col, pd.SparseDtype) for col in bunch.data.dtypes])
|
| 114 |
+
|
| 115 |
+
# Check a small subset of features
|
| 116 |
+
for expected_feature in [
|
| 117 |
+
"beginner",
|
| 118 |
+
"beginners",
|
| 119 |
+
"beginning",
|
| 120 |
+
"beginnings",
|
| 121 |
+
"begins",
|
| 122 |
+
"begley",
|
| 123 |
+
"begone",
|
| 124 |
+
]:
|
| 125 |
+
assert expected_feature in frame.keys()
|
| 126 |
+
assert "category_class" in frame.keys()
|
| 127 |
+
assert bunch.target.name == "category_class"
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def test_as_frame_no_pandas(fetch_20newsgroups_vectorized_fxt, hide_available_pandas):
|
| 131 |
+
check_pandas_dependency_message(fetch_20newsgroups_vectorized_fxt)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def test_outdated_pickle(fetch_20newsgroups_vectorized_fxt):
|
| 135 |
+
with patch("os.path.exists") as mock_is_exist:
|
| 136 |
+
with patch("joblib.load") as mock_load:
|
| 137 |
+
# mock that the dataset was cached
|
| 138 |
+
mock_is_exist.return_value = True
|
| 139 |
+
# mock that we have an outdated pickle with only X and y returned
|
| 140 |
+
mock_load.return_value = ("X", "y")
|
| 141 |
+
err_msg = "The cached dataset located in"
|
| 142 |
+
with pytest.raises(ValueError, match=err_msg):
|
| 143 |
+
fetch_20newsgroups_vectorized_fxt(as_frame=True)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_arff_parser.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import textwrap
|
| 2 |
+
from io import BytesIO
|
| 3 |
+
|
| 4 |
+
import pytest
|
| 5 |
+
|
| 6 |
+
from sklearn.datasets._arff_parser import (
|
| 7 |
+
_liac_arff_parser,
|
| 8 |
+
_pandas_arff_parser,
|
| 9 |
+
_post_process_frame,
|
| 10 |
+
load_arff_from_gzip_file,
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@pytest.mark.parametrize(
|
| 15 |
+
"feature_names, target_names",
|
| 16 |
+
[
|
| 17 |
+
(
|
| 18 |
+
[
|
| 19 |
+
"col_int_as_integer",
|
| 20 |
+
"col_int_as_numeric",
|
| 21 |
+
"col_float_as_real",
|
| 22 |
+
"col_float_as_numeric",
|
| 23 |
+
],
|
| 24 |
+
["col_categorical", "col_string"],
|
| 25 |
+
),
|
| 26 |
+
(
|
| 27 |
+
[
|
| 28 |
+
"col_int_as_integer",
|
| 29 |
+
"col_int_as_numeric",
|
| 30 |
+
"col_float_as_real",
|
| 31 |
+
"col_float_as_numeric",
|
| 32 |
+
],
|
| 33 |
+
["col_categorical"],
|
| 34 |
+
),
|
| 35 |
+
(
|
| 36 |
+
[
|
| 37 |
+
"col_int_as_integer",
|
| 38 |
+
"col_int_as_numeric",
|
| 39 |
+
"col_float_as_real",
|
| 40 |
+
"col_float_as_numeric",
|
| 41 |
+
],
|
| 42 |
+
[],
|
| 43 |
+
),
|
| 44 |
+
],
|
| 45 |
+
)
|
| 46 |
+
def test_post_process_frame(feature_names, target_names):
|
| 47 |
+
"""Check the behaviour of the post-processing function for splitting a dataframe."""
|
| 48 |
+
pd = pytest.importorskip("pandas")
|
| 49 |
+
|
| 50 |
+
X_original = pd.DataFrame(
|
| 51 |
+
{
|
| 52 |
+
"col_int_as_integer": [1, 2, 3],
|
| 53 |
+
"col_int_as_numeric": [1, 2, 3],
|
| 54 |
+
"col_float_as_real": [1.0, 2.0, 3.0],
|
| 55 |
+
"col_float_as_numeric": [1.0, 2.0, 3.0],
|
| 56 |
+
"col_categorical": ["a", "b", "c"],
|
| 57 |
+
"col_string": ["a", "b", "c"],
|
| 58 |
+
}
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
X, y = _post_process_frame(X_original, feature_names, target_names)
|
| 62 |
+
assert isinstance(X, pd.DataFrame)
|
| 63 |
+
if len(target_names) >= 2:
|
| 64 |
+
assert isinstance(y, pd.DataFrame)
|
| 65 |
+
elif len(target_names) == 1:
|
| 66 |
+
assert isinstance(y, pd.Series)
|
| 67 |
+
else:
|
| 68 |
+
assert y is None
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def test_load_arff_from_gzip_file_error_parser():
|
| 72 |
+
"""An error will be raised if the parser is not known."""
|
| 73 |
+
# None of the input parameters are required to be accurate since the check
|
| 74 |
+
# of the parser will be carried out first.
|
| 75 |
+
|
| 76 |
+
err_msg = "Unknown parser: 'xxx'. Should be 'liac-arff' or 'pandas'"
|
| 77 |
+
with pytest.raises(ValueError, match=err_msg):
|
| 78 |
+
load_arff_from_gzip_file("xxx", "xxx", "xxx", "xxx", "xxx", "xxx")
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
@pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser])
|
| 82 |
+
def test_pandas_arff_parser_strip_single_quotes(parser_func):
|
| 83 |
+
"""Check that we properly strip single quotes from the data."""
|
| 84 |
+
pd = pytest.importorskip("pandas")
|
| 85 |
+
|
| 86 |
+
arff_file = BytesIO(
|
| 87 |
+
textwrap.dedent(
|
| 88 |
+
"""
|
| 89 |
+
@relation 'toy'
|
| 90 |
+
@attribute 'cat_single_quote' {'A', 'B', 'C'}
|
| 91 |
+
@attribute 'str_single_quote' string
|
| 92 |
+
@attribute 'str_nested_quote' string
|
| 93 |
+
@attribute 'class' numeric
|
| 94 |
+
@data
|
| 95 |
+
'A','some text','\"expect double quotes\"',0
|
| 96 |
+
"""
|
| 97 |
+
).encode("utf-8")
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
columns_info = {
|
| 101 |
+
"cat_single_quote": {
|
| 102 |
+
"data_type": "nominal",
|
| 103 |
+
"name": "cat_single_quote",
|
| 104 |
+
},
|
| 105 |
+
"str_single_quote": {
|
| 106 |
+
"data_type": "string",
|
| 107 |
+
"name": "str_single_quote",
|
| 108 |
+
},
|
| 109 |
+
"str_nested_quote": {
|
| 110 |
+
"data_type": "string",
|
| 111 |
+
"name": "str_nested_quote",
|
| 112 |
+
},
|
| 113 |
+
"class": {
|
| 114 |
+
"data_type": "numeric",
|
| 115 |
+
"name": "class",
|
| 116 |
+
},
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
feature_names = [
|
| 120 |
+
"cat_single_quote",
|
| 121 |
+
"str_single_quote",
|
| 122 |
+
"str_nested_quote",
|
| 123 |
+
]
|
| 124 |
+
target_names = ["class"]
|
| 125 |
+
|
| 126 |
+
# We don't strip single quotes for string columns with the pandas parser.
|
| 127 |
+
expected_values = {
|
| 128 |
+
"cat_single_quote": "A",
|
| 129 |
+
"str_single_quote": (
|
| 130 |
+
"some text" if parser_func is _liac_arff_parser else "'some text'"
|
| 131 |
+
),
|
| 132 |
+
"str_nested_quote": (
|
| 133 |
+
'"expect double quotes"'
|
| 134 |
+
if parser_func is _liac_arff_parser
|
| 135 |
+
else "'\"expect double quotes\"'"
|
| 136 |
+
),
|
| 137 |
+
"class": 0,
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
_, _, frame, _ = parser_func(
|
| 141 |
+
arff_file,
|
| 142 |
+
output_arrays_type="pandas",
|
| 143 |
+
openml_columns_info=columns_info,
|
| 144 |
+
feature_names_to_select=feature_names,
|
| 145 |
+
target_names_to_select=target_names,
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
assert frame.columns.tolist() == feature_names + target_names
|
| 149 |
+
pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0))
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
@pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser])
|
| 153 |
+
def test_pandas_arff_parser_strip_double_quotes(parser_func):
|
| 154 |
+
"""Check that we properly strip double quotes from the data."""
|
| 155 |
+
pd = pytest.importorskip("pandas")
|
| 156 |
+
|
| 157 |
+
arff_file = BytesIO(
|
| 158 |
+
textwrap.dedent(
|
| 159 |
+
"""
|
| 160 |
+
@relation 'toy'
|
| 161 |
+
@attribute 'cat_double_quote' {"A", "B", "C"}
|
| 162 |
+
@attribute 'str_double_quote' string
|
| 163 |
+
@attribute 'str_nested_quote' string
|
| 164 |
+
@attribute 'class' numeric
|
| 165 |
+
@data
|
| 166 |
+
"A","some text","\'expect double quotes\'",0
|
| 167 |
+
"""
|
| 168 |
+
).encode("utf-8")
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
columns_info = {
|
| 172 |
+
"cat_double_quote": {
|
| 173 |
+
"data_type": "nominal",
|
| 174 |
+
"name": "cat_double_quote",
|
| 175 |
+
},
|
| 176 |
+
"str_double_quote": {
|
| 177 |
+
"data_type": "string",
|
| 178 |
+
"name": "str_double_quote",
|
| 179 |
+
},
|
| 180 |
+
"str_nested_quote": {
|
| 181 |
+
"data_type": "string",
|
| 182 |
+
"name": "str_nested_quote",
|
| 183 |
+
},
|
| 184 |
+
"class": {
|
| 185 |
+
"data_type": "numeric",
|
| 186 |
+
"name": "class",
|
| 187 |
+
},
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
feature_names = [
|
| 191 |
+
"cat_double_quote",
|
| 192 |
+
"str_double_quote",
|
| 193 |
+
"str_nested_quote",
|
| 194 |
+
]
|
| 195 |
+
target_names = ["class"]
|
| 196 |
+
|
| 197 |
+
expected_values = {
|
| 198 |
+
"cat_double_quote": "A",
|
| 199 |
+
"str_double_quote": "some text",
|
| 200 |
+
"str_nested_quote": "'expect double quotes'",
|
| 201 |
+
"class": 0,
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
_, _, frame, _ = parser_func(
|
| 205 |
+
arff_file,
|
| 206 |
+
output_arrays_type="pandas",
|
| 207 |
+
openml_columns_info=columns_info,
|
| 208 |
+
feature_names_to_select=feature_names,
|
| 209 |
+
target_names_to_select=target_names,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
assert frame.columns.tolist() == feature_names + target_names
|
| 213 |
+
pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0))
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
@pytest.mark.parametrize(
|
| 217 |
+
"parser_func",
|
| 218 |
+
[
|
| 219 |
+
# internal quotes are not considered to follow the ARFF spec in LIAC ARFF
|
| 220 |
+
pytest.param(_liac_arff_parser, marks=pytest.mark.xfail),
|
| 221 |
+
_pandas_arff_parser,
|
| 222 |
+
],
|
| 223 |
+
)
|
| 224 |
+
def test_pandas_arff_parser_strip_no_quotes(parser_func):
|
| 225 |
+
"""Check that we properly parse with no quotes characters."""
|
| 226 |
+
pd = pytest.importorskip("pandas")
|
| 227 |
+
|
| 228 |
+
arff_file = BytesIO(
|
| 229 |
+
textwrap.dedent(
|
| 230 |
+
"""
|
| 231 |
+
@relation 'toy'
|
| 232 |
+
@attribute 'cat_without_quote' {A, B, C}
|
| 233 |
+
@attribute 'str_without_quote' string
|
| 234 |
+
@attribute 'str_internal_quote' string
|
| 235 |
+
@attribute 'class' numeric
|
| 236 |
+
@data
|
| 237 |
+
A,some text,'internal' quote,0
|
| 238 |
+
"""
|
| 239 |
+
).encode("utf-8")
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
columns_info = {
|
| 243 |
+
"cat_without_quote": {
|
| 244 |
+
"data_type": "nominal",
|
| 245 |
+
"name": "cat_without_quote",
|
| 246 |
+
},
|
| 247 |
+
"str_without_quote": {
|
| 248 |
+
"data_type": "string",
|
| 249 |
+
"name": "str_without_quote",
|
| 250 |
+
},
|
| 251 |
+
"str_internal_quote": {
|
| 252 |
+
"data_type": "string",
|
| 253 |
+
"name": "str_internal_quote",
|
| 254 |
+
},
|
| 255 |
+
"class": {
|
| 256 |
+
"data_type": "numeric",
|
| 257 |
+
"name": "class",
|
| 258 |
+
},
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
feature_names = [
|
| 262 |
+
"cat_without_quote",
|
| 263 |
+
"str_without_quote",
|
| 264 |
+
"str_internal_quote",
|
| 265 |
+
]
|
| 266 |
+
target_names = ["class"]
|
| 267 |
+
|
| 268 |
+
expected_values = {
|
| 269 |
+
"cat_without_quote": "A",
|
| 270 |
+
"str_without_quote": "some text",
|
| 271 |
+
"str_internal_quote": "'internal' quote",
|
| 272 |
+
"class": 0,
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
_, _, frame, _ = parser_func(
|
| 276 |
+
arff_file,
|
| 277 |
+
output_arrays_type="pandas",
|
| 278 |
+
openml_columns_info=columns_info,
|
| 279 |
+
feature_names_to_select=feature_names,
|
| 280 |
+
target_names_to_select=target_names,
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
assert frame.columns.tolist() == feature_names + target_names
|
| 284 |
+
pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0))
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_base.py
ADDED
|
@@ -0,0 +1,659 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import io
|
| 3 |
+
import os
|
| 4 |
+
import re
|
| 5 |
+
import shutil
|
| 6 |
+
import tempfile
|
| 7 |
+
import warnings
|
| 8 |
+
from functools import partial
|
| 9 |
+
from importlib import resources
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from pickle import dumps, loads
|
| 12 |
+
from unittest.mock import Mock
|
| 13 |
+
from urllib.error import HTTPError
|
| 14 |
+
from urllib.parse import urlparse
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import pytest
|
| 18 |
+
|
| 19 |
+
from sklearn.datasets import (
|
| 20 |
+
clear_data_home,
|
| 21 |
+
fetch_file,
|
| 22 |
+
get_data_home,
|
| 23 |
+
load_breast_cancer,
|
| 24 |
+
load_diabetes,
|
| 25 |
+
load_digits,
|
| 26 |
+
load_files,
|
| 27 |
+
load_iris,
|
| 28 |
+
load_linnerud,
|
| 29 |
+
load_sample_image,
|
| 30 |
+
load_sample_images,
|
| 31 |
+
load_wine,
|
| 32 |
+
)
|
| 33 |
+
from sklearn.datasets._base import (
|
| 34 |
+
RemoteFileMetadata,
|
| 35 |
+
_derive_folder_and_filename_from_url,
|
| 36 |
+
_fetch_remote,
|
| 37 |
+
load_csv_data,
|
| 38 |
+
load_gzip_compressed_csv_data,
|
| 39 |
+
)
|
| 40 |
+
from sklearn.datasets.tests.test_common import check_as_frame
|
| 41 |
+
from sklearn.preprocessing import scale
|
| 42 |
+
from sklearn.utils import Bunch
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class _DummyPath:
|
| 46 |
+
"""Minimal class that implements the os.PathLike interface."""
|
| 47 |
+
|
| 48 |
+
def __init__(self, path):
|
| 49 |
+
self.path = path
|
| 50 |
+
|
| 51 |
+
def __fspath__(self):
|
| 52 |
+
return self.path
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _remove_dir(path):
|
| 56 |
+
if os.path.isdir(path):
|
| 57 |
+
shutil.rmtree(path)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@pytest.fixture(scope="module")
|
| 61 |
+
def data_home(tmpdir_factory):
|
| 62 |
+
tmp_file = str(tmpdir_factory.mktemp("scikit_learn_data_home_test"))
|
| 63 |
+
yield tmp_file
|
| 64 |
+
_remove_dir(tmp_file)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
@pytest.fixture(scope="module")
|
| 68 |
+
def load_files_root(tmpdir_factory):
|
| 69 |
+
tmp_file = str(tmpdir_factory.mktemp("scikit_learn_load_files_test"))
|
| 70 |
+
yield tmp_file
|
| 71 |
+
_remove_dir(tmp_file)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@pytest.fixture
|
| 75 |
+
def test_category_dir_1(load_files_root):
|
| 76 |
+
test_category_dir1 = tempfile.mkdtemp(dir=load_files_root)
|
| 77 |
+
sample_file = tempfile.NamedTemporaryFile(dir=test_category_dir1, delete=False)
|
| 78 |
+
sample_file.write(b"Hello World!\n")
|
| 79 |
+
sample_file.close()
|
| 80 |
+
yield str(test_category_dir1)
|
| 81 |
+
_remove_dir(test_category_dir1)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@pytest.fixture
|
| 85 |
+
def test_category_dir_2(load_files_root):
|
| 86 |
+
test_category_dir2 = tempfile.mkdtemp(dir=load_files_root)
|
| 87 |
+
yield str(test_category_dir2)
|
| 88 |
+
_remove_dir(test_category_dir2)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
@pytest.mark.thread_unsafe
|
| 92 |
+
@pytest.mark.parametrize("path_container", [None, Path, _DummyPath])
|
| 93 |
+
def test_data_home(path_container, data_home):
|
| 94 |
+
# get_data_home will point to a pre-existing folder
|
| 95 |
+
if path_container is not None:
|
| 96 |
+
data_home = path_container(data_home)
|
| 97 |
+
data_home = get_data_home(data_home=data_home)
|
| 98 |
+
assert data_home == data_home
|
| 99 |
+
assert os.path.exists(data_home)
|
| 100 |
+
|
| 101 |
+
# clear_data_home will delete both the content and the folder it-self
|
| 102 |
+
if path_container is not None:
|
| 103 |
+
data_home = path_container(data_home)
|
| 104 |
+
clear_data_home(data_home=data_home)
|
| 105 |
+
assert not os.path.exists(data_home)
|
| 106 |
+
|
| 107 |
+
# if the folder is missing it will be created again
|
| 108 |
+
data_home = get_data_home(data_home=data_home)
|
| 109 |
+
assert os.path.exists(data_home)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def test_default_empty_load_files(load_files_root):
|
| 113 |
+
res = load_files(load_files_root)
|
| 114 |
+
assert len(res.filenames) == 0
|
| 115 |
+
assert len(res.target_names) == 0
|
| 116 |
+
assert res.DESCR is None
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def test_default_load_files(test_category_dir_1, test_category_dir_2, load_files_root):
|
| 120 |
+
res = load_files(load_files_root)
|
| 121 |
+
assert len(res.filenames) == 1
|
| 122 |
+
assert len(res.target_names) == 2
|
| 123 |
+
assert res.DESCR is None
|
| 124 |
+
assert res.data == [b"Hello World!\n"]
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def test_load_files_w_categories_desc_and_encoding(
|
| 128 |
+
test_category_dir_1, test_category_dir_2, load_files_root
|
| 129 |
+
):
|
| 130 |
+
category = os.path.abspath(test_category_dir_1).split(os.sep).pop()
|
| 131 |
+
res = load_files(
|
| 132 |
+
load_files_root, description="test", categories=[category], encoding="utf-8"
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
assert len(res.filenames) == 1
|
| 136 |
+
assert len(res.target_names) == 1
|
| 137 |
+
assert res.DESCR == "test"
|
| 138 |
+
assert res.data == ["Hello World!\n"]
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def test_load_files_wo_load_content(
|
| 142 |
+
test_category_dir_1, test_category_dir_2, load_files_root
|
| 143 |
+
):
|
| 144 |
+
res = load_files(load_files_root, load_content=False)
|
| 145 |
+
assert len(res.filenames) == 1
|
| 146 |
+
assert len(res.target_names) == 2
|
| 147 |
+
assert res.DESCR is None
|
| 148 |
+
assert res.get("data") is None
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@pytest.mark.parametrize("allowed_extensions", ([".txt"], [".txt", ".json"]))
|
| 152 |
+
def test_load_files_allowed_extensions(tmp_path, allowed_extensions):
|
| 153 |
+
"""Check the behaviour of `allowed_extension` in `load_files`."""
|
| 154 |
+
d = tmp_path / "sub"
|
| 155 |
+
d.mkdir()
|
| 156 |
+
files = ("file1.txt", "file2.json", "file3.json", "file4.md")
|
| 157 |
+
paths = [d / f for f in files]
|
| 158 |
+
for p in paths:
|
| 159 |
+
p.write_bytes(b"hello")
|
| 160 |
+
res = load_files(tmp_path, allowed_extensions=allowed_extensions)
|
| 161 |
+
assert set([str(p) for p in paths if p.suffix in allowed_extensions]) == set(
|
| 162 |
+
res.filenames
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
@pytest.mark.parametrize(
|
| 167 |
+
"filename, expected_n_samples, expected_n_features, expected_target_names",
|
| 168 |
+
[
|
| 169 |
+
("wine_data.csv", 178, 13, ["class_0", "class_1", "class_2"]),
|
| 170 |
+
("iris.csv", 150, 4, ["setosa", "versicolor", "virginica"]),
|
| 171 |
+
("breast_cancer.csv", 569, 30, ["malignant", "benign"]),
|
| 172 |
+
],
|
| 173 |
+
)
|
| 174 |
+
def test_load_csv_data(
|
| 175 |
+
filename, expected_n_samples, expected_n_features, expected_target_names
|
| 176 |
+
):
|
| 177 |
+
actual_data, actual_target, actual_target_names = load_csv_data(filename)
|
| 178 |
+
assert actual_data.shape[0] == expected_n_samples
|
| 179 |
+
assert actual_data.shape[1] == expected_n_features
|
| 180 |
+
assert actual_target.shape[0] == expected_n_samples
|
| 181 |
+
np.testing.assert_array_equal(actual_target_names, expected_target_names)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def test_load_csv_data_with_descr():
|
| 185 |
+
data_file_name = "iris.csv"
|
| 186 |
+
descr_file_name = "iris.rst"
|
| 187 |
+
|
| 188 |
+
res_without_descr = load_csv_data(data_file_name=data_file_name)
|
| 189 |
+
res_with_descr = load_csv_data(
|
| 190 |
+
data_file_name=data_file_name, descr_file_name=descr_file_name
|
| 191 |
+
)
|
| 192 |
+
assert len(res_with_descr) == 4
|
| 193 |
+
assert len(res_without_descr) == 3
|
| 194 |
+
|
| 195 |
+
np.testing.assert_array_equal(res_with_descr[0], res_without_descr[0])
|
| 196 |
+
np.testing.assert_array_equal(res_with_descr[1], res_without_descr[1])
|
| 197 |
+
np.testing.assert_array_equal(res_with_descr[2], res_without_descr[2])
|
| 198 |
+
|
| 199 |
+
assert res_with_descr[-1].startswith(".. _iris_dataset:")
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
@pytest.mark.parametrize(
|
| 203 |
+
"filename, kwargs, expected_shape",
|
| 204 |
+
[
|
| 205 |
+
("diabetes_data_raw.csv.gz", {}, [442, 10]),
|
| 206 |
+
("diabetes_target.csv.gz", {}, [442]),
|
| 207 |
+
("digits.csv.gz", {"delimiter": ","}, [1797, 65]),
|
| 208 |
+
],
|
| 209 |
+
)
|
| 210 |
+
def test_load_gzip_compressed_csv_data(filename, kwargs, expected_shape):
|
| 211 |
+
actual_data = load_gzip_compressed_csv_data(filename, **kwargs)
|
| 212 |
+
assert actual_data.shape == tuple(expected_shape)
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def test_load_gzip_compressed_csv_data_with_descr():
|
| 216 |
+
data_file_name = "diabetes_target.csv.gz"
|
| 217 |
+
descr_file_name = "diabetes.rst"
|
| 218 |
+
|
| 219 |
+
expected_data = load_gzip_compressed_csv_data(data_file_name=data_file_name)
|
| 220 |
+
actual_data, descr = load_gzip_compressed_csv_data(
|
| 221 |
+
data_file_name=data_file_name,
|
| 222 |
+
descr_file_name=descr_file_name,
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
np.testing.assert_array_equal(actual_data, expected_data)
|
| 226 |
+
assert descr.startswith(".. _diabetes_dataset:")
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def test_load_sample_images():
|
| 230 |
+
try:
|
| 231 |
+
res = load_sample_images()
|
| 232 |
+
assert len(res.images) == 2
|
| 233 |
+
assert len(res.filenames) == 2
|
| 234 |
+
images = res.images
|
| 235 |
+
|
| 236 |
+
# assert is china image
|
| 237 |
+
assert np.all(images[0][0, 0, :] == np.array([174, 201, 231], dtype=np.uint8))
|
| 238 |
+
# assert is flower image
|
| 239 |
+
assert np.all(images[1][0, 0, :] == np.array([2, 19, 13], dtype=np.uint8))
|
| 240 |
+
assert res.DESCR
|
| 241 |
+
except ImportError:
|
| 242 |
+
warnings.warn("Could not load sample images, PIL is not available.")
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def test_load_sample_image():
|
| 246 |
+
try:
|
| 247 |
+
china = load_sample_image("china.jpg")
|
| 248 |
+
assert china.dtype == "uint8"
|
| 249 |
+
assert china.shape == (427, 640, 3)
|
| 250 |
+
except ImportError:
|
| 251 |
+
warnings.warn("Could not load sample images, PIL is not available.")
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def test_load_diabetes_raw():
|
| 255 |
+
"""Test to check that we load a scaled version by default but that we can
|
| 256 |
+
get an unscaled version when setting `scaled=False`."""
|
| 257 |
+
diabetes_raw = load_diabetes(scaled=False)
|
| 258 |
+
assert diabetes_raw.data.shape == (442, 10)
|
| 259 |
+
assert diabetes_raw.target.size == 442
|
| 260 |
+
assert len(diabetes_raw.feature_names) == 10
|
| 261 |
+
assert diabetes_raw.DESCR
|
| 262 |
+
|
| 263 |
+
diabetes_default = load_diabetes()
|
| 264 |
+
|
| 265 |
+
np.testing.assert_allclose(
|
| 266 |
+
scale(diabetes_raw.data) / (442**0.5), diabetes_default.data, atol=1e-04
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
@pytest.mark.parametrize(
|
| 271 |
+
"loader_func, data_shape, target_shape, n_target, has_descr, filenames",
|
| 272 |
+
[
|
| 273 |
+
(load_breast_cancer, (569, 30), (569,), 2, True, ["filename"]),
|
| 274 |
+
(load_wine, (178, 13), (178,), 3, True, []),
|
| 275 |
+
(load_iris, (150, 4), (150,), 3, True, ["filename"]),
|
| 276 |
+
(
|
| 277 |
+
load_linnerud,
|
| 278 |
+
(20, 3),
|
| 279 |
+
(20, 3),
|
| 280 |
+
3,
|
| 281 |
+
True,
|
| 282 |
+
["data_filename", "target_filename"],
|
| 283 |
+
),
|
| 284 |
+
(load_diabetes, (442, 10), (442,), None, True, []),
|
| 285 |
+
(load_digits, (1797, 64), (1797,), 10, True, []),
|
| 286 |
+
(partial(load_digits, n_class=9), (1617, 64), (1617,), 10, True, []),
|
| 287 |
+
],
|
| 288 |
+
)
|
| 289 |
+
def test_loader(loader_func, data_shape, target_shape, n_target, has_descr, filenames):
|
| 290 |
+
bunch = loader_func()
|
| 291 |
+
|
| 292 |
+
assert isinstance(bunch, Bunch)
|
| 293 |
+
assert bunch.data.shape == data_shape
|
| 294 |
+
assert bunch.target.shape == target_shape
|
| 295 |
+
if hasattr(bunch, "feature_names"):
|
| 296 |
+
assert len(bunch.feature_names) == data_shape[1]
|
| 297 |
+
if n_target is not None:
|
| 298 |
+
assert len(bunch.target_names) == n_target
|
| 299 |
+
if has_descr:
|
| 300 |
+
assert bunch.DESCR
|
| 301 |
+
if filenames:
|
| 302 |
+
assert "data_module" in bunch
|
| 303 |
+
assert all(
|
| 304 |
+
[
|
| 305 |
+
f in bunch
|
| 306 |
+
and (resources.files(bunch["data_module"]) / bunch[f]).is_file()
|
| 307 |
+
for f in filenames
|
| 308 |
+
]
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
@pytest.mark.parametrize(
|
| 313 |
+
"loader_func, data_dtype, target_dtype",
|
| 314 |
+
[
|
| 315 |
+
(load_breast_cancer, np.float64, int),
|
| 316 |
+
(load_diabetes, np.float64, np.float64),
|
| 317 |
+
(load_digits, np.float64, int),
|
| 318 |
+
(load_iris, np.float64, int),
|
| 319 |
+
(load_linnerud, np.float64, np.float64),
|
| 320 |
+
(load_wine, np.float64, int),
|
| 321 |
+
],
|
| 322 |
+
)
|
| 323 |
+
def test_toy_dataset_frame_dtype(loader_func, data_dtype, target_dtype):
|
| 324 |
+
default_result = loader_func()
|
| 325 |
+
check_as_frame(
|
| 326 |
+
default_result,
|
| 327 |
+
loader_func,
|
| 328 |
+
expected_data_dtype=data_dtype,
|
| 329 |
+
expected_target_dtype=target_dtype,
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def test_loads_dumps_bunch():
|
| 334 |
+
bunch = Bunch(x="x")
|
| 335 |
+
bunch_from_pkl = loads(dumps(bunch))
|
| 336 |
+
bunch_from_pkl.x = "y"
|
| 337 |
+
assert bunch_from_pkl["x"] == bunch_from_pkl.x
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def test_bunch_pickle_generated_with_0_16_and_read_with_0_17():
|
| 341 |
+
bunch = Bunch(key="original")
|
| 342 |
+
# This reproduces a problem when Bunch pickles have been created
|
| 343 |
+
# with scikit-learn 0.16 and are read with 0.17. Basically there
|
| 344 |
+
# is a surprising behaviour because reading bunch.key uses
|
| 345 |
+
# bunch.__dict__ (which is non empty for 0.16 Bunch objects)
|
| 346 |
+
# whereas assigning into bunch.key uses bunch.__setattr__. See
|
| 347 |
+
# https://github.com/scikit-learn/scikit-learn/issues/6196 for
|
| 348 |
+
# more details
|
| 349 |
+
bunch.__dict__["key"] = "set from __dict__"
|
| 350 |
+
bunch_from_pkl = loads(dumps(bunch))
|
| 351 |
+
# After loading from pickle the __dict__ should have been ignored
|
| 352 |
+
assert bunch_from_pkl.key == "original"
|
| 353 |
+
assert bunch_from_pkl["key"] == "original"
|
| 354 |
+
# Making sure that changing the attr does change the value
|
| 355 |
+
# associated with __getitem__ as well
|
| 356 |
+
bunch_from_pkl.key = "changed"
|
| 357 |
+
assert bunch_from_pkl.key == "changed"
|
| 358 |
+
assert bunch_from_pkl["key"] == "changed"
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def test_bunch_dir():
|
| 362 |
+
# check that dir (important for autocomplete) shows attributes
|
| 363 |
+
data = load_iris()
|
| 364 |
+
assert "data" in dir(data)
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def test_load_boston_error():
|
| 368 |
+
"""Check that we raise the ethical warning when trying to import `load_boston`."""
|
| 369 |
+
msg = "The Boston housing prices dataset has an ethical problem"
|
| 370 |
+
with pytest.raises(ImportError, match=msg):
|
| 371 |
+
from sklearn.datasets import load_boston # noqa: F401
|
| 372 |
+
|
| 373 |
+
# other non-existing function should raise the usual import error
|
| 374 |
+
msg = "cannot import name 'non_existing_function' from 'sklearn.datasets'"
|
| 375 |
+
with pytest.raises(ImportError, match=msg):
|
| 376 |
+
from sklearn.datasets import non_existing_function # noqa: F401
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def test_fetch_remote_raise_warnings_with_invalid_url(monkeypatch):
|
| 380 |
+
"""Check retry mechanism in _fetch_remote."""
|
| 381 |
+
|
| 382 |
+
url = "https://scikit-learn.org/this_file_does_not_exist.tar.gz"
|
| 383 |
+
invalid_remote_file = RemoteFileMetadata("invalid_file", url, None)
|
| 384 |
+
urlretrieve_mock = Mock(
|
| 385 |
+
side_effect=HTTPError(
|
| 386 |
+
url=url, code=404, msg="Not Found", hdrs=None, fp=io.BytesIO()
|
| 387 |
+
)
|
| 388 |
+
)
|
| 389 |
+
monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
|
| 390 |
+
|
| 391 |
+
with pytest.warns(UserWarning, match="Retry downloading") as record:
|
| 392 |
+
with pytest.raises(HTTPError, match="HTTP Error 404"):
|
| 393 |
+
_fetch_remote(invalid_remote_file, n_retries=3, delay=0)
|
| 394 |
+
|
| 395 |
+
assert urlretrieve_mock.call_count == 4
|
| 396 |
+
|
| 397 |
+
for r in record:
|
| 398 |
+
assert str(r.message) == f"Retry downloading from url: {url}"
|
| 399 |
+
assert len(record) == 3
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def test_derive_folder_and_filename_from_url():
|
| 403 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 404 |
+
"https://example.com/file.tar.gz"
|
| 405 |
+
)
|
| 406 |
+
assert folder == "example.com"
|
| 407 |
+
assert filename == "file.tar.gz"
|
| 408 |
+
|
| 409 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 410 |
+
"https://example.com/نمونه نماینده.data"
|
| 411 |
+
)
|
| 412 |
+
assert folder == "example.com"
|
| 413 |
+
assert filename == "نمونه-نماینده.data"
|
| 414 |
+
|
| 415 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 416 |
+
"https://example.com/path/to-/.file.tar.gz"
|
| 417 |
+
)
|
| 418 |
+
assert folder == "example.com/path_to"
|
| 419 |
+
assert filename == "file.tar.gz"
|
| 420 |
+
|
| 421 |
+
folder, filename = _derive_folder_and_filename_from_url("https://example.com/")
|
| 422 |
+
assert folder == "example.com"
|
| 423 |
+
assert filename == "downloaded_file"
|
| 424 |
+
|
| 425 |
+
folder, filename = _derive_folder_and_filename_from_url("https://example.com")
|
| 426 |
+
assert folder == "example.com"
|
| 427 |
+
assert filename == "downloaded_file"
|
| 428 |
+
|
| 429 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 430 |
+
"https://example.com/path/@to/data.json?param=value"
|
| 431 |
+
)
|
| 432 |
+
assert folder == "example.com/path_to"
|
| 433 |
+
assert filename == "data.json"
|
| 434 |
+
|
| 435 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 436 |
+
"https://example.com/path/@@to._/-_.data.json.#anchor"
|
| 437 |
+
)
|
| 438 |
+
assert folder == "example.com/path_to"
|
| 439 |
+
assert filename == "data.json"
|
| 440 |
+
|
| 441 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 442 |
+
"https://example.com//some_file.txt"
|
| 443 |
+
)
|
| 444 |
+
assert folder == "example.com"
|
| 445 |
+
assert filename == "some_file.txt"
|
| 446 |
+
|
| 447 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 448 |
+
"http://example/../some_file.txt"
|
| 449 |
+
)
|
| 450 |
+
assert folder == "example"
|
| 451 |
+
assert filename == "some_file.txt"
|
| 452 |
+
|
| 453 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 454 |
+
"https://example.com/!.'.,/some_file.txt"
|
| 455 |
+
)
|
| 456 |
+
assert folder == "example.com"
|
| 457 |
+
assert filename == "some_file.txt"
|
| 458 |
+
|
| 459 |
+
folder, filename = _derive_folder_and_filename_from_url(
|
| 460 |
+
"https://example.com/a/!.'.,/b/some_file.txt"
|
| 461 |
+
)
|
| 462 |
+
assert folder == "example.com/a_b"
|
| 463 |
+
assert filename == "some_file.txt"
|
| 464 |
+
|
| 465 |
+
folder, filename = _derive_folder_and_filename_from_url("https://example.com/!.'.,")
|
| 466 |
+
assert folder == "example.com"
|
| 467 |
+
assert filename == "downloaded_file"
|
| 468 |
+
|
| 469 |
+
with pytest.raises(ValueError, match="Invalid URL"):
|
| 470 |
+
_derive_folder_and_filename_from_url("https:/../")
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def _mock_urlretrieve(server_side):
|
| 474 |
+
def _urlretrieve_mock(url, local_path):
|
| 475 |
+
server_root = Path(server_side)
|
| 476 |
+
file_path = urlparse(url).path.strip("/")
|
| 477 |
+
if not (server_root / file_path).exists():
|
| 478 |
+
raise HTTPError(url, 404, "Not Found", None, None)
|
| 479 |
+
shutil.copy(server_root / file_path, local_path)
|
| 480 |
+
|
| 481 |
+
return Mock(side_effect=_urlretrieve_mock)
|
| 482 |
+
|
| 483 |
+
|
| 484 |
+
def test_fetch_file_using_data_home(monkeypatch, tmpdir):
|
| 485 |
+
tmpdir = Path(tmpdir)
|
| 486 |
+
server_side = tmpdir / "server_side"
|
| 487 |
+
server_side.mkdir()
|
| 488 |
+
data_file = server_side / "data.jsonl"
|
| 489 |
+
server_data = '{"a": 1, "b": 2}\n'
|
| 490 |
+
data_file.write_text(server_data, encoding="utf-8")
|
| 491 |
+
|
| 492 |
+
server_subfolder = server_side / "subfolder"
|
| 493 |
+
server_subfolder.mkdir()
|
| 494 |
+
other_data_file = server_subfolder / "other_file.txt"
|
| 495 |
+
other_data_file.write_text("Some important text data.", encoding="utf-8")
|
| 496 |
+
|
| 497 |
+
data_home = tmpdir / "data_home"
|
| 498 |
+
data_home.mkdir()
|
| 499 |
+
|
| 500 |
+
urlretrieve_mock = _mock_urlretrieve(server_side)
|
| 501 |
+
monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
|
| 502 |
+
|
| 503 |
+
monkeypatch.setattr(
|
| 504 |
+
"sklearn.datasets._base.get_data_home", Mock(return_value=data_home)
|
| 505 |
+
)
|
| 506 |
+
fetched_file_path = fetch_file(
|
| 507 |
+
"https://example.com/data.jsonl",
|
| 508 |
+
)
|
| 509 |
+
assert fetched_file_path == data_home / "example.com" / "data.jsonl"
|
| 510 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 511 |
+
|
| 512 |
+
fetched_file_path = fetch_file(
|
| 513 |
+
"https://example.com/subfolder/other_file.txt",
|
| 514 |
+
)
|
| 515 |
+
assert (
|
| 516 |
+
fetched_file_path == data_home / "example.com" / "subfolder" / "other_file.txt"
|
| 517 |
+
)
|
| 518 |
+
assert fetched_file_path.read_text(encoding="utf-8") == other_data_file.read_text(
|
| 519 |
+
"utf-8"
|
| 520 |
+
)
|
| 521 |
+
|
| 522 |
+
expected_warning_msg = re.escape(
|
| 523 |
+
"Retry downloading from url: https://example.com/subfolder/invalid.txt"
|
| 524 |
+
)
|
| 525 |
+
with pytest.raises(HTTPError):
|
| 526 |
+
with pytest.warns(match=expected_warning_msg):
|
| 527 |
+
fetch_file(
|
| 528 |
+
"https://example.com/subfolder/invalid.txt",
|
| 529 |
+
delay=0,
|
| 530 |
+
)
|
| 531 |
+
|
| 532 |
+
local_subfolder = data_home / "example.com" / "subfolder"
|
| 533 |
+
assert sorted(local_subfolder.iterdir()) == [local_subfolder / "other_file.txt"]
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
def test_fetch_file_without_sha256(monkeypatch, tmpdir):
|
| 537 |
+
server_side = tmpdir.mkdir("server_side")
|
| 538 |
+
data_file = Path(server_side / "data.jsonl")
|
| 539 |
+
server_data = '{"a": 1, "b": 2}\n'
|
| 540 |
+
data_file.write_text(server_data, encoding="utf-8")
|
| 541 |
+
|
| 542 |
+
client_side = tmpdir.mkdir("client_side")
|
| 543 |
+
|
| 544 |
+
urlretrieve_mock = _mock_urlretrieve(server_side)
|
| 545 |
+
monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
|
| 546 |
+
|
| 547 |
+
# The first call should trigger a download:
|
| 548 |
+
fetched_file_path = fetch_file(
|
| 549 |
+
"https://example.com/data.jsonl",
|
| 550 |
+
folder=client_side,
|
| 551 |
+
)
|
| 552 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 553 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 554 |
+
assert urlretrieve_mock.call_count == 1
|
| 555 |
+
|
| 556 |
+
# Fetching again the same file to the same folder should do nothing:
|
| 557 |
+
fetched_file_path = fetch_file(
|
| 558 |
+
"https://example.com/data.jsonl",
|
| 559 |
+
folder=client_side,
|
| 560 |
+
)
|
| 561 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 562 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 563 |
+
assert urlretrieve_mock.call_count == 1
|
| 564 |
+
|
| 565 |
+
# Deleting and calling again should re-download
|
| 566 |
+
fetched_file_path.unlink()
|
| 567 |
+
fetched_file_path = fetch_file(
|
| 568 |
+
"https://example.com/data.jsonl",
|
| 569 |
+
folder=client_side,
|
| 570 |
+
)
|
| 571 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 572 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 573 |
+
assert urlretrieve_mock.call_count == 2
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
def test_fetch_file_with_sha256(monkeypatch, tmpdir):
|
| 577 |
+
server_side = tmpdir.mkdir("server_side")
|
| 578 |
+
data_file = Path(server_side / "data.jsonl")
|
| 579 |
+
server_data = '{"a": 1, "b": 2}\n'
|
| 580 |
+
data_file.write_text(server_data, encoding="utf-8")
|
| 581 |
+
expected_sha256 = hashlib.sha256(data_file.read_bytes()).hexdigest()
|
| 582 |
+
|
| 583 |
+
client_side = tmpdir.mkdir("client_side")
|
| 584 |
+
|
| 585 |
+
urlretrieve_mock = _mock_urlretrieve(server_side)
|
| 586 |
+
monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock)
|
| 587 |
+
|
| 588 |
+
# The first call should trigger a download.
|
| 589 |
+
fetched_file_path = fetch_file(
|
| 590 |
+
"https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
|
| 591 |
+
)
|
| 592 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 593 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 594 |
+
assert urlretrieve_mock.call_count == 1
|
| 595 |
+
|
| 596 |
+
# Fetching again the same file to the same folder should do nothing when
|
| 597 |
+
# the sha256 match:
|
| 598 |
+
fetched_file_path = fetch_file(
|
| 599 |
+
"https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
|
| 600 |
+
)
|
| 601 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 602 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 603 |
+
assert urlretrieve_mock.call_count == 1
|
| 604 |
+
|
| 605 |
+
# Corrupting the local data should yield a warning and trigger a new download:
|
| 606 |
+
fetched_file_path.write_text("corrupted contents", encoding="utf-8")
|
| 607 |
+
expected_msg = (
|
| 608 |
+
r"SHA256 checksum of existing local file data.jsonl "
|
| 609 |
+
rf"\(.*\) differs from expected \({expected_sha256}\): "
|
| 610 |
+
r"re-downloading from https://example.com/data.jsonl \."
|
| 611 |
+
)
|
| 612 |
+
with pytest.warns(match=expected_msg):
|
| 613 |
+
fetched_file_path = fetch_file(
|
| 614 |
+
"https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
|
| 615 |
+
)
|
| 616 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 617 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 618 |
+
assert urlretrieve_mock.call_count == 2
|
| 619 |
+
|
| 620 |
+
# Calling again should do nothing:
|
| 621 |
+
fetched_file_path = fetch_file(
|
| 622 |
+
"https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
|
| 623 |
+
)
|
| 624 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 625 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 626 |
+
assert urlretrieve_mock.call_count == 2
|
| 627 |
+
|
| 628 |
+
# Deleting the local file and calling again should redownload without warning:
|
| 629 |
+
fetched_file_path.unlink()
|
| 630 |
+
fetched_file_path = fetch_file(
|
| 631 |
+
"https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256
|
| 632 |
+
)
|
| 633 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 634 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 635 |
+
assert urlretrieve_mock.call_count == 3
|
| 636 |
+
|
| 637 |
+
# Calling without a sha256 should also work without redownloading:
|
| 638 |
+
fetched_file_path = fetch_file(
|
| 639 |
+
"https://example.com/data.jsonl",
|
| 640 |
+
folder=client_side,
|
| 641 |
+
)
|
| 642 |
+
assert fetched_file_path == client_side / "data.jsonl"
|
| 643 |
+
assert fetched_file_path.read_text(encoding="utf-8") == server_data
|
| 644 |
+
assert urlretrieve_mock.call_count == 3
|
| 645 |
+
|
| 646 |
+
# Calling with a wrong sha256 should raise an informative exception:
|
| 647 |
+
non_matching_sha256 = "deadbabecafebeef"
|
| 648 |
+
expected_warning_msg = "differs from expected"
|
| 649 |
+
expected_error_msg = re.escape(
|
| 650 |
+
f"The SHA256 checksum of data.jsonl ({expected_sha256}) differs from "
|
| 651 |
+
f"expected ({non_matching_sha256})."
|
| 652 |
+
)
|
| 653 |
+
with pytest.raises(OSError, match=expected_error_msg):
|
| 654 |
+
with pytest.warns(match=expected_warning_msg):
|
| 655 |
+
fetch_file(
|
| 656 |
+
"https://example.com/data.jsonl",
|
| 657 |
+
folder=client_side,
|
| 658 |
+
sha256=non_matching_sha256,
|
| 659 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_california_housing.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test the california_housing loader, if the data is available,
|
| 2 |
+
or if specifically requested via environment variable
|
| 3 |
+
(e.g. for CI jobs)."""
|
| 4 |
+
|
| 5 |
+
from functools import partial
|
| 6 |
+
|
| 7 |
+
import pytest
|
| 8 |
+
|
| 9 |
+
from sklearn.datasets.tests.test_common import check_return_X_y
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def test_fetch(fetch_california_housing_fxt):
|
| 13 |
+
data = fetch_california_housing_fxt()
|
| 14 |
+
assert (20640, 8) == data.data.shape
|
| 15 |
+
assert (20640,) == data.target.shape
|
| 16 |
+
assert data.DESCR.startswith(".. _california_housing_dataset:")
|
| 17 |
+
|
| 18 |
+
# test return_X_y option
|
| 19 |
+
fetch_func = partial(fetch_california_housing_fxt)
|
| 20 |
+
check_return_X_y(data, fetch_func)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_fetch_asframe(fetch_california_housing_fxt):
|
| 24 |
+
pd = pytest.importorskip("pandas")
|
| 25 |
+
bunch = fetch_california_housing_fxt(as_frame=True)
|
| 26 |
+
frame = bunch.frame
|
| 27 |
+
assert hasattr(bunch, "frame") is True
|
| 28 |
+
assert frame.shape == (20640, 9)
|
| 29 |
+
assert isinstance(bunch.data, pd.DataFrame)
|
| 30 |
+
assert isinstance(bunch.target, pd.Series)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def test_pandas_dependency_message(fetch_california_housing_fxt, hide_available_pandas):
|
| 34 |
+
# Check that pandas is imported lazily and that an informative error
|
| 35 |
+
# message is raised when pandas is missing:
|
| 36 |
+
expected_msg = "fetch_california_housing with as_frame=True requires pandas"
|
| 37 |
+
with pytest.raises(ImportError, match=expected_msg):
|
| 38 |
+
fetch_california_housing_fxt(as_frame=True)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_common.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test loaders for common functionality."""
|
| 2 |
+
|
| 3 |
+
import inspect
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import pytest
|
| 8 |
+
|
| 9 |
+
import sklearn.datasets
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def is_pillow_installed():
|
| 13 |
+
try:
|
| 14 |
+
import PIL # noqa: F401
|
| 15 |
+
|
| 16 |
+
return True
|
| 17 |
+
except ImportError:
|
| 18 |
+
return False
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
FETCH_PYTEST_MARKERS = {
|
| 22 |
+
"return_X_y": {
|
| 23 |
+
"fetch_20newsgroups": pytest.mark.xfail(
|
| 24 |
+
reason="X is a list and does not have a shape argument"
|
| 25 |
+
),
|
| 26 |
+
"fetch_openml": pytest.mark.xfail(
|
| 27 |
+
reason="fetch_opeml requires a dataset name or id"
|
| 28 |
+
),
|
| 29 |
+
"fetch_lfw_people": pytest.mark.skipif(
|
| 30 |
+
not is_pillow_installed(), reason="pillow is not installed"
|
| 31 |
+
),
|
| 32 |
+
},
|
| 33 |
+
"as_frame": {
|
| 34 |
+
"fetch_openml": pytest.mark.xfail(
|
| 35 |
+
reason="fetch_opeml requires a dataset name or id"
|
| 36 |
+
),
|
| 37 |
+
},
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def check_pandas_dependency_message(fetch_func):
|
| 42 |
+
try:
|
| 43 |
+
import pandas # noqa: F401
|
| 44 |
+
|
| 45 |
+
pytest.skip("This test requires pandas to not be installed")
|
| 46 |
+
except ImportError:
|
| 47 |
+
# Check that pandas is imported lazily and that an informative error
|
| 48 |
+
# message is raised when pandas is missing:
|
| 49 |
+
name = fetch_func.__name__
|
| 50 |
+
expected_msg = f"{name} with as_frame=True requires pandas"
|
| 51 |
+
with pytest.raises(ImportError, match=expected_msg):
|
| 52 |
+
fetch_func(as_frame=True)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def check_return_X_y(bunch, dataset_func):
|
| 56 |
+
X_y_tuple = dataset_func(return_X_y=True)
|
| 57 |
+
assert isinstance(X_y_tuple, tuple)
|
| 58 |
+
assert X_y_tuple[0].shape == bunch.data.shape
|
| 59 |
+
assert X_y_tuple[1].shape == bunch.target.shape
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def check_as_frame(
|
| 63 |
+
bunch, dataset_func, expected_data_dtype=None, expected_target_dtype=None
|
| 64 |
+
):
|
| 65 |
+
pd = pytest.importorskip("pandas")
|
| 66 |
+
frame_bunch = dataset_func(as_frame=True)
|
| 67 |
+
assert hasattr(frame_bunch, "frame")
|
| 68 |
+
assert isinstance(frame_bunch.frame, pd.DataFrame)
|
| 69 |
+
assert isinstance(frame_bunch.data, pd.DataFrame)
|
| 70 |
+
assert frame_bunch.data.shape == bunch.data.shape
|
| 71 |
+
if frame_bunch.target.ndim > 1:
|
| 72 |
+
assert isinstance(frame_bunch.target, pd.DataFrame)
|
| 73 |
+
else:
|
| 74 |
+
assert isinstance(frame_bunch.target, pd.Series)
|
| 75 |
+
assert frame_bunch.target.shape[0] == bunch.target.shape[0]
|
| 76 |
+
if expected_data_dtype is not None:
|
| 77 |
+
assert np.all(frame_bunch.data.dtypes == expected_data_dtype)
|
| 78 |
+
if expected_target_dtype is not None:
|
| 79 |
+
assert np.all(frame_bunch.target.dtypes == expected_target_dtype)
|
| 80 |
+
|
| 81 |
+
# Test for return_X_y and as_frame=True
|
| 82 |
+
frame_X, frame_y = dataset_func(as_frame=True, return_X_y=True)
|
| 83 |
+
assert isinstance(frame_X, pd.DataFrame)
|
| 84 |
+
if frame_y.ndim > 1:
|
| 85 |
+
assert isinstance(frame_y, pd.DataFrame)
|
| 86 |
+
else:
|
| 87 |
+
assert isinstance(frame_y, pd.Series)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _skip_network_tests():
|
| 91 |
+
return os.environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "1"
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _generate_func_supporting_param(param, dataset_type=("load", "fetch")):
|
| 95 |
+
markers_fetch = FETCH_PYTEST_MARKERS.get(param, {})
|
| 96 |
+
for name, obj in inspect.getmembers(sklearn.datasets):
|
| 97 |
+
if not inspect.isfunction(obj):
|
| 98 |
+
continue
|
| 99 |
+
|
| 100 |
+
is_dataset_type = any([name.startswith(t) for t in dataset_type])
|
| 101 |
+
is_support_param = param in inspect.signature(obj).parameters
|
| 102 |
+
if is_dataset_type and is_support_param:
|
| 103 |
+
# check if we should skip if we don't have network support
|
| 104 |
+
marks = [
|
| 105 |
+
pytest.mark.skipif(
|
| 106 |
+
condition=name.startswith("fetch") and _skip_network_tests(),
|
| 107 |
+
reason="Skip because fetcher requires internet network",
|
| 108 |
+
)
|
| 109 |
+
]
|
| 110 |
+
if name in markers_fetch:
|
| 111 |
+
marks.append(markers_fetch[name])
|
| 112 |
+
|
| 113 |
+
yield pytest.param(name, obj, marks=marks)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
@pytest.mark.parametrize(
|
| 117 |
+
"name, dataset_func", _generate_func_supporting_param("return_X_y")
|
| 118 |
+
)
|
| 119 |
+
def test_common_check_return_X_y(name, dataset_func):
|
| 120 |
+
bunch = dataset_func()
|
| 121 |
+
check_return_X_y(bunch, dataset_func)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@pytest.mark.parametrize(
|
| 125 |
+
"name, dataset_func", _generate_func_supporting_param("as_frame")
|
| 126 |
+
)
|
| 127 |
+
def test_common_check_as_frame(name, dataset_func):
|
| 128 |
+
bunch = dataset_func()
|
| 129 |
+
check_as_frame(bunch, dataset_func)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
@pytest.mark.parametrize(
|
| 133 |
+
"name, dataset_func", _generate_func_supporting_param("as_frame")
|
| 134 |
+
)
|
| 135 |
+
def test_common_check_pandas_dependency(name, dataset_func):
|
| 136 |
+
check_pandas_dependency_message(dataset_func)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_covtype.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test the covtype loader, if the data is available,
|
| 2 |
+
or if specifically requested via environment variable
|
| 3 |
+
(e.g. for CI jobs)."""
|
| 4 |
+
|
| 5 |
+
from functools import partial
|
| 6 |
+
|
| 7 |
+
import pytest
|
| 8 |
+
|
| 9 |
+
from sklearn.datasets.tests.test_common import check_return_X_y
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def test_fetch(fetch_covtype_fxt, global_random_seed):
|
| 13 |
+
data1 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed)
|
| 14 |
+
data2 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed + 1)
|
| 15 |
+
|
| 16 |
+
X1, X2 = data1["data"], data2["data"]
|
| 17 |
+
assert (581012, 54) == X1.shape
|
| 18 |
+
assert X1.shape == X2.shape
|
| 19 |
+
|
| 20 |
+
assert X1.sum() == X2.sum()
|
| 21 |
+
|
| 22 |
+
y1, y2 = data1["target"], data2["target"]
|
| 23 |
+
assert (X1.shape[0],) == y1.shape
|
| 24 |
+
assert (X1.shape[0],) == y2.shape
|
| 25 |
+
|
| 26 |
+
descr_prefix = ".. _covtype_dataset:"
|
| 27 |
+
assert data1.DESCR.startswith(descr_prefix)
|
| 28 |
+
assert data2.DESCR.startswith(descr_prefix)
|
| 29 |
+
|
| 30 |
+
# test return_X_y option
|
| 31 |
+
fetch_func = partial(fetch_covtype_fxt)
|
| 32 |
+
check_return_X_y(data1, fetch_func)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_fetch_asframe(fetch_covtype_fxt):
|
| 36 |
+
pytest.importorskip("pandas")
|
| 37 |
+
|
| 38 |
+
bunch = fetch_covtype_fxt(as_frame=True)
|
| 39 |
+
assert hasattr(bunch, "frame")
|
| 40 |
+
frame = bunch.frame
|
| 41 |
+
assert frame.shape == (581012, 55)
|
| 42 |
+
assert bunch.data.shape == (581012, 54)
|
| 43 |
+
assert bunch.target.shape == (581012,)
|
| 44 |
+
|
| 45 |
+
column_names = set(frame.columns)
|
| 46 |
+
|
| 47 |
+
# enumerated names are added correctly
|
| 48 |
+
assert set(f"Wilderness_Area_{i}" for i in range(4)) < column_names
|
| 49 |
+
assert set(f"Soil_Type_{i}" for i in range(40)) < column_names
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def test_pandas_dependency_message(fetch_covtype_fxt, hide_available_pandas):
|
| 53 |
+
expected_msg = "fetch_covtype with as_frame=True requires pandas"
|
| 54 |
+
with pytest.raises(ImportError, match=expected_msg):
|
| 55 |
+
fetch_covtype_fxt(as_frame=True)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_kddcup99.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test kddcup99 loader, if the data is available,
|
| 2 |
+
or if specifically requested via environment variable
|
| 3 |
+
(e.g. for CI jobs).
|
| 4 |
+
|
| 5 |
+
Only 'percent10' mode is tested, as the full data
|
| 6 |
+
is too big to use in unit-testing.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from functools import partial
|
| 10 |
+
|
| 11 |
+
import pytest
|
| 12 |
+
|
| 13 |
+
from sklearn.datasets.tests.test_common import (
|
| 14 |
+
check_as_frame,
|
| 15 |
+
check_pandas_dependency_message,
|
| 16 |
+
check_return_X_y,
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@pytest.mark.parametrize("as_frame", [True, False])
|
| 21 |
+
@pytest.mark.parametrize(
|
| 22 |
+
"subset, n_samples, n_features",
|
| 23 |
+
[
|
| 24 |
+
(None, 494021, 41),
|
| 25 |
+
("SA", 100655, 41),
|
| 26 |
+
("SF", 73237, 4),
|
| 27 |
+
("http", 58725, 3),
|
| 28 |
+
("smtp", 9571, 3),
|
| 29 |
+
],
|
| 30 |
+
)
|
| 31 |
+
def test_fetch_kddcup99_percent10(
|
| 32 |
+
fetch_kddcup99_fxt, as_frame, subset, n_samples, n_features
|
| 33 |
+
):
|
| 34 |
+
data = fetch_kddcup99_fxt(subset=subset, as_frame=as_frame)
|
| 35 |
+
assert data.data.shape == (n_samples, n_features)
|
| 36 |
+
assert data.target.shape == (n_samples,)
|
| 37 |
+
if as_frame:
|
| 38 |
+
assert data.frame.shape == (n_samples, n_features + 1)
|
| 39 |
+
assert data.DESCR.startswith(".. _kddcup99_dataset:")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def test_fetch_kddcup99_return_X_y(fetch_kddcup99_fxt):
|
| 43 |
+
fetch_func = partial(fetch_kddcup99_fxt, subset="smtp")
|
| 44 |
+
data = fetch_func()
|
| 45 |
+
check_return_X_y(data, fetch_func)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def test_fetch_kddcup99_as_frame(fetch_kddcup99_fxt):
|
| 49 |
+
bunch = fetch_kddcup99_fxt()
|
| 50 |
+
check_as_frame(bunch, fetch_kddcup99_fxt)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def test_fetch_kddcup99_shuffle(fetch_kddcup99_fxt):
|
| 54 |
+
dataset = fetch_kddcup99_fxt(
|
| 55 |
+
random_state=0,
|
| 56 |
+
subset="SA",
|
| 57 |
+
percent10=True,
|
| 58 |
+
)
|
| 59 |
+
dataset_shuffled = fetch_kddcup99_fxt(
|
| 60 |
+
random_state=0,
|
| 61 |
+
subset="SA",
|
| 62 |
+
shuffle=True,
|
| 63 |
+
percent10=True,
|
| 64 |
+
)
|
| 65 |
+
assert set(dataset["target"]) == set(dataset_shuffled["target"])
|
| 66 |
+
assert dataset_shuffled.data.shape == dataset.data.shape
|
| 67 |
+
assert dataset_shuffled.target.shape == dataset.target.shape
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def test_pandas_dependency_message(fetch_kddcup99_fxt, hide_available_pandas):
|
| 71 |
+
check_pandas_dependency_message(fetch_kddcup99_fxt)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def test_corrupted_file_error_message(fetch_kddcup99_fxt, tmp_path):
|
| 75 |
+
"""Check that a nice error message is raised when cache is corrupted."""
|
| 76 |
+
kddcup99_dir = tmp_path / "kddcup99_10-py3"
|
| 77 |
+
kddcup99_dir.mkdir()
|
| 78 |
+
samples_path = kddcup99_dir / "samples"
|
| 79 |
+
|
| 80 |
+
with samples_path.open("wb") as f:
|
| 81 |
+
f.write(b"THIS IS CORRUPTED")
|
| 82 |
+
|
| 83 |
+
msg = (
|
| 84 |
+
"The cache for fetch_kddcup99 is invalid, please "
|
| 85 |
+
f"delete {kddcup99_dir} and run the fetch_kddcup99 again"
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
with pytest.raises(OSError, match=msg):
|
| 89 |
+
fetch_kddcup99_fxt(data_home=str(tmp_path))
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_lfw.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This test for the LFW require medium-size data downloading and processing
|
| 2 |
+
|
| 3 |
+
If the data has not been already downloaded by running the examples,
|
| 4 |
+
the tests won't run (skipped).
|
| 5 |
+
|
| 6 |
+
If the test are run, the first execution will be long (typically a bit
|
| 7 |
+
more than a couple of minutes) but as the dataset loader is leveraging
|
| 8 |
+
joblib, successive runs will be fast (less than 200ms).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import random
|
| 12 |
+
from functools import partial
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pytest
|
| 16 |
+
|
| 17 |
+
from sklearn.datasets import fetch_lfw_pairs, fetch_lfw_people
|
| 18 |
+
from sklearn.datasets.tests.test_common import check_return_X_y
|
| 19 |
+
from sklearn.utils._testing import assert_array_equal
|
| 20 |
+
|
| 21 |
+
FAKE_NAMES = [
|
| 22 |
+
"Abdelatif_Smith",
|
| 23 |
+
"Abhati_Kepler",
|
| 24 |
+
"Camara_Alvaro",
|
| 25 |
+
"Chen_Dupont",
|
| 26 |
+
"John_Lee",
|
| 27 |
+
"Lin_Bauman",
|
| 28 |
+
"Onur_Lopez",
|
| 29 |
+
]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@pytest.fixture(scope="module")
|
| 33 |
+
def mock_empty_data_home(tmp_path_factory):
|
| 34 |
+
data_dir = tmp_path_factory.mktemp("scikit_learn_empty_test")
|
| 35 |
+
|
| 36 |
+
yield data_dir
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@pytest.fixture(scope="module")
|
| 40 |
+
def mock_data_home(tmp_path_factory):
|
| 41 |
+
"""Test fixture run once and common to all tests of this module"""
|
| 42 |
+
Image = pytest.importorskip("PIL.Image")
|
| 43 |
+
|
| 44 |
+
data_dir = tmp_path_factory.mktemp("scikit_learn_lfw_test")
|
| 45 |
+
lfw_home = data_dir / "lfw_home"
|
| 46 |
+
lfw_home.mkdir(parents=True, exist_ok=True)
|
| 47 |
+
|
| 48 |
+
random_state = random.Random(42)
|
| 49 |
+
np_rng = np.random.RandomState(42)
|
| 50 |
+
|
| 51 |
+
# generate some random jpeg files for each person
|
| 52 |
+
counts = {}
|
| 53 |
+
for name in FAKE_NAMES:
|
| 54 |
+
folder_name = lfw_home / "lfw_funneled" / name
|
| 55 |
+
folder_name.mkdir(parents=True, exist_ok=True)
|
| 56 |
+
|
| 57 |
+
n_faces = np_rng.randint(1, 5)
|
| 58 |
+
counts[name] = n_faces
|
| 59 |
+
for i in range(n_faces):
|
| 60 |
+
file_path = folder_name / (name + "_%04d.jpg" % i)
|
| 61 |
+
uniface = np_rng.randint(0, 255, size=(250, 250, 3))
|
| 62 |
+
img = Image.fromarray(uniface.astype(np.uint8))
|
| 63 |
+
img.save(file_path)
|
| 64 |
+
|
| 65 |
+
# add some random file pollution to test robustness
|
| 66 |
+
(lfw_home / "lfw_funneled" / ".test.swp").write_bytes(
|
| 67 |
+
b"Text file to be ignored by the dataset loader."
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
# generate some pairing metadata files using the same format as LFW
|
| 71 |
+
with open(lfw_home / "pairsDevTrain.txt", "wb") as f:
|
| 72 |
+
f.write(b"10\n")
|
| 73 |
+
more_than_two = [name for name, count in counts.items() if count >= 2]
|
| 74 |
+
for i in range(5):
|
| 75 |
+
name = random_state.choice(more_than_two)
|
| 76 |
+
first, second = random_state.sample(range(counts[name]), 2)
|
| 77 |
+
f.write(("%s\t%d\t%d\n" % (name, first, second)).encode())
|
| 78 |
+
|
| 79 |
+
for i in range(5):
|
| 80 |
+
first_name, second_name = random_state.sample(FAKE_NAMES, 2)
|
| 81 |
+
first_index = np_rng.choice(np.arange(counts[first_name]))
|
| 82 |
+
second_index = np_rng.choice(np.arange(counts[second_name]))
|
| 83 |
+
f.write(
|
| 84 |
+
(
|
| 85 |
+
"%s\t%d\t%s\t%d\n"
|
| 86 |
+
% (first_name, first_index, second_name, second_index)
|
| 87 |
+
).encode()
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
(lfw_home / "pairsDevTest.txt").write_bytes(
|
| 91 |
+
b"Fake place holder that won't be tested"
|
| 92 |
+
)
|
| 93 |
+
(lfw_home / "pairs.txt").write_bytes(b"Fake place holder that won't be tested")
|
| 94 |
+
|
| 95 |
+
yield data_dir
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def test_load_empty_lfw_people(mock_empty_data_home):
|
| 99 |
+
with pytest.raises(OSError):
|
| 100 |
+
fetch_lfw_people(data_home=mock_empty_data_home, download_if_missing=False)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def test_load_fake_lfw_people(mock_data_home):
|
| 104 |
+
lfw_people = fetch_lfw_people(
|
| 105 |
+
data_home=mock_data_home, min_faces_per_person=3, download_if_missing=False
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
# The data is croped around the center as a rectangular bounding box
|
| 109 |
+
# around the face. Colors are converted to gray levels:
|
| 110 |
+
assert lfw_people.images.shape == (10, 62, 47)
|
| 111 |
+
assert lfw_people.data.shape == (10, 2914)
|
| 112 |
+
|
| 113 |
+
# the target is array of person integer ids
|
| 114 |
+
assert_array_equal(lfw_people.target, [2, 0, 1, 0, 2, 0, 2, 1, 1, 2])
|
| 115 |
+
|
| 116 |
+
# names of the persons can be found using the target_names array
|
| 117 |
+
expected_classes = ["Abdelatif Smith", "Abhati Kepler", "Onur Lopez"]
|
| 118 |
+
assert_array_equal(lfw_people.target_names, expected_classes)
|
| 119 |
+
|
| 120 |
+
# It is possible to ask for the original data without any croping or color
|
| 121 |
+
# conversion and not limit on the number of picture per person
|
| 122 |
+
lfw_people = fetch_lfw_people(
|
| 123 |
+
data_home=mock_data_home,
|
| 124 |
+
resize=None,
|
| 125 |
+
slice_=None,
|
| 126 |
+
color=True,
|
| 127 |
+
download_if_missing=False,
|
| 128 |
+
)
|
| 129 |
+
assert lfw_people.images.shape == (17, 250, 250, 3)
|
| 130 |
+
assert lfw_people.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:")
|
| 131 |
+
|
| 132 |
+
# the ids and class names are the same as previously
|
| 133 |
+
assert_array_equal(
|
| 134 |
+
lfw_people.target, [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2]
|
| 135 |
+
)
|
| 136 |
+
assert_array_equal(
|
| 137 |
+
lfw_people.target_names,
|
| 138 |
+
[
|
| 139 |
+
"Abdelatif Smith",
|
| 140 |
+
"Abhati Kepler",
|
| 141 |
+
"Camara Alvaro",
|
| 142 |
+
"Chen Dupont",
|
| 143 |
+
"John Lee",
|
| 144 |
+
"Lin Bauman",
|
| 145 |
+
"Onur Lopez",
|
| 146 |
+
],
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
# test return_X_y option
|
| 150 |
+
fetch_func = partial(
|
| 151 |
+
fetch_lfw_people,
|
| 152 |
+
data_home=mock_data_home,
|
| 153 |
+
resize=None,
|
| 154 |
+
slice_=None,
|
| 155 |
+
color=True,
|
| 156 |
+
download_if_missing=False,
|
| 157 |
+
)
|
| 158 |
+
check_return_X_y(lfw_people, fetch_func)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def test_load_fake_lfw_people_too_restrictive(mock_data_home):
|
| 162 |
+
with pytest.raises(ValueError):
|
| 163 |
+
fetch_lfw_people(
|
| 164 |
+
data_home=mock_data_home,
|
| 165 |
+
min_faces_per_person=100,
|
| 166 |
+
download_if_missing=False,
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def test_load_empty_lfw_pairs(mock_empty_data_home):
|
| 171 |
+
with pytest.raises(OSError):
|
| 172 |
+
fetch_lfw_pairs(data_home=mock_empty_data_home, download_if_missing=False)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def test_load_fake_lfw_pairs(mock_data_home):
|
| 176 |
+
lfw_pairs_train = fetch_lfw_pairs(
|
| 177 |
+
data_home=mock_data_home, download_if_missing=False
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
# The data is croped around the center as a rectangular bounding box
|
| 181 |
+
# around the face. Colors are converted to gray levels:
|
| 182 |
+
assert lfw_pairs_train.pairs.shape == (10, 2, 62, 47)
|
| 183 |
+
|
| 184 |
+
# the target is whether the person is the same or not
|
| 185 |
+
assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
|
| 186 |
+
|
| 187 |
+
# names of the persons can be found using the target_names array
|
| 188 |
+
expected_classes = ["Different persons", "Same person"]
|
| 189 |
+
assert_array_equal(lfw_pairs_train.target_names, expected_classes)
|
| 190 |
+
|
| 191 |
+
# It is possible to ask for the original data without any croping or color
|
| 192 |
+
# conversion
|
| 193 |
+
lfw_pairs_train = fetch_lfw_pairs(
|
| 194 |
+
data_home=mock_data_home,
|
| 195 |
+
resize=None,
|
| 196 |
+
slice_=None,
|
| 197 |
+
color=True,
|
| 198 |
+
download_if_missing=False,
|
| 199 |
+
)
|
| 200 |
+
assert lfw_pairs_train.pairs.shape == (10, 2, 250, 250, 3)
|
| 201 |
+
|
| 202 |
+
# the ids and class names are the same as previously
|
| 203 |
+
assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
|
| 204 |
+
assert_array_equal(lfw_pairs_train.target_names, expected_classes)
|
| 205 |
+
|
| 206 |
+
assert lfw_pairs_train.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:")
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def test_fetch_lfw_people_internal_cropping(mock_data_home):
|
| 210 |
+
"""Check that we properly crop the images.
|
| 211 |
+
|
| 212 |
+
Non-regression test for:
|
| 213 |
+
https://github.com/scikit-learn/scikit-learn/issues/24942
|
| 214 |
+
"""
|
| 215 |
+
# If cropping was not done properly and we don't resize the images, the images would
|
| 216 |
+
# have their original size (250x250) and the image would not fit in the NumPy array
|
| 217 |
+
# pre-allocated based on `slice_` parameter.
|
| 218 |
+
slice_ = (slice(70, 195), slice(78, 172))
|
| 219 |
+
lfw = fetch_lfw_people(
|
| 220 |
+
data_home=mock_data_home,
|
| 221 |
+
min_faces_per_person=3,
|
| 222 |
+
download_if_missing=False,
|
| 223 |
+
resize=None,
|
| 224 |
+
slice_=slice_,
|
| 225 |
+
)
|
| 226 |
+
assert lfw.images[0].shape == (
|
| 227 |
+
slice_[0].stop - slice_[0].start,
|
| 228 |
+
slice_[1].stop - slice_[1].start,
|
| 229 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_olivetti_faces.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test Olivetti faces fetcher, if the data is available,
|
| 2 |
+
or if specifically requested via environment variable
|
| 3 |
+
(e.g. for CI jobs)."""
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
from sklearn.datasets.tests.test_common import check_return_X_y
|
| 8 |
+
from sklearn.utils import Bunch
|
| 9 |
+
from sklearn.utils._testing import assert_array_equal
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def test_olivetti_faces(fetch_olivetti_faces_fxt):
|
| 13 |
+
data = fetch_olivetti_faces_fxt(shuffle=True, random_state=0)
|
| 14 |
+
|
| 15 |
+
assert isinstance(data, Bunch)
|
| 16 |
+
for expected_keys in ("data", "images", "target", "DESCR"):
|
| 17 |
+
assert expected_keys in data.keys()
|
| 18 |
+
|
| 19 |
+
assert data.data.shape == (400, 4096)
|
| 20 |
+
assert data.images.shape == (400, 64, 64)
|
| 21 |
+
assert data.target.shape == (400,)
|
| 22 |
+
assert_array_equal(np.unique(np.sort(data.target)), np.arange(40))
|
| 23 |
+
assert data.DESCR.startswith(".. _olivetti_faces_dataset:")
|
| 24 |
+
|
| 25 |
+
# test the return_X_y option
|
| 26 |
+
check_return_X_y(data, fetch_olivetti_faces_fxt)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_openml.py
ADDED
|
@@ -0,0 +1,1636 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test the openml loader."""
|
| 2 |
+
|
| 3 |
+
import gzip
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import re
|
| 7 |
+
from functools import partial
|
| 8 |
+
from importlib import resources
|
| 9 |
+
from io import BytesIO
|
| 10 |
+
from urllib.error import HTTPError
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import pytest
|
| 14 |
+
import scipy.sparse
|
| 15 |
+
|
| 16 |
+
import sklearn
|
| 17 |
+
from sklearn import config_context
|
| 18 |
+
from sklearn.datasets import fetch_openml as fetch_openml_orig
|
| 19 |
+
from sklearn.datasets._openml import (
|
| 20 |
+
_get_local_path,
|
| 21 |
+
_open_openml_url,
|
| 22 |
+
_retry_with_clean_cache,
|
| 23 |
+
)
|
| 24 |
+
from sklearn.utils import Bunch
|
| 25 |
+
from sklearn.utils._optional_dependencies import check_pandas_support
|
| 26 |
+
from sklearn.utils._testing import (
|
| 27 |
+
SkipTest,
|
| 28 |
+
assert_allclose,
|
| 29 |
+
assert_array_equal,
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
OPENML_TEST_DATA_MODULE = "sklearn.datasets.tests.data.openml"
|
| 33 |
+
# if True, urlopen will be monkey patched to only use local files
|
| 34 |
+
test_offline = True
|
| 35 |
+
_MONKEY_PATCH_LOCAL_OPENML_PATH = "data/v1/download/{}"
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class _MockHTTPResponse:
|
| 39 |
+
def __init__(self, data, is_gzip):
|
| 40 |
+
self.data = data
|
| 41 |
+
self.is_gzip = is_gzip
|
| 42 |
+
|
| 43 |
+
def read(self, amt=-1):
|
| 44 |
+
return self.data.read(amt)
|
| 45 |
+
|
| 46 |
+
def close(self):
|
| 47 |
+
self.data.close()
|
| 48 |
+
|
| 49 |
+
def info(self):
|
| 50 |
+
if self.is_gzip:
|
| 51 |
+
return {"Content-Encoding": "gzip"}
|
| 52 |
+
return {}
|
| 53 |
+
|
| 54 |
+
def __iter__(self):
|
| 55 |
+
return iter(self.data)
|
| 56 |
+
|
| 57 |
+
def __enter__(self):
|
| 58 |
+
return self
|
| 59 |
+
|
| 60 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 61 |
+
return False
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
# Disable the disk-based cache when testing `fetch_openml`:
|
| 65 |
+
# the mock data in sklearn/datasets/tests/data/openml/ is not always consistent
|
| 66 |
+
# with the version on openml.org. If one were to load the dataset outside of
|
| 67 |
+
# the tests, it may result in data that does not represent openml.org.
|
| 68 |
+
fetch_openml = partial(fetch_openml_orig, data_home=None)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _monkey_patch_webbased_functions(context, data_id, gzip_response):
|
| 72 |
+
# monkey patches the urlopen function. Important note: Do NOT use this
|
| 73 |
+
# in combination with a regular cache directory, as the files that are
|
| 74 |
+
# stored as cache should not be mixed up with real openml datasets
|
| 75 |
+
url_prefix_data_description = "https://api.openml.org/api/v1/json/data/"
|
| 76 |
+
url_prefix_data_features = "https://api.openml.org/api/v1/json/data/features/"
|
| 77 |
+
url_prefix_download_data = "https://www.openml.org/data/v1/download"
|
| 78 |
+
url_prefix_data_list = "https://api.openml.org/api/v1/json/data/list/"
|
| 79 |
+
|
| 80 |
+
path_suffix = ".gz"
|
| 81 |
+
read_fn = gzip.open
|
| 82 |
+
|
| 83 |
+
data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}"
|
| 84 |
+
|
| 85 |
+
def _file_name(url, suffix):
|
| 86 |
+
output = (
|
| 87 |
+
re.sub(r"\W", "-", url[len("https://api.openml.org/") :])
|
| 88 |
+
+ suffix
|
| 89 |
+
+ path_suffix
|
| 90 |
+
)
|
| 91 |
+
# Shorten the filenames to have better compatibility with windows 10
|
| 92 |
+
# and filenames > 260 characters
|
| 93 |
+
return (
|
| 94 |
+
output.replace("-json-data-list", "-jdl")
|
| 95 |
+
.replace("-json-data-features", "-jdf")
|
| 96 |
+
.replace("-json-data-qualities", "-jdq")
|
| 97 |
+
.replace("-json-data", "-jd")
|
| 98 |
+
.replace("-data_name", "-dn")
|
| 99 |
+
.replace("-download", "-dl")
|
| 100 |
+
.replace("-limit", "-l")
|
| 101 |
+
.replace("-data_version", "-dv")
|
| 102 |
+
.replace("-status", "-s")
|
| 103 |
+
.replace("-deactivated", "-dact")
|
| 104 |
+
.replace("-active", "-act")
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
def _mock_urlopen_shared(url, has_gzip_header, expected_prefix, suffix):
|
| 108 |
+
assert url.startswith(expected_prefix), (
|
| 109 |
+
f"{expected_prefix!r} does not match {url!r}"
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
data_file_name = _file_name(url, suffix)
|
| 113 |
+
data_file_path = resources.files(data_module) / data_file_name
|
| 114 |
+
|
| 115 |
+
with data_file_path.open("rb") as f:
|
| 116 |
+
if has_gzip_header and gzip_response:
|
| 117 |
+
fp = BytesIO(f.read())
|
| 118 |
+
return _MockHTTPResponse(fp, True)
|
| 119 |
+
else:
|
| 120 |
+
decompressed_f = read_fn(f, "rb")
|
| 121 |
+
fp = BytesIO(decompressed_f.read())
|
| 122 |
+
return _MockHTTPResponse(fp, False)
|
| 123 |
+
|
| 124 |
+
def _mock_urlopen_data_description(url, has_gzip_header):
|
| 125 |
+
return _mock_urlopen_shared(
|
| 126 |
+
url=url,
|
| 127 |
+
has_gzip_header=has_gzip_header,
|
| 128 |
+
expected_prefix=url_prefix_data_description,
|
| 129 |
+
suffix=".json",
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
def _mock_urlopen_data_features(url, has_gzip_header):
|
| 133 |
+
return _mock_urlopen_shared(
|
| 134 |
+
url=url,
|
| 135 |
+
has_gzip_header=has_gzip_header,
|
| 136 |
+
expected_prefix=url_prefix_data_features,
|
| 137 |
+
suffix=".json",
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
def _mock_urlopen_download_data(url, has_gzip_header):
|
| 141 |
+
# For simplicity the mock filenames don't contain the filename, i.e.
|
| 142 |
+
# the last part of the data description url after the last /.
|
| 143 |
+
# For example for id_1, data description download url is:
|
| 144 |
+
# gunzip -c sklearn/datasets/tests/data/openml/id_1/api-v1-jd-1.json.gz | grep '"url" # noqa: E501
|
| 145 |
+
# "https:\/\/www.openml.org\/data\/v1\/download\/1\/anneal.arff"
|
| 146 |
+
# but the mock filename does not contain anneal.arff and is:
|
| 147 |
+
# sklearn/datasets/tests/data/openml/id_1/data-v1-dl-1.arff.gz.
|
| 148 |
+
# We only keep the part of the url before the last /
|
| 149 |
+
url_without_filename = url.rsplit("/", 1)[0]
|
| 150 |
+
|
| 151 |
+
return _mock_urlopen_shared(
|
| 152 |
+
url=url_without_filename,
|
| 153 |
+
has_gzip_header=has_gzip_header,
|
| 154 |
+
expected_prefix=url_prefix_download_data,
|
| 155 |
+
suffix=".arff",
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
def _mock_urlopen_data_list(url, has_gzip_header):
|
| 159 |
+
assert url.startswith(url_prefix_data_list), (
|
| 160 |
+
f"{url_prefix_data_list!r} does not match {url!r}"
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
data_file_name = _file_name(url, ".json")
|
| 164 |
+
data_file_path = resources.files(data_module) / data_file_name
|
| 165 |
+
|
| 166 |
+
# load the file itself, to simulate a http error
|
| 167 |
+
with data_file_path.open("rb") as f:
|
| 168 |
+
decompressed_f = read_fn(f, "rb")
|
| 169 |
+
decoded_s = decompressed_f.read().decode("utf-8")
|
| 170 |
+
json_data = json.loads(decoded_s)
|
| 171 |
+
if "error" in json_data:
|
| 172 |
+
raise HTTPError(
|
| 173 |
+
url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO()
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
with data_file_path.open("rb") as f:
|
| 177 |
+
if has_gzip_header:
|
| 178 |
+
fp = BytesIO(f.read())
|
| 179 |
+
return _MockHTTPResponse(fp, True)
|
| 180 |
+
else:
|
| 181 |
+
decompressed_f = read_fn(f, "rb")
|
| 182 |
+
fp = BytesIO(decompressed_f.read())
|
| 183 |
+
return _MockHTTPResponse(fp, False)
|
| 184 |
+
|
| 185 |
+
def _mock_urlopen(request, *args, **kwargs):
|
| 186 |
+
url = request.get_full_url()
|
| 187 |
+
has_gzip_header = request.get_header("Accept-encoding") == "gzip"
|
| 188 |
+
if url.startswith(url_prefix_data_list):
|
| 189 |
+
return _mock_urlopen_data_list(url, has_gzip_header)
|
| 190 |
+
elif url.startswith(url_prefix_data_features):
|
| 191 |
+
return _mock_urlopen_data_features(url, has_gzip_header)
|
| 192 |
+
elif url.startswith(url_prefix_download_data):
|
| 193 |
+
return _mock_urlopen_download_data(url, has_gzip_header)
|
| 194 |
+
elif url.startswith(url_prefix_data_description):
|
| 195 |
+
return _mock_urlopen_data_description(url, has_gzip_header)
|
| 196 |
+
else:
|
| 197 |
+
raise ValueError("Unknown mocking URL pattern: %s" % url)
|
| 198 |
+
|
| 199 |
+
# XXX: Global variable
|
| 200 |
+
if test_offline:
|
| 201 |
+
context.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
###############################################################################
|
| 205 |
+
# Test the behaviour of `fetch_openml` depending of the input parameters.
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
@pytest.mark.parametrize(
|
| 209 |
+
"data_id, dataset_params, n_samples, n_features, n_targets",
|
| 210 |
+
[
|
| 211 |
+
# iris
|
| 212 |
+
(61, {"data_id": 61}, 150, 4, 1),
|
| 213 |
+
(61, {"name": "iris", "version": 1}, 150, 4, 1),
|
| 214 |
+
# anneal
|
| 215 |
+
(2, {"data_id": 2}, 11, 38, 1),
|
| 216 |
+
(2, {"name": "anneal", "version": 1}, 11, 38, 1),
|
| 217 |
+
# cpu
|
| 218 |
+
(561, {"data_id": 561}, 209, 7, 1),
|
| 219 |
+
(561, {"name": "cpu", "version": 1}, 209, 7, 1),
|
| 220 |
+
# emotions
|
| 221 |
+
(40589, {"data_id": 40589}, 13, 72, 6),
|
| 222 |
+
# adult-census
|
| 223 |
+
(1119, {"data_id": 1119}, 10, 14, 1),
|
| 224 |
+
(1119, {"name": "adult-census"}, 10, 14, 1),
|
| 225 |
+
# miceprotein
|
| 226 |
+
(40966, {"data_id": 40966}, 7, 77, 1),
|
| 227 |
+
(40966, {"name": "MiceProtein"}, 7, 77, 1),
|
| 228 |
+
# titanic
|
| 229 |
+
(40945, {"data_id": 40945}, 1309, 13, 1),
|
| 230 |
+
],
|
| 231 |
+
)
|
| 232 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 233 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 234 |
+
def test_fetch_openml_as_frame_true(
|
| 235 |
+
monkeypatch,
|
| 236 |
+
data_id,
|
| 237 |
+
dataset_params,
|
| 238 |
+
n_samples,
|
| 239 |
+
n_features,
|
| 240 |
+
n_targets,
|
| 241 |
+
parser,
|
| 242 |
+
gzip_response,
|
| 243 |
+
):
|
| 244 |
+
"""Check the behaviour of `fetch_openml` with `as_frame=True`.
|
| 245 |
+
|
| 246 |
+
Fetch by ID and/or name (depending if the file was previously cached).
|
| 247 |
+
"""
|
| 248 |
+
pd = pytest.importorskip("pandas")
|
| 249 |
+
|
| 250 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response)
|
| 251 |
+
bunch = fetch_openml(
|
| 252 |
+
as_frame=True,
|
| 253 |
+
cache=False,
|
| 254 |
+
parser=parser,
|
| 255 |
+
**dataset_params,
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
assert int(bunch.details["id"]) == data_id
|
| 259 |
+
assert isinstance(bunch, Bunch)
|
| 260 |
+
|
| 261 |
+
assert isinstance(bunch.frame, pd.DataFrame)
|
| 262 |
+
assert bunch.frame.shape == (n_samples, n_features + n_targets)
|
| 263 |
+
|
| 264 |
+
assert isinstance(bunch.data, pd.DataFrame)
|
| 265 |
+
assert bunch.data.shape == (n_samples, n_features)
|
| 266 |
+
|
| 267 |
+
if n_targets == 1:
|
| 268 |
+
assert isinstance(bunch.target, pd.Series)
|
| 269 |
+
assert bunch.target.shape == (n_samples,)
|
| 270 |
+
else:
|
| 271 |
+
assert isinstance(bunch.target, pd.DataFrame)
|
| 272 |
+
assert bunch.target.shape == (n_samples, n_targets)
|
| 273 |
+
|
| 274 |
+
assert bunch.categories is None
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
@pytest.mark.parametrize(
|
| 278 |
+
"data_id, dataset_params, n_samples, n_features, n_targets",
|
| 279 |
+
[
|
| 280 |
+
# iris
|
| 281 |
+
(61, {"data_id": 61}, 150, 4, 1),
|
| 282 |
+
(61, {"name": "iris", "version": 1}, 150, 4, 1),
|
| 283 |
+
# anneal
|
| 284 |
+
(2, {"data_id": 2}, 11, 38, 1),
|
| 285 |
+
(2, {"name": "anneal", "version": 1}, 11, 38, 1),
|
| 286 |
+
# cpu
|
| 287 |
+
(561, {"data_id": 561}, 209, 7, 1),
|
| 288 |
+
(561, {"name": "cpu", "version": 1}, 209, 7, 1),
|
| 289 |
+
# emotions
|
| 290 |
+
(40589, {"data_id": 40589}, 13, 72, 6),
|
| 291 |
+
# adult-census
|
| 292 |
+
(1119, {"data_id": 1119}, 10, 14, 1),
|
| 293 |
+
(1119, {"name": "adult-census"}, 10, 14, 1),
|
| 294 |
+
# miceprotein
|
| 295 |
+
(40966, {"data_id": 40966}, 7, 77, 1),
|
| 296 |
+
(40966, {"name": "MiceProtein"}, 7, 77, 1),
|
| 297 |
+
],
|
| 298 |
+
)
|
| 299 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 300 |
+
def test_fetch_openml_as_frame_false(
|
| 301 |
+
monkeypatch,
|
| 302 |
+
data_id,
|
| 303 |
+
dataset_params,
|
| 304 |
+
n_samples,
|
| 305 |
+
n_features,
|
| 306 |
+
n_targets,
|
| 307 |
+
parser,
|
| 308 |
+
):
|
| 309 |
+
"""Check the behaviour of `fetch_openml` with `as_frame=False`.
|
| 310 |
+
|
| 311 |
+
Fetch both by ID and/or name + version.
|
| 312 |
+
"""
|
| 313 |
+
pytest.importorskip("pandas")
|
| 314 |
+
|
| 315 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
|
| 316 |
+
bunch = fetch_openml(
|
| 317 |
+
as_frame=False,
|
| 318 |
+
cache=False,
|
| 319 |
+
parser=parser,
|
| 320 |
+
**dataset_params,
|
| 321 |
+
)
|
| 322 |
+
assert int(bunch.details["id"]) == data_id
|
| 323 |
+
assert isinstance(bunch, Bunch)
|
| 324 |
+
|
| 325 |
+
assert bunch.frame is None
|
| 326 |
+
|
| 327 |
+
assert isinstance(bunch.data, np.ndarray)
|
| 328 |
+
assert bunch.data.shape == (n_samples, n_features)
|
| 329 |
+
|
| 330 |
+
assert isinstance(bunch.target, np.ndarray)
|
| 331 |
+
if n_targets == 1:
|
| 332 |
+
assert bunch.target.shape == (n_samples,)
|
| 333 |
+
else:
|
| 334 |
+
assert bunch.target.shape == (n_samples, n_targets)
|
| 335 |
+
|
| 336 |
+
assert isinstance(bunch.categories, dict)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
@pytest.mark.parametrize("data_id", [61, 1119, 40945])
|
| 340 |
+
def test_fetch_openml_consistency_parser(monkeypatch, data_id):
|
| 341 |
+
"""Check the consistency of the LIAC-ARFF and pandas parsers."""
|
| 342 |
+
pd = pytest.importorskip("pandas")
|
| 343 |
+
|
| 344 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
|
| 345 |
+
bunch_liac = fetch_openml(
|
| 346 |
+
data_id=data_id,
|
| 347 |
+
as_frame=True,
|
| 348 |
+
cache=False,
|
| 349 |
+
parser="liac-arff",
|
| 350 |
+
)
|
| 351 |
+
bunch_pandas = fetch_openml(
|
| 352 |
+
data_id=data_id,
|
| 353 |
+
as_frame=True,
|
| 354 |
+
cache=False,
|
| 355 |
+
parser="pandas",
|
| 356 |
+
)
|
| 357 |
+
|
| 358 |
+
# The data frames for the input features should match up to some numerical
|
| 359 |
+
# dtype conversions (e.g. float64 <=> Int64) due to limitations of the
|
| 360 |
+
# LIAC-ARFF parser.
|
| 361 |
+
data_liac, data_pandas = bunch_liac.data, bunch_pandas.data
|
| 362 |
+
|
| 363 |
+
def convert_numerical_dtypes(series):
|
| 364 |
+
pandas_series = data_pandas[series.name]
|
| 365 |
+
if pd.api.types.is_numeric_dtype(pandas_series):
|
| 366 |
+
return series.astype(pandas_series.dtype)
|
| 367 |
+
else:
|
| 368 |
+
return series
|
| 369 |
+
|
| 370 |
+
data_liac_with_fixed_dtypes = data_liac.apply(convert_numerical_dtypes)
|
| 371 |
+
pd.testing.assert_frame_equal(data_liac_with_fixed_dtypes, data_pandas)
|
| 372 |
+
|
| 373 |
+
# Let's also check that the .frame attributes also match
|
| 374 |
+
frame_liac, frame_pandas = bunch_liac.frame, bunch_pandas.frame
|
| 375 |
+
|
| 376 |
+
# Note that the .frame attribute is a superset of the .data attribute:
|
| 377 |
+
pd.testing.assert_frame_equal(frame_pandas[bunch_pandas.feature_names], data_pandas)
|
| 378 |
+
|
| 379 |
+
# However the remaining columns, typically the target(s), are not necessarily
|
| 380 |
+
# dtyped similarly by both parsers due to limitations of the LIAC-ARFF parser.
|
| 381 |
+
# Therefore, extra dtype conversions are required for those columns:
|
| 382 |
+
|
| 383 |
+
def convert_numerical_and_categorical_dtypes(series):
|
| 384 |
+
pandas_series = frame_pandas[series.name]
|
| 385 |
+
if pd.api.types.is_numeric_dtype(pandas_series):
|
| 386 |
+
return series.astype(pandas_series.dtype)
|
| 387 |
+
elif isinstance(pandas_series.dtype, pd.CategoricalDtype):
|
| 388 |
+
# Compare categorical features by converting categorical liac uses
|
| 389 |
+
# strings to denote the categories, we rename the categories to make
|
| 390 |
+
# them comparable to the pandas parser. Fixing this behavior in
|
| 391 |
+
# LIAC-ARFF would allow to check the consistency in the future but
|
| 392 |
+
# we do not plan to maintain the LIAC-ARFF on the long term.
|
| 393 |
+
return series.cat.rename_categories(pandas_series.cat.categories)
|
| 394 |
+
else:
|
| 395 |
+
return series
|
| 396 |
+
|
| 397 |
+
frame_liac_with_fixed_dtypes = frame_liac.apply(
|
| 398 |
+
convert_numerical_and_categorical_dtypes
|
| 399 |
+
)
|
| 400 |
+
pd.testing.assert_frame_equal(frame_liac_with_fixed_dtypes, frame_pandas)
|
| 401 |
+
|
| 402 |
+
|
| 403 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 404 |
+
def test_fetch_openml_equivalence_array_dataframe(monkeypatch, parser):
|
| 405 |
+
"""Check the equivalence of the dataset when using `as_frame=False` and
|
| 406 |
+
`as_frame=True`.
|
| 407 |
+
"""
|
| 408 |
+
pytest.importorskip("pandas")
|
| 409 |
+
|
| 410 |
+
data_id = 61
|
| 411 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
|
| 412 |
+
bunch_as_frame_true = fetch_openml(
|
| 413 |
+
data_id=data_id,
|
| 414 |
+
as_frame=True,
|
| 415 |
+
cache=False,
|
| 416 |
+
parser=parser,
|
| 417 |
+
)
|
| 418 |
+
|
| 419 |
+
bunch_as_frame_false = fetch_openml(
|
| 420 |
+
data_id=data_id,
|
| 421 |
+
as_frame=False,
|
| 422 |
+
cache=False,
|
| 423 |
+
parser=parser,
|
| 424 |
+
)
|
| 425 |
+
|
| 426 |
+
assert_allclose(bunch_as_frame_false.data, bunch_as_frame_true.data)
|
| 427 |
+
assert_array_equal(bunch_as_frame_false.target, bunch_as_frame_true.target)
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 431 |
+
def test_fetch_openml_iris_pandas(monkeypatch, parser):
|
| 432 |
+
"""Check fetching on a numerical only dataset with string labels."""
|
| 433 |
+
pd = pytest.importorskip("pandas")
|
| 434 |
+
CategoricalDtype = pd.api.types.CategoricalDtype
|
| 435 |
+
data_id = 61
|
| 436 |
+
data_shape = (150, 4)
|
| 437 |
+
target_shape = (150,)
|
| 438 |
+
frame_shape = (150, 5)
|
| 439 |
+
|
| 440 |
+
target_dtype = CategoricalDtype(
|
| 441 |
+
["Iris-setosa", "Iris-versicolor", "Iris-virginica"]
|
| 442 |
+
)
|
| 443 |
+
data_dtypes = [np.float64] * 4
|
| 444 |
+
data_names = ["sepallength", "sepalwidth", "petallength", "petalwidth"]
|
| 445 |
+
target_name = "class"
|
| 446 |
+
|
| 447 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 448 |
+
|
| 449 |
+
bunch = fetch_openml(
|
| 450 |
+
data_id=data_id,
|
| 451 |
+
as_frame=True,
|
| 452 |
+
cache=False,
|
| 453 |
+
parser=parser,
|
| 454 |
+
)
|
| 455 |
+
data = bunch.data
|
| 456 |
+
target = bunch.target
|
| 457 |
+
frame = bunch.frame
|
| 458 |
+
|
| 459 |
+
assert isinstance(data, pd.DataFrame)
|
| 460 |
+
assert np.all(data.dtypes == data_dtypes)
|
| 461 |
+
assert data.shape == data_shape
|
| 462 |
+
assert np.all(data.columns == data_names)
|
| 463 |
+
assert np.all(bunch.feature_names == data_names)
|
| 464 |
+
assert bunch.target_names == [target_name]
|
| 465 |
+
|
| 466 |
+
assert isinstance(target, pd.Series)
|
| 467 |
+
assert target.dtype == target_dtype
|
| 468 |
+
assert target.shape == target_shape
|
| 469 |
+
assert target.name == target_name
|
| 470 |
+
assert target.index.is_unique
|
| 471 |
+
|
| 472 |
+
assert isinstance(frame, pd.DataFrame)
|
| 473 |
+
assert frame.shape == frame_shape
|
| 474 |
+
assert np.all(frame.dtypes == data_dtypes + [target_dtype])
|
| 475 |
+
assert frame.index.is_unique
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 479 |
+
@pytest.mark.parametrize("target_column", ["petalwidth", ["petalwidth", "petallength"]])
|
| 480 |
+
def test_fetch_openml_forcing_targets(monkeypatch, parser, target_column):
|
| 481 |
+
"""Check that we can force the target to not be the default target."""
|
| 482 |
+
pd = pytest.importorskip("pandas")
|
| 483 |
+
|
| 484 |
+
data_id = 61
|
| 485 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 486 |
+
bunch_forcing_target = fetch_openml(
|
| 487 |
+
data_id=data_id,
|
| 488 |
+
as_frame=True,
|
| 489 |
+
cache=False,
|
| 490 |
+
target_column=target_column,
|
| 491 |
+
parser=parser,
|
| 492 |
+
)
|
| 493 |
+
bunch_default = fetch_openml(
|
| 494 |
+
data_id=data_id,
|
| 495 |
+
as_frame=True,
|
| 496 |
+
cache=False,
|
| 497 |
+
parser=parser,
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
pd.testing.assert_frame_equal(bunch_forcing_target.frame, bunch_default.frame)
|
| 501 |
+
if isinstance(target_column, list):
|
| 502 |
+
pd.testing.assert_index_equal(
|
| 503 |
+
bunch_forcing_target.target.columns, pd.Index(target_column)
|
| 504 |
+
)
|
| 505 |
+
assert bunch_forcing_target.data.shape == (150, 3)
|
| 506 |
+
else:
|
| 507 |
+
assert bunch_forcing_target.target.name == target_column
|
| 508 |
+
assert bunch_forcing_target.data.shape == (150, 4)
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
@pytest.mark.parametrize("data_id", [61, 2, 561, 40589, 1119])
|
| 512 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 513 |
+
def test_fetch_openml_equivalence_frame_return_X_y(monkeypatch, data_id, parser):
|
| 514 |
+
"""Check the behaviour of `return_X_y=True` when `as_frame=True`."""
|
| 515 |
+
pd = pytest.importorskip("pandas")
|
| 516 |
+
|
| 517 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
|
| 518 |
+
bunch = fetch_openml(
|
| 519 |
+
data_id=data_id,
|
| 520 |
+
as_frame=True,
|
| 521 |
+
cache=False,
|
| 522 |
+
return_X_y=False,
|
| 523 |
+
parser=parser,
|
| 524 |
+
)
|
| 525 |
+
X, y = fetch_openml(
|
| 526 |
+
data_id=data_id,
|
| 527 |
+
as_frame=True,
|
| 528 |
+
cache=False,
|
| 529 |
+
return_X_y=True,
|
| 530 |
+
parser=parser,
|
| 531 |
+
)
|
| 532 |
+
|
| 533 |
+
pd.testing.assert_frame_equal(bunch.data, X)
|
| 534 |
+
if isinstance(y, pd.Series):
|
| 535 |
+
pd.testing.assert_series_equal(bunch.target, y)
|
| 536 |
+
else:
|
| 537 |
+
pd.testing.assert_frame_equal(bunch.target, y)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
@pytest.mark.parametrize("data_id", [61, 561, 40589, 1119])
|
| 541 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 542 |
+
def test_fetch_openml_equivalence_array_return_X_y(monkeypatch, data_id, parser):
|
| 543 |
+
"""Check the behaviour of `return_X_y=True` when `as_frame=False`."""
|
| 544 |
+
pytest.importorskip("pandas")
|
| 545 |
+
|
| 546 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
|
| 547 |
+
bunch = fetch_openml(
|
| 548 |
+
data_id=data_id,
|
| 549 |
+
as_frame=False,
|
| 550 |
+
cache=False,
|
| 551 |
+
return_X_y=False,
|
| 552 |
+
parser=parser,
|
| 553 |
+
)
|
| 554 |
+
X, y = fetch_openml(
|
| 555 |
+
data_id=data_id,
|
| 556 |
+
as_frame=False,
|
| 557 |
+
cache=False,
|
| 558 |
+
return_X_y=True,
|
| 559 |
+
parser=parser,
|
| 560 |
+
)
|
| 561 |
+
|
| 562 |
+
assert_array_equal(bunch.data, X)
|
| 563 |
+
assert_array_equal(bunch.target, y)
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
def test_fetch_openml_difference_parsers(monkeypatch):
|
| 567 |
+
"""Check the difference between liac-arff and pandas parser."""
|
| 568 |
+
pytest.importorskip("pandas")
|
| 569 |
+
|
| 570 |
+
data_id = 1119
|
| 571 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True)
|
| 572 |
+
# When `as_frame=False`, the categories will be ordinally encoded with
|
| 573 |
+
# liac-arff parser while this is not the case with pandas parser.
|
| 574 |
+
as_frame = False
|
| 575 |
+
bunch_liac_arff = fetch_openml(
|
| 576 |
+
data_id=data_id,
|
| 577 |
+
as_frame=as_frame,
|
| 578 |
+
cache=False,
|
| 579 |
+
parser="liac-arff",
|
| 580 |
+
)
|
| 581 |
+
bunch_pandas = fetch_openml(
|
| 582 |
+
data_id=data_id,
|
| 583 |
+
as_frame=as_frame,
|
| 584 |
+
cache=False,
|
| 585 |
+
parser="pandas",
|
| 586 |
+
)
|
| 587 |
+
|
| 588 |
+
assert bunch_liac_arff.data.dtype.kind == "f"
|
| 589 |
+
assert bunch_pandas.data.dtype == "O"
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
###############################################################################
|
| 593 |
+
# Test the ARFF parsing on several dataset to check if detect the correct
|
| 594 |
+
# types (categories, integers, floats).
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
@pytest.fixture(scope="module")
|
| 598 |
+
def datasets_column_names():
|
| 599 |
+
"""Returns the columns names for each dataset."""
|
| 600 |
+
return {
|
| 601 |
+
61: ["sepallength", "sepalwidth", "petallength", "petalwidth", "class"],
|
| 602 |
+
2: [
|
| 603 |
+
"family",
|
| 604 |
+
"product-type",
|
| 605 |
+
"steel",
|
| 606 |
+
"carbon",
|
| 607 |
+
"hardness",
|
| 608 |
+
"temper_rolling",
|
| 609 |
+
"condition",
|
| 610 |
+
"formability",
|
| 611 |
+
"strength",
|
| 612 |
+
"non-ageing",
|
| 613 |
+
"surface-finish",
|
| 614 |
+
"surface-quality",
|
| 615 |
+
"enamelability",
|
| 616 |
+
"bc",
|
| 617 |
+
"bf",
|
| 618 |
+
"bt",
|
| 619 |
+
"bw%2Fme",
|
| 620 |
+
"bl",
|
| 621 |
+
"m",
|
| 622 |
+
"chrom",
|
| 623 |
+
"phos",
|
| 624 |
+
"cbond",
|
| 625 |
+
"marvi",
|
| 626 |
+
"exptl",
|
| 627 |
+
"ferro",
|
| 628 |
+
"corr",
|
| 629 |
+
"blue%2Fbright%2Fvarn%2Fclean",
|
| 630 |
+
"lustre",
|
| 631 |
+
"jurofm",
|
| 632 |
+
"s",
|
| 633 |
+
"p",
|
| 634 |
+
"shape",
|
| 635 |
+
"thick",
|
| 636 |
+
"width",
|
| 637 |
+
"len",
|
| 638 |
+
"oil",
|
| 639 |
+
"bore",
|
| 640 |
+
"packing",
|
| 641 |
+
"class",
|
| 642 |
+
],
|
| 643 |
+
561: ["vendor", "MYCT", "MMIN", "MMAX", "CACH", "CHMIN", "CHMAX", "class"],
|
| 644 |
+
40589: [
|
| 645 |
+
"Mean_Acc1298_Mean_Mem40_Centroid",
|
| 646 |
+
"Mean_Acc1298_Mean_Mem40_Rolloff",
|
| 647 |
+
"Mean_Acc1298_Mean_Mem40_Flux",
|
| 648 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_0",
|
| 649 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_1",
|
| 650 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_2",
|
| 651 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_3",
|
| 652 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_4",
|
| 653 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_5",
|
| 654 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_6",
|
| 655 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_7",
|
| 656 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_8",
|
| 657 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_9",
|
| 658 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_10",
|
| 659 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_11",
|
| 660 |
+
"Mean_Acc1298_Mean_Mem40_MFCC_12",
|
| 661 |
+
"Mean_Acc1298_Std_Mem40_Centroid",
|
| 662 |
+
"Mean_Acc1298_Std_Mem40_Rolloff",
|
| 663 |
+
"Mean_Acc1298_Std_Mem40_Flux",
|
| 664 |
+
"Mean_Acc1298_Std_Mem40_MFCC_0",
|
| 665 |
+
"Mean_Acc1298_Std_Mem40_MFCC_1",
|
| 666 |
+
"Mean_Acc1298_Std_Mem40_MFCC_2",
|
| 667 |
+
"Mean_Acc1298_Std_Mem40_MFCC_3",
|
| 668 |
+
"Mean_Acc1298_Std_Mem40_MFCC_4",
|
| 669 |
+
"Mean_Acc1298_Std_Mem40_MFCC_5",
|
| 670 |
+
"Mean_Acc1298_Std_Mem40_MFCC_6",
|
| 671 |
+
"Mean_Acc1298_Std_Mem40_MFCC_7",
|
| 672 |
+
"Mean_Acc1298_Std_Mem40_MFCC_8",
|
| 673 |
+
"Mean_Acc1298_Std_Mem40_MFCC_9",
|
| 674 |
+
"Mean_Acc1298_Std_Mem40_MFCC_10",
|
| 675 |
+
"Mean_Acc1298_Std_Mem40_MFCC_11",
|
| 676 |
+
"Mean_Acc1298_Std_Mem40_MFCC_12",
|
| 677 |
+
"Std_Acc1298_Mean_Mem40_Centroid",
|
| 678 |
+
"Std_Acc1298_Mean_Mem40_Rolloff",
|
| 679 |
+
"Std_Acc1298_Mean_Mem40_Flux",
|
| 680 |
+
"Std_Acc1298_Mean_Mem40_MFCC_0",
|
| 681 |
+
"Std_Acc1298_Mean_Mem40_MFCC_1",
|
| 682 |
+
"Std_Acc1298_Mean_Mem40_MFCC_2",
|
| 683 |
+
"Std_Acc1298_Mean_Mem40_MFCC_3",
|
| 684 |
+
"Std_Acc1298_Mean_Mem40_MFCC_4",
|
| 685 |
+
"Std_Acc1298_Mean_Mem40_MFCC_5",
|
| 686 |
+
"Std_Acc1298_Mean_Mem40_MFCC_6",
|
| 687 |
+
"Std_Acc1298_Mean_Mem40_MFCC_7",
|
| 688 |
+
"Std_Acc1298_Mean_Mem40_MFCC_8",
|
| 689 |
+
"Std_Acc1298_Mean_Mem40_MFCC_9",
|
| 690 |
+
"Std_Acc1298_Mean_Mem40_MFCC_10",
|
| 691 |
+
"Std_Acc1298_Mean_Mem40_MFCC_11",
|
| 692 |
+
"Std_Acc1298_Mean_Mem40_MFCC_12",
|
| 693 |
+
"Std_Acc1298_Std_Mem40_Centroid",
|
| 694 |
+
"Std_Acc1298_Std_Mem40_Rolloff",
|
| 695 |
+
"Std_Acc1298_Std_Mem40_Flux",
|
| 696 |
+
"Std_Acc1298_Std_Mem40_MFCC_0",
|
| 697 |
+
"Std_Acc1298_Std_Mem40_MFCC_1",
|
| 698 |
+
"Std_Acc1298_Std_Mem40_MFCC_2",
|
| 699 |
+
"Std_Acc1298_Std_Mem40_MFCC_3",
|
| 700 |
+
"Std_Acc1298_Std_Mem40_MFCC_4",
|
| 701 |
+
"Std_Acc1298_Std_Mem40_MFCC_5",
|
| 702 |
+
"Std_Acc1298_Std_Mem40_MFCC_6",
|
| 703 |
+
"Std_Acc1298_Std_Mem40_MFCC_7",
|
| 704 |
+
"Std_Acc1298_Std_Mem40_MFCC_8",
|
| 705 |
+
"Std_Acc1298_Std_Mem40_MFCC_9",
|
| 706 |
+
"Std_Acc1298_Std_Mem40_MFCC_10",
|
| 707 |
+
"Std_Acc1298_Std_Mem40_MFCC_11",
|
| 708 |
+
"Std_Acc1298_Std_Mem40_MFCC_12",
|
| 709 |
+
"BH_LowPeakAmp",
|
| 710 |
+
"BH_LowPeakBPM",
|
| 711 |
+
"BH_HighPeakAmp",
|
| 712 |
+
"BH_HighPeakBPM",
|
| 713 |
+
"BH_HighLowRatio",
|
| 714 |
+
"BHSUM1",
|
| 715 |
+
"BHSUM2",
|
| 716 |
+
"BHSUM3",
|
| 717 |
+
"amazed.suprised",
|
| 718 |
+
"happy.pleased",
|
| 719 |
+
"relaxing.calm",
|
| 720 |
+
"quiet.still",
|
| 721 |
+
"sad.lonely",
|
| 722 |
+
"angry.aggresive",
|
| 723 |
+
],
|
| 724 |
+
1119: [
|
| 725 |
+
"age",
|
| 726 |
+
"workclass",
|
| 727 |
+
"fnlwgt:",
|
| 728 |
+
"education:",
|
| 729 |
+
"education-num:",
|
| 730 |
+
"marital-status:",
|
| 731 |
+
"occupation:",
|
| 732 |
+
"relationship:",
|
| 733 |
+
"race:",
|
| 734 |
+
"sex:",
|
| 735 |
+
"capital-gain:",
|
| 736 |
+
"capital-loss:",
|
| 737 |
+
"hours-per-week:",
|
| 738 |
+
"native-country:",
|
| 739 |
+
"class",
|
| 740 |
+
],
|
| 741 |
+
40966: [
|
| 742 |
+
"DYRK1A_N",
|
| 743 |
+
"ITSN1_N",
|
| 744 |
+
"BDNF_N",
|
| 745 |
+
"NR1_N",
|
| 746 |
+
"NR2A_N",
|
| 747 |
+
"pAKT_N",
|
| 748 |
+
"pBRAF_N",
|
| 749 |
+
"pCAMKII_N",
|
| 750 |
+
"pCREB_N",
|
| 751 |
+
"pELK_N",
|
| 752 |
+
"pERK_N",
|
| 753 |
+
"pJNK_N",
|
| 754 |
+
"PKCA_N",
|
| 755 |
+
"pMEK_N",
|
| 756 |
+
"pNR1_N",
|
| 757 |
+
"pNR2A_N",
|
| 758 |
+
"pNR2B_N",
|
| 759 |
+
"pPKCAB_N",
|
| 760 |
+
"pRSK_N",
|
| 761 |
+
"AKT_N",
|
| 762 |
+
"BRAF_N",
|
| 763 |
+
"CAMKII_N",
|
| 764 |
+
"CREB_N",
|
| 765 |
+
"ELK_N",
|
| 766 |
+
"ERK_N",
|
| 767 |
+
"GSK3B_N",
|
| 768 |
+
"JNK_N",
|
| 769 |
+
"MEK_N",
|
| 770 |
+
"TRKA_N",
|
| 771 |
+
"RSK_N",
|
| 772 |
+
"APP_N",
|
| 773 |
+
"Bcatenin_N",
|
| 774 |
+
"SOD1_N",
|
| 775 |
+
"MTOR_N",
|
| 776 |
+
"P38_N",
|
| 777 |
+
"pMTOR_N",
|
| 778 |
+
"DSCR1_N",
|
| 779 |
+
"AMPKA_N",
|
| 780 |
+
"NR2B_N",
|
| 781 |
+
"pNUMB_N",
|
| 782 |
+
"RAPTOR_N",
|
| 783 |
+
"TIAM1_N",
|
| 784 |
+
"pP70S6_N",
|
| 785 |
+
"NUMB_N",
|
| 786 |
+
"P70S6_N",
|
| 787 |
+
"pGSK3B_N",
|
| 788 |
+
"pPKCG_N",
|
| 789 |
+
"CDK5_N",
|
| 790 |
+
"S6_N",
|
| 791 |
+
"ADARB1_N",
|
| 792 |
+
"AcetylH3K9_N",
|
| 793 |
+
"RRP1_N",
|
| 794 |
+
"BAX_N",
|
| 795 |
+
"ARC_N",
|
| 796 |
+
"ERBB4_N",
|
| 797 |
+
"nNOS_N",
|
| 798 |
+
"Tau_N",
|
| 799 |
+
"GFAP_N",
|
| 800 |
+
"GluR3_N",
|
| 801 |
+
"GluR4_N",
|
| 802 |
+
"IL1B_N",
|
| 803 |
+
"P3525_N",
|
| 804 |
+
"pCASP9_N",
|
| 805 |
+
"PSD95_N",
|
| 806 |
+
"SNCA_N",
|
| 807 |
+
"Ubiquitin_N",
|
| 808 |
+
"pGSK3B_Tyr216_N",
|
| 809 |
+
"SHH_N",
|
| 810 |
+
"BAD_N",
|
| 811 |
+
"BCL2_N",
|
| 812 |
+
"pS6_N",
|
| 813 |
+
"pCFOS_N",
|
| 814 |
+
"SYP_N",
|
| 815 |
+
"H3AcK18_N",
|
| 816 |
+
"EGR1_N",
|
| 817 |
+
"H3MeK4_N",
|
| 818 |
+
"CaNA_N",
|
| 819 |
+
"class",
|
| 820 |
+
],
|
| 821 |
+
40945: [
|
| 822 |
+
"pclass",
|
| 823 |
+
"survived",
|
| 824 |
+
"name",
|
| 825 |
+
"sex",
|
| 826 |
+
"age",
|
| 827 |
+
"sibsp",
|
| 828 |
+
"parch",
|
| 829 |
+
"ticket",
|
| 830 |
+
"fare",
|
| 831 |
+
"cabin",
|
| 832 |
+
"embarked",
|
| 833 |
+
"boat",
|
| 834 |
+
"body",
|
| 835 |
+
"home.dest",
|
| 836 |
+
],
|
| 837 |
+
}
|
| 838 |
+
|
| 839 |
+
|
| 840 |
+
@pytest.fixture(scope="module")
|
| 841 |
+
def datasets_missing_values():
|
| 842 |
+
return {
|
| 843 |
+
61: {},
|
| 844 |
+
2: {
|
| 845 |
+
"family": 11,
|
| 846 |
+
"temper_rolling": 9,
|
| 847 |
+
"condition": 2,
|
| 848 |
+
"formability": 4,
|
| 849 |
+
"non-ageing": 10,
|
| 850 |
+
"surface-finish": 11,
|
| 851 |
+
"enamelability": 11,
|
| 852 |
+
"bc": 11,
|
| 853 |
+
"bf": 10,
|
| 854 |
+
"bt": 11,
|
| 855 |
+
"bw%2Fme": 8,
|
| 856 |
+
"bl": 9,
|
| 857 |
+
"m": 11,
|
| 858 |
+
"chrom": 11,
|
| 859 |
+
"phos": 11,
|
| 860 |
+
"cbond": 10,
|
| 861 |
+
"marvi": 11,
|
| 862 |
+
"exptl": 11,
|
| 863 |
+
"ferro": 11,
|
| 864 |
+
"corr": 11,
|
| 865 |
+
"blue%2Fbright%2Fvarn%2Fclean": 11,
|
| 866 |
+
"lustre": 8,
|
| 867 |
+
"jurofm": 11,
|
| 868 |
+
"s": 11,
|
| 869 |
+
"p": 11,
|
| 870 |
+
"oil": 10,
|
| 871 |
+
"packing": 11,
|
| 872 |
+
},
|
| 873 |
+
561: {},
|
| 874 |
+
40589: {},
|
| 875 |
+
1119: {},
|
| 876 |
+
40966: {"BCL2_N": 7},
|
| 877 |
+
40945: {
|
| 878 |
+
"age": 263,
|
| 879 |
+
"fare": 1,
|
| 880 |
+
"cabin": 1014,
|
| 881 |
+
"embarked": 2,
|
| 882 |
+
"boat": 823,
|
| 883 |
+
"body": 1188,
|
| 884 |
+
"home.dest": 564,
|
| 885 |
+
},
|
| 886 |
+
}
|
| 887 |
+
|
| 888 |
+
|
| 889 |
+
@pytest.mark.parametrize(
|
| 890 |
+
"data_id, parser, expected_n_categories, expected_n_floats, expected_n_ints",
|
| 891 |
+
[
|
| 892 |
+
# iris dataset
|
| 893 |
+
(61, "liac-arff", 1, 4, 0),
|
| 894 |
+
(61, "pandas", 1, 4, 0),
|
| 895 |
+
# anneal dataset
|
| 896 |
+
(2, "liac-arff", 33, 6, 0),
|
| 897 |
+
(2, "pandas", 33, 2, 4),
|
| 898 |
+
# cpu dataset
|
| 899 |
+
(561, "liac-arff", 1, 7, 0),
|
| 900 |
+
(561, "pandas", 1, 0, 7),
|
| 901 |
+
# emotions dataset
|
| 902 |
+
(40589, "liac-arff", 6, 72, 0),
|
| 903 |
+
(40589, "pandas", 6, 69, 3),
|
| 904 |
+
# adult-census dataset
|
| 905 |
+
(1119, "liac-arff", 9, 6, 0),
|
| 906 |
+
(1119, "pandas", 9, 0, 6),
|
| 907 |
+
# miceprotein
|
| 908 |
+
(40966, "liac-arff", 1, 77, 0),
|
| 909 |
+
(40966, "pandas", 1, 77, 0),
|
| 910 |
+
# titanic
|
| 911 |
+
(40945, "liac-arff", 3, 6, 0),
|
| 912 |
+
(40945, "pandas", 3, 3, 3),
|
| 913 |
+
],
|
| 914 |
+
)
|
| 915 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 916 |
+
def test_fetch_openml_types_inference(
|
| 917 |
+
monkeypatch,
|
| 918 |
+
data_id,
|
| 919 |
+
parser,
|
| 920 |
+
expected_n_categories,
|
| 921 |
+
expected_n_floats,
|
| 922 |
+
expected_n_ints,
|
| 923 |
+
gzip_response,
|
| 924 |
+
datasets_column_names,
|
| 925 |
+
datasets_missing_values,
|
| 926 |
+
):
|
| 927 |
+
"""Check that `fetch_openml` infer the right number of categories, integers, and
|
| 928 |
+
floats."""
|
| 929 |
+
pd = pytest.importorskip("pandas")
|
| 930 |
+
CategoricalDtype = pd.api.types.CategoricalDtype
|
| 931 |
+
|
| 932 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response)
|
| 933 |
+
|
| 934 |
+
bunch = fetch_openml(
|
| 935 |
+
data_id=data_id,
|
| 936 |
+
as_frame=True,
|
| 937 |
+
cache=False,
|
| 938 |
+
parser=parser,
|
| 939 |
+
)
|
| 940 |
+
frame = bunch.frame
|
| 941 |
+
|
| 942 |
+
n_categories = len(
|
| 943 |
+
[dtype for dtype in frame.dtypes if isinstance(dtype, CategoricalDtype)]
|
| 944 |
+
)
|
| 945 |
+
n_floats = len([dtype for dtype in frame.dtypes if dtype.kind == "f"])
|
| 946 |
+
n_ints = len([dtype for dtype in frame.dtypes if dtype.kind == "i"])
|
| 947 |
+
|
| 948 |
+
assert n_categories == expected_n_categories
|
| 949 |
+
assert n_floats == expected_n_floats
|
| 950 |
+
assert n_ints == expected_n_ints
|
| 951 |
+
|
| 952 |
+
assert frame.columns.tolist() == datasets_column_names[data_id]
|
| 953 |
+
|
| 954 |
+
frame_feature_to_n_nan = frame.isna().sum().to_dict()
|
| 955 |
+
for name, n_missing in frame_feature_to_n_nan.items():
|
| 956 |
+
expected_missing = datasets_missing_values[data_id].get(name, 0)
|
| 957 |
+
assert n_missing == expected_missing
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
###############################################################################
|
| 961 |
+
# Test some more specific behaviour
|
| 962 |
+
|
| 963 |
+
|
| 964 |
+
@pytest.mark.parametrize(
|
| 965 |
+
"params, err_msg",
|
| 966 |
+
[
|
| 967 |
+
(
|
| 968 |
+
{"parser": "unknown"},
|
| 969 |
+
"The 'parser' parameter of fetch_openml must be a str among",
|
| 970 |
+
),
|
| 971 |
+
(
|
| 972 |
+
{"as_frame": "unknown"},
|
| 973 |
+
"The 'as_frame' parameter of fetch_openml must be an instance",
|
| 974 |
+
),
|
| 975 |
+
],
|
| 976 |
+
)
|
| 977 |
+
def test_fetch_openml_validation_parameter(monkeypatch, params, err_msg):
|
| 978 |
+
data_id = 1119
|
| 979 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 980 |
+
with pytest.raises(ValueError, match=err_msg):
|
| 981 |
+
fetch_openml(data_id=data_id, **params)
|
| 982 |
+
|
| 983 |
+
|
| 984 |
+
@pytest.mark.parametrize(
|
| 985 |
+
"params",
|
| 986 |
+
[
|
| 987 |
+
{"as_frame": True, "parser": "auto"},
|
| 988 |
+
{"as_frame": "auto", "parser": "auto"},
|
| 989 |
+
{"as_frame": False, "parser": "pandas"},
|
| 990 |
+
{"as_frame": False, "parser": "auto"},
|
| 991 |
+
],
|
| 992 |
+
)
|
| 993 |
+
def test_fetch_openml_requires_pandas_error(monkeypatch, params):
|
| 994 |
+
"""Check that we raise the proper errors when we require pandas."""
|
| 995 |
+
data_id = 1119
|
| 996 |
+
try:
|
| 997 |
+
check_pandas_support("test_fetch_openml_requires_pandas")
|
| 998 |
+
except ImportError:
|
| 999 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 1000 |
+
err_msg = "requires pandas to be installed. Alternatively, explicitly"
|
| 1001 |
+
with pytest.raises(ImportError, match=err_msg):
|
| 1002 |
+
fetch_openml(data_id=data_id, **params)
|
| 1003 |
+
else:
|
| 1004 |
+
raise SkipTest("This test requires pandas to not be installed.")
|
| 1005 |
+
|
| 1006 |
+
|
| 1007 |
+
@pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive")
|
| 1008 |
+
@pytest.mark.parametrize(
|
| 1009 |
+
"params, err_msg",
|
| 1010 |
+
[
|
| 1011 |
+
(
|
| 1012 |
+
{"parser": "pandas"},
|
| 1013 |
+
"Sparse ARFF datasets cannot be loaded with parser='pandas'",
|
| 1014 |
+
),
|
| 1015 |
+
(
|
| 1016 |
+
{"as_frame": True},
|
| 1017 |
+
"Sparse ARFF datasets cannot be loaded with as_frame=True.",
|
| 1018 |
+
),
|
| 1019 |
+
(
|
| 1020 |
+
{"parser": "pandas", "as_frame": True},
|
| 1021 |
+
"Sparse ARFF datasets cannot be loaded with as_frame=True.",
|
| 1022 |
+
),
|
| 1023 |
+
],
|
| 1024 |
+
)
|
| 1025 |
+
def test_fetch_openml_sparse_arff_error(monkeypatch, params, err_msg):
|
| 1026 |
+
"""Check that we raise the expected error for sparse ARFF datasets and
|
| 1027 |
+
a wrong set of incompatible parameters.
|
| 1028 |
+
"""
|
| 1029 |
+
pytest.importorskip("pandas")
|
| 1030 |
+
data_id = 292
|
| 1031 |
+
|
| 1032 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 1033 |
+
with pytest.raises(ValueError, match=err_msg):
|
| 1034 |
+
fetch_openml(
|
| 1035 |
+
data_id=data_id,
|
| 1036 |
+
cache=False,
|
| 1037 |
+
**params,
|
| 1038 |
+
)
|
| 1039 |
+
|
| 1040 |
+
|
| 1041 |
+
@pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive")
|
| 1042 |
+
@pytest.mark.parametrize(
|
| 1043 |
+
"data_id, data_type",
|
| 1044 |
+
[
|
| 1045 |
+
(61, "dataframe"), # iris dataset version 1
|
| 1046 |
+
(292, "sparse"), # Australian dataset version 1
|
| 1047 |
+
],
|
| 1048 |
+
)
|
| 1049 |
+
def test_fetch_openml_auto_mode(monkeypatch, data_id, data_type):
|
| 1050 |
+
"""Check the auto mode of `fetch_openml`."""
|
| 1051 |
+
pd = pytest.importorskip("pandas")
|
| 1052 |
+
|
| 1053 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 1054 |
+
data = fetch_openml(data_id=data_id, as_frame="auto", cache=False)
|
| 1055 |
+
klass = pd.DataFrame if data_type == "dataframe" else scipy.sparse.csr_matrix
|
| 1056 |
+
assert isinstance(data.data, klass)
|
| 1057 |
+
|
| 1058 |
+
|
| 1059 |
+
def test_convert_arff_data_dataframe_warning_low_memory_pandas(monkeypatch):
|
| 1060 |
+
"""Check that we raise a warning regarding the working memory when using
|
| 1061 |
+
LIAC-ARFF parser."""
|
| 1062 |
+
pytest.importorskip("pandas")
|
| 1063 |
+
|
| 1064 |
+
data_id = 1119
|
| 1065 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 1066 |
+
|
| 1067 |
+
msg = "Could not adhere to working_memory config."
|
| 1068 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1069 |
+
with config_context(working_memory=1e-6):
|
| 1070 |
+
fetch_openml(
|
| 1071 |
+
data_id=data_id,
|
| 1072 |
+
as_frame=True,
|
| 1073 |
+
cache=False,
|
| 1074 |
+
parser="liac-arff",
|
| 1075 |
+
)
|
| 1076 |
+
|
| 1077 |
+
|
| 1078 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1079 |
+
def test_fetch_openml_iris_warn_multiple_version(monkeypatch, gzip_response):
|
| 1080 |
+
"""Check that a warning is raised when multiple versions exist and no version is
|
| 1081 |
+
requested."""
|
| 1082 |
+
data_id = 61
|
| 1083 |
+
data_name = "iris"
|
| 1084 |
+
|
| 1085 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1086 |
+
|
| 1087 |
+
msg = re.escape(
|
| 1088 |
+
"Multiple active versions of the dataset matching the name"
|
| 1089 |
+
" iris exist. Versions may be fundamentally different, "
|
| 1090 |
+
"returning version 1. Available versions:\n"
|
| 1091 |
+
"- version 1, status: active\n"
|
| 1092 |
+
" url: https://www.openml.org/search?type=data&id=61\n"
|
| 1093 |
+
"- version 3, status: active\n"
|
| 1094 |
+
" url: https://www.openml.org/search?type=data&id=969\n"
|
| 1095 |
+
)
|
| 1096 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1097 |
+
fetch_openml(
|
| 1098 |
+
name=data_name,
|
| 1099 |
+
as_frame=False,
|
| 1100 |
+
cache=False,
|
| 1101 |
+
parser="liac-arff",
|
| 1102 |
+
)
|
| 1103 |
+
|
| 1104 |
+
|
| 1105 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1106 |
+
def test_fetch_openml_no_target(monkeypatch, gzip_response):
|
| 1107 |
+
"""Check that we can get a dataset without target."""
|
| 1108 |
+
data_id = 61
|
| 1109 |
+
target_column = None
|
| 1110 |
+
expected_observations = 150
|
| 1111 |
+
expected_features = 5
|
| 1112 |
+
|
| 1113 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1114 |
+
data = fetch_openml(
|
| 1115 |
+
data_id=data_id,
|
| 1116 |
+
target_column=target_column,
|
| 1117 |
+
cache=False,
|
| 1118 |
+
as_frame=False,
|
| 1119 |
+
parser="liac-arff",
|
| 1120 |
+
)
|
| 1121 |
+
assert data.data.shape == (expected_observations, expected_features)
|
| 1122 |
+
assert data.target is None
|
| 1123 |
+
|
| 1124 |
+
|
| 1125 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1126 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 1127 |
+
def test_missing_values_pandas(monkeypatch, gzip_response, parser):
|
| 1128 |
+
"""check that missing values in categories are compatible with pandas
|
| 1129 |
+
categorical"""
|
| 1130 |
+
pytest.importorskip("pandas")
|
| 1131 |
+
|
| 1132 |
+
data_id = 42585
|
| 1133 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response)
|
| 1134 |
+
penguins = fetch_openml(
|
| 1135 |
+
data_id=data_id,
|
| 1136 |
+
cache=False,
|
| 1137 |
+
as_frame=True,
|
| 1138 |
+
parser=parser,
|
| 1139 |
+
)
|
| 1140 |
+
|
| 1141 |
+
cat_dtype = penguins.data.dtypes["sex"]
|
| 1142 |
+
# there are nans in the categorical
|
| 1143 |
+
assert penguins.data["sex"].isna().any()
|
| 1144 |
+
assert_array_equal(cat_dtype.categories, ["FEMALE", "MALE", "_"])
|
| 1145 |
+
|
| 1146 |
+
|
| 1147 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1148 |
+
@pytest.mark.parametrize(
|
| 1149 |
+
"dataset_params",
|
| 1150 |
+
[
|
| 1151 |
+
{"data_id": 40675},
|
| 1152 |
+
{"data_id": None, "name": "glass2", "version": 1},
|
| 1153 |
+
],
|
| 1154 |
+
)
|
| 1155 |
+
def test_fetch_openml_inactive(monkeypatch, gzip_response, dataset_params):
|
| 1156 |
+
"""Check that we raise a warning when the dataset is inactive."""
|
| 1157 |
+
data_id = 40675
|
| 1158 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1159 |
+
msg = "Version 1 of dataset glass2 is inactive,"
|
| 1160 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1161 |
+
glass2 = fetch_openml(
|
| 1162 |
+
cache=False, as_frame=False, parser="liac-arff", **dataset_params
|
| 1163 |
+
)
|
| 1164 |
+
assert glass2.data.shape == (163, 9)
|
| 1165 |
+
assert glass2.details["id"] == "40675"
|
| 1166 |
+
|
| 1167 |
+
|
| 1168 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1169 |
+
@pytest.mark.parametrize(
|
| 1170 |
+
"data_id, params, err_type, err_msg",
|
| 1171 |
+
[
|
| 1172 |
+
(40675, {"name": "glass2"}, ValueError, "No active dataset glass2 found"),
|
| 1173 |
+
(
|
| 1174 |
+
61,
|
| 1175 |
+
{"data_id": 61, "target_column": ["sepalwidth", "class"]},
|
| 1176 |
+
ValueError,
|
| 1177 |
+
"Can only handle homogeneous multi-target datasets",
|
| 1178 |
+
),
|
| 1179 |
+
(
|
| 1180 |
+
40945,
|
| 1181 |
+
{"data_id": 40945, "as_frame": False},
|
| 1182 |
+
ValueError,
|
| 1183 |
+
(
|
| 1184 |
+
"STRING attributes are not supported for array representation. Try"
|
| 1185 |
+
" as_frame=True"
|
| 1186 |
+
),
|
| 1187 |
+
),
|
| 1188 |
+
(
|
| 1189 |
+
2,
|
| 1190 |
+
{"data_id": 2, "target_column": "family", "as_frame": True},
|
| 1191 |
+
ValueError,
|
| 1192 |
+
"Target column 'family'",
|
| 1193 |
+
),
|
| 1194 |
+
(
|
| 1195 |
+
2,
|
| 1196 |
+
{"data_id": 2, "target_column": "family", "as_frame": False},
|
| 1197 |
+
ValueError,
|
| 1198 |
+
"Target column 'family'",
|
| 1199 |
+
),
|
| 1200 |
+
(
|
| 1201 |
+
61,
|
| 1202 |
+
{"data_id": 61, "target_column": "undefined"},
|
| 1203 |
+
KeyError,
|
| 1204 |
+
"Could not find target_column='undefined'",
|
| 1205 |
+
),
|
| 1206 |
+
(
|
| 1207 |
+
61,
|
| 1208 |
+
{"data_id": 61, "target_column": ["undefined", "class"]},
|
| 1209 |
+
KeyError,
|
| 1210 |
+
"Could not find target_column='undefined'",
|
| 1211 |
+
),
|
| 1212 |
+
],
|
| 1213 |
+
)
|
| 1214 |
+
@pytest.mark.parametrize("parser", ["liac-arff", "pandas"])
|
| 1215 |
+
def test_fetch_openml_error(
|
| 1216 |
+
monkeypatch, gzip_response, data_id, params, err_type, err_msg, parser
|
| 1217 |
+
):
|
| 1218 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1219 |
+
if params.get("as_frame", True) or parser == "pandas":
|
| 1220 |
+
pytest.importorskip("pandas")
|
| 1221 |
+
with pytest.raises(err_type, match=err_msg):
|
| 1222 |
+
fetch_openml(cache=False, parser=parser, **params)
|
| 1223 |
+
|
| 1224 |
+
|
| 1225 |
+
@pytest.mark.parametrize(
|
| 1226 |
+
"params, err_type, err_msg",
|
| 1227 |
+
[
|
| 1228 |
+
(
|
| 1229 |
+
{"data_id": -1, "name": None, "version": "version"},
|
| 1230 |
+
ValueError,
|
| 1231 |
+
"The 'version' parameter of fetch_openml must be an int in the range",
|
| 1232 |
+
),
|
| 1233 |
+
(
|
| 1234 |
+
{"data_id": -1, "name": "nAmE"},
|
| 1235 |
+
ValueError,
|
| 1236 |
+
"The 'data_id' parameter of fetch_openml must be an int in the range",
|
| 1237 |
+
),
|
| 1238 |
+
(
|
| 1239 |
+
{"data_id": -1, "name": "nAmE", "version": "version"},
|
| 1240 |
+
ValueError,
|
| 1241 |
+
"The 'version' parameter of fetch_openml must be an int",
|
| 1242 |
+
),
|
| 1243 |
+
(
|
| 1244 |
+
{},
|
| 1245 |
+
ValueError,
|
| 1246 |
+
"Neither name nor data_id are provided. Please provide name or data_id.",
|
| 1247 |
+
),
|
| 1248 |
+
],
|
| 1249 |
+
)
|
| 1250 |
+
def test_fetch_openml_raises_illegal_argument(params, err_type, err_msg):
|
| 1251 |
+
with pytest.raises(err_type, match=err_msg):
|
| 1252 |
+
fetch_openml(**params)
|
| 1253 |
+
|
| 1254 |
+
|
| 1255 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1256 |
+
def test_warn_ignore_attribute(monkeypatch, gzip_response):
|
| 1257 |
+
data_id = 40966
|
| 1258 |
+
expected_row_id_msg = "target_column='{}' has flag is_row_identifier."
|
| 1259 |
+
expected_ignore_msg = "target_column='{}' has flag is_ignore."
|
| 1260 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1261 |
+
# single column test
|
| 1262 |
+
target_col = "MouseID"
|
| 1263 |
+
msg = expected_row_id_msg.format(target_col)
|
| 1264 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1265 |
+
fetch_openml(
|
| 1266 |
+
data_id=data_id,
|
| 1267 |
+
target_column=target_col,
|
| 1268 |
+
cache=False,
|
| 1269 |
+
as_frame=False,
|
| 1270 |
+
parser="liac-arff",
|
| 1271 |
+
)
|
| 1272 |
+
target_col = "Genotype"
|
| 1273 |
+
msg = expected_ignore_msg.format(target_col)
|
| 1274 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1275 |
+
fetch_openml(
|
| 1276 |
+
data_id=data_id,
|
| 1277 |
+
target_column=target_col,
|
| 1278 |
+
cache=False,
|
| 1279 |
+
as_frame=False,
|
| 1280 |
+
parser="liac-arff",
|
| 1281 |
+
)
|
| 1282 |
+
# multi column test
|
| 1283 |
+
target_col = "MouseID"
|
| 1284 |
+
msg = expected_row_id_msg.format(target_col)
|
| 1285 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1286 |
+
fetch_openml(
|
| 1287 |
+
data_id=data_id,
|
| 1288 |
+
target_column=[target_col, "class"],
|
| 1289 |
+
cache=False,
|
| 1290 |
+
as_frame=False,
|
| 1291 |
+
parser="liac-arff",
|
| 1292 |
+
)
|
| 1293 |
+
target_col = "Genotype"
|
| 1294 |
+
msg = expected_ignore_msg.format(target_col)
|
| 1295 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1296 |
+
fetch_openml(
|
| 1297 |
+
data_id=data_id,
|
| 1298 |
+
target_column=[target_col, "class"],
|
| 1299 |
+
cache=False,
|
| 1300 |
+
as_frame=False,
|
| 1301 |
+
parser="liac-arff",
|
| 1302 |
+
)
|
| 1303 |
+
|
| 1304 |
+
|
| 1305 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1306 |
+
def test_dataset_with_openml_error(monkeypatch, gzip_response):
|
| 1307 |
+
data_id = 1
|
| 1308 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1309 |
+
msg = "OpenML registered a problem with the dataset. It might be unusable. Error:"
|
| 1310 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1311 |
+
fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff")
|
| 1312 |
+
|
| 1313 |
+
|
| 1314 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1315 |
+
def test_dataset_with_openml_warning(monkeypatch, gzip_response):
|
| 1316 |
+
data_id = 3
|
| 1317 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1318 |
+
msg = "OpenML raised a warning on the dataset. It might be unusable. Warning:"
|
| 1319 |
+
with pytest.warns(UserWarning, match=msg):
|
| 1320 |
+
fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff")
|
| 1321 |
+
|
| 1322 |
+
|
| 1323 |
+
def test_fetch_openml_overwrite_default_params_read_csv(monkeypatch):
|
| 1324 |
+
"""Check that we can overwrite the default parameters of `read_csv`."""
|
| 1325 |
+
pytest.importorskip("pandas")
|
| 1326 |
+
data_id = 1590
|
| 1327 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
|
| 1328 |
+
|
| 1329 |
+
common_params = {
|
| 1330 |
+
"data_id": data_id,
|
| 1331 |
+
"as_frame": True,
|
| 1332 |
+
"cache": False,
|
| 1333 |
+
"parser": "pandas",
|
| 1334 |
+
}
|
| 1335 |
+
|
| 1336 |
+
# By default, the initial spaces are skipped. We checked that setting the parameter
|
| 1337 |
+
# `skipinitialspace` to False will have an effect.
|
| 1338 |
+
adult_without_spaces = fetch_openml(**common_params)
|
| 1339 |
+
adult_with_spaces = fetch_openml(
|
| 1340 |
+
**common_params, read_csv_kwargs={"skipinitialspace": False}
|
| 1341 |
+
)
|
| 1342 |
+
assert all(
|
| 1343 |
+
cat.startswith(" ") for cat in adult_with_spaces.frame["class"].cat.categories
|
| 1344 |
+
)
|
| 1345 |
+
assert not any(
|
| 1346 |
+
cat.startswith(" ")
|
| 1347 |
+
for cat in adult_without_spaces.frame["class"].cat.categories
|
| 1348 |
+
)
|
| 1349 |
+
|
| 1350 |
+
|
| 1351 |
+
###############################################################################
|
| 1352 |
+
# Test cache, retry mechanisms, checksum, etc.
|
| 1353 |
+
|
| 1354 |
+
|
| 1355 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1356 |
+
def test_open_openml_url_cache(monkeypatch, gzip_response, tmpdir):
|
| 1357 |
+
data_id = 61
|
| 1358 |
+
|
| 1359 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1360 |
+
openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff"
|
| 1361 |
+
url = f"https://www.openml.org/{openml_path}"
|
| 1362 |
+
cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
|
| 1363 |
+
# first fill the cache
|
| 1364 |
+
response1 = _open_openml_url(url, cache_directory)
|
| 1365 |
+
# assert file exists
|
| 1366 |
+
location = _get_local_path(openml_path, cache_directory)
|
| 1367 |
+
assert os.path.isfile(location)
|
| 1368 |
+
# redownload, to utilize cache
|
| 1369 |
+
response2 = _open_openml_url(url, cache_directory)
|
| 1370 |
+
assert response1.read() == response2.read()
|
| 1371 |
+
|
| 1372 |
+
|
| 1373 |
+
@pytest.mark.parametrize("write_to_disk", [True, False])
|
| 1374 |
+
def test_open_openml_url_unlinks_local_path(monkeypatch, tmpdir, write_to_disk):
|
| 1375 |
+
data_id = 61
|
| 1376 |
+
openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff"
|
| 1377 |
+
url = f"https://www.openml.org/{openml_path}"
|
| 1378 |
+
cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
|
| 1379 |
+
location = _get_local_path(openml_path, cache_directory)
|
| 1380 |
+
|
| 1381 |
+
def _mock_urlopen(request, *args, **kwargs):
|
| 1382 |
+
if write_to_disk:
|
| 1383 |
+
with open(location, "w") as f:
|
| 1384 |
+
f.write("")
|
| 1385 |
+
raise ValueError("Invalid request")
|
| 1386 |
+
|
| 1387 |
+
monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen)
|
| 1388 |
+
|
| 1389 |
+
with pytest.raises(ValueError, match="Invalid request"):
|
| 1390 |
+
_open_openml_url(url, cache_directory)
|
| 1391 |
+
|
| 1392 |
+
assert not os.path.exists(location)
|
| 1393 |
+
|
| 1394 |
+
|
| 1395 |
+
def test_retry_with_clean_cache(tmpdir):
|
| 1396 |
+
data_id = 61
|
| 1397 |
+
openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id)
|
| 1398 |
+
cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
|
| 1399 |
+
location = _get_local_path(openml_path, cache_directory)
|
| 1400 |
+
os.makedirs(os.path.dirname(location))
|
| 1401 |
+
|
| 1402 |
+
with open(location, "w") as f:
|
| 1403 |
+
f.write("")
|
| 1404 |
+
|
| 1405 |
+
@_retry_with_clean_cache(openml_path, cache_directory)
|
| 1406 |
+
def _load_data():
|
| 1407 |
+
# The first call will raise an error since location exists
|
| 1408 |
+
if os.path.exists(location):
|
| 1409 |
+
raise Exception("File exist!")
|
| 1410 |
+
return 1
|
| 1411 |
+
|
| 1412 |
+
warn_msg = "Invalid cache, redownloading file"
|
| 1413 |
+
with pytest.warns(RuntimeWarning, match=warn_msg):
|
| 1414 |
+
result = _load_data()
|
| 1415 |
+
assert result == 1
|
| 1416 |
+
|
| 1417 |
+
|
| 1418 |
+
def test_retry_with_clean_cache_http_error(tmpdir):
|
| 1419 |
+
data_id = 61
|
| 1420 |
+
openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id)
|
| 1421 |
+
cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
|
| 1422 |
+
|
| 1423 |
+
@_retry_with_clean_cache(openml_path, cache_directory)
|
| 1424 |
+
def _load_data():
|
| 1425 |
+
raise HTTPError(
|
| 1426 |
+
url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO()
|
| 1427 |
+
)
|
| 1428 |
+
|
| 1429 |
+
error_msg = "Simulated mock error"
|
| 1430 |
+
with pytest.raises(HTTPError, match=error_msg):
|
| 1431 |
+
_load_data()
|
| 1432 |
+
|
| 1433 |
+
|
| 1434 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1435 |
+
def test_fetch_openml_cache(monkeypatch, gzip_response, tmpdir):
|
| 1436 |
+
def _mock_urlopen_raise(request, *args, **kwargs):
|
| 1437 |
+
raise ValueError(
|
| 1438 |
+
"This mechanism intends to test correct cache"
|
| 1439 |
+
"handling. As such, urlopen should never be "
|
| 1440 |
+
"accessed. URL: %s" % request.get_full_url()
|
| 1441 |
+
)
|
| 1442 |
+
|
| 1443 |
+
data_id = 61
|
| 1444 |
+
cache_directory = str(tmpdir.mkdir("scikit_learn_data"))
|
| 1445 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1446 |
+
X_fetched, y_fetched = fetch_openml(
|
| 1447 |
+
data_id=data_id,
|
| 1448 |
+
cache=True,
|
| 1449 |
+
data_home=cache_directory,
|
| 1450 |
+
return_X_y=True,
|
| 1451 |
+
as_frame=False,
|
| 1452 |
+
parser="liac-arff",
|
| 1453 |
+
)
|
| 1454 |
+
|
| 1455 |
+
monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen_raise)
|
| 1456 |
+
|
| 1457 |
+
X_cached, y_cached = fetch_openml(
|
| 1458 |
+
data_id=data_id,
|
| 1459 |
+
cache=True,
|
| 1460 |
+
data_home=cache_directory,
|
| 1461 |
+
return_X_y=True,
|
| 1462 |
+
as_frame=False,
|
| 1463 |
+
parser="liac-arff",
|
| 1464 |
+
)
|
| 1465 |
+
np.testing.assert_array_equal(X_fetched, X_cached)
|
| 1466 |
+
np.testing.assert_array_equal(y_fetched, y_cached)
|
| 1467 |
+
|
| 1468 |
+
|
| 1469 |
+
@pytest.mark.parametrize(
|
| 1470 |
+
"as_frame, parser",
|
| 1471 |
+
[
|
| 1472 |
+
(True, "liac-arff"),
|
| 1473 |
+
(False, "liac-arff"),
|
| 1474 |
+
(True, "pandas"),
|
| 1475 |
+
(False, "pandas"),
|
| 1476 |
+
],
|
| 1477 |
+
)
|
| 1478 |
+
def test_fetch_openml_verify_checksum(monkeypatch, as_frame, tmpdir, parser):
|
| 1479 |
+
"""Check that the checksum is working as expected."""
|
| 1480 |
+
if as_frame or parser == "pandas":
|
| 1481 |
+
pytest.importorskip("pandas")
|
| 1482 |
+
|
| 1483 |
+
data_id = 2
|
| 1484 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, True)
|
| 1485 |
+
|
| 1486 |
+
# create a temporary modified arff file
|
| 1487 |
+
original_data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}"
|
| 1488 |
+
original_data_file_name = "data-v1-dl-1666876.arff.gz"
|
| 1489 |
+
original_data_path = resources.files(original_data_module) / original_data_file_name
|
| 1490 |
+
corrupt_copy_path = tmpdir / "test_invalid_checksum.arff"
|
| 1491 |
+
with original_data_path.open("rb") as orig_file:
|
| 1492 |
+
orig_gzip = gzip.open(orig_file, "rb")
|
| 1493 |
+
data = bytearray(orig_gzip.read())
|
| 1494 |
+
data[len(data) - 1] = 37
|
| 1495 |
+
|
| 1496 |
+
with gzip.GzipFile(corrupt_copy_path, "wb") as modified_gzip:
|
| 1497 |
+
modified_gzip.write(data)
|
| 1498 |
+
|
| 1499 |
+
# Requests are already mocked by monkey_patch_webbased_functions.
|
| 1500 |
+
# We want to reuse that mock for all requests except file download,
|
| 1501 |
+
# hence creating a thin mock over the original mock
|
| 1502 |
+
mocked_openml_url = sklearn.datasets._openml.urlopen
|
| 1503 |
+
|
| 1504 |
+
def swap_file_mock(request, *args, **kwargs):
|
| 1505 |
+
url = request.get_full_url()
|
| 1506 |
+
if url.endswith("data/v1/download/1666876/anneal.arff"):
|
| 1507 |
+
with open(corrupt_copy_path, "rb") as f:
|
| 1508 |
+
corrupted_data = f.read()
|
| 1509 |
+
return _MockHTTPResponse(BytesIO(corrupted_data), is_gzip=True)
|
| 1510 |
+
else:
|
| 1511 |
+
return mocked_openml_url(request)
|
| 1512 |
+
|
| 1513 |
+
monkeypatch.setattr(sklearn.datasets._openml, "urlopen", swap_file_mock)
|
| 1514 |
+
|
| 1515 |
+
# validate failed checksum
|
| 1516 |
+
with pytest.raises(ValueError) as exc:
|
| 1517 |
+
sklearn.datasets.fetch_openml(
|
| 1518 |
+
data_id=data_id, cache=False, as_frame=as_frame, parser=parser
|
| 1519 |
+
)
|
| 1520 |
+
# exception message should have file-path
|
| 1521 |
+
assert exc.match("1666876")
|
| 1522 |
+
|
| 1523 |
+
|
| 1524 |
+
def test_open_openml_url_retry_on_network_error(monkeypatch):
|
| 1525 |
+
def _mock_urlopen_network_error(request, *args, **kwargs):
|
| 1526 |
+
raise HTTPError(
|
| 1527 |
+
url=None, code=404, msg="Simulated network error", hdrs=None, fp=BytesIO()
|
| 1528 |
+
)
|
| 1529 |
+
|
| 1530 |
+
monkeypatch.setattr(
|
| 1531 |
+
sklearn.datasets._openml, "urlopen", _mock_urlopen_network_error
|
| 1532 |
+
)
|
| 1533 |
+
|
| 1534 |
+
invalid_openml_url = "https://api.openml.org/invalid-url"
|
| 1535 |
+
|
| 1536 |
+
with pytest.warns(
|
| 1537 |
+
UserWarning,
|
| 1538 |
+
match=re.escape(
|
| 1539 |
+
"A network error occurred while downloading"
|
| 1540 |
+
f" {invalid_openml_url}. Retrying..."
|
| 1541 |
+
),
|
| 1542 |
+
) as record:
|
| 1543 |
+
with pytest.raises(HTTPError, match="Simulated network error") as exc_info:
|
| 1544 |
+
_open_openml_url(invalid_openml_url, None, delay=0)
|
| 1545 |
+
assert len(record) == 3
|
| 1546 |
+
# Avoid a ResourceWarning on Python 3.14 and later.
|
| 1547 |
+
exc_info.value.close()
|
| 1548 |
+
|
| 1549 |
+
|
| 1550 |
+
###############################################################################
|
| 1551 |
+
# Non-regressiont tests
|
| 1552 |
+
|
| 1553 |
+
|
| 1554 |
+
@pytest.mark.parametrize("gzip_response", [True, False])
|
| 1555 |
+
@pytest.mark.parametrize("parser", ("liac-arff", "pandas"))
|
| 1556 |
+
def test_fetch_openml_with_ignored_feature(monkeypatch, gzip_response, parser):
|
| 1557 |
+
"""Check that we can load the "zoo" dataset.
|
| 1558 |
+
Non-regression test for:
|
| 1559 |
+
https://github.com/scikit-learn/scikit-learn/issues/14340
|
| 1560 |
+
"""
|
| 1561 |
+
if parser == "pandas":
|
| 1562 |
+
pytest.importorskip("pandas")
|
| 1563 |
+
data_id = 62
|
| 1564 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response)
|
| 1565 |
+
|
| 1566 |
+
dataset = sklearn.datasets.fetch_openml(
|
| 1567 |
+
data_id=data_id, cache=False, as_frame=False, parser=parser
|
| 1568 |
+
)
|
| 1569 |
+
assert dataset is not None
|
| 1570 |
+
# The dataset has 17 features, including 1 ignored (animal),
|
| 1571 |
+
# so we assert that we don't have the ignored feature in the final Bunch
|
| 1572 |
+
assert dataset["data"].shape == (101, 16)
|
| 1573 |
+
assert "animal" not in dataset["feature_names"]
|
| 1574 |
+
|
| 1575 |
+
|
| 1576 |
+
def test_fetch_openml_strip_quotes(monkeypatch):
|
| 1577 |
+
"""Check that we strip the single quotes when used as a string delimiter.
|
| 1578 |
+
|
| 1579 |
+
Non-regression test for:
|
| 1580 |
+
https://github.com/scikit-learn/scikit-learn/issues/23381
|
| 1581 |
+
"""
|
| 1582 |
+
pd = pytest.importorskip("pandas")
|
| 1583 |
+
data_id = 40966
|
| 1584 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
|
| 1585 |
+
|
| 1586 |
+
common_params = {"as_frame": True, "cache": False, "data_id": data_id}
|
| 1587 |
+
mice_pandas = fetch_openml(parser="pandas", **common_params)
|
| 1588 |
+
mice_liac_arff = fetch_openml(parser="liac-arff", **common_params)
|
| 1589 |
+
pd.testing.assert_series_equal(mice_pandas.target, mice_liac_arff.target)
|
| 1590 |
+
assert not mice_pandas.target.str.startswith("'").any()
|
| 1591 |
+
assert not mice_pandas.target.str.endswith("'").any()
|
| 1592 |
+
|
| 1593 |
+
# similar behaviour should be observed when the column is not the target
|
| 1594 |
+
mice_pandas = fetch_openml(parser="pandas", target_column="NUMB_N", **common_params)
|
| 1595 |
+
mice_liac_arff = fetch_openml(
|
| 1596 |
+
parser="liac-arff", target_column="NUMB_N", **common_params
|
| 1597 |
+
)
|
| 1598 |
+
pd.testing.assert_series_equal(
|
| 1599 |
+
mice_pandas.frame["class"], mice_liac_arff.frame["class"]
|
| 1600 |
+
)
|
| 1601 |
+
assert not mice_pandas.frame["class"].str.startswith("'").any()
|
| 1602 |
+
assert not mice_pandas.frame["class"].str.endswith("'").any()
|
| 1603 |
+
|
| 1604 |
+
|
| 1605 |
+
def test_fetch_openml_leading_whitespace(monkeypatch):
|
| 1606 |
+
"""Check that we can strip leading whitespace in pandas parser.
|
| 1607 |
+
|
| 1608 |
+
Non-regression test for:
|
| 1609 |
+
https://github.com/scikit-learn/scikit-learn/issues/25311
|
| 1610 |
+
"""
|
| 1611 |
+
pd = pytest.importorskip("pandas")
|
| 1612 |
+
data_id = 1590
|
| 1613 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
|
| 1614 |
+
|
| 1615 |
+
common_params = {"as_frame": True, "cache": False, "data_id": data_id}
|
| 1616 |
+
adult_pandas = fetch_openml(parser="pandas", **common_params)
|
| 1617 |
+
adult_liac_arff = fetch_openml(parser="liac-arff", **common_params)
|
| 1618 |
+
pd.testing.assert_series_equal(
|
| 1619 |
+
adult_pandas.frame["class"], adult_liac_arff.frame["class"]
|
| 1620 |
+
)
|
| 1621 |
+
|
| 1622 |
+
|
| 1623 |
+
def test_fetch_openml_quotechar_escapechar(monkeypatch):
|
| 1624 |
+
"""Check that we can handle escapechar and single/double quotechar.
|
| 1625 |
+
|
| 1626 |
+
Non-regression test for:
|
| 1627 |
+
https://github.com/scikit-learn/scikit-learn/issues/25478
|
| 1628 |
+
"""
|
| 1629 |
+
pd = pytest.importorskip("pandas")
|
| 1630 |
+
data_id = 42074
|
| 1631 |
+
_monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False)
|
| 1632 |
+
|
| 1633 |
+
common_params = {"as_frame": True, "cache": False, "data_id": data_id}
|
| 1634 |
+
adult_pandas = fetch_openml(parser="pandas", **common_params)
|
| 1635 |
+
adult_liac_arff = fetch_openml(parser="liac-arff", **common_params)
|
| 1636 |
+
pd.testing.assert_frame_equal(adult_pandas.frame, adult_liac_arff.frame)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_rcv1.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test the rcv1 loader, if the data is available,
|
| 2 |
+
or if specifically requested via environment variable
|
| 3 |
+
(e.g. for CI jobs)."""
|
| 4 |
+
|
| 5 |
+
from functools import partial
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import scipy.sparse as sp
|
| 9 |
+
|
| 10 |
+
from sklearn.datasets.tests.test_common import check_return_X_y
|
| 11 |
+
from sklearn.utils._testing import assert_almost_equal, assert_array_equal
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def test_fetch_rcv1(fetch_rcv1_fxt, global_random_seed):
|
| 15 |
+
data1 = fetch_rcv1_fxt(shuffle=False)
|
| 16 |
+
X1, Y1 = data1.data, data1.target
|
| 17 |
+
cat_list, s1 = data1.target_names.tolist(), data1.sample_id
|
| 18 |
+
|
| 19 |
+
# test sparsity
|
| 20 |
+
assert sp.issparse(X1)
|
| 21 |
+
assert sp.issparse(Y1)
|
| 22 |
+
assert 60915113 == X1.data.size
|
| 23 |
+
assert 2606875 == Y1.data.size
|
| 24 |
+
|
| 25 |
+
# test shapes
|
| 26 |
+
assert (804414, 47236) == X1.shape
|
| 27 |
+
assert (804414, 103) == Y1.shape
|
| 28 |
+
assert (804414,) == s1.shape
|
| 29 |
+
assert 103 == len(cat_list)
|
| 30 |
+
|
| 31 |
+
# test descr
|
| 32 |
+
assert data1.DESCR.startswith(".. _rcv1_dataset:")
|
| 33 |
+
|
| 34 |
+
# test ordering of categories
|
| 35 |
+
first_categories = ["C11", "C12", "C13", "C14", "C15", "C151"]
|
| 36 |
+
assert_array_equal(first_categories, cat_list[:6])
|
| 37 |
+
|
| 38 |
+
# test number of sample for some categories
|
| 39 |
+
some_categories = ("GMIL", "E143", "CCAT")
|
| 40 |
+
number_non_zero_in_cat = (5, 1206, 381327)
|
| 41 |
+
for num, cat in zip(number_non_zero_in_cat, some_categories):
|
| 42 |
+
j = cat_list.index(cat)
|
| 43 |
+
assert num == Y1[:, j].data.size
|
| 44 |
+
|
| 45 |
+
# test shuffling and subset
|
| 46 |
+
data2 = fetch_rcv1_fxt(
|
| 47 |
+
shuffle=True, subset="train", random_state=global_random_seed
|
| 48 |
+
)
|
| 49 |
+
X2, Y2 = data2.data, data2.target
|
| 50 |
+
s2 = data2.sample_id
|
| 51 |
+
|
| 52 |
+
# test return_X_y option
|
| 53 |
+
fetch_func = partial(fetch_rcv1_fxt, shuffle=False, subset="train")
|
| 54 |
+
check_return_X_y(data2, fetch_func)
|
| 55 |
+
|
| 56 |
+
# The first 23149 samples are the training samples
|
| 57 |
+
assert_array_equal(np.sort(s1[:23149]), np.sort(s2))
|
| 58 |
+
|
| 59 |
+
# test some precise values
|
| 60 |
+
some_sample_ids = (2286, 3274, 14042)
|
| 61 |
+
for sample_id in some_sample_ids:
|
| 62 |
+
idx1 = s1.tolist().index(sample_id)
|
| 63 |
+
idx2 = s2.tolist().index(sample_id)
|
| 64 |
+
|
| 65 |
+
feature_values_1 = X1[idx1, :].toarray()
|
| 66 |
+
feature_values_2 = X2[idx2, :].toarray()
|
| 67 |
+
assert_almost_equal(feature_values_1, feature_values_2)
|
| 68 |
+
|
| 69 |
+
target_values_1 = Y1[idx1, :].toarray()
|
| 70 |
+
target_values_2 = Y2[idx2, :].toarray()
|
| 71 |
+
assert_almost_equal(target_values_1, target_values_2)
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_samples_generator.py
ADDED
|
@@ -0,0 +1,744 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
from collections import defaultdict
|
| 3 |
+
from functools import partial
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import pytest
|
| 7 |
+
import scipy.sparse as sp
|
| 8 |
+
|
| 9 |
+
from sklearn.datasets import (
|
| 10 |
+
make_biclusters,
|
| 11 |
+
make_blobs,
|
| 12 |
+
make_checkerboard,
|
| 13 |
+
make_circles,
|
| 14 |
+
make_classification,
|
| 15 |
+
make_friedman1,
|
| 16 |
+
make_friedman2,
|
| 17 |
+
make_friedman3,
|
| 18 |
+
make_hastie_10_2,
|
| 19 |
+
make_low_rank_matrix,
|
| 20 |
+
make_moons,
|
| 21 |
+
make_multilabel_classification,
|
| 22 |
+
make_regression,
|
| 23 |
+
make_s_curve,
|
| 24 |
+
make_sparse_coded_signal,
|
| 25 |
+
make_sparse_spd_matrix,
|
| 26 |
+
make_sparse_uncorrelated,
|
| 27 |
+
make_spd_matrix,
|
| 28 |
+
make_swiss_roll,
|
| 29 |
+
)
|
| 30 |
+
from sklearn.utils._testing import (
|
| 31 |
+
assert_allclose,
|
| 32 |
+
assert_allclose_dense_sparse,
|
| 33 |
+
assert_almost_equal,
|
| 34 |
+
assert_array_almost_equal,
|
| 35 |
+
assert_array_equal,
|
| 36 |
+
)
|
| 37 |
+
from sklearn.utils.validation import assert_all_finite
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def test_make_classification():
|
| 41 |
+
weights = [0.1, 0.25]
|
| 42 |
+
X, y = make_classification(
|
| 43 |
+
n_samples=100,
|
| 44 |
+
n_features=20,
|
| 45 |
+
n_informative=5,
|
| 46 |
+
n_redundant=1,
|
| 47 |
+
n_repeated=1,
|
| 48 |
+
n_classes=3,
|
| 49 |
+
n_clusters_per_class=1,
|
| 50 |
+
hypercube=False,
|
| 51 |
+
shift=None,
|
| 52 |
+
scale=None,
|
| 53 |
+
weights=weights,
|
| 54 |
+
random_state=0,
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
assert weights == [0.1, 0.25]
|
| 58 |
+
assert X.shape == (100, 20), "X shape mismatch"
|
| 59 |
+
assert y.shape == (100,), "y shape mismatch"
|
| 60 |
+
assert np.unique(y).shape == (3,), "Unexpected number of classes"
|
| 61 |
+
assert sum(y == 0) == 10, "Unexpected number of samples in class #0"
|
| 62 |
+
assert sum(y == 1) == 25, "Unexpected number of samples in class #1"
|
| 63 |
+
assert sum(y == 2) == 65, "Unexpected number of samples in class #2"
|
| 64 |
+
|
| 65 |
+
# Test for n_features > 30
|
| 66 |
+
X, y = make_classification(
|
| 67 |
+
n_samples=2000,
|
| 68 |
+
n_features=31,
|
| 69 |
+
n_informative=31,
|
| 70 |
+
n_redundant=0,
|
| 71 |
+
n_repeated=0,
|
| 72 |
+
hypercube=True,
|
| 73 |
+
scale=0.5,
|
| 74 |
+
random_state=0,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
assert X.shape == (2000, 31), "X shape mismatch"
|
| 78 |
+
assert y.shape == (2000,), "y shape mismatch"
|
| 79 |
+
assert (
|
| 80 |
+
np.unique(X.view([("", X.dtype)] * X.shape[1]))
|
| 81 |
+
.view(X.dtype)
|
| 82 |
+
.reshape(-1, X.shape[1])
|
| 83 |
+
.shape[0]
|
| 84 |
+
== 2000
|
| 85 |
+
), "Unexpected number of unique rows"
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def test_make_classification_informative_features():
|
| 89 |
+
"""Test the construction of informative features in make_classification
|
| 90 |
+
|
| 91 |
+
Also tests `n_clusters_per_class`, `n_classes`, `hypercube` and
|
| 92 |
+
fully-specified `weights`.
|
| 93 |
+
"""
|
| 94 |
+
# Create very separate clusters; check that vertices are unique and
|
| 95 |
+
# correspond to classes
|
| 96 |
+
class_sep = 1e6
|
| 97 |
+
make = partial(
|
| 98 |
+
make_classification,
|
| 99 |
+
class_sep=class_sep,
|
| 100 |
+
n_redundant=0,
|
| 101 |
+
n_repeated=0,
|
| 102 |
+
flip_y=0,
|
| 103 |
+
shift=0,
|
| 104 |
+
scale=1,
|
| 105 |
+
shuffle=False,
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
for n_informative, weights, n_clusters_per_class in [
|
| 109 |
+
(2, [1], 1),
|
| 110 |
+
(2, [1 / 3] * 3, 1),
|
| 111 |
+
(2, [1 / 4] * 4, 1),
|
| 112 |
+
(2, [1 / 2] * 2, 2),
|
| 113 |
+
(2, [3 / 4, 1 / 4], 2),
|
| 114 |
+
(10, [1 / 3] * 3, 10),
|
| 115 |
+
(64, [1], 1),
|
| 116 |
+
]:
|
| 117 |
+
n_classes = len(weights)
|
| 118 |
+
n_clusters = n_classes * n_clusters_per_class
|
| 119 |
+
n_samples = n_clusters * 50
|
| 120 |
+
|
| 121 |
+
for hypercube in (False, True):
|
| 122 |
+
X, y = make(
|
| 123 |
+
n_samples=n_samples,
|
| 124 |
+
n_classes=n_classes,
|
| 125 |
+
weights=weights,
|
| 126 |
+
n_features=n_informative,
|
| 127 |
+
n_informative=n_informative,
|
| 128 |
+
n_clusters_per_class=n_clusters_per_class,
|
| 129 |
+
hypercube=hypercube,
|
| 130 |
+
random_state=0,
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
assert X.shape == (n_samples, n_informative)
|
| 134 |
+
assert y.shape == (n_samples,)
|
| 135 |
+
|
| 136 |
+
# Cluster by sign, viewed as strings to allow uniquing
|
| 137 |
+
signs = np.sign(X)
|
| 138 |
+
signs = signs.view(dtype="|S{0}".format(signs.strides[0])).ravel()
|
| 139 |
+
unique_signs, cluster_index = np.unique(signs, return_inverse=True)
|
| 140 |
+
|
| 141 |
+
assert len(unique_signs) == n_clusters, (
|
| 142 |
+
"Wrong number of clusters, or not in distinct quadrants"
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
clusters_by_class = defaultdict(set)
|
| 146 |
+
for cluster, cls in zip(cluster_index, y):
|
| 147 |
+
clusters_by_class[cls].add(cluster)
|
| 148 |
+
for clusters in clusters_by_class.values():
|
| 149 |
+
assert len(clusters) == n_clusters_per_class, (
|
| 150 |
+
"Wrong number of clusters per class"
|
| 151 |
+
)
|
| 152 |
+
assert len(clusters_by_class) == n_classes, "Wrong number of classes"
|
| 153 |
+
|
| 154 |
+
assert_array_almost_equal(
|
| 155 |
+
np.bincount(y) / len(y) // weights,
|
| 156 |
+
[1] * n_classes,
|
| 157 |
+
err_msg="Wrong number of samples per class",
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
# Ensure on vertices of hypercube
|
| 161 |
+
for cluster in range(len(unique_signs)):
|
| 162 |
+
centroid = X[cluster_index == cluster].mean(axis=0)
|
| 163 |
+
if hypercube:
|
| 164 |
+
assert_array_almost_equal(
|
| 165 |
+
np.abs(centroid) / class_sep,
|
| 166 |
+
np.ones(n_informative),
|
| 167 |
+
decimal=5,
|
| 168 |
+
err_msg="Clusters are not centered on hypercube vertices",
|
| 169 |
+
)
|
| 170 |
+
else:
|
| 171 |
+
with pytest.raises(AssertionError):
|
| 172 |
+
assert_array_almost_equal(
|
| 173 |
+
np.abs(centroid) / class_sep,
|
| 174 |
+
np.ones(n_informative),
|
| 175 |
+
decimal=5,
|
| 176 |
+
err_msg=(
|
| 177 |
+
"Clusters should not be centered on hypercube vertices"
|
| 178 |
+
),
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
with pytest.raises(ValueError):
|
| 182 |
+
make(n_features=2, n_informative=2, n_classes=5, n_clusters_per_class=1)
|
| 183 |
+
with pytest.raises(ValueError):
|
| 184 |
+
make(n_features=2, n_informative=2, n_classes=3, n_clusters_per_class=2)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def test_make_classification_return_x_y():
|
| 188 |
+
"""
|
| 189 |
+
Test that make_classification returns a Bunch when return_X_y is False.
|
| 190 |
+
|
| 191 |
+
Also that bunch.X is the same as X
|
| 192 |
+
"""
|
| 193 |
+
|
| 194 |
+
kwargs = {
|
| 195 |
+
"n_samples": 100,
|
| 196 |
+
"n_features": 20,
|
| 197 |
+
"n_informative": 5,
|
| 198 |
+
"n_redundant": 1,
|
| 199 |
+
"n_repeated": 1,
|
| 200 |
+
"n_classes": 3,
|
| 201 |
+
"n_clusters_per_class": 2,
|
| 202 |
+
"weights": None,
|
| 203 |
+
"flip_y": 0.01,
|
| 204 |
+
"class_sep": 1.0,
|
| 205 |
+
"hypercube": True,
|
| 206 |
+
"shift": 0.0,
|
| 207 |
+
"scale": 1.0,
|
| 208 |
+
"shuffle": True,
|
| 209 |
+
"random_state": 42,
|
| 210 |
+
"return_X_y": True,
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
X, y = make_classification(**kwargs)
|
| 214 |
+
|
| 215 |
+
kwargs["return_X_y"] = False
|
| 216 |
+
bunch = make_classification(**kwargs)
|
| 217 |
+
|
| 218 |
+
assert (
|
| 219 |
+
hasattr(bunch, "DESCR")
|
| 220 |
+
and hasattr(bunch, "parameters")
|
| 221 |
+
and hasattr(bunch, "feature_info")
|
| 222 |
+
and hasattr(bunch, "X")
|
| 223 |
+
and hasattr(bunch, "y")
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
def count(str_):
|
| 227 |
+
return bunch.feature_info.count(str_)
|
| 228 |
+
|
| 229 |
+
assert np.array_equal(X, bunch.X)
|
| 230 |
+
assert np.array_equal(y, bunch.y)
|
| 231 |
+
assert bunch.DESCR == make_classification.__doc__
|
| 232 |
+
assert bunch.parameters == kwargs
|
| 233 |
+
assert count("informative") == kwargs["n_informative"]
|
| 234 |
+
assert count("redundant") == kwargs["n_redundant"]
|
| 235 |
+
assert count("repeated") == kwargs["n_repeated"]
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
@pytest.mark.parametrize(
|
| 239 |
+
"weights, err_type, err_msg",
|
| 240 |
+
[
|
| 241 |
+
([], ValueError, "Weights specified but incompatible with number of classes."),
|
| 242 |
+
(
|
| 243 |
+
[0.25, 0.75, 0.1],
|
| 244 |
+
ValueError,
|
| 245 |
+
"Weights specified but incompatible with number of classes.",
|
| 246 |
+
),
|
| 247 |
+
(
|
| 248 |
+
np.array([]),
|
| 249 |
+
ValueError,
|
| 250 |
+
"Weights specified but incompatible with number of classes.",
|
| 251 |
+
),
|
| 252 |
+
(
|
| 253 |
+
np.array([0.25, 0.75, 0.1]),
|
| 254 |
+
ValueError,
|
| 255 |
+
"Weights specified but incompatible with number of classes.",
|
| 256 |
+
),
|
| 257 |
+
(
|
| 258 |
+
np.random.random(3),
|
| 259 |
+
ValueError,
|
| 260 |
+
"Weights specified but incompatible with number of classes.",
|
| 261 |
+
),
|
| 262 |
+
],
|
| 263 |
+
)
|
| 264 |
+
def test_make_classification_weights_type(weights, err_type, err_msg):
|
| 265 |
+
with pytest.raises(err_type, match=err_msg):
|
| 266 |
+
make_classification(weights=weights)
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
@pytest.mark.parametrize("kwargs", [{}, {"n_classes": 3, "n_informative": 3}])
|
| 270 |
+
def test_make_classification_weights_array_or_list_ok(kwargs):
|
| 271 |
+
X1, y1 = make_classification(weights=[0.1, 0.9], random_state=0, **kwargs)
|
| 272 |
+
X2, y2 = make_classification(weights=np.array([0.1, 0.9]), random_state=0, **kwargs)
|
| 273 |
+
assert_almost_equal(X1, X2)
|
| 274 |
+
assert_almost_equal(y1, y2)
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def test_make_multilabel_classification_return_sequences():
|
| 278 |
+
for allow_unlabeled, min_length in zip((True, False), (0, 1)):
|
| 279 |
+
X, Y = make_multilabel_classification(
|
| 280 |
+
n_samples=100,
|
| 281 |
+
n_features=20,
|
| 282 |
+
n_classes=3,
|
| 283 |
+
random_state=0,
|
| 284 |
+
return_indicator=False,
|
| 285 |
+
allow_unlabeled=allow_unlabeled,
|
| 286 |
+
)
|
| 287 |
+
assert X.shape == (100, 20), "X shape mismatch"
|
| 288 |
+
if not allow_unlabeled:
|
| 289 |
+
assert max([max(y) for y in Y]) == 2
|
| 290 |
+
assert min([len(y) for y in Y]) == min_length
|
| 291 |
+
assert max([len(y) for y in Y]) <= 3
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def test_make_multilabel_classification_return_indicator():
|
| 295 |
+
for allow_unlabeled, min_length in zip((True, False), (0, 1)):
|
| 296 |
+
X, Y = make_multilabel_classification(
|
| 297 |
+
n_samples=25,
|
| 298 |
+
n_features=20,
|
| 299 |
+
n_classes=3,
|
| 300 |
+
random_state=0,
|
| 301 |
+
allow_unlabeled=allow_unlabeled,
|
| 302 |
+
)
|
| 303 |
+
assert X.shape == (25, 20), "X shape mismatch"
|
| 304 |
+
assert Y.shape == (25, 3), "Y shape mismatch"
|
| 305 |
+
assert np.all(np.sum(Y, axis=0) > min_length)
|
| 306 |
+
|
| 307 |
+
# Also test return_distributions and return_indicator with True
|
| 308 |
+
X2, Y2, p_c, p_w_c = make_multilabel_classification(
|
| 309 |
+
n_samples=25,
|
| 310 |
+
n_features=20,
|
| 311 |
+
n_classes=3,
|
| 312 |
+
random_state=0,
|
| 313 |
+
allow_unlabeled=allow_unlabeled,
|
| 314 |
+
return_distributions=True,
|
| 315 |
+
)
|
| 316 |
+
|
| 317 |
+
assert_array_almost_equal(X, X2)
|
| 318 |
+
assert_array_equal(Y, Y2)
|
| 319 |
+
assert p_c.shape == (3,)
|
| 320 |
+
assert_almost_equal(p_c.sum(), 1)
|
| 321 |
+
assert p_w_c.shape == (20, 3)
|
| 322 |
+
assert_almost_equal(p_w_c.sum(axis=0), [1] * 3)
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def test_make_multilabel_classification_return_indicator_sparse():
|
| 326 |
+
for allow_unlabeled, min_length in zip((True, False), (0, 1)):
|
| 327 |
+
X, Y = make_multilabel_classification(
|
| 328 |
+
n_samples=25,
|
| 329 |
+
n_features=20,
|
| 330 |
+
n_classes=3,
|
| 331 |
+
random_state=0,
|
| 332 |
+
return_indicator="sparse",
|
| 333 |
+
allow_unlabeled=allow_unlabeled,
|
| 334 |
+
)
|
| 335 |
+
assert X.shape == (25, 20), "X shape mismatch"
|
| 336 |
+
assert Y.shape == (25, 3), "Y shape mismatch"
|
| 337 |
+
assert sp.issparse(Y)
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def test_make_hastie_10_2():
|
| 341 |
+
X, y = make_hastie_10_2(n_samples=100, random_state=0)
|
| 342 |
+
assert X.shape == (100, 10), "X shape mismatch"
|
| 343 |
+
assert y.shape == (100,), "y shape mismatch"
|
| 344 |
+
assert np.unique(y).shape == (2,), "Unexpected number of classes"
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def test_make_regression(global_random_seed):
|
| 348 |
+
X, y, c = make_regression(
|
| 349 |
+
n_samples=200,
|
| 350 |
+
n_features=10,
|
| 351 |
+
n_informative=3,
|
| 352 |
+
effective_rank=5,
|
| 353 |
+
coef=True,
|
| 354 |
+
bias=0.0,
|
| 355 |
+
noise=1.0,
|
| 356 |
+
random_state=global_random_seed,
|
| 357 |
+
)
|
| 358 |
+
|
| 359 |
+
assert X.shape == (200, 10), "X shape mismatch"
|
| 360 |
+
assert y.shape == (200,), "y shape mismatch"
|
| 361 |
+
assert c.shape == (10,), "coef shape mismatch"
|
| 362 |
+
assert sum(c != 0.0) == 3, "Unexpected number of informative features"
|
| 363 |
+
|
| 364 |
+
# Test that y ~= np.dot(X, c) + bias + N(0, 1.0).
|
| 365 |
+
assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1)
|
| 366 |
+
|
| 367 |
+
# Test with small number of features.
|
| 368 |
+
X, y = make_regression(n_samples=100, n_features=1) # n_informative=3
|
| 369 |
+
assert X.shape == (100, 1)
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def test_make_regression_multitarget(global_random_seed):
|
| 373 |
+
X, y, c = make_regression(
|
| 374 |
+
n_samples=100,
|
| 375 |
+
n_features=10,
|
| 376 |
+
n_informative=3,
|
| 377 |
+
n_targets=3,
|
| 378 |
+
coef=True,
|
| 379 |
+
noise=1.0,
|
| 380 |
+
random_state=global_random_seed,
|
| 381 |
+
)
|
| 382 |
+
|
| 383 |
+
assert X.shape == (100, 10), "X shape mismatch"
|
| 384 |
+
assert y.shape == (100, 3), "y shape mismatch"
|
| 385 |
+
assert c.shape == (10, 3), "coef shape mismatch"
|
| 386 |
+
assert_array_equal(sum(c != 0.0), 3, "Unexpected number of informative features")
|
| 387 |
+
|
| 388 |
+
# Test that y ~= np.dot(X, c) + bias + N(0, 1.0)
|
| 389 |
+
assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def test_make_blobs(global_random_seed):
|
| 393 |
+
cluster_stds = np.array([0.05, 0.2, 0.4])
|
| 394 |
+
cluster_centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
|
| 395 |
+
X, y = make_blobs(
|
| 396 |
+
random_state=global_random_seed,
|
| 397 |
+
n_samples=50,
|
| 398 |
+
n_features=2,
|
| 399 |
+
centers=cluster_centers,
|
| 400 |
+
cluster_std=cluster_stds,
|
| 401 |
+
)
|
| 402 |
+
|
| 403 |
+
assert X.shape == (50, 2), "X shape mismatch"
|
| 404 |
+
assert y.shape == (50,), "y shape mismatch"
|
| 405 |
+
assert np.unique(y).shape == (3,), "Unexpected number of blobs"
|
| 406 |
+
for i, (ctr, std) in enumerate(zip(cluster_centers, cluster_stds)):
|
| 407 |
+
assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std")
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
def test_make_blobs_n_samples_list():
|
| 411 |
+
n_samples = [50, 30, 20]
|
| 412 |
+
X, y = make_blobs(n_samples=n_samples, n_features=2, random_state=0)
|
| 413 |
+
|
| 414 |
+
assert X.shape == (sum(n_samples), 2), "X shape mismatch"
|
| 415 |
+
assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), (
|
| 416 |
+
"Incorrect number of samples per blob"
|
| 417 |
+
)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def test_make_blobs_n_samples_list_with_centers(global_random_seed):
|
| 421 |
+
n_samples = [20, 20, 20]
|
| 422 |
+
centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
|
| 423 |
+
cluster_stds = np.array([0.05, 0.2, 0.4])
|
| 424 |
+
X, y = make_blobs(
|
| 425 |
+
n_samples=n_samples,
|
| 426 |
+
centers=centers,
|
| 427 |
+
cluster_std=cluster_stds,
|
| 428 |
+
random_state=global_random_seed,
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
assert X.shape == (sum(n_samples), 2), "X shape mismatch"
|
| 432 |
+
assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), (
|
| 433 |
+
"Incorrect number of samples per blob"
|
| 434 |
+
)
|
| 435 |
+
for i, (ctr, std) in enumerate(zip(centers, cluster_stds)):
|
| 436 |
+
assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std")
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
@pytest.mark.parametrize(
|
| 440 |
+
"n_samples", [[5, 3, 0], np.array([5, 3, 0]), tuple([5, 3, 0])]
|
| 441 |
+
)
|
| 442 |
+
def test_make_blobs_n_samples_centers_none(n_samples):
|
| 443 |
+
centers = None
|
| 444 |
+
X, y = make_blobs(n_samples=n_samples, centers=centers, random_state=0)
|
| 445 |
+
|
| 446 |
+
assert X.shape == (sum(n_samples), 2), "X shape mismatch"
|
| 447 |
+
assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), (
|
| 448 |
+
"Incorrect number of samples per blob"
|
| 449 |
+
)
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
def test_make_blobs_return_centers():
|
| 453 |
+
n_samples = [10, 20]
|
| 454 |
+
n_features = 3
|
| 455 |
+
X, y, centers = make_blobs(
|
| 456 |
+
n_samples=n_samples, n_features=n_features, return_centers=True, random_state=0
|
| 457 |
+
)
|
| 458 |
+
|
| 459 |
+
assert centers.shape == (len(n_samples), n_features)
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
def test_make_blobs_error():
|
| 463 |
+
n_samples = [20, 20, 20]
|
| 464 |
+
centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
|
| 465 |
+
cluster_stds = np.array([0.05, 0.2, 0.4])
|
| 466 |
+
wrong_centers_msg = re.escape(
|
| 467 |
+
"Length of `n_samples` not consistent with number of centers. "
|
| 468 |
+
f"Got n_samples = {n_samples} and centers = {centers[:-1]}"
|
| 469 |
+
)
|
| 470 |
+
with pytest.raises(ValueError, match=wrong_centers_msg):
|
| 471 |
+
make_blobs(n_samples, centers=centers[:-1])
|
| 472 |
+
wrong_std_msg = re.escape(
|
| 473 |
+
"Length of `clusters_std` not consistent with number of centers. "
|
| 474 |
+
f"Got centers = {centers} and cluster_std = {cluster_stds[:-1]}"
|
| 475 |
+
)
|
| 476 |
+
with pytest.raises(ValueError, match=wrong_std_msg):
|
| 477 |
+
make_blobs(n_samples, centers=centers, cluster_std=cluster_stds[:-1])
|
| 478 |
+
wrong_type_msg = "Parameter `centers` must be array-like. Got {!r} instead".format(
|
| 479 |
+
3
|
| 480 |
+
)
|
| 481 |
+
with pytest.raises(ValueError, match=wrong_type_msg):
|
| 482 |
+
make_blobs(n_samples, centers=3)
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
def test_make_friedman1(global_random_seed):
|
| 486 |
+
X, y = make_friedman1(
|
| 487 |
+
n_samples=5, n_features=10, noise=0.0, random_state=global_random_seed
|
| 488 |
+
)
|
| 489 |
+
|
| 490 |
+
assert X.shape == (5, 10), "X shape mismatch"
|
| 491 |
+
assert y.shape == (5,), "y shape mismatch"
|
| 492 |
+
|
| 493 |
+
assert_array_almost_equal(
|
| 494 |
+
y,
|
| 495 |
+
10 * np.sin(np.pi * X[:, 0] * X[:, 1])
|
| 496 |
+
+ 20 * (X[:, 2] - 0.5) ** 2
|
| 497 |
+
+ 10 * X[:, 3]
|
| 498 |
+
+ 5 * X[:, 4],
|
| 499 |
+
)
|
| 500 |
+
|
| 501 |
+
|
| 502 |
+
def test_make_friedman2(global_random_seed):
|
| 503 |
+
X, y = make_friedman2(n_samples=5, noise=0.0, random_state=global_random_seed)
|
| 504 |
+
|
| 505 |
+
assert X.shape == (5, 4), "X shape mismatch"
|
| 506 |
+
assert y.shape == (5,), "y shape mismatch"
|
| 507 |
+
|
| 508 |
+
assert_array_almost_equal(
|
| 509 |
+
y, (X[:, 0] ** 2 + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5
|
| 510 |
+
)
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def test_make_friedman3(global_random_seed):
|
| 514 |
+
X, y = make_friedman3(n_samples=5, noise=0.0, random_state=global_random_seed)
|
| 515 |
+
|
| 516 |
+
assert X.shape == (5, 4), "X shape mismatch"
|
| 517 |
+
assert y.shape == (5,), "y shape mismatch"
|
| 518 |
+
|
| 519 |
+
assert_array_almost_equal(
|
| 520 |
+
y, np.arctan((X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) / X[:, 0])
|
| 521 |
+
)
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def test_make_low_rank_matrix():
|
| 525 |
+
X = make_low_rank_matrix(
|
| 526 |
+
n_samples=50,
|
| 527 |
+
n_features=25,
|
| 528 |
+
effective_rank=5,
|
| 529 |
+
tail_strength=0.01,
|
| 530 |
+
random_state=0,
|
| 531 |
+
)
|
| 532 |
+
|
| 533 |
+
assert X.shape == (50, 25), "X shape mismatch"
|
| 534 |
+
|
| 535 |
+
from numpy.linalg import svd
|
| 536 |
+
|
| 537 |
+
u, s, v = svd(X)
|
| 538 |
+
assert sum(s) - 5 < 0.1, "X rank is not approximately 5"
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def test_make_sparse_coded_signal(global_random_seed):
|
| 542 |
+
Y, D, X = make_sparse_coded_signal(
|
| 543 |
+
n_samples=5,
|
| 544 |
+
n_components=8,
|
| 545 |
+
n_features=10,
|
| 546 |
+
n_nonzero_coefs=3,
|
| 547 |
+
random_state=global_random_seed,
|
| 548 |
+
)
|
| 549 |
+
assert Y.shape == (5, 10), "Y shape mismatch"
|
| 550 |
+
assert D.shape == (8, 10), "D shape mismatch"
|
| 551 |
+
assert X.shape == (5, 8), "X shape mismatch"
|
| 552 |
+
for row in X:
|
| 553 |
+
assert len(np.flatnonzero(row)) == 3, "Non-zero coefs mismatch"
|
| 554 |
+
assert_allclose(Y, X @ D)
|
| 555 |
+
assert_allclose(np.sqrt((D**2).sum(axis=1)), np.ones(D.shape[0]))
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
def test_make_sparse_uncorrelated():
|
| 559 |
+
X, y = make_sparse_uncorrelated(n_samples=5, n_features=10, random_state=0)
|
| 560 |
+
|
| 561 |
+
assert X.shape == (5, 10), "X shape mismatch"
|
| 562 |
+
assert y.shape == (5,), "y shape mismatch"
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
def test_make_spd_matrix(global_random_seed):
|
| 566 |
+
X = make_spd_matrix(n_dim=5, random_state=global_random_seed)
|
| 567 |
+
|
| 568 |
+
assert X.shape == (5, 5), "X shape mismatch"
|
| 569 |
+
assert_array_almost_equal(X, X.T)
|
| 570 |
+
|
| 571 |
+
from numpy.linalg import eig
|
| 572 |
+
|
| 573 |
+
eigenvalues, _ = eig(X)
|
| 574 |
+
assert np.all(eigenvalues > 0), "X is not positive-definite"
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
@pytest.mark.parametrize("norm_diag", [True, False])
|
| 578 |
+
@pytest.mark.parametrize(
|
| 579 |
+
"sparse_format", [None, "bsr", "coo", "csc", "csr", "dia", "dok", "lil"]
|
| 580 |
+
)
|
| 581 |
+
def test_make_sparse_spd_matrix(norm_diag, sparse_format, global_random_seed):
|
| 582 |
+
n_dim = 5
|
| 583 |
+
X = make_sparse_spd_matrix(
|
| 584 |
+
n_dim=n_dim,
|
| 585 |
+
norm_diag=norm_diag,
|
| 586 |
+
sparse_format=sparse_format,
|
| 587 |
+
random_state=global_random_seed,
|
| 588 |
+
)
|
| 589 |
+
|
| 590 |
+
assert X.shape == (n_dim, n_dim), "X shape mismatch"
|
| 591 |
+
if sparse_format is None:
|
| 592 |
+
assert not sp.issparse(X)
|
| 593 |
+
assert_allclose(X, X.T)
|
| 594 |
+
Xarr = X
|
| 595 |
+
else:
|
| 596 |
+
assert sp.issparse(X) and X.format == sparse_format
|
| 597 |
+
assert_allclose_dense_sparse(X, X.T)
|
| 598 |
+
Xarr = X.toarray()
|
| 599 |
+
|
| 600 |
+
from numpy.linalg import eig
|
| 601 |
+
|
| 602 |
+
# Do not use scipy.sparse.linalg.eigs because it cannot find all eigenvalues
|
| 603 |
+
eigenvalues, _ = eig(Xarr)
|
| 604 |
+
assert np.all(eigenvalues > 0), "X is not positive-definite"
|
| 605 |
+
|
| 606 |
+
if norm_diag:
|
| 607 |
+
# Check that leading diagonal elements are 1
|
| 608 |
+
assert_array_almost_equal(Xarr.diagonal(), np.ones(n_dim))
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
@pytest.mark.parametrize("hole", [False, True])
|
| 612 |
+
def test_make_swiss_roll(global_random_seed, hole):
|
| 613 |
+
X, t = make_swiss_roll(
|
| 614 |
+
n_samples=5, noise=0.0, random_state=global_random_seed, hole=hole
|
| 615 |
+
)
|
| 616 |
+
|
| 617 |
+
assert X.shape == (5, 3)
|
| 618 |
+
assert t.shape == (5,)
|
| 619 |
+
assert_array_almost_equal(X[:, 0], t * np.cos(t))
|
| 620 |
+
assert_array_almost_equal(X[:, 2], t * np.sin(t))
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
def test_make_s_curve(global_random_seed):
|
| 624 |
+
X, t = make_s_curve(n_samples=5, noise=0.0, random_state=global_random_seed)
|
| 625 |
+
|
| 626 |
+
assert X.shape == (5, 3), "X shape mismatch"
|
| 627 |
+
assert t.shape == (5,), "t shape mismatch"
|
| 628 |
+
assert_array_almost_equal(X[:, 0], np.sin(t))
|
| 629 |
+
assert_array_almost_equal(X[:, 2], np.sign(t) * (np.cos(t) - 1))
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
def test_make_biclusters():
|
| 633 |
+
X, rows, cols = make_biclusters(
|
| 634 |
+
shape=(100, 100), n_clusters=4, shuffle=True, random_state=0
|
| 635 |
+
)
|
| 636 |
+
assert X.shape == (100, 100), "X shape mismatch"
|
| 637 |
+
assert rows.shape == (4, 100), "rows shape mismatch"
|
| 638 |
+
assert cols.shape == (
|
| 639 |
+
4,
|
| 640 |
+
100,
|
| 641 |
+
), "columns shape mismatch"
|
| 642 |
+
assert_all_finite(X)
|
| 643 |
+
assert_all_finite(rows)
|
| 644 |
+
assert_all_finite(cols)
|
| 645 |
+
|
| 646 |
+
X2, _, _ = make_biclusters(
|
| 647 |
+
shape=(100, 100), n_clusters=4, shuffle=True, random_state=0
|
| 648 |
+
)
|
| 649 |
+
assert_array_almost_equal(X, X2)
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
def test_make_checkerboard():
|
| 653 |
+
X, rows, cols = make_checkerboard(
|
| 654 |
+
shape=(100, 100), n_clusters=(20, 5), shuffle=True, random_state=0
|
| 655 |
+
)
|
| 656 |
+
assert X.shape == (100, 100), "X shape mismatch"
|
| 657 |
+
assert rows.shape == (100, 100), "rows shape mismatch"
|
| 658 |
+
assert cols.shape == (
|
| 659 |
+
100,
|
| 660 |
+
100,
|
| 661 |
+
), "columns shape mismatch"
|
| 662 |
+
|
| 663 |
+
X, rows, cols = make_checkerboard(
|
| 664 |
+
shape=(100, 100), n_clusters=2, shuffle=True, random_state=0
|
| 665 |
+
)
|
| 666 |
+
assert_all_finite(X)
|
| 667 |
+
assert_all_finite(rows)
|
| 668 |
+
assert_all_finite(cols)
|
| 669 |
+
|
| 670 |
+
X1, _, _ = make_checkerboard(
|
| 671 |
+
shape=(100, 100), n_clusters=2, shuffle=True, random_state=0
|
| 672 |
+
)
|
| 673 |
+
X2, _, _ = make_checkerboard(
|
| 674 |
+
shape=(100, 100), n_clusters=2, shuffle=True, random_state=0
|
| 675 |
+
)
|
| 676 |
+
assert_array_almost_equal(X1, X2)
|
| 677 |
+
|
| 678 |
+
|
| 679 |
+
def test_make_moons(global_random_seed):
|
| 680 |
+
X, y = make_moons(3, shuffle=False, random_state=global_random_seed)
|
| 681 |
+
for x, label in zip(X, y):
|
| 682 |
+
center = [0.0, 0.0] if label == 0 else [1.0, 0.5]
|
| 683 |
+
dist_sqr = ((x - center) ** 2).sum()
|
| 684 |
+
assert_almost_equal(
|
| 685 |
+
dist_sqr, 1.0, err_msg="Point is not on expected unit circle"
|
| 686 |
+
)
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
def test_make_moons_unbalanced():
|
| 690 |
+
X, y = make_moons(n_samples=(7, 5))
|
| 691 |
+
assert np.sum(y == 0) == 7 and np.sum(y == 1) == 5, (
|
| 692 |
+
"Number of samples in a moon is wrong"
|
| 693 |
+
)
|
| 694 |
+
assert X.shape == (12, 2), "X shape mismatch"
|
| 695 |
+
assert y.shape == (12,), "y shape mismatch"
|
| 696 |
+
|
| 697 |
+
with pytest.raises(
|
| 698 |
+
ValueError,
|
| 699 |
+
match=r"`n_samples` can be either an int or a two-element tuple.",
|
| 700 |
+
):
|
| 701 |
+
make_moons(n_samples=(10,))
|
| 702 |
+
|
| 703 |
+
|
| 704 |
+
def test_make_circles():
|
| 705 |
+
factor = 0.3
|
| 706 |
+
|
| 707 |
+
for n_samples, n_outer, n_inner in [(7, 3, 4), (8, 4, 4)]:
|
| 708 |
+
# Testing odd and even case, because in the past make_circles always
|
| 709 |
+
# created an even number of samples.
|
| 710 |
+
X, y = make_circles(n_samples, shuffle=False, noise=None, factor=factor)
|
| 711 |
+
assert X.shape == (n_samples, 2), "X shape mismatch"
|
| 712 |
+
assert y.shape == (n_samples,), "y shape mismatch"
|
| 713 |
+
center = [0.0, 0.0]
|
| 714 |
+
for x, label in zip(X, y):
|
| 715 |
+
dist_sqr = ((x - center) ** 2).sum()
|
| 716 |
+
dist_exp = 1.0 if label == 0 else factor**2
|
| 717 |
+
dist_exp = 1.0 if label == 0 else factor**2
|
| 718 |
+
assert_almost_equal(
|
| 719 |
+
dist_sqr, dist_exp, err_msg="Point is not on expected circle"
|
| 720 |
+
)
|
| 721 |
+
|
| 722 |
+
assert X[y == 0].shape == (
|
| 723 |
+
n_outer,
|
| 724 |
+
2,
|
| 725 |
+
), "Samples not correctly distributed across circles."
|
| 726 |
+
assert X[y == 1].shape == (
|
| 727 |
+
n_inner,
|
| 728 |
+
2,
|
| 729 |
+
), "Samples not correctly distributed across circles."
|
| 730 |
+
|
| 731 |
+
|
| 732 |
+
def test_make_circles_unbalanced():
|
| 733 |
+
X, y = make_circles(n_samples=(2, 8))
|
| 734 |
+
|
| 735 |
+
assert np.sum(y == 0) == 2, "Number of samples in inner circle is wrong"
|
| 736 |
+
assert np.sum(y == 1) == 8, "Number of samples in outer circle is wrong"
|
| 737 |
+
assert X.shape == (10, 2), "X shape mismatch"
|
| 738 |
+
assert y.shape == (10,), "y shape mismatch"
|
| 739 |
+
|
| 740 |
+
with pytest.raises(
|
| 741 |
+
ValueError,
|
| 742 |
+
match="When a tuple, n_samples must have exactly two elements.",
|
| 743 |
+
):
|
| 744 |
+
make_circles(n_samples=(10,))
|
loan-advisor-agent/.venv/Lib/site-packages/sklearn/datasets/tests/test_svmlight_format.py
ADDED
|
@@ -0,0 +1,613 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gzip
|
| 2 |
+
import os
|
| 3 |
+
import shutil
|
| 4 |
+
from bz2 import BZ2File
|
| 5 |
+
from importlib import resources
|
| 6 |
+
from io import BytesIO
|
| 7 |
+
from tempfile import NamedTemporaryFile
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import pytest
|
| 11 |
+
import scipy.sparse as sp
|
| 12 |
+
|
| 13 |
+
import sklearn
|
| 14 |
+
from sklearn.datasets import dump_svmlight_file, load_svmlight_file, load_svmlight_files
|
| 15 |
+
from sklearn.utils._testing import (
|
| 16 |
+
assert_allclose,
|
| 17 |
+
assert_array_almost_equal,
|
| 18 |
+
assert_array_equal,
|
| 19 |
+
create_memmap_backed_data,
|
| 20 |
+
)
|
| 21 |
+
from sklearn.utils.fixes import CSR_CONTAINERS
|
| 22 |
+
|
| 23 |
+
TEST_DATA_MODULE = "sklearn.datasets.tests.data"
|
| 24 |
+
datafile = "svmlight_classification.txt"
|
| 25 |
+
multifile = "svmlight_multilabel.txt"
|
| 26 |
+
invalidfile = "svmlight_invalid.txt"
|
| 27 |
+
invalidfile2 = "svmlight_invalid_order.txt"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _svmlight_local_test_file_path(filename):
|
| 31 |
+
return resources.files(TEST_DATA_MODULE) / filename
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _load_svmlight_local_test_file(filename, **kwargs):
|
| 35 |
+
"""
|
| 36 |
+
Helper to load resource `filename` with `importlib.resources`
|
| 37 |
+
"""
|
| 38 |
+
data_path = _svmlight_local_test_file_path(filename)
|
| 39 |
+
with data_path.open("rb") as f:
|
| 40 |
+
return load_svmlight_file(f, **kwargs)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_load_svmlight_file():
|
| 44 |
+
X, y = _load_svmlight_local_test_file(datafile)
|
| 45 |
+
|
| 46 |
+
# test X's shape
|
| 47 |
+
assert X.indptr.shape[0] == 7
|
| 48 |
+
assert X.shape[0] == 6
|
| 49 |
+
assert X.shape[1] == 21
|
| 50 |
+
assert y.shape[0] == 6
|
| 51 |
+
|
| 52 |
+
# test X's non-zero values
|
| 53 |
+
for i, j, val in (
|
| 54 |
+
(0, 2, 2.5),
|
| 55 |
+
(0, 10, -5.2),
|
| 56 |
+
(0, 15, 1.5),
|
| 57 |
+
(1, 5, 1.0),
|
| 58 |
+
(1, 12, -3),
|
| 59 |
+
(2, 20, 27),
|
| 60 |
+
):
|
| 61 |
+
assert X[i, j] == val
|
| 62 |
+
|
| 63 |
+
# tests X's zero values
|
| 64 |
+
assert X[0, 3] == 0
|
| 65 |
+
assert X[0, 5] == 0
|
| 66 |
+
assert X[1, 8] == 0
|
| 67 |
+
assert X[1, 16] == 0
|
| 68 |
+
assert X[2, 18] == 0
|
| 69 |
+
|
| 70 |
+
# test can change X's values
|
| 71 |
+
X[0, 2] *= 2
|
| 72 |
+
assert X[0, 2] == 5
|
| 73 |
+
|
| 74 |
+
# test y
|
| 75 |
+
assert_array_equal(y, [1, 2, 3, 4, 1, 2])
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def test_load_svmlight_file_fd():
|
| 79 |
+
# test loading from file descriptor
|
| 80 |
+
|
| 81 |
+
# GH20081: testing equality between path-based and
|
| 82 |
+
# fd-based load_svmlight_file
|
| 83 |
+
|
| 84 |
+
data_path = resources.files(TEST_DATA_MODULE) / datafile
|
| 85 |
+
data_path = str(data_path)
|
| 86 |
+
X1, y1 = load_svmlight_file(data_path)
|
| 87 |
+
|
| 88 |
+
fd = os.open(data_path, os.O_RDONLY)
|
| 89 |
+
try:
|
| 90 |
+
X2, y2 = load_svmlight_file(fd)
|
| 91 |
+
assert_array_almost_equal(X1.data, X2.data)
|
| 92 |
+
assert_array_almost_equal(y1, y2)
|
| 93 |
+
finally:
|
| 94 |
+
os.close(fd)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def test_load_svmlight_pathlib():
|
| 98 |
+
# test loading from file descriptor
|
| 99 |
+
data_path = _svmlight_local_test_file_path(datafile)
|
| 100 |
+
X1, y1 = load_svmlight_file(str(data_path))
|
| 101 |
+
X2, y2 = load_svmlight_file(data_path)
|
| 102 |
+
|
| 103 |
+
assert_allclose(X1.data, X2.data)
|
| 104 |
+
assert_allclose(y1, y2)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def test_load_svmlight_file_multilabel():
|
| 108 |
+
X, y = _load_svmlight_local_test_file(multifile, multilabel=True)
|
| 109 |
+
assert y == [(0, 1), (2,), (), (1, 2)]
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def test_load_svmlight_files():
|
| 113 |
+
data_path = _svmlight_local_test_file_path(datafile)
|
| 114 |
+
X_train, y_train, X_test, y_test = load_svmlight_files(
|
| 115 |
+
[str(data_path)] * 2, dtype=np.float32
|
| 116 |
+
)
|
| 117 |
+
assert_array_equal(X_train.toarray(), X_test.toarray())
|
| 118 |
+
assert_array_almost_equal(y_train, y_test)
|
| 119 |
+
assert X_train.dtype == np.float32
|
| 120 |
+
assert X_test.dtype == np.float32
|
| 121 |
+
|
| 122 |
+
X1, y1, X2, y2, X3, y3 = load_svmlight_files([str(data_path)] * 3, dtype=np.float64)
|
| 123 |
+
assert X1.dtype == X2.dtype
|
| 124 |
+
assert X2.dtype == X3.dtype
|
| 125 |
+
assert X3.dtype == np.float64
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def test_load_svmlight_file_n_features():
|
| 129 |
+
X, y = _load_svmlight_local_test_file(datafile, n_features=22)
|
| 130 |
+
|
| 131 |
+
# test X'shape
|
| 132 |
+
assert X.indptr.shape[0] == 7
|
| 133 |
+
assert X.shape[0] == 6
|
| 134 |
+
assert X.shape[1] == 22
|
| 135 |
+
|
| 136 |
+
# test X's non-zero values
|
| 137 |
+
for i, j, val in ((0, 2, 2.5), (0, 10, -5.2), (1, 5, 1.0), (1, 12, -3)):
|
| 138 |
+
assert X[i, j] == val
|
| 139 |
+
|
| 140 |
+
# 21 features in file
|
| 141 |
+
with pytest.raises(ValueError):
|
| 142 |
+
_load_svmlight_local_test_file(datafile, n_features=20)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def test_load_compressed():
|
| 146 |
+
X, y = _load_svmlight_local_test_file(datafile)
|
| 147 |
+
|
| 148 |
+
with NamedTemporaryFile(prefix="sklearn-test", suffix=".gz") as tmp:
|
| 149 |
+
tmp.close() # necessary under windows
|
| 150 |
+
with _svmlight_local_test_file_path(datafile).open("rb") as f:
|
| 151 |
+
with gzip.open(tmp.name, "wb") as fh_out:
|
| 152 |
+
shutil.copyfileobj(f, fh_out)
|
| 153 |
+
Xgz, ygz = load_svmlight_file(tmp.name)
|
| 154 |
+
# because we "close" it manually and write to it,
|
| 155 |
+
# we need to remove it manually.
|
| 156 |
+
os.remove(tmp.name)
|
| 157 |
+
assert_array_almost_equal(X.toarray(), Xgz.toarray())
|
| 158 |
+
assert_array_almost_equal(y, ygz)
|
| 159 |
+
|
| 160 |
+
with NamedTemporaryFile(prefix="sklearn-test", suffix=".bz2") as tmp:
|
| 161 |
+
tmp.close() # necessary under windows
|
| 162 |
+
with _svmlight_local_test_file_path(datafile).open("rb") as f:
|
| 163 |
+
with BZ2File(tmp.name, "wb") as fh_out:
|
| 164 |
+
shutil.copyfileobj(f, fh_out)
|
| 165 |
+
Xbz, ybz = load_svmlight_file(tmp.name)
|
| 166 |
+
# because we "close" it manually and write to it,
|
| 167 |
+
# we need to remove it manually.
|
| 168 |
+
os.remove(tmp.name)
|
| 169 |
+
assert_array_almost_equal(X.toarray(), Xbz.toarray())
|
| 170 |
+
assert_array_almost_equal(y, ybz)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def test_load_invalid_file():
|
| 174 |
+
with pytest.raises(ValueError):
|
| 175 |
+
_load_svmlight_local_test_file(invalidfile)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def test_load_invalid_order_file():
|
| 179 |
+
with pytest.raises(ValueError):
|
| 180 |
+
_load_svmlight_local_test_file(invalidfile2)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def test_load_zero_based():
|
| 184 |
+
f = BytesIO(b"-1 4:1.\n1 0:1\n")
|
| 185 |
+
with pytest.raises(ValueError):
|
| 186 |
+
load_svmlight_file(f, zero_based=False)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def test_load_zero_based_auto():
|
| 190 |
+
data1 = b"-1 1:1 2:2 3:3\n"
|
| 191 |
+
data2 = b"-1 0:0 1:1\n"
|
| 192 |
+
|
| 193 |
+
f1 = BytesIO(data1)
|
| 194 |
+
X, y = load_svmlight_file(f1, zero_based="auto")
|
| 195 |
+
assert X.shape == (1, 3)
|
| 196 |
+
|
| 197 |
+
f1 = BytesIO(data1)
|
| 198 |
+
f2 = BytesIO(data2)
|
| 199 |
+
X1, y1, X2, y2 = load_svmlight_files([f1, f2], zero_based="auto")
|
| 200 |
+
assert X1.shape == (1, 4)
|
| 201 |
+
assert X2.shape == (1, 4)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def test_load_with_qid():
|
| 205 |
+
# load svmfile with qid attribute
|
| 206 |
+
data = b"""
|
| 207 |
+
3 qid:1 1:0.53 2:0.12
|
| 208 |
+
2 qid:1 1:0.13 2:0.1
|
| 209 |
+
7 qid:2 1:0.87 2:0.12"""
|
| 210 |
+
X, y = load_svmlight_file(BytesIO(data), query_id=False)
|
| 211 |
+
assert_array_equal(y, [3, 2, 7])
|
| 212 |
+
assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]])
|
| 213 |
+
res1 = load_svmlight_files([BytesIO(data)], query_id=True)
|
| 214 |
+
res2 = load_svmlight_file(BytesIO(data), query_id=True)
|
| 215 |
+
for X, y, qid in (res1, res2):
|
| 216 |
+
assert_array_equal(y, [3, 2, 7])
|
| 217 |
+
assert_array_equal(qid, [1, 1, 2])
|
| 218 |
+
assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]])
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
@pytest.mark.skip(
|
| 222 |
+
"testing the overflow of 32 bit sparse indexing requires a large amount of memory"
|
| 223 |
+
)
|
| 224 |
+
def test_load_large_qid():
|
| 225 |
+
"""
|
| 226 |
+
load large libsvm / svmlight file with qid attribute. Tests 64-bit query ID
|
| 227 |
+
"""
|
| 228 |
+
data = b"\n".join(
|
| 229 |
+
(
|
| 230 |
+
"3 qid:{0} 1:0.53 2:0.12\n2 qid:{0} 1:0.13 2:0.1".format(i).encode()
|
| 231 |
+
for i in range(1, 40 * 1000 * 1000)
|
| 232 |
+
)
|
| 233 |
+
)
|
| 234 |
+
X, y, qid = load_svmlight_file(BytesIO(data), query_id=True)
|
| 235 |
+
assert_array_equal(y[-4:], [3, 2, 3, 2])
|
| 236 |
+
assert_array_equal(np.unique(qid), np.arange(1, 40 * 1000 * 1000))
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def test_load_invalid_file2():
|
| 240 |
+
with pytest.raises(ValueError):
|
| 241 |
+
data_path = _svmlight_local_test_file_path(datafile)
|
| 242 |
+
invalid_path = _svmlight_local_test_file_path(invalidfile)
|
| 243 |
+
load_svmlight_files([str(data_path), str(invalid_path), str(data_path)])
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def test_not_a_filename():
|
| 247 |
+
# in python 3 integers are valid file opening arguments (taken as unix
|
| 248 |
+
# file descriptors)
|
| 249 |
+
with pytest.raises(TypeError):
|
| 250 |
+
load_svmlight_file(0.42)
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def test_invalid_filename():
|
| 254 |
+
with pytest.raises(OSError):
|
| 255 |
+
load_svmlight_file("trou pic nic douille")
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
|
| 259 |
+
def test_dump(csr_container):
|
| 260 |
+
X_sparse, y_dense = _load_svmlight_local_test_file(datafile)
|
| 261 |
+
X_dense = X_sparse.toarray()
|
| 262 |
+
y_sparse = csr_container(np.atleast_2d(y_dense))
|
| 263 |
+
|
| 264 |
+
# slicing a csr_matrix can unsort its .indices, so test that we sort
|
| 265 |
+
# those correctly
|
| 266 |
+
X_sliced = X_sparse[np.arange(X_sparse.shape[0])]
|
| 267 |
+
y_sliced = y_sparse[np.arange(y_sparse.shape[0])]
|
| 268 |
+
|
| 269 |
+
for X in (X_sparse, X_dense, X_sliced):
|
| 270 |
+
for y in (y_sparse, y_dense, y_sliced):
|
| 271 |
+
for zero_based in (True, False):
|
| 272 |
+
for dtype in [np.float32, np.float64, np.int32, np.int64]:
|
| 273 |
+
f = BytesIO()
|
| 274 |
+
# we need to pass a comment to get the version info in;
|
| 275 |
+
# LibSVM doesn't grok comments so they're not put in by
|
| 276 |
+
# default anymore.
|
| 277 |
+
|
| 278 |
+
if sp.issparse(y) and y.shape[0] == 1:
|
| 279 |
+
# make sure y's shape is: (n_samples, n_labels)
|
| 280 |
+
# when it is sparse
|
| 281 |
+
y = y.T
|
| 282 |
+
|
| 283 |
+
# Note: with dtype=np.int32 we are performing unsafe casts,
|
| 284 |
+
# where X.astype(dtype) overflows. The result is
|
| 285 |
+
# then platform dependent and X_dense.astype(dtype) may be
|
| 286 |
+
# different from X_sparse.astype(dtype).asarray().
|
| 287 |
+
X_input = X.astype(dtype)
|
| 288 |
+
|
| 289 |
+
dump_svmlight_file(
|
| 290 |
+
X_input, y, f, comment="test", zero_based=zero_based
|
| 291 |
+
)
|
| 292 |
+
f.seek(0)
|
| 293 |
+
|
| 294 |
+
comment = f.readline()
|
| 295 |
+
comment = str(comment, "utf-8")
|
| 296 |
+
|
| 297 |
+
assert "scikit-learn %s" % sklearn.__version__ in comment
|
| 298 |
+
|
| 299 |
+
comment = f.readline()
|
| 300 |
+
comment = str(comment, "utf-8")
|
| 301 |
+
|
| 302 |
+
assert ["one", "zero"][zero_based] + "-based" in comment
|
| 303 |
+
|
| 304 |
+
X2, y2 = load_svmlight_file(f, dtype=dtype, zero_based=zero_based)
|
| 305 |
+
assert X2.dtype == dtype
|
| 306 |
+
assert_array_equal(X2.sorted_indices().indices, X2.indices)
|
| 307 |
+
|
| 308 |
+
X2_dense = X2.toarray()
|
| 309 |
+
if sp.issparse(X_input):
|
| 310 |
+
X_input_dense = X_input.toarray()
|
| 311 |
+
else:
|
| 312 |
+
X_input_dense = X_input
|
| 313 |
+
|
| 314 |
+
if dtype == np.float32:
|
| 315 |
+
# allow a rounding error at the last decimal place
|
| 316 |
+
assert_array_almost_equal(X_input_dense, X2_dense, 4)
|
| 317 |
+
assert_array_almost_equal(
|
| 318 |
+
y_dense.astype(dtype, copy=False), y2, 4
|
| 319 |
+
)
|
| 320 |
+
else:
|
| 321 |
+
# allow a rounding error at the last decimal place
|
| 322 |
+
assert_array_almost_equal(X_input_dense, X2_dense, 15)
|
| 323 |
+
assert_array_almost_equal(
|
| 324 |
+
y_dense.astype(dtype, copy=False), y2, 15
|
| 325 |
+
)
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
|
| 329 |
+
def test_dump_multilabel(csr_container):
|
| 330 |
+
X = [[1, 0, 3, 0, 5], [0, 0, 0, 0, 0], [0, 5, 0, 1, 0]]
|
| 331 |
+
y_dense = [[0, 1, 0], [1, 0, 1], [1, 1, 0]]
|
| 332 |
+
y_sparse = csr_container(y_dense)
|
| 333 |
+
for y in [y_dense, y_sparse]:
|
| 334 |
+
f = BytesIO()
|
| 335 |
+
dump_svmlight_file(X, y, f, multilabel=True)
|
| 336 |
+
f.seek(0)
|
| 337 |
+
# make sure it dumps multilabel correctly
|
| 338 |
+
assert f.readline() == b"1 0:1 2:3 4:5\n"
|
| 339 |
+
assert f.readline() == b"0,2 \n"
|
| 340 |
+
assert f.readline() == b"0,1 1:5 3:1\n"
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def test_dump_concise():
|
| 344 |
+
one = 1
|
| 345 |
+
two = 2.1
|
| 346 |
+
three = 3.01
|
| 347 |
+
exact = 1.000000000000001
|
| 348 |
+
# loses the last decimal place
|
| 349 |
+
almost = 1.0000000000000001
|
| 350 |
+
X = [
|
| 351 |
+
[one, two, three, exact, almost],
|
| 352 |
+
[1e9, 2e18, 3e27, 0, 0],
|
| 353 |
+
[0, 0, 0, 0, 0],
|
| 354 |
+
[0, 0, 0, 0, 0],
|
| 355 |
+
[0, 0, 0, 0, 0],
|
| 356 |
+
]
|
| 357 |
+
y = [one, two, three, exact, almost]
|
| 358 |
+
f = BytesIO()
|
| 359 |
+
dump_svmlight_file(X, y, f)
|
| 360 |
+
f.seek(0)
|
| 361 |
+
# make sure it's using the most concise format possible
|
| 362 |
+
assert f.readline() == b"1 0:1 1:2.1 2:3.01 3:1.000000000000001 4:1\n"
|
| 363 |
+
assert f.readline() == b"2.1 0:1000000000 1:2e+18 2:3e+27\n"
|
| 364 |
+
assert f.readline() == b"3.01 \n"
|
| 365 |
+
assert f.readline() == b"1.000000000000001 \n"
|
| 366 |
+
assert f.readline() == b"1 \n"
|
| 367 |
+
f.seek(0)
|
| 368 |
+
# make sure it's correct too :)
|
| 369 |
+
X2, y2 = load_svmlight_file(f)
|
| 370 |
+
assert_array_almost_equal(X, X2.toarray())
|
| 371 |
+
assert_array_almost_equal(y, y2)
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def test_dump_comment():
|
| 375 |
+
X, y = _load_svmlight_local_test_file(datafile)
|
| 376 |
+
X = X.toarray()
|
| 377 |
+
|
| 378 |
+
f = BytesIO()
|
| 379 |
+
ascii_comment = "This is a comment\nspanning multiple lines."
|
| 380 |
+
dump_svmlight_file(X, y, f, comment=ascii_comment, zero_based=False)
|
| 381 |
+
f.seek(0)
|
| 382 |
+
|
| 383 |
+
X2, y2 = load_svmlight_file(f, zero_based=False)
|
| 384 |
+
assert_array_almost_equal(X, X2.toarray())
|
| 385 |
+
assert_array_almost_equal(y, y2)
|
| 386 |
+
|
| 387 |
+
# XXX we have to update this to support Python 3.x
|
| 388 |
+
utf8_comment = b"It is true that\n\xc2\xbd\xc2\xb2 = \xc2\xbc"
|
| 389 |
+
f = BytesIO()
|
| 390 |
+
with pytest.raises(UnicodeDecodeError):
|
| 391 |
+
dump_svmlight_file(X, y, f, comment=utf8_comment)
|
| 392 |
+
|
| 393 |
+
unicode_comment = utf8_comment.decode("utf-8")
|
| 394 |
+
f = BytesIO()
|
| 395 |
+
dump_svmlight_file(X, y, f, comment=unicode_comment, zero_based=False)
|
| 396 |
+
f.seek(0)
|
| 397 |
+
|
| 398 |
+
X2, y2 = load_svmlight_file(f, zero_based=False)
|
| 399 |
+
assert_array_almost_equal(X, X2.toarray())
|
| 400 |
+
assert_array_almost_equal(y, y2)
|
| 401 |
+
|
| 402 |
+
f = BytesIO()
|
| 403 |
+
with pytest.raises(ValueError):
|
| 404 |
+
dump_svmlight_file(X, y, f, comment="I've got a \0.")
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def test_dump_invalid():
|
| 408 |
+
X, y = _load_svmlight_local_test_file(datafile)
|
| 409 |
+
|
| 410 |
+
f = BytesIO()
|
| 411 |
+
y2d = [y]
|
| 412 |
+
with pytest.raises(ValueError):
|
| 413 |
+
dump_svmlight_file(X, y2d, f)
|
| 414 |
+
|
| 415 |
+
f = BytesIO()
|
| 416 |
+
with pytest.raises(ValueError):
|
| 417 |
+
dump_svmlight_file(X, y[:-1], f)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def test_dump_query_id():
|
| 421 |
+
# test dumping a file with query_id
|
| 422 |
+
X, y = _load_svmlight_local_test_file(datafile)
|
| 423 |
+
X = X.toarray()
|
| 424 |
+
query_id = np.arange(X.shape[0]) // 2
|
| 425 |
+
f = BytesIO()
|
| 426 |
+
dump_svmlight_file(X, y, f, query_id=query_id, zero_based=True)
|
| 427 |
+
|
| 428 |
+
f.seek(0)
|
| 429 |
+
X1, y1, query_id1 = load_svmlight_file(f, query_id=True, zero_based=True)
|
| 430 |
+
assert_array_almost_equal(X, X1.toarray())
|
| 431 |
+
assert_array_almost_equal(y, y1)
|
| 432 |
+
assert_array_almost_equal(query_id, query_id1)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def test_load_with_long_qid():
|
| 436 |
+
# load svmfile with longint qid attribute
|
| 437 |
+
data = b"""
|
| 438 |
+
1 qid:0 0:1 1:2 2:3
|
| 439 |
+
0 qid:72048431380967004 0:1440446648 1:72048431380967004 2:236784985
|
| 440 |
+
0 qid:-9223372036854775807 0:1440446648 1:72048431380967004 2:236784985
|
| 441 |
+
3 qid:9223372036854775807 0:1440446648 1:72048431380967004 2:236784985"""
|
| 442 |
+
X, y, qid = load_svmlight_file(BytesIO(data), query_id=True)
|
| 443 |
+
|
| 444 |
+
true_X = [
|
| 445 |
+
[1, 2, 3],
|
| 446 |
+
[1440446648, 72048431380967004, 236784985],
|
| 447 |
+
[1440446648, 72048431380967004, 236784985],
|
| 448 |
+
[1440446648, 72048431380967004, 236784985],
|
| 449 |
+
]
|
| 450 |
+
|
| 451 |
+
true_y = [1, 0, 0, 3]
|
| 452 |
+
trueQID = [0, 72048431380967004, -9223372036854775807, 9223372036854775807]
|
| 453 |
+
assert_array_equal(y, true_y)
|
| 454 |
+
assert_array_equal(X.toarray(), true_X)
|
| 455 |
+
assert_array_equal(qid, trueQID)
|
| 456 |
+
|
| 457 |
+
f = BytesIO()
|
| 458 |
+
dump_svmlight_file(X, y, f, query_id=qid, zero_based=True)
|
| 459 |
+
f.seek(0)
|
| 460 |
+
X, y, qid = load_svmlight_file(f, query_id=True, zero_based=True)
|
| 461 |
+
assert_array_equal(y, true_y)
|
| 462 |
+
assert_array_equal(X.toarray(), true_X)
|
| 463 |
+
assert_array_equal(qid, trueQID)
|
| 464 |
+
|
| 465 |
+
f.seek(0)
|
| 466 |
+
X, y = load_svmlight_file(f, query_id=False, zero_based=True)
|
| 467 |
+
assert_array_equal(y, true_y)
|
| 468 |
+
assert_array_equal(X.toarray(), true_X)
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
|
| 472 |
+
def test_load_zeros(csr_container):
|
| 473 |
+
f = BytesIO()
|
| 474 |
+
true_X = csr_container(np.zeros(shape=(3, 4)))
|
| 475 |
+
true_y = np.array([0, 1, 0])
|
| 476 |
+
dump_svmlight_file(true_X, true_y, f)
|
| 477 |
+
|
| 478 |
+
for zero_based in ["auto", True, False]:
|
| 479 |
+
f.seek(0)
|
| 480 |
+
X, y = load_svmlight_file(f, n_features=4, zero_based=zero_based)
|
| 481 |
+
assert_array_almost_equal(y, true_y)
|
| 482 |
+
assert_array_almost_equal(X.toarray(), true_X.toarray())
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
@pytest.mark.parametrize("sparsity", [0, 0.1, 0.5, 0.99, 1])
|
| 486 |
+
@pytest.mark.parametrize("n_samples", [13, 101])
|
| 487 |
+
@pytest.mark.parametrize("n_features", [2, 7, 41])
|
| 488 |
+
@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
|
| 489 |
+
def test_load_with_offsets(sparsity, n_samples, n_features, csr_container):
|
| 490 |
+
rng = np.random.RandomState(0)
|
| 491 |
+
X = rng.uniform(low=0.0, high=1.0, size=(n_samples, n_features))
|
| 492 |
+
if sparsity:
|
| 493 |
+
X[X < sparsity] = 0.0
|
| 494 |
+
X = csr_container(X)
|
| 495 |
+
y = rng.randint(low=0, high=2, size=n_samples)
|
| 496 |
+
|
| 497 |
+
f = BytesIO()
|
| 498 |
+
dump_svmlight_file(X, y, f)
|
| 499 |
+
f.seek(0)
|
| 500 |
+
|
| 501 |
+
size = len(f.getvalue())
|
| 502 |
+
|
| 503 |
+
# put some marks that are likely to happen anywhere in a row
|
| 504 |
+
mark_0 = 0
|
| 505 |
+
mark_1 = size // 3
|
| 506 |
+
length_0 = mark_1 - mark_0
|
| 507 |
+
mark_2 = 4 * size // 5
|
| 508 |
+
length_1 = mark_2 - mark_1
|
| 509 |
+
|
| 510 |
+
# load the original sparse matrix into 3 independent CSR matrices
|
| 511 |
+
X_0, y_0 = load_svmlight_file(
|
| 512 |
+
f, n_features=n_features, offset=mark_0, length=length_0
|
| 513 |
+
)
|
| 514 |
+
X_1, y_1 = load_svmlight_file(
|
| 515 |
+
f, n_features=n_features, offset=mark_1, length=length_1
|
| 516 |
+
)
|
| 517 |
+
X_2, y_2 = load_svmlight_file(f, n_features=n_features, offset=mark_2)
|
| 518 |
+
|
| 519 |
+
y_concat = np.concatenate([y_0, y_1, y_2])
|
| 520 |
+
X_concat = sp.vstack([X_0, X_1, X_2])
|
| 521 |
+
assert_array_almost_equal(y, y_concat)
|
| 522 |
+
assert_array_almost_equal(X.toarray(), X_concat.toarray())
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
|
| 526 |
+
def test_load_offset_exhaustive_splits(csr_container):
|
| 527 |
+
rng = np.random.RandomState(0)
|
| 528 |
+
X = np.array(
|
| 529 |
+
[
|
| 530 |
+
[0, 0, 0, 0, 0, 0],
|
| 531 |
+
[1, 2, 3, 4, 0, 6],
|
| 532 |
+
[1, 2, 3, 4, 0, 6],
|
| 533 |
+
[0, 0, 0, 0, 0, 0],
|
| 534 |
+
[1, 0, 3, 0, 0, 0],
|
| 535 |
+
[0, 0, 0, 0, 0, 1],
|
| 536 |
+
[1, 0, 0, 0, 0, 0],
|
| 537 |
+
]
|
| 538 |
+
)
|
| 539 |
+
X = csr_container(X)
|
| 540 |
+
n_samples, n_features = X.shape
|
| 541 |
+
y = rng.randint(low=0, high=2, size=n_samples)
|
| 542 |
+
query_id = np.arange(n_samples) // 2
|
| 543 |
+
|
| 544 |
+
f = BytesIO()
|
| 545 |
+
dump_svmlight_file(X, y, f, query_id=query_id)
|
| 546 |
+
f.seek(0)
|
| 547 |
+
|
| 548 |
+
size = len(f.getvalue())
|
| 549 |
+
|
| 550 |
+
# load the same data in 2 parts with all the possible byte offsets to
|
| 551 |
+
# locate the split so has to test for particular boundary cases
|
| 552 |
+
for mark in range(size):
|
| 553 |
+
f.seek(0)
|
| 554 |
+
X_0, y_0, q_0 = load_svmlight_file(
|
| 555 |
+
f, n_features=n_features, query_id=True, offset=0, length=mark
|
| 556 |
+
)
|
| 557 |
+
X_1, y_1, q_1 = load_svmlight_file(
|
| 558 |
+
f, n_features=n_features, query_id=True, offset=mark, length=-1
|
| 559 |
+
)
|
| 560 |
+
q_concat = np.concatenate([q_0, q_1])
|
| 561 |
+
y_concat = np.concatenate([y_0, y_1])
|
| 562 |
+
X_concat = sp.vstack([X_0, X_1])
|
| 563 |
+
assert_array_almost_equal(y, y_concat)
|
| 564 |
+
assert_array_equal(query_id, q_concat)
|
| 565 |
+
assert_array_almost_equal(X.toarray(), X_concat.toarray())
|
| 566 |
+
|
| 567 |
+
|
| 568 |
+
def test_load_with_offsets_error():
|
| 569 |
+
with pytest.raises(ValueError, match="n_features is required"):
|
| 570 |
+
_load_svmlight_local_test_file(datafile, offset=3, length=3)
|
| 571 |
+
|
| 572 |
+
|
| 573 |
+
@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
|
| 574 |
+
def test_multilabel_y_explicit_zeros(tmp_path, csr_container):
|
| 575 |
+
"""
|
| 576 |
+
Ensure that if y contains explicit zeros (i.e. elements of y.data equal to
|
| 577 |
+
0) then those explicit zeros are not encoded.
|
| 578 |
+
"""
|
| 579 |
+
save_path = str(tmp_path / "svm_explicit_zero")
|
| 580 |
+
rng = np.random.RandomState(42)
|
| 581 |
+
X = rng.randn(3, 5).astype(np.float64)
|
| 582 |
+
indptr = np.array([0, 2, 3, 6])
|
| 583 |
+
indices = np.array([0, 2, 2, 0, 1, 2])
|
| 584 |
+
# The first and last element are explicit zeros.
|
| 585 |
+
data = np.array([0, 1, 1, 1, 1, 0])
|
| 586 |
+
y = csr_container((data, indices, indptr), shape=(3, 3))
|
| 587 |
+
# y as a dense array would look like
|
| 588 |
+
# [[0, 0, 1],
|
| 589 |
+
# [0, 0, 1],
|
| 590 |
+
# [1, 1, 0]]
|
| 591 |
+
|
| 592 |
+
dump_svmlight_file(X, y, save_path, multilabel=True)
|
| 593 |
+
|
| 594 |
+
_, y_load = load_svmlight_file(save_path, multilabel=True)
|
| 595 |
+
y_true = [(2.0,), (2.0,), (0.0, 1.0)]
|
| 596 |
+
assert y_load == y_true
|
| 597 |
+
|
| 598 |
+
|
| 599 |
+
def test_dump_read_only(tmp_path):
|
| 600 |
+
"""Ensure that there is no ValueError when dumping a read-only `X`.
|
| 601 |
+
|
| 602 |
+
Non-regression test for:
|
| 603 |
+
https://github.com/scikit-learn/scikit-learn/issues/28026
|
| 604 |
+
"""
|
| 605 |
+
rng = np.random.RandomState(42)
|
| 606 |
+
X = rng.randn(5, 2)
|
| 607 |
+
y = rng.randn(5)
|
| 608 |
+
|
| 609 |
+
# Convert to memmap-backed which are read-only
|
| 610 |
+
X, y = create_memmap_backed_data([X, y])
|
| 611 |
+
|
| 612 |
+
save_path = str(tmp_path / "svm_read_only")
|
| 613 |
+
dump_svmlight_file(X, y, save_path)
|