giancap61 commited on
Commit
cc1decc
·
verified ·
1 Parent(s): 9d00a3b

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. TTS/Kokoro/.venv/Lib/site-packages/spacy_legacy/tests/pipeline/__pycache__/__init__.cpython-312.pyc +0 -0
  2. TTS/Kokoro/.venv/Lib/site-packages/spacy_legacy/tests/pipeline/__pycache__/test_tok2vec.cpython-312.pyc +0 -0
  3. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/__init__.cpython-312.pyc +0 -0
  4. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/chain.cpython-312.pyc +0 -0
  5. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/clearml.cpython-312.pyc +0 -0
  6. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/cupy.cpython-312.pyc +0 -0
  7. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/lookup.cpython-312.pyc +0 -0
  8. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/mlflow.cpython-312.pyc +0 -0
  9. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/pytorch.cpython-312.pyc +0 -0
  10. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/util.cpython-312.pyc +0 -0
  11. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/wandb.cpython-312.pyc +0 -0
  12. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__init__.py +0 -0
  13. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/__init__.cpython-312.pyc +0 -0
  14. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_chain.cpython-312.pyc +0 -0
  15. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_cupy.cpython-312.pyc +0 -0
  16. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_lookup.cpython-312.pyc +0 -0
  17. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_registry.cpython-312.pyc +0 -0
  18. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/util.cpython-312.pyc +0 -0
  19. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_chain.py +47 -0
  20. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_cupy.py +44 -0
  21. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_lookup.py +91 -0
  22. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_registry.py +24 -0
  23. TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/util.py +20 -0
  24. TTS/Kokoro/.venv/Lib/site-packages/srsly-2.5.2.dist-info/licenses/LICENSE +21 -0
  25. TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/__init__.cpython-312.pyc +0 -0
  26. TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_json_api.cpython-312.pyc +0 -0
  27. TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_msgpack_api.cpython-312.pyc +0 -0
  28. TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_pickle_api.cpython-312.pyc +0 -0
  29. TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_yaml_api.cpython-312.pyc +0 -0
  30. TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/about.cpython-312.pyc +0 -0
  31. TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/util.cpython-312.pyc +0 -0
  32. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__init__.py +8 -0
  33. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/__init__.cpython-312.pyc +0 -0
  34. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/cloudpickle.cpython-312.pyc +0 -0
  35. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/cloudpickle_fast.cpython-312.pyc +0 -0
  36. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/compat.cpython-312.pyc +0 -0
  37. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/cloudpickle.py +948 -0
  38. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/cloudpickle_fast.py +844 -0
  39. TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/compat.py +18 -0
  40. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__init__.py +93 -0
  41. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/__init__.cpython-312.pyc +0 -0
  42. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/_msgpack_numpy.cpython-312.pyc +0 -0
  43. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/_version.cpython-312.pyc +0 -0
  44. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/exceptions.cpython-312.pyc +0 -0
  45. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/ext.cpython-312.pyc +0 -0
  46. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/fallback.cpython-312.pyc +0 -0
  47. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/util.cpython-312.pyc +0 -0
  48. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/_epoch.cp312-win_amd64.pyd +0 -0
  49. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/_epoch.cpp +0 -0
  50. TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/_epoch.pyx +7 -0
TTS/Kokoro/.venv/Lib/site-packages/spacy_legacy/tests/pipeline/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (216 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_legacy/tests/pipeline/__pycache__/test_tok2vec.cpython-312.pyc ADDED
Binary file (2.3 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (202 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/chain.cpython-312.pyc ADDED
Binary file (2.44 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/clearml.cpython-312.pyc ADDED
Binary file (12.8 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/cupy.cpython-312.pyc ADDED
Binary file (3.58 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/lookup.cpython-312.pyc ADDED
Binary file (2.03 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/mlflow.cpython-312.pyc ADDED
Binary file (10.8 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/pytorch.cpython-312.pyc ADDED
Binary file (3.76 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/util.cpython-312.pyc ADDED
Binary file (4.45 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/__pycache__/wandb.cpython-312.pyc ADDED
Binary file (19.2 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__init__.py ADDED
File without changes
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (208 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_chain.cpython-312.pyc ADDED
Binary file (1.32 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_cupy.cpython-312.pyc ADDED
Binary file (2.04 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_lookup.cpython-312.pyc ADDED
Binary file (2.47 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/test_registry.cpython-312.pyc ADDED
Binary file (1.1 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/__pycache__/util.cpython-312.pyc ADDED
Binary file (1.05 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_chain.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pytest
2
+
3
+ from .util import load_logger_from_config
4
+
5
+
6
+ valid_config_string = """
7
+ [nlp]
8
+ lang = "en"
9
+ pipeline = ["tok2vec"]
10
+
11
+ [components]
12
+
13
+ [components.tok2vec]
14
+ factory = "tok2vec"
15
+
16
+ [training]
17
+
18
+ [training.logger]
19
+ @loggers = "spacy.ChainLogger.v1"
20
+ logger1 = {"@loggers": "spacy.ConsoleLogger.v1", "progress_bar": "true"}
21
+ logger9 = {"@loggers": "spacy.LookupLogger.v1", "patterns": ["test"]}
22
+ """
23
+
24
+ invalid_config_string = """
25
+ [nlp]
26
+ lang = "en"
27
+ pipeline = ["tok2vec"]
28
+
29
+ [components]
30
+
31
+ [components.tok2vec]
32
+ factory = "tok2vec"
33
+
34
+ [training]
35
+
36
+ [training.logger]
37
+ @loggers = "spacy.ChainLogger.v1"
38
+ """
39
+
40
+
41
+ def test_load_from_config():
42
+ valid_logger, nlp = load_logger_from_config(valid_config_string)
43
+ valid_logger(nlp)
44
+
45
+ with pytest.raises(ValueError, match="No loggers"):
46
+ invalid_logger, nlp = load_logger_from_config(invalid_config_string)
47
+ invalid_logger(nlp)
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_cupy.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pytest
2
+ from thinc.util import has_cupy_gpu, has_torch, has_tensorflow, require_gpu
3
+ from thinc.backends import use_pytorch_for_gpu_memory, use_tensorflow_for_gpu_memory
4
+
5
+ from spacy_loggers.cupy import cupy_logger_v1
6
+
7
+
8
+ @pytest.fixture
9
+ def logger():
10
+ setup_logger = cupy_logger_v1()
11
+ step, _ = setup_logger(None)
12
+ return step
13
+
14
+
15
+ @pytest.mark.skipif(not has_cupy_gpu, reason="CuPy support required")
16
+ def test_cupy_allocator_source_default(logger):
17
+ # Needs to be executed first as modifications to the memory pool
18
+ # persist for the entire session.
19
+ require_gpu()
20
+ info = {}
21
+ logger(info)
22
+ assert info["cupy.pool.source"] == "default"
23
+
24
+
25
+ @pytest.mark.skipif(
26
+ not has_cupy_gpu or not has_torch, reason="CuPy/PyTorch support required"
27
+ )
28
+ def test_cupy_allocator_source_torch(logger):
29
+ require_gpu()
30
+ use_pytorch_for_gpu_memory()
31
+ info = {}
32
+ logger(info)
33
+ assert info["cupy.pool.source"] == "pytorch"
34
+
35
+
36
+ @pytest.mark.skipif(
37
+ not has_cupy_gpu or not has_tensorflow, reason="CuPy/TensorFlow support required"
38
+ )
39
+ def test_cupy_allocator_source_tf(logger):
40
+ require_gpu()
41
+ use_tensorflow_for_gpu_memory()
42
+ info = {}
43
+ logger(info)
44
+ assert info["cupy.pool.source"] == "tensorflow"
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_lookup.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pytest
2
+ import re
3
+
4
+ from spacy_loggers.util import matcher_for_regex_patterns
5
+ from .util import load_logger_from_config
6
+
7
+
8
+ valid_config_string = """
9
+ [nlp]
10
+ lang = "en"
11
+ pipeline = ["tok2vec"]
12
+
13
+ [components]
14
+
15
+ [components.tok2vec]
16
+ factory = "tok2vec"
17
+
18
+ [training]
19
+
20
+ [training.logger]
21
+ @loggers = "spacy.LookupLogger.v1"
22
+ patterns = ["^[pP]ytorch", "zeppelin" ]
23
+ """
24
+
25
+ invalid_config_string_empty = """
26
+ [nlp]
27
+ lang = "en"
28
+ pipeline = ["tok2vec"]
29
+
30
+ [components]
31
+
32
+ [components.tok2vec]
33
+ factory = "tok2vec"
34
+
35
+ [training]
36
+
37
+ [training.logger]
38
+ @loggers = "spacy.LookupLogger.v1"
39
+ patterns = []
40
+ """
41
+
42
+ invalid_config_string_incorrect_pattern = """
43
+ [nlp]
44
+ lang = "en"
45
+ pipeline = ["tok2vec"]
46
+
47
+ [components]
48
+
49
+ [components.tok2vec]
50
+ factory = "tok2vec"
51
+
52
+ [training]
53
+
54
+ [training.logger]
55
+ @loggers = "spacy.LookupLogger.v1"
56
+ patterns = [")"]
57
+ """
58
+
59
+
60
+ def test_load_from_config():
61
+ valid_logger, nlp = load_logger_from_config(valid_config_string)
62
+ valid_logger(nlp)
63
+
64
+ with pytest.raises(ValueError, match="at least one pattern"):
65
+ invalid_logger, nlp = load_logger_from_config(invalid_config_string_empty)
66
+ invalid_logger(nlp)
67
+
68
+ with pytest.raises(ValueError, match="couldn't be compiled"):
69
+ invalid_logger, nlp = load_logger_from_config(
70
+ invalid_config_string_incorrect_pattern
71
+ )
72
+ invalid_logger(nlp)
73
+
74
+
75
+ def test_custom_stats_matcher():
76
+ patterns = ["^[pP]ytorch", "zeppelin$"]
77
+ inputs = [
78
+ "no match",
79
+ "torch",
80
+ "pYtorch",
81
+ "pytorch",
82
+ "Pytorch 1.13",
83
+ "led zeppelin",
84
+ ]
85
+ outputs = [False, False, False, True, True, True]
86
+
87
+ matcher = matcher_for_regex_patterns(patterns)
88
+ assert [matcher(x) for x in inputs] == outputs
89
+
90
+ with pytest.raises(ValueError, match="couldn't be compiled"):
91
+ matcher_for_regex_patterns([")"])
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/test_registry.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pytest
2
+ from spacy import registry
3
+
4
+ FUNCTIONS = [
5
+ ("loggers", "spacy.WandbLogger.v1"),
6
+ ("loggers", "spacy.WandbLogger.v2"),
7
+ ("loggers", "spacy.WandbLogger.v3"),
8
+ ("loggers", "spacy.WandbLogger.v4"),
9
+ ("loggers", "spacy.WandbLogger.v5"),
10
+ ("loggers", "spacy.MLflowLogger.v1"),
11
+ ("loggers", "spacy.MLflowLogger.v2"),
12
+ ("loggers", "spacy.ClearMLLogger.v1"),
13
+ ("loggers", "spacy.ClearMLLogger.v2"),
14
+ ("loggers", "spacy.ChainLogger.v1"),
15
+ ("loggers", "spacy.PyTorchLogger.v1"),
16
+ ("loggers", "spacy.LookupLogger.v1"),
17
+ ("loggers", "spacy.CupyLogger.v1"),
18
+ ]
19
+
20
+
21
+ @pytest.mark.parametrize("reg_name,func_name", FUNCTIONS)
22
+ def test_registry(reg_name, func_name):
23
+ assert registry.has(reg_name, func_name)
24
+ assert registry.get(reg_name, func_name)
TTS/Kokoro/.venv/Lib/site-packages/spacy_loggers/tests/util.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Tuple
2
+
3
+ from spacy import Language
4
+ from spacy.util import (
5
+ load_model_from_config,
6
+ registry,
7
+ load_config_from_str,
8
+ )
9
+ from spacy.schemas import ConfigSchemaTraining
10
+
11
+ from spacy_loggers.util import LoggerT
12
+
13
+
14
+ def load_logger_from_config(config_str: str) -> Tuple[LoggerT, Language]:
15
+ config = load_config_from_str(config_str)
16
+ nlp = load_model_from_config(config, auto_fill=True)
17
+ T = registry.resolve(
18
+ nlp.config.interpolate()["training"], schema=ConfigSchemaTraining
19
+ )
20
+ return T["logger"], nlp
TTS/Kokoro/.venv/Lib/site-packages/srsly-2.5.2.dist-info/licenses/LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ The MIT License (MIT)
2
+
3
+ Copyright (C) 2018 ExplosionAI UG (haftungsbeschränkt)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (986 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_json_api.cpython-312.pyc ADDED
Binary file (9.88 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_msgpack_api.cpython-312.pyc ADDED
Binary file (2.76 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_pickle_api.cpython-312.pyc ADDED
Binary file (1.12 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/_yaml_api.cpython-312.pyc ADDED
Binary file (5.2 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/about.cpython-312.pyc ADDED
Binary file (217 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/__pycache__/util.cpython-312.pyc ADDED
Binary file (1.48 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__init__.py ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ from .cloudpickle import * # noqa
2
+ from .cloudpickle_fast import CloudPickler, dumps, dump # noqa
3
+
4
+ # Conform to the convention used by python serialization libraries, which
5
+ # expose their Pickler subclass at top-level under the "Pickler" name.
6
+ Pickler = CloudPickler
7
+
8
+ __version__ = '2.2.0'
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (376 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/cloudpickle.cpython-312.pyc ADDED
Binary file (35 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/cloudpickle_fast.cpython-312.pyc ADDED
Binary file (32.9 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/__pycache__/compat.cpython-312.pyc ADDED
Binary file (501 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/cloudpickle.py ADDED
@@ -0,0 +1,948 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ This class is defined to override standard pickle functionality
3
+
4
+ The goals of it follow:
5
+ -Serialize lambdas and nested functions to compiled byte code
6
+ -Deal with main module correctly
7
+ -Deal with other non-serializable objects
8
+
9
+ It does not include an unpickler, as standard python unpickling suffices.
10
+
11
+ This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
12
+ <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
13
+
14
+ Copyright (c) 2012, Regents of the University of California.
15
+ Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
16
+ All rights reserved.
17
+
18
+ Redistribution and use in source and binary forms, with or without
19
+ modification, are permitted provided that the following conditions
20
+ are met:
21
+ * Redistributions of source code must retain the above copyright
22
+ notice, this list of conditions and the following disclaimer.
23
+ * Redistributions in binary form must reproduce the above copyright
24
+ notice, this list of conditions and the following disclaimer in the
25
+ documentation and/or other materials provided with the distribution.
26
+ * Neither the name of the University of California, Berkeley nor the
27
+ names of its contributors may be used to endorse or promote
28
+ products derived from this software without specific prior written
29
+ permission.
30
+
31
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35
+ HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
37
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
38
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
39
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
40
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
41
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42
+ """
43
+
44
+ import builtins
45
+ import dis
46
+ import opcode
47
+ import platform
48
+ import sys
49
+ import types
50
+ import weakref
51
+ import uuid
52
+ import threading
53
+ import typing
54
+ import warnings
55
+
56
+ from .compat import pickle
57
+ from collections import OrderedDict
58
+ from typing import ClassVar, Generic, Union, Tuple, Callable
59
+ from pickle import _getattribute
60
+ from importlib._bootstrap import _find_spec
61
+
62
+ try: # pragma: no branch
63
+ import typing_extensions as _typing_extensions
64
+ from typing_extensions import Literal, Final
65
+ except ImportError:
66
+ _typing_extensions = Literal = Final = None
67
+
68
+ if sys.version_info >= (3, 8):
69
+ from types import CellType
70
+ else:
71
+ def f():
72
+ a = 1
73
+
74
+ def g():
75
+ return a
76
+ return g
77
+ CellType = type(f().__closure__[0])
78
+
79
+
80
+ # cloudpickle is meant for inter process communication: we expect all
81
+ # communicating processes to run the same Python version hence we favor
82
+ # communication speed over compatibility:
83
+ DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
84
+
85
+ # Names of modules whose resources should be treated as dynamic.
86
+ _PICKLE_BY_VALUE_MODULES = set()
87
+
88
+ # Track the provenance of reconstructed dynamic classes to make it possible to
89
+ # reconstruct instances from the matching singleton class definition when
90
+ # appropriate and preserve the usual "isinstance" semantics of Python objects.
91
+ _DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
92
+ _DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
93
+ _DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
94
+
95
+ PYPY = platform.python_implementation() == "PyPy"
96
+
97
+ builtin_code_type = None
98
+ if PYPY:
99
+ # builtin-code objects only exist in pypy
100
+ builtin_code_type = type(float.__new__.__code__)
101
+
102
+ _extract_code_globals_cache = weakref.WeakKeyDictionary()
103
+
104
+
105
+ def _get_or_create_tracker_id(class_def):
106
+ with _DYNAMIC_CLASS_TRACKER_LOCK:
107
+ class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
108
+ if class_tracker_id is None:
109
+ class_tracker_id = uuid.uuid4().hex
110
+ _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
111
+ _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
112
+ return class_tracker_id
113
+
114
+
115
+ def _lookup_class_or_track(class_tracker_id, class_def):
116
+ if class_tracker_id is not None:
117
+ with _DYNAMIC_CLASS_TRACKER_LOCK:
118
+ class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
119
+ class_tracker_id, class_def)
120
+ _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
121
+ return class_def
122
+
123
+
124
+ def register_pickle_by_value(module):
125
+ """Register a module to make it functions and classes picklable by value.
126
+
127
+ By default, functions and classes that are attributes of an importable
128
+ module are to be pickled by reference, that is relying on re-importing
129
+ the attribute from the module at load time.
130
+
131
+ If `register_pickle_by_value(module)` is called, all its functions and
132
+ classes are subsequently to be pickled by value, meaning that they can
133
+ be loaded in Python processes where the module is not importable.
134
+
135
+ This is especially useful when developing a module in a distributed
136
+ execution environment: restarting the client Python process with the new
137
+ source code is enough: there is no need to re-install the new version
138
+ of the module on all the worker nodes nor to restart the workers.
139
+
140
+ Note: this feature is considered experimental. See the cloudpickle
141
+ README.md file for more details and limitations.
142
+ """
143
+ if not isinstance(module, types.ModuleType):
144
+ raise ValueError(
145
+ f"Input should be a module object, got {str(module)} instead"
146
+ )
147
+ # In the future, cloudpickle may need a way to access any module registered
148
+ # for pickling by value in order to introspect relative imports inside
149
+ # functions pickled by value. (see
150
+ # https://github.com/cloudpipe/cloudpickle/pull/417#issuecomment-873684633).
151
+ # This access can be ensured by checking that module is present in
152
+ # sys.modules at registering time and assuming that it will still be in
153
+ # there when accessed during pickling. Another alternative would be to
154
+ # store a weakref to the module. Even though cloudpickle does not implement
155
+ # this introspection yet, in order to avoid a possible breaking change
156
+ # later, we still enforce the presence of module inside sys.modules.
157
+ if module.__name__ not in sys.modules:
158
+ raise ValueError(
159
+ f"{module} was not imported correctly, have you used an "
160
+ f"`import` statement to access it?"
161
+ )
162
+ _PICKLE_BY_VALUE_MODULES.add(module.__name__)
163
+
164
+
165
+ def unregister_pickle_by_value(module):
166
+ """Unregister that the input module should be pickled by value."""
167
+ if not isinstance(module, types.ModuleType):
168
+ raise ValueError(
169
+ f"Input should be a module object, got {str(module)} instead"
170
+ )
171
+ if module.__name__ not in _PICKLE_BY_VALUE_MODULES:
172
+ raise ValueError(f"{module} is not registered for pickle by value")
173
+ else:
174
+ _PICKLE_BY_VALUE_MODULES.remove(module.__name__)
175
+
176
+
177
+ def list_registry_pickle_by_value():
178
+ return _PICKLE_BY_VALUE_MODULES.copy()
179
+
180
+
181
+ def _is_registered_pickle_by_value(module):
182
+ module_name = module.__name__
183
+ if module_name in _PICKLE_BY_VALUE_MODULES:
184
+ return True
185
+ while True:
186
+ parent_name = module_name.rsplit(".", 1)[0]
187
+ if parent_name == module_name:
188
+ break
189
+ if parent_name in _PICKLE_BY_VALUE_MODULES:
190
+ return True
191
+ module_name = parent_name
192
+ return False
193
+
194
+
195
+ def _whichmodule(obj, name):
196
+ """Find the module an object belongs to.
197
+
198
+ This function differs from ``pickle.whichmodule`` in two ways:
199
+ - it does not mangle the cases where obj's module is __main__ and obj was
200
+ not found in any module.
201
+ - Errors arising during module introspection are ignored, as those errors
202
+ are considered unwanted side effects.
203
+ """
204
+ if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa
205
+ # Workaround bug in old Python versions: prior to Python 3.7,
206
+ # T.__module__ would always be set to "typing" even when the TypeVar T
207
+ # would be defined in a different module.
208
+ if name is not None and getattr(typing, name, None) is obj:
209
+ # Built-in TypeVar defined in typing such as AnyStr
210
+ return 'typing'
211
+ else:
212
+ # User defined or third-party TypeVar: __module__ attribute is
213
+ # irrelevant, thus trigger a exhaustive search for obj in all
214
+ # modules.
215
+ module_name = None
216
+ else:
217
+ module_name = getattr(obj, '__module__', None)
218
+
219
+ if module_name is not None:
220
+ return module_name
221
+ # Protect the iteration by using a copy of sys.modules against dynamic
222
+ # modules that trigger imports of other modules upon calls to getattr or
223
+ # other threads importing at the same time.
224
+ for module_name, module in sys.modules.copy().items():
225
+ # Some modules such as coverage can inject non-module objects inside
226
+ # sys.modules
227
+ if (
228
+ module_name == '__main__' or
229
+ module is None or
230
+ not isinstance(module, types.ModuleType)
231
+ ):
232
+ continue
233
+ try:
234
+ if _getattribute(module, name)[0] is obj:
235
+ return module_name
236
+ except Exception:
237
+ pass
238
+ return None
239
+
240
+
241
+ def _should_pickle_by_reference(obj, name=None):
242
+ """Test whether an function or a class should be pickled by reference
243
+
244
+ Pickling by reference means by that the object (typically a function or a
245
+ class) is an attribute of a module that is assumed to be importable in the
246
+ target Python environment. Loading will therefore rely on importing the
247
+ module and then calling `getattr` on it to access the function or class.
248
+
249
+ Pickling by reference is the only option to pickle functions and classes
250
+ in the standard library. In cloudpickle the alternative option is to
251
+ pickle by value (for instance for interactively or locally defined
252
+ functions and classes or for attributes of modules that have been
253
+ explicitly registered to be pickled by value.
254
+ """
255
+ if isinstance(obj, types.FunctionType) or issubclass(type(obj), type):
256
+ module_and_name = _lookup_module_and_qualname(obj, name=name)
257
+ if module_and_name is None:
258
+ return False
259
+ module, name = module_and_name
260
+ return not _is_registered_pickle_by_value(module)
261
+
262
+ elif isinstance(obj, types.ModuleType):
263
+ # We assume that sys.modules is primarily used as a cache mechanism for
264
+ # the Python import machinery. Checking if a module has been added in
265
+ # is sys.modules therefore a cheap and simple heuristic to tell us
266
+ # whether we can assume that a given module could be imported by name
267
+ # in another Python process.
268
+ if _is_registered_pickle_by_value(obj):
269
+ return False
270
+ return obj.__name__ in sys.modules
271
+ else:
272
+ raise TypeError(
273
+ "cannot check importability of {} instances".format(
274
+ type(obj).__name__)
275
+ )
276
+
277
+
278
+ def _lookup_module_and_qualname(obj, name=None):
279
+ if name is None:
280
+ name = getattr(obj, '__qualname__', None)
281
+ if name is None: # pragma: no cover
282
+ # This used to be needed for Python 2.7 support but is probably not
283
+ # needed anymore. However we keep the __name__ introspection in case
284
+ # users of cloudpickle rely on this old behavior for unknown reasons.
285
+ name = getattr(obj, '__name__', None)
286
+
287
+ module_name = _whichmodule(obj, name)
288
+
289
+ if module_name is None:
290
+ # In this case, obj.__module__ is None AND obj was not found in any
291
+ # imported module. obj is thus treated as dynamic.
292
+ return None
293
+
294
+ if module_name == "__main__":
295
+ return None
296
+
297
+ # Note: if module_name is in sys.modules, the corresponding module is
298
+ # assumed importable at unpickling time. See #357
299
+ module = sys.modules.get(module_name, None)
300
+ if module is None:
301
+ # The main reason why obj's module would not be imported is that this
302
+ # module has been dynamically created, using for example
303
+ # types.ModuleType. The other possibility is that module was removed
304
+ # from sys.modules after obj was created/imported. But this case is not
305
+ # supported, as the standard pickle does not support it either.
306
+ return None
307
+
308
+ try:
309
+ obj2, parent = _getattribute(module, name)
310
+ except AttributeError:
311
+ # obj was not found inside the module it points to
312
+ return None
313
+ if obj2 is not obj:
314
+ return None
315
+ return module, name
316
+
317
+
318
+ def _extract_code_globals(co):
319
+ """
320
+ Find all globals names read or written to by codeblock co
321
+ """
322
+ out_names = _extract_code_globals_cache.get(co)
323
+ if out_names is None:
324
+ # We use a dict with None values instead of a set to get a
325
+ # deterministic order (assuming Python 3.6+) and avoid introducing
326
+ # non-deterministic pickle bytes as a results.
327
+ out_names = {name: None for name in _walk_global_ops(co)}
328
+
329
+ # Declaring a function inside another one using the "def ..."
330
+ # syntax generates a constant code object corresponding to the one
331
+ # of the nested function's As the nested function may itself need
332
+ # global variables, we need to introspect its code, extract its
333
+ # globals, (look for code object in it's co_consts attribute..) and
334
+ # add the result to code_globals
335
+ if co.co_consts:
336
+ for const in co.co_consts:
337
+ if isinstance(const, types.CodeType):
338
+ out_names.update(_extract_code_globals(const))
339
+
340
+ _extract_code_globals_cache[co] = out_names
341
+
342
+ return out_names
343
+
344
+
345
+ def _find_imported_submodules(code, top_level_dependencies):
346
+ """
347
+ Find currently imported submodules used by a function.
348
+
349
+ Submodules used by a function need to be detected and referenced for the
350
+ function to work correctly at depickling time. Because submodules can be
351
+ referenced as attribute of their parent package (``package.submodule``), we
352
+ need a special introspection technique that does not rely on GLOBAL-related
353
+ opcodes to find references of them in a code object.
354
+
355
+ Example:
356
+ ```
357
+ import concurrent.futures
358
+ import cloudpickle
359
+ def func():
360
+ x = concurrent.futures.ThreadPoolExecutor
361
+ if __name__ == '__main__':
362
+ cloudpickle.dumps(func)
363
+ ```
364
+ The globals extracted by cloudpickle in the function's state include the
365
+ concurrent package, but not its submodule (here, concurrent.futures), which
366
+ is the module used by func. Find_imported_submodules will detect the usage
367
+ of concurrent.futures. Saving this module alongside with func will ensure
368
+ that calling func once depickled does not fail due to concurrent.futures
369
+ not being imported
370
+ """
371
+
372
+ subimports = []
373
+ # check if any known dependency is an imported package
374
+ for x in top_level_dependencies:
375
+ if (isinstance(x, types.ModuleType) and
376
+ hasattr(x, '__package__') and x.__package__):
377
+ # check if the package has any currently loaded sub-imports
378
+ prefix = x.__name__ + '.'
379
+ # A concurrent thread could mutate sys.modules,
380
+ # make sure we iterate over a copy to avoid exceptions
381
+ for name in list(sys.modules):
382
+ # Older versions of pytest will add a "None" module to
383
+ # sys.modules.
384
+ if name is not None and name.startswith(prefix):
385
+ # check whether the function can address the sub-module
386
+ tokens = set(name[len(prefix):].split('.'))
387
+ if not tokens - set(code.co_names):
388
+ subimports.append(sys.modules[name])
389
+ return subimports
390
+
391
+
392
+ def cell_set(cell, value):
393
+ """Set the value of a closure cell.
394
+
395
+ The point of this function is to set the cell_contents attribute of a cell
396
+ after its creation. This operation is necessary in case the cell contains a
397
+ reference to the function the cell belongs to, as when calling the
398
+ function's constructor
399
+ ``f = types.FunctionType(code, globals, name, argdefs, closure)``,
400
+ closure will not be able to contain the yet-to-be-created f.
401
+
402
+ In Python3.7, cell_contents is writeable, so setting the contents of a cell
403
+ can be done simply using
404
+ >>> cell.cell_contents = value
405
+
406
+ In earlier Python3 versions, the cell_contents attribute of a cell is read
407
+ only, but this limitation can be worked around by leveraging the Python 3
408
+ ``nonlocal`` keyword.
409
+
410
+ In Python2 however, this attribute is read only, and there is no
411
+ ``nonlocal`` keyword. For this reason, we need to come up with more
412
+ complicated hacks to set this attribute.
413
+
414
+ The chosen approach is to create a function with a STORE_DEREF opcode,
415
+ which sets the content of a closure variable. Typically:
416
+
417
+ >>> def inner(value):
418
+ ... lambda: cell # the lambda makes cell a closure
419
+ ... cell = value # cell is a closure, so this triggers a STORE_DEREF
420
+
421
+ (Note that in Python2, A STORE_DEREF can never be triggered from an inner
422
+ function. The function g for example here
423
+ >>> def f(var):
424
+ ... def g():
425
+ ... var += 1
426
+ ... return g
427
+
428
+ will not modify the closure variable ``var```inplace, but instead try to
429
+ load a local variable var and increment it. As g does not assign the local
430
+ variable ``var`` any initial value, calling f(1)() will fail at runtime.)
431
+
432
+ Our objective is to set the value of a given cell ``cell``. So we need to
433
+ somewhat reference our ``cell`` object into the ``inner`` function so that
434
+ this object (and not the smoke cell of the lambda function) gets affected
435
+ by the STORE_DEREF operation.
436
+
437
+ In inner, ``cell`` is referenced as a cell variable (an enclosing variable
438
+ that is referenced by the inner function). If we create a new function
439
+ cell_set with the exact same code as ``inner``, but with ``cell`` marked as
440
+ a free variable instead, the STORE_DEREF will be applied on its closure -
441
+ ``cell``, which we can specify explicitly during construction! The new
442
+ cell_set variable thus actually sets the contents of a specified cell!
443
+
444
+ Note: we do not make use of the ``nonlocal`` keyword to set the contents of
445
+ a cell in early python3 versions to limit possible syntax errors in case
446
+ test and checker libraries decide to parse the whole file.
447
+ """
448
+
449
+ if sys.version_info[:2] >= (3, 7): # pragma: no branch
450
+ cell.cell_contents = value
451
+ else:
452
+ _cell_set = types.FunctionType(
453
+ _cell_set_template_code, {}, '_cell_set', (), (cell,),)
454
+ _cell_set(value)
455
+
456
+
457
+ def _make_cell_set_template_code():
458
+ def _cell_set_factory(value):
459
+ lambda: cell
460
+ cell = value
461
+
462
+ co = _cell_set_factory.__code__
463
+
464
+ _cell_set_template_code = types.CodeType(
465
+ co.co_argcount,
466
+ co.co_kwonlyargcount, # Python 3 only argument
467
+ co.co_nlocals,
468
+ co.co_stacksize,
469
+ co.co_flags,
470
+ co.co_code,
471
+ co.co_consts,
472
+ co.co_names,
473
+ co.co_varnames,
474
+ co.co_filename,
475
+ co.co_name,
476
+ co.co_firstlineno,
477
+ co.co_lnotab,
478
+ co.co_cellvars, # co_freevars is initialized with co_cellvars
479
+ (), # co_cellvars is made empty
480
+ )
481
+ return _cell_set_template_code
482
+
483
+
484
+ if sys.version_info[:2] < (3, 7):
485
+ _cell_set_template_code = _make_cell_set_template_code()
486
+
487
+ # relevant opcodes
488
+ STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
489
+ DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
490
+ LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
491
+ GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
492
+ HAVE_ARGUMENT = dis.HAVE_ARGUMENT
493
+ EXTENDED_ARG = dis.EXTENDED_ARG
494
+
495
+
496
+ _BUILTIN_TYPE_NAMES = {}
497
+ for k, v in types.__dict__.items():
498
+ if type(v) is type:
499
+ _BUILTIN_TYPE_NAMES[v] = k
500
+
501
+
502
+ def _builtin_type(name):
503
+ if name == "ClassType": # pragma: no cover
504
+ # Backward compat to load pickle files generated with cloudpickle
505
+ # < 1.3 even if loading pickle files from older versions is not
506
+ # officially supported.
507
+ return type
508
+ return getattr(types, name)
509
+
510
+
511
+ def _walk_global_ops(code):
512
+ """
513
+ Yield referenced name for all global-referencing instructions in *code*.
514
+ """
515
+ for instr in dis.get_instructions(code):
516
+ op = instr.opcode
517
+ if op in GLOBAL_OPS:
518
+ yield instr.argval
519
+
520
+
521
+ def _extract_class_dict(cls):
522
+ """Retrieve a copy of the dict of a class without the inherited methods"""
523
+ clsdict = dict(cls.__dict__) # copy dict proxy to a dict
524
+ if len(cls.__bases__) == 1:
525
+ inherited_dict = cls.__bases__[0].__dict__
526
+ else:
527
+ inherited_dict = {}
528
+ for base in reversed(cls.__bases__):
529
+ inherited_dict.update(base.__dict__)
530
+ to_remove = []
531
+ for name, value in clsdict.items():
532
+ try:
533
+ base_value = inherited_dict[name]
534
+ if value is base_value:
535
+ to_remove.append(name)
536
+ except KeyError:
537
+ pass
538
+ for name in to_remove:
539
+ clsdict.pop(name)
540
+ return clsdict
541
+
542
+
543
+ if sys.version_info[:2] < (3, 7): # pragma: no branch
544
+ def _is_parametrized_type_hint(obj):
545
+ # This is very cheap but might generate false positives. So try to
546
+ # narrow it down is good as possible.
547
+ type_module = getattr(type(obj), '__module__', None)
548
+ from_typing_extensions = type_module == 'typing_extensions'
549
+ from_typing = type_module == 'typing'
550
+
551
+ # general typing Constructs
552
+ is_typing = getattr(obj, '__origin__', None) is not None
553
+
554
+ # typing_extensions.Literal
555
+ is_literal = (
556
+ (getattr(obj, '__values__', None) is not None)
557
+ and from_typing_extensions
558
+ )
559
+
560
+ # typing_extensions.Final
561
+ is_final = (
562
+ (getattr(obj, '__type__', None) is not None)
563
+ and from_typing_extensions
564
+ )
565
+
566
+ # typing.ClassVar
567
+ is_classvar = (
568
+ (getattr(obj, '__type__', None) is not None) and from_typing
569
+ )
570
+
571
+ # typing.Union/Tuple for old Python 3.5
572
+ is_union = getattr(obj, '__union_params__', None) is not None
573
+ is_tuple = getattr(obj, '__tuple_params__', None) is not None
574
+ is_callable = (
575
+ getattr(obj, '__result__', None) is not None and
576
+ getattr(obj, '__args__', None) is not None
577
+ )
578
+ return any((is_typing, is_literal, is_final, is_classvar, is_union,
579
+ is_tuple, is_callable))
580
+
581
+ def _create_parametrized_type_hint(origin, args):
582
+ return origin[args]
583
+ else:
584
+ _is_parametrized_type_hint = None
585
+ _create_parametrized_type_hint = None
586
+
587
+
588
+ def parametrized_type_hint_getinitargs(obj):
589
+ # The distorted type check sematic for typing construct becomes:
590
+ # ``type(obj) is type(TypeHint)``, which means "obj is a
591
+ # parametrized TypeHint"
592
+ if type(obj) is type(Literal): # pragma: no branch
593
+ initargs = (Literal, obj.__values__)
594
+ elif type(obj) is type(Final): # pragma: no branch
595
+ initargs = (Final, obj.__type__)
596
+ elif type(obj) is type(ClassVar):
597
+ initargs = (ClassVar, obj.__type__)
598
+ elif type(obj) is type(Generic):
599
+ initargs = (obj.__origin__, obj.__args__)
600
+ elif type(obj) is type(Union):
601
+ initargs = (Union, obj.__args__)
602
+ elif type(obj) is type(Tuple):
603
+ initargs = (Tuple, obj.__args__)
604
+ elif type(obj) is type(Callable):
605
+ (*args, result) = obj.__args__
606
+ if len(args) == 1 and args[0] is Ellipsis:
607
+ args = Ellipsis
608
+ else:
609
+ args = list(args)
610
+ initargs = (Callable, (args, result))
611
+ else: # pragma: no cover
612
+ raise pickle.PicklingError(
613
+ f"Cloudpickle Error: Unknown type {type(obj)}"
614
+ )
615
+ return initargs
616
+
617
+
618
+ # Tornado support
619
+
620
+ def is_tornado_coroutine(func):
621
+ """
622
+ Return whether *func* is a Tornado coroutine function.
623
+ Running coroutines are not supported.
624
+ """
625
+ if 'tornado.gen' not in sys.modules:
626
+ return False
627
+ gen = sys.modules['tornado.gen']
628
+ if not hasattr(gen, "is_coroutine_function"):
629
+ # Tornado version is too old
630
+ return False
631
+ return gen.is_coroutine_function(func)
632
+
633
+
634
+ def _rebuild_tornado_coroutine(func):
635
+ from tornado import gen
636
+ return gen.coroutine(func)
637
+
638
+
639
+ # including pickles unloading functions in this namespace
640
+ load = pickle.load
641
+ loads = pickle.loads
642
+
643
+
644
+ def subimport(name):
645
+ # We cannot do simply: `return __import__(name)`: Indeed, if ``name`` is
646
+ # the name of a submodule, __import__ will return the top-level root module
647
+ # of this submodule. For instance, __import__('os.path') returns the `os`
648
+ # module.
649
+ __import__(name)
650
+ return sys.modules[name]
651
+
652
+
653
+ def dynamic_subimport(name, vars):
654
+ mod = types.ModuleType(name)
655
+ mod.__dict__.update(vars)
656
+ mod.__dict__['__builtins__'] = builtins.__dict__
657
+ return mod
658
+
659
+
660
+ def _gen_ellipsis():
661
+ return Ellipsis
662
+
663
+
664
+ def _gen_not_implemented():
665
+ return NotImplemented
666
+
667
+
668
+ def _get_cell_contents(cell):
669
+ try:
670
+ return cell.cell_contents
671
+ except ValueError:
672
+ # sentinel used by ``_fill_function`` which will leave the cell empty
673
+ return _empty_cell_value
674
+
675
+
676
+ def instance(cls):
677
+ """Create a new instance of a class.
678
+
679
+ Parameters
680
+ ----------
681
+ cls : type
682
+ The class to create an instance of.
683
+
684
+ Returns
685
+ -------
686
+ instance : cls
687
+ A new instance of ``cls``.
688
+ """
689
+ return cls()
690
+
691
+
692
+ @instance
693
+ class _empty_cell_value:
694
+ """sentinel for empty closures
695
+ """
696
+ @classmethod
697
+ def __reduce__(cls):
698
+ return cls.__name__
699
+
700
+
701
+ def _fill_function(*args):
702
+ """Fills in the rest of function data into the skeleton function object
703
+
704
+ The skeleton itself is create by _make_skel_func().
705
+ """
706
+ if len(args) == 2:
707
+ func = args[0]
708
+ state = args[1]
709
+ elif len(args) == 5:
710
+ # Backwards compat for cloudpickle v0.4.0, after which the `module`
711
+ # argument was introduced
712
+ func = args[0]
713
+ keys = ['globals', 'defaults', 'dict', 'closure_values']
714
+ state = dict(zip(keys, args[1:]))
715
+ elif len(args) == 6:
716
+ # Backwards compat for cloudpickle v0.4.1, after which the function
717
+ # state was passed as a dict to the _fill_function it-self.
718
+ func = args[0]
719
+ keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
720
+ state = dict(zip(keys, args[1:]))
721
+ else:
722
+ raise ValueError(f'Unexpected _fill_value arguments: {args!r}')
723
+
724
+ # - At pickling time, any dynamic global variable used by func is
725
+ # serialized by value (in state['globals']).
726
+ # - At unpickling time, func's __globals__ attribute is initialized by
727
+ # first retrieving an empty isolated namespace that will be shared
728
+ # with other functions pickled from the same original module
729
+ # by the same CloudPickler instance and then updated with the
730
+ # content of state['globals'] to populate the shared isolated
731
+ # namespace with all the global variables that are specifically
732
+ # referenced for this function.
733
+ func.__globals__.update(state['globals'])
734
+
735
+ func.__defaults__ = state['defaults']
736
+ func.__dict__ = state['dict']
737
+ if 'annotations' in state:
738
+ func.__annotations__ = state['annotations']
739
+ if 'doc' in state:
740
+ func.__doc__ = state['doc']
741
+ if 'name' in state:
742
+ func.__name__ = state['name']
743
+ if 'module' in state:
744
+ func.__module__ = state['module']
745
+ if 'qualname' in state:
746
+ func.__qualname__ = state['qualname']
747
+ if 'kwdefaults' in state:
748
+ func.__kwdefaults__ = state['kwdefaults']
749
+ # _cloudpickle_subimports is a set of submodules that must be loaded for
750
+ # the pickled function to work correctly at unpickling time. Now that these
751
+ # submodules are depickled (hence imported), they can be removed from the
752
+ # object's state (the object state only served as a reference holder to
753
+ # these submodules)
754
+ if '_cloudpickle_submodules' in state:
755
+ state.pop('_cloudpickle_submodules')
756
+
757
+ cells = func.__closure__
758
+ if cells is not None:
759
+ for cell, value in zip(cells, state['closure_values']):
760
+ if value is not _empty_cell_value:
761
+ cell_set(cell, value)
762
+
763
+ return func
764
+
765
+
766
+ def _make_function(code, globals, name, argdefs, closure):
767
+ # Setting __builtins__ in globals is needed for nogil CPython.
768
+ globals["__builtins__"] = __builtins__
769
+ return types.FunctionType(code, globals, name, argdefs, closure)
770
+
771
+
772
+ def _make_empty_cell():
773
+ if False:
774
+ # trick the compiler into creating an empty cell in our lambda
775
+ cell = None
776
+ raise AssertionError('this route should not be executed')
777
+
778
+ return (lambda: cell).__closure__[0]
779
+
780
+
781
+ def _make_cell(value=_empty_cell_value):
782
+ cell = _make_empty_cell()
783
+ if value is not _empty_cell_value:
784
+ cell_set(cell, value)
785
+ return cell
786
+
787
+
788
+ def _make_skel_func(code, cell_count, base_globals=None):
789
+ """ Creates a skeleton function object that contains just the provided
790
+ code and the correct number of cells in func_closure. All other
791
+ func attributes (e.g. func_globals) are empty.
792
+ """
793
+ # This function is deprecated and should be removed in cloudpickle 1.7
794
+ warnings.warn(
795
+ "A pickle file created using an old (<=1.4.1) version of cloudpickle "
796
+ "is currently being loaded. This is not supported by cloudpickle and "
797
+ "will break in cloudpickle 1.7", category=UserWarning
798
+ )
799
+ # This is backward-compatibility code: for cloudpickle versions between
800
+ # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
801
+ # should now always be a dictionary.
802
+ if base_globals is None or isinstance(base_globals, str):
803
+ base_globals = {}
804
+
805
+ base_globals['__builtins__'] = __builtins__
806
+
807
+ closure = (
808
+ tuple(_make_empty_cell() for _ in range(cell_count))
809
+ if cell_count >= 0 else
810
+ None
811
+ )
812
+ return types.FunctionType(code, base_globals, None, None, closure)
813
+
814
+
815
+ def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
816
+ class_tracker_id, extra):
817
+ """Build dynamic class with an empty __dict__ to be filled once memoized
818
+
819
+ If class_tracker_id is not None, try to lookup an existing class definition
820
+ matching that id. If none is found, track a newly reconstructed class
821
+ definition under that id so that other instances stemming from the same
822
+ class id will also reuse this class definition.
823
+
824
+ The "extra" variable is meant to be a dict (or None) that can be used for
825
+ forward compatibility shall the need arise.
826
+ """
827
+ skeleton_class = types.new_class(
828
+ name, bases, {'metaclass': type_constructor},
829
+ lambda ns: ns.update(type_kwargs)
830
+ )
831
+ return _lookup_class_or_track(class_tracker_id, skeleton_class)
832
+
833
+
834
+ def _rehydrate_skeleton_class(skeleton_class, class_dict):
835
+ """Put attributes from `class_dict` back on `skeleton_class`.
836
+
837
+ See CloudPickler.save_dynamic_class for more info.
838
+ """
839
+ registry = None
840
+ for attrname, attr in class_dict.items():
841
+ if attrname == "_abc_impl":
842
+ registry = attr
843
+ else:
844
+ setattr(skeleton_class, attrname, attr)
845
+ if registry is not None:
846
+ for subclass in registry:
847
+ skeleton_class.register(subclass)
848
+
849
+ return skeleton_class
850
+
851
+
852
+ def _make_skeleton_enum(bases, name, qualname, members, module,
853
+ class_tracker_id, extra):
854
+ """Build dynamic enum with an empty __dict__ to be filled once memoized
855
+
856
+ The creation of the enum class is inspired by the code of
857
+ EnumMeta._create_.
858
+
859
+ If class_tracker_id is not None, try to lookup an existing enum definition
860
+ matching that id. If none is found, track a newly reconstructed enum
861
+ definition under that id so that other instances stemming from the same
862
+ class id will also reuse this enum definition.
863
+
864
+ The "extra" variable is meant to be a dict (or None) that can be used for
865
+ forward compatibility shall the need arise.
866
+ """
867
+ # enums always inherit from their base Enum class at the last position in
868
+ # the list of base classes:
869
+ enum_base = bases[-1]
870
+ metacls = enum_base.__class__
871
+ classdict = metacls.__prepare__(name, bases)
872
+
873
+ for member_name, member_value in members.items():
874
+ classdict[member_name] = member_value
875
+ enum_class = metacls.__new__(metacls, name, bases, classdict)
876
+ enum_class.__module__ = module
877
+ enum_class.__qualname__ = qualname
878
+
879
+ return _lookup_class_or_track(class_tracker_id, enum_class)
880
+
881
+
882
+ def _make_typevar(name, bound, constraints, covariant, contravariant,
883
+ class_tracker_id):
884
+ tv = typing.TypeVar(
885
+ name, *constraints, bound=bound,
886
+ covariant=covariant, contravariant=contravariant
887
+ )
888
+ if class_tracker_id is not None:
889
+ return _lookup_class_or_track(class_tracker_id, tv)
890
+ else: # pragma: nocover
891
+ # Only for Python 3.5.3 compat.
892
+ return tv
893
+
894
+
895
+ def _decompose_typevar(obj):
896
+ return (
897
+ obj.__name__, obj.__bound__, obj.__constraints__,
898
+ obj.__covariant__, obj.__contravariant__,
899
+ _get_or_create_tracker_id(obj),
900
+ )
901
+
902
+
903
+ def _typevar_reduce(obj):
904
+ # TypeVar instances require the module information hence why we
905
+ # are not using the _should_pickle_by_reference directly
906
+ module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__)
907
+
908
+ if module_and_name is None:
909
+ return (_make_typevar, _decompose_typevar(obj))
910
+ elif _is_registered_pickle_by_value(module_and_name[0]):
911
+ return (_make_typevar, _decompose_typevar(obj))
912
+
913
+ return (getattr, module_and_name)
914
+
915
+
916
+ def _get_bases(typ):
917
+ if '__orig_bases__' in getattr(typ, '__dict__', {}):
918
+ # For generic types (see PEP 560)
919
+ # Note that simply checking `hasattr(typ, '__orig_bases__')` is not
920
+ # correct. Subclasses of a fully-parameterized generic class does not
921
+ # have `__orig_bases__` defined, but `hasattr(typ, '__orig_bases__')`
922
+ # will return True because it's defined in the base class.
923
+ bases_attr = '__orig_bases__'
924
+ else:
925
+ # For regular class objects
926
+ bases_attr = '__bases__'
927
+ return getattr(typ, bases_attr)
928
+
929
+
930
+ def _make_dict_keys(obj, is_ordered=False):
931
+ if is_ordered:
932
+ return OrderedDict.fromkeys(obj).keys()
933
+ else:
934
+ return dict.fromkeys(obj).keys()
935
+
936
+
937
+ def _make_dict_values(obj, is_ordered=False):
938
+ if is_ordered:
939
+ return OrderedDict((i, _) for i, _ in enumerate(obj)).values()
940
+ else:
941
+ return {i: _ for i, _ in enumerate(obj)}.values()
942
+
943
+
944
+ def _make_dict_items(obj, is_ordered=False):
945
+ if is_ordered:
946
+ return OrderedDict(obj).items()
947
+ else:
948
+ return obj.items()
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/cloudpickle_fast.py ADDED
@@ -0,0 +1,844 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ New, fast version of the CloudPickler.
3
+
4
+ This new CloudPickler class can now extend the fast C Pickler instead of the
5
+ previous Python implementation of the Pickler class. Because this functionality
6
+ is only available for Python versions 3.8+, a lot of backward-compatibility
7
+ code is also removed.
8
+
9
+ Note that the C Pickler subclassing API is CPython-specific. Therefore, some
10
+ guards present in cloudpickle.py that were written to handle PyPy specificities
11
+ are not present in cloudpickle_fast.py
12
+ """
13
+ import _collections_abc
14
+ import abc
15
+ import copyreg
16
+ import io
17
+ import itertools
18
+ import logging
19
+ import sys
20
+ import struct
21
+ import types
22
+ import weakref
23
+ import typing
24
+
25
+ from enum import Enum
26
+ from collections import ChainMap, OrderedDict
27
+
28
+ from .compat import pickle, Pickler
29
+ from .cloudpickle import (
30
+ _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL,
31
+ _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference,
32
+ _builtin_type, _get_or_create_tracker_id, _make_skeleton_class,
33
+ _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport,
34
+ _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType,
35
+ _is_parametrized_type_hint, PYPY, cell_set,
36
+ parametrized_type_hint_getinitargs, _create_parametrized_type_hint,
37
+ builtin_code_type,
38
+ _make_dict_keys, _make_dict_values, _make_dict_items, _make_function,
39
+ )
40
+
41
+
42
+ if pickle.HIGHEST_PROTOCOL >= 5:
43
+ # Shorthands similar to pickle.dump/pickle.dumps
44
+
45
+ def dump(obj, file, protocol=None, buffer_callback=None):
46
+ """Serialize obj as bytes streamed into file
47
+
48
+ protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
49
+ pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
50
+ speed between processes running the same Python version.
51
+
52
+ Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
53
+ compatibility with older versions of Python.
54
+ """
55
+ CloudPickler(
56
+ file, protocol=protocol, buffer_callback=buffer_callback
57
+ ).dump(obj)
58
+
59
+ def dumps(obj, protocol=None, buffer_callback=None):
60
+ """Serialize obj as a string of bytes allocated in memory
61
+
62
+ protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
63
+ pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
64
+ speed between processes running the same Python version.
65
+
66
+ Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
67
+ compatibility with older versions of Python.
68
+ """
69
+ with io.BytesIO() as file:
70
+ cp = CloudPickler(
71
+ file, protocol=protocol, buffer_callback=buffer_callback
72
+ )
73
+ cp.dump(obj)
74
+ return file.getvalue()
75
+
76
+ else:
77
+ # Shorthands similar to pickle.dump/pickle.dumps
78
+ def dump(obj, file, protocol=None):
79
+ """Serialize obj as bytes streamed into file
80
+
81
+ protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
82
+ pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
83
+ speed between processes running the same Python version.
84
+
85
+ Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
86
+ compatibility with older versions of Python.
87
+ """
88
+ CloudPickler(file, protocol=protocol).dump(obj)
89
+
90
+ def dumps(obj, protocol=None):
91
+ """Serialize obj as a string of bytes allocated in memory
92
+
93
+ protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
94
+ pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
95
+ speed between processes running the same Python version.
96
+
97
+ Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
98
+ compatibility with older versions of Python.
99
+ """
100
+ with io.BytesIO() as file:
101
+ cp = CloudPickler(file, protocol=protocol)
102
+ cp.dump(obj)
103
+ return file.getvalue()
104
+
105
+
106
+ load, loads = pickle.load, pickle.loads
107
+
108
+
109
+ # COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS
110
+ # -------------------------------------------------
111
+
112
+ def _class_getnewargs(obj):
113
+ type_kwargs = {}
114
+ if "__slots__" in obj.__dict__:
115
+ type_kwargs["__slots__"] = obj.__slots__
116
+
117
+ __dict__ = obj.__dict__.get('__dict__', None)
118
+ if isinstance(__dict__, property):
119
+ type_kwargs['__dict__'] = __dict__
120
+
121
+ return (type(obj), obj.__name__, _get_bases(obj), type_kwargs,
122
+ _get_or_create_tracker_id(obj), None)
123
+
124
+
125
+ def _enum_getnewargs(obj):
126
+ members = {e.name: e.value for e in obj}
127
+ return (obj.__bases__, obj.__name__, obj.__qualname__, members,
128
+ obj.__module__, _get_or_create_tracker_id(obj), None)
129
+
130
+
131
+ # COLLECTION OF OBJECTS RECONSTRUCTORS
132
+ # ------------------------------------
133
+ def _file_reconstructor(retval):
134
+ return retval
135
+
136
+
137
+ # COLLECTION OF OBJECTS STATE GETTERS
138
+ # -----------------------------------
139
+ def _function_getstate(func):
140
+ # - Put func's dynamic attributes (stored in func.__dict__) in state. These
141
+ # attributes will be restored at unpickling time using
142
+ # f.__dict__.update(state)
143
+ # - Put func's members into slotstate. Such attributes will be restored at
144
+ # unpickling time by iterating over slotstate and calling setattr(func,
145
+ # slotname, slotvalue)
146
+ slotstate = {
147
+ "__name__": func.__name__,
148
+ "__qualname__": func.__qualname__,
149
+ "__annotations__": func.__annotations__,
150
+ "__kwdefaults__": func.__kwdefaults__,
151
+ "__defaults__": func.__defaults__,
152
+ "__module__": func.__module__,
153
+ "__doc__": func.__doc__,
154
+ "__closure__": func.__closure__,
155
+ }
156
+
157
+ f_globals_ref = _extract_code_globals(func.__code__)
158
+ f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in
159
+ func.__globals__}
160
+
161
+ closure_values = (
162
+ list(map(_get_cell_contents, func.__closure__))
163
+ if func.__closure__ is not None else ()
164
+ )
165
+
166
+ # Extract currently-imported submodules used by func. Storing these modules
167
+ # in a smoke _cloudpickle_subimports attribute of the object's state will
168
+ # trigger the side effect of importing these modules at unpickling time
169
+ # (which is necessary for func to work correctly once depickled)
170
+ slotstate["_cloudpickle_submodules"] = _find_imported_submodules(
171
+ func.__code__, itertools.chain(f_globals.values(), closure_values))
172
+ slotstate["__globals__"] = f_globals
173
+
174
+ state = func.__dict__
175
+ return state, slotstate
176
+
177
+
178
+ def _class_getstate(obj):
179
+ clsdict = _extract_class_dict(obj)
180
+ clsdict.pop('__weakref__', None)
181
+
182
+ if issubclass(type(obj), abc.ABCMeta):
183
+ # If obj is an instance of an ABCMeta subclass, don't pickle the
184
+ # cache/negative caches populated during isinstance/issubclass
185
+ # checks, but pickle the list of registered subclasses of obj.
186
+ clsdict.pop('_abc_cache', None)
187
+ clsdict.pop('_abc_negative_cache', None)
188
+ clsdict.pop('_abc_negative_cache_version', None)
189
+ registry = clsdict.pop('_abc_registry', None)
190
+ if registry is None:
191
+ # in Python3.7+, the abc caches and registered subclasses of a
192
+ # class are bundled into the single _abc_impl attribute
193
+ clsdict.pop('_abc_impl', None)
194
+ (registry, _, _, _) = abc._get_dump(obj)
195
+
196
+ clsdict["_abc_impl"] = [subclass_weakref()
197
+ for subclass_weakref in registry]
198
+ else:
199
+ # In the above if clause, registry is a set of weakrefs -- in
200
+ # this case, registry is a WeakSet
201
+ clsdict["_abc_impl"] = [type_ for type_ in registry]
202
+
203
+ if "__slots__" in clsdict:
204
+ # pickle string length optimization: member descriptors of obj are
205
+ # created automatically from obj's __slots__ attribute, no need to
206
+ # save them in obj's state
207
+ if isinstance(obj.__slots__, str):
208
+ clsdict.pop(obj.__slots__)
209
+ else:
210
+ for k in obj.__slots__:
211
+ clsdict.pop(k, None)
212
+
213
+ clsdict.pop('__dict__', None) # unpicklable property object
214
+
215
+ return (clsdict, {})
216
+
217
+
218
+ def _enum_getstate(obj):
219
+ clsdict, slotstate = _class_getstate(obj)
220
+
221
+ members = {e.name: e.value for e in obj}
222
+ # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class:
223
+ # Those attributes are already handled by the metaclass.
224
+ for attrname in ["_generate_next_value_", "_member_names_",
225
+ "_member_map_", "_member_type_",
226
+ "_value2member_map_"]:
227
+ clsdict.pop(attrname, None)
228
+ for member in members:
229
+ clsdict.pop(member)
230
+ # Special handling of Enum subclasses
231
+ return clsdict, slotstate
232
+
233
+
234
+ # COLLECTIONS OF OBJECTS REDUCERS
235
+ # -------------------------------
236
+ # A reducer is a function taking a single argument (obj), and that returns a
237
+ # tuple with all the necessary data to re-construct obj. Apart from a few
238
+ # exceptions (list, dict, bytes, int, etc.), a reducer is necessary to
239
+ # correctly pickle an object.
240
+ # While many built-in objects (Exceptions objects, instances of the "object"
241
+ # class, etc), are shipped with their own built-in reducer (invoked using
242
+ # obj.__reduce__), some do not. The following methods were created to "fill
243
+ # these holes".
244
+
245
+ def _code_reduce(obj):
246
+ """codeobject reducer"""
247
+ # If you are not sure about the order of arguments, take a look at help
248
+ # of the specific type from types, for example:
249
+ # >>> from types import CodeType
250
+ # >>> help(CodeType)
251
+ if hasattr(obj, "co_exceptiontable"): # pragma: no branch
252
+ # Python 3.11 and later: there are some new attributes
253
+ # related to the enhanced exceptions.
254
+ args = (
255
+ obj.co_argcount, obj.co_posonlyargcount,
256
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
257
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
258
+ obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
259
+ obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable,
260
+ obj.co_freevars, obj.co_cellvars,
261
+ )
262
+ elif hasattr(obj, "co_linetable"): # pragma: no branch
263
+ # Python 3.10 and later: obj.co_lnotab is deprecated and constructor
264
+ # expects obj.co_linetable instead.
265
+ args = (
266
+ obj.co_argcount, obj.co_posonlyargcount,
267
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
268
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
269
+ obj.co_varnames, obj.co_filename, obj.co_name,
270
+ obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
271
+ obj.co_cellvars
272
+ )
273
+ elif hasattr(obj, "co_nmeta"): # pragma: no cover
274
+ # "nogil" Python: modified attributes from 3.9
275
+ args = (
276
+ obj.co_argcount, obj.co_posonlyargcount,
277
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_framesize,
278
+ obj.co_ndefaultargs, obj.co_nmeta,
279
+ obj.co_flags, obj.co_code, obj.co_consts,
280
+ obj.co_varnames, obj.co_filename, obj.co_name,
281
+ obj.co_firstlineno, obj.co_lnotab, obj.co_exc_handlers,
282
+ obj.co_jump_table, obj.co_freevars, obj.co_cellvars,
283
+ obj.co_free2reg, obj.co_cell2reg
284
+ )
285
+ elif hasattr(obj, "co_posonlyargcount"):
286
+ # Backward compat for 3.9 and older
287
+ args = (
288
+ obj.co_argcount, obj.co_posonlyargcount,
289
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
290
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
291
+ obj.co_varnames, obj.co_filename, obj.co_name,
292
+ obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
293
+ obj.co_cellvars
294
+ )
295
+ else:
296
+ # Backward compat for even older versions of Python
297
+ args = (
298
+ obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
299
+ obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
300
+ obj.co_names, obj.co_varnames, obj.co_filename,
301
+ obj.co_name, obj.co_firstlineno, obj.co_lnotab,
302
+ obj.co_freevars, obj.co_cellvars
303
+ )
304
+ return types.CodeType, args
305
+
306
+
307
+ def _cell_reduce(obj):
308
+ """Cell (containing values of a function's free variables) reducer"""
309
+ try:
310
+ obj.cell_contents
311
+ except ValueError: # cell is empty
312
+ return _make_empty_cell, ()
313
+ else:
314
+ return _make_cell, (obj.cell_contents, )
315
+
316
+
317
+ def _classmethod_reduce(obj):
318
+ orig_func = obj.__func__
319
+ return type(obj), (orig_func,)
320
+
321
+
322
+ def _file_reduce(obj):
323
+ """Save a file"""
324
+ import io
325
+
326
+ if not hasattr(obj, "name") or not hasattr(obj, "mode"):
327
+ raise pickle.PicklingError(
328
+ "Cannot pickle files that do not map to an actual file"
329
+ )
330
+ if obj is sys.stdout:
331
+ return getattr, (sys, "stdout")
332
+ if obj is sys.stderr:
333
+ return getattr, (sys, "stderr")
334
+ if obj is sys.stdin:
335
+ raise pickle.PicklingError("Cannot pickle standard input")
336
+ if obj.closed:
337
+ raise pickle.PicklingError("Cannot pickle closed files")
338
+ if hasattr(obj, "isatty") and obj.isatty():
339
+ raise pickle.PicklingError(
340
+ "Cannot pickle files that map to tty objects"
341
+ )
342
+ if "r" not in obj.mode and "+" not in obj.mode:
343
+ raise pickle.PicklingError(
344
+ "Cannot pickle files that are not opened for reading: %s"
345
+ % obj.mode
346
+ )
347
+
348
+ name = obj.name
349
+
350
+ retval = io.StringIO()
351
+
352
+ try:
353
+ # Read the whole file
354
+ curloc = obj.tell()
355
+ obj.seek(0)
356
+ contents = obj.read()
357
+ obj.seek(curloc)
358
+ except IOError as e:
359
+ raise pickle.PicklingError(
360
+ "Cannot pickle file %s as it cannot be read" % name
361
+ ) from e
362
+ retval.write(contents)
363
+ retval.seek(curloc)
364
+
365
+ retval.name = name
366
+ return _file_reconstructor, (retval,)
367
+
368
+
369
+ def _getset_descriptor_reduce(obj):
370
+ return getattr, (obj.__objclass__, obj.__name__)
371
+
372
+
373
+ def _mappingproxy_reduce(obj):
374
+ return types.MappingProxyType, (dict(obj),)
375
+
376
+
377
+ def _memoryview_reduce(obj):
378
+ return bytes, (obj.tobytes(),)
379
+
380
+
381
+ def _module_reduce(obj):
382
+ if _should_pickle_by_reference(obj):
383
+ return subimport, (obj.__name__,)
384
+ else:
385
+ # Some external libraries can populate the "__builtins__" entry of a
386
+ # module's `__dict__` with unpicklable objects (see #316). For that
387
+ # reason, we do not attempt to pickle the "__builtins__" entry, and
388
+ # restore a default value for it at unpickling time.
389
+ state = obj.__dict__.copy()
390
+ state.pop('__builtins__', None)
391
+ return dynamic_subimport, (obj.__name__, state)
392
+
393
+
394
+ def _method_reduce(obj):
395
+ return (types.MethodType, (obj.__func__, obj.__self__))
396
+
397
+
398
+ def _logger_reduce(obj):
399
+ return logging.getLogger, (obj.name,)
400
+
401
+
402
+ def _root_logger_reduce(obj):
403
+ return logging.getLogger, ()
404
+
405
+
406
+ def _property_reduce(obj):
407
+ return property, (obj.fget, obj.fset, obj.fdel, obj.__doc__)
408
+
409
+
410
+ def _weakset_reduce(obj):
411
+ return weakref.WeakSet, (list(obj),)
412
+
413
+
414
+ def _dynamic_class_reduce(obj):
415
+ """
416
+ Save a class that can't be stored as module global.
417
+
418
+ This method is used to serialize classes that are defined inside
419
+ functions, or that otherwise can't be serialized as attribute lookups
420
+ from global modules.
421
+ """
422
+ if Enum is not None and issubclass(obj, Enum):
423
+ return (
424
+ _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj),
425
+ None, None, _class_setstate
426
+ )
427
+ else:
428
+ return (
429
+ _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj),
430
+ None, None, _class_setstate
431
+ )
432
+
433
+
434
+ def _class_reduce(obj):
435
+ """Select the reducer depending on the dynamic nature of the class obj"""
436
+ if obj is type(None): # noqa
437
+ return type, (None,)
438
+ elif obj is type(Ellipsis):
439
+ return type, (Ellipsis,)
440
+ elif obj is type(NotImplemented):
441
+ return type, (NotImplemented,)
442
+ elif obj in _BUILTIN_TYPE_NAMES:
443
+ return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],)
444
+ elif not _should_pickle_by_reference(obj):
445
+ return _dynamic_class_reduce(obj)
446
+ return NotImplemented
447
+
448
+
449
+ def _dict_keys_reduce(obj):
450
+ # Safer not to ship the full dict as sending the rest might
451
+ # be unintended and could potentially cause leaking of
452
+ # sensitive information
453
+ return _make_dict_keys, (list(obj), )
454
+
455
+
456
+ def _dict_values_reduce(obj):
457
+ # Safer not to ship the full dict as sending the rest might
458
+ # be unintended and could potentially cause leaking of
459
+ # sensitive information
460
+ return _make_dict_values, (list(obj), )
461
+
462
+
463
+ def _dict_items_reduce(obj):
464
+ return _make_dict_items, (dict(obj), )
465
+
466
+
467
+ def _odict_keys_reduce(obj):
468
+ # Safer not to ship the full dict as sending the rest might
469
+ # be unintended and could potentially cause leaking of
470
+ # sensitive information
471
+ return _make_dict_keys, (list(obj), True)
472
+
473
+
474
+ def _odict_values_reduce(obj):
475
+ # Safer not to ship the full dict as sending the rest might
476
+ # be unintended and could potentially cause leaking of
477
+ # sensitive information
478
+ return _make_dict_values, (list(obj), True)
479
+
480
+
481
+ def _odict_items_reduce(obj):
482
+ return _make_dict_items, (dict(obj), True)
483
+
484
+
485
+ # COLLECTIONS OF OBJECTS STATE SETTERS
486
+ # ------------------------------------
487
+ # state setters are called at unpickling time, once the object is created and
488
+ # it has to be updated to how it was at unpickling time.
489
+
490
+
491
+ def _function_setstate(obj, state):
492
+ """Update the state of a dynamic function.
493
+
494
+ As __closure__ and __globals__ are readonly attributes of a function, we
495
+ cannot rely on the native setstate routine of pickle.load_build, that calls
496
+ setattr on items of the slotstate. Instead, we have to modify them inplace.
497
+ """
498
+ state, slotstate = state
499
+ obj.__dict__.update(state)
500
+
501
+ obj_globals = slotstate.pop("__globals__")
502
+ obj_closure = slotstate.pop("__closure__")
503
+ # _cloudpickle_subimports is a set of submodules that must be loaded for
504
+ # the pickled function to work correctly at unpickling time. Now that these
505
+ # submodules are depickled (hence imported), they can be removed from the
506
+ # object's state (the object state only served as a reference holder to
507
+ # these submodules)
508
+ slotstate.pop("_cloudpickle_submodules")
509
+
510
+ obj.__globals__.update(obj_globals)
511
+ obj.__globals__["__builtins__"] = __builtins__
512
+
513
+ if obj_closure is not None:
514
+ for i, cell in enumerate(obj_closure):
515
+ try:
516
+ value = cell.cell_contents
517
+ except ValueError: # cell is empty
518
+ continue
519
+ cell_set(obj.__closure__[i], value)
520
+
521
+ for k, v in slotstate.items():
522
+ setattr(obj, k, v)
523
+
524
+
525
+ def _class_setstate(obj, state):
526
+ state, slotstate = state
527
+ registry = None
528
+ for attrname, attr in state.items():
529
+ if attrname == "_abc_impl":
530
+ registry = attr
531
+ else:
532
+ setattr(obj, attrname, attr)
533
+ if registry is not None:
534
+ for subclass in registry:
535
+ obj.register(subclass)
536
+
537
+ return obj
538
+
539
+
540
+ class CloudPickler(Pickler):
541
+ # set of reducers defined and used by cloudpickle (private)
542
+ _dispatch_table = {}
543
+ _dispatch_table[classmethod] = _classmethod_reduce
544
+ _dispatch_table[io.TextIOWrapper] = _file_reduce
545
+ _dispatch_table[logging.Logger] = _logger_reduce
546
+ _dispatch_table[logging.RootLogger] = _root_logger_reduce
547
+ _dispatch_table[memoryview] = _memoryview_reduce
548
+ _dispatch_table[property] = _property_reduce
549
+ _dispatch_table[staticmethod] = _classmethod_reduce
550
+ _dispatch_table[CellType] = _cell_reduce
551
+ _dispatch_table[types.CodeType] = _code_reduce
552
+ _dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce
553
+ _dispatch_table[types.ModuleType] = _module_reduce
554
+ _dispatch_table[types.MethodType] = _method_reduce
555
+ _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce
556
+ _dispatch_table[weakref.WeakSet] = _weakset_reduce
557
+ _dispatch_table[typing.TypeVar] = _typevar_reduce
558
+ _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce
559
+ _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce
560
+ _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce
561
+ _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce
562
+ _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce
563
+ _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce
564
+ _dispatch_table[abc.abstractmethod] = _classmethod_reduce
565
+ _dispatch_table[abc.abstractclassmethod] = _classmethod_reduce
566
+ _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce
567
+ _dispatch_table[abc.abstractproperty] = _property_reduce
568
+
569
+ dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table)
570
+
571
+ # function reducers are defined as instance methods of CloudPickler
572
+ # objects, as they rely on a CloudPickler attribute (globals_ref)
573
+ def _dynamic_function_reduce(self, func):
574
+ """Reduce a function that is not pickleable via attribute lookup."""
575
+ newargs = self._function_getnewargs(func)
576
+ state = _function_getstate(func)
577
+ return (_make_function, newargs, state, None, None,
578
+ _function_setstate)
579
+
580
+ def _function_reduce(self, obj):
581
+ """Reducer for function objects.
582
+
583
+ If obj is a top-level attribute of a file-backed module, this
584
+ reducer returns NotImplemented, making the CloudPickler fallback to
585
+ traditional _pickle.Pickler routines to save obj. Otherwise, it reduces
586
+ obj using a custom cloudpickle reducer designed specifically to handle
587
+ dynamic functions.
588
+
589
+ As opposed to cloudpickle.py, There no special handling for builtin
590
+ pypy functions because cloudpickle_fast is CPython-specific.
591
+ """
592
+ if _should_pickle_by_reference(obj):
593
+ return NotImplemented
594
+ else:
595
+ return self._dynamic_function_reduce(obj)
596
+
597
+ def _function_getnewargs(self, func):
598
+ code = func.__code__
599
+
600
+ # base_globals represents the future global namespace of func at
601
+ # unpickling time. Looking it up and storing it in
602
+ # CloudpiPickler.globals_ref allow functions sharing the same globals
603
+ # at pickling time to also share them once unpickled, at one condition:
604
+ # since globals_ref is an attribute of a CloudPickler instance, and
605
+ # that a new CloudPickler is created each time pickle.dump or
606
+ # pickle.dumps is called, functions also need to be saved within the
607
+ # same invocation of cloudpickle.dump/cloudpickle.dumps (for example:
608
+ # cloudpickle.dumps([f1, f2])). There is no such limitation when using
609
+ # CloudPickler.dump, as long as the multiple invocations are bound to
610
+ # the same CloudPickler.
611
+ base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
612
+
613
+ if base_globals == {}:
614
+ # Add module attributes used to resolve relative imports
615
+ # instructions inside func.
616
+ for k in ["__package__", "__name__", "__path__", "__file__"]:
617
+ if k in func.__globals__:
618
+ base_globals[k] = func.__globals__[k]
619
+
620
+ # Do not bind the free variables before the function is created to
621
+ # avoid infinite recursion.
622
+ if func.__closure__ is None:
623
+ closure = None
624
+ else:
625
+ closure = tuple(
626
+ _make_empty_cell() for _ in range(len(code.co_freevars)))
627
+
628
+ return code, base_globals, None, None, closure
629
+
630
+ def dump(self, obj):
631
+ try:
632
+ return Pickler.dump(self, obj)
633
+ except RuntimeError as e:
634
+ if "recursion" in e.args[0]:
635
+ msg = (
636
+ "Could not pickle object as excessively deep recursion "
637
+ "required."
638
+ )
639
+ raise pickle.PicklingError(msg) from e
640
+ else:
641
+ raise
642
+
643
+ if pickle.HIGHEST_PROTOCOL >= 5:
644
+ def __init__(self, file, protocol=None, buffer_callback=None):
645
+ if protocol is None:
646
+ protocol = DEFAULT_PROTOCOL
647
+ Pickler.__init__(
648
+ self, file, protocol=protocol, buffer_callback=buffer_callback
649
+ )
650
+ # map functions __globals__ attribute ids, to ensure that functions
651
+ # sharing the same global namespace at pickling time also share
652
+ # their global namespace at unpickling time.
653
+ self.globals_ref = {}
654
+ self.proto = int(protocol)
655
+ else:
656
+ def __init__(self, file, protocol=None):
657
+ if protocol is None:
658
+ protocol = DEFAULT_PROTOCOL
659
+ Pickler.__init__(self, file, protocol=protocol)
660
+ # map functions __globals__ attribute ids, to ensure that functions
661
+ # sharing the same global namespace at pickling time also share
662
+ # their global namespace at unpickling time.
663
+ self.globals_ref = {}
664
+ assert hasattr(self, 'proto')
665
+
666
+ if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY:
667
+ # Pickler is the C implementation of the CPython pickler and therefore
668
+ # we rely on reduce_override method to customize the pickler behavior.
669
+
670
+ # `CloudPickler.dispatch` is only left for backward compatibility - note
671
+ # that when using protocol 5, `CloudPickler.dispatch` is not an
672
+ # extension of `Pickler.dispatch` dictionary, because CloudPickler
673
+ # subclasses the C-implemented Pickler, which does not expose a
674
+ # `dispatch` attribute. Earlier versions of the protocol 5 CloudPickler
675
+ # used `CloudPickler.dispatch` as a class-level attribute storing all
676
+ # reducers implemented by cloudpickle, but the attribute name was not a
677
+ # great choice given the meaning of `CloudPickler.dispatch` when
678
+ # `CloudPickler` extends the pure-python pickler.
679
+ dispatch = dispatch_table
680
+
681
+ # Implementation of the reducer_override callback, in order to
682
+ # efficiently serialize dynamic functions and classes by subclassing
683
+ # the C-implemented Pickler.
684
+ # TODO: decorrelate reducer_override (which is tied to CPython's
685
+ # implementation - would it make sense to backport it to pypy? - and
686
+ # pickle's protocol 5 which is implementation agnostic. Currently, the
687
+ # availability of both notions coincide on CPython's pickle and the
688
+ # pickle5 backport, but it may not be the case anymore when pypy
689
+ # implements protocol 5
690
+
691
+ def reducer_override(self, obj):
692
+ """Type-agnostic reducing callback for function and classes.
693
+
694
+ For performance reasons, subclasses of the C _pickle.Pickler class
695
+ cannot register custom reducers for functions and classes in the
696
+ dispatch_table. Reducer for such types must instead implemented in
697
+ the special reducer_override method.
698
+
699
+ Note that method will be called for any object except a few
700
+ builtin-types (int, lists, dicts etc.), which differs from reducers
701
+ in the Pickler's dispatch_table, each of them being invoked for
702
+ objects of a specific type only.
703
+
704
+ This property comes in handy for classes: although most classes are
705
+ instances of the ``type`` metaclass, some of them can be instances
706
+ of other custom metaclasses (such as enum.EnumMeta for example). In
707
+ particular, the metaclass will likely not be known in advance, and
708
+ thus cannot be special-cased using an entry in the dispatch_table.
709
+ reducer_override, among other things, allows us to register a
710
+ reducer that will be called for any class, independently of its
711
+ type.
712
+
713
+
714
+ Notes:
715
+
716
+ * reducer_override has the priority over dispatch_table-registered
717
+ reducers.
718
+ * reducer_override can be used to fix other limitations of
719
+ cloudpickle for other types that suffered from type-specific
720
+ reducers, such as Exceptions. See
721
+ https://github.com/cloudpipe/cloudpickle/issues/248
722
+ """
723
+ if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch
724
+ return (
725
+ _create_parametrized_type_hint,
726
+ parametrized_type_hint_getinitargs(obj)
727
+ )
728
+ t = type(obj)
729
+ try:
730
+ is_anyclass = issubclass(t, type)
731
+ except TypeError: # t is not a class (old Boost; see SF #502085)
732
+ is_anyclass = False
733
+
734
+ if is_anyclass:
735
+ return _class_reduce(obj)
736
+ elif isinstance(obj, types.FunctionType):
737
+ return self._function_reduce(obj)
738
+ else:
739
+ # fallback to save_global, including the Pickler's
740
+ # dispatch_table
741
+ return NotImplemented
742
+
743
+ else:
744
+ # When reducer_override is not available, hack the pure-Python
745
+ # Pickler's types.FunctionType and type savers. Note: the type saver
746
+ # must override Pickler.save_global, because pickle.py contains a
747
+ # hard-coded call to save_global when pickling meta-classes.
748
+ dispatch = Pickler.dispatch.copy()
749
+
750
+ def _save_reduce_pickle5(self, func, args, state=None, listitems=None,
751
+ dictitems=None, state_setter=None, obj=None):
752
+ save = self.save
753
+ write = self.write
754
+ self.save_reduce(
755
+ func, args, state=None, listitems=listitems,
756
+ dictitems=dictitems, obj=obj
757
+ )
758
+ # backport of the Python 3.8 state_setter pickle operations
759
+ save(state_setter)
760
+ save(obj) # simple BINGET opcode as obj is already memoized.
761
+ save(state)
762
+ write(pickle.TUPLE2)
763
+ # Trigger a state_setter(obj, state) function call.
764
+ write(pickle.REDUCE)
765
+ # The purpose of state_setter is to carry-out an
766
+ # inplace modification of obj. We do not care about what the
767
+ # method might return, so its output is eventually removed from
768
+ # the stack.
769
+ write(pickle.POP)
770
+
771
+ def save_global(self, obj, name=None, pack=struct.pack):
772
+ """
773
+ Save a "global".
774
+
775
+ The name of this method is somewhat misleading: all types get
776
+ dispatched here.
777
+ """
778
+ if obj is type(None): # noqa
779
+ return self.save_reduce(type, (None,), obj=obj)
780
+ elif obj is type(Ellipsis):
781
+ return self.save_reduce(type, (Ellipsis,), obj=obj)
782
+ elif obj is type(NotImplemented):
783
+ return self.save_reduce(type, (NotImplemented,), obj=obj)
784
+ elif obj in _BUILTIN_TYPE_NAMES:
785
+ return self.save_reduce(
786
+ _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
787
+
788
+ if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch
789
+ # Parametrized typing constructs in Python < 3.7 are not
790
+ # compatible with type checks and ``isinstance`` semantics. For
791
+ # this reason, it is easier to detect them using a
792
+ # duck-typing-based check (``_is_parametrized_type_hint``) than
793
+ # to populate the Pickler's dispatch with type-specific savers.
794
+ self.save_reduce(
795
+ _create_parametrized_type_hint,
796
+ parametrized_type_hint_getinitargs(obj),
797
+ obj=obj
798
+ )
799
+ elif name is not None:
800
+ Pickler.save_global(self, obj, name=name)
801
+ elif not _should_pickle_by_reference(obj, name=name):
802
+ self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj)
803
+ else:
804
+ Pickler.save_global(self, obj, name=name)
805
+ dispatch[type] = save_global
806
+
807
+ def save_function(self, obj, name=None):
808
+ """ Registered with the dispatch to handle all function types.
809
+
810
+ Determines what kind of function obj is (e.g. lambda, defined at
811
+ interactive prompt, etc) and handles the pickling appropriately.
812
+ """
813
+ if _should_pickle_by_reference(obj, name=name):
814
+ return Pickler.save_global(self, obj, name=name)
815
+ elif PYPY and isinstance(obj.__code__, builtin_code_type):
816
+ return self.save_pypy_builtin_func(obj)
817
+ else:
818
+ return self._save_reduce_pickle5(
819
+ *self._dynamic_function_reduce(obj), obj=obj
820
+ )
821
+
822
+ def save_pypy_builtin_func(self, obj):
823
+ """Save pypy equivalent of builtin functions.
824
+ PyPy does not have the concept of builtin-functions. Instead,
825
+ builtin-functions are simple function instances, but with a
826
+ builtin-code attribute.
827
+ Most of the time, builtin functions should be pickled by attribute.
828
+ But PyPy has flaky support for __qualname__, so some builtin
829
+ functions such as float.__new__ will be classified as dynamic. For
830
+ this reason only, we created this special routine. Because
831
+ builtin-functions are not expected to have closure or globals,
832
+ there is no additional hack (compared the one already implemented
833
+ in pickle) to protect ourselves from reference cycles. A simple
834
+ (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note
835
+ also that PyPy improved their support for __qualname__ in v3.6, so
836
+ this routing should be removed when cloudpickle supports only PyPy
837
+ 3.6 and later.
838
+ """
839
+ rv = (types.FunctionType, (obj.__code__, {}, obj.__name__,
840
+ obj.__defaults__, obj.__closure__),
841
+ obj.__dict__)
842
+ self.save_reduce(*rv, obj=obj)
843
+
844
+ dispatch[types.FunctionType] = save_function
TTS/Kokoro/.venv/Lib/site-packages/srsly/cloudpickle/compat.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import sys
2
+
3
+
4
+ if sys.version_info < (3, 8):
5
+ try:
6
+ import pickle5 as pickle # noqa: F401
7
+ from pickle5 import Pickler # noqa: F401
8
+ except ImportError:
9
+ import pickle # noqa: F401
10
+
11
+ # Use the Python pickler for old CPython versions
12
+ from pickle import _Pickler as Pickler # noqa: F401
13
+ else:
14
+ import pickle # noqa: F401
15
+
16
+ # Pickler will the C implementation in CPython and the Python
17
+ # implementation in PyPy
18
+ from pickle import Pickler # noqa: F401
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__init__.py ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding: utf-8
2
+
3
+ import functools
4
+ import catalogue
5
+
6
+ # These need to be imported before packer and unpacker
7
+ from ._epoch import utc, epoch # noqa
8
+
9
+ from ._version import version
10
+ from .exceptions import *
11
+
12
+ # In msgpack-python these are put under a _cmsgpack module that textually includes
13
+ # them. I dislike this so I refactored it.
14
+ from ._packer import Packer as _Packer
15
+ from ._unpacker import unpackb as _unpackb
16
+ from ._unpacker import Unpacker as _Unpacker
17
+ from .ext import ExtType
18
+ from ._msgpack_numpy import encode_numpy as _encode_numpy
19
+ from ._msgpack_numpy import decode_numpy as _decode_numpy
20
+
21
+
22
+ msgpack_encoders = catalogue.create("srsly", "msgpack_encoders", entry_points=True)
23
+ msgpack_decoders = catalogue.create("srsly", "msgpack_decoders", entry_points=True)
24
+
25
+ msgpack_encoders.register("numpy", func=_encode_numpy)
26
+ msgpack_decoders.register("numpy", func=_decode_numpy)
27
+
28
+
29
+ # msgpack_numpy extensions
30
+ class Packer(_Packer):
31
+ def __init__(self, *args, **kwargs):
32
+ default = kwargs.get("default")
33
+ for encoder in msgpack_encoders.get_all().values():
34
+ default = functools.partial(encoder, chain=default)
35
+ kwargs["default"] = default
36
+ super(Packer, self).__init__(*args, **kwargs)
37
+
38
+
39
+ class Unpacker(_Unpacker):
40
+ def __init__(self, *args, **kwargs):
41
+ object_hook = kwargs.get("object_hook")
42
+ for decoder in msgpack_decoders.get_all().values():
43
+ object_hook = functools.partial(decoder, chain=object_hook)
44
+ kwargs["object_hook"] = object_hook
45
+ super(Unpacker, self).__init__(*args, **kwargs)
46
+
47
+
48
+ def pack(o, stream, **kwargs):
49
+ """
50
+ Pack an object and write it to a stream.
51
+ """
52
+ packer = Packer(**kwargs)
53
+ stream.write(packer.pack(o))
54
+
55
+
56
+ def packb(o, **kwargs):
57
+ """
58
+ Pack an object and return the packed bytes.
59
+ """
60
+ return Packer(**kwargs).pack(o)
61
+
62
+
63
+ def unpack(stream, **kwargs):
64
+ """
65
+ Unpack a packed object from a stream.
66
+ """
67
+ if "object_pairs_hook" not in kwargs:
68
+ object_hook = kwargs.get("object_hook")
69
+ for decoder in msgpack_decoders.get_all().values():
70
+ object_hook = functools.partial(decoder, chain=object_hook)
71
+ kwargs["object_hook"] = object_hook
72
+ data = stream.read()
73
+ return _unpackb(data, **kwargs)
74
+
75
+
76
+ def unpackb(packed, **kwargs):
77
+ """
78
+ Unpack a packed object.
79
+ """
80
+ if "object_pairs_hook" not in kwargs:
81
+ object_hook = kwargs.get("object_hook")
82
+ for decoder in msgpack_decoders.get_all().values():
83
+ object_hook = functools.partial(decoder, chain=object_hook)
84
+ kwargs["object_hook"] = object_hook
85
+ return _unpackb(packed, **kwargs)
86
+
87
+
88
+ # alias for compatibility to simplejson/marshal/pickle.
89
+ load = unpack
90
+ loads = unpackb
91
+
92
+ dump = pack
93
+ dumps = packb
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (4.1 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/_msgpack_numpy.cpython-312.pyc ADDED
Binary file (3.57 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/_version.cpython-312.pyc ADDED
Binary file (234 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/exceptions.cpython-312.pyc ADDED
Binary file (2.04 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/ext.cpython-312.pyc ADDED
Binary file (8.3 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/fallback.cpython-312.pyc ADDED
Binary file (41.5 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/__pycache__/util.cpython-312.pyc ADDED
Binary file (662 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/_epoch.cp312-win_amd64.pyd ADDED
Binary file (21.5 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/_epoch.cpp ADDED
The diff for this file is too large to render. See raw diff
 
TTS/Kokoro/.venv/Lib/site-packages/srsly/msgpack/_epoch.pyx ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ from cpython.datetime cimport import_datetime, datetime_new
2
+
3
+ import_datetime()
4
+ import datetime
5
+
6
+ utc = datetime.timezone.utc
7
+ epoch = datetime_new(1970, 1, 1, 0, 0, 0, 0, tz=utc)