techprotrade commited on
Commit
2b47bd3
·
verified ·
1 Parent(s): bc5eb4d

Add loan-advisor-agent

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +27 -0
  2. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pxd +42 -0
  3. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.py +459 -0
  4. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pyi +29 -0
  5. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/__init__.cpython-312.pyc +0 -0
  6. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_compute_docstrings.cpython-312.pyc +0 -0
  7. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_generated_version.cpython-312.pyc +0 -0
  8. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/compute.cpython-312.pyc +0 -0
  9. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/ipc.cpython-312.pyc +0 -0
  10. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/types.cpython-312.pyc +0 -0
  11. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/util.cpython-312.pyc +0 -0
  12. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.cp312-win_amd64.pyd +3 -0
  13. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pxd +44 -0
  14. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pyx +610 -0
  15. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.cp312-win_amd64.pyd +0 -0
  16. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.pyx +179 -0
  17. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.cp312-win_amd64.pyd +3 -0
  18. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pxd +72 -0
  19. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pyx +0 -0
  20. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute_docstrings.py +257 -0
  21. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.cp312-win_amd64.pyd +3 -0
  22. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pxd +55 -0
  23. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pyx +1626 -0
  24. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pxd +67 -0
  25. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pyx +1084 -0
  26. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.cp312-win_amd64.pyd +3 -0
  27. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pxd +183 -0
  28. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pyx +0 -0
  29. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.cp312-win_amd64.pyd +0 -0
  30. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.pyx +51 -0
  31. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.cp312-win_amd64.pyd +3 -0
  32. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pxd +43 -0
  33. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pyx +1113 -0
  34. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.cp312-win_amd64.pyd +0 -0
  35. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.pyx +178 -0
  36. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dlpack.pxi +46 -0
  37. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.cp312-win_amd64.pyd +0 -0
  38. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.pyx +117 -0
  39. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.cp312-win_amd64.pyd +3 -0
  40. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.pyx +0 -0
  41. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.cp312-win_amd64.pyd +3 -0
  42. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pxd +91 -0
  43. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pyx +1653 -0
  44. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.cp312-win_amd64.pyd +0 -0
  45. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.pyx +209 -0
  46. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_generated_version.py +34 -0
  47. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.cp312-win_amd64.pyd +0 -0
  48. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.pyx +157 -0
  49. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.cp312-win_amd64.pyd +3 -0
  50. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.pxd +36 -0
.gitattributes CHANGED
@@ -510,3 +510,30 @@ loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/pytables.cpytho
510
  loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/sql.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
511
  loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/stata.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
512
  loan-advisor-agent/.venv/Lib/site-packages/pandas.libs/msvcp140-a4c2229bdc2a2a630acdc095b4d86008.dll filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
510
  loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/sql.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
511
  loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/stata.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
512
  loan-advisor-agent/.venv/Lib/site-packages/pandas.libs/msvcp140-a4c2229bdc2a2a630acdc095b4d86008.dll filter=lfs diff=lfs merge=lfs -text
513
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
514
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
515
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
516
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
517
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
518
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
519
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
520
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
521
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_orc.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
522
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_parquet.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
523
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_parquet_encryption.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
524
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_s3fs.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
525
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_substrait.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
526
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow.dll filter=lfs diff=lfs merge=lfs -text
527
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow.lib filter=lfs diff=lfs merge=lfs -text
528
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_acero.dll filter=lfs diff=lfs merge=lfs -text
529
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_acero.lib filter=lfs diff=lfs merge=lfs -text
530
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_compute.dll filter=lfs diff=lfs merge=lfs -text
531
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_compute.lib filter=lfs diff=lfs merge=lfs -text
532
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_dataset.dll filter=lfs diff=lfs merge=lfs -text
533
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_dataset.lib filter=lfs diff=lfs merge=lfs -text
534
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_flight.dll filter=lfs diff=lfs merge=lfs -text
535
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_flight.lib filter=lfs diff=lfs merge=lfs -text
536
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.dll filter=lfs diff=lfs merge=lfs -text
537
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.lib filter=lfs diff=lfs merge=lfs -text
538
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.dll filter=lfs diff=lfs merge=lfs -text
539
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.lib filter=lfs diff=lfs merge=lfs -text
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pxd ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ from libcpp.memory cimport shared_ptr
19
+ from pyarrow.includes.libarrow cimport (CArray, CBuffer, CDataType,
20
+ CField, CRecordBatch, CSchema,
21
+ CTable, CTensor, CSparseCOOTensor,
22
+ CSparseCSRMatrix, CSparseCSCMatrix,
23
+ CSparseCSFTensor)
24
+
25
+ cdef extern from "arrow/python/pyarrow.h" namespace "arrow::py":
26
+ cdef int import_pyarrow() except -1
27
+ cdef object wrap_buffer(const shared_ptr[CBuffer]& buffer)
28
+ cdef object wrap_data_type(const shared_ptr[CDataType]& type)
29
+ cdef object wrap_field(const shared_ptr[CField]& field)
30
+ cdef object wrap_schema(const shared_ptr[CSchema]& schema)
31
+ cdef object wrap_array(const shared_ptr[CArray]& sp_array)
32
+ cdef object wrap_tensor(const shared_ptr[CTensor]& sp_tensor)
33
+ cdef object wrap_sparse_tensor_coo(
34
+ const shared_ptr[CSparseCOOTensor]& sp_sparse_tensor)
35
+ cdef object wrap_sparse_tensor_csr(
36
+ const shared_ptr[CSparseCSRMatrix]& sp_sparse_tensor)
37
+ cdef object wrap_sparse_tensor_csc(
38
+ const shared_ptr[CSparseCSCMatrix]& sp_sparse_tensor)
39
+ cdef object wrap_sparse_tensor_csf(
40
+ const shared_ptr[CSparseCSFTensor]& sp_sparse_tensor)
41
+ cdef object wrap_table(const shared_ptr[CTable]& ctable)
42
+ cdef object wrap_batch(const shared_ptr[CRecordBatch]& cbatch)
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.py ADDED
@@ -0,0 +1,459 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # flake8: noqa
19
+
20
+ """
21
+ PyArrow is the python implementation of Apache Arrow.
22
+
23
+ Apache Arrow is a cross-language development platform for in-memory data.
24
+ It specifies a standardized language-independent columnar memory format for
25
+ flat and hierarchical data, organized for efficient analytic operations on
26
+ modern hardware. It also provides computational libraries and zero-copy
27
+ streaming messaging and interprocess communication.
28
+
29
+ For more information see the official page at https://arrow.apache.org
30
+ """
31
+
32
+
33
+ # start delvewheel patch
34
+ def _delvewheel_patch_1_12_0():
35
+ import os
36
+ if os.path.isdir(libs_dir := os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'pyarrow.libs'))):
37
+ os.add_dll_directory(libs_dir)
38
+
39
+
40
+ _delvewheel_patch_1_12_0()
41
+ del _delvewheel_patch_1_12_0
42
+ # end delvewheel patch
43
+
44
+ import importlib as _importlib
45
+ import os as _os
46
+ import platform as _platform
47
+ import sys as _sys
48
+
49
+ try:
50
+ from ._generated_version import version as __version__
51
+ except ImportError:
52
+ # Package is not installed, parse git tag at runtime
53
+ try:
54
+ import setuptools_scm
55
+ # Code duplicated from setup.py to avoid a dependency on each other
56
+
57
+ def parse_git(root, **kwargs):
58
+ """
59
+ Parse function for setuptools_scm that ignores tags for non-C++
60
+ subprojects, e.g. apache-arrow-js-XXX tags.
61
+ """
62
+ from setuptools_scm.git import parse
63
+ kwargs['describe_command'] = \
64
+ "git describe --dirty --tags --long --match 'apache-arrow-[0-9]*.*'"
65
+ return parse(root, **kwargs)
66
+ __version__ = setuptools_scm.get_version('../',
67
+ parse=parse_git)
68
+ except ImportError:
69
+ __version__ = None
70
+
71
+ from pyarrow.lib import (BuildInfo, CppBuildInfo, RuntimeInfo, set_timezone_db_path,
72
+ MonthDayNano, VersionInfo, build_info, cpp_build_info,
73
+ cpp_version, cpp_version_info, runtime_info,
74
+ cpu_count, set_cpu_count, enable_signal_handlers,
75
+ io_thread_count, is_opentelemetry_enabled, set_io_thread_count)
76
+
77
+
78
+ def show_versions():
79
+ """
80
+ Print various version information, to help with error reporting.
81
+ """
82
+ def print_entry(label, value):
83
+ print(f"{label: <26}: {value: <8}")
84
+
85
+ print("pyarrow version info\n--------------------")
86
+ print_entry("Package kind", build_info.cpp_build_info.package_kind
87
+ if len(build_info.cpp_build_info.package_kind) > 0
88
+ else "not indicated")
89
+ print_entry("Arrow C++ library version", build_info.cpp_build_info.version)
90
+ print_entry("Arrow C++ compiler",
91
+ (f"{build_info.cpp_build_info.compiler_id} "
92
+ f"{build_info.cpp_build_info.compiler_version}"))
93
+ print_entry("Arrow C++ compiler flags", build_info.cpp_build_info.compiler_flags)
94
+ print_entry("Arrow C++ git revision", build_info.cpp_build_info.git_id)
95
+ print_entry("Arrow C++ git description", build_info.cpp_build_info.git_description)
96
+ print_entry("Arrow C++ build type", build_info.cpp_build_info.build_type)
97
+ print_entry("PyArrow build type", build_info.build_type)
98
+
99
+
100
+ def _module_is_available(module):
101
+ try:
102
+ _importlib.import_module(f'pyarrow.{module}')
103
+ except ImportError:
104
+ return False
105
+ else:
106
+ return True
107
+
108
+
109
+ def _filesystem_is_available(fs):
110
+ try:
111
+ import pyarrow.fs
112
+ except ImportError:
113
+ return False
114
+
115
+ try:
116
+ getattr(pyarrow.fs, fs)
117
+ except (ImportError, AttributeError):
118
+ return False
119
+ else:
120
+ return True
121
+
122
+
123
+ def show_info():
124
+ """
125
+ Print detailed version and platform information, for error reporting
126
+ """
127
+ show_versions()
128
+
129
+ def print_entry(label, value):
130
+ print(f" {label: <20}: {value: <8}")
131
+
132
+ print("\nPlatform:")
133
+ print_entry("OS / Arch", f"{_platform.system()} {_platform.machine()}")
134
+ print_entry("SIMD Level", runtime_info().simd_level)
135
+ print_entry("Detected SIMD Level", runtime_info().detected_simd_level)
136
+
137
+ pool = default_memory_pool()
138
+ print("\nMemory:")
139
+ print_entry("Default backend", pool.backend_name)
140
+ print_entry("Bytes allocated", f"{pool.bytes_allocated()} bytes")
141
+ print_entry("Max memory", f"{pool.max_memory()} bytes")
142
+ print_entry("Supported Backends", ', '.join(supported_memory_backends()))
143
+
144
+ print("\nOptional modules:")
145
+ modules = ["csv", "cuda", "dataset", "feather", "flight", "fs", "gandiva", "json",
146
+ "orc", "parquet"]
147
+ for module in modules:
148
+ status = "Enabled" if _module_is_available(module) else "-"
149
+ print(f" {module: <20}: {status: <8}")
150
+ print(f" {'opentelemetry': <20}: {'Enabled' if is_opentelemetry_enabled() else '-': <8}")
151
+
152
+ print("\nFilesystems:")
153
+ filesystems = ["AzureFileSystem", "GcsFileSystem",
154
+ "HadoopFileSystem", "S3FileSystem"]
155
+ for fs in filesystems:
156
+ status = "Enabled" if _filesystem_is_available(fs) else "-"
157
+ print(f" {fs: <20}: {status: <8}")
158
+
159
+ print("\nCompression Codecs:")
160
+ codecs = ["brotli", "bz2", "gzip", "lz4_frame", "lz4", "snappy", "zstd"]
161
+ for codec in codecs:
162
+ status = "Enabled" if Codec.is_available(codec) else "-"
163
+ print(f" {codec: <20}: {status: <8}")
164
+
165
+
166
+ from pyarrow.lib import (null, bool_,
167
+ int8, int16, int32, int64,
168
+ uint8, uint16, uint32, uint64,
169
+ time32, time64, timestamp, date32, date64, duration,
170
+ month_day_nano_interval,
171
+ float16, float32, float64,
172
+ binary, string, utf8, binary_view, string_view,
173
+ large_binary, large_string, large_utf8,
174
+ decimal32, decimal64, decimal128, decimal256,
175
+ list_, large_list, list_view, large_list_view,
176
+ map_, struct,
177
+ union, sparse_union, dense_union,
178
+ dictionary,
179
+ run_end_encoded,
180
+ bool8, fixed_shape_tensor, json_, opaque, uuid,
181
+ field,
182
+ type_for_alias,
183
+ DataType, DictionaryType, StructType,
184
+ ListType, LargeListType, FixedSizeListType,
185
+ ListViewType, LargeListViewType,
186
+ MapType, UnionType, SparseUnionType, DenseUnionType,
187
+ TimestampType, Time32Type, Time64Type, DurationType,
188
+ FixedSizeBinaryType,
189
+ Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,
190
+ BaseExtensionType, ExtensionType,
191
+ RunEndEncodedType, Bool8Type, FixedShapeTensorType,
192
+ JsonType, OpaqueType, UuidType,
193
+ UnknownExtensionType,
194
+ register_extension_type, unregister_extension_type,
195
+ DictionaryMemo,
196
+ KeyValueMetadata,
197
+ Field,
198
+ Schema,
199
+ schema,
200
+ unify_schemas,
201
+ Array, Tensor,
202
+ array, chunked_array, record_batch, nulls, repeat,
203
+ SparseCOOTensor, SparseCSRMatrix, SparseCSCMatrix,
204
+ SparseCSFTensor,
205
+ infer_type, from_numpy_dtype,
206
+ arange,
207
+ NullArray,
208
+ NumericArray, IntegerArray, FloatingPointArray,
209
+ BooleanArray,
210
+ Int8Array, UInt8Array,
211
+ Int16Array, UInt16Array,
212
+ Int32Array, UInt32Array,
213
+ Int64Array, UInt64Array,
214
+ HalfFloatArray, FloatArray, DoubleArray,
215
+ ListArray, LargeListArray, FixedSizeListArray,
216
+ ListViewArray, LargeListViewArray,
217
+ MapArray, UnionArray,
218
+ BinaryArray, StringArray,
219
+ LargeBinaryArray, LargeStringArray,
220
+ BinaryViewArray, StringViewArray,
221
+ FixedSizeBinaryArray,
222
+ DictionaryArray,
223
+ Date32Array, Date64Array, TimestampArray,
224
+ Time32Array, Time64Array, DurationArray,
225
+ MonthDayNanoIntervalArray,
226
+ Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
227
+ StructArray, ExtensionArray,
228
+ RunEndEncodedArray, Bool8Array, FixedShapeTensorArray,
229
+ JsonArray, OpaqueArray, UuidArray,
230
+ scalar, NA, _NULL as NULL, Scalar,
231
+ NullScalar, BooleanScalar,
232
+ Int8Scalar, Int16Scalar, Int32Scalar, Int64Scalar,
233
+ UInt8Scalar, UInt16Scalar, UInt32Scalar, UInt64Scalar,
234
+ HalfFloatScalar, FloatScalar, DoubleScalar,
235
+ Decimal32Scalar, Decimal64Scalar, Decimal128Scalar, Decimal256Scalar,
236
+ ListScalar, LargeListScalar, FixedSizeListScalar,
237
+ ListViewScalar, LargeListViewScalar,
238
+ Date32Scalar, Date64Scalar,
239
+ Time32Scalar, Time64Scalar,
240
+ TimestampScalar, DurationScalar,
241
+ MonthDayNanoIntervalScalar,
242
+ BinaryScalar, LargeBinaryScalar, BinaryViewScalar,
243
+ StringScalar, LargeStringScalar, StringViewScalar,
244
+ FixedSizeBinaryScalar, DictionaryScalar,
245
+ MapScalar, StructScalar, UnionScalar,
246
+ RunEndEncodedScalar, Bool8Scalar, ExtensionScalar,
247
+ FixedShapeTensorScalar, JsonScalar, OpaqueScalar, UuidScalar)
248
+
249
+ # Buffers, allocation
250
+ from pyarrow.lib import (DeviceAllocationType, Device, MemoryManager,
251
+ default_cpu_memory_manager)
252
+
253
+ from pyarrow.lib import (Buffer, ResizableBuffer, foreign_buffer, py_buffer,
254
+ Codec, compress, decompress, allocate_buffer)
255
+
256
+ from pyarrow.lib import (MemoryPool, LoggingMemoryPool, ProxyMemoryPool,
257
+ total_allocated_bytes, set_memory_pool,
258
+ default_memory_pool, system_memory_pool,
259
+ jemalloc_memory_pool, mimalloc_memory_pool,
260
+ logging_memory_pool, proxy_memory_pool,
261
+ log_memory_allocations, jemalloc_set_decay_ms,
262
+ supported_memory_backends)
263
+
264
+ # I/O
265
+ from pyarrow.lib import (NativeFile, PythonFile,
266
+ BufferedInputStream, BufferedOutputStream, CacheOptions,
267
+ CompressedInputStream, CompressedOutputStream,
268
+ TransformInputStream, transcoding_input_stream,
269
+ FixedSizeBufferWriter,
270
+ BufferReader, BufferOutputStream,
271
+ OSFile, MemoryMappedFile, memory_map,
272
+ create_memory_map, MockOutputStream,
273
+ input_stream, output_stream,
274
+ have_libhdfs)
275
+
276
+ from pyarrow.lib import (ChunkedArray, RecordBatch, Table, table,
277
+ concat_arrays, concat_tables, TableGroupBy,
278
+ RecordBatchReader, concat_batches)
279
+
280
+ # Exceptions
281
+ from pyarrow.lib import (ArrowCancelled,
282
+ ArrowCapacityError,
283
+ ArrowException,
284
+ ArrowKeyError,
285
+ ArrowIndexError,
286
+ ArrowInvalid,
287
+ ArrowIOError,
288
+ ArrowMemoryError,
289
+ ArrowNotImplementedError,
290
+ ArrowTypeError,
291
+ ArrowSerializationError)
292
+
293
+ from pyarrow.ipc import serialize_pandas, deserialize_pandas
294
+ import pyarrow.ipc as ipc
295
+
296
+ import pyarrow.types as types
297
+
298
+
299
+ # ----------------------------------------------------------------------
300
+ # Deprecations
301
+
302
+ from pyarrow.util import _deprecate_api, _deprecate_class
303
+
304
+
305
+ # TODO: Deprecate these somehow in the pyarrow namespace
306
+ from pyarrow.ipc import (Message, MessageReader, MetadataVersion,
307
+ RecordBatchFileReader, RecordBatchFileWriter,
308
+ RecordBatchStreamReader, RecordBatchStreamWriter)
309
+
310
+ # ----------------------------------------------------------------------
311
+ # Returning absolute path to the pyarrow include directory (if bundled, e.g. in
312
+ # wheels)
313
+
314
+
315
+ def get_include():
316
+ """
317
+ Return absolute path to directory containing Arrow C++ include
318
+ headers. Similar to numpy.get_include
319
+ """
320
+ # Use pyarrow.lib location instead of just __file__. That works
321
+ # for both editable and non-editable builds as it points
322
+ # to the actual location of the compiled C++ extension.
323
+ from pyarrow import lib as _lib
324
+ return _os.path.join(_os.path.dirname(_lib.__file__), 'include')
325
+
326
+
327
+ def _get_pkg_config_executable():
328
+ return _os.environ.get('PKG_CONFIG', 'pkg-config')
329
+
330
+
331
+ def _has_pkg_config(pkgname):
332
+ import subprocess
333
+ try:
334
+ return subprocess.call([_get_pkg_config_executable(),
335
+ '--exists', pkgname]) == 0
336
+ except FileNotFoundError:
337
+ return False
338
+
339
+
340
+ def _read_pkg_config_variable(pkgname, cli_args):
341
+ import subprocess
342
+ cmd = [_get_pkg_config_executable(), pkgname] + cli_args
343
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
344
+ stderr=subprocess.PIPE)
345
+ out, err = proc.communicate()
346
+ if proc.returncode != 0:
347
+ raise RuntimeError("pkg-config failed: " + err.decode('utf8'))
348
+ return out.rstrip().decode('utf8')
349
+
350
+
351
+ def get_libraries():
352
+ """
353
+ Return list of library names to include in the `libraries` argument for C
354
+ or Cython extensions using pyarrow
355
+ """
356
+ return ['arrow_python', 'arrow']
357
+
358
+
359
+ def create_library_symlinks():
360
+ """
361
+ With Linux and macOS wheels, the bundled shared libraries have an embedded
362
+ ABI version like libarrow.so.17 or libarrow.17.dylib and so linking to them
363
+ with -larrow won't work unless we create symlinks at locations like
364
+ site-packages/pyarrow/libarrow.so. This unfortunate workaround addresses
365
+ prior problems we had with shipping two copies of the shared libraries to
366
+ permit third party projects like turbodbc to build their C++ extensions
367
+ against the pyarrow wheels.
368
+
369
+ This function must only be invoked once and only when the shared libraries
370
+ are bundled with the Python package, which should only apply to wheel-based
371
+ installs. It requires write access to the site-packages/pyarrow directory
372
+ and so depending on your system may need to be run with root.
373
+ """
374
+ import glob
375
+ if _sys.platform == 'win32':
376
+ return
377
+ package_cwd = _os.path.dirname(__file__)
378
+
379
+ if _sys.platform == 'linux':
380
+ bundled_libs = glob.glob(_os.path.join(package_cwd, '*.so.*'))
381
+
382
+ def get_symlink_path(hard_path):
383
+ return hard_path.rsplit('.', 1)[0]
384
+ else:
385
+ bundled_libs = glob.glob(_os.path.join(package_cwd, '*.*.dylib'))
386
+
387
+ def get_symlink_path(hard_path):
388
+ return '.'.join((hard_path.rsplit('.', 2)[0], 'dylib'))
389
+
390
+ for lib_hard_path in bundled_libs:
391
+ symlink_path = get_symlink_path(lib_hard_path)
392
+ if _os.path.exists(symlink_path):
393
+ continue
394
+ try:
395
+ _os.symlink(lib_hard_path, symlink_path)
396
+ except PermissionError:
397
+ print("Tried creating symlink {}. If you need to link to "
398
+ "bundled shared libraries, run "
399
+ "pyarrow.create_library_symlinks() as root")
400
+
401
+
402
+ def get_library_dirs():
403
+ """
404
+ Return lists of directories likely to contain Arrow C++ libraries for
405
+ linking C or Cython extensions using pyarrow
406
+ """
407
+ # Use pyarrow.lib location instead of just __file__. That works
408
+ # for both editable and non-editable builds as it points
409
+ # to the actual location of the compiled C++ extension.
410
+ from pyarrow import lib as _lib
411
+ package_cwd = _os.path.dirname(_lib.__file__)
412
+ library_dirs = [package_cwd]
413
+
414
+ def append_library_dir(library_dir):
415
+ if library_dir not in library_dirs:
416
+ library_dirs.append(library_dir)
417
+
418
+ # Search library paths via pkg-config. This is necessary if the user
419
+ # installed libarrow and the other shared libraries manually and they
420
+ # are not shipped inside the pyarrow package (see also ARROW-2976).
421
+ pkg_config_executable = _os.environ.get('PKG_CONFIG') or 'pkg-config'
422
+ for pkgname in ["arrow", "arrow_python"]:
423
+ if _has_pkg_config(pkgname):
424
+ library_dir = _read_pkg_config_variable(pkgname,
425
+ ["--libs-only-L"])
426
+ # pkg-config output could be empty if Arrow is installed
427
+ # as a system package.
428
+ if library_dir:
429
+ if not library_dir.startswith("-L"):
430
+ raise ValueError(
431
+ "pkg-config --libs-only-L returned unexpected "
432
+ f"value {library_dir!r}")
433
+ append_library_dir(library_dir[2:])
434
+
435
+ if _sys.platform == 'win32':
436
+ # TODO(wesm): Is this necessary, or does setuptools within a conda
437
+ # installation add Library\lib to the linker path for MSVC?
438
+ python_base_install = _os.path.dirname(_sys.executable)
439
+ library_dir = _os.path.join(python_base_install, 'Library', 'lib')
440
+
441
+ if _os.path.exists(_os.path.join(library_dir, 'arrow.lib')):
442
+ append_library_dir(library_dir)
443
+
444
+ # GH-45530: Add pyarrow.libs dir containing delvewheel-mangled
445
+ # msvcp140.dll
446
+ pyarrow_libs_dir = _os.path.abspath(
447
+ _os.path.join(_os.path.dirname(__file__), _os.pardir, "pyarrow.libs")
448
+ )
449
+ if _os.path.exists(pyarrow_libs_dir):
450
+ append_library_dir(pyarrow_libs_dir)
451
+
452
+ # ARROW-4074: Allow for ARROW_HOME to be set to some other directory
453
+ if _os.environ.get('ARROW_HOME'):
454
+ append_library_dir(_os.path.join(_os.environ['ARROW_HOME'], 'lib'))
455
+ else:
456
+ # Python wheels bundle the Arrow libraries in the pyarrow directory.
457
+ append_library_dir(_os.path.dirname(_os.path.abspath(__file__)))
458
+
459
+ return library_dirs
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pyi ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ """Type stubs for PyArrow.
19
+
20
+ This is a placeholder stub file.
21
+ Complete type annotations will be added in subsequent PRs.
22
+ """
23
+
24
+ from typing import Any
25
+
26
+ # TODO(GH-48970): remove __getattr__ before release as this
27
+ # will annotate non-existing attributes as Any.
28
+ # https://github.com/apache/arrow/issues/48970
29
+ def __getattr__(name: str) -> Any: ...
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (22.4 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_compute_docstrings.cpython-312.pyc ADDED
Binary file (7.36 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_generated_version.cpython-312.pyc ADDED
Binary file (803 Bytes). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/compute.cpython-312.pyc ADDED
Binary file (27.2 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/ipc.cpython-312.pyc ADDED
Binary file (10.9 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/types.cpython-312.pyc ADDED
Binary file (18.1 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/util.cpython-312.pyc ADDED
Binary file (11.5 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e99622df3dd58eb8c038d9a1d7af9ee0c33008e8516bceaf5fc15b74291ab43a
3
+ size 183808
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pxd ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.lib cimport *
21
+ from pyarrow.includes.common cimport *
22
+ from pyarrow.includes.libarrow cimport *
23
+ from pyarrow.includes.libarrow_acero cimport *
24
+
25
+
26
+ cdef class ExecNodeOptions(_Weakrefable):
27
+ cdef:
28
+ shared_ptr[CExecNodeOptions] wrapped
29
+
30
+ cdef void init(self, const shared_ptr[CExecNodeOptions]& sp)
31
+ cdef inline shared_ptr[CExecNodeOptions] unwrap(self) nogil
32
+
33
+
34
+ cdef class Declaration(_Weakrefable):
35
+
36
+ cdef:
37
+ CDeclaration decl
38
+
39
+ cdef void init(self, const CDeclaration& c_decl)
40
+
41
+ @staticmethod
42
+ cdef wrap(const CDeclaration& c_decl)
43
+
44
+ cdef inline CDeclaration unwrap(self) nogil
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pyx ADDED
@@ -0,0 +1,610 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # ---------------------------------------------------------------------
19
+ # Low-level Acero bindings
20
+
21
+ # cython: profile=False
22
+ # distutils: language = c++
23
+ # cython: language_level = 3
24
+
25
+ from pyarrow.includes.common cimport *
26
+ from pyarrow.includes.libarrow cimport *
27
+ from pyarrow.includes.libarrow_acero cimport *
28
+ from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table,
29
+ RecordBatchReader)
30
+ from pyarrow.lib import frombytes, tobytes
31
+ from pyarrow._compute cimport (
32
+ Expression, FunctionOptions, _ensure_field_ref, _true,
33
+ unwrap_null_placement, unwrap_sort_keys
34
+ )
35
+
36
+
37
+ cdef class ExecNodeOptions(_Weakrefable):
38
+ """
39
+ Base class for the node options.
40
+
41
+ Use one of the subclasses to construct an options object.
42
+ """
43
+ __slots__ = () # avoid mistakingly creating attributes
44
+
45
+ cdef void init(self, const shared_ptr[CExecNodeOptions]& sp):
46
+ self.wrapped = sp
47
+
48
+ cdef inline shared_ptr[CExecNodeOptions] unwrap(self) nogil:
49
+ return self.wrapped
50
+
51
+
52
+ cdef class _TableSourceNodeOptions(ExecNodeOptions):
53
+
54
+ def _set_options(self, Table table):
55
+ cdef:
56
+ shared_ptr[CTable] c_table
57
+
58
+ c_table = pyarrow_unwrap_table(table)
59
+ self.wrapped.reset(
60
+ new CTableSourceNodeOptions(c_table)
61
+ )
62
+
63
+
64
+ class TableSourceNodeOptions(_TableSourceNodeOptions):
65
+ """
66
+ A Source node which accepts a table.
67
+
68
+ This is the option class for the "table_source" node factory.
69
+
70
+ Parameters
71
+ ----------
72
+ table : pyarrow.Table
73
+ The table which acts as the data source.
74
+ """
75
+
76
+ def __init__(self, Table table):
77
+ self._set_options(table)
78
+
79
+
80
+ cdef class _FilterNodeOptions(ExecNodeOptions):
81
+
82
+ def _set_options(self, Expression filter_expression not None):
83
+ self.wrapped.reset(
84
+ new CFilterNodeOptions(<CExpression>filter_expression.unwrap())
85
+ )
86
+
87
+
88
+ class FilterNodeOptions(_FilterNodeOptions):
89
+ """
90
+ Make a node which excludes some rows from batches passed through it.
91
+
92
+ This is the option class for the "filter" node factory.
93
+
94
+ The "filter" operation provides an option to define data filtering
95
+ criteria. It selects rows where the given expression evaluates to true.
96
+ Filters can be written using pyarrow.compute.Expression, and the
97
+ expression must have a return type of boolean.
98
+
99
+ Parameters
100
+ ----------
101
+ filter_expression : pyarrow.compute.Expression
102
+ """
103
+
104
+ def __init__(self, Expression filter_expression):
105
+ self._set_options(filter_expression)
106
+
107
+
108
+ cdef class _ProjectNodeOptions(ExecNodeOptions):
109
+
110
+ def _set_options(self, expressions, names=None):
111
+ cdef:
112
+ Expression expr
113
+ vector[CExpression] c_expressions
114
+ vector[c_string] c_names
115
+
116
+ for expr in expressions:
117
+ c_expressions.push_back(expr.unwrap())
118
+
119
+ if names is not None:
120
+ if len(names) != len(expressions):
121
+ raise ValueError(
122
+ "The number of names should be equal to the number of expressions"
123
+ )
124
+
125
+ for name in names:
126
+ c_names.push_back(<c_string>tobytes(name))
127
+
128
+ self.wrapped.reset(
129
+ new CProjectNodeOptions(c_expressions, c_names)
130
+ )
131
+ else:
132
+ self.wrapped.reset(
133
+ new CProjectNodeOptions(c_expressions)
134
+ )
135
+
136
+
137
+ class ProjectNodeOptions(_ProjectNodeOptions):
138
+ """
139
+ Make a node which executes expressions on input batches,
140
+ producing batches of the same length with new columns.
141
+
142
+ This is the option class for the "project" node factory.
143
+
144
+ The "project" operation rearranges, deletes, transforms, and
145
+ creates columns. Each output column is computed by evaluating
146
+ an expression against the source record batch. These must be
147
+ scalar expressions (expressions consisting of scalar literals,
148
+ field references and scalar functions, i.e. elementwise functions
149
+ that return one value for each input row independent of the value
150
+ of all other rows).
151
+
152
+ Parameters
153
+ ----------
154
+ expressions : list of pyarrow.compute.Expression
155
+ List of expressions to evaluate against the source batch. This must
156
+ be scalar expressions.
157
+ names : list of str, optional
158
+ List of names for each of the output columns (same length as
159
+ `expressions`). If `names` is not provided, the string
160
+ representations of exprs will be used.
161
+ """
162
+
163
+ def __init__(self, expressions, names=None):
164
+ self._set_options(expressions, names)
165
+
166
+
167
+ cdef class _AggregateNodeOptions(ExecNodeOptions):
168
+
169
+ def _set_options(self, aggregates, keys=None):
170
+ cdef:
171
+ CAggregate c_aggr
172
+ vector[CAggregate] c_aggregations
173
+ vector[CFieldRef] c_keys
174
+
175
+ for arg_names, func_name, opts, name in aggregates:
176
+ c_aggr.function = tobytes(func_name)
177
+ if opts is not None:
178
+ c_aggr.options = (<FunctionOptions?>opts).wrapped
179
+ else:
180
+ c_aggr.options = <shared_ptr[CFunctionOptions]>nullptr
181
+ if not isinstance(arg_names, (list, tuple)):
182
+ arg_names = [arg_names]
183
+ for arg in arg_names:
184
+ c_aggr.target.push_back(_ensure_field_ref(arg))
185
+ c_aggr.name = tobytes(name)
186
+
187
+ c_aggregations.push_back(move(c_aggr))
188
+
189
+ if keys is None:
190
+ keys = []
191
+ for name in keys:
192
+ c_keys.push_back(_ensure_field_ref(name))
193
+
194
+ self.wrapped.reset(
195
+ new CAggregateNodeOptions(c_aggregations, c_keys)
196
+ )
197
+
198
+
199
+ class AggregateNodeOptions(_AggregateNodeOptions):
200
+ """
201
+ Make a node which aggregates input batches, optionally grouped by keys.
202
+
203
+ This is the option class for the "aggregate" node factory.
204
+
205
+ Acero supports two types of aggregates: "scalar" aggregates,
206
+ and "hash" aggregates. Scalar aggregates reduce an array or scalar
207
+ input to a single scalar output (e.g. computing the mean of a column).
208
+ Hash aggregates act like GROUP BY in SQL and first partition data
209
+ based on one or more key columns, then reduce the data in each partition.
210
+ The aggregate node supports both types of computation, and can compute
211
+ any number of aggregations at once.
212
+
213
+ Parameters
214
+ ----------
215
+ aggregates : list of tuples
216
+ Aggregations which will be applied to the targeted fields.
217
+ Specified as a list of tuples, where each tuple is one aggregation
218
+ specification and consists of: aggregation target column(s) followed
219
+ by function name, aggregation function options object and the
220
+ output field name.
221
+ The target column(s) specification can be a single field reference,
222
+ an empty list or a list of fields unary, nullary and n-ary aggregation
223
+ functions respectively. Each field reference can be a string
224
+ column name or expression.
225
+ keys : list of field references, optional
226
+ Keys by which aggregations will be grouped. Each key can reference
227
+ a field using a string name or expression.
228
+ """
229
+
230
+ def __init__(self, aggregates, keys=None):
231
+ self._set_options(aggregates, keys)
232
+
233
+
234
+ cdef class _OrderByNodeOptions(ExecNodeOptions):
235
+
236
+ def _set_options(self, sort_keys, null_placement):
237
+ self.wrapped.reset(
238
+ new COrderByNodeOptions(
239
+ COrdering(unwrap_sort_keys(sort_keys, allow_str=False),
240
+ unwrap_null_placement(null_placement))
241
+ )
242
+ )
243
+
244
+
245
+ class OrderByNodeOptions(_OrderByNodeOptions):
246
+ """
247
+ Make a node which applies a new ordering to the data.
248
+
249
+ Currently this node works by accumulating all data, sorting, and then
250
+ emitting the new data with an updated batch index.
251
+ Larger-than-memory sort is not currently supported.
252
+
253
+ This is the option class for the "order_by" node factory.
254
+
255
+ Parameters
256
+ ----------
257
+ sort_keys : sequence of (name, order) tuples
258
+ Names of field/column keys to sort the input on,
259
+ along with the order each field/column is sorted in.
260
+ Accepted values for `order` are "ascending", "descending".
261
+ Each field reference can be a string column name or expression.
262
+ null_placement : str, default "at_end"
263
+ Where nulls in input should be sorted, only applying to
264
+ columns/fields mentioned in `sort_keys`.
265
+ Accepted values are "at_start", "at_end".
266
+ """
267
+
268
+ def __init__(self, sort_keys=(), *, null_placement="at_end"):
269
+ self._set_options(sort_keys, null_placement)
270
+
271
+
272
+ cdef class _HashJoinNodeOptions(ExecNodeOptions):
273
+
274
+ def _set_options(
275
+ self, join_type, left_keys, right_keys, left_output=None, right_output=None,
276
+ output_suffix_for_left="", output_suffix_for_right="", Expression filter_expression=None,
277
+ ):
278
+ cdef:
279
+ CJoinType c_join_type
280
+ vector[CFieldRef] c_left_keys
281
+ vector[CFieldRef] c_right_keys
282
+ vector[CFieldRef] c_left_output
283
+ vector[CFieldRef] c_right_output
284
+ CExpression c_filter_expression
285
+
286
+ # join type
287
+ if join_type == "left semi":
288
+ c_join_type = CJoinType_LEFT_SEMI
289
+ elif join_type == "right semi":
290
+ c_join_type = CJoinType_RIGHT_SEMI
291
+ elif join_type == "left anti":
292
+ c_join_type = CJoinType_LEFT_ANTI
293
+ elif join_type == "right anti":
294
+ c_join_type = CJoinType_RIGHT_ANTI
295
+ elif join_type == "inner":
296
+ c_join_type = CJoinType_INNER
297
+ elif join_type == "left outer":
298
+ c_join_type = CJoinType_LEFT_OUTER
299
+ elif join_type == "right outer":
300
+ c_join_type = CJoinType_RIGHT_OUTER
301
+ elif join_type == "full outer":
302
+ c_join_type = CJoinType_FULL_OUTER
303
+ else:
304
+ raise ValueError("Unsupported join type")
305
+
306
+ # left/right keys
307
+ if not isinstance(left_keys, (list, tuple)):
308
+ left_keys = [left_keys]
309
+ for key in left_keys:
310
+ c_left_keys.push_back(_ensure_field_ref(key))
311
+ if not isinstance(right_keys, (list, tuple)):
312
+ right_keys = [right_keys]
313
+ for key in right_keys:
314
+ c_right_keys.push_back(_ensure_field_ref(key))
315
+
316
+ if filter_expression is None:
317
+ c_filter_expression = _true
318
+ else:
319
+ c_filter_expression = filter_expression.unwrap()
320
+
321
+ # left/right output fields
322
+ if left_output is not None and right_output is not None:
323
+ for colname in left_output:
324
+ c_left_output.push_back(_ensure_field_ref(colname))
325
+ for colname in right_output:
326
+ c_right_output.push_back(_ensure_field_ref(colname))
327
+
328
+ self.wrapped.reset(
329
+ new CHashJoinNodeOptions(
330
+ c_join_type, c_left_keys, c_right_keys,
331
+ c_left_output, c_right_output,
332
+ c_filter_expression,
333
+ <c_string>tobytes(output_suffix_for_left),
334
+ <c_string>tobytes(output_suffix_for_right)
335
+ )
336
+ )
337
+ else:
338
+ self.wrapped.reset(
339
+ new CHashJoinNodeOptions(
340
+ c_join_type, c_left_keys, c_right_keys,
341
+ c_filter_expression,
342
+ <c_string>tobytes(output_suffix_for_left),
343
+ <c_string>tobytes(output_suffix_for_right)
344
+ )
345
+ )
346
+
347
+
348
+ class HashJoinNodeOptions(_HashJoinNodeOptions):
349
+ """
350
+ Make a node which implements join operation using hash join strategy.
351
+
352
+ This is the option class for the "hashjoin" node factory.
353
+
354
+ Parameters
355
+ ----------
356
+ join_type : str
357
+ Type of join. One of "left semi", "right semi", "left anti",
358
+ "right anti", "inner", "left outer", "right outer", "full outer".
359
+ left_keys : str, Expression or list
360
+ Key fields from left input. Each key can be a string column name
361
+ or a field expression, or a list of such field references.
362
+ right_keys : str, Expression or list
363
+ Key fields from right input. See `left_keys` for details.
364
+ left_output : list, optional
365
+ List of output fields passed from left input. If left and right
366
+ output fields are not specified, all valid fields from both left and
367
+ right input will be output. Each field can be a string column name
368
+ or a field expression.
369
+ right_output : list, optional
370
+ List of output fields passed from right input. If left and right
371
+ output fields are not specified, all valid fields from both left and
372
+ right input will be output. Each field can be a string column name
373
+ or a field expression.
374
+ output_suffix_for_left : str
375
+ Suffix added to names of output fields coming from left input
376
+ (used to distinguish, if necessary, between fields of the same
377
+ name in left and right input and can be left empty if there are
378
+ no name collisions).
379
+ output_suffix_for_right : str
380
+ Suffix added to names of output fields coming from right input,
381
+ see `output_suffix_for_left` for details.
382
+ filter_expression : pyarrow.compute.Expression
383
+ Residual filter which is applied to matching row.
384
+ """
385
+
386
+ def __init__(
387
+ self, join_type, left_keys, right_keys, left_output=None, right_output=None,
388
+ output_suffix_for_left="", output_suffix_for_right="", filter_expression=None,
389
+ ):
390
+ self._set_options(
391
+ join_type, left_keys, right_keys, left_output, right_output,
392
+ output_suffix_for_left, output_suffix_for_right, filter_expression
393
+ )
394
+
395
+
396
+ cdef class _AsofJoinNodeOptions(ExecNodeOptions):
397
+
398
+ def _set_options(self, left_on, left_by, right_on, right_by, tolerance):
399
+ cdef:
400
+ vector[CFieldRef] c_left_by
401
+ vector[CFieldRef] c_right_by
402
+ CAsofJoinKeys c_left_keys
403
+ CAsofJoinKeys c_right_keys
404
+ vector[CAsofJoinKeys] c_input_keys
405
+
406
+ # Prepare left AsofJoinNodeOption::Keys
407
+ if not isinstance(left_by, (list, tuple)):
408
+ left_by = [left_by]
409
+ for key in left_by:
410
+ c_left_by.push_back(_ensure_field_ref(key))
411
+
412
+ c_left_keys.on_key = _ensure_field_ref(left_on)
413
+ c_left_keys.by_key = c_left_by
414
+
415
+ c_input_keys.push_back(c_left_keys)
416
+
417
+ # Prepare right AsofJoinNodeOption::Keys
418
+ if not isinstance(right_by, (list, tuple)):
419
+ right_by = [right_by]
420
+ for key in right_by:
421
+ c_right_by.push_back(_ensure_field_ref(key))
422
+
423
+ c_right_keys.on_key = _ensure_field_ref(right_on)
424
+ c_right_keys.by_key = c_right_by
425
+
426
+ c_input_keys.push_back(c_right_keys)
427
+
428
+ self.wrapped.reset(
429
+ new CAsofJoinNodeOptions(
430
+ c_input_keys,
431
+ tolerance,
432
+ )
433
+ )
434
+
435
+
436
+ class AsofJoinNodeOptions(_AsofJoinNodeOptions):
437
+ """
438
+ Make a node which implements 'as of join' operation.
439
+
440
+ This is the option class for the "asofjoin" node factory.
441
+
442
+ Parameters
443
+ ----------
444
+ left_on : str, Expression
445
+ The left key on which the join operation should be performed.
446
+ Can be a string column name or a field expression.
447
+
448
+ An inexact match is used on the "on" key, i.e. a row is considered a
449
+ match if and only if ``right.on - left.on`` is in the range
450
+ ``[min(0, tolerance), max(0, tolerance)]``.
451
+
452
+ The input dataset must be sorted by the "on" key. Must be a single
453
+ field of a common type.
454
+
455
+ Currently, the "on" key must be an integer, date, or timestamp type.
456
+ left_by: str, Expression or list
457
+ The left keys on which the join operation should be performed.
458
+ Exact equality is used for each field of the "by" keys.
459
+ Each key can be a string column name or a field expression,
460
+ or a list of such field references.
461
+ right_on : str, Expression
462
+ The right key on which the join operation should be performed.
463
+ See `left_on` for details.
464
+ right_by: str, Expression or list
465
+ The right keys on which the join operation should be performed.
466
+ See `left_by` for details.
467
+ tolerance : int
468
+ The tolerance to use for the asof join. The tolerance is interpreted in
469
+ the same units as the "on" key.
470
+ """
471
+
472
+ def __init__(self, left_on, left_by, right_on, right_by, tolerance):
473
+ self._set_options(left_on, left_by, right_on, right_by, tolerance)
474
+
475
+
476
+ cdef class Declaration(_Weakrefable):
477
+ """
478
+ Helper class for declaring the nodes of an ExecPlan.
479
+
480
+ A Declaration represents an unconstructed ExecNode, and potentially
481
+ more since its inputs may also be Declarations or when constructed
482
+ with ``from_sequence``.
483
+
484
+ The possible ExecNodes to use are registered with a name,
485
+ the "factory name", and need to be specified using this name, together
486
+ with its corresponding ExecNodeOptions subclass.
487
+
488
+ Parameters
489
+ ----------
490
+ factory_name : str
491
+ The ExecNode factory name, such as "table_source", "filter",
492
+ "project" etc. See the ExecNodeOptions subclasses for the exact
493
+ factory names to use.
494
+ options : ExecNodeOptions
495
+ Corresponding ExecNodeOptions subclass (matching the factory name).
496
+ inputs : list of Declaration, optional
497
+ Input nodes for this declaration. Optional if the node is a source
498
+ node, or when the declaration gets combined later with
499
+ ``from_sequence``.
500
+
501
+ Returns
502
+ -------
503
+ Declaration
504
+ """
505
+ cdef void init(self, const CDeclaration& c_decl):
506
+ self.decl = c_decl
507
+
508
+ @staticmethod
509
+ cdef wrap(const CDeclaration& c_decl):
510
+ cdef Declaration self = Declaration.__new__(Declaration)
511
+ self.init(c_decl)
512
+ return self
513
+
514
+ cdef inline CDeclaration unwrap(self) nogil:
515
+ return self.decl
516
+
517
+ def __init__(self, factory_name, ExecNodeOptions options, inputs=None):
518
+ cdef:
519
+ c_string c_factory_name
520
+ CDeclaration c_decl
521
+ vector[CDeclaration.Input] c_inputs
522
+
523
+ c_factory_name = tobytes(factory_name)
524
+
525
+ if inputs is not None:
526
+ for ipt in inputs:
527
+ c_inputs.push_back(
528
+ CDeclaration.Input((<Declaration>ipt).unwrap())
529
+ )
530
+
531
+ c_decl = CDeclaration(c_factory_name, c_inputs, options.unwrap())
532
+ self.init(c_decl)
533
+
534
+ @staticmethod
535
+ def from_sequence(decls):
536
+ """
537
+ Convenience factory for the common case of a simple sequence of nodes.
538
+
539
+ Each of the declarations will be appended to the inputs of the
540
+ subsequent declaration, and the final modified declaration will
541
+ be returned.
542
+
543
+ Parameters
544
+ ----------
545
+ decls : list of Declaration
546
+
547
+ Returns
548
+ -------
549
+ Declaration
550
+ """
551
+ cdef:
552
+ vector[CDeclaration] c_decls
553
+ CDeclaration c_decl
554
+
555
+ for decl in decls:
556
+ c_decls.push_back((<Declaration> decl).unwrap())
557
+
558
+ c_decl = CDeclaration.Sequence(c_decls)
559
+ return Declaration.wrap(c_decl)
560
+
561
+ def __str__(self):
562
+ return frombytes(GetResultValue(DeclarationToString(self.decl)))
563
+
564
+ def __repr__(self):
565
+ return f"<pyarrow.acero.Declaration>\n{self}"
566
+
567
+ def to_table(self, bint use_threads=True):
568
+ """
569
+ Run the declaration and collect the results into a table.
570
+
571
+ This method will implicitly add a sink node to the declaration
572
+ to collect results into a table. It will then create an ExecPlan
573
+ from the declaration, start the exec plan, block until the plan
574
+ has finished, and return the created table.
575
+
576
+ Parameters
577
+ ----------
578
+ use_threads : bool, default True
579
+ If set to False, then all CPU work will be done on the calling
580
+ thread. I/O tasks will still happen on the I/O executor
581
+ and may be multi-threaded (but should not use significant CPU
582
+ resources).
583
+
584
+ Returns
585
+ -------
586
+ pyarrow.Table
587
+ """
588
+ cdef:
589
+ shared_ptr[CTable] c_table
590
+
591
+ with nogil:
592
+ c_table = GetResultValue(DeclarationToTable(self.unwrap(), use_threads))
593
+ return pyarrow_wrap_table(c_table)
594
+
595
+ def to_reader(self, bint use_threads=True):
596
+ """Run the declaration and return results as a RecordBatchReader.
597
+
598
+ For details about the parameters, see `to_table`.
599
+
600
+ Returns
601
+ -------
602
+ pyarrow.RecordBatchReader
603
+ """
604
+ cdef:
605
+ RecordBatchReader reader
606
+ reader = RecordBatchReader.__new__(RecordBatchReader)
607
+ reader.reader.reset(
608
+ GetResultValue(DeclarationToReader(self.unwrap(), use_threads)).release()
609
+ )
610
+ return reader
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.cp312-win_amd64.pyd ADDED
Binary file (91.6 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.pyx ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+
21
+ from pyarrow.lib import frombytes, tobytes
22
+ from pyarrow.includes.libarrow_fs cimport *
23
+ from pyarrow._fs cimport FileSystem
24
+
25
+
26
+ cdef class AzureFileSystem(FileSystem):
27
+ """
28
+ Azure Blob Storage backed FileSystem implementation
29
+
30
+ This implementation supports flat namespace and hierarchical namespace (HNS) a.k.a.
31
+ Data Lake Gen2 storage accounts. HNS will be automatically detected and HNS specific
32
+ features will be used when they provide a performance advantage. Azurite emulator is
33
+ also supported. Note: `/` is the only supported delimiter.
34
+
35
+ The storage account is considered the root of the filesystem. When enabled, containers
36
+ will be created or deleted during relevant directory operations. Obviously, this also
37
+ requires authentication with the additional permissions.
38
+
39
+ By default `DefaultAzureCredential <https://github.com/Azure/azure-sdk-for-cpp/blob/main/sdk/identity/azure-identity/README.md#defaultazurecredential>`__
40
+ is used for authentication. This means it will try several types of authentication
41
+ and go with the first one that works. If any authentication parameters are provided when
42
+ initialising the FileSystem, they will be used instead of the default credential.
43
+
44
+ Parameters
45
+ ----------
46
+ account_name : str
47
+ Azure Blob Storage account name. This is the globally unique identifier for the
48
+ storage account.
49
+ account_key : str, default None
50
+ Account key of the storage account. If sas_token and account_key are None the
51
+ default credential will be used. The parameters account_key and sas_token are
52
+ mutually exclusive.
53
+ blob_storage_authority : str, default None
54
+ hostname[:port] of the Blob Service. Defaults to `.blob.core.windows.net`. Useful
55
+ for connecting to a local emulator, like Azurite.
56
+ blob_storage_scheme : str, default None
57
+ Either `http` or `https`. Defaults to `https`. Useful for connecting to a local
58
+ emulator, like Azurite.
59
+ client_id : str, default None
60
+ The client ID (Application ID) for Azure Active Directory authentication.
61
+ Its interpretation depends on the credential type being used:
62
+
63
+ - For `ClientSecretCredential`: It is the Application (client) ID of your
64
+ registered Azure AD application (Service Principal). It must be provided
65
+ together with `tenant_id` and `client_secret` to use ClientSecretCredential.
66
+ - For `ManagedIdentityCredential`: It is the client ID of a specific
67
+ user-assigned managed identity. This is only necessary if you are using a
68
+ user-assigned managed identity and need to explicitly specify which one
69
+ (e.g., if the resource has multiple user-assigned identities). For
70
+ system-assigned managed identities, this parameter is typically not required.
71
+
72
+ client_secret : str, default None
73
+ Client secret for Azure Active Directory authentication. Must be provided together
74
+ with `tenant_id` and `client_id` to use ClientSecretCredential.
75
+ dfs_storage_authority : str, default None
76
+ hostname[:port] of the Data Lake Gen 2 Service. Defaults to
77
+ `.dfs.core.windows.net`. Useful for connecting to a local emulator, like Azurite.
78
+ dfs_storage_scheme : str, default None
79
+ Either `http` or `https`. Defaults to `https`. Useful for connecting to a local
80
+ emulator, like Azurite.
81
+ sas_token : str, default None
82
+ SAS token for the storage account, used as an alternative to account_key. If sas_token
83
+ and account_key are None the default credential will be used. The parameters
84
+ account_key and sas_token are mutually exclusive.
85
+ tenant_id : str, default None
86
+ Tenant ID for Azure Active Directory authentication. Must be provided together with
87
+ `client_id` and `client_secret` to use ClientSecretCredential.
88
+
89
+ Examples
90
+ --------
91
+ >>> from pyarrow import fs
92
+ >>> azure_fs = fs.AzureFileSystem(account_name='myaccount')
93
+ >>> azurite_fs = fs.AzureFileSystem(
94
+ ... account_name='devstoreaccount1',
95
+ ... account_key='Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
96
+ ... blob_storage_authority='127.0.0.1:10000',
97
+ ... dfs_storage_authority='127.0.0.1:10000',
98
+ ... blob_storage_scheme='http',
99
+ ... dfs_storage_scheme='http',
100
+ ... )
101
+
102
+ For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
103
+ """
104
+ cdef:
105
+ CAzureFileSystem* azurefs
106
+
107
+ def __init__(self, account_name, *, account_key=None, blob_storage_authority=None,
108
+ blob_storage_scheme=None, client_id=None, client_secret=None,
109
+ dfs_storage_authority=None, dfs_storage_scheme=None,
110
+ sas_token=None, tenant_id=None):
111
+ cdef:
112
+ CAzureOptions options
113
+ shared_ptr[CAzureFileSystem] wrapped
114
+
115
+ options.account_name = tobytes(account_name)
116
+ if blob_storage_authority:
117
+ options.blob_storage_authority = tobytes(blob_storage_authority)
118
+ if dfs_storage_authority:
119
+ options.dfs_storage_authority = tobytes(dfs_storage_authority)
120
+ if blob_storage_scheme:
121
+ options.blob_storage_scheme = tobytes(blob_storage_scheme)
122
+ if dfs_storage_scheme:
123
+ options.dfs_storage_scheme = tobytes(dfs_storage_scheme)
124
+
125
+ if account_key and sas_token:
126
+ raise ValueError("Cannot specify both account_key and sas_token.")
127
+
128
+ if (tenant_id or client_id or client_secret):
129
+ if not client_id:
130
+ raise ValueError("client_id must be specified")
131
+ if not tenant_id and not client_secret:
132
+ options.ConfigureManagedIdentityCredential(tobytes(client_id))
133
+ elif tenant_id and client_secret:
134
+ options.ConfigureClientSecretCredential(
135
+ tobytes(tenant_id), tobytes(client_id), tobytes(client_secret)
136
+ )
137
+ else:
138
+ raise ValueError(
139
+ "Invalid Azure credential configuration: "
140
+ "For ManagedIdentityCredential, provide only client_id. "
141
+ "For ClientSecretCredential, provide tenant_id, client_id, and client_secret."
142
+ )
143
+ elif account_key:
144
+ options.ConfigureAccountKeyCredential(tobytes(account_key))
145
+ elif sas_token:
146
+ options.ConfigureSASCredential(tobytes(sas_token))
147
+ else:
148
+ options.ConfigureDefaultCredential()
149
+
150
+ with nogil:
151
+ wrapped = GetResultValue(CAzureFileSystem.Make(options))
152
+
153
+ self.init(<shared_ptr[CFileSystem]> wrapped)
154
+
155
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped):
156
+ FileSystem.init(self, wrapped)
157
+ self.azurefs = <CAzureFileSystem*> wrapped.get()
158
+
159
+ @staticmethod
160
+ def _reconstruct(kwargs):
161
+ # __reduce__ doesn't allow passing named arguments directly to the
162
+ # reconstructor, hence this wrapper.
163
+ return AzureFileSystem(**kwargs)
164
+
165
+ def __reduce__(self):
166
+ cdef CAzureOptions opts = self.azurefs.options()
167
+ return (
168
+ AzureFileSystem._reconstruct, (dict(
169
+ account_name=frombytes(opts.account_name),
170
+ account_key=frombytes(opts.AccountKey()),
171
+ blob_storage_authority=frombytes(opts.blob_storage_authority),
172
+ blob_storage_scheme=frombytes(opts.blob_storage_scheme),
173
+ client_id=frombytes(opts.ClientId()),
174
+ client_secret=frombytes(opts.ClientSecret()),
175
+ dfs_storage_authority=frombytes(opts.dfs_storage_authority),
176
+ dfs_storage_scheme=frombytes(opts.dfs_storage_scheme),
177
+ sas_token=frombytes(opts.SasToken()),
178
+ tenant_id=frombytes(opts.TenantId())
179
+ ),))
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed827e1771ac06884deef9368797fc21fe7028ce2e1efb91d6be237514dfe771
3
+ size 748032
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pxd ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.lib cimport *
21
+ from pyarrow.includes.common cimport *
22
+ from pyarrow.includes.libarrow cimport *
23
+
24
+ cdef class UdfContext(_Weakrefable):
25
+ cdef:
26
+ CUdfContext c_context
27
+
28
+ cdef void init(self, const CUdfContext& c_context)
29
+
30
+
31
+ cdef class FunctionOptions(_Weakrefable):
32
+ cdef:
33
+ shared_ptr[CFunctionOptions] wrapped
34
+
35
+ cdef const CFunctionOptions* get_options(self) except NULL
36
+ cdef void init(self, const shared_ptr[CFunctionOptions]& sp)
37
+
38
+ cdef inline shared_ptr[CFunctionOptions] unwrap(self)
39
+
40
+
41
+ cdef class _SortOptions(FunctionOptions):
42
+ pass
43
+
44
+
45
+ cdef CExpression _bind(Expression filter, Schema schema) except *
46
+
47
+
48
+ cdef class Expression(_Weakrefable):
49
+
50
+ cdef:
51
+ CExpression expr
52
+
53
+ cdef void init(self, const CExpression& sp)
54
+
55
+ @staticmethod
56
+ cdef wrap(const CExpression& sp)
57
+
58
+ cdef inline CExpression unwrap(self)
59
+
60
+ @staticmethod
61
+ cdef Expression _expr_or_scalar(object expr)
62
+
63
+
64
+ cdef CExpression _true
65
+
66
+ cdef CFieldRef _ensure_field_ref(value) except *
67
+
68
+ cdef vector[CSortKey] unwrap_sort_keys(sort_keys, allow_str=*) except *
69
+
70
+ cdef CSortOrder unwrap_sort_order(order) except *
71
+
72
+ cdef CNullPlacement unwrap_null_placement(null_placement) except *
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pyx ADDED
The diff for this file is too large to render. See raw diff
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute_docstrings.py ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ """
19
+ Custom documentation additions for compute functions.
20
+ """
21
+
22
+ function_doc_additions = {}
23
+
24
+ function_doc_additions["filter"] = """
25
+ Examples
26
+ --------
27
+ >>> import pyarrow as pa
28
+ >>> arr = pa.array(["a", "b", "c", None, "e"])
29
+ >>> mask = pa.array([True, False, None, False, True])
30
+ >>> arr.filter(mask)
31
+ <pyarrow.lib.StringArray object at ...>
32
+ [
33
+ "a",
34
+ "e"
35
+ ]
36
+ >>> arr.filter(mask, null_selection_behavior='emit_null')
37
+ <pyarrow.lib.StringArray object at ...>
38
+ [
39
+ "a",
40
+ null,
41
+ "e"
42
+ ]
43
+ """
44
+
45
+ function_doc_additions["mode"] = """
46
+ Examples
47
+ --------
48
+ >>> import pyarrow as pa
49
+ >>> import pyarrow.compute as pc
50
+ >>> arr = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
51
+ >>> modes = pc.mode(arr, 2)
52
+ >>> modes[0]
53
+ <pyarrow.StructScalar: [('mode', 2), ('count', 5)]>
54
+ >>> modes[1]
55
+ <pyarrow.StructScalar: [('mode', 1), ('count', 2)]>
56
+ """
57
+
58
+ function_doc_additions["min"] = """
59
+ Examples
60
+ --------
61
+ >>> import pyarrow as pa
62
+ >>> import pyarrow.compute as pc
63
+ >>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
64
+ >>> pc.min(arr1)
65
+ <pyarrow.Int64Scalar: 1>
66
+
67
+ Using ``skip_nulls`` to handle null values.
68
+
69
+ >>> arr2 = pa.array([1.0, None, 2.0, 3.0])
70
+ >>> pc.min(arr2)
71
+ <pyarrow.DoubleScalar: 1.0>
72
+ >>> pc.min(arr2, skip_nulls=False)
73
+ <pyarrow.DoubleScalar: None>
74
+
75
+ Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
76
+
77
+ >>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
78
+ >>> pc.min(arr3)
79
+ <pyarrow.DoubleScalar: 1.0>
80
+ >>> pc.min(arr3, options=pc.ScalarAggregateOptions(min_count=3))
81
+ <pyarrow.DoubleScalar: 1.0>
82
+ >>> pc.min(arr3, options=pc.ScalarAggregateOptions(min_count=4))
83
+ <pyarrow.DoubleScalar: None>
84
+
85
+ This function also works with string values.
86
+
87
+ >>> arr4 = pa.array(["z", None, "y", "x"])
88
+ >>> pc.min(arr4)
89
+ <pyarrow.StringScalar: 'x'>
90
+ """
91
+
92
+ function_doc_additions["max"] = """
93
+ Examples
94
+ --------
95
+ >>> import pyarrow as pa
96
+ >>> import pyarrow.compute as pc
97
+ >>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
98
+ >>> pc.max(arr1)
99
+ <pyarrow.Int64Scalar: 3>
100
+
101
+ Using ``skip_nulls`` to handle null values.
102
+
103
+ >>> arr2 = pa.array([1.0, None, 2.0, 3.0])
104
+ >>> pc.max(arr2)
105
+ <pyarrow.DoubleScalar: 3.0>
106
+ >>> pc.max(arr2, skip_nulls=False)
107
+ <pyarrow.DoubleScalar: None>
108
+
109
+ Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
110
+
111
+ >>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
112
+ >>> pc.max(arr3)
113
+ <pyarrow.DoubleScalar: 3.0>
114
+ >>> pc.max(arr3, options=pc.ScalarAggregateOptions(min_count=3))
115
+ <pyarrow.DoubleScalar: 3.0>
116
+ >>> pc.max(arr3, options=pc.ScalarAggregateOptions(min_count=4))
117
+ <pyarrow.DoubleScalar: None>
118
+
119
+ This function also works with string values.
120
+
121
+ >>> arr4 = pa.array(["z", None, "y", "x"])
122
+ >>> pc.max(arr4)
123
+ <pyarrow.StringScalar: 'z'>
124
+ """
125
+
126
+ function_doc_additions["min_max"] = """
127
+ Examples
128
+ --------
129
+ >>> import pyarrow as pa
130
+ >>> import pyarrow.compute as pc
131
+ >>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
132
+ >>> pc.min_max(arr1)
133
+ <pyarrow.StructScalar: [('min', 1), ('max', 3)]>
134
+
135
+ Using ``skip_nulls`` to handle null values.
136
+
137
+ >>> arr2 = pa.array([1.0, None, 2.0, 3.0])
138
+ >>> pc.min_max(arr2)
139
+ <pyarrow.StructScalar: [('min', 1.0), ('max', 3.0)]>
140
+ >>> pc.min_max(arr2, skip_nulls=False)
141
+ <pyarrow.StructScalar: [('min', None), ('max', None)]>
142
+
143
+ Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
144
+
145
+ >>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
146
+ >>> pc.min_max(arr3)
147
+ <pyarrow.StructScalar: [('min', 1.0), ('max', 3.0)]>
148
+ >>> pc.min_max(arr3, options=pc.ScalarAggregateOptions(min_count=3))
149
+ <pyarrow.StructScalar: [('min', 1.0), ('max', 3.0)]>
150
+ >>> pc.min_max(arr3, options=pc.ScalarAggregateOptions(min_count=4))
151
+ <pyarrow.StructScalar: [('min', None), ('max', None)]>
152
+
153
+ This function also works with string values.
154
+
155
+ >>> arr4 = pa.array(["z", None, "y", "x"])
156
+ >>> pc.min_max(arr4)
157
+ <pyarrow.StructScalar: [('min', 'x'), ('max', 'z')]>
158
+ """
159
+
160
+ function_doc_additions["first"] = """
161
+ Examples
162
+ --------
163
+ >>> import pyarrow as pa
164
+ >>> import pyarrow.compute as pc
165
+ >>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
166
+ >>> pc.first(arr1)
167
+ <pyarrow.Int64Scalar: 1>
168
+
169
+ Using ``skip_nulls`` to handle null values.
170
+
171
+ >>> arr2 = pa.array([None, 1.0, 2.0, 3.0])
172
+ >>> pc.first(arr2)
173
+ <pyarrow.DoubleScalar: 1.0>
174
+ >>> pc.first(arr2, skip_nulls=False)
175
+ <pyarrow.DoubleScalar: None>
176
+
177
+ Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
178
+
179
+ >>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
180
+ >>> pc.first(arr3)
181
+ <pyarrow.DoubleScalar: 1.0>
182
+ >>> pc.first(arr3, options=pc.ScalarAggregateOptions(min_count=3))
183
+ <pyarrow.DoubleScalar: 1.0>
184
+ >>> pc.first(arr3, options=pc.ScalarAggregateOptions(min_count=4))
185
+ <pyarrow.DoubleScalar: None>
186
+
187
+ See Also
188
+ --------
189
+ pyarrow.compute.first_last
190
+ pyarrow.compute.last
191
+ """
192
+
193
+ function_doc_additions["last"] = """
194
+ Examples
195
+ --------
196
+ >>> import pyarrow as pa
197
+ >>> import pyarrow.compute as pc
198
+ >>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
199
+ >>> pc.last(arr1)
200
+ <pyarrow.Int64Scalar: 2>
201
+
202
+ Using ``skip_nulls`` to handle null values.
203
+
204
+ >>> arr2 = pa.array([1.0, 2.0, 3.0, None])
205
+ >>> pc.last(arr2)
206
+ <pyarrow.DoubleScalar: 3.0>
207
+ >>> pc.last(arr2, skip_nulls=False)
208
+ <pyarrow.DoubleScalar: None>
209
+
210
+ Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
211
+
212
+ >>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
213
+ >>> pc.last(arr3)
214
+ <pyarrow.DoubleScalar: 3.0>
215
+ >>> pc.last(arr3, options=pc.ScalarAggregateOptions(min_count=3))
216
+ <pyarrow.DoubleScalar: 3.0>
217
+ >>> pc.last(arr3, options=pc.ScalarAggregateOptions(min_count=4))
218
+ <pyarrow.DoubleScalar: None>
219
+
220
+ See Also
221
+ --------
222
+ pyarrow.compute.first
223
+ pyarrow.compute.first_last
224
+ """
225
+
226
+ function_doc_additions["first_last"] = """
227
+ Examples
228
+ --------
229
+ >>> import pyarrow as pa
230
+ >>> import pyarrow.compute as pc
231
+ >>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
232
+ >>> pc.first_last(arr1)
233
+ <pyarrow.StructScalar: [('first', 1), ('last', 2)]>
234
+
235
+ Using ``skip_nulls`` to handle null values.
236
+
237
+ >>> arr2 = pa.array([None, 2.0, 3.0, None])
238
+ >>> pc.first_last(arr2)
239
+ <pyarrow.StructScalar: [('first', 2.0), ('last', 3.0)]>
240
+ >>> pc.first_last(arr2, skip_nulls=False)
241
+ <pyarrow.StructScalar: [('first', None), ('last', None)]>
242
+
243
+ Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
244
+
245
+ >>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
246
+ >>> pc.first_last(arr3)
247
+ <pyarrow.StructScalar: [('first', 1.0), ('last', 3.0)]>
248
+ >>> pc.first_last(arr3, options=pc.ScalarAggregateOptions(min_count=3))
249
+ <pyarrow.StructScalar: [('first', 1.0), ('last', 3.0)]>
250
+ >>> pc.first_last(arr3, options=pc.ScalarAggregateOptions(min_count=4))
251
+ <pyarrow.StructScalar: [('first', None), ('last', None)]>
252
+
253
+ See Also
254
+ --------
255
+ pyarrow.compute.first
256
+ pyarrow.compute.last
257
+ """
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5dcfd55b795fc02b51d0cb4801cebd830b6f90ba674f0b2d49115db488ae2594
3
+ size 270336
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pxd ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.includes.libarrow cimport *
21
+ from pyarrow.lib cimport _Weakrefable
22
+
23
+
24
+ cdef class ConvertOptions(_Weakrefable):
25
+ cdef:
26
+ unique_ptr[CCSVConvertOptions] options
27
+
28
+ @staticmethod
29
+ cdef ConvertOptions wrap(CCSVConvertOptions options)
30
+
31
+
32
+ cdef class ParseOptions(_Weakrefable):
33
+ cdef:
34
+ unique_ptr[CCSVParseOptions] options
35
+ object _invalid_row_handler
36
+
37
+ @staticmethod
38
+ cdef ParseOptions wrap(CCSVParseOptions options)
39
+
40
+
41
+ cdef class ReadOptions(_Weakrefable):
42
+ cdef:
43
+ unique_ptr[CCSVReadOptions] options
44
+ public object encoding
45
+
46
+ @staticmethod
47
+ cdef ReadOptions wrap(CCSVReadOptions options)
48
+
49
+
50
+ cdef class WriteOptions(_Weakrefable):
51
+ cdef:
52
+ unique_ptr[CCSVWriteOptions] options
53
+
54
+ @staticmethod
55
+ cdef WriteOptions wrap(CCSVWriteOptions options)
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pyx ADDED
@@ -0,0 +1,1626 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: profile=False
19
+ # distutils: language = c++
20
+ # cython: language_level = 3
21
+
22
+ from cython.operator cimport dereference as deref
23
+
24
+ from collections import namedtuple
25
+ from collections.abc import Mapping
26
+
27
+ from pyarrow.includes.common cimport *
28
+ from pyarrow.includes.libarrow cimport *
29
+ from pyarrow.includes.libarrow_python cimport *
30
+ from pyarrow.lib cimport (check_status, Field, MemoryPool, Schema,
31
+ RecordBatchReader, ensure_type,
32
+ maybe_unbox_memory_pool, get_input_stream,
33
+ get_writer, native_transcoding_input_stream,
34
+ pyarrow_unwrap_batch, pyarrow_unwrap_schema,
35
+ pyarrow_unwrap_table, pyarrow_wrap_schema,
36
+ pyarrow_wrap_table, pyarrow_wrap_data_type,
37
+ pyarrow_unwrap_data_type, Table, RecordBatch,
38
+ StopToken, _CRecordBatchWriter)
39
+ from pyarrow.lib import frombytes, tobytes, SignalStopHandler
40
+
41
+
42
+ cdef unsigned char _single_char(s) except 0:
43
+ val = ord(s)
44
+ if val == 0 or val > 127:
45
+ raise ValueError("Expecting an ASCII character")
46
+ return <unsigned char> val
47
+
48
+
49
+ _InvalidRow = namedtuple(
50
+ "_InvalidRow", ("expected_columns", "actual_columns", "number", "text"),
51
+ module=__name__)
52
+
53
+
54
+ class InvalidRow(_InvalidRow):
55
+ """
56
+ Description of an invalid row in a CSV file.
57
+
58
+ Parameters
59
+ ----------
60
+ expected_columns : int
61
+ The expected number of columns in the row.
62
+ actual_columns : int
63
+ The actual number of columns in the row.
64
+ number : int or None
65
+ The physical row number if known, otherwise None.
66
+ text : str
67
+ The contents of the row.
68
+ """
69
+ __slots__ = ()
70
+
71
+
72
+ cdef CInvalidRowResult _handle_invalid_row(
73
+ handler, const CCSVInvalidRow& c_row) except CInvalidRowResult_Error:
74
+ # A negative row number means undetermined (because of parallel reading)
75
+ row_number = c_row.number if c_row.number >= 0 else None
76
+ row = InvalidRow(c_row.expected_columns, c_row.actual_columns,
77
+ row_number, frombytes(<c_string> c_row.text))
78
+ result = handler(row)
79
+ if result == 'error':
80
+ return CInvalidRowResult_Error
81
+ elif result == 'skip':
82
+ return CInvalidRowResult_Skip
83
+ else:
84
+ raise ValueError("Invalid return value for invalid row handler: "
85
+ f"expected 'error' or 'skip', got {result!r}")
86
+
87
+
88
+ cdef class ReadOptions(_Weakrefable):
89
+ """
90
+ Options for reading CSV files.
91
+
92
+ Parameters
93
+ ----------
94
+ use_threads : bool, optional (default True)
95
+ Whether to use multiple threads to accelerate reading
96
+ block_size : int, optional
97
+ How much bytes to process at a time from the input stream.
98
+ This will determine multi-threading granularity as well as
99
+ the size of individual record batches or table chunks.
100
+ Minimum valid value for block size is 1
101
+ skip_rows : int, optional (default 0)
102
+ The number of rows to skip before the column names (if any)
103
+ and the CSV data.
104
+ skip_rows_after_names : int, optional (default 0)
105
+ The number of rows to skip after the column names.
106
+ This number can be larger than the number of rows in one
107
+ block, and empty rows are counted.
108
+ The order of application is as follows:
109
+ - `skip_rows` is applied (if non-zero);
110
+ - column names are read (unless `column_names` is set);
111
+ - `skip_rows_after_names` is applied (if non-zero).
112
+ column_names : list, optional
113
+ The column names of the target table. If empty, fall back on
114
+ `autogenerate_column_names`.
115
+ autogenerate_column_names : bool, optional (default False)
116
+ Whether to autogenerate column names if `column_names` is empty.
117
+ If true, column names will be of the form "f0", "f1"...
118
+ If false, column names will be read from the first CSV row
119
+ after `skip_rows`.
120
+ encoding : str, optional (default 'utf8')
121
+ The character encoding of the CSV data. Columns that cannot
122
+ decode using this encoding can still be read as Binary.
123
+
124
+ Examples
125
+ --------
126
+
127
+ Defining an example data:
128
+
129
+ >>> import io
130
+ >>> s = "1,2,3\\nFlamingo,2,2022-03-01\\nHorse,4,2022-03-02\\nBrittle stars,5,2022-03-03\\nCentipede,100,2022-03-04"
131
+ >>> print(s)
132
+ 1,2,3
133
+ Flamingo,2,2022-03-01
134
+ Horse,4,2022-03-02
135
+ Brittle stars,5,2022-03-03
136
+ Centipede,100,2022-03-04
137
+
138
+ Ignore the first numbered row and substitute it with defined
139
+ or autogenerated column names:
140
+
141
+ >>> from pyarrow import csv
142
+ >>> read_options = csv.ReadOptions(
143
+ ... column_names=["animals", "n_legs", "entry"],
144
+ ... skip_rows=1)
145
+ >>> csv.read_csv(io.BytesIO(s.encode()), read_options=read_options)
146
+ pyarrow.Table
147
+ animals: string
148
+ n_legs: int64
149
+ entry: date32[day]
150
+ ----
151
+ animals: [["Flamingo","Horse","Brittle stars","Centipede"]]
152
+ n_legs: [[2,4,5,100]]
153
+ entry: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
154
+
155
+ >>> read_options = csv.ReadOptions(autogenerate_column_names=True,
156
+ ... skip_rows=1)
157
+ >>> csv.read_csv(io.BytesIO(s.encode()), read_options=read_options)
158
+ pyarrow.Table
159
+ f0: string
160
+ f1: int64
161
+ f2: date32[day]
162
+ ----
163
+ f0: [["Flamingo","Horse","Brittle stars","Centipede"]]
164
+ f1: [[2,4,5,100]]
165
+ f2: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
166
+
167
+ Remove the first 2 rows of the data:
168
+
169
+ >>> read_options = csv.ReadOptions(skip_rows_after_names=2)
170
+ >>> csv.read_csv(io.BytesIO(s.encode()), read_options=read_options)
171
+ pyarrow.Table
172
+ 1: string
173
+ 2: int64
174
+ 3: date32[day]
175
+ ----
176
+ 1: [["Brittle stars","Centipede"]]
177
+ 2: [[5,100]]
178
+ 3: [[2022-03-03,2022-03-04]]
179
+ """
180
+
181
+ # Avoid mistakingly creating attributes
182
+ __slots__ = ()
183
+
184
+ # __init__() is not called when unpickling, initialize storage here
185
+ def __cinit__(self, *argw, **kwargs):
186
+ self.options.reset(new CCSVReadOptions(CCSVReadOptions.Defaults()))
187
+
188
+ def __init__(self, *, use_threads=None, block_size=None, skip_rows=None,
189
+ skip_rows_after_names=None, column_names=None,
190
+ autogenerate_column_names=None, encoding='utf8'):
191
+ if use_threads is not None:
192
+ self.use_threads = use_threads
193
+ if block_size is not None:
194
+ self.block_size = block_size
195
+ if skip_rows is not None:
196
+ self.skip_rows = skip_rows
197
+ if skip_rows_after_names is not None:
198
+ self.skip_rows_after_names = skip_rows_after_names
199
+ if column_names is not None:
200
+ self.column_names = column_names
201
+ if autogenerate_column_names is not None:
202
+ self.autogenerate_column_names= autogenerate_column_names
203
+ # Python-specific option
204
+ self.encoding = encoding
205
+
206
+ @property
207
+ def use_threads(self):
208
+ """
209
+ Whether to use multiple threads to accelerate reading.
210
+ """
211
+ return deref(self.options).use_threads
212
+
213
+ @use_threads.setter
214
+ def use_threads(self, value):
215
+ deref(self.options).use_threads = value
216
+
217
+ @property
218
+ def block_size(self):
219
+ """
220
+ How much bytes to process at a time from the input stream.
221
+ This will determine multi-threading granularity as well as
222
+ the size of individual record batches or table chunks.
223
+ """
224
+ return deref(self.options).block_size
225
+
226
+ @block_size.setter
227
+ def block_size(self, value):
228
+ deref(self.options).block_size = value
229
+
230
+ @property
231
+ def skip_rows(self):
232
+ """
233
+ The number of rows to skip before the column names (if any)
234
+ and the CSV data.
235
+ See `skip_rows_after_names` for interaction description
236
+ """
237
+ return deref(self.options).skip_rows
238
+
239
+ @skip_rows.setter
240
+ def skip_rows(self, value):
241
+ deref(self.options).skip_rows = value
242
+
243
+ @property
244
+ def skip_rows_after_names(self):
245
+ """
246
+ The number of rows to skip after the column names.
247
+ This number can be larger than the number of rows in one
248
+ block, and empty rows are counted.
249
+ The order of application is as follows:
250
+ - `skip_rows` is applied (if non-zero);
251
+ - column names are read (unless `column_names` is set);
252
+ - `skip_rows_after_names` is applied (if non-zero).
253
+ """
254
+ return deref(self.options).skip_rows_after_names
255
+
256
+ @skip_rows_after_names.setter
257
+ def skip_rows_after_names(self, value):
258
+ deref(self.options).skip_rows_after_names = value
259
+
260
+ @property
261
+ def column_names(self):
262
+ """
263
+ The column names of the target table. If empty, fall back on
264
+ `autogenerate_column_names`.
265
+ """
266
+ return [frombytes(s) for s in deref(self.options).column_names]
267
+
268
+ @column_names.setter
269
+ def column_names(self, value):
270
+ deref(self.options).column_names.clear()
271
+ for item in value:
272
+ deref(self.options).column_names.push_back(tobytes(item))
273
+
274
+ @property
275
+ def autogenerate_column_names(self):
276
+ """
277
+ Whether to autogenerate column names if `column_names` is empty.
278
+ If true, column names will be of the form "f0", "f1"...
279
+ If false, column names will be read from the first CSV row
280
+ after `skip_rows`.
281
+ """
282
+ return deref(self.options).autogenerate_column_names
283
+
284
+ @autogenerate_column_names.setter
285
+ def autogenerate_column_names(self, value):
286
+ deref(self.options).autogenerate_column_names = value
287
+
288
+ def validate(self):
289
+ check_status(deref(self.options).Validate())
290
+
291
+ def equals(self, ReadOptions other):
292
+ """
293
+ Parameters
294
+ ----------
295
+ other : pyarrow.csv.ReadOptions
296
+
297
+ Returns
298
+ -------
299
+ bool
300
+ """
301
+ return (
302
+ self.use_threads == other.use_threads and
303
+ self.block_size == other.block_size and
304
+ self.skip_rows == other.skip_rows and
305
+ self.skip_rows_after_names == other.skip_rows_after_names and
306
+ self.column_names == other.column_names and
307
+ self.autogenerate_column_names ==
308
+ other.autogenerate_column_names and
309
+ self.encoding == other.encoding
310
+ )
311
+
312
+ @staticmethod
313
+ cdef ReadOptions wrap(CCSVReadOptions options):
314
+ out = ReadOptions()
315
+ out.options.reset(new CCSVReadOptions(move(options)))
316
+ out.encoding = 'utf8' # No way to know this
317
+ return out
318
+
319
+ def __getstate__(self):
320
+ return (self.use_threads, self.block_size, self.skip_rows,
321
+ self.column_names, self.autogenerate_column_names,
322
+ self.encoding, self.skip_rows_after_names)
323
+
324
+ def __setstate__(self, state):
325
+ (self.use_threads, self.block_size, self.skip_rows,
326
+ self.column_names, self.autogenerate_column_names,
327
+ self.encoding, self.skip_rows_after_names) = state
328
+
329
+ def __eq__(self, other):
330
+ try:
331
+ return self.equals(other)
332
+ except TypeError:
333
+ return False
334
+
335
+ def _repr_base(self):
336
+ return (f"""
337
+ use_threads={self.use_threads},
338
+ block_size={self.block_size},
339
+ skip_rows={self.skip_rows},
340
+ skip_rows_after_names={self.skip_rows_after_names},
341
+ column_names={self.column_names},
342
+ autogenerate_column_names={self.autogenerate_column_names},
343
+ encoding={self.encoding!r}""")
344
+
345
+ def __repr__(self):
346
+ return (f"<pyarrow.csv.ReadOptions>({self._repr_base()})")
347
+
348
+ def __str__(self):
349
+ return (f"ReadOptions({self._repr_base()})")
350
+
351
+
352
+ cdef class ParseOptions(_Weakrefable):
353
+ """
354
+ Options for parsing CSV files.
355
+
356
+ Parameters
357
+ ----------
358
+ delimiter : 1-character string, optional (default ',')
359
+ The character delimiting individual cells in the CSV data.
360
+ quote_char : 1-character string or False, optional (default '"')
361
+ The character used optionally for quoting CSV values
362
+ (False if quoting is not allowed).
363
+ double_quote : bool, optional (default True)
364
+ Whether two quotes in a quoted CSV value denote a single quote
365
+ in the data.
366
+ escape_char : 1-character string or False, optional (default False)
367
+ The character used optionally for escaping special characters
368
+ (False if escaping is not allowed).
369
+ newlines_in_values : bool, optional (default False)
370
+ Whether newline characters are allowed in CSV values.
371
+ Setting this to True reduces the performance of multi-threaded
372
+ CSV reading.
373
+ ignore_empty_lines : bool, optional (default True)
374
+ Whether empty lines are ignored in CSV input.
375
+ If False, an empty line is interpreted as containing a single empty
376
+ value (assuming a one-column CSV file).
377
+ invalid_row_handler : callable, optional (default None)
378
+ If not None, this object is called for each CSV row that fails
379
+ parsing (because of a mismatching number of columns).
380
+ It should accept a single InvalidRow argument and return either
381
+ "skip" or "error" depending on the desired outcome.
382
+
383
+ Examples
384
+ --------
385
+
386
+ Defining an example file from bytes object:
387
+
388
+ >>> import io
389
+ >>> s = (
390
+ ... "animals;n_legs;entry\\n"
391
+ ... "Flamingo;2;2022-03-01\\n"
392
+ ... "# Comment here:\\n"
393
+ ... "Horse;4;2022-03-02\\n"
394
+ ... "Brittle stars;5;2022-03-03\\n"
395
+ ... "Centipede;100;2022-03-04"
396
+ ... )
397
+ >>> print(s)
398
+ animals;n_legs;entry
399
+ Flamingo;2;2022-03-01
400
+ # Comment here:
401
+ Horse;4;2022-03-02
402
+ Brittle stars;5;2022-03-03
403
+ Centipede;100;2022-03-04
404
+ >>> source = io.BytesIO(s.encode())
405
+
406
+ Read the data from a file skipping rows with comments
407
+ and defining the delimiter:
408
+
409
+ >>> from pyarrow import csv
410
+ >>> def skip_comment(row):
411
+ ... if row.text.startswith("# "):
412
+ ... return 'skip'
413
+ ... else:
414
+ ... return 'error'
415
+ ...
416
+ >>> parse_options = csv.ParseOptions(delimiter=";", invalid_row_handler=skip_comment)
417
+ >>> csv.read_csv(source, parse_options=parse_options)
418
+ pyarrow.Table
419
+ animals: string
420
+ n_legs: int64
421
+ entry: date32[day]
422
+ ----
423
+ animals: [["Flamingo","Horse","Brittle stars","Centipede"]]
424
+ n_legs: [[2,4,5,100]]
425
+ entry: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
426
+ """
427
+ __slots__ = ()
428
+
429
+ def __cinit__(self, *argw, **kwargs):
430
+ self._invalid_row_handler = None
431
+ self.options.reset(new CCSVParseOptions(CCSVParseOptions.Defaults()))
432
+
433
+ def __init__(self, *, delimiter=None, quote_char=None, double_quote=None,
434
+ escape_char=None, newlines_in_values=None,
435
+ ignore_empty_lines=None, invalid_row_handler=None):
436
+ if delimiter is not None:
437
+ self.delimiter = delimiter
438
+ if quote_char is not None:
439
+ self.quote_char = quote_char
440
+ if double_quote is not None:
441
+ self.double_quote = double_quote
442
+ if escape_char is not None:
443
+ self.escape_char = escape_char
444
+ if newlines_in_values is not None:
445
+ self.newlines_in_values = newlines_in_values
446
+ if ignore_empty_lines is not None:
447
+ self.ignore_empty_lines = ignore_empty_lines
448
+ if invalid_row_handler is not None:
449
+ self.invalid_row_handler = invalid_row_handler
450
+
451
+ @property
452
+ def delimiter(self):
453
+ """
454
+ The character delimiting individual cells in the CSV data.
455
+ """
456
+ return chr(deref(self.options).delimiter)
457
+
458
+ @delimiter.setter
459
+ def delimiter(self, value):
460
+ deref(self.options).delimiter = _single_char(value)
461
+
462
+ @property
463
+ def quote_char(self):
464
+ """
465
+ The character used optionally for quoting CSV values
466
+ (False if quoting is not allowed).
467
+ """
468
+ if deref(self.options).quoting:
469
+ return chr(deref(self.options).quote_char)
470
+ else:
471
+ return False
472
+
473
+ @quote_char.setter
474
+ def quote_char(self, value):
475
+ if value is False:
476
+ deref(self.options).quoting = False
477
+ else:
478
+ deref(self.options).quote_char = _single_char(value)
479
+ deref(self.options).quoting = True
480
+
481
+ @property
482
+ def double_quote(self):
483
+ """
484
+ Whether two quotes in a quoted CSV value denote a single quote
485
+ in the data.
486
+ """
487
+ return deref(self.options).double_quote
488
+
489
+ @double_quote.setter
490
+ def double_quote(self, value):
491
+ deref(self.options).double_quote = value
492
+
493
+ @property
494
+ def escape_char(self):
495
+ """
496
+ The character used optionally for escaping special characters
497
+ (False if escaping is not allowed).
498
+ """
499
+ if deref(self.options).escaping:
500
+ return chr(deref(self.options).escape_char)
501
+ else:
502
+ return False
503
+
504
+ @escape_char.setter
505
+ def escape_char(self, value):
506
+ if value is False:
507
+ deref(self.options).escaping = False
508
+ else:
509
+ deref(self.options).escape_char = _single_char(value)
510
+ deref(self.options).escaping = True
511
+
512
+ @property
513
+ def newlines_in_values(self):
514
+ """
515
+ Whether newline characters are allowed in CSV values.
516
+ Setting this to True reduces the performance of multi-threaded
517
+ CSV reading.
518
+ """
519
+ return deref(self.options).newlines_in_values
520
+
521
+ @newlines_in_values.setter
522
+ def newlines_in_values(self, value):
523
+ deref(self.options).newlines_in_values = value
524
+
525
+ @property
526
+ def ignore_empty_lines(self):
527
+ """
528
+ Whether empty lines are ignored in CSV input.
529
+ If False, an empty line is interpreted as containing a single empty
530
+ value (assuming a one-column CSV file).
531
+ """
532
+ return deref(self.options).ignore_empty_lines
533
+
534
+ @property
535
+ def invalid_row_handler(self):
536
+ """
537
+ Optional handler for invalid rows.
538
+
539
+ If not None, this object is called for each CSV row that fails
540
+ parsing (because of a mismatching number of columns).
541
+ It should accept a single InvalidRow argument and return either
542
+ "skip" or "error" depending on the desired outcome.
543
+ """
544
+ return self._invalid_row_handler
545
+
546
+ @invalid_row_handler.setter
547
+ def invalid_row_handler(self, value):
548
+ if value is not None and not callable(value):
549
+ raise TypeError("Expected callable or None, "
550
+ f"got instance of {type(value)!r}")
551
+ self._invalid_row_handler = value
552
+ deref(self.options).invalid_row_handler = MakeInvalidRowHandler(
553
+ <function[PyInvalidRowCallback]> &_handle_invalid_row, value)
554
+
555
+ @ignore_empty_lines.setter
556
+ def ignore_empty_lines(self, value):
557
+ deref(self.options).ignore_empty_lines = value
558
+
559
+ def validate(self):
560
+ check_status(deref(self.options).Validate())
561
+
562
+ def equals(self, ParseOptions other):
563
+ """
564
+ Parameters
565
+ ----------
566
+ other : pyarrow.csv.ParseOptions
567
+
568
+ Returns
569
+ -------
570
+ bool
571
+ """
572
+ return (
573
+ self.delimiter == other.delimiter and
574
+ self.quote_char == other.quote_char and
575
+ self.double_quote == other.double_quote and
576
+ self.escape_char == other.escape_char and
577
+ self.newlines_in_values == other.newlines_in_values and
578
+ self.ignore_empty_lines == other.ignore_empty_lines and
579
+ self._invalid_row_handler == other._invalid_row_handler
580
+ )
581
+
582
+ @staticmethod
583
+ cdef ParseOptions wrap(CCSVParseOptions options):
584
+ out = ParseOptions()
585
+ out.options.reset(new CCSVParseOptions(move(options)))
586
+ return out
587
+
588
+ def __getstate__(self):
589
+ return (self.delimiter, self.quote_char, self.double_quote,
590
+ self.escape_char, self.newlines_in_values,
591
+ self.ignore_empty_lines, self.invalid_row_handler)
592
+
593
+ def __setstate__(self, state):
594
+ (self.delimiter, self.quote_char, self.double_quote,
595
+ self.escape_char, self.newlines_in_values,
596
+ self.ignore_empty_lines, self.invalid_row_handler) = state
597
+
598
+ def __eq__(self, other):
599
+ try:
600
+ return self.equals(other)
601
+ except TypeError:
602
+ return False
603
+
604
+ def _repr_base(self):
605
+ return (f"""
606
+ delimiter={self.delimiter!r},
607
+ quote_char={self.quote_char!r},
608
+ double_quote={self.double_quote},
609
+ escape_char={self.escape_char!r},
610
+ newlines_in_values={self.newlines_in_values},
611
+ ignore_empty_lines={self.ignore_empty_lines},
612
+ invalid_row_handler={getattr(self.invalid_row_handler, '__name__',
613
+ self.invalid_row_handler)}""")
614
+
615
+ def __repr__(self):
616
+ return (f"<pyarrow.csv.ParseOptions>({self._repr_base()})")
617
+
618
+ def __str__(self):
619
+ return (f"ParseOptions({self._repr_base()})")
620
+
621
+
622
+ cdef class _ISO8601(_Weakrefable):
623
+ """
624
+ A special object indicating ISO-8601 parsing.
625
+ """
626
+ __slots__ = ()
627
+
628
+ def __str__(self):
629
+ return 'ISO8601'
630
+
631
+ def __eq__(self, other):
632
+ return isinstance(other, _ISO8601)
633
+
634
+
635
+ ISO8601 = _ISO8601()
636
+
637
+
638
+ cdef class ConvertOptions(_Weakrefable):
639
+ """
640
+ Options for converting CSV data.
641
+
642
+ Parameters
643
+ ----------
644
+ check_utf8 : bool, optional (default True)
645
+ Whether to check UTF8 validity of string columns.
646
+ column_types : pyarrow.Schema or dict, optional
647
+ Explicitly map column names to column types. Passing this argument
648
+ disables type inference on the defined columns.
649
+ null_values : list, optional
650
+ A sequence of strings that denote nulls in the data
651
+ (defaults are appropriate in most cases). Note that by default,
652
+ string columns are not checked for null values. To enable
653
+ null checking for those, specify ``strings_can_be_null=True``.
654
+ true_values : list, optional
655
+ A sequence of strings that denote true booleans in the data
656
+ (defaults are appropriate in most cases).
657
+ false_values : list, optional
658
+ A sequence of strings that denote false booleans in the data
659
+ (defaults are appropriate in most cases).
660
+ decimal_point : 1-character string, optional (default '.')
661
+ The character used as decimal point in floating-point and decimal
662
+ data.
663
+ strings_can_be_null : bool, optional (default False)
664
+ Whether string / binary columns can have null values.
665
+ If true, then strings in null_values are considered null for
666
+ string columns.
667
+ If false, then all strings are valid string values.
668
+ quoted_strings_can_be_null : bool, optional (default True)
669
+ Whether quoted values can be null.
670
+ If true, then strings in "null_values" are also considered null
671
+ when they appear quoted in the CSV file. Otherwise, quoted values
672
+ are never considered null.
673
+ include_columns : list, optional
674
+ The names of columns to include in the Table.
675
+ If empty, the Table will include all columns from the CSV file.
676
+ If not empty, only these columns will be included, in this order.
677
+ include_missing_columns : bool, optional (default False)
678
+ If false, columns in `include_columns` but not in the CSV file will
679
+ error out.
680
+ If true, columns in `include_columns` but not in the CSV file will
681
+ produce a column of nulls (whose type is selected using
682
+ `column_types`, or null by default).
683
+ This option is ignored if `include_columns` is empty.
684
+ auto_dict_encode : bool, optional (default False)
685
+ Whether to try to automatically dict-encode string / binary data.
686
+ If true, then when type inference detects a string or binary column,
687
+ it it dict-encoded up to `auto_dict_max_cardinality` distinct values
688
+ (per chunk), after which it switches to regular encoding.
689
+ This setting is ignored for non-inferred columns (those in
690
+ `column_types`).
691
+ auto_dict_max_cardinality : int, optional
692
+ The maximum dictionary cardinality for `auto_dict_encode`.
693
+ This value is per chunk.
694
+ timestamp_parsers : list, optional
695
+ A sequence of strptime()-compatible format strings, tried in order
696
+ when attempting to infer or convert timestamp values (the special
697
+ value ISO8601() can also be given). By default, a fast built-in
698
+ ISO-8601 parser is used.
699
+
700
+ Examples
701
+ --------
702
+
703
+ Defining an example data:
704
+
705
+ >>> import io
706
+ >>> s = (
707
+ ... "animals,n_legs,entry,fast\\n"
708
+ ... "Flamingo,2,01/03/2022,Yes\\n"
709
+ ... "Horse,4,02/03/2022,Yes\\n"
710
+ ... "Brittle stars,5,03/03/2022,No\\n"
711
+ ... "Centipede,100,04/03/2022,No\\n"
712
+ ... ",6,05/03/2022,"
713
+ ... )
714
+ >>> print(s)
715
+ animals,n_legs,entry,fast
716
+ Flamingo,2,01/03/2022,Yes
717
+ Horse,4,02/03/2022,Yes
718
+ Brittle stars,5,03/03/2022,No
719
+ Centipede,100,04/03/2022,No
720
+ ,6,05/03/2022,
721
+
722
+ Change the type of a column:
723
+
724
+ >>> import pyarrow as pa
725
+ >>> from pyarrow import csv
726
+ >>> convert_options = csv.ConvertOptions(column_types={"n_legs": pa.float64()})
727
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
728
+ pyarrow.Table
729
+ animals: string
730
+ n_legs: double
731
+ entry: string
732
+ fast: string
733
+ ----
734
+ animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
735
+ n_legs: [[2,4,5,100,6]]
736
+ entry: [["01/03/2022","02/03/2022","03/03/2022","04/03/2022","05/03/2022"]]
737
+ fast: [["Yes","Yes","No","No",""]]
738
+
739
+ Define a date parsing format to get a timestamp type column
740
+ (in case dates are not in ISO format and not converted by default):
741
+
742
+ >>> convert_options = csv.ConvertOptions(
743
+ ... timestamp_parsers=["%m/%d/%Y", "%m-%d-%Y"])
744
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
745
+ pyarrow.Table
746
+ animals: string
747
+ n_legs: int64
748
+ entry: timestamp[s]
749
+ fast: string
750
+ ----
751
+ animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
752
+ n_legs: [[2,4,5,100,6]]
753
+ entry: [[2022-01-03 00:00:00,2022-02-03 00:00:00,2022-03-03 00:00:00,2022-04-03 00:00:00,2022-05-03 00:00:00]]
754
+ fast: [["Yes","Yes","No","No",""]]
755
+
756
+ Specify a subset of columns to be read:
757
+
758
+ >>> convert_options = csv.ConvertOptions(
759
+ ... include_columns=["animals", "n_legs"])
760
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
761
+ pyarrow.Table
762
+ animals: string
763
+ n_legs: int64
764
+ ----
765
+ animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
766
+ n_legs: [[2,4,5,100,6]]
767
+
768
+ List additional column to be included as a null typed column:
769
+
770
+ >>> convert_options = csv.ConvertOptions(
771
+ ... include_columns=["animals", "n_legs", "location"],
772
+ ... include_missing_columns=True)
773
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
774
+ pyarrow.Table
775
+ animals: string
776
+ n_legs: int64
777
+ location: null
778
+ ----
779
+ animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
780
+ n_legs: [[2,4,5,100,6]]
781
+ location: [5 nulls]
782
+
783
+ Define columns as dictionary type (by default only the
784
+ string/binary columns are dictionary encoded):
785
+
786
+ >>> convert_options = csv.ConvertOptions(
787
+ ... timestamp_parsers=["%m/%d/%Y", "%m-%d-%Y"],
788
+ ... auto_dict_encode=True)
789
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
790
+ pyarrow.Table
791
+ animals: dictionary<values=string, indices=int32, ordered=0>
792
+ n_legs: int64
793
+ entry: timestamp[s]
794
+ fast: dictionary<values=string, indices=int32, ordered=0>
795
+ ----
796
+ animals: [ -- dictionary:
797
+ ["Flamingo","Horse","Brittle stars","Centipede",""] -- indices:
798
+ [0,1,2,3,4]]
799
+ n_legs: [[2,4,5,100,6]]
800
+ entry: [[2022-01-03 00:00:00,2022-02-03 00:00:00,2022-03-03 00:00:00,2022-04-03 00:00:00,2022-05-03 00:00:00]]
801
+ fast: [ -- dictionary:
802
+ ["Yes","No",""] -- indices:
803
+ [0,0,1,1,2]]
804
+
805
+ Set upper limit for the number of categories. If the categories
806
+ is more than the limit, the conversion to dictionary will not
807
+ happen:
808
+
809
+ >>> convert_options = csv.ConvertOptions(
810
+ ... include_columns=["animals"],
811
+ ... auto_dict_encode=True,
812
+ ... auto_dict_max_cardinality=2)
813
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
814
+ pyarrow.Table
815
+ animals: string
816
+ ----
817
+ animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
818
+
819
+ Set empty strings to missing values:
820
+
821
+ >>> convert_options = csv.ConvertOptions(include_columns=["animals", "n_legs"],
822
+ ... strings_can_be_null=True)
823
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
824
+ pyarrow.Table
825
+ animals: string
826
+ n_legs: int64
827
+ ----
828
+ animals: [["Flamingo","Horse","Brittle stars","Centipede",null]]
829
+ n_legs: [[2,4,5,100,6]]
830
+
831
+ Define values to be True and False when converting a column
832
+ into a bool type:
833
+
834
+ >>> convert_options = csv.ConvertOptions(
835
+ ... include_columns=["fast"],
836
+ ... false_values=["No"],
837
+ ... true_values=["Yes"])
838
+ >>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
839
+ pyarrow.Table
840
+ fast: bool
841
+ ----
842
+ fast: [[true,true,false,false,null]]
843
+ """
844
+
845
+ # Avoid mistakingly creating attributes
846
+ __slots__ = ()
847
+
848
+ def __cinit__(self, *argw, **kwargs):
849
+ self.options.reset(
850
+ new CCSVConvertOptions(CCSVConvertOptions.Defaults()))
851
+
852
+ def __init__(self, *, check_utf8=None, column_types=None, null_values=None,
853
+ true_values=None, false_values=None, decimal_point=None,
854
+ strings_can_be_null=None, quoted_strings_can_be_null=None,
855
+ include_columns=None, include_missing_columns=None,
856
+ auto_dict_encode=None, auto_dict_max_cardinality=None,
857
+ timestamp_parsers=None):
858
+ if check_utf8 is not None:
859
+ self.check_utf8 = check_utf8
860
+ if column_types is not None:
861
+ self.column_types = column_types
862
+ if null_values is not None:
863
+ self.null_values = null_values
864
+ if true_values is not None:
865
+ self.true_values = true_values
866
+ if false_values is not None:
867
+ self.false_values = false_values
868
+ if decimal_point is not None:
869
+ self.decimal_point = decimal_point
870
+ if strings_can_be_null is not None:
871
+ self.strings_can_be_null = strings_can_be_null
872
+ if quoted_strings_can_be_null is not None:
873
+ self.quoted_strings_can_be_null = quoted_strings_can_be_null
874
+ if include_columns is not None:
875
+ self.include_columns = include_columns
876
+ if include_missing_columns is not None:
877
+ self.include_missing_columns = include_missing_columns
878
+ if auto_dict_encode is not None:
879
+ self.auto_dict_encode = auto_dict_encode
880
+ if auto_dict_max_cardinality is not None:
881
+ self.auto_dict_max_cardinality = auto_dict_max_cardinality
882
+ if timestamp_parsers is not None:
883
+ self.timestamp_parsers = timestamp_parsers
884
+
885
+ @property
886
+ def check_utf8(self):
887
+ """
888
+ Whether to check UTF8 validity of string columns.
889
+ """
890
+ return deref(self.options).check_utf8
891
+
892
+ @check_utf8.setter
893
+ def check_utf8(self, value):
894
+ deref(self.options).check_utf8 = value
895
+
896
+ @property
897
+ def strings_can_be_null(self):
898
+ """
899
+ Whether string / binary columns can have null values.
900
+ """
901
+ return deref(self.options).strings_can_be_null
902
+
903
+ @strings_can_be_null.setter
904
+ def strings_can_be_null(self, value):
905
+ deref(self.options).strings_can_be_null = value
906
+
907
+ @property
908
+ def quoted_strings_can_be_null(self):
909
+ """
910
+ Whether quoted values can be null.
911
+ """
912
+ return deref(self.options).quoted_strings_can_be_null
913
+
914
+ @quoted_strings_can_be_null.setter
915
+ def quoted_strings_can_be_null(self, value):
916
+ deref(self.options).quoted_strings_can_be_null = value
917
+
918
+ @property
919
+ def column_types(self):
920
+ """
921
+ Explicitly map column names to column types.
922
+ """
923
+ d = {frombytes(item.first): pyarrow_wrap_data_type(item.second)
924
+ for item in deref(self.options).column_types}
925
+ return d
926
+
927
+ @column_types.setter
928
+ def column_types(self, value):
929
+ cdef:
930
+ shared_ptr[CDataType] typ
931
+
932
+ if isinstance(value, Mapping):
933
+ value = value.items()
934
+
935
+ deref(self.options).column_types.clear()
936
+ for item in value:
937
+ if isinstance(item, Field):
938
+ k = item.name
939
+ v = item.type
940
+ else:
941
+ k, v = item
942
+ typ = pyarrow_unwrap_data_type(ensure_type(v))
943
+ assert typ != NULL
944
+ deref(self.options).column_types[tobytes(k)] = typ
945
+
946
+ @property
947
+ def null_values(self):
948
+ """
949
+ A sequence of strings that denote nulls in the data.
950
+ """
951
+ return [frombytes(x) for x in deref(self.options).null_values]
952
+
953
+ @null_values.setter
954
+ def null_values(self, value):
955
+ deref(self.options).null_values = [tobytes(x) for x in value]
956
+
957
+ @property
958
+ def true_values(self):
959
+ """
960
+ A sequence of strings that denote true booleans in the data.
961
+ """
962
+ return [frombytes(x) for x in deref(self.options).true_values]
963
+
964
+ @true_values.setter
965
+ def true_values(self, value):
966
+ deref(self.options).true_values = [tobytes(x) for x in value]
967
+
968
+ @property
969
+ def false_values(self):
970
+ """
971
+ A sequence of strings that denote false booleans in the data.
972
+ """
973
+ return [frombytes(x) for x in deref(self.options).false_values]
974
+
975
+ @false_values.setter
976
+ def false_values(self, value):
977
+ deref(self.options).false_values = [tobytes(x) for x in value]
978
+
979
+ @property
980
+ def decimal_point(self):
981
+ """
982
+ The character used as decimal point in floating-point and decimal
983
+ data.
984
+ """
985
+ return chr(deref(self.options).decimal_point)
986
+
987
+ @decimal_point.setter
988
+ def decimal_point(self, value):
989
+ deref(self.options).decimal_point = _single_char(value)
990
+
991
+ @property
992
+ def auto_dict_encode(self):
993
+ """
994
+ Whether to try to automatically dict-encode string / binary data.
995
+ """
996
+ return deref(self.options).auto_dict_encode
997
+
998
+ @auto_dict_encode.setter
999
+ def auto_dict_encode(self, value):
1000
+ deref(self.options).auto_dict_encode = value
1001
+
1002
+ @property
1003
+ def auto_dict_max_cardinality(self):
1004
+ """
1005
+ The maximum dictionary cardinality for `auto_dict_encode`.
1006
+
1007
+ This value is per chunk.
1008
+ """
1009
+ return deref(self.options).auto_dict_max_cardinality
1010
+
1011
+ @auto_dict_max_cardinality.setter
1012
+ def auto_dict_max_cardinality(self, value):
1013
+ deref(self.options).auto_dict_max_cardinality = value
1014
+
1015
+ @property
1016
+ def include_columns(self):
1017
+ """
1018
+ The names of columns to include in the Table.
1019
+
1020
+ If empty, the Table will include all columns from the CSV file.
1021
+ If not empty, only these columns will be included, in this order.
1022
+ """
1023
+ return [frombytes(s) for s in deref(self.options).include_columns]
1024
+
1025
+ @include_columns.setter
1026
+ def include_columns(self, value):
1027
+ deref(self.options).include_columns.clear()
1028
+ for item in value:
1029
+ deref(self.options).include_columns.push_back(tobytes(item))
1030
+
1031
+ @property
1032
+ def include_missing_columns(self):
1033
+ """
1034
+ If false, columns in `include_columns` but not in the CSV file will
1035
+ error out.
1036
+ If true, columns in `include_columns` but not in the CSV file will
1037
+ produce a null column (whose type is selected using `column_types`,
1038
+ or null by default).
1039
+ This option is ignored if `include_columns` is empty.
1040
+ """
1041
+ return deref(self.options).include_missing_columns
1042
+
1043
+ @include_missing_columns.setter
1044
+ def include_missing_columns(self, value):
1045
+ deref(self.options).include_missing_columns = value
1046
+
1047
+ @property
1048
+ def timestamp_parsers(self):
1049
+ """
1050
+ A sequence of strptime()-compatible format strings, tried in order
1051
+ when attempting to infer or convert timestamp values (the special
1052
+ value ISO8601() can also be given). By default, a fast built-in
1053
+ ISO-8601 parser is used.
1054
+ """
1055
+ cdef:
1056
+ shared_ptr[CTimestampParser] c_parser
1057
+ c_string kind
1058
+
1059
+ parsers = []
1060
+ for c_parser in deref(self.options).timestamp_parsers:
1061
+ kind = deref(c_parser).kind()
1062
+ if kind == b'strptime':
1063
+ parsers.append(frombytes(deref(c_parser).format()))
1064
+ else:
1065
+ assert kind == b'iso8601'
1066
+ parsers.append(ISO8601)
1067
+
1068
+ return parsers
1069
+
1070
+ @timestamp_parsers.setter
1071
+ def timestamp_parsers(self, value):
1072
+ cdef:
1073
+ vector[shared_ptr[CTimestampParser]] c_parsers
1074
+
1075
+ for v in value:
1076
+ if isinstance(v, str):
1077
+ c_parsers.push_back(CTimestampParser.MakeStrptime(tobytes(v)))
1078
+ elif v == ISO8601:
1079
+ c_parsers.push_back(CTimestampParser.MakeISO8601())
1080
+ else:
1081
+ raise TypeError("Expected list of str or ISO8601 objects")
1082
+
1083
+ deref(self.options).timestamp_parsers = move(c_parsers)
1084
+
1085
+ @staticmethod
1086
+ cdef ConvertOptions wrap(CCSVConvertOptions options):
1087
+ out = ConvertOptions()
1088
+ out.options.reset(new CCSVConvertOptions(move(options)))
1089
+ return out
1090
+
1091
+ def validate(self):
1092
+ check_status(deref(self.options).Validate())
1093
+
1094
+ def equals(self, ConvertOptions other):
1095
+ """
1096
+ Parameters
1097
+ ----------
1098
+ other : pyarrow.csv.ConvertOptions
1099
+
1100
+ Returns
1101
+ -------
1102
+ bool
1103
+ """
1104
+ return (
1105
+ self.check_utf8 == other.check_utf8 and
1106
+ self.column_types == other.column_types and
1107
+ self.null_values == other.null_values and
1108
+ self.true_values == other.true_values and
1109
+ self.false_values == other.false_values and
1110
+ self.decimal_point == other.decimal_point and
1111
+ self.timestamp_parsers == other.timestamp_parsers and
1112
+ self.strings_can_be_null == other.strings_can_be_null and
1113
+ self.quoted_strings_can_be_null ==
1114
+ other.quoted_strings_can_be_null and
1115
+ self.auto_dict_encode == other.auto_dict_encode and
1116
+ self.auto_dict_max_cardinality ==
1117
+ other.auto_dict_max_cardinality and
1118
+ self.include_columns == other.include_columns and
1119
+ self.include_missing_columns == other.include_missing_columns
1120
+ )
1121
+
1122
+ def __getstate__(self):
1123
+ return (self.check_utf8, self.column_types, self.null_values,
1124
+ self.true_values, self.false_values, self.decimal_point,
1125
+ self.timestamp_parsers, self.strings_can_be_null,
1126
+ self.quoted_strings_can_be_null, self.auto_dict_encode,
1127
+ self.auto_dict_max_cardinality, self.include_columns,
1128
+ self.include_missing_columns)
1129
+
1130
+ def __setstate__(self, state):
1131
+ (self.check_utf8, self.column_types, self.null_values,
1132
+ self.true_values, self.false_values, self.decimal_point,
1133
+ self.timestamp_parsers, self.strings_can_be_null,
1134
+ self.quoted_strings_can_be_null, self.auto_dict_encode,
1135
+ self.auto_dict_max_cardinality, self.include_columns,
1136
+ self.include_missing_columns) = state
1137
+
1138
+ def __eq__(self, other):
1139
+ try:
1140
+ return self.equals(other)
1141
+ except TypeError:
1142
+ return False
1143
+
1144
+ def _repr_base(self):
1145
+ return (f"""
1146
+ check_utf8={self.check_utf8},
1147
+ column_types={self.column_types},
1148
+ null_values={self.null_values},
1149
+ true_values={self.true_values},
1150
+ false_values={self.false_values},
1151
+ decimal_point={self.decimal_point!r},
1152
+ strings_can_be_null={self.strings_can_be_null},
1153
+ quoted_strings_can_be_null={self.quoted_strings_can_be_null},
1154
+ include_columns={self.include_columns},
1155
+ include_missing_columns={self.include_missing_columns},
1156
+ auto_dict_encode={self.auto_dict_encode},
1157
+ auto_dict_max_cardinality={self.auto_dict_max_cardinality},
1158
+ timestamp_parsers={[str(i) for i in self.timestamp_parsers]}""")
1159
+
1160
+ def __repr__(self):
1161
+ return (f"<pyarrow.csv.ConvertOptions>({self._repr_base()})")
1162
+
1163
+ def __str__(self):
1164
+ return (f"ConvertOptions({self._repr_base()})")
1165
+
1166
+
1167
+ cdef _get_reader(input_file, ReadOptions read_options,
1168
+ shared_ptr[CInputStream]* out):
1169
+ use_memory_map = False
1170
+ get_input_stream(input_file, use_memory_map, out)
1171
+ if read_options is not None:
1172
+ out[0] = native_transcoding_input_stream(out[0],
1173
+ read_options.encoding,
1174
+ 'utf8')
1175
+
1176
+
1177
+ cdef _get_read_options(ReadOptions read_options, CCSVReadOptions* out):
1178
+ if read_options is None:
1179
+ out[0] = CCSVReadOptions.Defaults()
1180
+ else:
1181
+ out[0] = deref(read_options.options)
1182
+
1183
+
1184
+ cdef _get_parse_options(ParseOptions parse_options, CCSVParseOptions* out):
1185
+ if parse_options is None:
1186
+ out[0] = CCSVParseOptions.Defaults()
1187
+ else:
1188
+ out[0] = deref(parse_options.options)
1189
+
1190
+
1191
+ cdef _get_convert_options(ConvertOptions convert_options,
1192
+ CCSVConvertOptions* out):
1193
+ if convert_options is None:
1194
+ out[0] = CCSVConvertOptions.Defaults()
1195
+ else:
1196
+ out[0] = deref(convert_options.options)
1197
+
1198
+
1199
+ cdef class CSVStreamingReader(RecordBatchReader):
1200
+ """An object that reads record batches incrementally from a CSV file.
1201
+
1202
+ Should not be instantiated directly by user code.
1203
+ """
1204
+ cdef readonly:
1205
+ Schema schema
1206
+
1207
+ def __init__(self):
1208
+ raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
1209
+ "use pyarrow.csv.open_csv() instead.")
1210
+
1211
+ # Note about cancellation: we cannot create a SignalStopHandler
1212
+ # by default here, as several CSVStreamingReader instances may be
1213
+ # created (including by the same thread). Handling cancellation
1214
+ # would require having the user pass the SignalStopHandler.
1215
+ # (in addition to solving ARROW-11853)
1216
+
1217
+ cdef _open(self, shared_ptr[CInputStream] stream,
1218
+ CCSVReadOptions c_read_options,
1219
+ CCSVParseOptions c_parse_options,
1220
+ CCSVConvertOptions c_convert_options,
1221
+ MemoryPool memory_pool):
1222
+ cdef:
1223
+ shared_ptr[CSchema] c_schema
1224
+ CIOContext io_context
1225
+
1226
+ io_context = CIOContext(maybe_unbox_memory_pool(memory_pool))
1227
+
1228
+ with nogil:
1229
+ self.reader = <shared_ptr[CRecordBatchReader]> GetResultValue(
1230
+ CCSVStreamingReader.Make(
1231
+ io_context, stream,
1232
+ move(c_read_options), move(c_parse_options),
1233
+ move(c_convert_options)))
1234
+ c_schema = self.reader.get().schema()
1235
+
1236
+ self.schema = pyarrow_wrap_schema(c_schema)
1237
+
1238
+
1239
+ def read_csv(input_file, read_options=None, parse_options=None,
1240
+ convert_options=None, MemoryPool memory_pool=None):
1241
+ """
1242
+ Read a Table from a stream of CSV data.
1243
+
1244
+ Parameters
1245
+ ----------
1246
+ input_file : string, path or file-like object
1247
+ The location of CSV data. If a string or path, and if it ends
1248
+ with a recognized compressed file extension (e.g. ".gz" or ".bz2"),
1249
+ the data is automatically decompressed when reading.
1250
+ read_options : pyarrow.csv.ReadOptions, optional
1251
+ Options for the CSV reader (see pyarrow.csv.ReadOptions constructor
1252
+ for defaults)
1253
+ parse_options : pyarrow.csv.ParseOptions, optional
1254
+ Options for the CSV parser
1255
+ (see pyarrow.csv.ParseOptions constructor for defaults)
1256
+ convert_options : pyarrow.csv.ConvertOptions, optional
1257
+ Options for converting CSV data
1258
+ (see pyarrow.csv.ConvertOptions constructor for defaults)
1259
+ memory_pool : MemoryPool, optional
1260
+ Pool to allocate Table memory from
1261
+
1262
+ Returns
1263
+ -------
1264
+ :class:`pyarrow.Table`
1265
+ Contents of the CSV file as a in-memory table.
1266
+
1267
+ Examples
1268
+ --------
1269
+
1270
+ Defining an example file from bytes object:
1271
+
1272
+ >>> import io
1273
+ >>> s = (
1274
+ ... "animals,n_legs,entry\\n"
1275
+ ... "Flamingo,2,2022-03-01\\n"
1276
+ ... "Horse,4,2022-03-02\\n"
1277
+ ... "Brittle stars,5,2022-03-03\\n"
1278
+ ... "Centipede,100,2022-03-04"
1279
+ ... )
1280
+ >>> print(s)
1281
+ animals,n_legs,entry
1282
+ Flamingo,2,2022-03-01
1283
+ Horse,4,2022-03-02
1284
+ Brittle stars,5,2022-03-03
1285
+ Centipede,100,2022-03-04
1286
+ >>> source = io.BytesIO(s.encode())
1287
+
1288
+ Reading from the file
1289
+
1290
+ >>> from pyarrow import csv
1291
+ >>> csv.read_csv(source)
1292
+ pyarrow.Table
1293
+ animals: string
1294
+ n_legs: int64
1295
+ entry: date32[day]
1296
+ ----
1297
+ animals: [["Flamingo","Horse","Brittle stars","Centipede"]]
1298
+ n_legs: [[2,4,5,100]]
1299
+ entry: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
1300
+ """
1301
+ cdef:
1302
+ shared_ptr[CInputStream] stream
1303
+ CCSVReadOptions c_read_options
1304
+ CCSVParseOptions c_parse_options
1305
+ CCSVConvertOptions c_convert_options
1306
+ CIOContext io_context
1307
+ SharedPtrNoGIL[CCSVReader] reader
1308
+ shared_ptr[CTable] table
1309
+
1310
+ _get_reader(input_file, read_options, &stream)
1311
+ _get_read_options(read_options, &c_read_options)
1312
+ _get_parse_options(parse_options, &c_parse_options)
1313
+ _get_convert_options(convert_options, &c_convert_options)
1314
+
1315
+ with SignalStopHandler() as stop_handler:
1316
+ io_context = CIOContext(
1317
+ maybe_unbox_memory_pool(memory_pool),
1318
+ (<StopToken> stop_handler.stop_token).stop_token)
1319
+ reader = GetResultValue(CCSVReader.Make(
1320
+ io_context, stream,
1321
+ c_read_options, c_parse_options, c_convert_options))
1322
+
1323
+ with nogil:
1324
+ table = GetResultValue(reader.get().Read())
1325
+
1326
+ return pyarrow_wrap_table(table)
1327
+
1328
+
1329
+ def open_csv(input_file, read_options=None, parse_options=None,
1330
+ convert_options=None, MemoryPool memory_pool=None):
1331
+ """
1332
+ Open a streaming reader of CSV data.
1333
+
1334
+ Reading using this function is always single-threaded.
1335
+
1336
+ Parameters
1337
+ ----------
1338
+ input_file : string, path or file-like object
1339
+ The location of CSV data. If a string or path, and if it ends
1340
+ with a recognized compressed file extension (e.g. ".gz" or ".bz2"),
1341
+ the data is automatically decompressed when reading.
1342
+ read_options : pyarrow.csv.ReadOptions, optional
1343
+ Options for the CSV reader (see pyarrow.csv.ReadOptions constructor
1344
+ for defaults)
1345
+ parse_options : pyarrow.csv.ParseOptions, optional
1346
+ Options for the CSV parser
1347
+ (see pyarrow.csv.ParseOptions constructor for defaults)
1348
+ convert_options : pyarrow.csv.ConvertOptions, optional
1349
+ Options for converting CSV data
1350
+ (see pyarrow.csv.ConvertOptions constructor for defaults)
1351
+ memory_pool : MemoryPool, optional
1352
+ Pool to allocate RecordBatch memory from
1353
+
1354
+ Returns
1355
+ -------
1356
+ :class:`pyarrow.csv.CSVStreamingReader`
1357
+ """
1358
+ cdef:
1359
+ shared_ptr[CInputStream] stream
1360
+ CCSVReadOptions c_read_options
1361
+ CCSVParseOptions c_parse_options
1362
+ CCSVConvertOptions c_convert_options
1363
+ CSVStreamingReader reader
1364
+
1365
+ _get_reader(input_file, read_options, &stream)
1366
+ _get_read_options(read_options, &c_read_options)
1367
+ _get_parse_options(parse_options, &c_parse_options)
1368
+ _get_convert_options(convert_options, &c_convert_options)
1369
+
1370
+ reader = CSVStreamingReader.__new__(CSVStreamingReader)
1371
+ reader._open(stream, move(c_read_options), move(c_parse_options),
1372
+ move(c_convert_options), memory_pool)
1373
+ return reader
1374
+
1375
+
1376
+ def _raise_invalid_function_option(value, description, *,
1377
+ exception_class=ValueError):
1378
+ raise exception_class(f"\"{value}\" is not a valid {description}")
1379
+
1380
+
1381
+ cdef CQuotingStyle unwrap_quoting_style(quoting_style) except *:
1382
+ if quoting_style == "needed":
1383
+ return CQuotingStyle_Needed
1384
+ elif quoting_style == "all_valid":
1385
+ return CQuotingStyle_AllValid
1386
+ elif quoting_style == "none":
1387
+ return CQuotingStyle_None
1388
+ _raise_invalid_function_option(quoting_style, "quoting style")
1389
+
1390
+
1391
+ cdef wrap_quoting_style(quoting_style):
1392
+ if quoting_style == CQuotingStyle_Needed:
1393
+ return 'needed'
1394
+ elif quoting_style == CQuotingStyle_AllValid:
1395
+ return 'all_valid'
1396
+ elif quoting_style == CQuotingStyle_None:
1397
+ return 'none'
1398
+
1399
+
1400
+ cdef class WriteOptions(_Weakrefable):
1401
+ """
1402
+ Options for writing CSV files.
1403
+
1404
+ Parameters
1405
+ ----------
1406
+ include_header : bool, optional (default True)
1407
+ Whether to write an initial header line with column names
1408
+ batch_size : int, optional (default 1024)
1409
+ How many rows to process together when converting and writing
1410
+ CSV data
1411
+ delimiter : 1-character string, optional (default ",")
1412
+ The character delimiting individual cells in the CSV data.
1413
+ quoting_style : str, optional (default "needed")
1414
+ Whether to quote values, and if so, which quoting style to use.
1415
+ The following values are accepted:
1416
+
1417
+ - "needed" (default): only enclose values in quotes when needed.
1418
+ - "all_valid": enclose all valid values in quotes; nulls are not quoted.
1419
+ - "none": do not enclose any values in quotes; values containing
1420
+ special characters (such as quotes, cell delimiters or line endings)
1421
+ will raise an error.
1422
+ quoting_header : str, optional (default "needed")
1423
+ Same as quoting_style, but for header column names. Accepts same values.
1424
+ Note : both "needed" and "all_valid" have the same effect of quoting all column names.
1425
+ """
1426
+
1427
+ # Avoid mistakingly creating attributes
1428
+ __slots__ = ()
1429
+
1430
+ def __init__(self, *, include_header=None, batch_size=None,
1431
+ delimiter=None, quoting_style=None, quoting_header=None):
1432
+ self.options.reset(new CCSVWriteOptions(CCSVWriteOptions.Defaults()))
1433
+ if include_header is not None:
1434
+ self.include_header = include_header
1435
+ if batch_size is not None:
1436
+ self.batch_size = batch_size
1437
+ if delimiter is not None:
1438
+ self.delimiter = delimiter
1439
+ if quoting_style is not None:
1440
+ self.quoting_style = quoting_style
1441
+ if quoting_header is not None:
1442
+ self.quoting_header = quoting_header
1443
+
1444
+ @property
1445
+ def include_header(self):
1446
+ """
1447
+ Whether to write an initial header line with column names.
1448
+ """
1449
+ return deref(self.options).include_header
1450
+
1451
+ @include_header.setter
1452
+ def include_header(self, value):
1453
+ deref(self.options).include_header = value
1454
+
1455
+ @property
1456
+ def batch_size(self):
1457
+ """
1458
+ How many rows to process together when converting and writing
1459
+ CSV data.
1460
+ """
1461
+ return deref(self.options).batch_size
1462
+
1463
+ @batch_size.setter
1464
+ def batch_size(self, value):
1465
+ deref(self.options).batch_size = value
1466
+
1467
+ @property
1468
+ def delimiter(self):
1469
+ """
1470
+ The character delimiting individual cells in the CSV data.
1471
+ """
1472
+ return chr(deref(self.options).delimiter)
1473
+
1474
+ @delimiter.setter
1475
+ def delimiter(self, value):
1476
+ deref(self.options).delimiter = _single_char(value)
1477
+
1478
+ @property
1479
+ def quoting_style(self):
1480
+ """
1481
+ Whether to quote values, and if so, which quoting style to use.
1482
+ The following values are accepted:
1483
+
1484
+ - "needed" (default): only enclose values in quotes when needed.
1485
+ - "all_valid": enclose all valid values in quotes; nulls are not quoted.
1486
+ - "none": do not enclose any values in quotes; values containing
1487
+ special characters (such as quotes, cell delimiters or line endings)
1488
+ will raise an error.
1489
+ """
1490
+ return wrap_quoting_style(deref(self.options).quoting_style)
1491
+
1492
+ @quoting_style.setter
1493
+ def quoting_style(self, value):
1494
+ deref(self.options).quoting_style = unwrap_quoting_style(value)
1495
+
1496
+ @property
1497
+ def quoting_header(self):
1498
+ """
1499
+ Same as quoting_style, but for header column names.
1500
+ Note : both "needed" and "all_valid" have the same effect of quoting all column names.
1501
+ """
1502
+ return wrap_quoting_style(deref(self.options).quoting_header)
1503
+
1504
+ @quoting_header.setter
1505
+ def quoting_header(self, value):
1506
+ deref(self.options).quoting_header = unwrap_quoting_style(value)
1507
+
1508
+ @staticmethod
1509
+ cdef WriteOptions wrap(CCSVWriteOptions options):
1510
+ out = WriteOptions()
1511
+ out.options.reset(new CCSVWriteOptions(move(options)))
1512
+ return out
1513
+
1514
+ def validate(self):
1515
+ check_status(self.options.get().Validate())
1516
+
1517
+ def _repr_base(self):
1518
+ return (f"""
1519
+ include_header={self.include_header},
1520
+ batch_size={self.batch_size},
1521
+ delimiter={self.delimiter!r},
1522
+ quoting_style={self.quoting_style!r}""")
1523
+
1524
+ def __repr__(self):
1525
+ return (f"<pyarrow.csv.WriteOptions>({self._repr_base()})")
1526
+
1527
+ def __str__(self):
1528
+ return (f"WriteOptions({self._repr_base()})")
1529
+
1530
+
1531
+ cdef _get_write_options(WriteOptions write_options, CCSVWriteOptions* out):
1532
+ if write_options is None:
1533
+ out[0] = CCSVWriteOptions.Defaults()
1534
+ else:
1535
+ out[0] = deref(write_options.options)
1536
+
1537
+
1538
+ def write_csv(data, output_file, write_options=None,
1539
+ MemoryPool memory_pool=None):
1540
+ """
1541
+ Write record batch or table to a CSV file.
1542
+
1543
+ Parameters
1544
+ ----------
1545
+ data : pyarrow.RecordBatch or pyarrow.Table
1546
+ The data to write.
1547
+ output_file : string, path, pyarrow.NativeFile, or file-like object
1548
+ The location where to write the CSV data.
1549
+ write_options : pyarrow.csv.WriteOptions
1550
+ Options to configure writing the CSV data.
1551
+ memory_pool : MemoryPool, optional
1552
+ Pool for temporary allocations.
1553
+
1554
+ Examples
1555
+ --------
1556
+
1557
+ >>> import pyarrow as pa
1558
+ >>> from pyarrow import csv
1559
+
1560
+ >>> legs = pa.array([2, 4, 5, 100])
1561
+ >>> animals = pa.array(["Flamingo", "Horse", "Brittle stars", "Centipede"])
1562
+ >>> entry_date = pa.array(["01/03/2022", "02/03/2022",
1563
+ ... "03/03/2022", "04/03/2022"])
1564
+ >>> table = pa.table([animals, legs, entry_date],
1565
+ ... names=["animals", "n_legs", "entry"])
1566
+
1567
+ >>> csv.write_csv(table, "animals.csv")
1568
+
1569
+ >>> write_options = csv.WriteOptions(include_header=False)
1570
+ >>> csv.write_csv(table, "animals.csv", write_options=write_options)
1571
+
1572
+ >>> write_options = csv.WriteOptions(delimiter=";")
1573
+ >>> csv.write_csv(table, "animals.csv", write_options=write_options)
1574
+ """
1575
+ cdef:
1576
+ shared_ptr[COutputStream] stream
1577
+ CCSVWriteOptions c_write_options
1578
+ CMemoryPool* c_memory_pool
1579
+ CRecordBatch* batch
1580
+ CTable* table
1581
+ _get_write_options(write_options, &c_write_options)
1582
+
1583
+ get_writer(output_file, &stream)
1584
+ c_memory_pool = maybe_unbox_memory_pool(memory_pool)
1585
+ c_write_options.io_context = CIOContext(c_memory_pool)
1586
+ if isinstance(data, RecordBatch):
1587
+ batch = pyarrow_unwrap_batch(data).get()
1588
+ with nogil:
1589
+ check_status(WriteCSV(deref(batch), c_write_options, stream.get()))
1590
+ elif isinstance(data, Table):
1591
+ table = pyarrow_unwrap_table(data).get()
1592
+ with nogil:
1593
+ check_status(WriteCSV(deref(table), c_write_options, stream.get()))
1594
+ else:
1595
+ raise TypeError(f"Expected Table or RecordBatch, got '{type(data)}'")
1596
+
1597
+
1598
+ cdef class CSVWriter(_CRecordBatchWriter):
1599
+ """
1600
+ Writer to create a CSV file.
1601
+
1602
+ Parameters
1603
+ ----------
1604
+ sink : str, path, pyarrow.OutputStream or file-like object
1605
+ The location where to write the CSV data.
1606
+ schema : pyarrow.Schema
1607
+ The schema of the data to be written.
1608
+ write_options : pyarrow.csv.WriteOptions
1609
+ Options to configure writing the CSV data.
1610
+ memory_pool : MemoryPool, optional
1611
+ Pool for temporary allocations.
1612
+ """
1613
+
1614
+ def __init__(self, sink, Schema schema, *,
1615
+ WriteOptions write_options=None, MemoryPool memory_pool=None):
1616
+ cdef:
1617
+ shared_ptr[COutputStream] c_stream
1618
+ shared_ptr[CSchema] c_schema = pyarrow_unwrap_schema(schema)
1619
+ CCSVWriteOptions c_write_options
1620
+ CMemoryPool* c_memory_pool = maybe_unbox_memory_pool(memory_pool)
1621
+ _get_write_options(write_options, &c_write_options)
1622
+ c_write_options.io_context = CIOContext(c_memory_pool)
1623
+ get_writer(sink, &c_stream)
1624
+ with nogil:
1625
+ self.writer = GetResultValue(MakeCSVWriter(
1626
+ c_stream, c_schema, c_write_options))
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pxd ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.lib cimport *
21
+ from pyarrow.includes.common cimport *
22
+ from pyarrow.includes.libarrow cimport *
23
+ from pyarrow.includes.libarrow_cuda cimport *
24
+
25
+
26
+ cdef class Context(_Weakrefable):
27
+ cdef:
28
+ shared_ptr[CCudaContext] context
29
+ int device_number
30
+
31
+ cdef void init(self, const shared_ptr[CCudaContext]& ctx)
32
+
33
+
34
+ cdef class IpcMemHandle(_Weakrefable):
35
+ cdef:
36
+ shared_ptr[CCudaIpcMemHandle] handle
37
+
38
+ cdef void init(self, shared_ptr[CCudaIpcMemHandle]& h)
39
+
40
+
41
+ cdef class CudaBuffer(Buffer):
42
+ cdef:
43
+ shared_ptr[CCudaBuffer] cuda_buffer
44
+ object base
45
+
46
+ cdef void init_cuda(self,
47
+ const shared_ptr[CCudaBuffer]& buffer,
48
+ object base)
49
+
50
+
51
+ cdef class HostBuffer(Buffer):
52
+ cdef:
53
+ shared_ptr[CCudaHostBuffer] host_buffer
54
+
55
+ cdef void init_host(self, const shared_ptr[CCudaHostBuffer]& buffer)
56
+
57
+
58
+ cdef class BufferReader(NativeFile):
59
+ cdef:
60
+ CCudaBufferReader* reader
61
+ CudaBuffer buffer
62
+
63
+
64
+ cdef class BufferWriter(NativeFile):
65
+ cdef:
66
+ CCudaBufferWriter* writer
67
+ CudaBuffer buffer
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pyx ADDED
@@ -0,0 +1,1084 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+
19
+ from pyarrow.lib cimport *
20
+ from pyarrow.includes.libarrow_cuda cimport *
21
+ from pyarrow.lib import allocate_buffer, as_buffer, ArrowTypeError
22
+ from pyarrow.util import get_contiguous_span
23
+ cimport cpython as cp
24
+
25
+
26
+ cdef class Context(_Weakrefable):
27
+ """
28
+ CUDA driver context.
29
+ """
30
+
31
+ def __init__(self, *args, **kwargs):
32
+ """
33
+ Create a CUDA driver context for a particular device.
34
+
35
+ If a CUDA context handle is passed, it is wrapped, otherwise
36
+ a default CUDA context for the given device is requested.
37
+
38
+ Parameters
39
+ ----------
40
+ device_number : int (default 0)
41
+ Specify the GPU device for which the CUDA driver context is
42
+ requested.
43
+ handle : int, optional
44
+ Specify CUDA handle for a shared context that has been created
45
+ by another library.
46
+ """
47
+ # This method exposed because autodoc doesn't pick __cinit__
48
+
49
+ def __cinit__(self, int device_number=0, uintptr_t handle=0):
50
+ cdef CCudaDeviceManager* manager
51
+ manager = GetResultValue(CCudaDeviceManager.Instance())
52
+ cdef int n = manager.num_devices()
53
+ if device_number >= n or device_number < 0:
54
+ self.context.reset()
55
+ raise ValueError('device_number argument must be '
56
+ 'non-negative less than %s' % (n))
57
+ if handle == 0:
58
+ self.context = GetResultValue(manager.GetContext(device_number))
59
+ else:
60
+ self.context = GetResultValue(manager.GetSharedContext(
61
+ device_number, <void*>handle))
62
+ self.device_number = device_number
63
+
64
+ @staticmethod
65
+ def from_numba(context=None):
66
+ """
67
+ Create a Context instance from a Numba CUDA context.
68
+
69
+ Parameters
70
+ ----------
71
+ context : {numba.cuda.cudadrv.driver.Context, None}
72
+ A Numba CUDA context instance.
73
+ If None, the current Numba context is used.
74
+
75
+ Returns
76
+ -------
77
+ shared_context : pyarrow.cuda.Context
78
+ Context instance.
79
+ """
80
+ if context is None:
81
+ import numba.cuda
82
+ context = numba.cuda.current_context()
83
+ return Context(device_number=context.device.id,
84
+ handle=context.handle.value)
85
+
86
+ def to_numba(self):
87
+ """
88
+ Convert Context to a Numba CUDA context.
89
+
90
+ Returns
91
+ -------
92
+ context : numba.cuda.cudadrv.driver.Context
93
+ Numba CUDA context instance.
94
+ """
95
+ import ctypes
96
+ import numba.cuda
97
+ device = numba.cuda.gpus[self.device_number]
98
+ handle = ctypes.c_void_p(self.handle)
99
+ context = numba.cuda.cudadrv.driver.Context(device, handle)
100
+
101
+ class DummyPendingDeallocs(object):
102
+ # Context is managed by pyarrow
103
+ def add_item(self, *args, **kwargs):
104
+ pass
105
+
106
+ context.deallocations = DummyPendingDeallocs()
107
+ return context
108
+
109
+ @staticmethod
110
+ def get_num_devices():
111
+ """ Return the number of GPU devices.
112
+ """
113
+ cdef CCudaDeviceManager* manager
114
+ manager = GetResultValue(CCudaDeviceManager.Instance())
115
+ return manager.num_devices()
116
+
117
+ @property
118
+ def device_number(self):
119
+ """ Return context device number.
120
+ """
121
+ return self.device_number
122
+
123
+ @property
124
+ def handle(self):
125
+ """ Return pointer to context handle.
126
+ """
127
+ return <uintptr_t>self.context.get().handle()
128
+
129
+ cdef void init(self, const shared_ptr[CCudaContext]& ctx):
130
+ self.context = ctx
131
+
132
+ def synchronize(self):
133
+ """Blocks until the device has completed all preceding requested
134
+ tasks.
135
+ """
136
+ check_status(self.context.get().Synchronize())
137
+
138
+ @property
139
+ def bytes_allocated(self):
140
+ """Return the number of allocated bytes.
141
+ """
142
+ return self.context.get().bytes_allocated()
143
+
144
+ def get_device_address(self, uintptr_t address):
145
+ """Return the device address that is reachable from kernels running in
146
+ the context
147
+
148
+ Parameters
149
+ ----------
150
+ address : int
151
+ Specify memory address value
152
+
153
+ Returns
154
+ -------
155
+ device_address : int
156
+ Device address accessible from device context
157
+
158
+ Notes
159
+ -----
160
+ The device address is defined as a memory address accessible
161
+ by device. While it is often a device memory address but it
162
+ can be also a host memory address, for instance, when the
163
+ memory is allocated as host memory (using cudaMallocHost or
164
+ cudaHostAlloc) or as managed memory (using cudaMallocManaged)
165
+ or the host memory is page-locked (using cudaHostRegister).
166
+ """
167
+ return GetResultValue(self.context.get().GetDeviceAddress(address))
168
+
169
+ def new_buffer(self, int64_t nbytes):
170
+ """Return new device buffer.
171
+
172
+ Parameters
173
+ ----------
174
+ nbytes : int
175
+ Specify the number of bytes to be allocated.
176
+
177
+ Returns
178
+ -------
179
+ buf : CudaBuffer
180
+ Allocated buffer.
181
+ """
182
+ cdef:
183
+ shared_ptr[CCudaBuffer] cudabuf
184
+ with nogil:
185
+ cudabuf = GetResultValue(self.context.get().Allocate(nbytes))
186
+ return pyarrow_wrap_cudabuffer(cudabuf)
187
+
188
+ @property
189
+ def memory_manager(self):
190
+ """
191
+ The default memory manager tied to this context's device.
192
+
193
+ Returns
194
+ -------
195
+ MemoryManager
196
+ """
197
+ return MemoryManager.wrap(self.context.get().memory_manager())
198
+
199
+ @property
200
+ def device(self):
201
+ """
202
+ The device instance associated with this context.
203
+
204
+ Returns
205
+ -------
206
+ Device
207
+ """
208
+ return Device.wrap(self.context.get().device())
209
+
210
+ def foreign_buffer(self, address, size, base=None):
211
+ """
212
+ Create device buffer from address and size as a view.
213
+
214
+ The caller is responsible for allocating and freeing the
215
+ memory. When `address==size==0` then a new zero-sized buffer
216
+ is returned.
217
+
218
+ Parameters
219
+ ----------
220
+ address : int
221
+ Specify the starting address of the buffer. The address can
222
+ refer to both device or host memory but it must be
223
+ accessible from device after mapping it with
224
+ `get_device_address` method.
225
+ size : int
226
+ Specify the size of device buffer in bytes.
227
+ base : {None, object}
228
+ Specify object that owns the referenced memory.
229
+
230
+ Returns
231
+ -------
232
+ cbuf : CudaBuffer
233
+ Device buffer as a view of device reachable memory.
234
+
235
+ """
236
+ if not address and size == 0:
237
+ return self.new_buffer(0)
238
+ cdef:
239
+ uintptr_t c_addr = self.get_device_address(address)
240
+ int64_t c_size = size
241
+ shared_ptr[CCudaBuffer] cudabuf
242
+
243
+ cudabuf = GetResultValue(self.context.get().View(
244
+ <uint8_t*>c_addr, c_size))
245
+ return pyarrow_wrap_cudabuffer_base(cudabuf, base)
246
+
247
+ def open_ipc_buffer(self, ipc_handle):
248
+ """ Open existing CUDA IPC memory handle
249
+
250
+ Parameters
251
+ ----------
252
+ ipc_handle : IpcMemHandle
253
+ Specify opaque pointer to CUipcMemHandle (driver API).
254
+
255
+ Returns
256
+ -------
257
+ buf : CudaBuffer
258
+ referencing device buffer
259
+ """
260
+ handle = pyarrow_unwrap_cudaipcmemhandle(ipc_handle)
261
+ cdef shared_ptr[CCudaBuffer] cudabuf
262
+ with nogil:
263
+ cudabuf = GetResultValue(
264
+ self.context.get().OpenIpcBuffer(handle.get()[0]))
265
+ return pyarrow_wrap_cudabuffer(cudabuf)
266
+
267
+ def buffer_from_data(self, object data, int64_t offset=0, int64_t size=-1):
268
+ """Create device buffer and initialize with data.
269
+
270
+ Parameters
271
+ ----------
272
+ data : {CudaBuffer, HostBuffer, Buffer, array-like}
273
+ Specify data to be copied to device buffer.
274
+ offset : int
275
+ Specify the offset of input buffer for device data
276
+ buffering. Default: 0.
277
+ size : int
278
+ Specify the size of device buffer in bytes. Default: all
279
+ (starting from input offset)
280
+
281
+ Returns
282
+ -------
283
+ cbuf : CudaBuffer
284
+ Device buffer with copied data.
285
+ """
286
+ is_host_data = not pyarrow_is_cudabuffer(data)
287
+ buf = as_buffer(data) if is_host_data else data
288
+
289
+ bsize = buf.size
290
+ if offset < 0 or (bsize and offset >= bsize):
291
+ raise ValueError('offset argument is out-of-range')
292
+ if size < 0:
293
+ size = bsize - offset
294
+ elif offset + size > bsize:
295
+ raise ValueError(
296
+ 'requested larger slice than available in device buffer')
297
+
298
+ if offset != 0 or size != bsize:
299
+ buf = buf.slice(offset, size)
300
+
301
+ result = self.new_buffer(size)
302
+ if is_host_data:
303
+ result.copy_from_host(buf, position=0, nbytes=size)
304
+ else:
305
+ result.copy_from_device(buf, position=0, nbytes=size)
306
+ return result
307
+
308
+ def buffer_from_object(self, obj):
309
+ """Create device buffer view of arbitrary object that references
310
+ device accessible memory.
311
+
312
+ When the object contains a non-contiguous view of device
313
+ accessible memory then the returned device buffer will contain
314
+ contiguous view of the memory, that is, including the
315
+ intermediate data that is otherwise invisible to the input
316
+ object.
317
+
318
+ Parameters
319
+ ----------
320
+ obj : {object, Buffer, HostBuffer, CudaBuffer, ...}
321
+ Specify an object that holds (device or host) address that
322
+ can be accessed from device. This includes objects with
323
+ types defined in pyarrow.cuda as well as arbitrary objects
324
+ that implement the CUDA array interface as defined by numba.
325
+
326
+ Returns
327
+ -------
328
+ cbuf : CudaBuffer
329
+ Device buffer as a view of device accessible memory.
330
+
331
+ """
332
+ if isinstance(obj, HostBuffer):
333
+ return self.foreign_buffer(obj.address, obj.size, base=obj)
334
+ elif isinstance(obj, Buffer):
335
+ return CudaBuffer.from_buffer(obj)
336
+ elif isinstance(obj, CudaBuffer):
337
+ return obj
338
+ elif hasattr(obj, '__cuda_array_interface__'):
339
+ desc = obj.__cuda_array_interface__
340
+ addr = desc['data'][0]
341
+ if addr is None:
342
+ return self.new_buffer(0)
343
+ import numpy as np
344
+ start, end = get_contiguous_span(
345
+ desc['shape'], desc.get('strides'),
346
+ np.dtype(desc['typestr']).itemsize)
347
+ return self.foreign_buffer(addr + start, end - start, base=obj)
348
+ raise ArrowTypeError('cannot create device buffer view from'
349
+ ' `%s` object' % (type(obj)))
350
+
351
+
352
+ cdef class IpcMemHandle(_Weakrefable):
353
+ """A serializable container for a CUDA IPC handle.
354
+ """
355
+ cdef void init(self, shared_ptr[CCudaIpcMemHandle]& h):
356
+ self.handle = h
357
+
358
+ @staticmethod
359
+ def from_buffer(Buffer opaque_handle):
360
+ """Create IpcMemHandle from opaque buffer (e.g. from another
361
+ process)
362
+
363
+ Parameters
364
+ ----------
365
+ opaque_handle :
366
+ a CUipcMemHandle as a const void*
367
+
368
+ Returns
369
+ -------
370
+ ipc_handle : IpcMemHandle
371
+ """
372
+ c_buf = pyarrow_unwrap_buffer(opaque_handle)
373
+ cdef:
374
+ shared_ptr[CCudaIpcMemHandle] handle
375
+
376
+ handle = GetResultValue(
377
+ CCudaIpcMemHandle.FromBuffer(c_buf.get().data()))
378
+ return pyarrow_wrap_cudaipcmemhandle(handle)
379
+
380
+ def serialize(self, pool=None):
381
+ """Write IpcMemHandle to a Buffer
382
+
383
+ Parameters
384
+ ----------
385
+ pool : {MemoryPool, None}
386
+ Specify a pool to allocate memory from
387
+
388
+ Returns
389
+ -------
390
+ buf : Buffer
391
+ The serialized buffer.
392
+ """
393
+ cdef CMemoryPool* pool_ = maybe_unbox_memory_pool(pool)
394
+ cdef shared_ptr[CBuffer] buf
395
+ cdef CCudaIpcMemHandle* h = self.handle.get()
396
+ with nogil:
397
+ buf = GetResultValue(h.Serialize(pool_))
398
+ return pyarrow_wrap_buffer(buf)
399
+
400
+
401
+ cdef class CudaBuffer(Buffer):
402
+ """An Arrow buffer with data located in a GPU device.
403
+
404
+ To create a CudaBuffer instance, use Context.device_buffer().
405
+
406
+ The memory allocated in a CudaBuffer is freed when the buffer object
407
+ is deleted.
408
+ """
409
+
410
+ def __init__(self):
411
+ raise TypeError("Do not call CudaBuffer's constructor directly, use "
412
+ "`<pyarrow.Context instance>.device_buffer`"
413
+ " method instead.")
414
+
415
+ cdef void init_cuda(self,
416
+ const shared_ptr[CCudaBuffer]& buffer,
417
+ object base):
418
+ self.cuda_buffer = buffer
419
+ self.init(<shared_ptr[CBuffer]> buffer)
420
+ self.base = base
421
+
422
+ @staticmethod
423
+ def from_buffer(buf):
424
+ """ Convert back generic buffer into CudaBuffer
425
+
426
+ Parameters
427
+ ----------
428
+ buf : Buffer
429
+ Specify buffer containing CudaBuffer
430
+
431
+ Returns
432
+ -------
433
+ dbuf : CudaBuffer
434
+ Resulting device buffer.
435
+ """
436
+ c_buf = pyarrow_unwrap_buffer(buf)
437
+ cuda_buffer = GetResultValue(CCudaBuffer.FromBuffer(c_buf))
438
+ return pyarrow_wrap_cudabuffer(cuda_buffer)
439
+
440
+ @staticmethod
441
+ def from_numba(mem):
442
+ """Create a CudaBuffer view from numba MemoryPointer instance.
443
+
444
+ Parameters
445
+ ----------
446
+ mem : numba.cuda.cudadrv.driver.MemoryPointer
447
+
448
+ Returns
449
+ -------
450
+ cbuf : CudaBuffer
451
+ Device buffer as a view of numba MemoryPointer.
452
+ """
453
+ ctx = Context.from_numba(mem.context)
454
+ if mem.device_pointer_value is None and mem.size==0:
455
+ return ctx.new_buffer(0)
456
+ return ctx.foreign_buffer(mem.device_pointer_value, mem.size, base=mem)
457
+
458
+ def to_numba(self):
459
+ """Return numba memory pointer of CudaBuffer instance.
460
+ """
461
+ import ctypes
462
+ from numba.cuda.cudadrv.driver import MemoryPointer
463
+ try:
464
+ return MemoryPointer(self.context.to_numba(),
465
+ pointer=ctypes.c_void_p(self.address),
466
+ size=self.size)
467
+ except TypeError:
468
+ # Newer Numba does not take a context argument anymore
469
+ return MemoryPointer(pointer=ctypes.c_void_p(self.address),
470
+ size=self.size)
471
+
472
+ cdef getitem(self, int64_t i):
473
+ return self.copy_to_host(position=i, nbytes=1)[0]
474
+
475
+ def copy_to_host(self, int64_t position=0, int64_t nbytes=-1,
476
+ Buffer buf=None,
477
+ MemoryPool memory_pool=None, c_bool resizable=False):
478
+ """Copy memory from GPU device to CPU host
479
+
480
+ Caller is responsible for ensuring that all tasks affecting
481
+ the memory are finished. Use
482
+
483
+ `<CudaBuffer instance>.context.synchronize()`
484
+
485
+ when needed.
486
+
487
+ Parameters
488
+ ----------
489
+ position : int
490
+ Specify the starting position of the source data in GPU
491
+ device buffer. Default: 0.
492
+ nbytes : int
493
+ Specify the number of bytes to copy. Default: -1 (all from
494
+ the position until host buffer is full).
495
+ buf : Buffer
496
+ Specify a pre-allocated output buffer in host. Default: None
497
+ (allocate new output buffer).
498
+ memory_pool : MemoryPool
499
+ resizable : bool
500
+ Specify extra arguments to allocate_buffer. Used only when
501
+ buf is None.
502
+
503
+ Returns
504
+ -------
505
+ buf : Buffer
506
+ Output buffer in host.
507
+
508
+ """
509
+ if position < 0 or (self.size and position > self.size) \
510
+ or (self.size == 0 and position != 0):
511
+ raise ValueError('position argument is out-of-range')
512
+ cdef:
513
+ int64_t c_nbytes
514
+ if buf is None:
515
+ if nbytes < 0:
516
+ # copy all starting from position to new host buffer
517
+ c_nbytes = self.size - position
518
+ else:
519
+ if nbytes > self.size - position:
520
+ raise ValueError(
521
+ 'requested more to copy than available from '
522
+ 'device buffer')
523
+ # copy nbytes starting from position to new host buffer
524
+ c_nbytes = nbytes
525
+ buf = allocate_buffer(c_nbytes, memory_pool=memory_pool,
526
+ resizable=resizable)
527
+ else:
528
+ if nbytes < 0:
529
+ # copy all from position until given host buffer is full
530
+ c_nbytes = min(self.size - position, buf.size)
531
+ else:
532
+ if nbytes > buf.size:
533
+ raise ValueError(
534
+ 'requested copy does not fit into host buffer')
535
+ # copy nbytes from position to given host buffer
536
+ c_nbytes = nbytes
537
+
538
+ cdef:
539
+ shared_ptr[CBuffer] c_buf = pyarrow_unwrap_buffer(buf)
540
+ int64_t c_position = position
541
+ with nogil:
542
+ check_status(self.cuda_buffer.get()
543
+ .CopyToHost(c_position, c_nbytes,
544
+ c_buf.get().mutable_data()))
545
+ return buf
546
+
547
+ def copy_from_host(self, data, int64_t position=0, int64_t nbytes=-1):
548
+ """Copy data from host to device.
549
+
550
+ The device buffer must be pre-allocated.
551
+
552
+ Parameters
553
+ ----------
554
+ data : {Buffer, array-like}
555
+ Specify data in host. It can be array-like that is valid
556
+ argument to py_buffer
557
+ position : int
558
+ Specify the starting position of the copy in device buffer.
559
+ Default: 0.
560
+ nbytes : int
561
+ Specify the number of bytes to copy. Default: -1 (all from
562
+ source until device buffer, starting from position, is full)
563
+
564
+ Returns
565
+ -------
566
+ nbytes : int
567
+ Number of bytes copied.
568
+ """
569
+ if position < 0 or position > self.size:
570
+ raise ValueError('position argument is out-of-range')
571
+ cdef:
572
+ int64_t c_nbytes
573
+ buf = as_buffer(data)
574
+
575
+ if nbytes < 0:
576
+ # copy from host buffer to device buffer starting from
577
+ # position until device buffer is full
578
+ c_nbytes = min(self.size - position, buf.size)
579
+ else:
580
+ if nbytes > buf.size:
581
+ raise ValueError(
582
+ 'requested more to copy than available from host buffer')
583
+ if nbytes > self.size - position:
584
+ raise ValueError(
585
+ 'requested more to copy than available in device buffer')
586
+ # copy nbytes from host buffer to device buffer starting
587
+ # from position
588
+ c_nbytes = nbytes
589
+
590
+ cdef:
591
+ shared_ptr[CBuffer] c_buf = pyarrow_unwrap_buffer(buf)
592
+ int64_t c_position = position
593
+ with nogil:
594
+ check_status(self.cuda_buffer.get().
595
+ CopyFromHost(c_position, c_buf.get().data(),
596
+ c_nbytes))
597
+ return c_nbytes
598
+
599
+ def copy_from_device(self, buf, int64_t position=0, int64_t nbytes=-1):
600
+ """Copy data from device to device.
601
+
602
+ Parameters
603
+ ----------
604
+ buf : CudaBuffer
605
+ Specify source device buffer.
606
+ position : int
607
+ Specify the starting position of the copy in device buffer.
608
+ Default: 0.
609
+ nbytes : int
610
+ Specify the number of bytes to copy. Default: -1 (all from
611
+ source until device buffer, starting from position, is full)
612
+
613
+ Returns
614
+ -------
615
+ nbytes : int
616
+ Number of bytes copied.
617
+
618
+ """
619
+ if position < 0 or position > self.size:
620
+ raise ValueError('position argument is out-of-range')
621
+ cdef:
622
+ int64_t c_nbytes
623
+
624
+ if nbytes < 0:
625
+ # copy from source device buffer to device buffer starting
626
+ # from position until device buffer is full
627
+ c_nbytes = min(self.size - position, buf.size)
628
+ else:
629
+ if nbytes > buf.size:
630
+ raise ValueError(
631
+ 'requested more to copy than available from device buffer')
632
+ if nbytes > self.size - position:
633
+ raise ValueError(
634
+ 'requested more to copy than available in device buffer')
635
+ # copy nbytes from source device buffer to device buffer
636
+ # starting from position
637
+ c_nbytes = nbytes
638
+
639
+ cdef:
640
+ shared_ptr[CCudaBuffer] c_buf = pyarrow_unwrap_cudabuffer(buf)
641
+ int64_t c_position = position
642
+ shared_ptr[CCudaContext] c_src_ctx = pyarrow_unwrap_cudacontext(
643
+ buf.context)
644
+ void* c_source_data = <void*>(c_buf.get().address())
645
+
646
+ if self.context.handle != buf.context.handle:
647
+ with nogil:
648
+ check_status(self.cuda_buffer.get().
649
+ CopyFromAnotherDevice(c_src_ctx, c_position,
650
+ c_source_data, c_nbytes))
651
+ else:
652
+ with nogil:
653
+ check_status(self.cuda_buffer.get().
654
+ CopyFromDevice(c_position, c_source_data,
655
+ c_nbytes))
656
+ return c_nbytes
657
+
658
+ def export_for_ipc(self):
659
+ """
660
+ Expose this device buffer as IPC memory which can be used in other
661
+ processes.
662
+
663
+ After calling this function, this device memory will not be
664
+ freed when the CudaBuffer is destructed.
665
+
666
+ Returns
667
+ -------
668
+ ipc_handle : IpcMemHandle
669
+ The exported IPC handle
670
+
671
+ """
672
+ cdef shared_ptr[CCudaIpcMemHandle] handle
673
+ with nogil:
674
+ handle = GetResultValue(self.cuda_buffer.get().ExportForIpc())
675
+ return pyarrow_wrap_cudaipcmemhandle(handle)
676
+
677
+ @property
678
+ def context(self):
679
+ """Returns the CUDA driver context of this buffer.
680
+ """
681
+ return pyarrow_wrap_cudacontext(self.cuda_buffer.get().context())
682
+
683
+ def slice(self, offset=0, length=None):
684
+ """Return slice of device buffer
685
+
686
+ Parameters
687
+ ----------
688
+ offset : int, default 0
689
+ Specify offset from the start of device buffer to slice
690
+ length : int, default None
691
+ Specify the length of slice (default is until end of device
692
+ buffer starting from offset). If the length is larger than
693
+ the data available, the returned slice will have a size of
694
+ the available data starting from the offset.
695
+
696
+ Returns
697
+ -------
698
+ sliced : CudaBuffer
699
+ Zero-copy slice of device buffer.
700
+
701
+ """
702
+ if offset < 0 or (self.size and offset >= self.size):
703
+ raise ValueError('offset argument is out-of-range')
704
+ cdef int64_t offset_ = offset
705
+ cdef int64_t size
706
+ if length is None:
707
+ size = self.size - offset_
708
+ elif offset + length <= self.size:
709
+ size = length
710
+ else:
711
+ size = self.size - offset
712
+ parent = pyarrow_unwrap_cudabuffer(self)
713
+ return pyarrow_wrap_cudabuffer(make_shared[CCudaBuffer](parent,
714
+ offset_, size))
715
+
716
+ def to_pybytes(self):
717
+ """Return device buffer content as Python bytes.
718
+ """
719
+ return self.copy_to_host().to_pybytes()
720
+
721
+ def __getbuffer__(self, cp.Py_buffer* buffer, int flags):
722
+ # Device buffer contains data pointers on the device. Hence,
723
+ # cannot support buffer protocol PEP-3118 for CudaBuffer.
724
+ raise BufferError('buffer protocol for device buffer not supported')
725
+
726
+
727
+ cdef class HostBuffer(Buffer):
728
+ """Device-accessible CPU memory created using cudaHostAlloc.
729
+
730
+ To create a HostBuffer instance, use
731
+
732
+ cuda.new_host_buffer(<nbytes>)
733
+ """
734
+
735
+ def __init__(self):
736
+ raise TypeError("Do not call HostBuffer's constructor directly,"
737
+ " use `cuda.new_host_buffer` function instead.")
738
+
739
+ cdef void init_host(self, const shared_ptr[CCudaHostBuffer]& buffer):
740
+ self.host_buffer = buffer
741
+ self.init(<shared_ptr[CBuffer]> buffer)
742
+
743
+ @property
744
+ def size(self):
745
+ return self.host_buffer.get().size()
746
+
747
+
748
+ cdef class BufferReader(NativeFile):
749
+ """File interface for zero-copy read from CUDA buffers.
750
+
751
+ Note: Read methods return pointers to device memory. This means
752
+ you must be careful using this interface with any Arrow code which
753
+ may expect to be able to do anything other than pointer arithmetic
754
+ on the returned buffers.
755
+ """
756
+
757
+ def __cinit__(self, CudaBuffer obj):
758
+ self.buffer = obj
759
+ self.reader = new CCudaBufferReader(self.buffer.buffer)
760
+ self.set_random_access_file(
761
+ shared_ptr[CRandomAccessFile](self.reader))
762
+ self.is_readable = True
763
+
764
+ def read_buffer(self, nbytes=None):
765
+ """Return a slice view of the underlying device buffer.
766
+
767
+ The slice will start at the current reader position and will
768
+ have specified size in bytes.
769
+
770
+ Parameters
771
+ ----------
772
+ nbytes : int, default None
773
+ Specify the number of bytes to read. Default: None (read all
774
+ remaining bytes).
775
+
776
+ Returns
777
+ -------
778
+ cbuf : CudaBuffer
779
+ New device buffer.
780
+
781
+ """
782
+ cdef:
783
+ int64_t c_nbytes
784
+ shared_ptr[CCudaBuffer] output
785
+
786
+ if nbytes is None:
787
+ c_nbytes = self.size() - self.tell()
788
+ else:
789
+ c_nbytes = nbytes
790
+
791
+ with nogil:
792
+ output = static_pointer_cast[CCudaBuffer, CBuffer](
793
+ GetResultValue(self.reader.Read(c_nbytes)))
794
+
795
+ return pyarrow_wrap_cudabuffer(output)
796
+
797
+
798
+ cdef class BufferWriter(NativeFile):
799
+ """File interface for writing to CUDA buffers.
800
+
801
+ By default writes are unbuffered. Use set_buffer_size to enable
802
+ buffering.
803
+ """
804
+
805
+ def __cinit__(self, CudaBuffer buffer):
806
+ self.buffer = buffer
807
+ self.writer = new CCudaBufferWriter(self.buffer.cuda_buffer)
808
+ self.set_output_stream(shared_ptr[COutputStream](self.writer))
809
+ self.is_writable = True
810
+
811
+ def writeat(self, int64_t position, object data):
812
+ """Write data to buffer starting from position.
813
+
814
+ Parameters
815
+ ----------
816
+ position : int
817
+ Specify device buffer position where the data will be
818
+ written.
819
+ data : array-like
820
+ Specify data, the data instance must implement buffer
821
+ protocol.
822
+ """
823
+ cdef:
824
+ Buffer buf = as_buffer(data)
825
+ const uint8_t* c_data = buf.buffer.get().data()
826
+ int64_t c_size = buf.buffer.get().size()
827
+
828
+ with nogil:
829
+ check_status(self.writer.WriteAt(position, c_data, c_size))
830
+
831
+ def flush(self):
832
+ """ Flush the buffer stream """
833
+ with nogil:
834
+ check_status(self.writer.Flush())
835
+
836
+ def seek(self, int64_t position, int whence=0):
837
+ # TODO: remove this method after NativeFile.seek supports
838
+ # writable files.
839
+ cdef int64_t offset
840
+
841
+ with nogil:
842
+ if whence == 0:
843
+ offset = position
844
+ elif whence == 1:
845
+ offset = GetResultValue(self.writer.Tell())
846
+ offset = offset + position
847
+ else:
848
+ with gil:
849
+ raise ValueError(f"Invalid value of whence: {whence}")
850
+ check_status(self.writer.Seek(offset))
851
+ return self.tell()
852
+
853
+ @property
854
+ def buffer_size(self):
855
+ """Returns size of host (CPU) buffer, 0 for unbuffered
856
+ """
857
+ return self.writer.buffer_size()
858
+
859
+ @buffer_size.setter
860
+ def buffer_size(self, int64_t buffer_size):
861
+ """Set CPU buffer size to limit calls to cudaMemcpy
862
+
863
+ Parameters
864
+ ----------
865
+ buffer_size : int
866
+ Specify the size of CPU buffer to allocate in bytes.
867
+ """
868
+ with nogil:
869
+ check_status(self.writer.SetBufferSize(buffer_size))
870
+
871
+ @property
872
+ def num_bytes_buffered(self):
873
+ """Returns number of bytes buffered on host
874
+ """
875
+ return self.writer.num_bytes_buffered()
876
+
877
+ # Functions
878
+
879
+
880
+ def new_host_buffer(const int64_t size, int device=0):
881
+ """Return buffer with CUDA-accessible memory on CPU host
882
+
883
+ Parameters
884
+ ----------
885
+ size : int
886
+ Specify the number of bytes to be allocated.
887
+ device : int
888
+ Specify GPU device number.
889
+
890
+ Returns
891
+ -------
892
+ dbuf : HostBuffer
893
+ Allocated host buffer
894
+ """
895
+ cdef shared_ptr[CCudaHostBuffer] buffer
896
+ with nogil:
897
+ buffer = GetResultValue(AllocateCudaHostBuffer(device, size))
898
+ return pyarrow_wrap_cudahostbuffer(buffer)
899
+
900
+
901
+ def serialize_record_batch(object batch, object ctx):
902
+ """ Write record batch message to GPU device memory
903
+
904
+ Parameters
905
+ ----------
906
+ batch : RecordBatch
907
+ Record batch to write
908
+ ctx : Context
909
+ CUDA Context to allocate device memory from
910
+
911
+ Returns
912
+ -------
913
+ dbuf : CudaBuffer
914
+ device buffer which contains the record batch message
915
+ """
916
+ cdef shared_ptr[CCudaBuffer] buffer
917
+ cdef CRecordBatch* batch_ = pyarrow_unwrap_batch(batch).get()
918
+ cdef CCudaContext* ctx_ = pyarrow_unwrap_cudacontext(ctx).get()
919
+ with nogil:
920
+ buffer = GetResultValue(CudaSerializeRecordBatch(batch_[0], ctx_))
921
+ return pyarrow_wrap_cudabuffer(buffer)
922
+
923
+
924
+ def read_message(object source, pool=None):
925
+ """ Read Arrow IPC message located on GPU device
926
+
927
+ Parameters
928
+ ----------
929
+ source : {CudaBuffer, cuda.BufferReader}
930
+ Device buffer or reader of device buffer.
931
+ pool : MemoryPool (optional)
932
+ Pool to allocate CPU memory for the metadata
933
+
934
+ Returns
935
+ -------
936
+ message : Message
937
+ The deserialized message, body still on device
938
+ """
939
+ cdef:
940
+ Message result = Message.__new__(Message)
941
+ cdef CMemoryPool* pool_ = maybe_unbox_memory_pool(pool)
942
+ if not isinstance(source, BufferReader):
943
+ reader = BufferReader(source)
944
+ with nogil:
945
+ result.message = move(
946
+ GetResultValue(ReadMessage(reader.reader, pool_)))
947
+ return result
948
+
949
+
950
+ def read_record_batch(object buffer, object schema, *,
951
+ DictionaryMemo dictionary_memo=None, pool=None):
952
+ """Construct RecordBatch referencing IPC message located on CUDA device.
953
+
954
+ While the metadata is copied to host memory for deserialization,
955
+ the record batch data remains on the device.
956
+
957
+ Parameters
958
+ ----------
959
+ buffer :
960
+ Device buffer containing the complete IPC message
961
+ schema : Schema
962
+ The schema for the record batch
963
+ dictionary_memo : DictionaryMemo, optional
964
+ If message contains dictionaries, must pass a populated
965
+ DictionaryMemo
966
+ pool : MemoryPool (optional)
967
+ Pool to allocate metadata from
968
+
969
+ Returns
970
+ -------
971
+ batch : RecordBatch
972
+ Reconstructed record batch, with device pointers
973
+
974
+ """
975
+ cdef:
976
+ shared_ptr[CSchema] schema_ = pyarrow_unwrap_schema(schema)
977
+ shared_ptr[CCudaBuffer] buffer_ = pyarrow_unwrap_cudabuffer(buffer)
978
+ CDictionaryMemo temp_memo
979
+ CDictionaryMemo* arg_dict_memo
980
+ CMemoryPool* pool_ = maybe_unbox_memory_pool(pool)
981
+ shared_ptr[CRecordBatch] batch
982
+
983
+ if dictionary_memo is not None:
984
+ arg_dict_memo = dictionary_memo.memo
985
+ else:
986
+ arg_dict_memo = &temp_memo
987
+
988
+ with nogil:
989
+ batch = GetResultValue(CudaReadRecordBatch(
990
+ schema_, arg_dict_memo, buffer_, pool_))
991
+ return pyarrow_wrap_batch(batch)
992
+
993
+
994
+ # Public API
995
+
996
+
997
+ cdef public api bint pyarrow_is_buffer(object buffer):
998
+ return isinstance(buffer, Buffer)
999
+
1000
+ # cudabuffer
1001
+
1002
+ cdef public api bint pyarrow_is_cudabuffer(object buffer):
1003
+ return isinstance(buffer, CudaBuffer)
1004
+
1005
+
1006
+ cdef public api object \
1007
+ pyarrow_wrap_cudabuffer_base(const shared_ptr[CCudaBuffer]& buf, base):
1008
+ cdef CudaBuffer result = CudaBuffer.__new__(CudaBuffer)
1009
+ result.init_cuda(buf, base)
1010
+ return result
1011
+
1012
+
1013
+ cdef public api object \
1014
+ pyarrow_wrap_cudabuffer(const shared_ptr[CCudaBuffer]& buf):
1015
+ cdef CudaBuffer result = CudaBuffer.__new__(CudaBuffer)
1016
+ result.init_cuda(buf, None)
1017
+ return result
1018
+
1019
+
1020
+ cdef public api shared_ptr[CCudaBuffer] pyarrow_unwrap_cudabuffer(object obj):
1021
+ if pyarrow_is_cudabuffer(obj):
1022
+ return (<CudaBuffer>obj).cuda_buffer
1023
+ raise TypeError('expected CudaBuffer instance, got %s'
1024
+ % (type(obj).__name__))
1025
+
1026
+ # cudahostbuffer
1027
+
1028
+ cdef public api bint pyarrow_is_cudahostbuffer(object buffer):
1029
+ return isinstance(buffer, HostBuffer)
1030
+
1031
+
1032
+ cdef public api object \
1033
+ pyarrow_wrap_cudahostbuffer(const shared_ptr[CCudaHostBuffer]& buf):
1034
+ cdef HostBuffer result = HostBuffer.__new__(HostBuffer)
1035
+ result.init_host(buf)
1036
+ return result
1037
+
1038
+
1039
+ cdef public api shared_ptr[CCudaHostBuffer] \
1040
+ pyarrow_unwrap_cudahostbuffer(object obj):
1041
+ if pyarrow_is_cudahostbuffer(obj):
1042
+ return (<HostBuffer>obj).host_buffer
1043
+ raise TypeError('expected HostBuffer instance, got %s'
1044
+ % (type(obj).__name__))
1045
+
1046
+ # cudacontext
1047
+
1048
+ cdef public api bint pyarrow_is_cudacontext(object ctx):
1049
+ return isinstance(ctx, Context)
1050
+
1051
+
1052
+ cdef public api object \
1053
+ pyarrow_wrap_cudacontext(const shared_ptr[CCudaContext]& ctx):
1054
+ cdef Context result = Context.__new__(Context)
1055
+ result.init(ctx)
1056
+ return result
1057
+
1058
+
1059
+ cdef public api shared_ptr[CCudaContext] \
1060
+ pyarrow_unwrap_cudacontext(object obj):
1061
+ if pyarrow_is_cudacontext(obj):
1062
+ return (<Context>obj).context
1063
+ raise TypeError('expected Context instance, got %s'
1064
+ % (type(obj).__name__))
1065
+
1066
+ # cudaipcmemhandle
1067
+
1068
+ cdef public api bint pyarrow_is_cudaipcmemhandle(object handle):
1069
+ return isinstance(handle, IpcMemHandle)
1070
+
1071
+
1072
+ cdef public api object \
1073
+ pyarrow_wrap_cudaipcmemhandle(shared_ptr[CCudaIpcMemHandle]& h):
1074
+ cdef IpcMemHandle result = IpcMemHandle.__new__(IpcMemHandle)
1075
+ result.init(h)
1076
+ return result
1077
+
1078
+
1079
+ cdef public api shared_ptr[CCudaIpcMemHandle] \
1080
+ pyarrow_unwrap_cudaipcmemhandle(object obj):
1081
+ if pyarrow_is_cudaipcmemhandle(obj):
1082
+ return (<IpcMemHandle>obj).handle
1083
+ raise TypeError('expected IpcMemHandle instance, got %s'
1084
+ % (type(obj).__name__))
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0df3f1c21a22218c9122e5cf8412636584bf9a49387db67939ccccf61d52241
3
+ size 704000
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pxd ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ """Dataset is currently unstable. APIs subject to change without notice."""
21
+
22
+ from pyarrow.includes.common cimport *
23
+ from pyarrow.includes.libarrow_dataset cimport *
24
+ from pyarrow.lib cimport *
25
+ from pyarrow._fs cimport FileSystem, FileInfo
26
+
27
+
28
+ cdef CFileSource _make_file_source(object file, FileSystem filesystem=*, object file_size=*)
29
+
30
+ cdef class DatasetFactory(_Weakrefable):
31
+
32
+ cdef:
33
+ SharedPtrNoGIL[CDatasetFactory] wrapped
34
+ CDatasetFactory* factory
35
+
36
+ cdef init(self, const shared_ptr[CDatasetFactory]& sp)
37
+
38
+ @staticmethod
39
+ cdef wrap(const shared_ptr[CDatasetFactory]& sp)
40
+
41
+ cdef inline shared_ptr[CDatasetFactory] unwrap(self) nogil
42
+
43
+
44
+ cdef class Dataset(_Weakrefable):
45
+
46
+ cdef:
47
+ SharedPtrNoGIL[CDataset] wrapped
48
+ CDataset* dataset
49
+ public dict _scan_options
50
+
51
+ cdef void init(self, const shared_ptr[CDataset]& sp)
52
+
53
+ @staticmethod
54
+ cdef wrap(const shared_ptr[CDataset]& sp)
55
+
56
+ cdef shared_ptr[CDataset] unwrap(self) nogil
57
+
58
+
59
+ cdef class Scanner(_Weakrefable):
60
+ cdef:
61
+ SharedPtrNoGIL[CScanner] wrapped
62
+ CScanner* scanner
63
+
64
+ cdef void init(self, const shared_ptr[CScanner]& sp)
65
+
66
+ @staticmethod
67
+ cdef wrap(const shared_ptr[CScanner]& sp)
68
+
69
+ cdef shared_ptr[CScanner] unwrap(self)
70
+
71
+ @staticmethod
72
+ cdef shared_ptr[CScanOptions] _make_scan_options(Dataset dataset, dict py_scanoptions) except *
73
+
74
+
75
+ cdef class FragmentScanOptions(_Weakrefable):
76
+
77
+ cdef:
78
+ shared_ptr[CFragmentScanOptions] wrapped
79
+
80
+ cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp)
81
+
82
+ @staticmethod
83
+ cdef wrap(const shared_ptr[CFragmentScanOptions]& sp)
84
+
85
+
86
+ cdef class FileFormat(_Weakrefable):
87
+
88
+ cdef:
89
+ shared_ptr[CFileFormat] wrapped
90
+ CFileFormat* format
91
+
92
+ cdef void init(self, const shared_ptr[CFileFormat]& sp)
93
+
94
+ @staticmethod
95
+ cdef wrap(const shared_ptr[CFileFormat]& sp)
96
+
97
+ cdef inline shared_ptr[CFileFormat] unwrap(self)
98
+
99
+ cdef _set_default_fragment_scan_options(self, FragmentScanOptions options)
100
+
101
+ # Return a WrittenFile after a file was written.
102
+ # May be overridden by subclasses, e.g. to add metadata.
103
+ cdef WrittenFile _finish_write(self, path, base_dir,
104
+ CFileWriter* file_writer)
105
+
106
+
107
+ cdef class FileWriteOptions(_Weakrefable):
108
+
109
+ cdef:
110
+ shared_ptr[CFileWriteOptions] wrapped
111
+ CFileWriteOptions* c_options
112
+
113
+ cdef void init(self, const shared_ptr[CFileWriteOptions]& sp)
114
+
115
+ @staticmethod
116
+ cdef wrap(const shared_ptr[CFileWriteOptions]& sp)
117
+
118
+ cdef inline shared_ptr[CFileWriteOptions] unwrap(self)
119
+
120
+
121
+ cdef class Fragment(_Weakrefable):
122
+
123
+ cdef:
124
+ SharedPtrNoGIL[CFragment] wrapped
125
+ CFragment* fragment
126
+
127
+ cdef void init(self, const shared_ptr[CFragment]& sp)
128
+
129
+ @staticmethod
130
+ cdef wrap(const shared_ptr[CFragment]& sp)
131
+
132
+ cdef inline shared_ptr[CFragment] unwrap(self)
133
+
134
+
135
+ cdef class FileFragment(Fragment):
136
+
137
+ cdef:
138
+ CFileFragment* file_fragment
139
+
140
+ cdef void init(self, const shared_ptr[CFragment]& sp)
141
+
142
+
143
+ cdef class Partitioning(_Weakrefable):
144
+
145
+ cdef:
146
+ shared_ptr[CPartitioning] wrapped
147
+ CPartitioning* partitioning
148
+
149
+ cdef init(self, const shared_ptr[CPartitioning]& sp)
150
+
151
+ @staticmethod
152
+ cdef wrap(const shared_ptr[CPartitioning]& sp)
153
+
154
+ cdef inline shared_ptr[CPartitioning] unwrap(self)
155
+
156
+
157
+ cdef class PartitioningFactory(_Weakrefable):
158
+
159
+ cdef:
160
+ shared_ptr[CPartitioningFactory] wrapped
161
+ CPartitioningFactory* factory
162
+ object constructor
163
+ object options
164
+
165
+ cdef init(self, const shared_ptr[CPartitioningFactory]& sp)
166
+
167
+ @staticmethod
168
+ cdef wrap(const shared_ptr[CPartitioningFactory]& sp,
169
+ object constructor, object options)
170
+
171
+ cdef inline shared_ptr[CPartitioningFactory] unwrap(self)
172
+
173
+
174
+ cdef class WrittenFile(_Weakrefable):
175
+
176
+ # The full path to the created file
177
+ cdef public str path
178
+ # Optional Parquet metadata
179
+ # This metadata will have the file path attribute set to the path of
180
+ # the written file.
181
+ cdef public object metadata
182
+ # The size of the file in bytes
183
+ cdef public int64_t size
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pyx ADDED
The diff for this file is too large to render. See raw diff
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.cp312-win_amd64.pyd ADDED
Binary file (69.6 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.pyx ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ """Dataset support for ORC file format."""
21
+
22
+ from pyarrow.lib cimport *
23
+ from pyarrow.includes.libarrow cimport *
24
+ from pyarrow.includes.libarrow_dataset cimport *
25
+
26
+ from pyarrow._dataset cimport FileFormat
27
+
28
+
29
+ cdef class OrcFileFormat(FileFormat):
30
+
31
+ def __init__(self):
32
+ self.init(shared_ptr[CFileFormat](new COrcFileFormat()))
33
+
34
+ def equals(self, OrcFileFormat other):
35
+ """
36
+ Parameters
37
+ ----------
38
+ other : pyarrow.dataset.OrcFileFormat
39
+
40
+ Returns
41
+ -------
42
+ True
43
+ """
44
+ return True
45
+
46
+ @property
47
+ def default_extname(self):
48
+ return "orc"
49
+
50
+ def __reduce__(self):
51
+ return OrcFileFormat, tuple()
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c33edeeee46454a49d78803f343a137e1b6cf5e8ed512712309bc1023f881f0e
3
+ size 253952
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pxd ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ """Dataset support for Parquet file format."""
21
+
22
+ from pyarrow.includes.libarrow_dataset cimport *
23
+ from pyarrow.includes.libarrow_dataset_parquet cimport *
24
+
25
+ from pyarrow._dataset cimport FragmentScanOptions, FileWriteOptions
26
+
27
+
28
+ cdef class ParquetFragmentScanOptions(FragmentScanOptions):
29
+ cdef:
30
+ CParquetFragmentScanOptions* parquet_options
31
+ object _parquet_decryption_config
32
+ object _decryption_properties
33
+
34
+ cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp)
35
+ cdef CReaderProperties* reader_properties(self)
36
+ cdef ArrowReaderProperties* arrow_reader_properties(self)
37
+
38
+
39
+ cdef class ParquetFileWriteOptions(FileWriteOptions):
40
+
41
+ cdef:
42
+ CParquetFileWriteOptions* parquet_options
43
+ object _properties
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pyx ADDED
@@ -0,0 +1,1113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ """Dataset support for Parquet file format."""
21
+
22
+ from cython.operator cimport dereference as deref
23
+
24
+ import os
25
+ import warnings
26
+
27
+ import pyarrow as pa
28
+ from pyarrow.lib cimport *
29
+ from pyarrow.lib import frombytes, tobytes, is_threading_enabled
30
+ from pyarrow.includes.libarrow cimport *
31
+ from pyarrow.includes.libarrow_dataset cimport *
32
+ from pyarrow.includes.libarrow_dataset_parquet cimport *
33
+ from pyarrow._fs cimport FileSystem
34
+
35
+ from pyarrow._compute cimport Expression, _bind
36
+ from pyarrow._dataset cimport (
37
+ _make_file_source,
38
+ DatasetFactory,
39
+ FileFormat,
40
+ FileFragment,
41
+ FileWriteOptions,
42
+ Fragment,
43
+ FragmentScanOptions,
44
+ CacheOptions,
45
+ Partitioning,
46
+ PartitioningFactory,
47
+ WrittenFile
48
+ )
49
+
50
+ from pyarrow._parquet cimport (
51
+ _create_writer_properties, _create_arrow_writer_properties,
52
+ _unwrap_list_type, FileMetaData,
53
+ )
54
+
55
+
56
+ try:
57
+ from pyarrow._dataset_parquet_encryption import (
58
+ set_encryption_config, set_decryption_config, set_decryption_properties
59
+ )
60
+ parquet_encryption_enabled = True
61
+ except ImportError:
62
+ parquet_encryption_enabled = False
63
+
64
+
65
+ cdef Expression _true = Expression._scalar(True)
66
+
67
+ ctypedef CParquetFileWriter* _CParquetFileWriterPtr
68
+
69
+
70
+ cdef class ParquetFileFormat(FileFormat):
71
+ """
72
+ FileFormat for Parquet
73
+
74
+ Parameters
75
+ ----------
76
+ read_options : ParquetReadOptions
77
+ Read options for the file.
78
+ default_fragment_scan_options : ParquetFragmentScanOptions
79
+ Scan Options for the file.
80
+ **kwargs : dict
81
+ Additional options for read option or scan option
82
+ """
83
+
84
+ cdef:
85
+ CParquetFileFormat* parquet_format
86
+
87
+ def __init__(self, read_options=None,
88
+ default_fragment_scan_options=None,
89
+ **kwargs):
90
+ cdef:
91
+ shared_ptr[CParquetFileFormat] wrapped
92
+ CParquetFileFormatReaderOptions* options
93
+
94
+ # Read/scan options
95
+ read_options_args = {option: kwargs[option] for option in kwargs
96
+ if option in _PARQUET_READ_OPTIONS}
97
+ scan_args = {option: kwargs[option] for option in kwargs
98
+ if option not in _PARQUET_READ_OPTIONS}
99
+ if read_options and read_options_args:
100
+ duplicates = ', '.join(sorted(read_options_args))
101
+ raise ValueError(f'If `read_options` is given, '
102
+ f'cannot specify {duplicates}')
103
+ if default_fragment_scan_options and scan_args:
104
+ duplicates = ', '.join(sorted(scan_args))
105
+ raise ValueError(f'If `default_fragment_scan_options` is given, '
106
+ f'cannot specify {duplicates}')
107
+
108
+ if read_options is None:
109
+ read_options = ParquetReadOptions(**read_options_args)
110
+ elif isinstance(read_options, dict):
111
+ # For backwards compatibility
112
+ duplicates = []
113
+ for option, value in read_options.items():
114
+ if option in _PARQUET_READ_OPTIONS:
115
+ read_options_args[option] = value
116
+ else:
117
+ duplicates.append(option)
118
+ scan_args[option] = value
119
+ if duplicates:
120
+ duplicates = ", ".join(duplicates)
121
+ warnings.warn(f'The scan options {duplicates} should be '
122
+ 'specified directly as keyword arguments')
123
+ read_options = ParquetReadOptions(**read_options_args)
124
+ elif not isinstance(read_options, ParquetReadOptions):
125
+ raise TypeError('`read_options` must be either a dictionary or an '
126
+ 'instance of ParquetReadOptions')
127
+
128
+ if default_fragment_scan_options is None:
129
+ default_fragment_scan_options = ParquetFragmentScanOptions(**scan_args)
130
+ elif isinstance(default_fragment_scan_options, dict):
131
+ default_fragment_scan_options = ParquetFragmentScanOptions(
132
+ **default_fragment_scan_options)
133
+ elif not isinstance(default_fragment_scan_options,
134
+ ParquetFragmentScanOptions):
135
+ raise TypeError('`default_fragment_scan_options` must be either a '
136
+ 'dictionary or an instance of '
137
+ 'ParquetFragmentScanOptions')
138
+
139
+ wrapped = make_shared[CParquetFileFormat]()
140
+
141
+ options = &(wrapped.get().reader_options)
142
+ if read_options.dictionary_columns is not None:
143
+ for column in read_options.dictionary_columns:
144
+ options.dict_columns.insert(tobytes(column))
145
+ options.coerce_int96_timestamp_unit = \
146
+ read_options._coerce_int96_timestamp_unit
147
+ options.binary_type = read_options._binary_type
148
+ options.list_type = read_options._list_type
149
+
150
+ self.init(<shared_ptr[CFileFormat]> wrapped)
151
+ self.default_fragment_scan_options = default_fragment_scan_options
152
+
153
+ cdef void init(self, const shared_ptr[CFileFormat]& sp):
154
+ FileFormat.init(self, sp)
155
+ self.parquet_format = <CParquetFileFormat*> sp.get()
156
+
157
+ cdef WrittenFile _finish_write(self, path, base_dir,
158
+ CFileWriter* file_writer):
159
+ cdef:
160
+ FileMetaData parquet_metadata
161
+ CParquetFileWriter* parquet_file_writer
162
+
163
+ parquet_metadata = None
164
+ parquet_file_writer = dynamic_cast[_CParquetFileWriterPtr](file_writer)
165
+ with nogil:
166
+ metadata = deref(
167
+ deref(parquet_file_writer).parquet_writer()).metadata()
168
+ if metadata:
169
+ parquet_metadata = FileMetaData.__new__(FileMetaData)
170
+ parquet_metadata.init(metadata)
171
+ parquet_metadata.set_file_path(os.path.relpath(path, base_dir))
172
+
173
+ size = GetResultValue(file_writer.GetBytesWritten())
174
+
175
+ return WrittenFile(path, parquet_metadata, size)
176
+
177
+ @property
178
+ def read_options(self):
179
+ cdef CParquetFileFormatReaderOptions* options
180
+ options = &self.parquet_format.reader_options
181
+ parquet_read_options = ParquetReadOptions(
182
+ dictionary_columns={frombytes(col)
183
+ for col in options.dict_columns},
184
+ )
185
+ # Read options getter/setter works with strings so setting
186
+ # the private property which uses the C Type
187
+ parquet_read_options._coerce_int96_timestamp_unit = \
188
+ options.coerce_int96_timestamp_unit
189
+ parquet_read_options._binary_type = options.binary_type
190
+ parquet_read_options._list_type = options.list_type
191
+ return parquet_read_options
192
+
193
+ def make_write_options(self, **kwargs):
194
+ """
195
+ Parameters
196
+ ----------
197
+ **kwargs : dict
198
+
199
+ Returns
200
+ -------
201
+ pyarrow.dataset.FileWriteOptions
202
+ """
203
+ # Safeguard from calling make_write_options as a static class method
204
+ if not isinstance(self, ParquetFileFormat):
205
+ raise TypeError("make_write_options() should be called on "
206
+ "an instance of ParquetFileFormat")
207
+ opts = FileFormat.make_write_options(self)
208
+ (<ParquetFileWriteOptions> opts).update(**kwargs)
209
+ return opts
210
+
211
+ cdef _set_default_fragment_scan_options(self, FragmentScanOptions options):
212
+ if options.type_name == 'parquet':
213
+ self.parquet_format.default_fragment_scan_options = options.wrapped
214
+ else:
215
+ super()._set_default_fragment_scan_options(options)
216
+
217
+ def equals(self, ParquetFileFormat other):
218
+ """
219
+ Parameters
220
+ ----------
221
+ other : pyarrow.dataset.ParquetFileFormat
222
+
223
+ Returns
224
+ -------
225
+ bool
226
+ """
227
+ return (
228
+ self.read_options.equals(other.read_options) and
229
+ self.default_fragment_scan_options ==
230
+ other.default_fragment_scan_options
231
+ )
232
+
233
+ @property
234
+ def default_extname(self):
235
+ return "parquet"
236
+
237
+ def __reduce__(self):
238
+ return ParquetFileFormat, (self.read_options,
239
+ self.default_fragment_scan_options)
240
+
241
+ def __repr__(self):
242
+ return f"<ParquetFileFormat read_options={self.read_options}>"
243
+
244
+ def make_fragment(self, file, filesystem=None,
245
+ Expression partition_expression=None, row_groups=None, *, file_size=None):
246
+ """
247
+ Make a FileFragment from a given file.
248
+
249
+ Parameters
250
+ ----------
251
+ file : file-like object, path-like or str
252
+ The file or file path to make a fragment from.
253
+ filesystem : Filesystem, optional
254
+ If `filesystem` is given, `file` must be a string and specifies
255
+ the path of the file to read from the filesystem.
256
+ partition_expression : Expression, optional
257
+ An expression that is guaranteed true for all rows in the fragment. Allows
258
+ fragment to be potentially skipped while scanning with a filter.
259
+ row_groups : Iterable, optional
260
+ The indices of the row groups to include
261
+ file_size : int, optional
262
+ The size of the file in bytes. Can improve performance with high-latency filesystems
263
+ when file size needs to be known before reading.
264
+
265
+ Returns
266
+ -------
267
+ fragment : Fragment
268
+ The file fragment
269
+ """
270
+ cdef:
271
+ vector[int] c_row_groups
272
+ if partition_expression is None:
273
+ partition_expression = _true
274
+ if row_groups is None:
275
+ return super().make_fragment(file, filesystem,
276
+ partition_expression, file_size=file_size)
277
+
278
+ c_source = _make_file_source(file, filesystem, file_size)
279
+ c_row_groups = [<int> row_group for row_group in set(row_groups)]
280
+
281
+ c_fragment = <shared_ptr[CFragment]> GetResultValue(
282
+ self.parquet_format.MakeFragment(move(c_source),
283
+ partition_expression.unwrap(),
284
+ <shared_ptr[CSchema]>nullptr,
285
+ move(c_row_groups)))
286
+ return Fragment.wrap(move(c_fragment))
287
+
288
+
289
+ class RowGroupInfo:
290
+ """
291
+ A wrapper class for RowGroup information
292
+
293
+ Parameters
294
+ ----------
295
+ id : integer
296
+ The group ID.
297
+ metadata : FileMetaData
298
+ The rowgroup metadata.
299
+ schema : Schema
300
+ Schema of the rows.
301
+ """
302
+
303
+ def __init__(self, id, metadata, schema):
304
+ self.id = id
305
+ self.metadata = metadata
306
+ self.schema = schema
307
+
308
+ @property
309
+ def num_rows(self):
310
+ return self.metadata.num_rows
311
+
312
+ @property
313
+ def total_byte_size(self):
314
+ return self.metadata.total_byte_size
315
+
316
+ @property
317
+ def statistics(self):
318
+ def name_stats(i):
319
+ col = self.metadata.column(i)
320
+
321
+ stats = col.statistics
322
+ if stats is None or not stats.has_min_max:
323
+ return None, None
324
+
325
+ name = col.path_in_schema
326
+ field_index = self.schema.get_field_index(name)
327
+ if field_index < 0:
328
+ return None, None
329
+
330
+ typ = self.schema.field(field_index).type
331
+ return col.path_in_schema, {
332
+ 'min': pa.scalar(stats.min, type=typ).as_py(),
333
+ 'max': pa.scalar(stats.max, type=typ).as_py()
334
+ }
335
+
336
+ return {
337
+ name: stats for name, stats
338
+ in map(name_stats, range(self.metadata.num_columns))
339
+ if stats is not None
340
+ }
341
+
342
+ def __repr__(self):
343
+ return f"RowGroupInfo({self.id})"
344
+
345
+ def __eq__(self, other):
346
+ if isinstance(other, int):
347
+ return self.id == other
348
+ if not isinstance(other, RowGroupInfo):
349
+ return False
350
+ return self.id == other.id
351
+
352
+
353
+ cdef class ParquetFileFragment(FileFragment):
354
+ """A Fragment representing a parquet file."""
355
+
356
+ cdef:
357
+ CParquetFileFragment* parquet_file_fragment
358
+
359
+ cdef void init(self, const shared_ptr[CFragment]& sp):
360
+ FileFragment.init(self, sp)
361
+ self.parquet_file_fragment = <CParquetFileFragment*> sp.get()
362
+
363
+ def __reduce__(self):
364
+ buffer = self.buffer
365
+ # parquet_file_fragment.row_groups() is empty if the metadata
366
+ # information of the file is not yet populated
367
+ if not bool(self.parquet_file_fragment.row_groups()):
368
+ row_groups = None
369
+ else:
370
+ row_groups = [row_group.id for row_group in self.row_groups]
371
+
372
+ return self.format.make_fragment, (
373
+ self.path if buffer is None else buffer,
374
+ self.filesystem,
375
+ self.partition_expression,
376
+ row_groups
377
+ )
378
+
379
+ def ensure_complete_metadata(self):
380
+ """
381
+ Ensure that all metadata (statistics, physical schema, ...) have
382
+ been read and cached in this fragment.
383
+ """
384
+ with nogil:
385
+ check_status(self.parquet_file_fragment.EnsureCompleteMetadata())
386
+
387
+ @property
388
+ def row_groups(self):
389
+ metadata = self.metadata
390
+ cdef vector[int] row_groups = self.parquet_file_fragment.row_groups()
391
+ return [RowGroupInfo(i, metadata.row_group(i), self.physical_schema)
392
+ for i in row_groups]
393
+
394
+ @property
395
+ def metadata(self):
396
+ self.ensure_complete_metadata()
397
+ cdef FileMetaData metadata = FileMetaData.__new__(FileMetaData)
398
+ metadata.init(self.parquet_file_fragment.metadata())
399
+ return metadata
400
+
401
+ @property
402
+ def num_row_groups(self):
403
+ """
404
+ Return the number of row groups viewed by this fragment (not the
405
+ number of row groups in the origin file).
406
+ """
407
+ self.ensure_complete_metadata()
408
+ return self.parquet_file_fragment.row_groups().size()
409
+
410
+ def split_by_row_group(self, Expression filter=None,
411
+ Schema schema=None):
412
+ """
413
+ Split the fragment into multiple fragments.
414
+
415
+ Yield a Fragment wrapping each row group in this ParquetFileFragment.
416
+ Row groups will be excluded whose metadata contradicts the optional
417
+ filter.
418
+
419
+ Parameters
420
+ ----------
421
+ filter : Expression, default None
422
+ Only include the row groups which satisfy this predicate (using
423
+ the Parquet RowGroup statistics).
424
+ schema : Schema, default None
425
+ Schema to use when filtering row groups. Defaults to the
426
+ Fragment's physical schema
427
+
428
+ Returns
429
+ -------
430
+ A list of Fragments
431
+ """
432
+ cdef:
433
+ vector[shared_ptr[CFragment]] c_fragments
434
+ CExpression c_filter
435
+ shared_ptr[CFragment] c_fragment
436
+
437
+ schema = schema or self.physical_schema
438
+ c_filter = _bind(filter, schema)
439
+ with nogil:
440
+ c_fragments = move(GetResultValue(
441
+ self.parquet_file_fragment.SplitByRowGroup(move(c_filter))))
442
+
443
+ return [Fragment.wrap(c_fragment) for c_fragment in c_fragments]
444
+
445
+ def subset(self, Expression filter=None, Schema schema=None,
446
+ object row_group_ids=None):
447
+ """
448
+ Create a subset of the fragment (viewing a subset of the row groups).
449
+
450
+ Subset can be specified by either a filter predicate (with optional
451
+ schema) or by a list of row group IDs. Note that when using a filter,
452
+ the resulting fragment can be empty (viewing no row groups).
453
+
454
+ Parameters
455
+ ----------
456
+ filter : Expression, default None
457
+ Only include the row groups which satisfy this predicate (using
458
+ the Parquet RowGroup statistics).
459
+ schema : Schema, default None
460
+ Schema to use when filtering row groups. Defaults to the
461
+ Fragment's physical schema
462
+ row_group_ids : list of ints
463
+ The row group IDs to include in the subset. Can only be specified
464
+ if `filter` is None.
465
+
466
+ Returns
467
+ -------
468
+ ParquetFileFragment
469
+ """
470
+ cdef:
471
+ CExpression c_filter
472
+ vector[int] c_row_group_ids
473
+ shared_ptr[CFragment] c_fragment
474
+
475
+ if filter is not None and row_group_ids is not None:
476
+ raise ValueError(
477
+ "Cannot specify both 'filter' and 'row_group_ids'."
478
+ )
479
+
480
+ if filter is not None:
481
+ schema = schema or self.physical_schema
482
+ c_filter = _bind(filter, schema)
483
+ with nogil:
484
+ c_fragment = move(GetResultValue(
485
+ self.parquet_file_fragment.SubsetWithFilter(
486
+ move(c_filter))))
487
+ elif row_group_ids is not None:
488
+ c_row_group_ids = [
489
+ <int> row_group for row_group in sorted(set(row_group_ids))
490
+ ]
491
+ with nogil:
492
+ c_fragment = move(GetResultValue(
493
+ self.parquet_file_fragment.SubsetWithIds(
494
+ move(c_row_group_ids))))
495
+ else:
496
+ raise ValueError(
497
+ "Need to specify one of 'filter' or 'row_group_ids'"
498
+ )
499
+
500
+ return Fragment.wrap(c_fragment)
501
+
502
+
503
+ cdef class ParquetReadOptions(_Weakrefable):
504
+ """
505
+ Parquet format specific options for reading.
506
+
507
+ Parameters
508
+ ----------
509
+ dictionary_columns : list of string, default None
510
+ Names of columns which should be dictionary encoded as
511
+ they are read
512
+ coerce_int96_timestamp_unit : str, default None
513
+ Cast timestamps that are stored in INT96 format to a particular
514
+ resolution (e.g. 'ms'). Setting to None is equivalent to 'ns'
515
+ and therefore INT96 timestamps will be inferred as timestamps
516
+ in nanoseconds
517
+ binary_type : pyarrow.DataType, default None
518
+ If given, Parquet binary columns will be read as this datatype.
519
+ This setting is ignored if a serialized Arrow schema is found in
520
+ the Parquet metadata.
521
+ list_type : subclass of pyarrow.DataType, default None
522
+ If given, non-MAP repeated columns will be read as an instance of
523
+ this datatype (either pyarrow.ListType or pyarrow.LargeListType).
524
+ This setting is ignored if a serialized Arrow schema is found in
525
+ the Parquet metadata.
526
+ """
527
+
528
+ cdef public:
529
+ set dictionary_columns
530
+ TimeUnit _coerce_int96_timestamp_unit
531
+ Type _binary_type
532
+ Type _list_type
533
+
534
+ # Also see _PARQUET_READ_OPTIONS
535
+ def __init__(self, dictionary_columns=None,
536
+ coerce_int96_timestamp_unit=None,
537
+ binary_type=None, list_type=None):
538
+ self.dictionary_columns = set(dictionary_columns or set())
539
+ self.coerce_int96_timestamp_unit = coerce_int96_timestamp_unit
540
+ self.binary_type = binary_type
541
+ self.list_type = list_type
542
+
543
+ @property
544
+ def binary_type(self):
545
+ return primitive_type(self._binary_type)
546
+
547
+ @binary_type.setter
548
+ def binary_type(self, ty):
549
+ if ty is not None:
550
+ self._binary_type = pyarrow_unwrap_data_type(ty).get().id()
551
+ else:
552
+ self._binary_type = _Type_BINARY
553
+
554
+ @property
555
+ def list_type(self):
556
+ return (pa.LargeListType if self._list_type == _Type_LARGE_LIST
557
+ else pa.ListType)
558
+
559
+ @list_type.setter
560
+ def list_type(self, ty):
561
+ if ty is not None:
562
+ self._list_type = _unwrap_list_type(ty)
563
+ else:
564
+ self._list_type = _Type_LIST
565
+
566
+ @property
567
+ def coerce_int96_timestamp_unit(self):
568
+ return timeunit_to_string(self._coerce_int96_timestamp_unit)
569
+
570
+ @coerce_int96_timestamp_unit.setter
571
+ def coerce_int96_timestamp_unit(self, unit):
572
+ if unit is not None:
573
+ self._coerce_int96_timestamp_unit = string_to_timeunit(unit)
574
+ else:
575
+ self._coerce_int96_timestamp_unit = TimeUnit_NANO
576
+
577
+ def equals(self, ParquetReadOptions other):
578
+ """
579
+ Parameters
580
+ ----------
581
+ other : pyarrow.dataset.ParquetReadOptions
582
+
583
+ Returns
584
+ -------
585
+ bool
586
+ """
587
+ return (self.dictionary_columns == other.dictionary_columns and
588
+ self._coerce_int96_timestamp_unit ==
589
+ other._coerce_int96_timestamp_unit and
590
+ self._binary_type == other._binary_type and
591
+ self._list_type == other._list_type)
592
+
593
+ def __eq__(self, other):
594
+ try:
595
+ return self.equals(other)
596
+ except TypeError:
597
+ return False
598
+
599
+ def __repr__(self):
600
+ return (
601
+ f"<ParquetReadOptions"
602
+ f" dictionary_columns={self.dictionary_columns}"
603
+ f" coerce_int96_timestamp_unit={self.coerce_int96_timestamp_unit}"
604
+ f" binary_type={self.binary_type}"
605
+ f" list_type={self.list_type}"
606
+ f">"
607
+ )
608
+
609
+
610
+ cdef class ParquetFileWriteOptions(FileWriteOptions):
611
+
612
+ def update(self, **kwargs):
613
+ """
614
+ Parameters
615
+ ----------
616
+ **kwargs : dict
617
+ """
618
+ arrow_fields = {
619
+ "use_deprecated_int96_timestamps",
620
+ "coerce_timestamps",
621
+ "allow_truncated_timestamps",
622
+ "use_compliant_nested_type",
623
+ }
624
+
625
+ setters = set()
626
+ for name, value in kwargs.items():
627
+ if name not in self._properties:
628
+ raise TypeError("unexpected parquet write option: " + name)
629
+ self._properties[name] = value
630
+ if name in arrow_fields:
631
+ setters.add(self._set_arrow_properties)
632
+ elif name == "encryption_config" and value is not None:
633
+ setters.add(self._set_encryption_config)
634
+ else:
635
+ setters.add(self._set_properties)
636
+
637
+ for setter in setters:
638
+ setter()
639
+
640
+ def _set_properties(self):
641
+ cdef CParquetFileWriteOptions* opts = self.parquet_options
642
+
643
+ opts.writer_properties = _create_writer_properties(
644
+ use_dictionary=self._properties["use_dictionary"],
645
+ compression=self._properties["compression"],
646
+ version=self._properties["version"],
647
+ write_statistics=self._properties["write_statistics"],
648
+ data_page_size=self._properties["data_page_size"],
649
+ max_rows_per_page=self._properties["max_rows_per_page"],
650
+ compression_level=self._properties["compression_level"],
651
+ use_byte_stream_split=(
652
+ self._properties["use_byte_stream_split"]
653
+ ),
654
+ column_encoding=self._properties["column_encoding"],
655
+ data_page_version=self._properties["data_page_version"],
656
+ encryption_properties=self._properties["encryption_properties"],
657
+ write_batch_size=self._properties["write_batch_size"],
658
+ dictionary_pagesize_limit=self._properties["dictionary_pagesize_limit"],
659
+ write_page_index=self._properties["write_page_index"],
660
+ write_page_checksum=self._properties["write_page_checksum"],
661
+ sorting_columns=self._properties["sorting_columns"],
662
+ store_decimal_as_integer=self._properties["store_decimal_as_integer"],
663
+ )
664
+
665
+ def _set_arrow_properties(self):
666
+ cdef CParquetFileWriteOptions* opts = self.parquet_options
667
+
668
+ opts.arrow_writer_properties = _create_arrow_writer_properties(
669
+ use_deprecated_int96_timestamps=(
670
+ self._properties["use_deprecated_int96_timestamps"]
671
+ ),
672
+ coerce_timestamps=self._properties["coerce_timestamps"],
673
+ allow_truncated_timestamps=(
674
+ self._properties["allow_truncated_timestamps"]
675
+ ),
676
+ writer_engine_version="V2",
677
+ use_compliant_nested_type=(
678
+ self._properties["use_compliant_nested_type"]
679
+ )
680
+ )
681
+
682
+ def _set_encryption_config(self):
683
+ if not parquet_encryption_enabled:
684
+ raise NotImplementedError(
685
+ "Encryption is not enabled in your installation of pyarrow, but an "
686
+ "encryption_config was provided."
687
+ )
688
+ set_encryption_config(self, self._properties["encryption_config"])
689
+
690
+ cdef void init(self, const shared_ptr[CFileWriteOptions]& sp):
691
+ FileWriteOptions.init(self, sp)
692
+ self.parquet_options = <CParquetFileWriteOptions*> sp.get()
693
+ self._properties = dict(
694
+ use_dictionary=True,
695
+ compression="snappy",
696
+ version="2.6",
697
+ write_statistics=None,
698
+ data_page_size=None,
699
+ max_rows_per_page=None,
700
+ compression_level=None,
701
+ use_byte_stream_split=False,
702
+ column_encoding=None,
703
+ data_page_version="1.0",
704
+ use_deprecated_int96_timestamps=False,
705
+ coerce_timestamps=None,
706
+ allow_truncated_timestamps=False,
707
+ use_compliant_nested_type=True,
708
+ encryption_properties=None,
709
+ write_batch_size=None,
710
+ dictionary_pagesize_limit=None,
711
+ write_page_index=False,
712
+ encryption_config=None,
713
+ write_page_checksum=False,
714
+ sorting_columns=None,
715
+ store_decimal_as_integer=False,
716
+ )
717
+
718
+ self._set_properties()
719
+ self._set_arrow_properties()
720
+
721
+ def __repr__(self):
722
+ return f"<pyarrow.dataset.ParquetFileWriteOptions {self._properties}>"
723
+
724
+
725
+ cdef set _PARQUET_READ_OPTIONS = {
726
+ 'dictionary_columns', 'coerce_int96_timestamp_unit', 'binary_type', 'list_type',
727
+ }
728
+
729
+
730
+ cdef class ParquetFragmentScanOptions(FragmentScanOptions):
731
+ """
732
+ Scan-specific options for Parquet fragments.
733
+
734
+ Parameters
735
+ ----------
736
+ use_buffered_stream : bool, default False
737
+ Read files through buffered input streams rather than loading entire
738
+ row groups at once. This may be enabled to reduce memory overhead.
739
+ Disabled by default.
740
+ buffer_size : int, default 8192
741
+ Size of buffered stream, if enabled. Default is 8KB.
742
+ pre_buffer : bool, default True
743
+ If enabled, pre-buffer the raw Parquet data instead of issuing one
744
+ read per column chunk. This can improve performance on high-latency
745
+ filesystems (e.g. S3, GCS) by coalescing and issuing file reads in
746
+ parallel using a background I/O thread pool.
747
+ Set to False if you want to prioritize minimal memory usage
748
+ over maximum speed.
749
+ cache_options : pyarrow.CacheOptions, default None
750
+ Cache options used when pre_buffer is enabled. The default values should
751
+ be good for most use cases. You may want to adjust these for example if
752
+ you have exceptionally high latency to the file system.
753
+ thrift_string_size_limit : int, default None
754
+ If not None, override the maximum total string size allocated
755
+ when decoding Thrift structures. The default limit should be
756
+ sufficient for most Parquet files.
757
+ thrift_container_size_limit : int, default None
758
+ If not None, override the maximum total size of containers allocated
759
+ when decoding Thrift structures. The default limit should be
760
+ sufficient for most Parquet files.
761
+ decryption_config : pyarrow.dataset.ParquetDecryptionConfig, default None
762
+ If not None, use the provided ParquetDecryptionConfig to decrypt the
763
+ Parquet file.
764
+ decryption_properties : pyarrow.parquet.FileDecryptionProperties, default None
765
+ If not None, use the provided FileDecryptionProperties to decrypt encrypted
766
+ Parquet file.
767
+ page_checksum_verification : bool, default False
768
+ If True, verify the page checksum for each page read from the file.
769
+ arrow_extensions_enabled : bool, default False
770
+ If True, read Parquet logical types as Arrow extension types where possible,
771
+ (e.g., read JSON as the canonical `arrow.json` extension type or UUID as
772
+ the canonical `arrow.uuid` extension type).
773
+ """
774
+
775
+ # Avoid mistakingly creating attributes
776
+ __slots__ = ()
777
+
778
+ def __init__(self, *, bint use_buffered_stream=False,
779
+ buffer_size=8192,
780
+ bint pre_buffer=True,
781
+ cache_options=None,
782
+ thrift_string_size_limit=None,
783
+ thrift_container_size_limit=None,
784
+ decryption_config=None,
785
+ decryption_properties=None,
786
+ bint page_checksum_verification=False,
787
+ bint arrow_extensions_enabled=False):
788
+ self.init(shared_ptr[CFragmentScanOptions](
789
+ new CParquetFragmentScanOptions()))
790
+ self.use_buffered_stream = use_buffered_stream
791
+ self.buffer_size = buffer_size
792
+ if pre_buffer and not is_threading_enabled():
793
+ pre_buffer = False
794
+ self.pre_buffer = pre_buffer
795
+ if cache_options is not None:
796
+ self.cache_options = cache_options
797
+ if thrift_string_size_limit is not None:
798
+ self.thrift_string_size_limit = thrift_string_size_limit
799
+ if thrift_container_size_limit is not None:
800
+ self.thrift_container_size_limit = thrift_container_size_limit
801
+ if decryption_config is not None:
802
+ self.parquet_decryption_config = decryption_config
803
+ if decryption_properties is not None:
804
+ self.decryption_properties = decryption_properties
805
+ self.page_checksum_verification = page_checksum_verification
806
+ self.arrow_extensions_enabled = arrow_extensions_enabled
807
+
808
+ cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp):
809
+ FragmentScanOptions.init(self, sp)
810
+ self.parquet_options = <CParquetFragmentScanOptions*> sp.get()
811
+
812
+ cdef CReaderProperties* reader_properties(self):
813
+ return self.parquet_options.reader_properties.get()
814
+
815
+ cdef ArrowReaderProperties* arrow_reader_properties(self):
816
+ return self.parquet_options.arrow_reader_properties.get()
817
+
818
+ @property
819
+ def use_buffered_stream(self):
820
+ return self.reader_properties().is_buffered_stream_enabled()
821
+
822
+ @use_buffered_stream.setter
823
+ def use_buffered_stream(self, bint use_buffered_stream):
824
+ if use_buffered_stream:
825
+ self.reader_properties().enable_buffered_stream()
826
+ else:
827
+ self.reader_properties().disable_buffered_stream()
828
+
829
+ @property
830
+ def buffer_size(self):
831
+ return self.reader_properties().buffer_size()
832
+
833
+ @buffer_size.setter
834
+ def buffer_size(self, buffer_size):
835
+ if buffer_size <= 0:
836
+ raise ValueError("Buffer size must be larger than zero")
837
+ self.reader_properties().set_buffer_size(buffer_size)
838
+
839
+ @property
840
+ def pre_buffer(self):
841
+ return self.arrow_reader_properties().pre_buffer()
842
+
843
+ @pre_buffer.setter
844
+ def pre_buffer(self, bint pre_buffer):
845
+ if pre_buffer and not is_threading_enabled():
846
+ return
847
+ self.arrow_reader_properties().set_pre_buffer(pre_buffer)
848
+
849
+ @property
850
+ def cache_options(self):
851
+ return CacheOptions.wrap(self.arrow_reader_properties().cache_options())
852
+
853
+ @cache_options.setter
854
+ def cache_options(self, CacheOptions options):
855
+ self.arrow_reader_properties().set_cache_options(options.unwrap())
856
+
857
+ @property
858
+ def thrift_string_size_limit(self):
859
+ return self.reader_properties().thrift_string_size_limit()
860
+
861
+ @thrift_string_size_limit.setter
862
+ def thrift_string_size_limit(self, size):
863
+ if size <= 0:
864
+ raise ValueError("size must be larger than zero")
865
+ self.reader_properties().set_thrift_string_size_limit(size)
866
+
867
+ @property
868
+ def thrift_container_size_limit(self):
869
+ return self.reader_properties().thrift_container_size_limit()
870
+
871
+ @thrift_container_size_limit.setter
872
+ def thrift_container_size_limit(self, size):
873
+ if size <= 0:
874
+ raise ValueError("size must be larger than zero")
875
+ self.reader_properties().set_thrift_container_size_limit(size)
876
+
877
+ @property
878
+ def decryption_properties(self):
879
+ if not parquet_encryption_enabled:
880
+ raise NotImplementedError(
881
+ "Unable to access encryption features. "
882
+ "Encryption is not enabled in your installation of pyarrow."
883
+ )
884
+ return self._decryption_properties
885
+
886
+ @decryption_properties.setter
887
+ def decryption_properties(self, config):
888
+ if not parquet_encryption_enabled:
889
+ raise NotImplementedError(
890
+ "Encryption is not enabled in your installation of pyarrow, but "
891
+ "decryption_properties were provided."
892
+ )
893
+ set_decryption_properties(self, config)
894
+ self._decryption_properties = config
895
+
896
+ @property
897
+ def parquet_decryption_config(self):
898
+ if not parquet_encryption_enabled:
899
+ raise NotImplementedError(
900
+ "Unable to access encryption features. "
901
+ "Encryption is not enabled in your installation of pyarrow."
902
+ )
903
+ return self._parquet_decryption_config
904
+
905
+ @parquet_decryption_config.setter
906
+ def parquet_decryption_config(self, config):
907
+ if not parquet_encryption_enabled:
908
+ raise NotImplementedError(
909
+ "Encryption is not enabled in your installation of pyarrow, but a "
910
+ "decryption_config was provided."
911
+ )
912
+ set_decryption_config(self, config)
913
+ self._parquet_decryption_config = config
914
+
915
+ @property
916
+ def page_checksum_verification(self):
917
+ return self.reader_properties().page_checksum_verification()
918
+
919
+ @page_checksum_verification.setter
920
+ def page_checksum_verification(self, bint page_checksum_verification):
921
+ self.reader_properties().set_page_checksum_verification(page_checksum_verification)
922
+
923
+ @property
924
+ def arrow_extensions_enabled(self):
925
+ return self.arrow_reader_properties().get_arrow_extensions_enabled()
926
+
927
+ @arrow_extensions_enabled.setter
928
+ def arrow_extensions_enabled(self, bint arrow_extensions_enabled):
929
+ self.arrow_reader_properties().set_arrow_extensions_enabled(arrow_extensions_enabled)
930
+
931
+ def equals(self, ParquetFragmentScanOptions other):
932
+ """
933
+ Parameters
934
+ ----------
935
+ other : pyarrow.dataset.ParquetFragmentScanOptions
936
+
937
+ Returns
938
+ -------
939
+ bool
940
+ """
941
+ attrs = (
942
+ self.use_buffered_stream, self.buffer_size, self.pre_buffer, self.cache_options,
943
+ self.thrift_string_size_limit, self.thrift_container_size_limit,
944
+ self.page_checksum_verification, self.arrow_extensions_enabled)
945
+ other_attrs = (
946
+ other.use_buffered_stream, other.buffer_size, other.pre_buffer, other.cache_options,
947
+ other.thrift_string_size_limit,
948
+ other.thrift_container_size_limit, other.page_checksum_verification,
949
+ other.arrow_extensions_enabled)
950
+ return attrs == other_attrs
951
+
952
+ @staticmethod
953
+ def _reconstruct(kwargs):
954
+ # __reduce__ doesn't allow passing named arguments directly to the
955
+ # reconstructor, hence this wrapper.
956
+ return ParquetFragmentScanOptions(**kwargs)
957
+
958
+ def __reduce__(self):
959
+ kwargs = dict(
960
+ use_buffered_stream=self.use_buffered_stream,
961
+ buffer_size=self.buffer_size,
962
+ pre_buffer=self.pre_buffer,
963
+ cache_options=self.cache_options,
964
+ thrift_string_size_limit=self.thrift_string_size_limit,
965
+ thrift_container_size_limit=self.thrift_container_size_limit,
966
+ page_checksum_verification=self.page_checksum_verification,
967
+ arrow_extensions_enabled=self.arrow_extensions_enabled
968
+ )
969
+ return ParquetFragmentScanOptions._reconstruct, (kwargs,)
970
+
971
+
972
+ cdef class ParquetFactoryOptions(_Weakrefable):
973
+ """
974
+ Influences the discovery of parquet dataset.
975
+
976
+ Parameters
977
+ ----------
978
+ partition_base_dir : str, optional
979
+ For the purposes of applying the partitioning, paths will be
980
+ stripped of the partition_base_dir. Files not matching the
981
+ partition_base_dir prefix will be skipped for partitioning discovery.
982
+ The ignored files will still be part of the Dataset, but will not
983
+ have partition information.
984
+ partitioning : Partitioning, PartitioningFactory, optional
985
+ The partitioning scheme applied to fragments, see ``Partitioning``.
986
+ validate_column_chunk_paths : bool, default False
987
+ Assert that all ColumnChunk paths are consistent. The parquet spec
988
+ allows for ColumnChunk data to be stored in multiple files, but
989
+ ParquetDatasetFactory supports only a single file with all ColumnChunk
990
+ data. If this flag is set construction of a ParquetDatasetFactory will
991
+ raise an error if ColumnChunk data is not resident in a single file.
992
+ """
993
+
994
+ cdef:
995
+ CParquetFactoryOptions options
996
+
997
+ __slots__ = () # avoid mistakingly creating attributes
998
+
999
+ def __init__(self, partition_base_dir=None, partitioning=None,
1000
+ validate_column_chunk_paths=False):
1001
+ if isinstance(partitioning, PartitioningFactory):
1002
+ self.partitioning_factory = partitioning
1003
+ elif isinstance(partitioning, Partitioning):
1004
+ self.partitioning = partitioning
1005
+
1006
+ if partition_base_dir is not None:
1007
+ self.partition_base_dir = partition_base_dir
1008
+
1009
+ self.options.validate_column_chunk_paths = validate_column_chunk_paths
1010
+
1011
+ cdef inline CParquetFactoryOptions unwrap(self):
1012
+ return self.options
1013
+
1014
+ @property
1015
+ def partitioning(self):
1016
+ """Partitioning to apply to discovered files.
1017
+
1018
+ NOTE: setting this property will overwrite partitioning_factory.
1019
+ """
1020
+ c_partitioning = self.options.partitioning.partitioning()
1021
+ if c_partitioning.get() == nullptr:
1022
+ return None
1023
+ return Partitioning.wrap(c_partitioning)
1024
+
1025
+ @partitioning.setter
1026
+ def partitioning(self, Partitioning value):
1027
+ self.options.partitioning = (<Partitioning> value).unwrap()
1028
+
1029
+ @property
1030
+ def partitioning_factory(self):
1031
+ """PartitioningFactory to apply to discovered files and
1032
+ discover a Partitioning.
1033
+
1034
+ NOTE: setting this property will overwrite partitioning.
1035
+ """
1036
+ c_factory = self.options.partitioning.factory()
1037
+ if c_factory.get() == nullptr:
1038
+ return None
1039
+ return PartitioningFactory.wrap(c_factory, None, None)
1040
+
1041
+ @partitioning_factory.setter
1042
+ def partitioning_factory(self, PartitioningFactory value):
1043
+ self.options.partitioning = (<PartitioningFactory> value).unwrap()
1044
+
1045
+ @property
1046
+ def partition_base_dir(self):
1047
+ """
1048
+ Base directory to strip paths before applying the partitioning.
1049
+ """
1050
+ return frombytes(self.options.partition_base_dir)
1051
+
1052
+ @partition_base_dir.setter
1053
+ def partition_base_dir(self, value):
1054
+ self.options.partition_base_dir = tobytes(value)
1055
+
1056
+ @property
1057
+ def validate_column_chunk_paths(self):
1058
+ """
1059
+ Base directory to strip paths before applying the partitioning.
1060
+ """
1061
+ return self.options.validate_column_chunk_paths
1062
+
1063
+ @validate_column_chunk_paths.setter
1064
+ def validate_column_chunk_paths(self, value):
1065
+ self.options.validate_column_chunk_paths = value
1066
+
1067
+
1068
+ cdef class ParquetDatasetFactory(DatasetFactory):
1069
+ """
1070
+ Create a ParquetDatasetFactory from a Parquet `_metadata` file.
1071
+
1072
+ Parameters
1073
+ ----------
1074
+ metadata_path : str
1075
+ Path to the `_metadata` parquet metadata-only file generated with
1076
+ `pyarrow.parquet.write_metadata`.
1077
+ filesystem : pyarrow.fs.FileSystem
1078
+ Filesystem to read the metadata_path from, and subsequent parquet
1079
+ files.
1080
+ format : ParquetFileFormat
1081
+ Parquet format options.
1082
+ options : ParquetFactoryOptions, optional
1083
+ Various flags influencing the discovery of filesystem paths.
1084
+ """
1085
+
1086
+ cdef:
1087
+ CParquetDatasetFactory* parquet_factory
1088
+
1089
+ def __init__(self, metadata_path, FileSystem filesystem not None,
1090
+ FileFormat format not None,
1091
+ ParquetFactoryOptions options=None):
1092
+ cdef:
1093
+ c_string c_path
1094
+ shared_ptr[CFileSystem] c_filesystem
1095
+ shared_ptr[CParquetFileFormat] c_format
1096
+ CResult[shared_ptr[CDatasetFactory]] result
1097
+ CParquetFactoryOptions c_options
1098
+
1099
+ c_path = tobytes(metadata_path)
1100
+ c_filesystem = filesystem.unwrap()
1101
+ c_format = static_pointer_cast[CParquetFileFormat, CFileFormat](
1102
+ format.unwrap())
1103
+ options = options or ParquetFactoryOptions()
1104
+ c_options = options.unwrap()
1105
+
1106
+ with nogil:
1107
+ result = CParquetDatasetFactory.MakeFromMetaDataPath(
1108
+ c_path, c_filesystem, c_format, c_options)
1109
+ self.init(GetResultValue(result))
1110
+
1111
+ cdef init(self, shared_ptr[CDatasetFactory]& sp):
1112
+ DatasetFactory.init(self, sp)
1113
+ self.parquet_factory = <CParquetDatasetFactory*> sp.get()
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.cp312-win_amd64.pyd ADDED
Binary file (92.7 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.pyx ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ """Dataset support for Parquet encryption."""
21
+
22
+ from pyarrow.includes.libarrow_dataset_parquet cimport *
23
+ from pyarrow._parquet_encryption cimport *
24
+ from pyarrow._dataset_parquet cimport ParquetFragmentScanOptions, ParquetFileWriteOptions
25
+
26
+
27
+ cdef class ParquetEncryptionConfig(_Weakrefable):
28
+ """
29
+ Core configuration class encapsulating parameters for high-level encryption
30
+ within the Parquet framework.
31
+
32
+ The ParquetEncryptionConfig class serves as a bridge for passing encryption-related
33
+ parameters to the appropriate components within the Parquet library. It maintains references
34
+ to objects that define the encryption strategy, Key Management Service (KMS) configuration,
35
+ and specific encryption configurations for Parquet data.
36
+
37
+ Parameters
38
+ ----------
39
+ crypto_factory : pyarrow.parquet.encryption.CryptoFactory
40
+ Shared pointer to a `CryptoFactory` object. The `CryptoFactory` is responsible for
41
+ creating cryptographic components, such as encryptors and decryptors.
42
+ kms_connection_config : pyarrow.parquet.encryption.KmsConnectionConfig
43
+ Shared pointer to a `KmsConnectionConfig` object. This object holds the configuration
44
+ parameters necessary for connecting to a Key Management Service (KMS).
45
+ encryption_config : pyarrow.parquet.encryption.EncryptionConfiguration
46
+ Shared pointer to an `EncryptionConfiguration` object. This object defines specific
47
+ encryption settings for Parquet data, including the keys assigned to different columns.
48
+
49
+ Raises
50
+ ------
51
+ ValueError
52
+ Raised if `encryption_config` is None.
53
+ """
54
+ cdef:
55
+ shared_ptr[CParquetEncryptionConfig] c_config
56
+
57
+ # Avoid mistakenly creating attributes
58
+ __slots__ = ()
59
+
60
+ def __cinit__(self, CryptoFactory crypto_factory, KmsConnectionConfig kms_connection_config,
61
+ EncryptionConfiguration encryption_config):
62
+
63
+ cdef shared_ptr[CEncryptionConfiguration] c_encryption_config
64
+
65
+ if crypto_factory is None:
66
+ raise ValueError("crypto_factory cannot be None")
67
+
68
+ if kms_connection_config is None:
69
+ raise ValueError("kms_connection_config cannot be None")
70
+
71
+ if encryption_config is None:
72
+ raise ValueError("encryption_config cannot be None")
73
+
74
+ self.c_config.reset(new CParquetEncryptionConfig())
75
+
76
+ c_encryption_config = pyarrow_unwrap_encryptionconfig(
77
+ encryption_config)
78
+
79
+ self.c_config.get().crypto_factory = pyarrow_unwrap_cryptofactory(crypto_factory)
80
+ self.c_config.get().kms_connection_config = pyarrow_unwrap_kmsconnectionconfig(
81
+ kms_connection_config)
82
+ self.c_config.get().encryption_config = c_encryption_config
83
+
84
+ @staticmethod
85
+ cdef wrap(shared_ptr[CParquetEncryptionConfig] c_config):
86
+ cdef ParquetEncryptionConfig python_config = ParquetEncryptionConfig.__new__(ParquetEncryptionConfig)
87
+ python_config.c_config = c_config
88
+ return python_config
89
+
90
+ cdef shared_ptr[CParquetEncryptionConfig] unwrap(self):
91
+ return self.c_config
92
+
93
+
94
+ cdef class ParquetDecryptionConfig(_Weakrefable):
95
+ """
96
+ Core configuration class encapsulating parameters for high-level decryption
97
+ within the Parquet framework.
98
+
99
+ ParquetDecryptionConfig is designed to pass decryption-related parameters to
100
+ the appropriate decryption components within the Parquet library. It holds references to
101
+ objects that define the decryption strategy, Key Management Service (KMS) configuration,
102
+ and specific decryption configurations for reading encrypted Parquet data.
103
+
104
+ Parameters
105
+ ----------
106
+ crypto_factory : pyarrow.parquet.encryption.CryptoFactory
107
+ Shared pointer to a `CryptoFactory` object, pivotal in creating cryptographic
108
+ components for the decryption process.
109
+ kms_connection_config : pyarrow.parquet.encryption.KmsConnectionConfig
110
+ Shared pointer to a `KmsConnectionConfig` object, containing parameters necessary
111
+ for connecting to a Key Management Service (KMS) during decryption.
112
+ decryption_config : pyarrow.parquet.encryption.DecryptionConfiguration
113
+ Shared pointer to a `DecryptionConfiguration` object, specifying decryption settings
114
+ for reading encrypted Parquet data.
115
+
116
+ Raises
117
+ ------
118
+ ValueError
119
+ Raised if `decryption_config` is None.
120
+ """
121
+
122
+ cdef:
123
+ shared_ptr[CParquetDecryptionConfig] c_config
124
+
125
+ # Avoid mistakingly creating attributes
126
+ __slots__ = ()
127
+
128
+ def __cinit__(self, CryptoFactory crypto_factory, KmsConnectionConfig kms_connection_config,
129
+ DecryptionConfiguration decryption_config):
130
+
131
+ cdef shared_ptr[CDecryptionConfiguration] c_decryption_config
132
+
133
+ if decryption_config is None:
134
+ raise ValueError(
135
+ "decryption_config cannot be None")
136
+
137
+ self.c_config.reset(new CParquetDecryptionConfig())
138
+
139
+ c_decryption_config = pyarrow_unwrap_decryptionconfig(
140
+ decryption_config)
141
+
142
+ self.c_config.get().crypto_factory = pyarrow_unwrap_cryptofactory(crypto_factory)
143
+ self.c_config.get().kms_connection_config = pyarrow_unwrap_kmsconnectionconfig(
144
+ kms_connection_config)
145
+ self.c_config.get().decryption_config = c_decryption_config
146
+
147
+ @staticmethod
148
+ cdef wrap(shared_ptr[CParquetDecryptionConfig] c_config):
149
+ cdef ParquetDecryptionConfig python_config = ParquetDecryptionConfig.__new__(ParquetDecryptionConfig)
150
+ python_config.c_config = c_config
151
+ return python_config
152
+
153
+ cdef shared_ptr[CParquetDecryptionConfig] unwrap(self):
154
+ return self.c_config
155
+
156
+
157
+ def set_encryption_config(
158
+ ParquetFileWriteOptions opts not None,
159
+ ParquetEncryptionConfig config not None
160
+ ):
161
+ cdef shared_ptr[CParquetEncryptionConfig] c_config = config.unwrap()
162
+ opts.parquet_options.parquet_encryption_config = c_config
163
+
164
+
165
+ def set_decryption_properties(
166
+ ParquetFragmentScanOptions opts not None,
167
+ FileDecryptionProperties config not None
168
+ ):
169
+ cdef CReaderProperties* reader_props = opts.reader_properties()
170
+ reader_props.file_decryption_properties(config.unwrap())
171
+
172
+
173
+ def set_decryption_config(
174
+ ParquetFragmentScanOptions opts not None,
175
+ ParquetDecryptionConfig config not None
176
+ ):
177
+ cdef shared_ptr[CParquetDecryptionConfig] c_config = config.unwrap()
178
+ opts.parquet_options.parquet_decryption_config = c_config
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dlpack.pxi ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ cimport cpython
19
+ from cpython.pycapsule cimport PyCapsule_New
20
+
21
+
22
+ cdef void dlpack_pycapsule_deleter(object dltensor) noexcept:
23
+ cdef DLManagedTensor* dlm_tensor
24
+ cdef PyObject* err_type
25
+ cdef PyObject* err_value
26
+ cdef PyObject* err_traceback
27
+
28
+ # Do nothing if the capsule has been consumed
29
+ if cpython.PyCapsule_IsValid(dltensor, "used_dltensor"):
30
+ return
31
+
32
+ # An exception may be in-flight, we must save it in case
33
+ # we create another one
34
+ cpython.PyErr_Fetch(&err_type, &err_value, &err_traceback)
35
+
36
+ dlm_tensor = <DLManagedTensor*>cpython.PyCapsule_GetPointer(dltensor, 'dltensor')
37
+ if dlm_tensor == NULL:
38
+ cpython.PyErr_WriteUnraisable(dltensor)
39
+ # The deleter can be NULL if there is no way for the caller
40
+ # to provide a reasonable destructor
41
+ elif dlm_tensor.deleter:
42
+ dlm_tensor.deleter(dlm_tensor)
43
+ assert (not cpython.PyErr_Occurred())
44
+
45
+ # Set the error indicator from err_type, err_value, err_traceback
46
+ cpython.PyErr_Restore(err_type, err_value, err_traceback)
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.cp312-win_amd64.pyd ADDED
Binary file (76.8 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.pyx ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # ---------------------------------------------------------------------
19
+ # Implement Feather file format
20
+
21
+ # cython: profile=False
22
+ # distutils: language = c++
23
+ # cython: language_level=3
24
+
25
+ from cython.operator cimport dereference as deref
26
+ from pyarrow.includes.common cimport *
27
+ from pyarrow.includes.libarrow cimport *
28
+ from pyarrow.includes.libarrow_feather cimport *
29
+ from pyarrow.lib cimport (check_status, Table, _Weakrefable,
30
+ get_writer, get_reader, pyarrow_wrap_table)
31
+ from pyarrow.lib import tobytes
32
+
33
+
34
+ class FeatherError(Exception):
35
+ pass
36
+
37
+
38
+ def write_feather(Table table, object dest, compression=None,
39
+ compression_level=None, chunksize=None, version=2):
40
+ cdef shared_ptr[COutputStream] sink
41
+ get_writer(dest, &sink)
42
+
43
+ cdef CFeatherProperties properties
44
+ if version == 2:
45
+ properties.version = kFeatherV2Version
46
+ else:
47
+ properties.version = kFeatherV1Version
48
+
49
+ if compression == 'zstd':
50
+ properties.compression = CCompressionType_ZSTD
51
+ elif compression == 'lz4':
52
+ properties.compression = CCompressionType_LZ4_FRAME
53
+ else:
54
+ properties.compression = CCompressionType_UNCOMPRESSED
55
+
56
+ if chunksize is not None:
57
+ properties.chunksize = chunksize
58
+
59
+ if compression_level is not None:
60
+ properties.compression_level = compression_level
61
+
62
+ with nogil:
63
+ check_status(WriteFeather(deref(table.table), sink.get(),
64
+ properties))
65
+
66
+
67
+ cdef class FeatherReader(_Weakrefable):
68
+ cdef:
69
+ shared_ptr[CFeatherReader] reader
70
+
71
+ def __cinit__(self, source, c_bool use_memory_map, c_bool use_threads):
72
+ cdef:
73
+ shared_ptr[CRandomAccessFile] reader
74
+ CIpcReadOptions options = CIpcReadOptions.Defaults()
75
+ options.use_threads = use_threads
76
+
77
+ get_reader(source, use_memory_map, &reader)
78
+ with nogil:
79
+ self.reader = GetResultValue(CFeatherReader.Open(reader, options))
80
+
81
+ @property
82
+ def version(self):
83
+ return self.reader.get().version()
84
+
85
+ def read(self):
86
+ cdef shared_ptr[CTable] sp_table
87
+ with nogil:
88
+ check_status(self.reader.get()
89
+ .Read(&sp_table))
90
+
91
+ return pyarrow_wrap_table(sp_table)
92
+
93
+ def read_indices(self, indices):
94
+ cdef:
95
+ shared_ptr[CTable] sp_table
96
+ vector[int] c_indices
97
+
98
+ for index in indices:
99
+ c_indices.push_back(index)
100
+ with nogil:
101
+ check_status(self.reader.get()
102
+ .Read(c_indices, &sp_table))
103
+
104
+ return pyarrow_wrap_table(sp_table)
105
+
106
+ def read_names(self, names):
107
+ cdef:
108
+ shared_ptr[CTable] sp_table
109
+ vector[c_string] c_names
110
+
111
+ for name in names:
112
+ c_names.push_back(tobytes(name))
113
+ with nogil:
114
+ check_status(self.reader.get()
115
+ .Read(c_names, &sp_table))
116
+
117
+ return pyarrow_wrap_table(sp_table)
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c6a830239a0932aeb1f25e902d4883d25c5fbda493af209537a692ba9b7952e
3
+ size 852480
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.pyx ADDED
The diff for this file is too large to render. See raw diff
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a73589e40c08c676cab65fbf245bd25fd6b4f68d09518395dc492b6f6461d358
3
+ size 303616
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pxd ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.includes.common cimport *
21
+ from pyarrow.includes.libarrow_fs cimport *
22
+ from pyarrow.lib import _detect_compression, frombytes, tobytes
23
+ from pyarrow.lib cimport *
24
+
25
+
26
+ cpdef enum FileType:
27
+ NotFound = <int8_t> CFileType_NotFound
28
+ Unknown = <int8_t> CFileType_Unknown
29
+ File = <int8_t> CFileType_File
30
+ Directory = <int8_t> CFileType_Directory
31
+
32
+
33
+ cdef class FileInfo(_Weakrefable):
34
+ cdef:
35
+ CFileInfo info
36
+
37
+ @staticmethod
38
+ cdef wrap(CFileInfo info)
39
+
40
+ cdef inline CFileInfo unwrap(self) nogil
41
+
42
+ @staticmethod
43
+ cdef CFileInfo unwrap_safe(obj)
44
+
45
+
46
+ cdef class FileSelector(_Weakrefable):
47
+ cdef:
48
+ CFileSelector selector
49
+
50
+ @staticmethod
51
+ cdef FileSelector wrap(CFileSelector selector)
52
+
53
+ cdef inline CFileSelector unwrap(self) nogil
54
+
55
+
56
+ cdef class FileSystem(_Weakrefable):
57
+ cdef:
58
+ shared_ptr[CFileSystem] wrapped
59
+ CFileSystem* fs
60
+
61
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped)
62
+
63
+ @staticmethod
64
+ cdef wrap(const shared_ptr[CFileSystem]& sp)
65
+
66
+ cdef inline shared_ptr[CFileSystem] unwrap(self) nogil
67
+
68
+
69
+ cdef class LocalFileSystem(FileSystem):
70
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped)
71
+
72
+
73
+ cdef class SubTreeFileSystem(FileSystem):
74
+ cdef:
75
+ CSubTreeFileSystem* subtreefs
76
+
77
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped)
78
+
79
+
80
+ cdef class _MockFileSystem(FileSystem):
81
+ cdef:
82
+ CMockFileSystem* mockfs
83
+
84
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped)
85
+
86
+
87
+ cdef class PyFileSystem(FileSystem):
88
+ cdef:
89
+ CPyFileSystem* pyfs
90
+
91
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped)
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pyx ADDED
@@ -0,0 +1,1653 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from cpython.datetime cimport datetime, PyDateTime_DateTime
21
+
22
+ from pyarrow.includes.common cimport *
23
+ from pyarrow.includes.libarrow_python cimport PyDateTime_to_TimePoint
24
+ from pyarrow.lib import _detect_compression, frombytes, tobytes
25
+ from pyarrow.lib cimport *
26
+ from pyarrow.util import _stringify_path
27
+
28
+ from abc import ABC, abstractmethod
29
+ from datetime import datetime, timezone
30
+ import os
31
+ import pathlib
32
+ import sys
33
+
34
+
35
+ cdef _init_ca_paths():
36
+ cdef CFileSystemGlobalOptions options
37
+
38
+ import ssl
39
+ paths = ssl.get_default_verify_paths()
40
+ if paths.cafile:
41
+ options.tls_ca_file_path = os.fsencode(paths.cafile)
42
+ if paths.capath:
43
+ options.tls_ca_dir_path = os.fsencode(paths.capath)
44
+ check_status(CFileSystemsInitialize(options))
45
+
46
+
47
+ if sys.platform == 'linux':
48
+ # ARROW-9261: On Linux, we may need to fixup the paths to TLS CA certs
49
+ # (especially in manylinux packages) since the values hardcoded at
50
+ # compile-time in libcurl may be wrong.
51
+ _init_ca_paths()
52
+
53
+
54
+ cdef inline c_string _path_as_bytes(path) except *:
55
+ # handle only abstract paths, not bound to any filesystem like pathlib is,
56
+ # so we only accept plain strings
57
+ if not isinstance(path, (bytes, str)):
58
+ raise TypeError('Path must be a string')
59
+ # tobytes always uses utf-8, which is more or less ok, at least on Windows
60
+ # since the C++ side then decodes from utf-8. On Unix, os.fsencode may be
61
+ # better.
62
+ return tobytes(path)
63
+
64
+
65
+ cdef object _wrap_file_type(CFileType ty):
66
+ return FileType(<int8_t> ty)
67
+
68
+
69
+ cdef CFileType _unwrap_file_type(FileType ty) except *:
70
+ if ty == FileType.Unknown:
71
+ return CFileType_Unknown
72
+ elif ty == FileType.NotFound:
73
+ return CFileType_NotFound
74
+ elif ty == FileType.File:
75
+ return CFileType_File
76
+ elif ty == FileType.Directory:
77
+ return CFileType_Directory
78
+ assert 0
79
+
80
+
81
+ def _file_type_to_string(ty):
82
+ # Python 3.11 changed str(IntEnum) to return the string representation
83
+ # of the integer value: https://github.com/python/cpython/issues/94763
84
+ return f"{ty.__class__.__name__}.{ty._name_}"
85
+
86
+
87
+ cdef class FileInfo(_Weakrefable):
88
+ """
89
+ FileSystem entry info.
90
+
91
+ Parameters
92
+ ----------
93
+ path : str
94
+ The full path to the filesystem entry.
95
+ type : FileType
96
+ The type of the filesystem entry.
97
+ mtime : datetime or float, default None
98
+ If given, the modification time of the filesystem entry.
99
+ If a float is given, it is the number of seconds since the
100
+ Unix epoch.
101
+ mtime_ns : int, default None
102
+ If given, the modification time of the filesystem entry,
103
+ in nanoseconds since the Unix epoch.
104
+ `mtime` and `mtime_ns` are mutually exclusive.
105
+ size : int, default None
106
+ If given, the filesystem entry size in bytes. This should only
107
+ be given if `type` is `FileType.File`.
108
+
109
+ Examples
110
+ --------
111
+ Generate a file:
112
+
113
+ >>> from pyarrow import fs
114
+ >>> local = fs.LocalFileSystem()
115
+ >>> path_fs = local_path + '/pyarrow-fs-example.dat'
116
+ >>> with local.open_output_stream(path_fs) as stream:
117
+ ... stream.write(b'data')
118
+ 4
119
+
120
+ Get FileInfo object using ``get_file_info()``:
121
+
122
+ >>> file_info = local.get_file_info(path_fs)
123
+ >>> file_info
124
+ <FileInfo for '.../pyarrow-fs-example.dat': type=FileType.File, size=4>
125
+
126
+ Inspect FileInfo attributes:
127
+
128
+ >>> file_info.type
129
+ <FileType.File: 2>
130
+
131
+ >>> file_info.is_file
132
+ True
133
+
134
+ >>> file_info.path
135
+ '/.../pyarrow-fs-example.dat'
136
+
137
+ >>> file_info.base_name
138
+ 'pyarrow-fs-example.dat'
139
+
140
+ >>> file_info.size
141
+ 4
142
+
143
+ >>> file_info.extension
144
+ 'dat'
145
+
146
+ >>> file_info.mtime # doctest: +SKIP
147
+ datetime.datetime(2022, 6, 29, 7, 56, 10, 873922, tzinfo=datetime.timezone.utc)
148
+
149
+ >>> file_info.mtime_ns # doctest: +SKIP
150
+ 1656489370873922073
151
+ """
152
+
153
+ def __init__(self, path, FileType type=FileType.Unknown, *,
154
+ mtime=None, mtime_ns=None, size=None):
155
+ self.info.set_path(tobytes(path))
156
+ self.info.set_type(_unwrap_file_type(type))
157
+ if mtime is not None:
158
+ if mtime_ns is not None:
159
+ raise TypeError("Only one of mtime and mtime_ns "
160
+ "can be given")
161
+ if isinstance(mtime, datetime):
162
+ self.info.set_mtime(PyDateTime_to_TimePoint(
163
+ <PyDateTime_DateTime*> mtime))
164
+ else:
165
+ self.info.set_mtime(TimePoint_from_s(mtime))
166
+ elif mtime_ns is not None:
167
+ self.info.set_mtime(TimePoint_from_ns(mtime_ns))
168
+ if size is not None:
169
+ self.info.set_size(size)
170
+
171
+ @staticmethod
172
+ cdef wrap(CFileInfo info):
173
+ cdef FileInfo self = FileInfo.__new__(FileInfo)
174
+ self.info = move(info)
175
+ return self
176
+
177
+ cdef inline CFileInfo unwrap(self) nogil:
178
+ return self.info
179
+
180
+ @staticmethod
181
+ cdef CFileInfo unwrap_safe(obj):
182
+ if not isinstance(obj, FileInfo):
183
+ raise TypeError(f"Expected FileInfo instance, got {type(obj)}")
184
+ return (<FileInfo> obj).unwrap()
185
+
186
+ def __repr__(self):
187
+ def getvalue(attr):
188
+ try:
189
+ return getattr(self, attr)
190
+ except ValueError:
191
+ return ''
192
+
193
+ s = (f'<FileInfo for {self.path!r}: '
194
+ f'type={_file_type_to_string(self.type)}')
195
+ if self.is_file:
196
+ s += f', size={self.size}'
197
+ s += '>'
198
+ return s
199
+
200
+ @property
201
+ def type(self):
202
+ """
203
+ Type of the file.
204
+
205
+ The returned enum values can be the following:
206
+
207
+ - FileType.NotFound: target does not exist
208
+ - FileType.Unknown: target exists but its type is unknown (could be a
209
+ special file such as a Unix socket or character device, or
210
+ Windows NUL / CON / ...)
211
+ - FileType.File: target is a regular file
212
+ - FileType.Directory: target is a regular directory
213
+
214
+ Returns
215
+ -------
216
+ type : FileType
217
+ """
218
+ return _wrap_file_type(self.info.type())
219
+
220
+ @property
221
+ def is_file(self):
222
+ """
223
+ """
224
+ return self.type == FileType.File
225
+
226
+ @property
227
+ def path(self):
228
+ """
229
+ The full file path in the filesystem.
230
+
231
+ Examples
232
+ --------
233
+ >>> file_info = local.get_file_info(path)
234
+ >>> file_info.path
235
+ '/.../pyarrow-fs-example.dat'
236
+ """
237
+ return frombytes(self.info.path())
238
+
239
+ @property
240
+ def base_name(self):
241
+ """
242
+ The file base name.
243
+
244
+ Component after the last directory separator.
245
+
246
+ Examples
247
+ --------
248
+ >>> file_info = local.get_file_info(path)
249
+ >>> file_info.base_name
250
+ 'pyarrow-fs-example.dat'
251
+ """
252
+ return frombytes(self.info.base_name())
253
+
254
+ @property
255
+ def size(self):
256
+ """
257
+ The size in bytes, if available.
258
+
259
+ Only regular files are guaranteed to have a size.
260
+
261
+ Returns
262
+ -------
263
+ size : int or None
264
+ """
265
+ cdef int64_t size
266
+ size = self.info.size()
267
+ return (size if size != -1 else None)
268
+
269
+ @property
270
+ def extension(self):
271
+ """
272
+ The file extension.
273
+
274
+ Examples
275
+ --------
276
+ >>> file_info = local.get_file_info(path)
277
+ >>> file_info.extension
278
+ 'dat'
279
+ """
280
+ return frombytes(self.info.extension())
281
+
282
+ @property
283
+ def mtime(self):
284
+ """
285
+ The time of last modification, if available.
286
+
287
+ Returns
288
+ -------
289
+ mtime : datetime.datetime or None
290
+
291
+ Examples
292
+ --------
293
+ >>> file_info = local.get_file_info(path)
294
+ >>> file_info.mtime # doctest: +SKIP
295
+ datetime.datetime(2022, 6, 29, 7, 56, 10, 873922, tzinfo=datetime.timezone.utc)
296
+ """
297
+ cdef int64_t nanoseconds
298
+ nanoseconds = TimePoint_to_ns(self.info.mtime())
299
+ return (datetime.fromtimestamp(nanoseconds / 1.0e9, timezone.utc)
300
+ if nanoseconds != -1 else None)
301
+
302
+ @property
303
+ def mtime_ns(self):
304
+ """
305
+ The time of last modification, if available, expressed in nanoseconds
306
+ since the Unix epoch.
307
+
308
+ Returns
309
+ -------
310
+ mtime_ns : int or None
311
+
312
+ Examples
313
+ --------
314
+ >>> file_info = local.get_file_info(path)
315
+ >>> file_info.mtime_ns # doctest: +SKIP
316
+ 1656489370873922073
317
+ """
318
+ cdef int64_t nanoseconds
319
+ nanoseconds = TimePoint_to_ns(self.info.mtime())
320
+ return (nanoseconds if nanoseconds != -1 else None)
321
+
322
+
323
+ cdef class FileSelector(_Weakrefable):
324
+ """
325
+ File and directory selector.
326
+
327
+ It contains a set of options that describes how to search for files and
328
+ directories.
329
+
330
+ Parameters
331
+ ----------
332
+ base_dir : str
333
+ The directory in which to select files. Relative paths also work, use
334
+ '.' for the current directory and '..' for the parent.
335
+ allow_not_found : bool, default False
336
+ The behavior if `base_dir` doesn't exist in the filesystem.
337
+ If false, an error is returned.
338
+ If true, an empty selection is returned.
339
+ recursive : bool, default False
340
+ Whether to recurse into subdirectories.
341
+
342
+ Examples
343
+ --------
344
+ List the contents of a directory and subdirectories:
345
+
346
+ >>> selector_1 = fs.FileSelector(local_path, recursive=True)
347
+ >>> local.get_file_info(selector_1) # doctest: +SKIP
348
+ [<FileInfo for 'tmp/alphabet/example.dat': type=FileType.File, size=4>,
349
+ <FileInfo for 'tmp/alphabet/subdir': type=FileType.Directory>,
350
+ <FileInfo for 'tmp/alphabet/subdir/example_copy.dat': type=FileType.File, size=4>]
351
+
352
+ List only the contents of the base directory:
353
+
354
+ >>> selector_2 = fs.FileSelector(local_path)
355
+ >>> local.get_file_info(selector_2) # doctest: +SKIP
356
+ [<FileInfo for 'tmp/alphabet/example.dat': type=FileType.File, size=4>,
357
+ <FileInfo for 'tmp/alphabet/subdir': type=FileType.Directory>]
358
+
359
+ Return empty selection if the directory doesn't exist:
360
+
361
+ >>> selector_not_found = fs.FileSelector(local_path + '/missing',
362
+ ... recursive=True,
363
+ ... allow_not_found=True)
364
+ >>> local.get_file_info(selector_not_found)
365
+ []
366
+ """
367
+
368
+ def __init__(self, base_dir, bint allow_not_found=False,
369
+ bint recursive=False):
370
+ self.base_dir = base_dir
371
+ self.recursive = recursive
372
+ self.allow_not_found = allow_not_found
373
+
374
+ @staticmethod
375
+ cdef FileSelector wrap(CFileSelector wrapped):
376
+ cdef FileSelector self = FileSelector.__new__(FileSelector)
377
+ self.selector = move(wrapped)
378
+ return self
379
+
380
+ cdef inline CFileSelector unwrap(self) nogil:
381
+ return self.selector
382
+
383
+ @property
384
+ def base_dir(self):
385
+ return frombytes(self.selector.base_dir)
386
+
387
+ @base_dir.setter
388
+ def base_dir(self, base_dir):
389
+ self.selector.base_dir = _path_as_bytes(base_dir)
390
+
391
+ @property
392
+ def allow_not_found(self):
393
+ return self.selector.allow_not_found
394
+
395
+ @allow_not_found.setter
396
+ def allow_not_found(self, bint allow_not_found):
397
+ self.selector.allow_not_found = allow_not_found
398
+
399
+ @property
400
+ def recursive(self):
401
+ return self.selector.recursive
402
+
403
+ @recursive.setter
404
+ def recursive(self, bint recursive):
405
+ self.selector.recursive = recursive
406
+
407
+ def __repr__(self):
408
+ return f"<FileSelector base_dir={self.base_dir!r} recursive={self.recursive}>"
409
+
410
+
411
+ cdef class FileSystem(_Weakrefable):
412
+ """
413
+ Abstract file system API.
414
+ """
415
+
416
+ def __init__(self):
417
+ raise TypeError("FileSystem is an abstract class, instantiate one of "
418
+ "the subclasses instead: LocalFileSystem or "
419
+ "SubTreeFileSystem")
420
+
421
+ @staticmethod
422
+ def _from_uri(uri):
423
+ fs, _path = FileSystem.from_uri(uri)
424
+ return fs
425
+
426
+ @staticmethod
427
+ def _fsspec_from_uri(uri):
428
+ """Instantiate FSSpecHandler and path for the given URI."""
429
+ try:
430
+ import fsspec
431
+ except ImportError:
432
+ raise ImportError(
433
+ "`fsspec` is required to handle `fsspec+<filesystem>://` and `hf://` URIs."
434
+ )
435
+ from .fs import FSSpecHandler
436
+
437
+ uri = uri.removeprefix("fsspec+")
438
+ fs, path = fsspec.url_to_fs(uri)
439
+ fs = PyFileSystem(FSSpecHandler(fs))
440
+
441
+ return fs, path
442
+
443
+ @staticmethod
444
+ def _native_from_uri(uri):
445
+ """Instantiate native FileSystem and path for the given URI."""
446
+ cdef:
447
+ c_string c_path
448
+ c_string c_uri
449
+ CResult[shared_ptr[CFileSystem]] result
450
+
451
+ if isinstance(uri, pathlib.Path):
452
+ # Make absolute
453
+ uri = uri.resolve().absolute()
454
+ c_uri = tobytes(_stringify_path(uri))
455
+ with nogil:
456
+ result = CFileSystemFromUriOrPath(c_uri, &c_path)
457
+ return FileSystem.wrap(GetResultValue(result)), frombytes(c_path)
458
+
459
+ @staticmethod
460
+ def from_uri(uri):
461
+ """
462
+ Create a new FileSystem from URI or Path.
463
+
464
+ Recognized URI schemes are "file", "mock", "s3fs", "gs", "gcs", "hdfs" and "viewfs".
465
+ In addition, the argument can be a pathlib.Path object, or a string
466
+ describing an absolute local path.
467
+
468
+ Parameters
469
+ ----------
470
+ uri : string
471
+ URI-based path, for example: file:///some/local/path.
472
+
473
+ Returns
474
+ -------
475
+ tuple of (FileSystem, str path)
476
+ With (filesystem, path) tuple where path is the abstract path
477
+ inside the FileSystem instance.
478
+
479
+ Examples
480
+ --------
481
+ Create a new FileSystem subclass from a URI:
482
+
483
+ >>> uri = f'file:///{local_path}/pyarrow-fs-example.dat'
484
+ >>> local_new, path_new = fs.FileSystem.from_uri(uri)
485
+ >>> local_new
486
+ <pyarrow._fs.LocalFileSystem object at ...
487
+ >>> path_new
488
+ '/.../pyarrow-fs-example.dat'
489
+
490
+ Or from a s3 bucket:
491
+
492
+ >>> fs.FileSystem.from_uri("s3://usgs-landsat/collection02/")
493
+ (<pyarrow._s3fs.S3FileSystem object at ...>, 'usgs-landsat/collection02')
494
+
495
+ Or from an fsspec+ URI:
496
+
497
+ >>> fs.FileSystem.from_uri("fsspec+memory:///path/to/file")
498
+ (<pyarrow._fs.PyFileSystem object at ...>, '/path/to/file')
499
+ """
500
+ if isinstance(uri, str) and uri.startswith(("fsspec+", "hf://")):
501
+ return FileSystem._fsspec_from_uri(uri)
502
+ else:
503
+ return FileSystem._native_from_uri(uri)
504
+
505
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped):
506
+ self.wrapped = wrapped
507
+ self.fs = wrapped.get()
508
+
509
+ @staticmethod
510
+ cdef wrap(const shared_ptr[CFileSystem]& sp):
511
+ cdef FileSystem self
512
+
513
+ typ = frombytes(sp.get().type_name())
514
+ if typ == 'local':
515
+ self = LocalFileSystem.__new__(LocalFileSystem)
516
+ elif typ == 'mock':
517
+ self = _MockFileSystem.__new__(_MockFileSystem)
518
+ elif typ == 'subtree':
519
+ self = SubTreeFileSystem.__new__(SubTreeFileSystem)
520
+ elif typ == 's3':
521
+ from pyarrow._s3fs import S3FileSystem
522
+ self = S3FileSystem.__new__(S3FileSystem)
523
+ elif typ == 'gcs':
524
+ from pyarrow._gcsfs import GcsFileSystem
525
+ self = GcsFileSystem.__new__(GcsFileSystem)
526
+ elif typ == 'abfs':
527
+ from pyarrow._azurefs import AzureFileSystem
528
+ self = AzureFileSystem.__new__(AzureFileSystem)
529
+ elif typ == 'hdfs':
530
+ from pyarrow._hdfs import HadoopFileSystem
531
+ self = HadoopFileSystem.__new__(HadoopFileSystem)
532
+ elif typ.startswith('py::'):
533
+ self = PyFileSystem.__new__(PyFileSystem)
534
+ else:
535
+ raise TypeError('Cannot wrap FileSystem pointer')
536
+
537
+ self.init(sp)
538
+ return self
539
+
540
+ cdef inline shared_ptr[CFileSystem] unwrap(self) nogil:
541
+ return self.wrapped
542
+
543
+ def equals(self, FileSystem other not None):
544
+ """
545
+ Parameters
546
+ ----------
547
+ other : pyarrow.fs.FileSystem
548
+
549
+ Returns
550
+ -------
551
+ bool
552
+ """
553
+ return self.fs.Equals(other.unwrap())
554
+
555
+ def __eq__(self, other):
556
+ try:
557
+ return self.equals(other)
558
+ except TypeError:
559
+ return NotImplemented
560
+
561
+ @property
562
+ def type_name(self):
563
+ """
564
+ The filesystem's type name.
565
+ """
566
+ return frombytes(self.fs.type_name())
567
+
568
+ def get_file_info(self, paths_or_selector):
569
+ """
570
+ Get info for the given files.
571
+
572
+ Any symlink is automatically dereferenced, recursively. A non-existing
573
+ or unreachable file returns a FileStat object and has a FileType of
574
+ value NotFound. An exception indicates a truly exceptional condition
575
+ (low-level I/O error, etc.).
576
+
577
+ Parameters
578
+ ----------
579
+ paths_or_selector : FileSelector, path-like or list of path-likes
580
+ Either a selector object, a path-like object or a list of
581
+ path-like objects. The selector's base directory will not be
582
+ part of the results, even if it exists. If it doesn't exist,
583
+ use `allow_not_found`.
584
+
585
+ Returns
586
+ -------
587
+ FileInfo or list of FileInfo
588
+ Single FileInfo object is returned for a single path, otherwise
589
+ a list of FileInfo objects is returned.
590
+
591
+ Examples
592
+ --------
593
+ >>> local
594
+ <pyarrow._fs.LocalFileSystem object at ...>
595
+ >>> local.get_file_info(f"/{local_path}/pyarrow-fs-example.dat")
596
+ <FileInfo for '/.../pyarrow-fs-example.dat': type=FileType.File, size=4>
597
+ """
598
+ cdef:
599
+ CFileInfo info
600
+ c_string path
601
+ vector[CFileInfo] infos
602
+ vector[c_string] paths
603
+ CFileSelector selector
604
+
605
+ if isinstance(paths_or_selector, FileSelector):
606
+ with nogil:
607
+ selector = (<FileSelector>paths_or_selector).selector
608
+ infos = GetResultValue(self.fs.GetFileInfo(selector))
609
+ elif isinstance(paths_or_selector, (list, tuple)):
610
+ paths = [_path_as_bytes(s) for s in paths_or_selector]
611
+ with nogil:
612
+ infos = GetResultValue(self.fs.GetFileInfo(paths))
613
+ elif isinstance(paths_or_selector, (bytes, str)):
614
+ path =_path_as_bytes(paths_or_selector)
615
+ with nogil:
616
+ info = GetResultValue(self.fs.GetFileInfo(path))
617
+ return FileInfo.wrap(info)
618
+ else:
619
+ raise TypeError('Must pass either path(s) or a FileSelector')
620
+
621
+ return [FileInfo.wrap(info) for info in infos]
622
+
623
+ def create_dir(self, path, *, bint recursive=True):
624
+ """
625
+ Create a directory and subdirectories.
626
+
627
+ This function succeeds if the directory already exists.
628
+
629
+ Parameters
630
+ ----------
631
+ path : str
632
+ The path of the new directory.
633
+ recursive : bool, default True
634
+ Create nested directories as well.
635
+ """
636
+ cdef c_string directory = _path_as_bytes(path)
637
+ with nogil:
638
+ check_status(self.fs.CreateDir(directory, recursive=recursive))
639
+
640
+ def delete_dir(self, path):
641
+ """
642
+ Delete a directory and its contents, recursively.
643
+
644
+ Parameters
645
+ ----------
646
+ path : str
647
+ The path of the directory to be deleted.
648
+ """
649
+ cdef c_string directory = _path_as_bytes(path)
650
+ with nogil:
651
+ check_status(self.fs.DeleteDir(directory))
652
+
653
+ def delete_dir_contents(self, path, *,
654
+ bint accept_root_dir=False,
655
+ bint missing_dir_ok=False):
656
+ """
657
+ Delete a directory's contents, recursively.
658
+
659
+ Like delete_dir, but doesn't delete the directory itself.
660
+
661
+ Parameters
662
+ ----------
663
+ path : str
664
+ The path of the directory to be deleted.
665
+ accept_root_dir : boolean, default False
666
+ Allow deleting the root directory's contents
667
+ (if path is empty or "/")
668
+ missing_dir_ok : boolean, default False
669
+ If False then an error is raised if path does
670
+ not exist
671
+ """
672
+ cdef c_string directory = _path_as_bytes(path)
673
+ if accept_root_dir and directory.strip(b"/") == b"":
674
+ with nogil:
675
+ check_status(self.fs.DeleteRootDirContents())
676
+ else:
677
+ with nogil:
678
+ check_status(self.fs.DeleteDirContents(directory,
679
+ missing_dir_ok))
680
+
681
+ def move(self, src, dest):
682
+ """
683
+ Move / rename a file or directory.
684
+
685
+ If the destination exists:
686
+ - if it is a non-empty directory, an error is returned
687
+ - otherwise, if it has the same type as the source, it is replaced
688
+ - otherwise, behavior is unspecified (implementation-dependent).
689
+
690
+ Parameters
691
+ ----------
692
+ src : str
693
+ The path of the file or the directory to be moved.
694
+ dest : str
695
+ The destination path where the file or directory is moved to.
696
+
697
+ Examples
698
+ --------
699
+ Create a new folder with a file:
700
+
701
+ >>> local.create_dir('/tmp/other_dir')
702
+ >>> local.copy_file(path,'/tmp/move_example.dat')
703
+
704
+ Move the file:
705
+
706
+ >>> local.move('/tmp/move_example.dat',
707
+ ... '/tmp/other_dir/move_example_2.dat')
708
+
709
+ Inspect the file info:
710
+
711
+ >>> local.get_file_info('/tmp/other_dir/move_example_2.dat')
712
+ <FileInfo for '/tmp/other_dir/move_example_2.dat': type=FileType.File, size=4>
713
+ >>> local.get_file_info('/tmp/move_example.dat')
714
+ <FileInfo for '/tmp/move_example.dat': type=FileType.NotFound>
715
+
716
+ Delete the folder:
717
+ >>> local.delete_dir('/tmp/other_dir')
718
+ """
719
+ cdef:
720
+ c_string source = _path_as_bytes(src)
721
+ c_string destination = _path_as_bytes(dest)
722
+ with nogil:
723
+ check_status(self.fs.Move(source, destination))
724
+
725
+ def copy_file(self, src, dest):
726
+ """
727
+ Copy a file.
728
+
729
+ If the destination exists and is a directory, an error is returned.
730
+ Otherwise, it is replaced.
731
+
732
+ Parameters
733
+ ----------
734
+ src : str
735
+ The path of the file to be copied from.
736
+ dest : str
737
+ The destination path where the file is copied to.
738
+
739
+ Examples
740
+ --------
741
+ >>> local.copy_file(path,
742
+ ... local_path + '/pyarrow-fs-example_copy.dat')
743
+
744
+ Inspect the file info:
745
+
746
+ >>> local.get_file_info(local_path + '/pyarrow-fs-example_copy.dat')
747
+ <FileInfo for '/.../pyarrow-fs-example_copy.dat': type=FileType.File, size=4>
748
+ >>> local.get_file_info(path)
749
+ <FileInfo for '/.../pyarrow-fs-example.dat': type=FileType.File, size=4>
750
+ """
751
+ cdef:
752
+ c_string source = _path_as_bytes(src)
753
+ c_string destination = _path_as_bytes(dest)
754
+ with nogil:
755
+ check_status(self.fs.CopyFile(source, destination))
756
+
757
+ def delete_file(self, path):
758
+ """
759
+ Delete a file.
760
+
761
+ Parameters
762
+ ----------
763
+ path : str
764
+ The path of the file to be deleted.
765
+ """
766
+ cdef c_string file = _path_as_bytes(path)
767
+ with nogil:
768
+ check_status(self.fs.DeleteFile(file))
769
+
770
+ def _wrap_input_stream(self, stream, path, compression, buffer_size):
771
+ if buffer_size is not None and buffer_size != 0:
772
+ stream = BufferedInputStream(stream, buffer_size)
773
+ if compression == 'detect':
774
+ compression = _detect_compression(path)
775
+ if compression is not None:
776
+ stream = CompressedInputStream(stream, compression)
777
+ return stream
778
+
779
+ def _wrap_output_stream(self, stream, path, compression, buffer_size):
780
+ if buffer_size is not None and buffer_size != 0:
781
+ stream = BufferedOutputStream(stream, buffer_size)
782
+ if compression == 'detect':
783
+ compression = _detect_compression(path)
784
+ if compression is not None:
785
+ stream = CompressedOutputStream(stream, compression)
786
+ return stream
787
+
788
+ def open_input_file(self, path):
789
+ """
790
+ Open an input file for random access reading.
791
+
792
+ Parameters
793
+ ----------
794
+ path : str
795
+ The source to open for reading.
796
+
797
+ Returns
798
+ -------
799
+ stream : NativeFile
800
+
801
+ Examples
802
+ --------
803
+ Print the data from the file with `open_input_file()`:
804
+
805
+ >>> with local.open_input_file(path) as f:
806
+ ... print(f.readall())
807
+ b'data'
808
+ """
809
+ cdef:
810
+ c_string pathstr = _path_as_bytes(path)
811
+ NativeFile stream = NativeFile()
812
+ shared_ptr[CRandomAccessFile] in_handle
813
+
814
+ with nogil:
815
+ in_handle = GetResultValue(self.fs.OpenInputFile(pathstr))
816
+
817
+ stream.set_random_access_file(in_handle)
818
+ stream.is_readable = True
819
+ return stream
820
+
821
+ def open_input_stream(self, path, compression='detect', buffer_size=None):
822
+ """
823
+ Open an input stream for sequential reading.
824
+
825
+ Parameters
826
+ ----------
827
+ path : str
828
+ The source to open for reading.
829
+ compression : str optional, default 'detect'
830
+ The compression algorithm to use for on-the-fly decompression.
831
+ If "detect" and source is a file path, then compression will be
832
+ chosen based on the file extension.
833
+ If None, no compression will be applied. Otherwise, a well-known
834
+ algorithm name must be supplied (e.g. "gzip").
835
+ buffer_size : int optional, default None
836
+ If None or 0, no buffering will happen. Otherwise the size of the
837
+ temporary read buffer.
838
+
839
+ Returns
840
+ -------
841
+ stream : NativeFile
842
+
843
+ Examples
844
+ --------
845
+ Print the data from the file with `open_input_stream()`:
846
+
847
+ >>> with local.open_input_stream(path) as f:
848
+ ... print(f.readall())
849
+ b'data'
850
+ """
851
+ cdef:
852
+ c_string pathstr = _path_as_bytes(path)
853
+ NativeFile stream = NativeFile()
854
+ shared_ptr[CInputStream] in_handle
855
+
856
+ with nogil:
857
+ in_handle = GetResultValue(self.fs.OpenInputStream(pathstr))
858
+
859
+ stream.set_input_stream(in_handle)
860
+ stream.is_readable = True
861
+
862
+ return self._wrap_input_stream(
863
+ stream, path=path, compression=compression, buffer_size=buffer_size
864
+ )
865
+
866
+ def open_output_stream(self, path, compression='detect',
867
+ buffer_size=None, metadata=None):
868
+ """
869
+ Open an output stream for sequential writing.
870
+
871
+ If the target already exists, existing data is truncated.
872
+
873
+ Parameters
874
+ ----------
875
+ path : str
876
+ The source to open for writing.
877
+ compression : str optional, default 'detect'
878
+ The compression algorithm to use for on-the-fly compression.
879
+ If "detect" and source is a file path, then compression will be
880
+ chosen based on the file extension.
881
+ If None, no compression will be applied. Otherwise, a well-known
882
+ algorithm name must be supplied (e.g. "gzip").
883
+ buffer_size : int optional, default None
884
+ If None or 0, no buffering will happen. Otherwise the size of the
885
+ temporary write buffer.
886
+ metadata : dict optional, default None
887
+ If not None, a mapping of string keys to string values.
888
+ Some filesystems support storing metadata along the file
889
+ (such as "Content-Type").
890
+ Unsupported metadata keys will be ignored.
891
+
892
+ Returns
893
+ -------
894
+ stream : NativeFile
895
+
896
+ Examples
897
+ --------
898
+ >>> local = fs.LocalFileSystem()
899
+ >>> with local.open_output_stream(path) as stream:
900
+ ... stream.write(b'data')
901
+ 4
902
+ """
903
+ cdef:
904
+ c_string pathstr = _path_as_bytes(path)
905
+ NativeFile stream = NativeFile()
906
+ shared_ptr[COutputStream] out_handle
907
+ shared_ptr[const CKeyValueMetadata] c_metadata
908
+
909
+ if metadata is not None:
910
+ c_metadata = pyarrow_unwrap_metadata(KeyValueMetadata(metadata))
911
+
912
+ with nogil:
913
+ out_handle = GetResultValue(
914
+ self.fs.OpenOutputStream(pathstr, c_metadata))
915
+
916
+ stream.set_output_stream(out_handle)
917
+ stream.is_writable = True
918
+
919
+ return self._wrap_output_stream(
920
+ stream, path=path, compression=compression, buffer_size=buffer_size
921
+ )
922
+
923
+ def open_append_stream(self, path, compression='detect',
924
+ buffer_size=None, metadata=None):
925
+ """
926
+ Open an output stream for appending.
927
+
928
+ If the target doesn't exist, a new empty file is created.
929
+
930
+ .. note::
931
+ Some filesystem implementations do not support efficient
932
+ appending to an existing file, in which case this method will
933
+ raise NotImplementedError.
934
+ Consider writing to multiple files (using e.g. the dataset layer)
935
+ instead.
936
+
937
+ Parameters
938
+ ----------
939
+ path : str
940
+ The source to open for writing.
941
+ compression : str optional, default 'detect'
942
+ The compression algorithm to use for on-the-fly compression.
943
+ If "detect" and source is a file path, then compression will be
944
+ chosen based on the file extension.
945
+ If None, no compression will be applied. Otherwise, a well-known
946
+ algorithm name must be supplied (e.g. "gzip").
947
+ buffer_size : int optional, default None
948
+ If None or 0, no buffering will happen. Otherwise the size of the
949
+ temporary write buffer.
950
+ metadata : dict optional, default None
951
+ If not None, a mapping of string keys to string values.
952
+ Some filesystems support storing metadata along the file
953
+ (such as "Content-Type").
954
+ Unsupported metadata keys will be ignored.
955
+
956
+ Returns
957
+ -------
958
+ stream : NativeFile
959
+
960
+ Examples
961
+ --------
962
+ Append new data to a FileSystem subclass with nonempty file:
963
+
964
+ >>> with local.open_append_stream(path) as f:
965
+ ... f.write(b'+newly added')
966
+ 12
967
+
968
+ Print out the content to the file:
969
+
970
+ >>> with local.open_input_file(path) as f:
971
+ ... print(f.readall())
972
+ b'data+newly added'
973
+ """
974
+ cdef:
975
+ c_string pathstr = _path_as_bytes(path)
976
+ NativeFile stream = NativeFile()
977
+ shared_ptr[COutputStream] out_handle
978
+ shared_ptr[const CKeyValueMetadata] c_metadata
979
+
980
+ if metadata is not None:
981
+ c_metadata = pyarrow_unwrap_metadata(KeyValueMetadata(metadata))
982
+
983
+ with nogil:
984
+ out_handle = GetResultValue(
985
+ self.fs.OpenAppendStream(pathstr, c_metadata))
986
+
987
+ stream.set_output_stream(out_handle)
988
+ stream.is_writable = True
989
+
990
+ return self._wrap_output_stream(
991
+ stream, path=path, compression=compression, buffer_size=buffer_size
992
+ )
993
+
994
+ def normalize_path(self, path):
995
+ """
996
+ Normalize filesystem path.
997
+
998
+ Parameters
999
+ ----------
1000
+ path : str
1001
+ The path to normalize
1002
+
1003
+ Returns
1004
+ -------
1005
+ normalized_path : str
1006
+ The normalized path
1007
+ """
1008
+ cdef:
1009
+ c_string c_path = _path_as_bytes(path)
1010
+ c_string c_path_normalized
1011
+
1012
+ c_path_normalized = GetResultValue(self.fs.NormalizePath(c_path))
1013
+ return frombytes(c_path_normalized)
1014
+
1015
+
1016
+ cdef class LocalFileSystem(FileSystem):
1017
+ """
1018
+ A FileSystem implementation accessing files on the local machine.
1019
+
1020
+ Details such as symlinks are abstracted away (symlinks are always followed,
1021
+ except when deleting an entry).
1022
+
1023
+ Parameters
1024
+ ----------
1025
+ use_mmap : bool, default False
1026
+ Whether open_input_stream and open_input_file should return
1027
+ a mmap'ed file or a regular file.
1028
+
1029
+ Examples
1030
+ --------
1031
+ Create a FileSystem object with LocalFileSystem constructor:
1032
+
1033
+ >>> from pyarrow import fs
1034
+ >>> local = fs.LocalFileSystem()
1035
+ >>> local
1036
+ <pyarrow._fs.LocalFileSystem object at ...>
1037
+
1038
+ and write data on to the file:
1039
+
1040
+ >>> with local.open_output_stream('/tmp/local_fs.dat') as stream:
1041
+ ... stream.write(b'data')
1042
+ 4
1043
+ >>> with local.open_input_stream('/tmp/local_fs.dat') as stream:
1044
+ ... print(stream.readall())
1045
+ b'data'
1046
+
1047
+ Create a FileSystem object inferred from a URI of the saved file:
1048
+
1049
+ >>> local_new, path = fs.LocalFileSystem.from_uri('/tmp/local_fs.dat')
1050
+ >>> local_new
1051
+ <pyarrow._fs.LocalFileSystem object at ...
1052
+ >>> path
1053
+ '/tmp/local_fs.dat'
1054
+
1055
+ Check if FileSystems `local` and `local_new` are equal:
1056
+
1057
+ >>> local.equals(local_new)
1058
+ True
1059
+
1060
+ Compare two different FileSystems:
1061
+
1062
+ >>> local2 = fs.LocalFileSystem(use_mmap=True)
1063
+ >>> local.equals(local2)
1064
+ False
1065
+
1066
+ Copy a file and print out the data:
1067
+
1068
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/local_fs-copy.dat')
1069
+ >>> with local.open_input_stream('/tmp/local_fs-copy.dat') as stream:
1070
+ ... print(stream.readall())
1071
+ ...
1072
+ b'data'
1073
+
1074
+ Open an output stream for appending, add text and print the new data:
1075
+
1076
+ >>> with local.open_append_stream('/tmp/local_fs-copy.dat') as f:
1077
+ ... f.write(b'+newly added')
1078
+ 12
1079
+
1080
+ >>> with local.open_input_stream('/tmp/local_fs-copy.dat') as f:
1081
+ ... print(f.readall())
1082
+ b'data+newly added'
1083
+
1084
+ Create a directory, copy a file into it and then delete the whole directory:
1085
+
1086
+ >>> local.create_dir('/tmp/new_folder')
1087
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
1088
+ >>> local.get_file_info('/tmp/new_folder')
1089
+ <FileInfo for '/tmp/new_folder': type=FileType.Directory>
1090
+ >>> local.delete_dir('/tmp/new_folder')
1091
+ >>> local.get_file_info('/tmp/new_folder')
1092
+ <FileInfo for '/tmp/new_folder': type=FileType.NotFound>
1093
+
1094
+ Create a directory, copy a file into it and then delete
1095
+ the content of the directory:
1096
+
1097
+ >>> local.create_dir('/tmp/new_folder')
1098
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
1099
+ >>> local.get_file_info('/tmp/new_folder/local_fs.dat')
1100
+ <FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.File, size=4>
1101
+ >>> local.delete_dir_contents('/tmp/new_folder')
1102
+ >>> local.get_file_info('/tmp/new_folder')
1103
+ <FileInfo for '/tmp/new_folder': type=FileType.Directory>
1104
+ >>> local.get_file_info('/tmp/new_folder/local_fs.dat')
1105
+ <FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.NotFound>
1106
+
1107
+ Create a directory, copy a file into it and then delete
1108
+ the file from the directory:
1109
+
1110
+ >>> local.create_dir('/tmp/new_folder')
1111
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
1112
+ >>> local.delete_file('/tmp/new_folder/local_fs.dat')
1113
+ >>> local.get_file_info('/tmp/new_folder/local_fs.dat')
1114
+ <FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.NotFound>
1115
+ >>> local.get_file_info('/tmp/new_folder')
1116
+ <FileInfo for '/tmp/new_folder': type=FileType.Directory>
1117
+
1118
+ Move the file:
1119
+
1120
+ >>> local.move('/tmp/local_fs-copy.dat', '/tmp/new_folder/local_fs-copy.dat')
1121
+ >>> local.get_file_info('/tmp/new_folder/local_fs-copy.dat')
1122
+ <FileInfo for '/tmp/new_folder/local_fs-copy.dat': type=FileType.File, size=16>
1123
+ >>> local.get_file_info('/tmp/local_fs-copy.dat')
1124
+ <FileInfo for '/tmp/local_fs-copy.dat': type=FileType.NotFound>
1125
+
1126
+ To finish delete the file left:
1127
+ >>> local.delete_file('/tmp/local_fs.dat')
1128
+ """
1129
+
1130
+ def __init__(self, *, use_mmap=False):
1131
+ cdef:
1132
+ shared_ptr[CFileSystem] fs
1133
+ c_string c_uri
1134
+
1135
+ # from_uri needs a non-empty path, so just use a placeholder of /_
1136
+ c_uri = tobytes(f"file:///_?use_mmap={int(use_mmap)}")
1137
+ with nogil:
1138
+ fs = GetResultValue(CFileSystemFromUri(c_uri))
1139
+ self.init(<shared_ptr[CFileSystem]> fs)
1140
+
1141
+ def __reduce__(self):
1142
+ uri = frombytes(GetResultValue(self.fs.MakeUri(b"/_")))
1143
+ return FileSystem._from_uri, (uri,)
1144
+
1145
+
1146
+ cdef class SubTreeFileSystem(FileSystem):
1147
+ """
1148
+ Delegates to another implementation after prepending a fixed base path.
1149
+
1150
+ This is useful to expose a logical view of a subtree of a filesystem,
1151
+ for example a directory in a LocalFileSystem.
1152
+
1153
+ Note, that this makes no security guarantee. For example, symlinks may
1154
+ allow to "escape" the subtree and access other parts of the underlying
1155
+ filesystem.
1156
+
1157
+ Parameters
1158
+ ----------
1159
+ base_path : str
1160
+ The root of the subtree.
1161
+ base_fs : FileSystem
1162
+ FileSystem object the operations delegated to.
1163
+
1164
+ Examples
1165
+ --------
1166
+ Create a LocalFileSystem instance:
1167
+
1168
+ >>> from pyarrow import fs
1169
+ >>> local = fs.LocalFileSystem()
1170
+ >>> with local.open_output_stream('/tmp/local_fs.dat') as stream:
1171
+ ... stream.write(b'data')
1172
+ 4
1173
+
1174
+ Create a directory and a SubTreeFileSystem instance:
1175
+
1176
+ >>> local.create_dir('/tmp/sub_tree')
1177
+ >>> subtree = fs.SubTreeFileSystem('/tmp/sub_tree', local)
1178
+
1179
+ Write data into the existing file:
1180
+
1181
+ >>> with subtree.open_append_stream('sub_tree_fs.dat') as f:
1182
+ ... f.write(b'+newly added')
1183
+ 12
1184
+
1185
+ Print out the attributes:
1186
+
1187
+ >>> subtree.base_fs
1188
+ <pyarrow._fs.LocalFileSystem object at ...>
1189
+ >>> subtree.base_path
1190
+ '/tmp/sub_tree/'
1191
+
1192
+ Get info for the given directory or given file:
1193
+
1194
+ >>> subtree.get_file_info('')
1195
+ <FileInfo for '': type=FileType.Directory>
1196
+ >>> subtree.get_file_info('sub_tree_fs.dat')
1197
+ <FileInfo for 'sub_tree_fs.dat': type=FileType.File, size=12>
1198
+
1199
+ Delete the file and directory:
1200
+
1201
+ >>> subtree.delete_file('sub_tree_fs.dat')
1202
+ >>> local.delete_dir('/tmp/sub_tree')
1203
+ >>> local.delete_file('/tmp/local_fs.dat')
1204
+
1205
+ For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
1206
+ """
1207
+
1208
+ def __init__(self, base_path, FileSystem base_fs):
1209
+ cdef:
1210
+ c_string pathstr
1211
+ shared_ptr[CSubTreeFileSystem] wrapped
1212
+
1213
+ pathstr = _path_as_bytes(base_path)
1214
+ wrapped = make_shared[CSubTreeFileSystem](pathstr, base_fs.wrapped)
1215
+
1216
+ self.init(<shared_ptr[CFileSystem]> wrapped)
1217
+
1218
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped):
1219
+ FileSystem.init(self, wrapped)
1220
+ self.subtreefs = <CSubTreeFileSystem*> wrapped.get()
1221
+
1222
+ def __repr__(self):
1223
+ return f"SubTreeFileSystem(base_path={self.base_path}, base_fs={self.base_fs})"
1224
+
1225
+ def __reduce__(self):
1226
+ return SubTreeFileSystem, (
1227
+ frombytes(self.subtreefs.base_path()),
1228
+ FileSystem.wrap(self.subtreefs.base_fs())
1229
+ )
1230
+
1231
+ @property
1232
+ def base_path(self):
1233
+ return frombytes(self.subtreefs.base_path())
1234
+
1235
+ @property
1236
+ def base_fs(self):
1237
+ return FileSystem.wrap(self.subtreefs.base_fs())
1238
+
1239
+
1240
+ cdef class _MockFileSystem(FileSystem):
1241
+
1242
+ def __init__(self, datetime current_time=None):
1243
+ cdef shared_ptr[CMockFileSystem] wrapped
1244
+
1245
+ current_time = current_time or datetime.now()
1246
+ wrapped = make_shared[CMockFileSystem](
1247
+ PyDateTime_to_TimePoint(<PyDateTime_DateTime*> current_time)
1248
+ )
1249
+
1250
+ self.init(<shared_ptr[CFileSystem]> wrapped)
1251
+
1252
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped):
1253
+ FileSystem.init(self, wrapped)
1254
+ self.mockfs = <CMockFileSystem*> wrapped.get()
1255
+
1256
+
1257
+ cdef class PyFileSystem(FileSystem):
1258
+ """
1259
+ A FileSystem with behavior implemented in Python.
1260
+
1261
+ Parameters
1262
+ ----------
1263
+ handler : FileSystemHandler
1264
+ The handler object implementing custom filesystem behavior.
1265
+
1266
+ Examples
1267
+ --------
1268
+ Create an fsspec-based filesystem object for GitHub:
1269
+
1270
+ >>> from fsspec.implementations import github
1271
+ >>> gfs = github.GithubFileSystem('apache', 'arrow') # doctest: +SKIP
1272
+
1273
+ Get a PyArrow FileSystem object:
1274
+
1275
+ >>> from pyarrow.fs import PyFileSystem, FSSpecHandler
1276
+ >>> pa_fs = PyFileSystem(FSSpecHandler(gfs)) # doctest: +SKIP
1277
+
1278
+ Use :func:`~pyarrow.fs.FileSystem` functionality ``get_file_info()``:
1279
+
1280
+ >>> pa_fs.get_file_info('README.md') # doctest: +SKIP
1281
+ <FileInfo for 'README.md': type=FileType.File, size=...>
1282
+ """
1283
+
1284
+ def __init__(self, handler):
1285
+ cdef:
1286
+ CPyFileSystemVtable vtable
1287
+ shared_ptr[CPyFileSystem] wrapped
1288
+
1289
+ if not isinstance(handler, FileSystemHandler):
1290
+ raise TypeError(
1291
+ f"Expected a FileSystemHandler instance, got {type(handler)}")
1292
+
1293
+ vtable.get_type_name = _cb_get_type_name
1294
+ vtable.equals = _cb_equals
1295
+ vtable.get_file_info = _cb_get_file_info
1296
+ vtable.get_file_info_vector = _cb_get_file_info_vector
1297
+ vtable.get_file_info_selector = _cb_get_file_info_selector
1298
+ vtable.create_dir = _cb_create_dir
1299
+ vtable.delete_dir = _cb_delete_dir
1300
+ vtable.delete_dir_contents = _cb_delete_dir_contents
1301
+ vtable.delete_root_dir_contents = _cb_delete_root_dir_contents
1302
+ vtable.delete_file = _cb_delete_file
1303
+ vtable.move = _cb_move
1304
+ vtable.copy_file = _cb_copy_file
1305
+ vtable.open_input_stream = _cb_open_input_stream
1306
+ vtable.open_input_file = _cb_open_input_file
1307
+ vtable.open_output_stream = _cb_open_output_stream
1308
+ vtable.open_append_stream = _cb_open_append_stream
1309
+ vtable.normalize_path = _cb_normalize_path
1310
+
1311
+ wrapped = CPyFileSystem.Make(handler, move(vtable))
1312
+ self.init(<shared_ptr[CFileSystem]> wrapped)
1313
+
1314
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped):
1315
+ FileSystem.init(self, wrapped)
1316
+ self.pyfs = <CPyFileSystem*> wrapped.get()
1317
+
1318
+ @property
1319
+ def handler(self):
1320
+ """
1321
+ The filesystem's underlying handler.
1322
+
1323
+ Returns
1324
+ -------
1325
+ handler : FileSystemHandler
1326
+ """
1327
+ return <object> self.pyfs.handler()
1328
+
1329
+ def __reduce__(self):
1330
+ return PyFileSystem, (self.handler,)
1331
+
1332
+
1333
+ class FileSystemHandler(ABC):
1334
+ """
1335
+ An abstract class exposing methods to implement PyFileSystem's behavior.
1336
+ """
1337
+
1338
+ @abstractmethod
1339
+ def get_type_name(self):
1340
+ """
1341
+ Implement PyFileSystem.type_name.
1342
+ """
1343
+
1344
+ @abstractmethod
1345
+ def get_file_info(self, paths):
1346
+ """
1347
+ Implement PyFileSystem.get_file_info(paths).
1348
+
1349
+ Parameters
1350
+ ----------
1351
+ paths : list of str
1352
+ paths for which we want to retrieve the info.
1353
+ """
1354
+
1355
+ @abstractmethod
1356
+ def get_file_info_selector(self, selector):
1357
+ """
1358
+ Implement PyFileSystem.get_file_info(selector).
1359
+
1360
+ Parameters
1361
+ ----------
1362
+ selector : FileSelector
1363
+ selector for which we want to retrieve the info.
1364
+ """
1365
+
1366
+ @abstractmethod
1367
+ def create_dir(self, path, recursive):
1368
+ """
1369
+ Implement PyFileSystem.create_dir(...).
1370
+
1371
+ Parameters
1372
+ ----------
1373
+ path : str
1374
+ path of the directory.
1375
+ recursive : bool
1376
+ if the parent directories should be created too.
1377
+ """
1378
+
1379
+ @abstractmethod
1380
+ def delete_dir(self, path):
1381
+ """
1382
+ Implement PyFileSystem.delete_dir(...).
1383
+
1384
+ Parameters
1385
+ ----------
1386
+ path : str
1387
+ path of the directory.
1388
+ """
1389
+
1390
+ @abstractmethod
1391
+ def delete_dir_contents(self, path, missing_dir_ok=False):
1392
+ """
1393
+ Implement PyFileSystem.delete_dir_contents(...).
1394
+
1395
+ Parameters
1396
+ ----------
1397
+ path : str
1398
+ path of the directory.
1399
+ missing_dir_ok : bool
1400
+ if False an error should be raised if path does not exist
1401
+ """
1402
+
1403
+ @abstractmethod
1404
+ def delete_root_dir_contents(self):
1405
+ """
1406
+ Implement PyFileSystem.delete_dir_contents("/", accept_root_dir=True).
1407
+ """
1408
+
1409
+ @abstractmethod
1410
+ def delete_file(self, path):
1411
+ """
1412
+ Implement PyFileSystem.delete_file(...).
1413
+
1414
+ Parameters
1415
+ ----------
1416
+ path : str
1417
+ path of the file.
1418
+ """
1419
+
1420
+ @abstractmethod
1421
+ def move(self, src, dest):
1422
+ """
1423
+ Implement PyFileSystem.move(...).
1424
+
1425
+ Parameters
1426
+ ----------
1427
+ src : str
1428
+ path of what should be moved.
1429
+ dest : str
1430
+ path of where it should be moved to.
1431
+ """
1432
+
1433
+ @abstractmethod
1434
+ def copy_file(self, src, dest):
1435
+ """
1436
+ Implement PyFileSystem.copy_file(...).
1437
+
1438
+ Parameters
1439
+ ----------
1440
+ src : str
1441
+ path of what should be copied.
1442
+ dest : str
1443
+ path of where it should be copied to.
1444
+ """
1445
+
1446
+ @abstractmethod
1447
+ def open_input_stream(self, path):
1448
+ """
1449
+ Implement PyFileSystem.open_input_stream(...).
1450
+
1451
+ Parameters
1452
+ ----------
1453
+ path : str
1454
+ path of what should be opened.
1455
+ """
1456
+
1457
+ @abstractmethod
1458
+ def open_input_file(self, path):
1459
+ """
1460
+ Implement PyFileSystem.open_input_file(...).
1461
+
1462
+ Parameters
1463
+ ----------
1464
+ path : str
1465
+ path of what should be opened.
1466
+ """
1467
+
1468
+ @abstractmethod
1469
+ def open_output_stream(self, path, metadata):
1470
+ """
1471
+ Implement PyFileSystem.open_output_stream(...).
1472
+
1473
+ Parameters
1474
+ ----------
1475
+ path : str
1476
+ path of what should be opened.
1477
+ metadata : mapping
1478
+ Mapping of string keys to string values.
1479
+ Some filesystems support storing metadata along the file
1480
+ (such as "Content-Type").
1481
+ """
1482
+
1483
+ @abstractmethod
1484
+ def open_append_stream(self, path, metadata):
1485
+ """
1486
+ Implement PyFileSystem.open_append_stream(...).
1487
+
1488
+ Parameters
1489
+ ----------
1490
+ path : str
1491
+ path of what should be opened.
1492
+ metadata : mapping
1493
+ Mapping of string keys to string values.
1494
+ Some filesystems support storing metadata along the file
1495
+ (such as "Content-Type").
1496
+ """
1497
+
1498
+ @abstractmethod
1499
+ def normalize_path(self, path):
1500
+ """
1501
+ Implement PyFileSystem.normalize_path(...).
1502
+
1503
+ Parameters
1504
+ ----------
1505
+ path : str
1506
+ path of what should be normalized.
1507
+ """
1508
+
1509
+ # Callback definitions for CPyFileSystemVtable
1510
+
1511
+
1512
+ cdef void _cb_get_type_name(handler, c_string* out) except *:
1513
+ out[0] = tobytes("py::" + handler.get_type_name())
1514
+
1515
+ cdef c_bool _cb_equals(handler, const CFileSystem& c_other) except False:
1516
+ if c_other.type_name().startswith(b"py::"):
1517
+ return <object> (<const CPyFileSystem&> c_other).handler() == handler
1518
+
1519
+ return False
1520
+
1521
+ cdef void _cb_get_file_info(handler, const c_string& path,
1522
+ CFileInfo* out) except *:
1523
+ infos = handler.get_file_info([frombytes(path)])
1524
+ if not isinstance(infos, list) or len(infos) != 1:
1525
+ raise TypeError("get_file_info should have returned a 1-element list")
1526
+ out[0] = FileInfo.unwrap_safe(infos[0])
1527
+
1528
+ cdef void _cb_get_file_info_vector(handler, const vector[c_string]& paths,
1529
+ vector[CFileInfo]* out) except *:
1530
+ py_paths = [frombytes(paths[i]) for i in range(len(paths))]
1531
+ infos = handler.get_file_info(py_paths)
1532
+ if not isinstance(infos, list):
1533
+ raise TypeError("get_file_info should have returned a list")
1534
+ out[0].clear()
1535
+ out[0].reserve(len(infos))
1536
+ for info in infos:
1537
+ out[0].push_back(FileInfo.unwrap_safe(info))
1538
+
1539
+ cdef void _cb_get_file_info_selector(handler, const CFileSelector& selector,
1540
+ vector[CFileInfo]* out) except *:
1541
+ infos = handler.get_file_info_selector(FileSelector.wrap(selector))
1542
+ if not isinstance(infos, list):
1543
+ raise TypeError("get_file_info_selector should have returned a list")
1544
+ out[0].clear()
1545
+ out[0].reserve(len(infos))
1546
+ for info in infos:
1547
+ out[0].push_back(FileInfo.unwrap_safe(info))
1548
+
1549
+ cdef void _cb_create_dir(handler, const c_string& path,
1550
+ c_bool recursive) except *:
1551
+ handler.create_dir(frombytes(path), recursive)
1552
+
1553
+ cdef void _cb_delete_dir(handler, const c_string& path) except *:
1554
+ handler.delete_dir(frombytes(path))
1555
+
1556
+ cdef void _cb_delete_dir_contents(handler, const c_string& path,
1557
+ c_bool missing_dir_ok) except *:
1558
+ handler.delete_dir_contents(frombytes(path), missing_dir_ok)
1559
+
1560
+ cdef void _cb_delete_root_dir_contents(handler) except *:
1561
+ handler.delete_root_dir_contents()
1562
+
1563
+ cdef void _cb_delete_file(handler, const c_string& path) except *:
1564
+ handler.delete_file(frombytes(path))
1565
+
1566
+ cdef void _cb_move(handler, const c_string& src,
1567
+ const c_string& dest) except *:
1568
+ handler.move(frombytes(src), frombytes(dest))
1569
+
1570
+ cdef void _cb_copy_file(handler, const c_string& src,
1571
+ const c_string& dest) except *:
1572
+ handler.copy_file(frombytes(src), frombytes(dest))
1573
+
1574
+ cdef void _cb_open_input_stream(handler, const c_string& path,
1575
+ shared_ptr[CInputStream]* out) except *:
1576
+ stream = handler.open_input_stream(frombytes(path))
1577
+ if not isinstance(stream, NativeFile):
1578
+ raise TypeError("open_input_stream should have returned "
1579
+ "a PyArrow file")
1580
+ out[0] = (<NativeFile> stream).get_input_stream()
1581
+
1582
+ cdef void _cb_open_input_file(handler, const c_string& path,
1583
+ shared_ptr[CRandomAccessFile]* out) except *:
1584
+ stream = handler.open_input_file(frombytes(path))
1585
+ if not isinstance(stream, NativeFile):
1586
+ raise TypeError("open_input_file should have returned "
1587
+ "a PyArrow file")
1588
+ out[0] = (<NativeFile> stream).get_random_access_file()
1589
+
1590
+ cdef void _cb_open_output_stream(
1591
+ handler, const c_string& path,
1592
+ const shared_ptr[const CKeyValueMetadata]& metadata,
1593
+ shared_ptr[COutputStream]* out) except *:
1594
+ stream = handler.open_output_stream(
1595
+ frombytes(path), pyarrow_wrap_metadata(metadata))
1596
+ if not isinstance(stream, NativeFile):
1597
+ raise TypeError("open_output_stream should have returned "
1598
+ "a PyArrow file")
1599
+ out[0] = (<NativeFile> stream).get_output_stream()
1600
+
1601
+ cdef void _cb_open_append_stream(
1602
+ handler, const c_string& path,
1603
+ const shared_ptr[const CKeyValueMetadata]& metadata,
1604
+ shared_ptr[COutputStream]* out) except *:
1605
+ stream = handler.open_append_stream(
1606
+ frombytes(path), pyarrow_wrap_metadata(metadata))
1607
+ if not isinstance(stream, NativeFile):
1608
+ raise TypeError("open_append_stream should have returned "
1609
+ "a PyArrow file")
1610
+ out[0] = (<NativeFile> stream).get_output_stream()
1611
+
1612
+ cdef void _cb_normalize_path(handler, const c_string& path,
1613
+ c_string* out) except *:
1614
+ out[0] = tobytes(handler.normalize_path(frombytes(path)))
1615
+
1616
+
1617
+ def _copy_files(FileSystem source_fs, str source_path,
1618
+ FileSystem destination_fs, str destination_path,
1619
+ int64_t chunk_size, c_bool use_threads):
1620
+ # low-level helper exposed through pyarrow/fs.py::copy_files
1621
+ cdef:
1622
+ CFileLocator c_source
1623
+ vector[CFileLocator] c_sources
1624
+ CFileLocator c_destination
1625
+ vector[CFileLocator] c_destinations
1626
+
1627
+ c_source.filesystem = source_fs.unwrap()
1628
+ c_source.path = tobytes(source_path)
1629
+ c_sources.push_back(c_source)
1630
+
1631
+ c_destination.filesystem = destination_fs.unwrap()
1632
+ c_destination.path = tobytes(destination_path)
1633
+ c_destinations.push_back(c_destination)
1634
+
1635
+ with nogil:
1636
+ check_status(CCopyFiles(
1637
+ c_sources, c_destinations,
1638
+ c_default_io_context(), chunk_size, use_threads,
1639
+ ))
1640
+
1641
+
1642
+ def _copy_files_selector(FileSystem source_fs, FileSelector source_sel,
1643
+ FileSystem destination_fs, str destination_base_dir,
1644
+ int64_t chunk_size, c_bool use_threads):
1645
+ # low-level helper exposed through pyarrow/fs.py::copy_files
1646
+ cdef c_string c_destination_base_dir = tobytes(destination_base_dir)
1647
+
1648
+ with nogil:
1649
+ check_status(CCopyFilesWithSelector(
1650
+ source_fs.unwrap(), source_sel.unwrap(),
1651
+ destination_fs.unwrap(), c_destination_base_dir,
1652
+ c_default_io_context(), chunk_size, use_threads,
1653
+ ))
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.cp312-win_amd64.pyd ADDED
Binary file (94.2 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.pyx ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.lib cimport (pyarrow_wrap_metadata,
21
+ pyarrow_unwrap_metadata)
22
+ from pyarrow.lib import frombytes, tobytes, ensure_metadata
23
+ from pyarrow.includes.common cimport *
24
+ from pyarrow.includes.libarrow cimport *
25
+ from pyarrow.includes.libarrow_fs cimport *
26
+ from pyarrow._fs cimport FileSystem, TimePoint_to_ns, PyDateTime_to_TimePoint
27
+
28
+ from datetime import datetime, timedelta, timezone
29
+
30
+
31
+ cdef class GcsFileSystem(FileSystem):
32
+ """
33
+ Google Cloud Storage (GCS) backed FileSystem implementation
34
+
35
+ By default uses the process described in https://google.aip.dev/auth/4110
36
+ to resolve credentials. If not running on Google Cloud Platform (GCP),
37
+ this generally requires the environment variable
38
+ GOOGLE_APPLICATION_CREDENTIALS to point to a JSON file
39
+ containing credentials.
40
+
41
+ Note: GCS buckets are special and the operations available on them may be
42
+ limited or more expensive than expected compared to local file systems.
43
+
44
+ Note: When pickling a GcsFileSystem that uses default credentials, resolution
45
+ credentials are not stored in the serialized data. Therefore, when unpickling
46
+ it is assumed that the necessary credentials are in place for the target
47
+ process.
48
+
49
+ Parameters
50
+ ----------
51
+ anonymous : boolean, default False
52
+ Whether to connect anonymously.
53
+ If true, will not attempt to look up credentials using standard GCP
54
+ configuration methods.
55
+ access_token : str, default None
56
+ GCP access token. If provided, temporary credentials will be fetched by
57
+ assuming this role; also, a `credential_token_expiration` must be
58
+ specified as well.
59
+ target_service_account : str, default None
60
+ An optional service account to try to impersonate when accessing GCS. This
61
+ requires the specified credential user or service account to have the necessary
62
+ permissions.
63
+ credential_token_expiration : datetime, default None
64
+ Expiration for credential generated with an access token. Must be specified
65
+ if `access_token` is specified.
66
+ default_bucket_location : str, default 'US'
67
+ GCP region to create buckets in.
68
+ scheme : str, default 'https'
69
+ GCS connection transport scheme.
70
+ endpoint_override : str, default None
71
+ Override endpoint with a connect string such as "localhost:9000"
72
+ default_metadata : mapping or pyarrow.KeyValueMetadata, default None
73
+ Default metadata for `open_output_stream`. This will be ignored if
74
+ non-empty metadata is passed to `open_output_stream`.
75
+ retry_time_limit : timedelta, default None
76
+ Set the maximum amount of time the GCS client will attempt to retry
77
+ transient errors. Subsecond granularity is ignored.
78
+ project_id : str, default None
79
+ The GCP project identifier to use for creating buckets.
80
+ If not set, the library uses the GOOGLE_CLOUD_PROJECT environment
81
+ variable. Most I/O operations do not need a project id, only applications
82
+ that create new buckets need a project id.
83
+ """
84
+
85
+ cdef:
86
+ CGcsFileSystem* gcsfs
87
+
88
+ def __init__(self, *, bint anonymous=False, access_token=None,
89
+ target_service_account=None, credential_token_expiration=None,
90
+ default_bucket_location='US',
91
+ scheme=None,
92
+ endpoint_override=None,
93
+ default_metadata=None,
94
+ retry_time_limit=None,
95
+ project_id=None):
96
+ cdef:
97
+ CGcsOptions options
98
+ shared_ptr[CGcsFileSystem] wrapped
99
+ double time_limit_seconds
100
+
101
+ # Intentional use of truthiness because empty strings aren't valid and
102
+ # for reconstruction from pickling will give empty strings.
103
+ if anonymous and (target_service_account or access_token):
104
+ raise ValueError(
105
+ 'anonymous option is not compatible with target_service_account and '
106
+ 'access_token'
107
+ )
108
+ elif bool(access_token) != bool(credential_token_expiration):
109
+ raise ValueError(
110
+ 'access_token and credential_token_expiration must be '
111
+ 'specified together'
112
+ )
113
+
114
+ elif anonymous:
115
+ options = CGcsOptions.Anonymous()
116
+ elif access_token:
117
+ if not isinstance(credential_token_expiration, datetime):
118
+ raise ValueError(
119
+ "credential_token_expiration must be a datetime")
120
+ options = CGcsOptions.FromAccessToken(
121
+ tobytes(access_token),
122
+ PyDateTime_to_TimePoint(<PyDateTime_DateTime*>credential_token_expiration))
123
+ else:
124
+ options = CGcsOptions.Defaults()
125
+
126
+ # Target service account requires base credentials so
127
+ # it is not part of the if/else chain above which only
128
+ # handles base credentials.
129
+ if target_service_account:
130
+ options = CGcsOptions.FromImpersonatedServiceAccount(
131
+ options.credentials, tobytes(target_service_account))
132
+
133
+ options.default_bucket_location = tobytes(default_bucket_location)
134
+
135
+ if scheme is not None:
136
+ options.scheme = tobytes(scheme)
137
+ if endpoint_override is not None:
138
+ options.endpoint_override = tobytes(endpoint_override)
139
+ if default_metadata is not None:
140
+ options.default_metadata = pyarrow_unwrap_metadata(
141
+ ensure_metadata(default_metadata))
142
+ if retry_time_limit is not None:
143
+ time_limit_seconds = retry_time_limit.total_seconds()
144
+ options.retry_limit_seconds = time_limit_seconds
145
+ if project_id is not None:
146
+ options.project_id = <c_string>tobytes(project_id)
147
+
148
+ with nogil:
149
+ wrapped = GetResultValue(CGcsFileSystem.Make(options))
150
+
151
+ self.init(<shared_ptr[CFileSystem]> wrapped)
152
+
153
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped):
154
+ FileSystem.init(self, wrapped)
155
+ self.gcsfs = <CGcsFileSystem*> wrapped.get()
156
+
157
+ def _expiration_datetime_from_options(self):
158
+ expiration_ns = TimePoint_to_ns(
159
+ self.gcsfs.options().credentials.expiration())
160
+ if expiration_ns == 0:
161
+ return None
162
+ return datetime.fromtimestamp(expiration_ns / 1.0e9, timezone.utc)
163
+
164
+ @staticmethod
165
+ def _reconstruct(kwargs):
166
+ # __reduce__ doesn't allow passing named arguments directly to the
167
+ # reconstructor, hence this wrapper.
168
+ return GcsFileSystem(**kwargs)
169
+
170
+ def __reduce__(self):
171
+ cdef CGcsOptions opts = self.gcsfs.options()
172
+ service_account = frombytes(opts.credentials.target_service_account())
173
+ expiration_dt = self._expiration_datetime_from_options()
174
+ retry_time_limit = None
175
+ if opts.retry_limit_seconds.has_value():
176
+ retry_time_limit = timedelta(
177
+ seconds=opts.retry_limit_seconds.value())
178
+ project_id = None
179
+ if opts.project_id.has_value():
180
+ project_id = frombytes(opts.project_id.value())
181
+ return (
182
+ GcsFileSystem._reconstruct, (dict(
183
+ access_token=frombytes(opts.credentials.access_token()),
184
+ anonymous=opts.credentials.anonymous(),
185
+ credential_token_expiration=expiration_dt,
186
+ target_service_account=service_account,
187
+ scheme=frombytes(opts.scheme),
188
+ endpoint_override=frombytes(opts.endpoint_override),
189
+ default_bucket_location=frombytes(
190
+ opts.default_bucket_location),
191
+ default_metadata=pyarrow_wrap_metadata(opts.default_metadata),
192
+ retry_time_limit=retry_time_limit,
193
+ project_id=project_id
194
+ ),))
195
+
196
+ @property
197
+ def default_bucket_location(self):
198
+ """
199
+ The GCP location this filesystem will write to.
200
+ """
201
+ return frombytes(self.gcsfs.options().default_bucket_location)
202
+
203
+ @property
204
+ def project_id(self):
205
+ """
206
+ The GCP project id this filesystem will use.
207
+ """
208
+ if self.gcsfs.options().project_id.has_value():
209
+ return frombytes(self.gcsfs.options().project_id.value())
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_generated_version.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # file generated by setuptools-scm
2
+ # don't change, don't track in version control
3
+
4
+ __all__ = [
5
+ "__version__",
6
+ "__version_tuple__",
7
+ "version",
8
+ "version_tuple",
9
+ "__commit_id__",
10
+ "commit_id",
11
+ ]
12
+
13
+ TYPE_CHECKING = False
14
+ if TYPE_CHECKING:
15
+ from typing import Tuple
16
+ from typing import Union
17
+
18
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
19
+ COMMIT_ID = Union[str, None]
20
+ else:
21
+ VERSION_TUPLE = object
22
+ COMMIT_ID = object
23
+
24
+ version: str
25
+ __version__: str
26
+ __version_tuple__: VERSION_TUPLE
27
+ version_tuple: VERSION_TUPLE
28
+ commit_id: COMMIT_ID
29
+ __commit_id__: COMMIT_ID
30
+
31
+ __version__ = version = '24.0.0'
32
+ __version_tuple__ = version_tuple = (24, 0, 0)
33
+
34
+ __commit_id__ = commit_id = None
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.cp312-win_amd64.pyd ADDED
Binary file (92.7 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.pyx ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.includes.common cimport *
21
+ from pyarrow.includes.libarrow cimport *
22
+ from pyarrow.includes.libarrow_fs cimport *
23
+ from pyarrow._fs cimport FileSystem
24
+
25
+ from pyarrow.lib import frombytes, tobytes
26
+ from pyarrow.util import _stringify_path
27
+
28
+
29
+ cdef class HadoopFileSystem(FileSystem):
30
+ """
31
+ HDFS backed FileSystem implementation
32
+
33
+ Parameters
34
+ ----------
35
+ host : str
36
+ HDFS host to connect to. Set to "default" for fs.defaultFS from
37
+ core-site.xml.
38
+ port : int, default 8020
39
+ HDFS port to connect to. Set to 0 for default or logical (HA) nodes.
40
+ user : str, default None
41
+ Username when connecting to HDFS; None implies login user.
42
+ replication : int, default 3
43
+ Number of copies each block will have.
44
+ buffer_size : int, default 0
45
+ If 0, no buffering will happen otherwise the size of the temporary read
46
+ and write buffer.
47
+ default_block_size : int, default None
48
+ None means the default configuration for HDFS, a typical block size is
49
+ 128 MB.
50
+ kerb_ticket : string or path, default None
51
+ If not None, the path to the Kerberos ticket cache.
52
+ extra_conf : dict, default None
53
+ Extra key/value pairs for configuration; will override any
54
+ hdfs-site.xml properties.
55
+
56
+ Examples
57
+ --------
58
+ >>> from pyarrow import fs
59
+ >>> hdfs = fs.HadoopFileSystem(host, port, user=user, kerb_ticket=ticket_cache_path) # doctest: +SKIP
60
+
61
+ For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
62
+ """
63
+
64
+ cdef:
65
+ CHadoopFileSystem* hdfs
66
+
67
+ def __init__(self, str host, int port=8020, *, str user=None,
68
+ int replication=3, int buffer_size=0,
69
+ default_block_size=None, kerb_ticket=None,
70
+ extra_conf=None):
71
+ cdef:
72
+ CHdfsOptions options
73
+ shared_ptr[CHadoopFileSystem] wrapped
74
+
75
+ if not host.startswith(('hdfs://', 'viewfs://')) and host != "default":
76
+ # TODO(kszucs): do more sanitization
77
+ host = f'hdfs://{host}'
78
+
79
+ options.ConfigureEndPoint(tobytes(host), int(port))
80
+ options.ConfigureReplication(replication)
81
+ options.ConfigureBufferSize(buffer_size)
82
+
83
+ if user is not None:
84
+ options.ConfigureUser(tobytes(user))
85
+ if default_block_size is not None:
86
+ options.ConfigureBlockSize(default_block_size)
87
+ if kerb_ticket is not None:
88
+ options.ConfigureKerberosTicketCachePath(
89
+ tobytes(_stringify_path(kerb_ticket)))
90
+ if extra_conf is not None:
91
+ for k, v in extra_conf.items():
92
+ options.ConfigureExtraConf(tobytes(k), tobytes(v))
93
+
94
+ with nogil:
95
+ wrapped = GetResultValue(CHadoopFileSystem.Make(options))
96
+ self.init(<shared_ptr[CFileSystem]> wrapped)
97
+
98
+ cdef init(self, const shared_ptr[CFileSystem]& wrapped):
99
+ FileSystem.init(self, wrapped)
100
+ self.hdfs = <CHadoopFileSystem*> wrapped.get()
101
+
102
+ @staticmethod
103
+ def from_uri(uri):
104
+ """
105
+ Instantiate HadoopFileSystem object from an URI string.
106
+
107
+ The following two calls are equivalent
108
+
109
+ * ``HadoopFileSystem.from_uri('hdfs://localhost:8020/?user=test\
110
+ &replication=1')``
111
+ * ``HadoopFileSystem('localhost', port=8020, user='test', \
112
+ replication=1)``
113
+
114
+ Parameters
115
+ ----------
116
+ uri : str
117
+ A string URI describing the connection to HDFS.
118
+ In order to change the user, replication, buffer_size or
119
+ default_block_size pass the values as query parts.
120
+
121
+ Returns
122
+ -------
123
+ HadoopFileSystem
124
+ """
125
+ cdef:
126
+ HadoopFileSystem self = HadoopFileSystem.__new__(HadoopFileSystem)
127
+ shared_ptr[CHadoopFileSystem] wrapped
128
+ CHdfsOptions options
129
+
130
+ options = GetResultValue(CHdfsOptions.FromUriString(tobytes(uri)))
131
+ with nogil:
132
+ wrapped = GetResultValue(CHadoopFileSystem.Make(options))
133
+
134
+ self.init(<shared_ptr[CFileSystem]> wrapped)
135
+ return self
136
+
137
+ @staticmethod
138
+ def _reconstruct(kwargs):
139
+ # __reduce__ doesn't allow passing named arguments directly to the
140
+ # reconstructor, hence this wrapper.
141
+ return HadoopFileSystem(**kwargs)
142
+
143
+ def __reduce__(self):
144
+ cdef CHdfsOptions opts = self.hdfs.options()
145
+ return (
146
+ HadoopFileSystem._reconstruct, (dict(
147
+ host=frombytes(opts.connection_config.host),
148
+ port=opts.connection_config.port,
149
+ user=frombytes(opts.connection_config.user),
150
+ replication=opts.replication,
151
+ buffer_size=opts.buffer_size,
152
+ default_block_size=opts.default_block_size,
153
+ kerb_ticket=frombytes(opts.connection_config.kerb_ticket),
154
+ extra_conf={frombytes(k): frombytes(v)
155
+ for k, v in opts.connection_config.extra_conf},
156
+ ),)
157
+ )
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:630be0ef2d4c9b08c78fe7784b0829079def49227d3fb91a373ad9d47f37f40b
3
+ size 102912
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.pxd ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # cython: language_level = 3
19
+
20
+ from pyarrow.includes.libarrow cimport *
21
+ from pyarrow.lib cimport _Weakrefable
22
+
23
+
24
+ cdef class ParseOptions(_Weakrefable):
25
+ cdef:
26
+ CJSONParseOptions options
27
+
28
+ @staticmethod
29
+ cdef ParseOptions wrap(CJSONParseOptions options)
30
+
31
+ cdef class ReadOptions(_Weakrefable):
32
+ cdef:
33
+ CJSONReadOptions options
34
+
35
+ @staticmethod
36
+ cdef ReadOptions wrap(CJSONReadOptions options)