Add loan-advisor-agent
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +27 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pxd +42 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.py +459 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pyi +29 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_compute_docstrings.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_generated_version.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/compute.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/ipc.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/types.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/util.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pxd +44 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pyx +610 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.pyx +179 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pxd +72 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pyx +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute_docstrings.py +257 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pxd +55 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pyx +1626 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pxd +67 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pyx +1084 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pxd +183 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pyx +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.pyx +51 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pxd +43 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pyx +1113 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.pyx +178 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dlpack.pxi +46 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.pyx +117 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.pyx +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pxd +91 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pyx +1653 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.pyx +209 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_generated_version.py +34 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.cp312-win_amd64.pyd +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.pyx +157 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.pxd +36 -0
.gitattributes
CHANGED
|
@@ -510,3 +510,30 @@ loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/pytables.cpytho
|
|
| 510 |
loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/sql.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
|
| 511 |
loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/stata.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
|
| 512 |
loan-advisor-agent/.venv/Lib/site-packages/pandas.libs/msvcp140-a4c2229bdc2a2a630acdc095b4d86008.dll filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 510 |
loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/sql.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
|
| 511 |
loan-advisor-agent/.venv/Lib/site-packages/pandas/io/__pycache__/stata.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
|
| 512 |
loan-advisor-agent/.venv/Lib/site-packages/pandas.libs/msvcp140-a4c2229bdc2a2a630acdc095b4d86008.dll filter=lfs diff=lfs merge=lfs -text
|
| 513 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 514 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 515 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 516 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 517 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 518 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 519 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 520 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 521 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_orc.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 522 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_parquet.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 523 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_parquet_encryption.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 524 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_s3fs.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 525 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_substrait.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 526 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow.dll filter=lfs diff=lfs merge=lfs -text
|
| 527 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow.lib filter=lfs diff=lfs merge=lfs -text
|
| 528 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_acero.dll filter=lfs diff=lfs merge=lfs -text
|
| 529 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_acero.lib filter=lfs diff=lfs merge=lfs -text
|
| 530 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_compute.dll filter=lfs diff=lfs merge=lfs -text
|
| 531 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_compute.lib filter=lfs diff=lfs merge=lfs -text
|
| 532 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_dataset.dll filter=lfs diff=lfs merge=lfs -text
|
| 533 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_dataset.lib filter=lfs diff=lfs merge=lfs -text
|
| 534 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_flight.dll filter=lfs diff=lfs merge=lfs -text
|
| 535 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_flight.lib filter=lfs diff=lfs merge=lfs -text
|
| 536 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.dll filter=lfs diff=lfs merge=lfs -text
|
| 537 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.lib filter=lfs diff=lfs merge=lfs -text
|
| 538 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.dll filter=lfs diff=lfs merge=lfs -text
|
| 539 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.lib filter=lfs diff=lfs merge=lfs -text
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pxd
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
from libcpp.memory cimport shared_ptr
|
| 19 |
+
from pyarrow.includes.libarrow cimport (CArray, CBuffer, CDataType,
|
| 20 |
+
CField, CRecordBatch, CSchema,
|
| 21 |
+
CTable, CTensor, CSparseCOOTensor,
|
| 22 |
+
CSparseCSRMatrix, CSparseCSCMatrix,
|
| 23 |
+
CSparseCSFTensor)
|
| 24 |
+
|
| 25 |
+
cdef extern from "arrow/python/pyarrow.h" namespace "arrow::py":
|
| 26 |
+
cdef int import_pyarrow() except -1
|
| 27 |
+
cdef object wrap_buffer(const shared_ptr[CBuffer]& buffer)
|
| 28 |
+
cdef object wrap_data_type(const shared_ptr[CDataType]& type)
|
| 29 |
+
cdef object wrap_field(const shared_ptr[CField]& field)
|
| 30 |
+
cdef object wrap_schema(const shared_ptr[CSchema]& schema)
|
| 31 |
+
cdef object wrap_array(const shared_ptr[CArray]& sp_array)
|
| 32 |
+
cdef object wrap_tensor(const shared_ptr[CTensor]& sp_tensor)
|
| 33 |
+
cdef object wrap_sparse_tensor_coo(
|
| 34 |
+
const shared_ptr[CSparseCOOTensor]& sp_sparse_tensor)
|
| 35 |
+
cdef object wrap_sparse_tensor_csr(
|
| 36 |
+
const shared_ptr[CSparseCSRMatrix]& sp_sparse_tensor)
|
| 37 |
+
cdef object wrap_sparse_tensor_csc(
|
| 38 |
+
const shared_ptr[CSparseCSCMatrix]& sp_sparse_tensor)
|
| 39 |
+
cdef object wrap_sparse_tensor_csf(
|
| 40 |
+
const shared_ptr[CSparseCSFTensor]& sp_sparse_tensor)
|
| 41 |
+
cdef object wrap_table(const shared_ptr[CTable]& ctable)
|
| 42 |
+
cdef object wrap_batch(const shared_ptr[CRecordBatch]& cbatch)
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.py
ADDED
|
@@ -0,0 +1,459 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# flake8: noqa
|
| 19 |
+
|
| 20 |
+
"""
|
| 21 |
+
PyArrow is the python implementation of Apache Arrow.
|
| 22 |
+
|
| 23 |
+
Apache Arrow is a cross-language development platform for in-memory data.
|
| 24 |
+
It specifies a standardized language-independent columnar memory format for
|
| 25 |
+
flat and hierarchical data, organized for efficient analytic operations on
|
| 26 |
+
modern hardware. It also provides computational libraries and zero-copy
|
| 27 |
+
streaming messaging and interprocess communication.
|
| 28 |
+
|
| 29 |
+
For more information see the official page at https://arrow.apache.org
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# start delvewheel patch
|
| 34 |
+
def _delvewheel_patch_1_12_0():
|
| 35 |
+
import os
|
| 36 |
+
if os.path.isdir(libs_dir := os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'pyarrow.libs'))):
|
| 37 |
+
os.add_dll_directory(libs_dir)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
_delvewheel_patch_1_12_0()
|
| 41 |
+
del _delvewheel_patch_1_12_0
|
| 42 |
+
# end delvewheel patch
|
| 43 |
+
|
| 44 |
+
import importlib as _importlib
|
| 45 |
+
import os as _os
|
| 46 |
+
import platform as _platform
|
| 47 |
+
import sys as _sys
|
| 48 |
+
|
| 49 |
+
try:
|
| 50 |
+
from ._generated_version import version as __version__
|
| 51 |
+
except ImportError:
|
| 52 |
+
# Package is not installed, parse git tag at runtime
|
| 53 |
+
try:
|
| 54 |
+
import setuptools_scm
|
| 55 |
+
# Code duplicated from setup.py to avoid a dependency on each other
|
| 56 |
+
|
| 57 |
+
def parse_git(root, **kwargs):
|
| 58 |
+
"""
|
| 59 |
+
Parse function for setuptools_scm that ignores tags for non-C++
|
| 60 |
+
subprojects, e.g. apache-arrow-js-XXX tags.
|
| 61 |
+
"""
|
| 62 |
+
from setuptools_scm.git import parse
|
| 63 |
+
kwargs['describe_command'] = \
|
| 64 |
+
"git describe --dirty --tags --long --match 'apache-arrow-[0-9]*.*'"
|
| 65 |
+
return parse(root, **kwargs)
|
| 66 |
+
__version__ = setuptools_scm.get_version('../',
|
| 67 |
+
parse=parse_git)
|
| 68 |
+
except ImportError:
|
| 69 |
+
__version__ = None
|
| 70 |
+
|
| 71 |
+
from pyarrow.lib import (BuildInfo, CppBuildInfo, RuntimeInfo, set_timezone_db_path,
|
| 72 |
+
MonthDayNano, VersionInfo, build_info, cpp_build_info,
|
| 73 |
+
cpp_version, cpp_version_info, runtime_info,
|
| 74 |
+
cpu_count, set_cpu_count, enable_signal_handlers,
|
| 75 |
+
io_thread_count, is_opentelemetry_enabled, set_io_thread_count)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def show_versions():
|
| 79 |
+
"""
|
| 80 |
+
Print various version information, to help with error reporting.
|
| 81 |
+
"""
|
| 82 |
+
def print_entry(label, value):
|
| 83 |
+
print(f"{label: <26}: {value: <8}")
|
| 84 |
+
|
| 85 |
+
print("pyarrow version info\n--------------------")
|
| 86 |
+
print_entry("Package kind", build_info.cpp_build_info.package_kind
|
| 87 |
+
if len(build_info.cpp_build_info.package_kind) > 0
|
| 88 |
+
else "not indicated")
|
| 89 |
+
print_entry("Arrow C++ library version", build_info.cpp_build_info.version)
|
| 90 |
+
print_entry("Arrow C++ compiler",
|
| 91 |
+
(f"{build_info.cpp_build_info.compiler_id} "
|
| 92 |
+
f"{build_info.cpp_build_info.compiler_version}"))
|
| 93 |
+
print_entry("Arrow C++ compiler flags", build_info.cpp_build_info.compiler_flags)
|
| 94 |
+
print_entry("Arrow C++ git revision", build_info.cpp_build_info.git_id)
|
| 95 |
+
print_entry("Arrow C++ git description", build_info.cpp_build_info.git_description)
|
| 96 |
+
print_entry("Arrow C++ build type", build_info.cpp_build_info.build_type)
|
| 97 |
+
print_entry("PyArrow build type", build_info.build_type)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _module_is_available(module):
|
| 101 |
+
try:
|
| 102 |
+
_importlib.import_module(f'pyarrow.{module}')
|
| 103 |
+
except ImportError:
|
| 104 |
+
return False
|
| 105 |
+
else:
|
| 106 |
+
return True
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _filesystem_is_available(fs):
|
| 110 |
+
try:
|
| 111 |
+
import pyarrow.fs
|
| 112 |
+
except ImportError:
|
| 113 |
+
return False
|
| 114 |
+
|
| 115 |
+
try:
|
| 116 |
+
getattr(pyarrow.fs, fs)
|
| 117 |
+
except (ImportError, AttributeError):
|
| 118 |
+
return False
|
| 119 |
+
else:
|
| 120 |
+
return True
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def show_info():
|
| 124 |
+
"""
|
| 125 |
+
Print detailed version and platform information, for error reporting
|
| 126 |
+
"""
|
| 127 |
+
show_versions()
|
| 128 |
+
|
| 129 |
+
def print_entry(label, value):
|
| 130 |
+
print(f" {label: <20}: {value: <8}")
|
| 131 |
+
|
| 132 |
+
print("\nPlatform:")
|
| 133 |
+
print_entry("OS / Arch", f"{_platform.system()} {_platform.machine()}")
|
| 134 |
+
print_entry("SIMD Level", runtime_info().simd_level)
|
| 135 |
+
print_entry("Detected SIMD Level", runtime_info().detected_simd_level)
|
| 136 |
+
|
| 137 |
+
pool = default_memory_pool()
|
| 138 |
+
print("\nMemory:")
|
| 139 |
+
print_entry("Default backend", pool.backend_name)
|
| 140 |
+
print_entry("Bytes allocated", f"{pool.bytes_allocated()} bytes")
|
| 141 |
+
print_entry("Max memory", f"{pool.max_memory()} bytes")
|
| 142 |
+
print_entry("Supported Backends", ', '.join(supported_memory_backends()))
|
| 143 |
+
|
| 144 |
+
print("\nOptional modules:")
|
| 145 |
+
modules = ["csv", "cuda", "dataset", "feather", "flight", "fs", "gandiva", "json",
|
| 146 |
+
"orc", "parquet"]
|
| 147 |
+
for module in modules:
|
| 148 |
+
status = "Enabled" if _module_is_available(module) else "-"
|
| 149 |
+
print(f" {module: <20}: {status: <8}")
|
| 150 |
+
print(f" {'opentelemetry': <20}: {'Enabled' if is_opentelemetry_enabled() else '-': <8}")
|
| 151 |
+
|
| 152 |
+
print("\nFilesystems:")
|
| 153 |
+
filesystems = ["AzureFileSystem", "GcsFileSystem",
|
| 154 |
+
"HadoopFileSystem", "S3FileSystem"]
|
| 155 |
+
for fs in filesystems:
|
| 156 |
+
status = "Enabled" if _filesystem_is_available(fs) else "-"
|
| 157 |
+
print(f" {fs: <20}: {status: <8}")
|
| 158 |
+
|
| 159 |
+
print("\nCompression Codecs:")
|
| 160 |
+
codecs = ["brotli", "bz2", "gzip", "lz4_frame", "lz4", "snappy", "zstd"]
|
| 161 |
+
for codec in codecs:
|
| 162 |
+
status = "Enabled" if Codec.is_available(codec) else "-"
|
| 163 |
+
print(f" {codec: <20}: {status: <8}")
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
from pyarrow.lib import (null, bool_,
|
| 167 |
+
int8, int16, int32, int64,
|
| 168 |
+
uint8, uint16, uint32, uint64,
|
| 169 |
+
time32, time64, timestamp, date32, date64, duration,
|
| 170 |
+
month_day_nano_interval,
|
| 171 |
+
float16, float32, float64,
|
| 172 |
+
binary, string, utf8, binary_view, string_view,
|
| 173 |
+
large_binary, large_string, large_utf8,
|
| 174 |
+
decimal32, decimal64, decimal128, decimal256,
|
| 175 |
+
list_, large_list, list_view, large_list_view,
|
| 176 |
+
map_, struct,
|
| 177 |
+
union, sparse_union, dense_union,
|
| 178 |
+
dictionary,
|
| 179 |
+
run_end_encoded,
|
| 180 |
+
bool8, fixed_shape_tensor, json_, opaque, uuid,
|
| 181 |
+
field,
|
| 182 |
+
type_for_alias,
|
| 183 |
+
DataType, DictionaryType, StructType,
|
| 184 |
+
ListType, LargeListType, FixedSizeListType,
|
| 185 |
+
ListViewType, LargeListViewType,
|
| 186 |
+
MapType, UnionType, SparseUnionType, DenseUnionType,
|
| 187 |
+
TimestampType, Time32Type, Time64Type, DurationType,
|
| 188 |
+
FixedSizeBinaryType,
|
| 189 |
+
Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,
|
| 190 |
+
BaseExtensionType, ExtensionType,
|
| 191 |
+
RunEndEncodedType, Bool8Type, FixedShapeTensorType,
|
| 192 |
+
JsonType, OpaqueType, UuidType,
|
| 193 |
+
UnknownExtensionType,
|
| 194 |
+
register_extension_type, unregister_extension_type,
|
| 195 |
+
DictionaryMemo,
|
| 196 |
+
KeyValueMetadata,
|
| 197 |
+
Field,
|
| 198 |
+
Schema,
|
| 199 |
+
schema,
|
| 200 |
+
unify_schemas,
|
| 201 |
+
Array, Tensor,
|
| 202 |
+
array, chunked_array, record_batch, nulls, repeat,
|
| 203 |
+
SparseCOOTensor, SparseCSRMatrix, SparseCSCMatrix,
|
| 204 |
+
SparseCSFTensor,
|
| 205 |
+
infer_type, from_numpy_dtype,
|
| 206 |
+
arange,
|
| 207 |
+
NullArray,
|
| 208 |
+
NumericArray, IntegerArray, FloatingPointArray,
|
| 209 |
+
BooleanArray,
|
| 210 |
+
Int8Array, UInt8Array,
|
| 211 |
+
Int16Array, UInt16Array,
|
| 212 |
+
Int32Array, UInt32Array,
|
| 213 |
+
Int64Array, UInt64Array,
|
| 214 |
+
HalfFloatArray, FloatArray, DoubleArray,
|
| 215 |
+
ListArray, LargeListArray, FixedSizeListArray,
|
| 216 |
+
ListViewArray, LargeListViewArray,
|
| 217 |
+
MapArray, UnionArray,
|
| 218 |
+
BinaryArray, StringArray,
|
| 219 |
+
LargeBinaryArray, LargeStringArray,
|
| 220 |
+
BinaryViewArray, StringViewArray,
|
| 221 |
+
FixedSizeBinaryArray,
|
| 222 |
+
DictionaryArray,
|
| 223 |
+
Date32Array, Date64Array, TimestampArray,
|
| 224 |
+
Time32Array, Time64Array, DurationArray,
|
| 225 |
+
MonthDayNanoIntervalArray,
|
| 226 |
+
Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
|
| 227 |
+
StructArray, ExtensionArray,
|
| 228 |
+
RunEndEncodedArray, Bool8Array, FixedShapeTensorArray,
|
| 229 |
+
JsonArray, OpaqueArray, UuidArray,
|
| 230 |
+
scalar, NA, _NULL as NULL, Scalar,
|
| 231 |
+
NullScalar, BooleanScalar,
|
| 232 |
+
Int8Scalar, Int16Scalar, Int32Scalar, Int64Scalar,
|
| 233 |
+
UInt8Scalar, UInt16Scalar, UInt32Scalar, UInt64Scalar,
|
| 234 |
+
HalfFloatScalar, FloatScalar, DoubleScalar,
|
| 235 |
+
Decimal32Scalar, Decimal64Scalar, Decimal128Scalar, Decimal256Scalar,
|
| 236 |
+
ListScalar, LargeListScalar, FixedSizeListScalar,
|
| 237 |
+
ListViewScalar, LargeListViewScalar,
|
| 238 |
+
Date32Scalar, Date64Scalar,
|
| 239 |
+
Time32Scalar, Time64Scalar,
|
| 240 |
+
TimestampScalar, DurationScalar,
|
| 241 |
+
MonthDayNanoIntervalScalar,
|
| 242 |
+
BinaryScalar, LargeBinaryScalar, BinaryViewScalar,
|
| 243 |
+
StringScalar, LargeStringScalar, StringViewScalar,
|
| 244 |
+
FixedSizeBinaryScalar, DictionaryScalar,
|
| 245 |
+
MapScalar, StructScalar, UnionScalar,
|
| 246 |
+
RunEndEncodedScalar, Bool8Scalar, ExtensionScalar,
|
| 247 |
+
FixedShapeTensorScalar, JsonScalar, OpaqueScalar, UuidScalar)
|
| 248 |
+
|
| 249 |
+
# Buffers, allocation
|
| 250 |
+
from pyarrow.lib import (DeviceAllocationType, Device, MemoryManager,
|
| 251 |
+
default_cpu_memory_manager)
|
| 252 |
+
|
| 253 |
+
from pyarrow.lib import (Buffer, ResizableBuffer, foreign_buffer, py_buffer,
|
| 254 |
+
Codec, compress, decompress, allocate_buffer)
|
| 255 |
+
|
| 256 |
+
from pyarrow.lib import (MemoryPool, LoggingMemoryPool, ProxyMemoryPool,
|
| 257 |
+
total_allocated_bytes, set_memory_pool,
|
| 258 |
+
default_memory_pool, system_memory_pool,
|
| 259 |
+
jemalloc_memory_pool, mimalloc_memory_pool,
|
| 260 |
+
logging_memory_pool, proxy_memory_pool,
|
| 261 |
+
log_memory_allocations, jemalloc_set_decay_ms,
|
| 262 |
+
supported_memory_backends)
|
| 263 |
+
|
| 264 |
+
# I/O
|
| 265 |
+
from pyarrow.lib import (NativeFile, PythonFile,
|
| 266 |
+
BufferedInputStream, BufferedOutputStream, CacheOptions,
|
| 267 |
+
CompressedInputStream, CompressedOutputStream,
|
| 268 |
+
TransformInputStream, transcoding_input_stream,
|
| 269 |
+
FixedSizeBufferWriter,
|
| 270 |
+
BufferReader, BufferOutputStream,
|
| 271 |
+
OSFile, MemoryMappedFile, memory_map,
|
| 272 |
+
create_memory_map, MockOutputStream,
|
| 273 |
+
input_stream, output_stream,
|
| 274 |
+
have_libhdfs)
|
| 275 |
+
|
| 276 |
+
from pyarrow.lib import (ChunkedArray, RecordBatch, Table, table,
|
| 277 |
+
concat_arrays, concat_tables, TableGroupBy,
|
| 278 |
+
RecordBatchReader, concat_batches)
|
| 279 |
+
|
| 280 |
+
# Exceptions
|
| 281 |
+
from pyarrow.lib import (ArrowCancelled,
|
| 282 |
+
ArrowCapacityError,
|
| 283 |
+
ArrowException,
|
| 284 |
+
ArrowKeyError,
|
| 285 |
+
ArrowIndexError,
|
| 286 |
+
ArrowInvalid,
|
| 287 |
+
ArrowIOError,
|
| 288 |
+
ArrowMemoryError,
|
| 289 |
+
ArrowNotImplementedError,
|
| 290 |
+
ArrowTypeError,
|
| 291 |
+
ArrowSerializationError)
|
| 292 |
+
|
| 293 |
+
from pyarrow.ipc import serialize_pandas, deserialize_pandas
|
| 294 |
+
import pyarrow.ipc as ipc
|
| 295 |
+
|
| 296 |
+
import pyarrow.types as types
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
# ----------------------------------------------------------------------
|
| 300 |
+
# Deprecations
|
| 301 |
+
|
| 302 |
+
from pyarrow.util import _deprecate_api, _deprecate_class
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
# TODO: Deprecate these somehow in the pyarrow namespace
|
| 306 |
+
from pyarrow.ipc import (Message, MessageReader, MetadataVersion,
|
| 307 |
+
RecordBatchFileReader, RecordBatchFileWriter,
|
| 308 |
+
RecordBatchStreamReader, RecordBatchStreamWriter)
|
| 309 |
+
|
| 310 |
+
# ----------------------------------------------------------------------
|
| 311 |
+
# Returning absolute path to the pyarrow include directory (if bundled, e.g. in
|
| 312 |
+
# wheels)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def get_include():
|
| 316 |
+
"""
|
| 317 |
+
Return absolute path to directory containing Arrow C++ include
|
| 318 |
+
headers. Similar to numpy.get_include
|
| 319 |
+
"""
|
| 320 |
+
# Use pyarrow.lib location instead of just __file__. That works
|
| 321 |
+
# for both editable and non-editable builds as it points
|
| 322 |
+
# to the actual location of the compiled C++ extension.
|
| 323 |
+
from pyarrow import lib as _lib
|
| 324 |
+
return _os.path.join(_os.path.dirname(_lib.__file__), 'include')
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
def _get_pkg_config_executable():
|
| 328 |
+
return _os.environ.get('PKG_CONFIG', 'pkg-config')
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def _has_pkg_config(pkgname):
|
| 332 |
+
import subprocess
|
| 333 |
+
try:
|
| 334 |
+
return subprocess.call([_get_pkg_config_executable(),
|
| 335 |
+
'--exists', pkgname]) == 0
|
| 336 |
+
except FileNotFoundError:
|
| 337 |
+
return False
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def _read_pkg_config_variable(pkgname, cli_args):
|
| 341 |
+
import subprocess
|
| 342 |
+
cmd = [_get_pkg_config_executable(), pkgname] + cli_args
|
| 343 |
+
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
| 344 |
+
stderr=subprocess.PIPE)
|
| 345 |
+
out, err = proc.communicate()
|
| 346 |
+
if proc.returncode != 0:
|
| 347 |
+
raise RuntimeError("pkg-config failed: " + err.decode('utf8'))
|
| 348 |
+
return out.rstrip().decode('utf8')
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def get_libraries():
|
| 352 |
+
"""
|
| 353 |
+
Return list of library names to include in the `libraries` argument for C
|
| 354 |
+
or Cython extensions using pyarrow
|
| 355 |
+
"""
|
| 356 |
+
return ['arrow_python', 'arrow']
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def create_library_symlinks():
|
| 360 |
+
"""
|
| 361 |
+
With Linux and macOS wheels, the bundled shared libraries have an embedded
|
| 362 |
+
ABI version like libarrow.so.17 or libarrow.17.dylib and so linking to them
|
| 363 |
+
with -larrow won't work unless we create symlinks at locations like
|
| 364 |
+
site-packages/pyarrow/libarrow.so. This unfortunate workaround addresses
|
| 365 |
+
prior problems we had with shipping two copies of the shared libraries to
|
| 366 |
+
permit third party projects like turbodbc to build their C++ extensions
|
| 367 |
+
against the pyarrow wheels.
|
| 368 |
+
|
| 369 |
+
This function must only be invoked once and only when the shared libraries
|
| 370 |
+
are bundled with the Python package, which should only apply to wheel-based
|
| 371 |
+
installs. It requires write access to the site-packages/pyarrow directory
|
| 372 |
+
and so depending on your system may need to be run with root.
|
| 373 |
+
"""
|
| 374 |
+
import glob
|
| 375 |
+
if _sys.platform == 'win32':
|
| 376 |
+
return
|
| 377 |
+
package_cwd = _os.path.dirname(__file__)
|
| 378 |
+
|
| 379 |
+
if _sys.platform == 'linux':
|
| 380 |
+
bundled_libs = glob.glob(_os.path.join(package_cwd, '*.so.*'))
|
| 381 |
+
|
| 382 |
+
def get_symlink_path(hard_path):
|
| 383 |
+
return hard_path.rsplit('.', 1)[0]
|
| 384 |
+
else:
|
| 385 |
+
bundled_libs = glob.glob(_os.path.join(package_cwd, '*.*.dylib'))
|
| 386 |
+
|
| 387 |
+
def get_symlink_path(hard_path):
|
| 388 |
+
return '.'.join((hard_path.rsplit('.', 2)[0], 'dylib'))
|
| 389 |
+
|
| 390 |
+
for lib_hard_path in bundled_libs:
|
| 391 |
+
symlink_path = get_symlink_path(lib_hard_path)
|
| 392 |
+
if _os.path.exists(symlink_path):
|
| 393 |
+
continue
|
| 394 |
+
try:
|
| 395 |
+
_os.symlink(lib_hard_path, symlink_path)
|
| 396 |
+
except PermissionError:
|
| 397 |
+
print("Tried creating symlink {}. If you need to link to "
|
| 398 |
+
"bundled shared libraries, run "
|
| 399 |
+
"pyarrow.create_library_symlinks() as root")
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def get_library_dirs():
|
| 403 |
+
"""
|
| 404 |
+
Return lists of directories likely to contain Arrow C++ libraries for
|
| 405 |
+
linking C or Cython extensions using pyarrow
|
| 406 |
+
"""
|
| 407 |
+
# Use pyarrow.lib location instead of just __file__. That works
|
| 408 |
+
# for both editable and non-editable builds as it points
|
| 409 |
+
# to the actual location of the compiled C++ extension.
|
| 410 |
+
from pyarrow import lib as _lib
|
| 411 |
+
package_cwd = _os.path.dirname(_lib.__file__)
|
| 412 |
+
library_dirs = [package_cwd]
|
| 413 |
+
|
| 414 |
+
def append_library_dir(library_dir):
|
| 415 |
+
if library_dir not in library_dirs:
|
| 416 |
+
library_dirs.append(library_dir)
|
| 417 |
+
|
| 418 |
+
# Search library paths via pkg-config. This is necessary if the user
|
| 419 |
+
# installed libarrow and the other shared libraries manually and they
|
| 420 |
+
# are not shipped inside the pyarrow package (see also ARROW-2976).
|
| 421 |
+
pkg_config_executable = _os.environ.get('PKG_CONFIG') or 'pkg-config'
|
| 422 |
+
for pkgname in ["arrow", "arrow_python"]:
|
| 423 |
+
if _has_pkg_config(pkgname):
|
| 424 |
+
library_dir = _read_pkg_config_variable(pkgname,
|
| 425 |
+
["--libs-only-L"])
|
| 426 |
+
# pkg-config output could be empty if Arrow is installed
|
| 427 |
+
# as a system package.
|
| 428 |
+
if library_dir:
|
| 429 |
+
if not library_dir.startswith("-L"):
|
| 430 |
+
raise ValueError(
|
| 431 |
+
"pkg-config --libs-only-L returned unexpected "
|
| 432 |
+
f"value {library_dir!r}")
|
| 433 |
+
append_library_dir(library_dir[2:])
|
| 434 |
+
|
| 435 |
+
if _sys.platform == 'win32':
|
| 436 |
+
# TODO(wesm): Is this necessary, or does setuptools within a conda
|
| 437 |
+
# installation add Library\lib to the linker path for MSVC?
|
| 438 |
+
python_base_install = _os.path.dirname(_sys.executable)
|
| 439 |
+
library_dir = _os.path.join(python_base_install, 'Library', 'lib')
|
| 440 |
+
|
| 441 |
+
if _os.path.exists(_os.path.join(library_dir, 'arrow.lib')):
|
| 442 |
+
append_library_dir(library_dir)
|
| 443 |
+
|
| 444 |
+
# GH-45530: Add pyarrow.libs dir containing delvewheel-mangled
|
| 445 |
+
# msvcp140.dll
|
| 446 |
+
pyarrow_libs_dir = _os.path.abspath(
|
| 447 |
+
_os.path.join(_os.path.dirname(__file__), _os.pardir, "pyarrow.libs")
|
| 448 |
+
)
|
| 449 |
+
if _os.path.exists(pyarrow_libs_dir):
|
| 450 |
+
append_library_dir(pyarrow_libs_dir)
|
| 451 |
+
|
| 452 |
+
# ARROW-4074: Allow for ARROW_HOME to be set to some other directory
|
| 453 |
+
if _os.environ.get('ARROW_HOME'):
|
| 454 |
+
append_library_dir(_os.path.join(_os.environ['ARROW_HOME'], 'lib'))
|
| 455 |
+
else:
|
| 456 |
+
# Python wheels bundle the Arrow libraries in the pyarrow directory.
|
| 457 |
+
append_library_dir(_os.path.dirname(_os.path.abspath(__file__)))
|
| 458 |
+
|
| 459 |
+
return library_dirs
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__init__.pyi
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
"""Type stubs for PyArrow.
|
| 19 |
+
|
| 20 |
+
This is a placeholder stub file.
|
| 21 |
+
Complete type annotations will be added in subsequent PRs.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from typing import Any
|
| 25 |
+
|
| 26 |
+
# TODO(GH-48970): remove __getattr__ before release as this
|
| 27 |
+
# will annotate non-existing attributes as Any.
|
| 28 |
+
# https://github.com/apache/arrow/issues/48970
|
| 29 |
+
def __getattr__(name: str) -> Any: ...
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (22.4 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_compute_docstrings.cpython-312.pyc
ADDED
|
Binary file (7.36 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/_generated_version.cpython-312.pyc
ADDED
|
Binary file (803 Bytes). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/compute.cpython-312.pyc
ADDED
|
Binary file (27.2 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/ipc.cpython-312.pyc
ADDED
|
Binary file (10.9 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/types.cpython-312.pyc
ADDED
|
Binary file (18.1 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/__pycache__/util.cpython-312.pyc
ADDED
|
Binary file (11.5 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e99622df3dd58eb8c038d9a1d7af9ee0c33008e8516bceaf5fc15b74291ab43a
|
| 3 |
+
size 183808
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pxd
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.lib cimport *
|
| 21 |
+
from pyarrow.includes.common cimport *
|
| 22 |
+
from pyarrow.includes.libarrow cimport *
|
| 23 |
+
from pyarrow.includes.libarrow_acero cimport *
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
cdef class ExecNodeOptions(_Weakrefable):
|
| 27 |
+
cdef:
|
| 28 |
+
shared_ptr[CExecNodeOptions] wrapped
|
| 29 |
+
|
| 30 |
+
cdef void init(self, const shared_ptr[CExecNodeOptions]& sp)
|
| 31 |
+
cdef inline shared_ptr[CExecNodeOptions] unwrap(self) nogil
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
cdef class Declaration(_Weakrefable):
|
| 35 |
+
|
| 36 |
+
cdef:
|
| 37 |
+
CDeclaration decl
|
| 38 |
+
|
| 39 |
+
cdef void init(self, const CDeclaration& c_decl)
|
| 40 |
+
|
| 41 |
+
@staticmethod
|
| 42 |
+
cdef wrap(const CDeclaration& c_decl)
|
| 43 |
+
|
| 44 |
+
cdef inline CDeclaration unwrap(self) nogil
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_acero.pyx
ADDED
|
@@ -0,0 +1,610 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# ---------------------------------------------------------------------
|
| 19 |
+
# Low-level Acero bindings
|
| 20 |
+
|
| 21 |
+
# cython: profile=False
|
| 22 |
+
# distutils: language = c++
|
| 23 |
+
# cython: language_level = 3
|
| 24 |
+
|
| 25 |
+
from pyarrow.includes.common cimport *
|
| 26 |
+
from pyarrow.includes.libarrow cimport *
|
| 27 |
+
from pyarrow.includes.libarrow_acero cimport *
|
| 28 |
+
from pyarrow.lib cimport (Table, pyarrow_unwrap_table, pyarrow_wrap_table,
|
| 29 |
+
RecordBatchReader)
|
| 30 |
+
from pyarrow.lib import frombytes, tobytes
|
| 31 |
+
from pyarrow._compute cimport (
|
| 32 |
+
Expression, FunctionOptions, _ensure_field_ref, _true,
|
| 33 |
+
unwrap_null_placement, unwrap_sort_keys
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
cdef class ExecNodeOptions(_Weakrefable):
|
| 38 |
+
"""
|
| 39 |
+
Base class for the node options.
|
| 40 |
+
|
| 41 |
+
Use one of the subclasses to construct an options object.
|
| 42 |
+
"""
|
| 43 |
+
__slots__ = () # avoid mistakingly creating attributes
|
| 44 |
+
|
| 45 |
+
cdef void init(self, const shared_ptr[CExecNodeOptions]& sp):
|
| 46 |
+
self.wrapped = sp
|
| 47 |
+
|
| 48 |
+
cdef inline shared_ptr[CExecNodeOptions] unwrap(self) nogil:
|
| 49 |
+
return self.wrapped
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
cdef class _TableSourceNodeOptions(ExecNodeOptions):
|
| 53 |
+
|
| 54 |
+
def _set_options(self, Table table):
|
| 55 |
+
cdef:
|
| 56 |
+
shared_ptr[CTable] c_table
|
| 57 |
+
|
| 58 |
+
c_table = pyarrow_unwrap_table(table)
|
| 59 |
+
self.wrapped.reset(
|
| 60 |
+
new CTableSourceNodeOptions(c_table)
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class TableSourceNodeOptions(_TableSourceNodeOptions):
|
| 65 |
+
"""
|
| 66 |
+
A Source node which accepts a table.
|
| 67 |
+
|
| 68 |
+
This is the option class for the "table_source" node factory.
|
| 69 |
+
|
| 70 |
+
Parameters
|
| 71 |
+
----------
|
| 72 |
+
table : pyarrow.Table
|
| 73 |
+
The table which acts as the data source.
|
| 74 |
+
"""
|
| 75 |
+
|
| 76 |
+
def __init__(self, Table table):
|
| 77 |
+
self._set_options(table)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
cdef class _FilterNodeOptions(ExecNodeOptions):
|
| 81 |
+
|
| 82 |
+
def _set_options(self, Expression filter_expression not None):
|
| 83 |
+
self.wrapped.reset(
|
| 84 |
+
new CFilterNodeOptions(<CExpression>filter_expression.unwrap())
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class FilterNodeOptions(_FilterNodeOptions):
|
| 89 |
+
"""
|
| 90 |
+
Make a node which excludes some rows from batches passed through it.
|
| 91 |
+
|
| 92 |
+
This is the option class for the "filter" node factory.
|
| 93 |
+
|
| 94 |
+
The "filter" operation provides an option to define data filtering
|
| 95 |
+
criteria. It selects rows where the given expression evaluates to true.
|
| 96 |
+
Filters can be written using pyarrow.compute.Expression, and the
|
| 97 |
+
expression must have a return type of boolean.
|
| 98 |
+
|
| 99 |
+
Parameters
|
| 100 |
+
----------
|
| 101 |
+
filter_expression : pyarrow.compute.Expression
|
| 102 |
+
"""
|
| 103 |
+
|
| 104 |
+
def __init__(self, Expression filter_expression):
|
| 105 |
+
self._set_options(filter_expression)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
cdef class _ProjectNodeOptions(ExecNodeOptions):
|
| 109 |
+
|
| 110 |
+
def _set_options(self, expressions, names=None):
|
| 111 |
+
cdef:
|
| 112 |
+
Expression expr
|
| 113 |
+
vector[CExpression] c_expressions
|
| 114 |
+
vector[c_string] c_names
|
| 115 |
+
|
| 116 |
+
for expr in expressions:
|
| 117 |
+
c_expressions.push_back(expr.unwrap())
|
| 118 |
+
|
| 119 |
+
if names is not None:
|
| 120 |
+
if len(names) != len(expressions):
|
| 121 |
+
raise ValueError(
|
| 122 |
+
"The number of names should be equal to the number of expressions"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
for name in names:
|
| 126 |
+
c_names.push_back(<c_string>tobytes(name))
|
| 127 |
+
|
| 128 |
+
self.wrapped.reset(
|
| 129 |
+
new CProjectNodeOptions(c_expressions, c_names)
|
| 130 |
+
)
|
| 131 |
+
else:
|
| 132 |
+
self.wrapped.reset(
|
| 133 |
+
new CProjectNodeOptions(c_expressions)
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
class ProjectNodeOptions(_ProjectNodeOptions):
|
| 138 |
+
"""
|
| 139 |
+
Make a node which executes expressions on input batches,
|
| 140 |
+
producing batches of the same length with new columns.
|
| 141 |
+
|
| 142 |
+
This is the option class for the "project" node factory.
|
| 143 |
+
|
| 144 |
+
The "project" operation rearranges, deletes, transforms, and
|
| 145 |
+
creates columns. Each output column is computed by evaluating
|
| 146 |
+
an expression against the source record batch. These must be
|
| 147 |
+
scalar expressions (expressions consisting of scalar literals,
|
| 148 |
+
field references and scalar functions, i.e. elementwise functions
|
| 149 |
+
that return one value for each input row independent of the value
|
| 150 |
+
of all other rows).
|
| 151 |
+
|
| 152 |
+
Parameters
|
| 153 |
+
----------
|
| 154 |
+
expressions : list of pyarrow.compute.Expression
|
| 155 |
+
List of expressions to evaluate against the source batch. This must
|
| 156 |
+
be scalar expressions.
|
| 157 |
+
names : list of str, optional
|
| 158 |
+
List of names for each of the output columns (same length as
|
| 159 |
+
`expressions`). If `names` is not provided, the string
|
| 160 |
+
representations of exprs will be used.
|
| 161 |
+
"""
|
| 162 |
+
|
| 163 |
+
def __init__(self, expressions, names=None):
|
| 164 |
+
self._set_options(expressions, names)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
cdef class _AggregateNodeOptions(ExecNodeOptions):
|
| 168 |
+
|
| 169 |
+
def _set_options(self, aggregates, keys=None):
|
| 170 |
+
cdef:
|
| 171 |
+
CAggregate c_aggr
|
| 172 |
+
vector[CAggregate] c_aggregations
|
| 173 |
+
vector[CFieldRef] c_keys
|
| 174 |
+
|
| 175 |
+
for arg_names, func_name, opts, name in aggregates:
|
| 176 |
+
c_aggr.function = tobytes(func_name)
|
| 177 |
+
if opts is not None:
|
| 178 |
+
c_aggr.options = (<FunctionOptions?>opts).wrapped
|
| 179 |
+
else:
|
| 180 |
+
c_aggr.options = <shared_ptr[CFunctionOptions]>nullptr
|
| 181 |
+
if not isinstance(arg_names, (list, tuple)):
|
| 182 |
+
arg_names = [arg_names]
|
| 183 |
+
for arg in arg_names:
|
| 184 |
+
c_aggr.target.push_back(_ensure_field_ref(arg))
|
| 185 |
+
c_aggr.name = tobytes(name)
|
| 186 |
+
|
| 187 |
+
c_aggregations.push_back(move(c_aggr))
|
| 188 |
+
|
| 189 |
+
if keys is None:
|
| 190 |
+
keys = []
|
| 191 |
+
for name in keys:
|
| 192 |
+
c_keys.push_back(_ensure_field_ref(name))
|
| 193 |
+
|
| 194 |
+
self.wrapped.reset(
|
| 195 |
+
new CAggregateNodeOptions(c_aggregations, c_keys)
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
class AggregateNodeOptions(_AggregateNodeOptions):
|
| 200 |
+
"""
|
| 201 |
+
Make a node which aggregates input batches, optionally grouped by keys.
|
| 202 |
+
|
| 203 |
+
This is the option class for the "aggregate" node factory.
|
| 204 |
+
|
| 205 |
+
Acero supports two types of aggregates: "scalar" aggregates,
|
| 206 |
+
and "hash" aggregates. Scalar aggregates reduce an array or scalar
|
| 207 |
+
input to a single scalar output (e.g. computing the mean of a column).
|
| 208 |
+
Hash aggregates act like GROUP BY in SQL and first partition data
|
| 209 |
+
based on one or more key columns, then reduce the data in each partition.
|
| 210 |
+
The aggregate node supports both types of computation, and can compute
|
| 211 |
+
any number of aggregations at once.
|
| 212 |
+
|
| 213 |
+
Parameters
|
| 214 |
+
----------
|
| 215 |
+
aggregates : list of tuples
|
| 216 |
+
Aggregations which will be applied to the targeted fields.
|
| 217 |
+
Specified as a list of tuples, where each tuple is one aggregation
|
| 218 |
+
specification and consists of: aggregation target column(s) followed
|
| 219 |
+
by function name, aggregation function options object and the
|
| 220 |
+
output field name.
|
| 221 |
+
The target column(s) specification can be a single field reference,
|
| 222 |
+
an empty list or a list of fields unary, nullary and n-ary aggregation
|
| 223 |
+
functions respectively. Each field reference can be a string
|
| 224 |
+
column name or expression.
|
| 225 |
+
keys : list of field references, optional
|
| 226 |
+
Keys by which aggregations will be grouped. Each key can reference
|
| 227 |
+
a field using a string name or expression.
|
| 228 |
+
"""
|
| 229 |
+
|
| 230 |
+
def __init__(self, aggregates, keys=None):
|
| 231 |
+
self._set_options(aggregates, keys)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
cdef class _OrderByNodeOptions(ExecNodeOptions):
|
| 235 |
+
|
| 236 |
+
def _set_options(self, sort_keys, null_placement):
|
| 237 |
+
self.wrapped.reset(
|
| 238 |
+
new COrderByNodeOptions(
|
| 239 |
+
COrdering(unwrap_sort_keys(sort_keys, allow_str=False),
|
| 240 |
+
unwrap_null_placement(null_placement))
|
| 241 |
+
)
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
class OrderByNodeOptions(_OrderByNodeOptions):
|
| 246 |
+
"""
|
| 247 |
+
Make a node which applies a new ordering to the data.
|
| 248 |
+
|
| 249 |
+
Currently this node works by accumulating all data, sorting, and then
|
| 250 |
+
emitting the new data with an updated batch index.
|
| 251 |
+
Larger-than-memory sort is not currently supported.
|
| 252 |
+
|
| 253 |
+
This is the option class for the "order_by" node factory.
|
| 254 |
+
|
| 255 |
+
Parameters
|
| 256 |
+
----------
|
| 257 |
+
sort_keys : sequence of (name, order) tuples
|
| 258 |
+
Names of field/column keys to sort the input on,
|
| 259 |
+
along with the order each field/column is sorted in.
|
| 260 |
+
Accepted values for `order` are "ascending", "descending".
|
| 261 |
+
Each field reference can be a string column name or expression.
|
| 262 |
+
null_placement : str, default "at_end"
|
| 263 |
+
Where nulls in input should be sorted, only applying to
|
| 264 |
+
columns/fields mentioned in `sort_keys`.
|
| 265 |
+
Accepted values are "at_start", "at_end".
|
| 266 |
+
"""
|
| 267 |
+
|
| 268 |
+
def __init__(self, sort_keys=(), *, null_placement="at_end"):
|
| 269 |
+
self._set_options(sort_keys, null_placement)
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
cdef class _HashJoinNodeOptions(ExecNodeOptions):
|
| 273 |
+
|
| 274 |
+
def _set_options(
|
| 275 |
+
self, join_type, left_keys, right_keys, left_output=None, right_output=None,
|
| 276 |
+
output_suffix_for_left="", output_suffix_for_right="", Expression filter_expression=None,
|
| 277 |
+
):
|
| 278 |
+
cdef:
|
| 279 |
+
CJoinType c_join_type
|
| 280 |
+
vector[CFieldRef] c_left_keys
|
| 281 |
+
vector[CFieldRef] c_right_keys
|
| 282 |
+
vector[CFieldRef] c_left_output
|
| 283 |
+
vector[CFieldRef] c_right_output
|
| 284 |
+
CExpression c_filter_expression
|
| 285 |
+
|
| 286 |
+
# join type
|
| 287 |
+
if join_type == "left semi":
|
| 288 |
+
c_join_type = CJoinType_LEFT_SEMI
|
| 289 |
+
elif join_type == "right semi":
|
| 290 |
+
c_join_type = CJoinType_RIGHT_SEMI
|
| 291 |
+
elif join_type == "left anti":
|
| 292 |
+
c_join_type = CJoinType_LEFT_ANTI
|
| 293 |
+
elif join_type == "right anti":
|
| 294 |
+
c_join_type = CJoinType_RIGHT_ANTI
|
| 295 |
+
elif join_type == "inner":
|
| 296 |
+
c_join_type = CJoinType_INNER
|
| 297 |
+
elif join_type == "left outer":
|
| 298 |
+
c_join_type = CJoinType_LEFT_OUTER
|
| 299 |
+
elif join_type == "right outer":
|
| 300 |
+
c_join_type = CJoinType_RIGHT_OUTER
|
| 301 |
+
elif join_type == "full outer":
|
| 302 |
+
c_join_type = CJoinType_FULL_OUTER
|
| 303 |
+
else:
|
| 304 |
+
raise ValueError("Unsupported join type")
|
| 305 |
+
|
| 306 |
+
# left/right keys
|
| 307 |
+
if not isinstance(left_keys, (list, tuple)):
|
| 308 |
+
left_keys = [left_keys]
|
| 309 |
+
for key in left_keys:
|
| 310 |
+
c_left_keys.push_back(_ensure_field_ref(key))
|
| 311 |
+
if not isinstance(right_keys, (list, tuple)):
|
| 312 |
+
right_keys = [right_keys]
|
| 313 |
+
for key in right_keys:
|
| 314 |
+
c_right_keys.push_back(_ensure_field_ref(key))
|
| 315 |
+
|
| 316 |
+
if filter_expression is None:
|
| 317 |
+
c_filter_expression = _true
|
| 318 |
+
else:
|
| 319 |
+
c_filter_expression = filter_expression.unwrap()
|
| 320 |
+
|
| 321 |
+
# left/right output fields
|
| 322 |
+
if left_output is not None and right_output is not None:
|
| 323 |
+
for colname in left_output:
|
| 324 |
+
c_left_output.push_back(_ensure_field_ref(colname))
|
| 325 |
+
for colname in right_output:
|
| 326 |
+
c_right_output.push_back(_ensure_field_ref(colname))
|
| 327 |
+
|
| 328 |
+
self.wrapped.reset(
|
| 329 |
+
new CHashJoinNodeOptions(
|
| 330 |
+
c_join_type, c_left_keys, c_right_keys,
|
| 331 |
+
c_left_output, c_right_output,
|
| 332 |
+
c_filter_expression,
|
| 333 |
+
<c_string>tobytes(output_suffix_for_left),
|
| 334 |
+
<c_string>tobytes(output_suffix_for_right)
|
| 335 |
+
)
|
| 336 |
+
)
|
| 337 |
+
else:
|
| 338 |
+
self.wrapped.reset(
|
| 339 |
+
new CHashJoinNodeOptions(
|
| 340 |
+
c_join_type, c_left_keys, c_right_keys,
|
| 341 |
+
c_filter_expression,
|
| 342 |
+
<c_string>tobytes(output_suffix_for_left),
|
| 343 |
+
<c_string>tobytes(output_suffix_for_right)
|
| 344 |
+
)
|
| 345 |
+
)
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
class HashJoinNodeOptions(_HashJoinNodeOptions):
|
| 349 |
+
"""
|
| 350 |
+
Make a node which implements join operation using hash join strategy.
|
| 351 |
+
|
| 352 |
+
This is the option class for the "hashjoin" node factory.
|
| 353 |
+
|
| 354 |
+
Parameters
|
| 355 |
+
----------
|
| 356 |
+
join_type : str
|
| 357 |
+
Type of join. One of "left semi", "right semi", "left anti",
|
| 358 |
+
"right anti", "inner", "left outer", "right outer", "full outer".
|
| 359 |
+
left_keys : str, Expression or list
|
| 360 |
+
Key fields from left input. Each key can be a string column name
|
| 361 |
+
or a field expression, or a list of such field references.
|
| 362 |
+
right_keys : str, Expression or list
|
| 363 |
+
Key fields from right input. See `left_keys` for details.
|
| 364 |
+
left_output : list, optional
|
| 365 |
+
List of output fields passed from left input. If left and right
|
| 366 |
+
output fields are not specified, all valid fields from both left and
|
| 367 |
+
right input will be output. Each field can be a string column name
|
| 368 |
+
or a field expression.
|
| 369 |
+
right_output : list, optional
|
| 370 |
+
List of output fields passed from right input. If left and right
|
| 371 |
+
output fields are not specified, all valid fields from both left and
|
| 372 |
+
right input will be output. Each field can be a string column name
|
| 373 |
+
or a field expression.
|
| 374 |
+
output_suffix_for_left : str
|
| 375 |
+
Suffix added to names of output fields coming from left input
|
| 376 |
+
(used to distinguish, if necessary, between fields of the same
|
| 377 |
+
name in left and right input and can be left empty if there are
|
| 378 |
+
no name collisions).
|
| 379 |
+
output_suffix_for_right : str
|
| 380 |
+
Suffix added to names of output fields coming from right input,
|
| 381 |
+
see `output_suffix_for_left` for details.
|
| 382 |
+
filter_expression : pyarrow.compute.Expression
|
| 383 |
+
Residual filter which is applied to matching row.
|
| 384 |
+
"""
|
| 385 |
+
|
| 386 |
+
def __init__(
|
| 387 |
+
self, join_type, left_keys, right_keys, left_output=None, right_output=None,
|
| 388 |
+
output_suffix_for_left="", output_suffix_for_right="", filter_expression=None,
|
| 389 |
+
):
|
| 390 |
+
self._set_options(
|
| 391 |
+
join_type, left_keys, right_keys, left_output, right_output,
|
| 392 |
+
output_suffix_for_left, output_suffix_for_right, filter_expression
|
| 393 |
+
)
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
cdef class _AsofJoinNodeOptions(ExecNodeOptions):
|
| 397 |
+
|
| 398 |
+
def _set_options(self, left_on, left_by, right_on, right_by, tolerance):
|
| 399 |
+
cdef:
|
| 400 |
+
vector[CFieldRef] c_left_by
|
| 401 |
+
vector[CFieldRef] c_right_by
|
| 402 |
+
CAsofJoinKeys c_left_keys
|
| 403 |
+
CAsofJoinKeys c_right_keys
|
| 404 |
+
vector[CAsofJoinKeys] c_input_keys
|
| 405 |
+
|
| 406 |
+
# Prepare left AsofJoinNodeOption::Keys
|
| 407 |
+
if not isinstance(left_by, (list, tuple)):
|
| 408 |
+
left_by = [left_by]
|
| 409 |
+
for key in left_by:
|
| 410 |
+
c_left_by.push_back(_ensure_field_ref(key))
|
| 411 |
+
|
| 412 |
+
c_left_keys.on_key = _ensure_field_ref(left_on)
|
| 413 |
+
c_left_keys.by_key = c_left_by
|
| 414 |
+
|
| 415 |
+
c_input_keys.push_back(c_left_keys)
|
| 416 |
+
|
| 417 |
+
# Prepare right AsofJoinNodeOption::Keys
|
| 418 |
+
if not isinstance(right_by, (list, tuple)):
|
| 419 |
+
right_by = [right_by]
|
| 420 |
+
for key in right_by:
|
| 421 |
+
c_right_by.push_back(_ensure_field_ref(key))
|
| 422 |
+
|
| 423 |
+
c_right_keys.on_key = _ensure_field_ref(right_on)
|
| 424 |
+
c_right_keys.by_key = c_right_by
|
| 425 |
+
|
| 426 |
+
c_input_keys.push_back(c_right_keys)
|
| 427 |
+
|
| 428 |
+
self.wrapped.reset(
|
| 429 |
+
new CAsofJoinNodeOptions(
|
| 430 |
+
c_input_keys,
|
| 431 |
+
tolerance,
|
| 432 |
+
)
|
| 433 |
+
)
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
class AsofJoinNodeOptions(_AsofJoinNodeOptions):
|
| 437 |
+
"""
|
| 438 |
+
Make a node which implements 'as of join' operation.
|
| 439 |
+
|
| 440 |
+
This is the option class for the "asofjoin" node factory.
|
| 441 |
+
|
| 442 |
+
Parameters
|
| 443 |
+
----------
|
| 444 |
+
left_on : str, Expression
|
| 445 |
+
The left key on which the join operation should be performed.
|
| 446 |
+
Can be a string column name or a field expression.
|
| 447 |
+
|
| 448 |
+
An inexact match is used on the "on" key, i.e. a row is considered a
|
| 449 |
+
match if and only if ``right.on - left.on`` is in the range
|
| 450 |
+
``[min(0, tolerance), max(0, tolerance)]``.
|
| 451 |
+
|
| 452 |
+
The input dataset must be sorted by the "on" key. Must be a single
|
| 453 |
+
field of a common type.
|
| 454 |
+
|
| 455 |
+
Currently, the "on" key must be an integer, date, or timestamp type.
|
| 456 |
+
left_by: str, Expression or list
|
| 457 |
+
The left keys on which the join operation should be performed.
|
| 458 |
+
Exact equality is used for each field of the "by" keys.
|
| 459 |
+
Each key can be a string column name or a field expression,
|
| 460 |
+
or a list of such field references.
|
| 461 |
+
right_on : str, Expression
|
| 462 |
+
The right key on which the join operation should be performed.
|
| 463 |
+
See `left_on` for details.
|
| 464 |
+
right_by: str, Expression or list
|
| 465 |
+
The right keys on which the join operation should be performed.
|
| 466 |
+
See `left_by` for details.
|
| 467 |
+
tolerance : int
|
| 468 |
+
The tolerance to use for the asof join. The tolerance is interpreted in
|
| 469 |
+
the same units as the "on" key.
|
| 470 |
+
"""
|
| 471 |
+
|
| 472 |
+
def __init__(self, left_on, left_by, right_on, right_by, tolerance):
|
| 473 |
+
self._set_options(left_on, left_by, right_on, right_by, tolerance)
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
cdef class Declaration(_Weakrefable):
|
| 477 |
+
"""
|
| 478 |
+
Helper class for declaring the nodes of an ExecPlan.
|
| 479 |
+
|
| 480 |
+
A Declaration represents an unconstructed ExecNode, and potentially
|
| 481 |
+
more since its inputs may also be Declarations or when constructed
|
| 482 |
+
with ``from_sequence``.
|
| 483 |
+
|
| 484 |
+
The possible ExecNodes to use are registered with a name,
|
| 485 |
+
the "factory name", and need to be specified using this name, together
|
| 486 |
+
with its corresponding ExecNodeOptions subclass.
|
| 487 |
+
|
| 488 |
+
Parameters
|
| 489 |
+
----------
|
| 490 |
+
factory_name : str
|
| 491 |
+
The ExecNode factory name, such as "table_source", "filter",
|
| 492 |
+
"project" etc. See the ExecNodeOptions subclasses for the exact
|
| 493 |
+
factory names to use.
|
| 494 |
+
options : ExecNodeOptions
|
| 495 |
+
Corresponding ExecNodeOptions subclass (matching the factory name).
|
| 496 |
+
inputs : list of Declaration, optional
|
| 497 |
+
Input nodes for this declaration. Optional if the node is a source
|
| 498 |
+
node, or when the declaration gets combined later with
|
| 499 |
+
``from_sequence``.
|
| 500 |
+
|
| 501 |
+
Returns
|
| 502 |
+
-------
|
| 503 |
+
Declaration
|
| 504 |
+
"""
|
| 505 |
+
cdef void init(self, const CDeclaration& c_decl):
|
| 506 |
+
self.decl = c_decl
|
| 507 |
+
|
| 508 |
+
@staticmethod
|
| 509 |
+
cdef wrap(const CDeclaration& c_decl):
|
| 510 |
+
cdef Declaration self = Declaration.__new__(Declaration)
|
| 511 |
+
self.init(c_decl)
|
| 512 |
+
return self
|
| 513 |
+
|
| 514 |
+
cdef inline CDeclaration unwrap(self) nogil:
|
| 515 |
+
return self.decl
|
| 516 |
+
|
| 517 |
+
def __init__(self, factory_name, ExecNodeOptions options, inputs=None):
|
| 518 |
+
cdef:
|
| 519 |
+
c_string c_factory_name
|
| 520 |
+
CDeclaration c_decl
|
| 521 |
+
vector[CDeclaration.Input] c_inputs
|
| 522 |
+
|
| 523 |
+
c_factory_name = tobytes(factory_name)
|
| 524 |
+
|
| 525 |
+
if inputs is not None:
|
| 526 |
+
for ipt in inputs:
|
| 527 |
+
c_inputs.push_back(
|
| 528 |
+
CDeclaration.Input((<Declaration>ipt).unwrap())
|
| 529 |
+
)
|
| 530 |
+
|
| 531 |
+
c_decl = CDeclaration(c_factory_name, c_inputs, options.unwrap())
|
| 532 |
+
self.init(c_decl)
|
| 533 |
+
|
| 534 |
+
@staticmethod
|
| 535 |
+
def from_sequence(decls):
|
| 536 |
+
"""
|
| 537 |
+
Convenience factory for the common case of a simple sequence of nodes.
|
| 538 |
+
|
| 539 |
+
Each of the declarations will be appended to the inputs of the
|
| 540 |
+
subsequent declaration, and the final modified declaration will
|
| 541 |
+
be returned.
|
| 542 |
+
|
| 543 |
+
Parameters
|
| 544 |
+
----------
|
| 545 |
+
decls : list of Declaration
|
| 546 |
+
|
| 547 |
+
Returns
|
| 548 |
+
-------
|
| 549 |
+
Declaration
|
| 550 |
+
"""
|
| 551 |
+
cdef:
|
| 552 |
+
vector[CDeclaration] c_decls
|
| 553 |
+
CDeclaration c_decl
|
| 554 |
+
|
| 555 |
+
for decl in decls:
|
| 556 |
+
c_decls.push_back((<Declaration> decl).unwrap())
|
| 557 |
+
|
| 558 |
+
c_decl = CDeclaration.Sequence(c_decls)
|
| 559 |
+
return Declaration.wrap(c_decl)
|
| 560 |
+
|
| 561 |
+
def __str__(self):
|
| 562 |
+
return frombytes(GetResultValue(DeclarationToString(self.decl)))
|
| 563 |
+
|
| 564 |
+
def __repr__(self):
|
| 565 |
+
return f"<pyarrow.acero.Declaration>\n{self}"
|
| 566 |
+
|
| 567 |
+
def to_table(self, bint use_threads=True):
|
| 568 |
+
"""
|
| 569 |
+
Run the declaration and collect the results into a table.
|
| 570 |
+
|
| 571 |
+
This method will implicitly add a sink node to the declaration
|
| 572 |
+
to collect results into a table. It will then create an ExecPlan
|
| 573 |
+
from the declaration, start the exec plan, block until the plan
|
| 574 |
+
has finished, and return the created table.
|
| 575 |
+
|
| 576 |
+
Parameters
|
| 577 |
+
----------
|
| 578 |
+
use_threads : bool, default True
|
| 579 |
+
If set to False, then all CPU work will be done on the calling
|
| 580 |
+
thread. I/O tasks will still happen on the I/O executor
|
| 581 |
+
and may be multi-threaded (but should not use significant CPU
|
| 582 |
+
resources).
|
| 583 |
+
|
| 584 |
+
Returns
|
| 585 |
+
-------
|
| 586 |
+
pyarrow.Table
|
| 587 |
+
"""
|
| 588 |
+
cdef:
|
| 589 |
+
shared_ptr[CTable] c_table
|
| 590 |
+
|
| 591 |
+
with nogil:
|
| 592 |
+
c_table = GetResultValue(DeclarationToTable(self.unwrap(), use_threads))
|
| 593 |
+
return pyarrow_wrap_table(c_table)
|
| 594 |
+
|
| 595 |
+
def to_reader(self, bint use_threads=True):
|
| 596 |
+
"""Run the declaration and return results as a RecordBatchReader.
|
| 597 |
+
|
| 598 |
+
For details about the parameters, see `to_table`.
|
| 599 |
+
|
| 600 |
+
Returns
|
| 601 |
+
-------
|
| 602 |
+
pyarrow.RecordBatchReader
|
| 603 |
+
"""
|
| 604 |
+
cdef:
|
| 605 |
+
RecordBatchReader reader
|
| 606 |
+
reader = RecordBatchReader.__new__(RecordBatchReader)
|
| 607 |
+
reader.reader.reset(
|
| 608 |
+
GetResultValue(DeclarationToReader(self.unwrap(), use_threads)).release()
|
| 609 |
+
)
|
| 610 |
+
return reader
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.cp312-win_amd64.pyd
ADDED
|
Binary file (91.6 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_azurefs.pyx
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
from pyarrow.lib import frombytes, tobytes
|
| 22 |
+
from pyarrow.includes.libarrow_fs cimport *
|
| 23 |
+
from pyarrow._fs cimport FileSystem
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
cdef class AzureFileSystem(FileSystem):
|
| 27 |
+
"""
|
| 28 |
+
Azure Blob Storage backed FileSystem implementation
|
| 29 |
+
|
| 30 |
+
This implementation supports flat namespace and hierarchical namespace (HNS) a.k.a.
|
| 31 |
+
Data Lake Gen2 storage accounts. HNS will be automatically detected and HNS specific
|
| 32 |
+
features will be used when they provide a performance advantage. Azurite emulator is
|
| 33 |
+
also supported. Note: `/` is the only supported delimiter.
|
| 34 |
+
|
| 35 |
+
The storage account is considered the root of the filesystem. When enabled, containers
|
| 36 |
+
will be created or deleted during relevant directory operations. Obviously, this also
|
| 37 |
+
requires authentication with the additional permissions.
|
| 38 |
+
|
| 39 |
+
By default `DefaultAzureCredential <https://github.com/Azure/azure-sdk-for-cpp/blob/main/sdk/identity/azure-identity/README.md#defaultazurecredential>`__
|
| 40 |
+
is used for authentication. This means it will try several types of authentication
|
| 41 |
+
and go with the first one that works. If any authentication parameters are provided when
|
| 42 |
+
initialising the FileSystem, they will be used instead of the default credential.
|
| 43 |
+
|
| 44 |
+
Parameters
|
| 45 |
+
----------
|
| 46 |
+
account_name : str
|
| 47 |
+
Azure Blob Storage account name. This is the globally unique identifier for the
|
| 48 |
+
storage account.
|
| 49 |
+
account_key : str, default None
|
| 50 |
+
Account key of the storage account. If sas_token and account_key are None the
|
| 51 |
+
default credential will be used. The parameters account_key and sas_token are
|
| 52 |
+
mutually exclusive.
|
| 53 |
+
blob_storage_authority : str, default None
|
| 54 |
+
hostname[:port] of the Blob Service. Defaults to `.blob.core.windows.net`. Useful
|
| 55 |
+
for connecting to a local emulator, like Azurite.
|
| 56 |
+
blob_storage_scheme : str, default None
|
| 57 |
+
Either `http` or `https`. Defaults to `https`. Useful for connecting to a local
|
| 58 |
+
emulator, like Azurite.
|
| 59 |
+
client_id : str, default None
|
| 60 |
+
The client ID (Application ID) for Azure Active Directory authentication.
|
| 61 |
+
Its interpretation depends on the credential type being used:
|
| 62 |
+
|
| 63 |
+
- For `ClientSecretCredential`: It is the Application (client) ID of your
|
| 64 |
+
registered Azure AD application (Service Principal). It must be provided
|
| 65 |
+
together with `tenant_id` and `client_secret` to use ClientSecretCredential.
|
| 66 |
+
- For `ManagedIdentityCredential`: It is the client ID of a specific
|
| 67 |
+
user-assigned managed identity. This is only necessary if you are using a
|
| 68 |
+
user-assigned managed identity and need to explicitly specify which one
|
| 69 |
+
(e.g., if the resource has multiple user-assigned identities). For
|
| 70 |
+
system-assigned managed identities, this parameter is typically not required.
|
| 71 |
+
|
| 72 |
+
client_secret : str, default None
|
| 73 |
+
Client secret for Azure Active Directory authentication. Must be provided together
|
| 74 |
+
with `tenant_id` and `client_id` to use ClientSecretCredential.
|
| 75 |
+
dfs_storage_authority : str, default None
|
| 76 |
+
hostname[:port] of the Data Lake Gen 2 Service. Defaults to
|
| 77 |
+
`.dfs.core.windows.net`. Useful for connecting to a local emulator, like Azurite.
|
| 78 |
+
dfs_storage_scheme : str, default None
|
| 79 |
+
Either `http` or `https`. Defaults to `https`. Useful for connecting to a local
|
| 80 |
+
emulator, like Azurite.
|
| 81 |
+
sas_token : str, default None
|
| 82 |
+
SAS token for the storage account, used as an alternative to account_key. If sas_token
|
| 83 |
+
and account_key are None the default credential will be used. The parameters
|
| 84 |
+
account_key and sas_token are mutually exclusive.
|
| 85 |
+
tenant_id : str, default None
|
| 86 |
+
Tenant ID for Azure Active Directory authentication. Must be provided together with
|
| 87 |
+
`client_id` and `client_secret` to use ClientSecretCredential.
|
| 88 |
+
|
| 89 |
+
Examples
|
| 90 |
+
--------
|
| 91 |
+
>>> from pyarrow import fs
|
| 92 |
+
>>> azure_fs = fs.AzureFileSystem(account_name='myaccount')
|
| 93 |
+
>>> azurite_fs = fs.AzureFileSystem(
|
| 94 |
+
... account_name='devstoreaccount1',
|
| 95 |
+
... account_key='Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
|
| 96 |
+
... blob_storage_authority='127.0.0.1:10000',
|
| 97 |
+
... dfs_storage_authority='127.0.0.1:10000',
|
| 98 |
+
... blob_storage_scheme='http',
|
| 99 |
+
... dfs_storage_scheme='http',
|
| 100 |
+
... )
|
| 101 |
+
|
| 102 |
+
For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
|
| 103 |
+
"""
|
| 104 |
+
cdef:
|
| 105 |
+
CAzureFileSystem* azurefs
|
| 106 |
+
|
| 107 |
+
def __init__(self, account_name, *, account_key=None, blob_storage_authority=None,
|
| 108 |
+
blob_storage_scheme=None, client_id=None, client_secret=None,
|
| 109 |
+
dfs_storage_authority=None, dfs_storage_scheme=None,
|
| 110 |
+
sas_token=None, tenant_id=None):
|
| 111 |
+
cdef:
|
| 112 |
+
CAzureOptions options
|
| 113 |
+
shared_ptr[CAzureFileSystem] wrapped
|
| 114 |
+
|
| 115 |
+
options.account_name = tobytes(account_name)
|
| 116 |
+
if blob_storage_authority:
|
| 117 |
+
options.blob_storage_authority = tobytes(blob_storage_authority)
|
| 118 |
+
if dfs_storage_authority:
|
| 119 |
+
options.dfs_storage_authority = tobytes(dfs_storage_authority)
|
| 120 |
+
if blob_storage_scheme:
|
| 121 |
+
options.blob_storage_scheme = tobytes(blob_storage_scheme)
|
| 122 |
+
if dfs_storage_scheme:
|
| 123 |
+
options.dfs_storage_scheme = tobytes(dfs_storage_scheme)
|
| 124 |
+
|
| 125 |
+
if account_key and sas_token:
|
| 126 |
+
raise ValueError("Cannot specify both account_key and sas_token.")
|
| 127 |
+
|
| 128 |
+
if (tenant_id or client_id or client_secret):
|
| 129 |
+
if not client_id:
|
| 130 |
+
raise ValueError("client_id must be specified")
|
| 131 |
+
if not tenant_id and not client_secret:
|
| 132 |
+
options.ConfigureManagedIdentityCredential(tobytes(client_id))
|
| 133 |
+
elif tenant_id and client_secret:
|
| 134 |
+
options.ConfigureClientSecretCredential(
|
| 135 |
+
tobytes(tenant_id), tobytes(client_id), tobytes(client_secret)
|
| 136 |
+
)
|
| 137 |
+
else:
|
| 138 |
+
raise ValueError(
|
| 139 |
+
"Invalid Azure credential configuration: "
|
| 140 |
+
"For ManagedIdentityCredential, provide only client_id. "
|
| 141 |
+
"For ClientSecretCredential, provide tenant_id, client_id, and client_secret."
|
| 142 |
+
)
|
| 143 |
+
elif account_key:
|
| 144 |
+
options.ConfigureAccountKeyCredential(tobytes(account_key))
|
| 145 |
+
elif sas_token:
|
| 146 |
+
options.ConfigureSASCredential(tobytes(sas_token))
|
| 147 |
+
else:
|
| 148 |
+
options.ConfigureDefaultCredential()
|
| 149 |
+
|
| 150 |
+
with nogil:
|
| 151 |
+
wrapped = GetResultValue(CAzureFileSystem.Make(options))
|
| 152 |
+
|
| 153 |
+
self.init(<shared_ptr[CFileSystem]> wrapped)
|
| 154 |
+
|
| 155 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped):
|
| 156 |
+
FileSystem.init(self, wrapped)
|
| 157 |
+
self.azurefs = <CAzureFileSystem*> wrapped.get()
|
| 158 |
+
|
| 159 |
+
@staticmethod
|
| 160 |
+
def _reconstruct(kwargs):
|
| 161 |
+
# __reduce__ doesn't allow passing named arguments directly to the
|
| 162 |
+
# reconstructor, hence this wrapper.
|
| 163 |
+
return AzureFileSystem(**kwargs)
|
| 164 |
+
|
| 165 |
+
def __reduce__(self):
|
| 166 |
+
cdef CAzureOptions opts = self.azurefs.options()
|
| 167 |
+
return (
|
| 168 |
+
AzureFileSystem._reconstruct, (dict(
|
| 169 |
+
account_name=frombytes(opts.account_name),
|
| 170 |
+
account_key=frombytes(opts.AccountKey()),
|
| 171 |
+
blob_storage_authority=frombytes(opts.blob_storage_authority),
|
| 172 |
+
blob_storage_scheme=frombytes(opts.blob_storage_scheme),
|
| 173 |
+
client_id=frombytes(opts.ClientId()),
|
| 174 |
+
client_secret=frombytes(opts.ClientSecret()),
|
| 175 |
+
dfs_storage_authority=frombytes(opts.dfs_storage_authority),
|
| 176 |
+
dfs_storage_scheme=frombytes(opts.dfs_storage_scheme),
|
| 177 |
+
sas_token=frombytes(opts.SasToken()),
|
| 178 |
+
tenant_id=frombytes(opts.TenantId())
|
| 179 |
+
),))
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed827e1771ac06884deef9368797fc21fe7028ce2e1efb91d6be237514dfe771
|
| 3 |
+
size 748032
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pxd
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.lib cimport *
|
| 21 |
+
from pyarrow.includes.common cimport *
|
| 22 |
+
from pyarrow.includes.libarrow cimport *
|
| 23 |
+
|
| 24 |
+
cdef class UdfContext(_Weakrefable):
|
| 25 |
+
cdef:
|
| 26 |
+
CUdfContext c_context
|
| 27 |
+
|
| 28 |
+
cdef void init(self, const CUdfContext& c_context)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
cdef class FunctionOptions(_Weakrefable):
|
| 32 |
+
cdef:
|
| 33 |
+
shared_ptr[CFunctionOptions] wrapped
|
| 34 |
+
|
| 35 |
+
cdef const CFunctionOptions* get_options(self) except NULL
|
| 36 |
+
cdef void init(self, const shared_ptr[CFunctionOptions]& sp)
|
| 37 |
+
|
| 38 |
+
cdef inline shared_ptr[CFunctionOptions] unwrap(self)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
cdef class _SortOptions(FunctionOptions):
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
cdef CExpression _bind(Expression filter, Schema schema) except *
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
cdef class Expression(_Weakrefable):
|
| 49 |
+
|
| 50 |
+
cdef:
|
| 51 |
+
CExpression expr
|
| 52 |
+
|
| 53 |
+
cdef void init(self, const CExpression& sp)
|
| 54 |
+
|
| 55 |
+
@staticmethod
|
| 56 |
+
cdef wrap(const CExpression& sp)
|
| 57 |
+
|
| 58 |
+
cdef inline CExpression unwrap(self)
|
| 59 |
+
|
| 60 |
+
@staticmethod
|
| 61 |
+
cdef Expression _expr_or_scalar(object expr)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
cdef CExpression _true
|
| 65 |
+
|
| 66 |
+
cdef CFieldRef _ensure_field_ref(value) except *
|
| 67 |
+
|
| 68 |
+
cdef vector[CSortKey] unwrap_sort_keys(sort_keys, allow_str=*) except *
|
| 69 |
+
|
| 70 |
+
cdef CSortOrder unwrap_sort_order(order) except *
|
| 71 |
+
|
| 72 |
+
cdef CNullPlacement unwrap_null_placement(null_placement) except *
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute.pyx
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_compute_docstrings.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
"""
|
| 19 |
+
Custom documentation additions for compute functions.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
function_doc_additions = {}
|
| 23 |
+
|
| 24 |
+
function_doc_additions["filter"] = """
|
| 25 |
+
Examples
|
| 26 |
+
--------
|
| 27 |
+
>>> import pyarrow as pa
|
| 28 |
+
>>> arr = pa.array(["a", "b", "c", None, "e"])
|
| 29 |
+
>>> mask = pa.array([True, False, None, False, True])
|
| 30 |
+
>>> arr.filter(mask)
|
| 31 |
+
<pyarrow.lib.StringArray object at ...>
|
| 32 |
+
[
|
| 33 |
+
"a",
|
| 34 |
+
"e"
|
| 35 |
+
]
|
| 36 |
+
>>> arr.filter(mask, null_selection_behavior='emit_null')
|
| 37 |
+
<pyarrow.lib.StringArray object at ...>
|
| 38 |
+
[
|
| 39 |
+
"a",
|
| 40 |
+
null,
|
| 41 |
+
"e"
|
| 42 |
+
]
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
function_doc_additions["mode"] = """
|
| 46 |
+
Examples
|
| 47 |
+
--------
|
| 48 |
+
>>> import pyarrow as pa
|
| 49 |
+
>>> import pyarrow.compute as pc
|
| 50 |
+
>>> arr = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
|
| 51 |
+
>>> modes = pc.mode(arr, 2)
|
| 52 |
+
>>> modes[0]
|
| 53 |
+
<pyarrow.StructScalar: [('mode', 2), ('count', 5)]>
|
| 54 |
+
>>> modes[1]
|
| 55 |
+
<pyarrow.StructScalar: [('mode', 1), ('count', 2)]>
|
| 56 |
+
"""
|
| 57 |
+
|
| 58 |
+
function_doc_additions["min"] = """
|
| 59 |
+
Examples
|
| 60 |
+
--------
|
| 61 |
+
>>> import pyarrow as pa
|
| 62 |
+
>>> import pyarrow.compute as pc
|
| 63 |
+
>>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
|
| 64 |
+
>>> pc.min(arr1)
|
| 65 |
+
<pyarrow.Int64Scalar: 1>
|
| 66 |
+
|
| 67 |
+
Using ``skip_nulls`` to handle null values.
|
| 68 |
+
|
| 69 |
+
>>> arr2 = pa.array([1.0, None, 2.0, 3.0])
|
| 70 |
+
>>> pc.min(arr2)
|
| 71 |
+
<pyarrow.DoubleScalar: 1.0>
|
| 72 |
+
>>> pc.min(arr2, skip_nulls=False)
|
| 73 |
+
<pyarrow.DoubleScalar: None>
|
| 74 |
+
|
| 75 |
+
Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
|
| 76 |
+
|
| 77 |
+
>>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
|
| 78 |
+
>>> pc.min(arr3)
|
| 79 |
+
<pyarrow.DoubleScalar: 1.0>
|
| 80 |
+
>>> pc.min(arr3, options=pc.ScalarAggregateOptions(min_count=3))
|
| 81 |
+
<pyarrow.DoubleScalar: 1.0>
|
| 82 |
+
>>> pc.min(arr3, options=pc.ScalarAggregateOptions(min_count=4))
|
| 83 |
+
<pyarrow.DoubleScalar: None>
|
| 84 |
+
|
| 85 |
+
This function also works with string values.
|
| 86 |
+
|
| 87 |
+
>>> arr4 = pa.array(["z", None, "y", "x"])
|
| 88 |
+
>>> pc.min(arr4)
|
| 89 |
+
<pyarrow.StringScalar: 'x'>
|
| 90 |
+
"""
|
| 91 |
+
|
| 92 |
+
function_doc_additions["max"] = """
|
| 93 |
+
Examples
|
| 94 |
+
--------
|
| 95 |
+
>>> import pyarrow as pa
|
| 96 |
+
>>> import pyarrow.compute as pc
|
| 97 |
+
>>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
|
| 98 |
+
>>> pc.max(arr1)
|
| 99 |
+
<pyarrow.Int64Scalar: 3>
|
| 100 |
+
|
| 101 |
+
Using ``skip_nulls`` to handle null values.
|
| 102 |
+
|
| 103 |
+
>>> arr2 = pa.array([1.0, None, 2.0, 3.0])
|
| 104 |
+
>>> pc.max(arr2)
|
| 105 |
+
<pyarrow.DoubleScalar: 3.0>
|
| 106 |
+
>>> pc.max(arr2, skip_nulls=False)
|
| 107 |
+
<pyarrow.DoubleScalar: None>
|
| 108 |
+
|
| 109 |
+
Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
|
| 110 |
+
|
| 111 |
+
>>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
|
| 112 |
+
>>> pc.max(arr3)
|
| 113 |
+
<pyarrow.DoubleScalar: 3.0>
|
| 114 |
+
>>> pc.max(arr3, options=pc.ScalarAggregateOptions(min_count=3))
|
| 115 |
+
<pyarrow.DoubleScalar: 3.0>
|
| 116 |
+
>>> pc.max(arr3, options=pc.ScalarAggregateOptions(min_count=4))
|
| 117 |
+
<pyarrow.DoubleScalar: None>
|
| 118 |
+
|
| 119 |
+
This function also works with string values.
|
| 120 |
+
|
| 121 |
+
>>> arr4 = pa.array(["z", None, "y", "x"])
|
| 122 |
+
>>> pc.max(arr4)
|
| 123 |
+
<pyarrow.StringScalar: 'z'>
|
| 124 |
+
"""
|
| 125 |
+
|
| 126 |
+
function_doc_additions["min_max"] = """
|
| 127 |
+
Examples
|
| 128 |
+
--------
|
| 129 |
+
>>> import pyarrow as pa
|
| 130 |
+
>>> import pyarrow.compute as pc
|
| 131 |
+
>>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
|
| 132 |
+
>>> pc.min_max(arr1)
|
| 133 |
+
<pyarrow.StructScalar: [('min', 1), ('max', 3)]>
|
| 134 |
+
|
| 135 |
+
Using ``skip_nulls`` to handle null values.
|
| 136 |
+
|
| 137 |
+
>>> arr2 = pa.array([1.0, None, 2.0, 3.0])
|
| 138 |
+
>>> pc.min_max(arr2)
|
| 139 |
+
<pyarrow.StructScalar: [('min', 1.0), ('max', 3.0)]>
|
| 140 |
+
>>> pc.min_max(arr2, skip_nulls=False)
|
| 141 |
+
<pyarrow.StructScalar: [('min', None), ('max', None)]>
|
| 142 |
+
|
| 143 |
+
Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
|
| 144 |
+
|
| 145 |
+
>>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
|
| 146 |
+
>>> pc.min_max(arr3)
|
| 147 |
+
<pyarrow.StructScalar: [('min', 1.0), ('max', 3.0)]>
|
| 148 |
+
>>> pc.min_max(arr3, options=pc.ScalarAggregateOptions(min_count=3))
|
| 149 |
+
<pyarrow.StructScalar: [('min', 1.0), ('max', 3.0)]>
|
| 150 |
+
>>> pc.min_max(arr3, options=pc.ScalarAggregateOptions(min_count=4))
|
| 151 |
+
<pyarrow.StructScalar: [('min', None), ('max', None)]>
|
| 152 |
+
|
| 153 |
+
This function also works with string values.
|
| 154 |
+
|
| 155 |
+
>>> arr4 = pa.array(["z", None, "y", "x"])
|
| 156 |
+
>>> pc.min_max(arr4)
|
| 157 |
+
<pyarrow.StructScalar: [('min', 'x'), ('max', 'z')]>
|
| 158 |
+
"""
|
| 159 |
+
|
| 160 |
+
function_doc_additions["first"] = """
|
| 161 |
+
Examples
|
| 162 |
+
--------
|
| 163 |
+
>>> import pyarrow as pa
|
| 164 |
+
>>> import pyarrow.compute as pc
|
| 165 |
+
>>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
|
| 166 |
+
>>> pc.first(arr1)
|
| 167 |
+
<pyarrow.Int64Scalar: 1>
|
| 168 |
+
|
| 169 |
+
Using ``skip_nulls`` to handle null values.
|
| 170 |
+
|
| 171 |
+
>>> arr2 = pa.array([None, 1.0, 2.0, 3.0])
|
| 172 |
+
>>> pc.first(arr2)
|
| 173 |
+
<pyarrow.DoubleScalar: 1.0>
|
| 174 |
+
>>> pc.first(arr2, skip_nulls=False)
|
| 175 |
+
<pyarrow.DoubleScalar: None>
|
| 176 |
+
|
| 177 |
+
Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
|
| 178 |
+
|
| 179 |
+
>>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
|
| 180 |
+
>>> pc.first(arr3)
|
| 181 |
+
<pyarrow.DoubleScalar: 1.0>
|
| 182 |
+
>>> pc.first(arr3, options=pc.ScalarAggregateOptions(min_count=3))
|
| 183 |
+
<pyarrow.DoubleScalar: 1.0>
|
| 184 |
+
>>> pc.first(arr3, options=pc.ScalarAggregateOptions(min_count=4))
|
| 185 |
+
<pyarrow.DoubleScalar: None>
|
| 186 |
+
|
| 187 |
+
See Also
|
| 188 |
+
--------
|
| 189 |
+
pyarrow.compute.first_last
|
| 190 |
+
pyarrow.compute.last
|
| 191 |
+
"""
|
| 192 |
+
|
| 193 |
+
function_doc_additions["last"] = """
|
| 194 |
+
Examples
|
| 195 |
+
--------
|
| 196 |
+
>>> import pyarrow as pa
|
| 197 |
+
>>> import pyarrow.compute as pc
|
| 198 |
+
>>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
|
| 199 |
+
>>> pc.last(arr1)
|
| 200 |
+
<pyarrow.Int64Scalar: 2>
|
| 201 |
+
|
| 202 |
+
Using ``skip_nulls`` to handle null values.
|
| 203 |
+
|
| 204 |
+
>>> arr2 = pa.array([1.0, 2.0, 3.0, None])
|
| 205 |
+
>>> pc.last(arr2)
|
| 206 |
+
<pyarrow.DoubleScalar: 3.0>
|
| 207 |
+
>>> pc.last(arr2, skip_nulls=False)
|
| 208 |
+
<pyarrow.DoubleScalar: None>
|
| 209 |
+
|
| 210 |
+
Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
|
| 211 |
+
|
| 212 |
+
>>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
|
| 213 |
+
>>> pc.last(arr3)
|
| 214 |
+
<pyarrow.DoubleScalar: 3.0>
|
| 215 |
+
>>> pc.last(arr3, options=pc.ScalarAggregateOptions(min_count=3))
|
| 216 |
+
<pyarrow.DoubleScalar: 3.0>
|
| 217 |
+
>>> pc.last(arr3, options=pc.ScalarAggregateOptions(min_count=4))
|
| 218 |
+
<pyarrow.DoubleScalar: None>
|
| 219 |
+
|
| 220 |
+
See Also
|
| 221 |
+
--------
|
| 222 |
+
pyarrow.compute.first
|
| 223 |
+
pyarrow.compute.first_last
|
| 224 |
+
"""
|
| 225 |
+
|
| 226 |
+
function_doc_additions["first_last"] = """
|
| 227 |
+
Examples
|
| 228 |
+
--------
|
| 229 |
+
>>> import pyarrow as pa
|
| 230 |
+
>>> import pyarrow.compute as pc
|
| 231 |
+
>>> arr1 = pa.array([1, 1, 2, 2, 3, 2, 2, 2])
|
| 232 |
+
>>> pc.first_last(arr1)
|
| 233 |
+
<pyarrow.StructScalar: [('first', 1), ('last', 2)]>
|
| 234 |
+
|
| 235 |
+
Using ``skip_nulls`` to handle null values.
|
| 236 |
+
|
| 237 |
+
>>> arr2 = pa.array([None, 2.0, 3.0, None])
|
| 238 |
+
>>> pc.first_last(arr2)
|
| 239 |
+
<pyarrow.StructScalar: [('first', 2.0), ('last', 3.0)]>
|
| 240 |
+
>>> pc.first_last(arr2, skip_nulls=False)
|
| 241 |
+
<pyarrow.StructScalar: [('first', None), ('last', None)]>
|
| 242 |
+
|
| 243 |
+
Using ``ScalarAggregateOptions`` to control minimum number of non-null values.
|
| 244 |
+
|
| 245 |
+
>>> arr3 = pa.array([1.0, None, float("nan"), 3.0])
|
| 246 |
+
>>> pc.first_last(arr3)
|
| 247 |
+
<pyarrow.StructScalar: [('first', 1.0), ('last', 3.0)]>
|
| 248 |
+
>>> pc.first_last(arr3, options=pc.ScalarAggregateOptions(min_count=3))
|
| 249 |
+
<pyarrow.StructScalar: [('first', 1.0), ('last', 3.0)]>
|
| 250 |
+
>>> pc.first_last(arr3, options=pc.ScalarAggregateOptions(min_count=4))
|
| 251 |
+
<pyarrow.StructScalar: [('first', None), ('last', None)]>
|
| 252 |
+
|
| 253 |
+
See Also
|
| 254 |
+
--------
|
| 255 |
+
pyarrow.compute.first
|
| 256 |
+
pyarrow.compute.last
|
| 257 |
+
"""
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5dcfd55b795fc02b51d0cb4801cebd830b6f90ba674f0b2d49115db488ae2594
|
| 3 |
+
size 270336
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pxd
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.includes.libarrow cimport *
|
| 21 |
+
from pyarrow.lib cimport _Weakrefable
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
cdef class ConvertOptions(_Weakrefable):
|
| 25 |
+
cdef:
|
| 26 |
+
unique_ptr[CCSVConvertOptions] options
|
| 27 |
+
|
| 28 |
+
@staticmethod
|
| 29 |
+
cdef ConvertOptions wrap(CCSVConvertOptions options)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
cdef class ParseOptions(_Weakrefable):
|
| 33 |
+
cdef:
|
| 34 |
+
unique_ptr[CCSVParseOptions] options
|
| 35 |
+
object _invalid_row_handler
|
| 36 |
+
|
| 37 |
+
@staticmethod
|
| 38 |
+
cdef ParseOptions wrap(CCSVParseOptions options)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
cdef class ReadOptions(_Weakrefable):
|
| 42 |
+
cdef:
|
| 43 |
+
unique_ptr[CCSVReadOptions] options
|
| 44 |
+
public object encoding
|
| 45 |
+
|
| 46 |
+
@staticmethod
|
| 47 |
+
cdef ReadOptions wrap(CCSVReadOptions options)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
cdef class WriteOptions(_Weakrefable):
|
| 51 |
+
cdef:
|
| 52 |
+
unique_ptr[CCSVWriteOptions] options
|
| 53 |
+
|
| 54 |
+
@staticmethod
|
| 55 |
+
cdef WriteOptions wrap(CCSVWriteOptions options)
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_csv.pyx
ADDED
|
@@ -0,0 +1,1626 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: profile=False
|
| 19 |
+
# distutils: language = c++
|
| 20 |
+
# cython: language_level = 3
|
| 21 |
+
|
| 22 |
+
from cython.operator cimport dereference as deref
|
| 23 |
+
|
| 24 |
+
from collections import namedtuple
|
| 25 |
+
from collections.abc import Mapping
|
| 26 |
+
|
| 27 |
+
from pyarrow.includes.common cimport *
|
| 28 |
+
from pyarrow.includes.libarrow cimport *
|
| 29 |
+
from pyarrow.includes.libarrow_python cimport *
|
| 30 |
+
from pyarrow.lib cimport (check_status, Field, MemoryPool, Schema,
|
| 31 |
+
RecordBatchReader, ensure_type,
|
| 32 |
+
maybe_unbox_memory_pool, get_input_stream,
|
| 33 |
+
get_writer, native_transcoding_input_stream,
|
| 34 |
+
pyarrow_unwrap_batch, pyarrow_unwrap_schema,
|
| 35 |
+
pyarrow_unwrap_table, pyarrow_wrap_schema,
|
| 36 |
+
pyarrow_wrap_table, pyarrow_wrap_data_type,
|
| 37 |
+
pyarrow_unwrap_data_type, Table, RecordBatch,
|
| 38 |
+
StopToken, _CRecordBatchWriter)
|
| 39 |
+
from pyarrow.lib import frombytes, tobytes, SignalStopHandler
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
cdef unsigned char _single_char(s) except 0:
|
| 43 |
+
val = ord(s)
|
| 44 |
+
if val == 0 or val > 127:
|
| 45 |
+
raise ValueError("Expecting an ASCII character")
|
| 46 |
+
return <unsigned char> val
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
_InvalidRow = namedtuple(
|
| 50 |
+
"_InvalidRow", ("expected_columns", "actual_columns", "number", "text"),
|
| 51 |
+
module=__name__)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class InvalidRow(_InvalidRow):
|
| 55 |
+
"""
|
| 56 |
+
Description of an invalid row in a CSV file.
|
| 57 |
+
|
| 58 |
+
Parameters
|
| 59 |
+
----------
|
| 60 |
+
expected_columns : int
|
| 61 |
+
The expected number of columns in the row.
|
| 62 |
+
actual_columns : int
|
| 63 |
+
The actual number of columns in the row.
|
| 64 |
+
number : int or None
|
| 65 |
+
The physical row number if known, otherwise None.
|
| 66 |
+
text : str
|
| 67 |
+
The contents of the row.
|
| 68 |
+
"""
|
| 69 |
+
__slots__ = ()
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
cdef CInvalidRowResult _handle_invalid_row(
|
| 73 |
+
handler, const CCSVInvalidRow& c_row) except CInvalidRowResult_Error:
|
| 74 |
+
# A negative row number means undetermined (because of parallel reading)
|
| 75 |
+
row_number = c_row.number if c_row.number >= 0 else None
|
| 76 |
+
row = InvalidRow(c_row.expected_columns, c_row.actual_columns,
|
| 77 |
+
row_number, frombytes(<c_string> c_row.text))
|
| 78 |
+
result = handler(row)
|
| 79 |
+
if result == 'error':
|
| 80 |
+
return CInvalidRowResult_Error
|
| 81 |
+
elif result == 'skip':
|
| 82 |
+
return CInvalidRowResult_Skip
|
| 83 |
+
else:
|
| 84 |
+
raise ValueError("Invalid return value for invalid row handler: "
|
| 85 |
+
f"expected 'error' or 'skip', got {result!r}")
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
cdef class ReadOptions(_Weakrefable):
|
| 89 |
+
"""
|
| 90 |
+
Options for reading CSV files.
|
| 91 |
+
|
| 92 |
+
Parameters
|
| 93 |
+
----------
|
| 94 |
+
use_threads : bool, optional (default True)
|
| 95 |
+
Whether to use multiple threads to accelerate reading
|
| 96 |
+
block_size : int, optional
|
| 97 |
+
How much bytes to process at a time from the input stream.
|
| 98 |
+
This will determine multi-threading granularity as well as
|
| 99 |
+
the size of individual record batches or table chunks.
|
| 100 |
+
Minimum valid value for block size is 1
|
| 101 |
+
skip_rows : int, optional (default 0)
|
| 102 |
+
The number of rows to skip before the column names (if any)
|
| 103 |
+
and the CSV data.
|
| 104 |
+
skip_rows_after_names : int, optional (default 0)
|
| 105 |
+
The number of rows to skip after the column names.
|
| 106 |
+
This number can be larger than the number of rows in one
|
| 107 |
+
block, and empty rows are counted.
|
| 108 |
+
The order of application is as follows:
|
| 109 |
+
- `skip_rows` is applied (if non-zero);
|
| 110 |
+
- column names are read (unless `column_names` is set);
|
| 111 |
+
- `skip_rows_after_names` is applied (if non-zero).
|
| 112 |
+
column_names : list, optional
|
| 113 |
+
The column names of the target table. If empty, fall back on
|
| 114 |
+
`autogenerate_column_names`.
|
| 115 |
+
autogenerate_column_names : bool, optional (default False)
|
| 116 |
+
Whether to autogenerate column names if `column_names` is empty.
|
| 117 |
+
If true, column names will be of the form "f0", "f1"...
|
| 118 |
+
If false, column names will be read from the first CSV row
|
| 119 |
+
after `skip_rows`.
|
| 120 |
+
encoding : str, optional (default 'utf8')
|
| 121 |
+
The character encoding of the CSV data. Columns that cannot
|
| 122 |
+
decode using this encoding can still be read as Binary.
|
| 123 |
+
|
| 124 |
+
Examples
|
| 125 |
+
--------
|
| 126 |
+
|
| 127 |
+
Defining an example data:
|
| 128 |
+
|
| 129 |
+
>>> import io
|
| 130 |
+
>>> s = "1,2,3\\nFlamingo,2,2022-03-01\\nHorse,4,2022-03-02\\nBrittle stars,5,2022-03-03\\nCentipede,100,2022-03-04"
|
| 131 |
+
>>> print(s)
|
| 132 |
+
1,2,3
|
| 133 |
+
Flamingo,2,2022-03-01
|
| 134 |
+
Horse,4,2022-03-02
|
| 135 |
+
Brittle stars,5,2022-03-03
|
| 136 |
+
Centipede,100,2022-03-04
|
| 137 |
+
|
| 138 |
+
Ignore the first numbered row and substitute it with defined
|
| 139 |
+
or autogenerated column names:
|
| 140 |
+
|
| 141 |
+
>>> from pyarrow import csv
|
| 142 |
+
>>> read_options = csv.ReadOptions(
|
| 143 |
+
... column_names=["animals", "n_legs", "entry"],
|
| 144 |
+
... skip_rows=1)
|
| 145 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), read_options=read_options)
|
| 146 |
+
pyarrow.Table
|
| 147 |
+
animals: string
|
| 148 |
+
n_legs: int64
|
| 149 |
+
entry: date32[day]
|
| 150 |
+
----
|
| 151 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede"]]
|
| 152 |
+
n_legs: [[2,4,5,100]]
|
| 153 |
+
entry: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
|
| 154 |
+
|
| 155 |
+
>>> read_options = csv.ReadOptions(autogenerate_column_names=True,
|
| 156 |
+
... skip_rows=1)
|
| 157 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), read_options=read_options)
|
| 158 |
+
pyarrow.Table
|
| 159 |
+
f0: string
|
| 160 |
+
f1: int64
|
| 161 |
+
f2: date32[day]
|
| 162 |
+
----
|
| 163 |
+
f0: [["Flamingo","Horse","Brittle stars","Centipede"]]
|
| 164 |
+
f1: [[2,4,5,100]]
|
| 165 |
+
f2: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
|
| 166 |
+
|
| 167 |
+
Remove the first 2 rows of the data:
|
| 168 |
+
|
| 169 |
+
>>> read_options = csv.ReadOptions(skip_rows_after_names=2)
|
| 170 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), read_options=read_options)
|
| 171 |
+
pyarrow.Table
|
| 172 |
+
1: string
|
| 173 |
+
2: int64
|
| 174 |
+
3: date32[day]
|
| 175 |
+
----
|
| 176 |
+
1: [["Brittle stars","Centipede"]]
|
| 177 |
+
2: [[5,100]]
|
| 178 |
+
3: [[2022-03-03,2022-03-04]]
|
| 179 |
+
"""
|
| 180 |
+
|
| 181 |
+
# Avoid mistakingly creating attributes
|
| 182 |
+
__slots__ = ()
|
| 183 |
+
|
| 184 |
+
# __init__() is not called when unpickling, initialize storage here
|
| 185 |
+
def __cinit__(self, *argw, **kwargs):
|
| 186 |
+
self.options.reset(new CCSVReadOptions(CCSVReadOptions.Defaults()))
|
| 187 |
+
|
| 188 |
+
def __init__(self, *, use_threads=None, block_size=None, skip_rows=None,
|
| 189 |
+
skip_rows_after_names=None, column_names=None,
|
| 190 |
+
autogenerate_column_names=None, encoding='utf8'):
|
| 191 |
+
if use_threads is not None:
|
| 192 |
+
self.use_threads = use_threads
|
| 193 |
+
if block_size is not None:
|
| 194 |
+
self.block_size = block_size
|
| 195 |
+
if skip_rows is not None:
|
| 196 |
+
self.skip_rows = skip_rows
|
| 197 |
+
if skip_rows_after_names is not None:
|
| 198 |
+
self.skip_rows_after_names = skip_rows_after_names
|
| 199 |
+
if column_names is not None:
|
| 200 |
+
self.column_names = column_names
|
| 201 |
+
if autogenerate_column_names is not None:
|
| 202 |
+
self.autogenerate_column_names= autogenerate_column_names
|
| 203 |
+
# Python-specific option
|
| 204 |
+
self.encoding = encoding
|
| 205 |
+
|
| 206 |
+
@property
|
| 207 |
+
def use_threads(self):
|
| 208 |
+
"""
|
| 209 |
+
Whether to use multiple threads to accelerate reading.
|
| 210 |
+
"""
|
| 211 |
+
return deref(self.options).use_threads
|
| 212 |
+
|
| 213 |
+
@use_threads.setter
|
| 214 |
+
def use_threads(self, value):
|
| 215 |
+
deref(self.options).use_threads = value
|
| 216 |
+
|
| 217 |
+
@property
|
| 218 |
+
def block_size(self):
|
| 219 |
+
"""
|
| 220 |
+
How much bytes to process at a time from the input stream.
|
| 221 |
+
This will determine multi-threading granularity as well as
|
| 222 |
+
the size of individual record batches or table chunks.
|
| 223 |
+
"""
|
| 224 |
+
return deref(self.options).block_size
|
| 225 |
+
|
| 226 |
+
@block_size.setter
|
| 227 |
+
def block_size(self, value):
|
| 228 |
+
deref(self.options).block_size = value
|
| 229 |
+
|
| 230 |
+
@property
|
| 231 |
+
def skip_rows(self):
|
| 232 |
+
"""
|
| 233 |
+
The number of rows to skip before the column names (if any)
|
| 234 |
+
and the CSV data.
|
| 235 |
+
See `skip_rows_after_names` for interaction description
|
| 236 |
+
"""
|
| 237 |
+
return deref(self.options).skip_rows
|
| 238 |
+
|
| 239 |
+
@skip_rows.setter
|
| 240 |
+
def skip_rows(self, value):
|
| 241 |
+
deref(self.options).skip_rows = value
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def skip_rows_after_names(self):
|
| 245 |
+
"""
|
| 246 |
+
The number of rows to skip after the column names.
|
| 247 |
+
This number can be larger than the number of rows in one
|
| 248 |
+
block, and empty rows are counted.
|
| 249 |
+
The order of application is as follows:
|
| 250 |
+
- `skip_rows` is applied (if non-zero);
|
| 251 |
+
- column names are read (unless `column_names` is set);
|
| 252 |
+
- `skip_rows_after_names` is applied (if non-zero).
|
| 253 |
+
"""
|
| 254 |
+
return deref(self.options).skip_rows_after_names
|
| 255 |
+
|
| 256 |
+
@skip_rows_after_names.setter
|
| 257 |
+
def skip_rows_after_names(self, value):
|
| 258 |
+
deref(self.options).skip_rows_after_names = value
|
| 259 |
+
|
| 260 |
+
@property
|
| 261 |
+
def column_names(self):
|
| 262 |
+
"""
|
| 263 |
+
The column names of the target table. If empty, fall back on
|
| 264 |
+
`autogenerate_column_names`.
|
| 265 |
+
"""
|
| 266 |
+
return [frombytes(s) for s in deref(self.options).column_names]
|
| 267 |
+
|
| 268 |
+
@column_names.setter
|
| 269 |
+
def column_names(self, value):
|
| 270 |
+
deref(self.options).column_names.clear()
|
| 271 |
+
for item in value:
|
| 272 |
+
deref(self.options).column_names.push_back(tobytes(item))
|
| 273 |
+
|
| 274 |
+
@property
|
| 275 |
+
def autogenerate_column_names(self):
|
| 276 |
+
"""
|
| 277 |
+
Whether to autogenerate column names if `column_names` is empty.
|
| 278 |
+
If true, column names will be of the form "f0", "f1"...
|
| 279 |
+
If false, column names will be read from the first CSV row
|
| 280 |
+
after `skip_rows`.
|
| 281 |
+
"""
|
| 282 |
+
return deref(self.options).autogenerate_column_names
|
| 283 |
+
|
| 284 |
+
@autogenerate_column_names.setter
|
| 285 |
+
def autogenerate_column_names(self, value):
|
| 286 |
+
deref(self.options).autogenerate_column_names = value
|
| 287 |
+
|
| 288 |
+
def validate(self):
|
| 289 |
+
check_status(deref(self.options).Validate())
|
| 290 |
+
|
| 291 |
+
def equals(self, ReadOptions other):
|
| 292 |
+
"""
|
| 293 |
+
Parameters
|
| 294 |
+
----------
|
| 295 |
+
other : pyarrow.csv.ReadOptions
|
| 296 |
+
|
| 297 |
+
Returns
|
| 298 |
+
-------
|
| 299 |
+
bool
|
| 300 |
+
"""
|
| 301 |
+
return (
|
| 302 |
+
self.use_threads == other.use_threads and
|
| 303 |
+
self.block_size == other.block_size and
|
| 304 |
+
self.skip_rows == other.skip_rows and
|
| 305 |
+
self.skip_rows_after_names == other.skip_rows_after_names and
|
| 306 |
+
self.column_names == other.column_names and
|
| 307 |
+
self.autogenerate_column_names ==
|
| 308 |
+
other.autogenerate_column_names and
|
| 309 |
+
self.encoding == other.encoding
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
@staticmethod
|
| 313 |
+
cdef ReadOptions wrap(CCSVReadOptions options):
|
| 314 |
+
out = ReadOptions()
|
| 315 |
+
out.options.reset(new CCSVReadOptions(move(options)))
|
| 316 |
+
out.encoding = 'utf8' # No way to know this
|
| 317 |
+
return out
|
| 318 |
+
|
| 319 |
+
def __getstate__(self):
|
| 320 |
+
return (self.use_threads, self.block_size, self.skip_rows,
|
| 321 |
+
self.column_names, self.autogenerate_column_names,
|
| 322 |
+
self.encoding, self.skip_rows_after_names)
|
| 323 |
+
|
| 324 |
+
def __setstate__(self, state):
|
| 325 |
+
(self.use_threads, self.block_size, self.skip_rows,
|
| 326 |
+
self.column_names, self.autogenerate_column_names,
|
| 327 |
+
self.encoding, self.skip_rows_after_names) = state
|
| 328 |
+
|
| 329 |
+
def __eq__(self, other):
|
| 330 |
+
try:
|
| 331 |
+
return self.equals(other)
|
| 332 |
+
except TypeError:
|
| 333 |
+
return False
|
| 334 |
+
|
| 335 |
+
def _repr_base(self):
|
| 336 |
+
return (f"""
|
| 337 |
+
use_threads={self.use_threads},
|
| 338 |
+
block_size={self.block_size},
|
| 339 |
+
skip_rows={self.skip_rows},
|
| 340 |
+
skip_rows_after_names={self.skip_rows_after_names},
|
| 341 |
+
column_names={self.column_names},
|
| 342 |
+
autogenerate_column_names={self.autogenerate_column_names},
|
| 343 |
+
encoding={self.encoding!r}""")
|
| 344 |
+
|
| 345 |
+
def __repr__(self):
|
| 346 |
+
return (f"<pyarrow.csv.ReadOptions>({self._repr_base()})")
|
| 347 |
+
|
| 348 |
+
def __str__(self):
|
| 349 |
+
return (f"ReadOptions({self._repr_base()})")
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
cdef class ParseOptions(_Weakrefable):
|
| 353 |
+
"""
|
| 354 |
+
Options for parsing CSV files.
|
| 355 |
+
|
| 356 |
+
Parameters
|
| 357 |
+
----------
|
| 358 |
+
delimiter : 1-character string, optional (default ',')
|
| 359 |
+
The character delimiting individual cells in the CSV data.
|
| 360 |
+
quote_char : 1-character string or False, optional (default '"')
|
| 361 |
+
The character used optionally for quoting CSV values
|
| 362 |
+
(False if quoting is not allowed).
|
| 363 |
+
double_quote : bool, optional (default True)
|
| 364 |
+
Whether two quotes in a quoted CSV value denote a single quote
|
| 365 |
+
in the data.
|
| 366 |
+
escape_char : 1-character string or False, optional (default False)
|
| 367 |
+
The character used optionally for escaping special characters
|
| 368 |
+
(False if escaping is not allowed).
|
| 369 |
+
newlines_in_values : bool, optional (default False)
|
| 370 |
+
Whether newline characters are allowed in CSV values.
|
| 371 |
+
Setting this to True reduces the performance of multi-threaded
|
| 372 |
+
CSV reading.
|
| 373 |
+
ignore_empty_lines : bool, optional (default True)
|
| 374 |
+
Whether empty lines are ignored in CSV input.
|
| 375 |
+
If False, an empty line is interpreted as containing a single empty
|
| 376 |
+
value (assuming a one-column CSV file).
|
| 377 |
+
invalid_row_handler : callable, optional (default None)
|
| 378 |
+
If not None, this object is called for each CSV row that fails
|
| 379 |
+
parsing (because of a mismatching number of columns).
|
| 380 |
+
It should accept a single InvalidRow argument and return either
|
| 381 |
+
"skip" or "error" depending on the desired outcome.
|
| 382 |
+
|
| 383 |
+
Examples
|
| 384 |
+
--------
|
| 385 |
+
|
| 386 |
+
Defining an example file from bytes object:
|
| 387 |
+
|
| 388 |
+
>>> import io
|
| 389 |
+
>>> s = (
|
| 390 |
+
... "animals;n_legs;entry\\n"
|
| 391 |
+
... "Flamingo;2;2022-03-01\\n"
|
| 392 |
+
... "# Comment here:\\n"
|
| 393 |
+
... "Horse;4;2022-03-02\\n"
|
| 394 |
+
... "Brittle stars;5;2022-03-03\\n"
|
| 395 |
+
... "Centipede;100;2022-03-04"
|
| 396 |
+
... )
|
| 397 |
+
>>> print(s)
|
| 398 |
+
animals;n_legs;entry
|
| 399 |
+
Flamingo;2;2022-03-01
|
| 400 |
+
# Comment here:
|
| 401 |
+
Horse;4;2022-03-02
|
| 402 |
+
Brittle stars;5;2022-03-03
|
| 403 |
+
Centipede;100;2022-03-04
|
| 404 |
+
>>> source = io.BytesIO(s.encode())
|
| 405 |
+
|
| 406 |
+
Read the data from a file skipping rows with comments
|
| 407 |
+
and defining the delimiter:
|
| 408 |
+
|
| 409 |
+
>>> from pyarrow import csv
|
| 410 |
+
>>> def skip_comment(row):
|
| 411 |
+
... if row.text.startswith("# "):
|
| 412 |
+
... return 'skip'
|
| 413 |
+
... else:
|
| 414 |
+
... return 'error'
|
| 415 |
+
...
|
| 416 |
+
>>> parse_options = csv.ParseOptions(delimiter=";", invalid_row_handler=skip_comment)
|
| 417 |
+
>>> csv.read_csv(source, parse_options=parse_options)
|
| 418 |
+
pyarrow.Table
|
| 419 |
+
animals: string
|
| 420 |
+
n_legs: int64
|
| 421 |
+
entry: date32[day]
|
| 422 |
+
----
|
| 423 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede"]]
|
| 424 |
+
n_legs: [[2,4,5,100]]
|
| 425 |
+
entry: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
|
| 426 |
+
"""
|
| 427 |
+
__slots__ = ()
|
| 428 |
+
|
| 429 |
+
def __cinit__(self, *argw, **kwargs):
|
| 430 |
+
self._invalid_row_handler = None
|
| 431 |
+
self.options.reset(new CCSVParseOptions(CCSVParseOptions.Defaults()))
|
| 432 |
+
|
| 433 |
+
def __init__(self, *, delimiter=None, quote_char=None, double_quote=None,
|
| 434 |
+
escape_char=None, newlines_in_values=None,
|
| 435 |
+
ignore_empty_lines=None, invalid_row_handler=None):
|
| 436 |
+
if delimiter is not None:
|
| 437 |
+
self.delimiter = delimiter
|
| 438 |
+
if quote_char is not None:
|
| 439 |
+
self.quote_char = quote_char
|
| 440 |
+
if double_quote is not None:
|
| 441 |
+
self.double_quote = double_quote
|
| 442 |
+
if escape_char is not None:
|
| 443 |
+
self.escape_char = escape_char
|
| 444 |
+
if newlines_in_values is not None:
|
| 445 |
+
self.newlines_in_values = newlines_in_values
|
| 446 |
+
if ignore_empty_lines is not None:
|
| 447 |
+
self.ignore_empty_lines = ignore_empty_lines
|
| 448 |
+
if invalid_row_handler is not None:
|
| 449 |
+
self.invalid_row_handler = invalid_row_handler
|
| 450 |
+
|
| 451 |
+
@property
|
| 452 |
+
def delimiter(self):
|
| 453 |
+
"""
|
| 454 |
+
The character delimiting individual cells in the CSV data.
|
| 455 |
+
"""
|
| 456 |
+
return chr(deref(self.options).delimiter)
|
| 457 |
+
|
| 458 |
+
@delimiter.setter
|
| 459 |
+
def delimiter(self, value):
|
| 460 |
+
deref(self.options).delimiter = _single_char(value)
|
| 461 |
+
|
| 462 |
+
@property
|
| 463 |
+
def quote_char(self):
|
| 464 |
+
"""
|
| 465 |
+
The character used optionally for quoting CSV values
|
| 466 |
+
(False if quoting is not allowed).
|
| 467 |
+
"""
|
| 468 |
+
if deref(self.options).quoting:
|
| 469 |
+
return chr(deref(self.options).quote_char)
|
| 470 |
+
else:
|
| 471 |
+
return False
|
| 472 |
+
|
| 473 |
+
@quote_char.setter
|
| 474 |
+
def quote_char(self, value):
|
| 475 |
+
if value is False:
|
| 476 |
+
deref(self.options).quoting = False
|
| 477 |
+
else:
|
| 478 |
+
deref(self.options).quote_char = _single_char(value)
|
| 479 |
+
deref(self.options).quoting = True
|
| 480 |
+
|
| 481 |
+
@property
|
| 482 |
+
def double_quote(self):
|
| 483 |
+
"""
|
| 484 |
+
Whether two quotes in a quoted CSV value denote a single quote
|
| 485 |
+
in the data.
|
| 486 |
+
"""
|
| 487 |
+
return deref(self.options).double_quote
|
| 488 |
+
|
| 489 |
+
@double_quote.setter
|
| 490 |
+
def double_quote(self, value):
|
| 491 |
+
deref(self.options).double_quote = value
|
| 492 |
+
|
| 493 |
+
@property
|
| 494 |
+
def escape_char(self):
|
| 495 |
+
"""
|
| 496 |
+
The character used optionally for escaping special characters
|
| 497 |
+
(False if escaping is not allowed).
|
| 498 |
+
"""
|
| 499 |
+
if deref(self.options).escaping:
|
| 500 |
+
return chr(deref(self.options).escape_char)
|
| 501 |
+
else:
|
| 502 |
+
return False
|
| 503 |
+
|
| 504 |
+
@escape_char.setter
|
| 505 |
+
def escape_char(self, value):
|
| 506 |
+
if value is False:
|
| 507 |
+
deref(self.options).escaping = False
|
| 508 |
+
else:
|
| 509 |
+
deref(self.options).escape_char = _single_char(value)
|
| 510 |
+
deref(self.options).escaping = True
|
| 511 |
+
|
| 512 |
+
@property
|
| 513 |
+
def newlines_in_values(self):
|
| 514 |
+
"""
|
| 515 |
+
Whether newline characters are allowed in CSV values.
|
| 516 |
+
Setting this to True reduces the performance of multi-threaded
|
| 517 |
+
CSV reading.
|
| 518 |
+
"""
|
| 519 |
+
return deref(self.options).newlines_in_values
|
| 520 |
+
|
| 521 |
+
@newlines_in_values.setter
|
| 522 |
+
def newlines_in_values(self, value):
|
| 523 |
+
deref(self.options).newlines_in_values = value
|
| 524 |
+
|
| 525 |
+
@property
|
| 526 |
+
def ignore_empty_lines(self):
|
| 527 |
+
"""
|
| 528 |
+
Whether empty lines are ignored in CSV input.
|
| 529 |
+
If False, an empty line is interpreted as containing a single empty
|
| 530 |
+
value (assuming a one-column CSV file).
|
| 531 |
+
"""
|
| 532 |
+
return deref(self.options).ignore_empty_lines
|
| 533 |
+
|
| 534 |
+
@property
|
| 535 |
+
def invalid_row_handler(self):
|
| 536 |
+
"""
|
| 537 |
+
Optional handler for invalid rows.
|
| 538 |
+
|
| 539 |
+
If not None, this object is called for each CSV row that fails
|
| 540 |
+
parsing (because of a mismatching number of columns).
|
| 541 |
+
It should accept a single InvalidRow argument and return either
|
| 542 |
+
"skip" or "error" depending on the desired outcome.
|
| 543 |
+
"""
|
| 544 |
+
return self._invalid_row_handler
|
| 545 |
+
|
| 546 |
+
@invalid_row_handler.setter
|
| 547 |
+
def invalid_row_handler(self, value):
|
| 548 |
+
if value is not None and not callable(value):
|
| 549 |
+
raise TypeError("Expected callable or None, "
|
| 550 |
+
f"got instance of {type(value)!r}")
|
| 551 |
+
self._invalid_row_handler = value
|
| 552 |
+
deref(self.options).invalid_row_handler = MakeInvalidRowHandler(
|
| 553 |
+
<function[PyInvalidRowCallback]> &_handle_invalid_row, value)
|
| 554 |
+
|
| 555 |
+
@ignore_empty_lines.setter
|
| 556 |
+
def ignore_empty_lines(self, value):
|
| 557 |
+
deref(self.options).ignore_empty_lines = value
|
| 558 |
+
|
| 559 |
+
def validate(self):
|
| 560 |
+
check_status(deref(self.options).Validate())
|
| 561 |
+
|
| 562 |
+
def equals(self, ParseOptions other):
|
| 563 |
+
"""
|
| 564 |
+
Parameters
|
| 565 |
+
----------
|
| 566 |
+
other : pyarrow.csv.ParseOptions
|
| 567 |
+
|
| 568 |
+
Returns
|
| 569 |
+
-------
|
| 570 |
+
bool
|
| 571 |
+
"""
|
| 572 |
+
return (
|
| 573 |
+
self.delimiter == other.delimiter and
|
| 574 |
+
self.quote_char == other.quote_char and
|
| 575 |
+
self.double_quote == other.double_quote and
|
| 576 |
+
self.escape_char == other.escape_char and
|
| 577 |
+
self.newlines_in_values == other.newlines_in_values and
|
| 578 |
+
self.ignore_empty_lines == other.ignore_empty_lines and
|
| 579 |
+
self._invalid_row_handler == other._invalid_row_handler
|
| 580 |
+
)
|
| 581 |
+
|
| 582 |
+
@staticmethod
|
| 583 |
+
cdef ParseOptions wrap(CCSVParseOptions options):
|
| 584 |
+
out = ParseOptions()
|
| 585 |
+
out.options.reset(new CCSVParseOptions(move(options)))
|
| 586 |
+
return out
|
| 587 |
+
|
| 588 |
+
def __getstate__(self):
|
| 589 |
+
return (self.delimiter, self.quote_char, self.double_quote,
|
| 590 |
+
self.escape_char, self.newlines_in_values,
|
| 591 |
+
self.ignore_empty_lines, self.invalid_row_handler)
|
| 592 |
+
|
| 593 |
+
def __setstate__(self, state):
|
| 594 |
+
(self.delimiter, self.quote_char, self.double_quote,
|
| 595 |
+
self.escape_char, self.newlines_in_values,
|
| 596 |
+
self.ignore_empty_lines, self.invalid_row_handler) = state
|
| 597 |
+
|
| 598 |
+
def __eq__(self, other):
|
| 599 |
+
try:
|
| 600 |
+
return self.equals(other)
|
| 601 |
+
except TypeError:
|
| 602 |
+
return False
|
| 603 |
+
|
| 604 |
+
def _repr_base(self):
|
| 605 |
+
return (f"""
|
| 606 |
+
delimiter={self.delimiter!r},
|
| 607 |
+
quote_char={self.quote_char!r},
|
| 608 |
+
double_quote={self.double_quote},
|
| 609 |
+
escape_char={self.escape_char!r},
|
| 610 |
+
newlines_in_values={self.newlines_in_values},
|
| 611 |
+
ignore_empty_lines={self.ignore_empty_lines},
|
| 612 |
+
invalid_row_handler={getattr(self.invalid_row_handler, '__name__',
|
| 613 |
+
self.invalid_row_handler)}""")
|
| 614 |
+
|
| 615 |
+
def __repr__(self):
|
| 616 |
+
return (f"<pyarrow.csv.ParseOptions>({self._repr_base()})")
|
| 617 |
+
|
| 618 |
+
def __str__(self):
|
| 619 |
+
return (f"ParseOptions({self._repr_base()})")
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
cdef class _ISO8601(_Weakrefable):
|
| 623 |
+
"""
|
| 624 |
+
A special object indicating ISO-8601 parsing.
|
| 625 |
+
"""
|
| 626 |
+
__slots__ = ()
|
| 627 |
+
|
| 628 |
+
def __str__(self):
|
| 629 |
+
return 'ISO8601'
|
| 630 |
+
|
| 631 |
+
def __eq__(self, other):
|
| 632 |
+
return isinstance(other, _ISO8601)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
ISO8601 = _ISO8601()
|
| 636 |
+
|
| 637 |
+
|
| 638 |
+
cdef class ConvertOptions(_Weakrefable):
|
| 639 |
+
"""
|
| 640 |
+
Options for converting CSV data.
|
| 641 |
+
|
| 642 |
+
Parameters
|
| 643 |
+
----------
|
| 644 |
+
check_utf8 : bool, optional (default True)
|
| 645 |
+
Whether to check UTF8 validity of string columns.
|
| 646 |
+
column_types : pyarrow.Schema or dict, optional
|
| 647 |
+
Explicitly map column names to column types. Passing this argument
|
| 648 |
+
disables type inference on the defined columns.
|
| 649 |
+
null_values : list, optional
|
| 650 |
+
A sequence of strings that denote nulls in the data
|
| 651 |
+
(defaults are appropriate in most cases). Note that by default,
|
| 652 |
+
string columns are not checked for null values. To enable
|
| 653 |
+
null checking for those, specify ``strings_can_be_null=True``.
|
| 654 |
+
true_values : list, optional
|
| 655 |
+
A sequence of strings that denote true booleans in the data
|
| 656 |
+
(defaults are appropriate in most cases).
|
| 657 |
+
false_values : list, optional
|
| 658 |
+
A sequence of strings that denote false booleans in the data
|
| 659 |
+
(defaults are appropriate in most cases).
|
| 660 |
+
decimal_point : 1-character string, optional (default '.')
|
| 661 |
+
The character used as decimal point in floating-point and decimal
|
| 662 |
+
data.
|
| 663 |
+
strings_can_be_null : bool, optional (default False)
|
| 664 |
+
Whether string / binary columns can have null values.
|
| 665 |
+
If true, then strings in null_values are considered null for
|
| 666 |
+
string columns.
|
| 667 |
+
If false, then all strings are valid string values.
|
| 668 |
+
quoted_strings_can_be_null : bool, optional (default True)
|
| 669 |
+
Whether quoted values can be null.
|
| 670 |
+
If true, then strings in "null_values" are also considered null
|
| 671 |
+
when they appear quoted in the CSV file. Otherwise, quoted values
|
| 672 |
+
are never considered null.
|
| 673 |
+
include_columns : list, optional
|
| 674 |
+
The names of columns to include in the Table.
|
| 675 |
+
If empty, the Table will include all columns from the CSV file.
|
| 676 |
+
If not empty, only these columns will be included, in this order.
|
| 677 |
+
include_missing_columns : bool, optional (default False)
|
| 678 |
+
If false, columns in `include_columns` but not in the CSV file will
|
| 679 |
+
error out.
|
| 680 |
+
If true, columns in `include_columns` but not in the CSV file will
|
| 681 |
+
produce a column of nulls (whose type is selected using
|
| 682 |
+
`column_types`, or null by default).
|
| 683 |
+
This option is ignored if `include_columns` is empty.
|
| 684 |
+
auto_dict_encode : bool, optional (default False)
|
| 685 |
+
Whether to try to automatically dict-encode string / binary data.
|
| 686 |
+
If true, then when type inference detects a string or binary column,
|
| 687 |
+
it it dict-encoded up to `auto_dict_max_cardinality` distinct values
|
| 688 |
+
(per chunk), after which it switches to regular encoding.
|
| 689 |
+
This setting is ignored for non-inferred columns (those in
|
| 690 |
+
`column_types`).
|
| 691 |
+
auto_dict_max_cardinality : int, optional
|
| 692 |
+
The maximum dictionary cardinality for `auto_dict_encode`.
|
| 693 |
+
This value is per chunk.
|
| 694 |
+
timestamp_parsers : list, optional
|
| 695 |
+
A sequence of strptime()-compatible format strings, tried in order
|
| 696 |
+
when attempting to infer or convert timestamp values (the special
|
| 697 |
+
value ISO8601() can also be given). By default, a fast built-in
|
| 698 |
+
ISO-8601 parser is used.
|
| 699 |
+
|
| 700 |
+
Examples
|
| 701 |
+
--------
|
| 702 |
+
|
| 703 |
+
Defining an example data:
|
| 704 |
+
|
| 705 |
+
>>> import io
|
| 706 |
+
>>> s = (
|
| 707 |
+
... "animals,n_legs,entry,fast\\n"
|
| 708 |
+
... "Flamingo,2,01/03/2022,Yes\\n"
|
| 709 |
+
... "Horse,4,02/03/2022,Yes\\n"
|
| 710 |
+
... "Brittle stars,5,03/03/2022,No\\n"
|
| 711 |
+
... "Centipede,100,04/03/2022,No\\n"
|
| 712 |
+
... ",6,05/03/2022,"
|
| 713 |
+
... )
|
| 714 |
+
>>> print(s)
|
| 715 |
+
animals,n_legs,entry,fast
|
| 716 |
+
Flamingo,2,01/03/2022,Yes
|
| 717 |
+
Horse,4,02/03/2022,Yes
|
| 718 |
+
Brittle stars,5,03/03/2022,No
|
| 719 |
+
Centipede,100,04/03/2022,No
|
| 720 |
+
,6,05/03/2022,
|
| 721 |
+
|
| 722 |
+
Change the type of a column:
|
| 723 |
+
|
| 724 |
+
>>> import pyarrow as pa
|
| 725 |
+
>>> from pyarrow import csv
|
| 726 |
+
>>> convert_options = csv.ConvertOptions(column_types={"n_legs": pa.float64()})
|
| 727 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 728 |
+
pyarrow.Table
|
| 729 |
+
animals: string
|
| 730 |
+
n_legs: double
|
| 731 |
+
entry: string
|
| 732 |
+
fast: string
|
| 733 |
+
----
|
| 734 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
|
| 735 |
+
n_legs: [[2,4,5,100,6]]
|
| 736 |
+
entry: [["01/03/2022","02/03/2022","03/03/2022","04/03/2022","05/03/2022"]]
|
| 737 |
+
fast: [["Yes","Yes","No","No",""]]
|
| 738 |
+
|
| 739 |
+
Define a date parsing format to get a timestamp type column
|
| 740 |
+
(in case dates are not in ISO format and not converted by default):
|
| 741 |
+
|
| 742 |
+
>>> convert_options = csv.ConvertOptions(
|
| 743 |
+
... timestamp_parsers=["%m/%d/%Y", "%m-%d-%Y"])
|
| 744 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 745 |
+
pyarrow.Table
|
| 746 |
+
animals: string
|
| 747 |
+
n_legs: int64
|
| 748 |
+
entry: timestamp[s]
|
| 749 |
+
fast: string
|
| 750 |
+
----
|
| 751 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
|
| 752 |
+
n_legs: [[2,4,5,100,6]]
|
| 753 |
+
entry: [[2022-01-03 00:00:00,2022-02-03 00:00:00,2022-03-03 00:00:00,2022-04-03 00:00:00,2022-05-03 00:00:00]]
|
| 754 |
+
fast: [["Yes","Yes","No","No",""]]
|
| 755 |
+
|
| 756 |
+
Specify a subset of columns to be read:
|
| 757 |
+
|
| 758 |
+
>>> convert_options = csv.ConvertOptions(
|
| 759 |
+
... include_columns=["animals", "n_legs"])
|
| 760 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 761 |
+
pyarrow.Table
|
| 762 |
+
animals: string
|
| 763 |
+
n_legs: int64
|
| 764 |
+
----
|
| 765 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
|
| 766 |
+
n_legs: [[2,4,5,100,6]]
|
| 767 |
+
|
| 768 |
+
List additional column to be included as a null typed column:
|
| 769 |
+
|
| 770 |
+
>>> convert_options = csv.ConvertOptions(
|
| 771 |
+
... include_columns=["animals", "n_legs", "location"],
|
| 772 |
+
... include_missing_columns=True)
|
| 773 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 774 |
+
pyarrow.Table
|
| 775 |
+
animals: string
|
| 776 |
+
n_legs: int64
|
| 777 |
+
location: null
|
| 778 |
+
----
|
| 779 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
|
| 780 |
+
n_legs: [[2,4,5,100,6]]
|
| 781 |
+
location: [5 nulls]
|
| 782 |
+
|
| 783 |
+
Define columns as dictionary type (by default only the
|
| 784 |
+
string/binary columns are dictionary encoded):
|
| 785 |
+
|
| 786 |
+
>>> convert_options = csv.ConvertOptions(
|
| 787 |
+
... timestamp_parsers=["%m/%d/%Y", "%m-%d-%Y"],
|
| 788 |
+
... auto_dict_encode=True)
|
| 789 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 790 |
+
pyarrow.Table
|
| 791 |
+
animals: dictionary<values=string, indices=int32, ordered=0>
|
| 792 |
+
n_legs: int64
|
| 793 |
+
entry: timestamp[s]
|
| 794 |
+
fast: dictionary<values=string, indices=int32, ordered=0>
|
| 795 |
+
----
|
| 796 |
+
animals: [ -- dictionary:
|
| 797 |
+
["Flamingo","Horse","Brittle stars","Centipede",""] -- indices:
|
| 798 |
+
[0,1,2,3,4]]
|
| 799 |
+
n_legs: [[2,4,5,100,6]]
|
| 800 |
+
entry: [[2022-01-03 00:00:00,2022-02-03 00:00:00,2022-03-03 00:00:00,2022-04-03 00:00:00,2022-05-03 00:00:00]]
|
| 801 |
+
fast: [ -- dictionary:
|
| 802 |
+
["Yes","No",""] -- indices:
|
| 803 |
+
[0,0,1,1,2]]
|
| 804 |
+
|
| 805 |
+
Set upper limit for the number of categories. If the categories
|
| 806 |
+
is more than the limit, the conversion to dictionary will not
|
| 807 |
+
happen:
|
| 808 |
+
|
| 809 |
+
>>> convert_options = csv.ConvertOptions(
|
| 810 |
+
... include_columns=["animals"],
|
| 811 |
+
... auto_dict_encode=True,
|
| 812 |
+
... auto_dict_max_cardinality=2)
|
| 813 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 814 |
+
pyarrow.Table
|
| 815 |
+
animals: string
|
| 816 |
+
----
|
| 817 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede",""]]
|
| 818 |
+
|
| 819 |
+
Set empty strings to missing values:
|
| 820 |
+
|
| 821 |
+
>>> convert_options = csv.ConvertOptions(include_columns=["animals", "n_legs"],
|
| 822 |
+
... strings_can_be_null=True)
|
| 823 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 824 |
+
pyarrow.Table
|
| 825 |
+
animals: string
|
| 826 |
+
n_legs: int64
|
| 827 |
+
----
|
| 828 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede",null]]
|
| 829 |
+
n_legs: [[2,4,5,100,6]]
|
| 830 |
+
|
| 831 |
+
Define values to be True and False when converting a column
|
| 832 |
+
into a bool type:
|
| 833 |
+
|
| 834 |
+
>>> convert_options = csv.ConvertOptions(
|
| 835 |
+
... include_columns=["fast"],
|
| 836 |
+
... false_values=["No"],
|
| 837 |
+
... true_values=["Yes"])
|
| 838 |
+
>>> csv.read_csv(io.BytesIO(s.encode()), convert_options=convert_options)
|
| 839 |
+
pyarrow.Table
|
| 840 |
+
fast: bool
|
| 841 |
+
----
|
| 842 |
+
fast: [[true,true,false,false,null]]
|
| 843 |
+
"""
|
| 844 |
+
|
| 845 |
+
# Avoid mistakingly creating attributes
|
| 846 |
+
__slots__ = ()
|
| 847 |
+
|
| 848 |
+
def __cinit__(self, *argw, **kwargs):
|
| 849 |
+
self.options.reset(
|
| 850 |
+
new CCSVConvertOptions(CCSVConvertOptions.Defaults()))
|
| 851 |
+
|
| 852 |
+
def __init__(self, *, check_utf8=None, column_types=None, null_values=None,
|
| 853 |
+
true_values=None, false_values=None, decimal_point=None,
|
| 854 |
+
strings_can_be_null=None, quoted_strings_can_be_null=None,
|
| 855 |
+
include_columns=None, include_missing_columns=None,
|
| 856 |
+
auto_dict_encode=None, auto_dict_max_cardinality=None,
|
| 857 |
+
timestamp_parsers=None):
|
| 858 |
+
if check_utf8 is not None:
|
| 859 |
+
self.check_utf8 = check_utf8
|
| 860 |
+
if column_types is not None:
|
| 861 |
+
self.column_types = column_types
|
| 862 |
+
if null_values is not None:
|
| 863 |
+
self.null_values = null_values
|
| 864 |
+
if true_values is not None:
|
| 865 |
+
self.true_values = true_values
|
| 866 |
+
if false_values is not None:
|
| 867 |
+
self.false_values = false_values
|
| 868 |
+
if decimal_point is not None:
|
| 869 |
+
self.decimal_point = decimal_point
|
| 870 |
+
if strings_can_be_null is not None:
|
| 871 |
+
self.strings_can_be_null = strings_can_be_null
|
| 872 |
+
if quoted_strings_can_be_null is not None:
|
| 873 |
+
self.quoted_strings_can_be_null = quoted_strings_can_be_null
|
| 874 |
+
if include_columns is not None:
|
| 875 |
+
self.include_columns = include_columns
|
| 876 |
+
if include_missing_columns is not None:
|
| 877 |
+
self.include_missing_columns = include_missing_columns
|
| 878 |
+
if auto_dict_encode is not None:
|
| 879 |
+
self.auto_dict_encode = auto_dict_encode
|
| 880 |
+
if auto_dict_max_cardinality is not None:
|
| 881 |
+
self.auto_dict_max_cardinality = auto_dict_max_cardinality
|
| 882 |
+
if timestamp_parsers is not None:
|
| 883 |
+
self.timestamp_parsers = timestamp_parsers
|
| 884 |
+
|
| 885 |
+
@property
|
| 886 |
+
def check_utf8(self):
|
| 887 |
+
"""
|
| 888 |
+
Whether to check UTF8 validity of string columns.
|
| 889 |
+
"""
|
| 890 |
+
return deref(self.options).check_utf8
|
| 891 |
+
|
| 892 |
+
@check_utf8.setter
|
| 893 |
+
def check_utf8(self, value):
|
| 894 |
+
deref(self.options).check_utf8 = value
|
| 895 |
+
|
| 896 |
+
@property
|
| 897 |
+
def strings_can_be_null(self):
|
| 898 |
+
"""
|
| 899 |
+
Whether string / binary columns can have null values.
|
| 900 |
+
"""
|
| 901 |
+
return deref(self.options).strings_can_be_null
|
| 902 |
+
|
| 903 |
+
@strings_can_be_null.setter
|
| 904 |
+
def strings_can_be_null(self, value):
|
| 905 |
+
deref(self.options).strings_can_be_null = value
|
| 906 |
+
|
| 907 |
+
@property
|
| 908 |
+
def quoted_strings_can_be_null(self):
|
| 909 |
+
"""
|
| 910 |
+
Whether quoted values can be null.
|
| 911 |
+
"""
|
| 912 |
+
return deref(self.options).quoted_strings_can_be_null
|
| 913 |
+
|
| 914 |
+
@quoted_strings_can_be_null.setter
|
| 915 |
+
def quoted_strings_can_be_null(self, value):
|
| 916 |
+
deref(self.options).quoted_strings_can_be_null = value
|
| 917 |
+
|
| 918 |
+
@property
|
| 919 |
+
def column_types(self):
|
| 920 |
+
"""
|
| 921 |
+
Explicitly map column names to column types.
|
| 922 |
+
"""
|
| 923 |
+
d = {frombytes(item.first): pyarrow_wrap_data_type(item.second)
|
| 924 |
+
for item in deref(self.options).column_types}
|
| 925 |
+
return d
|
| 926 |
+
|
| 927 |
+
@column_types.setter
|
| 928 |
+
def column_types(self, value):
|
| 929 |
+
cdef:
|
| 930 |
+
shared_ptr[CDataType] typ
|
| 931 |
+
|
| 932 |
+
if isinstance(value, Mapping):
|
| 933 |
+
value = value.items()
|
| 934 |
+
|
| 935 |
+
deref(self.options).column_types.clear()
|
| 936 |
+
for item in value:
|
| 937 |
+
if isinstance(item, Field):
|
| 938 |
+
k = item.name
|
| 939 |
+
v = item.type
|
| 940 |
+
else:
|
| 941 |
+
k, v = item
|
| 942 |
+
typ = pyarrow_unwrap_data_type(ensure_type(v))
|
| 943 |
+
assert typ != NULL
|
| 944 |
+
deref(self.options).column_types[tobytes(k)] = typ
|
| 945 |
+
|
| 946 |
+
@property
|
| 947 |
+
def null_values(self):
|
| 948 |
+
"""
|
| 949 |
+
A sequence of strings that denote nulls in the data.
|
| 950 |
+
"""
|
| 951 |
+
return [frombytes(x) for x in deref(self.options).null_values]
|
| 952 |
+
|
| 953 |
+
@null_values.setter
|
| 954 |
+
def null_values(self, value):
|
| 955 |
+
deref(self.options).null_values = [tobytes(x) for x in value]
|
| 956 |
+
|
| 957 |
+
@property
|
| 958 |
+
def true_values(self):
|
| 959 |
+
"""
|
| 960 |
+
A sequence of strings that denote true booleans in the data.
|
| 961 |
+
"""
|
| 962 |
+
return [frombytes(x) for x in deref(self.options).true_values]
|
| 963 |
+
|
| 964 |
+
@true_values.setter
|
| 965 |
+
def true_values(self, value):
|
| 966 |
+
deref(self.options).true_values = [tobytes(x) for x in value]
|
| 967 |
+
|
| 968 |
+
@property
|
| 969 |
+
def false_values(self):
|
| 970 |
+
"""
|
| 971 |
+
A sequence of strings that denote false booleans in the data.
|
| 972 |
+
"""
|
| 973 |
+
return [frombytes(x) for x in deref(self.options).false_values]
|
| 974 |
+
|
| 975 |
+
@false_values.setter
|
| 976 |
+
def false_values(self, value):
|
| 977 |
+
deref(self.options).false_values = [tobytes(x) for x in value]
|
| 978 |
+
|
| 979 |
+
@property
|
| 980 |
+
def decimal_point(self):
|
| 981 |
+
"""
|
| 982 |
+
The character used as decimal point in floating-point and decimal
|
| 983 |
+
data.
|
| 984 |
+
"""
|
| 985 |
+
return chr(deref(self.options).decimal_point)
|
| 986 |
+
|
| 987 |
+
@decimal_point.setter
|
| 988 |
+
def decimal_point(self, value):
|
| 989 |
+
deref(self.options).decimal_point = _single_char(value)
|
| 990 |
+
|
| 991 |
+
@property
|
| 992 |
+
def auto_dict_encode(self):
|
| 993 |
+
"""
|
| 994 |
+
Whether to try to automatically dict-encode string / binary data.
|
| 995 |
+
"""
|
| 996 |
+
return deref(self.options).auto_dict_encode
|
| 997 |
+
|
| 998 |
+
@auto_dict_encode.setter
|
| 999 |
+
def auto_dict_encode(self, value):
|
| 1000 |
+
deref(self.options).auto_dict_encode = value
|
| 1001 |
+
|
| 1002 |
+
@property
|
| 1003 |
+
def auto_dict_max_cardinality(self):
|
| 1004 |
+
"""
|
| 1005 |
+
The maximum dictionary cardinality for `auto_dict_encode`.
|
| 1006 |
+
|
| 1007 |
+
This value is per chunk.
|
| 1008 |
+
"""
|
| 1009 |
+
return deref(self.options).auto_dict_max_cardinality
|
| 1010 |
+
|
| 1011 |
+
@auto_dict_max_cardinality.setter
|
| 1012 |
+
def auto_dict_max_cardinality(self, value):
|
| 1013 |
+
deref(self.options).auto_dict_max_cardinality = value
|
| 1014 |
+
|
| 1015 |
+
@property
|
| 1016 |
+
def include_columns(self):
|
| 1017 |
+
"""
|
| 1018 |
+
The names of columns to include in the Table.
|
| 1019 |
+
|
| 1020 |
+
If empty, the Table will include all columns from the CSV file.
|
| 1021 |
+
If not empty, only these columns will be included, in this order.
|
| 1022 |
+
"""
|
| 1023 |
+
return [frombytes(s) for s in deref(self.options).include_columns]
|
| 1024 |
+
|
| 1025 |
+
@include_columns.setter
|
| 1026 |
+
def include_columns(self, value):
|
| 1027 |
+
deref(self.options).include_columns.clear()
|
| 1028 |
+
for item in value:
|
| 1029 |
+
deref(self.options).include_columns.push_back(tobytes(item))
|
| 1030 |
+
|
| 1031 |
+
@property
|
| 1032 |
+
def include_missing_columns(self):
|
| 1033 |
+
"""
|
| 1034 |
+
If false, columns in `include_columns` but not in the CSV file will
|
| 1035 |
+
error out.
|
| 1036 |
+
If true, columns in `include_columns` but not in the CSV file will
|
| 1037 |
+
produce a null column (whose type is selected using `column_types`,
|
| 1038 |
+
or null by default).
|
| 1039 |
+
This option is ignored if `include_columns` is empty.
|
| 1040 |
+
"""
|
| 1041 |
+
return deref(self.options).include_missing_columns
|
| 1042 |
+
|
| 1043 |
+
@include_missing_columns.setter
|
| 1044 |
+
def include_missing_columns(self, value):
|
| 1045 |
+
deref(self.options).include_missing_columns = value
|
| 1046 |
+
|
| 1047 |
+
@property
|
| 1048 |
+
def timestamp_parsers(self):
|
| 1049 |
+
"""
|
| 1050 |
+
A sequence of strptime()-compatible format strings, tried in order
|
| 1051 |
+
when attempting to infer or convert timestamp values (the special
|
| 1052 |
+
value ISO8601() can also be given). By default, a fast built-in
|
| 1053 |
+
ISO-8601 parser is used.
|
| 1054 |
+
"""
|
| 1055 |
+
cdef:
|
| 1056 |
+
shared_ptr[CTimestampParser] c_parser
|
| 1057 |
+
c_string kind
|
| 1058 |
+
|
| 1059 |
+
parsers = []
|
| 1060 |
+
for c_parser in deref(self.options).timestamp_parsers:
|
| 1061 |
+
kind = deref(c_parser).kind()
|
| 1062 |
+
if kind == b'strptime':
|
| 1063 |
+
parsers.append(frombytes(deref(c_parser).format()))
|
| 1064 |
+
else:
|
| 1065 |
+
assert kind == b'iso8601'
|
| 1066 |
+
parsers.append(ISO8601)
|
| 1067 |
+
|
| 1068 |
+
return parsers
|
| 1069 |
+
|
| 1070 |
+
@timestamp_parsers.setter
|
| 1071 |
+
def timestamp_parsers(self, value):
|
| 1072 |
+
cdef:
|
| 1073 |
+
vector[shared_ptr[CTimestampParser]] c_parsers
|
| 1074 |
+
|
| 1075 |
+
for v in value:
|
| 1076 |
+
if isinstance(v, str):
|
| 1077 |
+
c_parsers.push_back(CTimestampParser.MakeStrptime(tobytes(v)))
|
| 1078 |
+
elif v == ISO8601:
|
| 1079 |
+
c_parsers.push_back(CTimestampParser.MakeISO8601())
|
| 1080 |
+
else:
|
| 1081 |
+
raise TypeError("Expected list of str or ISO8601 objects")
|
| 1082 |
+
|
| 1083 |
+
deref(self.options).timestamp_parsers = move(c_parsers)
|
| 1084 |
+
|
| 1085 |
+
@staticmethod
|
| 1086 |
+
cdef ConvertOptions wrap(CCSVConvertOptions options):
|
| 1087 |
+
out = ConvertOptions()
|
| 1088 |
+
out.options.reset(new CCSVConvertOptions(move(options)))
|
| 1089 |
+
return out
|
| 1090 |
+
|
| 1091 |
+
def validate(self):
|
| 1092 |
+
check_status(deref(self.options).Validate())
|
| 1093 |
+
|
| 1094 |
+
def equals(self, ConvertOptions other):
|
| 1095 |
+
"""
|
| 1096 |
+
Parameters
|
| 1097 |
+
----------
|
| 1098 |
+
other : pyarrow.csv.ConvertOptions
|
| 1099 |
+
|
| 1100 |
+
Returns
|
| 1101 |
+
-------
|
| 1102 |
+
bool
|
| 1103 |
+
"""
|
| 1104 |
+
return (
|
| 1105 |
+
self.check_utf8 == other.check_utf8 and
|
| 1106 |
+
self.column_types == other.column_types and
|
| 1107 |
+
self.null_values == other.null_values and
|
| 1108 |
+
self.true_values == other.true_values and
|
| 1109 |
+
self.false_values == other.false_values and
|
| 1110 |
+
self.decimal_point == other.decimal_point and
|
| 1111 |
+
self.timestamp_parsers == other.timestamp_parsers and
|
| 1112 |
+
self.strings_can_be_null == other.strings_can_be_null and
|
| 1113 |
+
self.quoted_strings_can_be_null ==
|
| 1114 |
+
other.quoted_strings_can_be_null and
|
| 1115 |
+
self.auto_dict_encode == other.auto_dict_encode and
|
| 1116 |
+
self.auto_dict_max_cardinality ==
|
| 1117 |
+
other.auto_dict_max_cardinality and
|
| 1118 |
+
self.include_columns == other.include_columns and
|
| 1119 |
+
self.include_missing_columns == other.include_missing_columns
|
| 1120 |
+
)
|
| 1121 |
+
|
| 1122 |
+
def __getstate__(self):
|
| 1123 |
+
return (self.check_utf8, self.column_types, self.null_values,
|
| 1124 |
+
self.true_values, self.false_values, self.decimal_point,
|
| 1125 |
+
self.timestamp_parsers, self.strings_can_be_null,
|
| 1126 |
+
self.quoted_strings_can_be_null, self.auto_dict_encode,
|
| 1127 |
+
self.auto_dict_max_cardinality, self.include_columns,
|
| 1128 |
+
self.include_missing_columns)
|
| 1129 |
+
|
| 1130 |
+
def __setstate__(self, state):
|
| 1131 |
+
(self.check_utf8, self.column_types, self.null_values,
|
| 1132 |
+
self.true_values, self.false_values, self.decimal_point,
|
| 1133 |
+
self.timestamp_parsers, self.strings_can_be_null,
|
| 1134 |
+
self.quoted_strings_can_be_null, self.auto_dict_encode,
|
| 1135 |
+
self.auto_dict_max_cardinality, self.include_columns,
|
| 1136 |
+
self.include_missing_columns) = state
|
| 1137 |
+
|
| 1138 |
+
def __eq__(self, other):
|
| 1139 |
+
try:
|
| 1140 |
+
return self.equals(other)
|
| 1141 |
+
except TypeError:
|
| 1142 |
+
return False
|
| 1143 |
+
|
| 1144 |
+
def _repr_base(self):
|
| 1145 |
+
return (f"""
|
| 1146 |
+
check_utf8={self.check_utf8},
|
| 1147 |
+
column_types={self.column_types},
|
| 1148 |
+
null_values={self.null_values},
|
| 1149 |
+
true_values={self.true_values},
|
| 1150 |
+
false_values={self.false_values},
|
| 1151 |
+
decimal_point={self.decimal_point!r},
|
| 1152 |
+
strings_can_be_null={self.strings_can_be_null},
|
| 1153 |
+
quoted_strings_can_be_null={self.quoted_strings_can_be_null},
|
| 1154 |
+
include_columns={self.include_columns},
|
| 1155 |
+
include_missing_columns={self.include_missing_columns},
|
| 1156 |
+
auto_dict_encode={self.auto_dict_encode},
|
| 1157 |
+
auto_dict_max_cardinality={self.auto_dict_max_cardinality},
|
| 1158 |
+
timestamp_parsers={[str(i) for i in self.timestamp_parsers]}""")
|
| 1159 |
+
|
| 1160 |
+
def __repr__(self):
|
| 1161 |
+
return (f"<pyarrow.csv.ConvertOptions>({self._repr_base()})")
|
| 1162 |
+
|
| 1163 |
+
def __str__(self):
|
| 1164 |
+
return (f"ConvertOptions({self._repr_base()})")
|
| 1165 |
+
|
| 1166 |
+
|
| 1167 |
+
cdef _get_reader(input_file, ReadOptions read_options,
|
| 1168 |
+
shared_ptr[CInputStream]* out):
|
| 1169 |
+
use_memory_map = False
|
| 1170 |
+
get_input_stream(input_file, use_memory_map, out)
|
| 1171 |
+
if read_options is not None:
|
| 1172 |
+
out[0] = native_transcoding_input_stream(out[0],
|
| 1173 |
+
read_options.encoding,
|
| 1174 |
+
'utf8')
|
| 1175 |
+
|
| 1176 |
+
|
| 1177 |
+
cdef _get_read_options(ReadOptions read_options, CCSVReadOptions* out):
|
| 1178 |
+
if read_options is None:
|
| 1179 |
+
out[0] = CCSVReadOptions.Defaults()
|
| 1180 |
+
else:
|
| 1181 |
+
out[0] = deref(read_options.options)
|
| 1182 |
+
|
| 1183 |
+
|
| 1184 |
+
cdef _get_parse_options(ParseOptions parse_options, CCSVParseOptions* out):
|
| 1185 |
+
if parse_options is None:
|
| 1186 |
+
out[0] = CCSVParseOptions.Defaults()
|
| 1187 |
+
else:
|
| 1188 |
+
out[0] = deref(parse_options.options)
|
| 1189 |
+
|
| 1190 |
+
|
| 1191 |
+
cdef _get_convert_options(ConvertOptions convert_options,
|
| 1192 |
+
CCSVConvertOptions* out):
|
| 1193 |
+
if convert_options is None:
|
| 1194 |
+
out[0] = CCSVConvertOptions.Defaults()
|
| 1195 |
+
else:
|
| 1196 |
+
out[0] = deref(convert_options.options)
|
| 1197 |
+
|
| 1198 |
+
|
| 1199 |
+
cdef class CSVStreamingReader(RecordBatchReader):
|
| 1200 |
+
"""An object that reads record batches incrementally from a CSV file.
|
| 1201 |
+
|
| 1202 |
+
Should not be instantiated directly by user code.
|
| 1203 |
+
"""
|
| 1204 |
+
cdef readonly:
|
| 1205 |
+
Schema schema
|
| 1206 |
+
|
| 1207 |
+
def __init__(self):
|
| 1208 |
+
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
|
| 1209 |
+
"use pyarrow.csv.open_csv() instead.")
|
| 1210 |
+
|
| 1211 |
+
# Note about cancellation: we cannot create a SignalStopHandler
|
| 1212 |
+
# by default here, as several CSVStreamingReader instances may be
|
| 1213 |
+
# created (including by the same thread). Handling cancellation
|
| 1214 |
+
# would require having the user pass the SignalStopHandler.
|
| 1215 |
+
# (in addition to solving ARROW-11853)
|
| 1216 |
+
|
| 1217 |
+
cdef _open(self, shared_ptr[CInputStream] stream,
|
| 1218 |
+
CCSVReadOptions c_read_options,
|
| 1219 |
+
CCSVParseOptions c_parse_options,
|
| 1220 |
+
CCSVConvertOptions c_convert_options,
|
| 1221 |
+
MemoryPool memory_pool):
|
| 1222 |
+
cdef:
|
| 1223 |
+
shared_ptr[CSchema] c_schema
|
| 1224 |
+
CIOContext io_context
|
| 1225 |
+
|
| 1226 |
+
io_context = CIOContext(maybe_unbox_memory_pool(memory_pool))
|
| 1227 |
+
|
| 1228 |
+
with nogil:
|
| 1229 |
+
self.reader = <shared_ptr[CRecordBatchReader]> GetResultValue(
|
| 1230 |
+
CCSVStreamingReader.Make(
|
| 1231 |
+
io_context, stream,
|
| 1232 |
+
move(c_read_options), move(c_parse_options),
|
| 1233 |
+
move(c_convert_options)))
|
| 1234 |
+
c_schema = self.reader.get().schema()
|
| 1235 |
+
|
| 1236 |
+
self.schema = pyarrow_wrap_schema(c_schema)
|
| 1237 |
+
|
| 1238 |
+
|
| 1239 |
+
def read_csv(input_file, read_options=None, parse_options=None,
|
| 1240 |
+
convert_options=None, MemoryPool memory_pool=None):
|
| 1241 |
+
"""
|
| 1242 |
+
Read a Table from a stream of CSV data.
|
| 1243 |
+
|
| 1244 |
+
Parameters
|
| 1245 |
+
----------
|
| 1246 |
+
input_file : string, path or file-like object
|
| 1247 |
+
The location of CSV data. If a string or path, and if it ends
|
| 1248 |
+
with a recognized compressed file extension (e.g. ".gz" or ".bz2"),
|
| 1249 |
+
the data is automatically decompressed when reading.
|
| 1250 |
+
read_options : pyarrow.csv.ReadOptions, optional
|
| 1251 |
+
Options for the CSV reader (see pyarrow.csv.ReadOptions constructor
|
| 1252 |
+
for defaults)
|
| 1253 |
+
parse_options : pyarrow.csv.ParseOptions, optional
|
| 1254 |
+
Options for the CSV parser
|
| 1255 |
+
(see pyarrow.csv.ParseOptions constructor for defaults)
|
| 1256 |
+
convert_options : pyarrow.csv.ConvertOptions, optional
|
| 1257 |
+
Options for converting CSV data
|
| 1258 |
+
(see pyarrow.csv.ConvertOptions constructor for defaults)
|
| 1259 |
+
memory_pool : MemoryPool, optional
|
| 1260 |
+
Pool to allocate Table memory from
|
| 1261 |
+
|
| 1262 |
+
Returns
|
| 1263 |
+
-------
|
| 1264 |
+
:class:`pyarrow.Table`
|
| 1265 |
+
Contents of the CSV file as a in-memory table.
|
| 1266 |
+
|
| 1267 |
+
Examples
|
| 1268 |
+
--------
|
| 1269 |
+
|
| 1270 |
+
Defining an example file from bytes object:
|
| 1271 |
+
|
| 1272 |
+
>>> import io
|
| 1273 |
+
>>> s = (
|
| 1274 |
+
... "animals,n_legs,entry\\n"
|
| 1275 |
+
... "Flamingo,2,2022-03-01\\n"
|
| 1276 |
+
... "Horse,4,2022-03-02\\n"
|
| 1277 |
+
... "Brittle stars,5,2022-03-03\\n"
|
| 1278 |
+
... "Centipede,100,2022-03-04"
|
| 1279 |
+
... )
|
| 1280 |
+
>>> print(s)
|
| 1281 |
+
animals,n_legs,entry
|
| 1282 |
+
Flamingo,2,2022-03-01
|
| 1283 |
+
Horse,4,2022-03-02
|
| 1284 |
+
Brittle stars,5,2022-03-03
|
| 1285 |
+
Centipede,100,2022-03-04
|
| 1286 |
+
>>> source = io.BytesIO(s.encode())
|
| 1287 |
+
|
| 1288 |
+
Reading from the file
|
| 1289 |
+
|
| 1290 |
+
>>> from pyarrow import csv
|
| 1291 |
+
>>> csv.read_csv(source)
|
| 1292 |
+
pyarrow.Table
|
| 1293 |
+
animals: string
|
| 1294 |
+
n_legs: int64
|
| 1295 |
+
entry: date32[day]
|
| 1296 |
+
----
|
| 1297 |
+
animals: [["Flamingo","Horse","Brittle stars","Centipede"]]
|
| 1298 |
+
n_legs: [[2,4,5,100]]
|
| 1299 |
+
entry: [[2022-03-01,2022-03-02,2022-03-03,2022-03-04]]
|
| 1300 |
+
"""
|
| 1301 |
+
cdef:
|
| 1302 |
+
shared_ptr[CInputStream] stream
|
| 1303 |
+
CCSVReadOptions c_read_options
|
| 1304 |
+
CCSVParseOptions c_parse_options
|
| 1305 |
+
CCSVConvertOptions c_convert_options
|
| 1306 |
+
CIOContext io_context
|
| 1307 |
+
SharedPtrNoGIL[CCSVReader] reader
|
| 1308 |
+
shared_ptr[CTable] table
|
| 1309 |
+
|
| 1310 |
+
_get_reader(input_file, read_options, &stream)
|
| 1311 |
+
_get_read_options(read_options, &c_read_options)
|
| 1312 |
+
_get_parse_options(parse_options, &c_parse_options)
|
| 1313 |
+
_get_convert_options(convert_options, &c_convert_options)
|
| 1314 |
+
|
| 1315 |
+
with SignalStopHandler() as stop_handler:
|
| 1316 |
+
io_context = CIOContext(
|
| 1317 |
+
maybe_unbox_memory_pool(memory_pool),
|
| 1318 |
+
(<StopToken> stop_handler.stop_token).stop_token)
|
| 1319 |
+
reader = GetResultValue(CCSVReader.Make(
|
| 1320 |
+
io_context, stream,
|
| 1321 |
+
c_read_options, c_parse_options, c_convert_options))
|
| 1322 |
+
|
| 1323 |
+
with nogil:
|
| 1324 |
+
table = GetResultValue(reader.get().Read())
|
| 1325 |
+
|
| 1326 |
+
return pyarrow_wrap_table(table)
|
| 1327 |
+
|
| 1328 |
+
|
| 1329 |
+
def open_csv(input_file, read_options=None, parse_options=None,
|
| 1330 |
+
convert_options=None, MemoryPool memory_pool=None):
|
| 1331 |
+
"""
|
| 1332 |
+
Open a streaming reader of CSV data.
|
| 1333 |
+
|
| 1334 |
+
Reading using this function is always single-threaded.
|
| 1335 |
+
|
| 1336 |
+
Parameters
|
| 1337 |
+
----------
|
| 1338 |
+
input_file : string, path or file-like object
|
| 1339 |
+
The location of CSV data. If a string or path, and if it ends
|
| 1340 |
+
with a recognized compressed file extension (e.g. ".gz" or ".bz2"),
|
| 1341 |
+
the data is automatically decompressed when reading.
|
| 1342 |
+
read_options : pyarrow.csv.ReadOptions, optional
|
| 1343 |
+
Options for the CSV reader (see pyarrow.csv.ReadOptions constructor
|
| 1344 |
+
for defaults)
|
| 1345 |
+
parse_options : pyarrow.csv.ParseOptions, optional
|
| 1346 |
+
Options for the CSV parser
|
| 1347 |
+
(see pyarrow.csv.ParseOptions constructor for defaults)
|
| 1348 |
+
convert_options : pyarrow.csv.ConvertOptions, optional
|
| 1349 |
+
Options for converting CSV data
|
| 1350 |
+
(see pyarrow.csv.ConvertOptions constructor for defaults)
|
| 1351 |
+
memory_pool : MemoryPool, optional
|
| 1352 |
+
Pool to allocate RecordBatch memory from
|
| 1353 |
+
|
| 1354 |
+
Returns
|
| 1355 |
+
-------
|
| 1356 |
+
:class:`pyarrow.csv.CSVStreamingReader`
|
| 1357 |
+
"""
|
| 1358 |
+
cdef:
|
| 1359 |
+
shared_ptr[CInputStream] stream
|
| 1360 |
+
CCSVReadOptions c_read_options
|
| 1361 |
+
CCSVParseOptions c_parse_options
|
| 1362 |
+
CCSVConvertOptions c_convert_options
|
| 1363 |
+
CSVStreamingReader reader
|
| 1364 |
+
|
| 1365 |
+
_get_reader(input_file, read_options, &stream)
|
| 1366 |
+
_get_read_options(read_options, &c_read_options)
|
| 1367 |
+
_get_parse_options(parse_options, &c_parse_options)
|
| 1368 |
+
_get_convert_options(convert_options, &c_convert_options)
|
| 1369 |
+
|
| 1370 |
+
reader = CSVStreamingReader.__new__(CSVStreamingReader)
|
| 1371 |
+
reader._open(stream, move(c_read_options), move(c_parse_options),
|
| 1372 |
+
move(c_convert_options), memory_pool)
|
| 1373 |
+
return reader
|
| 1374 |
+
|
| 1375 |
+
|
| 1376 |
+
def _raise_invalid_function_option(value, description, *,
|
| 1377 |
+
exception_class=ValueError):
|
| 1378 |
+
raise exception_class(f"\"{value}\" is not a valid {description}")
|
| 1379 |
+
|
| 1380 |
+
|
| 1381 |
+
cdef CQuotingStyle unwrap_quoting_style(quoting_style) except *:
|
| 1382 |
+
if quoting_style == "needed":
|
| 1383 |
+
return CQuotingStyle_Needed
|
| 1384 |
+
elif quoting_style == "all_valid":
|
| 1385 |
+
return CQuotingStyle_AllValid
|
| 1386 |
+
elif quoting_style == "none":
|
| 1387 |
+
return CQuotingStyle_None
|
| 1388 |
+
_raise_invalid_function_option(quoting_style, "quoting style")
|
| 1389 |
+
|
| 1390 |
+
|
| 1391 |
+
cdef wrap_quoting_style(quoting_style):
|
| 1392 |
+
if quoting_style == CQuotingStyle_Needed:
|
| 1393 |
+
return 'needed'
|
| 1394 |
+
elif quoting_style == CQuotingStyle_AllValid:
|
| 1395 |
+
return 'all_valid'
|
| 1396 |
+
elif quoting_style == CQuotingStyle_None:
|
| 1397 |
+
return 'none'
|
| 1398 |
+
|
| 1399 |
+
|
| 1400 |
+
cdef class WriteOptions(_Weakrefable):
|
| 1401 |
+
"""
|
| 1402 |
+
Options for writing CSV files.
|
| 1403 |
+
|
| 1404 |
+
Parameters
|
| 1405 |
+
----------
|
| 1406 |
+
include_header : bool, optional (default True)
|
| 1407 |
+
Whether to write an initial header line with column names
|
| 1408 |
+
batch_size : int, optional (default 1024)
|
| 1409 |
+
How many rows to process together when converting and writing
|
| 1410 |
+
CSV data
|
| 1411 |
+
delimiter : 1-character string, optional (default ",")
|
| 1412 |
+
The character delimiting individual cells in the CSV data.
|
| 1413 |
+
quoting_style : str, optional (default "needed")
|
| 1414 |
+
Whether to quote values, and if so, which quoting style to use.
|
| 1415 |
+
The following values are accepted:
|
| 1416 |
+
|
| 1417 |
+
- "needed" (default): only enclose values in quotes when needed.
|
| 1418 |
+
- "all_valid": enclose all valid values in quotes; nulls are not quoted.
|
| 1419 |
+
- "none": do not enclose any values in quotes; values containing
|
| 1420 |
+
special characters (such as quotes, cell delimiters or line endings)
|
| 1421 |
+
will raise an error.
|
| 1422 |
+
quoting_header : str, optional (default "needed")
|
| 1423 |
+
Same as quoting_style, but for header column names. Accepts same values.
|
| 1424 |
+
Note : both "needed" and "all_valid" have the same effect of quoting all column names.
|
| 1425 |
+
"""
|
| 1426 |
+
|
| 1427 |
+
# Avoid mistakingly creating attributes
|
| 1428 |
+
__slots__ = ()
|
| 1429 |
+
|
| 1430 |
+
def __init__(self, *, include_header=None, batch_size=None,
|
| 1431 |
+
delimiter=None, quoting_style=None, quoting_header=None):
|
| 1432 |
+
self.options.reset(new CCSVWriteOptions(CCSVWriteOptions.Defaults()))
|
| 1433 |
+
if include_header is not None:
|
| 1434 |
+
self.include_header = include_header
|
| 1435 |
+
if batch_size is not None:
|
| 1436 |
+
self.batch_size = batch_size
|
| 1437 |
+
if delimiter is not None:
|
| 1438 |
+
self.delimiter = delimiter
|
| 1439 |
+
if quoting_style is not None:
|
| 1440 |
+
self.quoting_style = quoting_style
|
| 1441 |
+
if quoting_header is not None:
|
| 1442 |
+
self.quoting_header = quoting_header
|
| 1443 |
+
|
| 1444 |
+
@property
|
| 1445 |
+
def include_header(self):
|
| 1446 |
+
"""
|
| 1447 |
+
Whether to write an initial header line with column names.
|
| 1448 |
+
"""
|
| 1449 |
+
return deref(self.options).include_header
|
| 1450 |
+
|
| 1451 |
+
@include_header.setter
|
| 1452 |
+
def include_header(self, value):
|
| 1453 |
+
deref(self.options).include_header = value
|
| 1454 |
+
|
| 1455 |
+
@property
|
| 1456 |
+
def batch_size(self):
|
| 1457 |
+
"""
|
| 1458 |
+
How many rows to process together when converting and writing
|
| 1459 |
+
CSV data.
|
| 1460 |
+
"""
|
| 1461 |
+
return deref(self.options).batch_size
|
| 1462 |
+
|
| 1463 |
+
@batch_size.setter
|
| 1464 |
+
def batch_size(self, value):
|
| 1465 |
+
deref(self.options).batch_size = value
|
| 1466 |
+
|
| 1467 |
+
@property
|
| 1468 |
+
def delimiter(self):
|
| 1469 |
+
"""
|
| 1470 |
+
The character delimiting individual cells in the CSV data.
|
| 1471 |
+
"""
|
| 1472 |
+
return chr(deref(self.options).delimiter)
|
| 1473 |
+
|
| 1474 |
+
@delimiter.setter
|
| 1475 |
+
def delimiter(self, value):
|
| 1476 |
+
deref(self.options).delimiter = _single_char(value)
|
| 1477 |
+
|
| 1478 |
+
@property
|
| 1479 |
+
def quoting_style(self):
|
| 1480 |
+
"""
|
| 1481 |
+
Whether to quote values, and if so, which quoting style to use.
|
| 1482 |
+
The following values are accepted:
|
| 1483 |
+
|
| 1484 |
+
- "needed" (default): only enclose values in quotes when needed.
|
| 1485 |
+
- "all_valid": enclose all valid values in quotes; nulls are not quoted.
|
| 1486 |
+
- "none": do not enclose any values in quotes; values containing
|
| 1487 |
+
special characters (such as quotes, cell delimiters or line endings)
|
| 1488 |
+
will raise an error.
|
| 1489 |
+
"""
|
| 1490 |
+
return wrap_quoting_style(deref(self.options).quoting_style)
|
| 1491 |
+
|
| 1492 |
+
@quoting_style.setter
|
| 1493 |
+
def quoting_style(self, value):
|
| 1494 |
+
deref(self.options).quoting_style = unwrap_quoting_style(value)
|
| 1495 |
+
|
| 1496 |
+
@property
|
| 1497 |
+
def quoting_header(self):
|
| 1498 |
+
"""
|
| 1499 |
+
Same as quoting_style, but for header column names.
|
| 1500 |
+
Note : both "needed" and "all_valid" have the same effect of quoting all column names.
|
| 1501 |
+
"""
|
| 1502 |
+
return wrap_quoting_style(deref(self.options).quoting_header)
|
| 1503 |
+
|
| 1504 |
+
@quoting_header.setter
|
| 1505 |
+
def quoting_header(self, value):
|
| 1506 |
+
deref(self.options).quoting_header = unwrap_quoting_style(value)
|
| 1507 |
+
|
| 1508 |
+
@staticmethod
|
| 1509 |
+
cdef WriteOptions wrap(CCSVWriteOptions options):
|
| 1510 |
+
out = WriteOptions()
|
| 1511 |
+
out.options.reset(new CCSVWriteOptions(move(options)))
|
| 1512 |
+
return out
|
| 1513 |
+
|
| 1514 |
+
def validate(self):
|
| 1515 |
+
check_status(self.options.get().Validate())
|
| 1516 |
+
|
| 1517 |
+
def _repr_base(self):
|
| 1518 |
+
return (f"""
|
| 1519 |
+
include_header={self.include_header},
|
| 1520 |
+
batch_size={self.batch_size},
|
| 1521 |
+
delimiter={self.delimiter!r},
|
| 1522 |
+
quoting_style={self.quoting_style!r}""")
|
| 1523 |
+
|
| 1524 |
+
def __repr__(self):
|
| 1525 |
+
return (f"<pyarrow.csv.WriteOptions>({self._repr_base()})")
|
| 1526 |
+
|
| 1527 |
+
def __str__(self):
|
| 1528 |
+
return (f"WriteOptions({self._repr_base()})")
|
| 1529 |
+
|
| 1530 |
+
|
| 1531 |
+
cdef _get_write_options(WriteOptions write_options, CCSVWriteOptions* out):
|
| 1532 |
+
if write_options is None:
|
| 1533 |
+
out[0] = CCSVWriteOptions.Defaults()
|
| 1534 |
+
else:
|
| 1535 |
+
out[0] = deref(write_options.options)
|
| 1536 |
+
|
| 1537 |
+
|
| 1538 |
+
def write_csv(data, output_file, write_options=None,
|
| 1539 |
+
MemoryPool memory_pool=None):
|
| 1540 |
+
"""
|
| 1541 |
+
Write record batch or table to a CSV file.
|
| 1542 |
+
|
| 1543 |
+
Parameters
|
| 1544 |
+
----------
|
| 1545 |
+
data : pyarrow.RecordBatch or pyarrow.Table
|
| 1546 |
+
The data to write.
|
| 1547 |
+
output_file : string, path, pyarrow.NativeFile, or file-like object
|
| 1548 |
+
The location where to write the CSV data.
|
| 1549 |
+
write_options : pyarrow.csv.WriteOptions
|
| 1550 |
+
Options to configure writing the CSV data.
|
| 1551 |
+
memory_pool : MemoryPool, optional
|
| 1552 |
+
Pool for temporary allocations.
|
| 1553 |
+
|
| 1554 |
+
Examples
|
| 1555 |
+
--------
|
| 1556 |
+
|
| 1557 |
+
>>> import pyarrow as pa
|
| 1558 |
+
>>> from pyarrow import csv
|
| 1559 |
+
|
| 1560 |
+
>>> legs = pa.array([2, 4, 5, 100])
|
| 1561 |
+
>>> animals = pa.array(["Flamingo", "Horse", "Brittle stars", "Centipede"])
|
| 1562 |
+
>>> entry_date = pa.array(["01/03/2022", "02/03/2022",
|
| 1563 |
+
... "03/03/2022", "04/03/2022"])
|
| 1564 |
+
>>> table = pa.table([animals, legs, entry_date],
|
| 1565 |
+
... names=["animals", "n_legs", "entry"])
|
| 1566 |
+
|
| 1567 |
+
>>> csv.write_csv(table, "animals.csv")
|
| 1568 |
+
|
| 1569 |
+
>>> write_options = csv.WriteOptions(include_header=False)
|
| 1570 |
+
>>> csv.write_csv(table, "animals.csv", write_options=write_options)
|
| 1571 |
+
|
| 1572 |
+
>>> write_options = csv.WriteOptions(delimiter=";")
|
| 1573 |
+
>>> csv.write_csv(table, "animals.csv", write_options=write_options)
|
| 1574 |
+
"""
|
| 1575 |
+
cdef:
|
| 1576 |
+
shared_ptr[COutputStream] stream
|
| 1577 |
+
CCSVWriteOptions c_write_options
|
| 1578 |
+
CMemoryPool* c_memory_pool
|
| 1579 |
+
CRecordBatch* batch
|
| 1580 |
+
CTable* table
|
| 1581 |
+
_get_write_options(write_options, &c_write_options)
|
| 1582 |
+
|
| 1583 |
+
get_writer(output_file, &stream)
|
| 1584 |
+
c_memory_pool = maybe_unbox_memory_pool(memory_pool)
|
| 1585 |
+
c_write_options.io_context = CIOContext(c_memory_pool)
|
| 1586 |
+
if isinstance(data, RecordBatch):
|
| 1587 |
+
batch = pyarrow_unwrap_batch(data).get()
|
| 1588 |
+
with nogil:
|
| 1589 |
+
check_status(WriteCSV(deref(batch), c_write_options, stream.get()))
|
| 1590 |
+
elif isinstance(data, Table):
|
| 1591 |
+
table = pyarrow_unwrap_table(data).get()
|
| 1592 |
+
with nogil:
|
| 1593 |
+
check_status(WriteCSV(deref(table), c_write_options, stream.get()))
|
| 1594 |
+
else:
|
| 1595 |
+
raise TypeError(f"Expected Table or RecordBatch, got '{type(data)}'")
|
| 1596 |
+
|
| 1597 |
+
|
| 1598 |
+
cdef class CSVWriter(_CRecordBatchWriter):
|
| 1599 |
+
"""
|
| 1600 |
+
Writer to create a CSV file.
|
| 1601 |
+
|
| 1602 |
+
Parameters
|
| 1603 |
+
----------
|
| 1604 |
+
sink : str, path, pyarrow.OutputStream or file-like object
|
| 1605 |
+
The location where to write the CSV data.
|
| 1606 |
+
schema : pyarrow.Schema
|
| 1607 |
+
The schema of the data to be written.
|
| 1608 |
+
write_options : pyarrow.csv.WriteOptions
|
| 1609 |
+
Options to configure writing the CSV data.
|
| 1610 |
+
memory_pool : MemoryPool, optional
|
| 1611 |
+
Pool for temporary allocations.
|
| 1612 |
+
"""
|
| 1613 |
+
|
| 1614 |
+
def __init__(self, sink, Schema schema, *,
|
| 1615 |
+
WriteOptions write_options=None, MemoryPool memory_pool=None):
|
| 1616 |
+
cdef:
|
| 1617 |
+
shared_ptr[COutputStream] c_stream
|
| 1618 |
+
shared_ptr[CSchema] c_schema = pyarrow_unwrap_schema(schema)
|
| 1619 |
+
CCSVWriteOptions c_write_options
|
| 1620 |
+
CMemoryPool* c_memory_pool = maybe_unbox_memory_pool(memory_pool)
|
| 1621 |
+
_get_write_options(write_options, &c_write_options)
|
| 1622 |
+
c_write_options.io_context = CIOContext(c_memory_pool)
|
| 1623 |
+
get_writer(sink, &c_stream)
|
| 1624 |
+
with nogil:
|
| 1625 |
+
self.writer = GetResultValue(MakeCSVWriter(
|
| 1626 |
+
c_stream, c_schema, c_write_options))
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pxd
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.lib cimport *
|
| 21 |
+
from pyarrow.includes.common cimport *
|
| 22 |
+
from pyarrow.includes.libarrow cimport *
|
| 23 |
+
from pyarrow.includes.libarrow_cuda cimport *
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
cdef class Context(_Weakrefable):
|
| 27 |
+
cdef:
|
| 28 |
+
shared_ptr[CCudaContext] context
|
| 29 |
+
int device_number
|
| 30 |
+
|
| 31 |
+
cdef void init(self, const shared_ptr[CCudaContext]& ctx)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
cdef class IpcMemHandle(_Weakrefable):
|
| 35 |
+
cdef:
|
| 36 |
+
shared_ptr[CCudaIpcMemHandle] handle
|
| 37 |
+
|
| 38 |
+
cdef void init(self, shared_ptr[CCudaIpcMemHandle]& h)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
cdef class CudaBuffer(Buffer):
|
| 42 |
+
cdef:
|
| 43 |
+
shared_ptr[CCudaBuffer] cuda_buffer
|
| 44 |
+
object base
|
| 45 |
+
|
| 46 |
+
cdef void init_cuda(self,
|
| 47 |
+
const shared_ptr[CCudaBuffer]& buffer,
|
| 48 |
+
object base)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
cdef class HostBuffer(Buffer):
|
| 52 |
+
cdef:
|
| 53 |
+
shared_ptr[CCudaHostBuffer] host_buffer
|
| 54 |
+
|
| 55 |
+
cdef void init_host(self, const shared_ptr[CCudaHostBuffer]& buffer)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
cdef class BufferReader(NativeFile):
|
| 59 |
+
cdef:
|
| 60 |
+
CCudaBufferReader* reader
|
| 61 |
+
CudaBuffer buffer
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
cdef class BufferWriter(NativeFile):
|
| 65 |
+
cdef:
|
| 66 |
+
CCudaBufferWriter* writer
|
| 67 |
+
CudaBuffer buffer
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_cuda.pyx
ADDED
|
@@ -0,0 +1,1084 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
from pyarrow.lib cimport *
|
| 20 |
+
from pyarrow.includes.libarrow_cuda cimport *
|
| 21 |
+
from pyarrow.lib import allocate_buffer, as_buffer, ArrowTypeError
|
| 22 |
+
from pyarrow.util import get_contiguous_span
|
| 23 |
+
cimport cpython as cp
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
cdef class Context(_Weakrefable):
|
| 27 |
+
"""
|
| 28 |
+
CUDA driver context.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
def __init__(self, *args, **kwargs):
|
| 32 |
+
"""
|
| 33 |
+
Create a CUDA driver context for a particular device.
|
| 34 |
+
|
| 35 |
+
If a CUDA context handle is passed, it is wrapped, otherwise
|
| 36 |
+
a default CUDA context for the given device is requested.
|
| 37 |
+
|
| 38 |
+
Parameters
|
| 39 |
+
----------
|
| 40 |
+
device_number : int (default 0)
|
| 41 |
+
Specify the GPU device for which the CUDA driver context is
|
| 42 |
+
requested.
|
| 43 |
+
handle : int, optional
|
| 44 |
+
Specify CUDA handle for a shared context that has been created
|
| 45 |
+
by another library.
|
| 46 |
+
"""
|
| 47 |
+
# This method exposed because autodoc doesn't pick __cinit__
|
| 48 |
+
|
| 49 |
+
def __cinit__(self, int device_number=0, uintptr_t handle=0):
|
| 50 |
+
cdef CCudaDeviceManager* manager
|
| 51 |
+
manager = GetResultValue(CCudaDeviceManager.Instance())
|
| 52 |
+
cdef int n = manager.num_devices()
|
| 53 |
+
if device_number >= n or device_number < 0:
|
| 54 |
+
self.context.reset()
|
| 55 |
+
raise ValueError('device_number argument must be '
|
| 56 |
+
'non-negative less than %s' % (n))
|
| 57 |
+
if handle == 0:
|
| 58 |
+
self.context = GetResultValue(manager.GetContext(device_number))
|
| 59 |
+
else:
|
| 60 |
+
self.context = GetResultValue(manager.GetSharedContext(
|
| 61 |
+
device_number, <void*>handle))
|
| 62 |
+
self.device_number = device_number
|
| 63 |
+
|
| 64 |
+
@staticmethod
|
| 65 |
+
def from_numba(context=None):
|
| 66 |
+
"""
|
| 67 |
+
Create a Context instance from a Numba CUDA context.
|
| 68 |
+
|
| 69 |
+
Parameters
|
| 70 |
+
----------
|
| 71 |
+
context : {numba.cuda.cudadrv.driver.Context, None}
|
| 72 |
+
A Numba CUDA context instance.
|
| 73 |
+
If None, the current Numba context is used.
|
| 74 |
+
|
| 75 |
+
Returns
|
| 76 |
+
-------
|
| 77 |
+
shared_context : pyarrow.cuda.Context
|
| 78 |
+
Context instance.
|
| 79 |
+
"""
|
| 80 |
+
if context is None:
|
| 81 |
+
import numba.cuda
|
| 82 |
+
context = numba.cuda.current_context()
|
| 83 |
+
return Context(device_number=context.device.id,
|
| 84 |
+
handle=context.handle.value)
|
| 85 |
+
|
| 86 |
+
def to_numba(self):
|
| 87 |
+
"""
|
| 88 |
+
Convert Context to a Numba CUDA context.
|
| 89 |
+
|
| 90 |
+
Returns
|
| 91 |
+
-------
|
| 92 |
+
context : numba.cuda.cudadrv.driver.Context
|
| 93 |
+
Numba CUDA context instance.
|
| 94 |
+
"""
|
| 95 |
+
import ctypes
|
| 96 |
+
import numba.cuda
|
| 97 |
+
device = numba.cuda.gpus[self.device_number]
|
| 98 |
+
handle = ctypes.c_void_p(self.handle)
|
| 99 |
+
context = numba.cuda.cudadrv.driver.Context(device, handle)
|
| 100 |
+
|
| 101 |
+
class DummyPendingDeallocs(object):
|
| 102 |
+
# Context is managed by pyarrow
|
| 103 |
+
def add_item(self, *args, **kwargs):
|
| 104 |
+
pass
|
| 105 |
+
|
| 106 |
+
context.deallocations = DummyPendingDeallocs()
|
| 107 |
+
return context
|
| 108 |
+
|
| 109 |
+
@staticmethod
|
| 110 |
+
def get_num_devices():
|
| 111 |
+
""" Return the number of GPU devices.
|
| 112 |
+
"""
|
| 113 |
+
cdef CCudaDeviceManager* manager
|
| 114 |
+
manager = GetResultValue(CCudaDeviceManager.Instance())
|
| 115 |
+
return manager.num_devices()
|
| 116 |
+
|
| 117 |
+
@property
|
| 118 |
+
def device_number(self):
|
| 119 |
+
""" Return context device number.
|
| 120 |
+
"""
|
| 121 |
+
return self.device_number
|
| 122 |
+
|
| 123 |
+
@property
|
| 124 |
+
def handle(self):
|
| 125 |
+
""" Return pointer to context handle.
|
| 126 |
+
"""
|
| 127 |
+
return <uintptr_t>self.context.get().handle()
|
| 128 |
+
|
| 129 |
+
cdef void init(self, const shared_ptr[CCudaContext]& ctx):
|
| 130 |
+
self.context = ctx
|
| 131 |
+
|
| 132 |
+
def synchronize(self):
|
| 133 |
+
"""Blocks until the device has completed all preceding requested
|
| 134 |
+
tasks.
|
| 135 |
+
"""
|
| 136 |
+
check_status(self.context.get().Synchronize())
|
| 137 |
+
|
| 138 |
+
@property
|
| 139 |
+
def bytes_allocated(self):
|
| 140 |
+
"""Return the number of allocated bytes.
|
| 141 |
+
"""
|
| 142 |
+
return self.context.get().bytes_allocated()
|
| 143 |
+
|
| 144 |
+
def get_device_address(self, uintptr_t address):
|
| 145 |
+
"""Return the device address that is reachable from kernels running in
|
| 146 |
+
the context
|
| 147 |
+
|
| 148 |
+
Parameters
|
| 149 |
+
----------
|
| 150 |
+
address : int
|
| 151 |
+
Specify memory address value
|
| 152 |
+
|
| 153 |
+
Returns
|
| 154 |
+
-------
|
| 155 |
+
device_address : int
|
| 156 |
+
Device address accessible from device context
|
| 157 |
+
|
| 158 |
+
Notes
|
| 159 |
+
-----
|
| 160 |
+
The device address is defined as a memory address accessible
|
| 161 |
+
by device. While it is often a device memory address but it
|
| 162 |
+
can be also a host memory address, for instance, when the
|
| 163 |
+
memory is allocated as host memory (using cudaMallocHost or
|
| 164 |
+
cudaHostAlloc) or as managed memory (using cudaMallocManaged)
|
| 165 |
+
or the host memory is page-locked (using cudaHostRegister).
|
| 166 |
+
"""
|
| 167 |
+
return GetResultValue(self.context.get().GetDeviceAddress(address))
|
| 168 |
+
|
| 169 |
+
def new_buffer(self, int64_t nbytes):
|
| 170 |
+
"""Return new device buffer.
|
| 171 |
+
|
| 172 |
+
Parameters
|
| 173 |
+
----------
|
| 174 |
+
nbytes : int
|
| 175 |
+
Specify the number of bytes to be allocated.
|
| 176 |
+
|
| 177 |
+
Returns
|
| 178 |
+
-------
|
| 179 |
+
buf : CudaBuffer
|
| 180 |
+
Allocated buffer.
|
| 181 |
+
"""
|
| 182 |
+
cdef:
|
| 183 |
+
shared_ptr[CCudaBuffer] cudabuf
|
| 184 |
+
with nogil:
|
| 185 |
+
cudabuf = GetResultValue(self.context.get().Allocate(nbytes))
|
| 186 |
+
return pyarrow_wrap_cudabuffer(cudabuf)
|
| 187 |
+
|
| 188 |
+
@property
|
| 189 |
+
def memory_manager(self):
|
| 190 |
+
"""
|
| 191 |
+
The default memory manager tied to this context's device.
|
| 192 |
+
|
| 193 |
+
Returns
|
| 194 |
+
-------
|
| 195 |
+
MemoryManager
|
| 196 |
+
"""
|
| 197 |
+
return MemoryManager.wrap(self.context.get().memory_manager())
|
| 198 |
+
|
| 199 |
+
@property
|
| 200 |
+
def device(self):
|
| 201 |
+
"""
|
| 202 |
+
The device instance associated with this context.
|
| 203 |
+
|
| 204 |
+
Returns
|
| 205 |
+
-------
|
| 206 |
+
Device
|
| 207 |
+
"""
|
| 208 |
+
return Device.wrap(self.context.get().device())
|
| 209 |
+
|
| 210 |
+
def foreign_buffer(self, address, size, base=None):
|
| 211 |
+
"""
|
| 212 |
+
Create device buffer from address and size as a view.
|
| 213 |
+
|
| 214 |
+
The caller is responsible for allocating and freeing the
|
| 215 |
+
memory. When `address==size==0` then a new zero-sized buffer
|
| 216 |
+
is returned.
|
| 217 |
+
|
| 218 |
+
Parameters
|
| 219 |
+
----------
|
| 220 |
+
address : int
|
| 221 |
+
Specify the starting address of the buffer. The address can
|
| 222 |
+
refer to both device or host memory but it must be
|
| 223 |
+
accessible from device after mapping it with
|
| 224 |
+
`get_device_address` method.
|
| 225 |
+
size : int
|
| 226 |
+
Specify the size of device buffer in bytes.
|
| 227 |
+
base : {None, object}
|
| 228 |
+
Specify object that owns the referenced memory.
|
| 229 |
+
|
| 230 |
+
Returns
|
| 231 |
+
-------
|
| 232 |
+
cbuf : CudaBuffer
|
| 233 |
+
Device buffer as a view of device reachable memory.
|
| 234 |
+
|
| 235 |
+
"""
|
| 236 |
+
if not address and size == 0:
|
| 237 |
+
return self.new_buffer(0)
|
| 238 |
+
cdef:
|
| 239 |
+
uintptr_t c_addr = self.get_device_address(address)
|
| 240 |
+
int64_t c_size = size
|
| 241 |
+
shared_ptr[CCudaBuffer] cudabuf
|
| 242 |
+
|
| 243 |
+
cudabuf = GetResultValue(self.context.get().View(
|
| 244 |
+
<uint8_t*>c_addr, c_size))
|
| 245 |
+
return pyarrow_wrap_cudabuffer_base(cudabuf, base)
|
| 246 |
+
|
| 247 |
+
def open_ipc_buffer(self, ipc_handle):
|
| 248 |
+
""" Open existing CUDA IPC memory handle
|
| 249 |
+
|
| 250 |
+
Parameters
|
| 251 |
+
----------
|
| 252 |
+
ipc_handle : IpcMemHandle
|
| 253 |
+
Specify opaque pointer to CUipcMemHandle (driver API).
|
| 254 |
+
|
| 255 |
+
Returns
|
| 256 |
+
-------
|
| 257 |
+
buf : CudaBuffer
|
| 258 |
+
referencing device buffer
|
| 259 |
+
"""
|
| 260 |
+
handle = pyarrow_unwrap_cudaipcmemhandle(ipc_handle)
|
| 261 |
+
cdef shared_ptr[CCudaBuffer] cudabuf
|
| 262 |
+
with nogil:
|
| 263 |
+
cudabuf = GetResultValue(
|
| 264 |
+
self.context.get().OpenIpcBuffer(handle.get()[0]))
|
| 265 |
+
return pyarrow_wrap_cudabuffer(cudabuf)
|
| 266 |
+
|
| 267 |
+
def buffer_from_data(self, object data, int64_t offset=0, int64_t size=-1):
|
| 268 |
+
"""Create device buffer and initialize with data.
|
| 269 |
+
|
| 270 |
+
Parameters
|
| 271 |
+
----------
|
| 272 |
+
data : {CudaBuffer, HostBuffer, Buffer, array-like}
|
| 273 |
+
Specify data to be copied to device buffer.
|
| 274 |
+
offset : int
|
| 275 |
+
Specify the offset of input buffer for device data
|
| 276 |
+
buffering. Default: 0.
|
| 277 |
+
size : int
|
| 278 |
+
Specify the size of device buffer in bytes. Default: all
|
| 279 |
+
(starting from input offset)
|
| 280 |
+
|
| 281 |
+
Returns
|
| 282 |
+
-------
|
| 283 |
+
cbuf : CudaBuffer
|
| 284 |
+
Device buffer with copied data.
|
| 285 |
+
"""
|
| 286 |
+
is_host_data = not pyarrow_is_cudabuffer(data)
|
| 287 |
+
buf = as_buffer(data) if is_host_data else data
|
| 288 |
+
|
| 289 |
+
bsize = buf.size
|
| 290 |
+
if offset < 0 or (bsize and offset >= bsize):
|
| 291 |
+
raise ValueError('offset argument is out-of-range')
|
| 292 |
+
if size < 0:
|
| 293 |
+
size = bsize - offset
|
| 294 |
+
elif offset + size > bsize:
|
| 295 |
+
raise ValueError(
|
| 296 |
+
'requested larger slice than available in device buffer')
|
| 297 |
+
|
| 298 |
+
if offset != 0 or size != bsize:
|
| 299 |
+
buf = buf.slice(offset, size)
|
| 300 |
+
|
| 301 |
+
result = self.new_buffer(size)
|
| 302 |
+
if is_host_data:
|
| 303 |
+
result.copy_from_host(buf, position=0, nbytes=size)
|
| 304 |
+
else:
|
| 305 |
+
result.copy_from_device(buf, position=0, nbytes=size)
|
| 306 |
+
return result
|
| 307 |
+
|
| 308 |
+
def buffer_from_object(self, obj):
|
| 309 |
+
"""Create device buffer view of arbitrary object that references
|
| 310 |
+
device accessible memory.
|
| 311 |
+
|
| 312 |
+
When the object contains a non-contiguous view of device
|
| 313 |
+
accessible memory then the returned device buffer will contain
|
| 314 |
+
contiguous view of the memory, that is, including the
|
| 315 |
+
intermediate data that is otherwise invisible to the input
|
| 316 |
+
object.
|
| 317 |
+
|
| 318 |
+
Parameters
|
| 319 |
+
----------
|
| 320 |
+
obj : {object, Buffer, HostBuffer, CudaBuffer, ...}
|
| 321 |
+
Specify an object that holds (device or host) address that
|
| 322 |
+
can be accessed from device. This includes objects with
|
| 323 |
+
types defined in pyarrow.cuda as well as arbitrary objects
|
| 324 |
+
that implement the CUDA array interface as defined by numba.
|
| 325 |
+
|
| 326 |
+
Returns
|
| 327 |
+
-------
|
| 328 |
+
cbuf : CudaBuffer
|
| 329 |
+
Device buffer as a view of device accessible memory.
|
| 330 |
+
|
| 331 |
+
"""
|
| 332 |
+
if isinstance(obj, HostBuffer):
|
| 333 |
+
return self.foreign_buffer(obj.address, obj.size, base=obj)
|
| 334 |
+
elif isinstance(obj, Buffer):
|
| 335 |
+
return CudaBuffer.from_buffer(obj)
|
| 336 |
+
elif isinstance(obj, CudaBuffer):
|
| 337 |
+
return obj
|
| 338 |
+
elif hasattr(obj, '__cuda_array_interface__'):
|
| 339 |
+
desc = obj.__cuda_array_interface__
|
| 340 |
+
addr = desc['data'][0]
|
| 341 |
+
if addr is None:
|
| 342 |
+
return self.new_buffer(0)
|
| 343 |
+
import numpy as np
|
| 344 |
+
start, end = get_contiguous_span(
|
| 345 |
+
desc['shape'], desc.get('strides'),
|
| 346 |
+
np.dtype(desc['typestr']).itemsize)
|
| 347 |
+
return self.foreign_buffer(addr + start, end - start, base=obj)
|
| 348 |
+
raise ArrowTypeError('cannot create device buffer view from'
|
| 349 |
+
' `%s` object' % (type(obj)))
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
cdef class IpcMemHandle(_Weakrefable):
|
| 353 |
+
"""A serializable container for a CUDA IPC handle.
|
| 354 |
+
"""
|
| 355 |
+
cdef void init(self, shared_ptr[CCudaIpcMemHandle]& h):
|
| 356 |
+
self.handle = h
|
| 357 |
+
|
| 358 |
+
@staticmethod
|
| 359 |
+
def from_buffer(Buffer opaque_handle):
|
| 360 |
+
"""Create IpcMemHandle from opaque buffer (e.g. from another
|
| 361 |
+
process)
|
| 362 |
+
|
| 363 |
+
Parameters
|
| 364 |
+
----------
|
| 365 |
+
opaque_handle :
|
| 366 |
+
a CUipcMemHandle as a const void*
|
| 367 |
+
|
| 368 |
+
Returns
|
| 369 |
+
-------
|
| 370 |
+
ipc_handle : IpcMemHandle
|
| 371 |
+
"""
|
| 372 |
+
c_buf = pyarrow_unwrap_buffer(opaque_handle)
|
| 373 |
+
cdef:
|
| 374 |
+
shared_ptr[CCudaIpcMemHandle] handle
|
| 375 |
+
|
| 376 |
+
handle = GetResultValue(
|
| 377 |
+
CCudaIpcMemHandle.FromBuffer(c_buf.get().data()))
|
| 378 |
+
return pyarrow_wrap_cudaipcmemhandle(handle)
|
| 379 |
+
|
| 380 |
+
def serialize(self, pool=None):
|
| 381 |
+
"""Write IpcMemHandle to a Buffer
|
| 382 |
+
|
| 383 |
+
Parameters
|
| 384 |
+
----------
|
| 385 |
+
pool : {MemoryPool, None}
|
| 386 |
+
Specify a pool to allocate memory from
|
| 387 |
+
|
| 388 |
+
Returns
|
| 389 |
+
-------
|
| 390 |
+
buf : Buffer
|
| 391 |
+
The serialized buffer.
|
| 392 |
+
"""
|
| 393 |
+
cdef CMemoryPool* pool_ = maybe_unbox_memory_pool(pool)
|
| 394 |
+
cdef shared_ptr[CBuffer] buf
|
| 395 |
+
cdef CCudaIpcMemHandle* h = self.handle.get()
|
| 396 |
+
with nogil:
|
| 397 |
+
buf = GetResultValue(h.Serialize(pool_))
|
| 398 |
+
return pyarrow_wrap_buffer(buf)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
cdef class CudaBuffer(Buffer):
|
| 402 |
+
"""An Arrow buffer with data located in a GPU device.
|
| 403 |
+
|
| 404 |
+
To create a CudaBuffer instance, use Context.device_buffer().
|
| 405 |
+
|
| 406 |
+
The memory allocated in a CudaBuffer is freed when the buffer object
|
| 407 |
+
is deleted.
|
| 408 |
+
"""
|
| 409 |
+
|
| 410 |
+
def __init__(self):
|
| 411 |
+
raise TypeError("Do not call CudaBuffer's constructor directly, use "
|
| 412 |
+
"`<pyarrow.Context instance>.device_buffer`"
|
| 413 |
+
" method instead.")
|
| 414 |
+
|
| 415 |
+
cdef void init_cuda(self,
|
| 416 |
+
const shared_ptr[CCudaBuffer]& buffer,
|
| 417 |
+
object base):
|
| 418 |
+
self.cuda_buffer = buffer
|
| 419 |
+
self.init(<shared_ptr[CBuffer]> buffer)
|
| 420 |
+
self.base = base
|
| 421 |
+
|
| 422 |
+
@staticmethod
|
| 423 |
+
def from_buffer(buf):
|
| 424 |
+
""" Convert back generic buffer into CudaBuffer
|
| 425 |
+
|
| 426 |
+
Parameters
|
| 427 |
+
----------
|
| 428 |
+
buf : Buffer
|
| 429 |
+
Specify buffer containing CudaBuffer
|
| 430 |
+
|
| 431 |
+
Returns
|
| 432 |
+
-------
|
| 433 |
+
dbuf : CudaBuffer
|
| 434 |
+
Resulting device buffer.
|
| 435 |
+
"""
|
| 436 |
+
c_buf = pyarrow_unwrap_buffer(buf)
|
| 437 |
+
cuda_buffer = GetResultValue(CCudaBuffer.FromBuffer(c_buf))
|
| 438 |
+
return pyarrow_wrap_cudabuffer(cuda_buffer)
|
| 439 |
+
|
| 440 |
+
@staticmethod
|
| 441 |
+
def from_numba(mem):
|
| 442 |
+
"""Create a CudaBuffer view from numba MemoryPointer instance.
|
| 443 |
+
|
| 444 |
+
Parameters
|
| 445 |
+
----------
|
| 446 |
+
mem : numba.cuda.cudadrv.driver.MemoryPointer
|
| 447 |
+
|
| 448 |
+
Returns
|
| 449 |
+
-------
|
| 450 |
+
cbuf : CudaBuffer
|
| 451 |
+
Device buffer as a view of numba MemoryPointer.
|
| 452 |
+
"""
|
| 453 |
+
ctx = Context.from_numba(mem.context)
|
| 454 |
+
if mem.device_pointer_value is None and mem.size==0:
|
| 455 |
+
return ctx.new_buffer(0)
|
| 456 |
+
return ctx.foreign_buffer(mem.device_pointer_value, mem.size, base=mem)
|
| 457 |
+
|
| 458 |
+
def to_numba(self):
|
| 459 |
+
"""Return numba memory pointer of CudaBuffer instance.
|
| 460 |
+
"""
|
| 461 |
+
import ctypes
|
| 462 |
+
from numba.cuda.cudadrv.driver import MemoryPointer
|
| 463 |
+
try:
|
| 464 |
+
return MemoryPointer(self.context.to_numba(),
|
| 465 |
+
pointer=ctypes.c_void_p(self.address),
|
| 466 |
+
size=self.size)
|
| 467 |
+
except TypeError:
|
| 468 |
+
# Newer Numba does not take a context argument anymore
|
| 469 |
+
return MemoryPointer(pointer=ctypes.c_void_p(self.address),
|
| 470 |
+
size=self.size)
|
| 471 |
+
|
| 472 |
+
cdef getitem(self, int64_t i):
|
| 473 |
+
return self.copy_to_host(position=i, nbytes=1)[0]
|
| 474 |
+
|
| 475 |
+
def copy_to_host(self, int64_t position=0, int64_t nbytes=-1,
|
| 476 |
+
Buffer buf=None,
|
| 477 |
+
MemoryPool memory_pool=None, c_bool resizable=False):
|
| 478 |
+
"""Copy memory from GPU device to CPU host
|
| 479 |
+
|
| 480 |
+
Caller is responsible for ensuring that all tasks affecting
|
| 481 |
+
the memory are finished. Use
|
| 482 |
+
|
| 483 |
+
`<CudaBuffer instance>.context.synchronize()`
|
| 484 |
+
|
| 485 |
+
when needed.
|
| 486 |
+
|
| 487 |
+
Parameters
|
| 488 |
+
----------
|
| 489 |
+
position : int
|
| 490 |
+
Specify the starting position of the source data in GPU
|
| 491 |
+
device buffer. Default: 0.
|
| 492 |
+
nbytes : int
|
| 493 |
+
Specify the number of bytes to copy. Default: -1 (all from
|
| 494 |
+
the position until host buffer is full).
|
| 495 |
+
buf : Buffer
|
| 496 |
+
Specify a pre-allocated output buffer in host. Default: None
|
| 497 |
+
(allocate new output buffer).
|
| 498 |
+
memory_pool : MemoryPool
|
| 499 |
+
resizable : bool
|
| 500 |
+
Specify extra arguments to allocate_buffer. Used only when
|
| 501 |
+
buf is None.
|
| 502 |
+
|
| 503 |
+
Returns
|
| 504 |
+
-------
|
| 505 |
+
buf : Buffer
|
| 506 |
+
Output buffer in host.
|
| 507 |
+
|
| 508 |
+
"""
|
| 509 |
+
if position < 0 or (self.size and position > self.size) \
|
| 510 |
+
or (self.size == 0 and position != 0):
|
| 511 |
+
raise ValueError('position argument is out-of-range')
|
| 512 |
+
cdef:
|
| 513 |
+
int64_t c_nbytes
|
| 514 |
+
if buf is None:
|
| 515 |
+
if nbytes < 0:
|
| 516 |
+
# copy all starting from position to new host buffer
|
| 517 |
+
c_nbytes = self.size - position
|
| 518 |
+
else:
|
| 519 |
+
if nbytes > self.size - position:
|
| 520 |
+
raise ValueError(
|
| 521 |
+
'requested more to copy than available from '
|
| 522 |
+
'device buffer')
|
| 523 |
+
# copy nbytes starting from position to new host buffer
|
| 524 |
+
c_nbytes = nbytes
|
| 525 |
+
buf = allocate_buffer(c_nbytes, memory_pool=memory_pool,
|
| 526 |
+
resizable=resizable)
|
| 527 |
+
else:
|
| 528 |
+
if nbytes < 0:
|
| 529 |
+
# copy all from position until given host buffer is full
|
| 530 |
+
c_nbytes = min(self.size - position, buf.size)
|
| 531 |
+
else:
|
| 532 |
+
if nbytes > buf.size:
|
| 533 |
+
raise ValueError(
|
| 534 |
+
'requested copy does not fit into host buffer')
|
| 535 |
+
# copy nbytes from position to given host buffer
|
| 536 |
+
c_nbytes = nbytes
|
| 537 |
+
|
| 538 |
+
cdef:
|
| 539 |
+
shared_ptr[CBuffer] c_buf = pyarrow_unwrap_buffer(buf)
|
| 540 |
+
int64_t c_position = position
|
| 541 |
+
with nogil:
|
| 542 |
+
check_status(self.cuda_buffer.get()
|
| 543 |
+
.CopyToHost(c_position, c_nbytes,
|
| 544 |
+
c_buf.get().mutable_data()))
|
| 545 |
+
return buf
|
| 546 |
+
|
| 547 |
+
def copy_from_host(self, data, int64_t position=0, int64_t nbytes=-1):
|
| 548 |
+
"""Copy data from host to device.
|
| 549 |
+
|
| 550 |
+
The device buffer must be pre-allocated.
|
| 551 |
+
|
| 552 |
+
Parameters
|
| 553 |
+
----------
|
| 554 |
+
data : {Buffer, array-like}
|
| 555 |
+
Specify data in host. It can be array-like that is valid
|
| 556 |
+
argument to py_buffer
|
| 557 |
+
position : int
|
| 558 |
+
Specify the starting position of the copy in device buffer.
|
| 559 |
+
Default: 0.
|
| 560 |
+
nbytes : int
|
| 561 |
+
Specify the number of bytes to copy. Default: -1 (all from
|
| 562 |
+
source until device buffer, starting from position, is full)
|
| 563 |
+
|
| 564 |
+
Returns
|
| 565 |
+
-------
|
| 566 |
+
nbytes : int
|
| 567 |
+
Number of bytes copied.
|
| 568 |
+
"""
|
| 569 |
+
if position < 0 or position > self.size:
|
| 570 |
+
raise ValueError('position argument is out-of-range')
|
| 571 |
+
cdef:
|
| 572 |
+
int64_t c_nbytes
|
| 573 |
+
buf = as_buffer(data)
|
| 574 |
+
|
| 575 |
+
if nbytes < 0:
|
| 576 |
+
# copy from host buffer to device buffer starting from
|
| 577 |
+
# position until device buffer is full
|
| 578 |
+
c_nbytes = min(self.size - position, buf.size)
|
| 579 |
+
else:
|
| 580 |
+
if nbytes > buf.size:
|
| 581 |
+
raise ValueError(
|
| 582 |
+
'requested more to copy than available from host buffer')
|
| 583 |
+
if nbytes > self.size - position:
|
| 584 |
+
raise ValueError(
|
| 585 |
+
'requested more to copy than available in device buffer')
|
| 586 |
+
# copy nbytes from host buffer to device buffer starting
|
| 587 |
+
# from position
|
| 588 |
+
c_nbytes = nbytes
|
| 589 |
+
|
| 590 |
+
cdef:
|
| 591 |
+
shared_ptr[CBuffer] c_buf = pyarrow_unwrap_buffer(buf)
|
| 592 |
+
int64_t c_position = position
|
| 593 |
+
with nogil:
|
| 594 |
+
check_status(self.cuda_buffer.get().
|
| 595 |
+
CopyFromHost(c_position, c_buf.get().data(),
|
| 596 |
+
c_nbytes))
|
| 597 |
+
return c_nbytes
|
| 598 |
+
|
| 599 |
+
def copy_from_device(self, buf, int64_t position=0, int64_t nbytes=-1):
|
| 600 |
+
"""Copy data from device to device.
|
| 601 |
+
|
| 602 |
+
Parameters
|
| 603 |
+
----------
|
| 604 |
+
buf : CudaBuffer
|
| 605 |
+
Specify source device buffer.
|
| 606 |
+
position : int
|
| 607 |
+
Specify the starting position of the copy in device buffer.
|
| 608 |
+
Default: 0.
|
| 609 |
+
nbytes : int
|
| 610 |
+
Specify the number of bytes to copy. Default: -1 (all from
|
| 611 |
+
source until device buffer, starting from position, is full)
|
| 612 |
+
|
| 613 |
+
Returns
|
| 614 |
+
-------
|
| 615 |
+
nbytes : int
|
| 616 |
+
Number of bytes copied.
|
| 617 |
+
|
| 618 |
+
"""
|
| 619 |
+
if position < 0 or position > self.size:
|
| 620 |
+
raise ValueError('position argument is out-of-range')
|
| 621 |
+
cdef:
|
| 622 |
+
int64_t c_nbytes
|
| 623 |
+
|
| 624 |
+
if nbytes < 0:
|
| 625 |
+
# copy from source device buffer to device buffer starting
|
| 626 |
+
# from position until device buffer is full
|
| 627 |
+
c_nbytes = min(self.size - position, buf.size)
|
| 628 |
+
else:
|
| 629 |
+
if nbytes > buf.size:
|
| 630 |
+
raise ValueError(
|
| 631 |
+
'requested more to copy than available from device buffer')
|
| 632 |
+
if nbytes > self.size - position:
|
| 633 |
+
raise ValueError(
|
| 634 |
+
'requested more to copy than available in device buffer')
|
| 635 |
+
# copy nbytes from source device buffer to device buffer
|
| 636 |
+
# starting from position
|
| 637 |
+
c_nbytes = nbytes
|
| 638 |
+
|
| 639 |
+
cdef:
|
| 640 |
+
shared_ptr[CCudaBuffer] c_buf = pyarrow_unwrap_cudabuffer(buf)
|
| 641 |
+
int64_t c_position = position
|
| 642 |
+
shared_ptr[CCudaContext] c_src_ctx = pyarrow_unwrap_cudacontext(
|
| 643 |
+
buf.context)
|
| 644 |
+
void* c_source_data = <void*>(c_buf.get().address())
|
| 645 |
+
|
| 646 |
+
if self.context.handle != buf.context.handle:
|
| 647 |
+
with nogil:
|
| 648 |
+
check_status(self.cuda_buffer.get().
|
| 649 |
+
CopyFromAnotherDevice(c_src_ctx, c_position,
|
| 650 |
+
c_source_data, c_nbytes))
|
| 651 |
+
else:
|
| 652 |
+
with nogil:
|
| 653 |
+
check_status(self.cuda_buffer.get().
|
| 654 |
+
CopyFromDevice(c_position, c_source_data,
|
| 655 |
+
c_nbytes))
|
| 656 |
+
return c_nbytes
|
| 657 |
+
|
| 658 |
+
def export_for_ipc(self):
|
| 659 |
+
"""
|
| 660 |
+
Expose this device buffer as IPC memory which can be used in other
|
| 661 |
+
processes.
|
| 662 |
+
|
| 663 |
+
After calling this function, this device memory will not be
|
| 664 |
+
freed when the CudaBuffer is destructed.
|
| 665 |
+
|
| 666 |
+
Returns
|
| 667 |
+
-------
|
| 668 |
+
ipc_handle : IpcMemHandle
|
| 669 |
+
The exported IPC handle
|
| 670 |
+
|
| 671 |
+
"""
|
| 672 |
+
cdef shared_ptr[CCudaIpcMemHandle] handle
|
| 673 |
+
with nogil:
|
| 674 |
+
handle = GetResultValue(self.cuda_buffer.get().ExportForIpc())
|
| 675 |
+
return pyarrow_wrap_cudaipcmemhandle(handle)
|
| 676 |
+
|
| 677 |
+
@property
|
| 678 |
+
def context(self):
|
| 679 |
+
"""Returns the CUDA driver context of this buffer.
|
| 680 |
+
"""
|
| 681 |
+
return pyarrow_wrap_cudacontext(self.cuda_buffer.get().context())
|
| 682 |
+
|
| 683 |
+
def slice(self, offset=0, length=None):
|
| 684 |
+
"""Return slice of device buffer
|
| 685 |
+
|
| 686 |
+
Parameters
|
| 687 |
+
----------
|
| 688 |
+
offset : int, default 0
|
| 689 |
+
Specify offset from the start of device buffer to slice
|
| 690 |
+
length : int, default None
|
| 691 |
+
Specify the length of slice (default is until end of device
|
| 692 |
+
buffer starting from offset). If the length is larger than
|
| 693 |
+
the data available, the returned slice will have a size of
|
| 694 |
+
the available data starting from the offset.
|
| 695 |
+
|
| 696 |
+
Returns
|
| 697 |
+
-------
|
| 698 |
+
sliced : CudaBuffer
|
| 699 |
+
Zero-copy slice of device buffer.
|
| 700 |
+
|
| 701 |
+
"""
|
| 702 |
+
if offset < 0 or (self.size and offset >= self.size):
|
| 703 |
+
raise ValueError('offset argument is out-of-range')
|
| 704 |
+
cdef int64_t offset_ = offset
|
| 705 |
+
cdef int64_t size
|
| 706 |
+
if length is None:
|
| 707 |
+
size = self.size - offset_
|
| 708 |
+
elif offset + length <= self.size:
|
| 709 |
+
size = length
|
| 710 |
+
else:
|
| 711 |
+
size = self.size - offset
|
| 712 |
+
parent = pyarrow_unwrap_cudabuffer(self)
|
| 713 |
+
return pyarrow_wrap_cudabuffer(make_shared[CCudaBuffer](parent,
|
| 714 |
+
offset_, size))
|
| 715 |
+
|
| 716 |
+
def to_pybytes(self):
|
| 717 |
+
"""Return device buffer content as Python bytes.
|
| 718 |
+
"""
|
| 719 |
+
return self.copy_to_host().to_pybytes()
|
| 720 |
+
|
| 721 |
+
def __getbuffer__(self, cp.Py_buffer* buffer, int flags):
|
| 722 |
+
# Device buffer contains data pointers on the device. Hence,
|
| 723 |
+
# cannot support buffer protocol PEP-3118 for CudaBuffer.
|
| 724 |
+
raise BufferError('buffer protocol for device buffer not supported')
|
| 725 |
+
|
| 726 |
+
|
| 727 |
+
cdef class HostBuffer(Buffer):
|
| 728 |
+
"""Device-accessible CPU memory created using cudaHostAlloc.
|
| 729 |
+
|
| 730 |
+
To create a HostBuffer instance, use
|
| 731 |
+
|
| 732 |
+
cuda.new_host_buffer(<nbytes>)
|
| 733 |
+
"""
|
| 734 |
+
|
| 735 |
+
def __init__(self):
|
| 736 |
+
raise TypeError("Do not call HostBuffer's constructor directly,"
|
| 737 |
+
" use `cuda.new_host_buffer` function instead.")
|
| 738 |
+
|
| 739 |
+
cdef void init_host(self, const shared_ptr[CCudaHostBuffer]& buffer):
|
| 740 |
+
self.host_buffer = buffer
|
| 741 |
+
self.init(<shared_ptr[CBuffer]> buffer)
|
| 742 |
+
|
| 743 |
+
@property
|
| 744 |
+
def size(self):
|
| 745 |
+
return self.host_buffer.get().size()
|
| 746 |
+
|
| 747 |
+
|
| 748 |
+
cdef class BufferReader(NativeFile):
|
| 749 |
+
"""File interface for zero-copy read from CUDA buffers.
|
| 750 |
+
|
| 751 |
+
Note: Read methods return pointers to device memory. This means
|
| 752 |
+
you must be careful using this interface with any Arrow code which
|
| 753 |
+
may expect to be able to do anything other than pointer arithmetic
|
| 754 |
+
on the returned buffers.
|
| 755 |
+
"""
|
| 756 |
+
|
| 757 |
+
def __cinit__(self, CudaBuffer obj):
|
| 758 |
+
self.buffer = obj
|
| 759 |
+
self.reader = new CCudaBufferReader(self.buffer.buffer)
|
| 760 |
+
self.set_random_access_file(
|
| 761 |
+
shared_ptr[CRandomAccessFile](self.reader))
|
| 762 |
+
self.is_readable = True
|
| 763 |
+
|
| 764 |
+
def read_buffer(self, nbytes=None):
|
| 765 |
+
"""Return a slice view of the underlying device buffer.
|
| 766 |
+
|
| 767 |
+
The slice will start at the current reader position and will
|
| 768 |
+
have specified size in bytes.
|
| 769 |
+
|
| 770 |
+
Parameters
|
| 771 |
+
----------
|
| 772 |
+
nbytes : int, default None
|
| 773 |
+
Specify the number of bytes to read. Default: None (read all
|
| 774 |
+
remaining bytes).
|
| 775 |
+
|
| 776 |
+
Returns
|
| 777 |
+
-------
|
| 778 |
+
cbuf : CudaBuffer
|
| 779 |
+
New device buffer.
|
| 780 |
+
|
| 781 |
+
"""
|
| 782 |
+
cdef:
|
| 783 |
+
int64_t c_nbytes
|
| 784 |
+
shared_ptr[CCudaBuffer] output
|
| 785 |
+
|
| 786 |
+
if nbytes is None:
|
| 787 |
+
c_nbytes = self.size() - self.tell()
|
| 788 |
+
else:
|
| 789 |
+
c_nbytes = nbytes
|
| 790 |
+
|
| 791 |
+
with nogil:
|
| 792 |
+
output = static_pointer_cast[CCudaBuffer, CBuffer](
|
| 793 |
+
GetResultValue(self.reader.Read(c_nbytes)))
|
| 794 |
+
|
| 795 |
+
return pyarrow_wrap_cudabuffer(output)
|
| 796 |
+
|
| 797 |
+
|
| 798 |
+
cdef class BufferWriter(NativeFile):
|
| 799 |
+
"""File interface for writing to CUDA buffers.
|
| 800 |
+
|
| 801 |
+
By default writes are unbuffered. Use set_buffer_size to enable
|
| 802 |
+
buffering.
|
| 803 |
+
"""
|
| 804 |
+
|
| 805 |
+
def __cinit__(self, CudaBuffer buffer):
|
| 806 |
+
self.buffer = buffer
|
| 807 |
+
self.writer = new CCudaBufferWriter(self.buffer.cuda_buffer)
|
| 808 |
+
self.set_output_stream(shared_ptr[COutputStream](self.writer))
|
| 809 |
+
self.is_writable = True
|
| 810 |
+
|
| 811 |
+
def writeat(self, int64_t position, object data):
|
| 812 |
+
"""Write data to buffer starting from position.
|
| 813 |
+
|
| 814 |
+
Parameters
|
| 815 |
+
----------
|
| 816 |
+
position : int
|
| 817 |
+
Specify device buffer position where the data will be
|
| 818 |
+
written.
|
| 819 |
+
data : array-like
|
| 820 |
+
Specify data, the data instance must implement buffer
|
| 821 |
+
protocol.
|
| 822 |
+
"""
|
| 823 |
+
cdef:
|
| 824 |
+
Buffer buf = as_buffer(data)
|
| 825 |
+
const uint8_t* c_data = buf.buffer.get().data()
|
| 826 |
+
int64_t c_size = buf.buffer.get().size()
|
| 827 |
+
|
| 828 |
+
with nogil:
|
| 829 |
+
check_status(self.writer.WriteAt(position, c_data, c_size))
|
| 830 |
+
|
| 831 |
+
def flush(self):
|
| 832 |
+
""" Flush the buffer stream """
|
| 833 |
+
with nogil:
|
| 834 |
+
check_status(self.writer.Flush())
|
| 835 |
+
|
| 836 |
+
def seek(self, int64_t position, int whence=0):
|
| 837 |
+
# TODO: remove this method after NativeFile.seek supports
|
| 838 |
+
# writable files.
|
| 839 |
+
cdef int64_t offset
|
| 840 |
+
|
| 841 |
+
with nogil:
|
| 842 |
+
if whence == 0:
|
| 843 |
+
offset = position
|
| 844 |
+
elif whence == 1:
|
| 845 |
+
offset = GetResultValue(self.writer.Tell())
|
| 846 |
+
offset = offset + position
|
| 847 |
+
else:
|
| 848 |
+
with gil:
|
| 849 |
+
raise ValueError(f"Invalid value of whence: {whence}")
|
| 850 |
+
check_status(self.writer.Seek(offset))
|
| 851 |
+
return self.tell()
|
| 852 |
+
|
| 853 |
+
@property
|
| 854 |
+
def buffer_size(self):
|
| 855 |
+
"""Returns size of host (CPU) buffer, 0 for unbuffered
|
| 856 |
+
"""
|
| 857 |
+
return self.writer.buffer_size()
|
| 858 |
+
|
| 859 |
+
@buffer_size.setter
|
| 860 |
+
def buffer_size(self, int64_t buffer_size):
|
| 861 |
+
"""Set CPU buffer size to limit calls to cudaMemcpy
|
| 862 |
+
|
| 863 |
+
Parameters
|
| 864 |
+
----------
|
| 865 |
+
buffer_size : int
|
| 866 |
+
Specify the size of CPU buffer to allocate in bytes.
|
| 867 |
+
"""
|
| 868 |
+
with nogil:
|
| 869 |
+
check_status(self.writer.SetBufferSize(buffer_size))
|
| 870 |
+
|
| 871 |
+
@property
|
| 872 |
+
def num_bytes_buffered(self):
|
| 873 |
+
"""Returns number of bytes buffered on host
|
| 874 |
+
"""
|
| 875 |
+
return self.writer.num_bytes_buffered()
|
| 876 |
+
|
| 877 |
+
# Functions
|
| 878 |
+
|
| 879 |
+
|
| 880 |
+
def new_host_buffer(const int64_t size, int device=0):
|
| 881 |
+
"""Return buffer with CUDA-accessible memory on CPU host
|
| 882 |
+
|
| 883 |
+
Parameters
|
| 884 |
+
----------
|
| 885 |
+
size : int
|
| 886 |
+
Specify the number of bytes to be allocated.
|
| 887 |
+
device : int
|
| 888 |
+
Specify GPU device number.
|
| 889 |
+
|
| 890 |
+
Returns
|
| 891 |
+
-------
|
| 892 |
+
dbuf : HostBuffer
|
| 893 |
+
Allocated host buffer
|
| 894 |
+
"""
|
| 895 |
+
cdef shared_ptr[CCudaHostBuffer] buffer
|
| 896 |
+
with nogil:
|
| 897 |
+
buffer = GetResultValue(AllocateCudaHostBuffer(device, size))
|
| 898 |
+
return pyarrow_wrap_cudahostbuffer(buffer)
|
| 899 |
+
|
| 900 |
+
|
| 901 |
+
def serialize_record_batch(object batch, object ctx):
|
| 902 |
+
""" Write record batch message to GPU device memory
|
| 903 |
+
|
| 904 |
+
Parameters
|
| 905 |
+
----------
|
| 906 |
+
batch : RecordBatch
|
| 907 |
+
Record batch to write
|
| 908 |
+
ctx : Context
|
| 909 |
+
CUDA Context to allocate device memory from
|
| 910 |
+
|
| 911 |
+
Returns
|
| 912 |
+
-------
|
| 913 |
+
dbuf : CudaBuffer
|
| 914 |
+
device buffer which contains the record batch message
|
| 915 |
+
"""
|
| 916 |
+
cdef shared_ptr[CCudaBuffer] buffer
|
| 917 |
+
cdef CRecordBatch* batch_ = pyarrow_unwrap_batch(batch).get()
|
| 918 |
+
cdef CCudaContext* ctx_ = pyarrow_unwrap_cudacontext(ctx).get()
|
| 919 |
+
with nogil:
|
| 920 |
+
buffer = GetResultValue(CudaSerializeRecordBatch(batch_[0], ctx_))
|
| 921 |
+
return pyarrow_wrap_cudabuffer(buffer)
|
| 922 |
+
|
| 923 |
+
|
| 924 |
+
def read_message(object source, pool=None):
|
| 925 |
+
""" Read Arrow IPC message located on GPU device
|
| 926 |
+
|
| 927 |
+
Parameters
|
| 928 |
+
----------
|
| 929 |
+
source : {CudaBuffer, cuda.BufferReader}
|
| 930 |
+
Device buffer or reader of device buffer.
|
| 931 |
+
pool : MemoryPool (optional)
|
| 932 |
+
Pool to allocate CPU memory for the metadata
|
| 933 |
+
|
| 934 |
+
Returns
|
| 935 |
+
-------
|
| 936 |
+
message : Message
|
| 937 |
+
The deserialized message, body still on device
|
| 938 |
+
"""
|
| 939 |
+
cdef:
|
| 940 |
+
Message result = Message.__new__(Message)
|
| 941 |
+
cdef CMemoryPool* pool_ = maybe_unbox_memory_pool(pool)
|
| 942 |
+
if not isinstance(source, BufferReader):
|
| 943 |
+
reader = BufferReader(source)
|
| 944 |
+
with nogil:
|
| 945 |
+
result.message = move(
|
| 946 |
+
GetResultValue(ReadMessage(reader.reader, pool_)))
|
| 947 |
+
return result
|
| 948 |
+
|
| 949 |
+
|
| 950 |
+
def read_record_batch(object buffer, object schema, *,
|
| 951 |
+
DictionaryMemo dictionary_memo=None, pool=None):
|
| 952 |
+
"""Construct RecordBatch referencing IPC message located on CUDA device.
|
| 953 |
+
|
| 954 |
+
While the metadata is copied to host memory for deserialization,
|
| 955 |
+
the record batch data remains on the device.
|
| 956 |
+
|
| 957 |
+
Parameters
|
| 958 |
+
----------
|
| 959 |
+
buffer :
|
| 960 |
+
Device buffer containing the complete IPC message
|
| 961 |
+
schema : Schema
|
| 962 |
+
The schema for the record batch
|
| 963 |
+
dictionary_memo : DictionaryMemo, optional
|
| 964 |
+
If message contains dictionaries, must pass a populated
|
| 965 |
+
DictionaryMemo
|
| 966 |
+
pool : MemoryPool (optional)
|
| 967 |
+
Pool to allocate metadata from
|
| 968 |
+
|
| 969 |
+
Returns
|
| 970 |
+
-------
|
| 971 |
+
batch : RecordBatch
|
| 972 |
+
Reconstructed record batch, with device pointers
|
| 973 |
+
|
| 974 |
+
"""
|
| 975 |
+
cdef:
|
| 976 |
+
shared_ptr[CSchema] schema_ = pyarrow_unwrap_schema(schema)
|
| 977 |
+
shared_ptr[CCudaBuffer] buffer_ = pyarrow_unwrap_cudabuffer(buffer)
|
| 978 |
+
CDictionaryMemo temp_memo
|
| 979 |
+
CDictionaryMemo* arg_dict_memo
|
| 980 |
+
CMemoryPool* pool_ = maybe_unbox_memory_pool(pool)
|
| 981 |
+
shared_ptr[CRecordBatch] batch
|
| 982 |
+
|
| 983 |
+
if dictionary_memo is not None:
|
| 984 |
+
arg_dict_memo = dictionary_memo.memo
|
| 985 |
+
else:
|
| 986 |
+
arg_dict_memo = &temp_memo
|
| 987 |
+
|
| 988 |
+
with nogil:
|
| 989 |
+
batch = GetResultValue(CudaReadRecordBatch(
|
| 990 |
+
schema_, arg_dict_memo, buffer_, pool_))
|
| 991 |
+
return pyarrow_wrap_batch(batch)
|
| 992 |
+
|
| 993 |
+
|
| 994 |
+
# Public API
|
| 995 |
+
|
| 996 |
+
|
| 997 |
+
cdef public api bint pyarrow_is_buffer(object buffer):
|
| 998 |
+
return isinstance(buffer, Buffer)
|
| 999 |
+
|
| 1000 |
+
# cudabuffer
|
| 1001 |
+
|
| 1002 |
+
cdef public api bint pyarrow_is_cudabuffer(object buffer):
|
| 1003 |
+
return isinstance(buffer, CudaBuffer)
|
| 1004 |
+
|
| 1005 |
+
|
| 1006 |
+
cdef public api object \
|
| 1007 |
+
pyarrow_wrap_cudabuffer_base(const shared_ptr[CCudaBuffer]& buf, base):
|
| 1008 |
+
cdef CudaBuffer result = CudaBuffer.__new__(CudaBuffer)
|
| 1009 |
+
result.init_cuda(buf, base)
|
| 1010 |
+
return result
|
| 1011 |
+
|
| 1012 |
+
|
| 1013 |
+
cdef public api object \
|
| 1014 |
+
pyarrow_wrap_cudabuffer(const shared_ptr[CCudaBuffer]& buf):
|
| 1015 |
+
cdef CudaBuffer result = CudaBuffer.__new__(CudaBuffer)
|
| 1016 |
+
result.init_cuda(buf, None)
|
| 1017 |
+
return result
|
| 1018 |
+
|
| 1019 |
+
|
| 1020 |
+
cdef public api shared_ptr[CCudaBuffer] pyarrow_unwrap_cudabuffer(object obj):
|
| 1021 |
+
if pyarrow_is_cudabuffer(obj):
|
| 1022 |
+
return (<CudaBuffer>obj).cuda_buffer
|
| 1023 |
+
raise TypeError('expected CudaBuffer instance, got %s'
|
| 1024 |
+
% (type(obj).__name__))
|
| 1025 |
+
|
| 1026 |
+
# cudahostbuffer
|
| 1027 |
+
|
| 1028 |
+
cdef public api bint pyarrow_is_cudahostbuffer(object buffer):
|
| 1029 |
+
return isinstance(buffer, HostBuffer)
|
| 1030 |
+
|
| 1031 |
+
|
| 1032 |
+
cdef public api object \
|
| 1033 |
+
pyarrow_wrap_cudahostbuffer(const shared_ptr[CCudaHostBuffer]& buf):
|
| 1034 |
+
cdef HostBuffer result = HostBuffer.__new__(HostBuffer)
|
| 1035 |
+
result.init_host(buf)
|
| 1036 |
+
return result
|
| 1037 |
+
|
| 1038 |
+
|
| 1039 |
+
cdef public api shared_ptr[CCudaHostBuffer] \
|
| 1040 |
+
pyarrow_unwrap_cudahostbuffer(object obj):
|
| 1041 |
+
if pyarrow_is_cudahostbuffer(obj):
|
| 1042 |
+
return (<HostBuffer>obj).host_buffer
|
| 1043 |
+
raise TypeError('expected HostBuffer instance, got %s'
|
| 1044 |
+
% (type(obj).__name__))
|
| 1045 |
+
|
| 1046 |
+
# cudacontext
|
| 1047 |
+
|
| 1048 |
+
cdef public api bint pyarrow_is_cudacontext(object ctx):
|
| 1049 |
+
return isinstance(ctx, Context)
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
+
cdef public api object \
|
| 1053 |
+
pyarrow_wrap_cudacontext(const shared_ptr[CCudaContext]& ctx):
|
| 1054 |
+
cdef Context result = Context.__new__(Context)
|
| 1055 |
+
result.init(ctx)
|
| 1056 |
+
return result
|
| 1057 |
+
|
| 1058 |
+
|
| 1059 |
+
cdef public api shared_ptr[CCudaContext] \
|
| 1060 |
+
pyarrow_unwrap_cudacontext(object obj):
|
| 1061 |
+
if pyarrow_is_cudacontext(obj):
|
| 1062 |
+
return (<Context>obj).context
|
| 1063 |
+
raise TypeError('expected Context instance, got %s'
|
| 1064 |
+
% (type(obj).__name__))
|
| 1065 |
+
|
| 1066 |
+
# cudaipcmemhandle
|
| 1067 |
+
|
| 1068 |
+
cdef public api bint pyarrow_is_cudaipcmemhandle(object handle):
|
| 1069 |
+
return isinstance(handle, IpcMemHandle)
|
| 1070 |
+
|
| 1071 |
+
|
| 1072 |
+
cdef public api object \
|
| 1073 |
+
pyarrow_wrap_cudaipcmemhandle(shared_ptr[CCudaIpcMemHandle]& h):
|
| 1074 |
+
cdef IpcMemHandle result = IpcMemHandle.__new__(IpcMemHandle)
|
| 1075 |
+
result.init(h)
|
| 1076 |
+
return result
|
| 1077 |
+
|
| 1078 |
+
|
| 1079 |
+
cdef public api shared_ptr[CCudaIpcMemHandle] \
|
| 1080 |
+
pyarrow_unwrap_cudaipcmemhandle(object obj):
|
| 1081 |
+
if pyarrow_is_cudaipcmemhandle(obj):
|
| 1082 |
+
return (<IpcMemHandle>obj).handle
|
| 1083 |
+
raise TypeError('expected IpcMemHandle instance, got %s'
|
| 1084 |
+
% (type(obj).__name__))
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b0df3f1c21a22218c9122e5cf8412636584bf9a49387db67939ccccf61d52241
|
| 3 |
+
size 704000
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pxd
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
"""Dataset is currently unstable. APIs subject to change without notice."""
|
| 21 |
+
|
| 22 |
+
from pyarrow.includes.common cimport *
|
| 23 |
+
from pyarrow.includes.libarrow_dataset cimport *
|
| 24 |
+
from pyarrow.lib cimport *
|
| 25 |
+
from pyarrow._fs cimport FileSystem, FileInfo
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
cdef CFileSource _make_file_source(object file, FileSystem filesystem=*, object file_size=*)
|
| 29 |
+
|
| 30 |
+
cdef class DatasetFactory(_Weakrefable):
|
| 31 |
+
|
| 32 |
+
cdef:
|
| 33 |
+
SharedPtrNoGIL[CDatasetFactory] wrapped
|
| 34 |
+
CDatasetFactory* factory
|
| 35 |
+
|
| 36 |
+
cdef init(self, const shared_ptr[CDatasetFactory]& sp)
|
| 37 |
+
|
| 38 |
+
@staticmethod
|
| 39 |
+
cdef wrap(const shared_ptr[CDatasetFactory]& sp)
|
| 40 |
+
|
| 41 |
+
cdef inline shared_ptr[CDatasetFactory] unwrap(self) nogil
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
cdef class Dataset(_Weakrefable):
|
| 45 |
+
|
| 46 |
+
cdef:
|
| 47 |
+
SharedPtrNoGIL[CDataset] wrapped
|
| 48 |
+
CDataset* dataset
|
| 49 |
+
public dict _scan_options
|
| 50 |
+
|
| 51 |
+
cdef void init(self, const shared_ptr[CDataset]& sp)
|
| 52 |
+
|
| 53 |
+
@staticmethod
|
| 54 |
+
cdef wrap(const shared_ptr[CDataset]& sp)
|
| 55 |
+
|
| 56 |
+
cdef shared_ptr[CDataset] unwrap(self) nogil
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
cdef class Scanner(_Weakrefable):
|
| 60 |
+
cdef:
|
| 61 |
+
SharedPtrNoGIL[CScanner] wrapped
|
| 62 |
+
CScanner* scanner
|
| 63 |
+
|
| 64 |
+
cdef void init(self, const shared_ptr[CScanner]& sp)
|
| 65 |
+
|
| 66 |
+
@staticmethod
|
| 67 |
+
cdef wrap(const shared_ptr[CScanner]& sp)
|
| 68 |
+
|
| 69 |
+
cdef shared_ptr[CScanner] unwrap(self)
|
| 70 |
+
|
| 71 |
+
@staticmethod
|
| 72 |
+
cdef shared_ptr[CScanOptions] _make_scan_options(Dataset dataset, dict py_scanoptions) except *
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
cdef class FragmentScanOptions(_Weakrefable):
|
| 76 |
+
|
| 77 |
+
cdef:
|
| 78 |
+
shared_ptr[CFragmentScanOptions] wrapped
|
| 79 |
+
|
| 80 |
+
cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp)
|
| 81 |
+
|
| 82 |
+
@staticmethod
|
| 83 |
+
cdef wrap(const shared_ptr[CFragmentScanOptions]& sp)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
cdef class FileFormat(_Weakrefable):
|
| 87 |
+
|
| 88 |
+
cdef:
|
| 89 |
+
shared_ptr[CFileFormat] wrapped
|
| 90 |
+
CFileFormat* format
|
| 91 |
+
|
| 92 |
+
cdef void init(self, const shared_ptr[CFileFormat]& sp)
|
| 93 |
+
|
| 94 |
+
@staticmethod
|
| 95 |
+
cdef wrap(const shared_ptr[CFileFormat]& sp)
|
| 96 |
+
|
| 97 |
+
cdef inline shared_ptr[CFileFormat] unwrap(self)
|
| 98 |
+
|
| 99 |
+
cdef _set_default_fragment_scan_options(self, FragmentScanOptions options)
|
| 100 |
+
|
| 101 |
+
# Return a WrittenFile after a file was written.
|
| 102 |
+
# May be overridden by subclasses, e.g. to add metadata.
|
| 103 |
+
cdef WrittenFile _finish_write(self, path, base_dir,
|
| 104 |
+
CFileWriter* file_writer)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
cdef class FileWriteOptions(_Weakrefable):
|
| 108 |
+
|
| 109 |
+
cdef:
|
| 110 |
+
shared_ptr[CFileWriteOptions] wrapped
|
| 111 |
+
CFileWriteOptions* c_options
|
| 112 |
+
|
| 113 |
+
cdef void init(self, const shared_ptr[CFileWriteOptions]& sp)
|
| 114 |
+
|
| 115 |
+
@staticmethod
|
| 116 |
+
cdef wrap(const shared_ptr[CFileWriteOptions]& sp)
|
| 117 |
+
|
| 118 |
+
cdef inline shared_ptr[CFileWriteOptions] unwrap(self)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
cdef class Fragment(_Weakrefable):
|
| 122 |
+
|
| 123 |
+
cdef:
|
| 124 |
+
SharedPtrNoGIL[CFragment] wrapped
|
| 125 |
+
CFragment* fragment
|
| 126 |
+
|
| 127 |
+
cdef void init(self, const shared_ptr[CFragment]& sp)
|
| 128 |
+
|
| 129 |
+
@staticmethod
|
| 130 |
+
cdef wrap(const shared_ptr[CFragment]& sp)
|
| 131 |
+
|
| 132 |
+
cdef inline shared_ptr[CFragment] unwrap(self)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
cdef class FileFragment(Fragment):
|
| 136 |
+
|
| 137 |
+
cdef:
|
| 138 |
+
CFileFragment* file_fragment
|
| 139 |
+
|
| 140 |
+
cdef void init(self, const shared_ptr[CFragment]& sp)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
cdef class Partitioning(_Weakrefable):
|
| 144 |
+
|
| 145 |
+
cdef:
|
| 146 |
+
shared_ptr[CPartitioning] wrapped
|
| 147 |
+
CPartitioning* partitioning
|
| 148 |
+
|
| 149 |
+
cdef init(self, const shared_ptr[CPartitioning]& sp)
|
| 150 |
+
|
| 151 |
+
@staticmethod
|
| 152 |
+
cdef wrap(const shared_ptr[CPartitioning]& sp)
|
| 153 |
+
|
| 154 |
+
cdef inline shared_ptr[CPartitioning] unwrap(self)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
cdef class PartitioningFactory(_Weakrefable):
|
| 158 |
+
|
| 159 |
+
cdef:
|
| 160 |
+
shared_ptr[CPartitioningFactory] wrapped
|
| 161 |
+
CPartitioningFactory* factory
|
| 162 |
+
object constructor
|
| 163 |
+
object options
|
| 164 |
+
|
| 165 |
+
cdef init(self, const shared_ptr[CPartitioningFactory]& sp)
|
| 166 |
+
|
| 167 |
+
@staticmethod
|
| 168 |
+
cdef wrap(const shared_ptr[CPartitioningFactory]& sp,
|
| 169 |
+
object constructor, object options)
|
| 170 |
+
|
| 171 |
+
cdef inline shared_ptr[CPartitioningFactory] unwrap(self)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
cdef class WrittenFile(_Weakrefable):
|
| 175 |
+
|
| 176 |
+
# The full path to the created file
|
| 177 |
+
cdef public str path
|
| 178 |
+
# Optional Parquet metadata
|
| 179 |
+
# This metadata will have the file path attribute set to the path of
|
| 180 |
+
# the written file.
|
| 181 |
+
cdef public object metadata
|
| 182 |
+
# The size of the file in bytes
|
| 183 |
+
cdef public int64_t size
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset.pyx
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.cp312-win_amd64.pyd
ADDED
|
Binary file (69.6 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_orc.pyx
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
"""Dataset support for ORC file format."""
|
| 21 |
+
|
| 22 |
+
from pyarrow.lib cimport *
|
| 23 |
+
from pyarrow.includes.libarrow cimport *
|
| 24 |
+
from pyarrow.includes.libarrow_dataset cimport *
|
| 25 |
+
|
| 26 |
+
from pyarrow._dataset cimport FileFormat
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
cdef class OrcFileFormat(FileFormat):
|
| 30 |
+
|
| 31 |
+
def __init__(self):
|
| 32 |
+
self.init(shared_ptr[CFileFormat](new COrcFileFormat()))
|
| 33 |
+
|
| 34 |
+
def equals(self, OrcFileFormat other):
|
| 35 |
+
"""
|
| 36 |
+
Parameters
|
| 37 |
+
----------
|
| 38 |
+
other : pyarrow.dataset.OrcFileFormat
|
| 39 |
+
|
| 40 |
+
Returns
|
| 41 |
+
-------
|
| 42 |
+
True
|
| 43 |
+
"""
|
| 44 |
+
return True
|
| 45 |
+
|
| 46 |
+
@property
|
| 47 |
+
def default_extname(self):
|
| 48 |
+
return "orc"
|
| 49 |
+
|
| 50 |
+
def __reduce__(self):
|
| 51 |
+
return OrcFileFormat, tuple()
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c33edeeee46454a49d78803f343a137e1b6cf5e8ed512712309bc1023f881f0e
|
| 3 |
+
size 253952
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pxd
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
"""Dataset support for Parquet file format."""
|
| 21 |
+
|
| 22 |
+
from pyarrow.includes.libarrow_dataset cimport *
|
| 23 |
+
from pyarrow.includes.libarrow_dataset_parquet cimport *
|
| 24 |
+
|
| 25 |
+
from pyarrow._dataset cimport FragmentScanOptions, FileWriteOptions
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
cdef class ParquetFragmentScanOptions(FragmentScanOptions):
|
| 29 |
+
cdef:
|
| 30 |
+
CParquetFragmentScanOptions* parquet_options
|
| 31 |
+
object _parquet_decryption_config
|
| 32 |
+
object _decryption_properties
|
| 33 |
+
|
| 34 |
+
cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp)
|
| 35 |
+
cdef CReaderProperties* reader_properties(self)
|
| 36 |
+
cdef ArrowReaderProperties* arrow_reader_properties(self)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
cdef class ParquetFileWriteOptions(FileWriteOptions):
|
| 40 |
+
|
| 41 |
+
cdef:
|
| 42 |
+
CParquetFileWriteOptions* parquet_options
|
| 43 |
+
object _properties
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet.pyx
ADDED
|
@@ -0,0 +1,1113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
"""Dataset support for Parquet file format."""
|
| 21 |
+
|
| 22 |
+
from cython.operator cimport dereference as deref
|
| 23 |
+
|
| 24 |
+
import os
|
| 25 |
+
import warnings
|
| 26 |
+
|
| 27 |
+
import pyarrow as pa
|
| 28 |
+
from pyarrow.lib cimport *
|
| 29 |
+
from pyarrow.lib import frombytes, tobytes, is_threading_enabled
|
| 30 |
+
from pyarrow.includes.libarrow cimport *
|
| 31 |
+
from pyarrow.includes.libarrow_dataset cimport *
|
| 32 |
+
from pyarrow.includes.libarrow_dataset_parquet cimport *
|
| 33 |
+
from pyarrow._fs cimport FileSystem
|
| 34 |
+
|
| 35 |
+
from pyarrow._compute cimport Expression, _bind
|
| 36 |
+
from pyarrow._dataset cimport (
|
| 37 |
+
_make_file_source,
|
| 38 |
+
DatasetFactory,
|
| 39 |
+
FileFormat,
|
| 40 |
+
FileFragment,
|
| 41 |
+
FileWriteOptions,
|
| 42 |
+
Fragment,
|
| 43 |
+
FragmentScanOptions,
|
| 44 |
+
CacheOptions,
|
| 45 |
+
Partitioning,
|
| 46 |
+
PartitioningFactory,
|
| 47 |
+
WrittenFile
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
from pyarrow._parquet cimport (
|
| 51 |
+
_create_writer_properties, _create_arrow_writer_properties,
|
| 52 |
+
_unwrap_list_type, FileMetaData,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
try:
|
| 57 |
+
from pyarrow._dataset_parquet_encryption import (
|
| 58 |
+
set_encryption_config, set_decryption_config, set_decryption_properties
|
| 59 |
+
)
|
| 60 |
+
parquet_encryption_enabled = True
|
| 61 |
+
except ImportError:
|
| 62 |
+
parquet_encryption_enabled = False
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
cdef Expression _true = Expression._scalar(True)
|
| 66 |
+
|
| 67 |
+
ctypedef CParquetFileWriter* _CParquetFileWriterPtr
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
cdef class ParquetFileFormat(FileFormat):
|
| 71 |
+
"""
|
| 72 |
+
FileFormat for Parquet
|
| 73 |
+
|
| 74 |
+
Parameters
|
| 75 |
+
----------
|
| 76 |
+
read_options : ParquetReadOptions
|
| 77 |
+
Read options for the file.
|
| 78 |
+
default_fragment_scan_options : ParquetFragmentScanOptions
|
| 79 |
+
Scan Options for the file.
|
| 80 |
+
**kwargs : dict
|
| 81 |
+
Additional options for read option or scan option
|
| 82 |
+
"""
|
| 83 |
+
|
| 84 |
+
cdef:
|
| 85 |
+
CParquetFileFormat* parquet_format
|
| 86 |
+
|
| 87 |
+
def __init__(self, read_options=None,
|
| 88 |
+
default_fragment_scan_options=None,
|
| 89 |
+
**kwargs):
|
| 90 |
+
cdef:
|
| 91 |
+
shared_ptr[CParquetFileFormat] wrapped
|
| 92 |
+
CParquetFileFormatReaderOptions* options
|
| 93 |
+
|
| 94 |
+
# Read/scan options
|
| 95 |
+
read_options_args = {option: kwargs[option] for option in kwargs
|
| 96 |
+
if option in _PARQUET_READ_OPTIONS}
|
| 97 |
+
scan_args = {option: kwargs[option] for option in kwargs
|
| 98 |
+
if option not in _PARQUET_READ_OPTIONS}
|
| 99 |
+
if read_options and read_options_args:
|
| 100 |
+
duplicates = ', '.join(sorted(read_options_args))
|
| 101 |
+
raise ValueError(f'If `read_options` is given, '
|
| 102 |
+
f'cannot specify {duplicates}')
|
| 103 |
+
if default_fragment_scan_options and scan_args:
|
| 104 |
+
duplicates = ', '.join(sorted(scan_args))
|
| 105 |
+
raise ValueError(f'If `default_fragment_scan_options` is given, '
|
| 106 |
+
f'cannot specify {duplicates}')
|
| 107 |
+
|
| 108 |
+
if read_options is None:
|
| 109 |
+
read_options = ParquetReadOptions(**read_options_args)
|
| 110 |
+
elif isinstance(read_options, dict):
|
| 111 |
+
# For backwards compatibility
|
| 112 |
+
duplicates = []
|
| 113 |
+
for option, value in read_options.items():
|
| 114 |
+
if option in _PARQUET_READ_OPTIONS:
|
| 115 |
+
read_options_args[option] = value
|
| 116 |
+
else:
|
| 117 |
+
duplicates.append(option)
|
| 118 |
+
scan_args[option] = value
|
| 119 |
+
if duplicates:
|
| 120 |
+
duplicates = ", ".join(duplicates)
|
| 121 |
+
warnings.warn(f'The scan options {duplicates} should be '
|
| 122 |
+
'specified directly as keyword arguments')
|
| 123 |
+
read_options = ParquetReadOptions(**read_options_args)
|
| 124 |
+
elif not isinstance(read_options, ParquetReadOptions):
|
| 125 |
+
raise TypeError('`read_options` must be either a dictionary or an '
|
| 126 |
+
'instance of ParquetReadOptions')
|
| 127 |
+
|
| 128 |
+
if default_fragment_scan_options is None:
|
| 129 |
+
default_fragment_scan_options = ParquetFragmentScanOptions(**scan_args)
|
| 130 |
+
elif isinstance(default_fragment_scan_options, dict):
|
| 131 |
+
default_fragment_scan_options = ParquetFragmentScanOptions(
|
| 132 |
+
**default_fragment_scan_options)
|
| 133 |
+
elif not isinstance(default_fragment_scan_options,
|
| 134 |
+
ParquetFragmentScanOptions):
|
| 135 |
+
raise TypeError('`default_fragment_scan_options` must be either a '
|
| 136 |
+
'dictionary or an instance of '
|
| 137 |
+
'ParquetFragmentScanOptions')
|
| 138 |
+
|
| 139 |
+
wrapped = make_shared[CParquetFileFormat]()
|
| 140 |
+
|
| 141 |
+
options = &(wrapped.get().reader_options)
|
| 142 |
+
if read_options.dictionary_columns is not None:
|
| 143 |
+
for column in read_options.dictionary_columns:
|
| 144 |
+
options.dict_columns.insert(tobytes(column))
|
| 145 |
+
options.coerce_int96_timestamp_unit = \
|
| 146 |
+
read_options._coerce_int96_timestamp_unit
|
| 147 |
+
options.binary_type = read_options._binary_type
|
| 148 |
+
options.list_type = read_options._list_type
|
| 149 |
+
|
| 150 |
+
self.init(<shared_ptr[CFileFormat]> wrapped)
|
| 151 |
+
self.default_fragment_scan_options = default_fragment_scan_options
|
| 152 |
+
|
| 153 |
+
cdef void init(self, const shared_ptr[CFileFormat]& sp):
|
| 154 |
+
FileFormat.init(self, sp)
|
| 155 |
+
self.parquet_format = <CParquetFileFormat*> sp.get()
|
| 156 |
+
|
| 157 |
+
cdef WrittenFile _finish_write(self, path, base_dir,
|
| 158 |
+
CFileWriter* file_writer):
|
| 159 |
+
cdef:
|
| 160 |
+
FileMetaData parquet_metadata
|
| 161 |
+
CParquetFileWriter* parquet_file_writer
|
| 162 |
+
|
| 163 |
+
parquet_metadata = None
|
| 164 |
+
parquet_file_writer = dynamic_cast[_CParquetFileWriterPtr](file_writer)
|
| 165 |
+
with nogil:
|
| 166 |
+
metadata = deref(
|
| 167 |
+
deref(parquet_file_writer).parquet_writer()).metadata()
|
| 168 |
+
if metadata:
|
| 169 |
+
parquet_metadata = FileMetaData.__new__(FileMetaData)
|
| 170 |
+
parquet_metadata.init(metadata)
|
| 171 |
+
parquet_metadata.set_file_path(os.path.relpath(path, base_dir))
|
| 172 |
+
|
| 173 |
+
size = GetResultValue(file_writer.GetBytesWritten())
|
| 174 |
+
|
| 175 |
+
return WrittenFile(path, parquet_metadata, size)
|
| 176 |
+
|
| 177 |
+
@property
|
| 178 |
+
def read_options(self):
|
| 179 |
+
cdef CParquetFileFormatReaderOptions* options
|
| 180 |
+
options = &self.parquet_format.reader_options
|
| 181 |
+
parquet_read_options = ParquetReadOptions(
|
| 182 |
+
dictionary_columns={frombytes(col)
|
| 183 |
+
for col in options.dict_columns},
|
| 184 |
+
)
|
| 185 |
+
# Read options getter/setter works with strings so setting
|
| 186 |
+
# the private property which uses the C Type
|
| 187 |
+
parquet_read_options._coerce_int96_timestamp_unit = \
|
| 188 |
+
options.coerce_int96_timestamp_unit
|
| 189 |
+
parquet_read_options._binary_type = options.binary_type
|
| 190 |
+
parquet_read_options._list_type = options.list_type
|
| 191 |
+
return parquet_read_options
|
| 192 |
+
|
| 193 |
+
def make_write_options(self, **kwargs):
|
| 194 |
+
"""
|
| 195 |
+
Parameters
|
| 196 |
+
----------
|
| 197 |
+
**kwargs : dict
|
| 198 |
+
|
| 199 |
+
Returns
|
| 200 |
+
-------
|
| 201 |
+
pyarrow.dataset.FileWriteOptions
|
| 202 |
+
"""
|
| 203 |
+
# Safeguard from calling make_write_options as a static class method
|
| 204 |
+
if not isinstance(self, ParquetFileFormat):
|
| 205 |
+
raise TypeError("make_write_options() should be called on "
|
| 206 |
+
"an instance of ParquetFileFormat")
|
| 207 |
+
opts = FileFormat.make_write_options(self)
|
| 208 |
+
(<ParquetFileWriteOptions> opts).update(**kwargs)
|
| 209 |
+
return opts
|
| 210 |
+
|
| 211 |
+
cdef _set_default_fragment_scan_options(self, FragmentScanOptions options):
|
| 212 |
+
if options.type_name == 'parquet':
|
| 213 |
+
self.parquet_format.default_fragment_scan_options = options.wrapped
|
| 214 |
+
else:
|
| 215 |
+
super()._set_default_fragment_scan_options(options)
|
| 216 |
+
|
| 217 |
+
def equals(self, ParquetFileFormat other):
|
| 218 |
+
"""
|
| 219 |
+
Parameters
|
| 220 |
+
----------
|
| 221 |
+
other : pyarrow.dataset.ParquetFileFormat
|
| 222 |
+
|
| 223 |
+
Returns
|
| 224 |
+
-------
|
| 225 |
+
bool
|
| 226 |
+
"""
|
| 227 |
+
return (
|
| 228 |
+
self.read_options.equals(other.read_options) and
|
| 229 |
+
self.default_fragment_scan_options ==
|
| 230 |
+
other.default_fragment_scan_options
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
@property
|
| 234 |
+
def default_extname(self):
|
| 235 |
+
return "parquet"
|
| 236 |
+
|
| 237 |
+
def __reduce__(self):
|
| 238 |
+
return ParquetFileFormat, (self.read_options,
|
| 239 |
+
self.default_fragment_scan_options)
|
| 240 |
+
|
| 241 |
+
def __repr__(self):
|
| 242 |
+
return f"<ParquetFileFormat read_options={self.read_options}>"
|
| 243 |
+
|
| 244 |
+
def make_fragment(self, file, filesystem=None,
|
| 245 |
+
Expression partition_expression=None, row_groups=None, *, file_size=None):
|
| 246 |
+
"""
|
| 247 |
+
Make a FileFragment from a given file.
|
| 248 |
+
|
| 249 |
+
Parameters
|
| 250 |
+
----------
|
| 251 |
+
file : file-like object, path-like or str
|
| 252 |
+
The file or file path to make a fragment from.
|
| 253 |
+
filesystem : Filesystem, optional
|
| 254 |
+
If `filesystem` is given, `file` must be a string and specifies
|
| 255 |
+
the path of the file to read from the filesystem.
|
| 256 |
+
partition_expression : Expression, optional
|
| 257 |
+
An expression that is guaranteed true for all rows in the fragment. Allows
|
| 258 |
+
fragment to be potentially skipped while scanning with a filter.
|
| 259 |
+
row_groups : Iterable, optional
|
| 260 |
+
The indices of the row groups to include
|
| 261 |
+
file_size : int, optional
|
| 262 |
+
The size of the file in bytes. Can improve performance with high-latency filesystems
|
| 263 |
+
when file size needs to be known before reading.
|
| 264 |
+
|
| 265 |
+
Returns
|
| 266 |
+
-------
|
| 267 |
+
fragment : Fragment
|
| 268 |
+
The file fragment
|
| 269 |
+
"""
|
| 270 |
+
cdef:
|
| 271 |
+
vector[int] c_row_groups
|
| 272 |
+
if partition_expression is None:
|
| 273 |
+
partition_expression = _true
|
| 274 |
+
if row_groups is None:
|
| 275 |
+
return super().make_fragment(file, filesystem,
|
| 276 |
+
partition_expression, file_size=file_size)
|
| 277 |
+
|
| 278 |
+
c_source = _make_file_source(file, filesystem, file_size)
|
| 279 |
+
c_row_groups = [<int> row_group for row_group in set(row_groups)]
|
| 280 |
+
|
| 281 |
+
c_fragment = <shared_ptr[CFragment]> GetResultValue(
|
| 282 |
+
self.parquet_format.MakeFragment(move(c_source),
|
| 283 |
+
partition_expression.unwrap(),
|
| 284 |
+
<shared_ptr[CSchema]>nullptr,
|
| 285 |
+
move(c_row_groups)))
|
| 286 |
+
return Fragment.wrap(move(c_fragment))
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
class RowGroupInfo:
|
| 290 |
+
"""
|
| 291 |
+
A wrapper class for RowGroup information
|
| 292 |
+
|
| 293 |
+
Parameters
|
| 294 |
+
----------
|
| 295 |
+
id : integer
|
| 296 |
+
The group ID.
|
| 297 |
+
metadata : FileMetaData
|
| 298 |
+
The rowgroup metadata.
|
| 299 |
+
schema : Schema
|
| 300 |
+
Schema of the rows.
|
| 301 |
+
"""
|
| 302 |
+
|
| 303 |
+
def __init__(self, id, metadata, schema):
|
| 304 |
+
self.id = id
|
| 305 |
+
self.metadata = metadata
|
| 306 |
+
self.schema = schema
|
| 307 |
+
|
| 308 |
+
@property
|
| 309 |
+
def num_rows(self):
|
| 310 |
+
return self.metadata.num_rows
|
| 311 |
+
|
| 312 |
+
@property
|
| 313 |
+
def total_byte_size(self):
|
| 314 |
+
return self.metadata.total_byte_size
|
| 315 |
+
|
| 316 |
+
@property
|
| 317 |
+
def statistics(self):
|
| 318 |
+
def name_stats(i):
|
| 319 |
+
col = self.metadata.column(i)
|
| 320 |
+
|
| 321 |
+
stats = col.statistics
|
| 322 |
+
if stats is None or not stats.has_min_max:
|
| 323 |
+
return None, None
|
| 324 |
+
|
| 325 |
+
name = col.path_in_schema
|
| 326 |
+
field_index = self.schema.get_field_index(name)
|
| 327 |
+
if field_index < 0:
|
| 328 |
+
return None, None
|
| 329 |
+
|
| 330 |
+
typ = self.schema.field(field_index).type
|
| 331 |
+
return col.path_in_schema, {
|
| 332 |
+
'min': pa.scalar(stats.min, type=typ).as_py(),
|
| 333 |
+
'max': pa.scalar(stats.max, type=typ).as_py()
|
| 334 |
+
}
|
| 335 |
+
|
| 336 |
+
return {
|
| 337 |
+
name: stats for name, stats
|
| 338 |
+
in map(name_stats, range(self.metadata.num_columns))
|
| 339 |
+
if stats is not None
|
| 340 |
+
}
|
| 341 |
+
|
| 342 |
+
def __repr__(self):
|
| 343 |
+
return f"RowGroupInfo({self.id})"
|
| 344 |
+
|
| 345 |
+
def __eq__(self, other):
|
| 346 |
+
if isinstance(other, int):
|
| 347 |
+
return self.id == other
|
| 348 |
+
if not isinstance(other, RowGroupInfo):
|
| 349 |
+
return False
|
| 350 |
+
return self.id == other.id
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
cdef class ParquetFileFragment(FileFragment):
|
| 354 |
+
"""A Fragment representing a parquet file."""
|
| 355 |
+
|
| 356 |
+
cdef:
|
| 357 |
+
CParquetFileFragment* parquet_file_fragment
|
| 358 |
+
|
| 359 |
+
cdef void init(self, const shared_ptr[CFragment]& sp):
|
| 360 |
+
FileFragment.init(self, sp)
|
| 361 |
+
self.parquet_file_fragment = <CParquetFileFragment*> sp.get()
|
| 362 |
+
|
| 363 |
+
def __reduce__(self):
|
| 364 |
+
buffer = self.buffer
|
| 365 |
+
# parquet_file_fragment.row_groups() is empty if the metadata
|
| 366 |
+
# information of the file is not yet populated
|
| 367 |
+
if not bool(self.parquet_file_fragment.row_groups()):
|
| 368 |
+
row_groups = None
|
| 369 |
+
else:
|
| 370 |
+
row_groups = [row_group.id for row_group in self.row_groups]
|
| 371 |
+
|
| 372 |
+
return self.format.make_fragment, (
|
| 373 |
+
self.path if buffer is None else buffer,
|
| 374 |
+
self.filesystem,
|
| 375 |
+
self.partition_expression,
|
| 376 |
+
row_groups
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
def ensure_complete_metadata(self):
|
| 380 |
+
"""
|
| 381 |
+
Ensure that all metadata (statistics, physical schema, ...) have
|
| 382 |
+
been read and cached in this fragment.
|
| 383 |
+
"""
|
| 384 |
+
with nogil:
|
| 385 |
+
check_status(self.parquet_file_fragment.EnsureCompleteMetadata())
|
| 386 |
+
|
| 387 |
+
@property
|
| 388 |
+
def row_groups(self):
|
| 389 |
+
metadata = self.metadata
|
| 390 |
+
cdef vector[int] row_groups = self.parquet_file_fragment.row_groups()
|
| 391 |
+
return [RowGroupInfo(i, metadata.row_group(i), self.physical_schema)
|
| 392 |
+
for i in row_groups]
|
| 393 |
+
|
| 394 |
+
@property
|
| 395 |
+
def metadata(self):
|
| 396 |
+
self.ensure_complete_metadata()
|
| 397 |
+
cdef FileMetaData metadata = FileMetaData.__new__(FileMetaData)
|
| 398 |
+
metadata.init(self.parquet_file_fragment.metadata())
|
| 399 |
+
return metadata
|
| 400 |
+
|
| 401 |
+
@property
|
| 402 |
+
def num_row_groups(self):
|
| 403 |
+
"""
|
| 404 |
+
Return the number of row groups viewed by this fragment (not the
|
| 405 |
+
number of row groups in the origin file).
|
| 406 |
+
"""
|
| 407 |
+
self.ensure_complete_metadata()
|
| 408 |
+
return self.parquet_file_fragment.row_groups().size()
|
| 409 |
+
|
| 410 |
+
def split_by_row_group(self, Expression filter=None,
|
| 411 |
+
Schema schema=None):
|
| 412 |
+
"""
|
| 413 |
+
Split the fragment into multiple fragments.
|
| 414 |
+
|
| 415 |
+
Yield a Fragment wrapping each row group in this ParquetFileFragment.
|
| 416 |
+
Row groups will be excluded whose metadata contradicts the optional
|
| 417 |
+
filter.
|
| 418 |
+
|
| 419 |
+
Parameters
|
| 420 |
+
----------
|
| 421 |
+
filter : Expression, default None
|
| 422 |
+
Only include the row groups which satisfy this predicate (using
|
| 423 |
+
the Parquet RowGroup statistics).
|
| 424 |
+
schema : Schema, default None
|
| 425 |
+
Schema to use when filtering row groups. Defaults to the
|
| 426 |
+
Fragment's physical schema
|
| 427 |
+
|
| 428 |
+
Returns
|
| 429 |
+
-------
|
| 430 |
+
A list of Fragments
|
| 431 |
+
"""
|
| 432 |
+
cdef:
|
| 433 |
+
vector[shared_ptr[CFragment]] c_fragments
|
| 434 |
+
CExpression c_filter
|
| 435 |
+
shared_ptr[CFragment] c_fragment
|
| 436 |
+
|
| 437 |
+
schema = schema or self.physical_schema
|
| 438 |
+
c_filter = _bind(filter, schema)
|
| 439 |
+
with nogil:
|
| 440 |
+
c_fragments = move(GetResultValue(
|
| 441 |
+
self.parquet_file_fragment.SplitByRowGroup(move(c_filter))))
|
| 442 |
+
|
| 443 |
+
return [Fragment.wrap(c_fragment) for c_fragment in c_fragments]
|
| 444 |
+
|
| 445 |
+
def subset(self, Expression filter=None, Schema schema=None,
|
| 446 |
+
object row_group_ids=None):
|
| 447 |
+
"""
|
| 448 |
+
Create a subset of the fragment (viewing a subset of the row groups).
|
| 449 |
+
|
| 450 |
+
Subset can be specified by either a filter predicate (with optional
|
| 451 |
+
schema) or by a list of row group IDs. Note that when using a filter,
|
| 452 |
+
the resulting fragment can be empty (viewing no row groups).
|
| 453 |
+
|
| 454 |
+
Parameters
|
| 455 |
+
----------
|
| 456 |
+
filter : Expression, default None
|
| 457 |
+
Only include the row groups which satisfy this predicate (using
|
| 458 |
+
the Parquet RowGroup statistics).
|
| 459 |
+
schema : Schema, default None
|
| 460 |
+
Schema to use when filtering row groups. Defaults to the
|
| 461 |
+
Fragment's physical schema
|
| 462 |
+
row_group_ids : list of ints
|
| 463 |
+
The row group IDs to include in the subset. Can only be specified
|
| 464 |
+
if `filter` is None.
|
| 465 |
+
|
| 466 |
+
Returns
|
| 467 |
+
-------
|
| 468 |
+
ParquetFileFragment
|
| 469 |
+
"""
|
| 470 |
+
cdef:
|
| 471 |
+
CExpression c_filter
|
| 472 |
+
vector[int] c_row_group_ids
|
| 473 |
+
shared_ptr[CFragment] c_fragment
|
| 474 |
+
|
| 475 |
+
if filter is not None and row_group_ids is not None:
|
| 476 |
+
raise ValueError(
|
| 477 |
+
"Cannot specify both 'filter' and 'row_group_ids'."
|
| 478 |
+
)
|
| 479 |
+
|
| 480 |
+
if filter is not None:
|
| 481 |
+
schema = schema or self.physical_schema
|
| 482 |
+
c_filter = _bind(filter, schema)
|
| 483 |
+
with nogil:
|
| 484 |
+
c_fragment = move(GetResultValue(
|
| 485 |
+
self.parquet_file_fragment.SubsetWithFilter(
|
| 486 |
+
move(c_filter))))
|
| 487 |
+
elif row_group_ids is not None:
|
| 488 |
+
c_row_group_ids = [
|
| 489 |
+
<int> row_group for row_group in sorted(set(row_group_ids))
|
| 490 |
+
]
|
| 491 |
+
with nogil:
|
| 492 |
+
c_fragment = move(GetResultValue(
|
| 493 |
+
self.parquet_file_fragment.SubsetWithIds(
|
| 494 |
+
move(c_row_group_ids))))
|
| 495 |
+
else:
|
| 496 |
+
raise ValueError(
|
| 497 |
+
"Need to specify one of 'filter' or 'row_group_ids'"
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
return Fragment.wrap(c_fragment)
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
cdef class ParquetReadOptions(_Weakrefable):
|
| 504 |
+
"""
|
| 505 |
+
Parquet format specific options for reading.
|
| 506 |
+
|
| 507 |
+
Parameters
|
| 508 |
+
----------
|
| 509 |
+
dictionary_columns : list of string, default None
|
| 510 |
+
Names of columns which should be dictionary encoded as
|
| 511 |
+
they are read
|
| 512 |
+
coerce_int96_timestamp_unit : str, default None
|
| 513 |
+
Cast timestamps that are stored in INT96 format to a particular
|
| 514 |
+
resolution (e.g. 'ms'). Setting to None is equivalent to 'ns'
|
| 515 |
+
and therefore INT96 timestamps will be inferred as timestamps
|
| 516 |
+
in nanoseconds
|
| 517 |
+
binary_type : pyarrow.DataType, default None
|
| 518 |
+
If given, Parquet binary columns will be read as this datatype.
|
| 519 |
+
This setting is ignored if a serialized Arrow schema is found in
|
| 520 |
+
the Parquet metadata.
|
| 521 |
+
list_type : subclass of pyarrow.DataType, default None
|
| 522 |
+
If given, non-MAP repeated columns will be read as an instance of
|
| 523 |
+
this datatype (either pyarrow.ListType or pyarrow.LargeListType).
|
| 524 |
+
This setting is ignored if a serialized Arrow schema is found in
|
| 525 |
+
the Parquet metadata.
|
| 526 |
+
"""
|
| 527 |
+
|
| 528 |
+
cdef public:
|
| 529 |
+
set dictionary_columns
|
| 530 |
+
TimeUnit _coerce_int96_timestamp_unit
|
| 531 |
+
Type _binary_type
|
| 532 |
+
Type _list_type
|
| 533 |
+
|
| 534 |
+
# Also see _PARQUET_READ_OPTIONS
|
| 535 |
+
def __init__(self, dictionary_columns=None,
|
| 536 |
+
coerce_int96_timestamp_unit=None,
|
| 537 |
+
binary_type=None, list_type=None):
|
| 538 |
+
self.dictionary_columns = set(dictionary_columns or set())
|
| 539 |
+
self.coerce_int96_timestamp_unit = coerce_int96_timestamp_unit
|
| 540 |
+
self.binary_type = binary_type
|
| 541 |
+
self.list_type = list_type
|
| 542 |
+
|
| 543 |
+
@property
|
| 544 |
+
def binary_type(self):
|
| 545 |
+
return primitive_type(self._binary_type)
|
| 546 |
+
|
| 547 |
+
@binary_type.setter
|
| 548 |
+
def binary_type(self, ty):
|
| 549 |
+
if ty is not None:
|
| 550 |
+
self._binary_type = pyarrow_unwrap_data_type(ty).get().id()
|
| 551 |
+
else:
|
| 552 |
+
self._binary_type = _Type_BINARY
|
| 553 |
+
|
| 554 |
+
@property
|
| 555 |
+
def list_type(self):
|
| 556 |
+
return (pa.LargeListType if self._list_type == _Type_LARGE_LIST
|
| 557 |
+
else pa.ListType)
|
| 558 |
+
|
| 559 |
+
@list_type.setter
|
| 560 |
+
def list_type(self, ty):
|
| 561 |
+
if ty is not None:
|
| 562 |
+
self._list_type = _unwrap_list_type(ty)
|
| 563 |
+
else:
|
| 564 |
+
self._list_type = _Type_LIST
|
| 565 |
+
|
| 566 |
+
@property
|
| 567 |
+
def coerce_int96_timestamp_unit(self):
|
| 568 |
+
return timeunit_to_string(self._coerce_int96_timestamp_unit)
|
| 569 |
+
|
| 570 |
+
@coerce_int96_timestamp_unit.setter
|
| 571 |
+
def coerce_int96_timestamp_unit(self, unit):
|
| 572 |
+
if unit is not None:
|
| 573 |
+
self._coerce_int96_timestamp_unit = string_to_timeunit(unit)
|
| 574 |
+
else:
|
| 575 |
+
self._coerce_int96_timestamp_unit = TimeUnit_NANO
|
| 576 |
+
|
| 577 |
+
def equals(self, ParquetReadOptions other):
|
| 578 |
+
"""
|
| 579 |
+
Parameters
|
| 580 |
+
----------
|
| 581 |
+
other : pyarrow.dataset.ParquetReadOptions
|
| 582 |
+
|
| 583 |
+
Returns
|
| 584 |
+
-------
|
| 585 |
+
bool
|
| 586 |
+
"""
|
| 587 |
+
return (self.dictionary_columns == other.dictionary_columns and
|
| 588 |
+
self._coerce_int96_timestamp_unit ==
|
| 589 |
+
other._coerce_int96_timestamp_unit and
|
| 590 |
+
self._binary_type == other._binary_type and
|
| 591 |
+
self._list_type == other._list_type)
|
| 592 |
+
|
| 593 |
+
def __eq__(self, other):
|
| 594 |
+
try:
|
| 595 |
+
return self.equals(other)
|
| 596 |
+
except TypeError:
|
| 597 |
+
return False
|
| 598 |
+
|
| 599 |
+
def __repr__(self):
|
| 600 |
+
return (
|
| 601 |
+
f"<ParquetReadOptions"
|
| 602 |
+
f" dictionary_columns={self.dictionary_columns}"
|
| 603 |
+
f" coerce_int96_timestamp_unit={self.coerce_int96_timestamp_unit}"
|
| 604 |
+
f" binary_type={self.binary_type}"
|
| 605 |
+
f" list_type={self.list_type}"
|
| 606 |
+
f">"
|
| 607 |
+
)
|
| 608 |
+
|
| 609 |
+
|
| 610 |
+
cdef class ParquetFileWriteOptions(FileWriteOptions):
|
| 611 |
+
|
| 612 |
+
def update(self, **kwargs):
|
| 613 |
+
"""
|
| 614 |
+
Parameters
|
| 615 |
+
----------
|
| 616 |
+
**kwargs : dict
|
| 617 |
+
"""
|
| 618 |
+
arrow_fields = {
|
| 619 |
+
"use_deprecated_int96_timestamps",
|
| 620 |
+
"coerce_timestamps",
|
| 621 |
+
"allow_truncated_timestamps",
|
| 622 |
+
"use_compliant_nested_type",
|
| 623 |
+
}
|
| 624 |
+
|
| 625 |
+
setters = set()
|
| 626 |
+
for name, value in kwargs.items():
|
| 627 |
+
if name not in self._properties:
|
| 628 |
+
raise TypeError("unexpected parquet write option: " + name)
|
| 629 |
+
self._properties[name] = value
|
| 630 |
+
if name in arrow_fields:
|
| 631 |
+
setters.add(self._set_arrow_properties)
|
| 632 |
+
elif name == "encryption_config" and value is not None:
|
| 633 |
+
setters.add(self._set_encryption_config)
|
| 634 |
+
else:
|
| 635 |
+
setters.add(self._set_properties)
|
| 636 |
+
|
| 637 |
+
for setter in setters:
|
| 638 |
+
setter()
|
| 639 |
+
|
| 640 |
+
def _set_properties(self):
|
| 641 |
+
cdef CParquetFileWriteOptions* opts = self.parquet_options
|
| 642 |
+
|
| 643 |
+
opts.writer_properties = _create_writer_properties(
|
| 644 |
+
use_dictionary=self._properties["use_dictionary"],
|
| 645 |
+
compression=self._properties["compression"],
|
| 646 |
+
version=self._properties["version"],
|
| 647 |
+
write_statistics=self._properties["write_statistics"],
|
| 648 |
+
data_page_size=self._properties["data_page_size"],
|
| 649 |
+
max_rows_per_page=self._properties["max_rows_per_page"],
|
| 650 |
+
compression_level=self._properties["compression_level"],
|
| 651 |
+
use_byte_stream_split=(
|
| 652 |
+
self._properties["use_byte_stream_split"]
|
| 653 |
+
),
|
| 654 |
+
column_encoding=self._properties["column_encoding"],
|
| 655 |
+
data_page_version=self._properties["data_page_version"],
|
| 656 |
+
encryption_properties=self._properties["encryption_properties"],
|
| 657 |
+
write_batch_size=self._properties["write_batch_size"],
|
| 658 |
+
dictionary_pagesize_limit=self._properties["dictionary_pagesize_limit"],
|
| 659 |
+
write_page_index=self._properties["write_page_index"],
|
| 660 |
+
write_page_checksum=self._properties["write_page_checksum"],
|
| 661 |
+
sorting_columns=self._properties["sorting_columns"],
|
| 662 |
+
store_decimal_as_integer=self._properties["store_decimal_as_integer"],
|
| 663 |
+
)
|
| 664 |
+
|
| 665 |
+
def _set_arrow_properties(self):
|
| 666 |
+
cdef CParquetFileWriteOptions* opts = self.parquet_options
|
| 667 |
+
|
| 668 |
+
opts.arrow_writer_properties = _create_arrow_writer_properties(
|
| 669 |
+
use_deprecated_int96_timestamps=(
|
| 670 |
+
self._properties["use_deprecated_int96_timestamps"]
|
| 671 |
+
),
|
| 672 |
+
coerce_timestamps=self._properties["coerce_timestamps"],
|
| 673 |
+
allow_truncated_timestamps=(
|
| 674 |
+
self._properties["allow_truncated_timestamps"]
|
| 675 |
+
),
|
| 676 |
+
writer_engine_version="V2",
|
| 677 |
+
use_compliant_nested_type=(
|
| 678 |
+
self._properties["use_compliant_nested_type"]
|
| 679 |
+
)
|
| 680 |
+
)
|
| 681 |
+
|
| 682 |
+
def _set_encryption_config(self):
|
| 683 |
+
if not parquet_encryption_enabled:
|
| 684 |
+
raise NotImplementedError(
|
| 685 |
+
"Encryption is not enabled in your installation of pyarrow, but an "
|
| 686 |
+
"encryption_config was provided."
|
| 687 |
+
)
|
| 688 |
+
set_encryption_config(self, self._properties["encryption_config"])
|
| 689 |
+
|
| 690 |
+
cdef void init(self, const shared_ptr[CFileWriteOptions]& sp):
|
| 691 |
+
FileWriteOptions.init(self, sp)
|
| 692 |
+
self.parquet_options = <CParquetFileWriteOptions*> sp.get()
|
| 693 |
+
self._properties = dict(
|
| 694 |
+
use_dictionary=True,
|
| 695 |
+
compression="snappy",
|
| 696 |
+
version="2.6",
|
| 697 |
+
write_statistics=None,
|
| 698 |
+
data_page_size=None,
|
| 699 |
+
max_rows_per_page=None,
|
| 700 |
+
compression_level=None,
|
| 701 |
+
use_byte_stream_split=False,
|
| 702 |
+
column_encoding=None,
|
| 703 |
+
data_page_version="1.0",
|
| 704 |
+
use_deprecated_int96_timestamps=False,
|
| 705 |
+
coerce_timestamps=None,
|
| 706 |
+
allow_truncated_timestamps=False,
|
| 707 |
+
use_compliant_nested_type=True,
|
| 708 |
+
encryption_properties=None,
|
| 709 |
+
write_batch_size=None,
|
| 710 |
+
dictionary_pagesize_limit=None,
|
| 711 |
+
write_page_index=False,
|
| 712 |
+
encryption_config=None,
|
| 713 |
+
write_page_checksum=False,
|
| 714 |
+
sorting_columns=None,
|
| 715 |
+
store_decimal_as_integer=False,
|
| 716 |
+
)
|
| 717 |
+
|
| 718 |
+
self._set_properties()
|
| 719 |
+
self._set_arrow_properties()
|
| 720 |
+
|
| 721 |
+
def __repr__(self):
|
| 722 |
+
return f"<pyarrow.dataset.ParquetFileWriteOptions {self._properties}>"
|
| 723 |
+
|
| 724 |
+
|
| 725 |
+
cdef set _PARQUET_READ_OPTIONS = {
|
| 726 |
+
'dictionary_columns', 'coerce_int96_timestamp_unit', 'binary_type', 'list_type',
|
| 727 |
+
}
|
| 728 |
+
|
| 729 |
+
|
| 730 |
+
cdef class ParquetFragmentScanOptions(FragmentScanOptions):
|
| 731 |
+
"""
|
| 732 |
+
Scan-specific options for Parquet fragments.
|
| 733 |
+
|
| 734 |
+
Parameters
|
| 735 |
+
----------
|
| 736 |
+
use_buffered_stream : bool, default False
|
| 737 |
+
Read files through buffered input streams rather than loading entire
|
| 738 |
+
row groups at once. This may be enabled to reduce memory overhead.
|
| 739 |
+
Disabled by default.
|
| 740 |
+
buffer_size : int, default 8192
|
| 741 |
+
Size of buffered stream, if enabled. Default is 8KB.
|
| 742 |
+
pre_buffer : bool, default True
|
| 743 |
+
If enabled, pre-buffer the raw Parquet data instead of issuing one
|
| 744 |
+
read per column chunk. This can improve performance on high-latency
|
| 745 |
+
filesystems (e.g. S3, GCS) by coalescing and issuing file reads in
|
| 746 |
+
parallel using a background I/O thread pool.
|
| 747 |
+
Set to False if you want to prioritize minimal memory usage
|
| 748 |
+
over maximum speed.
|
| 749 |
+
cache_options : pyarrow.CacheOptions, default None
|
| 750 |
+
Cache options used when pre_buffer is enabled. The default values should
|
| 751 |
+
be good for most use cases. You may want to adjust these for example if
|
| 752 |
+
you have exceptionally high latency to the file system.
|
| 753 |
+
thrift_string_size_limit : int, default None
|
| 754 |
+
If not None, override the maximum total string size allocated
|
| 755 |
+
when decoding Thrift structures. The default limit should be
|
| 756 |
+
sufficient for most Parquet files.
|
| 757 |
+
thrift_container_size_limit : int, default None
|
| 758 |
+
If not None, override the maximum total size of containers allocated
|
| 759 |
+
when decoding Thrift structures. The default limit should be
|
| 760 |
+
sufficient for most Parquet files.
|
| 761 |
+
decryption_config : pyarrow.dataset.ParquetDecryptionConfig, default None
|
| 762 |
+
If not None, use the provided ParquetDecryptionConfig to decrypt the
|
| 763 |
+
Parquet file.
|
| 764 |
+
decryption_properties : pyarrow.parquet.FileDecryptionProperties, default None
|
| 765 |
+
If not None, use the provided FileDecryptionProperties to decrypt encrypted
|
| 766 |
+
Parquet file.
|
| 767 |
+
page_checksum_verification : bool, default False
|
| 768 |
+
If True, verify the page checksum for each page read from the file.
|
| 769 |
+
arrow_extensions_enabled : bool, default False
|
| 770 |
+
If True, read Parquet logical types as Arrow extension types where possible,
|
| 771 |
+
(e.g., read JSON as the canonical `arrow.json` extension type or UUID as
|
| 772 |
+
the canonical `arrow.uuid` extension type).
|
| 773 |
+
"""
|
| 774 |
+
|
| 775 |
+
# Avoid mistakingly creating attributes
|
| 776 |
+
__slots__ = ()
|
| 777 |
+
|
| 778 |
+
def __init__(self, *, bint use_buffered_stream=False,
|
| 779 |
+
buffer_size=8192,
|
| 780 |
+
bint pre_buffer=True,
|
| 781 |
+
cache_options=None,
|
| 782 |
+
thrift_string_size_limit=None,
|
| 783 |
+
thrift_container_size_limit=None,
|
| 784 |
+
decryption_config=None,
|
| 785 |
+
decryption_properties=None,
|
| 786 |
+
bint page_checksum_verification=False,
|
| 787 |
+
bint arrow_extensions_enabled=False):
|
| 788 |
+
self.init(shared_ptr[CFragmentScanOptions](
|
| 789 |
+
new CParquetFragmentScanOptions()))
|
| 790 |
+
self.use_buffered_stream = use_buffered_stream
|
| 791 |
+
self.buffer_size = buffer_size
|
| 792 |
+
if pre_buffer and not is_threading_enabled():
|
| 793 |
+
pre_buffer = False
|
| 794 |
+
self.pre_buffer = pre_buffer
|
| 795 |
+
if cache_options is not None:
|
| 796 |
+
self.cache_options = cache_options
|
| 797 |
+
if thrift_string_size_limit is not None:
|
| 798 |
+
self.thrift_string_size_limit = thrift_string_size_limit
|
| 799 |
+
if thrift_container_size_limit is not None:
|
| 800 |
+
self.thrift_container_size_limit = thrift_container_size_limit
|
| 801 |
+
if decryption_config is not None:
|
| 802 |
+
self.parquet_decryption_config = decryption_config
|
| 803 |
+
if decryption_properties is not None:
|
| 804 |
+
self.decryption_properties = decryption_properties
|
| 805 |
+
self.page_checksum_verification = page_checksum_verification
|
| 806 |
+
self.arrow_extensions_enabled = arrow_extensions_enabled
|
| 807 |
+
|
| 808 |
+
cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp):
|
| 809 |
+
FragmentScanOptions.init(self, sp)
|
| 810 |
+
self.parquet_options = <CParquetFragmentScanOptions*> sp.get()
|
| 811 |
+
|
| 812 |
+
cdef CReaderProperties* reader_properties(self):
|
| 813 |
+
return self.parquet_options.reader_properties.get()
|
| 814 |
+
|
| 815 |
+
cdef ArrowReaderProperties* arrow_reader_properties(self):
|
| 816 |
+
return self.parquet_options.arrow_reader_properties.get()
|
| 817 |
+
|
| 818 |
+
@property
|
| 819 |
+
def use_buffered_stream(self):
|
| 820 |
+
return self.reader_properties().is_buffered_stream_enabled()
|
| 821 |
+
|
| 822 |
+
@use_buffered_stream.setter
|
| 823 |
+
def use_buffered_stream(self, bint use_buffered_stream):
|
| 824 |
+
if use_buffered_stream:
|
| 825 |
+
self.reader_properties().enable_buffered_stream()
|
| 826 |
+
else:
|
| 827 |
+
self.reader_properties().disable_buffered_stream()
|
| 828 |
+
|
| 829 |
+
@property
|
| 830 |
+
def buffer_size(self):
|
| 831 |
+
return self.reader_properties().buffer_size()
|
| 832 |
+
|
| 833 |
+
@buffer_size.setter
|
| 834 |
+
def buffer_size(self, buffer_size):
|
| 835 |
+
if buffer_size <= 0:
|
| 836 |
+
raise ValueError("Buffer size must be larger than zero")
|
| 837 |
+
self.reader_properties().set_buffer_size(buffer_size)
|
| 838 |
+
|
| 839 |
+
@property
|
| 840 |
+
def pre_buffer(self):
|
| 841 |
+
return self.arrow_reader_properties().pre_buffer()
|
| 842 |
+
|
| 843 |
+
@pre_buffer.setter
|
| 844 |
+
def pre_buffer(self, bint pre_buffer):
|
| 845 |
+
if pre_buffer and not is_threading_enabled():
|
| 846 |
+
return
|
| 847 |
+
self.arrow_reader_properties().set_pre_buffer(pre_buffer)
|
| 848 |
+
|
| 849 |
+
@property
|
| 850 |
+
def cache_options(self):
|
| 851 |
+
return CacheOptions.wrap(self.arrow_reader_properties().cache_options())
|
| 852 |
+
|
| 853 |
+
@cache_options.setter
|
| 854 |
+
def cache_options(self, CacheOptions options):
|
| 855 |
+
self.arrow_reader_properties().set_cache_options(options.unwrap())
|
| 856 |
+
|
| 857 |
+
@property
|
| 858 |
+
def thrift_string_size_limit(self):
|
| 859 |
+
return self.reader_properties().thrift_string_size_limit()
|
| 860 |
+
|
| 861 |
+
@thrift_string_size_limit.setter
|
| 862 |
+
def thrift_string_size_limit(self, size):
|
| 863 |
+
if size <= 0:
|
| 864 |
+
raise ValueError("size must be larger than zero")
|
| 865 |
+
self.reader_properties().set_thrift_string_size_limit(size)
|
| 866 |
+
|
| 867 |
+
@property
|
| 868 |
+
def thrift_container_size_limit(self):
|
| 869 |
+
return self.reader_properties().thrift_container_size_limit()
|
| 870 |
+
|
| 871 |
+
@thrift_container_size_limit.setter
|
| 872 |
+
def thrift_container_size_limit(self, size):
|
| 873 |
+
if size <= 0:
|
| 874 |
+
raise ValueError("size must be larger than zero")
|
| 875 |
+
self.reader_properties().set_thrift_container_size_limit(size)
|
| 876 |
+
|
| 877 |
+
@property
|
| 878 |
+
def decryption_properties(self):
|
| 879 |
+
if not parquet_encryption_enabled:
|
| 880 |
+
raise NotImplementedError(
|
| 881 |
+
"Unable to access encryption features. "
|
| 882 |
+
"Encryption is not enabled in your installation of pyarrow."
|
| 883 |
+
)
|
| 884 |
+
return self._decryption_properties
|
| 885 |
+
|
| 886 |
+
@decryption_properties.setter
|
| 887 |
+
def decryption_properties(self, config):
|
| 888 |
+
if not parquet_encryption_enabled:
|
| 889 |
+
raise NotImplementedError(
|
| 890 |
+
"Encryption is not enabled in your installation of pyarrow, but "
|
| 891 |
+
"decryption_properties were provided."
|
| 892 |
+
)
|
| 893 |
+
set_decryption_properties(self, config)
|
| 894 |
+
self._decryption_properties = config
|
| 895 |
+
|
| 896 |
+
@property
|
| 897 |
+
def parquet_decryption_config(self):
|
| 898 |
+
if not parquet_encryption_enabled:
|
| 899 |
+
raise NotImplementedError(
|
| 900 |
+
"Unable to access encryption features. "
|
| 901 |
+
"Encryption is not enabled in your installation of pyarrow."
|
| 902 |
+
)
|
| 903 |
+
return self._parquet_decryption_config
|
| 904 |
+
|
| 905 |
+
@parquet_decryption_config.setter
|
| 906 |
+
def parquet_decryption_config(self, config):
|
| 907 |
+
if not parquet_encryption_enabled:
|
| 908 |
+
raise NotImplementedError(
|
| 909 |
+
"Encryption is not enabled in your installation of pyarrow, but a "
|
| 910 |
+
"decryption_config was provided."
|
| 911 |
+
)
|
| 912 |
+
set_decryption_config(self, config)
|
| 913 |
+
self._parquet_decryption_config = config
|
| 914 |
+
|
| 915 |
+
@property
|
| 916 |
+
def page_checksum_verification(self):
|
| 917 |
+
return self.reader_properties().page_checksum_verification()
|
| 918 |
+
|
| 919 |
+
@page_checksum_verification.setter
|
| 920 |
+
def page_checksum_verification(self, bint page_checksum_verification):
|
| 921 |
+
self.reader_properties().set_page_checksum_verification(page_checksum_verification)
|
| 922 |
+
|
| 923 |
+
@property
|
| 924 |
+
def arrow_extensions_enabled(self):
|
| 925 |
+
return self.arrow_reader_properties().get_arrow_extensions_enabled()
|
| 926 |
+
|
| 927 |
+
@arrow_extensions_enabled.setter
|
| 928 |
+
def arrow_extensions_enabled(self, bint arrow_extensions_enabled):
|
| 929 |
+
self.arrow_reader_properties().set_arrow_extensions_enabled(arrow_extensions_enabled)
|
| 930 |
+
|
| 931 |
+
def equals(self, ParquetFragmentScanOptions other):
|
| 932 |
+
"""
|
| 933 |
+
Parameters
|
| 934 |
+
----------
|
| 935 |
+
other : pyarrow.dataset.ParquetFragmentScanOptions
|
| 936 |
+
|
| 937 |
+
Returns
|
| 938 |
+
-------
|
| 939 |
+
bool
|
| 940 |
+
"""
|
| 941 |
+
attrs = (
|
| 942 |
+
self.use_buffered_stream, self.buffer_size, self.pre_buffer, self.cache_options,
|
| 943 |
+
self.thrift_string_size_limit, self.thrift_container_size_limit,
|
| 944 |
+
self.page_checksum_verification, self.arrow_extensions_enabled)
|
| 945 |
+
other_attrs = (
|
| 946 |
+
other.use_buffered_stream, other.buffer_size, other.pre_buffer, other.cache_options,
|
| 947 |
+
other.thrift_string_size_limit,
|
| 948 |
+
other.thrift_container_size_limit, other.page_checksum_verification,
|
| 949 |
+
other.arrow_extensions_enabled)
|
| 950 |
+
return attrs == other_attrs
|
| 951 |
+
|
| 952 |
+
@staticmethod
|
| 953 |
+
def _reconstruct(kwargs):
|
| 954 |
+
# __reduce__ doesn't allow passing named arguments directly to the
|
| 955 |
+
# reconstructor, hence this wrapper.
|
| 956 |
+
return ParquetFragmentScanOptions(**kwargs)
|
| 957 |
+
|
| 958 |
+
def __reduce__(self):
|
| 959 |
+
kwargs = dict(
|
| 960 |
+
use_buffered_stream=self.use_buffered_stream,
|
| 961 |
+
buffer_size=self.buffer_size,
|
| 962 |
+
pre_buffer=self.pre_buffer,
|
| 963 |
+
cache_options=self.cache_options,
|
| 964 |
+
thrift_string_size_limit=self.thrift_string_size_limit,
|
| 965 |
+
thrift_container_size_limit=self.thrift_container_size_limit,
|
| 966 |
+
page_checksum_verification=self.page_checksum_verification,
|
| 967 |
+
arrow_extensions_enabled=self.arrow_extensions_enabled
|
| 968 |
+
)
|
| 969 |
+
return ParquetFragmentScanOptions._reconstruct, (kwargs,)
|
| 970 |
+
|
| 971 |
+
|
| 972 |
+
cdef class ParquetFactoryOptions(_Weakrefable):
|
| 973 |
+
"""
|
| 974 |
+
Influences the discovery of parquet dataset.
|
| 975 |
+
|
| 976 |
+
Parameters
|
| 977 |
+
----------
|
| 978 |
+
partition_base_dir : str, optional
|
| 979 |
+
For the purposes of applying the partitioning, paths will be
|
| 980 |
+
stripped of the partition_base_dir. Files not matching the
|
| 981 |
+
partition_base_dir prefix will be skipped for partitioning discovery.
|
| 982 |
+
The ignored files will still be part of the Dataset, but will not
|
| 983 |
+
have partition information.
|
| 984 |
+
partitioning : Partitioning, PartitioningFactory, optional
|
| 985 |
+
The partitioning scheme applied to fragments, see ``Partitioning``.
|
| 986 |
+
validate_column_chunk_paths : bool, default False
|
| 987 |
+
Assert that all ColumnChunk paths are consistent. The parquet spec
|
| 988 |
+
allows for ColumnChunk data to be stored in multiple files, but
|
| 989 |
+
ParquetDatasetFactory supports only a single file with all ColumnChunk
|
| 990 |
+
data. If this flag is set construction of a ParquetDatasetFactory will
|
| 991 |
+
raise an error if ColumnChunk data is not resident in a single file.
|
| 992 |
+
"""
|
| 993 |
+
|
| 994 |
+
cdef:
|
| 995 |
+
CParquetFactoryOptions options
|
| 996 |
+
|
| 997 |
+
__slots__ = () # avoid mistakingly creating attributes
|
| 998 |
+
|
| 999 |
+
def __init__(self, partition_base_dir=None, partitioning=None,
|
| 1000 |
+
validate_column_chunk_paths=False):
|
| 1001 |
+
if isinstance(partitioning, PartitioningFactory):
|
| 1002 |
+
self.partitioning_factory = partitioning
|
| 1003 |
+
elif isinstance(partitioning, Partitioning):
|
| 1004 |
+
self.partitioning = partitioning
|
| 1005 |
+
|
| 1006 |
+
if partition_base_dir is not None:
|
| 1007 |
+
self.partition_base_dir = partition_base_dir
|
| 1008 |
+
|
| 1009 |
+
self.options.validate_column_chunk_paths = validate_column_chunk_paths
|
| 1010 |
+
|
| 1011 |
+
cdef inline CParquetFactoryOptions unwrap(self):
|
| 1012 |
+
return self.options
|
| 1013 |
+
|
| 1014 |
+
@property
|
| 1015 |
+
def partitioning(self):
|
| 1016 |
+
"""Partitioning to apply to discovered files.
|
| 1017 |
+
|
| 1018 |
+
NOTE: setting this property will overwrite partitioning_factory.
|
| 1019 |
+
"""
|
| 1020 |
+
c_partitioning = self.options.partitioning.partitioning()
|
| 1021 |
+
if c_partitioning.get() == nullptr:
|
| 1022 |
+
return None
|
| 1023 |
+
return Partitioning.wrap(c_partitioning)
|
| 1024 |
+
|
| 1025 |
+
@partitioning.setter
|
| 1026 |
+
def partitioning(self, Partitioning value):
|
| 1027 |
+
self.options.partitioning = (<Partitioning> value).unwrap()
|
| 1028 |
+
|
| 1029 |
+
@property
|
| 1030 |
+
def partitioning_factory(self):
|
| 1031 |
+
"""PartitioningFactory to apply to discovered files and
|
| 1032 |
+
discover a Partitioning.
|
| 1033 |
+
|
| 1034 |
+
NOTE: setting this property will overwrite partitioning.
|
| 1035 |
+
"""
|
| 1036 |
+
c_factory = self.options.partitioning.factory()
|
| 1037 |
+
if c_factory.get() == nullptr:
|
| 1038 |
+
return None
|
| 1039 |
+
return PartitioningFactory.wrap(c_factory, None, None)
|
| 1040 |
+
|
| 1041 |
+
@partitioning_factory.setter
|
| 1042 |
+
def partitioning_factory(self, PartitioningFactory value):
|
| 1043 |
+
self.options.partitioning = (<PartitioningFactory> value).unwrap()
|
| 1044 |
+
|
| 1045 |
+
@property
|
| 1046 |
+
def partition_base_dir(self):
|
| 1047 |
+
"""
|
| 1048 |
+
Base directory to strip paths before applying the partitioning.
|
| 1049 |
+
"""
|
| 1050 |
+
return frombytes(self.options.partition_base_dir)
|
| 1051 |
+
|
| 1052 |
+
@partition_base_dir.setter
|
| 1053 |
+
def partition_base_dir(self, value):
|
| 1054 |
+
self.options.partition_base_dir = tobytes(value)
|
| 1055 |
+
|
| 1056 |
+
@property
|
| 1057 |
+
def validate_column_chunk_paths(self):
|
| 1058 |
+
"""
|
| 1059 |
+
Base directory to strip paths before applying the partitioning.
|
| 1060 |
+
"""
|
| 1061 |
+
return self.options.validate_column_chunk_paths
|
| 1062 |
+
|
| 1063 |
+
@validate_column_chunk_paths.setter
|
| 1064 |
+
def validate_column_chunk_paths(self, value):
|
| 1065 |
+
self.options.validate_column_chunk_paths = value
|
| 1066 |
+
|
| 1067 |
+
|
| 1068 |
+
cdef class ParquetDatasetFactory(DatasetFactory):
|
| 1069 |
+
"""
|
| 1070 |
+
Create a ParquetDatasetFactory from a Parquet `_metadata` file.
|
| 1071 |
+
|
| 1072 |
+
Parameters
|
| 1073 |
+
----------
|
| 1074 |
+
metadata_path : str
|
| 1075 |
+
Path to the `_metadata` parquet metadata-only file generated with
|
| 1076 |
+
`pyarrow.parquet.write_metadata`.
|
| 1077 |
+
filesystem : pyarrow.fs.FileSystem
|
| 1078 |
+
Filesystem to read the metadata_path from, and subsequent parquet
|
| 1079 |
+
files.
|
| 1080 |
+
format : ParquetFileFormat
|
| 1081 |
+
Parquet format options.
|
| 1082 |
+
options : ParquetFactoryOptions, optional
|
| 1083 |
+
Various flags influencing the discovery of filesystem paths.
|
| 1084 |
+
"""
|
| 1085 |
+
|
| 1086 |
+
cdef:
|
| 1087 |
+
CParquetDatasetFactory* parquet_factory
|
| 1088 |
+
|
| 1089 |
+
def __init__(self, metadata_path, FileSystem filesystem not None,
|
| 1090 |
+
FileFormat format not None,
|
| 1091 |
+
ParquetFactoryOptions options=None):
|
| 1092 |
+
cdef:
|
| 1093 |
+
c_string c_path
|
| 1094 |
+
shared_ptr[CFileSystem] c_filesystem
|
| 1095 |
+
shared_ptr[CParquetFileFormat] c_format
|
| 1096 |
+
CResult[shared_ptr[CDatasetFactory]] result
|
| 1097 |
+
CParquetFactoryOptions c_options
|
| 1098 |
+
|
| 1099 |
+
c_path = tobytes(metadata_path)
|
| 1100 |
+
c_filesystem = filesystem.unwrap()
|
| 1101 |
+
c_format = static_pointer_cast[CParquetFileFormat, CFileFormat](
|
| 1102 |
+
format.unwrap())
|
| 1103 |
+
options = options or ParquetFactoryOptions()
|
| 1104 |
+
c_options = options.unwrap()
|
| 1105 |
+
|
| 1106 |
+
with nogil:
|
| 1107 |
+
result = CParquetDatasetFactory.MakeFromMetaDataPath(
|
| 1108 |
+
c_path, c_filesystem, c_format, c_options)
|
| 1109 |
+
self.init(GetResultValue(result))
|
| 1110 |
+
|
| 1111 |
+
cdef init(self, shared_ptr[CDatasetFactory]& sp):
|
| 1112 |
+
DatasetFactory.init(self, sp)
|
| 1113 |
+
self.parquet_factory = <CParquetDatasetFactory*> sp.get()
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.cp312-win_amd64.pyd
ADDED
|
Binary file (92.7 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dataset_parquet_encryption.pyx
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
"""Dataset support for Parquet encryption."""
|
| 21 |
+
|
| 22 |
+
from pyarrow.includes.libarrow_dataset_parquet cimport *
|
| 23 |
+
from pyarrow._parquet_encryption cimport *
|
| 24 |
+
from pyarrow._dataset_parquet cimport ParquetFragmentScanOptions, ParquetFileWriteOptions
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
cdef class ParquetEncryptionConfig(_Weakrefable):
|
| 28 |
+
"""
|
| 29 |
+
Core configuration class encapsulating parameters for high-level encryption
|
| 30 |
+
within the Parquet framework.
|
| 31 |
+
|
| 32 |
+
The ParquetEncryptionConfig class serves as a bridge for passing encryption-related
|
| 33 |
+
parameters to the appropriate components within the Parquet library. It maintains references
|
| 34 |
+
to objects that define the encryption strategy, Key Management Service (KMS) configuration,
|
| 35 |
+
and specific encryption configurations for Parquet data.
|
| 36 |
+
|
| 37 |
+
Parameters
|
| 38 |
+
----------
|
| 39 |
+
crypto_factory : pyarrow.parquet.encryption.CryptoFactory
|
| 40 |
+
Shared pointer to a `CryptoFactory` object. The `CryptoFactory` is responsible for
|
| 41 |
+
creating cryptographic components, such as encryptors and decryptors.
|
| 42 |
+
kms_connection_config : pyarrow.parquet.encryption.KmsConnectionConfig
|
| 43 |
+
Shared pointer to a `KmsConnectionConfig` object. This object holds the configuration
|
| 44 |
+
parameters necessary for connecting to a Key Management Service (KMS).
|
| 45 |
+
encryption_config : pyarrow.parquet.encryption.EncryptionConfiguration
|
| 46 |
+
Shared pointer to an `EncryptionConfiguration` object. This object defines specific
|
| 47 |
+
encryption settings for Parquet data, including the keys assigned to different columns.
|
| 48 |
+
|
| 49 |
+
Raises
|
| 50 |
+
------
|
| 51 |
+
ValueError
|
| 52 |
+
Raised if `encryption_config` is None.
|
| 53 |
+
"""
|
| 54 |
+
cdef:
|
| 55 |
+
shared_ptr[CParquetEncryptionConfig] c_config
|
| 56 |
+
|
| 57 |
+
# Avoid mistakenly creating attributes
|
| 58 |
+
__slots__ = ()
|
| 59 |
+
|
| 60 |
+
def __cinit__(self, CryptoFactory crypto_factory, KmsConnectionConfig kms_connection_config,
|
| 61 |
+
EncryptionConfiguration encryption_config):
|
| 62 |
+
|
| 63 |
+
cdef shared_ptr[CEncryptionConfiguration] c_encryption_config
|
| 64 |
+
|
| 65 |
+
if crypto_factory is None:
|
| 66 |
+
raise ValueError("crypto_factory cannot be None")
|
| 67 |
+
|
| 68 |
+
if kms_connection_config is None:
|
| 69 |
+
raise ValueError("kms_connection_config cannot be None")
|
| 70 |
+
|
| 71 |
+
if encryption_config is None:
|
| 72 |
+
raise ValueError("encryption_config cannot be None")
|
| 73 |
+
|
| 74 |
+
self.c_config.reset(new CParquetEncryptionConfig())
|
| 75 |
+
|
| 76 |
+
c_encryption_config = pyarrow_unwrap_encryptionconfig(
|
| 77 |
+
encryption_config)
|
| 78 |
+
|
| 79 |
+
self.c_config.get().crypto_factory = pyarrow_unwrap_cryptofactory(crypto_factory)
|
| 80 |
+
self.c_config.get().kms_connection_config = pyarrow_unwrap_kmsconnectionconfig(
|
| 81 |
+
kms_connection_config)
|
| 82 |
+
self.c_config.get().encryption_config = c_encryption_config
|
| 83 |
+
|
| 84 |
+
@staticmethod
|
| 85 |
+
cdef wrap(shared_ptr[CParquetEncryptionConfig] c_config):
|
| 86 |
+
cdef ParquetEncryptionConfig python_config = ParquetEncryptionConfig.__new__(ParquetEncryptionConfig)
|
| 87 |
+
python_config.c_config = c_config
|
| 88 |
+
return python_config
|
| 89 |
+
|
| 90 |
+
cdef shared_ptr[CParquetEncryptionConfig] unwrap(self):
|
| 91 |
+
return self.c_config
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
cdef class ParquetDecryptionConfig(_Weakrefable):
|
| 95 |
+
"""
|
| 96 |
+
Core configuration class encapsulating parameters for high-level decryption
|
| 97 |
+
within the Parquet framework.
|
| 98 |
+
|
| 99 |
+
ParquetDecryptionConfig is designed to pass decryption-related parameters to
|
| 100 |
+
the appropriate decryption components within the Parquet library. It holds references to
|
| 101 |
+
objects that define the decryption strategy, Key Management Service (KMS) configuration,
|
| 102 |
+
and specific decryption configurations for reading encrypted Parquet data.
|
| 103 |
+
|
| 104 |
+
Parameters
|
| 105 |
+
----------
|
| 106 |
+
crypto_factory : pyarrow.parquet.encryption.CryptoFactory
|
| 107 |
+
Shared pointer to a `CryptoFactory` object, pivotal in creating cryptographic
|
| 108 |
+
components for the decryption process.
|
| 109 |
+
kms_connection_config : pyarrow.parquet.encryption.KmsConnectionConfig
|
| 110 |
+
Shared pointer to a `KmsConnectionConfig` object, containing parameters necessary
|
| 111 |
+
for connecting to a Key Management Service (KMS) during decryption.
|
| 112 |
+
decryption_config : pyarrow.parquet.encryption.DecryptionConfiguration
|
| 113 |
+
Shared pointer to a `DecryptionConfiguration` object, specifying decryption settings
|
| 114 |
+
for reading encrypted Parquet data.
|
| 115 |
+
|
| 116 |
+
Raises
|
| 117 |
+
------
|
| 118 |
+
ValueError
|
| 119 |
+
Raised if `decryption_config` is None.
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
cdef:
|
| 123 |
+
shared_ptr[CParquetDecryptionConfig] c_config
|
| 124 |
+
|
| 125 |
+
# Avoid mistakingly creating attributes
|
| 126 |
+
__slots__ = ()
|
| 127 |
+
|
| 128 |
+
def __cinit__(self, CryptoFactory crypto_factory, KmsConnectionConfig kms_connection_config,
|
| 129 |
+
DecryptionConfiguration decryption_config):
|
| 130 |
+
|
| 131 |
+
cdef shared_ptr[CDecryptionConfiguration] c_decryption_config
|
| 132 |
+
|
| 133 |
+
if decryption_config is None:
|
| 134 |
+
raise ValueError(
|
| 135 |
+
"decryption_config cannot be None")
|
| 136 |
+
|
| 137 |
+
self.c_config.reset(new CParquetDecryptionConfig())
|
| 138 |
+
|
| 139 |
+
c_decryption_config = pyarrow_unwrap_decryptionconfig(
|
| 140 |
+
decryption_config)
|
| 141 |
+
|
| 142 |
+
self.c_config.get().crypto_factory = pyarrow_unwrap_cryptofactory(crypto_factory)
|
| 143 |
+
self.c_config.get().kms_connection_config = pyarrow_unwrap_kmsconnectionconfig(
|
| 144 |
+
kms_connection_config)
|
| 145 |
+
self.c_config.get().decryption_config = c_decryption_config
|
| 146 |
+
|
| 147 |
+
@staticmethod
|
| 148 |
+
cdef wrap(shared_ptr[CParquetDecryptionConfig] c_config):
|
| 149 |
+
cdef ParquetDecryptionConfig python_config = ParquetDecryptionConfig.__new__(ParquetDecryptionConfig)
|
| 150 |
+
python_config.c_config = c_config
|
| 151 |
+
return python_config
|
| 152 |
+
|
| 153 |
+
cdef shared_ptr[CParquetDecryptionConfig] unwrap(self):
|
| 154 |
+
return self.c_config
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def set_encryption_config(
|
| 158 |
+
ParquetFileWriteOptions opts not None,
|
| 159 |
+
ParquetEncryptionConfig config not None
|
| 160 |
+
):
|
| 161 |
+
cdef shared_ptr[CParquetEncryptionConfig] c_config = config.unwrap()
|
| 162 |
+
opts.parquet_options.parquet_encryption_config = c_config
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def set_decryption_properties(
|
| 166 |
+
ParquetFragmentScanOptions opts not None,
|
| 167 |
+
FileDecryptionProperties config not None
|
| 168 |
+
):
|
| 169 |
+
cdef CReaderProperties* reader_props = opts.reader_properties()
|
| 170 |
+
reader_props.file_decryption_properties(config.unwrap())
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def set_decryption_config(
|
| 174 |
+
ParquetFragmentScanOptions opts not None,
|
| 175 |
+
ParquetDecryptionConfig config not None
|
| 176 |
+
):
|
| 177 |
+
cdef shared_ptr[CParquetDecryptionConfig] c_config = config.unwrap()
|
| 178 |
+
opts.parquet_options.parquet_decryption_config = c_config
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_dlpack.pxi
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
cimport cpython
|
| 19 |
+
from cpython.pycapsule cimport PyCapsule_New
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
cdef void dlpack_pycapsule_deleter(object dltensor) noexcept:
|
| 23 |
+
cdef DLManagedTensor* dlm_tensor
|
| 24 |
+
cdef PyObject* err_type
|
| 25 |
+
cdef PyObject* err_value
|
| 26 |
+
cdef PyObject* err_traceback
|
| 27 |
+
|
| 28 |
+
# Do nothing if the capsule has been consumed
|
| 29 |
+
if cpython.PyCapsule_IsValid(dltensor, "used_dltensor"):
|
| 30 |
+
return
|
| 31 |
+
|
| 32 |
+
# An exception may be in-flight, we must save it in case
|
| 33 |
+
# we create another one
|
| 34 |
+
cpython.PyErr_Fetch(&err_type, &err_value, &err_traceback)
|
| 35 |
+
|
| 36 |
+
dlm_tensor = <DLManagedTensor*>cpython.PyCapsule_GetPointer(dltensor, 'dltensor')
|
| 37 |
+
if dlm_tensor == NULL:
|
| 38 |
+
cpython.PyErr_WriteUnraisable(dltensor)
|
| 39 |
+
# The deleter can be NULL if there is no way for the caller
|
| 40 |
+
# to provide a reasonable destructor
|
| 41 |
+
elif dlm_tensor.deleter:
|
| 42 |
+
dlm_tensor.deleter(dlm_tensor)
|
| 43 |
+
assert (not cpython.PyErr_Occurred())
|
| 44 |
+
|
| 45 |
+
# Set the error indicator from err_type, err_value, err_traceback
|
| 46 |
+
cpython.PyErr_Restore(err_type, err_value, err_traceback)
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.cp312-win_amd64.pyd
ADDED
|
Binary file (76.8 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_feather.pyx
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# ---------------------------------------------------------------------
|
| 19 |
+
# Implement Feather file format
|
| 20 |
+
|
| 21 |
+
# cython: profile=False
|
| 22 |
+
# distutils: language = c++
|
| 23 |
+
# cython: language_level=3
|
| 24 |
+
|
| 25 |
+
from cython.operator cimport dereference as deref
|
| 26 |
+
from pyarrow.includes.common cimport *
|
| 27 |
+
from pyarrow.includes.libarrow cimport *
|
| 28 |
+
from pyarrow.includes.libarrow_feather cimport *
|
| 29 |
+
from pyarrow.lib cimport (check_status, Table, _Weakrefable,
|
| 30 |
+
get_writer, get_reader, pyarrow_wrap_table)
|
| 31 |
+
from pyarrow.lib import tobytes
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class FeatherError(Exception):
|
| 35 |
+
pass
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def write_feather(Table table, object dest, compression=None,
|
| 39 |
+
compression_level=None, chunksize=None, version=2):
|
| 40 |
+
cdef shared_ptr[COutputStream] sink
|
| 41 |
+
get_writer(dest, &sink)
|
| 42 |
+
|
| 43 |
+
cdef CFeatherProperties properties
|
| 44 |
+
if version == 2:
|
| 45 |
+
properties.version = kFeatherV2Version
|
| 46 |
+
else:
|
| 47 |
+
properties.version = kFeatherV1Version
|
| 48 |
+
|
| 49 |
+
if compression == 'zstd':
|
| 50 |
+
properties.compression = CCompressionType_ZSTD
|
| 51 |
+
elif compression == 'lz4':
|
| 52 |
+
properties.compression = CCompressionType_LZ4_FRAME
|
| 53 |
+
else:
|
| 54 |
+
properties.compression = CCompressionType_UNCOMPRESSED
|
| 55 |
+
|
| 56 |
+
if chunksize is not None:
|
| 57 |
+
properties.chunksize = chunksize
|
| 58 |
+
|
| 59 |
+
if compression_level is not None:
|
| 60 |
+
properties.compression_level = compression_level
|
| 61 |
+
|
| 62 |
+
with nogil:
|
| 63 |
+
check_status(WriteFeather(deref(table.table), sink.get(),
|
| 64 |
+
properties))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
cdef class FeatherReader(_Weakrefable):
|
| 68 |
+
cdef:
|
| 69 |
+
shared_ptr[CFeatherReader] reader
|
| 70 |
+
|
| 71 |
+
def __cinit__(self, source, c_bool use_memory_map, c_bool use_threads):
|
| 72 |
+
cdef:
|
| 73 |
+
shared_ptr[CRandomAccessFile] reader
|
| 74 |
+
CIpcReadOptions options = CIpcReadOptions.Defaults()
|
| 75 |
+
options.use_threads = use_threads
|
| 76 |
+
|
| 77 |
+
get_reader(source, use_memory_map, &reader)
|
| 78 |
+
with nogil:
|
| 79 |
+
self.reader = GetResultValue(CFeatherReader.Open(reader, options))
|
| 80 |
+
|
| 81 |
+
@property
|
| 82 |
+
def version(self):
|
| 83 |
+
return self.reader.get().version()
|
| 84 |
+
|
| 85 |
+
def read(self):
|
| 86 |
+
cdef shared_ptr[CTable] sp_table
|
| 87 |
+
with nogil:
|
| 88 |
+
check_status(self.reader.get()
|
| 89 |
+
.Read(&sp_table))
|
| 90 |
+
|
| 91 |
+
return pyarrow_wrap_table(sp_table)
|
| 92 |
+
|
| 93 |
+
def read_indices(self, indices):
|
| 94 |
+
cdef:
|
| 95 |
+
shared_ptr[CTable] sp_table
|
| 96 |
+
vector[int] c_indices
|
| 97 |
+
|
| 98 |
+
for index in indices:
|
| 99 |
+
c_indices.push_back(index)
|
| 100 |
+
with nogil:
|
| 101 |
+
check_status(self.reader.get()
|
| 102 |
+
.Read(c_indices, &sp_table))
|
| 103 |
+
|
| 104 |
+
return pyarrow_wrap_table(sp_table)
|
| 105 |
+
|
| 106 |
+
def read_names(self, names):
|
| 107 |
+
cdef:
|
| 108 |
+
shared_ptr[CTable] sp_table
|
| 109 |
+
vector[c_string] c_names
|
| 110 |
+
|
| 111 |
+
for name in names:
|
| 112 |
+
c_names.push_back(tobytes(name))
|
| 113 |
+
with nogil:
|
| 114 |
+
check_status(self.reader.get()
|
| 115 |
+
.Read(c_names, &sp_table))
|
| 116 |
+
|
| 117 |
+
return pyarrow_wrap_table(sp_table)
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5c6a830239a0932aeb1f25e902d4883d25c5fbda493af209537a692ba9b7952e
|
| 3 |
+
size 852480
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_flight.pyx
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a73589e40c08c676cab65fbf245bd25fd6b4f68d09518395dc492b6f6461d358
|
| 3 |
+
size 303616
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pxd
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.includes.common cimport *
|
| 21 |
+
from pyarrow.includes.libarrow_fs cimport *
|
| 22 |
+
from pyarrow.lib import _detect_compression, frombytes, tobytes
|
| 23 |
+
from pyarrow.lib cimport *
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
cpdef enum FileType:
|
| 27 |
+
NotFound = <int8_t> CFileType_NotFound
|
| 28 |
+
Unknown = <int8_t> CFileType_Unknown
|
| 29 |
+
File = <int8_t> CFileType_File
|
| 30 |
+
Directory = <int8_t> CFileType_Directory
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
cdef class FileInfo(_Weakrefable):
|
| 34 |
+
cdef:
|
| 35 |
+
CFileInfo info
|
| 36 |
+
|
| 37 |
+
@staticmethod
|
| 38 |
+
cdef wrap(CFileInfo info)
|
| 39 |
+
|
| 40 |
+
cdef inline CFileInfo unwrap(self) nogil
|
| 41 |
+
|
| 42 |
+
@staticmethod
|
| 43 |
+
cdef CFileInfo unwrap_safe(obj)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
cdef class FileSelector(_Weakrefable):
|
| 47 |
+
cdef:
|
| 48 |
+
CFileSelector selector
|
| 49 |
+
|
| 50 |
+
@staticmethod
|
| 51 |
+
cdef FileSelector wrap(CFileSelector selector)
|
| 52 |
+
|
| 53 |
+
cdef inline CFileSelector unwrap(self) nogil
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
cdef class FileSystem(_Weakrefable):
|
| 57 |
+
cdef:
|
| 58 |
+
shared_ptr[CFileSystem] wrapped
|
| 59 |
+
CFileSystem* fs
|
| 60 |
+
|
| 61 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped)
|
| 62 |
+
|
| 63 |
+
@staticmethod
|
| 64 |
+
cdef wrap(const shared_ptr[CFileSystem]& sp)
|
| 65 |
+
|
| 66 |
+
cdef inline shared_ptr[CFileSystem] unwrap(self) nogil
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
cdef class LocalFileSystem(FileSystem):
|
| 70 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
cdef class SubTreeFileSystem(FileSystem):
|
| 74 |
+
cdef:
|
| 75 |
+
CSubTreeFileSystem* subtreefs
|
| 76 |
+
|
| 77 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
cdef class _MockFileSystem(FileSystem):
|
| 81 |
+
cdef:
|
| 82 |
+
CMockFileSystem* mockfs
|
| 83 |
+
|
| 84 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
cdef class PyFileSystem(FileSystem):
|
| 88 |
+
cdef:
|
| 89 |
+
CPyFileSystem* pyfs
|
| 90 |
+
|
| 91 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped)
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_fs.pyx
ADDED
|
@@ -0,0 +1,1653 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from cpython.datetime cimport datetime, PyDateTime_DateTime
|
| 21 |
+
|
| 22 |
+
from pyarrow.includes.common cimport *
|
| 23 |
+
from pyarrow.includes.libarrow_python cimport PyDateTime_to_TimePoint
|
| 24 |
+
from pyarrow.lib import _detect_compression, frombytes, tobytes
|
| 25 |
+
from pyarrow.lib cimport *
|
| 26 |
+
from pyarrow.util import _stringify_path
|
| 27 |
+
|
| 28 |
+
from abc import ABC, abstractmethod
|
| 29 |
+
from datetime import datetime, timezone
|
| 30 |
+
import os
|
| 31 |
+
import pathlib
|
| 32 |
+
import sys
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
cdef _init_ca_paths():
|
| 36 |
+
cdef CFileSystemGlobalOptions options
|
| 37 |
+
|
| 38 |
+
import ssl
|
| 39 |
+
paths = ssl.get_default_verify_paths()
|
| 40 |
+
if paths.cafile:
|
| 41 |
+
options.tls_ca_file_path = os.fsencode(paths.cafile)
|
| 42 |
+
if paths.capath:
|
| 43 |
+
options.tls_ca_dir_path = os.fsencode(paths.capath)
|
| 44 |
+
check_status(CFileSystemsInitialize(options))
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
if sys.platform == 'linux':
|
| 48 |
+
# ARROW-9261: On Linux, we may need to fixup the paths to TLS CA certs
|
| 49 |
+
# (especially in manylinux packages) since the values hardcoded at
|
| 50 |
+
# compile-time in libcurl may be wrong.
|
| 51 |
+
_init_ca_paths()
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
cdef inline c_string _path_as_bytes(path) except *:
|
| 55 |
+
# handle only abstract paths, not bound to any filesystem like pathlib is,
|
| 56 |
+
# so we only accept plain strings
|
| 57 |
+
if not isinstance(path, (bytes, str)):
|
| 58 |
+
raise TypeError('Path must be a string')
|
| 59 |
+
# tobytes always uses utf-8, which is more or less ok, at least on Windows
|
| 60 |
+
# since the C++ side then decodes from utf-8. On Unix, os.fsencode may be
|
| 61 |
+
# better.
|
| 62 |
+
return tobytes(path)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
cdef object _wrap_file_type(CFileType ty):
|
| 66 |
+
return FileType(<int8_t> ty)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
cdef CFileType _unwrap_file_type(FileType ty) except *:
|
| 70 |
+
if ty == FileType.Unknown:
|
| 71 |
+
return CFileType_Unknown
|
| 72 |
+
elif ty == FileType.NotFound:
|
| 73 |
+
return CFileType_NotFound
|
| 74 |
+
elif ty == FileType.File:
|
| 75 |
+
return CFileType_File
|
| 76 |
+
elif ty == FileType.Directory:
|
| 77 |
+
return CFileType_Directory
|
| 78 |
+
assert 0
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _file_type_to_string(ty):
|
| 82 |
+
# Python 3.11 changed str(IntEnum) to return the string representation
|
| 83 |
+
# of the integer value: https://github.com/python/cpython/issues/94763
|
| 84 |
+
return f"{ty.__class__.__name__}.{ty._name_}"
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
cdef class FileInfo(_Weakrefable):
|
| 88 |
+
"""
|
| 89 |
+
FileSystem entry info.
|
| 90 |
+
|
| 91 |
+
Parameters
|
| 92 |
+
----------
|
| 93 |
+
path : str
|
| 94 |
+
The full path to the filesystem entry.
|
| 95 |
+
type : FileType
|
| 96 |
+
The type of the filesystem entry.
|
| 97 |
+
mtime : datetime or float, default None
|
| 98 |
+
If given, the modification time of the filesystem entry.
|
| 99 |
+
If a float is given, it is the number of seconds since the
|
| 100 |
+
Unix epoch.
|
| 101 |
+
mtime_ns : int, default None
|
| 102 |
+
If given, the modification time of the filesystem entry,
|
| 103 |
+
in nanoseconds since the Unix epoch.
|
| 104 |
+
`mtime` and `mtime_ns` are mutually exclusive.
|
| 105 |
+
size : int, default None
|
| 106 |
+
If given, the filesystem entry size in bytes. This should only
|
| 107 |
+
be given if `type` is `FileType.File`.
|
| 108 |
+
|
| 109 |
+
Examples
|
| 110 |
+
--------
|
| 111 |
+
Generate a file:
|
| 112 |
+
|
| 113 |
+
>>> from pyarrow import fs
|
| 114 |
+
>>> local = fs.LocalFileSystem()
|
| 115 |
+
>>> path_fs = local_path + '/pyarrow-fs-example.dat'
|
| 116 |
+
>>> with local.open_output_stream(path_fs) as stream:
|
| 117 |
+
... stream.write(b'data')
|
| 118 |
+
4
|
| 119 |
+
|
| 120 |
+
Get FileInfo object using ``get_file_info()``:
|
| 121 |
+
|
| 122 |
+
>>> file_info = local.get_file_info(path_fs)
|
| 123 |
+
>>> file_info
|
| 124 |
+
<FileInfo for '.../pyarrow-fs-example.dat': type=FileType.File, size=4>
|
| 125 |
+
|
| 126 |
+
Inspect FileInfo attributes:
|
| 127 |
+
|
| 128 |
+
>>> file_info.type
|
| 129 |
+
<FileType.File: 2>
|
| 130 |
+
|
| 131 |
+
>>> file_info.is_file
|
| 132 |
+
True
|
| 133 |
+
|
| 134 |
+
>>> file_info.path
|
| 135 |
+
'/.../pyarrow-fs-example.dat'
|
| 136 |
+
|
| 137 |
+
>>> file_info.base_name
|
| 138 |
+
'pyarrow-fs-example.dat'
|
| 139 |
+
|
| 140 |
+
>>> file_info.size
|
| 141 |
+
4
|
| 142 |
+
|
| 143 |
+
>>> file_info.extension
|
| 144 |
+
'dat'
|
| 145 |
+
|
| 146 |
+
>>> file_info.mtime # doctest: +SKIP
|
| 147 |
+
datetime.datetime(2022, 6, 29, 7, 56, 10, 873922, tzinfo=datetime.timezone.utc)
|
| 148 |
+
|
| 149 |
+
>>> file_info.mtime_ns # doctest: +SKIP
|
| 150 |
+
1656489370873922073
|
| 151 |
+
"""
|
| 152 |
+
|
| 153 |
+
def __init__(self, path, FileType type=FileType.Unknown, *,
|
| 154 |
+
mtime=None, mtime_ns=None, size=None):
|
| 155 |
+
self.info.set_path(tobytes(path))
|
| 156 |
+
self.info.set_type(_unwrap_file_type(type))
|
| 157 |
+
if mtime is not None:
|
| 158 |
+
if mtime_ns is not None:
|
| 159 |
+
raise TypeError("Only one of mtime and mtime_ns "
|
| 160 |
+
"can be given")
|
| 161 |
+
if isinstance(mtime, datetime):
|
| 162 |
+
self.info.set_mtime(PyDateTime_to_TimePoint(
|
| 163 |
+
<PyDateTime_DateTime*> mtime))
|
| 164 |
+
else:
|
| 165 |
+
self.info.set_mtime(TimePoint_from_s(mtime))
|
| 166 |
+
elif mtime_ns is not None:
|
| 167 |
+
self.info.set_mtime(TimePoint_from_ns(mtime_ns))
|
| 168 |
+
if size is not None:
|
| 169 |
+
self.info.set_size(size)
|
| 170 |
+
|
| 171 |
+
@staticmethod
|
| 172 |
+
cdef wrap(CFileInfo info):
|
| 173 |
+
cdef FileInfo self = FileInfo.__new__(FileInfo)
|
| 174 |
+
self.info = move(info)
|
| 175 |
+
return self
|
| 176 |
+
|
| 177 |
+
cdef inline CFileInfo unwrap(self) nogil:
|
| 178 |
+
return self.info
|
| 179 |
+
|
| 180 |
+
@staticmethod
|
| 181 |
+
cdef CFileInfo unwrap_safe(obj):
|
| 182 |
+
if not isinstance(obj, FileInfo):
|
| 183 |
+
raise TypeError(f"Expected FileInfo instance, got {type(obj)}")
|
| 184 |
+
return (<FileInfo> obj).unwrap()
|
| 185 |
+
|
| 186 |
+
def __repr__(self):
|
| 187 |
+
def getvalue(attr):
|
| 188 |
+
try:
|
| 189 |
+
return getattr(self, attr)
|
| 190 |
+
except ValueError:
|
| 191 |
+
return ''
|
| 192 |
+
|
| 193 |
+
s = (f'<FileInfo for {self.path!r}: '
|
| 194 |
+
f'type={_file_type_to_string(self.type)}')
|
| 195 |
+
if self.is_file:
|
| 196 |
+
s += f', size={self.size}'
|
| 197 |
+
s += '>'
|
| 198 |
+
return s
|
| 199 |
+
|
| 200 |
+
@property
|
| 201 |
+
def type(self):
|
| 202 |
+
"""
|
| 203 |
+
Type of the file.
|
| 204 |
+
|
| 205 |
+
The returned enum values can be the following:
|
| 206 |
+
|
| 207 |
+
- FileType.NotFound: target does not exist
|
| 208 |
+
- FileType.Unknown: target exists but its type is unknown (could be a
|
| 209 |
+
special file such as a Unix socket or character device, or
|
| 210 |
+
Windows NUL / CON / ...)
|
| 211 |
+
- FileType.File: target is a regular file
|
| 212 |
+
- FileType.Directory: target is a regular directory
|
| 213 |
+
|
| 214 |
+
Returns
|
| 215 |
+
-------
|
| 216 |
+
type : FileType
|
| 217 |
+
"""
|
| 218 |
+
return _wrap_file_type(self.info.type())
|
| 219 |
+
|
| 220 |
+
@property
|
| 221 |
+
def is_file(self):
|
| 222 |
+
"""
|
| 223 |
+
"""
|
| 224 |
+
return self.type == FileType.File
|
| 225 |
+
|
| 226 |
+
@property
|
| 227 |
+
def path(self):
|
| 228 |
+
"""
|
| 229 |
+
The full file path in the filesystem.
|
| 230 |
+
|
| 231 |
+
Examples
|
| 232 |
+
--------
|
| 233 |
+
>>> file_info = local.get_file_info(path)
|
| 234 |
+
>>> file_info.path
|
| 235 |
+
'/.../pyarrow-fs-example.dat'
|
| 236 |
+
"""
|
| 237 |
+
return frombytes(self.info.path())
|
| 238 |
+
|
| 239 |
+
@property
|
| 240 |
+
def base_name(self):
|
| 241 |
+
"""
|
| 242 |
+
The file base name.
|
| 243 |
+
|
| 244 |
+
Component after the last directory separator.
|
| 245 |
+
|
| 246 |
+
Examples
|
| 247 |
+
--------
|
| 248 |
+
>>> file_info = local.get_file_info(path)
|
| 249 |
+
>>> file_info.base_name
|
| 250 |
+
'pyarrow-fs-example.dat'
|
| 251 |
+
"""
|
| 252 |
+
return frombytes(self.info.base_name())
|
| 253 |
+
|
| 254 |
+
@property
|
| 255 |
+
def size(self):
|
| 256 |
+
"""
|
| 257 |
+
The size in bytes, if available.
|
| 258 |
+
|
| 259 |
+
Only regular files are guaranteed to have a size.
|
| 260 |
+
|
| 261 |
+
Returns
|
| 262 |
+
-------
|
| 263 |
+
size : int or None
|
| 264 |
+
"""
|
| 265 |
+
cdef int64_t size
|
| 266 |
+
size = self.info.size()
|
| 267 |
+
return (size if size != -1 else None)
|
| 268 |
+
|
| 269 |
+
@property
|
| 270 |
+
def extension(self):
|
| 271 |
+
"""
|
| 272 |
+
The file extension.
|
| 273 |
+
|
| 274 |
+
Examples
|
| 275 |
+
--------
|
| 276 |
+
>>> file_info = local.get_file_info(path)
|
| 277 |
+
>>> file_info.extension
|
| 278 |
+
'dat'
|
| 279 |
+
"""
|
| 280 |
+
return frombytes(self.info.extension())
|
| 281 |
+
|
| 282 |
+
@property
|
| 283 |
+
def mtime(self):
|
| 284 |
+
"""
|
| 285 |
+
The time of last modification, if available.
|
| 286 |
+
|
| 287 |
+
Returns
|
| 288 |
+
-------
|
| 289 |
+
mtime : datetime.datetime or None
|
| 290 |
+
|
| 291 |
+
Examples
|
| 292 |
+
--------
|
| 293 |
+
>>> file_info = local.get_file_info(path)
|
| 294 |
+
>>> file_info.mtime # doctest: +SKIP
|
| 295 |
+
datetime.datetime(2022, 6, 29, 7, 56, 10, 873922, tzinfo=datetime.timezone.utc)
|
| 296 |
+
"""
|
| 297 |
+
cdef int64_t nanoseconds
|
| 298 |
+
nanoseconds = TimePoint_to_ns(self.info.mtime())
|
| 299 |
+
return (datetime.fromtimestamp(nanoseconds / 1.0e9, timezone.utc)
|
| 300 |
+
if nanoseconds != -1 else None)
|
| 301 |
+
|
| 302 |
+
@property
|
| 303 |
+
def mtime_ns(self):
|
| 304 |
+
"""
|
| 305 |
+
The time of last modification, if available, expressed in nanoseconds
|
| 306 |
+
since the Unix epoch.
|
| 307 |
+
|
| 308 |
+
Returns
|
| 309 |
+
-------
|
| 310 |
+
mtime_ns : int or None
|
| 311 |
+
|
| 312 |
+
Examples
|
| 313 |
+
--------
|
| 314 |
+
>>> file_info = local.get_file_info(path)
|
| 315 |
+
>>> file_info.mtime_ns # doctest: +SKIP
|
| 316 |
+
1656489370873922073
|
| 317 |
+
"""
|
| 318 |
+
cdef int64_t nanoseconds
|
| 319 |
+
nanoseconds = TimePoint_to_ns(self.info.mtime())
|
| 320 |
+
return (nanoseconds if nanoseconds != -1 else None)
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
cdef class FileSelector(_Weakrefable):
|
| 324 |
+
"""
|
| 325 |
+
File and directory selector.
|
| 326 |
+
|
| 327 |
+
It contains a set of options that describes how to search for files and
|
| 328 |
+
directories.
|
| 329 |
+
|
| 330 |
+
Parameters
|
| 331 |
+
----------
|
| 332 |
+
base_dir : str
|
| 333 |
+
The directory in which to select files. Relative paths also work, use
|
| 334 |
+
'.' for the current directory and '..' for the parent.
|
| 335 |
+
allow_not_found : bool, default False
|
| 336 |
+
The behavior if `base_dir` doesn't exist in the filesystem.
|
| 337 |
+
If false, an error is returned.
|
| 338 |
+
If true, an empty selection is returned.
|
| 339 |
+
recursive : bool, default False
|
| 340 |
+
Whether to recurse into subdirectories.
|
| 341 |
+
|
| 342 |
+
Examples
|
| 343 |
+
--------
|
| 344 |
+
List the contents of a directory and subdirectories:
|
| 345 |
+
|
| 346 |
+
>>> selector_1 = fs.FileSelector(local_path, recursive=True)
|
| 347 |
+
>>> local.get_file_info(selector_1) # doctest: +SKIP
|
| 348 |
+
[<FileInfo for 'tmp/alphabet/example.dat': type=FileType.File, size=4>,
|
| 349 |
+
<FileInfo for 'tmp/alphabet/subdir': type=FileType.Directory>,
|
| 350 |
+
<FileInfo for 'tmp/alphabet/subdir/example_copy.dat': type=FileType.File, size=4>]
|
| 351 |
+
|
| 352 |
+
List only the contents of the base directory:
|
| 353 |
+
|
| 354 |
+
>>> selector_2 = fs.FileSelector(local_path)
|
| 355 |
+
>>> local.get_file_info(selector_2) # doctest: +SKIP
|
| 356 |
+
[<FileInfo for 'tmp/alphabet/example.dat': type=FileType.File, size=4>,
|
| 357 |
+
<FileInfo for 'tmp/alphabet/subdir': type=FileType.Directory>]
|
| 358 |
+
|
| 359 |
+
Return empty selection if the directory doesn't exist:
|
| 360 |
+
|
| 361 |
+
>>> selector_not_found = fs.FileSelector(local_path + '/missing',
|
| 362 |
+
... recursive=True,
|
| 363 |
+
... allow_not_found=True)
|
| 364 |
+
>>> local.get_file_info(selector_not_found)
|
| 365 |
+
[]
|
| 366 |
+
"""
|
| 367 |
+
|
| 368 |
+
def __init__(self, base_dir, bint allow_not_found=False,
|
| 369 |
+
bint recursive=False):
|
| 370 |
+
self.base_dir = base_dir
|
| 371 |
+
self.recursive = recursive
|
| 372 |
+
self.allow_not_found = allow_not_found
|
| 373 |
+
|
| 374 |
+
@staticmethod
|
| 375 |
+
cdef FileSelector wrap(CFileSelector wrapped):
|
| 376 |
+
cdef FileSelector self = FileSelector.__new__(FileSelector)
|
| 377 |
+
self.selector = move(wrapped)
|
| 378 |
+
return self
|
| 379 |
+
|
| 380 |
+
cdef inline CFileSelector unwrap(self) nogil:
|
| 381 |
+
return self.selector
|
| 382 |
+
|
| 383 |
+
@property
|
| 384 |
+
def base_dir(self):
|
| 385 |
+
return frombytes(self.selector.base_dir)
|
| 386 |
+
|
| 387 |
+
@base_dir.setter
|
| 388 |
+
def base_dir(self, base_dir):
|
| 389 |
+
self.selector.base_dir = _path_as_bytes(base_dir)
|
| 390 |
+
|
| 391 |
+
@property
|
| 392 |
+
def allow_not_found(self):
|
| 393 |
+
return self.selector.allow_not_found
|
| 394 |
+
|
| 395 |
+
@allow_not_found.setter
|
| 396 |
+
def allow_not_found(self, bint allow_not_found):
|
| 397 |
+
self.selector.allow_not_found = allow_not_found
|
| 398 |
+
|
| 399 |
+
@property
|
| 400 |
+
def recursive(self):
|
| 401 |
+
return self.selector.recursive
|
| 402 |
+
|
| 403 |
+
@recursive.setter
|
| 404 |
+
def recursive(self, bint recursive):
|
| 405 |
+
self.selector.recursive = recursive
|
| 406 |
+
|
| 407 |
+
def __repr__(self):
|
| 408 |
+
return f"<FileSelector base_dir={self.base_dir!r} recursive={self.recursive}>"
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
cdef class FileSystem(_Weakrefable):
|
| 412 |
+
"""
|
| 413 |
+
Abstract file system API.
|
| 414 |
+
"""
|
| 415 |
+
|
| 416 |
+
def __init__(self):
|
| 417 |
+
raise TypeError("FileSystem is an abstract class, instantiate one of "
|
| 418 |
+
"the subclasses instead: LocalFileSystem or "
|
| 419 |
+
"SubTreeFileSystem")
|
| 420 |
+
|
| 421 |
+
@staticmethod
|
| 422 |
+
def _from_uri(uri):
|
| 423 |
+
fs, _path = FileSystem.from_uri(uri)
|
| 424 |
+
return fs
|
| 425 |
+
|
| 426 |
+
@staticmethod
|
| 427 |
+
def _fsspec_from_uri(uri):
|
| 428 |
+
"""Instantiate FSSpecHandler and path for the given URI."""
|
| 429 |
+
try:
|
| 430 |
+
import fsspec
|
| 431 |
+
except ImportError:
|
| 432 |
+
raise ImportError(
|
| 433 |
+
"`fsspec` is required to handle `fsspec+<filesystem>://` and `hf://` URIs."
|
| 434 |
+
)
|
| 435 |
+
from .fs import FSSpecHandler
|
| 436 |
+
|
| 437 |
+
uri = uri.removeprefix("fsspec+")
|
| 438 |
+
fs, path = fsspec.url_to_fs(uri)
|
| 439 |
+
fs = PyFileSystem(FSSpecHandler(fs))
|
| 440 |
+
|
| 441 |
+
return fs, path
|
| 442 |
+
|
| 443 |
+
@staticmethod
|
| 444 |
+
def _native_from_uri(uri):
|
| 445 |
+
"""Instantiate native FileSystem and path for the given URI."""
|
| 446 |
+
cdef:
|
| 447 |
+
c_string c_path
|
| 448 |
+
c_string c_uri
|
| 449 |
+
CResult[shared_ptr[CFileSystem]] result
|
| 450 |
+
|
| 451 |
+
if isinstance(uri, pathlib.Path):
|
| 452 |
+
# Make absolute
|
| 453 |
+
uri = uri.resolve().absolute()
|
| 454 |
+
c_uri = tobytes(_stringify_path(uri))
|
| 455 |
+
with nogil:
|
| 456 |
+
result = CFileSystemFromUriOrPath(c_uri, &c_path)
|
| 457 |
+
return FileSystem.wrap(GetResultValue(result)), frombytes(c_path)
|
| 458 |
+
|
| 459 |
+
@staticmethod
|
| 460 |
+
def from_uri(uri):
|
| 461 |
+
"""
|
| 462 |
+
Create a new FileSystem from URI or Path.
|
| 463 |
+
|
| 464 |
+
Recognized URI schemes are "file", "mock", "s3fs", "gs", "gcs", "hdfs" and "viewfs".
|
| 465 |
+
In addition, the argument can be a pathlib.Path object, or a string
|
| 466 |
+
describing an absolute local path.
|
| 467 |
+
|
| 468 |
+
Parameters
|
| 469 |
+
----------
|
| 470 |
+
uri : string
|
| 471 |
+
URI-based path, for example: file:///some/local/path.
|
| 472 |
+
|
| 473 |
+
Returns
|
| 474 |
+
-------
|
| 475 |
+
tuple of (FileSystem, str path)
|
| 476 |
+
With (filesystem, path) tuple where path is the abstract path
|
| 477 |
+
inside the FileSystem instance.
|
| 478 |
+
|
| 479 |
+
Examples
|
| 480 |
+
--------
|
| 481 |
+
Create a new FileSystem subclass from a URI:
|
| 482 |
+
|
| 483 |
+
>>> uri = f'file:///{local_path}/pyarrow-fs-example.dat'
|
| 484 |
+
>>> local_new, path_new = fs.FileSystem.from_uri(uri)
|
| 485 |
+
>>> local_new
|
| 486 |
+
<pyarrow._fs.LocalFileSystem object at ...
|
| 487 |
+
>>> path_new
|
| 488 |
+
'/.../pyarrow-fs-example.dat'
|
| 489 |
+
|
| 490 |
+
Or from a s3 bucket:
|
| 491 |
+
|
| 492 |
+
>>> fs.FileSystem.from_uri("s3://usgs-landsat/collection02/")
|
| 493 |
+
(<pyarrow._s3fs.S3FileSystem object at ...>, 'usgs-landsat/collection02')
|
| 494 |
+
|
| 495 |
+
Or from an fsspec+ URI:
|
| 496 |
+
|
| 497 |
+
>>> fs.FileSystem.from_uri("fsspec+memory:///path/to/file")
|
| 498 |
+
(<pyarrow._fs.PyFileSystem object at ...>, '/path/to/file')
|
| 499 |
+
"""
|
| 500 |
+
if isinstance(uri, str) and uri.startswith(("fsspec+", "hf://")):
|
| 501 |
+
return FileSystem._fsspec_from_uri(uri)
|
| 502 |
+
else:
|
| 503 |
+
return FileSystem._native_from_uri(uri)
|
| 504 |
+
|
| 505 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped):
|
| 506 |
+
self.wrapped = wrapped
|
| 507 |
+
self.fs = wrapped.get()
|
| 508 |
+
|
| 509 |
+
@staticmethod
|
| 510 |
+
cdef wrap(const shared_ptr[CFileSystem]& sp):
|
| 511 |
+
cdef FileSystem self
|
| 512 |
+
|
| 513 |
+
typ = frombytes(sp.get().type_name())
|
| 514 |
+
if typ == 'local':
|
| 515 |
+
self = LocalFileSystem.__new__(LocalFileSystem)
|
| 516 |
+
elif typ == 'mock':
|
| 517 |
+
self = _MockFileSystem.__new__(_MockFileSystem)
|
| 518 |
+
elif typ == 'subtree':
|
| 519 |
+
self = SubTreeFileSystem.__new__(SubTreeFileSystem)
|
| 520 |
+
elif typ == 's3':
|
| 521 |
+
from pyarrow._s3fs import S3FileSystem
|
| 522 |
+
self = S3FileSystem.__new__(S3FileSystem)
|
| 523 |
+
elif typ == 'gcs':
|
| 524 |
+
from pyarrow._gcsfs import GcsFileSystem
|
| 525 |
+
self = GcsFileSystem.__new__(GcsFileSystem)
|
| 526 |
+
elif typ == 'abfs':
|
| 527 |
+
from pyarrow._azurefs import AzureFileSystem
|
| 528 |
+
self = AzureFileSystem.__new__(AzureFileSystem)
|
| 529 |
+
elif typ == 'hdfs':
|
| 530 |
+
from pyarrow._hdfs import HadoopFileSystem
|
| 531 |
+
self = HadoopFileSystem.__new__(HadoopFileSystem)
|
| 532 |
+
elif typ.startswith('py::'):
|
| 533 |
+
self = PyFileSystem.__new__(PyFileSystem)
|
| 534 |
+
else:
|
| 535 |
+
raise TypeError('Cannot wrap FileSystem pointer')
|
| 536 |
+
|
| 537 |
+
self.init(sp)
|
| 538 |
+
return self
|
| 539 |
+
|
| 540 |
+
cdef inline shared_ptr[CFileSystem] unwrap(self) nogil:
|
| 541 |
+
return self.wrapped
|
| 542 |
+
|
| 543 |
+
def equals(self, FileSystem other not None):
|
| 544 |
+
"""
|
| 545 |
+
Parameters
|
| 546 |
+
----------
|
| 547 |
+
other : pyarrow.fs.FileSystem
|
| 548 |
+
|
| 549 |
+
Returns
|
| 550 |
+
-------
|
| 551 |
+
bool
|
| 552 |
+
"""
|
| 553 |
+
return self.fs.Equals(other.unwrap())
|
| 554 |
+
|
| 555 |
+
def __eq__(self, other):
|
| 556 |
+
try:
|
| 557 |
+
return self.equals(other)
|
| 558 |
+
except TypeError:
|
| 559 |
+
return NotImplemented
|
| 560 |
+
|
| 561 |
+
@property
|
| 562 |
+
def type_name(self):
|
| 563 |
+
"""
|
| 564 |
+
The filesystem's type name.
|
| 565 |
+
"""
|
| 566 |
+
return frombytes(self.fs.type_name())
|
| 567 |
+
|
| 568 |
+
def get_file_info(self, paths_or_selector):
|
| 569 |
+
"""
|
| 570 |
+
Get info for the given files.
|
| 571 |
+
|
| 572 |
+
Any symlink is automatically dereferenced, recursively. A non-existing
|
| 573 |
+
or unreachable file returns a FileStat object and has a FileType of
|
| 574 |
+
value NotFound. An exception indicates a truly exceptional condition
|
| 575 |
+
(low-level I/O error, etc.).
|
| 576 |
+
|
| 577 |
+
Parameters
|
| 578 |
+
----------
|
| 579 |
+
paths_or_selector : FileSelector, path-like or list of path-likes
|
| 580 |
+
Either a selector object, a path-like object or a list of
|
| 581 |
+
path-like objects. The selector's base directory will not be
|
| 582 |
+
part of the results, even if it exists. If it doesn't exist,
|
| 583 |
+
use `allow_not_found`.
|
| 584 |
+
|
| 585 |
+
Returns
|
| 586 |
+
-------
|
| 587 |
+
FileInfo or list of FileInfo
|
| 588 |
+
Single FileInfo object is returned for a single path, otherwise
|
| 589 |
+
a list of FileInfo objects is returned.
|
| 590 |
+
|
| 591 |
+
Examples
|
| 592 |
+
--------
|
| 593 |
+
>>> local
|
| 594 |
+
<pyarrow._fs.LocalFileSystem object at ...>
|
| 595 |
+
>>> local.get_file_info(f"/{local_path}/pyarrow-fs-example.dat")
|
| 596 |
+
<FileInfo for '/.../pyarrow-fs-example.dat': type=FileType.File, size=4>
|
| 597 |
+
"""
|
| 598 |
+
cdef:
|
| 599 |
+
CFileInfo info
|
| 600 |
+
c_string path
|
| 601 |
+
vector[CFileInfo] infos
|
| 602 |
+
vector[c_string] paths
|
| 603 |
+
CFileSelector selector
|
| 604 |
+
|
| 605 |
+
if isinstance(paths_or_selector, FileSelector):
|
| 606 |
+
with nogil:
|
| 607 |
+
selector = (<FileSelector>paths_or_selector).selector
|
| 608 |
+
infos = GetResultValue(self.fs.GetFileInfo(selector))
|
| 609 |
+
elif isinstance(paths_or_selector, (list, tuple)):
|
| 610 |
+
paths = [_path_as_bytes(s) for s in paths_or_selector]
|
| 611 |
+
with nogil:
|
| 612 |
+
infos = GetResultValue(self.fs.GetFileInfo(paths))
|
| 613 |
+
elif isinstance(paths_or_selector, (bytes, str)):
|
| 614 |
+
path =_path_as_bytes(paths_or_selector)
|
| 615 |
+
with nogil:
|
| 616 |
+
info = GetResultValue(self.fs.GetFileInfo(path))
|
| 617 |
+
return FileInfo.wrap(info)
|
| 618 |
+
else:
|
| 619 |
+
raise TypeError('Must pass either path(s) or a FileSelector')
|
| 620 |
+
|
| 621 |
+
return [FileInfo.wrap(info) for info in infos]
|
| 622 |
+
|
| 623 |
+
def create_dir(self, path, *, bint recursive=True):
|
| 624 |
+
"""
|
| 625 |
+
Create a directory and subdirectories.
|
| 626 |
+
|
| 627 |
+
This function succeeds if the directory already exists.
|
| 628 |
+
|
| 629 |
+
Parameters
|
| 630 |
+
----------
|
| 631 |
+
path : str
|
| 632 |
+
The path of the new directory.
|
| 633 |
+
recursive : bool, default True
|
| 634 |
+
Create nested directories as well.
|
| 635 |
+
"""
|
| 636 |
+
cdef c_string directory = _path_as_bytes(path)
|
| 637 |
+
with nogil:
|
| 638 |
+
check_status(self.fs.CreateDir(directory, recursive=recursive))
|
| 639 |
+
|
| 640 |
+
def delete_dir(self, path):
|
| 641 |
+
"""
|
| 642 |
+
Delete a directory and its contents, recursively.
|
| 643 |
+
|
| 644 |
+
Parameters
|
| 645 |
+
----------
|
| 646 |
+
path : str
|
| 647 |
+
The path of the directory to be deleted.
|
| 648 |
+
"""
|
| 649 |
+
cdef c_string directory = _path_as_bytes(path)
|
| 650 |
+
with nogil:
|
| 651 |
+
check_status(self.fs.DeleteDir(directory))
|
| 652 |
+
|
| 653 |
+
def delete_dir_contents(self, path, *,
|
| 654 |
+
bint accept_root_dir=False,
|
| 655 |
+
bint missing_dir_ok=False):
|
| 656 |
+
"""
|
| 657 |
+
Delete a directory's contents, recursively.
|
| 658 |
+
|
| 659 |
+
Like delete_dir, but doesn't delete the directory itself.
|
| 660 |
+
|
| 661 |
+
Parameters
|
| 662 |
+
----------
|
| 663 |
+
path : str
|
| 664 |
+
The path of the directory to be deleted.
|
| 665 |
+
accept_root_dir : boolean, default False
|
| 666 |
+
Allow deleting the root directory's contents
|
| 667 |
+
(if path is empty or "/")
|
| 668 |
+
missing_dir_ok : boolean, default False
|
| 669 |
+
If False then an error is raised if path does
|
| 670 |
+
not exist
|
| 671 |
+
"""
|
| 672 |
+
cdef c_string directory = _path_as_bytes(path)
|
| 673 |
+
if accept_root_dir and directory.strip(b"/") == b"":
|
| 674 |
+
with nogil:
|
| 675 |
+
check_status(self.fs.DeleteRootDirContents())
|
| 676 |
+
else:
|
| 677 |
+
with nogil:
|
| 678 |
+
check_status(self.fs.DeleteDirContents(directory,
|
| 679 |
+
missing_dir_ok))
|
| 680 |
+
|
| 681 |
+
def move(self, src, dest):
|
| 682 |
+
"""
|
| 683 |
+
Move / rename a file or directory.
|
| 684 |
+
|
| 685 |
+
If the destination exists:
|
| 686 |
+
- if it is a non-empty directory, an error is returned
|
| 687 |
+
- otherwise, if it has the same type as the source, it is replaced
|
| 688 |
+
- otherwise, behavior is unspecified (implementation-dependent).
|
| 689 |
+
|
| 690 |
+
Parameters
|
| 691 |
+
----------
|
| 692 |
+
src : str
|
| 693 |
+
The path of the file or the directory to be moved.
|
| 694 |
+
dest : str
|
| 695 |
+
The destination path where the file or directory is moved to.
|
| 696 |
+
|
| 697 |
+
Examples
|
| 698 |
+
--------
|
| 699 |
+
Create a new folder with a file:
|
| 700 |
+
|
| 701 |
+
>>> local.create_dir('/tmp/other_dir')
|
| 702 |
+
>>> local.copy_file(path,'/tmp/move_example.dat')
|
| 703 |
+
|
| 704 |
+
Move the file:
|
| 705 |
+
|
| 706 |
+
>>> local.move('/tmp/move_example.dat',
|
| 707 |
+
... '/tmp/other_dir/move_example_2.dat')
|
| 708 |
+
|
| 709 |
+
Inspect the file info:
|
| 710 |
+
|
| 711 |
+
>>> local.get_file_info('/tmp/other_dir/move_example_2.dat')
|
| 712 |
+
<FileInfo for '/tmp/other_dir/move_example_2.dat': type=FileType.File, size=4>
|
| 713 |
+
>>> local.get_file_info('/tmp/move_example.dat')
|
| 714 |
+
<FileInfo for '/tmp/move_example.dat': type=FileType.NotFound>
|
| 715 |
+
|
| 716 |
+
Delete the folder:
|
| 717 |
+
>>> local.delete_dir('/tmp/other_dir')
|
| 718 |
+
"""
|
| 719 |
+
cdef:
|
| 720 |
+
c_string source = _path_as_bytes(src)
|
| 721 |
+
c_string destination = _path_as_bytes(dest)
|
| 722 |
+
with nogil:
|
| 723 |
+
check_status(self.fs.Move(source, destination))
|
| 724 |
+
|
| 725 |
+
def copy_file(self, src, dest):
|
| 726 |
+
"""
|
| 727 |
+
Copy a file.
|
| 728 |
+
|
| 729 |
+
If the destination exists and is a directory, an error is returned.
|
| 730 |
+
Otherwise, it is replaced.
|
| 731 |
+
|
| 732 |
+
Parameters
|
| 733 |
+
----------
|
| 734 |
+
src : str
|
| 735 |
+
The path of the file to be copied from.
|
| 736 |
+
dest : str
|
| 737 |
+
The destination path where the file is copied to.
|
| 738 |
+
|
| 739 |
+
Examples
|
| 740 |
+
--------
|
| 741 |
+
>>> local.copy_file(path,
|
| 742 |
+
... local_path + '/pyarrow-fs-example_copy.dat')
|
| 743 |
+
|
| 744 |
+
Inspect the file info:
|
| 745 |
+
|
| 746 |
+
>>> local.get_file_info(local_path + '/pyarrow-fs-example_copy.dat')
|
| 747 |
+
<FileInfo for '/.../pyarrow-fs-example_copy.dat': type=FileType.File, size=4>
|
| 748 |
+
>>> local.get_file_info(path)
|
| 749 |
+
<FileInfo for '/.../pyarrow-fs-example.dat': type=FileType.File, size=4>
|
| 750 |
+
"""
|
| 751 |
+
cdef:
|
| 752 |
+
c_string source = _path_as_bytes(src)
|
| 753 |
+
c_string destination = _path_as_bytes(dest)
|
| 754 |
+
with nogil:
|
| 755 |
+
check_status(self.fs.CopyFile(source, destination))
|
| 756 |
+
|
| 757 |
+
def delete_file(self, path):
|
| 758 |
+
"""
|
| 759 |
+
Delete a file.
|
| 760 |
+
|
| 761 |
+
Parameters
|
| 762 |
+
----------
|
| 763 |
+
path : str
|
| 764 |
+
The path of the file to be deleted.
|
| 765 |
+
"""
|
| 766 |
+
cdef c_string file = _path_as_bytes(path)
|
| 767 |
+
with nogil:
|
| 768 |
+
check_status(self.fs.DeleteFile(file))
|
| 769 |
+
|
| 770 |
+
def _wrap_input_stream(self, stream, path, compression, buffer_size):
|
| 771 |
+
if buffer_size is not None and buffer_size != 0:
|
| 772 |
+
stream = BufferedInputStream(stream, buffer_size)
|
| 773 |
+
if compression == 'detect':
|
| 774 |
+
compression = _detect_compression(path)
|
| 775 |
+
if compression is not None:
|
| 776 |
+
stream = CompressedInputStream(stream, compression)
|
| 777 |
+
return stream
|
| 778 |
+
|
| 779 |
+
def _wrap_output_stream(self, stream, path, compression, buffer_size):
|
| 780 |
+
if buffer_size is not None and buffer_size != 0:
|
| 781 |
+
stream = BufferedOutputStream(stream, buffer_size)
|
| 782 |
+
if compression == 'detect':
|
| 783 |
+
compression = _detect_compression(path)
|
| 784 |
+
if compression is not None:
|
| 785 |
+
stream = CompressedOutputStream(stream, compression)
|
| 786 |
+
return stream
|
| 787 |
+
|
| 788 |
+
def open_input_file(self, path):
|
| 789 |
+
"""
|
| 790 |
+
Open an input file for random access reading.
|
| 791 |
+
|
| 792 |
+
Parameters
|
| 793 |
+
----------
|
| 794 |
+
path : str
|
| 795 |
+
The source to open for reading.
|
| 796 |
+
|
| 797 |
+
Returns
|
| 798 |
+
-------
|
| 799 |
+
stream : NativeFile
|
| 800 |
+
|
| 801 |
+
Examples
|
| 802 |
+
--------
|
| 803 |
+
Print the data from the file with `open_input_file()`:
|
| 804 |
+
|
| 805 |
+
>>> with local.open_input_file(path) as f:
|
| 806 |
+
... print(f.readall())
|
| 807 |
+
b'data'
|
| 808 |
+
"""
|
| 809 |
+
cdef:
|
| 810 |
+
c_string pathstr = _path_as_bytes(path)
|
| 811 |
+
NativeFile stream = NativeFile()
|
| 812 |
+
shared_ptr[CRandomAccessFile] in_handle
|
| 813 |
+
|
| 814 |
+
with nogil:
|
| 815 |
+
in_handle = GetResultValue(self.fs.OpenInputFile(pathstr))
|
| 816 |
+
|
| 817 |
+
stream.set_random_access_file(in_handle)
|
| 818 |
+
stream.is_readable = True
|
| 819 |
+
return stream
|
| 820 |
+
|
| 821 |
+
def open_input_stream(self, path, compression='detect', buffer_size=None):
|
| 822 |
+
"""
|
| 823 |
+
Open an input stream for sequential reading.
|
| 824 |
+
|
| 825 |
+
Parameters
|
| 826 |
+
----------
|
| 827 |
+
path : str
|
| 828 |
+
The source to open for reading.
|
| 829 |
+
compression : str optional, default 'detect'
|
| 830 |
+
The compression algorithm to use for on-the-fly decompression.
|
| 831 |
+
If "detect" and source is a file path, then compression will be
|
| 832 |
+
chosen based on the file extension.
|
| 833 |
+
If None, no compression will be applied. Otherwise, a well-known
|
| 834 |
+
algorithm name must be supplied (e.g. "gzip").
|
| 835 |
+
buffer_size : int optional, default None
|
| 836 |
+
If None or 0, no buffering will happen. Otherwise the size of the
|
| 837 |
+
temporary read buffer.
|
| 838 |
+
|
| 839 |
+
Returns
|
| 840 |
+
-------
|
| 841 |
+
stream : NativeFile
|
| 842 |
+
|
| 843 |
+
Examples
|
| 844 |
+
--------
|
| 845 |
+
Print the data from the file with `open_input_stream()`:
|
| 846 |
+
|
| 847 |
+
>>> with local.open_input_stream(path) as f:
|
| 848 |
+
... print(f.readall())
|
| 849 |
+
b'data'
|
| 850 |
+
"""
|
| 851 |
+
cdef:
|
| 852 |
+
c_string pathstr = _path_as_bytes(path)
|
| 853 |
+
NativeFile stream = NativeFile()
|
| 854 |
+
shared_ptr[CInputStream] in_handle
|
| 855 |
+
|
| 856 |
+
with nogil:
|
| 857 |
+
in_handle = GetResultValue(self.fs.OpenInputStream(pathstr))
|
| 858 |
+
|
| 859 |
+
stream.set_input_stream(in_handle)
|
| 860 |
+
stream.is_readable = True
|
| 861 |
+
|
| 862 |
+
return self._wrap_input_stream(
|
| 863 |
+
stream, path=path, compression=compression, buffer_size=buffer_size
|
| 864 |
+
)
|
| 865 |
+
|
| 866 |
+
def open_output_stream(self, path, compression='detect',
|
| 867 |
+
buffer_size=None, metadata=None):
|
| 868 |
+
"""
|
| 869 |
+
Open an output stream for sequential writing.
|
| 870 |
+
|
| 871 |
+
If the target already exists, existing data is truncated.
|
| 872 |
+
|
| 873 |
+
Parameters
|
| 874 |
+
----------
|
| 875 |
+
path : str
|
| 876 |
+
The source to open for writing.
|
| 877 |
+
compression : str optional, default 'detect'
|
| 878 |
+
The compression algorithm to use for on-the-fly compression.
|
| 879 |
+
If "detect" and source is a file path, then compression will be
|
| 880 |
+
chosen based on the file extension.
|
| 881 |
+
If None, no compression will be applied. Otherwise, a well-known
|
| 882 |
+
algorithm name must be supplied (e.g. "gzip").
|
| 883 |
+
buffer_size : int optional, default None
|
| 884 |
+
If None or 0, no buffering will happen. Otherwise the size of the
|
| 885 |
+
temporary write buffer.
|
| 886 |
+
metadata : dict optional, default None
|
| 887 |
+
If not None, a mapping of string keys to string values.
|
| 888 |
+
Some filesystems support storing metadata along the file
|
| 889 |
+
(such as "Content-Type").
|
| 890 |
+
Unsupported metadata keys will be ignored.
|
| 891 |
+
|
| 892 |
+
Returns
|
| 893 |
+
-------
|
| 894 |
+
stream : NativeFile
|
| 895 |
+
|
| 896 |
+
Examples
|
| 897 |
+
--------
|
| 898 |
+
>>> local = fs.LocalFileSystem()
|
| 899 |
+
>>> with local.open_output_stream(path) as stream:
|
| 900 |
+
... stream.write(b'data')
|
| 901 |
+
4
|
| 902 |
+
"""
|
| 903 |
+
cdef:
|
| 904 |
+
c_string pathstr = _path_as_bytes(path)
|
| 905 |
+
NativeFile stream = NativeFile()
|
| 906 |
+
shared_ptr[COutputStream] out_handle
|
| 907 |
+
shared_ptr[const CKeyValueMetadata] c_metadata
|
| 908 |
+
|
| 909 |
+
if metadata is not None:
|
| 910 |
+
c_metadata = pyarrow_unwrap_metadata(KeyValueMetadata(metadata))
|
| 911 |
+
|
| 912 |
+
with nogil:
|
| 913 |
+
out_handle = GetResultValue(
|
| 914 |
+
self.fs.OpenOutputStream(pathstr, c_metadata))
|
| 915 |
+
|
| 916 |
+
stream.set_output_stream(out_handle)
|
| 917 |
+
stream.is_writable = True
|
| 918 |
+
|
| 919 |
+
return self._wrap_output_stream(
|
| 920 |
+
stream, path=path, compression=compression, buffer_size=buffer_size
|
| 921 |
+
)
|
| 922 |
+
|
| 923 |
+
def open_append_stream(self, path, compression='detect',
|
| 924 |
+
buffer_size=None, metadata=None):
|
| 925 |
+
"""
|
| 926 |
+
Open an output stream for appending.
|
| 927 |
+
|
| 928 |
+
If the target doesn't exist, a new empty file is created.
|
| 929 |
+
|
| 930 |
+
.. note::
|
| 931 |
+
Some filesystem implementations do not support efficient
|
| 932 |
+
appending to an existing file, in which case this method will
|
| 933 |
+
raise NotImplementedError.
|
| 934 |
+
Consider writing to multiple files (using e.g. the dataset layer)
|
| 935 |
+
instead.
|
| 936 |
+
|
| 937 |
+
Parameters
|
| 938 |
+
----------
|
| 939 |
+
path : str
|
| 940 |
+
The source to open for writing.
|
| 941 |
+
compression : str optional, default 'detect'
|
| 942 |
+
The compression algorithm to use for on-the-fly compression.
|
| 943 |
+
If "detect" and source is a file path, then compression will be
|
| 944 |
+
chosen based on the file extension.
|
| 945 |
+
If None, no compression will be applied. Otherwise, a well-known
|
| 946 |
+
algorithm name must be supplied (e.g. "gzip").
|
| 947 |
+
buffer_size : int optional, default None
|
| 948 |
+
If None or 0, no buffering will happen. Otherwise the size of the
|
| 949 |
+
temporary write buffer.
|
| 950 |
+
metadata : dict optional, default None
|
| 951 |
+
If not None, a mapping of string keys to string values.
|
| 952 |
+
Some filesystems support storing metadata along the file
|
| 953 |
+
(such as "Content-Type").
|
| 954 |
+
Unsupported metadata keys will be ignored.
|
| 955 |
+
|
| 956 |
+
Returns
|
| 957 |
+
-------
|
| 958 |
+
stream : NativeFile
|
| 959 |
+
|
| 960 |
+
Examples
|
| 961 |
+
--------
|
| 962 |
+
Append new data to a FileSystem subclass with nonempty file:
|
| 963 |
+
|
| 964 |
+
>>> with local.open_append_stream(path) as f:
|
| 965 |
+
... f.write(b'+newly added')
|
| 966 |
+
12
|
| 967 |
+
|
| 968 |
+
Print out the content to the file:
|
| 969 |
+
|
| 970 |
+
>>> with local.open_input_file(path) as f:
|
| 971 |
+
... print(f.readall())
|
| 972 |
+
b'data+newly added'
|
| 973 |
+
"""
|
| 974 |
+
cdef:
|
| 975 |
+
c_string pathstr = _path_as_bytes(path)
|
| 976 |
+
NativeFile stream = NativeFile()
|
| 977 |
+
shared_ptr[COutputStream] out_handle
|
| 978 |
+
shared_ptr[const CKeyValueMetadata] c_metadata
|
| 979 |
+
|
| 980 |
+
if metadata is not None:
|
| 981 |
+
c_metadata = pyarrow_unwrap_metadata(KeyValueMetadata(metadata))
|
| 982 |
+
|
| 983 |
+
with nogil:
|
| 984 |
+
out_handle = GetResultValue(
|
| 985 |
+
self.fs.OpenAppendStream(pathstr, c_metadata))
|
| 986 |
+
|
| 987 |
+
stream.set_output_stream(out_handle)
|
| 988 |
+
stream.is_writable = True
|
| 989 |
+
|
| 990 |
+
return self._wrap_output_stream(
|
| 991 |
+
stream, path=path, compression=compression, buffer_size=buffer_size
|
| 992 |
+
)
|
| 993 |
+
|
| 994 |
+
def normalize_path(self, path):
|
| 995 |
+
"""
|
| 996 |
+
Normalize filesystem path.
|
| 997 |
+
|
| 998 |
+
Parameters
|
| 999 |
+
----------
|
| 1000 |
+
path : str
|
| 1001 |
+
The path to normalize
|
| 1002 |
+
|
| 1003 |
+
Returns
|
| 1004 |
+
-------
|
| 1005 |
+
normalized_path : str
|
| 1006 |
+
The normalized path
|
| 1007 |
+
"""
|
| 1008 |
+
cdef:
|
| 1009 |
+
c_string c_path = _path_as_bytes(path)
|
| 1010 |
+
c_string c_path_normalized
|
| 1011 |
+
|
| 1012 |
+
c_path_normalized = GetResultValue(self.fs.NormalizePath(c_path))
|
| 1013 |
+
return frombytes(c_path_normalized)
|
| 1014 |
+
|
| 1015 |
+
|
| 1016 |
+
cdef class LocalFileSystem(FileSystem):
|
| 1017 |
+
"""
|
| 1018 |
+
A FileSystem implementation accessing files on the local machine.
|
| 1019 |
+
|
| 1020 |
+
Details such as symlinks are abstracted away (symlinks are always followed,
|
| 1021 |
+
except when deleting an entry).
|
| 1022 |
+
|
| 1023 |
+
Parameters
|
| 1024 |
+
----------
|
| 1025 |
+
use_mmap : bool, default False
|
| 1026 |
+
Whether open_input_stream and open_input_file should return
|
| 1027 |
+
a mmap'ed file or a regular file.
|
| 1028 |
+
|
| 1029 |
+
Examples
|
| 1030 |
+
--------
|
| 1031 |
+
Create a FileSystem object with LocalFileSystem constructor:
|
| 1032 |
+
|
| 1033 |
+
>>> from pyarrow import fs
|
| 1034 |
+
>>> local = fs.LocalFileSystem()
|
| 1035 |
+
>>> local
|
| 1036 |
+
<pyarrow._fs.LocalFileSystem object at ...>
|
| 1037 |
+
|
| 1038 |
+
and write data on to the file:
|
| 1039 |
+
|
| 1040 |
+
>>> with local.open_output_stream('/tmp/local_fs.dat') as stream:
|
| 1041 |
+
... stream.write(b'data')
|
| 1042 |
+
4
|
| 1043 |
+
>>> with local.open_input_stream('/tmp/local_fs.dat') as stream:
|
| 1044 |
+
... print(stream.readall())
|
| 1045 |
+
b'data'
|
| 1046 |
+
|
| 1047 |
+
Create a FileSystem object inferred from a URI of the saved file:
|
| 1048 |
+
|
| 1049 |
+
>>> local_new, path = fs.LocalFileSystem.from_uri('/tmp/local_fs.dat')
|
| 1050 |
+
>>> local_new
|
| 1051 |
+
<pyarrow._fs.LocalFileSystem object at ...
|
| 1052 |
+
>>> path
|
| 1053 |
+
'/tmp/local_fs.dat'
|
| 1054 |
+
|
| 1055 |
+
Check if FileSystems `local` and `local_new` are equal:
|
| 1056 |
+
|
| 1057 |
+
>>> local.equals(local_new)
|
| 1058 |
+
True
|
| 1059 |
+
|
| 1060 |
+
Compare two different FileSystems:
|
| 1061 |
+
|
| 1062 |
+
>>> local2 = fs.LocalFileSystem(use_mmap=True)
|
| 1063 |
+
>>> local.equals(local2)
|
| 1064 |
+
False
|
| 1065 |
+
|
| 1066 |
+
Copy a file and print out the data:
|
| 1067 |
+
|
| 1068 |
+
>>> local.copy_file('/tmp/local_fs.dat', '/tmp/local_fs-copy.dat')
|
| 1069 |
+
>>> with local.open_input_stream('/tmp/local_fs-copy.dat') as stream:
|
| 1070 |
+
... print(stream.readall())
|
| 1071 |
+
...
|
| 1072 |
+
b'data'
|
| 1073 |
+
|
| 1074 |
+
Open an output stream for appending, add text and print the new data:
|
| 1075 |
+
|
| 1076 |
+
>>> with local.open_append_stream('/tmp/local_fs-copy.dat') as f:
|
| 1077 |
+
... f.write(b'+newly added')
|
| 1078 |
+
12
|
| 1079 |
+
|
| 1080 |
+
>>> with local.open_input_stream('/tmp/local_fs-copy.dat') as f:
|
| 1081 |
+
... print(f.readall())
|
| 1082 |
+
b'data+newly added'
|
| 1083 |
+
|
| 1084 |
+
Create a directory, copy a file into it and then delete the whole directory:
|
| 1085 |
+
|
| 1086 |
+
>>> local.create_dir('/tmp/new_folder')
|
| 1087 |
+
>>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
|
| 1088 |
+
>>> local.get_file_info('/tmp/new_folder')
|
| 1089 |
+
<FileInfo for '/tmp/new_folder': type=FileType.Directory>
|
| 1090 |
+
>>> local.delete_dir('/tmp/new_folder')
|
| 1091 |
+
>>> local.get_file_info('/tmp/new_folder')
|
| 1092 |
+
<FileInfo for '/tmp/new_folder': type=FileType.NotFound>
|
| 1093 |
+
|
| 1094 |
+
Create a directory, copy a file into it and then delete
|
| 1095 |
+
the content of the directory:
|
| 1096 |
+
|
| 1097 |
+
>>> local.create_dir('/tmp/new_folder')
|
| 1098 |
+
>>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
|
| 1099 |
+
>>> local.get_file_info('/tmp/new_folder/local_fs.dat')
|
| 1100 |
+
<FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.File, size=4>
|
| 1101 |
+
>>> local.delete_dir_contents('/tmp/new_folder')
|
| 1102 |
+
>>> local.get_file_info('/tmp/new_folder')
|
| 1103 |
+
<FileInfo for '/tmp/new_folder': type=FileType.Directory>
|
| 1104 |
+
>>> local.get_file_info('/tmp/new_folder/local_fs.dat')
|
| 1105 |
+
<FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.NotFound>
|
| 1106 |
+
|
| 1107 |
+
Create a directory, copy a file into it and then delete
|
| 1108 |
+
the file from the directory:
|
| 1109 |
+
|
| 1110 |
+
>>> local.create_dir('/tmp/new_folder')
|
| 1111 |
+
>>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
|
| 1112 |
+
>>> local.delete_file('/tmp/new_folder/local_fs.dat')
|
| 1113 |
+
>>> local.get_file_info('/tmp/new_folder/local_fs.dat')
|
| 1114 |
+
<FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.NotFound>
|
| 1115 |
+
>>> local.get_file_info('/tmp/new_folder')
|
| 1116 |
+
<FileInfo for '/tmp/new_folder': type=FileType.Directory>
|
| 1117 |
+
|
| 1118 |
+
Move the file:
|
| 1119 |
+
|
| 1120 |
+
>>> local.move('/tmp/local_fs-copy.dat', '/tmp/new_folder/local_fs-copy.dat')
|
| 1121 |
+
>>> local.get_file_info('/tmp/new_folder/local_fs-copy.dat')
|
| 1122 |
+
<FileInfo for '/tmp/new_folder/local_fs-copy.dat': type=FileType.File, size=16>
|
| 1123 |
+
>>> local.get_file_info('/tmp/local_fs-copy.dat')
|
| 1124 |
+
<FileInfo for '/tmp/local_fs-copy.dat': type=FileType.NotFound>
|
| 1125 |
+
|
| 1126 |
+
To finish delete the file left:
|
| 1127 |
+
>>> local.delete_file('/tmp/local_fs.dat')
|
| 1128 |
+
"""
|
| 1129 |
+
|
| 1130 |
+
def __init__(self, *, use_mmap=False):
|
| 1131 |
+
cdef:
|
| 1132 |
+
shared_ptr[CFileSystem] fs
|
| 1133 |
+
c_string c_uri
|
| 1134 |
+
|
| 1135 |
+
# from_uri needs a non-empty path, so just use a placeholder of /_
|
| 1136 |
+
c_uri = tobytes(f"file:///_?use_mmap={int(use_mmap)}")
|
| 1137 |
+
with nogil:
|
| 1138 |
+
fs = GetResultValue(CFileSystemFromUri(c_uri))
|
| 1139 |
+
self.init(<shared_ptr[CFileSystem]> fs)
|
| 1140 |
+
|
| 1141 |
+
def __reduce__(self):
|
| 1142 |
+
uri = frombytes(GetResultValue(self.fs.MakeUri(b"/_")))
|
| 1143 |
+
return FileSystem._from_uri, (uri,)
|
| 1144 |
+
|
| 1145 |
+
|
| 1146 |
+
cdef class SubTreeFileSystem(FileSystem):
|
| 1147 |
+
"""
|
| 1148 |
+
Delegates to another implementation after prepending a fixed base path.
|
| 1149 |
+
|
| 1150 |
+
This is useful to expose a logical view of a subtree of a filesystem,
|
| 1151 |
+
for example a directory in a LocalFileSystem.
|
| 1152 |
+
|
| 1153 |
+
Note, that this makes no security guarantee. For example, symlinks may
|
| 1154 |
+
allow to "escape" the subtree and access other parts of the underlying
|
| 1155 |
+
filesystem.
|
| 1156 |
+
|
| 1157 |
+
Parameters
|
| 1158 |
+
----------
|
| 1159 |
+
base_path : str
|
| 1160 |
+
The root of the subtree.
|
| 1161 |
+
base_fs : FileSystem
|
| 1162 |
+
FileSystem object the operations delegated to.
|
| 1163 |
+
|
| 1164 |
+
Examples
|
| 1165 |
+
--------
|
| 1166 |
+
Create a LocalFileSystem instance:
|
| 1167 |
+
|
| 1168 |
+
>>> from pyarrow import fs
|
| 1169 |
+
>>> local = fs.LocalFileSystem()
|
| 1170 |
+
>>> with local.open_output_stream('/tmp/local_fs.dat') as stream:
|
| 1171 |
+
... stream.write(b'data')
|
| 1172 |
+
4
|
| 1173 |
+
|
| 1174 |
+
Create a directory and a SubTreeFileSystem instance:
|
| 1175 |
+
|
| 1176 |
+
>>> local.create_dir('/tmp/sub_tree')
|
| 1177 |
+
>>> subtree = fs.SubTreeFileSystem('/tmp/sub_tree', local)
|
| 1178 |
+
|
| 1179 |
+
Write data into the existing file:
|
| 1180 |
+
|
| 1181 |
+
>>> with subtree.open_append_stream('sub_tree_fs.dat') as f:
|
| 1182 |
+
... f.write(b'+newly added')
|
| 1183 |
+
12
|
| 1184 |
+
|
| 1185 |
+
Print out the attributes:
|
| 1186 |
+
|
| 1187 |
+
>>> subtree.base_fs
|
| 1188 |
+
<pyarrow._fs.LocalFileSystem object at ...>
|
| 1189 |
+
>>> subtree.base_path
|
| 1190 |
+
'/tmp/sub_tree/'
|
| 1191 |
+
|
| 1192 |
+
Get info for the given directory or given file:
|
| 1193 |
+
|
| 1194 |
+
>>> subtree.get_file_info('')
|
| 1195 |
+
<FileInfo for '': type=FileType.Directory>
|
| 1196 |
+
>>> subtree.get_file_info('sub_tree_fs.dat')
|
| 1197 |
+
<FileInfo for 'sub_tree_fs.dat': type=FileType.File, size=12>
|
| 1198 |
+
|
| 1199 |
+
Delete the file and directory:
|
| 1200 |
+
|
| 1201 |
+
>>> subtree.delete_file('sub_tree_fs.dat')
|
| 1202 |
+
>>> local.delete_dir('/tmp/sub_tree')
|
| 1203 |
+
>>> local.delete_file('/tmp/local_fs.dat')
|
| 1204 |
+
|
| 1205 |
+
For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
|
| 1206 |
+
"""
|
| 1207 |
+
|
| 1208 |
+
def __init__(self, base_path, FileSystem base_fs):
|
| 1209 |
+
cdef:
|
| 1210 |
+
c_string pathstr
|
| 1211 |
+
shared_ptr[CSubTreeFileSystem] wrapped
|
| 1212 |
+
|
| 1213 |
+
pathstr = _path_as_bytes(base_path)
|
| 1214 |
+
wrapped = make_shared[CSubTreeFileSystem](pathstr, base_fs.wrapped)
|
| 1215 |
+
|
| 1216 |
+
self.init(<shared_ptr[CFileSystem]> wrapped)
|
| 1217 |
+
|
| 1218 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped):
|
| 1219 |
+
FileSystem.init(self, wrapped)
|
| 1220 |
+
self.subtreefs = <CSubTreeFileSystem*> wrapped.get()
|
| 1221 |
+
|
| 1222 |
+
def __repr__(self):
|
| 1223 |
+
return f"SubTreeFileSystem(base_path={self.base_path}, base_fs={self.base_fs})"
|
| 1224 |
+
|
| 1225 |
+
def __reduce__(self):
|
| 1226 |
+
return SubTreeFileSystem, (
|
| 1227 |
+
frombytes(self.subtreefs.base_path()),
|
| 1228 |
+
FileSystem.wrap(self.subtreefs.base_fs())
|
| 1229 |
+
)
|
| 1230 |
+
|
| 1231 |
+
@property
|
| 1232 |
+
def base_path(self):
|
| 1233 |
+
return frombytes(self.subtreefs.base_path())
|
| 1234 |
+
|
| 1235 |
+
@property
|
| 1236 |
+
def base_fs(self):
|
| 1237 |
+
return FileSystem.wrap(self.subtreefs.base_fs())
|
| 1238 |
+
|
| 1239 |
+
|
| 1240 |
+
cdef class _MockFileSystem(FileSystem):
|
| 1241 |
+
|
| 1242 |
+
def __init__(self, datetime current_time=None):
|
| 1243 |
+
cdef shared_ptr[CMockFileSystem] wrapped
|
| 1244 |
+
|
| 1245 |
+
current_time = current_time or datetime.now()
|
| 1246 |
+
wrapped = make_shared[CMockFileSystem](
|
| 1247 |
+
PyDateTime_to_TimePoint(<PyDateTime_DateTime*> current_time)
|
| 1248 |
+
)
|
| 1249 |
+
|
| 1250 |
+
self.init(<shared_ptr[CFileSystem]> wrapped)
|
| 1251 |
+
|
| 1252 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped):
|
| 1253 |
+
FileSystem.init(self, wrapped)
|
| 1254 |
+
self.mockfs = <CMockFileSystem*> wrapped.get()
|
| 1255 |
+
|
| 1256 |
+
|
| 1257 |
+
cdef class PyFileSystem(FileSystem):
|
| 1258 |
+
"""
|
| 1259 |
+
A FileSystem with behavior implemented in Python.
|
| 1260 |
+
|
| 1261 |
+
Parameters
|
| 1262 |
+
----------
|
| 1263 |
+
handler : FileSystemHandler
|
| 1264 |
+
The handler object implementing custom filesystem behavior.
|
| 1265 |
+
|
| 1266 |
+
Examples
|
| 1267 |
+
--------
|
| 1268 |
+
Create an fsspec-based filesystem object for GitHub:
|
| 1269 |
+
|
| 1270 |
+
>>> from fsspec.implementations import github
|
| 1271 |
+
>>> gfs = github.GithubFileSystem('apache', 'arrow') # doctest: +SKIP
|
| 1272 |
+
|
| 1273 |
+
Get a PyArrow FileSystem object:
|
| 1274 |
+
|
| 1275 |
+
>>> from pyarrow.fs import PyFileSystem, FSSpecHandler
|
| 1276 |
+
>>> pa_fs = PyFileSystem(FSSpecHandler(gfs)) # doctest: +SKIP
|
| 1277 |
+
|
| 1278 |
+
Use :func:`~pyarrow.fs.FileSystem` functionality ``get_file_info()``:
|
| 1279 |
+
|
| 1280 |
+
>>> pa_fs.get_file_info('README.md') # doctest: +SKIP
|
| 1281 |
+
<FileInfo for 'README.md': type=FileType.File, size=...>
|
| 1282 |
+
"""
|
| 1283 |
+
|
| 1284 |
+
def __init__(self, handler):
|
| 1285 |
+
cdef:
|
| 1286 |
+
CPyFileSystemVtable vtable
|
| 1287 |
+
shared_ptr[CPyFileSystem] wrapped
|
| 1288 |
+
|
| 1289 |
+
if not isinstance(handler, FileSystemHandler):
|
| 1290 |
+
raise TypeError(
|
| 1291 |
+
f"Expected a FileSystemHandler instance, got {type(handler)}")
|
| 1292 |
+
|
| 1293 |
+
vtable.get_type_name = _cb_get_type_name
|
| 1294 |
+
vtable.equals = _cb_equals
|
| 1295 |
+
vtable.get_file_info = _cb_get_file_info
|
| 1296 |
+
vtable.get_file_info_vector = _cb_get_file_info_vector
|
| 1297 |
+
vtable.get_file_info_selector = _cb_get_file_info_selector
|
| 1298 |
+
vtable.create_dir = _cb_create_dir
|
| 1299 |
+
vtable.delete_dir = _cb_delete_dir
|
| 1300 |
+
vtable.delete_dir_contents = _cb_delete_dir_contents
|
| 1301 |
+
vtable.delete_root_dir_contents = _cb_delete_root_dir_contents
|
| 1302 |
+
vtable.delete_file = _cb_delete_file
|
| 1303 |
+
vtable.move = _cb_move
|
| 1304 |
+
vtable.copy_file = _cb_copy_file
|
| 1305 |
+
vtable.open_input_stream = _cb_open_input_stream
|
| 1306 |
+
vtable.open_input_file = _cb_open_input_file
|
| 1307 |
+
vtable.open_output_stream = _cb_open_output_stream
|
| 1308 |
+
vtable.open_append_stream = _cb_open_append_stream
|
| 1309 |
+
vtable.normalize_path = _cb_normalize_path
|
| 1310 |
+
|
| 1311 |
+
wrapped = CPyFileSystem.Make(handler, move(vtable))
|
| 1312 |
+
self.init(<shared_ptr[CFileSystem]> wrapped)
|
| 1313 |
+
|
| 1314 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped):
|
| 1315 |
+
FileSystem.init(self, wrapped)
|
| 1316 |
+
self.pyfs = <CPyFileSystem*> wrapped.get()
|
| 1317 |
+
|
| 1318 |
+
@property
|
| 1319 |
+
def handler(self):
|
| 1320 |
+
"""
|
| 1321 |
+
The filesystem's underlying handler.
|
| 1322 |
+
|
| 1323 |
+
Returns
|
| 1324 |
+
-------
|
| 1325 |
+
handler : FileSystemHandler
|
| 1326 |
+
"""
|
| 1327 |
+
return <object> self.pyfs.handler()
|
| 1328 |
+
|
| 1329 |
+
def __reduce__(self):
|
| 1330 |
+
return PyFileSystem, (self.handler,)
|
| 1331 |
+
|
| 1332 |
+
|
| 1333 |
+
class FileSystemHandler(ABC):
|
| 1334 |
+
"""
|
| 1335 |
+
An abstract class exposing methods to implement PyFileSystem's behavior.
|
| 1336 |
+
"""
|
| 1337 |
+
|
| 1338 |
+
@abstractmethod
|
| 1339 |
+
def get_type_name(self):
|
| 1340 |
+
"""
|
| 1341 |
+
Implement PyFileSystem.type_name.
|
| 1342 |
+
"""
|
| 1343 |
+
|
| 1344 |
+
@abstractmethod
|
| 1345 |
+
def get_file_info(self, paths):
|
| 1346 |
+
"""
|
| 1347 |
+
Implement PyFileSystem.get_file_info(paths).
|
| 1348 |
+
|
| 1349 |
+
Parameters
|
| 1350 |
+
----------
|
| 1351 |
+
paths : list of str
|
| 1352 |
+
paths for which we want to retrieve the info.
|
| 1353 |
+
"""
|
| 1354 |
+
|
| 1355 |
+
@abstractmethod
|
| 1356 |
+
def get_file_info_selector(self, selector):
|
| 1357 |
+
"""
|
| 1358 |
+
Implement PyFileSystem.get_file_info(selector).
|
| 1359 |
+
|
| 1360 |
+
Parameters
|
| 1361 |
+
----------
|
| 1362 |
+
selector : FileSelector
|
| 1363 |
+
selector for which we want to retrieve the info.
|
| 1364 |
+
"""
|
| 1365 |
+
|
| 1366 |
+
@abstractmethod
|
| 1367 |
+
def create_dir(self, path, recursive):
|
| 1368 |
+
"""
|
| 1369 |
+
Implement PyFileSystem.create_dir(...).
|
| 1370 |
+
|
| 1371 |
+
Parameters
|
| 1372 |
+
----------
|
| 1373 |
+
path : str
|
| 1374 |
+
path of the directory.
|
| 1375 |
+
recursive : bool
|
| 1376 |
+
if the parent directories should be created too.
|
| 1377 |
+
"""
|
| 1378 |
+
|
| 1379 |
+
@abstractmethod
|
| 1380 |
+
def delete_dir(self, path):
|
| 1381 |
+
"""
|
| 1382 |
+
Implement PyFileSystem.delete_dir(...).
|
| 1383 |
+
|
| 1384 |
+
Parameters
|
| 1385 |
+
----------
|
| 1386 |
+
path : str
|
| 1387 |
+
path of the directory.
|
| 1388 |
+
"""
|
| 1389 |
+
|
| 1390 |
+
@abstractmethod
|
| 1391 |
+
def delete_dir_contents(self, path, missing_dir_ok=False):
|
| 1392 |
+
"""
|
| 1393 |
+
Implement PyFileSystem.delete_dir_contents(...).
|
| 1394 |
+
|
| 1395 |
+
Parameters
|
| 1396 |
+
----------
|
| 1397 |
+
path : str
|
| 1398 |
+
path of the directory.
|
| 1399 |
+
missing_dir_ok : bool
|
| 1400 |
+
if False an error should be raised if path does not exist
|
| 1401 |
+
"""
|
| 1402 |
+
|
| 1403 |
+
@abstractmethod
|
| 1404 |
+
def delete_root_dir_contents(self):
|
| 1405 |
+
"""
|
| 1406 |
+
Implement PyFileSystem.delete_dir_contents("/", accept_root_dir=True).
|
| 1407 |
+
"""
|
| 1408 |
+
|
| 1409 |
+
@abstractmethod
|
| 1410 |
+
def delete_file(self, path):
|
| 1411 |
+
"""
|
| 1412 |
+
Implement PyFileSystem.delete_file(...).
|
| 1413 |
+
|
| 1414 |
+
Parameters
|
| 1415 |
+
----------
|
| 1416 |
+
path : str
|
| 1417 |
+
path of the file.
|
| 1418 |
+
"""
|
| 1419 |
+
|
| 1420 |
+
@abstractmethod
|
| 1421 |
+
def move(self, src, dest):
|
| 1422 |
+
"""
|
| 1423 |
+
Implement PyFileSystem.move(...).
|
| 1424 |
+
|
| 1425 |
+
Parameters
|
| 1426 |
+
----------
|
| 1427 |
+
src : str
|
| 1428 |
+
path of what should be moved.
|
| 1429 |
+
dest : str
|
| 1430 |
+
path of where it should be moved to.
|
| 1431 |
+
"""
|
| 1432 |
+
|
| 1433 |
+
@abstractmethod
|
| 1434 |
+
def copy_file(self, src, dest):
|
| 1435 |
+
"""
|
| 1436 |
+
Implement PyFileSystem.copy_file(...).
|
| 1437 |
+
|
| 1438 |
+
Parameters
|
| 1439 |
+
----------
|
| 1440 |
+
src : str
|
| 1441 |
+
path of what should be copied.
|
| 1442 |
+
dest : str
|
| 1443 |
+
path of where it should be copied to.
|
| 1444 |
+
"""
|
| 1445 |
+
|
| 1446 |
+
@abstractmethod
|
| 1447 |
+
def open_input_stream(self, path):
|
| 1448 |
+
"""
|
| 1449 |
+
Implement PyFileSystem.open_input_stream(...).
|
| 1450 |
+
|
| 1451 |
+
Parameters
|
| 1452 |
+
----------
|
| 1453 |
+
path : str
|
| 1454 |
+
path of what should be opened.
|
| 1455 |
+
"""
|
| 1456 |
+
|
| 1457 |
+
@abstractmethod
|
| 1458 |
+
def open_input_file(self, path):
|
| 1459 |
+
"""
|
| 1460 |
+
Implement PyFileSystem.open_input_file(...).
|
| 1461 |
+
|
| 1462 |
+
Parameters
|
| 1463 |
+
----------
|
| 1464 |
+
path : str
|
| 1465 |
+
path of what should be opened.
|
| 1466 |
+
"""
|
| 1467 |
+
|
| 1468 |
+
@abstractmethod
|
| 1469 |
+
def open_output_stream(self, path, metadata):
|
| 1470 |
+
"""
|
| 1471 |
+
Implement PyFileSystem.open_output_stream(...).
|
| 1472 |
+
|
| 1473 |
+
Parameters
|
| 1474 |
+
----------
|
| 1475 |
+
path : str
|
| 1476 |
+
path of what should be opened.
|
| 1477 |
+
metadata : mapping
|
| 1478 |
+
Mapping of string keys to string values.
|
| 1479 |
+
Some filesystems support storing metadata along the file
|
| 1480 |
+
(such as "Content-Type").
|
| 1481 |
+
"""
|
| 1482 |
+
|
| 1483 |
+
@abstractmethod
|
| 1484 |
+
def open_append_stream(self, path, metadata):
|
| 1485 |
+
"""
|
| 1486 |
+
Implement PyFileSystem.open_append_stream(...).
|
| 1487 |
+
|
| 1488 |
+
Parameters
|
| 1489 |
+
----------
|
| 1490 |
+
path : str
|
| 1491 |
+
path of what should be opened.
|
| 1492 |
+
metadata : mapping
|
| 1493 |
+
Mapping of string keys to string values.
|
| 1494 |
+
Some filesystems support storing metadata along the file
|
| 1495 |
+
(such as "Content-Type").
|
| 1496 |
+
"""
|
| 1497 |
+
|
| 1498 |
+
@abstractmethod
|
| 1499 |
+
def normalize_path(self, path):
|
| 1500 |
+
"""
|
| 1501 |
+
Implement PyFileSystem.normalize_path(...).
|
| 1502 |
+
|
| 1503 |
+
Parameters
|
| 1504 |
+
----------
|
| 1505 |
+
path : str
|
| 1506 |
+
path of what should be normalized.
|
| 1507 |
+
"""
|
| 1508 |
+
|
| 1509 |
+
# Callback definitions for CPyFileSystemVtable
|
| 1510 |
+
|
| 1511 |
+
|
| 1512 |
+
cdef void _cb_get_type_name(handler, c_string* out) except *:
|
| 1513 |
+
out[0] = tobytes("py::" + handler.get_type_name())
|
| 1514 |
+
|
| 1515 |
+
cdef c_bool _cb_equals(handler, const CFileSystem& c_other) except False:
|
| 1516 |
+
if c_other.type_name().startswith(b"py::"):
|
| 1517 |
+
return <object> (<const CPyFileSystem&> c_other).handler() == handler
|
| 1518 |
+
|
| 1519 |
+
return False
|
| 1520 |
+
|
| 1521 |
+
cdef void _cb_get_file_info(handler, const c_string& path,
|
| 1522 |
+
CFileInfo* out) except *:
|
| 1523 |
+
infos = handler.get_file_info([frombytes(path)])
|
| 1524 |
+
if not isinstance(infos, list) or len(infos) != 1:
|
| 1525 |
+
raise TypeError("get_file_info should have returned a 1-element list")
|
| 1526 |
+
out[0] = FileInfo.unwrap_safe(infos[0])
|
| 1527 |
+
|
| 1528 |
+
cdef void _cb_get_file_info_vector(handler, const vector[c_string]& paths,
|
| 1529 |
+
vector[CFileInfo]* out) except *:
|
| 1530 |
+
py_paths = [frombytes(paths[i]) for i in range(len(paths))]
|
| 1531 |
+
infos = handler.get_file_info(py_paths)
|
| 1532 |
+
if not isinstance(infos, list):
|
| 1533 |
+
raise TypeError("get_file_info should have returned a list")
|
| 1534 |
+
out[0].clear()
|
| 1535 |
+
out[0].reserve(len(infos))
|
| 1536 |
+
for info in infos:
|
| 1537 |
+
out[0].push_back(FileInfo.unwrap_safe(info))
|
| 1538 |
+
|
| 1539 |
+
cdef void _cb_get_file_info_selector(handler, const CFileSelector& selector,
|
| 1540 |
+
vector[CFileInfo]* out) except *:
|
| 1541 |
+
infos = handler.get_file_info_selector(FileSelector.wrap(selector))
|
| 1542 |
+
if not isinstance(infos, list):
|
| 1543 |
+
raise TypeError("get_file_info_selector should have returned a list")
|
| 1544 |
+
out[0].clear()
|
| 1545 |
+
out[0].reserve(len(infos))
|
| 1546 |
+
for info in infos:
|
| 1547 |
+
out[0].push_back(FileInfo.unwrap_safe(info))
|
| 1548 |
+
|
| 1549 |
+
cdef void _cb_create_dir(handler, const c_string& path,
|
| 1550 |
+
c_bool recursive) except *:
|
| 1551 |
+
handler.create_dir(frombytes(path), recursive)
|
| 1552 |
+
|
| 1553 |
+
cdef void _cb_delete_dir(handler, const c_string& path) except *:
|
| 1554 |
+
handler.delete_dir(frombytes(path))
|
| 1555 |
+
|
| 1556 |
+
cdef void _cb_delete_dir_contents(handler, const c_string& path,
|
| 1557 |
+
c_bool missing_dir_ok) except *:
|
| 1558 |
+
handler.delete_dir_contents(frombytes(path), missing_dir_ok)
|
| 1559 |
+
|
| 1560 |
+
cdef void _cb_delete_root_dir_contents(handler) except *:
|
| 1561 |
+
handler.delete_root_dir_contents()
|
| 1562 |
+
|
| 1563 |
+
cdef void _cb_delete_file(handler, const c_string& path) except *:
|
| 1564 |
+
handler.delete_file(frombytes(path))
|
| 1565 |
+
|
| 1566 |
+
cdef void _cb_move(handler, const c_string& src,
|
| 1567 |
+
const c_string& dest) except *:
|
| 1568 |
+
handler.move(frombytes(src), frombytes(dest))
|
| 1569 |
+
|
| 1570 |
+
cdef void _cb_copy_file(handler, const c_string& src,
|
| 1571 |
+
const c_string& dest) except *:
|
| 1572 |
+
handler.copy_file(frombytes(src), frombytes(dest))
|
| 1573 |
+
|
| 1574 |
+
cdef void _cb_open_input_stream(handler, const c_string& path,
|
| 1575 |
+
shared_ptr[CInputStream]* out) except *:
|
| 1576 |
+
stream = handler.open_input_stream(frombytes(path))
|
| 1577 |
+
if not isinstance(stream, NativeFile):
|
| 1578 |
+
raise TypeError("open_input_stream should have returned "
|
| 1579 |
+
"a PyArrow file")
|
| 1580 |
+
out[0] = (<NativeFile> stream).get_input_stream()
|
| 1581 |
+
|
| 1582 |
+
cdef void _cb_open_input_file(handler, const c_string& path,
|
| 1583 |
+
shared_ptr[CRandomAccessFile]* out) except *:
|
| 1584 |
+
stream = handler.open_input_file(frombytes(path))
|
| 1585 |
+
if not isinstance(stream, NativeFile):
|
| 1586 |
+
raise TypeError("open_input_file should have returned "
|
| 1587 |
+
"a PyArrow file")
|
| 1588 |
+
out[0] = (<NativeFile> stream).get_random_access_file()
|
| 1589 |
+
|
| 1590 |
+
cdef void _cb_open_output_stream(
|
| 1591 |
+
handler, const c_string& path,
|
| 1592 |
+
const shared_ptr[const CKeyValueMetadata]& metadata,
|
| 1593 |
+
shared_ptr[COutputStream]* out) except *:
|
| 1594 |
+
stream = handler.open_output_stream(
|
| 1595 |
+
frombytes(path), pyarrow_wrap_metadata(metadata))
|
| 1596 |
+
if not isinstance(stream, NativeFile):
|
| 1597 |
+
raise TypeError("open_output_stream should have returned "
|
| 1598 |
+
"a PyArrow file")
|
| 1599 |
+
out[0] = (<NativeFile> stream).get_output_stream()
|
| 1600 |
+
|
| 1601 |
+
cdef void _cb_open_append_stream(
|
| 1602 |
+
handler, const c_string& path,
|
| 1603 |
+
const shared_ptr[const CKeyValueMetadata]& metadata,
|
| 1604 |
+
shared_ptr[COutputStream]* out) except *:
|
| 1605 |
+
stream = handler.open_append_stream(
|
| 1606 |
+
frombytes(path), pyarrow_wrap_metadata(metadata))
|
| 1607 |
+
if not isinstance(stream, NativeFile):
|
| 1608 |
+
raise TypeError("open_append_stream should have returned "
|
| 1609 |
+
"a PyArrow file")
|
| 1610 |
+
out[0] = (<NativeFile> stream).get_output_stream()
|
| 1611 |
+
|
| 1612 |
+
cdef void _cb_normalize_path(handler, const c_string& path,
|
| 1613 |
+
c_string* out) except *:
|
| 1614 |
+
out[0] = tobytes(handler.normalize_path(frombytes(path)))
|
| 1615 |
+
|
| 1616 |
+
|
| 1617 |
+
def _copy_files(FileSystem source_fs, str source_path,
|
| 1618 |
+
FileSystem destination_fs, str destination_path,
|
| 1619 |
+
int64_t chunk_size, c_bool use_threads):
|
| 1620 |
+
# low-level helper exposed through pyarrow/fs.py::copy_files
|
| 1621 |
+
cdef:
|
| 1622 |
+
CFileLocator c_source
|
| 1623 |
+
vector[CFileLocator] c_sources
|
| 1624 |
+
CFileLocator c_destination
|
| 1625 |
+
vector[CFileLocator] c_destinations
|
| 1626 |
+
|
| 1627 |
+
c_source.filesystem = source_fs.unwrap()
|
| 1628 |
+
c_source.path = tobytes(source_path)
|
| 1629 |
+
c_sources.push_back(c_source)
|
| 1630 |
+
|
| 1631 |
+
c_destination.filesystem = destination_fs.unwrap()
|
| 1632 |
+
c_destination.path = tobytes(destination_path)
|
| 1633 |
+
c_destinations.push_back(c_destination)
|
| 1634 |
+
|
| 1635 |
+
with nogil:
|
| 1636 |
+
check_status(CCopyFiles(
|
| 1637 |
+
c_sources, c_destinations,
|
| 1638 |
+
c_default_io_context(), chunk_size, use_threads,
|
| 1639 |
+
))
|
| 1640 |
+
|
| 1641 |
+
|
| 1642 |
+
def _copy_files_selector(FileSystem source_fs, FileSelector source_sel,
|
| 1643 |
+
FileSystem destination_fs, str destination_base_dir,
|
| 1644 |
+
int64_t chunk_size, c_bool use_threads):
|
| 1645 |
+
# low-level helper exposed through pyarrow/fs.py::copy_files
|
| 1646 |
+
cdef c_string c_destination_base_dir = tobytes(destination_base_dir)
|
| 1647 |
+
|
| 1648 |
+
with nogil:
|
| 1649 |
+
check_status(CCopyFilesWithSelector(
|
| 1650 |
+
source_fs.unwrap(), source_sel.unwrap(),
|
| 1651 |
+
destination_fs.unwrap(), c_destination_base_dir,
|
| 1652 |
+
c_default_io_context(), chunk_size, use_threads,
|
| 1653 |
+
))
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.cp312-win_amd64.pyd
ADDED
|
Binary file (94.2 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_gcsfs.pyx
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.lib cimport (pyarrow_wrap_metadata,
|
| 21 |
+
pyarrow_unwrap_metadata)
|
| 22 |
+
from pyarrow.lib import frombytes, tobytes, ensure_metadata
|
| 23 |
+
from pyarrow.includes.common cimport *
|
| 24 |
+
from pyarrow.includes.libarrow cimport *
|
| 25 |
+
from pyarrow.includes.libarrow_fs cimport *
|
| 26 |
+
from pyarrow._fs cimport FileSystem, TimePoint_to_ns, PyDateTime_to_TimePoint
|
| 27 |
+
|
| 28 |
+
from datetime import datetime, timedelta, timezone
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
cdef class GcsFileSystem(FileSystem):
|
| 32 |
+
"""
|
| 33 |
+
Google Cloud Storage (GCS) backed FileSystem implementation
|
| 34 |
+
|
| 35 |
+
By default uses the process described in https://google.aip.dev/auth/4110
|
| 36 |
+
to resolve credentials. If not running on Google Cloud Platform (GCP),
|
| 37 |
+
this generally requires the environment variable
|
| 38 |
+
GOOGLE_APPLICATION_CREDENTIALS to point to a JSON file
|
| 39 |
+
containing credentials.
|
| 40 |
+
|
| 41 |
+
Note: GCS buckets are special and the operations available on them may be
|
| 42 |
+
limited or more expensive than expected compared to local file systems.
|
| 43 |
+
|
| 44 |
+
Note: When pickling a GcsFileSystem that uses default credentials, resolution
|
| 45 |
+
credentials are not stored in the serialized data. Therefore, when unpickling
|
| 46 |
+
it is assumed that the necessary credentials are in place for the target
|
| 47 |
+
process.
|
| 48 |
+
|
| 49 |
+
Parameters
|
| 50 |
+
----------
|
| 51 |
+
anonymous : boolean, default False
|
| 52 |
+
Whether to connect anonymously.
|
| 53 |
+
If true, will not attempt to look up credentials using standard GCP
|
| 54 |
+
configuration methods.
|
| 55 |
+
access_token : str, default None
|
| 56 |
+
GCP access token. If provided, temporary credentials will be fetched by
|
| 57 |
+
assuming this role; also, a `credential_token_expiration` must be
|
| 58 |
+
specified as well.
|
| 59 |
+
target_service_account : str, default None
|
| 60 |
+
An optional service account to try to impersonate when accessing GCS. This
|
| 61 |
+
requires the specified credential user or service account to have the necessary
|
| 62 |
+
permissions.
|
| 63 |
+
credential_token_expiration : datetime, default None
|
| 64 |
+
Expiration for credential generated with an access token. Must be specified
|
| 65 |
+
if `access_token` is specified.
|
| 66 |
+
default_bucket_location : str, default 'US'
|
| 67 |
+
GCP region to create buckets in.
|
| 68 |
+
scheme : str, default 'https'
|
| 69 |
+
GCS connection transport scheme.
|
| 70 |
+
endpoint_override : str, default None
|
| 71 |
+
Override endpoint with a connect string such as "localhost:9000"
|
| 72 |
+
default_metadata : mapping or pyarrow.KeyValueMetadata, default None
|
| 73 |
+
Default metadata for `open_output_stream`. This will be ignored if
|
| 74 |
+
non-empty metadata is passed to `open_output_stream`.
|
| 75 |
+
retry_time_limit : timedelta, default None
|
| 76 |
+
Set the maximum amount of time the GCS client will attempt to retry
|
| 77 |
+
transient errors. Subsecond granularity is ignored.
|
| 78 |
+
project_id : str, default None
|
| 79 |
+
The GCP project identifier to use for creating buckets.
|
| 80 |
+
If not set, the library uses the GOOGLE_CLOUD_PROJECT environment
|
| 81 |
+
variable. Most I/O operations do not need a project id, only applications
|
| 82 |
+
that create new buckets need a project id.
|
| 83 |
+
"""
|
| 84 |
+
|
| 85 |
+
cdef:
|
| 86 |
+
CGcsFileSystem* gcsfs
|
| 87 |
+
|
| 88 |
+
def __init__(self, *, bint anonymous=False, access_token=None,
|
| 89 |
+
target_service_account=None, credential_token_expiration=None,
|
| 90 |
+
default_bucket_location='US',
|
| 91 |
+
scheme=None,
|
| 92 |
+
endpoint_override=None,
|
| 93 |
+
default_metadata=None,
|
| 94 |
+
retry_time_limit=None,
|
| 95 |
+
project_id=None):
|
| 96 |
+
cdef:
|
| 97 |
+
CGcsOptions options
|
| 98 |
+
shared_ptr[CGcsFileSystem] wrapped
|
| 99 |
+
double time_limit_seconds
|
| 100 |
+
|
| 101 |
+
# Intentional use of truthiness because empty strings aren't valid and
|
| 102 |
+
# for reconstruction from pickling will give empty strings.
|
| 103 |
+
if anonymous and (target_service_account or access_token):
|
| 104 |
+
raise ValueError(
|
| 105 |
+
'anonymous option is not compatible with target_service_account and '
|
| 106 |
+
'access_token'
|
| 107 |
+
)
|
| 108 |
+
elif bool(access_token) != bool(credential_token_expiration):
|
| 109 |
+
raise ValueError(
|
| 110 |
+
'access_token and credential_token_expiration must be '
|
| 111 |
+
'specified together'
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
elif anonymous:
|
| 115 |
+
options = CGcsOptions.Anonymous()
|
| 116 |
+
elif access_token:
|
| 117 |
+
if not isinstance(credential_token_expiration, datetime):
|
| 118 |
+
raise ValueError(
|
| 119 |
+
"credential_token_expiration must be a datetime")
|
| 120 |
+
options = CGcsOptions.FromAccessToken(
|
| 121 |
+
tobytes(access_token),
|
| 122 |
+
PyDateTime_to_TimePoint(<PyDateTime_DateTime*>credential_token_expiration))
|
| 123 |
+
else:
|
| 124 |
+
options = CGcsOptions.Defaults()
|
| 125 |
+
|
| 126 |
+
# Target service account requires base credentials so
|
| 127 |
+
# it is not part of the if/else chain above which only
|
| 128 |
+
# handles base credentials.
|
| 129 |
+
if target_service_account:
|
| 130 |
+
options = CGcsOptions.FromImpersonatedServiceAccount(
|
| 131 |
+
options.credentials, tobytes(target_service_account))
|
| 132 |
+
|
| 133 |
+
options.default_bucket_location = tobytes(default_bucket_location)
|
| 134 |
+
|
| 135 |
+
if scheme is not None:
|
| 136 |
+
options.scheme = tobytes(scheme)
|
| 137 |
+
if endpoint_override is not None:
|
| 138 |
+
options.endpoint_override = tobytes(endpoint_override)
|
| 139 |
+
if default_metadata is not None:
|
| 140 |
+
options.default_metadata = pyarrow_unwrap_metadata(
|
| 141 |
+
ensure_metadata(default_metadata))
|
| 142 |
+
if retry_time_limit is not None:
|
| 143 |
+
time_limit_seconds = retry_time_limit.total_seconds()
|
| 144 |
+
options.retry_limit_seconds = time_limit_seconds
|
| 145 |
+
if project_id is not None:
|
| 146 |
+
options.project_id = <c_string>tobytes(project_id)
|
| 147 |
+
|
| 148 |
+
with nogil:
|
| 149 |
+
wrapped = GetResultValue(CGcsFileSystem.Make(options))
|
| 150 |
+
|
| 151 |
+
self.init(<shared_ptr[CFileSystem]> wrapped)
|
| 152 |
+
|
| 153 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped):
|
| 154 |
+
FileSystem.init(self, wrapped)
|
| 155 |
+
self.gcsfs = <CGcsFileSystem*> wrapped.get()
|
| 156 |
+
|
| 157 |
+
def _expiration_datetime_from_options(self):
|
| 158 |
+
expiration_ns = TimePoint_to_ns(
|
| 159 |
+
self.gcsfs.options().credentials.expiration())
|
| 160 |
+
if expiration_ns == 0:
|
| 161 |
+
return None
|
| 162 |
+
return datetime.fromtimestamp(expiration_ns / 1.0e9, timezone.utc)
|
| 163 |
+
|
| 164 |
+
@staticmethod
|
| 165 |
+
def _reconstruct(kwargs):
|
| 166 |
+
# __reduce__ doesn't allow passing named arguments directly to the
|
| 167 |
+
# reconstructor, hence this wrapper.
|
| 168 |
+
return GcsFileSystem(**kwargs)
|
| 169 |
+
|
| 170 |
+
def __reduce__(self):
|
| 171 |
+
cdef CGcsOptions opts = self.gcsfs.options()
|
| 172 |
+
service_account = frombytes(opts.credentials.target_service_account())
|
| 173 |
+
expiration_dt = self._expiration_datetime_from_options()
|
| 174 |
+
retry_time_limit = None
|
| 175 |
+
if opts.retry_limit_seconds.has_value():
|
| 176 |
+
retry_time_limit = timedelta(
|
| 177 |
+
seconds=opts.retry_limit_seconds.value())
|
| 178 |
+
project_id = None
|
| 179 |
+
if opts.project_id.has_value():
|
| 180 |
+
project_id = frombytes(opts.project_id.value())
|
| 181 |
+
return (
|
| 182 |
+
GcsFileSystem._reconstruct, (dict(
|
| 183 |
+
access_token=frombytes(opts.credentials.access_token()),
|
| 184 |
+
anonymous=opts.credentials.anonymous(),
|
| 185 |
+
credential_token_expiration=expiration_dt,
|
| 186 |
+
target_service_account=service_account,
|
| 187 |
+
scheme=frombytes(opts.scheme),
|
| 188 |
+
endpoint_override=frombytes(opts.endpoint_override),
|
| 189 |
+
default_bucket_location=frombytes(
|
| 190 |
+
opts.default_bucket_location),
|
| 191 |
+
default_metadata=pyarrow_wrap_metadata(opts.default_metadata),
|
| 192 |
+
retry_time_limit=retry_time_limit,
|
| 193 |
+
project_id=project_id
|
| 194 |
+
),))
|
| 195 |
+
|
| 196 |
+
@property
|
| 197 |
+
def default_bucket_location(self):
|
| 198 |
+
"""
|
| 199 |
+
The GCP location this filesystem will write to.
|
| 200 |
+
"""
|
| 201 |
+
return frombytes(self.gcsfs.options().default_bucket_location)
|
| 202 |
+
|
| 203 |
+
@property
|
| 204 |
+
def project_id(self):
|
| 205 |
+
"""
|
| 206 |
+
The GCP project id this filesystem will use.
|
| 207 |
+
"""
|
| 208 |
+
if self.gcsfs.options().project_id.has_value():
|
| 209 |
+
return frombytes(self.gcsfs.options().project_id.value())
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_generated_version.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# file generated by setuptools-scm
|
| 2 |
+
# don't change, don't track in version control
|
| 3 |
+
|
| 4 |
+
__all__ = [
|
| 5 |
+
"__version__",
|
| 6 |
+
"__version_tuple__",
|
| 7 |
+
"version",
|
| 8 |
+
"version_tuple",
|
| 9 |
+
"__commit_id__",
|
| 10 |
+
"commit_id",
|
| 11 |
+
]
|
| 12 |
+
|
| 13 |
+
TYPE_CHECKING = False
|
| 14 |
+
if TYPE_CHECKING:
|
| 15 |
+
from typing import Tuple
|
| 16 |
+
from typing import Union
|
| 17 |
+
|
| 18 |
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
| 19 |
+
COMMIT_ID = Union[str, None]
|
| 20 |
+
else:
|
| 21 |
+
VERSION_TUPLE = object
|
| 22 |
+
COMMIT_ID = object
|
| 23 |
+
|
| 24 |
+
version: str
|
| 25 |
+
__version__: str
|
| 26 |
+
__version_tuple__: VERSION_TUPLE
|
| 27 |
+
version_tuple: VERSION_TUPLE
|
| 28 |
+
commit_id: COMMIT_ID
|
| 29 |
+
__commit_id__: COMMIT_ID
|
| 30 |
+
|
| 31 |
+
__version__ = version = '24.0.0'
|
| 32 |
+
__version_tuple__ = version_tuple = (24, 0, 0)
|
| 33 |
+
|
| 34 |
+
__commit_id__ = commit_id = None
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.cp312-win_amd64.pyd
ADDED
|
Binary file (92.7 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_hdfs.pyx
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.includes.common cimport *
|
| 21 |
+
from pyarrow.includes.libarrow cimport *
|
| 22 |
+
from pyarrow.includes.libarrow_fs cimport *
|
| 23 |
+
from pyarrow._fs cimport FileSystem
|
| 24 |
+
|
| 25 |
+
from pyarrow.lib import frombytes, tobytes
|
| 26 |
+
from pyarrow.util import _stringify_path
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
cdef class HadoopFileSystem(FileSystem):
|
| 30 |
+
"""
|
| 31 |
+
HDFS backed FileSystem implementation
|
| 32 |
+
|
| 33 |
+
Parameters
|
| 34 |
+
----------
|
| 35 |
+
host : str
|
| 36 |
+
HDFS host to connect to. Set to "default" for fs.defaultFS from
|
| 37 |
+
core-site.xml.
|
| 38 |
+
port : int, default 8020
|
| 39 |
+
HDFS port to connect to. Set to 0 for default or logical (HA) nodes.
|
| 40 |
+
user : str, default None
|
| 41 |
+
Username when connecting to HDFS; None implies login user.
|
| 42 |
+
replication : int, default 3
|
| 43 |
+
Number of copies each block will have.
|
| 44 |
+
buffer_size : int, default 0
|
| 45 |
+
If 0, no buffering will happen otherwise the size of the temporary read
|
| 46 |
+
and write buffer.
|
| 47 |
+
default_block_size : int, default None
|
| 48 |
+
None means the default configuration for HDFS, a typical block size is
|
| 49 |
+
128 MB.
|
| 50 |
+
kerb_ticket : string or path, default None
|
| 51 |
+
If not None, the path to the Kerberos ticket cache.
|
| 52 |
+
extra_conf : dict, default None
|
| 53 |
+
Extra key/value pairs for configuration; will override any
|
| 54 |
+
hdfs-site.xml properties.
|
| 55 |
+
|
| 56 |
+
Examples
|
| 57 |
+
--------
|
| 58 |
+
>>> from pyarrow import fs
|
| 59 |
+
>>> hdfs = fs.HadoopFileSystem(host, port, user=user, kerb_ticket=ticket_cache_path) # doctest: +SKIP
|
| 60 |
+
|
| 61 |
+
For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
cdef:
|
| 65 |
+
CHadoopFileSystem* hdfs
|
| 66 |
+
|
| 67 |
+
def __init__(self, str host, int port=8020, *, str user=None,
|
| 68 |
+
int replication=3, int buffer_size=0,
|
| 69 |
+
default_block_size=None, kerb_ticket=None,
|
| 70 |
+
extra_conf=None):
|
| 71 |
+
cdef:
|
| 72 |
+
CHdfsOptions options
|
| 73 |
+
shared_ptr[CHadoopFileSystem] wrapped
|
| 74 |
+
|
| 75 |
+
if not host.startswith(('hdfs://', 'viewfs://')) and host != "default":
|
| 76 |
+
# TODO(kszucs): do more sanitization
|
| 77 |
+
host = f'hdfs://{host}'
|
| 78 |
+
|
| 79 |
+
options.ConfigureEndPoint(tobytes(host), int(port))
|
| 80 |
+
options.ConfigureReplication(replication)
|
| 81 |
+
options.ConfigureBufferSize(buffer_size)
|
| 82 |
+
|
| 83 |
+
if user is not None:
|
| 84 |
+
options.ConfigureUser(tobytes(user))
|
| 85 |
+
if default_block_size is not None:
|
| 86 |
+
options.ConfigureBlockSize(default_block_size)
|
| 87 |
+
if kerb_ticket is not None:
|
| 88 |
+
options.ConfigureKerberosTicketCachePath(
|
| 89 |
+
tobytes(_stringify_path(kerb_ticket)))
|
| 90 |
+
if extra_conf is not None:
|
| 91 |
+
for k, v in extra_conf.items():
|
| 92 |
+
options.ConfigureExtraConf(tobytes(k), tobytes(v))
|
| 93 |
+
|
| 94 |
+
with nogil:
|
| 95 |
+
wrapped = GetResultValue(CHadoopFileSystem.Make(options))
|
| 96 |
+
self.init(<shared_ptr[CFileSystem]> wrapped)
|
| 97 |
+
|
| 98 |
+
cdef init(self, const shared_ptr[CFileSystem]& wrapped):
|
| 99 |
+
FileSystem.init(self, wrapped)
|
| 100 |
+
self.hdfs = <CHadoopFileSystem*> wrapped.get()
|
| 101 |
+
|
| 102 |
+
@staticmethod
|
| 103 |
+
def from_uri(uri):
|
| 104 |
+
"""
|
| 105 |
+
Instantiate HadoopFileSystem object from an URI string.
|
| 106 |
+
|
| 107 |
+
The following two calls are equivalent
|
| 108 |
+
|
| 109 |
+
* ``HadoopFileSystem.from_uri('hdfs://localhost:8020/?user=test\
|
| 110 |
+
&replication=1')``
|
| 111 |
+
* ``HadoopFileSystem('localhost', port=8020, user='test', \
|
| 112 |
+
replication=1)``
|
| 113 |
+
|
| 114 |
+
Parameters
|
| 115 |
+
----------
|
| 116 |
+
uri : str
|
| 117 |
+
A string URI describing the connection to HDFS.
|
| 118 |
+
In order to change the user, replication, buffer_size or
|
| 119 |
+
default_block_size pass the values as query parts.
|
| 120 |
+
|
| 121 |
+
Returns
|
| 122 |
+
-------
|
| 123 |
+
HadoopFileSystem
|
| 124 |
+
"""
|
| 125 |
+
cdef:
|
| 126 |
+
HadoopFileSystem self = HadoopFileSystem.__new__(HadoopFileSystem)
|
| 127 |
+
shared_ptr[CHadoopFileSystem] wrapped
|
| 128 |
+
CHdfsOptions options
|
| 129 |
+
|
| 130 |
+
options = GetResultValue(CHdfsOptions.FromUriString(tobytes(uri)))
|
| 131 |
+
with nogil:
|
| 132 |
+
wrapped = GetResultValue(CHadoopFileSystem.Make(options))
|
| 133 |
+
|
| 134 |
+
self.init(<shared_ptr[CFileSystem]> wrapped)
|
| 135 |
+
return self
|
| 136 |
+
|
| 137 |
+
@staticmethod
|
| 138 |
+
def _reconstruct(kwargs):
|
| 139 |
+
# __reduce__ doesn't allow passing named arguments directly to the
|
| 140 |
+
# reconstructor, hence this wrapper.
|
| 141 |
+
return HadoopFileSystem(**kwargs)
|
| 142 |
+
|
| 143 |
+
def __reduce__(self):
|
| 144 |
+
cdef CHdfsOptions opts = self.hdfs.options()
|
| 145 |
+
return (
|
| 146 |
+
HadoopFileSystem._reconstruct, (dict(
|
| 147 |
+
host=frombytes(opts.connection_config.host),
|
| 148 |
+
port=opts.connection_config.port,
|
| 149 |
+
user=frombytes(opts.connection_config.user),
|
| 150 |
+
replication=opts.replication,
|
| 151 |
+
buffer_size=opts.buffer_size,
|
| 152 |
+
default_block_size=opts.default_block_size,
|
| 153 |
+
kerb_ticket=frombytes(opts.connection_config.kerb_ticket),
|
| 154 |
+
extra_conf={frombytes(k): frombytes(v)
|
| 155 |
+
for k, v in opts.connection_config.extra_conf},
|
| 156 |
+
),)
|
| 157 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:630be0ef2d4c9b08c78fe7784b0829079def49227d3fb91a373ad9d47f37f40b
|
| 3 |
+
size 102912
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/_json.pxd
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
# or more contributor license agreements. See the NOTICE file
|
| 3 |
+
# distributed with this work for additional information
|
| 4 |
+
# regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
# to you under the Apache License, Version 2.0 (the
|
| 6 |
+
# "License"); you may not use this file except in compliance
|
| 7 |
+
# with the License. You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing,
|
| 12 |
+
# software distributed under the License is distributed on an
|
| 13 |
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
# KIND, either express or implied. See the License for the
|
| 15 |
+
# specific language governing permissions and limitations
|
| 16 |
+
# under the License.
|
| 17 |
+
|
| 18 |
+
# cython: language_level = 3
|
| 19 |
+
|
| 20 |
+
from pyarrow.includes.libarrow cimport *
|
| 21 |
+
from pyarrow.lib cimport _Weakrefable
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
cdef class ParseOptions(_Weakrefable):
|
| 25 |
+
cdef:
|
| 26 |
+
CJSONParseOptions options
|
| 27 |
+
|
| 28 |
+
@staticmethod
|
| 29 |
+
cdef ParseOptions wrap(CJSONParseOptions options)
|
| 30 |
+
|
| 31 |
+
cdef class ReadOptions(_Weakrefable):
|
| 32 |
+
cdef:
|
| 33 |
+
CJSONReadOptions options
|
| 34 |
+
|
| 35 |
+
@staticmethod
|
| 36 |
+
cdef ReadOptions wrap(CJSONReadOptions options)
|