Buckets:
| import"../chunks/DsnmJJEf.js";import{i as Xo,h as Go,C as Jo,H as n,D as a,E as Yo,s as Ko,a as Zo}from"../chunks/Cq6VUY_c.js";import{p as Ho,o as $o,s as e,f as Na,a as ba,b as Qo,c as t,d as ua,n as r,r as o}from"../chunks/Bb-LL0eD.js";import{E as Vo}from"../chunks/CEnmBKuJ.js";const er='{"title":"Table Classes","local":"table-classes","sections":[{"title":"Table","local":"datasets.table.Table","sections":[],"depth":2},{"title":"InMemoryTable","local":"datasets.table.InMemoryTable","sections":[],"depth":2},{"title":"MemoryMappedTable","local":"datasets.table.MemoryMappedTable","sections":[],"depth":2},{"title":"ConcatenationTable","local":"datasets.table.ConcatenationTable","sections":[],"depth":2},{"title":"Utils","local":"datasets.table.concat_tables","sections":[],"depth":2}],"depth":1}';var ar=ua('<meta name="hf:doc:metadata"/>'),tr=ua("<p>Examples:</p> <!>",1),or=ua(`<p></p> <!> <!> <p>Each <code>Dataset</code> object is backed by a PyArrow Table. | |
| A Table can be loaded from either the disk (memory mapped) or in memory. | |
| Several Table types are available, and they all inherit from <a href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table">table.Table</a>.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Wraps a pyarrow Table by using composition. | |
| This is the base class for <code>InMemoryTable</code>, <code>MemoryMappedTable</code> and <code>ConcatenationTable</code>.</p> <p>It implements all the basic attributes/methods of the pyarrow Table class except | |
| the Table transforms: <code>slice, filter, flatten, combine_chunks, cast, add_column, append_column, remove_column, set_column, rename_columns</code> and <code>drop</code>.</p> <p>The implementation of these methods differs for the subclasses.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Perform validation checks. An exception is raised if validation fails.</p> <p>By default only cheap validation checks are run. Pass <code>full=True</code> for thorough validation checks (potentially <code>O(n)</code>).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Check if contents of two tables are equal.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert Table to list of (contiguous) <code>RecordBatch</code> objects.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert the Table to a <code>dict</code> or <code>OrderedDict</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert to a pandas-compatible NumPy array or DataFrame, as appropriate.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a schema field by its column name or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a column by its column name, or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Iterator over all columns in their numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Schema of the table and its columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>List of all columns in numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of columns in this table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of rows in this table.</p> <p>Due to the definition of a table, all columns have the same number of | |
| rows.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dimensions of the table: (#rows, #columns).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Total number of bytes consumed by the elements of the table.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The table is said in-memory when it is loaded into the user’s RAM.</p> <p>Pickling it does copy all the data using memory. | |
| Its implementation is simple and uses the underlying pyarrow Table methods directly.</p> <p>This is different from the <code>MemoryMapped</code> table, for which pickling doesn’t copy all the | |
| data in memory. For a <code>MemoryMapped</code>, unpickling instead reloads the table from the disk.</p> <p><code>InMemoryTable</code> must be used when data fit in memory, while <code>MemoryMapped</code> are reserved for | |
| data bigger than memory or when you want the memory footprint of your application to | |
| stay low.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Perform validation checks. An exception is raised if validation fails.</p> <p>By default only cheap validation checks are run. Pass <code>full=True</code> for thorough validation checks (potentially <code>O(n)</code>).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Check if contents of two tables are equal.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert Table to list of (contiguous) <code>RecordBatch</code> objects.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert the Table to a <code>dict</code> or <code>OrderedDict</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert to a pandas-compatible NumPy array or DataFrame, as appropriate.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a schema field by its column name or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a column by its column name, or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Iterator over all columns in their numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Schema of the table and its columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>List of all columns in numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of columns in this table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of rows in this table.</p> <p>Due to the definition of a table, all columns have the same number of | |
| rows.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dimensions of the table: (#rows, #columns).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Total number of bytes consumed by the elements of the table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Names of the table’s columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compute zero-copy slice of this Table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select records from a Table. See <code>pyarrow.compute.filter</code> for full usage.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Flatten this Table. Each column with a struct type is flattened | |
| into one column per struct field. Other columns are left unchanged.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Make a new table by combining the chunks this table has.</p> <p>All the underlying chunks in the <code>ChunkedArray</code> of each column are | |
| concatenated into zero or one chunk.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Cast table values to another schema.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>EXPERIMENTAL: Create shallow copy of table by replacing schema | |
| key-value metadata with the indicated new metadata (which may be <code>None</code>, | |
| which deletes any existing metadata).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Add column to Table at position.</p> <p>A new table is returned with the column added, the original table | |
| object is left unchanged.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Append column at end of columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Create new Table with the indicated column removed.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Replace column in Table at position.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Create new table with columns renamed to provided names.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select columns of the table.</p> <p>Returns a new table with the specified columns, and metadata preserved.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Drop one or more columns and return a new table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert pandas.DataFrame to an Arrow Table.</p> <p>The column types in the resulting Arrow Table are inferred from the | |
| dtypes of the pandas.Series in the DataFrame. In the case of non-object | |
| Series, the NumPy dtype is translated to its Arrow equivalent. In the | |
| case of <code>object</code>, we need to guess the datatype by looking at the | |
| Python objects in this Series.</p> <p>Be aware that Series of the <code>object</code> dtype don’t carry enough | |
| information to always lead to a meaningful Arrow type. In the case that | |
| we cannot infer a type, e.g. because the DataFrame is of length 0 or | |
| the Series only contains <code>None/nan</code> objects, the type is set to | |
| null. This behavior can be avoided by constructing an explicit schema | |
| and passing it to this function.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Construct a Table from Arrow arrays.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Construct a Table from Arrow arrays or columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Construct a Table from a sequence or iterator of Arrow <code>RecordBatches</code>.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The table is said memory mapped when it doesn’t use the user’s RAM but loads the data | |
| from the disk instead.</p> <p>Pickling it doesn’t copy the data into memory. | |
| Instead, only the path to the memory mapped arrow file is pickled, as well as the list | |
| of transforms to “replay” when reloading the table from the disk.</p> <p>Its implementation requires to store an history of all the transforms that were applied | |
| to the underlying pyarrow Table, so that they can be “replayed” when reloading the Table | |
| from the disk.</p> <p>This is different from the <code>InMemoryTable</code> table, for which pickling does copy all the | |
| data in memory.</p> <p><code>InMemoryTable</code> must be used when data fit in memory, while <code>MemoryMapped</code> are reserved for | |
| data bigger than memory or when you want the memory footprint of your application to | |
| stay low.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Perform validation checks. An exception is raised if validation fails.</p> <p>By default only cheap validation checks are run. Pass <code>full=True</code> for thorough validation checks (potentially <code>O(n)</code>).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Check if contents of two tables are equal.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert Table to list of (contiguous) <code>RecordBatch</code> objects.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert the Table to a <code>dict</code> or <code>OrderedDict</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert to a pandas-compatible NumPy array or DataFrame, as appropriate.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a schema field by its column name or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a column by its column name, or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Iterator over all columns in their numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Schema of the table and its columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>List of all columns in numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of columns in this table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of rows in this table.</p> <p>Due to the definition of a table, all columns have the same number of | |
| rows.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dimensions of the table: (#rows, #columns).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Total number of bytes consumed by the elements of the table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Names of the table’s columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compute zero-copy slice of this Table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select records from a Table. See <code>pyarrow.compute.filter</code> for full usage.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Flatten this Table. Each column with a struct type is flattened | |
| into one column per struct field. Other columns are left unchanged.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Make a new table by combining the chunks this table has.</p> <p>All the underlying chunks in the ChunkedArray of each column are | |
| concatenated into zero or one chunk.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Cast table values to another schema</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>EXPERIMENTAL: Create shallow copy of table by replacing schema | |
| key-value metadata with the indicated new metadata (which may be None, | |
| which deletes any existing metadata.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Add column to Table at position.</p> <p>A new table is returned with the column added, the original table | |
| object is left unchanged.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Append column at end of columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Create new Table with the indicated column removed.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Replace column in Table at position.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Create new table with columns renamed to provided names.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select columns of the table.</p> <p>Returns a new table with the specified columns, and metadata preserved.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Drop one or more columns and return a new table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The table comes from the concatenation of several tables called blocks. | |
| It enables concatenation on both axis 0 (append rows) and axis 1 (append columns).</p> <p>The underlying tables are called “blocks” and can be either <code>InMemoryTable</code> or <code>MemoryMappedTable</code> objects. | |
| This allows to combine tables that come from memory or that are memory mapped. | |
| When a <code>ConcatenationTable</code> is pickled, then each block is pickled:</p> <ul><li>the <code>InMemoryTable</code> objects are pickled by copying all the data in memory.</li> <li>the MemoryMappedTable objects are pickled without copying the data into memory. | |
| Instead, only the path to the memory mapped arrow file is pickled, as well as the list | |
| of transforms to “replays” when reloading the table from the disk.</li></ul> <p>Its implementation requires to store each block separately. | |
| The <code>blocks</code> attributes stores a list of list of blocks. | |
| The first axis concatenates the tables along the axis 0 (it appends rows), | |
| while the second axis concatenates tables along the axis 1 (it appends columns).</p> <p>If some columns are missing when concatenating on axis 0, they are filled with null values. | |
| This is done using <code>pyarrow.concat_tables(tables, promote=True)</code>.</p> <p>You can access the fully combined table by accessing the <code>ConcatenationTable.table</code> attribute, | |
| and the blocks by accessing the <code>ConcatenationTable.blocks</code> attribute.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Perform validation checks. An exception is raised if validation fails.</p> <p>By default only cheap validation checks are run. Pass <code>full=True</code> for thorough validation checks (potentially <code>O(n)</code>).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Check if contents of two tables are equal.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert Table to list of (contiguous) <code>RecordBatch</code> objects.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert the Table to a <code>dict</code> or <code>OrderedDict</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Convert to a pandas-compatible NumPy array or DataFrame, as appropriate.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a schema field by its column name or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select a column by its column name, or numeric index.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Iterator over all columns in their numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Schema of the table and its columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>List of all columns in numerical order.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of columns in this table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Number of rows in this table.</p> <p>Due to the definition of a table, all columns have the same number of | |
| rows.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dimensions of the table: (#rows, #columns).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Total number of bytes consumed by the elements of the table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Names of the table’s columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compute zero-copy slice of this Table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select records from a Table. See <code>pyarrow.compute.filter</code> for full usage.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Flatten this Table. Each column with a struct type is flattened | |
| into one column per struct field. Other columns are left unchanged.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Make a new table by combining the chunks this table has.</p> <p>All the underlying chunks in the <code>ChunkedArray</code> of each column are | |
| concatenated into zero or one chunk.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Cast table values to another schema.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>EXPERIMENTAL: Create shallow copy of table by replacing schema | |
| key-value metadata with the indicated new metadata (which may be <code>None</code>, | |
| which deletes any existing metadata).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Add column to Table at position.</p> <p>A new table is returned with the column added, the original table | |
| object is left unchanged.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Append column at end of columns.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Create new Table with the indicated column removed.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Replace column in Table at position.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Create new table with columns renamed to provided names.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Select columns of the table.</p> <p>Returns a new table with the specified columns, and metadata preserved.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Drop one or more columns and return a new table.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Create <code>ConcatenationTable</code> from list of tables.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Concatenate tables.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Get the cache files that are loaded by the table. | |
| Cache file are used when parts of the table come from the disk via memory mapping.</p></div> <!> <p></p>`,1);function cr(Ea,Sa){Ho(Sa,!1),$o(()=>{new URLSearchParams(window.location.search).get("fw")}),Xo();var ha=or();Go("19y8qkx",s=>{var ma=ar();Ko(ma,"content",er),ba(s,ma)});var ga=e(Na(ha),2);Jo(ga,{containerStyle:"float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"});var ya=e(ga,2);n(ya,{title:"Table Classes",local:"table-classes",headingTag:"h1"});var va=e(ya,4);n(va,{title:"Table",local:"datasets.table.Table",headingTag:"h2"});var d=e(va,2),fa=t(d);a(fa,{name:"class datasets.table.Table",anchor:"datasets.table.Table",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L210",parameters:[{name:"table",val:": Table"}]});var c=e(fa,8),Ua=t(c);a(Ua,{name:"validate",anchor:"datasets.table.Table.validate",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L235",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.Table.validate.full",description:`<strong>full</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, run expensive checks, otherwise cheap checks only.`,name:"full"}],raiseDescription:` | |
| <ul> | |
| <li><code>pa.lib.ArrowInvalid</code> — if validation fails</li> | |
| </ul> | |
| `,raiseType:` | |
| <p><code>pa.lib.ArrowInvalid</code></p> | |
| `}),r(4),o(c);var l=e(c,2),Pa=t(l);a(Pa,{name:"equals",anchor:"datasets.table.Table.equals",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L251",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.Table.equals.other",description:`<strong>other</strong> (<a href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table">Table</a>) — | |
| Table to compare against.`,name:"other"},{anchor:"datasets.table.Table.equals.check_metadata",description:`<strong>check_metadata</strong> <code>bool</code>, defaults to <code>False</code>) — | |
| Whether schema metadata equality should be checked as well.`,name:"check_metadata"}],returnType:` | |
| <p><code>bool</code></p> | |
| `}),r(2),o(l);var i=e(l,2),Ra=t(i);a(Ra,{name:"to_batches",anchor:"datasets.table.Table.to_batches",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L268",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.Table.to_batches.max_chunksize",description:`<strong>max_chunksize</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Maximum size for <code>RecordBatch</code> chunks. Individual chunks may be | |
| smaller depending on the chunk layout of individual columns.`,name:"max_chunksize"}],returnDescription:` | |
| <p><code>List[pyarrow.RecordBatch]</code></p> | |
| `}),r(2),o(i);var p=e(i,2),qa=t(p);a(qa,{name:"to_pydict",anchor:"datasets.table.Table.to_pydict",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L282",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnType:` | |
| <p><code>dict</code></p> | |
| `}),r(2),o(p);var m=e(p,2),za=t(m);a(za,{name:"to_pandas",anchor:"datasets.table.Table.to_pandas",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L300",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.Table.to_pandas.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| Arrow MemoryPool to use for allocations. Uses the default memory | |
| pool is not passed.`,name:"memory_pool"},{anchor:"datasets.table.Table.to_pandas.strings_to_categorical",description:`<strong>strings_to_categorical</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Encode string (UTF8) and binary types to <code>pandas.Categorical</code>.`,name:"strings_to_categorical"},{anchor:"datasets.table.Table.to_pandas.categories",description:`<strong>categories</strong> (<code>list</code>, defaults to <code>empty</code>) — | |
| List of fields that should be returned as <code>pandas.Categorical</code>. Only | |
| applies to table-like data structures.`,name:"categories"},{anchor:"datasets.table.Table.to_pandas.zero_copy_only",description:`<strong>zero_copy_only</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Raise an <code>ArrowException</code> if this function call would require copying | |
| the underlying data.`,name:"zero_copy_only"},{anchor:"datasets.table.Table.to_pandas.integer_object_nulls",description:`<strong>integer_object_nulls</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast integers with nulls to objects.`,name:"integer_object_nulls"},{anchor:"datasets.table.Table.to_pandas.date_as_object",description:`<strong>date_as_object</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Cast dates to objects. If <code>False</code>, convert to <code>datetime64[ns]</code> dtype.`,name:"date_as_object"},{anchor:"datasets.table.Table.to_pandas.timestamp_as_object",description:`<strong>timestamp_as_object</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast non-nanosecond timestamps (<code>np.datetime64</code>) to objects. This is | |
| useful if you have timestamps that don’t fit in the normal date | |
| range of nanosecond timestamps (1678 CE-2262 CE). | |
| If <code>False</code>, all timestamps are converted to <code>datetime64[ns]</code> dtype.`,name:"timestamp_as_object"},{anchor:"datasets.table.Table.to_pandas.use_threads",description:`<strong>use_threads</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Whether to parallelize the conversion using multiple threads.`,name:"use_threads"},{anchor:"datasets.table.Table.to_pandas.deduplicate_objects",description:`<strong>deduplicate_objects</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Do not create multiple copies Python objects when created, to save | |
| on memory use. Conversion will be slower.`,name:"deduplicate_objects"},{anchor:"datasets.table.Table.to_pandas.ignore_metadata",description:`<strong>ignore_metadata</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, do not use the ‘pandas’ metadata to reconstruct the | |
| DataFrame index, if present.`,name:"ignore_metadata"},{anchor:"datasets.table.Table.to_pandas.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| For certain data types, a cast is needed in order to store the | |
| data in a pandas DataFrame or Series (e.g. timestamps are always | |
| stored as nanoseconds in pandas). This option controls whether it | |
| is a safe cast or not.`,name:"safe"},{anchor:"datasets.table.Table.to_pandas.split_blocks",description:`<strong>split_blocks</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, generate one internal “block” for each column when | |
| creating a pandas.DataFrame from a <code>RecordBatch</code> or <code>Table</code>. While this | |
| can temporarily reduce memory note that various pandas operations | |
| can trigger “consolidation” which may balloon memory use.`,name:"split_blocks"},{anchor:"datasets.table.Table.to_pandas.self_destruct",description:`<strong>self_destruct</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| EXPERIMENTAL: If <code>True</code>, attempt to deallocate the originating Arrow | |
| memory while converting the Arrow object to pandas. If you use the | |
| object after calling <code>to_pandas</code> with this option it will crash your | |
| program.`,name:"self_destruct"},{anchor:"datasets.table.Table.to_pandas.types_mapper",description:`<strong>types_mapper</strong> (<code>function</code>, defaults to <code>None</code>) — | |
| A function mapping a pyarrow DataType to a pandas <code>ExtensionDtype</code>. | |
| This can be used to override the default pandas type for conversion | |
| of built-in pyarrow types or in absence of <code>pandas_metadata</code> in the | |
| Table schema. The function receives a pyarrow DataType and is | |
| expected to return a pandas <code>ExtensionDtype</code> or <code>None</code> if the | |
| default conversion should be used for that type. If you have | |
| a dictionary mapping, you can pass <code>dict.get</code> as function.`,name:"types_mapper"}],returnDescription:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code> depending on type of object</p> | |
| `,returnType:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code></p> | |
| `}),r(2),o(m);var b=e(m,2),Ba=t(b);a(Ba,{name:"to_string",anchor:"datasets.table.Table.to_string",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L362",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),o(b);var u=e(b,2),Oa=t(u);a(Oa,{name:"field",anchor:"datasets.table.Table.field",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L381",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.Table.field.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the field to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.Field</code></p> | |
| `}),r(2),o(u);var h=e(u,2),Wa=t(h);a(Wa,{name:"column",anchor:"datasets.table.Table.column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L394",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.Table.column.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the column to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(h);var g=e(h,2),Xa=t(g);a(Xa,{name:"itercolumns",anchor:"datasets.table.Table.itercolumns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L407",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(g);var y=e(g,2),Ga=t(y);a(Ga,{name:"schema",anchor:"datasets.table.Table.schema",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L416",parameters:[],returnDescription:` | |
| <p><code>pyarrow.Schema</code></p> | |
| `}),r(2),o(y);var v=e(y,2),Ja=t(v);a(Ja,{name:"columns",anchor:"datasets.table.Table.columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L426",parameters:[],returnDescription:` | |
| <p><code>List[pa.ChunkedArray]</code></p> | |
| `}),r(2),o(v);var f=e(v,2),Ya=t(f);a(Ya,{name:"num_columns",anchor:"datasets.table.Table.num_columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L436",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(2),o(f);var _=e(f,2),Ka=t(_);a(Ka,{name:"num_rows",anchor:"datasets.table.Table.num_rows",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L446",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(4),o(_);var T=e(_,2),Za=t(T);a(Za,{name:"shape",anchor:"datasets.table.Table.shape",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L459",parameters:[],returnDescription:` | |
| <p>Number of rows and number of columns.</p> | |
| `,returnType:` | |
| <p><code>(int, int)</code></p> | |
| `}),r(2),o(T);var _a=e(T,2),Ha=t(_a);a(Ha,{name:"nbytes",anchor:"datasets.table.Table.nbytes",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L469",parameters:[]}),r(2),o(_a),o(d);var Ta=e(d,2);n(Ta,{title:"InMemoryTable",local:"datasets.table.InMemoryTable",headingTag:"h2"});var w=e(Ta,2),wa=t(w);a(wa,{name:"class datasets.table.InMemoryTable",anchor:"datasets.table.InMemoryTable",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L695",parameters:[{name:"table",val:": Table"}]});var x=e(wa,10),$a=t(x);a($a,{name:"validate",anchor:"datasets.table.InMemoryTable.validate",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L235",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.validate.full",description:`<strong>full</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, run expensive checks, otherwise cheap checks only.`,name:"full"}],raiseDescription:` | |
| <ul> | |
| <li><code>pa.lib.ArrowInvalid</code> — if validation fails</li> | |
| </ul> | |
| `,raiseType:` | |
| <p><code>pa.lib.ArrowInvalid</code></p> | |
| `}),r(4),o(x);var M=e(x,2),Qa=t(M);a(Qa,{name:"equals",anchor:"datasets.table.InMemoryTable.equals",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L251",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.equals.other",description:`<strong>other</strong> (<a href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table">Table</a>) — | |
| Table to compare against.`,name:"other"},{anchor:"datasets.table.InMemoryTable.equals.check_metadata",description:`<strong>check_metadata</strong> <code>bool</code>, defaults to <code>False</code>) — | |
| Whether schema metadata equality should be checked as well.`,name:"check_metadata"}],returnType:` | |
| <p><code>bool</code></p> | |
| `}),r(2),o(M);var k=e(M,2),Va=t(k);a(Va,{name:"to_batches",anchor:"datasets.table.InMemoryTable.to_batches",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L268",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.to_batches.max_chunksize",description:`<strong>max_chunksize</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Maximum size for <code>RecordBatch</code> chunks. Individual chunks may be | |
| smaller depending on the chunk layout of individual columns.`,name:"max_chunksize"}],returnDescription:` | |
| <p><code>List[pyarrow.RecordBatch]</code></p> | |
| `}),r(2),o(k);var D=e(k,2),et=t(D);a(et,{name:"to_pydict",anchor:"datasets.table.InMemoryTable.to_pydict",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L282",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnType:` | |
| <p><code>dict</code></p> | |
| `}),r(2),o(D);var C=e(D,2),at=t(C);a(at,{name:"to_pandas",anchor:"datasets.table.InMemoryTable.to_pandas",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L300",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.to_pandas.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| Arrow MemoryPool to use for allocations. Uses the default memory | |
| pool is not passed.`,name:"memory_pool"},{anchor:"datasets.table.InMemoryTable.to_pandas.strings_to_categorical",description:`<strong>strings_to_categorical</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Encode string (UTF8) and binary types to <code>pandas.Categorical</code>.`,name:"strings_to_categorical"},{anchor:"datasets.table.InMemoryTable.to_pandas.categories",description:`<strong>categories</strong> (<code>list</code>, defaults to <code>empty</code>) — | |
| List of fields that should be returned as <code>pandas.Categorical</code>. Only | |
| applies to table-like data structures.`,name:"categories"},{anchor:"datasets.table.InMemoryTable.to_pandas.zero_copy_only",description:`<strong>zero_copy_only</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Raise an <code>ArrowException</code> if this function call would require copying | |
| the underlying data.`,name:"zero_copy_only"},{anchor:"datasets.table.InMemoryTable.to_pandas.integer_object_nulls",description:`<strong>integer_object_nulls</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast integers with nulls to objects.`,name:"integer_object_nulls"},{anchor:"datasets.table.InMemoryTable.to_pandas.date_as_object",description:`<strong>date_as_object</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Cast dates to objects. If <code>False</code>, convert to <code>datetime64[ns]</code> dtype.`,name:"date_as_object"},{anchor:"datasets.table.InMemoryTable.to_pandas.timestamp_as_object",description:`<strong>timestamp_as_object</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast non-nanosecond timestamps (<code>np.datetime64</code>) to objects. This is | |
| useful if you have timestamps that don’t fit in the normal date | |
| range of nanosecond timestamps (1678 CE-2262 CE). | |
| If <code>False</code>, all timestamps are converted to <code>datetime64[ns]</code> dtype.`,name:"timestamp_as_object"},{anchor:"datasets.table.InMemoryTable.to_pandas.use_threads",description:`<strong>use_threads</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Whether to parallelize the conversion using multiple threads.`,name:"use_threads"},{anchor:"datasets.table.InMemoryTable.to_pandas.deduplicate_objects",description:`<strong>deduplicate_objects</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Do not create multiple copies Python objects when created, to save | |
| on memory use. Conversion will be slower.`,name:"deduplicate_objects"},{anchor:"datasets.table.InMemoryTable.to_pandas.ignore_metadata",description:`<strong>ignore_metadata</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, do not use the ‘pandas’ metadata to reconstruct the | |
| DataFrame index, if present.`,name:"ignore_metadata"},{anchor:"datasets.table.InMemoryTable.to_pandas.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| For certain data types, a cast is needed in order to store the | |
| data in a pandas DataFrame or Series (e.g. timestamps are always | |
| stored as nanoseconds in pandas). This option controls whether it | |
| is a safe cast or not.`,name:"safe"},{anchor:"datasets.table.InMemoryTable.to_pandas.split_blocks",description:`<strong>split_blocks</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, generate one internal “block” for each column when | |
| creating a pandas.DataFrame from a <code>RecordBatch</code> or <code>Table</code>. While this | |
| can temporarily reduce memory note that various pandas operations | |
| can trigger “consolidation” which may balloon memory use.`,name:"split_blocks"},{anchor:"datasets.table.InMemoryTable.to_pandas.self_destruct",description:`<strong>self_destruct</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| EXPERIMENTAL: If <code>True</code>, attempt to deallocate the originating Arrow | |
| memory while converting the Arrow object to pandas. If you use the | |
| object after calling <code>to_pandas</code> with this option it will crash your | |
| program.`,name:"self_destruct"},{anchor:"datasets.table.InMemoryTable.to_pandas.types_mapper",description:`<strong>types_mapper</strong> (<code>function</code>, defaults to <code>None</code>) — | |
| A function mapping a pyarrow DataType to a pandas <code>ExtensionDtype</code>. | |
| This can be used to override the default pandas type for conversion | |
| of built-in pyarrow types or in absence of <code>pandas_metadata</code> in the | |
| Table schema. The function receives a pyarrow DataType and is | |
| expected to return a pandas <code>ExtensionDtype</code> or <code>None</code> if the | |
| default conversion should be used for that type. If you have | |
| a dictionary mapping, you can pass <code>dict.get</code> as function.`,name:"types_mapper"}],returnDescription:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code> depending on type of object</p> | |
| `,returnType:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code></p> | |
| `}),r(2),o(C);var I=e(C,2),tt=t(I);a(tt,{name:"to_string",anchor:"datasets.table.InMemoryTable.to_string",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L362",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),o(I);var L=e(I,2),ot=t(L);a(ot,{name:"field",anchor:"datasets.table.InMemoryTable.field",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L381",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.field.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the field to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.Field</code></p> | |
| `}),r(2),o(L);var j=e(L,2),rt=t(j);a(rt,{name:"column",anchor:"datasets.table.InMemoryTable.column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L394",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.column.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the column to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(j);var F=e(j,2),nt=t(F);a(nt,{name:"itercolumns",anchor:"datasets.table.InMemoryTable.itercolumns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L407",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(F);var A=e(F,2),st=t(A);a(st,{name:"schema",anchor:"datasets.table.InMemoryTable.schema",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L416",parameters:[],returnDescription:` | |
| <p><code>pyarrow.Schema</code></p> | |
| `}),r(2),o(A);var N=e(A,2),dt=t(N);a(dt,{name:"columns",anchor:"datasets.table.InMemoryTable.columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L426",parameters:[],returnDescription:` | |
| <p><code>List[pa.ChunkedArray]</code></p> | |
| `}),r(2),o(N);var E=e(N,2),ct=t(E);a(ct,{name:"num_columns",anchor:"datasets.table.InMemoryTable.num_columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L436",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(2),o(E);var S=e(E,2),lt=t(S);a(lt,{name:"num_rows",anchor:"datasets.table.InMemoryTable.num_rows",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L446",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(4),o(S);var U=e(S,2),it=t(U);a(it,{name:"shape",anchor:"datasets.table.InMemoryTable.shape",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L459",parameters:[],returnDescription:` | |
| <p>Number of rows and number of columns.</p> | |
| `,returnType:` | |
| <p><code>(int, int)</code></p> | |
| `}),r(2),o(U);var P=e(U,2),pt=t(P);a(pt,{name:"nbytes",anchor:"datasets.table.InMemoryTable.nbytes",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L469",parameters:[]}),r(2),o(P);var R=e(P,2),mt=t(R);a(mt,{name:"column_names",anchor:"datasets.table.InMemoryTable.column_names",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L476",parameters:[]}),r(2),o(R);var q=e(R,2),bt=t(q);a(bt,{name:"slice",anchor:"datasets.table.InMemoryTable.slice",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L859",parameters:[{name:"offset",val:" = 0"},{name:"length",val:" = None"}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.slice.offset",description:`<strong>offset</strong> (<code>int</code>, defaults to <code>0</code>) — | |
| Offset from start of table to slice.`,name:"offset"},{anchor:"datasets.table.InMemoryTable.slice.length",description:`<strong>length</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Length of slice (default is until end of table starting from | |
| offset).`,name:"length"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(q);var z=e(q,2),ut=t(z);a(ut,{name:"filter",anchor:"datasets.table.InMemoryTable.filter",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L876",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),r(2),o(z);var B=e(z,2),ht=t(B);a(ht,{name:"flatten",anchor:"datasets.table.InMemoryTable.flatten",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L882",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.flatten.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| For memory allocations, if required, otherwise use default pool.`,name:"memory_pool"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(B);var O=e(B,2),gt=t(O);a(gt,{name:"combine_chunks",anchor:"datasets.table.InMemoryTable.combine_chunks",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L896",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.combine_chunks.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| For memory allocations, if required, otherwise use default pool.`,name:"memory_pool"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(4),o(O);var W=e(O,2),yt=t(W);a(yt,{name:"cast",anchor:"datasets.table.InMemoryTable.cast",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L912",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.cast.target_schema",description:`<strong>target_schema</strong> (<code>Schema</code>) — | |
| Schema to cast to, the names and order of fields must match.`,name:"target_schema"},{anchor:"datasets.table.InMemoryTable.cast.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Check for overflows or other unsafe conversions.`,name:"safe"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(W);var X=e(W,2),vt=t(X);a(vt,{name:"replace_schema_metadata",anchor:"datasets.table.InMemoryTable.replace_schema_metadata",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L927",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.replace_schema_metadata.metadata",description:"<strong>metadata</strong> (<code>dict</code>, defaults to <code>None</code>) —",name:"metadata"}],returnDescription:` | |
| <p>shallow_copy</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(X);var G=e(X,2),ft=t(G);a(ft,{name:"add_column",anchor:"datasets.table.InMemoryTable.add_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L941",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.add_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index to place the column at.`,name:"i"},{anchor:"datasets.table.InMemoryTable.add_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.InMemoryTable.add_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column added.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(4),o(G);var J=e(G,2),_t=t(J);a(_t,{name:"append_column",anchor:"datasets.table.InMemoryTable.append_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L962",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.append_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.InMemoryTable.append_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column added.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(J);var Y=e(J,2),Tt=t(Y);a(Tt,{name:"remove_column",anchor:"datasets.table.InMemoryTable.remove_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L979",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.remove_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index of column to remove.`,name:"i"}],returnDescription:` | |
| <p>New table without the column.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(Y);var K=e(Y,2),wt=t(K);a(wt,{name:"set_column",anchor:"datasets.table.InMemoryTable.set_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L993",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.set_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index to place the column at.`,name:"i"},{anchor:"datasets.table.InMemoryTable.set_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.InMemoryTable.set_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column set.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(K);var Z=e(K,2),xt=t(Z);a(xt,{name:"rename_columns",anchor:"datasets.table.InMemoryTable.rename_columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1012",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),r(2),o(Z);var H=e(Z,2),Mt=t(H);a(Mt,{name:"select",anchor:"datasets.table.InMemoryTable.select",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1035",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.select.columns",description:`<strong>columns</strong> (<code>Union[List[str], List[int]]</code>) — | |
| The column names or integer indices to select.`,name:"columns"}],returnDescription:` | |
| <p>New table with the specified columns, and metadata preserved.</p> | |
| `,returnType:` | |
| <p><a | |
| href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table" | |
| >datasets.table.Table</a></p> | |
| `}),r(4),o(H);var $=e(H,2),kt=t($);a(kt,{name:"drop",anchor:"datasets.table.InMemoryTable.drop",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1018",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.drop.columns",description:`<strong>columns</strong> (<code>List[str]</code>) — | |
| List of field names referencing existing columns.`,name:"columns"}],returnDescription:` | |
| <p>New table without the columns.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `,raiseDescription:` | |
| <ul> | |
| <li><code>KeyError</code> — : if any of the passed columns name are not existing.</li> | |
| </ul> | |
| `,raiseType:` | |
| <p><code>KeyError</code></p> | |
| `}),r(2),o($);var Q=e($,2),Dt=t(Q);a(Dt,{name:"from_file",anchor:"datasets.table.InMemoryTable.from_file",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L719",parameters:[{name:"filename",val:": str"}]}),o(Q);var V=e(Q,2),Ct=t(V);a(Ct,{name:"from_buffer",anchor:"datasets.table.InMemoryTable.from_buffer",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L724",parameters:[{name:"buffer",val:": Buffer"}]}),o(V);var ee=e(V,2),xa=t(ee);a(xa,{name:"from_pandas",anchor:"datasets.table.InMemoryTable.from_pandas",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L729",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.from_pandas.df",description:"<strong>df</strong> (<code>pandas.DataFrame</code>) —",name:"df"},{anchor:"datasets.table.InMemoryTable.from_pandas.schema",description:`<strong>schema</strong> (<code>pyarrow.Schema</code>, <em>optional</em>) — | |
| The expected schema of the Arrow Table. This can be used to | |
| indicate the type of columns if we cannot infer it automatically. | |
| If passed, the output will have exactly this schema. Columns | |
| specified in the schema that are not found in the DataFrame columns | |
| or its index will raise an error. Additional columns or index | |
| levels in the DataFrame which are not specified in the schema will | |
| be ignored.`,name:"schema"},{anchor:"datasets.table.InMemoryTable.from_pandas.preserve_index",description:`<strong>preserve_index</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to store the index as an additional column in the resulting | |
| <code>Table</code>. The default of None will store the index as a column, | |
| except for RangeIndex which is stored as metadata only. Use | |
| <code>preserve_index=True</code> to force it to be stored as a column.`,name:"preserve_index"},{anchor:"datasets.table.InMemoryTable.from_pandas.nthreads",description:`<strong>nthreads</strong> (<code>int</code>, defaults to <code>None</code> (may use up to system CPU count threads)) — | |
| If greater than 1, convert columns to Arrow in parallel using | |
| indicated number of threads.`,name:"nthreads"},{anchor:"datasets.table.InMemoryTable.from_pandas.columns",description:`<strong>columns</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| List of column to be converted. If <code>None</code>, use all columns.`,name:"columns"},{anchor:"datasets.table.InMemoryTable.from_pandas.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Check for overflows or other unsafe conversions,`,name:"safe"}],returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `});var It=e(xa,8);Vo(It,{anchor:"datasets.table.InMemoryTable.from_pandas.example",children:(s,ma)=>{var Aa=tr(),Wo=e(Na(Aa),2);Zo(Wo,{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQWltcG9ydCUyMHB5YXJyb3clMjBhcyUyMHBhJTBBZGYlMjAlM0QlMjBwZC5EYXRhRnJhbWUoJTdCJTBBcGEuVGFibGUuZnJvbV9wYW5kYXMoZGYp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa | |
| <span class="hljs-meta">>>> </span>df = pd.DataFrame({ | |
| ... <span class="hljs-string">'int'</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], | |
| ... <span class="hljs-string">'str'</span>: [<span class="hljs-string">'a'</span>, <span class="hljs-string">'b'</span>] | |
| ... }) | |
| <span class="hljs-meta">>>> </span>pa.Table.from_pandas(df) | |
| <pyarrow.lib.Table <span class="hljs-built_in">object</span> at <span class="hljs-number">0x7f05d1fb1b40</span>>`,lang:"python",wrap:!1}),ba(s,Aa)},$$slots:{default:!0}}),o(ee);var ae=e(ee,2),Lt=t(ae);a(Lt,{name:"from_arrays",anchor:"datasets.table.InMemoryTable.from_arrays",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L787",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.from_arrays.arrays",description:`<strong>arrays</strong> (<code>List[Union[pyarrow.Array, pyarrow.ChunkedArray]]</code>) — | |
| Equal-length arrays that should form the table.`,name:"arrays"},{anchor:"datasets.table.InMemoryTable.from_arrays.names",description:`<strong>names</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| Names for the table columns. If not passed, schema must be passed.`,name:"names"},{anchor:"datasets.table.InMemoryTable.from_arrays.schema",description:`<strong>schema</strong> (<code>Schema</code>, defaults to <code>None</code>) — | |
| Schema for the created table. If not passed, names must be passed.`,name:"schema"},{anchor:"datasets.table.InMemoryTable.from_arrays.metadata",description:`<strong>metadata</strong> (<code>Union[dict, Mapping]</code>, defaults to <code>None</code>) — | |
| Optional metadata for the schema (if inferred).`,name:"metadata"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(ae);var te=e(ae,2),jt=t(te);a(jt,{name:"from_pydict",anchor:"datasets.table.InMemoryTable.from_pydict",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L807",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.from_pydict.mapping",description:`<strong>mapping</strong> (<code>Union[dict, Mapping]</code>) — | |
| A mapping of strings to Arrays or Python lists.`,name:"mapping"},{anchor:"datasets.table.InMemoryTable.from_pydict.schema",description:`<strong>schema</strong> (<code>Schema</code>, defaults to <code>None</code>) — | |
| If not passed, will be inferred from the Mapping values`,name:"schema"},{anchor:"datasets.table.InMemoryTable.from_pydict.metadata",description:`<strong>metadata</strong> (<code>Union[dict, Mapping]</code>, defaults to <code>None</code>) — | |
| Optional metadata for the schema (if inferred).`,name:"metadata"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(te);var Ma=e(te,2),Ft=t(Ma);a(Ft,{name:"from_batches",anchor:"datasets.table.InMemoryTable.from_batches",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L843",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.InMemoryTable.from_batches.batches",description:`<strong>batches</strong> (<code>Union[Sequence[pyarrow.RecordBatch], Iterator[pyarrow.RecordBatch]]</code>) — | |
| Sequence of <code>RecordBatch</code> to be converted, all schemas must be equal.`,name:"batches"},{anchor:"datasets.table.InMemoryTable.from_batches.schema",description:`<strong>schema</strong> (<code>Schema</code>, defaults to <code>None</code>) — | |
| If not passed, will be inferred from the first <code>RecordBatch</code>.`,name:"schema"}],returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(Ma),o(w);var ka=e(w,2);n(ka,{title:"MemoryMappedTable",local:"datasets.table.MemoryMappedTable",headingTag:"h2"});var oe=e(ka,2),Da=t(oe);a(Da,{name:"class datasets.table.MemoryMappedTable",anchor:"datasets.table.MemoryMappedTable",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1055",parameters:[{name:"table",val:": Table"},{name:"path",val:": str"},{name:"replays",val:": typing.Optional[list[tuple[str, tuple, dict]]] = None"}]});var re=e(Da,12),At=t(re);a(At,{name:"validate",anchor:"datasets.table.MemoryMappedTable.validate",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L235",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.validate.full",description:`<strong>full</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, run expensive checks, otherwise cheap checks only.`,name:"full"}],raiseDescription:` | |
| <ul> | |
| <li><code>pa.lib.ArrowInvalid</code> — if validation fails</li> | |
| </ul> | |
| `,raiseType:` | |
| <p><code>pa.lib.ArrowInvalid</code></p> | |
| `}),r(4),o(re);var ne=e(re,2),Nt=t(ne);a(Nt,{name:"equals",anchor:"datasets.table.MemoryMappedTable.equals",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L251",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.equals.other",description:`<strong>other</strong> (<a href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table">Table</a>) — | |
| Table to compare against.`,name:"other"},{anchor:"datasets.table.MemoryMappedTable.equals.check_metadata",description:`<strong>check_metadata</strong> <code>bool</code>, defaults to <code>False</code>) — | |
| Whether schema metadata equality should be checked as well.`,name:"check_metadata"}],returnType:` | |
| <p><code>bool</code></p> | |
| `}),r(2),o(ne);var se=e(ne,2),Et=t(se);a(Et,{name:"to_batches",anchor:"datasets.table.MemoryMappedTable.to_batches",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L268",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.to_batches.max_chunksize",description:`<strong>max_chunksize</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Maximum size for <code>RecordBatch</code> chunks. Individual chunks may be | |
| smaller depending on the chunk layout of individual columns.`,name:"max_chunksize"}],returnDescription:` | |
| <p><code>List[pyarrow.RecordBatch]</code></p> | |
| `}),r(2),o(se);var de=e(se,2),St=t(de);a(St,{name:"to_pydict",anchor:"datasets.table.MemoryMappedTable.to_pydict",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L282",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnType:` | |
| <p><code>dict</code></p> | |
| `}),r(2),o(de);var ce=e(de,2),Ut=t(ce);a(Ut,{name:"to_pandas",anchor:"datasets.table.MemoryMappedTable.to_pandas",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L300",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.to_pandas.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| Arrow MemoryPool to use for allocations. Uses the default memory | |
| pool is not passed.`,name:"memory_pool"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.strings_to_categorical",description:`<strong>strings_to_categorical</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Encode string (UTF8) and binary types to <code>pandas.Categorical</code>.`,name:"strings_to_categorical"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.categories",description:`<strong>categories</strong> (<code>list</code>, defaults to <code>empty</code>) — | |
| List of fields that should be returned as <code>pandas.Categorical</code>. Only | |
| applies to table-like data structures.`,name:"categories"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.zero_copy_only",description:`<strong>zero_copy_only</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Raise an <code>ArrowException</code> if this function call would require copying | |
| the underlying data.`,name:"zero_copy_only"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.integer_object_nulls",description:`<strong>integer_object_nulls</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast integers with nulls to objects.`,name:"integer_object_nulls"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.date_as_object",description:`<strong>date_as_object</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Cast dates to objects. If <code>False</code>, convert to <code>datetime64[ns]</code> dtype.`,name:"date_as_object"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.timestamp_as_object",description:`<strong>timestamp_as_object</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast non-nanosecond timestamps (<code>np.datetime64</code>) to objects. This is | |
| useful if you have timestamps that don’t fit in the normal date | |
| range of nanosecond timestamps (1678 CE-2262 CE). | |
| If <code>False</code>, all timestamps are converted to <code>datetime64[ns]</code> dtype.`,name:"timestamp_as_object"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.use_threads",description:`<strong>use_threads</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Whether to parallelize the conversion using multiple threads.`,name:"use_threads"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.deduplicate_objects",description:`<strong>deduplicate_objects</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Do not create multiple copies Python objects when created, to save | |
| on memory use. Conversion will be slower.`,name:"deduplicate_objects"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.ignore_metadata",description:`<strong>ignore_metadata</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, do not use the ‘pandas’ metadata to reconstruct the | |
| DataFrame index, if present.`,name:"ignore_metadata"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| For certain data types, a cast is needed in order to store the | |
| data in a pandas DataFrame or Series (e.g. timestamps are always | |
| stored as nanoseconds in pandas). This option controls whether it | |
| is a safe cast or not.`,name:"safe"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.split_blocks",description:`<strong>split_blocks</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, generate one internal “block” for each column when | |
| creating a pandas.DataFrame from a <code>RecordBatch</code> or <code>Table</code>. While this | |
| can temporarily reduce memory note that various pandas operations | |
| can trigger “consolidation” which may balloon memory use.`,name:"split_blocks"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.self_destruct",description:`<strong>self_destruct</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| EXPERIMENTAL: If <code>True</code>, attempt to deallocate the originating Arrow | |
| memory while converting the Arrow object to pandas. If you use the | |
| object after calling <code>to_pandas</code> with this option it will crash your | |
| program.`,name:"self_destruct"},{anchor:"datasets.table.MemoryMappedTable.to_pandas.types_mapper",description:`<strong>types_mapper</strong> (<code>function</code>, defaults to <code>None</code>) — | |
| A function mapping a pyarrow DataType to a pandas <code>ExtensionDtype</code>. | |
| This can be used to override the default pandas type for conversion | |
| of built-in pyarrow types or in absence of <code>pandas_metadata</code> in the | |
| Table schema. The function receives a pyarrow DataType and is | |
| expected to return a pandas <code>ExtensionDtype</code> or <code>None</code> if the | |
| default conversion should be used for that type. If you have | |
| a dictionary mapping, you can pass <code>dict.get</code> as function.`,name:"types_mapper"}],returnDescription:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code> depending on type of object</p> | |
| `,returnType:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code></p> | |
| `}),r(2),o(ce);var le=e(ce,2),Pt=t(le);a(Pt,{name:"to_string",anchor:"datasets.table.MemoryMappedTable.to_string",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L362",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),o(le);var ie=e(le,2),Rt=t(ie);a(Rt,{name:"field",anchor:"datasets.table.MemoryMappedTable.field",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L381",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.field.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the field to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.Field</code></p> | |
| `}),r(2),o(ie);var pe=e(ie,2),qt=t(pe);a(qt,{name:"column",anchor:"datasets.table.MemoryMappedTable.column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L394",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.column.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the column to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(pe);var me=e(pe,2),zt=t(me);a(zt,{name:"itercolumns",anchor:"datasets.table.MemoryMappedTable.itercolumns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L407",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(me);var be=e(me,2),Bt=t(be);a(Bt,{name:"schema",anchor:"datasets.table.MemoryMappedTable.schema",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L416",parameters:[],returnDescription:` | |
| <p><code>pyarrow.Schema</code></p> | |
| `}),r(2),o(be);var ue=e(be,2),Ot=t(ue);a(Ot,{name:"columns",anchor:"datasets.table.MemoryMappedTable.columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L426",parameters:[],returnDescription:` | |
| <p><code>List[pa.ChunkedArray]</code></p> | |
| `}),r(2),o(ue);var he=e(ue,2),Wt=t(he);a(Wt,{name:"num_columns",anchor:"datasets.table.MemoryMappedTable.num_columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L436",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(2),o(he);var ge=e(he,2),Xt=t(ge);a(Xt,{name:"num_rows",anchor:"datasets.table.MemoryMappedTable.num_rows",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L446",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(4),o(ge);var ye=e(ge,2),Gt=t(ye);a(Gt,{name:"shape",anchor:"datasets.table.MemoryMappedTable.shape",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L459",parameters:[],returnDescription:` | |
| <p>Number of rows and number of columns.</p> | |
| `,returnType:` | |
| <p><code>(int, int)</code></p> | |
| `}),r(2),o(ye);var ve=e(ye,2),Jt=t(ve);a(Jt,{name:"nbytes",anchor:"datasets.table.MemoryMappedTable.nbytes",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L469",parameters:[]}),r(2),o(ve);var fe=e(ve,2),Yt=t(fe);a(Yt,{name:"column_names",anchor:"datasets.table.MemoryMappedTable.column_names",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L476",parameters:[]}),r(2),o(fe);var _e=e(fe,2),Kt=t(_e);a(Kt,{name:"slice",anchor:"datasets.table.MemoryMappedTable.slice",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1114",parameters:[{name:"offset",val:" = 0"},{name:"length",val:" = None"}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.slice.offset",description:`<strong>offset</strong> (<code>int</code>, defaults to <code>0</code>) — | |
| Offset from start of table to slice.`,name:"offset"},{anchor:"datasets.table.MemoryMappedTable.slice.length",description:`<strong>length</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Length of slice (default is until end of table starting from | |
| offset).`,name:"length"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(_e);var Te=e(_e,2),Zt=t(Te);a(Zt,{name:"filter",anchor:"datasets.table.MemoryMappedTable.filter",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1133",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),r(2),o(Te);var we=e(Te,2),Ht=t(we);a(Ht,{name:"flatten",anchor:"datasets.table.MemoryMappedTable.flatten",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1141",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.flatten.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| For memory allocations, if required, otherwise use default pool.`,name:"memory_pool"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(we);var xe=e(we,2),$t=t(xe);a($t,{name:"combine_chunks",anchor:"datasets.table.MemoryMappedTable.combine_chunks",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1157",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.combine_chunks.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| For memory allocations, if required, otherwise use default pool.`,name:"memory_pool"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(4),o(xe);var Me=e(xe,2),Qt=t(Me);a(Qt,{name:"cast",anchor:"datasets.table.MemoryMappedTable.cast",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1175",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.cast.target_schema",description:`<strong>target_schema</strong> (<code>Schema</code>) — | |
| Schema to cast to, the names and order of fields must match.`,name:"target_schema"},{anchor:"datasets.table.MemoryMappedTable.cast.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Check for overflows or other unsafe conversions.`,name:"safe"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(Me);var ke=e(Me,2),Vt=t(ke);a(Vt,{name:"replace_schema_metadata",anchor:"datasets.table.MemoryMappedTable.replace_schema_metadata",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1192",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.replace_schema_metadata.metadata",description:"<strong>metadata</strong> (<code>dict</code>, defaults to <code>None</code>) —",name:"metadata"}],returnDescription:` | |
| <p>shallow_copy</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(ke);var De=e(ke,2),eo=t(De);a(eo,{name:"add_column",anchor:"datasets.table.MemoryMappedTable.add_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1208",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.add_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index to place the column at.`,name:"i"},{anchor:"datasets.table.MemoryMappedTable.add_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.MemoryMappedTable.add_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column added.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(4),o(De);var Ce=e(De,2),ao=t(Ce);a(ao,{name:"append_column",anchor:"datasets.table.MemoryMappedTable.append_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1231",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.append_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.MemoryMappedTable.append_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column added.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(Ce);var Ie=e(Ce,2),to=t(Ie);a(to,{name:"remove_column",anchor:"datasets.table.MemoryMappedTable.remove_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1250",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.remove_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index of column to remove.`,name:"i"}],returnDescription:` | |
| <p>New table without the column.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(Ie);var Le=e(Ie,2),oo=t(Le);a(oo,{name:"set_column",anchor:"datasets.table.MemoryMappedTable.set_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1266",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.set_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index to place the column at.`,name:"i"},{anchor:"datasets.table.MemoryMappedTable.set_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.MemoryMappedTable.set_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column set.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(Le);var je=e(Le,2),ro=t(je);a(ro,{name:"rename_columns",anchor:"datasets.table.MemoryMappedTable.rename_columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1287",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),r(2),o(je);var Fe=e(je,2),no=t(Fe);a(no,{name:"select",anchor:"datasets.table.MemoryMappedTable.select",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1314",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.select.columns",description:`<strong>columns</strong> (<code>Union[List[str], List[int]]</code>) — | |
| The column names or integer indices to select.`,name:"columns"}],returnDescription:` | |
| <p>New table with the specified columns, and metadata preserved.</p> | |
| `,returnType:` | |
| <p><a | |
| href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table" | |
| >datasets.table.Table</a></p> | |
| `}),r(4),o(Fe);var Ae=e(Fe,2),so=t(Ae);a(so,{name:"drop",anchor:"datasets.table.MemoryMappedTable.drop",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1295",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.MemoryMappedTable.drop.columns",description:`<strong>columns</strong> (<code>List[str]</code>) — | |
| List of field names referencing existing columns.`,name:"columns"}],returnDescription:` | |
| <p>New table without the columns.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `,raiseDescription:` | |
| <ul> | |
| <li><code>KeyError</code> — : if any of the passed columns name are not existing.</li> | |
| </ul> | |
| `,raiseType:` | |
| <p><code>KeyError</code></p> | |
| `}),r(2),o(Ae);var Ca=e(Ae,2),co=t(Ca);a(co,{name:"from_file",anchor:"datasets.table.MemoryMappedTable.from_file",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1081",parameters:[{name:"filename",val:": str"},{name:"replays",val:" = None"}]}),o(Ca),o(oe);var Ia=e(oe,2);n(Ia,{title:"ConcatenationTable",local:"datasets.table.ConcatenationTable",headingTag:"h2"});var Ne=e(Ia,2),La=t(Ne);a(La,{name:"class datasets.table.ConcatenationTable",anchor:"datasets.table.ConcatenationTable",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1339",parameters:[{name:"table",val:": Table"},{name:"blocks",val:": list"}]});var Ee=e(La,14),lo=t(Ee);a(lo,{name:"validate",anchor:"datasets.table.ConcatenationTable.validate",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L235",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.validate.full",description:`<strong>full</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, run expensive checks, otherwise cheap checks only.`,name:"full"}],raiseDescription:` | |
| <ul> | |
| <li><code>pa.lib.ArrowInvalid</code> — if validation fails</li> | |
| </ul> | |
| `,raiseType:` | |
| <p><code>pa.lib.ArrowInvalid</code></p> | |
| `}),r(4),o(Ee);var Se=e(Ee,2),io=t(Se);a(io,{name:"equals",anchor:"datasets.table.ConcatenationTable.equals",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L251",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.equals.other",description:`<strong>other</strong> (<a href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table">Table</a>) — | |
| Table to compare against.`,name:"other"},{anchor:"datasets.table.ConcatenationTable.equals.check_metadata",description:`<strong>check_metadata</strong> <code>bool</code>, defaults to <code>False</code>) — | |
| Whether schema metadata equality should be checked as well.`,name:"check_metadata"}],returnType:` | |
| <p><code>bool</code></p> | |
| `}),r(2),o(Se);var Ue=e(Se,2),po=t(Ue);a(po,{name:"to_batches",anchor:"datasets.table.ConcatenationTable.to_batches",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L268",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.to_batches.max_chunksize",description:`<strong>max_chunksize</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Maximum size for <code>RecordBatch</code> chunks. Individual chunks may be | |
| smaller depending on the chunk layout of individual columns.`,name:"max_chunksize"}],returnDescription:` | |
| <p><code>List[pyarrow.RecordBatch]</code></p> | |
| `}),r(2),o(Ue);var Pe=e(Ue,2),mo=t(Pe);a(mo,{name:"to_pydict",anchor:"datasets.table.ConcatenationTable.to_pydict",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L282",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnType:` | |
| <p><code>dict</code></p> | |
| `}),r(2),o(Pe);var Re=e(Pe,2),bo=t(Re);a(bo,{name:"to_pandas",anchor:"datasets.table.ConcatenationTable.to_pandas",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L300",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.to_pandas.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| Arrow MemoryPool to use for allocations. Uses the default memory | |
| pool is not passed.`,name:"memory_pool"},{anchor:"datasets.table.ConcatenationTable.to_pandas.strings_to_categorical",description:`<strong>strings_to_categorical</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Encode string (UTF8) and binary types to <code>pandas.Categorical</code>.`,name:"strings_to_categorical"},{anchor:"datasets.table.ConcatenationTable.to_pandas.categories",description:`<strong>categories</strong> (<code>list</code>, defaults to <code>empty</code>) — | |
| List of fields that should be returned as <code>pandas.Categorical</code>. Only | |
| applies to table-like data structures.`,name:"categories"},{anchor:"datasets.table.ConcatenationTable.to_pandas.zero_copy_only",description:`<strong>zero_copy_only</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Raise an <code>ArrowException</code> if this function call would require copying | |
| the underlying data.`,name:"zero_copy_only"},{anchor:"datasets.table.ConcatenationTable.to_pandas.integer_object_nulls",description:`<strong>integer_object_nulls</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast integers with nulls to objects.`,name:"integer_object_nulls"},{anchor:"datasets.table.ConcatenationTable.to_pandas.date_as_object",description:`<strong>date_as_object</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Cast dates to objects. If <code>False</code>, convert to <code>datetime64[ns]</code> dtype.`,name:"date_as_object"},{anchor:"datasets.table.ConcatenationTable.to_pandas.timestamp_as_object",description:`<strong>timestamp_as_object</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Cast non-nanosecond timestamps (<code>np.datetime64</code>) to objects. This is | |
| useful if you have timestamps that don’t fit in the normal date | |
| range of nanosecond timestamps (1678 CE-2262 CE). | |
| If <code>False</code>, all timestamps are converted to <code>datetime64[ns]</code> dtype.`,name:"timestamp_as_object"},{anchor:"datasets.table.ConcatenationTable.to_pandas.use_threads",description:`<strong>use_threads</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Whether to parallelize the conversion using multiple threads.`,name:"use_threads"},{anchor:"datasets.table.ConcatenationTable.to_pandas.deduplicate_objects",description:`<strong>deduplicate_objects</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Do not create multiple copies Python objects when created, to save | |
| on memory use. Conversion will be slower.`,name:"deduplicate_objects"},{anchor:"datasets.table.ConcatenationTable.to_pandas.ignore_metadata",description:`<strong>ignore_metadata</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, do not use the ‘pandas’ metadata to reconstruct the | |
| DataFrame index, if present.`,name:"ignore_metadata"},{anchor:"datasets.table.ConcatenationTable.to_pandas.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| For certain data types, a cast is needed in order to store the | |
| data in a pandas DataFrame or Series (e.g. timestamps are always | |
| stored as nanoseconds in pandas). This option controls whether it | |
| is a safe cast or not.`,name:"safe"},{anchor:"datasets.table.ConcatenationTable.to_pandas.split_blocks",description:`<strong>split_blocks</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, generate one internal “block” for each column when | |
| creating a pandas.DataFrame from a <code>RecordBatch</code> or <code>Table</code>. While this | |
| can temporarily reduce memory note that various pandas operations | |
| can trigger “consolidation” which may balloon memory use.`,name:"split_blocks"},{anchor:"datasets.table.ConcatenationTable.to_pandas.self_destruct",description:`<strong>self_destruct</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| EXPERIMENTAL: If <code>True</code>, attempt to deallocate the originating Arrow | |
| memory while converting the Arrow object to pandas. If you use the | |
| object after calling <code>to_pandas</code> with this option it will crash your | |
| program.`,name:"self_destruct"},{anchor:"datasets.table.ConcatenationTable.to_pandas.types_mapper",description:`<strong>types_mapper</strong> (<code>function</code>, defaults to <code>None</code>) — | |
| A function mapping a pyarrow DataType to a pandas <code>ExtensionDtype</code>. | |
| This can be used to override the default pandas type for conversion | |
| of built-in pyarrow types or in absence of <code>pandas_metadata</code> in the | |
| Table schema. The function receives a pyarrow DataType and is | |
| expected to return a pandas <code>ExtensionDtype</code> or <code>None</code> if the | |
| default conversion should be used for that type. If you have | |
| a dictionary mapping, you can pass <code>dict.get</code> as function.`,name:"types_mapper"}],returnDescription:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code> depending on type of object</p> | |
| `,returnType:` | |
| <p><code>pandas.Series</code> or <code>pandas.DataFrame</code></p> | |
| `}),r(2),o(Re);var qe=e(Re,2),uo=t(qe);a(uo,{name:"to_string",anchor:"datasets.table.ConcatenationTable.to_string",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L362",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),o(qe);var ze=e(qe,2),ho=t(ze);a(ho,{name:"field",anchor:"datasets.table.ConcatenationTable.field",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L381",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.field.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the field to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.Field</code></p> | |
| `}),r(2),o(ze);var Be=e(ze,2),go=t(Be);a(go,{name:"column",anchor:"datasets.table.ConcatenationTable.column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L394",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.column.i",description:`<strong>i</strong> (<code>Union[int, str]</code>) — | |
| The index or name of the column to retrieve.`,name:"i"}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(Be);var Oe=e(Be,2),yo=t(Oe);a(yo,{name:"itercolumns",anchor:"datasets.table.ConcatenationTable.itercolumns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L407",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],returnDescription:` | |
| <p><code>pyarrow.ChunkedArray</code></p> | |
| `}),r(2),o(Oe);var We=e(Oe,2),vo=t(We);a(vo,{name:"schema",anchor:"datasets.table.ConcatenationTable.schema",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L416",parameters:[],returnDescription:` | |
| <p><code>pyarrow.Schema</code></p> | |
| `}),r(2),o(We);var Xe=e(We,2),fo=t(Xe);a(fo,{name:"columns",anchor:"datasets.table.ConcatenationTable.columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L426",parameters:[],returnDescription:` | |
| <p><code>List[pa.ChunkedArray]</code></p> | |
| `}),r(2),o(Xe);var Ge=e(Xe,2),_o=t(Ge);a(_o,{name:"num_columns",anchor:"datasets.table.ConcatenationTable.num_columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L436",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(2),o(Ge);var Je=e(Ge,2),To=t(Je);a(To,{name:"num_rows",anchor:"datasets.table.ConcatenationTable.num_rows",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L446",parameters:[],returnDescription:` | |
| <p>int</p> | |
| `}),r(4),o(Je);var Ye=e(Je,2),wo=t(Ye);a(wo,{name:"shape",anchor:"datasets.table.ConcatenationTable.shape",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L459",parameters:[],returnDescription:` | |
| <p>Number of rows and number of columns.</p> | |
| `,returnType:` | |
| <p><code>(int, int)</code></p> | |
| `}),r(2),o(Ye);var Ke=e(Ye,2),xo=t(Ke);a(xo,{name:"nbytes",anchor:"datasets.table.ConcatenationTable.nbytes",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L469",parameters:[]}),r(2),o(Ke);var Ze=e(Ke,2),Mo=t(Ze);a(Mo,{name:"column_names",anchor:"datasets.table.ConcatenationTable.column_names",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L476",parameters:[]}),r(2),o(Ze);var He=e(Ze,2),ko=t(He);a(ko,{name:"slice",anchor:"datasets.table.ConcatenationTable.slice",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1548",parameters:[{name:"offset",val:" = 0"},{name:"length",val:" = None"}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.slice.offset",description:`<strong>offset</strong> (<code>int</code>, defaults to <code>0</code>) — | |
| Offset from start of table to slice.`,name:"offset"},{anchor:"datasets.table.ConcatenationTable.slice.length",description:`<strong>length</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Length of slice (default is until end of table starting from | |
| offset).`,name:"length"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(He);var $e=e(He,2),Do=t($e);a(Do,{name:"filter",anchor:"datasets.table.ConcatenationTable.filter",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1579",parameters:[{name:"mask",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}]}),r(2),o($e);var Qe=e($e,2),Co=t(Qe);a(Co,{name:"flatten",anchor:"datasets.table.ConcatenationTable.flatten",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1590",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.flatten.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| For memory allocations, if required, otherwise use default pool.`,name:"memory_pool"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(Qe);var Ve=e(Qe,2),Io=t(Ve);a(Io,{name:"combine_chunks",anchor:"datasets.table.ConcatenationTable.combine_chunks",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1608",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.combine_chunks.memory_pool",description:`<strong>memory_pool</strong> (<code>MemoryPool</code>, defaults to <code>None</code>) — | |
| For memory allocations, if required, otherwise use default pool.`,name:"memory_pool"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(4),o(Ve);var ea=e(Ve,2),Lo=t(ea);a(Lo,{name:"cast",anchor:"datasets.table.ConcatenationTable.cast",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1628",parameters:[{name:"target_schema",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.cast.target_schema",description:`<strong>target_schema</strong> (<code>Schema</code>) — | |
| Schema to cast to, the names and order of fields must match.`,name:"target_schema"},{anchor:"datasets.table.ConcatenationTable.cast.safe",description:`<strong>safe</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Check for overflows or other unsafe conversions.`,name:"safe"}],returnDescription:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(ea);var aa=e(ea,2),jo=t(aa);a(jo,{name:"replace_schema_metadata",anchor:"datasets.table.ConcatenationTable.replace_schema_metadata",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1659",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.replace_schema_metadata.metadata",description:"<strong>metadata</strong> (<code>dict</code>, defaults to <code>None</code>) —",name:"metadata"}],returnDescription:` | |
| <p>shallow_copy</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(aa);var ta=e(aa,2),Fo=t(ta);a(Fo,{name:"add_column",anchor:"datasets.table.ConcatenationTable.add_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1677",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.add_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index to place the column at.`,name:"i"},{anchor:"datasets.table.ConcatenationTable.add_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.ConcatenationTable.add_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column added.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(4),o(ta);var oa=e(ta,2),Ao=t(oa);a(Ao,{name:"append_column",anchor:"datasets.table.ConcatenationTable.append_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1698",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.append_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.ConcatenationTable.append_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column added.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(oa);var ra=e(oa,2),No=t(ra);a(No,{name:"remove_column",anchor:"datasets.table.ConcatenationTable.remove_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1715",parameters:[{name:"i",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.remove_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index of column to remove.`,name:"i"}],returnDescription:` | |
| <p>New table without the column.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(ra);var na=e(ra,2),Eo=t(na);a(Eo,{name:"set_column",anchor:"datasets.table.ConcatenationTable.set_column",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1739",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.set_column.i",description:`<strong>i</strong> (<code>int</code>) — | |
| Index to place the column at.`,name:"i"},{anchor:"datasets.table.ConcatenationTable.set_column.field_",description:`<strong>field_</strong> (<code>Union[str, pyarrow.Field]</code>) — | |
| If a string is passed then the type is deduced from the column | |
| data.`,name:"field_"},{anchor:"datasets.table.ConcatenationTable.set_column.column",description:`<strong>column</strong> (<code>Union[pyarrow.Array, List[pyarrow.Array]]</code>) — | |
| Column data.`,name:"column"}],returnDescription:` | |
| <p>New table with the passed column set.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(na);var sa=e(na,2),So=t(sa);a(So,{name:"rename_columns",anchor:"datasets.table.ConcatenationTable.rename_columns",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1758",parameters:[{name:"names",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}]}),r(2),o(sa);var da=e(sa,2),Uo=t(da);a(Uo,{name:"select",anchor:"datasets.table.ConcatenationTable.select",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1792",parameters:[{name:"columns",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.select.columns",description:`<strong>columns</strong> (<code>Union[List[str], List[int]]</code>) — | |
| The column names or integer indices to select.`,name:"columns"}],returnDescription:` | |
| <p>New table with the specified columns, and metadata preserved.</p> | |
| `,returnType:` | |
| <p><a | |
| href="/docs/datasets/pr_8983/en/package_reference/table_classes#datasets.table.Table" | |
| >datasets.table.Table</a></p> | |
| `}),r(4),o(da);var ca=e(da,2),Po=t(ca);a(Po,{name:"drop",anchor:"datasets.table.ConcatenationTable.drop",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1771",parameters:[{name:"columns",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.drop.columns",description:`<strong>columns</strong> (<code>List[str]</code>) — | |
| List of field names referencing existing columns.`,name:"columns"}],returnDescription:` | |
| <p>New table without the columns.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `,raiseDescription:` | |
| <ul> | |
| <li><code>KeyError</code> — : if any of the passed columns name are not existing.</li> | |
| </ul> | |
| `,raiseType:` | |
| <p><code>KeyError</code></p> | |
| `}),r(2),o(ca);var la=e(ca,2),Ro=t(la);a(Ro,{name:"from_blocks",anchor:"datasets.table.ConcatenationTable.from_blocks",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1444",parameters:[{name:"blocks",val:": ~TableBlockContainer"}]}),o(la);var ja=e(la,2),qo=t(ja);a(qo,{name:"from_tables",anchor:"datasets.table.ConcatenationTable.from_tables",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1458",parameters:[{name:"tables",val:": list"},{name:"axis",val:": int = 0"}],parametersDescription:[{anchor:"datasets.table.ConcatenationTable.from_tables.tables",description:`<strong>tables</strong> (list of <code>Table</code> or list of <code>pyarrow.Table</code>) — | |
| List of tables.`,name:"tables"},{anchor:"datasets.table.ConcatenationTable.from_tables.axis",description:`<strong>axis</strong> (<code>{0, 1}</code>, defaults to <code>0</code>, meaning over rows) — | |
| Axis to concatenate over, where <code>0</code> means over rows (vertically) and <code>1</code> means over columns | |
| (horizontally). | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 1.6.0</p> | |
| </div>`,name:"axis"}]}),r(2),o(ja),o(Ne);var Fa=e(Ne,2);n(Fa,{title:"Utils",local:"datasets.table.concat_tables",headingTag:"h2"});var ia=e(Fa,2),zo=t(ia);a(zo,{name:"datasets.table.concat_tables",anchor:"datasets.table.concat_tables",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1812",parameters:[{name:"tables",val:": list"},{name:"axis",val:": int = 0"}],parametersDescription:[{anchor:"datasets.table.concat_tables.tables",description:`<strong>tables</strong> (list of <code>Table</code>) — | |
| List of tables to be concatenated.`,name:"tables"},{anchor:"datasets.table.concat_tables.axis",description:`<strong>axis</strong> (<code>{0, 1}</code>, defaults to <code>0</code>, meaning over rows) — | |
| Axis to concatenate over, where <code>0</code> means over rows (vertically) and <code>1</code> means over columns | |
| (horizontally). | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 1.6.0</p> | |
| </div>`,name:"axis"}],returnDescription:` | |
| <p>If the number of input tables is > 1, then the returned table is a <code>datasets.table.ConcatenationTable</code>. | |
| Otherwise if there’s only one table, it is returned as is.</p> | |
| `,returnType:` | |
| <p><code>datasets.table.Table</code></p> | |
| `}),r(2),o(ia);var pa=e(ia,2),Bo=t(pa);a(Bo,{name:"datasets.table.list_table_cache_files",anchor:"datasets.table.list_table_cache_files",source:"https://github.com/huggingface/datasets/blob/r_8983/src/datasets/table.py#L1835",parameters:[{name:"table",val:": Table"}],returnDescription:` | |
| <p>A list of paths to the cache files loaded by the table.</p> | |
| `,returnType:` | |
| <p><code>List[str]</code></p> | |
| `}),r(2),o(pa);var Oo=e(pa,2);Yo(Oo,{source:"https://github.com/huggingface/datasets/blob/main/docs/source/package_reference/table_classes.mdx"}),r(2),ba(Ea,ha),Qo()}export{cr as component}; | |
Xet Storage Details
- Size:
- 111 kB
- Xet hash:
- a344b9694a09315f7d15eea391f3d8f8e9a2237818f968cd0c4ebcb95bb8de66
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.