Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_variable(self, name, expression, overwrite=True, unique=True):
        if unique or overwrite or name not in self.variables:
            existing_names = self.get_column_names(virtual=False) + list(self.variables.keys())
            name = vaex.utils.find_valid_name(name, used=[] if not unique else existing_names)
            self.variables[name] = expression
            self.signal_variable_changed.emit(self, name, "add")
            if unique:
                return name

Response: Add a virtual variable to a DataFrame.


Instruction:
Can you tell what is the following Python 3 function doing
def delete_variable(self, name):
        del self.variables[name]
        self.signal_variable_changed.emit(self, name, "delete")

Response: Deletes a variable from a DataFrame.


Instruction:
Make a summary of the following Python 3 code
def tail(self, n=10):
        N = len(self)
        # self.cat(i1=max(0, N-n), i2=min(len(self), N))
        return self[max(0, N - n):min(len(self), N)]

Response: Return a shallow copy a DataFrame with the last n rows.


Instruction:
Write a Python 3 function for
displaying the first and last n elements of a DataFrame.

Response:
def head_and_tail_print(self, n=5):
        """Display the first and last n elements of a DataFrame."""
        from IPython import display
        display.display(display.HTML(self._head_and_tail_table(n)))


Instruction:
Create a Python 3 function to
give a description of the DataFrame.

Response:
def describe(self, strings=True, virtual=True, selection=None):
        """Give a description of the DataFrame.

        >>> import vaex
        >>> df = vaex.example()[['x', 'y', 'z']]
        >>> df.describe()
                         x          y          z
        dtype      float64    float64    float64
        count       330000     330000     330000
        missing          0          0          0
        mean    -0.0671315 -0.0535899  0.0169582
        std        7.31746    7.78605    5.05521
        min       -128.294   -71.5524   -44.3342
        max        271.366    146.466    50.7185
        >>> df.describe(selection=df.x > 0)
                           x         y          z
        dtype        float64   float64    float64
        count         164060    164060     164060
        missing       165940    165940     165940
        mean         5.13572 -0.486786 -0.0868073
        std          5.18701   7.61621    5.02831
        min      1.51635e-05  -71.5524   -44.3342
        max          271.366   78.0724    40.2191

        :param bool strings: Describe string columns or not
        :param bool virtual: Describe virtual columns or not
        :param selection: Optional selection to use.
        :return: Pandas dataframe

        """
        import pandas as pd
        N = len(self)
        columns = {}
        for feature in self.get_column_names(strings=strings, virtual=virtual)[:]:
            dtype = str(self.dtype(feature)) if self.dtype(feature) != str else 'str'
            if self.dtype(feature) == str_type or self.dtype(feature).kind in ['S', 'U', 'O']:
                count = self.count(feature, selection=selection, delay=True)
                self.execute()
                count = count.get()
                columns[feature] = ((dtype, count, N-count, '--', '--', '--', '--'))
            else:
                count = self.count(feature, selection=selection, delay=True)
                mean = self.mean(feature, selection=selection, delay=True)
                std = self.std(feature, selection=selection, delay=True)
                minmax = self.minmax(feature, selection=selection, delay=True)
                self.execute()
                count, mean, std, minmax = count.get(), mean.get(), std.get(), minmax.get()
                count = int(count)
                columns[feature] = ((dtype, count, N-count, mean, std, minmax[0], minmax[1]))
        return pd.DataFrame(data=columns, index=['dtype', 'count', 'missing', 'mean', 'std', 'min', 'max'])


Instruction:
Can you implement a function in Python 3 that
displays the DataFrame from row i1 till i2

Response:
def cat(self, i1, i2, format='html'):
        """Display the DataFrame from row i1 till i2

        For format, see https://pypi.org/project/tabulate/

        :param int i1: Start row
        :param int i2: End row.
        :param str format: Format to use, e.g. 'html', 'plain', 'latex'
        """
        from IPython import display
        if format == 'html':
            output = self._as_html_table(i1, i2)
            display.display(display.HTML(output))
        else:
            output = self._as_table(i1, i2, format=format)
            print(output)


Instruction:
Can you write a function in Python 3 where it
sets the current row and emit the signal signal_pick.

Response:
def set_current_row(self, value):
        """Set the current row, and emit the signal signal_pick."""
        if (value is not None) and ((value < 0) or (value >= len(self))):
            raise IndexError("index %d out of range [0,%d]" % (value, len(self)))
        self._current_row = value
        self.signal_pick.emit(self, value)


Instruction:
Write a Python 3 script to
return a list of column names for the current instance of a table.

Response:
def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None):
        """Return a list of column names

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']

        :param virtual: If False, skip virtual columns
        :param hidden: If False, skip hidden columns
        :param strings: If False, skip string columns
        :param regex: Only return column names matching the (optional) regular expression
        :rtype: list of str

        Example:
        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']
        """
        def column_filter(name):
            '''Return True if column with specified name should be returned'''
            if regex and not re.match(regex, name):
                return False
            if not virtual and name in self.virtual_columns:
                return False
            if not strings and (self.dtype(name) == str_type or self.dtype(name).type == np.string_):
                return False
            if not hidden and name.startswith('__'):
                return False
            return True
        return [name for name in self.column_names if column_filter(name)]


Instruction:
Create a Python 3 function for
setting the active_fraction set picked row to None remove selection.

Response:
def set_active_fraction(self, value):
        """Sets the active_fraction, set picked row to None, and remove selection.

        TODO: we may be able to keep the selection, if we keep the expression, and also the picked row
        """
        if value != self._active_fraction:
            self._active_fraction = value
            # self._fraction_length = int(self._length * self._active_fraction)
            self.select(None)
            self.set_current_row(None)
            self._length_unfiltered = int(round(self._length_original * self._active_fraction))
            self._index_start = 0
            self._index_end = self._length_unfiltered
            self.signal_active_fraction_changed.emit(self, value)


Instruction:
How would you explain what the following Python 3 function does
def set_active_range(self, i1, i2):
        logger.debug("set active range to: %r", (i1, i2))
        self._active_fraction = (i2 - i1) / float(self.length_original())
        # self._fraction_length = int(self._length * self._active_fraction)
        self._index_start = i1
        self._index_end = i2
        self.select(None)
        self.set_current_row(None)
        self._length_unfiltered = i2 - i1
        self.signal_active_fraction_changed.emit(self, self._active_fraction)

Response: Sets the active_fraction set picked row to None remove selection.


Instruction:
Write a Python 3 script to
return a new DataFrame where all columns are trimmed by the active range.

Response:
def trim(self, inplace=False):
        '''Return a DataFrame, where all columns are 'trimmed' by the active range.

        For the returned DataFrame, df.get_active_range() returns (0, df.length_original()).

        {note_copy}

        :param inplace: {inplace}
        :rtype: DataFrame
        '''
        df = self if inplace else self.copy()
        for name in df:
            column = df.columns.get(name)
            if column is not None:
                if self._index_start == 0 and len(column) == self._index_end:
                    pass  # we already assigned it in .copy
                else:
                    if isinstance(column, np.ndarray):  # real array
                        df.columns[name] = column[self._index_start:self._index_end]
                    else:
                        df.columns[name] = column.trim(self._index_start, self._index_end)
        df._length_original = self.length_unfiltered()
        df._length_unfiltered = df._length_original
        df._index_start = 0
        df._index_end = df._length_original
        df._active_fraction = 1
        return df


Instruction:
How would you code a function in Python 3 to
return a DataFrame containing only rows indexed by indices

Response:
def take(self, indices):
        '''Returns a DataFrame containing only rows indexed by indices

        {note_copy}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df.take([0,2])
         #  s      x
         0  a      1
         1  c      3

        :param indices: sequence (list or numpy array) with row numbers
        :return: DataFrame which is a shallow copy of the original data.
        :rtype: DataFrame
        '''
        df = self.copy()
        # if the columns in ds already have a ColumnIndex
        # we could do, direct_indices = df.column['bla'].indices[indices]
        # which should be shared among multiple ColumnIndex'es, so we store
        # them in this dict
        direct_indices_map = {}
        indices = np.array(indices)
        for name in df:
            column = df.columns.get(name)
            if column is not None:
                # we optimize this somewhere, so we don't do multiple
                # levels of indirection
                if isinstance(column, ColumnIndexed):
                    # TODO: think about what happpens when the indices are masked.. ?
                    if id(column.indices) not in direct_indices_map:
                        direct_indices = column.indices[indices]
                        direct_indices_map[id(column.indices)] = direct_indices
                    else:
                        direct_indices = direct_indices_map[id(column.indices)]
                    df.columns[name] = ColumnIndexed(column.df, direct_indices, column.name)
                else:
                    df.columns[name] = ColumnIndexed(self, indices, name)
        df._length_original = len(indices)
        df._length_unfiltered = df._length_original
        df.set_selection(None, name=FILTER_SELECTION_NAME)
        return df


Instruction:
Implement a Python 3 function for
returning a DataFrame containing only the filtered rows.

Response:
def extract(self):
        '''Return a DataFrame containing only the filtered rows.

        {note_copy}

        The resulting DataFrame may be more efficient to work with when the original DataFrame is
        heavily filtered (contains just a small number of rows).

        If no filtering is applied, it returns a trimmed view.
        For the returned df, len(df) == df.length_original() == df.length_unfiltered()

        :rtype: DataFrame
        '''
        trimmed = self.trim()
        if trimmed.filtered:
            indices = trimmed._filtered_range_to_unfiltered_indices(0, len(trimmed))
            return trimmed.take(indices)
        else:
            return trimmed


Instruction:
Make a summary of the following Python 3 code
def sample(self, n=None, frac=None, replace=False, weights=None, random_state=None):
        '''Returns a DataFrame with a random set of rows

        {note_copy}

        Provide either n or frac.

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df
          #  s      x
          0  a      1
          1  b      2
          2  c      3
          3  d      4
        >>> df.sample(n=2, random_state=42) # 2 random rows, fixed seed
          #  s      x
          0  b      2
          1  d      4
        >>> df.sample(frac=1, random_state=42) # 'shuffling'
          #  s      x
          0  c      3
          1  a      1
          2  d      4
          3  b      2
        >>> df.sample(frac=1, replace=True, random_state=42) # useful for bootstrap (may contain repeated samples)
          #  s      x
          0  d      4
          1  a      1
          2  a      1
          3  d      4

        :param int n: number of samples to take (default 1 if frac is None)
        :param float frac: fractional number of takes to take
        :param bool replace: If true, a row may be drawn multiple times
        :param str or expression weights: (unnormalized) probability that a row can be drawn
        :param int or RandomState: seed or RandomState for reproducability, when None a random seed it chosen
        :return: {return_shallow_copy}
        :rtype: DataFrame
        '''
        self = self.extract()
        if type(random_state) == int or random_state is None:
            random_state = np.random.RandomState(seed=random_state)
        if n is None and frac is None:
            n = 1
        elif frac is not None:
            n = int(round(frac * len(self)))
        weights_values = None
        if weights is not None:
            weights_values = self.evaluate(weights)
            weights_values = weights_values / self.sum(weights)
        indices = random_state.choice(len(self), n, replace=replace, p=weights_values)
        return self.take(indices)

Response: Returns a DataFrame with a random set of rows with optional fractional number of takes.


Instruction:
Write a Python 3 script to
return a list containing random portions of the DataFrame.

Response:
def split_random(self, frac, random_state=None):
        '''Returns a list containing random portions of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex, import numpy as np
        >>> np.random.seed(111)
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split_random(frac=0.3, random_state=42):
        ...     print(dfs.x.values)
        ...
        [8 1 5]
        [0 7 2 9 4 3 6]
        >>> for split in df.split_random(frac=[0.2, 0.3, 0.5], random_state=42):
        ...     print(dfs.x.values)
        [8 1]
        [5 0 7]
        [2 9 4 3 6]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :param int random_state: (default, None) Random number seed for reproducibility.
        :return: A list of DataFrames.
        :rtype: list
        '''
        self = self.extract()
        if type(random_state) == int or random_state is None:
            random_state = np.random.RandomState(seed=random_state)
        indices = random_state.choice(len(self), len(self), replace=False)
        return self.take(indices).split(frac)


Instruction:
Can you generate the documentation for the following Python 3 function
def split(self, frac):
        '''Returns a list containing ordered subsets of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split(frac=0.3):
        ...     print(dfs.x.values)
        ...
        [0 1 3]
        [3 4 5 6 7 8 9]
        >>> for split in df.split(frac=[0.2, 0.3, 0.5]):
        ...     print(dfs.x.values)
        [0 1]
        [2 3 4]
        [5 6 7 8 9]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :return: A list of DataFrames.
        :rtype: list
        '''
        self = self.extract()
        if _issequence(frac):
            # make sure it is normalized
            total = sum(frac)
            frac = [k / total for k in frac]
        else:
            assert frac <= 1, "fraction should be <= 1"
            frac = [frac, 1 - frac]
        offsets = np.round(np.cumsum(frac) * len(self)).astype(np.int64)
        start = 0
        for offset in offsets:
            yield self[start:offset]
            start = offset

Response: Returns a list of ordered subsets of the DataFrame.


Instruction:
Implement a function in Python 3 to
return a sorted DataFrame sorted by the expression by

Response:
def sort(self, by, ascending=True, kind='quicksort'):
        '''Return a sorted DataFrame, sorted by the expression 'by'

        {note_copy}

        {note_filter}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df['y'] = (df.x-1.8)**2
        >>> df
          #  s      x     y
          0  a      1  0.64
          1  b      2  0.04
          2  c      3  1.44
          3  d      4  4.84
        >>> df.sort('y', ascending=False)  # Note: passing '(x-1.8)**2' gives the same result
          #  s      x     y
          0  d      4  4.84
          1  c      3  1.44
          2  a      1  0.64
          3  b      2  0.04

        :param str or expression by: expression to sort by
        :param bool ascending: ascending (default, True) or descending (False)
        :param str kind: kind of algorithm to use (passed to numpy.argsort)
        '''
        self = self.trim()
        values = self.evaluate(by, filtered=False)
        indices = np.argsort(values, kind=kind)
        if not ascending:
            indices = indices[::-1].copy()  # this may be used a lot, so copy for performance
        return self.take(indices)


Instruction:
Can you generate the documentation for the following Python 3 function
def fillna(self, value, fill_nan=True, fill_masked=True, column_names=None, prefix='__original_', inplace=False):
        '''Return a DataFrame, where missing values/NaN are filled with 'value'.

        The original columns will be renamed, and by default they will be hidden columns. No data is lost.

        {note_copy}

        {note_filter}

        Example:

        >>> import vaex
        >>> import numpy as np
        >>> x = np.array([3, 1, np.nan, 10, np.nan])
        >>> df = vaex.from_arrays(x=x)
        >>> df_filled = df.fillna(value=-1, column_names=['x'])
        >>> df_filled
          #    x
          0    3
          1    1
          2   -1
          3   10
          4   -1

        :param float value: The value to use for filling nan or masked values.
        :param bool fill_na: If True, fill np.nan values with `value`.
        :param bool fill_masked: If True, fill masked values with `values`.
        :param list column_names: List of column names in which to fill missing values.
        :param str prefix: The prefix to give the original columns.
        :param inplace: {inplace}
        '''
        df = self.trim(inplace=inplace)
        column_names = column_names or list(self)
        for name in column_names:
            column = df.columns.get(name)
            if column is not None:
                new_name = df.rename_column(name, prefix + name)
                expr = df[new_name]
                df[name] = df.func.fillna(expr, value, fill_nan=fill_nan, fill_masked=fill_masked)
            else:
                df[name] = df.func.fillna(df[name], value, fill_nan=fill_nan, fill_masked=fill_masked)
        return df

Response: Return a DataFrame where missing values or NaN are filled with value.


Instruction:
Write a Python 3 function that can
return a new DataFrame where the virtual column is turned into an in memory numpy array.

Response:
def materialize(self, virtual_column, inplace=False):
        '''Returns a new DataFrame where the virtual column is turned into an in memory numpy array.

        Example:

        >>> x = np.arange(1,4)
        >>> y = np.arange(2,5)
        >>> df = vaex.from_arrays(x=x, y=y)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5 # 'r' is a virtual column (computed on the fly)
        >>> df = df.materialize('r')  # now 'r' is a 'real' column (i.e. a numpy array)

        :param inplace: {inplace}
        '''
        df = self.trim(inplace=inplace)
        virtual_column = _ensure_string_from_expression(virtual_column)
        if virtual_column not in df.virtual_columns:
            raise KeyError('Virtual column not found: %r' % virtual_column)
        ar = df.evaluate(virtual_column, filtered=False)
        del df[virtual_column]
        df.add_column(virtual_column, ar)
        return df


Instruction:
Write a Python 3 function that can
get the current selection object.

Response:
def get_selection(self, name="default"):
        """Get the current selection object (mostly for internal use atm)."""
        name = _normalize_selection_name(name)
        selection_history = self.selection_histories[name]
        index = self.selection_history_indices[name]
        if index == -1:
            return None
        else:
            return selection_history[index]


Instruction:
Can you create a Python 3 function that
undoes selection for the name.

Response:
def selection_undo(self, name="default", executor=None):
        """Undo selection, for the name."""
        logger.debug("undo")
        executor = executor or self.executor
        assert self.selection_can_undo(name=name)
        selection_history = self.selection_histories[name]
        index = self.selection_history_indices[name]
        self.selection_history_indices[name] -= 1
        self.signal_selection_changed.emit(self)
        logger.debug("undo: selection history is %r, index is %r", selection_history, self.selection_history_indices[name])


Instruction:
Write a Python 3 function that can
redo selection for the name.

Response:
def selection_redo(self, name="default", executor=None):
        """Redo selection, for the name."""
        logger.debug("redo")
        executor = executor or self.executor
        assert self.selection_can_redo(name=name)
        selection_history = self.selection_histories[name]
        index = self.selection_history_indices[name]
        next = selection_history[index + 1]
        self.selection_history_indices[name] += 1
        self.signal_selection_changed.emit(self)
        logger.debug("redo: selection history is %r, index is %r", selection_history, index)


Instruction:
Can you tell what is the following Python 3 function doing
def select(self, boolean_expression, mode="replace", name="default", executor=None):
        boolean_expression = _ensure_string_from_expression(boolean_expression)
        if boolean_expression is None and not self.has_selection(name=name):
            pass  # we don't want to pollute the history with many None selections
            self.signal_selection_changed.emit(self)  # TODO: unittest want to know, does this make sense?
        else:
            def create(current):
                return selections.SelectionExpression(boolean_expression, current, mode) if boolean_expression else None
            self._selection(create, name)

Response: Perform a selection of the entries in the history tree with the given boolean expression.


Instruction:
Explain what the following Python 3 code does
def select_non_missing(self, drop_nan=True, drop_masked=True, column_names=None, mode="replace", name="default"):
        column_names = column_names or self.get_column_names(virtual=False)

        def create(current):
            return selections.SelectionDropNa(drop_nan, drop_masked, column_names, current, mode)
        self._selection(create, name)

Response: Create a selection that selects rows having non missing values for all columns in column_names.


Instruction:
Here you have a function in Python 3, explain what it does
def dropna(self, drop_nan=True, drop_masked=True, column_names=None):
        copy = self.copy()
        copy.select_non_missing(drop_nan=drop_nan, drop_masked=drop_masked, column_names=column_names,
                                name=FILTER_SELECTION_NAME, mode='and')
        return copy

Response: Create a shallow copy of the DataFrame with drop_nan drop_masked and column_names set to None.


Instruction:
Implement a Python 3 function for
selecting a 2d rectangular box in the space given by x y bounds by limits.

Response:
def select_rectangle(self, x, y, limits, mode="replace", name="default"):
        """Select a 2d rectangular box in the space given by x and y, bounds by limits.

        Example:

        >>> df.select_box('x', 'y', [(0, 10), (0, 1)])

        :param x: expression for the x space
        :param y: expression fo the y space
        :param limits: sequence of shape [(x1, x2), (y1, y2)]
        :param mode:
        """
        self.select_box([x, y], limits, mode=mode, name=name)


Instruction:
How would you code a function in Python 3 to
select a n - dimensional rectangular box bounded by limits.

Response:
def select_box(self, spaces, limits, mode="replace", name="default"):
        """Select a n-dimensional rectangular box bounded by limits.

        The following examples are equivalent:

        >>> df.select_box(['x', 'y'], [(0, 10), (0, 1)])
        >>> df.select_rectangle('x', 'y', [(0, 10), (0, 1)])

        :param spaces: list of expressions
        :param limits: sequence of shape [(x1, x2), (y1, y2)]
        :param mode:
        :param name:
        :return:
        """
        sorted_limits = [(min(l), max(l)) for l in limits]
        expressions = ["((%s) >= %f) & ((%s) <= %f)" % (expression, lmin, expression, lmax) for
                       (expression, (lmin, lmax)) in zip(spaces, sorted_limits)]
        self.select("&".join(expressions), mode=mode, name=name)


Instruction:
Write a Python 3 function for
selecting a circular region centred on xc yc with a radius of r.

Response:
def select_circle(self, x, y, xc, yc, r, mode="replace", name="default", inclusive=True):
        """
        Select a circular region centred on xc, yc, with a radius of r.

        Example:

        >>> df.select_circle('x','y',2,3,1)

        :param x: expression for the x space
        :param y: expression for the y space
        :param xc: location of the centre of the circle in x
        :param yc: location of the centre of the circle in y
        :param r: the radius of the circle
        :param name: name of the selection
        :param mode:
        :return:
        """

        # expr = "({x}-{xc})**2 + ({y}-{yc})**2 <={r}**2".format(**locals())
        if inclusive:
            expr = (self[x] - xc)**2 + (self[y] - yc)**2 <= r**2
        else:
            expr = (self[x] - xc)**2 + (self[y] - yc)**2 < r**2

        self.select(boolean_expression=expr, mode=mode, name=name)


Instruction:
Here you have a function in Python 3, explain what it does
def select_ellipse(self, x, y, xc, yc, width, height, angle=0, mode="replace", name="default", radians=False, inclusive=True):

        # Computing the properties of the ellipse prior to selection
        if radians:
            pass
        else:
            alpha = np.deg2rad(angle)
        xr = width / 2
        yr = height / 2
        r = max(xr, yr)
        a = xr / r
        b = yr / r

        expr = "(({x}-{xc})*cos({alpha})+({y}-{yc})*sin({alpha}))**2/{a}**2 + (({x}-{xc})*sin({alpha})-({y}-{yc})*cos({alpha}))**2/{b}**2 <= {r}**2".format(**locals())

        if inclusive:
            expr = ((self[x] - xc) * np.cos(alpha) + (self[y] - yc) * np.sin(alpha))**2 / a**2 + ((self[x] - xc) * np.sin(alpha) - (self[y] - yc) * np.cos(alpha))**2 / b**2 <= r**2
        else:
            expr = ((self[x] - xc) * np.cos(alpha) + (self[y] - yc) * np.sin(alpha))**2 / a**2 + ((self[x] - xc) * np.sin(alpha) - (self[y] - yc) * np.cos(alpha))**2 / b**2 < r**2

        self.select(boolean_expression=expr, mode=mode, name=name)

Response: Select an ellipse centred on x y with a certain width height and angle.


Instruction:
Can you tell what is the following Python 3 function doing
def select_lasso(self, expression_x, expression_y, xsequence, ysequence, mode="replace", name="default", executor=None):

        def create(current):
            return selections.SelectionLasso(expression_x, expression_y, xsequence, ysequence, current, mode)
        self._selection(create, name, executor=executor)

Response: Selects a lasso from the database.


Instruction:
Can you generate the documentation for the following Python 3 function
def select_inverse(self, name="default", executor=None):

        def create(current):
            return selections.SelectionInvert(current)
        self._selection(create, name, executor=executor)

Response: Invert the selection i. e. what is selected will not be and vice versa


Instruction:
Can you generate the documentation for the following Python 3 function
def set_selection(self, selection, name="default", executor=None):
        def create(current):
            return selection
        self._selection(create, name, executor=executor, execute_fully=True)

Response: Sets the selection object


Instruction:
Make a summary of the following Python 3 code
def _selection(self, create_selection, name, executor=None, execute_fully=False):
        selection_history = self.selection_histories[name]
        previous_index = self.selection_history_indices[name]
        current = selection_history[previous_index] if selection_history else None
        selection = create_selection(current)
        executor = executor or self.executor
        selection_history.append(selection)
        self.selection_history_indices[name] += 1
        # clip any redo history
        del selection_history[self.selection_history_indices[name]:-1]
        if 0:
            if self.is_local():
                if selection:
                    # result = selection.execute(executor=executor, execute_fully=execute_fully)
                    result = vaex.promise.Promise.fulfilled(None)
                    self.signal_selection_changed.emit(self)
                else:
                    result = vaex.promise.Promise.fulfilled(None)
                    self.signal_selection_changed.emit(self)
            else:
                self.signal_selection_changed.emit(self)
                result = vaex.promise.Promise.fulfilled(None)
        self.signal_selection_changed.emit(self)
        result = vaex.promise.Promise.fulfilled(None)
        logger.debug("select selection history is %r, index is %r", selection_history, self.selection_history_indices[name])
        return result

Response: select_lasso and select almost share the same code


Instruction:
Can you tell what is the following Python 3 function doing
def drop(self, columns, inplace=False, check=True):
        columns = _ensure_list(columns)
        columns = _ensure_strings_from_expressions(columns)
        df = self if inplace else self.copy()
        depending_columns = df._depending_columns(columns_exclude=columns)
        for column in columns:
            if check and column in depending_columns:
                df._hide_column(column)
            else:
                del df[column]
        return df

Response: Drop columns or a single column.


Instruction:
Write a Python 3 script for
hiding a column by prefixing the name with __

Response:
def _hide_column(self, column):
        '''Hides a column by prefixing the name with \'__\''''
        column = _ensure_string_from_expression(column)
        new_name = self._find_valid_name('__' + column)
        self._rename(column, new_name)


Instruction:
How would you code a function in Python 3 to
find a non - colliding name by optional postfixing

Response:
def _find_valid_name(self, initial_name):
        '''Finds a non-colliding name by optional postfixing'''
        return vaex.utils.find_valid_name(initial_name, used=self.get_column_names(hidden=True))


Instruction:
Write a Python 3 function for
finding all depending column for a set of column names minus the excluded ones

Response:
def _depending_columns(self, columns=None, columns_exclude=None, check_filter=True):
        '''Find all depending column for a set of column (default all), minus the excluded ones'''
        columns = set(columns or self.get_column_names(hidden=True))
        if columns_exclude:
            columns -= set(columns_exclude)
        depending_columns = set()
        for column in columns:
            expression = self._expr(column)
            depending_columns |= expression.variables()
        depending_columns -= set(columns)
        if check_filter:
            if self.filtered:
                selection = self.get_selection(FILTER_SELECTION_NAME)
                depending_columns |= selection._depending_columns(self)
        return depending_columns


Instruction:
Can you generate the documentation for the following Python 3 function
def _root_nodes(self):
        # these lists (~used as ordered set) keep track of leafes and root nodes
        # root nodes
        root_nodes = []
        leafes = []
        def walk(node):
            # this function recursively walks the expression graph
            if isinstance(node, six.string_types):
                # we end up at a leaf
                leafes.append(node)
                if node in root_nodes:  # so it cannot be a root node
                    root_nodes.remove(node)
            else:
                node_repr, fname, fobj, deps = node
                if node_repr in self.virtual_columns:
                    # we encountered a virtual column, similar behaviour as leaf
                    leafes.append(node_repr)
                    if node_repr in root_nodes:
                        root_nodes.remove(node_repr)
                # resursive part
                for dep in deps:
                    walk(dep)
        for column in self.virtual_columns.keys():
            if column not in leafes:
                root_nodes.append(column)
            node = self[column]._graph()
            # we don't do the virtual column itself, just it's depedencies
            node_repr, fname, fobj, deps = node
            for dep in deps:
                walk(dep)
        return root_nodes

Response: Returns a list of string which are the virtual columns that are not used in any other virtual column.


Instruction:
Write a Python 3 script to
return a graphviz. Digraph object with a graph of all virtual columns

Response:
def _graphviz(self, dot=None):
        """Return a graphviz.Digraph object with a graph of all virtual columns"""
        from graphviz import Digraph
        dot = dot or Digraph(comment='whole dataframe')
        root_nodes = self._root_nodes()
        for column in root_nodes:
            self[column]._graphviz(dot=dot)
        return dot


Instruction:
Implement a Python 3 function for
marking the given column as categorical with given labels assuming zero indexing.

Response:
def categorize(self, column, labels=None, check=True):
        """Mark column as categorical, with given labels, assuming zero indexing"""
        column = _ensure_string_from_expression(column)
        if check:
            vmin, vmax = self.minmax(column)
            if labels is None:
                N = int(vmax + 1)
                labels = list(map(str, range(N)))
            if (vmax - vmin) >= len(labels):
                raise ValueError('value of {} found, which is larger than number of labels {}'.format(vmax, len(labels)))
        self._categories[column] = dict(labels=labels, N=len(labels))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ordinal_encode(self, column, values=None, inplace=False):
        column = _ensure_string_from_expression(column)
        df = self if inplace else self.copy()
        # for the codes, we need to work on the unfiltered dataset, since the filter
        # may change, and we also cannot add an array that is smaller in length
        df_unfiltered = df.copy()
        # maybe we need some filter manipulation methods
        df_unfiltered.select_nothing(name=FILTER_SELECTION_NAME)
        df_unfiltered._length_unfiltered = df._length_original
        df_unfiltered.set_active_range(0, df._length_original)
        # codes point to the index of found_values
        # meaning: found_values[codes[0]] == ds[column].values[0]
        found_values, codes = df_unfiltered.unique(column, return_inverse=True)
        if values is None:
            values = found_values
        else:
            # we have specified which values we should support, anything
            # not found will be masked
            translation = np.zeros(len(found_values), dtype=np.uint64)
            # mark values that are in the column, but not in values with a special value
            missing_value = len(found_values)
            for i, found_value in enumerate(found_values):
                try:
                    found_value = found_value.decode('ascii')
                except:
                    pass
                if found_value not in values:  # not present, we need a missing value
                    translation[i] = missing_value
                else:
                    translation[i] = values.index(found_value)
            codes = translation[codes]
            if missing_value in translation:
                # all special values will be marked as missing
                codes = np.ma.masked_array(codes, codes==missing_value)

        original_column = df.rename_column(column, '__original_' + column, unique=True)
        labels = [str(k) for k in values]
        df.add_column(column, codes)
        df._categories[column] = dict(labels=labels, N=len(values), values=values)
        return df

Response: Encode column as ordinal values and mark it as categorical.


Instruction:
Can you tell what is the following Python 3 function doing
def data(self):
        class Datas(object):
            pass

        datas = Datas()
        for name, array in self.columns.items():
            setattr(datas, name, array)
        return datas

Response: Gives direct access to the data as numpy arrays.


Instruction:
Implement a function in Python 3 to
create a shallow copy of the DataFrame.

Response:
def shallow_copy(self, virtual=True, variables=True):
        """Creates a (shallow) copy of the DataFrame.

        It will link to the same data, but will have its own state, e.g. virtual columns, variables, selection etc.

        """
        df = DataFrameLocal(self.name, self.path, self.column_names)
        df.columns.update(self.columns)
        df._length_unfiltered = self._length_unfiltered
        df._length_original = self._length_original
        df._index_end = self._index_end
        df._index_start = self._index_start
        df._active_fraction = self._active_fraction
        if virtual:
            df.virtual_columns.update(self.virtual_columns)
        if variables:
            df.variables.update(self.variables)
        # half shallow/deep copy
        # for key, value in self.selection_histories.items():
        # df.selection_histories[key] = list(value)
        # for key, value in self.selection_history_indices.items():
        # df.selection_history_indices[key] = value
        return df


Instruction:
Given the following Python 3 function, write the documentation
def length(self, selection=False):
        if selection:
            return 0 if self.mask is None else np.sum(self.mask)
        else:
            return len(self)

Response: Get the length of the whole DataFrame for the selection of the whole DataFrame.


Instruction:
Write a Python 3 script for
joining the columns of the other DataFrame to this DataFrame and add them to the self.

Response:
def _hstack(self, other, prefix=None):
        """Join the columns of the other DataFrame to this one, assuming the ordering is the same"""
        assert len(self) == len(other), "does not make sense to horizontally stack DataFrames with different lengths"
        for name in other.get_column_names():
            if prefix:
                new_name = prefix + name
            else:
                new_name = name
            self.add_column(new_name, other.columns[name])


Instruction:
Can you create a Python 3 function that
concatenates two DataFrames adding the rows of one the other DataFrame to the current DataFrame.

Response:
def concat(self, other):
        """Concatenates two DataFrames, adding the rows of one the other DataFrame to the current, returned in a new DataFrame.

        No copy of the data is made.

        :param other: The other DataFrame that is concatenated with this DataFrame
        :return: New DataFrame with the rows concatenated
        :rtype: DataFrameConcatenated
        """
        dfs = []
        if isinstance(self, DataFrameConcatenated):
            dfs.extend(self.dfs)
        else:
            dfs.extend([self])
        if isinstance(other, DataFrameConcatenated):
            dfs.extend(other.dfs)
        else:
            dfs.extend([other])
        return DataFrameConcatenated(dfs)


Instruction:
Make a summary of the following Python 3 code
def evaluate(self, expression, i1=None, i2=None, out=None, selection=None, filtered=True, internal=False):
        expression = _ensure_string_from_expression(expression)
        selection = _ensure_strings_from_expressions(selection)
        i1 = i1 or 0
        i2 = i2 or (len(self) if (self.filtered and filtered) else self.length_unfiltered())
        mask = None

        if self.filtered and filtered:  # if we filter, i1:i2 has a different meaning
            indices = self._filtered_range_to_unfiltered_indices(i1, i2)
            i1 = indices[0]
            i2 = indices[-1] + 1  # +1 to make it inclusive
        # for both a selection or filtering we have a mask
        if selection is not None or (self.filtered and filtered):
            mask = self.evaluate_selection_mask(selection, i1, i2)
        scope = scopes._BlockScope(self, i1, i2, mask=mask, **self.variables)
        # value = value[mask]
        if out is not None:
            scope.buffers[expression] = out
        value = scope.evaluate(expression)
        if isinstance(value, ColumnString) and not internal:
            value = value.to_numpy()
        return value

Response: Evaluate the expression and return the result as a numpy array.


Instruction:
Implement a function in Python 3 to
compare two DataFrames and report their difference

Response:
def compare(self, other, report_missing=True, report_difference=False, show=10, orderby=None, column_names=None):
        """Compare two DataFrames and report their difference, use with care for large DataFrames"""
        if column_names is None:
            column_names = self.get_column_names(virtual=False)
            for other_column_name in other.get_column_names(virtual=False):
                if other_column_name not in column_names:
                    column_names.append(other_column_name)
        different_values = []
        missing = []
        type_mismatch = []
        meta_mismatch = []
        assert len(self) == len(other)
        if orderby:
            index1 = np.argsort(self.columns[orderby])
            index2 = np.argsort(other.columns[orderby])
        for column_name in column_names:
            if column_name not in self.get_column_names(virtual=False):
                missing.append(column_name)
                if report_missing:
                    print("%s missing from this DataFrame" % column_name)
            elif column_name not in other.get_column_names(virtual=False):
                missing.append(column_name)
                if report_missing:
                    print("%s missing from other DataFrame" % column_name)
            else:
                ucd1 = self.ucds.get(column_name)
                ucd2 = other.ucds.get(column_name)
                if ucd1 != ucd2:
                    print("ucd mismatch : %r vs %r for %s" % (ucd1, ucd2, column_name))
                    meta_mismatch.append(column_name)
                unit1 = self.units.get(column_name)
                unit2 = other.units.get(column_name)
                if unit1 != unit2:
                    print("unit mismatch : %r vs %r for %s" % (unit1, unit2, column_name))
                    meta_mismatch.append(column_name)
                type1 = self.dtype(column_name)
                if type1 != str_type:
                    type1 = type1.type
                type2 = other.dtype(column_name)
                if type2 != str_type:
                    type2 = type2.type
                if type1 != type2:
                    print("different dtypes: %s vs %s for %s" % (self.dtype(column_name), other.dtype(column_name), column_name))
                    type_mismatch.append(column_name)
                else:
                    # a = self.columns[column_name]
                    # b = other.columns[column_name]
                    # if self.filtered:
                    #   a = a[self.evaluate_selection_mask(None)]
                    # if other.filtered:
                    #   b = b[other.evaluate_selection_mask(None)]
                    a = self.evaluate(column_name)
                    b = other.evaluate(column_name)
                    if orderby:
                        a = a[index1]
                        b = b[index2]

                    def normalize(ar):
                        if ar.dtype == str_type:
                            return ar
                        if ar.dtype.kind == "f" and hasattr(ar, "mask"):
                            mask = ar.mask
                            ar = ar.copy()
                            ar[mask] = np.nan
                        if ar.dtype.kind in "SU":
                            if hasattr(ar, "mask"):
                                data = ar.data
                            else:
                                data = ar
                            values = [value.strip() for value in data.tolist()]
                            if hasattr(ar, "mask"):
                                ar = np.ma.masked_array(values, ar.mask)
                            else:
                                ar = np.array(values)
                        return ar

                    def equal_mask(a, b):
                        a = normalize(a)
                        b = normalize(b)
                        boolean_mask = (a == b)
                        if self.dtype(column_name) != str_type and self.dtype(column_name).kind == 'f':  # floats with nan won't equal itself, i.e. NaN != NaN
                            boolean_mask |= (np.isnan(a) & np.isnan(b))
                        return boolean_mask
                    boolean_mask = equal_mask(a, b)
                    all_equal = np.all(boolean_mask)
                    if not all_equal:
                        count = np.sum(~boolean_mask)
                        print("%s does not match for both DataFrames, %d rows are diffent out of %d" % (column_name, count, len(self)))
                        different_values.append(column_name)
                        if report_difference:
                            indices = np.arange(len(self))[~boolean_mask]
                            values1 = self.columns[column_name][:][~boolean_mask]
                            values2 = other.columns[column_name][:][~boolean_mask]
                            print("\tshowing difference for the first 10")
                            for i in range(min(len(values1), show)):
                                try:
                                    diff = values1[i] - values2[i]
                                except:
                                    diff = "does not exists"
                                print("%s[%d] == %s != %s other.%s[%d] (diff = %s)" % (column_name, indices[i], values1[i], values2[i], column_name, indices[i], diff))
        return different_values, missing, type_mismatch, meta_mismatch


Instruction:
Can you generate the documentation for the following Python 3 function
def join(self, other, on=None, left_on=None, right_on=None, lsuffix='', rsuffix='', how='left', inplace=False):
        ds = self if inplace else self.copy()
        if how == 'left':
            left = ds
            right = other
        elif how == 'right':
            left = other
            right = ds
            lsuffix, rsuffix = rsuffix, lsuffix
            left_on, right_on = right_on, left_on
        else:
            raise ValueError('join type not supported: {}, only left and right'.format(how))

        for name in right:
            if name in left and name + rsuffix == name + lsuffix:
                raise ValueError('column name collision: {} exists in both column, and no proper suffix given'
                                 .format(name))

        right = right.extract()  # get rid of filters and active_range
        assert left.length_unfiltered() == left.length_original()
        N = left.length_unfiltered()
        N_other = len(right)
        left_on = left_on or on
        right_on = right_on or on
        if left_on is None and right_on is None:
            for name in right:
                right_name = name
                if name in left:
                    left.rename_column(name, name + lsuffix)
                    right_name = name + rsuffix
                if name in right.virtual_columns:
                    left.add_virtual_column(right_name, right.virtual_columns[name])
                else:
                    left.add_column(right_name, right.columns[name])
        else:
            left_values = left.evaluate(left_on, filtered=False)
            right_values = right.evaluate(right_on)
            # maps from the left_values to row #
            if np.ma.isMaskedArray(left_values):
                mask = ~left_values.mask
                left_values = left_values.data
                index_left = dict(zip(left_values[mask], np.arange(N)[mask]))
            else:
                index_left = dict(zip(left_values, np.arange(N)))
            # idem for right
            if np.ma.isMaskedArray(right_values):
                mask = ~right_values.mask
                right_values = right_values.data
                index_other = dict(zip(right_values[mask], np.arange(N_other)[mask]))
            else:
                index_other = dict(zip(right_values, np.arange(N_other)))

            # we do a left join, find all rows of the right DataFrame
            # that has an entry on the left
            # for each row in the right
            # find which row it needs to go to in the right
            # from_indices = np.zeros(N_other, dtype=np.int64)  # row # of right
            # to_indices = np.zeros(N_other, dtype=np.int64)    # goes to row # on the left
            # keep a boolean mask of which rows are found
            left_mask = np.ones(N, dtype=np.bool)
            # and which row they point to in the right
            left_row_to_right = np.zeros(N, dtype=np.int64) - 1
            for i in range(N_other):
                left_row = index_left.get(right_values[i])
                if left_row is not None:
                    left_mask[left_row] = False  # unmask, it exists
                    left_row_to_right[left_row] = i

            lookup = np.ma.array(left_row_to_right, mask=left_mask)
            for name in right:
                right_name = name
                if name in left:
                    left.rename_column(name, name + lsuffix)
                    right_name = name + rsuffix
                if name in right.virtual_columns:
                    left.add_virtual_column(right_name, right.virtual_columns[name])
                else:
                    left.add_column(right_name, ColumnIndexed(right, lookup, name))
        return left

Response: Returns a new DataFrame with the same data and other.


Instruction:
How would you code a function in Python 3 to
export the DataFrame to a file

Response:
def export(self, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):
        """Exports the DataFrame to a file written with arrow

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param str byteorder: = for native, < for little endian and > for big endian (not supported for fits)
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:
        """
        if path.endswith('.arrow'):
            self.export_arrow(path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)
        elif path.endswith('.hdf5'):
            self.export_hdf5(path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)
        elif path.endswith('.fits'):
            self.export_fits(path, column_names, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)
        if path.endswith('.parquet'):
            self.export_parquet(path, column_names, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)


Instruction:
Can you tell what is the following Python 3 function doing
def export_arrow(self, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):
        import vaex_arrow.export
        vaex_arrow.export.export(self, path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)

Response: Exports the DataFrame to a file written with arrow.


Instruction:
Write a Python 3 script to
export the DataFrame to a vaex hdf5 file

Response:
def export_hdf5(self, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):
        """Exports the DataFrame to a vaex hdf5 file

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param str byteorder: = for native, < for little endian and > for big endian
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:
        """
        import vaex.export
        vaex.export.export_hdf5(self, path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)


Instruction:
Write a Python 3 script to
return a DataFrame object when groupby is not None

Response:
def groupby(self, by=None, agg=None):
        """Return a :class:`GroupBy` or :class:`DataFrame` object when agg is not None

        Examples:

        >>> import vaex
        >>> import numpy as np
        >>> np.random.seed(42)
        >>> x = np.random.randint(1, 5, 10)
        >>> y = x**2
        >>> df = vaex.from_arrays(x=x, y=y)
        >>> df.groupby(df.x, agg='count')
        #    x    y_count
        0    3          4
        1    4          2
        2    1          3
        3    2          1
        >>> df.groupby(df.x, agg=[vaex.agg.count('y'), vaex.agg.mean('y')])
        #    x    y_count    y_mean
        0    3          4         9
        1    4          2        16
        2    1          3         1
        3    2          1         4
        >>> df.groupby(df.x, agg={'z': [vaex.agg.count('y'), vaex.agg.mean('y')]})
        #    x    z_count    z_mean
        0    3          4         9
        1    4          2        16
        2    1          3         1
        3    2          1         4

        Example using datetime:

        >>> import vaex
        >>> import numpy as np
        >>> t = np.arange('2015-01-01', '2015-02-01', dtype=np.datetime64)
        >>> y = np.arange(len(t))
        >>> df = vaex.from_arrays(t=t, y=y)
        >>> df.groupby(vaex.BinnerTime.per_week(df.t)).agg({'y' : 'sum'})
        #  t                      y
        0  2015-01-01 00:00:00   21
        1  2015-01-08 00:00:00   70
        2  2015-01-15 00:00:00  119
        3  2015-01-22 00:00:00  168
        4  2015-01-29 00:00:00   87

        :param dict, list or agg agg: Aggregate operation in the form of a string, vaex.agg object, a dictionary
            where the keys indicate the target column names, and the values the operations, or the a list of aggregates.
            When not given, it will return the groupby object.
        :return: :class:`DataFrame` or :class:`GroupBy` object.
        """
        from .groupby import GroupBy
        groupby = GroupBy(self, by=by)
        if agg is None:
            return groupby
        else:
            return groupby.agg(agg)


Instruction:
Create a Python 3 function to
return a BinBy object when agg is not None

Response:
def binby(self, by=None, agg=None):
        """Return a :class:`BinBy` or :class:`DataArray` object when agg is not None

        The binby operations does not return a 'flat' DataFrame, instead it returns an N-d grid
        in the form of an xarray.

        :param dict, list or agg agg: Aggregate operation in the form of a string, vaex.agg object, a dictionary
            where the keys indicate the target column names, and the values the operations, or the a list of aggregates.
            When not given, it will return the binby object.
        :return: :class:`DataArray` or :class:`BinBy` object.
        """
        from .groupby import BinBy
        binby = BinBy(self, by=by)
        if agg is None:
            return binby
        else:
            return binby.agg(agg)


Instruction:
Write a Python 3 script for
adding a column to the DataFrame

Response:
def add_column(self, name, data):
        """Add a column to the DataFrame

        :param str name: name of column
        :param data: numpy array with the data
        """
        # assert _is_array_type_ok(data), "dtype not supported: %r, %r" % (data.dtype, data.dtype.type)
        # self._length = len(data)
        # if self._length_unfiltered is None:
        #     self._length_unfiltered = len(data)
        #     self._length_original = len(data)
        #     self._index_end = self._length_unfiltered
        super(DataFrameArrays, self).add_column(name, data)
        self._length_unfiltered = int(round(self._length_original * self._active_fraction))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def then(self, success=None, failure=None):
        ret = self.create_next()

        def callAndFulfill(v):
            """
            A callback to be invoked if the "self promise"
            is fulfilled.
            """
            try:
                if aplus._isFunction(success):
                    ret.fulfill(success(v))
                else:
                    ret.fulfill(v)
            except Exception as e:
                Promise.last_exc_info = sys.exc_info()
                e.exc_info = sys.exc_info()
                ret.reject(e)

        def callAndReject(r):
            """
            A callback to be invoked if the "self promise"
            is rejected.
            """
            try:
                if aplus._isFunction(failure):
                    ret.fulfill(failure(r))
                else:
                    ret.reject(r)
            except Exception as e:
                Promise.last_exc_info = sys.exc_info()
                e.exc_info = sys.exc_info()
                ret.reject(e)

        self.done(callAndFulfill, callAndReject)

        return ret

Response: A method that returns a promise that resolves with the value of the next entry in the cache.


Instruction:
Make a summary of the following Python 3 code
def patch(f):
    '''Adds method f to the DataFrame class'''
    name = f.__name__
    setattr(DataFrame, name, f)
    return f

Response: Adds a method f to the DataFrame class


Instruction:
Given the following Python 3 function, write the documentation
def plot1d(self, x=None, what="count(*)", grid=None, shape=64, facet=None, limits=None, figsize=None, f="identity", n=None, normalize_axis=None,
           xlabel=None, ylabel=None, label=None,
           selection=None, show=False, tight_layout=True, hardcopy=None,
           **kwargs):
    """Viz data in 1d (histograms, running means etc)

    Example

    >>> df.plot1d(df.x)
    >>> df.plot1d(df.x, limits=[0, 100], shape=100)
    >>> df.plot1d(df.x, what='mean(y)', limits=[0, 100], shape=100)

    If you want to do a computation yourself, pass the grid argument, but you are responsible for passing the
    same limits arguments:

    >>> counts = df.mean(df.y, binby=df.x, limits=[0, 100], shape=100)/100.
    >>> df.plot1d(df.x, limits=[0, 100], shape=100, grid=means, label='mean(y)/100')

    :param x: Expression to bin in the x direction
    :param what: What to plot, count(*) will show a N-d histogram, mean('x'), the mean of the x column, sum('x') the sum
    :param grid: If the binning is done before by yourself, you can pass it
    :param facet: Expression to produce facetted plots ( facet='x:0,1,12' will produce 12 plots with x in a range between 0 and 1)
    :param limits: list of [xmin, xmax], or a description such as 'minmax', '99%'
    :param figsize: (x, y) tuple passed to pylab.figure for setting the figure size
    :param f: transform values by: 'identity' does nothing 'log' or 'log10' will show the log of the value
    :param n: normalization function, currently only 'normalize' is supported, or None for no normalization
    :param normalize_axis: which axes to normalize on, None means normalize by the global maximum.
    :param normalize_axis:
    :param xlabel: String for label on x axis (may contain latex)
    :param ylabel: Same for y axis
    :param: tight_layout: call pylab.tight_layout or not
    :param kwargs: extra argument passed to pylab.plot
    :return:
    """

    import pylab
    f = _parse_f(f)
    n = _parse_n(n)
    if type(shape) == int:
        shape = (shape,)
    binby = []
    x = _ensure_strings_from_expressions(x)
    for expression in [x]:
        if expression is not None:
            binby = [expression] + binby
    limits = self.limits(binby, limits)
    if figsize is not None:
        pylab.figure(num=None, figsize=figsize, dpi=80, facecolor='w', edgecolor='k')
    fig = pylab.gcf()
    import re
    if facet is not None:
        match = re.match("(.*):(.*),(.*),(.*)", facet)
        if match:
            groups = match.groups()
            facet_expression = groups[0]
            facet_limits = [ast.literal_eval(groups[1]), ast.literal_eval(groups[2])]
            facet_count = ast.literal_eval(groups[3])
            limits.append(facet_limits)
            binby.append(facet_expression)
            shape = (facet_count,) + shape
        else:
            raise ValueError("Could not understand 'facet' argument %r, expected something in form: 'column:-1,10:5'" % facet)

    if grid is None:
        if what:
            if isinstance(what, (vaex.stat.Expression)):
                grid = what.calculate(self, binby=binby, limits=limits, shape=shape, selection=selection)
            else:
                what = what.strip()
                index = what.index("(")
                import re
                groups = re.match("(.*)\((.*)\)", what).groups()
                if groups and len(groups) == 2:
                    function = groups[0]
                    arguments = groups[1].strip()
                    functions = ["mean", "sum", "std", "count"]
                    if function in functions:
                        # grid = getattr(self, function)(arguments, binby, limits=limits, shape=shape, selection=selection)
                        grid = getattr(vaex.stat, function)(arguments).calculate(self, binby=binby, limits=limits, shape=shape, selection=selection)
                    elif function == "count" and arguments == "*":
                        grid = self.count(binby=binby, shape=shape, limits=limits, selection=selection)
                    elif function == "cumulative" and arguments == "*":
                        # TODO: comulative should also include the tails outside limits
                        grid = self.count(binby=binby, shape=shape, limits=limits, selection=selection)
                        grid = np.cumsum(grid)
                    else:
                        raise ValueError("Could not understand method: %s, expected one of %r'" % (function, functions))
                else:
                    raise ValueError("Could not understand 'what' argument %r, expected something in form: 'count(*)', 'mean(x)'" % what)
        else:
            grid = self.histogram(binby, size=shape, limits=limits, selection=selection)
    fgrid = f(grid)
    if n is not None:
        # ngrid = n(fgrid, axis=normalize_axis)
        ngrid = fgrid / fgrid.sum()
    else:
        ngrid = fgrid
        # reductions = [_parse_reduction(r, colormap, colors) for r in reduce]
        # rgrid = ngrid * 1.
        # for r in reduce:
        #   r = _parse_reduction(r, colormap, colors)
        #   rgrid = r(rgrid)
        # grid = self.reduce(grid, )
    xmin, xmax = limits[-1]
    if facet:
        N = len(grid[-1])
    else:
        N = len(grid)
    xexpression = binby[0]
    xar = np.arange(N + 1) / (N - 0.) * (xmax - xmin) + xmin
    label = str(label or selection or x)
    if facet:
        import math
        rows, columns = int(math.ceil(facet_count / 4.)), 4
        values = np.linspace(facet_limits[0], facet_limits[1], facet_count + 1)
        for i in range(facet_count):
            ax = pylab.subplot(rows, columns, i + 1)
            value = ax.plot(xar, ngrid[i], drawstyle="steps-mid", label=label, **kwargs)
            v1, v2 = values[i], values[i + 1]
            pylab.xlabel(xlabel or x)
            pylab.ylabel(ylabel or what)
            ax.set_title("%3f <= %s < %3f" % (v1, facet_expression, v2))
            if self.iscategory(xexpression):
                labels = self.category_labels(xexpression)
                step = len(labels) // max_labels
                pylab.xticks(range(len(labels))[::step], labels[::step], size='small')
    else:
        # im = pylab.imshow(rgrid, extent=np.array(limits[:2]).flatten(), origin="lower", aspect=aspect)
        pylab.xlabel(xlabel or self.label(x))
        pylab.ylabel(ylabel or what)
        # print(xar, ngrid)
        # repeat the first element, that's how plot/steps likes it..
        g = np.concatenate([ngrid[0:1], ngrid])
        value = pylab.plot(xar, g, drawstyle="steps-pre", label=label, **kwargs)
        if self.iscategory(xexpression):
            labels = self.category_labels(xexpression)
            step = len(labels) // max_labels
            pylab.xticks(range(len(labels))[::step], labels[::step], size='small')
    if tight_layout:
        pylab.tight_layout()
    if hardcopy:
        pylab.savefig(hardcopy)
    if show:
        pylab.show()
    return value

Response: Plot a 1D histogram of the data in a 1D grid.


Instruction:
Make a summary of the following Python 3 code
def scatter(self, x, y, xerr=None, yerr=None, cov=None, corr=None, s_expr=None, c_expr=None, labels=None, selection=None, length_limit=50000,
    length_check=True, label=None, xlabel=None, ylabel=None, errorbar_kwargs={}, ellipse_kwargs={}, **kwargs):
    """Viz (small amounts) of data in 2d using a scatter plot

    Convenience wrapper around pylab.scatter when for working with small DataFrames or selections

    :param x: Expression for x axis
    :param y: Idem for y
    :param s_expr: When given, use if for the s (size) argument of pylab.scatter
    :param c_expr: When given, use if for the c (color) argument of pylab.scatter
    :param labels: Annotate the points with these text values
    :param selection: Single selection expression, or None
    :param length_limit: maximum number of rows it will plot
    :param length_check: should we do the maximum row check or not?
    :param label: label for the legend
    :param xlabel: label for x axis, if None .label(x) is used
    :param ylabel: label for y axis, if None .label(y) is used
    :param errorbar_kwargs: extra dict with arguments passed to plt.errorbar
    :param kwargs: extra arguments passed to pylab.scatter
    :return:
    """
    import pylab as plt
    x = _ensure_strings_from_expressions(x)
    y = _ensure_strings_from_expressions(y)
    label = str(label or selection)
    selection = _ensure_strings_from_expressions(selection)
    if length_check:
        count = self.count(selection=selection)
        if count > length_limit:
            raise ValueError("the number of rows (%d) is above the limit (%d), pass length_check=False, or increase length_limit" % (count, length_limit))
    x_values = self.evaluate(x, selection=selection)
    y_values = self.evaluate(y, selection=selection)
    if s_expr:
        kwargs["s"] = self.evaluate(s_expr, selection=selection)
    if c_expr:
        kwargs["c"] = self.evaluate(c_expr, selection=selection)
    plt.xlabel(xlabel or self.label(x))
    plt.ylabel(ylabel or self.label(y))
    s = plt.scatter(x_values, y_values, label=label, **kwargs)
    if labels:
        label_values = self.evaluate(labels, selection=selection)
        for i, label_value in enumerate(label_values):
            plt.annotate(label_value, (x_values[i], y_values[i]))
    xerr_values = None
    yerr_values = None
    if cov is not None or corr is not None:
        from matplotlib.patches import Ellipse
        sx = self.evaluate(xerr, selection=selection)
        sy = self.evaluate(yerr, selection=selection)
        if corr is not None:
            sxy = self.evaluate(corr, selection=selection) * sx * sy
        elif cov is not None:
            sxy = self.evaluate(cov, selection=selection)
        cov_matrix = np.zeros((len(sx), 2, 2))
        cov_matrix[:,0,0] = sx**2
        cov_matrix[:,1,1] = sy**2
        cov_matrix[:,0,1] = cov_matrix[:,1,0] = sxy
        ax = plt.gca()
        ellipse_kwargs = dict(ellipse_kwargs)
        ellipse_kwargs['facecolor'] = ellipse_kwargs.get('facecolor', 'none')
        ellipse_kwargs['edgecolor'] = ellipse_kwargs.get('edgecolor', 'black')
        for i in range(len(sx)):
            eigen_values, eigen_vectors = np.linalg.eig(cov_matrix[i])
            indices = np.argsort(eigen_values)[::-1]
            eigen_values = eigen_values[indices]
            eigen_vectors = eigen_vectors[:,indices]
            v1 = eigen_vectors[:, 0]
            v2 = eigen_vectors[:, 1]
            varx = cov_matrix[i, 0, 0]
            vary = cov_matrix[i, 1, 1]
            angle = np.arctan2(v1[1], v1[0])
            # round off errors cause negative values?
            if eigen_values[1] < 0 and abs((eigen_values[1]/eigen_values[0])) < 1e-10:
                eigen_values[1] = 0
            if eigen_values[0] < 0 or eigen_values[1] < 0:
                raise ValueError('neg val')
            width, height = np.sqrt(np.max(eigen_values)), np.sqrt(np.min(eigen_values))
            e = Ellipse(xy=(x_values[i], y_values[i]), width=width, height=height, angle=np.degrees(angle), **ellipse_kwargs)
            ax.add_artist(e)
    else:
        if xerr is not None:
            if _issequence(xerr):
                assert len(xerr) == 2, "if xerr is a sequence it should be of length 2"
                xerr_values = [self.evaluate(xerr[0], selection=selection), self.evaluate(xerr[1], selection=selection)]
            else:
                xerr_values = self.evaluate(xerr, selection=selection)
        if yerr is not None:
            if _issequence(yerr):
                assert len(yerr) == 2, "if yerr is a sequence it should be of length 2"
                yerr_values = [self.evaluate(yerr[0], selection=selection), self.evaluate(yerr[1], selection=selection)]
            else:
                yerr_values = self.evaluate(yerr, selection=selection)
        if xerr_values is not None or yerr_values is not None:
            errorbar_kwargs = dict(errorbar_kwargs)
            errorbar_kwargs['fmt'] = errorbar_kwargs.get('fmt', 'none')
            plt.errorbar(x_values, y_values, yerr=yerr_values, xerr=xerr_values, **errorbar_kwargs)
    return s

Response: Plots a scatter plot of the data in 2d using a scatter plot.


Instruction:
How would you explain what the following Python 3 function does
def register_function(scope=None, as_property=False, name=None):
    prefix = ''
    if scope:
        prefix = scope + "_"
        if scope not in scopes:
            raise KeyError("unknown scope")
    def wrapper(f, name=name):
        name = name or f.__name__
        # remove possible prefix
        if name.startswith(prefix):
            name = name[len(prefix):]
        full_name = prefix + name
        if scope:
            def closure(name=name, full_name=full_name, function=f):
                def wrapper(self, *args, **kwargs):
                    lazy_func = getattr(self.expression.ds.func, full_name)
                    args = (self.expression, ) + args
                    return lazy_func(*args, **kwargs)
                return functools.wraps(function)(wrapper)
            if as_property:
                setattr(scopes[scope], name, property(closure()))
            else:
                setattr(scopes[scope], name, closure())
        else:
            def closure(name=name, full_name=full_name, function=f):
                def wrapper(self, *args, **kwargs):
                    lazy_func = getattr(self.ds.func, full_name)
                    args = (self, ) + args
                    return lazy_func(*args, **kwargs)
                return functools.wraps(function)(wrapper)
            setattr(vaex.expression.Expression, name, closure())
        vaex.expression.expression_namespace[prefix + name] = f
        return f  # we leave the original function as is
    return wrapper

Response: Decorator to register a new function with vaex.


Instruction:
Here you have a function in Python 3, explain what it does
def fillna(ar, value, fill_nan=True, fill_masked=True):
    '''Returns an array where missing values are replaced by value.

    If the dtype is object, nan values and 'nan' string values
    are replaced by value when fill_nan==True.
    '''
    ar = ar if not isinstance(ar, column.Column) else ar.to_numpy()
    if ar.dtype.kind in 'O' and fill_nan:
        strings = ar.astype(str)
        mask = strings == 'nan'
        ar = ar.copy()
        ar[mask] = value
    elif ar.dtype.kind in 'f' and fill_nan:
        mask = np.isnan(ar)
        if np.any(mask):
            ar = ar.copy()
            ar[mask] = value
    if fill_masked and np.ma.isMaskedArray(ar):
        mask = ar.mask
        if np.any(mask):
            ar = ar.data.copy()
            ar[mask] = value
    return ar

Response: Returns an array where missing values are replaced by value.


Instruction:
Can you generate the documentation for the following Python 3 function
def dt_dayofweek(x):
    import pandas as pd
    return pd.Series(x).dt.dayofweek.values

Response: Return the day of the week in a sequence of time series.


Instruction:
Write a Python 3 function that can
return an expression containing the ordinal day of the year.

Response:
def dt_dayofyear(x):
    """The ordinal day of the year.

    :returns: an expression containing the ordinal day of the year.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.dayofyear
    Expression = dt_dayofyear(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  285
    1   42
    2  316
    """
    import pandas as pd
    return pd.Series(x).dt.dayofyear.values


Instruction:
Write a Python 3 function for
checking whether a year is a leap year.

Response:
def dt_is_leap_year(x):
    """Check whether a year is a leap year.

    :returns: an expression which evaluates to True if a year is a leap year, and to False otherwise.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.is_leap_year
    Expression = dt_is_leap_year(date)
    Length: 3 dtype: bool (expression)
    ----------------------------------
    0  False
    1   True
    2  False
    """
    import pandas as pd
    return pd.Series(x).dt.is_leap_year.values


Instruction:
Implement a function in Python 3 to
extract the year out of a datetime column.

Response:
def dt_year(x):
    """Extracts the year out of a datetime sample.

    :returns: an expression containing the year extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.year
    Expression = dt_year(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  2009
    1  2016
    2  2015
    """
    import pandas as pd
    return pd.Series(x).dt.year.values


Instruction:
Can you generate a brief explanation for the following Python 3 code
def dt_month(x):
    import pandas as pd
    return pd.Series(x).dt.month.values

Response: Extracts the month out of a datetime column.


Instruction:
Can you tell what is the following Python 3 function doing
def dt_month_name(x):
    import pandas as pd
    return pd.Series(_pandas_dt_fix(x)).dt.month_name().values.astype(str)

Response: Returns the month names of a datetime sample in English.


Instruction:
Here you have a function in Python 3, explain what it does
def dt_day(x):
    import pandas as pd
    return pd.Series(x).dt.day.values

Response: Extracts the day from a datetime column.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def dt_day_name(x):
    import pandas as pd
    return pd.Series(_pandas_dt_fix(x)).dt.day_name().values.astype(str)

Response: Returns the day names of a datetime sample in English.


Instruction:
Can you write a function in Python 3 where it
returns an expression containing the week ordinal of the year extracted from a datetime column.

Response:
def dt_weekofyear(x):
    """Returns the week ordinal of the year.

    :returns: an expression containing the week ordinal of the year, extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.weekofyear
    Expression = dt_weekofyear(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  42
    1   6
    2  46
    """
    import pandas as pd
    return pd.Series(x).dt.weekofyear.values


Instruction:
Here you have a function in Python 3, explain what it does
def dt_hour(x):
    import pandas as pd
    return pd.Series(x).dt.hour.values

Response: Extracts the hour out of a datetime column.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def dt_minute(x):
    import pandas as pd
    return pd.Series(x).dt.minute.values

Response: Extracts the minute out of a datetime column.


Instruction:
Can you generate the documentation for the following Python 3 function
def dt_second(x):
    import pandas as pd
    return pd.Series(x).dt.second.values

Response: Extracts the second out of a datetime column.


Instruction:
Write a Python 3 function for
capitalizing the first letter of a string sample.

Response:
def str_capitalize(x):
    """Capitalize the first letter of a string sample.

    :returns: an expression containing the capitalized strings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.capitalize()
    Expression = str_capitalize(text)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    Something
    1  Very pretty
    2    Is coming
    3          Our
    4         Way.
    """
    sl = _to_string_sequence(x).capitalize()
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Write a Python 3 function for
concatenating two string columns on a row - by - row basis.

Response:
def str_cat(x, other):
    """Concatenate two string columns on a row-by-row basis.

    :param expression other: The expression of the other column to be concatenated.
    :returns: an expression containing the concatenated columns.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.cat(df.text)
    Expression = str_cat(text, text)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0      SomethingSomething
    1  very prettyvery pretty
    2      is comingis coming
    3                  ourour
    4                way.way.
    """
    sl1 = _to_string_sequence(x)
    sl2 = _to_string_sequence(other)
    sl = sl1.concat(sl2)
    return column.ColumnStringArrow.from_string_sequence(sl)


Instruction:
How would you implement a function in Python 3 that
checks if a string pattern or regex is contained within a sample of a string column.

Response:
def str_contains(x, pattern, regex=True):
    """Check if a string pattern or regex is contained within a sample of a string column.

    :param str pattern: A string or regex pattern
    :param bool regex: If True,
    :returns: an expression which is evaluated to True if the pattern is found in a given sample, and it is False otherwise.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.contains('very')
    Expression = str_contains(text, 'very')
    Length: 5 dtype: bool (expression)
    ----------------------------------
    0  False
    1   True
    2  False
    3  False
    4  False
    """
    return _to_string_sequence(x).search(pattern, regex)


Instruction:
Implement a function in Python 3 to
count the occurences of a pattern in a string column.

Response:
def str_count(x, pat, regex=False):
    """Count the occurences of a pattern in sample of a string column.

    :param str pat: A string or regex pattern
    :param bool regex: If True,
    :returns: an expression containing the number of times a pattern is found in each sample.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.count(pat="et", regex=False)
    Expression = str_count(text, pat='et', regex=False)
    Length: 5 dtype: int64 (expression)
    -----------------------------------
    0  1
    1  1
    2  0
    3  0
    4  0
    """
    return _to_string_sequence(x).count(pat, regex)


Instruction:
Here you have a function in Python 3, explain what it does
def str_find(x, sub, start=0, end=None):
    return _to_string_sequence(x).find(sub, start, 0 if end is None else end, end is None, True)

Response: Returns the lowest indices in each string in a column where the provided substring is fully contained between within the provided sample.


Instruction:
Can you write a function in Python 3 where it
extracts a character from each sample at the specified position from a string column.

Response:
def str_get(x, i):
    """Extract a character from each sample at the specified position from a string column.
    Note that if the specified position is out of bound of the string sample, this method returns '', while pandas retunrs nan.

    :param int i: The index location, at which to extract the character.
    :returns: an expression containing the extracted characters.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.get(5)
    Expression = str_get(text, 5)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    h
    1    p
    2    m
    3
    4
    """
    x = _to_string_sequence(x)
    if i == -1:
        sl = x.slice_string_end(-1)
    else:
        sl = x.slice_string(i, i+1)
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Make a summary of the following Python 3 code
def str_index(x, sub, start=0, end=None):
    return str_find(x, sub, start, end)

Response: Returns the lowest indices in each string in a column where the provided substring is fully contained between within the provided sample.


Instruction:
Given the following Python 3 function, write the documentation
def str_join(x, sep):
    sl = _to_string_list_sequence(x).join(sep)
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)

Response: Same as find but uses pandas to join strings


Instruction:
Can you create a Python 3 function that
converts string samples to lower case.

Response:
def str_lower(x):
    """Converts string samples to lower case.

    :returns: an expression containing the converted strings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.lower()
    Expression = str_lower(text)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    something
    1  very pretty
    2    is coming
    3          our
    4         way.
    """
    sl = _to_string_sequence(x).lower()
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Implement a Python 3 function for
removing leading characters from a string sample.

Response:
def str_lstrip(x, to_strip=None):
    """Remove leading characters from a string sample.

    :param str to_strip: The string to be removed
    :returns: an expression containing the modified string column.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.lstrip(to_strip='very ')
    Expression = str_lstrip(text, to_strip='very ')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0  Something
    1     pretty
    2  is coming
    3        our
    4       way.
    """
    # in c++ we give empty string the same meaning as None
    sl = _to_string_sequence(x).lstrip('' if to_strip is None else to_strip) if to_strip != '' else x
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Given the following Python 3 function, write the documentation
def str_pad(x, width, side='left', fillchar=' '):
    sl = _to_string_sequence(x).pad(width, fillchar, side in ['left', 'both'], side in ['right', 'both'])
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)

Response: Pads a string in a given column.


Instruction:
How would you code a function in Python 3 to
duplicate each string in a column.

Response:
def str_repeat(x, repeats):
    """Duplicate each string in a column.

    :param int repeats: number of times each string sample is to be duplicated.
    :returns: an expression containing the duplicated strings

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.repeat(3)
    Expression = str_repeat(text, 3)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0        SomethingSomethingSomething
    1  very prettyvery prettyvery pretty
    2        is comingis comingis coming
    3                          ourourour
    4                       way.way.way.
    """
    sl = _to_string_sequence(x).repeat(repeats)
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Create a Python 3 function for
replacing occurences of a pattern or a regex in a column with some other string.

Response:
def str_replace(x, pat, repl, n=-1, flags=0, regex=False):
    """Replace occurences of a pattern/regex in a column with some other string.

    :param str pattern: string or a regex pattern
    :param str replace: a replacement string
    :param int n: number of replacements to be made from the start. If -1 make all replacements.
    :param int flags: ??
    :param bool regex: If True, ...?
    :returns: an expression containing the string replacements.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.replace(pat='et', repl='__')
    Expression = str_replace(text, pat='et', repl='__')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    Som__hing
    1  very pr__ty
    2    is coming
    3          our
    4         way.
    """
    sl = _to_string_sequence(x).replace(pat, repl, n, flags, regex)
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Can you write a function in Python 3 where it
returns the highest indices in each string in a column where the provided substring is fully contained between within the provided start and end indices.

Response:
def str_rfind(x, sub, start=0, end=None):
    """Returns the highest indices in each string in a column, where the provided substring is fully contained between within a
    sample. If the substring is not found, -1 is returned.

    :param str sub: A substring to be found in the samples
    :param int start:
    :param int end:
    :returns: an expression containing the highest indices specifying the start of the substring.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.rfind(sub="et")
    Expression = str_rfind(text, sub='et')
    Length: 5 dtype: int64 (expression)
    -----------------------------------
    0   3
    1   7
    2  -1
    3  -1
    4  -1
    """
    return _to_string_sequence(x).find(sub, start, 0 if end is None else end, end is None, False)


Instruction:
How would you explain what the following Python 3 function does
def str_rindex(x, sub, start=0, end=None):
    return str_rfind(x, sub, start, end)

Response: Returns the highest indices in each string in a column where the provided substring is fully contained between within the provided start and end indices.


Instruction:
Can you tell what is the following Python 3 function doing
def str_rjust(x, width, fillchar=' '):
    sl = _to_string_sequence(x).pad(width, fillchar, True, False)
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)

Response: Fills the left side of a string sample with a specified character such that the strings are left - hand justified.


Instruction:
Can you create a Python 3 function that
removes trailing characters from a string sample.

Response:
def str_rstrip(x, to_strip=None):
    """Remove trailing characters from a string sample.

    :param str to_strip: The string to be removed
    :returns: an expression containing the modified string column.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.rstrip(to_strip='ing')
    Expression = str_rstrip(text, to_strip='ing')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0       Someth
    1  very pretty
    2       is com
    3          our
    4         way.
    """
    # in c++ we give empty string the same meaning as None
    sl = _to_string_sequence(x).rstrip('' if to_strip is None else to_strip) if to_strip != '' else x
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Can you implement a function in Python 3 that
slices substrings from each string element in a column.

Response:
def str_slice(x, start=0, stop=None):  # TODO: support n
    """Slice substrings from each string element in a column.

    :param int start: The start position for the slice operation.
    :param int end: The stop position for the slice operation.
    :returns: an expression containing the sliced substrings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.slice(start=2, stop=5)
    Expression = str_pandas_slice(text, start=2, stop=5)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0  met
    1   ry
    2   co
    3    r
    4   y.
    """
    if stop is None:
        sll = _to_string_sequence(x).slice_string_end(start)
    else:
        sll = _to_string_sequence(x).slice_string(start, stop)
    return sll


Instruction:
Make a summary of the following Python 3 code
def str_strip(x, to_strip=None):
    # in c++ we give empty string the same meaning as None
    sl = _to_string_sequence(x).strip('' if to_strip is None else to_strip) if to_strip != '' else x
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)

Response: Removes leading and trailing characters from each string saple in a column.


Instruction:
Implement a Python 3 function for
converting all string samples to titlecase.

Response:
def str_title(x):
    """Converts all string samples to titlecase.

    :returns: an expression containing the converted strings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.title()
    Expression = str_title(text)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    Something
    1  Very Pretty
    2    Is Coming
    3          Our
    4         Way.
    """
    sl = _to_string_sequence(x).title()
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)


Instruction:
Given the following Python 3 function, write the documentation
def str_upper(x):
    sl = _to_string_sequence(x).upper()
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)

Response: Converts all strings in a column to uppercase.


Instruction:
Can you tell what is the following Python 3 function doing
def format(x, format):
    # don't change the dtype, otherwise for each block the dtype may be different (string length)
    sl = vaex.strings.format(x, format)
    return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)

Response: Formats x with the given format.