Instruction:
How would you explain what the following Python 3 function does
def setup_user_email(request, user, addresses):
    from .models import EmailAddress

    assert not EmailAddress.objects.filter(user=user).exists()
    priority_addresses = []
    # Is there a stashed e-mail?
    adapter = get_adapter(request)
    stashed_email = adapter.unstash_verified_email(request)
    if stashed_email:
        priority_addresses.append(EmailAddress(user=user,
                                               email=stashed_email,
                                               primary=True,
                                               verified=True))
    email = user_email(user)
    if email:
        priority_addresses.append(EmailAddress(user=user,
                                               email=email,
                                               primary=True,
                                               verified=False))
    addresses, primary = cleanup_email_addresses(
        request,
        priority_addresses + addresses)
    for a in addresses:
        a.user = user
        a.save()
    EmailAddress.objects.fill_cache_for_user(user, addresses)
    if (primary and email and email.lower() != primary.email.lower()):
        user_email(user, primary.email)
        user.save()
    return primary

Response: Sets up proper EmailAddress for the user that was just signed up.


Instruction:
Can you tell what is the following Python 3 function doing
def send_email_confirmation(request, user, signup=False):
    from .models import EmailAddress, EmailConfirmation

    cooldown_period = timedelta(
        seconds=app_settings.EMAIL_CONFIRMATION_COOLDOWN
    )

    email = user_email(user)
    if email:
        try:
            email_address = EmailAddress.objects.get_for_user(user, email)
            if not email_address.verified:
                if app_settings.EMAIL_CONFIRMATION_HMAC:
                    send_email = True
                else:
                    send_email = not EmailConfirmation.objects.filter(
                        sent__gt=now() - cooldown_period,
                        email_address=email_address).exists()
                if send_email:
                    email_address.send_confirmation(request,
                                                    signup=signup)
            else:
                send_email = False
        except EmailAddress.DoesNotExist:
            send_email = True
            email_address = EmailAddress.objects.add_email(request,
                                                           user,
                                                           email,
                                                           signup=signup,
                                                           confirm=True)
            assert email_address
        # At this point, if we were supposed to send an email we have sent it.
        if send_email:
            get_adapter(request).add_message(
                request,
                messages.INFO,
                'account/messages/'
                'email_confirmation_sent.txt',
                {'email': email})
    if signup:
        get_adapter(request).stash_user(request, user_pk_to_url_str(user))

Response: Sends an email confirmation mail to the user.


Instruction:
Write a Python 3 script to
synchronize user. email with the user. emailaddress_set.

Response:
def sync_user_email_addresses(user):
    """
    Keep user.email in sync with user.emailaddress_set.

    Under some circumstances the user.email may not have ended up as
    an EmailAddress record, e.g. in the case of manually created admin
    users.
    """
    from .models import EmailAddress
    email = user_email(user)
    if email and not EmailAddress.objects.filter(user=user,
                                                 email__iexact=email).exists():
        if app_settings.UNIQUE_EMAIL \
                and EmailAddress.objects.filter(email__iexact=email).exists():
            # Bail out
            return
        EmailAddress.objects.create(user=user,
                                    email=email,
                                    primary=False,
                                    verified=False)


Instruction:
Explain what the following Python 3 code does
def filter_users_by_email(email):
    from .models import EmailAddress
    User = get_user_model()
    mails = EmailAddress.objects.filter(email__iexact=email)
    users = [e.user for e in mails.prefetch_related('user')]
    if app_settings.USER_MODEL_EMAIL_FIELD:
        q_dict = {app_settings.USER_MODEL_EMAIL_FIELD + '__iexact': email}
        users += list(User.objects.filter(**q_dict))
    return list(set(users))

Response: Return list of users by email address


Instruction:
How would you explain what the following Python 3 function does
def setup_dummy_social_apps(sender, **kwargs):
    from allauth.socialaccount.providers import registry
    from allauth.socialaccount.models import SocialApp
    from allauth.socialaccount.providers.oauth.provider import OAuthProvider
    from allauth.socialaccount.providers.oauth2.provider import OAuth2Provider
    from django.contrib.sites.models import Site

    site = Site.objects.get_current()
    for provider in registry.get_list():
        if (isinstance(provider, OAuth2Provider)
                or isinstance(provider, OAuthProvider)):
            try:
                SocialApp.objects.get(provider=provider.id,
                                      sites=site)
            except SocialApp.DoesNotExist:
                print ("Installing dummy application credentials for %s."
                       " Authentication via this provider will not work"
                       " until you configure proper credentials via the"
                       " Django admin (`SocialApp` models)" % provider.id)
                app = SocialApp.objects.create(
                    provider=provider.id,
                    secret='secret',
                    client_id='client-id',
                    name='Dummy %s app' % provider.id)
                app.sites.add(site)

Response: Setup dummy social applications for OAuth based providers.


Instruction:
Write a Python 3 script for
returning value dumped to string.

Response:
def value_from_object(self, obj):
        """Return value dumped to string."""
        val = super(JSONField, self).value_from_object(obj)
        return self.get_prep_value(val)


Instruction:
Given the following Python 3 function, write the documentation
def EMAIL_VERIFICATION(self):
        ret = self._setting("EMAIL_VERIFICATION",
                            self.EmailVerificationMethod.OPTIONAL)
        # Deal with legacy (boolean based) setting
        if ret is True:
            ret = self.EmailVerificationMethod.MANDATORY
        elif ret is False:
            ret = self.EmailVerificationMethod.OPTIONAL
        return ret

Response: Returns the e - mail verification method.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def PASSWORD_MIN_LENGTH(self):
        from django.conf import settings
        ret = None
        if not settings.AUTH_PASSWORD_VALIDATORS:
            ret = self._setting("PASSWORD_MIN_LENGTH", 6)
        return ret

Response: Returns the minimum password length for the user.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def calculate_best_chunk_size(self, data_length):
        chunk_size, extra = divmod(data_length, self.n_workers * 5)
        if extra:
            chunk_size += 1
        return chunk_size

Response: Calculates the best chunk size for a list of length data_length.


Instruction:
Here you have a function in Python 3, explain what it does
def map_reduce(self, map_function, data, function_kwargs=None, chunk_size=None, data_length=None):
        if data_length is None:
            data_length = len(data)

        if not chunk_size:
            chunk_size = self.calculate_best_chunk_size(data_length)

        chunk_generator = self.partition(data, chunk_size=chunk_size)

        map_kwargs = {"map_function": map_function, "kwargs": function_kwargs}

        if hasattr(self, "progressbar_title"):
            total_number_of_expected_results = math.ceil(data_length / chunk_size)
            result = tqdm(self.distribute(_function_with_partly_reduce, chunk_generator, map_kwargs),
                          total=total_number_of_expected_results,
                          desc=self.progressbar_title, disable=self.disable_progressbar)
        else:
            result = self.distribute(_function_with_partly_reduce, chunk_generator, map_kwargs),

        result = list(itertools.chain.from_iterable(result))

        return result

Response: This method maps a function to each element of the data and reduces the results to return a flattened list.


Instruction:
Make a summary of the following Python 3 code
def distribute(self, func, partitioned_chunks, kwargs):
        return map(partial(func, **kwargs), partitioned_chunks)

Response: Calculates the features in a sequential fashion by pythons map command


Instruction:
Can you generate a brief explanation for the following Python 3 code
def distribute(self, func, partitioned_chunks, kwargs):
        result = self.client.gather(self.client.map(partial(func, **kwargs), partitioned_chunks))
        return [item for sublist in result for item in sublist]

Response: Distribute the function func to the local cluster and return the features in a parallel fashion.


Instruction:
How would you code a function in Python 3 to
calculate the optimal chunk size for a dask cluster entry.

Response:
def calculate_best_chunk_size(self, data_length):
        """
        Uses the number of dask workers in the cluster (during execution time, meaning when you start the extraction)
        to find the optimal chunk_size.

        :param data_length: A length which defines how many calculations there need to be.
        :type data_length: int
        """
        n_workers = len(self.client.scheduler_info()["workers"])
        chunk_size, extra = divmod(data_length, n_workers * 5)
        if extra:
            chunk_size += 1
        return chunk_size


Instruction:
How would you code a function in Python 3 to
distribute the function to a thread pool and return the features in a parallel fashion.

Response:
def distribute(self, func, partitioned_chunks, kwargs):
        """
        Calculates the features in a parallel fashion by distributing the map command to a thread pool

        :param func: the function to send to each worker.
        :type func: callable
        :param partitioned_chunks: The list of data chunks - each element is again
            a list of chunks - and should be processed by one worker.
        :type partitioned_chunks: iterable
        :param kwargs: parameters for the map function
        :type kwargs: dict of string to parameter

        :return: The result of the calculation as a list - each item should be the result of the application of func
            to a single element.
        """
        return self.pool.imap_unordered(partial(func, **kwargs), partitioned_chunks)


Instruction:
Implement a Python 3 function for
closing the thread pool and terminate the thread pool.

Response:
def close(self):
        """
        Collects the result from the workers and closes the thread pool.
        """
        self.pool.close()
        self.pool.terminate()
        self.pool.join()


Instruction:
Given the following Python 3 function, write the documentation
def from_columns(columns, columns_to_ignore=None):

    kind_to_fc_parameters = {}

    if columns_to_ignore is None:
        columns_to_ignore = []

    for col in columns:
        if col in columns_to_ignore:
            continue

        if not isinstance(col, basestring):
            raise TypeError("Column name {} should be a string or unicode".format(col))

        # Split according to our separator into <col_name>, <feature_name>, <feature_params>
        parts = col.split('__')
        n_parts = len(parts)

        if n_parts == 1:
            raise ValueError("Splitting of columnname {} resulted in only one part.".format(col))

        kind = parts[0]
        feature_name = parts[1]

        if kind not in kind_to_fc_parameters:
            kind_to_fc_parameters[kind] = {}

        if not hasattr(feature_calculators, feature_name):
            raise ValueError("Unknown feature name {}".format(feature_name))

        config = get_config_from_string(parts)
        if config:
            if feature_name in kind_to_fc_parameters[kind]:
                kind_to_fc_parameters[kind][feature_name].append(config)
            else:
                kind_to_fc_parameters[kind][feature_name] = [config]
        else:
            kind_to_fc_parameters[kind][feature_name] = None

    return kind_to_fc_parameters

Response: Creates a mapping from kind names to fc_parameters objects.


Instruction:
Can you implement a function in Python 3 that
rolls 1D array elements.

Response:
def _roll(a, shift):
    """
    Roll 1D array elements. Improves the performance of numpy.roll() by reducing the overhead introduced from the
    flexibility of the numpy.roll() method such as the support for rolling over multiple dimensions.

    Elements that roll beyond the last position are re-introduced at the beginning. Similarly, elements that roll
    back beyond the first position are re-introduced at the end (with negative shift).

    Examples
    --------
    >>> x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    >>> _roll(x, shift=2)
    >>> array([8, 9, 0, 1, 2, 3, 4, 5, 6, 7])

    >>> x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    >>> _roll(x, shift=-2)
    >>> array([2, 3, 4, 5, 6, 7, 8, 9, 0, 1])

    >>> x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    >>> _roll(x, shift=12)
    >>> array([8, 9, 0, 1, 2, 3, 4, 5, 6, 7])

    Benchmark
    ---------
    >>> x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    >>> %timeit _roll(x, shift=2)
    >>> 1.89 µs ± 341 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

    >>> x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    >>> %timeit np.roll(x, shift=2)
    >>> 11.4 µs ± 776 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

    :param a: the input array
    :type a: array_like
    :param shift: the number of places by which elements are shifted
    :type shift: int

    :return: shifted array with the same shape as a
    :return type: ndarray
    """
    if not isinstance(a, np.ndarray):
        a = np.asarray(a)
    idx = shift % len(a)
    return np.concatenate([a[-idx:], a[:-idx]])


Instruction:
Implement a function in Python 3 to
estimate the coefficients of a random variational system of the given time series.

Response:
def _estimate_friedrich_coefficients(x, m, r):
    """
    Coefficients of polynomial :math:`h(x)`, which has been fitted to
    the deterministic dynamics of Langevin model
    .. math::
        \dot{x}(t) = h(x(t)) + \mathcal{N}(0,R)

    As described by

        Friedrich et al. (2000): Physics Letters A 271, p. 217-222
        *Extracting model equations from experimental data*

    For short time-series this method is highly dependent on the parameters.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param m: order of polynom to fit for estimating fixed points of dynamics
    :type m: int
    :param r: number of quantils to use for averaging
    :type r: float

    :return: coefficients of polynomial of deterministic dynamics
    :return type: ndarray
    """
    assert m > 0, "Order of polynomial need to be positive integer, found {}".format(m)

    df = pd.DataFrame({'signal': x[:-1], 'delta': np.diff(x)})
    try:
        df['quantiles'] = pd.qcut(df.signal, r)
    except ValueError:
        return [np.NaN] * (m + 1)

    quantiles = df.groupby('quantiles')

    result = pd.DataFrame({'x_mean': quantiles.signal.mean(), 'y_mean': quantiles.delta.mean()})
    result.dropna(inplace=True)

    try:
        return np.polyfit(result.x_mean, result.y_mean, deg=m)
    except (np.linalg.LinAlgError, ValueError):
        return [np.NaN] * (m + 1)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _aggregate_on_chunks(x, f_agg, chunk_len):
    return [getattr(x[i * chunk_len: (i + 1) * chunk_len], f_agg)() for i in range(int(np.ceil(len(x) / chunk_len)))]

Response: Takes the time series x and creates a lower sampled version of it by applying the aggregation function f_agg on consecutive chunks of length chunk_len and returns the list of the aggregation function over the chunks of length chunk_len


Instruction:
Make a summary of the following Python 3 code
def set_property(key, value):
    def decorate_func(func):
        setattr(func, key, value)
        if func.__doc__ and key == "fctype":
            func.__doc__ = func.__doc__ + "\n\n    *This function is of type: " + value + "*\n"
        return func
    return decorate_func

Response: This method returns a decorator that sets the property key of the function to value


Instruction:
Can you tell what is the following Python 3 function doing
def variance_larger_than_standard_deviation(x):
    y = np.var(x)
    return y > np.sqrt(y)

Response: Returns a boolean variable denoting if the variance of x is greater than its standard deviation.


Instruction:
How would you explain what the following Python 3 function does
def ratio_beyond_r_sigma(x, r):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return np.sum(np.abs(x - np.mean(x)) > r * np.std(x))/x.size

Response: Ratio of values that are more than r * std ( x so r sigma away from the mean of x.


Instruction:
Can you implement a function in Python 3 that
checks if the maximum value of x is observed more than once

Response:
def has_duplicate_max(x):
    """
    Checks if the maximum value of x is observed more than once

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :return: the value of this feature
    :return type: bool
    """
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return np.sum(x == np.max(x)) >= 2


Instruction:
Given the following Python 3 function, write the documentation
def has_duplicate_min(x):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return np.sum(x == np.min(x)) >= 2

Response: Checks if the minimal value of x is observed more than once


Instruction:
Can you generate the documentation for the following Python 3 function
def has_duplicate(x):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return x.size != np.unique(x).size

Response: Checks if any value in x occurs more than once


Instruction:
Can you implement a function in Python 3 that
calculates the partial autocorrelation function at the given lag.

Response:
def partial_autocorrelation(x, param):
    """
    Calculates the value of the partial autocorrelation function at the given lag. The lag `k` partial autocorrelation
    of a time series :math:`\\lbrace x_t, t = 1 \\ldots T \\rbrace` equals the partial correlation of :math:`x_t` and
    :math:`x_{t-k}`, adjusted for the intermediate variables
    :math:`\\lbrace x_{t-1}, \\ldots, x_{t-k+1} \\rbrace` ([1]).
    Following [2], it can be defined as

    .. math::

        \\alpha_k = \\frac{ Cov(x_t, x_{t-k} | x_{t-1}, \\ldots, x_{t-k+1})}
        {\\sqrt{ Var(x_t | x_{t-1}, \\ldots, x_{t-k+1}) Var(x_{t-k} | x_{t-1}, \\ldots, x_{t-k+1} )}}

    with (a) :math:`x_t = f(x_{t-1}, \\ldots, x_{t-k+1})` and (b) :math:`x_{t-k} = f(x_{t-1}, \\ldots, x_{t-k+1})`
    being AR(k-1) models that can be fitted by OLS. Be aware that in (a), the regression is done on past values to
    predict :math:`x_t` whereas in (b), future values are used to calculate the past value :math:`x_{t-k}`.
    It is said in [1] that "for an AR(p), the partial autocorrelations [ :math:`\\alpha_k` ] will be nonzero for `k<=p`
    and zero for `k>p`."
    With this property, it is used to determine the lag of an AR-Process.

    .. rubric:: References

    |  [1] Box, G. E., Jenkins, G. M., Reinsel, G. C., & Ljung, G. M. (2015).
    |  Time series analysis: forecasting and control. John Wiley & Sons.
    |  [2] https://onlinecourses.science.psu.edu/stat510/node/62

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param param: contains dictionaries {"lag": val} with int val indicating the lag to be returned
    :type param: list
    :return: the value of this feature
    :return type: float
    """
    # Check the difference between demanded lags by param and possible lags to calculate (depends on len(x))
    max_demanded_lag = max([lag["lag"] for lag in param])
    n = len(x)

    # Check if list is too short to make calculations
    if n <= 1:
        pacf_coeffs = [np.nan] * (max_demanded_lag + 1)
    else:
        if (n <= max_demanded_lag):
            max_lag = n - 1
        else:
            max_lag = max_demanded_lag
        pacf_coeffs = list(pacf(x, method="ld", nlags=max_lag))
        pacf_coeffs = pacf_coeffs + [np.nan] * max(0, (max_demanded_lag - max_lag))

    return [("lag_{}".format(lag["lag"]), pacf_coeffs[lag["lag"]]) for lag in param]


Instruction:
How would you explain what the following Python 3 function does
def abs_energy(x):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return np.dot(x, x)

Response: Returns the absolute energy of the time series which is the sum over the squared values


Instruction:
Make a summary of the following Python 3 code
def mean_second_derivative_central(x):

    diff = (_roll(x, 1) - 2 * np.array(x) + _roll(x, -1)) / 2.0
    return np.mean(diff[1:-1])

Response: Returns the mean value of a central approximation of the second derivative of the central approximation of the second derivative.


Instruction:
Can you tell what is the following Python 3 function doing
def skewness(x):
    if not isinstance(x, pd.Series):
        x = pd.Series(x)
    return pd.Series.skew(x)

Response: Returns the sample skewness of x.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def kurtosis(x):
    if not isinstance(x, pd.Series):
        x = pd.Series(x)
    return pd.Series.kurtosis(x)

Response: Returns the kurtosis of x


Instruction:
Can you create a Python 3 function that
returns the length of the longest consecutive subsequence in x that is smaller than the mean of x

Response:
def longest_strike_below_mean(x):
    """
    Returns the length of the longest consecutive subsequence in x that is smaller than the mean of x

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :return: the value of this feature
    :return type: float
    """
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return np.max(_get_length_sequences_where(x <= np.mean(x))) if x.size > 0 else 0


Instruction:
Explain what the following Python 3 code does
def count_above_mean(x):
    m = np.mean(x)
    return np.where(x > m)[0].size

Response: Returns the number of values in x that are higher than the mean of x


Instruction:
Can you implement a function in Python 3 that
returns the number of values in x that are lower than the mean of x

Response:
def count_below_mean(x):
    """
    Returns the number of values in x that are lower than the mean of x

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :return: the value of this feature
    :return type: float
    """
    m = np.mean(x)
    return np.where(x < m)[0].size


Instruction:
Write a Python 3 script to
return the relative last location of the maximum value of x.

Response:
def last_location_of_maximum(x):
    """
    Returns the relative last location of the maximum value of x.
    The position is calculated relatively to the length of x.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :return: the value of this feature
    :return type: float
    """
    x = np.asarray(x)
    return 1.0 - np.argmax(x[::-1]) / len(x) if len(x) > 0 else np.NaN


Instruction:
Given the following Python 3 function, write the documentation
def first_location_of_maximum(x):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return np.argmax(x) / len(x) if len(x) > 0 else np.NaN

Response: Returns the first location of the maximum value of x.


Instruction:
Can you write a function in Python 3 where it
returns the last location of the minimal value of x.

Response:
def last_location_of_minimum(x):
    """
    Returns the last location of the minimal value of x.
    The position is calculated relatively to the length of x.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :return: the value of this feature
    :return type: float
    """
    x = np.asarray(x)
    return 1.0 - np.argmin(x[::-1]) / len(x) if len(x) > 0 else np.NaN


Instruction:
Given the following Python 3 function, write the documentation
def first_location_of_minimum(x):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    return np.argmin(x) / len(x) if len(x) > 0 else np.NaN

Response: Returns the first location of the minimal value of x.


Instruction:
How would you implement a function in Python 3 that
returns the percentage of unique values that occurring more than once in the time series x.

Response:
def percentage_of_reoccurring_datapoints_to_all_datapoints(x):
    """
    Returns the percentage of unique values, that are present in the time series
    more than once.

        len(different values occurring more than once) / len(different values)

    This means the percentage is normalized to the number of unique values,
    in contrast to the percentage_of_reoccurring_values_to_all_values.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :return: the value of this feature
    :return type: float
    """
    if len(x) == 0:
        return np.nan

    unique, counts = np.unique(x, return_counts=True)

    if counts.shape[0] == 0:
        return 0

    return np.sum(counts > 1) / float(counts.shape[0])


Instruction:
Can you generate the documentation for the following Python 3 function
def percentage_of_reoccurring_values_to_all_values(x):
    if not isinstance(x, pd.Series):
        x = pd.Series(x)

    if x.size == 0:
        return np.nan

    value_counts = x.value_counts()
    reoccuring_values = value_counts[value_counts > 1].sum()

    if np.isnan(reoccuring_values):
        return 0

    return reoccuring_values / x.size

Response: Returns the ratio of unique values that occurring more than once to all values.


Instruction:
Write a Python 3 function for
returning the sum of all values that are present in the time series more than once.

Response:
def sum_of_reoccurring_values(x):
    """
    Returns the sum of all values, that are present in the time series
    more than once.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :return: the value of this feature
    :return type: float
    """
    unique, counts = np.unique(x, return_counts=True)
    counts[counts < 2] = 0
    counts[counts > 1] = 1
    return np.sum(counts * unique)


Instruction:
Here you have a function in Python 3, explain what it does
def sum_of_reoccurring_data_points(x):
    unique, counts = np.unique(x, return_counts=True)
    counts[counts < 2] = 0
    return np.sum(counts * unique)

Response:
Returns the sum of all data points that are present in the time series
    more than once.


Instruction:
Make a summary of the following Python 3 code
def ratio_value_number_to_time_series_length(x):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    if x.size == 0:
        return np.nan

    return np.unique(x).size / x.size

Response: Returns a factor which is 1 if all values in the time series occur only once and below one if all values in the time series occur only once and below one if all values in the time series occur only once and below one if this is not the case.


Instruction:
Write a Python 3 function that can
calculate the fourier coefficients of a one - dimensional discrete Fourier Transform for real input by fast fourier transformation algorithm.

Response:
def fft_coefficient(x, param):
    """
    Calculates the fourier coefficients of the one-dimensional discrete Fourier Transform for real input by fast
    fourier transformation algorithm

    .. math::
        A_k =  \\sum_{m=0}^{n-1} a_m \\exp \\left \\{ -2 \\pi i \\frac{m k}{n} \\right \\}, \\qquad k = 0,
        \\ldots , n-1.

    The resulting coefficients will be complex, this feature calculator can return the real part (attr=="real"),
    the imaginary part (attr=="imag), the absolute value (attr=""abs) and the angle in degrees (attr=="angle).

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param param: contains dictionaries {"coeff": x, "attr": s} with x int and x >= 0, s str and in ["real", "imag",
        "abs", "angle"]
    :type param: list
    :return: the different feature values
    :return type: pandas.Series
    """

    assert min([config["coeff"] for config in param]) >= 0, "Coefficients must be positive or zero."
    assert set([config["attr"] for config in param]) <= set(["imag", "real", "abs", "angle"]), \
        'Attribute must be "real", "imag", "angle" or "abs"'

    fft = np.fft.rfft(x)

    def complex_agg(x, agg):
        if agg == "real":
            return x.real
        elif agg == "imag":
            return x.imag
        elif agg == "abs":
            return np.abs(x)
        elif agg == "angle":
            return np.angle(x, deg=True)

    res = [complex_agg(fft[config["coeff"]], config["attr"]) if config["coeff"] < len(fft)
           else np.NaN for config in param]
    index = ['coeff_{}__attr_"{}"'.format(config["coeff"], config["attr"]) for config in param]
    return zip(index, res)


Instruction:
Can you write a function in Python 3 where it
calculates the spectral centroid variance skew and kurtosis of the absolute fourier transform spectrum.

Response:
def fft_aggregated(x, param):
    """
    Returns the spectral centroid (mean), variance, skew, and kurtosis of the absolute fourier transform spectrum.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param param: contains dictionaries {"aggtype": s} where s str and in ["centroid", "variance",
        "skew", "kurtosis"]
    :type param: list
    :return: the different feature values
    :return type: pandas.Series
    """

    assert set([config["aggtype"] for config in param]) <= set(["centroid", "variance", "skew", "kurtosis"]), \
        'Attribute must be "centroid", "variance", "skew", "kurtosis"'

    def get_moment(y, moment):
        """
        Returns the (non centered) moment of the distribution y:
        E[y**moment] = \sum_i[index(y_i)^moment * y_i] / \sum_i[y_i]

        :param y: the discrete distribution from which one wants to calculate the moment
        :type y: pandas.Series or np.array
        :param moment: the moment one wants to calcalate (choose 1,2,3, ... )
        :type moment: int
        :return: the moment requested
        :return type: float
        """
        return y.dot(np.arange(len(y))**moment) / y.sum()

    def get_centroid(y):
        """
        :param y: the discrete distribution from which one wants to calculate the centroid
        :type y: pandas.Series or np.array
        :return: the centroid of distribution y (aka distribution mean, first moment)
        :return type: float
        """
        return get_moment(y, 1)

    def get_variance(y):
        """
        :param y: the discrete distribution from which one wants to calculate the variance
        :type y: pandas.Series or np.array
        :return: the variance of distribution y
        :return type: float
        """
        return get_moment(y, 2) - get_centroid(y) ** 2

    def get_skew(y):
        """
        Calculates the skew as the third standardized moment.
        Ref: https://en.wikipedia.org/wiki/Skewness#Definition

        :param y: the discrete distribution from which one wants to calculate the skew
        :type y: pandas.Series or np.array
        :return: the skew of distribution y
        :return type: float
        """

        variance = get_variance(y)
        # In the limit of a dirac delta, skew should be 0 and variance 0.  However, in the discrete limit,
        # the skew blows up as variance --> 0, hence return nan when variance is smaller than a resolution of 0.5:
        if variance < 0.5:
            return np.nan
        else:
            return (
                get_moment(y, 3) - 3*get_centroid(y)*variance - get_centroid(y)**3
            ) / get_variance(y)**(1.5)

    def get_kurtosis(y):
        """
        Calculates the kurtosis as the fourth standardized moment.
        Ref: https://en.wikipedia.org/wiki/Kurtosis#Pearson_moments

        :param y: the discrete distribution from which one wants to calculate the kurtosis
        :type y: pandas.Series or np.array
        :return: the kurtosis of distribution y
        :return type: float
        """

        variance = get_variance(y)
        # In the limit of a dirac delta, kurtosis should be 3 and variance 0.  However, in the discrete limit,
        # the kurtosis blows up as variance --> 0, hence return nan when variance is smaller than a resolution of 0.5:
        if variance < 0.5:
            return np.nan
        else:
            return (
                get_moment(y, 4) - 4*get_centroid(y)*get_moment(y, 3)
                + 6*get_moment(y, 2)*get_centroid(y)**2 - 3*get_centroid(y)
            ) / get_variance(y)**2

    calculation = dict(
        centroid=get_centroid,
        variance=get_variance,
        skew=get_skew,
        kurtosis=get_kurtosis
    )

    fft_abs = np.abs(np.fft.rfft(x))

    res = [calculation[config["aggtype"]](fft_abs) for config in param]
    index = ['aggtype_"{}"'.format(config["aggtype"]) for config in param]
    return zip(index, res)


Instruction:
Can you implement a function in Python 3 that
calculates the number of peaks of at least support n in the time series x.

Response:
def number_peaks(x, n):
    """
    Calculates the number of peaks of at least support n in the time series x. A peak of support n is defined as a
    subsequence of x where a value occurs, which is bigger than its n neighbours to the left and to the right.

    Hence in the sequence

    >>> x = [3, 0, 0, 4, 0, 0, 13]

    4 is a peak of support 1 and 2 because in the subsequences

    >>> [0, 4, 0]
    >>> [0, 0, 4, 0, 0]

    4 is still the highest value. Here, 4 is not a peak of support 3 because 13 is the 3th neighbour to the right of 4
    and its bigger than 4.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param n: the support of the peak
    :type n: int
    :return: the value of this feature
    :return type: float
    """
    x_reduced = x[n:-n]

    res = None
    for i in range(1, n + 1):
        result_first = (x_reduced > _roll(x, i)[n:-n])

        if res is None:
            res = result_first
        else:
            res &= result_first

        res &= (x_reduced > _roll(x, -i)[n:-n])
    return np.sum(res)


Instruction:
Given the following Python 3 function, write the documentation
def index_mass_quantile(x, param):

    x = np.asarray(x)
    abs_x = np.abs(x)
    s = sum(abs_x)

    if s == 0:
        # all values in x are zero or it has length 0
        return [("q_{}".format(config["q"]), np.NaN) for config in param]
    else:
        # at least one value is not zero
        mass_centralized = np.cumsum(abs_x) / s
        return [("q_{}".format(config["q"]),
                (np.argmax(mass_centralized >= config["q"])+1)/len(x)) for config in param]

Response: This function calculates the relative index i where q% of the mass of the time series x lie left of i.


Instruction:
Write a Python 3 script to
calculate a linear least-squares regression for the values of the time series versus the sequence from 0 to length of the time series minus one. This feature assumes the signal to be uniformly sampled. It will not use the time stamps to fit the model. The parameters control which of the characteristics are returned. Possible extracted attributes are "pvalue", "rvalue", "intercept", "slope", "stderr", see the documentation of linregress for more information. :param x: the time series to calculate the feature of :type x: numpy.ndarray :param param: contains dictionaries {"attr": x} with x an string, the attribute name of the regression model :type param: list :return: the different feature values :return type: pandas.Series

Response:
def linear_trend(x, param):
    """
    Calculate a linear least-squares regression for the values of the time series versus the sequence from 0 to
    length of the time series minus one.
    This feature assumes the signal to be uniformly sampled. It will not use the time stamps to fit the model.
    The parameters control which of the characteristics are returned.

    Possible extracted attributes are "pvalue", "rvalue", "intercept", "slope", "stderr", see the documentation of
    linregress for more information.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param param: contains dictionaries {"attr": x} with x an string, the attribute name of the regression model
    :type param: list
    :return: the different feature values
    :return type: pandas.Series
    """
    # todo: we could use the index of the DataFrame here
    linReg = linregress(range(len(x)), x)

    return [("attr_\"{}\"".format(config["attr"]), getattr(linReg, config["attr"]))
            for config in param]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cwt_coefficients(x, param):

    calculated_cwt = {}
    res = []
    indices = []

    for parameter_combination in param:
        widths = parameter_combination["widths"]
        w = parameter_combination["w"]
        coeff = parameter_combination["coeff"]

        if widths not in calculated_cwt:
            calculated_cwt[widths] = cwt(x, ricker, widths)

        calculated_cwt_for_widths = calculated_cwt[widths]

        indices += ["widths_{}__coeff_{}__w_{}".format(widths, coeff, w)]

        i = widths.index(w)
        if calculated_cwt_for_widths.shape[1] <= coeff:
            res += [np.NaN]
        else:
            res += [calculated_cwt_for_widths[i, coeff]]

    return zip(indices, res)

Response: Calculates the Continuous wavelet transform for the Ricker wavelet transform.


Instruction:
Can you tell what is the following Python 3 function doing
def spkt_welch_density(x, param):

    freq, pxx = welch(x, nperseg=min(len(x), 256))
    coeff = [config["coeff"] for config in param]
    indices = ["coeff_{}".format(i) for i in coeff]

    if len(pxx) <= np.max(coeff):  # There are fewer data points in the time series than requested coefficients

        # filter coefficients that are not contained in pxx
        reduced_coeff = [coefficient for coefficient in coeff if len(pxx) > coefficient]
        not_calculated_coefficients = [coefficient for coefficient in coeff
                                       if coefficient not in reduced_coeff]

        # Fill up the rest of the requested coefficients with np.NaNs
        return zip(indices, list(pxx[reduced_coeff]) + [np.NaN] * len(not_calculated_coefficients))
    else:
        return zip(indices, pxx[coeff])

Response: This function calculates the cross power spectral density of the time series x at different frequencies.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def change_quantiles(x, ql, qh, isabs, f_agg):
    if ql >= qh:
        ValueError("ql={} should be lower than qh={}".format(ql, qh))

    div = np.diff(x)
    if isabs:
        div = np.abs(div)
    # All values that originate from the corridor between the quantiles ql and qh will have the category 0,
    # other will be np.NaN
    try:
        bin_cat = pd.qcut(x, [ql, qh], labels=False)
        bin_cat_0 = bin_cat == 0
    except ValueError:  # Occurs when ql are qh effectively equal, e.g. x is not long enough or is too categorical
        return 0
    # We only count changes that start and end inside the corridor
    ind = (bin_cat_0 & _roll(bin_cat_0, 1))[1:]
    if sum(ind) == 0:
        return 0
    else:
        ind_inside_corridor = np.where(ind == 1)
        aggregator = getattr(np, f_agg)
        return aggregator(div[ind_inside_corridor])

Response: This function fixes a corridor given by the quantiles ql and qh of the distribution of x and calculates the average absolute value of consecutive changes of the series x inside the corridor.


Instruction:
Can you generate the documentation for the following Python 3 function
def c3(x, lag):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    n = x.size
    if 2 * lag >= n:
        return 0
    else:
        return np.mean((_roll(x, 2 * -lag) * _roll(x, -lag) * x)[0:(n - 2 * lag)])

Response: This function calculates the value of the next nonlinearity in a time series.


Instruction:
Create a Python 3 function to
compute the binned entropy of a time series.

Response:
def binned_entropy(x, max_bins):
    """
    First bins the values of x into max_bins equidistant bins.
    Then calculates the value of

    .. math::

        - \\sum_{k=0}^{min(max\\_bins, len(x))} p_k log(p_k) \\cdot \\mathbf{1}_{(p_k > 0)}

    where :math:`p_k` is the percentage of samples in bin :math:`k`.

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param max_bins: the maximal number of bins
    :type max_bins: int
    :return: the value of this feature
    :return type: float
    """
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    hist, bin_edges = np.histogram(x, bins=max_bins)
    probs = hist / x.size
    return - np.sum(p * np.math.log(p) for p in probs if p != 0)


Instruction:
Can you tell what is the following Python 3 function doing
def sample_entropy(x):
    x = np.array(x)

    sample_length = 1 # number of sequential points of the time series
    tolerance = 0.2 * np.std(x) # 0.2 is a common value for r - why?

    n = len(x)
    prev = np.zeros(n)
    curr = np.zeros(n)
    A = np.zeros((1, 1))  # number of matches for m = [1,...,template_length - 1]
    B = np.zeros((1, 1))  # number of matches for m = [1,...,template_length]

    for i in range(n - 1):
        nj = n - i - 1
        ts1 = x[i]
        for jj in range(nj):
            j = jj + i + 1
            if abs(x[j] - ts1) < tolerance:  # distance between two vectors
                curr[jj] = prev[jj] + 1
                temp_ts_length = min(sample_length, curr[jj])
                for m in range(int(temp_ts_length)):
                    A[m] += 1
                    if j < n - 1:
                        B[m] += 1
            else:
                curr[jj] = 0
        for j in range(nj):
            prev[j] = curr[j]

    N = n * (n - 1) / 2
    B = np.vstack(([N], B[0]))

    # sample entropy = -1 * (log (A/B))
    similarity_ratio = A / B
    se = -1 * np.log(similarity_ratio)
    se = np.reshape(se, -1)
    return se[0]

Response: Calculate and return sample entropy of a time series.


Instruction:
Write a Python 3 function for
calculating the autocorrelation of the specified lag.

Response:
def autocorrelation(x, lag):
    """
    Calculates the autocorrelation of the specified lag, according to the formula [1]

    .. math::

        \\frac{1}{(n-l)\sigma^{2}} \\sum_{t=1}^{n-l}(X_{t}-\\mu )(X_{t+l}-\\mu)

    where :math:`n` is the length of the time series :math:`X_i`, :math:`\sigma^2` its variance and :math:`\mu` its
    mean. `l` denotes the lag.

    .. rubric:: References

    [1] https://en.wikipedia.org/wiki/Autocorrelation#Estimation

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param lag: the lag
    :type lag: int
    :return: the value of this feature
    :return type: float
    """
    # This is important: If a series is passed, the product below is calculated
    # based on the index, which corresponds to squaring the series.
    if type(x) is pd.Series:
        x = x.values
    if len(x) < lag:
        return np.nan
    # Slice the relevant subseries based on the lag
    y1 = x[:(len(x)-lag)]
    y2 = x[lag:]
    # Subtract the mean of the whole series x
    x_mean = np.mean(x)
    # The result is sometimes referred to as "covariation"
    sum_product = np.sum((y1 - x_mean) * (y2 - x_mean))
    # Return the normalized unbiased covariance
    v = np.var(x)
    if np.isclose(v, 0):
        return np.NaN
    else:
        return sum_product / ((len(x) - lag) * v)


Instruction:
Here you have a function in Python 3, explain what it does
def quantile(x, q):
    x = pd.Series(x)
    return pd.Series.quantile(x, q)

Response: Calculates the q quantile of x.


Instruction:
Make a summary of the following Python 3 code
def number_crossing_m(x, m):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)
    # From https://stackoverflow.com/questions/3843017/efficiently-detect-sign-changes-in-python
    positive = x > m
    return np.where(np.bitwise_xor(positive[1:], positive[:-1]))[0].size

Response: Calculates the number of crossings of x on m.


Instruction:
Make a summary of the following Python 3 code
def value_count(x, value):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)

    if np.isnan(value):
        return np.isnan(x).sum()
    else:
        return x[x == value].size

Response: Count occurrences of value in time series x.


Instruction:
Write a Python 3 function that can
count observed values within the interval [ min max ).

Response:
def range_count(x, min, max):
    """
    Count observed values within the interval [min, max).

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param min: the inclusive lower bound of the range
    :type min: int or float
    :param max: the exclusive upper bound of the range
    :type max: int or float
    :return: the count of values within the range
    :rtype: int
    """
    return np.sum((x >= min) & (x < max))


Instruction:
Given the following Python 3 function, write the documentation
def approximate_entropy(x, m, r):
    if not isinstance(x, (np.ndarray, pd.Series)):
        x = np.asarray(x)

    N = x.size
    r *= np.std(x)
    if r < 0:
        raise ValueError("Parameter r must be positive.")
    if N <= m+1:
        return 0

    def _phi(m):
        x_re = np.array([x[i:i+m] for i in range(N - m + 1)])
        C = np.sum(np.max(np.abs(x_re[:, np.newaxis] - x_re[np.newaxis, :]),
                          axis=2) <= r, axis=0) / (N-m+1)
        return np.sum(np.log(C)) / (N - m + 1.0)

    return np.abs(_phi(m) - _phi(m + 1))

Response: This method calculates the approximate entropy of a time series with a given level.


Instruction:
Create a Python 3 function for
calculating the friedrich coefficients of the current system of the system of the current system of the current system of the current system of the current system of the current system.

Response:
def friedrich_coefficients(x, param):
    """
    Coefficients of polynomial :math:`h(x)`, which has been fitted to
    the deterministic dynamics of Langevin model

    .. math::
        \dot{x}(t) = h(x(t)) + \mathcal{N}(0,R)

    as described by [1].

    For short time-series this method is highly dependent on the parameters.

    .. rubric:: References

    |  [1] Friedrich et al. (2000): Physics Letters A 271, p. 217-222
    |  *Extracting model equations from experimental data*

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param param: contains dictionaries {"m": x, "r": y, "coeff": z} with x being positive integer, the order of polynom to fit for estimating fixed points of
                    dynamics, y positive float, the number of quantils to use for averaging and finally z, a positive integer corresponding to the returned
                    coefficient
    :type param: list
    :return: the different feature values
    :return type: pandas.Series
    """
    calculated = {}  # calculated is dictionary storing the calculated coefficients {m: {r: friedrich_coefficients}}
    res = {}  # res is a dictionary containg the results {"m_10__r_2__coeff_3": 15.43}

    for parameter_combination in param:
        m = parameter_combination['m']
        r = parameter_combination['r']
        coeff = parameter_combination["coeff"]

        assert coeff >= 0, "Coefficients must be positive or zero. Found {}".format(coeff)

        # calculate the current friedrich coefficients if they do not exist yet
        if m not in calculated:
            calculated[m] = {r: _estimate_friedrich_coefficients(x, m, r)}
        else:
            if r not in calculated[m]:
                calculated[m] = {r: _estimate_friedrich_coefficients(x, m, r)}

        try:
            res["m_{}__r_{}__coeff_{}".format(m, r, coeff)] = calculated[m][r][coeff]
        except IndexError:
            res["m_{}__r_{}__coeff_{}".format(m, r, coeff)] = np.NaN
    return [(key, value) for key, value in res.items()]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def max_langevin_fixed_point(x, r, m):

    coeff = _estimate_friedrich_coefficients(x, m, r)

    try:
        max_fixed_point = np.max(np.real(np.roots(coeff)))
    except (np.linalg.LinAlgError, ValueError):
        return np.nan

    return max_fixed_point

Response: Estimate the maximum fixed point of the deterministic dynamics of Langevin model.


Instruction:
How would you code a function in Python 3 to
calculate a linear least - squares regression for values of the time series x.

Response:
def agg_linear_trend(x, param):
    """
    Calculates a linear least-squares regression for values of the time series that were aggregated over chunks versus
    the sequence from 0 up to the number of chunks minus one.

    This feature assumes the signal to be uniformly sampled. It will not use the time stamps to fit the model.

    The parameters attr controls which of the characteristics are returned. Possible extracted attributes are "pvalue",
    "rvalue", "intercept", "slope", "stderr", see the documentation of linregress for more information.

    The chunksize is regulated by "chunk_len". It specifies how many time series values are in each chunk.

    Further, the aggregation function is controlled by "f_agg", which can use "max", "min" or , "mean", "median"

    :param x: the time series to calculate the feature of
    :type x: numpy.ndarray
    :param param: contains dictionaries {"attr": x, "chunk_len": l, "f_agg": f} with x, f an string and l an int
    :type param: list
    :return: the different feature values
    :return type: pandas.Series
    """
    # todo: we could use the index of the DataFrame here

    calculated_agg = {}
    res_data = []
    res_index = []

    for parameter_combination in param:

        chunk_len = parameter_combination["chunk_len"]
        f_agg = parameter_combination["f_agg"]

        aggregate_result = _aggregate_on_chunks(x, f_agg, chunk_len)
        if f_agg not in calculated_agg or chunk_len not in calculated_agg[f_agg]:
            if chunk_len >= len(x):
                calculated_agg[f_agg] = {chunk_len: np.NaN}
            else:
                lin_reg_result = linregress(range(len(aggregate_result)), aggregate_result)
                calculated_agg[f_agg] = {chunk_len: lin_reg_result}

        attr = parameter_combination["attr"]

        if chunk_len >= len(x):
            res_data.append(np.NaN)
        else:
            res_data.append(getattr(calculated_agg[f_agg][chunk_len], attr))

        res_index.append("f_agg_\"{}\"__chunk_len_{}__attr_\"{}\"".format(f_agg, chunk_len, attr))

    return zip(res_index, res_data)


Instruction:
Given the following Python 3 function, write the documentation
def energy_ratio_by_chunks(x, param):
    res_data = []
    res_index = []
    full_series_energy = np.sum(x ** 2)

    for parameter_combination in param:
        num_segments = parameter_combination["num_segments"]
        segment_focus = parameter_combination["segment_focus"]
        assert segment_focus < num_segments
        assert num_segments > 0

        res_data.append(np.sum(np.array_split(x, num_segments)[segment_focus] ** 2.0)/full_series_energy)
        res_index.append("num_segments_{}__segment_focus_{}".format(num_segments, segment_focus))

    return list(zip(res_index, res_data))

Response: Calculates the sum of squares of chunk i out of N chunks expressed as a ratio with the sum of squares over the whole series.


Instruction:
Can you generate the documentation for the following Python 3 function
def linear_trend_timewise(x, param):
    ix = x.index

    # Get differences between each timestamp and the first timestamp in seconds.
    # Then convert to hours and reshape for linear regression
    times_seconds = (ix - ix[0]).total_seconds()
    times_hours = np.asarray(times_seconds / float(3600))

    linReg = linregress(times_hours, x.values)

    return [("attr_\"{}\"".format(config["attr"]), getattr(linReg, config["attr"]))
            for config in param]

Response: This function calculates a linear least - squares regression for the values of the time series versus the sequence from 0 to length of the time series minus one.


Instruction:
Can you tell what is the following Python 3 function doing
def fit(self, X, y):
        if not isinstance(X, pd.DataFrame):
            X = pd.DataFrame(X.copy())

        if not isinstance(y, pd.Series):
            y = pd.Series(y.copy())

        relevance_table = calculate_relevance_table(
                                X, y, ml_task=self.ml_task, n_jobs=self.n_jobs,
                                chunksize=self.chunksize, fdr_level=self.fdr_level,
                                hypotheses_independent=self.hypotheses_independent,
                                test_for_binary_target_real_feature=self.test_for_binary_target_real_feature)
        self.relevant_features = relevance_table.loc[relevance_table.relevant].feature.tolist()
        self.feature_importances_ = 1.0 - relevance_table.p_value.values
        self.p_values = relevance_table.p_value.values
        self.features = relevance_table.index.tolist()

        return self

Response: This method calculates the relevance table for the given features and returns the fitted estimator with the information which features are relevant.


Instruction:
Explain what the following Python 3 code does
def transform(self, X):
        if self.relevant_features is None:
            raise RuntimeError("You have to call fit before.")

        if isinstance(X, pd.DataFrame):
            return X.copy().loc[:, self.relevant_features]
        else:
            return X[:, self.relevant_features]

Response: Delete all features which were not relevant in the fit phase.


Instruction:
Can you generate the documentation for the following Python 3 function
def download_har_dataset():

    zipurl = 'https://github.com/MaxBenChrist/human-activity-dataset/blob/master/UCI%20HAR%20Dataset.zip?raw=true'

    if os.path.exists(data_file_name_dataset) and os.path.exists(data_file_name_classes):
        _logger.warning("You have already downloaded the Human Activity Data Set.")
        return

    with urlopen(zipurl) as zipresp:
        with ZipFile(BytesIO(zipresp.read())) as zfile:
            zfile.extractall(path=data_file_name)
        zfile.close()

Response: Download the Human Activity Recognition Dataset from UCI ML Repository and store it at the data_file_name_dataset and data_file_name_classes.


Instruction:
Can you implement a function in Python 3 that
adds the features calculated using the timeseries_container and add them to the corresponding rows in the input pandas.DataFrame X. To save some computing time, you should only include those time serieses in the container, that you need. You can set the timeseries container with the method :func:`set_timeseries_container`. :param X: the DataFrame to which the calculated timeseries features will be added. This is *not* the dataframe with the timeseries itself. :type X: pandas.DataFrame :return: The input DataFrame, but with added features. :rtype: pandas.DataFrame

Response:
def transform(self, X):
        """
        Add the features calculated using the timeseries_container and add them to the corresponding rows in the input
        pandas.DataFrame X.

        To save some computing time, you should only include those time serieses in the container, that you
        need. You can set the timeseries container with the method :func:`set_timeseries_container`.

        :param X: the DataFrame to which the calculated timeseries features will be added. This is *not* the
               dataframe with the timeseries itself.
        :type X: pandas.DataFrame

        :return: The input DataFrame, but with added features.
        :rtype: pandas.DataFrame
        """
        if self.timeseries_container is None:
            raise RuntimeError("You have to provide a time series using the set_timeseries_container function before.")

        # Extract only features for the IDs in X.index
        timeseries_container_X = restrict_input_to_index(self.timeseries_container, self.column_id, X.index)

        extracted_features = extract_features(timeseries_container_X,
                                              default_fc_parameters=self.default_fc_parameters,
                                              kind_to_fc_parameters=self.kind_to_fc_parameters,
                                              column_id=self.column_id, column_sort=self.column_sort,
                                              column_kind=self.column_kind, column_value=self.column_value,
                                              chunksize=self.chunksize,
                                              n_jobs=self.n_jobs, show_warnings=self.show_warnings,
                                              disable_progressbar=self.disable_progressbar,
                                              impute_function=self.impute_function,
                                              profile=self.profile,
                                              profiling_filename=self.profiling_filename,
                                              profiling_sorting=self.profiling_sorting)

        X = pd.merge(X, extracted_features, left_index=True, right_index=True, how="left")

        return X


Instruction:
Can you tell what is the following Python 3 function doing
def fit(self, X, y):
        if self.timeseries_container is None:
            raise RuntimeError("You have to provide a time series using the set_timeseries_container function before.")

        self.feature_extractor.set_timeseries_container(self.timeseries_container)

        if self.filter_only_tsfresh_features:
            # Do not merge the time series features to the old features
            X_tmp = pd.DataFrame(index=X.index)
        else:
            X_tmp = X

        X_augmented = self.feature_extractor.transform(X_tmp)

        self.col_to_max, self.col_to_min, self.col_to_median = get_range_values_per_column(X_augmented)
        X_augmented = impute_dataframe_range(X_augmented, col_to_max=self.col_to_max, col_to_median=self.col_to_median,
                                             col_to_min=self.col_to_min)

        self.feature_selector.fit(X_augmented, y)

        return self

Response: Fits the estimator with the information of the features of X and y.


Instruction:
Create a Python 3 function to
calculate the relevance table for the features contained in feature matrix X with respect to target vector y.

Response:
def calculate_relevance_table(X, y, ml_task='auto', n_jobs=defaults.N_PROCESSES, chunksize=defaults.CHUNKSIZE,
                   test_for_binary_target_binary_feature=defaults.TEST_FOR_BINARY_TARGET_BINARY_FEATURE,
                   test_for_binary_target_real_feature=defaults.TEST_FOR_BINARY_TARGET_REAL_FEATURE,
                   test_for_real_target_binary_feature=defaults.TEST_FOR_REAL_TARGET_BINARY_FEATURE,
                   test_for_real_target_real_feature=defaults.TEST_FOR_REAL_TARGET_REAL_FEATURE,
                   fdr_level=defaults.FDR_LEVEL, hypotheses_independent=defaults.HYPOTHESES_INDEPENDENT):
    """
    Calculate the relevance table for the features contained in feature matrix `X` with respect to target vector `y`.
    The relevance table is calculated for the intended machine learning task `ml_task`.

    To accomplish this for each feature from the input pandas.DataFrame an univariate feature significance test
    is conducted. Those tests generate p values that are then evaluated by the Benjamini Hochberg procedure to
    decide which features to keep and which to delete.

    We are testing

        :math:`H_0` = the Feature is not relevant and should not be added

    against

        :math:`H_1` = the Feature is relevant and should be kept

    or in other words

        :math:`H_0` = Target and Feature are independent / the Feature has no influence on the target

        :math:`H_1` = Target and Feature are associated / dependent

    When the target is binary this becomes

        :math:`H_0 = \\left( F_{\\text{target}=1} = F_{\\text{target}=0} \\right)`

        :math:`H_1 = \\left( F_{\\text{target}=1} \\neq F_{\\text{target}=0} \\right)`

    Where :math:`F` is the distribution of the target.

    In the same way we can state the hypothesis when the feature is binary

        :math:`H_0 =  \\left( T_{\\text{feature}=1} = T_{\\text{feature}=0} \\right)`

        :math:`H_1 = \\left( T_{\\text{feature}=1} \\neq T_{\\text{feature}=0} \\right)`

    Here :math:`T` is the distribution of the target.

    TODO: And for real valued?

    :param X: Feature matrix in the format mentioned before which will be reduced to only the relevant features.
              It can contain both binary or real-valued features at the same time.
    :type X: pandas.DataFrame

    :param y: Target vector which is needed to test which features are relevant. Can be binary or real-valued.
    :type y: pandas.Series or numpy.ndarray

    :param ml_task: The intended machine learning task. Either `'classification'`, `'regression'` or `'auto'`.
                    Defaults to `'auto'`, meaning the intended task is inferred from `y`.
                    If `y` has a boolean, integer or object dtype, the task is assumend to be classification,
                    else regression.
    :type ml_task: str

    :param test_for_binary_target_binary_feature: Which test to be used for binary target, binary feature (currently unused)
    :type test_for_binary_target_binary_feature: str

    :param test_for_binary_target_real_feature: Which test to be used for binary target, real feature
    :type test_for_binary_target_real_feature: str

    :param test_for_real_target_binary_feature: Which test to be used for real target, binary feature (currently unused)
    :type test_for_real_target_binary_feature: str

    :param test_for_real_target_real_feature: Which test to be used for real target, real feature (currently unused)
    :type test_for_real_target_real_feature: str

    :param fdr_level: The FDR level that should be respected, this is the theoretical expected percentage of irrelevant
                      features among all created features.
    :type fdr_level: float

    :param hypotheses_independent: Can the significance of the features be assumed to be independent?
                                   Normally, this should be set to False as the features are never
                                   independent (e.g. mean and median)
    :type hypotheses_independent: bool

    :param n_jobs: Number of processes to use during the p-value calculation
    :type n_jobs: int

    :param chunksize: The size of one chunk that is submitted to the worker
        process for the parallelisation.  Where one chunk is defined as a
        singular time series for one id and one kind. If you set the chunksize
        to 10, then it means that one task is to calculate all features for 10
        time series.  If it is set it to None, depending on distributor,
        heuristics are used to find the optimal chunksize. If you get out of
        memory exceptions, you can try it with the dask distributor and a
        smaller chunksize.
    :type chunksize: None or int

    :return: A pandas.DataFrame with each column of the input DataFrame X as index with information on the significance
             of this particular feature. The DataFrame has the columns
             "Feature",
             "type" (binary, real or const),
             "p_value" (the significance of this feature as a p-value, lower means more significant)
             "relevant" (True if the Benjamini Hochberg procedure rejected the null hypothesis [the feature is
             not relevant] for this feature)
    :rtype: pandas.DataFrame
    """
    if ml_task not in ['auto', 'classification', 'regression']:
        raise ValueError('ml_task must be one of: \'auto\', \'classification\', \'regression\'')
    elif ml_task == 'auto':
        ml_task = infer_ml_task(y)

    if n_jobs == 0:
        map_function = map
    else:
        pool = Pool(n_jobs)
        map_function = partial(pool.map, chunksize=chunksize)

    relevance_table = pd.DataFrame(index=pd.Series(X.columns, name='feature'))
    relevance_table['feature'] = relevance_table.index
    relevance_table['type'] = pd.Series(
        map_function(get_feature_type, [X[feature] for feature in relevance_table.index]),
        index=relevance_table.index
    )
    table_real = relevance_table[relevance_table.type == 'real'].copy()
    table_binary = relevance_table[relevance_table.type == 'binary'].copy()

    table_const = relevance_table[relevance_table.type == 'constant'].copy()
    table_const['p_value'] = np.NaN
    table_const['relevant'] = False

    if len(table_const) == len(relevance_table):
        return table_const

    if ml_task == 'classification':
        tables = []
        for label in y.unique():
            _test_real_feature = partial(target_binary_feature_real_test, y=(y == label),
                                              test=test_for_binary_target_real_feature)
            _test_binary_feature = partial(target_binary_feature_binary_test, y=(y == label))
            tmp = _calculate_relevance_table_for_implicit_target(
                table_real, table_binary, X, _test_real_feature, _test_binary_feature, hypotheses_independent,
                fdr_level, map_function
            )
            tables.append(tmp)
        relevance_table = combine_relevance_tables(tables)
    elif ml_task == 'regression':
        _test_real_feature = partial(target_real_feature_real_test, y=y)
        _test_binary_feature = partial(target_real_feature_binary_test, y=y)
        relevance_table = _calculate_relevance_table_for_implicit_target(
            table_real, table_binary, X, _test_real_feature, _test_binary_feature, hypotheses_independent, fdr_level,
            map_function
        )

    relevance_table = pd.concat([relevance_table, table_const], axis=0)

    if n_jobs != 0:
        pool.close()
        pool.terminate()
        pool.join()

    if sum(relevance_table['relevant']) == 0:
        _logger.warning("No feature was found relevant for {} for fdr level = {}. "
                        "Consider using a lower fdr level or other features.".format(ml_task, fdr_level))

    return relevance_table


Instruction:
Write a Python 3 script for
inferring the machine learning task to select for.

Response:
def infer_ml_task(y):
    """
    Infer the machine learning task to select for.
    The result will be either `'regression'` or `'classification'`.
    If the target vector only consists of integer typed values or objects, we assume the task is `'classification'`.
    Else `'regression'`.

    :param y: The target vector y.
    :type y: pandas.Series
    :return: 'classification' or 'regression'
    :rtype: str
    """
    if y.dtype.kind in np.typecodes['AllInteger'] or y.dtype == np.object:
        ml_task = 'classification'
    else:
        ml_task = 'regression'

    _logger.warning('Infered {} as machine learning task'.format(ml_task))
    return ml_task


Instruction:
Write a Python 3 script for
creating a combined relevance table out of a list of relevance tables.

Response:
def combine_relevance_tables(relevance_tables):
    """
    Create a combined relevance table out of a list of relevance tables,
    aggregating the p-values and the relevances.

    :param relevance_tables: A list of relevance tables
    :type relevance_tables: List[pd.DataFrame]
    :return: The combined relevance table
    :rtype: pandas.DataFrame
    """
    def _combine(a, b):
        a.relevant |= b.relevant
        a.p_value = a.p_value.combine(b.p_value, min, 1)
        return a

    return reduce(_combine, relevance_tables)


Instruction:
Given the following Python 3 function, write the documentation
def get_feature_type(feature_column):
    n_unique_values = len(set(feature_column.values))
    if n_unique_values == 1:
        _logger.warning("[test_feature_significance] Feature {} is constant".format(feature_column.name))
        return 'constant'
    elif n_unique_values == 2:
        return 'binary'
    else:
        return 'real'

Response: Determines if a given feature is real binary or real.


Instruction:
Here you have a function in Python 3, explain what it does
def extract_features(timeseries_container, default_fc_parameters=None,
                     kind_to_fc_parameters=None,
                     column_id=None, column_sort=None, column_kind=None, column_value=None,
                     chunksize=defaults.CHUNKSIZE,
                     n_jobs=defaults.N_PROCESSES, show_warnings=defaults.SHOW_WARNINGS,
                     disable_progressbar=defaults.DISABLE_PROGRESSBAR,
                     impute_function=defaults.IMPUTE_FUNCTION,
                     profile=defaults.PROFILING,
                     profiling_filename=defaults.PROFILING_FILENAME,
                     profiling_sorting=defaults.PROFILING_SORTING,
                     distributor=None):
    """
    Extract features from

    * a :class:`pandas.DataFrame` containing the different time series

    or

    * a dictionary of :class:`pandas.DataFrame` each containing one type of time series

    In both cases a :class:`pandas.DataFrame` with the calculated features will be returned.

    For a list of all the calculated time series features, please see the
    :class:`~tsfresh.feature_extraction.settings.ComprehensiveFCParameters` class,
    which is used to control which features with which parameters are calculated.

    For a detailed explanation of the different parameters and data formats please see :ref:`data-formats-label`.

    Examples
    ========

    >>> from tsfresh.examples import load_robot_execution_failures
    >>> from tsfresh import extract_features
    >>> df, _ = load_robot_execution_failures()
    >>> X = extract_features(df, column_id='id', column_sort='time')

    :param timeseries_container: The pandas.DataFrame with the time series to compute the features for, or a
            dictionary of pandas.DataFrames.
    :type timeseries_container: pandas.DataFrame or dict

    :param default_fc_parameters: mapping from feature calculator names to parameters. Only those names
           which are keys in this dict will be calculated. See the class:`ComprehensiveFCParameters` for
           more information.
    :type default_fc_parameters: dict

    :param kind_to_fc_parameters: mapping from kind names to objects of the same type as the ones for
            default_fc_parameters. If you put a kind as a key here, the fc_parameters
            object (which is the value), will be used instead of the default_fc_parameters. This means that kinds, for
            which kind_of_fc_parameters doe not have any entries, will be ignored by the feature selection.
    :type kind_to_fc_parameters: dict

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_sort: The name of the sort column.
    :type column_sort: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
    :type n_jobs: int

    :param chunksize: The size of one chunk that is submitted to the worker
        process for the parallelisation.  Where one chunk is defined as a
        singular time series for one id and one kind. If you set the chunksize
        to 10, then it means that one task is to calculate all features for 10
        time series.  If it is set it to None, depending on distributor,
        heuristics are used to find the optimal chunksize. If you get out of
        memory exceptions, you can try it with the dask distributor and a
        smaller chunksize.
    :type chunksize: None or int

    :param: show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
    :type show_warnings: bool

    :param disable_progressbar: Do not show a progressbar while doing the calculation.
    :type disable_progressbar: bool

    :param impute_function: None, if no imputing should happen or the function to call for imputing.
    :type impute_function: None or callable

    :param profile: Turn on profiling during feature extraction
    :type profile: bool

    :param profiling_sorting: How to sort the profiling results (see the documentation of the profiling package for
           more information)
    :type profiling_sorting: basestring

    :param profiling_filename: Where to save the profiling results.
    :type profiling_filename: basestring

    :param distributor: Advanced parameter: set this to a class name that you want to use as a
             distributor. See the utilities/distribution.py for more information. Leave to None, if you want
             TSFresh to choose the best distributor.
    :type distributor: class

    :return: The (maybe imputed) DataFrame containing extracted features.
    :rtype: pandas.DataFrame
    """

    # Always use the standardized way of storing the data.
    # See the function normalize_input_to_internal_representation for more information.
    df_melt, column_id, column_kind, column_value = \
        dataframe_functions._normalize_input_to_internal_representation(
            timeseries_container=timeseries_container,
            column_id=column_id, column_kind=column_kind,
            column_sort=column_sort,
            column_value=column_value)
    # Use the standard setting if the user did not supply ones himself.
    if default_fc_parameters is None and kind_to_fc_parameters is None:
        default_fc_parameters = ComprehensiveFCParameters()
    elif default_fc_parameters is None and kind_to_fc_parameters is not None:
        default_fc_parameters = {}

    # If requested, do profiling (advanced feature)
    if profile:
        profiler = profiling.start_profiling()

    with warnings.catch_warnings():
        if not show_warnings:
            warnings.simplefilter("ignore")
        else:
            warnings.simplefilter("default")

        result = _do_extraction(df=df_melt,
                                column_id=column_id, column_value=column_value,
                                column_kind=column_kind,
                                n_jobs=n_jobs, chunk_size=chunksize,
                                disable_progressbar=disable_progressbar,
                                default_fc_parameters=default_fc_parameters,
                                kind_to_fc_parameters=kind_to_fc_parameters,
                                distributor=distributor)

        # Impute the result if requested
        if impute_function is not None:
            impute_function(result)

    # Turn off profiling if it was turned on
    if profile:
        profiling.end_profiling(profiler, filename=profiling_filename,
                                sorting=profiling_sorting)

    return result

Response: Extract features from a time series container.


Instruction:
How would you implement a function in Python 3 that
generates the data in a list of individual time series containers.

Response:
def generate_data_chunk_format(df, column_id, column_kind, column_value):
    """Converts the dataframe df in into a list of individual time seriess.

    E.g. the DataFrame

        ====  ======  =========
          id  kind          val
        ====  ======  =========
           1  a       -0.21761
           1  a       -0.613667
           1  a       -2.07339
           2  b       -0.576254
           2  b       -1.21924
        ====  ======  =========

    into

        [(1, 'a', pd.Series([-0.217610, -0.613667, -2.073386]),
         (2, 'b', pd.Series([-0.576254, -1.219238])]

    The data is separated out into those single time series and the _do_extraction_on_chunk is
    called on each of them. The results are then combined into a single pandas DataFrame.

    The call is either happening in parallel or not and is showing a progress bar or not depending
    on the given flags.

    :param df: The dataframe in the normalized format which is used for extraction.
    :type df: pd.DataFrame

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :return: the data in chunks
    :rtype: list
    """
    MAX_VALUES_GROUPBY = 2147483647

    if df[[column_id, column_kind]].nunique().prod() >= MAX_VALUES_GROUPBY:
        _logger.error(
            "The time series container has {} different ids and {} different kind of time series, in total {} possible combinations. "
            "Due to a limitation in pandas we can only process a maximum of {} id/kind combinations. Please reduce your time series container and restart "
            "the calculation".format(
                df[column_id].nunique(), df[column_kind].nunique(),
                df[[column_id, column_kind]].nunique().prod(), MAX_VALUES_GROUPBY)
        )
        raise ValueError(
            "Number of ids/kinds are too high. Please reduce your data size and run feature extraction again.")
    data_in_chunks = [x + (y,) for x, y in
                      df.groupby([column_id, column_kind], as_index=True)[column_value]]

    return data_in_chunks


Instruction:
How would you implement a function in Python 3 that
wraps around the _do_extraction_on_chunk, which calls it on all chunks in the data frame. A chunk is a subset of the data, with a given kind and id - so a single time series. The data is separated out into those single time series and the _do_extraction_on_chunk is called on each of them. The results are then combined into a single pandas DataFrame. The call is either happening in parallel or not and is showing a progress bar or not depending on the given flags. :param df: The dataframe in the normalized format which is used for extraction. :type df: pd.DataFrame :param default_fc_parameters: mapping from feature calculator names to parameters. Only those names which are keys in this dict will be calculated. See the class:`ComprehensiveFCParameters` for more information. :type default_fc_parameters: dict :param kind_to_fc_parameters: mapping from kind names to objects of the same type as the ones for default_fc_parameters. If you put a kind as a key here, the fc_parameters object (which is the value), will be used instead of the default_fc_parameters. :type kind_to_fc_parameters: dict :param column_id: The name of the id column to group by. :type column_id: str :param column_kind: The name of the column keeping record on the kind of the value. :type column_kind: str :param column_value: The name for the column keeping the value itself. :type column_value: str :param chunk_size: The size of one chunk for the parallelization :type chunk_size: None or int :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used. :type n_jobs: int :param disable_progressbar: Do not show a progressbar while doing the calculation. :type disable_progressbar: bool :param distributor: Advanced parameter: See the utilities/distribution.py for more information. Leave to None, if you want TSFresh to choose the best distributor. :type distributor: DistributorBaseClass :return: the extracted features :rtype: pd.DataFrame

Response:
def _do_extraction(df, column_id, column_value, column_kind,
                   default_fc_parameters, kind_to_fc_parameters,
                   n_jobs, chunk_size, disable_progressbar, distributor):
    """
    Wrapper around the _do_extraction_on_chunk, which calls it on all chunks in the data frame.
    A chunk is a subset of the data, with a given kind and id - so a single time series.

    The data is separated out into those single time series and the _do_extraction_on_chunk is
    called on each of them. The results are then combined into a single pandas DataFrame.

    The call is either happening in parallel or not and is showing a progress bar or not depending
    on the given flags.

    :param df: The dataframe in the normalized format which is used for extraction.
    :type df: pd.DataFrame

    :param default_fc_parameters: mapping from feature calculator names to parameters. Only those names
           which are keys in this dict will be calculated. See the class:`ComprehensiveFCParameters` for
           more information.
    :type default_fc_parameters: dict

    :param kind_to_fc_parameters: mapping from kind names to objects of the same type as the ones for
            default_fc_parameters. If you put a kind as a key here, the fc_parameters
            object (which is the value), will be used instead of the default_fc_parameters.
    :type kind_to_fc_parameters: dict

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param chunk_size: The size of one chunk for the parallelization
    :type chunk_size: None or int

    :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
    :type n_jobs: int

    :param disable_progressbar: Do not show a progressbar while doing the calculation.
    :type disable_progressbar: bool

    :param distributor: Advanced parameter:  See the utilities/distribution.py for more information.
                         Leave to None, if you want TSFresh to choose the best distributor.
    :type distributor: DistributorBaseClass

    :return: the extracted features
    :rtype: pd.DataFrame
    """

    data_in_chunks = generate_data_chunk_format(df, column_id, column_kind, column_value)

    if distributor is None:
        if n_jobs == 0:
            distributor = MapDistributor(disable_progressbar=disable_progressbar,
                                         progressbar_title="Feature Extraction")
        else:
            distributor = MultiprocessingDistributor(n_workers=n_jobs,
                                                     disable_progressbar=disable_progressbar,
                                                     progressbar_title="Feature Extraction")

    if not isinstance(distributor, DistributorBaseClass):
        raise ValueError("the passed distributor is not an DistributorBaseClass object")

    kwargs = dict(default_fc_parameters=default_fc_parameters,
                  kind_to_fc_parameters=kind_to_fc_parameters)

    result = distributor.map_reduce(_do_extraction_on_chunk, data=data_in_chunks,
                                    chunk_size=chunk_size,
                                    function_kwargs=kwargs)
    distributor.close()

    # Return a dataframe in the typical form (id as index and feature names as columns)
    result = pd.DataFrame(result)
    if result.columns.contains("value"):
        result["value"] = result["value"].astype(float)

    if len(result) != 0:
        result = result.pivot("id", "variable", "value")
        result.index = result.index.astype(df[column_id].dtype)

    return result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _do_extraction_on_chunk(chunk, default_fc_parameters, kind_to_fc_parameters):
    sample_id, kind, data = chunk
    if kind_to_fc_parameters and kind in kind_to_fc_parameters:
        fc_parameters = kind_to_fc_parameters[kind]
    else:
        fc_parameters = default_fc_parameters

    def _f():
        for function_name, parameter_list in fc_parameters.items():
            func = getattr(feature_calculators, function_name)

            # If the function uses the index, pass is at as a pandas Series.
            # Otherwise, convert to numpy array
            if getattr(func, 'input', False) == 'pd.Series':
                # If it has a required index type, check that the data has the right index type.
                index_type = getattr(func, 'index_type', None)
                if index_type is not None:
                    try:
                        assert isinstance(data.index, index_type)
                    except AssertionError:
                        warnings.warn(
                            "{} requires the data to have a index of type {}. Results will "
                            "not be calculated".format(function_name, index_type)
                        )
                        continue
                x = data
            else:
                x = data.values

            if func.fctype == "combiner":
                result = func(x, param=parameter_list)
            else:
                if parameter_list:
                    result = ((convert_to_output_format(param), func(x, **param)) for param in
                              parameter_list)
                else:
                    result = [("", func(x))]

            for key, item in result:
                feature_name = str(kind) + "__" + func.__name__
                if key:
                    feature_name += "__" + str(key)
                yield {"variable": feature_name, "value": item, "id": sample_id}

    return list(_f())

Response: This function is used to extract all features from a chunk.


Instruction:
Here you have a function in Python 3, explain what it does
def get_config_from_string(parts):
    relevant_parts = parts[2:]
    if not relevant_parts:
        return

    config_kwargs = [s.rsplit("_", 1)[0] for s in relevant_parts]
    config_values = [s.rsplit("_", 1)[1] for s in relevant_parts]

    dict_if_configs = {}

    for key, value in zip(config_kwargs, config_values):
        if value.lower() == "nan":
            dict_if_configs[key] = np.NaN
        elif value.lower() == "-inf":
            dict_if_configs[key] = np.NINF
        elif value.lower() == "inf":
            dict_if_configs[key] = np.PINF
        else:
            dict_if_configs[key] = ast.literal_eval(value)

    return dict_if_configs

Response: This function extracts the configuration of a certain function from the column name.


Instruction:
Given the following Python 3 function, write the documentation
def convert_to_output_format(param):

    def add_parenthesis_if_string_value(x):
        if isinstance(x, string_types):
            return '"' + str(x) + '"'
        else:
            return str(x)

    return "__".join(str(key) + "_" + add_parenthesis_if_string_value(param[key]) for key in sorted(param.keys()))

Response: Helper function to convert parameters to a valid string in the output format.


Instruction:
Given the following Python 3 function, write the documentation
def end_profiling(profiler, filename, sorting=None):
    profiler.disable()
    s = six.StringIO()
    ps = pstats.Stats(profiler, stream=s).sort_stats(sorting)
    ps.print_stats()

    with open(filename, "w+") as f:
        _logger.info("[calculate_ts_features] Finished profiling of time series feature extraction")
        f.write(s.getvalue())

Response:
End the profiling process and write out the profiled
    data into a file.


Instruction:
Can you tell what is the following Python 3 function doing
def fit(self, X, y=None):
            if not isinstance(X, pd.DataFrame):
                X = pd.DataFrame(X)

            col_to_max, col_to_min, col_to_median = get_range_values_per_column(X)

            if self.col_to_NINF_repl_preset is not None:
                if not set(X.columns) >= set(self.col_to_NINF_repl_preset.keys()):
                    raise ValueError("Preset dictionary 'col_to_NINF_repl_preset' contain more keys "
                                     "than the column names in X")
                col_to_min.update(self.col_to_NINF_repl_preset)
            self._col_to_NINF_repl = col_to_min

            if self.col_to_PINF_repl_preset is not None:
                if not set(X.columns) >= set(self.col_to_PINF_repl_preset.keys()):
                    raise ValueError("Preset dictionary 'col_to_PINF_repl_preset' contain more keys "
                                     "than the column names in X")
                col_to_max.update(self.col_to_PINF_repl_preset)
            self._col_to_PINF_repl = col_to_max

            if self.col_to_NAN_repl_preset is not None:
                if not set(X.columns) >= set(self.col_to_NAN_repl_preset.keys()):
                    raise ValueError("Preset dictionary 'col_to_NAN_repl_preset' contain more keys "
                                     "than the column names in X")
                col_to_median.update(self.col_to_NAN_repl_preset)
            self._col_to_NAN_repl = col_to_median

            return self

Response: Compute the min max and median for all columns in the DataFrame.


Instruction:
Can you tell what is the following Python 3 function doing
def transform(self, X):

            if not isinstance(X, pd.DataFrame):
                X = pd.DataFrame(X)

            if self._col_to_NINF_repl is None or self._col_to_PINF_repl is None or self._col_to_NAN_repl is None:
                raise NotFittedError("PerColumnImputer is not fitted")

            X = impute_dataframe_range(X, self._col_to_PINF_repl, self._col_to_NINF_repl, self._col_to_NAN_repl)

            return X

Response:
Column - wise replace all NaNs - inf and + inf in the DataFrame X with average / extreme
            values from the provided dictionaries.


Instruction:
How would you implement a function in Python 3 that
downloads the Robot Execution Failures LP1 Data Set from the UCI Machine Learning Repository and store it locally.

Response:
def download_robot_execution_failures():
    """
    Download the Robot Execution Failures LP1 Data Set[#1] from the UCI Machine Learning Repository [#2] and store it
    locally.

    :return:

    Examples
    ========

    >>> from tsfresh.examples import download_robot_execution_failures
    >>> download_robot_execution_failures()
    """
    if os.path.exists(data_file_name):
        _logger.warning("You have already downloaded the Robot Execution Failures LP1 Data Set.")
        return

    if not os.access(module_path, os.W_OK):
        raise RuntimeError("You don't have the necessary permissions to download the Robot Execution Failures LP1 Data "
                           "Set into the module path. Consider installing the module in a virtualenv you "
                           "own or run this function with appropriate permissions.")

    os.makedirs(os.path.dirname(data_file_name))

    r = requests.get(UCI_MLD_REF_URL)

    if r.status_code != 200:
        raise RuntimeError("Could not download the Robot Execution Failures LP1 Data Set from the UCI Machine Learning "
                           "Repository. HTTP status code: {}".format(r.status_code))

    with open(data_file_name, "w") as f:
        f.write(r.text)


Instruction:
Can you write a function in Python 3 where it
loads the Robot Execution Failures LP1 Data Set [ 1 ].

Response:
def load_robot_execution_failures(multiclass=False):
    """
    Load the Robot Execution Failures LP1 Data Set[1].
    The Time series are passed as a flat DataFrame.

    Examples
    ========

    >>> from tsfresh.examples import load_robot_execution_failures
    >>> df, y = load_robot_execution_failures()
    >>> print(df.shape)
    (1320, 8)

    :param multiclass: If True, return all target labels. The default returns only "normal" vs all other labels.
    :type multiclass: bool
    :return: time series data as :class:`pandas.DataFrame` and target vector as :class:`pandas.Series`
    :rtype: tuple
    """
    if not os.path.exists(data_file_name):
        raise RuntimeError(UCI_MLD_REF_MSG)

    id_to_target = {}
    df_rows = []

    with open(data_file_name) as f:
        cur_id = 0
        time = 0

        for line in f.readlines():
            # New sample --> increase id, reset time and determine target
            if line[0] not in ['\t', '\n']:
                cur_id += 1
                time = 0
                if multiclass:
                    id_to_target[cur_id] = line.strip()
                else:
                    id_to_target[cur_id] = (line.strip() == 'normal')
            # Data row --> split and convert values, create complete df row
            elif line[0] == '\t':
                values = list(map(int, line.split('\t')[1:]))
                df_rows.append([cur_id, time] + values)
                time += 1

    df = pd.DataFrame(df_rows, columns=['id', 'time', 'F_x', 'F_y', 'F_z', 'T_x', 'T_y', 'T_z'])
    y = pd.Series(id_to_target)

    return df, y