Instruction:
Given the following Python 3 function, write the documentation
def score(self, idx, warp_matrix):
        # Transform source points with estimated transformation
        trg_fit = scipy.dot(warp_matrix, np.concatenate((self.src_pts[idx, :], np.ones((len(idx), 1))), axis=1).T).T
        # Compute error in transformation
        err_per_point = np.sqrt(np.sum((self.trg_pts[idx, :] - trg_fit[:, :2])**2, axis=1))  # sum squared error per row
        return err_per_point

Response: Estimate the registration error of the estimated transformation matrix


Instruction:
Can you implement a function in Python 3 that
creates a view of array that is larger than a given size window.

Response:
def rolling_window(array, window=(0,), asteps=None, wsteps=None, axes=None, toend=True):
    """Create a view of `array` which for every point gives the n-dimensional
    neighbourhood of size window. New dimensions are added at the end of
    `array` or after the corresponding original dimension.

    Parameters
    ----------
    array : array_like
        Array to which the rolling window is applied.
    window : int or tuple
        Either a single integer to create a window of only the last axis or a
        tuple to create it for the last len(window) axes. 0 can be used as a
        to ignore a dimension in the window.
    asteps : tuple
        Aligned at the last axis, new steps for the original array, ie. for
        creation of non-overlapping windows. (Equivalent to slicing result)
    wsteps : int or tuple (same size as window)
        steps for the added window dimensions. These can be 0 to repeat values
        along the axis.
    axes: int or tuple
        If given, must have the same size as window. In this case window is
        interpreted as the size in the dimension given by axes. IE. a window
        of (2, 1) is equivalent to window=2 and axis=-2.
    toend : bool
        If False, the new dimensions are right after the corresponding original
        dimension, instead of at the end of the array. Adding the new axes at the
        end makes it easier to get the neighborhood, however toend=False will give
        a more intuitive result if you view the whole array.

    Returns
    -------
    A view on `array` which is smaller to fit the windows and has windows added
    dimensions (0s not counting), ie. every point of `array` is an array of size
    window.

    Examples
    --------
    >>> a = np.arange(9).reshape(3,3)
    >>> rolling_window(a, (2,2))
    array([[[[0, 1],
             [3, 4]],

            [[1, 2],
             [4, 5]]],

           [[[3, 4],
             [6, 7]],

            [[4, 5],
             [7, 8]]]])

    Or to create non-overlapping windows, but only along the first dimension:
    >>> rolling_window(a, (2,0), asteps=(2,1))
    array([[[0, 3],
            [1, 4],
            [2, 5]]])

    Note that the 0 is discared, so that the output dimension is 3:
    >>> rolling_window(a, (2,0), asteps=(2,1)).shape
    (1, 3, 2)

    This is useful for example to calculate the maximum in all (overlapping)
    2x2 submatrixes:
    >>> rolling_window(a, (2,2)).max((2,3))
    array([[4, 5],
           [7, 8]])

    Or delay embedding (3D embedding with delay 2):
    >>> x = np.arange(10)
    >>> rolling_window(x, 3, wsteps=2)
    array([[0, 2, 4],
           [1, 3, 5],
           [2, 4, 6],
           [3, 5, 7],
           [4, 6, 8],
           [5, 7, 9]])
    """
    # pylint: disable=too-many-branches
    # pylint: disable=too-many-statements

    array = np.asarray(array)
    orig_shape = np.asarray(array.shape)
    window = np.atleast_1d(window).astype(int)  # maybe crude to cast to int...

    if axes is not None:
        axes = np.atleast_1d(axes)
        new_window = np.zeros(array.ndim, dtype=int)
        for axis, size in zip(axes, window):
            new_window[axis] = size
        window = new_window

    # Check if window is legal:
    if window.ndim > 1:
        raise ValueError("`window` must be one-dimensional.")
    if np.any(window < 0):
        raise ValueError("All elements of `window` must be larger then 1.")
    if len(array.shape) < len(window):
        raise ValueError("`window` length must be less or equal `array` dimension.")

    _asteps = np.ones_like(orig_shape)
    if asteps is not None:
        asteps = np.atleast_1d(asteps)
        if asteps.ndim != 1:
            raise ValueError("`asteps` must be either a scalar or one dimensional.")
        if len(asteps) > array.ndim:
            raise ValueError("`asteps` cannot be longer then the `array` dimension.")
        # does not enforce alignment, so that steps can be same as window too.
        _asteps[-len(asteps):] = asteps

        if np.any(asteps < 1):
            raise ValueError("All elements of `asteps` must be larger then 1.")
    asteps = _asteps

    _wsteps = np.ones_like(window)
    if wsteps is not None:
        wsteps = np.atleast_1d(wsteps)
        if wsteps.shape != window.shape:
            raise ValueError("`wsteps` must have the same shape as `window`.")
        if np.any(wsteps < 0):
            raise ValueError("All elements of `wsteps` must be larger then 0.")

        _wsteps[:] = wsteps
        _wsteps[window == 0] = 1  # make sure that steps are 1 for non-existing dims.
    wsteps = _wsteps

    # Check that the window would not be larger then the original:
    if np.any(orig_shape[-len(window):] < window * wsteps):
        raise ValueError("`window` * `wsteps` larger then `array` in at least one dimension.")

    new_shape = orig_shape  # just renaming...

    # For calculating the new shape 0s must act like 1s:
    _window = window.copy()
    _window[_window == 0] = 1

    new_shape[-len(window):] += wsteps - _window * wsteps
    new_shape = (new_shape + asteps - 1) // asteps
    # make sure the new_shape is at least 1 in any "old" dimension (ie. steps
    # is (too) large, but we do not care.
    new_shape[new_shape < 1] = 1
    shape = new_shape

    strides = np.asarray(array.strides)
    strides *= asteps
    new_strides = array.strides[-len(window):] * wsteps

    # The full new shape and strides:
    if toend:
        new_shape = np.concatenate((shape, window))
        new_strides = np.concatenate((strides, new_strides))
    else:
        _ = np.zeros_like(shape)
        _[-len(window):] = window
        _window = _.copy()
        _[-len(window):] = new_strides
        _new_strides = _

        new_shape = np.zeros(len(shape) * 2, dtype=int)
        new_strides = np.zeros(len(shape) * 2, dtype=int)

        new_shape[::2] = shape
        new_strides[::2] = strides
        new_shape[1::2] = _window
        new_strides[1::2] = _new_strides

    new_strides = new_strides[new_shape != 0]
    new_shape = new_shape[new_shape != 0]

    return np.lib.stride_tricks.as_strided(array, shape=new_shape, strides=new_strides)


Instruction:
Implement a function in Python 3 to
function to warp input image using a linear transformation

Response:
def warp(self, warp_matrix, img, iflag=cv2.INTER_NEAREST):
        """ Function to warp input image given an estimated 2D linear transformation

        :param warp_matrix: Linear 2x3 matrix to use to linearly warp the input images
        :type warp_matrix: ndarray
        :param img: Image to be warped with estimated transformation
        :type img: ndarray
        :param iflag: Interpolation flag, specified interpolation using during resampling of warped image
        :type iflag: cv2.INTER_*
        :return: Warped image using the linear matrix
        """

        height, width = img.shape[:2]
        warped_img = np.zeros_like(img, dtype=img.dtype)

        # Check if image to warp is 2D or 3D. If 3D need to loop over channels
        if (self.interpolation_type == InterpolationType.LINEAR) or img.ndim == 2:
            warped_img = cv2.warpAffine(img.astype(np.float32), warp_matrix, (width, height),
                                        flags=iflag).astype(img.dtype)

        elif img.ndim == 3:
            for idx in range(img.shape[-1]):
                warped_img[..., idx] = cv2.warpAffine(img[..., idx].astype(np.float32), warp_matrix, (width, height),
                                                      flags=iflag).astype(img.dtype)
        else:
            raise ValueError('Image has incorrect number of dimensions: {}'.format(img.ndim))

        return warped_img


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_registration_suspicious(warp_matrix):
        if warp_matrix is None:
            return 1

        cos_theta = np.trace(warp_matrix[:2, :2]) / 2
        rot_angle = np.arccos(cos_theta)
        transl_norm = np.linalg.norm(warp_matrix[:, 2])
        return 1 if int((rot_angle > MAX_ROTATION) or (transl_norm > MAX_TRANSLATION)) else 0

Response: Static method that checks if estimated linear transformation could be unplausible and returns 1 if it is not.


Instruction:
How would you explain what the following Python 3 function does
def register(self, src, trg, trg_mask=None, src_mask=None):
        # Initialise instance of CrossCorr object
        ccreg = registration.CrossCorr()
        # padding_value = 0
        # Compute translation between pair of images
        model = ccreg.fit(src, reference=trg)
        # Get translation as an array
        translation = [-x for x in model.toarray().tolist()[0]]
        # Fill in transformation matrix
        warp_matrix = np.eye(2, 3)
        warp_matrix[0, 2] = translation[1]
        warp_matrix[1, 2] = translation[0]
        # Return transformation matrix
        return warp_matrix

Response: Implementation of pair - wise registration using the CrossCorr model.


Instruction:
Write a Python 3 script to
normalise and scale image in 0 - 255 range

Response:
def rescale_image(image):
        """ Normalise and scale image in 0-255 range """
        s2_min_value, s2_max_value = 0, 1
        out_min_value, out_max_value = 0, 255
        # Clamp values in 0-1 range
        image[image > s2_max_value] = s2_max_value
        image[image < s2_min_value] = s2_min_value
        # Rescale to uint8 range
        out_image = out_max_value + (image-s2_min_value)*(out_max_value-out_min_value)/(s2_max_value-s2_min_value)
        return out_image.astype(np.uint8)


Instruction:
Given the following Python 3 function, write the documentation
def transform(self, X):
        if self.mode_ == 'target':
            return np.apply_along_axis(self._target, 1, np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2])))

        if self.mode_ == 'majority':
            return np.apply_along_axis(self._majority, 1, np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2])))

        print('Invalid mode! Set mode to majority or target. Returning input.')
        return X

Response: Returns a new array with the same shape as the original array.


Instruction:
Here you have a function in Python 3, explain what it does
def transform(self, X):

        if self.binary_:
            return (X & self.definition_ > 0).astype(int)
        return (X == self.definition_).astype(int)

Response: Returns True if the array - like X is a valid tag mask False otherwise.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_submap(self, eopatch):
        bbox_poly = eopatch.bbox.get_geometry()

        filtered_data = self.vector_data[self.vector_data.geometry.intersects(bbox_poly)].copy(deep=True)
        filtered_data.geometry = filtered_data.geometry.intersection(bbox_poly)

        return filtered_data

Response:
Returns a new geopandas dataframe with same structure as original one except that
        it contains only polygons that are contained within the bounding box.


Instruction:
Make a summary of the following Python 3 code
def execute(self, eopatch):
        bbox_map = self._get_submap(eopatch)
        height, width = self._get_shape(eopatch)
        data_transform = rasterio.transform.from_bounds(*eopatch.bbox, width=width, height=height)

        if self.feature_name in eopatch[self.feature_type]:
            raster = eopatch[self.feature_type][self.feature_name].squeeze()
        else:
            raster = np.ones((height, width), dtype=self.raster_dtype) * self.no_data_value

        if not bbox_map.empty:
            rasterio.features.rasterize([(bbox_map.cascaded_union.buffer(0), self.raster_value)], out=raster,
                                        transform=data_transform, dtype=self.raster_dtype)

        eopatch[self.feature_type][self.feature_name] = raster[..., np.newaxis]

        return eopatch

Response: Execute function which adds new vector layer to the EOPatch


Instruction:
Implement a Python 3 function for
executing function which adds new vector layer to the EOPatch

Response:
def execute(self, eopatch):
        """ Execute function which adds new vector layer to the EOPatch

        :param eopatch: input EOPatch
        :type eopatch: EOPatch
        :return: New EOPatch with added vector layer
        :rtype: EOPatch
        """
        for raster_ft, raster_fn, vector_fn in self.feature_gen(eopatch):
            vector_ft = FeatureType.VECTOR_TIMELESS if raster_ft.is_timeless() else FeatureType.VECTOR

            raster = eopatch[raster_ft][raster_fn]
            height, width = raster.shape[:2] if raster_ft.is_timeless() else raster.shape[1: 3]

            if self.raster_dtype:
                raster = raster.astype(self.raster_dtype)

            data_transform = rasterio.transform.from_bounds(*eopatch.bbox, width=width, height=height)

            crs = eopatch.bbox.get_crs()

            if raster_ft.is_timeless():
                eopatch[vector_ft][vector_fn] = self._vectorize_single_raster(raster, data_transform, crs)
            else:
                gpd_list = [self._vectorize_single_raster(raster[time_idx, ...], data_transform, crs,
                                                          timestamp=eopatch.timestamp[time_idx])
                            for time_idx in range(raster.shape[0])]

                eopatch[vector_ft][vector_fn] = GeoDataFrame(pd.concat(gpd_list, ignore_index=True),
                                                             crs=gpd_list[0].crs)

        return eopatch


Instruction:
How would you explain what the following Python 3 function does
def area(self, cc_index=None):
        if cc_index is not None:
            return self.areas[cc_index]
        return np.sum(self.areas)

Response: Returns the area of the selected polygon or all polygons if index is not provided.


Instruction:
Can you create a Python 3 function that
samples n points from the provided raster mask.

Response:
def sample(self, nsamples=1, weighted=True):
        """
        Sample n points from the provided raster mask. The number of points belonging
        to each class is proportional to the area this class covers in the raster mask,
        if weighted is set to True.

        TODO: If polygon has holes the number of sampled points will be less than nsamples

        :param nsamples: number of sampled samples
        :type nsamples: integer
        :param weighted: flag to apply weights proportional to total area of each class/polygon when sampling
        :type weighted: bool, default is True
        """
        weights = self.areas / np.sum(self.areas) if weighted else None
        index = np.random.choice(a=len(self.geometries), size=nsamples, p=weights)

        labels = []
        rows = []
        cols = []
        for idx in index:
            polygon = self.geometries[idx]['polygon']
            label = self.geometries[idx]['label']
            point = PointSampler.random_point(polygon.envelope.bounds)
            if PointSampler.contains(polygon, point):
                labels.append(label)
                rows.append(int(point.y))
                cols.append(int(point.x))
                # samples.append({'label':label, 'row':point.y, 'col':point.x})

        return labels, rows, cols


Instruction:
Explain what the following Python 3 code does
def sample_cc(self, nsamples=1, weighted=True):
        weights = self.areas / np.sum(self.areas) if weighted else None
        for index in np.random.choice(a=len(self.geometries), size=nsamples, p=weights):
            yield self.geometries[index]

Response: Returns a random polygon of any class.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def sample_within_cc(self, cc_index, nsamples=1):
        polygon = self.geometries[cc_index]['polygon']
        samples = []
        while len(samples) < nsamples:
            point = PointSampler.random_point(polygon.envelope.bounds)
            if PointSampler.contains(polygon, point):
                samples.append(point)
        return samples

Response: Returns randomly sampled points from a polygon.


Instruction:
Write a Python 3 script to
test whether a point lies within the polygon.

Response:
def contains(polygon, point):
        """
        Tests whether point lies within the polygon
        """
        in_hole = functools.reduce(
            lambda P, Q: P and Q,
            [interior.covers(point) for interior in polygon.interiors]
        ) if polygon.interiors else False
        return polygon.covers(point) and not in_hole


Instruction:
Can you write a function in Python 3 where it
selects a random point in interior of a rectangle

Response:
def random_coords(bounds):
        """ Selects a random point in interior of a rectangle

        :param bounds: Rectangle coordinates (x_min, y_min, x_max, y_max)
        :type bounds: tuple(float)
        :return: Random point from interior of rectangle
        :rtype: tuple of x and y coordinates
        """
        x_min, y_min, x_max, y_max = bounds
        x = np.random.randint(x_min, x_max)
        y = np.random.randint(y_min, y_max)
        return x, y


Instruction:
How would you explain what the following Python 3 function does
def random_point_triangle(triangle, use_int_coords=True):
        xs, ys = triangle.exterior.coords.xy
        A, B, C = zip(xs[:-1], ys[:-1])
        r1, r2 = np.random.rand(), np.random.rand()
        rx, ry = (1 - sqrt(r1)) * np.asarray(A) + sqrt(r1) * (1 - r2) * np.asarray(B) + sqrt(r1) * r2 * np.asarray(C)
        if use_int_coords:
            rx, ry = round(rx), round(ry)
            return Point(int(rx), int(ry))
        return Point(rx, ry)

Response: Selects a random point in interior of a triangle.


Instruction:
How would you explain what the following Python 3 function does
def _get_unknown_value(self):
        label_set = set(self.labels)
        value = 0
        while value in label_set:
            value += 1
        return value

Response: Finds the smallest integer value that is not in the labels


Instruction:
Create a Python 3 function to
sample nsamples_per_label points from the binary mask corresponding to label.

Response:
def _binary_sample(image, label, n_samples_per_label, label_count):
        """ Sample `nsamples_per_label` points from the binary mask corresponding to `label`

        Randomly sample `nsamples_per_label` point form the binary mask corresponding to `label`. Sampling with
        replacement is used if the required `nsamples_per_label` is larger than the available `label_count`

        :param image: Input 2D raster label image
        :type image: uint8 numpy array
        :param label: Scalar value of label to consider
        :type label: uint8
        :param n_samples_per_label: Number of points to sample form the binary mask
        :type n_samples_per_label: uint32
        :param label_count: Number of points available for `label`
        :type label_count: uint32
        :return: Sampled label value, row index of samples, col index of samples
        """
        h_idx, w_idx = np.where(image == label)

        rand_idx = np.random.choice(h_idx.size, size=n_samples_per_label, replace=label_count < n_samples_per_label)

        return h_idx[rand_idx], w_idx[rand_idx]


Instruction:
Write a Python 3 script to
sample the n_samples points of raster into a list of row indices of samples and column indices of samples.

Response:
def sample(self, raster, n_samples=1000):
        """ Sample `nsamples` points form raster

        :param raster: Input 2D or single-channel 3D label image
        :type raster: uint8 numpy array
        :param n_samples: Number of points to sample in total
        :type n_samples: uint32
        :return: List of row indices of samples, list of column indices of samples
        :rtype: numpy.array, numpy.array
        """
        # Check dimensionality and reshape to 2D
        raster = raster.copy()
        if raster.ndim == 3 and raster.shape[-1] == 1:
            raster = raster.squeeze()
        elif raster.ndim != 2:
            raise ValueError('Class operates on 2D or 3D single-channel raster images')

        # Calculate mask of all pixels which can be sampled
        mask = np.zeros(raster.shape, dtype=np.bool)
        for label in self.labels:
            label_mask = (raster == label)
            mask |= label_mask

        unique_labels, label_count = np.unique(raster[mask], return_counts=True)
        if not unique_labels.size:
            LOGGER.warning('No samples matching given parameters found in EOPatch')
            return np.empty((0,), dtype=np.uint32), np.empty((0,), dtype=np.uint32)

        if self.even_sampling:
            # Valid labels have the same (or off by one) number of samples
            n_samples_per_label = np.diff(np.round(np.linspace(0, n_samples,
                                                               num=label_count.size + 1))).astype(np.uint32)
        else:
            # Number of samples per label is proportional to label frequency
            label_ratio = label_count / np.sum(label_count)
            n_samples_per_label = (np.ceil(n_samples * label_ratio)).astype(np.uint32)

        # Apply mask to raster
        unknown_value = self._get_unknown_value()
        raster[~mask] = unknown_value
        if not np.array_equal(~mask, raster == unknown_value):
            raise ValueError('Failed to set unknown value. Too many labels for sampling reference mask of type '
                             '{}'.format(raster.dtype))

        # Sample raster
        samples = np.concatenate([self._binary_sample(raster, label, n_sample_label, label_count)
                                  for label, n_sample_label, label_count in zip(unique_labels, n_samples_per_label,
                                                                                label_count)], axis=1).T

        # Shuffle to mix labels in case they are fed directly to train a ML model
        np.random.shuffle(samples)

        # Return row index and col index. Return exactly `n_sample` values
        return samples[:n_samples, 0], samples[:n_samples, 1]


Instruction:
Write a Python 3 function that can
execute random spatial sampling of time - series stored in the input eopatch.

Response:
def execute(self, eopatch, seed=None):
        """ Execute random spatial sampling of time-series stored in the input eopatch

        :param eopatch: Input eopatch to be sampled
        :type eopatch: EOPatch
        :param seed: Setting seed of random sampling. If None no seed will be used.
        :type seed: int or None
        :return: An EOPatch with spatially sampled temporal features and associated labels
        :type eopatch: EOPatch
        """
        np.random.seed(seed)

        # Retrieve data and raster label image from eopatch
        f_type, f_name = next(self.ref_mask_feature(eopatch))
        raster = eopatch[f_type][f_name]

        # Initialise sampler
        sampler = PointRasterSampler(self.ref_labels, **self.sampling_params)
        # Perform sampling
        rows, cols = sampler.sample(raster, n_samples=self.n_samples)

        if self.return_new_eopatch:
            new_eopatch = EOPatch()
            new_eopatch.timestamp = eopatch.timestamp[:]
            new_eopatch.bbox = eopatch.bbox  # Should be copied
            new_eopatch.meta_info = eopatch.meta_info.copy()  # Should be deep copied - implement this in core
        else:
            new_eopatch = eopatch

        # Add sampled features
        for feature_type, feature_name, new_feature_name in self.sample_features(eopatch):

            if feature_type.is_time_dependent():
                sampled_data = eopatch[feature_type][feature_name][:, rows, cols, :]
            else:
                sampled_data = eopatch[feature_type][feature_name][rows, cols, :]
            new_eopatch[feature_type][new_feature_name] = sampled_data[..., np.newaxis, :]

        return new_eopatch


Instruction:
Explain what the following Python 3 code does
def execute(self, eopatch, *, eopatch_folder):
        eopatch.save(os.path.join(self.folder, eopatch_folder), *self.args, **self.kwargs)
        return eopatch

Response: Saves the EOPatch to disk.


Instruction:
Can you write a function in Python 3 where it
loads the EOPatch from disk.

Response:
def execute(self, *, eopatch_folder):
        """Loads the EOPatch from disk: `folder/eopatch_folder`.

        :param eopatch_folder: name of EOPatch folder containing data
        :type eopatch_folder: str
        :return: EOPatch loaded from disk
        :rtype: EOPatch
        """
        eopatch = EOPatch.load(os.path.join(self.folder, eopatch_folder), *self.args, **self.kwargs)
        return eopatch


Instruction:
Here you have a function in Python 3, explain what it does
def execute(self, eopatch, data):
        if self.feature_name is None:
            eopatch[self.feature_type] = data
        else:
            eopatch[self.feature_type][self.feature_name] = data

        return eopatch

Response: Returns the EOPatch with added features.


Instruction:
How would you code a function in Python 3 to
return the EOPatch with removed features.

Response:
def execute(self, eopatch):
        """Returns the EOPatch with removed features.

        :param eopatch: input EOPatch
        :type eopatch: EOPatch
        :return: input EOPatch without the specified feature
        :rtype: EOPatch
        """
        for feature_type, feature_name in self.feature_gen(eopatch):
            if feature_name is ...:
                eopatch.reset_feature_type(feature_type)
            else:
                del eopatch[feature_type][feature_name]

        return eopatch


Instruction:
How would you code a function in Python 3 to
return the EOPatch with renamed features.

Response:
def execute(self, eopatch):
        """Returns the EOPatch with renamed features.

        :param eopatch: input EOPatch
        :type eopatch: EOPatch
        :return: input EOPatch with the renamed features
        :rtype: EOPatch
        """
        for feature_type, feature_name, new_feature_name in self.feature_gen(eopatch):
            eopatch[feature_type][new_feature_name] = eopatch[feature_type][feature_name]
            del eopatch[feature_type][feature_name]

        return eopatch


Instruction:
Explain what the following Python 3 code does
def _numpy_index_by_percentile(self, data, percentile):
        data_perc_low = np.nanpercentile(data, percentile, axis=0, interpolation=self.interpolation)

        indices = np.empty(data_perc_low.shape, dtype=np.uint8)
        indices[:] = np.nan

        abs_diff = np.where(np.isnan(data_perc_low), np.inf, abs(data - data_perc_low))

        indices = np.where(np.isnan(data_perc_low), self.max_index, np.nanargmin(abs_diff, axis=0))

        return indices

Response: Calculate the percentile of numpy stack and return the index of the chosen pixel.


Instruction:
How would you explain what the following Python 3 function does
def _geoville_index_by_percentile(self, data, percentile):
        # no_obs = bn.allnan(arr_tmp["data"], axis=0)
        data_tmp = np.array(data, copy=True)
        valid_obs = np.sum(np.isfinite(data_tmp), axis=0)
        # replace NaN with maximum
        max_val = np.nanmax(data_tmp) + 1
        data_tmp[np.isnan(data_tmp)] = max_val
        # sort - former NaNs will move to the end
        ind_tmp = np.argsort(data_tmp, kind="mergesort", axis=0)
        # desired position as well as floor and ceiling of it
        k_arr = (valid_obs - 1) * (percentile / 100.0)
        k_arr = np.where(k_arr < 0, 0, k_arr)
        f_arr = np.floor(k_arr + 0.5)
        f_arr = f_arr.astype(np.int)
        # get floor value of reference band and index band
        ind = f_arr.astype("int16")
        y_val, x_val = ind_tmp.shape[1], ind_tmp.shape[2]
        y_val, x_val = np.ogrid[0:y_val, 0:x_val]
        idx = np.where(valid_obs == 0, self.max_index, ind_tmp[ind, y_val, x_val])
        return idx

Response: Calculate the index of the chosen pixel based on the percentile of numpy stack and return the index of the chosen pixel.


Instruction:
Explain what the following Python 3 code does
def _get_indices(self, data):
        indices = self._index_by_percentile(data, self.percentile)
        return indices

Response: Compute indices along temporal dimension corresponding to the sought percentile


Instruction:
How would you implement a function in Python 3 that
computes composite array merging temporal frames according to the compositing method

Response:
def execute(self, eopatch):
        """ Compute composite array merging temporal frames according to the compositing method

            :param eopatch: eopatch holding time-series
            :return: eopatch with composite image of time-series
        """
        feature_type, feature_name = next(self.feature(eopatch))
        data = eopatch[feature_type][feature_name].copy()

        # compute band according to compositing method (e.g. blue, maxNDVI, maxNDWI)
        reference_bands = self._get_reference_band(data)

        # find temporal indices corresponding to pre-defined percentile
        indices = self._get_indices(reference_bands)

        # compute composite image selecting values along temporal dimension corresponding to percentile indices
        composite_image = np.empty((data.shape[1:]), np.float32)
        composite_image[:] = self.no_data_value
        for scene_id, scene in enumerate(data):
            composite_image = np.where(np.dstack([indices]) == scene_id, scene, composite_image)

        eopatch[self.composite_type][self.composite_name] = composite_image

        return eopatch


Instruction:
Create a Python 3 function to
extract the NDVI band from time - series data

Response:
def _get_reference_band(self, data):
        """ Extract the NDVI band from time-series

        :param data: 4D array from which to compute the NDVI reference band
        :type data: numpy array
        :return: 3D array containing the NDVI reference band
        """
        nir = data[..., self.nir_idx].astype("float32")
        red = data[..., self.red_idx].astype("float32")
        return (nir - red) / (nir + red)


Instruction:
Write a Python 3 function that can
extract the NDWI band from time - series data

Response:
def _get_reference_band(self, data):
        """ Extract the NDWI band from time-series

        :param data: 4D array from which to compute the NDWI reference band
        :type data: numpy array
        :return: 3D array containing the NDWI reference band
        """
        nir = data[..., self.nir_idx].astype("float32")
        swir1 = data[..., self.swir1_idx].astype("float32")
        return (nir - swir1) / (nir + swir1)


Instruction:
Given the following Python 3 function, write the documentation
def _get_reference_band(self, data):
        blue = data[..., self.blue_idx].astype("float32")
        nir = data[..., self.nir_idx].astype("float32")
        swir1 = data[..., self.swir1_idx].astype("float32")
        return np.nanmax(np.array([nir, swir1]), axis=0) / blue

Response: Extract the max - ratio band from time - series


Instruction:
Write a Python 3 script for
performing histogram matching of the time - series with respect to a reference scene.

Response:
def execute(self, eopatch):
        """ Perform histogram matching of the time-series with respect to a reference scene

            :param eopatch: eopatch holding the time-series and reference data
            :type eopatch: EOPatch
            :return: The same eopatch instance with the normalised time-series
        """
        feature_type, feature_name, new_feature_name = next(self.feature(eopatch))
        reference_type, reference_name = next(self.reference(eopatch))

        reference_scene = eopatch[reference_type][reference_name]
        # check if band dimension matches
        if reference_scene.shape[-1] != eopatch[feature_type][feature_name].shape[-1]:
            raise ValueError('Time-series and reference scene must have corresponding bands')

        eopatch[feature_type][new_feature_name] = np.zeros_like(eopatch[feature_type][feature_name])
        for source_id, source in enumerate(eopatch[feature_type][feature_name]):
            # mask-out same invalid pixels
            src_masked = np.where(np.isnan(reference_scene), np.nan, source)
            ref_masked = np.where(np.isnan(source), np.nan, reference_scene)
            # compute statistics
            std_ref = np.nanstd(ref_masked, axis=(0, 1), dtype=np.float64)
            std_src = np.nanstd(src_masked, axis=(0, 1), dtype=np.float64)
            mean_ref = np.nanmean(ref_masked, axis=(0, 1), dtype=np.float64)
            mean_src = np.nanmean(src_masked, axis=(0, 1), dtype=np.float64)
            # normalise values
            eopatch[feature_type][new_feature_name][source_id] = \
                source * (std_ref / std_src) + (mean_ref - (mean_src * (std_ref / std_src)))

        return eopatch


Instruction:
Write a Python 3 function that can
check if the value fits the feature type and returns the corresponding object.

Response:
def _parse_feature_value(self, value):
        """ Checks if value fits the feature type. If not it tries to fix it or raise an error

        :raises: ValueError
        """
        if isinstance(value, _FileLoader):
            return value
        if not hasattr(self, 'ndim'):  # Because of serialization/deserialization during multiprocessing
            return value

        if self.ndim:
            if not isinstance(value, np.ndarray):
                raise ValueError('{} feature has to be a numpy array'.format(self.feature_type))
            if value.ndim != self.ndim:
                raise ValueError('Numpy array of {} feature has to have {} '
                                 'dimension{}'.format(self.feature_type, self.ndim, 's' if self.ndim > 1 else ''))

            if self.feature_type.is_discrete():
                if not issubclass(value.dtype.type, (np.integer, np.bool, np.bool_, np.bool8)):
                    msg = '{} is a discrete feature type therefore dtype of data should be a subtype of ' \
                          'numpy.integer or numpy.bool, found type {}. In the future an error will be raised because' \
                          'of this'.format(self.feature_type, value.dtype.type)
                    warnings.warn(msg, DeprecationWarning, stacklevel=3)

                    # raise ValueError('{} is a discrete feature type therefore dtype of data has to be a subtype of '
                    #                  'numpy.integer or numpy.bool, found type {}'.format(self.feature_type,
                    #                                                                      value.dtype.type))
            # This checking is disabled for now
            # else:
            #     if not issubclass(value.dtype.type, (np.floating, np.float)):
            #         raise ValueError('{} is a floating feature type therefore dtype of data has to be a subtype of '
            #                          'numpy.floating or numpy.float, found type {}'.format(self.feature_type,
            #                                                                                value.dtype.type))
            return value

        if self.is_vector:
            if isinstance(value, gpd.GeoSeries):
                value = gpd.GeoDataFrame(dict(geometry=value), crs=value.crs)

            if isinstance(value, gpd.GeoDataFrame):
                if self.feature_type is FeatureType.VECTOR:
                    if FeatureType.TIMESTAMP.value.upper() not in value:
                        raise ValueError("{} feature has to contain a column 'TIMESTAMP' with "
                                         "timestamps".format(self.feature_type))

                return value

            raise ValueError('{} feature works with data of type {}, parsing data type {} is not supported'
                             'given'.format(self.feature_type, gpd.GeoDataFrame.__name__, type(value)))

        return value


Instruction:
Can you tell what is the following Python 3 function doing
def _correctly_load_bbox(bbox, path, is_zipped=False):
        warnings.warn("Bounding box of your EOPatch is saved in old format which in the future won't be supported "
                      "anymore. Please save bounding box again, you can overwrite the existing one", DeprecationWarning,
                      stacklevel=4)

        with open(gzip.open(path) if is_zipped else path, 'rb') as pickle_file:
            crs_cnt = -1
            for _, arg, _ in pickletools.genops(pickle_file):
                if arg == 'sentinelhub.constants CRS':
                    crs_cnt = 2
                if crs_cnt == 0:
                    return sentinelhub.BBox(tuple(bbox), sentinelhub.CRS(arg))
                crs_cnt -= 1

        raise ValueError('Failed to correctly load BBox object, try downgrading sentinelhub package to <=2.4.7')

Response: Load a pickled BBox object from a file.


Instruction:
How would you explain what the following Python 3 function does
def load(self):
        # pylint: disable=too-many-return-statements
        if not os.path.isdir(self.patch_path):
            raise OSError('EOPatch does not exist in path {} anymore'.format(self.patch_path))

        path = self.get_file_path()
        if not os.path.exists(path):
            raise OSError('Feature in path {} does not exist anymore'.format(path))

        file_formats = FileFormat.split_by_extensions(path)[1:]

        if not file_formats or file_formats[-1] is FileFormat.PICKLE:
            with open(path, "rb") as infile:
                data = pickle.load(infile)

                if isinstance(data, sentinelhub.BBox) and not hasattr(data, 'crs'):
                    return self._correctly_load_bbox(data, path)
                return data

        if file_formats[-1] is FileFormat.NPY:
            if self.mmap:
                return np.load(path, mmap_mode='r')
            return np.load(path)

        if file_formats[-1] is FileFormat.GZIP:
            if file_formats[-2] is FileFormat.NPY:
                return np.load(gzip.open(path))

            if len(file_formats) == 1 or file_formats[-2] is FileFormat.PICKLE:
                data = pickle.load(gzip.open(path))

                if isinstance(data, sentinelhub.BBox) and not hasattr(data, 'crs'):
                    return self._correctly_load_bbox(data, path, is_zipped=True)
                return data

        raise ValueError('Could not load data from unsupported file format {}'.format(file_formats[-1]))

Response: Method which loads data from the file


Instruction:
How would you explain what the following Python 3 function does
def get_file_path(self, path):
        feature_filename = self._get_filename_path(path)

        feature_filename += self.file_format.extension()
        if self.compress_level:
            feature_filename += FileFormat.GZIP.extension()

        return feature_filename

Response: Returns a filename with file path


Instruction:
Make a summary of the following Python 3 code
def save(self, eopatch, use_tmp=True):
        filename = self.tmp_filename if use_tmp else self.final_filename

        if self.feature_name is None:
            data = eopatch[self.feature_type]
            if self.feature_type.has_dict():
                data = data.get_dict()

            if self.feature_type is FeatureType.BBOX:
                data = tuple(data) + (int(data.crs.value),)
        else:
            data = eopatch[self.feature_type][self.feature_name]

        file_dir = os.path.dirname(filename)
        os.makedirs(file_dir, exist_ok=True)

        if self.compress_level:
            file_handle = gzip.GzipFile(filename, 'w', self.compress_level)
        else:
            file_handle = open(filename, 'wb')

        with file_handle as outfile:
            LOGGER.debug("Saving (%s, %s) to %s", str(self.feature_type), str(self.feature_name), filename)

            if self.file_format is FileFormat.NPY:
                np.save(outfile, data)
            elif self.file_format is FileFormat.PICKLE:
                pickle.dump(data, outfile)
            else:
                ValueError('File {} was not saved because saving in file format {} is currently not '
                           'supported'.format(filename, self.file_format))

Response: Method which saves the data in the eopatch to the final location


Instruction:
Can you generate the documentation for the following Python 3 function
def _mask_feature_data(feature_data, mask, mask_type):

        if mask_type.is_spatial() and feature_data.shape[1: 3] != mask.shape[-3: -1]:
            raise ValueError('Spatial dimensions of interpolation and mask feature do not match: '
                             '{} {}'.format(feature_data.shape, mask.shape))

        if mask_type.is_time_dependent() and feature_data.shape[0] != mask.shape[0]:
            raise ValueError('Time dimension of interpolation and mask feature do not match: '
                             '{} {}'.format(feature_data.shape, mask.shape))

        # This allows masking each channel differently but causes some complications while masking with label
        if mask.shape[-1] != feature_data.shape[-1]:
            mask = mask[..., 0]

        if mask_type is FeatureType.MASK:
            feature_data[mask, ...] = np.nan

        elif mask_type is FeatureType.MASK_TIMELESS:
            feature_data[:, mask, ...] = np.nan

        elif mask_type is FeatureType.LABEL:
            np.swapaxes(feature_data, 1, 3)
            feature_data[mask, ..., :, :] = np.nan
            np.swapaxes(feature_data, 1, 3)

        return feature_data

Response: Masks data with a given mask of given mask type.


Instruction:
Write a Python 3 script for
functioning to find NaN values in data that start or end of time - series.

Response:
def _get_start_end_nans(data):
        """ Find NaN values in data that either start or end the time-series

        Function to return a binary array of same size as data where `True` values correspond to NaN values present at
        beginning or end of time-series. NaNs internal to the time-series are not included in the binary mask.

        :param data: Array of observations of size TxNOBS
        :type data: numpy.array
        :return: Binary array of shape TxNOBS. `True` values indicate NaNs present at beginning or end of time-series
        :rtype: numpy.array
        """
        # find NaNs that start a time-series
        start_nan = np.isnan(data)
        for idx, row in enumerate(start_nan[:-1]):
            start_nan[idx + 1] = np.logical_and(row, start_nan[idx + 1])
        # find NaNs that end a time-series
        end_nan = np.isnan(data)
        for idx, row in enumerate(end_nan[-2::-1]):
            end_nan[-idx-2] = np.logical_and(row, end_nan[-idx-1])

        return np.logical_or(start_nan, end_nan)


Instruction:
Given the following Python 3 function, write the documentation
def _get_unique_times(data, times):
        seen = set()
        duplicated_indices = np.array([idx for idx, item in enumerate(times) if item in seen or seen.add(item)],
                                      dtype=int)
        duplicated_times = np.unique(times[duplicated_indices])

        for time in duplicated_times:
            indices = np.where(times == time)[0]
            nan_mask = np.all(np.isnan(data[indices]), axis=0)
            data[indices[0], ~nan_mask] = np.nanmean(data[indices][:, ~nan_mask], axis=0)

        times = np.delete(times, duplicated_indices, axis=0)
        data = np.delete(data, duplicated_indices, axis=0)

        return data, times

Response: Replace duplicate acquisitions which have same values on the chosen time scale with their average.


Instruction:
Given the following Python 3 function, write the documentation
def _copy_old_features(new_eopatch, old_eopatch, copy_features):
        if copy_features:
            existing_features = set(new_eopatch.get_feature_list())

            for copy_feature_type, copy_feature_name, copy_new_feature_name in copy_features:
                new_feature = copy_feature_type, copy_new_feature_name

                if new_feature in existing_features:
                    raise ValueError('Feature {} of {} already exists in the new EOPatch! '
                                     'Use a different name!'.format(copy_new_feature_name, copy_feature_type))
                else:
                    existing_features.add(new_feature)

                    new_eopatch[copy_feature_type][copy_new_feature_name] = \
                        old_eopatch[copy_feature_type][copy_feature_name]

        return new_eopatch

Response: Copy features from old EOPatch to new EOPatch.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def interpolate_data(self, data, times, resampled_times):
        # pylint: disable=too-many-locals
        # get size of 2d array t x nobs
        nobs = data.shape[-1]
        if self.interpolate_pixel_wise:
            # initialise array of interpolated values
            new_data = data if self.resample_range is None else np.full(
                (len(resampled_times), nobs),
                np.nan, dtype=data.dtype)

            # Interpolate for each pixel, could be easily parallelized
            for obs in range(nobs):
                valid = ~np.isnan(data[:, obs])

                obs_interpolating_func = self.get_interpolation_function(times[valid], data[valid, obs])

                new_data[:, obs] = obs_interpolating_func(resampled_times[:, np.newaxis])

            # return interpolated values
            return new_data

        # mask representing overlap between reference and resampled times
        time_mask = (resampled_times >= np.min(times)) & (resampled_times <= np.max(times))

        # define time values as linear monotonically increasing over the observations
        const = int(self.filling_factor * (np.max(times) - np.min(times)))
        temp_values = (times[:, np.newaxis] +
                       const * np.arange(nobs)[np.newaxis, :].astype(np.float64))
        res_temp_values = (resampled_times[:, np.newaxis] +
                           const * np.arange(nobs)[np.newaxis, :].astype(np.float64))

        # initialise array of interpolated values
        new_data = np.full((len(resampled_times), nobs), np.nan, dtype=data.dtype)

        # array defining index correspondence between reference times and resampled times
        ori2res = np.array([np.abs(resampled_times - o).argmin()
                            if np.min(resampled_times) <= o <= np.max(resampled_times) else None for o in times])

        # find NaNs that start or end a time-series
        row_nans, col_nans = np.where(self._get_start_end_nans(data))
        nan_row_res_indices = np.array([index for index in ori2res[row_nans] if index is not None], dtype=np.int32)
        nan_col_res_indices = np.array([index is not None for index in ori2res[row_nans]],
                                       dtype=np.bool)
        if nan_row_res_indices.size:
            # mask out from output values the starting/ending NaNs
            res_temp_values[nan_row_res_indices, col_nans[nan_col_res_indices]] = np.nan
        # if temporal values outside the reference dates are required (extrapolation) masked them to NaN
        res_temp_values[~time_mask, :] = np.nan

        # build 1d array for interpolation. Spline functions require monotonically increasing values of x,
        # so .T is used
        input_x = temp_values.T[~np.isnan(data).T]
        input_y = data.T[~np.isnan(data).T]

        # build interpolation function
        if len(input_x) > 1:
            interp_func = self.get_interpolation_function(input_x, input_y)

            # interpolate non-NaN values in resampled time values
            new_data[~np.isnan(res_temp_values)] = interp_func(res_temp_values[~np.isnan(res_temp_values)])

        # return interpolated values
        return new_data

Response: Interpolates data feature containing the values of the specified time intervals.


Instruction:
Write a Python 3 function that can
initialize interpolation model :param times: Array of reference times in second relative to the first timestamp :type times: numpy.array :param series: One dimensional array of time series :type series: numpy.array :return: Initialized interpolation model class

Response:
def get_interpolation_function(self, times, series):
        """ Initializes interpolation model

        :param times: Array of reference times in second relative to the first timestamp
        :type times: numpy.array
        :param series: One dimensional array of time series
        :type series: numpy.array
        :return: Initialized interpolation model class
        """
        return self.interpolation_object(times, series, **self.interpolation_parameters)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_resampled_timestamp(self, timestamp):
        if self.resample_range is None:
            return timestamp

        if not isinstance(self.resample_range, (tuple, list)):
            raise ValueError('Invalid resample_range {}, expected tuple'.format(self.resample_range))

        if tuple(map(type, self.resample_range)) == (str, str, int):
            start_date = dateutil.parser.parse(self.resample_range[0])
            end_date = dateutil.parser.parse(self.resample_range[1])
            step = dt.timedelta(days=self.resample_range[2])
            days = [start_date]
            while days[-1] + step < end_date:
                days.append(days[-1] + step)

        elif self.resample_range and np.all([isinstance(date, str) for date in self.resample_range]):
            days = [dateutil.parser.parse(date) for date in self.resample_range]
        elif self.resample_range and np.all([isinstance(date, dt.datetime) for date in self.resample_range]):
            days = [date for date in self.resample_range]
        else:
            raise ValueError('Invalid format in {}, expected strings or datetimes'.format(self.resample_range))

        return days

Response: Takes a list of timestamps and generates a list of timestamps according to self. resample_range


Instruction:
Write a Python 3 script for
executing method that processes an EOPatch and returns EOPatch object

Response:
def execute(self, eopatch):
        """ Execute method that processes EOPatch and returns EOPatch
        """
        # pylint: disable=too-many-locals
        feature_type, feature_name, new_feature_name = next(self.feature(eopatch))

        # Make a copy not to change original numpy array
        feature_data = eopatch[feature_type][feature_name].copy()
        time_num, height, width, band_num = feature_data.shape
        if time_num <= 1:
            raise ValueError('Feature {} has time dimension of size {}, required at least size '
                             '2'.format((feature_type, feature_name), time_num))

        # Apply a mask on data
        if self.mask_feature is not None:
            for mask_type, mask_name in self.mask_feature(eopatch):
                negated_mask = ~eopatch[mask_type][mask_name].astype(np.bool)
                feature_data = self._mask_feature_data(feature_data, negated_mask, mask_type)

        # Flatten array
        feature_data = np.reshape(feature_data, (time_num, height * width * band_num))

        # If resampling create new EOPatch
        new_eopatch = EOPatch() if self.resample_range else eopatch

        # Resample times
        times = eopatch.time_series(scale_time=self.scale_time)
        new_eopatch.timestamp = self.get_resampled_timestamp(eopatch.timestamp)
        total_diff = int((new_eopatch.timestamp[0].date() - eopatch.timestamp[0].date()).total_seconds())
        resampled_times = new_eopatch.time_series(scale_time=self.scale_time) + total_diff // self.scale_time

        # Add BBox to eopatch if it was created anew
        if new_eopatch.bbox is None:
            new_eopatch.bbox = eopatch.bbox

        # Replace duplicate acquisitions which have same values on the chosen time scale with their average
        feature_data, times = self._get_unique_times(feature_data, times)

        # Interpolate
        feature_data = self.interpolate_data(feature_data, times, resampled_times)

        # Normalize
        if self.result_interval:
            min_val, max_val = self.result_interval
            valid_mask = ~np.isnan(feature_data)
            feature_data[valid_mask] = np.maximum(np.minimum(feature_data[valid_mask], max_val), min_val)

        # Replace unknown value
        if not np.isnan(self.unknown_value):
            feature_data[np.isnan(feature_data)] = self.unknown_value

        # Reshape back
        new_eopatch[feature_type][new_feature_name] = np.reshape(feature_data,
                                                                 (feature_data.shape[0], height, width, band_num))

        # append features from old patch
        new_eopatch = self._copy_old_features(new_eopatch, eopatch, self.copy_features)

        return new_eopatch


Instruction:
Here you have a function in Python 3, explain what it does
def interpolate_data(self, data, times, resampled_times):
        if True in np.unique(np.isnan(data)):
            raise ValueError('Data must not contain any masked/invalid pixels or NaN values')

        interp_func = self.get_interpolation_function(times, data)

        time_mask = (resampled_times >= np.min(times)) & (resampled_times <= np.max(times))
        new_data = np.full((resampled_times.size,) + data.shape[1:], np.nan, dtype=data.dtype)
        new_data[time_mask] = interp_func(resampled_times[time_mask])
        return new_data

Response: Interpolates data feature by time.


Instruction:
Given the following Python 3 function, write the documentation
def get_interpolation_function(self, times, data):
        return self.interpolation_object(times, data, axis=0, **self.interpolation_parameters)

Response:
Initializes interpolation model

        :param times: Array of reference times in second relative to the first timestamp
        :type times: numpy.array
        :param data: One dimensional array of time series
        :type data: numpy.array
        :return: Initialized interpolation model class


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_text(element):
    ''' return inner text in pyquery element '''
    _add_links_to_text(element)
    try:
        return element.text(squash_space=False)
    except TypeError:
        return element.text()

Response: Get inner text in pyquery element


Instruction:
Make a summary of the following Python 3 code
def download_file(url, local_filename):
    local_filename = os.path.abspath(local_filename)
    path = os.path.dirname(local_filename)
    if not os.path.isdir(path):
        os.makedirs(path)

    with tqdm(unit='B', unit_scale=True) as progress:
        def report(chunk, chunksize, total):
            progress.total = total
            progress.update(chunksize)
        return urlretrieve(url, local_filename, reporthook=report)

Response: Download a file from url to local_filename


Instruction:
How would you implement a function in Python 3 that
tries to find GCC on OSX for OpenMP support.

Response:
def extract_gcc_binaries():
    """Try to find GCC on OSX for OpenMP support."""
    patterns = ['/opt/local/bin/g++-mp-[0-9].[0-9]',
                '/opt/local/bin/g++-mp-[0-9]',
                '/usr/local/bin/g++-[0-9].[0-9]',
                '/usr/local/bin/g++-[0-9]']
    if 'darwin' in platform.platform().lower():
        gcc_binaries = []
        for pattern in patterns:
            gcc_binaries += glob.glob(pattern)
        gcc_binaries.sort()
        if gcc_binaries:
            _, gcc = os.path.split(gcc_binaries[-1])
            return gcc
        else:
            return None
    else:
        return None


Instruction:
Explain what the following Python 3 code does
def set_gcc():
    # For macports and homebrew

    if 'darwin' in platform.platform().lower():
        gcc = extract_gcc_binaries()

        if gcc is not None:
            os.environ["CC"] = gcc
            os.environ["CXX"] = gcc

        else:
            global use_openmp
            use_openmp = False
            logging.warning('No GCC available. Install gcc from Homebrew '
                            'using brew install gcc.')

Response: Try to use GCC on OS X for OpenMP support.


Instruction:
Can you create a Python 3 function that
returns the non zeroes of a row in csr_matrix

Response:
def nonzeros(m, row):
    """ returns the non zeroes of a row in csr_matrix """
    for index in range(m.indptr[row], m.indptr[row+1]):
        yield m.indices[index], m.data[index]


Instruction:
Make a summary of the following Python 3 code
def check_blas_config():
    # don't warn repeatedly
    global _checked_blas_config
    if _checked_blas_config:
        return
    _checked_blas_config = True

    if np.__config__.get_info('openblas_info') and os.environ.get('OPENBLAS_NUM_THREADS') != '1':
        logging.warning("OpenBLAS detected. Its highly recommend to set the environment variable "
                        "'export OPENBLAS_NUM_THREADS=1' to disable its internal multithreading")
    if np.__config__.get_info('blas_mkl_info') and os.environ.get('MKL_NUM_THREADS') != '1':
        logging.warning("Intel MKL BLAS detected. Its highly recommend to set the environment "
                        "variable 'export MKL_NUM_THREADS=1' to disable its internal "
                        "multithreading")

Response: checks to see if we are using OpenBlas and Intel MKL and if so warns if the number of threads isn t set to 1


Instruction:
Explain what the following Python 3 code does
def _read_dataframe(filename):
    import pandas

    # read in triples of user/artist/playcount from the input dataset
    # get a model based off the input params
    start = time.time()
    log.debug("reading data from %s", filename)
    data = pandas.read_csv(filename, delimiter='|', quotechar='\\')

    # map each artist and user to a unique numeric value
    data['uid'] = data['uid'].astype("category")
    data['mid'] = data['mid'].astype("category")

    # store as a CSR matrix
    log.debug("read data file in %s", time.time() - start)
    return data

Response: Reads the original dataset PSV as a pandas dataframe


Instruction:
Create a Python 3 function to
return the lastfm dataset downloaded locally if necessary.

Response:
def get_lastfm():
    """ Returns the lastfm360k dataset, downloading locally if necessary.
    Returns a tuple of (artistids, userids, plays) where plays is a CSR matrix """

    filename = os.path.join(_download.LOCAL_CACHE_DIR, "lastfm_360k.hdf5")
    if not os.path.isfile(filename):
        log.info("Downloading dataset to '%s'", filename)
        _download.download_file(URL, filename)
    else:
        log.info("Using cached dataset at '%s'", filename)

    with h5py.File(filename, 'r') as f:
        m = f.get('artist_user_plays')
        plays = csr_matrix((m.get('data'), m.get('indices'), m.get('indptr')))
        return np.array(f['artist']), np.array(f['user']), plays


Instruction:
How would you explain what the following Python 3 function does
def alternating_least_squares(Ciu, factors, **kwargs):
    log.warning("This method is deprecated. Please use the AlternatingLeastSquares"
                " class instead")

    model = AlternatingLeastSquares(factors=factors, **kwargs)
    model.fit(Ciu)
    return model.item_factors, model.user_factors

Response: factorizes the matrix Cui using an implicit alternating least squares algorithm


Instruction:
How would you implement a function in Python 3 that
calculates factors Xu for each user in Cui using least squares on Y.

Response:
def least_squares(Cui, X, Y, regularization, num_threads=0):
    """ For each user in Cui, calculate factors Xu for them
    using least squares on Y.

    Note: this is at least 10 times slower than the cython version included
    here.
    """
    users, n_factors = X.shape
    YtY = Y.T.dot(Y)

    for u in range(users):
        X[u] = user_factor(Y, YtY, Cui, u, regularization, n_factors)


Instruction:
How would you explain what the following Python 3 function does
def fit(self, item_users, show_progress=True):
        Ciu = item_users
        if not isinstance(Ciu, scipy.sparse.csr_matrix):
            s = time.time()
            log.debug("Converting input to CSR format")
            Ciu = Ciu.tocsr()
            log.debug("Converted input to CSR in %.3fs", time.time() - s)

        if Ciu.dtype != np.float32:
            Ciu = Ciu.astype(np.float32)

        s = time.time()
        Cui = Ciu.T.tocsr()
        log.debug("Calculated transpose in %.3fs", time.time() - s)

        items, users = Ciu.shape

        s = time.time()
        # Initialize the variables randomly if they haven't already been set
        if self.user_factors is None:
            self.user_factors = np.random.rand(users, self.factors).astype(self.dtype) * 0.01
        if self.item_factors is None:
            self.item_factors = np.random.rand(items, self.factors).astype(self.dtype) * 0.01

        log.debug("Initialized factors in %s", time.time() - s)

        # invalidate cached norms and squared factors
        self._item_norms = None
        self._YtY = None

        if self.use_gpu:
            return self._fit_gpu(Ciu, Cui, show_progress)

        solver = self.solver

        log.debug("Running %i ALS iterations", self.iterations)
        with tqdm.tqdm(total=self.iterations, disable=not show_progress) as progress:
            # alternate between learning the user_factors from the item_factors and vice-versa
            for iteration in range(self.iterations):
                s = time.time()
                solver(Cui, self.user_factors, self.item_factors, self.regularization,
                       num_threads=self.num_threads)
                progress.update(.5)

                solver(Ciu, self.item_factors, self.user_factors, self.regularization,
                       num_threads=self.num_threads)
                progress.update(.5)

                if self.calculate_training_loss:
                    loss = _als.calculate_loss(Cui, self.user_factors, self.item_factors,
                                               self.regularization, num_threads=self.num_threads)
                    progress.set_postfix({"loss": loss})

                if self.fit_callback:
                    self.fit_callback(iteration, time.time() - s)

        if self.calculate_training_loss:
            log.info("Final training loss %.4f", loss)

Response: Fits the latent factor model of the input data.


Instruction:
How would you explain what the following Python 3 function does
def _fit_gpu(self, Ciu_host, Cui_host, show_progress=True):
        if not implicit.cuda.HAS_CUDA:
            raise ValueError("No CUDA extension has been built, can't train on GPU.")

        if self.dtype == np.float64:
            log.warning("Factors of dtype float64 aren't supported with gpu fitting. "
                        "Converting factors to float32")
            self.item_factors = self.item_factors.astype(np.float32)
            self.user_factors = self.user_factors.astype(np.float32)

        Ciu = implicit.cuda.CuCSRMatrix(Ciu_host)
        Cui = implicit.cuda.CuCSRMatrix(Cui_host)
        X = implicit.cuda.CuDenseMatrix(self.user_factors.astype(np.float32))
        Y = implicit.cuda.CuDenseMatrix(self.item_factors.astype(np.float32))

        solver = implicit.cuda.CuLeastSquaresSolver(self.factors)
        log.debug("Running %i ALS iterations", self.iterations)
        with tqdm.tqdm(total=self.iterations, disable=not show_progress) as progress:
            for iteration in range(self.iterations):
                s = time.time()
                solver.least_squares(Cui, X, Y, self.regularization, self.cg_steps)
                progress.update(.5)
                solver.least_squares(Ciu, Y, X, self.regularization, self.cg_steps)
                progress.update(.5)

                if self.fit_callback:
                    self.fit_callback(iteration, time.time() - s)

                if self.calculate_training_loss:
                    loss = solver.calculate_loss(Cui, X, Y, self.regularization)
                    progress.set_postfix({"loss": loss})

        if self.calculate_training_loss:
            log.info("Final training loss %.4f", loss)

        X.to_host(self.user_factors)
        Y.to_host(self.item_factors)

Response: Fits the GPU on the GPU.


Instruction:
How would you explain what the following Python 3 function does
def explain(self, userid, user_items, itemid, user_weights=None, N=10):
        # user_weights = Cholesky decomposition of Wu^-1
        # from section 5 of the paper CF for Implicit Feedback Datasets
        user_items = user_items.tocsr()
        if user_weights is None:
            A, _ = user_linear_equation(self.item_factors, self.YtY,
                                        user_items, userid,
                                        self.regularization, self.factors)
            user_weights = scipy.linalg.cho_factor(A)
        seed_item = self.item_factors[itemid]

        # weighted_item = y_i^t W_u
        weighted_item = scipy.linalg.cho_solve(user_weights, seed_item)

        total_score = 0.0
        h = []
        for i, (itemid, confidence) in enumerate(nonzeros(user_items, userid)):
            if confidence < 0:
                continue

            factor = self.item_factors[itemid]
            # s_u^ij = (y_i^t W^u) y_j
            score = weighted_item.dot(factor) * confidence
            total_score += score
            contribution = (score, itemid)
            if i < N:
                heapq.heappush(h, contribution)
            else:
                heapq.heappushpop(h, contribution)

        items = (heapq.heappop(h) for i in range(len(h)))
        top_contributions = list((i, s) for s, i in items)[::-1]
        return total_score, top_contributions, user_weights

Response: Provides explanations for why the item is liked by the user.


Instruction:
Can you generate the documentation for the following Python 3 function
def augment_inner_product_matrix(factors):
    norms = numpy.linalg.norm(factors, axis=1)
    max_norm = norms.max()

    # add an extra dimension so that the norm of each row is the same
    # (max_norm)
    extra_dimension = numpy.sqrt(max_norm ** 2 - norms ** 2)
    return max_norm, numpy.append(factors, extra_dimension.reshape(norms.shape[0], 1), axis=1)

Response:
This function transforms a factor matrix such that an angular nearest neighbours search
    will return the top related items of the inner product.


Instruction:
How would you code a function in Python 3 to
convert a scipy sparse matrix to a spark dataframe

Response:
def convert_sparse_to_dataframe(spark, context, sparse_matrix):
    """ Converts a scipy sparse matrix to a spark dataframe """
    m = sparse_matrix.tocoo()
    data = context.parallelize(numpy.array([m.row, m.col, m.data]).T,
                               numSlices=len(m.row)/1024)
    return spark.createDataFrame(data.map(lambda p: Row(row=int(p[0]),
                                                        col=int(p[1]),
                                                        data=float(p[2]))))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_dataset(triplets_filename, summary_filename="msd_summary_file.h5",
                     outputfilename="msd_taste_profile.hdf5"):
    """ Generates a hdf5 datasetfile from the raw datafiles:

    You will need to download the train_triplets from here:
        https://labrosa.ee.columbia.edu/millionsong/tasteprofile#getting
    And the 'Summary File of the whole dataset' from here
    https://labrosa.ee.columbia.edu/millionsong/pages/getting-dataset

    You shouldn't have to run this yourself, and can instead just download the
    output using the 'get_msd_taste_profile' funciton
    """
    data = _read_triplets_dataframe(triplets_filename)
    track_info = _join_summary_file(data, summary_filename)

    _hfd5_from_dataframe(data, track_info, outputfilename)

Response: Generates a hdf5 datasetfile from the raw datafiles


Instruction:
Write a Python 3 function that can
read the original dataset TSV as a pandas dataframe

Response:
def _read_triplets_dataframe(filename):
    """ Reads the original dataset TSV as a pandas dataframe """
    # delay importing this to avoid another dependency
    import pandas

    # read in triples of user/artist/playcount from the input dataset
    # get a model based off the input params
    start = time.time()
    log.debug("reading data from %s", filename)
    data = pandas.read_table("train_triplets.txt", names=['user', 'track', 'plays'])

    # map each artist and user to a unique numeric value
    data['user'] = data['user'].astype("category")
    data['track'] = data['track'].astype("category")

    # store as a CSR matrix
    log.debug("read data file in %s", time.time() - start)

    return data


Instruction:
Can you write a function in Python 3 where it
gets the trackinfo array by joining taste profile to the track summary file

Response:
def _join_summary_file(data, summary_filename="msd_summary_file.h5"):
    """ Gets the trackinfo array by joining taste profile to the track summary file """
    msd = h5py.File(summary_filename)

    # create a lookup table of trackid -> position
    track_lookup = dict((t.encode("utf8"), i) for i, t in enumerate(data['track'].cat.categories))

    # join on trackid to the summary file to get the artist/album/songname
    track_info = np.empty(shape=(len(track_lookup), 4), dtype=np.object)
    with tqdm.tqdm(total=len(track_info)) as progress:
        for song in msd['metadata']['songs']:
            trackid = song[17]
            if trackid in track_lookup:
                pos = track_lookup[trackid]
                track_info[pos] = [x.decode("utf8") for x in (trackid, song[9], song[14], song[18])]
                progress.update(1)

    return track_info


Instruction:
Can you generate the documentation for the following Python 3 function
def tfidf_weight(X):
    X = coo_matrix(X)

    # calculate IDF
    N = float(X.shape[0])
    idf = log(N) - log1p(bincount(X.col))

    # apply TF-IDF adjustment
    X.data = sqrt(X.data) * idf[X.col]
    return X

Response: Weights a Sparse Matrix by TF - IDF Weighted


Instruction:
Make a summary of the following Python 3 code
def normalize(X):
    X = coo_matrix(X)
    X.data = X.data / sqrt(bincount(X.row, X.data ** 2))[X.row]
    return X

Response: normalize matrix X to be a sparse matrix


Instruction:
Write a Python 3 script for
weighing each row of a sparse matrix X by BM25 weighting

Response:
def bm25_weight(X, K1=100, B=0.8):
    """ Weighs each row of a sparse matrix X  by BM25 weighting """
    # calculate idf per term (user)
    X = coo_matrix(X)

    N = float(X.shape[0])
    idf = log(N) - log1p(bincount(X.col))

    # calculate length_norm per document (artist)
    row_sums = numpy.ravel(X.sum(axis=1))
    average_length = row_sums.mean()
    length_norm = (1.0 - B) + B * row_sums / average_length

    # weight matrix rows by bm25
    X.data = X.data * (K1 + 1.0) / (K1 * length_norm[X.row] + X.data) * idf[X.col]
    return X


Instruction:
Implement a function in Python 3 to
compute and stores the similarity matrix

Response:
def fit(self, weighted, show_progress=True):
        """ Computes and stores the similarity matrix """
        self.similarity = all_pairs_knn(weighted, self.K,
                                        show_progress=show_progress,
                                        num_threads=self.num_threads).tocsr()
        self.scorer = NearestNeighboursScorer(self.similarity)


Instruction:
Write a Python 3 script for
returning the best N recommendations for a user given its id

Response:
def recommend(self, userid, user_items,
                  N=10, filter_already_liked_items=True, filter_items=None, recalculate_user=False):
        """ returns the best N recommendations for a user given its id"""
        if userid >= user_items.shape[0]:
            raise ValueError("userid is out of bounds of the user_items matrix")

        # recalculate_user is ignored because this is not a model based algorithm
        items = N
        if filter_items:
            items += len(filter_items)

        indices, data = self.scorer.recommend(userid, user_items.indptr, user_items.indices,
                                              user_items.data, K=items,
                                              remove_own_likes=filter_already_liked_items)
        best = sorted(zip(indices, data), key=lambda x: -x[1])

        if not filter_items:
            return best

        liked = set(filter_items)
        return list(itertools.islice((rec for rec in best if rec[0] not in liked), N))


Instruction:
Write a Python 3 script for
ranking given items for a user and returns sorted item list

Response:
def rank_items(self, userid, user_items, selected_items, recalculate_user=False):
        """ Rank given items for a user and returns sorted item list """
        # check if selected_items contains itemids that are not in the model(user_items)
        if max(selected_items) >= user_items.shape[1] or min(selected_items) < 0:
            raise IndexError("Some of selected itemids are not in the model")

        # calculate the relevance scores
        liked_vector = user_items[userid]
        recommendations = liked_vector.dot(self.similarity)

        # remove items that are not in the selected_items
        best = sorted(zip(recommendations.indices, recommendations.data), key=lambda x: -x[1])
        ret = [rec for rec in best if rec[0] in selected_items]

        # returned items should be equal to input selected items
        for itemid in selected_items:
            if itemid not in recommendations.indices:
                ret.append((itemid, -1.0))
        return ret


Instruction:
Given the following Python 3 function, write the documentation
def similar_items(self, itemid, N=10):
        if itemid >= self.similarity.shape[0]:
            return []

        return sorted(list(nonzeros(self.similarity, itemid)), key=lambda x: -x[1])[:N]

Response: Returns a list of the most similar items


Instruction:
Can you tell what is the following Python 3 function doing
def get_reddit():

    filename = os.path.join(_download.LOCAL_CACHE_DIR, "reddit.hdf5")
    if not os.path.isfile(filename):
        log.info("Downloading dataset to '%s'", filename)
        _download.download_file(URL, filename)
    else:
        log.info("Using cached dataset at '%s'", filename)

    with h5py.File(filename, 'r') as f:
        m = f.get('item_user_ratings')
        return csr_matrix((m.get('data'), m.get('indices'), m.get('indptr')))

Response: Returns the reddit dataset downloaded locally if necessary.


Instruction:
Implement a Python 3 function for
reading the original dataset TSV as a pandas dataframe

Response:
def _read_dataframe(filename):
    """ Reads the original dataset TSV as a pandas dataframe """
    # delay importing this to avoid another dependency
    import pandas

    # read in triples of user/artist/playcount from the input dataset
    # get a model based off the input params
    start = time.time()
    log.debug("reading data from %s", filename)
    data = pandas.read_table(filename, usecols=[0, 1, 3], names=['user', 'item', 'rating'])

    # map each artist and user to a unique numeric value
    data['user'] = data['user'].astype("category")
    data['item'] = data['item'].astype("category")

    # store as a CSR matrix
    log.debug("read data file in %s", time.time() - start)
    return data


Instruction:
Can you tell what is the following Python 3 function doing
def get_movielens(variant="20m"):
    filename = "movielens_%s.hdf5" % variant

    path = os.path.join(_download.LOCAL_CACHE_DIR, filename)
    if not os.path.isfile(path):
        log.info("Downloading dataset to '%s'", path)
        _download.download_file(URL_BASE + filename, path)
    else:
        log.info("Using cached dataset at '%s'", path)

    with h5py.File(path, 'r') as f:
        m = f.get('movie_user_ratings')
        plays = csr_matrix((m.get('data'), m.get('indices'), m.get('indptr')))
        return np.array(f['movie']), plays

Response: Gets the movielens datasets for a given variant.


Instruction:
Write a Python 3 function for
generating a hdf5 movielens datasetfile from the raw datafiles found at path.

Response:
def generate_dataset(path, variant='20m', outputpath="."):
    """ Generates a hdf5 movielens datasetfile from the raw datafiles found at:
    https://grouplens.org/datasets/movielens/20m/

    You shouldn't have to run this yourself, and can instead just download the
    output using the 'get_movielens' funciton./
    """
    filename = os.path.join(outputpath, "movielens_%s.hdf5" % variant)

    if variant == '20m':
        ratings, movies = _read_dataframes_20M(path)
    elif variant == '100k':
        ratings, movies = _read_dataframes_100k(path)
    else:
        ratings, movies = _read_dataframes(path)

    _hfd5_from_dataframe(ratings, movies, filename)


Instruction:
How would you code a function in Python 3 to
read in the ratings and movies 20M

Response:
def _read_dataframes_20M(path):
    """ reads in the movielens 20M"""
    import pandas

    ratings = pandas.read_csv(os.path.join(path, "ratings.csv"))
    movies = pandas.read_csv(os.path.join(path, "movies.csv"))

    return ratings, movies


Instruction:
Can you tell what is the following Python 3 function doing
def _read_dataframes_100k(path):
    import pandas

    ratings = pandas.read_table(os.path.join(path, "u.data"),
                                names=['userId', 'movieId', 'rating', 'timestamp'])

    movies = pandas.read_csv(os.path.join(path, "u.item"),
                             names=['movieId', 'title'],
                             usecols=[0, 1],
                             delimiter='|',
                             encoding='ISO-8859-1')

    return ratings, movies

Response: reads in the movielens 100k dataset


Instruction:
Write a Python 3 function for
recommending items for a user Calculates the N best recommendations for a user, and returns a list of itemids, score. Parameters ---------- userid : int The userid to calculate recommendations for user_items : csr_matrix A sparse matrix of shape (number_users, number_items). This lets us look up the liked items and their weights for the user. This is used to filter out items that have already been liked from the output, and to also potentially calculate the best items for this user. N : int, optional The number of results to return filter_items : sequence of ints, optional List of extra item ids to filter out from the output recalculate_user : bool, optional When true, don't rely on stored user state and instead recalculate from the passed in user_items Returns ------- list List of (itemid, score) tuples

Response:
def recommend(self, userid, user_items,
                  N=10, filter_already_liked_items=True, filter_items=None, recalculate_user=False):
        """
        Recommends items for a user

        Calculates the N best recommendations for a user, and returns a list of itemids, score.

        Parameters
        ----------
        userid : int
            The userid to calculate recommendations for
        user_items : csr_matrix
            A sparse matrix of shape (number_users, number_items). This lets us look
            up the liked items and their weights for the user. This is used to filter out
            items that have already been liked from the output, and to also potentially
            calculate the best items for this user.
        N : int, optional
            The number of results to return
        filter_items : sequence of ints, optional
            List of extra item ids to filter out from the output
        recalculate_user : bool, optional
            When true, don't rely on stored user state and instead recalculate from the
            passed in user_items

        Returns
        -------
        list
            List of (itemid, score) tuples
        """
        pass


Instruction:
Here you have a function in Python 3, explain what it does
def find_in_path(name, path):
    "Find a file in a search path"
    # adapted fom http://code.activestate.com/recipes/52224-find-a-file-given-a-search-path/
    for dir in path.split(os.pathsep):
        binpath = os.path.join(dir, name)
        if os.path.exists(binpath):
            return os.path.abspath(binpath)
    return None

Response: Find a file in a search path


Instruction:
How would you explain what the following Python 3 function does
def locate_cuda():
    nvcc_bin = 'nvcc'
    if sys.platform.startswith("win"):
        nvcc_bin = 'nvcc.exe'

    # first check if the CUDAHOME env variable is in use
    if 'CUDAHOME' in os.environ:
        home = os.environ['CUDAHOME']
        nvcc = os.path.join(home, 'bin', nvcc_bin)
    elif 'CUDA_PATH' in os.environ:
        home = os.environ['CUDA_PATH']
        nvcc = os.path.join(home, 'bin', nvcc_bin)
    else:
        # otherwise, search the PATH for NVCC
        nvcc = find_in_path(nvcc_bin, os.environ['PATH'])
        if nvcc is None:
            logging.warning('The nvcc binary could not be located in your $PATH. Either add it to '
                            'your path, or set $CUDAHOME to enable CUDA extensions')
            return None
        home = os.path.dirname(os.path.dirname(nvcc))

    cudaconfig = {'home': home,
                  'nvcc': nvcc,
                  'include': os.path.join(home, 'include'),
                  'lib64':   os.path.join(home, 'lib64')}
    post_args = ['-gencode=arch=compute_30,code=sm_30',
                 '-gencode=arch=compute_50,code=sm_50',
                 '-gencode=arch=compute_60,code=sm_60',
                 '-gencode=arch=compute_60,code=compute_60',
                 '--ptxas-options=-v', '-O2']

    if sys.platform == "win32":
        cudaconfig['lib64'] = os.path.join(home, 'lib', 'x64')
        post_args += ['-Xcompiler', '/MD']
    else:
        post_args += ['-c', '--compiler-options', "'-fPIC'"]

    for k, v in cudaconfig.items():
        if not os.path.exists(v):
            logging.warning('The CUDA %s path could not be located in %s', k, v)
            return None

    cudaconfig['post_args'] = post_args
    return cudaconfig

Response: Locate the cuda installation on the system and return a dict with keys home nvcc include and lib64 and values giving the absolute path to each directory.


Instruction:
Write a Python 3 script for
generating a list of similar artists in lastfm by utiliizing the similar_items API of the models.

Response:
def calculate_similar_artists(output_filename, model_name="als"):
    """ generates a list of similar artists in lastfm by utiliizing the 'similar_items'
    api of the models """
    artists, users, plays = get_lastfm()

    # create a model from the input data
    model = get_model(model_name)

    # if we're training an ALS based model, weight input for last.fm
    # by bm25
    if issubclass(model.__class__, AlternatingLeastSquares):
        # lets weight these models by bm25weight.
        logging.debug("weighting matrix by bm25_weight")
        plays = bm25_weight(plays, K1=100, B=0.8)

        # also disable building approximate recommend index
        model.approximate_recommend = False

    # this is actually disturbingly expensive:
    plays = plays.tocsr()

    logging.debug("training model %s", model_name)
    start = time.time()
    model.fit(plays)
    logging.debug("trained model '%s' in %0.2fs", model_name, time.time() - start)

    # write out similar artists by popularity
    start = time.time()
    logging.debug("calculating top artists")

    user_count = np.ediff1d(plays.indptr)
    to_generate = sorted(np.arange(len(artists)), key=lambda x: -user_count[x])

    # write out as a TSV of artistid, otherartistid, score
    logging.debug("writing similar items")
    with tqdm.tqdm(total=len(to_generate)) as progress:
        with codecs.open(output_filename, "w", "utf8") as o:
            for artistid in to_generate:
                artist = artists[artistid]
                for other, score in model.similar_items(artistid, 11):
                    o.write("%s\t%s\t%s\n" % (artist, artists[other], score))
                progress.update(1)

    logging.debug("generated similar artists in %0.2fs",  time.time() - start)


Instruction:
How would you explain what the following Python 3 function does
def calculate_recommendations(output_filename, model_name="als"):
    # train the model based off input params
    artists, users, plays = get_lastfm()

    # create a model from the input data
    model = get_model(model_name)

    # if we're training an ALS based model, weight input for last.fm
    # by bm25
    if issubclass(model.__class__, AlternatingLeastSquares):
        # lets weight these models by bm25weight.
        logging.debug("weighting matrix by bm25_weight")
        plays = bm25_weight(plays, K1=100, B=0.8)

        # also disable building approximate recommend index
        model.approximate_similar_items = False

    # this is actually disturbingly expensive:
    plays = plays.tocsr()

    logging.debug("training model %s", model_name)
    start = time.time()
    model.fit(plays)
    logging.debug("trained model '%s' in %0.2fs", model_name, time.time() - start)

    # generate recommendations for each user and write out to a file
    start = time.time()
    user_plays = plays.T.tocsr()
    with tqdm.tqdm(total=len(users)) as progress:
        with codecs.open(output_filename, "w", "utf8") as o:
            for userid, username in enumerate(users):
                for artistid, score in model.recommend(userid, user_plays):
                    o.write("%s\t%s\t%s\n" % (username, artists[artistid], score))
                progress.update(1)
    logging.debug("generated recommendations in %0.2fs",  time.time() - start)

Response: Generates artist recommendations for each user in the dataset.


Instruction:
Given the following Python 3 function, write the documentation
def ion_or_solid_comp_object(formula):
    m = re.search(r"\[([^\[\]]+)\]|\(aq\)", formula)
    if m:
        comp_obj = Ion.from_formula(formula)
    elif re.search(r"\(s\)", formula):
        comp_obj = Composition(formula[:-3])
    else:
        comp_obj = Composition(formula)
    return comp_obj

Response:
Returns either an ion object given
    a formula.


Instruction:
Create a Python 3 function to
generate a label for the pourbaix plotter

Response:
def generate_entry_label(entry):
    """
    Generates a label for the pourbaix plotter

    Args:
        entry (PourbaixEntry or MultiEntry): entry to get a label for
    """
    if isinstance(entry, MultiEntry):
        return " + ".join([latexify_ion(e.name) for e in entry.entry_list])
    else:
        return latexify_ion(latexify(entry.name))


Instruction:
Can you generate the documentation for the following Python 3 function
def energy_at_conditions(self, pH, V):
        return self.energy + self.npH * PREFAC * pH + self.nPhi * V

Response: Returns the free energy at a given pH and voltage


Instruction:
Create a Python 3 function for
returning the energy at an electrochemical condition normalized by number of non - O atoms at the given conditions.

Response:
def normalized_energy_at_conditions(self, pH, V):
        """
        Energy at an electrochemical condition, compatible with
        numpy arrays for pH/V input

        Args:
            pH (float): pH at condition
            V (float): applied potential at condition

        Returns:
            energy normalized by number of non-O/H atoms at condition
        """
        return self.energy_at_conditions(pH, V) * self.normalization_factor


Instruction:
Create a Python 3 function to
return a dictionary which contains the Pourbaix Entry data.

Response:
def as_dict(self):
        """
        Returns dict which contains Pourbaix Entry data.
        Note that the pH, voltage, H2O factors are always calculated when
        constructing a PourbaixEntry object.
        """
        d = {"@module": self.__class__.__module__,
             "@class": self.__class__.__name__}
        if isinstance(self.entry, IonEntry):
            d["entry_type"] = "Ion"
        else:
            d["entry_type"] = "Solid"
        d["entry"] = self.entry.as_dict()
        d["concentration"] = self.concentration
        d["entry_id"] = self.entry_id
        return d


Instruction:
Create a Python 3 function for
creating a new entry from a dictionary.

Response:
def from_dict(cls, d):
        """
        Invokes
        """
        entry_type = d["entry_type"]
        if entry_type == "Ion":
            entry = IonEntry.from_dict(d["entry"])
        else:
            entry = PDEntry.from_dict(d["entry"])
        entry_id = d["entry_id"]
        concentration = d["concentration"]
        return PourbaixEntry(entry, entry_id, concentration)


Instruction:
Can you tell what is the following Python 3 function doing
def normalization_factor(self):
        return 1.0 / (self.num_atoms - self.composition.get('H', 0)
                      - self.composition.get('O', 0))

Response: Returns the normalization factor of the current locale.


Instruction:
Make a summary of the following Python 3 code
def from_dict(cls, d):
        return IonEntry(Ion.from_dict(d["ion"]), d["energy"], d.get("name", None))

Response: Returns an IonEntry object from a dict.


Instruction:
Make a summary of the following Python 3 code
def as_dict(self):
        d = {"ion": self.ion.as_dict(), "energy": self.energy,
             "name": self.name}
        return d

Response: Creates a dict of composition energy ion name


Instruction:
Can you generate the documentation for the following Python 3 function
def _generate_multielement_entries(self, entries, forced_include=None,
                                       nproc=None):
        """
        Create entries for multi-element Pourbaix construction.

        This works by finding all possible linear combinations
        of entries that can result in the specified composition
        from the initialized comp_dict.

        Args:
            entries ([PourbaixEntries]): list of pourbaix entries
                to process into MultiEntries
            forced_include ([PourbaixEntries]) list of pourbaix entries
                that must be included in multielement entries
            nproc (int): number of processes to be used in parallel
                treatment of entry combos
        """
        N = len(self._elt_comp)  # No. of elements
        total_comp = Composition(self._elt_comp)
        forced_include = forced_include or []

        # generate all combinations of compounds that have all elements
        entry_combos = [itertools.combinations(
            entries, j + 1 - len(forced_include)) for j in range(N)]
        entry_combos = itertools.chain.from_iterable(entry_combos)
        if forced_include:
            entry_combos = [forced_include + list(ec) for ec in entry_combos]
        entry_combos = filter(lambda x: total_comp < MultiEntry(x).composition,
                              entry_combos)

        # Generate and filter entries
        processed_entries = []
        total = sum([comb(len(entries), j + 1 - len(forced_include))
                     for j in range(N)])
        if total > 1e6:
            warnings.warn("Your pourbaix diagram includes {} entries and may "
                          "take a long time to generate.".format(total))

        # Parallel processing of multi-entry generation
        if nproc is not None:
            f = partial(self.process_multientry, prod_comp=total_comp)
            with Pool(nproc) as p:
                processed_entries = list(tqdm(p.imap(f, entry_combos),
                                              total=total))
            processed_entries = list(filter(bool, processed_entries))
        # Serial processing of multi-entry generation
        else:
            for entry_combo in entry_combos:
                processed_entry = self.process_multientry(entry_combo, total_comp)
                if processed_entry is not None:
                    processed_entries.append(processed_entry)

        return processed_entries

Response: Generate entries for multi - element Pourbaix construction.


Instruction:
Here you have a function in Python 3, explain what it does
def process_multientry(entry_list, prod_comp, coeff_threshold=1e-4):
        dummy_oh = [Composition("H"), Composition("O")]
        try:
            # Get balanced reaction coeffs, ensuring all < 0 or conc thresh
            # Note that we get reduced compositions for solids and non-reduced
            # compositions for ions because ions aren't normalized due to
            # their charge state.
            entry_comps = [e.composition for e in entry_list]
            rxn = Reaction(entry_comps + dummy_oh, [prod_comp])
            coeffs = -np.array([rxn.get_coeff(comp) for comp in entry_comps])
            # Return None if reaction coeff threshold is not met
            # TODO: this filtration step might be put somewhere else
            if (coeffs > coeff_threshold).all():
                return MultiEntry(entry_list, weights=coeffs.tolist())
            else:
                return None
        except ReactionError:
            return None

Response: This method is used to find a MultiEntry object based on a list of entries and a product composition and returns a MultiEntry object with weights according to the coefficients of the reaction coefficients.


Instruction:
Create a Python 3 function for
returning a set of pourbaix stable domains (i. e. polygons) in pH-V space from a list of pourbaix_entries This function works by using scipy's HalfspaceIntersection function to construct all of the 2-D polygons that form the boundaries of the planes corresponding to individual entry gibbs free energies as a function of pH and V. Hyperplanes of the form a*pH + b*V + 1 - g(0, 0) are constructed and supplied to HalfspaceIntersection, which then finds the boundaries of each pourbaix region using the intersection points. Args: pourbaix_entries ([PourbaixEntry]): Pourbaix entries with which to construct stable pourbaix domains limits ([[float]]): limits in which to do the pourbaix analysis Returns: Returns a dict of the form {entry: [boundary_points]}. The list of boundary points are the sides of the N-1 dim polytope bounding the allowable ph-V range of each entry.

Response:
def get_pourbaix_domains(pourbaix_entries, limits=None):
        """
        Returns a set of pourbaix stable domains (i. e. polygons) in
        pH-V space from a list of pourbaix_entries

        This function works by using scipy's HalfspaceIntersection
        function to construct all of the 2-D polygons that form the
        boundaries of the planes corresponding to individual entry
        gibbs free energies as a function of pH and V. Hyperplanes
        of the form a*pH + b*V + 1 - g(0, 0) are constructed and
        supplied to HalfspaceIntersection, which then finds the
        boundaries of each pourbaix region using the intersection
        points.

        Args:
            pourbaix_entries ([PourbaixEntry]): Pourbaix entries
                with which to construct stable pourbaix domains
            limits ([[float]]): limits in which to do the pourbaix
                analysis

        Returns:
            Returns a dict of the form {entry: [boundary_points]}.
            The list of boundary points are the sides of the N-1
            dim polytope bounding the allowable ph-V range of each entry.
        """
        if limits is None:
            limits = [[-2, 16], [-4, 4]]

        # Get hyperplanes
        hyperplanes = [np.array([-PREFAC * entry.npH, -entry.nPhi,
                                 0, -entry.energy]) * entry.normalization_factor
                       for entry in pourbaix_entries]
        hyperplanes = np.array(hyperplanes)
        hyperplanes[:, 2] = 1

        max_contribs = np.max(np.abs(hyperplanes), axis=0)
        g_max = np.dot(-max_contribs, [limits[0][1], limits[1][1], 0, 1])

        # Add border hyperplanes and generate HalfspaceIntersection
        border_hyperplanes = [[-1, 0, 0, limits[0][0]],
                              [1, 0, 0, -limits[0][1]],
                              [0, -1, 0, limits[1][0]],
                              [0, 1, 0, -limits[1][1]],
                              [0, 0, -1, 2 * g_max]]
        hs_hyperplanes = np.vstack([hyperplanes, border_hyperplanes])
        interior_point = np.average(limits, axis=1).tolist() + [g_max]
        hs_int = HalfspaceIntersection(hs_hyperplanes, np.array(interior_point))

        # organize the boundary points by entry
        pourbaix_domains = {entry: [] for entry in pourbaix_entries}
        for intersection, facet in zip(hs_int.intersections,
                                       hs_int.dual_facets):
            for v in facet:
                if v < len(pourbaix_entries):
                    this_entry = pourbaix_entries[v]
                    pourbaix_domains[this_entry].append(intersection)

        # Remove entries with no pourbaix region
        pourbaix_domains = {k: v for k, v in pourbaix_domains.items() if v}
        pourbaix_domain_vertices = {}

        for entry, points in pourbaix_domains.items():
            points = np.array(points)[:, :2]
            # Initial sort to ensure consistency
            points = points[np.lexsort(np.transpose(points))]
            center = np.average(points, axis=0)
            points_centered = points - center

            # Sort points by cross product of centered points,
            # isn't strictly necessary but useful for plotting tools
            point_comparator = lambda x, y: x[0] * y[1] - x[1] * y[0]
            points_centered = sorted(points_centered,
                                     key=cmp_to_key(point_comparator))
            points = points_centered + center

            # Create simplices corresponding to pourbaix boundary
            simplices = [Simplex(points[indices])
                         for indices in ConvexHull(points).simplices]
            pourbaix_domains[entry] = simplices
            pourbaix_domain_vertices[entry] = points

        return pourbaix_domains, pourbaix_domain_vertices