Instruction:
Implement a function in Python 3 to
search for Zomato locations by keyword.

Response:
def getLocations(self, query, **kwargs):
        """
        :param query: suggestion for location name
        :param lat: latitude
        :param lon: longitude
        :param count: number of max results to display
        :return: json response
        Search for Zomato locations by keyword. Provide coordinates to get better search results
        """
        params = {"query": query}
        optional_params = ["lat", "lon", "count"]

        for key in optional_params:
            if key in kwargs:
                params[key] = kwargs[key]
        locations = self.api.get("/locations", params)
        return locations


Instruction:
Can you generate the documentation for the following Python 3 function
def getDailyMenu(self, restaurant_id):
        params = {"res_id": restaurant_id}
        daily_menu = self.api.get("/dailymenu", params)
        return daily_menu

Response: Get daily menu using Zomato restaurant ID.


Instruction:
How would you code a function in Python 3 to
get detailed restaurant details

Response:
def getRestaurantDetails(self, restaurant_id):
        """
        :param restaurant_id: id of restaurant whose details are requested
        :return: json response
        Get detailed restaurant information using Zomato restaurant ID.
        Partner Access is required to access photos and reviews.
        """
        params = {"res_id": restaurant_id}
        restaurant_details = self.api.get("/restaurant", params)
        return restaurant_details


Instruction:
How would you code a function in Python 3 to
get the restaurant reviews using the Zomato restaurant ID

Response:
def getRestaurantReviews(self, restaurant_id, **kwargs):
        """
        :param restaurant_id: id of restaurant whose details are requested
        :param start: fetch results after this offset
        :param count: max number of results to retrieve
        :return: json response
        Get restaurant reviews using the Zomato restaurant ID
        """
        params = {"res_id": restaurant_id}
        optional_params = ["start", "count"]

        for key in optional_params:
            if key in kwargs:
                params[key] = kwargs[key]
        reviews = self.api.get("/reviews", params)
        return reviews


Instruction:
Can you implement a function in Python 3 that
searches for restaurants in a Zomato location.

Response:
def search(self, **kwargs):
        """
        :param entity_id: location id
        :param entity_type: location type (city, subzone, zone, lanmark, metro , group)
        :param q: search keyword
        :param start: fetch results after offset
        :param count: max number of results to display
        :param lat: latitude
        :param lon: longitude
        :param radius: radius around (lat,lon); to define search area, defined in meters(M)
        :param cuisines: list of cuisine id's separated by comma
        :param establishment_type: estblishment id obtained from establishments call
        :param collection_id: collection id obtained from collections call
        :param category: category ids obtained from categories call
        :param sort: sort restaurants by (cost, rating, real_distance)
        :param order: used with 'sort' parameter to define ascending / descending
        :return: json response
        The location input can be specified using Zomato location ID or coordinates. Cuisine / Establishment /
        Collection IDs can be obtained from respective api calls.

        Partner Access is required to access photos and reviews.

        Examples:
        - To search for 'Italian' restaurants in 'Manhattan, New York City',
        set cuisines = 55, entity_id = 94741 and entity_type = zone
        - To search for 'cafes' in 'Manhattan, New York City',
        set establishment_type = 1, entity_type = zone and entity_id = 94741
        - Get list of all restaurants in 'Trending this Week' collection in 'New York City' by using
        entity_id = 280, entity_type = city and collection_id = 1
        """
        params = {}
        available_params = [
            "entity_id", "entity_type", "q", "start",
            "count", "lat", "lon", "radius", "cuisines",
            "establishment_type", "collection_id",
            "category", "sort", "order"]

        for key in available_params:
            if key in kwargs:
                params[key] = kwargs[key]
        results = self.api.get("/search", params)
        return results


Instruction:
Given the following Python 3 function, write the documentation
def array(a, context=None, axis=(0,), dtype=None, npartitions=None):
        if dtype is None:
            arry = asarray(a)
            dtype = arry.dtype
        else:
            arry = asarray(a, dtype)
        shape = arry.shape
        ndim = len(shape)

        # handle the axes specification and transpose if necessary
        axes = ConstructSpark._format_axes(axis, arry.shape)
        key_axes, value_axes = get_kv_axes(arry.shape, axes)
        permutation = key_axes + value_axes
        arry = arry.transpose(*permutation)
        split = len(axes)

        if split < 1:
            raise ValueError("split axis must be greater than 0, got %g" % split)
        if split > len(shape):
            raise ValueError("split axis must not exceed number of axes %g, got %g" % (ndim, split))

        key_shape = shape[:split]
        val_shape = shape[split:]

        keys = zip(*unravel_index(arange(0, int(prod(key_shape))), key_shape))
        vals = arry.reshape((prod(key_shape),) + val_shape)

        rdd = context.parallelize(zip(keys, vals), npartitions)
        return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)

Response: Create a spark bolt array from an array - like object.


Instruction:
Can you tell what is the following Python 3 function doing
def ones(shape, context=None, axis=(0,), dtype=float64, npartitions=None):
        from numpy import ones
        return ConstructSpark._wrap(ones, shape, context, axis, dtype, npartitions)

Response: Construct a spark bolt array of ones.


Instruction:
Can you implement a function in Python 3 that
concatenates two bolt arrays together along a given axis.

Response:
def concatenate(arrays, axis=0):
        """
        Join two bolt arrays together, at least one of which is in spark.

        Parameters
        ----------
        arrays : tuple
            A pair of arrays. At least one must be a spark array,
            the other can be a local bolt array, a local numpy array,
            or an array-like.

        axis : int, optional, default=0
            The axis along which the arrays will be joined.

        Returns
        -------
        BoltArraySpark
        """
        if not isinstance(arrays, tuple):
            raise ValueError("data type not understood")
        if not len(arrays) == 2:
            raise NotImplementedError("spark concatenation only supports two arrays")

        first, second = arrays
        if isinstance(first, BoltArraySpark):
            return first.concatenate(second, axis)
        elif isinstance(second, BoltArraySpark):
            first = ConstructSpark.array(first, second._rdd.context)
            return first.concatenate(second, axis)
        else:
            raise ValueError("at least one array must be a spark bolt array")


Instruction:
Write a Python 3 function for
checking that the arguments are consistent with Spark array construction.

Response:
def _argcheck(*args, **kwargs):
        """
        Check that arguments are consistent with spark array construction.

        Conditions are:
        (1) a positional argument is a SparkContext
        (2) keyword arg 'context' is a SparkContext
        (3) an argument is a BoltArraySpark, or
        (4) an argument is a nested list containing a BoltArraySpark
        """
        try:
            from pyspark import SparkContext
        except ImportError:
            return False

        cond1 = any([isinstance(arg, SparkContext) for arg in args])
        cond2 = isinstance(kwargs.get('context', None), SparkContext)
        cond3 = any([isinstance(arg, BoltArraySpark) for arg in args])
        cond4 = any([any([isinstance(sub, BoltArraySpark) for sub in arg])
                     if isinstance(arg, (tuple, list)) else False for arg in args])
        return cond1 or cond2 or cond3 or cond4


Instruction:
Make a summary of the following Python 3 code
def _format_axes(axes, shape):
        if isinstance(axes, int):
            axes = (axes,)
        elif isinstance(axes, list) or hasattr(axes, '__iter__'):
            axes = tuple(axes)
        if not isinstance(axes, tuple):
            raise ValueError("axes argument %s in the constructor not specified correctly" % str(axes))
        if min(axes) < 0 or max(axes) > len(shape) - 1:
            raise ValueError("invalid key axes %s given shape %s" % (str(axes), str(shape)))
        return axes

Response: Format target axes given an array shape


Instruction:
Can you tell what is the following Python 3 function doing
def _wrap(func, shape, context=None, axis=(0,), dtype=None, npartitions=None):
        if isinstance(shape, int):
            shape = (shape,)
        key_shape, value_shape = get_kv_shape(shape, ConstructSpark._format_axes(axis, shape))
        split = len(key_shape)

        # make the keys
        rdd = context.parallelize(list(product(*[arange(x) for x in key_shape])), npartitions)

        # use a map to make the arrays in parallel
        rdd = rdd.map(lambda x: (x, func(value_shape, dtype, order='C')))
        return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)

Response: Wrap an existing numpy constructor in a parallelized construction


Instruction:
Create a Python 3 function to
align the local bolt array so that the specified axes for iteration are in the keys.

Response:
def _align(self, axes, key_shape=None):
        """
        Align local bolt array so that axes for iteration are in the keys.

        This operation is applied before most functional operators.
        It ensures that the specified axes are valid, and might transpose/reshape
        the underlying array so that the functional operators can be applied
        over the correct records.

        Parameters
        ----------
        axes: tuple[int]
            One or more axes that will be iterated over by a functional operator

        Returns
        -------
        BoltArrayLocal
        """

        # ensure that the key axes are valid for an ndarray of this shape
        inshape(self.shape, axes)

        # compute the set of dimensions/axes that will be used to reshape
        remaining = [dim for dim in range(len(self.shape)) if dim not in axes]
        key_shape = key_shape if key_shape else [self.shape[axis] for axis in axes]
        remaining_shape = [self.shape[axis] for axis in remaining]
        linearized_shape = [prod(key_shape)] + remaining_shape

        # compute the transpose permutation
        transpose_order = axes + remaining

        # transpose the array so that the keys being mapped over come first, then linearize keys
        reshaped = self.transpose(*transpose_order).reshape(*linearized_shape)

        return reshaped


Instruction:
Write a Python 3 script for
returning a new BoltArrayLocal containing only the entries in the log entry table that satisfy a function along a given axis.

Response:
def filter(self, func, axis=(0,)):
        """
        Filter array along an axis.

        Applies a function which should evaluate to boolean,
        along a single axis or multiple axes. Array will be
        aligned so that the desired set of axes are in the
        keys, which may require a transpose/reshape.

        Parameters
        ----------
        func : function
            Function to apply, should return boolean

        axis : tuple or int, optional, default=(0,)
            Axis or multiple axes to filter along.

        Returns
        -------
        BoltArrayLocal
        """
        axes = sorted(tupleize(axis))
        reshaped = self._align(axes)

        filtered = asarray(list(filter(func, reshaped)))

        return self._constructor(filtered)


Instruction:
Can you generate the documentation for the following Python 3 function
def map(self, func, axis=(0,)):
        axes = sorted(tupleize(axis))
        key_shape = [self.shape[axis] for axis in axes]
        reshaped = self._align(axes, key_shape=key_shape)

        mapped = asarray(list(map(func, reshaped)))
        elem_shape = mapped[0].shape

        # invert the previous reshape operation, using the shape of the map result
        linearized_shape_inv = key_shape + list(elem_shape)
        reordered = mapped.reshape(*linearized_shape_inv)

        return self._constructor(reordered)

Response: Applies a function across an axis and returns a BoltArrayLocal object with the resulting array.


Instruction:
How would you code a function in Python 3 to
reduce the array along an axis.

Response:
def reduce(self, func, axis=0):
        """
        Reduce an array along an axis.

        Applies an associative/commutative function of two arguments
        cumulatively to all arrays along an axis. Array will be aligned
        so that the desired set of axes are in the keys, which may
        require a transpose/reshape.

        Parameters
        ----------
        func : function
            Function of two arrays that returns a single array

        axis : tuple or int, optional, default=(0,)
            Axis or multiple axes to reduce along.

        Returns
        -------
        BoltArrayLocal
        """
        axes = sorted(tupleize(axis))

        # if the function is a ufunc, it can automatically handle reducing over multiple axes
        if isinstance(func, ufunc):
            inshape(self.shape, axes)
            reduced = func.reduce(self, axis=tuple(axes))
        else:
            reshaped = self._align(axes)
            reduced = reduce(func, reshaped)

        new_array = self._constructor(reduced)

        # ensure that the shape of the reduced array is valid
        expected_shape = [self.shape[i] for i in range(len(self.shape)) if i not in axes]
        if new_array.shape != tuple(expected_shape):
            raise ValueError("reduce did not yield a BoltArray with valid dimensions")

        return new_array


Instruction:
How would you implement a function in Python 3 that
concatenates this array with another array.

Response:
def concatenate(self, arry, axis=0):
        """
        Join this array with another array.

        Paramters
        ---------
        arry : ndarray or BoltArrayLocal
            Another array to concatenate with

        axis : int, optional, default=0
            The axis along which arrays will be joined.

        Returns
        -------
        BoltArrayLocal
        """
        if isinstance(arry, ndarray):
            from bolt import concatenate
            return concatenate((self, arry), axis)
        else:
            raise ValueError("other must be local array, got %s" % type(arry))


Instruction:
How would you code a function in Python 3 to
convert a BoltArrayLocal into a BoltArraySpark

Response:
def tospark(self, sc, axis=0):
        """
        Converts a BoltArrayLocal into a BoltArraySpark

        Parameters
        ----------
        sc : SparkContext
            The SparkContext which will be used to create the BoltArraySpark

        axis : tuple or int, optional, default=0
            The axis (or axes) across which this array will be parallelized

        Returns
        -------
        BoltArraySpark
        """
        from bolt import array
        return array(self.toarray(), sc, axis=axis)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def tordd(self, sc, axis=0):
        from bolt import array
        return array(self.toarray(), sc, axis=axis).tordd()

Response: Converts a BoltArrayLocal into an RDD of the Topology s unique keys.


Instruction:
Create a Python 3 function to
make an intermediate RDD where all records are combined into a list of keys and larger ndarray along a new 0th dimension.

Response:
def stack(self, size):
        """
        Make an intermediate RDD where all records are combined into a
        list of keys and larger ndarray along a new 0th dimension.
        """
        def tostacks(partition):
            keys = []
            arrs = []
            for key, arr in partition:
                keys.append(key)
                arrs.append(arr)
                if size and 0 <= size <= len(keys):
                    yield (keys, asarray(arrs))
                    keys, arrs = [], []
            if keys:
                yield (keys, asarray(arrs))

        rdd = self._rdd.mapPartitions(tostacks)
        return self._constructor(rdd).__finalize__(self)


Instruction:
Explain what the following Python 3 code does
def map(self, func):
        vshape = self.shape[self.split:]
        x = self._rdd.values().first()
        if x.shape == vshape:
            a, b = asarray([x]), asarray([x, x])
        else:
            a, b = x, concatenate((x, x))

        try:
            atest = func(a)
            btest = func(b)
        except Exception as e:
            raise RuntimeError("Error evaluating function on test array, got error:\n %s" % e)

        if not (isinstance(atest, ndarray) and isinstance(btest, ndarray)):
            raise ValueError("Function must return ndarray")

        # different shapes map to the same new shape
        elif atest.shape == btest.shape:
            if self._rekeyed is True:
                # we've already rekeyed
                rdd = self._rdd.map(lambda kv: (kv[0], func(kv[1])))
                shape = (self.shape[0],) + atest.shape
            else:
                # do the rekeying
                count, rdd = zip_with_index(self._rdd.values())
                rdd = rdd.map(lambda kv: ((kv[1],), func(kv[0])))
                shape = (count,) + atest.shape
            split = 1
            rekeyed = True

        # different shapes stay different (along the first dimension)
        elif atest.shape[0] == a.shape[0] and btest.shape[0] == b.shape[0]:
            shape = self.shape[0:self.split] + atest.shape[1:]
            split = self.split
            rdd = self._rdd.map(lambda kv: (kv[0], func(kv[1])))
            rekeyed = self._rekeyed

        else:
            raise ValueError("Cannot infer effect of function on shape")

        return self._constructor(rdd, rekeyed=rekeyed, shape=shape, split=split).__finalize__(self)

Response: Applies a function on each value in the intermediate RDD.


Instruction:
Can you create a Python 3 function that
splits the distributed array into chunks.

Response:
def _chunk(self, size="150", axis=None, padding=None):
        """
        Split values of distributed array into chunks.

        Transforms an underlying pair RDD of (key, value) into
        records of the form: (key, chunk id), (chunked value).
        Here, chunk id is a tuple identifying the chunk and
        chunked value is a subset of the data from each original value,
        that has been divided along the specified dimensions.

        Parameters
        ----------
        size : str or tuple or int
            If str, the average size (in KB) of the chunks in all value dimensions.
            If int or tuple, an explicit specification of the number chunks in
            each value dimension.

        axis : tuple, optional, default=None
            One or more axes to estimate chunks for, if provided any
            other axes will use one chunk.

        padding: tuple or int, default = None
            Number of elements per dimension that will overlap with the adjacent chunk.
            If a tuple, specifies padding along each chunked dimension; if a int, same
            padding will be applied to all chunked dimensions.
        """
        if self.split == len(self.shape) and padding is None:
            self._rdd = self._rdd.map(lambda kv: (kv[0]+(0,), array(kv[1], ndmin=1)))
            self._shape = self._shape + (1,)
            self._plan = (1,)
            self._padding = array([0])
            return self

        rdd = self._rdd
        self._plan, self._padding = self.getplan(size, axis, padding)

        if any([x + y > z for x, y, z in zip(self.plan, self.padding, self.vshape)]):
            raise ValueError("Chunk sizes %s plus padding sizes %s cannot exceed value dimensions %s along any axis"
                             % (tuple(self.plan), tuple(self.padding), tuple(self.vshape)))

        if any([x > y for x, y in zip(self.padding, self.plan)]):
            raise ValueError("Padding sizes %s cannot exceed chunk sizes %s along any axis"
                             % (tuple(self.padding), tuple(self.plan)))

        slices = self.getslices(self.plan, self.padding, self.vshape)
        labels = list(product(*[list(enumerate(s)) for s in slices]))
        scheme = [list(zip(*s)) for s in labels]

        def _chunk(record):
            k, v = record[0], record[1]
            for (chk, slc) in scheme:
                if type(k) is int:
                    k = (k,)
                yield k + chk, v[slc]

        rdd = rdd.flatMap(_chunk)
        return self._constructor(rdd, shape=self.shape, split=self.split,
                                 dtype=self.dtype, plan=self.plan, padding=self.padding, ordered=self._ordered)


Instruction:
Write a Python 3 function for
converting a chunked array back into a full array with ( key value ) pairs where key is a tuple of indices and value is an ndarray.

Response:
def unchunk(self):
        """
        Convert a chunked array back into a full array with (key,value) pairs
        where key is a tuple of indices, and value is an ndarray.
        """
        plan, padding, vshape, split = self.plan, self.padding, self.vshape, self.split
        nchunks = self.getnumber(plan, vshape)
        full_shape = concatenate((nchunks, plan))
        n = len(vshape)
        perm = concatenate(list(zip(range(n), range(n, 2*n))))

        if self.uniform:
            def _unchunk(it):
                ordered = sorted(it, key=lambda kv: kv[0][split:])
                keys, values = zip(*ordered)
                yield keys[0][:split], asarray(values).reshape(full_shape).transpose(perm).reshape(vshape)
        else:
            def _unchunk(it):
                ordered = sorted(it, key=lambda kv: kv[0][split:])
                keys, values = zip(*ordered)
                k_chks = [k[split:] for k in keys]
                arr = empty(nchunks, dtype='object')
                for (i, d) in zip(k_chks, values):
                    arr[i] = d
                yield keys[0][:split], allstack(arr.tolist())

        # remove padding
        if self.padded:
            removepad = self.removepad
            rdd = self._rdd.map(lambda kv: (kv[0], removepad(kv[0][split:], kv[1], nchunks, padding, axes=range(n))))
        else:
            rdd = self._rdd

        # skip partitionBy if there is not actually any chunking
        if array_equal(self.plan, self.vshape):
           rdd = rdd.map(lambda kv: (kv[0][:split], kv[1]))
           ordered = self._ordered
        else:
            ranges = self.kshape
            npartitions = int(prod(ranges))
            if len(self.kshape) == 0:
                partitioner = lambda k: 0
            else:
                partitioner = lambda k: ravel_multi_index(k[:split], ranges)
            rdd = rdd.partitionBy(numPartitions=npartitions, partitionFunc=partitioner).mapPartitions(_unchunk)
            ordered = True

        if array_equal(self.vshape, [1]):
            rdd = rdd.mapValues(lambda v: squeeze(v))
            newshape = self.shape[:-1]
        else:
            newshape = self.shape

        return BoltArraySpark(rdd, shape=newshape, split=self._split,
                              dtype=self.dtype, ordered=ordered)


Instruction:
How would you code a function in Python 3 to
return a new ChunkedArray containing the keys and values for the specified axes.

Response:
def keys_to_values(self, axes, size=None):
        """
        Move indices in the keys into the values.

        Padding on these new value-dimensions is not currently supported and is set to 0.

        Parameters
        ----------
        axes : tuple
            Axes from keys to move to values.

        size : tuple, optional, default=None
            Size of chunks for the values along the new dimensions.
            If None, then no chunking for all axes (number of chunks = 1)

        Returns
        -------
        ChunkedArray
        """
        if len(axes) == 0:
            return self

        kmask = self.kmask(axes)

        if size is None:
            size = self.kshape[kmask]

        # update properties
        newplan = r_[size, self.plan]
        newsplit = self._split - len(axes)
        newshape = tuple(r_[self.kshape[~kmask], self.kshape[kmask], self.vshape].astype(int).tolist())
        newpadding = r_[zeros(len(axes), dtype=int), self.padding]

        result = self._constructor(None, shape=newshape, split=newsplit,
                                   dtype=self.dtype, plan=newplan, padding=newpadding, ordered=True)

        # convert keys into chunk + within-chunk label
        split = self.split
        def _relabel(record):
            k, data = record
            keys, chks = asarray(k[:split], 'int'), k[split:]
            movingkeys, stationarykeys = keys[kmask], keys[~kmask]
            newchks = [int(m) for m in movingkeys/size]  # element-wise integer division that works in Python 2 and 3
            labels = mod(movingkeys, size)
            return tuple(stationarykeys) + tuple(newchks) + tuple(chks) + tuple(labels), data

        rdd = self._rdd.map(_relabel)

        # group the new chunks together
        nchunks = result.getnumber(result.plan, result.vshape)
        npartitions = int(prod(result.kshape) * prod(nchunks))
        ranges = tuple(result.kshape) + tuple(nchunks)
        n = len(axes)
        if n == 0:
            s = slice(None)
        else:
            s = slice(-n)
        partitioner = lambda k: ravel_multi_index(k[s], ranges)

        rdd = rdd.partitionBy(numPartitions=npartitions, partitionFunc=partitioner)

        # reassemble the pieces in the chunks by sorting and then stacking
        uniform = result.uniform

        def _rebuild(it):
            ordered = sorted(it, key=lambda kv: kv[0][n:])
            keys, data = zip(*ordered)

            k = keys[0][s]
            labels = asarray([x[-n:] for x in keys])

            if uniform:
                labelshape = tuple(size)
            else:
                labelshape = tuple(amax(labels, axis=0) - amin(labels, axis=0) + 1)

            valshape = data[0].shape
            fullshape = labelshape + valshape
            yield k, asarray(data).reshape(fullshape)

        result._rdd = rdd.mapPartitions(_rebuild)

        if array_equal(self.vshape, [1]):
            result._rdd = result._rdd.mapValues(lambda v: squeeze(v))
            result._shape = result.shape[:-1]
            result._plan = result.plan[:-1]

        return result


Instruction:
Write a Python 3 function that can
apply an array function to each subarray and returns a ChunkedArray containing the result.

Response:
def map(self, func, value_shape=None, dtype=None):
        """
        Apply an array -> array function on each subarray.

        The function can change the shape of the subarray, but only along
        dimensions that are not chunked.

        Parameters
        ----------
        func : function
            Function of a single subarray to apply

        value_shape:
            Known shape of chunking plan after the map

        dtype: numpy.dtype, optional, default=None
            Known dtype of values resulting from operation

        Returns
        -------
        ChunkedArray
        """

        if value_shape is None or dtype is None:
            # try to compute the size of each mapped element by applying func to a random array
            try:
                mapped = func(random.randn(*self.plan).astype(self.dtype))
            except Exception:
                first = self._rdd.first()
                if first:
                    # eval func on the first element
                    mapped = func(first[1])
            if value_shape is None:
                value_shape = mapped.shape
            if dtype is None:
                dtype = mapped.dtype

        chunked_dims = where(self.plan != self.vshape)[0]
        unchunked_dims = where(self.plan == self.vshape)[0]

        # check that no dimensions are dropped
        if len(value_shape) != len(self.plan):
            raise NotImplementedError('map on ChunkedArray cannot drop dimensions')

        # check that chunked dimensions did not change shape
        if any([value_shape[i] != self.plan[i] for i in chunked_dims]):
            raise ValueError('map cannot change the sizes of chunked dimensions')

        def check_and_apply(v):
            new = func(v)
            if len(unchunked_dims) > 0:
                if any([new.shape[i] != value_shape[i] for i in unchunked_dims]):
                    raise Exception("Map operation did not produce values of uniform shape.")
            if len(chunked_dims) > 0:
                if any([v.shape[i] != new.shape[i] for i in chunked_dims]):
                    raise Exception("Map operation changed the size of a chunked dimension")
            return new

        rdd = self._rdd.mapValues(check_and_apply)

        vshape = [value_shape[i] if i in unchunked_dims else self.vshape[i] for i in range(len(self.vshape))]
        newshape = r_[self.kshape, vshape].astype(int).tolist()

        return self._constructor(rdd, shape=tuple(newshape), dtype=dtype,
                                 plan=asarray(value_shape)).__finalize__(self)


Instruction:
Here you have a function in Python 3, explain what it does
def map_generic(self, func):
        def process_record(val):
            newval = empty(1, dtype="object")
            newval[0]  = func(val)
            return newval

        rdd = self._rdd.mapValues(process_record)

        nchunks = self.getnumber(self.plan, self.vshape)
        newshape = tuple([int(s) for s in r_[self.kshape, nchunks]])
        newsplit = len(self.shape)
        return BoltArraySpark(rdd, shape=newshape, split=newsplit, ordered=self._ordered, dtype="object")

Response: Applies a generic array - > object to each subarray


Instruction:
Here you have a function in Python 3, explain what it does
def getplan(self, size="150", axes=None, padding=None):
        from numpy import dtype as gettype

        # initialize with all elements in one chunk
        plan = self.vshape

        # check for subset of axes
        if axes is None:
            if isinstance(size, str):
                axes = arange(len(self.vshape))
            else:
                axes = arange(len(size))
        else:
            axes = asarray(axes, 'int')

        # set padding
        pad = array(len(self.vshape)*[0, ])
        if padding is not None:
            pad[axes] = padding

        # set the plan
        if isinstance(size, tuple):
            plan[axes] = size

        elif isinstance(size, str):
            # convert from kilobytes
            size = 1000.0 * float(size)

            # calculate from dtype
            elsize = gettype(self.dtype).itemsize
            nelements = prod(self.vshape)
            dims = self.vshape[self.vmask(axes)]

            if size <= elsize:
                s = ones(len(axes))

            else:
                remsize = 1.0 * nelements * elsize
                s = []
                for (i, d) in enumerate(dims):
                    minsize = remsize/d
                    if minsize >= size:
                        s.append(1)
                        remsize = minsize
                        continue
                    else:
                        s.append(min(d, floor(size/minsize)))
                        s[i+1:] = plan[i+1:]
                        break

            plan[axes] = s

        else:
            raise ValueError("Chunk size not understood, must be tuple or int")

        return plan, pad

Response: Return a numpy array with the size of the specified size along each axis.


Instruction:
Create a Python 3 function to
remove the padding from chunks.

Response:
def removepad(idx, value, number, padding, axes=None):
        """
        Remove the padding from chunks.

        Given a chunk and its corresponding index, use the plan and padding to remove any
        padding from the chunk along with specified axes.

        Parameters
        ----------
        idx: tuple or array-like
            The chunk index, indicating which chunk this is.

        value: ndarray
            The chunk that goes along with the index.

        number: ndarray or array-like
            The number of chunks along each dimension.

        padding: ndarray or array-like
            The padding scheme.

        axes: tuple, optional, default = None
            The axes (in the values) along which to remove padding.
        """
        if axes is None:
            axes = range(len(number))
        mask = len(number)*[False, ]
        for i in range(len(mask)):
            if i in axes and padding[i] != 0:
                mask[i] = True

        starts = [0 if (i == 0 or not m) else p for (i, m, p) in zip(idx, mask, padding)]
        stops = [None if (i == n-1 or not m) else -p for (i, m, p, n) in zip(idx, mask, padding, number)]
        slices = [slice(i1, i2) for (i1, i2) in zip(starts, stops)]

        return value[slices]


Instruction:
Explain what the following Python 3 code does
def getnumber(plan, shape):
        nchunks = []
        for size, d in zip(plan, shape):
            nchunks.append(int(ceil(1.0 * d/size)))
        return nchunks

Response: Returns the number of chunks for the given plan and shape.


Instruction:
Create a Python 3 function to
return a list of slices for the given dimensions padding and shape.

Response:
def getslices(plan, padding, shape):
        """
        Obtain slices for the given dimensions, padding, and chunks.

        Given a plan for the number of chunks along each dimension and the amount of padding,
        calculate a list of slices required to generate those chunks.

        Parameters
        ----------
        plan: tuple or array-like
            Size of chunks (in number of elements) along each dimensions.
            Length must be equal to the number of dimensions.

        padding: tuple or array-like
            Size of overlap (in number of elements) between chunks along each dimension.
            Length must be equal to the number of dimensions.

        shape: tuple
             Dimensions of axes to be chunked.
        """
        slices = []
        for size, pad, d in zip(plan, padding, shape):
            nchunks = int(floor(d/size))
            remainder = d % size
            start = 0
            dimslices = []
            for idx in range(nchunks):
                end = start + size
                # left endpoint
                if idx == 0:
                    left = start
                else:
                    left = start - pad
                # right endpoint
                if idx == nchunks:
                    right = end
                else:
                    right = end + pad
                dimslices.append(slice(left, right, 1))
                start = end
            if remainder:
                dimslices.append(slice(end - pad, d, 1))
            slices.append(dimslices)
        return slices


Instruction:
Make a summary of the following Python 3 code
def getmask(inds, n):
        inds = asarray(inds, 'int')
        mask = zeros(n, dtype=bool)
        mask[inds] = True
        return mask

Response: Returns a binary mask by setting a subset of entries to true.


Instruction:
Here you have a function in Python 3, explain what it does
def repartition(self, npartitions):

        rdd = self._rdd.repartition(npartitions)
        return self._constructor(rdd, ordered=False).__finalize__(self)

Response: Repartition the underlying RDD to a new number of partitions.


Instruction:
Implement a function in Python 3 to
return a new StackedArray object with the original records of the current distributed array.

Response:
def stack(self, size=None):
        """
        Aggregates records of a distributed array.

        Stacking should improve the performance of vectorized operations,
        but the resulting StackedArray object only exposes a restricted set
        of operations (e.g. map, reduce). The unstack method can be used
        to restore the full bolt array.

        Parameters
        ----------
        size : int, optional, default=None
            The maximum size for each stack (number of original records),
            will aggregate groups of records per partition up to this size,
            if None will aggregate all records on each partition.

        Returns
        -------
        StackedArray
        """
        stk = StackedArray(self._rdd, shape=self.shape, split=self.split)
        return stk.stack(size)


Instruction:
Implement a Python 3 function for
aligning spark bolt array so that the specified axes for iteration are in the keys and values.

Response:
def _align(self, axis):
        """
        Align spark bolt array so that axes for iteration are in the keys.

        This operation is applied before most functional operators.
        It ensures that the specified axes are valid, and swaps
        key/value axes so that functional operators can be applied
        over the correct records.

        Parameters
        ----------
        axis: tuple[int]
            One or more axes that wil be iterated over by a functional operator

        Returns
        -------
        BoltArraySpark
        """
        # ensure that the specified axes are valid
        inshape(self.shape, axis)

        # find the value axes that should be moved into the keys (axis >= split)
        tokeys = [(a - self.split) for a in axis if a >= self.split]

        # find the key axes that should be moved into the values (axis < split)
        tovalues = [a for a in range(self.split) if a not in axis]

        if tokeys or tovalues:
            return self.swap(tovalues, tokeys)
        else:
            return self


Instruction:
How would you implement a function in Python 3 that
returns the first element of an array

Response:
def first(self):
        """
        Return the first element of an array
        """
        from bolt.local.array import BoltArrayLocal
        rdd = self._rdd if self._ordered else self._rdd.sortByKey()
        return BoltArrayLocal(rdd.values().first())


Instruction:
Write a Python 3 function that can
apply a function across an axis and returns a BoltArraySpark object with the mapped elements.

Response:
def map(self, func, axis=(0,), value_shape=None, dtype=None, with_keys=False):
        """
        Apply a function across an axis.

        Array will be aligned so that the desired set of axes
        are in the keys, which may incur a swap.

        Parameters
        ----------
        func : function
            Function of a single array to apply. If with_keys=True,
            function should be of a (tuple, array) pair.

        axis : tuple or int, optional, default=(0,)
            Axis or multiple axes to apply function along.

        value_shape : tuple, optional, default=None
            Known shape of values resulting from operation

        dtype: numpy.dtype, optional, default=None
            Known dtype of values resulting from operation

        with_keys : bool, optional, default=False
            Include keys as an argument to the function

        Returns
        -------
        BoltArraySpark
        """
        axis = tupleize(axis)
        swapped = self._align(axis)

        if with_keys:
            test_func = lambda x: func(((0,), x))
        else:
            test_func = func

        if value_shape is None or dtype is None:
            # try to compute the size of each mapped element by applying func to a random array
            try:
                mapped = test_func(random.randn(*swapped.values.shape).astype(self.dtype))
            except Exception:
                first = swapped._rdd.first()
                if first:
                    # eval func on the first element
                    mapped = test_func(first[1])
            if value_shape is None:
                value_shape = mapped.shape
            if dtype is None:
                dtype = mapped.dtype

        shape = tuple([swapped._shape[ax] for ax in range(len(axis))]) + tupleize(value_shape)

        if with_keys:
            rdd = swapped._rdd.map(lambda kv: (kv[0], func(kv)))
        else:
            rdd = swapped._rdd.mapValues(func)

        # reshaping will fail if the elements aren't uniformly shaped
        def check(v):
            if len(v.shape) > 0 and v.shape != tupleize(value_shape):
                raise Exception("Map operation did not produce values of uniform shape.")
            return v

        rdd = rdd.mapValues(lambda v: check(v))

        return self._constructor(rdd, shape=shape, dtype=dtype, split=swapped.split).__finalize__(swapped)


Instruction:
Can you generate the documentation for the following Python 3 function
def filter(self, func, axis=(0,), sort=False):
        axis = tupleize(axis)

        swapped = self._align(axis)
        def f(record):
            return func(record[1])
        rdd = swapped._rdd.filter(f)
        if sort:
            rdd = rdd.sortByKey().values()
        else:
            rdd = rdd.values()

        # count the resulting array in order to reindex (linearize) the keys
        count, zipped = zip_with_index(rdd)
        if not count:
            count = zipped.count()
        reindexed = zipped.map(lambda kv: (tupleize(kv[1]), kv[0]))

        # since we can only filter over one axis, the remaining shape is always the following
        remaining = list(swapped.shape[len(axis):])
        if count != 0:
            shape = tuple([count] + remaining)
        else:
            shape = (0,)

        return self._constructor(reindexed, shape=shape, split=1).__finalize__(swapped)

Response: Filter array along an axis.


Instruction:
Write a Python 3 function that can
apply a commutative or associative function of two arrays along an axis and returns a single array along that axis.

Response:
def reduce(self, func, axis=(0,), keepdims=False):
        """
        Reduce an array along an axis.

        Applies a commutative/associative function of two
        arguments cumulatively to all arrays along an axis.
        Array will be aligned so that the desired set of axes
        are in the keys, which may incur a swap.

        Parameters
        ----------
        func : function
            Function of two arrays that returns a single array

        axis : tuple or int, optional, default=(0,)
            Axis or multiple axes to reduce along.

        Returns
        -------
        BoltArraySpark
        """
        from bolt.local.array import BoltArrayLocal
        from numpy import ndarray

        axis = tupleize(axis)
        swapped = self._align(axis)
        arr = swapped._rdd.values().treeReduce(func, depth=3)

        if keepdims:
            for i in axis:
                arr = expand_dims(arr, axis=i)

        if not isinstance(arr, ndarray):
            # the result of a reduce can also be a scalar
            return arr
        elif arr.shape == (1,):
            # ndarrays with single values in them should be converted into scalars
            return arr[0]

        return BoltArrayLocal(arr)


Instruction:
How would you code a function in Python 3 to
compute a statistic over an axis.

Response:
def _stat(self, axis=None, func=None, name=None, keepdims=False):
        """
        Compute a statistic over an axis.

        Can provide either a function (for use in a reduce)
        or a name (for use by a stat counter).

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        func : function, optional, default=None
            Function for reduce, see BoltArraySpark.reduce

        name : str
            A named statistic, see StatCounter

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        if axis is None:
            axis = list(range(len(self.shape)))
        axis = tupleize(axis)

        if func and not name:
            return self.reduce(func, axis, keepdims)

        if name and not func:
            from bolt.local.array import BoltArrayLocal

            swapped = self._align(axis)

            def reducer(left, right):
                return left.combine(right)

            counter = swapped._rdd.values()\
                             .mapPartitions(lambda i: [StatCounter(values=i, stats=name)])\
                             .treeReduce(reducer, depth=3)

            arr = getattr(counter, name)

            if keepdims:
                for i in axis:
                    arr = expand_dims(arr, axis=i)

            return BoltArrayLocal(arr).toscalar()

        else:
            raise ValueError('Must specify either a function or a statistic name.')


Instruction:
Can you generate the documentation for the following Python 3 function
def mean(self, axis=None, keepdims=False):
        return self._stat(axis, name='mean', keepdims=keepdims)

Response: Return the mean of the array over the given axis.


Instruction:
Write a Python 3 function for
returning the variance of the array over the given axis.

Response:
def var(self, axis=None, keepdims=False):
        """
        Return the variance of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        return self._stat(axis, name='variance', keepdims=keepdims)


Instruction:
Write a Python 3 script for
returning the standard deviation of the array over the given axis.

Response:
def std(self, axis=None, keepdims=False):
        """
        Return the standard deviation of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        return self._stat(axis, name='stdev', keepdims=keepdims)


Instruction:
Implement a Python 3 function for
returning the sum of the array over the given axis.

Response:
def sum(self, axis=None, keepdims=False):
        """
        Return the sum of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        from operator import add
        return self._stat(axis, func=add, keepdims=keepdims)


Instruction:
Can you generate the documentation for the following Python 3 function
def max(self, axis=None, keepdims=False):
        from numpy import maximum
        return self._stat(axis, func=maximum, keepdims=keepdims)

Response: Return the maximum value over the given axis.


Instruction:
Given the following Python 3 function, write the documentation
def min(self, axis=None, keepdims=False):
        from numpy import minimum
        return self._stat(axis, func=minimum, keepdims=keepdims)

Response: Return the minimum value over the given axis.


Instruction:
Can you write a function in Python 3 where it
concatenates this array with another array.

Response:
def concatenate(self, arry, axis=0):
        """
        Join this array with another array.

        Paramters
        ---------
        arry : ndarray, BoltArrayLocal, or BoltArraySpark
            Another array to concatenate with

        axis : int, optional, default=0
            The axis along which arrays will be joined.

        Returns
        -------
        BoltArraySpark
        """
        if isinstance(arry, ndarray):
            from bolt.spark.construct import ConstructSpark
            arry = ConstructSpark.array(arry, self._rdd.context, axis=range(0, self.split))
        else:
            if not isinstance(arry, BoltArraySpark):
                raise ValueError("other must be local array or spark array, got %s" % type(arry))

        if not all([x == y if not i == axis else True
                    for i, (x, y) in enumerate(zip(self.shape, arry.shape))]):
            raise ValueError("all the input array dimensions except for "
                             "the concatenation axis must match exactly")

        if not self.split == arry.split:
            raise NotImplementedError("two arrays must have the same split ")

        if axis < self.split:
            shape = self.keys.shape

            def key_func(key):
                key = list(key)
                key[axis] += shape[axis]
                return tuple(key)

            rdd = self._rdd.union(arry._rdd.map(lambda kv: (key_func(kv[0]), kv[1])))

        else:
            from numpy import concatenate as npconcatenate
            shift = axis - self.split
            rdd = self._rdd.join(arry._rdd).map(lambda kv: (kv[0], npconcatenate(kv[1], axis=shift)))

        shape = tuple([x + y if i == axis else x
                      for i, (x, y) in enumerate(zip(self.shape, arry.shape))])

        return self._constructor(rdd, shape=shape, ordered=False).__finalize__(self)


Instruction:
Given the following Python 3 function, write the documentation
def _getbasic(self, index):
        key_slices = index[0:self.split]
        value_slices = index[self.split:]

        def key_check(key):
            def inrange(k, s):
                if s.step > 0:
                    return s.start <= k < s.stop
                else:
                    return s.stop < k <= s.start
            def check(k, s):
                return inrange(k, s) and mod(k - s.start, s.step) == 0
            out = [check(k, s) for k, s in zip(key, key_slices)]
            return all(out)

        def key_func(key):
            return tuple([(k - s.start)/s.step for k, s in zip(key, key_slices)])

        filtered = self._rdd.filter(lambda kv: key_check(kv[0]))

        if self._split == self.ndim:
            rdd = filtered.map(lambda kv: (key_func(kv[0]), kv[1]))
        else:
            # handle use of use slice.stop = -1 for a special case (see utils.slicify)
            value_slices = [s if s.stop != -1 else slice(s.start, None, s.step) for s in value_slices]
            rdd = filtered.map(lambda kv: (key_func(kv[0]), kv[1][value_slices]))

        shape = tuple([int(ceil((s.stop - s.start) / float(s.step))) for s in index])
        split = self.split
        return rdd, shape, split

Response: Basic indexing for slices or ints.


Instruction:
Make a summary of the following Python 3 code
def _getadvanced(self, index):
        index = [asarray(i) for i in index]
        shape = index[0].shape
        if not all([i.shape == shape for i in index]):
            raise ValueError("shape mismatch: indexing arrays could not be broadcast "
                             "together with shapes " + ("%s " * self.ndim)
                             % tuple([i.shape for i in index]))

        index = tuple([listify(i, d) for (i, d) in zip(index, self.shape)])

        # build tuples with target indices
        key_tuples = list(zip(*index[0:self.split]))
        value_tuples = list(zip(*index[self.split:]))

        # build dictionary to look up targets in values
        d = {}
        for k, g in groupby(zip(value_tuples, key_tuples), lambda x: x[1]):
            d[k] = map(lambda x: x[0], list(g))

        def key_check(key):
            return key in key_tuples

        def key_func(key):
            return unravel_index(key, shape)

        # filter records based on key targets
        filtered = self._rdd.filter(lambda kv: key_check(kv[0]))

        # subselect and flatten records based on value targets (if they exist)
        if len(value_tuples) > 0:
            flattened = filtered.flatMap(lambda kv: [(kv[0], kv[1][i]) for i in d[kv[0]]])
        else:
            flattened = filtered

        # reindex
        indexed = flattened.zipWithIndex()
        rdd = indexed.map(lambda kkv: (key_func(kkv[1]), kkv[0][1]))
        split = len(shape)

        return rdd, shape, split

Response: Get the indices of the entries in the index.


Instruction:
Given the following Python 3 function, write the documentation
def _getmixed(self, index):
        # find the single advanced index
        loc = where([isinstance(i, (tuple, list, ndarray)) for i in index])[0][0]
        idx = list(index[loc])

        if isinstance(idx[0], (tuple, list, ndarray)):
            raise ValueError("When mixing basic and advanced indexing, "
                             "advanced index must be one-dimensional")

        # single advanced index is on a key -- filter and update key
        if loc < self.split:
            def newkey(key):
                newkey = list(key)
                newkey[loc] = idx.index(key[loc])
                return tuple(newkey)
            rdd = self._rdd.filter(lambda kv: kv[0][loc] in idx).map(lambda kv: (newkey(kv[0]), kv[1]))
        # single advanced index is on a value -- use NumPy indexing
        else:
            slices = [slice(0, None, None) for _ in self.values.shape]
            slices[loc - self.split] = idx
            rdd = self._rdd.map(lambda kv: (kv[0], kv[1][slices]))
        newshape = list(self.shape)
        newshape[loc] = len(idx)
        barray = self._constructor(rdd, shape=tuple(newshape)).__finalize__(self)

        # apply the rest of the simple indices
        new_index = index[:]
        new_index[loc] = slice(0, None, None)
        barray = barray[tuple(new_index)]
        return barray._rdd, barray.shape, barray.split

Response: Return a new RDD with the mixed entries.


Instruction:
How would you implement a function in Python 3 that
returns a chunked version of the distributed array.

Response:
def chunk(self, size="150", axis=None, padding=None):
        """
        Chunks records of a distributed array.

        Chunking breaks arrays into subarrays, using an specified
        size of chunks along each value dimension. Can alternatively
        specify an average chunk byte size (in kilobytes) and the size of
        chunks (as ints) will be computed automatically.

        Parameters
        ----------
        size : tuple, int, or str, optional, default = "150"
            A string giving the size in kilobytes, or a tuple with the size
            of chunks along each dimension.

        axis : int or tuple, optional, default = None
            One or more axis to chunk array along, if None
            will use all axes,

        padding: tuple or int, default = None
            Number of elements per dimension that will overlap with the adjacent chunk.
            If a tuple, specifies padding along each chunked dimension; if a int, same
            padding will be applied to all chunked dimensions.

        Returns
        -------
        ChunkedArray
        """
        if type(size) is not str:
            size = tupleize((size))
        axis = tupleize((axis))
        padding = tupleize((padding))

        from bolt.spark.chunk import ChunkedArray

        chnk = ChunkedArray(rdd=self._rdd, shape=self._shape, split=self._split, dtype=self._dtype)
        return chnk._chunk(size, axis, padding)


Instruction:
Can you write a function in Python 3 where it
swaps axes from keys to values. This is the core operation underlying shape manipulation on the Spark bolt array. It exchanges an arbitrary set of axes between the keys and the valeus. If either is None, will only move axes in one direction (from keys to values, or values to keys). Keys moved to values will be placed immediately after the split; values moved to keys will be placed immediately before the split. Parameters ---------- kaxes : tuple Axes from keys to move to values vaxes : tuple Axes from values to move to keys size : tuple or int, optional, default = "150" Can either provide a string giving the size in kilobytes, or a tuple with the number of chunks along each value dimension being moved Returns ------- BoltArraySpark

Response:
def swap(self, kaxes, vaxes, size="150"):
        """
        Swap axes from keys to values.

        This is the core operation underlying shape manipulation
        on the Spark bolt array. It exchanges an arbitrary set of axes
        between the keys and the valeus. If either is None, will only
        move axes in one direction (from keys to values, or values to keys).
        Keys moved to values will be placed immediately after the split;
        values moved to keys will be placed immediately before the split.

        Parameters
        ----------
        kaxes : tuple
            Axes from keys to move to values

        vaxes : tuple
            Axes from values to move to keys

        size : tuple or int, optional, default = "150"
            Can either provide a string giving the size in kilobytes,
            or a tuple with the number of chunks along each
            value dimension being moved

        Returns
        -------
        BoltArraySpark
        """
        kaxes = asarray(tupleize(kaxes), 'int')
        vaxes = asarray(tupleize(vaxes), 'int')
        if type(size) is not str:
            size = tupleize(size)

        if len(kaxes) == self.keys.ndim and len(vaxes) == 0:
            raise ValueError('Cannot perform a swap that would '
                             'end up with all data on a single key')

        if len(kaxes) == 0 and len(vaxes) == 0:
            return self

        from bolt.spark.chunk import ChunkedArray

        chunks = self.chunk(size)

        swapped = chunks.keys_to_values(kaxes).values_to_keys([v+len(kaxes) for v in vaxes])
        barray = swapped.unchunk()

        return barray


Instruction:
Can you generate a brief explanation for the following Python 3 code
def transpose(self, *axes):
        if len(axes) == 0:
            p = arange(self.ndim-1, -1, -1)
        else:
            p = asarray(argpack(axes))

        istransposeable(p, range(self.ndim))

        split = self.split

        # compute the keys/value axes that need to be swapped
        new_keys, new_values = p[:split], p[split:]
        swapping_keys = sort(new_values[new_values < split])
        swapping_values = sort(new_keys[new_keys >= split])
        stationary_keys = sort(new_keys[new_keys < split])
        stationary_values = sort(new_values[new_values >= split])

        # compute the permutation that the swap causes
        p_swap = r_[stationary_keys, swapping_values, swapping_keys, stationary_values]

        # compute the extra permutation (p_x)  on top of this that
        # needs to happen to get the full permutation desired
        p_swap_inv = argsort(p_swap)
        p_x = p_swap_inv[p]
        p_keys, p_values = p_x[:split], p_x[split:]-split

        # perform the swap and the the within key/value permutations
        arr = self.swap(swapping_keys, swapping_values-split)
        arr = arr.keys.transpose(tuple(p_keys.tolist()))
        arr = arr.values.transpose(tuple(p_values.tolist()))

        return arr

Response: Return an array with the axes transposed.


Instruction:
Write a Python 3 function that can
return the array with two axes interchanged.

Response:
def swapaxes(self, axis1, axis2):
        """
        Return the array with two axes interchanged.

        Parameters
        ----------
        axis1 : int
            The first axis to swap

        axis2 : int
            The second axis to swap
        """
        p = list(range(self.ndim))
        p[axis1] = axis2
        p[axis2] = axis1

        return self.transpose(p)


Instruction:
Write a Python 3 function that can
return an array with the same data but a new shape.

Response:
def reshape(self, *shape):
        """
        Return an array with the same data but a new shape.

        Currently only supports reshaping that independently
        reshapes the keys, or the values, or both.

        Parameters
        ----------
        shape :  tuple of ints, or n ints
            New shape
        """
        new = argpack(shape)
        isreshapeable(new, self.shape)

        if new == self.shape:
            return self

        i = self._reshapebasic(new)
        if i == -1:
            raise NotImplementedError("Currently no support for reshaping between "
                                      "keys and values for BoltArraySpark")
        else:
            new_key_shape, new_value_shape = new[:i], new[i:]
            return self.keys.reshape(new_key_shape).values.reshape(new_value_shape)


Instruction:
Can you tell what is the following Python 3 function doing
def _reshapebasic(self, shape):
        new = tupleize(shape)
        old_key_size = prod(self.keys.shape)
        old_value_size = prod(self.values.shape)

        for i in range(len(new)):
            new_key_size = prod(new[:i])
            new_value_size = prod(new[i:])
            if new_key_size == old_key_size and new_value_size == old_value_size:
                return i

        return -1

Response:
Check if the reshape can be broken into independant reshapes
        on the keys and values. If it can return the index in the new shape
        otherwise returns - 1.


Instruction:
Here you have a function in Python 3, explain what it does
def squeeze(self, axis=None):
        if not any([d == 1 for d in self.shape]):
            return self

        if axis is None:
            drop = where(asarray(self.shape) == 1)[0]
        elif isinstance(axis, int):
            drop = asarray((axis,))
        elif isinstance(axis, tuple):
            drop = asarray(axis)
        else:
            raise ValueError("an integer or tuple is required for the axis")

        if any([self.shape[i] > 1 for i in drop]):
            raise ValueError("cannot select an axis to squeeze out which has size greater than one")

        if any(asarray(drop) < self.split):
            kmask = set([d for d in drop if d < self.split])
            kfunc = lambda k: tuple([kk for ii, kk in enumerate(k) if ii not in kmask])
        else:
            kfunc = lambda k: k

        if any(asarray(drop) >= self.split):
            vmask = tuple([d - self.split for d in drop if d >= self.split])
            vfunc = lambda v: v.squeeze(vmask)
        else:
            vfunc = lambda v: v

        rdd = self._rdd.map(lambda kv: (kfunc(kv[0]), vfunc(kv[1])))
        shape = tuple([ss for ii, ss in enumerate(self.shape) if ii not in drop])
        split = len([d for d in range(self.keys.ndim) if d not in drop])
        return self._constructor(rdd, shape=shape, split=split).__finalize__(self)

Response: Return a new array with only one or more single - dimensional axes removed.


Instruction:
Can you generate the documentation for the following Python 3 function
def astype(self, dtype, casting='unsafe'):
        rdd = self._rdd.mapValues(lambda v: v.astype(dtype, 'K', casting))
        return self._constructor(rdd, dtype=dtype).__finalize__(self)

Response: Cast the array to a specified type.


Instruction:
Explain what the following Python 3 code does
def clip(self, min=None, max=None):
        rdd = self._rdd.mapValues(lambda v: v.clip(min=min, max=max))
        return self._constructor(rdd).__finalize__(self)

Response: Clip values above and below.


Instruction:
Can you write a function in Python 3 where it
returns the contents as a local array.

Response:
def toarray(self):
        """
        Returns the contents as a local array.

        Will likely cause memory problems for large objects.
        """
        rdd = self._rdd if self._ordered else self._rdd.sortByKey()
        x = rdd.values().collect()
        return asarray(x).reshape(self.shape)


Instruction:
Create a Python 3 function for
coercing singletons and lists and ndarrays to tuples.

Response:
def tupleize(arg):
    """
    Coerce singletons and lists and ndarrays to tuples.

    Parameters
    ----------
    arg : tuple, list, ndarray, or singleton
        Item to coerce
    """
    if arg is None:
        return None
    if not isinstance(arg, (tuple, list, ndarray, Iterable)):
        return tuple((arg,))
    elif isinstance(arg, (list, ndarray)):
        return tuple(arg)
    elif isinstance(arg, Iterable) and not isinstance(arg, str):
        return tuple(arg)
    else:
        return arg


Instruction:
Write a Python 3 function that can
coerce a list of arguments into a tuple.

Response:
def argpack(args):
    """
    Coerce a list of arguments to a tuple.

    Parameters
    ----------
    args : tuple or nested tuple
        Pack arguments into a tuple, converting ((,...),) or (,) -> (,)
    """
    if isinstance(args[0], (tuple, list, ndarray)):
        return tupleize(args[0])
    elif isinstance(args[0], Iterable) and not isinstance(args[0], str):
        # coerce any iterable into a list before calling tupleize (Python 3 compatibility)
        return tupleize(list(args[0]))
    else:
        return tuple(args)


Instruction:
How would you implement a function in Python 3 that
checks to see if a list of axes are contained within an array shape.

Response:
def inshape(shape, axes):
    """
    Checks to see if a list of axes are contained within an array shape.

    Parameters
    ----------
    shape : tuple[int]
        the shape of a BoltArray

    axes : tuple[int]
        the axes to check against shape
    """
    valid = all([(axis < len(shape)) and (axis >= 0) for axis in axes])
    if not valid:
        raise ValueError("axes not valid for an ndarray of shape: %s" % str(shape))


Instruction:
Can you generate the documentation for the following Python 3 function
def allclose(a, b):
    from numpy import allclose
    return (a.shape == b.shape) and allclose(a, b)

Response: Tests that a and b are all close and match in shape.


Instruction:
How would you explain what the following Python 3 function does
def listify(lst, dim):
    if not all([l.dtype == int for l in lst]):
        raise ValueError("indices must be integers")

    if npany(asarray(lst) >= dim):
        raise ValueError("indices out of bounds for axis with size %s" % dim)

    return lst.flatten()

Response: Flatten lists of indices and ensure bounded by a known dim.


Instruction:
Given the following Python 3 function, write the documentation
def slicify(slc, dim):
    if isinstance(slc, slice):

        # default limits
        start = 0 if slc.start is None else slc.start
        stop = dim if slc.stop is None else slc.stop
        step = 1 if slc.step is None else slc.step
        # account for negative indices
        if start < 0: start += dim
        if stop < 0: stop += dim
        # account for over-flowing the bounds
        if step > 0:
            if start < 0: start = 0
            if stop > dim: stop = dim
        else:
            if stop < 0: stop = -1
            if start > dim: start = dim-1

        return slice(start, stop, step)

    elif isinstance(slc, int):
        if slc < 0:
            slc += dim
        return slice(slc, slc+1, 1)

    else:
        raise ValueError("Type for slice %s not recongized" % type(slc))

Response: Convert a slice to a sequence of items.


Instruction:
Can you generate the documentation for the following Python 3 function
def istransposeable(new, old):

    new, old = tupleize(new), tupleize(old)

    if not len(new) == len(old):
        raise ValueError("Axes do not match axes of keys")

    if not len(set(new)) == len(set(old)):
        raise ValueError("Repeated axes")

    if any(n < 0 for n in new) or max(new) > len(old) - 1:
        raise ValueError("Invalid axes")

Response:
Checks to see if a proposed tuple of axes is a valid permutation
    of an old set of axes.


Instruction:
How would you implement a function in Python 3 that
checks to see if a proposed tuple of axes can be reshaped of the old axes.

Response:
def isreshapeable(new, old):
    """
    Check to see if a proposed tuple of axes is a valid reshaping of
    the old axes by ensuring that they can be factored.

    Parameters
    ----------
    new : tuple
        tuple of proposed axes

    old : tuple
        tuple of old axes
    """

    new, old = tupleize(new), tupleize(old)

    if not prod(new) == prod(old):
        raise ValueError("Total size of new keys must remain unchanged")


Instruction:
Given the following Python 3 function, write the documentation
def allstack(vals, depth=0):
    if type(vals[0]) is ndarray:
        return concatenate(vals, axis=depth)
    else:
        return concatenate([allstack(x, depth+1) for x in vals], axis=depth)

Response: This function creates a new array along a series of nested lists along a series of levels.


Instruction:
Can you create a Python 3 function that
expands dimensions by iteratively append empty axes.

Response:
def iterexpand(arry, extra):
    """
    Expand dimensions by iteratively append empty axes.

    Parameters
    ----------
    arry : ndarray
        The original array

    extra : int
        The number of empty axes to append
    """
    for d in range(arry.ndim, arry.ndim+extra):
        arry = expand_dims(arry, axis=d)
    return arry


Instruction:
Can you generate a brief explanation for the following Python 3 code
def zip_with_index(rdd):
    starts = [0]
    if rdd.getNumPartitions() > 1:
        nums = rdd.mapPartitions(lambda it: [sum(1 for _ in it)]).collect()
        count = sum(nums)
        for i in range(len(nums) - 1):
            starts.append(starts[-1] + nums[i])
    else:
        count = rdd.count()

    def func(k, it):
        for i, v in enumerate(it, starts[k]):
            yield v, i

    return count, rdd.mapPartitionsWithIndex(func)

Response: A version of Spark s zipWithIndex that eagerly returns count.


Instruction:
Here you have a function in Python 3, explain what it does
def wrapped(f):
    import inspect

    def extract(func):
        append = ""
        args = inspect.getargspec(func)
        for i, a in enumerate(args.args):
            if i < (len(args) - len(args.defaults)):
                append += str(a) + ", "
            else:
                default = args.defaults[i-len(args.defaults)]
                if hasattr(default, "__name__"):
                    default = default.__name__
                else:
                    default = str(default)
                append += str(a) + "=" + default + ", "
        append = append[:-2] + ")"
        return append

    doc = f.__doc__ + "\n"
    doc += "    local -> array(" + extract(getattr(ConstructLocal, f.__name__)) + "\n"
    doc += "    spark -> array(" + extract(getattr(ConstructSpark, f.__name__)) + "\n"
    f.__doc__ = doc
    return f

Response: Decorator to append routed docstrings


Instruction:
Implement a Python 3 function for
returning the first constructor that matches the given arguments.

Response:
def lookup(*args, **kwargs):
    """
    Use arguments to route constructor.

    Applies a series of checks on arguments to identify constructor,
    starting with known keyword arguments, and then applying
    constructor-specific checks
    """
    if 'mode' in kwargs:
        mode = kwargs['mode']
        if mode not in constructors:
            raise ValueError('Mode %s not supported' % mode)
        del kwargs['mode']
        return constructors[mode]
    else:
        for mode, constructor in constructors:
            if constructor._argcheck(*args, **kwargs):
                return constructor
    return ConstructLocal


Instruction:
Make a summary of the following Python 3 code
def transpose(self, *axes):
        new = argpack(axes)
        old = range(self.ndim)
        istransposeable(new, old)

        if new == old:
            return self._barray

        def f(k):
            return tuple(k[i] for i in new)

        newrdd = self._barray._rdd.map(lambda kv: (f(kv[0]), kv[1]))
        newshape = tuple(self.shape[i] for i in new) + self._barray.values.shape

        return BoltArraySpark(newrdd, shape=newshape, ordered=False).__finalize__(self._barray)

Response: Transposes just the keys of a BoltArraySpark returning a new BoltArraySpark object.


Instruction:
Can you tell what is the following Python 3 function doing
def reshape(self, *shape):
        new = argpack(shape)
        old = self.shape
        isreshapeable(new, old)

        if new == old:
            return self._barray

        def f(v):
            return v.reshape(new)

        newrdd = self._barray._rdd.mapValues(f)
        newshape = self._barray.keys.shape + new

        return BoltArraySpark(newrdd, shape=newshape).__finalize__(self._barray)

Response: Reshape just the values of a BoltArraySpark returning a new BoltArraySpark object with the new shape.


Instruction:
How would you implement a function in Python 3 that
transposes just the values of a BoltArraySpark returning a new BoltArraySpark object.

Response:
def transpose(self, *axes):
        """
        Transpose just the values of a BoltArraySpark, returning a
        new BoltArraySpark.

        Parameters
        ----------
        axes : tuple
             New proposed axes.
        """
        new = argpack(axes)
        old = range(self.ndim)
        istransposeable(new, old)

        if new == old:
            return self._barray

        def f(v):
            return v.transpose(new)

        newrdd = self._barray._rdd.mapValues(f)
        newshape = self._barray.keys.shape + tuple(self.shape[i] for i in new)

        return BoltArraySpark(newrdd, shape=newshape).__finalize__(self._barray)


Instruction:
Can you generate the documentation for the following Python 3 function
def ones(shape, dtype=float64, order='C'):
        from numpy import ones
        return ConstructLocal._wrap(ones, shape, dtype, order)

Response: Create a local bolt array of ones.


Instruction:
Here you have a function in Python 3, explain what it does
def zeros(shape, dtype=float64, order='C'):
        from numpy import zeros
        return ConstructLocal._wrap(zeros, shape, dtype, order)

Response: Create a local bolt array of zeros.


Instruction:
Given the following Python 3 function, write the documentation
def concatenate(arrays, axis=0):
        if not isinstance(arrays, tuple):
            raise ValueError("data type not understood")
        arrays = tuple([asarray(a) for a in arrays])
        from numpy import concatenate
        return BoltArrayLocal(concatenate(arrays, axis))

Response: Concatenate a sequence of array - like objects together.


Instruction:
Implement a Python 3 function for
returning A and B in y = Ax^B http://mathworld. wolfram. com / LeastSquaresFittingPowerLaw. html Returns A and B in y = Ax^B http://mathworld. wolfram. com / LeastSquaresFittingPowerLaw. html Returns A and B in y = Ax^B

Response:
def plfit_lsq(x,y):
    """
    Returns A and B in y=Ax^B
    http://mathworld.wolfram.com/LeastSquaresFittingPowerLaw.html
    """
    n = len(x)
    btop = n * (log(x)*log(y)).sum() - (log(x)).sum()*(log(y)).sum()
    bbottom = n*(log(x)**2).sum() - (log(x).sum())**2
    b = btop / bbottom
    a = ( log(y).sum() - b * log(x).sum() ) / n

    A = exp(a)
    return A,b


Instruction:
Can you generate a brief explanation for the following Python 3 code
def plfit(x,nosmall=False,finite=False):
    xmins = unique(x)
    xmins = xmins[1:-1]
    dat = xmins * 0
    z = sort(x)
    for xm in arange(len(xmins)):
        xmin = xmins[xm]
        z    = z[z>=xmin]
        n    = float(len(z))
        # estimate alpha using direct MLE
        a    =  n / sum( log(z/xmin) )
        if nosmall:
            # 4. For continuous data, PLFIT can return erroneously large estimates of
            #    alpha when xmin is so large that the number of obs x >= xmin is very
            #    small. To prevent this, we can truncate the search over xmin values
            #    before the finite-size bias becomes significant by calling PLFIT as
            if (a-1)/sqrt(n) > 0.1:
                #dat(xm:end) = [];
                dat = dat[:xm]
                xm = len(xmins)+1
                break
        # compute KS statistic
        cx   = arange(n)/float(n)  #data
        cf   = 1-(xmin/z)**a  # fitted
        dat[xm] = max( abs(cf-cx) )
    D     = min(dat);
    #xmin  = xmins(find(dat<=D,1,'first'));
    xmin  = xmins[argmin(dat)]
    z     = x[x>=xmin]
    n     = len(z)
    alpha = 1 + n / sum( log(z/xmin) )
    if finite:
        alpha = alpha*(n-1)/n+1/n
    if n < 50 and ~finite:
        print '(PLFIT) Warning: finite-size bias may be present.'
    L = n*log((alpha-1)/xmin) - alpha*sum(log(z/xmin));
    return xmin,alpha,L,dat

Response: Return a new object of the same size as the original object.


Instruction:
Given the following Python 3 function, write the documentation
def plotcdf(x,xmin,alpha):

    x=sort(x)
    n=len(x)
    xcdf = arange(n,0,-1,dtype='float')/float(n)

    q = x[x>=xmin]
    fcdf = (q/xmin)**(1-alpha)
    nc = xcdf[argmax(x>=xmin)]
    fcdf_norm = nc*fcdf

    loglog(x,xcdf)
    loglog(q,fcdf_norm)

Response: Plot CDF and powerlaw of a single node.


Instruction:
Here you have a function in Python 3, explain what it does
def plotpdf(x,xmin,alpha,nbins=30,dolog=False):

    x=sort(x)
    n=len(x)

    if dolog:
        hb = hist(x,bins=logspace(log10(min(x)),log10(max(x)),nbins),log=True)
        alpha += 1
    else:
        hb = hist(x,bins=linspace((min(x)),(max(x)),nbins))
    h,b=hb[0],hb[1]
    b = b[1:]

    q = x[x>=xmin]
    px = (alpha-1)/xmin * (q/xmin)**(-alpha)

    arg = argmin(abs(b-xmin))
    norm = mean( h[b>xmin] / ((alpha-1)/xmin * (b[b>xmin]/xmin)**(-alpha))  )
    px = px*norm

    loglog(q,px)

    gca().set_xlim(min(x),max(x))

Response: Plot PDF and powerlaw.


Instruction:
Make a summary of the following Python 3 code
def plexp(x,xm=1,a=2.5):

    C = 1/(-xm/(1 - a) - xm/a + math.exp(a)*xm/a)
    Ppl = lambda X: 1+C*(xm/(1-a)*(X/xm)**(1-a))
    Pexp = lambda X: C*xm/a*math.exp(a)-C*(xm/a)*math.exp(-a*(X/xm-1))
    d=Ppl(x)
    d[x<xm]=Pexp(x)
    return d

Response: Plexp is a function that returns a piecewise distribution exponential x<xmin powerlaw x > = xm


Instruction:
Can you write a function in Python 3 where it
inverses CDF for a piecewise PDF as defined in eqn. 3. 10

Response:
def plexp_inv(P,xm,a):
    """
    Inverse CDF for a piecewise PDF as defined in eqn. 3.10
    of Clauset et al.
    """

    C = 1/(-xm/(1 - a) - xm/a + math.exp(a)*xm/a)
    Pxm = 1+C*(xm/(1-a))
    pp = P
    x = xm*(pp-1)*(1-a)/(C*xm)**(1/(1-a)) if pp >= Pxm else (math.log( ((C*xm/a)*math.exp(a)-pp)/(C*xm/a)) - a) * (-xm/a)
    #x[P>=Pxm] = xm*( (P[P>=Pxm]-1) * (1-a)/(C*xm) )**(1/(1-a)) # powerlaw
    #x[P<Pxm] = (math.log( (C*xm/a*math.exp(a)-P[P<Pxm])/(C*xm/a) ) - a) * (-xm/a) # exp

    return x


Instruction:
Given the following Python 3 function, write the documentation
def alpha_(self,x):
        def alpha(xmin,x=x):
            """
            given a sorted data set and a minimum, returns power law MLE fit
            data is passed as a keyword parameter so that it can be vectorized
            """
            x = [i for i in x if i>=xmin]
            n = sum(x)
            divsum = sum([math.log(i/xmin) for i in x])
            if divsum == 0:
                return float('inf')
            # the "1+" here is unimportant because alpha_ is only used for minimization
            a = 1 + float(n) / divsum
            return a
        return alpha

Response: Create a mappable function alpha to apply to each xmin in a list of xmins.


Instruction:
Can you create a Python 3 function that
generates a mappable function alpha to apply to each xmin in a list of xmins.

Response:
def alpha_gen(x):
    """ Create a mappable function alpha to apply to each xmin in a list of xmins.
    This is essentially the slow version of fplfit/cplfit, though I bet it could
    be speeded up with a clever use of parellel_map.  Not intended to be used by users.

    Docstring for the generated alpha function::

        Given a sorted data set and a minimum, returns power law MLE fit
        data is passed as a keyword parameter so that it can be vectorized

        If there is only one element, return alpha=0
    """
    def alpha_(xmin,x=x):
        """
        Given a sorted data set and a minimum, returns power law MLE fit
        data is passed as a keyword parameter so that it can be vectorized

        If there is only one element, return alpha=0
        """
        gexmin = x>=xmin
        n = np.count_nonzero(gexmin)
        if n < 2:
            return 0
        x = x[gexmin]
        a = 1 + float(n) / sum(log(x/xmin))
        return a
    return alpha_


Instruction:
Can you tell what is the following Python 3 function doing
def plexp_cdf(x,xmin=1,alpha=2.5, pl_only=False, exp_only=False):

    x = np.array(x)
    C = 1/(-xmin/(1 - alpha) - xmin/alpha + exp(alpha)*xmin/alpha)
    Ppl = lambda X: 1+C*(xmin/(1-alpha)*(X/xmin)**(1-alpha))
    Pexp = lambda X: C*xmin/alpha*exp(alpha)-C*(xmin/alpha)*exp(-alpha*(X/xmin-1))

    if exp_only:
        return Pexp(x)
    elif pl_only:
        return Ppl(x)

    d=Ppl(x)
    d[x<xmin]=Pexp(x)[x<xmin]
    return d

Response: Returns the CDF version of the piecewise distribution exponential x<xmin and powerlaw x > = xmin


Instruction:
Write a Python 3 function that can
inverse CDF for a piecewise PDF.

Response:
def plexp_inv(P, xmin, alpha, guess=1.):
    """
    Inverse CDF for a piecewise PDF as defined in eqn. 3.10
    of Clauset et al.

    (previous version was incorrect and lead to weird discontinuities in the
    distribution function)
    """
    def equation(x,prob):
        return plexp_cdf(x, xmin, alpha)-prob
    # http://stackoverflow.com/questions/19840425/scipy-optimize-faster-root-finding-over-2d-grid
    def solver(y, x0=guess):
        return scipy.optimize.fsolve(equation, guess, args=(y,))
    f = np.vectorize(solver)
    return f(P)


Instruction:
Write a Python 3 function for
calculating the discrete likelihood of a single node.

Response:
def discrete_likelihood(data, xmin, alpha):
    """
    Equation B.8 in Clauset

    Given a data set, an xmin value, and an alpha "scaling parameter", computes
    the log-likelihood (the value to be maximized)
    """
    if not scipyOK:
        raise ImportError("Can't import scipy.  Need scipy for zeta function.")
    from scipy.special import zeta as zeta

    zz = data[data>=xmin]
    nn = len(zz)

    sum_log_data = np.log(zz).sum()

    zeta = zeta(alpha, xmin)

    L_of_alpha = -1*nn*log(zeta) - alpha * sum_log_data

    return L_of_alpha


Instruction:
Can you write a function in Python 3 where it
computes the discrete likelihood for all scaling parameters in the range alpharange

Response:
def discrete_likelihood_vector(data, xmin, alpharange=(1.5,3.5), n_alpha=201):
    """
    Compute the likelihood for all "scaling parameters" in the range (alpharange)
    for a given xmin.  This is only part of the discrete value likelihood
    maximization problem as described in Clauset et al
    (Equation B.8)

    *alpharange* [ 2-tuple ]
        Two floats specifying the upper and lower limits of the power law alpha to test
    """
    from scipy.special import zeta as zeta

    zz = data[data>=xmin]
    nn = len(zz)

    alpha_vector = np.linspace(alpharange[0],alpharange[1],n_alpha)
    sum_log_data = np.log(zz).sum()

    # alpha_vector is a vector, xmin is a scalar
    zeta_vector = zeta(alpha_vector, xmin)

    #xminvec = np.arange(1.0,xmin)

    #xminalphasum = np.sum([xm**(-alpha_vector) for xm in xminvec])
    #L = -1*alpha_vector*sum_log_data - nn*log(zeta_vector) - xminalphasum

    L_of_alpha = -1*nn*log(zeta_vector) - alpha_vector * sum_log_data

    return L_of_alpha


Instruction:
Can you tell what is the following Python 3 function doing
def discrete_max_likelihood_arg(data, xmin, alpharange=(1.5,3.5), n_alpha=201):
    likelihoods = discrete_likelihood_vector(data, xmin, alpharange=alpharange, n_alpha=n_alpha)
    Largmax = np.argmax(likelihoods)
    return Largmax

Response: Returns the maximum likelihood of the data given an input xmin


Instruction:
Can you tell what is the following Python 3 function doing
def discrete_max_likelihood(data, xmin, alpharange=(1.5,3.5), n_alpha=201):
    likelihoods = discrete_likelihood_vector(data, xmin, alpharange=alpharange, n_alpha=n_alpha)
    Lmax = np.max(likelihoods)
    return Lmax

Response: Returns the maximum likelihood of the data given an input xmin


Instruction:
Implement a function in Python 3 to
return the most likely alpha for the data given an xmin

Response:
def most_likely_alpha(data, xmin, alpharange=(1.5,3.5), n_alpha=201):
    """
    Return the most likely alpha for the data given an xmin
    """
    alpha_vector = np.linspace(alpharange[0],alpharange[1],n_alpha)
    return alpha_vector[discrete_max_likelihood_arg(data, xmin,
                                                    alpharange=alpharange,
                                                    n_alpha=n_alpha)]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def discrete_alpha_mle(data, xmin):
    # boolean indices of positive data
    gexmin = (data>=xmin)
    nn = gexmin.sum()
    if nn < 2:
        return 0
    xx = data[gexmin]
    alpha = 1.0 + float(nn) * (sum(log(xx/(float(xmin)-0.5))))**-1
    return alpha

Response: Return the discrete alpha in the continuous case


Instruction:
Can you generate the documentation for the following Python 3 function
def discrete_best_alpha(data, alpharangemults=(0.9,1.1), n_alpha=201, approximate=True, verbose=True):

    xmins = np.unique(data)
    if approximate:
        alpha_of_xmin = [ discrete_alpha_mle(data,xmin) for xmin in xmins ]
    else:
        alpha_approx = [ discrete_alpha_mle(data,xmin) for xmin in xmins ]
        alpharanges = [(0.9*a,1.1*a) for a in alpha_approx]
        alpha_of_xmin = [ most_likely_alpha(data,xmin,alpharange=ar,n_alpha=n_alpha) for xmin,ar in zip(xmins,alpharanges) ]
    ksvalues = [ discrete_ksD(data, xmin, alpha) for xmin,alpha in zip(xmins,alpha_of_xmin) ]

    best_index = argmin(ksvalues)
    best_alpha = alpha_of_xmin[best_index]
    best_xmin = xmins[best_index]
    best_ks = ksvalues[best_index]
    best_likelihood = discrete_likelihood(data, best_xmin, best_alpha)

    if verbose:
        print("alpha = %f   xmin = %f   ksD = %f   L = %f   (n<x) = %i  (n>=x) = %i" % (
                best_alpha, best_xmin, best_ks, best_likelihood,
                (data<best_xmin).sum(), (data>=best_xmin).sum()))

    return best_alpha,best_xmin,best_ks,best_likelihood

Response: Return the discrete best alpha for the given data.


Instruction:
Write a Python 3 function that can
return the power law ks - test D value w / data

Response:
def discrete_ksD(data, xmin, alpha):
    """
    given a sorted data set, a minimum, and an alpha, returns the power law ks-test
    D value w/data

    The returned value is the "D" parameter in the ks test

    (this is implemented differently from the continuous version because there
    are potentially multiple identical points that need comparison to the power
    law)
    """
    zz = np.sort(data[data>=xmin])
    nn = float(len(zz))
    if nn < 2:
        return np.inf
    #cx = np.arange(nn,dtype='float')/float(nn)
    #cf = 1.0-(zz/xmin)**(1.0-alpha)
    model_cdf = 1.0-(zz.astype('float')/float(xmin))**(1.0-alpha)
    data_cdf = np.searchsorted(zz,zz,side='left')/(float(nn))

    ks = max(abs(model_cdf-data_cdf))
    return ks