Instruction:
Write a Python 3 script to
measure the axis aligned bounding box (aabb) of a mesh in multiple coordinate systems. Args: fbasename (str): filename of input model log (str): filename of log file coord_system (enum in ['CARTESIAN', 'CYLINDRICAL'] Coordinate system to use: 'CARTESIAN': lists contain [x, y, z] 'CYLINDRICAL': lists contain [r, theta, z] Returns: dict: dictionary with the following aabb properties min (3 element list): minimum values max (3 element list): maximum values center (3 element list): the center point size (3 element list): size of the aabb in each coordinate (max-min) diagonal (float): the diagonal of the aabb

Response:
def measure_aabb(fbasename=None, log=None, coord_system='CARTESIAN'):
    """ Measure the axis aligned bounding box (aabb) of a mesh
    in multiple coordinate systems.

    Args:
        fbasename (str): filename of input model
        log (str): filename of log file
        coord_system (enum in ['CARTESIAN', 'CYLINDRICAL']
            Coordinate system to use:
                'CARTESIAN': lists contain [x, y, z]
                'CYLINDRICAL': lists contain [r, theta, z]
    Returns:
        dict: dictionary with the following aabb properties
            min (3 element list): minimum values
            max (3 element list): maximum values
            center (3 element list): the center point
            size (3 element list): size of the aabb in each coordinate (max-min)
            diagonal (float): the diagonal of the aabb
    """
    # TODO: add center point, spherical coordinate system
    fext = os.path.splitext(fbasename)[1][1:].strip().lower()
    if fext != 'xyz':
        fin = 'TEMP3D_aabb.xyz'
        run(log=log, file_in=fbasename, file_out=fin, script=None)
    else:
        fin = fbasename
    fread = open(fin, 'r')
    aabb = {'min': [999999.0, 999999.0, 999999.0], 'max': [-999999.0, -999999.0, -999999.0]}
    for line in fread:
        x_co, y_co, z_co = line.split()
        x_co = util.to_float(x_co)
        y_co = util.to_float(y_co)
        z_co = util.to_float(z_co)
        if coord_system == 'CARTESIAN':
            if x_co < aabb['min'][0]:
                aabb['min'][0] = x_co
            if y_co < aabb['min'][1]:
                aabb['min'][1] = y_co
            if z_co < aabb['min'][2]:
                aabb['min'][2] = z_co
            if x_co > aabb['max'][0]:
                aabb['max'][0] = x_co
            if y_co > aabb['max'][1]:
                aabb['max'][1] = y_co
            if z_co > aabb['max'][2]:
                aabb['max'][2] = z_co
        elif coord_system == 'CYLINDRICAL':
            radius = math.sqrt(x_co**2 + y_co**2)
            theta = math.degrees(math.atan2(y_co, x_co))
            if radius < aabb['min'][0]:
                aabb['min'][0] = radius
            if theta < aabb['min'][1]:
                aabb['min'][1] = theta
            if z_co < aabb['min'][2]:
                aabb['min'][2] = z_co
            if radius > aabb['max'][0]:
                aabb['max'][0] = radius
            if theta > aabb['max'][1]:
                aabb['max'][1] = theta
            if z_co > aabb['max'][2]:
                aabb['max'][2] = z_co
    fread.close()
    try:
        aabb['center'] = [(aabb['max'][0] + aabb['min'][0]) / 2,
                          (aabb['max'][1] + aabb['min'][1]) / 2,
                          (aabb['max'][2] + aabb['min'][2]) / 2]
        aabb['size'] = [aabb['max'][0] - aabb['min'][0], aabb['max'][1] - aabb['min'][1],
                        aabb['max'][2] - aabb['min'][2]]
        aabb['diagonal'] = math.sqrt(
            aabb['size'][0]**2 +
            aabb['size'][1]**2 +
            aabb['size'][2]**2)
    except UnboundLocalError:
        print('Error: aabb input file does not contain valid data. Exiting ...')
        sys.exit(1)
    for key, value in aabb.items():
        if log is None:
            print('{:10} = {}'.format(key, value))
        else:
            log_file = open(log, 'a')
            log_file.write('{:10} = {}\n'.format(key, value))
            log_file.close()
    """
    if log is not None:
        log_file = open(log, 'a')
        #log_file.write('***Axis Aligned Bounding Results for file "%s":\n' % fbasename)
        log_file.write('min = %s\n' % aabb['min'])
        log_file.write('max = %s\n' % aabb['max'])
        log_file.write('center = %s\n' % aabb['center'])
        log_file.write('size = %s\n' % aabb['size'])
        log_file.write('diagonal = %s\n\n' % aabb['diagonal'])
        log_file.close()
    # print(aabb)
    """
    return aabb


Instruction:
Here you have a function in Python 3, explain what it does
def measure_section(fbasename=None, log=None, axis='z', offset=0.0,
                    rotate_x_angle=None, ml_version=ml_version):
    """Measure a cross section of a mesh

    Perform a plane cut in one of the major axes (X, Y, Z). If you want to cut on
    a different plane you will need to rotate the model in place, perform the cut,
    and rotate it back.

    Args:
        fbasename (str): filename of input model
        log (str): filename of log file
        axis (str): axis perpendicular to the cutting plane, e.g. specify "z" to cut
            parallel to the XY plane.
        offset (float): amount to offset the cutting plane from the origin
        rotate_x_angle (float): degrees to rotate about the X axis. Useful for correcting "Up" direction: 90 to rotate Y to Z, and -90 to rotate Z to Y.

    Returns:
        dict: dictionary with the following keys for the aabb of the section:
            min (list): list of the x, y & z minimum values
            max (list): list of the x, y & z maximum values
            center (list): the x, y & z coordinates of the center of the aabb
            size (list): list of the x, y & z sizes (max - min)
            diagonal (float): the diagonal of the aabb
    """
    ml_script1_file = 'TEMP3D_measure_section.mlx'
    file_out = 'TEMP3D_sect_aabb.xyz'

    ml_script1 = mlx.FilterScript(file_in=fbasename, file_out=file_out, ml_version=ml_version)
    if rotate_x_angle is not None:
        transform.rotate(ml_script1, axis='x', angle=rotate_x_angle)
    compute.section(ml_script1, axis=axis, offset=offset)
    layers.delete_lower(ml_script1)
    ml_script1.save_to_file(ml_script1_file)
    ml_script1.run_script(log=log, script_file=ml_script1_file)
    aabb = measure_aabb(file_out, log)
    return aabb

Response: Measure a cross section of a mesh.


Instruction:
Here you have a function in Python 3, explain what it does
def polylinesort(fbasename=None, log=None):
    fext = os.path.splitext(fbasename)[1][1:].strip().lower()
    if fext != 'obj':
        print('Input file must be obj. Exiting ...')
        sys.exit(1)
    fread = open(fbasename, 'r')
    first = True
    polyline_vertices = []
    line_segments = []
    for line in fread:
        element, x_co, y_co, z_co = line.split()
        if element == 'v':
            polyline_vertices.append(
                [util.to_float(x_co), util.to_float(y_co), util.to_float(z_co)])
        elif element == 'l':
            p1 = x_co
            p2 = y_co
            line_segments.append([int(p1), int(p2)])

    fread.close()
    if log is not None:
        log_file = open(log, 'a')
        #log_file.write('***Axis Aligned Bounding Results for file "%s":\n' % fbasename)
        """log_file.write('min = %s\n' % aabb['min'])
        log_file.write('max = %s\n' % aabb['max'])
        log_file.write('center = %s\n' % aabb['center'])
        log_file.write('size = %s\n' % aabb['size'])
        log_file.write('diagonal = %s\n' % aabb['diagonal'])"""
        log_file.close()
    # print(aabb)
    return None

Response: Sort a continuous polyline or polylines.


Instruction:
Write a Python 3 script for
measuring the topology of a single mesh.

Response:
def measure_topology(fbasename=None, log=None, ml_version=ml_version):
    """Measures mesh topology

    Args:
        fbasename (str): input filename.
        log (str): filename to log output

    Returns:
        dict: dictionary with the following keys:
            vert_num (int): number of vertices
            edge_num (int): number of edges
            face_num (int): number of faces
            unref_vert_num (int): number or unreferenced vertices
            boundry_edge_num (int): number of boundary edges
            part_num (int): number of parts (components) in the mesh.
            manifold (bool): True if mesh is two-manifold, otherwise false.
            non_manifold_edge (int): number of non_manifold edges.
            non_manifold_vert (int): number of non-manifold verices
            genus (int or str): genus of the mesh, either a number or
                'undefined' if the mesh is non-manifold.
            holes (int or str): number of holes in the mesh, either a number
                or 'undefined' if the mesh is non-manifold.

    """
    ml_script1_file = 'TEMP3D_measure_topology.mlx'
    ml_script1 = mlx.FilterScript(file_in=fbasename, ml_version=ml_version)
    compute.measure_topology(ml_script1)
    ml_script1.save_to_file(ml_script1_file)
    ml_script1.run_script(log=log, script_file=ml_script1_file)
    topology = ml_script1.topology
    return topology


Instruction:
Can you implement a function in Python 3 that
measures mesh geometry aabb and topology.

Response:
def measure_all(fbasename=None, log=None, ml_version=ml_version):
    """Measures mesh geometry, aabb and topology."""
    ml_script1_file = 'TEMP3D_measure_gAndT.mlx'
    if ml_version == '1.3.4BETA':
        file_out = 'TEMP3D_aabb.xyz'
    else:
        file_out = None

    ml_script1 = mlx.FilterScript(file_in=fbasename, file_out=file_out, ml_version=ml_version)
    compute.measure_geometry(ml_script1)
    compute.measure_topology(ml_script1)
    ml_script1.save_to_file(ml_script1_file)
    ml_script1.run_script(log=log, script_file=ml_script1_file)
    geometry = ml_script1.geometry
    topology = ml_script1.topology

    if ml_version == '1.3.4BETA':
        if log is not None:
            log_file = open(log, 'a')
            log_file.write(
                '***Axis Aligned Bounding Results for file "%s":\n' %
                fbasename)
            log_file.close()
        aabb = measure_aabb(file_out, log)
    else:
        aabb = geometry['aabb']
    return aabb, geometry, topology


Instruction:
Explain what the following Python 3 code does
def measure_dimension(fbasename=None, log=None, axis1=None, offset1=0.0,
                      axis2=None, offset2=0.0, ml_version=ml_version):
    """Measure a dimension of a mesh"""
    axis1 = axis1.lower()
    axis2 = axis2.lower()
    ml_script1_file = 'TEMP3D_measure_dimension.mlx'
    file_out = 'TEMP3D_measure_dimension.xyz'

    ml_script1 = mlx.FilterScript(file_in=fbasename, file_out=file_out, ml_version=ml_version)
    compute.section(ml_script1, axis1, offset1, surface=True)
    compute.section(ml_script1, axis2, offset2, surface=False)
    layers.delete_lower(ml_script1)
    ml_script1.save_to_file(ml_script1_file)
    ml_script1.run_script(log=log, script_file=ml_script1_file)

    for val in ('x', 'y', 'z'):
        if val not in (axis1, axis2):
            axis = val
    # ord: Get number that represents letter in ASCII
    # Here we find the offset from 'x' to determine the list reference
    # i.e. 0 for x, 1 for y, 2 for z
    axis_num = ord(axis) - ord('x')
    aabb = measure_aabb(file_out, log)
    dimension = {'min': aabb['min'][axis_num], 'max': aabb['max'][axis_num],
                 'length': aabb['size'][axis_num], 'axis': axis}
    if log is None:
        print('\nFor file "%s"' % fbasename)
        print('Dimension parallel to %s with %s=%s & %s=%s:' % (axis, axis1, offset1,
                                                                axis2, offset2))
        print('  Min = %s, Max = %s, Total length = %s' % (dimension['min'],
                                                           dimension['max'], dimension['length']))
    else:
        log_file = open(log, 'a')
        log_file.write('\nFor file "%s"\n' % fbasename)
        log_file.write('Dimension parallel to %s with %s=%s & %s=%s:\n' % (axis, axis1, offset1,
                                                                           axis2, offset2))
        log_file.write('min = %s\n' % dimension['min'])
        log_file.write('max = %s\n' % dimension['max'])
        log_file.write('Total length = %s\n' % dimension['length'])
        log_file.close()
    return dimension

Response: Measure a dimension of a mesh


Instruction:
Can you tell what is the following Python 3 function doing
def lowercase_ext(filename):
    if '.' in filename:
        main, ext = os.path.splitext(filename)
        return main + ext.lower()
    # For consistency with os.path.splitext,
    # do not treat a filename without an extension as an extension.
    # That is, do not return filename.lower().
    return filename

Response: Lowercase the extension of a file.


Instruction:
Create a Python 3 function to
patch the request class of the app.

Response:
def patch_request_class(app, size=64 * 1024 * 1024):
    """
    By default, Flask will accept uploads to an arbitrary size. While Werkzeug
    switches uploads from memory to a temporary file when they hit 500 KiB,
    it's still possible for someone to overload your disk space with a
    gigantic file.

    This patches the app's request class's
    `~werkzeug.BaseRequest.max_content_length` attribute so that any upload
    larger than the given size is rejected with an HTTP error.

    .. note::

       In Flask 0.6, you can do this by setting the `MAX_CONTENT_LENGTH`
       setting, without patching the request class. To emulate this behavior,
       you can pass `None` as the size (you must pass it explicitly). That is
       the best way to call this function, as it won't break the Flask 0.6
       functionality if it exists.

    .. versionchanged:: 0.1.1

    :param app: The app to patch the request class of.
    :param size: The maximum size to accept, in bytes. The default is 64 MiB.
                 If it is `None`, the app's `MAX_CONTENT_LENGTH` configuration
                 setting will be used to patch.
    """
    if size is None:
        if isinstance(app.request_class.__dict__['max_content_length'],
                      property):
            return
        size = app.config.get('MAX_CONTENT_LENGTH')
    reqclass = app.request_class
    patched = type(reqclass.__name__, (reqclass,),
                   {'max_content_length': size})
    app.request_class = patched


Instruction:
Can you generate the documentation for the following Python 3 function
def config_for_set(uset, app, defaults=None):
    config = app.config
    prefix = 'UPLOADED_%s_' % uset.name.upper()
    using_defaults = False
    if defaults is None:
        defaults = dict(dest=None, url=None)

    allow_extns = tuple(config.get(prefix + 'ALLOW', ()))
    deny_extns = tuple(config.get(prefix + 'DENY', ()))
    destination = config.get(prefix + 'DEST')
    base_url = config.get(prefix + 'URL')

    if destination is None:
        # the upload set's destination wasn't given
        if uset.default_dest:
            # use the "default_dest" callable
            destination = uset.default_dest(app)
        if destination is None: # still
            # use the default dest from the config
            if defaults['dest'] is not None:
                using_defaults = True
                destination = os.path.join(defaults['dest'], uset.name)
            else:
                raise RuntimeError("no destination for set %s" % uset.name)

    if base_url is None and using_defaults and defaults['url']:
        base_url = addslash(defaults['url']) + uset.name + '/'

    return UploadConfiguration(destination, base_url, allow_extns, deny_extns)

Response: This function returns the configuration for a single set.


Instruction:
Write a Python 3 script for
configuring the uploads module for the given app.

Response:
def configure_uploads(app, upload_sets):
    """
    Call this after the app has been configured. It will go through all the
    upload sets, get their configuration, and store the configuration on the
    app. It will also register the uploads module if it hasn't been set. This
    can be called multiple times with different upload sets.

    .. versionchanged:: 0.1.3
       The uploads module/blueprint will only be registered if it is needed
       to serve the upload sets.

    :param app: The `~flask.Flask` instance to get the configuration from.
    :param upload_sets: The `UploadSet` instances to configure.
    """
    if isinstance(upload_sets, UploadSet):
        upload_sets = (upload_sets,)

    if not hasattr(app, 'upload_set_config'):
        app.upload_set_config = {}
    set_config = app.upload_set_config
    defaults = dict(dest=app.config.get('UPLOADS_DEFAULT_DEST'),
                    url=app.config.get('UPLOADS_DEFAULT_URL'))

    for uset in upload_sets:
        config = config_for_set(uset, app, defaults)
        set_config[uset.name] = config

    should_serve = any(s.base_url is None for s in set_config.values())
    if '_uploads' not in app.blueprints and should_serve:
        app.register_blueprint(uploads_mod)


Instruction:
Implement a Python 3 function for
getting the current configuration for the specified upload set.

Response:
def config(self):
        """
        This gets the current configuration. By default, it looks up the
        current application and gets the configuration from there. But if you
        don't want to go to the full effort of setting an application, or it's
        otherwise outside of a request context, set the `_config` attribute to
        an `UploadConfiguration` instance, then set it back to `None` when
        you're done.
        """
        if self._config is not None:
            return self._config
        try:
            return current_app.upload_set_config[self.name]
        except AttributeError:
            raise RuntimeError("cannot access configuration outside request")


Instruction:
Explain what the following Python 3 code does
def extension_allowed(self, ext):
        return ((ext in self.config.allow) or
                (ext in self.extensions and ext not in self.config.deny))

Response: Determines whether a specific extension is allowed.


Instruction:
Can you tell what is the following Python 3 function doing
def save(self, storage, folder=None, name=None):
        if not isinstance(storage, FileStorage):
            raise TypeError("storage must be a werkzeug.FileStorage")

        if folder is None and name is not None and "/" in name:
            folder, name = os.path.split(name)

        basename = self.get_basename(storage.filename)
        if name:
            if name.endswith('.'):
                basename = name + extension(basename)
            else:
                basename = name

        if not self.file_allowed(storage, basename):
            raise UploadNotAllowed()

        if folder:
            target_folder = os.path.join(self.config.destination, folder)
        else:
            target_folder = self.config.destination
        if not os.path.exists(target_folder):
            os.makedirs(target_folder)
        if os.path.exists(os.path.join(target_folder, basename)):
            basename = self.resolve_conflict(target_folder, basename)

        target = os.path.join(target_folder, basename)
        storage.save(target)
        if folder:
            return posixpath.join(folder, basename)
        else:
            return basename

Response: This method saves a file into a specific folder or name.


Instruction:
Create a Python 3 function to
return actual version specified in filename.

Response:
def get_vprof_version(filename):
    """Returns actual version specified in filename."""
    with open(filename) as src_file:
        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                                  src_file.read(), re.M)
        if version_match:
            return version_match.group(1)
        raise RuntimeError('Unable to find version info.')


Instruction:
Here you have a function in Python 3, explain what it does
def _remove_duplicates(objects):
    seen, uniq = set(), []
    for obj in objects:
        obj_id = id(obj)
        if obj_id in seen:
            continue
        seen.add(obj_id)
        uniq.append(obj)
    return uniq

Response:
Removes duplicate objects.

    http://www.peterbe.com/plog/uniqifiers-benchmark.


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_obj_count_difference(objs1, objs2):
    clean_obj_list1 = _process_in_memory_objects(objs1)
    clean_obj_list2 = _process_in_memory_objects(objs2)
    obj_count_1 = _get_object_count_by_type(clean_obj_list1)
    obj_count_2 = _get_object_count_by_type(clean_obj_list2)
    return obj_count_1 - obj_count_2

Response: Returns count difference in two collections of Python objects.


Instruction:
How would you explain what the following Python 3 function does
def _trace_memory_usage(self, frame, event, arg):  #pylint: disable=unused-argument
        if event == 'line' and frame.f_code.co_filename in self.target_modules:
            self._events_list.append(
                (frame.f_lineno, self._process.memory_info().rss,
                 frame.f_code.co_name, frame.f_code.co_filename))
        return self._trace_memory_usage

Response: Checks memory usage when line event occur.


Instruction:
Can you generate the documentation for the following Python 3 function
def code_events(self):
        if self._resulting_events:
            return self._resulting_events
        for i, (lineno, mem, func, fname) in enumerate(self._events_list):
            mem_in_mb = float(mem - self.mem_overhead) / _BYTES_IN_MB
            if (self._resulting_events and
                    self._resulting_events[-1][0] == lineno and
                    self._resulting_events[-1][2] == func and
                    self._resulting_events[-1][3] == fname and
                    self._resulting_events[-1][1] < mem_in_mb):
                self._resulting_events[-1][1] = mem_in_mb
            else:
                self._resulting_events.append(
                    [i + 1, lineno, mem_in_mb, func, fname])
        return self._resulting_events

Response: Returns processed memory usage.


Instruction:
Make a summary of the following Python 3 code
def obj_overhead(self):
        overhead = [
            self,
            self._resulting_events,
            self._events_list,
            self._process
        ]
        overhead_count = _get_object_count_by_type(overhead)
        # One for reference to __dict__ and one for reference to
        # the current module.
        overhead_count[dict] += 2
        return overhead_count

Response: Returns all objects that are considered a profiler overhead.


Instruction:
Can you write a function in Python 3 where it
computes the memory overhead of the process.

Response:
def compute_mem_overhead(self):
        """Returns memory overhead."""
        self.mem_overhead = (self._process.memory_info().rss -
                             builtins.initial_rss_size)


Instruction:
Given the following Python 3 function, write the documentation
def profile_package(self):
        target_modules = base_profiler.get_pkg_module_names(self._run_object)
        try:
            with _CodeEventsTracker(target_modules) as prof:
                prof.compute_mem_overhead()
                runpy.run_path(self._run_object, run_name='__main__')
        except SystemExit:
            pass
        return prof, None

Response: Returns memory stats for a package.


Instruction:
Write a Python 3 function for
returning memory stats for a module.

Response:
def profile_module(self):
        """Returns memory stats for a module."""
        target_modules = {self._run_object}
        try:
            with open(self._run_object, 'rb') as srcfile,\
                _CodeEventsTracker(target_modules) as prof:
                code = compile(srcfile.read(), self._run_object, 'exec')
                prof.compute_mem_overhead()
                exec(code, self._globs, None)
        except SystemExit:
            pass
        return prof, None


Instruction:
Can you tell what is the following Python 3 function doing
def profile_function(self):
        target_modules = {self._run_object.__code__.co_filename}
        with _CodeEventsTracker(target_modules) as prof:
            prof.compute_mem_overhead()
            result = self._run_object(*self._run_args, **self._run_kwargs)
        return prof, result

Response: Returns memory stats for a function.


Instruction:
How would you explain what the following Python 3 function does
def run(self):
        existing_objects = _get_in_memory_objects()
        prof, result = self.profile()
        new_objects = _get_in_memory_objects()

        new_obj_count = _get_obj_count_difference(new_objects, existing_objects)
        result_obj_count = new_obj_count - prof.obj_overhead

        # existing_objects list is also profiler overhead
        result_obj_count[list] -= 1
        pretty_obj_count = _format_obj_count(result_obj_count)
        return {
            'objectName': self._object_name,
            'codeEvents': prof.code_events,
            'totalEvents': len(prof.code_events),
            'objectsCount': pretty_obj_count,
            'result': result,
            'timestamp': int(time.time())
        }

Response: Collects memory stats for specified Python program.


Instruction:
Make a summary of the following Python 3 code
def get_pkg_module_names(package_path):
    module_names = set()
    for fobj, modname, _ in pkgutil.iter_modules(path=[package_path]):
        filename = os.path.join(fobj.path, '%s.py' % modname)
        if os.path.exists(filename):
            module_names.add(os.path.abspath(filename))
    return module_names

Response: Returns a set of module filenames from a Python package.


Instruction:
Write a Python 3 function that can
run function in separate process.

Response:
def run_in_separate_process(func, *args, **kwargs):
    """Runs function in separate process.

    This function is used instead of a decorator, since Python multiprocessing
    module can't serialize decorated function on all platforms.
    """
    manager = multiprocessing.Manager()
    manager_dict = manager.dict()
    process = ProcessWithException(
        manager_dict, target=func, args=args, kwargs=kwargs)
    process.start()
    process.join()
    exc = process.exception
    if exc:
        raise exc
    return process.output


Instruction:
Can you implement a function in Python 3 that
determines the type of run object.

Response:
def get_run_object_type(run_object):
        """Determines run object type."""
        if isinstance(run_object, tuple):
            return 'function'
        run_object, _, _ = run_object.partition(' ')
        if os.path.isdir(run_object):
            return 'package'
        return 'module'


Instruction:
Explain what the following Python 3 code does
def init_module(self, run_object):
        self.profile = self.profile_module
        self._run_object, _, self._run_args = run_object.partition(' ')
        self._object_name = '%s (module)' % self._run_object
        self._globs = {
            '__file__': self._run_object,
            '__name__': '__main__',
            '__package__': None,
        }
        program_path = os.path.dirname(self._run_object)
        if sys.path[0] != program_path:
            sys.path.insert(0, program_path)
        self._replace_sysargs()

Response: Initializes profiler with a module.


Instruction:
Create a Python 3 function for
initializing profiler with a package.

Response:
def init_package(self, run_object):
        """Initializes profiler with a package."""
        self.profile = self.profile_package
        self._run_object, _, self._run_args = run_object.partition(' ')
        self._object_name = '%s (package)' % self._run_object
        self._replace_sysargs()


Instruction:
Can you implement a function in Python 3 that
initializes profiler with a function.

Response:
def init_function(self, run_object):
        """Initializes profiler with a function."""
        self.profile = self.profile_function
        self._run_object, self._run_args, self._run_kwargs = run_object
        filename = inspect.getsourcefile(self._run_object)
        self._object_name = '%s @ %s (function)' % (
            self._run_object.__name__, filename)


Instruction:
Here you have a function in Python 3, explain what it does
def _replace_sysargs(self):
        sys.argv[:] = [self._run_object]
        if self._run_args:
            sys.argv += self._run_args.split()

Response: Replaces sys. argv with proper args to pass to script.


Instruction:
How would you implement a function in Python 3 that
samples the current stack and adds result in self. _stats.

Response:
def sample(self, signum, frame):  #pylint: disable=unused-argument
        """Samples current stack and adds result in self._stats.

        Args:
            signum: Signal that activates handler.
            frame: Frame on top of the stack when signal is handled.
        """
        stack = []
        while frame and frame != self.base_frame:
            stack.append((
                frame.f_code.co_name,
                frame.f_code.co_filename,
                frame.f_code.co_firstlineno))
            frame = frame.f_back
        self._stats[tuple(stack)] += 1
        signal.setitimer(signal.ITIMER_PROF, _SAMPLE_INTERVAL)


Instruction:
Write a Python 3 function that can
insert a stack into the call tree.

Response:
def _insert_stack(stack, sample_count, call_tree):
        """Inserts stack into the call tree.

        Args:
            stack: Call stack.
            sample_count: Sample count of call stack.
            call_tree: Call tree.
        """
        curr_level = call_tree
        for func in stack:
            next_level_index = {
                node['stack']: node for node in curr_level['children']}
            if func not in next_level_index:
                new_node = {'stack': func, 'children': [], 'sampleCount': 0}
                curr_level['children'].append(new_node)
                curr_level = new_node
            else:
                curr_level = next_level_index[func]
        curr_level['sampleCount'] = sample_count


Instruction:
Write a Python 3 script to
count and fills sample count inside call tree.

Response:
def _fill_sample_count(self, node):
        """Counts and fills sample counts inside call tree."""
        node['sampleCount'] += sum(
            self._fill_sample_count(child) for child in node['children'])
        return node['sampleCount']


Instruction:
Given the following Python 3 function, write the documentation
def _format_tree(self, node, total_samples):
        funcname, filename, _ = node['stack']
        sample_percent = self._get_percentage(
            node['sampleCount'], total_samples)
        color_hash = base_profiler.hash_name('%s @ %s' % (funcname, filename))
        return {
            'stack': node['stack'],
            'children': [self._format_tree(child, total_samples)
                         for child in node['children']],
            'sampleCount': node['sampleCount'],
            'samplePercentage': sample_percent,
            'colorHash': color_hash
        }

Response: Reformats the call tree for the UI.


Instruction:
Can you implement a function in Python 3 that
returns a call tree.

Response:
def call_tree(self):
        """Returns call tree."""
        call_tree = {'stack': 'base', 'sampleCount': 0, 'children': []}
        for stack, sample_count in self._stats.items():
            self._insert_stack(reversed(stack), sample_count, call_tree)
        self._fill_sample_count(call_tree)
        if not call_tree['children']:
            return {}
        return self._format_tree(
            call_tree['children'][0], call_tree['sampleCount'])


Instruction:
Write a Python 3 function that can
run statistical profiler on a package.

Response:
def _profile_package(self):
        """Runs statistical profiler on a package."""
        with _StatProfiler() as prof:
            prof.base_frame = inspect.currentframe()
            try:
                runpy.run_path(self._run_object, run_name='__main__')
            except SystemExit:
                pass

        call_tree = prof.call_tree
        return {
            'objectName': self._object_name,
            'sampleInterval': _SAMPLE_INTERVAL,
            'runTime': prof.run_time,
            'callStats': call_tree,
            'totalSamples': call_tree.get('sampleCount', 0),
            'timestamp': int(time.time())
        }


Instruction:
Can you write a function in Python 3 where it
runs statistical profiler on a module.

Response:
def _profile_module(self):
        """Runs statistical profiler on a module."""
        with open(self._run_object, 'rb') as srcfile, _StatProfiler() as prof:
            code = compile(srcfile.read(), self._run_object, 'exec')
            prof.base_frame = inspect.currentframe()
            try:
                exec(code, self._globs, None)
            except SystemExit:
                pass

        call_tree = prof.call_tree
        return {
            'objectName': self._object_name,
            'sampleInterval': _SAMPLE_INTERVAL,
            'runTime': prof.run_time,
            'callStats': call_tree,
            'totalSamples': call_tree.get('sampleCount', 0),
            'timestamp': int(time.time())
        }


Instruction:
Here you have a function in Python 3, explain what it does
def profile_function(self):
        with _StatProfiler() as prof:
            result = self._run_object(*self._run_args, **self._run_kwargs)

        call_tree = prof.call_tree
        return {
            'objectName': self._object_name,
            'sampleInterval': _SAMPLE_INTERVAL,
            'runTime': prof.run_time,
            'callStats': call_tree,
            'totalSamples': call_tree.get('sampleCount', 0),
            'result': result,
            'timestamp': int(time.time())
        }

Response: Runs statistical profiler on a function.


Instruction:
Given the following Python 3 function, write the documentation
def _transform_stats(prof):
        records = []
        for info, params in prof.stats.items():
            filename, lineno, funcname = info
            cum_calls, num_calls, time_per_call, cum_time, _ = params
            if prof.total_tt == 0:
                percentage = 0
            else:
                percentage = round(100 * (cum_time / prof.total_tt), 4)
            cum_time = round(cum_time, 4)
            func_name = '%s @ %s' % (funcname, filename)
            color_hash = base_profiler.hash_name(func_name)
            records.append(
                (filename, lineno, funcname, cum_time, percentage, num_calls,
                 cum_calls, time_per_call, filename, color_hash))
        return sorted(records, key=operator.itemgetter(4), reverse=True)

Response: Processes collected stats for UI.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _profile_package(self):
        prof = cProfile.Profile()
        prof.enable()
        try:
            runpy.run_path(self._run_object, run_name='__main__')
        except SystemExit:
            pass
        prof.disable()
        prof_stats = pstats.Stats(prof)
        prof_stats.calc_callees()
        return {
            'objectName': self._object_name,
            'callStats': self._transform_stats(prof_stats),
            'totalTime': prof_stats.total_tt,
            'primitiveCalls': prof_stats.prim_calls,
            'totalCalls': prof_stats.total_calls,
            'timestamp': int(time.time())
        }

Response: Runs cProfile on a package.


Instruction:
Can you tell what is the following Python 3 function doing
def _profile_module(self):
        prof = cProfile.Profile()
        try:
            with open(self._run_object, 'rb') as srcfile:
                code = compile(srcfile.read(), self._run_object, 'exec')
            prof.runctx(code, self._globs, None)
        except SystemExit:
            pass
        prof_stats = pstats.Stats(prof)
        prof_stats.calc_callees()
        return {
            'objectName': self._object_name,
            'callStats': self._transform_stats(prof_stats),
            'totalTime': prof_stats.total_tt,
            'primitiveCalls': prof_stats.prim_calls,
            'totalCalls': prof_stats.total_calls,
            'timestamp': int(time.time())
        }

Response: Runs cProfile on a module.


Instruction:
Create a Python 3 function for
running cProfile on a function.

Response:
def profile_function(self):
        """Runs cProfile on a function."""
        prof = cProfile.Profile()
        prof.enable()
        result = self._run_object(*self._run_args, **self._run_kwargs)
        prof.disable()
        prof_stats = pstats.Stats(prof)
        prof_stats.calc_callees()
        return {
            'objectName': self._object_name,
            'callStats': self._transform_stats(prof_stats),
            'totalTime': prof_stats.total_tt,
            'primitiveCalls': prof_stats.prim_calls,
            'totalCalls': prof_stats.total_calls,
            'result': result,
            'timestamp': int(time.time())
        }


Instruction:
Explain what the following Python 3 code does
def show_guestbook():
    cursor = flask.g.db.execute(
        'SELECT name, message FROM entry ORDER BY id DESC;')
    entries = [{'name': row[0], 'message': row[1]} for row in cursor.fetchall()]
    return jinja2.Template(LAYOUT).render(entries=entries)

Response: Returns all existing guestbook records.


Instruction:
Write a Python 3 function for
adding a single guestbook record.

Response:
def add_entry():
    """Adds single guestbook record."""
    name, msg = flask.request.form['name'], flask.request.form['message']
    flask.g.db.execute(
        'INSERT INTO entry (name, message) VALUES (?, ?)', (name, msg))
    flask.g.db.commit()
    return flask.redirect('/')


Instruction:
Explain what the following Python 3 code does
def start(host, port, profiler_stats, dont_start_browser, debug_mode):
    stats_handler = functools.partial(StatsHandler, profiler_stats)
    if not debug_mode:
        sys.stderr = open(os.devnull, 'w')
    print('Starting HTTP server...')
    if not dont_start_browser:
        webbrowser.open('http://{}:{}/'.format(host, port))
    try:
        StatsServer((host, port), stats_handler).serve_forever()
    except KeyboardInterrupt:
        print('Stopping...')
        sys.exit(0)

Response: Starts the HTTP server with specified parameters.


Instruction:
Create a Python 3 function to
handle index. html requests.

Response:
def _handle_root():
        """Handles index.html requests."""
        res_filename = os.path.join(
            os.path.dirname(__file__), _PROFILE_HTML)
        with io.open(res_filename, 'rb') as res_file:
            content = res_file.read()
        return content, 'text/html'


Instruction:
How would you implement a function in Python 3 that
handles other files requests.

Response:
def _handle_other(self):
        """Handles static files requests."""
        res_filename = os.path.join(
            os.path.dirname(__file__), _STATIC_DIR, self.path[1:])
        with io.open(res_filename, 'rb') as res_file:
            content = res_file.read()
        _, extension = os.path.splitext(self.path)
        return content, 'text/%s' % extension[1:]


Instruction:
Write a Python 3 function that can
handle HTTP GET requests.

Response:
def do_GET(self):
        """Handles HTTP GET requests."""
        handler = self.uri_map.get(self.path) or self._handle_other
        content, content_type = handler()
        compressed_content = gzip.compress(content)
        self._send_response(
            200, headers=(('Content-type', '%s; charset=utf-8' % content_type),
                          ('Content-Encoding', 'gzip'),
                          ('Content-Length', len(compressed_content))))
        self.wfile.write(compressed_content)


Instruction:
Create a Python 3 function to
handle HTTP POST requests.

Response:
def do_POST(self):
        """Handles HTTP POST requests."""
        post_data = self.rfile.read(int(self.headers['Content-Length']))
        json_data = gzip.decompress(post_data)
        self._profile_json.update(json.loads(json_data.decode('utf-8')))
        self._send_response(
            200, headers=(('Content-type', '%s; charset=utf-8' % 'text/json'),
                          ('Content-Encoding', 'gzip'),
                          ('Content-Length', len(post_data))))


Instruction:
How would you implement a function in Python 3 that
sends HTTP response code message and headers.

Response:
def _send_response(self, http_code, message=None, headers=None):
        """Sends HTTP response code, message and headers."""
        self.send_response(http_code, message)
        if headers:
            for header in headers:
                self.send_header(*header)
            self.end_headers()


Instruction:
Can you write a function in Python 3 where it
checks whether path belongs to standard library or installed modules.

Response:
def check_standard_dir(module_path):
    """Checks whether path belongs to standard library or installed modules."""
    if 'site-packages' in module_path:
        return True
    for stdlib_path in _STDLIB_PATHS:
        if fnmatch.fnmatchcase(module_path, stdlib_path + '*'):
            return True
    return False


Instruction:
How would you explain what the following Python 3 function does
def record_line(self, frame, event, arg):  # pylint: disable=unused-argument
        if event == 'line':
            if self.prev_timestamp:
                runtime = time.time() - self.prev_timestamp
                self.lines.append([self.prev_path, self.prev_lineno, runtime])
            self.prev_lineno = frame.f_lineno
            self.prev_path = frame.f_code.co_filename
            self.prev_timestamp = time.time()
        return self.record_line

Response: Records the execution time of the line.


Instruction:
How would you code a function in Python 3 to
filter code from standard library from self. lines.

Response:
def lines_without_stdlib(self):
        """Filters code from standard library from self.lines."""
        prev_line = None
        current_module_path = inspect.getabsfile(inspect.currentframe())
        for module_path, lineno, runtime in self.lines:
            module_abspath = os.path.abspath(module_path)
            if not prev_line:
                prev_line = [module_abspath, lineno, runtime]
            else:
                if (not check_standard_dir(module_path) and
                        module_abspath != current_module_path):
                    yield prev_line
                    prev_line = [module_abspath, lineno, runtime]
                else:
                    prev_line[2] += runtime
        yield prev_line


Instruction:
Can you write a function in Python 3 where it
fills code heatmap and execution count dictionaries.

Response:
def fill_heatmap(self):
        """Fills code heatmap and execution count dictionaries."""
        for module_path, lineno, runtime in self.lines_without_stdlib:
            self._execution_count[module_path][lineno] += 1
            self._heatmap[module_path][lineno] += runtime


Instruction:
How would you implement a function in Python 3 that
calculates the skip map for large sources.

Response:
def _calc_skips(self, heatmap, num_lines):
        """Calculates skip map for large sources.
        Skip map is a list of tuples where first element of tuple is line
        number and second is length of the skip region:
            [(1, 10), (15, 10)] means skipping 10 lines after line 1 and
            10 lines after line 15.
        """
        if num_lines < self.MIN_SKIP_SIZE:
            return []
        skips, prev_line = [], 0
        for line in sorted(heatmap):
            curr_skip = line - prev_line - 1
            if curr_skip > self.SKIP_LINES:
                skips.append((prev_line, curr_skip))
            prev_line = line
        if num_lines - prev_line > self.SKIP_LINES:
            skips.append((prev_line, num_lines - prev_line))
        return skips


Instruction:
Can you tell what is the following Python 3 function doing
def _skip_lines(src_code, skip_map):
        if not skip_map:
            return [['line', j + 1, l] for j, l in enumerate(src_code)]
        code_with_skips, i = [], 0
        for line, length in skip_map:
            code_with_skips.extend(
                ['line', i + j + 1, l] for j, l in enumerate(src_code[i:line]))
            if (code_with_skips
                    and code_with_skips[-1][0] == 'skip'):  # Merge skips.
                code_with_skips[-1][1] += length
            else:
                code_with_skips.append(['skip', length])
            i = line + length
        code_with_skips.extend(
            ['line', i + j + 1, l] for j, l in enumerate(src_code[i:]))
        return code_with_skips

Response: Skips lines in src_code specified by skip_map.


Instruction:
Create a Python 3 function to
calculate the run time for the given object.

Response:
def _profile_package(self):
        """Calculates heatmap for package."""
        with _CodeHeatmapCalculator() as prof:
            try:
                runpy.run_path(self._run_object, run_name='__main__')
            except SystemExit:
                pass

        heatmaps = []
        for filename, heatmap in prof.heatmap.items():
            if os.path.isfile(filename):
                heatmaps.append(
                    self._format_heatmap(
                        filename, heatmap, prof.execution_count[filename]))

        run_time = sum(heatmap['runTime'] for heatmap in heatmaps)
        return {
            'objectName': self._run_object,
            'runTime': run_time,
            'heatmaps': heatmaps
        }


Instruction:
Create a Python 3 function for
formatting heatmap for UI.

Response:
def _format_heatmap(self, filename, heatmap, execution_count):
        """Formats heatmap for UI."""
        with open(filename) as src_file:
            file_source = src_file.read().split('\n')
            skip_map = self._calc_skips(heatmap, len(file_source))
        run_time = sum(time for time in heatmap.values())
        return {
            'name': filename,
            'heatmap': heatmap,
            'executionCount': execution_count,
            'srcCode': self._skip_lines(file_source, skip_map),
            'runTime': run_time
        }


Instruction:
Explain what the following Python 3 code does
def _profile_module(self):
        with open(self._run_object, 'r') as srcfile:
            src_code = srcfile.read()
            code = compile(src_code, self._run_object, 'exec')
        try:
            with _CodeHeatmapCalculator() as prof:
                exec(code, self._globs, None)
        except SystemExit:
            pass

        heatmaps = []
        for filename, heatmap in prof.heatmap.items():
            if os.path.isfile(filename):
                heatmaps.append(
                    self._format_heatmap(
                        filename, heatmap, prof.execution_count[filename]))

        run_time = sum(heatmap['runTime'] for heatmap in heatmaps)
        return {
            'objectName': self._run_object,
            'runTime': run_time,
            'heatmaps': heatmaps
        }

Response: Calculates the heatmap for the module.


Instruction:
Can you create a Python 3 function that
calculates heatmap for function.

Response:
def profile_function(self):
        """Calculates heatmap for function."""
        with _CodeHeatmapCalculator() as prof:
            result = self._run_object(*self._run_args, **self._run_kwargs)
        code_lines, start_line = inspect.getsourcelines(self._run_object)

        source_lines = []
        for line in code_lines:
            source_lines.append(('line', start_line, line))
            start_line += 1

        filename = os.path.abspath(inspect.getsourcefile(self._run_object))
        heatmap = prof.heatmap[filename]
        run_time = sum(time for time in heatmap.values())
        return {
            'objectName': self._object_name,
            'runTime': run_time,
            'result': result,
            'timestamp': int(time.time()),
            'heatmaps': [{
                'name': self._object_name,
                'heatmap': heatmap,
                'executionCount': prof.execution_count[filename],
                'srcCode': source_lines,
                'runTime': run_time
            }]
        }


Instruction:
Can you generate the documentation for the following Python 3 function
def run_profilers(run_object, prof_config, verbose=False):
    if len(prof_config) > len(set(prof_config)):
        raise AmbiguousConfigurationError(
            'Profiler configuration %s is ambiguous' % prof_config)

    available_profilers = {opt for opt, _ in _PROFILERS}
    for option in prof_config:
        if option not in available_profilers:
            raise BadOptionError('Unknown option: %s' % option)

    run_stats = OrderedDict()
    present_profilers = ((o, p) for o, p in _PROFILERS if o in prof_config)
    for option, prof in present_profilers:
        curr_profiler = prof(run_object)
        if verbose:
            print('Running %s...' % curr_profiler.__class__.__name__)
        run_stats[option] = curr_profiler.run()
    return run_stats

Response: Runs profilers on a single object.


Instruction:
Write a Python 3 script to
run a function on a node.

Response:
def run(func, options, args=(), kwargs={}, host='localhost', port=8000):  # pylint: disable=dangerous-default-value
    """Runs profilers on a function.

    Args:
        func: A Python function.
        options: A string with profilers configuration (i.e. 'cmh').
        args: func non-keyword arguments.
        kwargs: func keyword arguments.
        host: Host name to send collected data.
        port: Port number to send collected data.

    Returns:
        A result of func execution.
    """
    run_stats = run_profilers((func, args, kwargs), options)

    result = None
    for prof in run_stats:
        if not result:
            result = run_stats[prof]['result']
        del run_stats[prof]['result']  # Don't send result to remote host

    post_data = gzip.compress(
        json.dumps(run_stats).encode('utf-8'))
    urllib.request.urlopen('http://%s:%s' % (host, port), post_data)
    return result


Instruction:
Write a Python 3 function that can
predict probability estimates for the test vectors X.

Response:
def predict_proba(self, X):
        """
        Return probability estimates for the RDD containing test vector X.

        Parameters
        ----------
        X : RDD containing array-like items, shape = [m_samples, n_features]

        Returns
        -------
        C : RDD with array-like items , shape = [n_samples, n_classes]
            Returns the probability of the samples for each class in
            the models for each RDD block. The columns correspond to the classes
            in sorted order, as they appear in the attribute `classes_`.
        """
        check_rdd(X, (sp.spmatrix, np.ndarray))
        return X.map(
            lambda X: super(SparkBaseNB, self).predict_proba(X))


Instruction:
Explain what the following Python 3 code does
def predict_log_proba(self, X):
        # required, scikit call self.predict_log_proba(X) in predict_proba
        # and thus this function is call, it must have the same behavior when
        # not called by sparkit-learn
        if not isinstance(X, BlockRDD):
            return super(SparkBaseNB, self).predict_log_proba(X)

        check_rdd(X, (sp.spmatrix, np.ndarray))
        return X.map(
            lambda X: super(SparkBaseNB, self).predict_log_proba(X))

Response: Predicts the log probability estimates for the test vectors X.


Instruction:
Can you tell what is the following Python 3 function doing
def fit(self, Z, classes=None):
        check_rdd(Z, {'X': (sp.spmatrix, np.ndarray), 'y': (sp.spmatrix, np.ndarray)})
        models = Z[:, ['X', 'y']].map(
            lambda X_y: self.partial_fit(X_y[0], X_y[1], classes))
        avg = models.reduce(operator.add)
        self.__dict__.update(avg.__dict__)
        return self

Response: Fit Gaussian Naive Bayes according to X y.


Instruction:
Implement a Python 3 function for
fitting Multinomial Naive Bayes according to X y.

Response:
def fit(self, Z, classes=None):
        """
        TODO fulibacsi fix docstring
        Fit Multinomial Naive Bayes according to (X,y) pair
        which is zipped into TupleRDD Z.

        Parameters
        ----------
        Z : TupleRDD containing X [array-like, shape (m_samples, n_features)]
            and y [array-like, shape (m_samples,)] tuple
            Training vectors, where ,_samples is the number of samples in the
            block and n_features is the number of features, and y contains
            the target values.

        Returns
        -------
        self : object
            Returns self.
        """
        check_rdd(Z, {'X': (sp.spmatrix, np.ndarray), 'y': (sp.spmatrix, np.ndarray)})
        if 'w' in Z.columns:
            models = Z[:, ['X', 'y', 'w']].map(
                lambda X_y_w: self.partial_fit(
                    X_y_w[0], X_y_w[1], classes, X_y_w[2]
                )
            )
        else:
            models = Z[:, ['X', 'y']].map(
                lambda X_y: self.partial_fit(X_y[0], X_y[1], classes))
        avg = models.sum()
        self.__dict__.update(avg.__dict__)
        return self


Instruction:
Write a Python 3 script to
create vocabulary for the current language.

Response:
def _init_vocab(self, analyzed_docs):
        """Create vocabulary
        """
        class SetAccum(AccumulatorParam):

            def zero(self, initialValue):
                return set(initialValue)

            def addInPlace(self, v1, v2):
                v1 |= v2
                return v1

        if not self.fixed_vocabulary_:
            accum = analyzed_docs._rdd.context.accumulator(set(), SetAccum())
            analyzed_docs.foreach(
                lambda x: accum.add(set(chain.from_iterable(x))))
            vocabulary = {t: i for i, t in enumerate(accum.value)}
        else:
            vocabulary = self.vocabulary_

        if not vocabulary:
            raise ValueError("empty vocabulary; perhaps the documents only"
                             " contain stop words")
        return vocabulary


Instruction:
Explain what the following Python 3 code does
def _count_vocab(self, analyzed_docs):
        vocabulary = self.vocabulary_
        j_indices = _make_int_array()
        indptr = _make_int_array()
        indptr.append(0)
        for doc in analyzed_docs:
            for feature in doc:
                try:
                    j_indices.append(vocabulary[feature])
                except KeyError:
                    # Ignore out-of-vocabulary items for fixed_vocab=True
                    continue
            indptr.append(len(j_indices))

        j_indices = frombuffer_empty(j_indices, dtype=np.intc)
        indptr = np.frombuffer(indptr, dtype=np.intc)
        values = np.ones(len(j_indices))

        X = sp.csr_matrix((values, j_indices, indptr),
                          shape=(len(indptr) - 1, len(vocabulary)),
                          dtype=self.dtype)
        X.sum_duplicates()

        if self.binary:
            X.data.fill(1)

        return X

Response:
Create sparse feature matrix and vocabulary where fixed_vocab = False
            is True


Instruction:
Create a Python 3 function for
sorting features by name Returns a reordered matrix and modifies the vocabulary in place

Response:
def _sort_features(self, vocabulary):
        """Sort features by name

        Returns a reordered matrix and modifies the vocabulary in place
        """
        sorted_features = sorted(six.iteritems(vocabulary))
        map_index = np.empty(len(sorted_features), dtype=np.int32)
        for new_val, (term, old_val) in enumerate(sorted_features):
            map_index[new_val] = old_val
            vocabulary[term] = new_val

        return map_index


Instruction:
Can you write a function in Python 3 where it
removes too rare or too common features.

Response:
def _limit_features(self, X, vocabulary, high=None, low=None,
                        limit=None):
        """Remove too rare or too common features.

        Prune features that are non zero in more samples than high or less
        documents than low, modifying the vocabulary, and restricting it to
        at most the limit most frequent.

        This does not prune samples with zero features.
        """
        if high is None and low is None and limit is None:
            return X, set()

        # Calculate a mask based on document frequencies
        dfs = X.map(_document_frequency).sum()
        tfs = X.map(lambda x: np.asarray(x.sum(axis=0))).sum().ravel()
        mask = np.ones(len(dfs), dtype=bool)
        if high is not None:
            mask &= dfs <= high
        if low is not None:
            mask &= dfs >= low
        if limit is not None and mask.sum() > limit:
            mask_inds = (-tfs[mask]).argsort()[:limit]
            new_mask = np.zeros(len(dfs), dtype=bool)
            new_mask[np.where(mask)[0][mask_inds]] = True
            mask = new_mask

        new_indices = np.cumsum(mask) - 1  # maps old indices to new
        removed_terms = set()
        for term, old_index in list(six.iteritems(vocabulary)):
            if mask[old_index]:
                vocabulary[term] = new_indices[old_index]
            else:
                del vocabulary[term]
                removed_terms.add(term)
        kept_indices = np.where(mask)[0]

        if len(kept_indices) == 0:
            raise ValueError("After pruning, no terms remain. Try a lower"
                             " min_df or a higher max_df.")

        return kept_indices, removed_terms


Instruction:
Can you create a Python 3 function that
learns the vocabulary dictionary and return the term - document matrix.

Response:
def fit_transform(self, Z):
        """Learn the vocabulary dictionary and return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        Z : iterable or DictRDD with column 'X'
            An iterable of raw_documents which yields either str, unicode or
            file objects; or a DictRDD with column 'X' containing such
            iterables.

        Returns
        -------
        X : array, [n_samples, n_features] or DictRDD
            Document-term matrix.
        """
        self._validate_vocabulary()

        # map analyzer and cache result
        analyze = self.build_analyzer()
        A = Z.transform(lambda X: list(map(analyze, X)), column='X').persist()

        # create vocabulary
        X = A[:, 'X'] if isinstance(A, DictRDD) else A
        self.vocabulary_ = self._init_vocab(X)

        # transform according to vocabulary
        mapper = self.broadcast(self._count_vocab, A.context)
        Z = A.transform(mapper, column='X', dtype=sp.spmatrix)

        if not self.fixed_vocabulary_:
            X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

            max_df = self.max_df
            min_df = self.min_df
            max_features = self.max_features

            # limit features according to min_df, max_df parameters
            n_doc = X.shape[0]
            max_doc_count = (max_df
                             if isinstance(max_df, numbers.Integral)
                             else max_df * n_doc)
            min_doc_count = (min_df
                             if isinstance(min_df, numbers.Integral)
                             else min_df * n_doc)
            if max_doc_count < min_doc_count:
                raise ValueError(
                    "max_df corresponds to < documents than min_df")
            kept_indices, self.stop_words_ = self._limit_features(
                X, self.vocabulary_, max_doc_count, min_doc_count, max_features)

            # sort features
            map_index = self._sort_features(self.vocabulary_)

            # combined mask
            mask = kept_indices[map_index]

            Z = Z.transform(lambda x: x[:, mask], column='X', dtype=sp.spmatrix)
        A.unpersist()
        return Z


Instruction:
Can you write a function in Python 3 where it
transforms documents to document - term matrix.

Response:
def transform(self, Z):
        """Transform documents to document-term matrix.

        Extract token counts out of raw text documents using the vocabulary
        fitted with fit or the one provided to the constructor.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : sparse matrix, [n_samples, n_features]
            Document-term matrix.
        """
        if not hasattr(self, 'vocabulary_'):
            self._validate_vocabulary()

        self._check_vocabulary()

        analyze = self.build_analyzer()
        mapper = self.broadcast(self._count_vocab, Z.context)

        Z = Z.transform(lambda X: list(map(analyze, X)), column='X') \
             .transform(mapper, column='X', dtype=sp.spmatrix)

        return Z


Instruction:
How would you implement a function in Python 3 that
transforms an ArrayRDD or DictRDD containing sequence of documents to a document - term matrix.

Response:
def transform(self, Z):
        """Transform an ArrayRDD (or DictRDD with column 'X') containing
        sequence of documents to a document-term matrix.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD with raw text documents
            Samples. Each sample must be a text document (either bytes or
            unicode strings) which will be tokenized and hashed.

        Returns
        -------
        Z : SparseRDD/DictRDD containg scipy.sparse matrix
            Document-term matrix.
        """
        mapper = super(SparkHashingVectorizer, self).transform
        return Z.transform(mapper, column='X', dtype=sp.spmatrix)


Instruction:
Can you tell what is the following Python 3 function doing
def fit(self, Z):

        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (sp.spmatrix, np.ndarray))

        def mapper(X, use_idf=self.use_idf):
            if not sp.issparse(X):
                X = sp.csc_matrix(X)
            if use_idf:
                return _document_frequency(X)

        if self.use_idf:
            n_samples, n_features = X.shape
            df = X.map(mapper).treeReduce(operator.add)

            # perform idf smoothing if required
            df += int(self.smooth_idf)
            n_samples += int(self.smooth_idf)

            # log1p instead of log makes sure terms with zero idf don't get
            # suppressed entirely
            idf = np.log(float(n_samples) / df) + 1.0
            self._idf_diag = sp.spdiags(idf,
                                        diags=0, m=n_features, n=n_features)
        return self

Response: Learn the idf vector for the current term counts and the term counts in Z.


Instruction:
How would you code a function in Python 3 to
fit the log - likelihood to the given set of data.

Response:
def fit(self, Z):
        """Compute the mean and std to be used for later scaling.
        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector.
                {array-like, sparse matrix}, shape [n_samples, n_features]
                The data used to compute the mean and standard deviation
                used for later scaling along the features axis.
            y - Target labels
                Passthrough for ``Pipeline`` compatibility.
        """

        # Reset internal state before fitting
        self._reset()
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        def mapper(X):
            """Calculate statistics for every numpy or scipy blocks."""
            X = check_array(X, ('csr', 'csc'), dtype=np.float64)
            if hasattr(X, "toarray"):   # sparse matrix
                mean, var = mean_variance_axis(X, axis=0)
            else:
                mean, var = np.mean(X, axis=0), np.var(X, axis=0)
            return X.shape[0], mean, var

        def reducer(a, b):
            """Calculate the combined statistics."""
            n_a, mean_a, var_a = a
            n_b, mean_b, var_b = b
            n_ab = n_a + n_b
            mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab
            var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) + \
                     ((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** 2)
            return (n_ab, mean_ab, var_ab)

        if check_rdd_dtype(X, (sp.spmatrix)):
            if self.with_mean:
                raise ValueError(
                    "Cannot center sparse matrices: pass `with_mean=False` "
                    "instead. See docstring for motivation and alternatives.")
        self.n_samples_seen_, self.mean_, self.var_ = X.map(mapper).treeReduce(reducer)

        if self.with_std:
            self.scale_ = _handle_zeros_in_scale(np.sqrt(self.var_))
        else:
            self.scale_ = None

        return self


Instruction:
Can you generate the documentation for the following Python 3 function
def transform(self, Z):
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        if check_rdd_dtype(X, (sp.spmatrix)):
            if self.with_mean:
                raise ValueError(
                    "Cannot center sparse matrices: pass `with_mean=False` "
                    "instead. See docstring for motivation and alternatives.")
            if self.scale_ is not None:
                def mapper(X):
                    inplace_column_scale(X, 1 / self.scale_)
                    return X
        else:
            if self.with_mean:
                if self.with_std:
                    def mapper(X):
                        X -= self.mean_
                        X /= self.scale_
                        return X
                else:
                    def mapper(X):
                        X -= self.mean_
                        return X
            else:
                if self.with_std:
                    def mapper(X):
                        X /= self.scale_
                        return X
                else:
                    raise ValueError("Need with_std or with_mean ")
        return Z.transform(mapper, column="X")

Response: Perform standardization by centering and scaling by centering and scaling.


Instruction:
Given the following Python 3 function, write the documentation
def to_scikit(self):
        scaler = StandardScaler(with_mean=self.with_mean,
                                with_std=self.with_std,
                                copy=self.copy)
        scaler.__dict__ = self.__dict__
        return scaler

Response: Convert to equivalent StandardScaler


Instruction:
Can you write a function in Python 3 where it
wraps a Scikit - learn Linear model s fit method to use with RDDs input.

Response:
def _spark_fit(self, cls, Z, *args, **kwargs):
        """Wraps a Scikit-learn Linear model's fit method to use with RDD
        input.

        Parameters
        ----------
        cls : class object
            The sklearn linear model's class to wrap.
        Z : TupleRDD or DictRDD
            The distributed train data in a DictRDD.

        Returns
        -------
        self: the wrapped class
        """
        mapper = lambda X_y: super(cls, self).fit(
            X_y[0], X_y[1], *args, **kwargs
        )
        models = Z.map(mapper)
        avg = models.reduce(operator.add) / models.count()
        self.__dict__.update(avg.__dict__)
        return self


Instruction:
Here you have a function in Python 3, explain what it does
def _spark_predict(self, cls, X, *args, **kwargs):
        return X.map(lambda X: super(cls, self).predict(X, *args, **kwargs))

Response:
Wraps a Scikit - learn Linear model s predict method to use with RDDs
            input.


Instruction:
Implement a Python 3 function for
fitting linear model. Parameters ---------- Z : DictRDD with (X, y) values X containing numpy array or sparse matrix - The training data y containing the target values Returns ------- self : returns an instance of self.

Response:
def fit(self, Z):
        """
        Fit linear model.

        Parameters
        ----------
        Z : DictRDD with (X, y) values
            X containing numpy array or sparse matrix - The training data
            y containing the target values

        Returns
        -------
        self : returns an instance of self.
        """
        check_rdd(Z, {'X': (sp.spmatrix, np.ndarray)})
        return self._spark_fit(SparkLinearRegression, Z)


Instruction:
How would you code a function in Python 3 to
fit all the transforms one after the other and transform the data using the final estimator.

Response:
def fit(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then fit the transformed data using the final estimator.

        Parameters
        ----------
        Z : ArrayRDD, TupleRDD or DictRDD
            Input data in blocked distributed format.

        Returns
        -------
        self : SparkPipeline
        """
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        self.steps[-1][-1].fit(Zt, **fit_params)
        Zt.unpersist()
        return self


Instruction:
Implement a Python 3 function for
fitting all the transforms one after the other and transform the data. Use fit_transform on transformed data using the final estimator.

Response:
def fit_transform(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then use fit_transform on transformed data using the final
        estimator."""
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        if hasattr(self.steps[-1][-1], 'fit_transform'):
            return self.steps[-1][-1].fit_transform(Zt, **fit_params)
        else:
            return self.steps[-1][-1].fit(Zt, **fit_params).transform(Zt)


Instruction:
Given the following Python 3 function, write the documentation
def score(self, Z):
        Zt = Z
        for name, transform in self.steps[:-1]:
            Zt = transform.transform(Zt)
        return self.steps[-1][-1].score(Zt)

Response: Applies transforms to the data and returns the score method of the final estimator. Valid only if the final estimator implements the score method. Valid only if the final estimator implements the score method. Valid only if the final estimator implements the score method.


Instruction:
Make a summary of the following Python 3 code
def fit(self, Z, **fit_params):
        fit_params_steps = dict((step, {})
                                for step, _ in self.transformer_list)

        for pname, pval in six.iteritems(fit_params):
            step, param = pname.split('__', 1)
            fit_params_steps[step][param] = pval

        transformers = Parallel(n_jobs=self.n_jobs, backend="threading")(
            delayed(_fit_one_transformer)(trans, Z, **fit_params_steps[name])
            for name, trans in self.transformer_list)
        self._update_transformer_list(transformers)
        return self

Response: Fits all transformers using X.


Instruction:
Here you have a function in Python 3, explain what it does
def fit_transform(self, Z, **fit_params):
        return self.fit(Z, **fit_params).transform(Z)

Response: Fit all transformers using X transform the data and concatenate the results.


Instruction:
Can you tell what is the following Python 3 function doing
def transform(self, Z):
        if isinstance(Z, DictRDD):
            X = Z[:, 'X']
        else:
            X = Z

        Zs = [_transform_one(trans, name, X, self.transformer_weights)
              for name, trans in self.transformer_list]
        X_rdd = reduce(lambda x, y: x.zip(y._rdd), Zs)
        X_rdd = X_rdd.map(flatten)
        mapper = np.hstack
        for item in X_rdd.first():
            if sp.issparse(item):
                mapper = sp.hstack
        X_rdd = X_rdd.map(lambda x: mapper(x))

        if isinstance(Z, DictRDD):
            return DictRDD([X_rdd, Z[:, 'y']],
                           columns=Z.columns,
                           dtype=Z.dtype,
                           bsize=Z.bsize)
        else:
            return X_rdd

Response: Transform X by each transformer.


Instruction:
Can you generate the documentation for the following Python 3 function
def fit(self, Z, classes=None):
        check_rdd(Z, {'X': (sp.spmatrix, np.ndarray)})
        mapper = lambda X_y: super(SparkRandomForestClassifier, self).fit(
            X_y[0], X_y[1]
        )

        models = Z.map(mapper).collect()

        self.__dict__ = models[0].__dict__
        self.estimators_ = []
        for m in models:
            self.estimators_ += m.estimators_
        self.n_estimators = len(self.estimators_)
        return self

Response: Fit the model according to the given training data.


Instruction:
Can you tell what is the following Python 3 function doing
def _fit(self, Z, parameter_iterable):
        self.scorer_ = check_scoring(self.estimator, scoring=self.scoring)

        cv = self.cv
        cv = _check_cv(cv, Z)

        if self.verbose > 0:
            if isinstance(parameter_iterable, Sized):
                n_candidates = len(parameter_iterable)
                print("Fitting {0} folds for each of {1} candidates, totalling"
                      " {2} fits".format(len(cv), n_candidates,
                                         n_candidates * len(cv)))

        base_estimator = clone(self.estimator)

        pre_dispatch = self.pre_dispatch

        out = Parallel(
            n_jobs=self.n_jobs, verbose=self.verbose,
            pre_dispatch=pre_dispatch, backend="threading"
        )(
            delayed(_fit_and_score)(clone(base_estimator), Z, self.scorer_,
                                    train, test, self.verbose, parameters,
                                    self.fit_params, return_parameters=True,
                                    error_score=self.error_score)
            for parameters in parameter_iterable
            for train, test in cv)

        # Out is a list of triplet: score, estimator, n_test_samples
        n_fits = len(out)
        n_folds = len(cv)

        scores = list()
        grid_scores = list()
        for grid_start in range(0, n_fits, n_folds):
            n_test_samples = 0
            score = 0
            all_scores = []
            for this_score, this_n_test_samples, _, parameters in \
                    out[grid_start:grid_start + n_folds]:
                all_scores.append(this_score)
                if self.iid:
                    this_score *= this_n_test_samples
                    n_test_samples += this_n_test_samples
                score += this_score
            if self.iid:
                score /= float(n_test_samples)
            else:
                score /= float(n_folds)
            scores.append((score, parameters))
            # TODO: shall we also store the test_fold_sizes?
            grid_scores.append(_CVScoreTuple(
                parameters,
                score,
                np.array(all_scores)))
        # Store the computed scores
        self.grid_scores_ = grid_scores

        # Find the best parameters by comparing on the mean validation score:
        # note that `sorted` is deterministic in the way it breaks ties
        best = sorted(grid_scores, key=lambda x: x.mean_validation_score,
                      reverse=True)[0]
        self.best_params_ = best.parameters
        self.best_score_ = best.mean_validation_score

        if self.refit:
            # fit the best estimator using the entire dataset
            # clone first to work around broken estimators
            best_estimator = clone(base_estimator).set_params(
                **best.parameters)
            best_estimator.fit(Z, **self.fit_params)
            self.best_estimator_ = best_estimator
        return self

Response: Perform the actual fitting of the parameters.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fit(self, y):

        def mapper(y):
            y = column_or_1d(y, warn=True)
            _check_numpy_unicode_bug(y)
            return np.unique(y)

        def reducer(a, b):
            return np.unique(np.concatenate((a, b)))

        self.classes_ = y.map(mapper).reduce(reducer)

        return self

Response: Fit label encoder to target values.


Instruction:
Write a Python 3 function that can
transform labels to normalized encoding.

Response:
def transform(self, y):
        """Transform labels to normalized encoding.
        Parameters
        ----------
        y : ArrayRDD [n_samples]
            Target values.
        Returns
        -------
        y : ArrayRDD [n_samples]
        """
        mapper = super(SparkLabelEncoder, self).transform
        mapper = self.broadcast(mapper, y.context)
        return y.transform(mapper)


Instruction:
Explain what the following Python 3 code does
def _score(estimator, Z_test, scorer):
    score = scorer(estimator, Z_test)
    if not isinstance(score, numbers.Number):
        raise ValueError("scoring must return a number, got %s (%s) instead."
                         % (str(score), type(score)))
    return score

Response: Compute the score of an estimator on a given test set.


Instruction:
Given the following Python 3 function, write the documentation
def fit(self, Z):
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))
        if self.init == 'k-means||':
            self._mllib_model = MLlibKMeans.train(
                X.unblock(),
                self.n_clusters,
                maxIterations=self.max_iter,
                initializationMode="k-means||")
            self.cluster_centers_ = self._mllib_model.centers
        else:
            models = X.map(lambda X: super(SparkKMeans, self).fit(X))
            models = models.map(lambda model: model.cluster_centers_).collect()
            return super(SparkKMeans, self).fit(np.concatenate(models))

Response: Compute k - means clustering.


Instruction:
Create a Python 3 function to
predict the closest cluster each sample in X belongs to.

Response:
def predict(self, X):
        """Predict the closest cluster each sample in X belongs to.

        In the vector quantization literature, `cluster_centers_` is called
        the code book and each value returned by `predict` is the index of
        the closest code in the code book.

        Parameters
        ----------
        X : ArrayRDD containing array-like, sparse matrix
            New data to predict.

        Returns
        -------
        labels : ArrayRDD with predictions
            Index of the cluster each sample belongs to.

        """
        check_rdd(X, (np.ndarray, sp.spmatrix))
        if hasattr(self, '_mllib_model'):
            if isinstance(X, ArrayRDD):
                X = X.unblock()
            return X.map(lambda x: self._mllib_model.predict(x))
        else:
            rdd = X.map(lambda X: super(SparkKMeans, self).predict(X))
            return ArrayRDD(rdd)


Instruction:
Here you have a function in Python 3, explain what it does
def fit(self, Z, classes=None):
        check_rdd(Z, {'X': (sp.spmatrix, np.ndarray)})
        self._classes_ = np.unique(classes)
        return self._spark_fit(SparkSGDClassifier, Z)

Response: Fit the model according to the given training data.


Instruction:
Explain what the following Python 3 code does
def predict(self, X):
        check_rdd(X, (sp.spmatrix, np.ndarray))
        return self._spark_predict(SparkSGDClassifier, X)

Response: Distributed method to predict class labels for samples in X.


Instruction:
Can you implement a function in Python 3 that
checks if the types of blocks in the RDD are in the expected types.

Response:
def check_rdd_dtype(rdd, expected_dtype):
    """Checks if the blocks in the RDD matches the expected types.

    Parameters:
    -----------
    rdd: splearn.BlockRDD
        The RDD to check
    expected_dtype: {type, list of types, tuple of types, dict of types}
        Expected type(s). If the RDD is a DictRDD the parameter type is
        restricted to dict.

    Returns:
    --------
    accept: bool
        Returns if the types are matched.
    """
    if not isinstance(rdd, BlockRDD):
        raise TypeError("Expected {0} for parameter rdd, got {1}."
                        .format(BlockRDD, type(rdd)))
    if isinstance(rdd, DictRDD):
        if not isinstance(expected_dtype, dict):
            raise TypeError('Expected {0} for parameter '
                            'expected_dtype, got {1}.'
                            .format(dict, type(expected_dtype)))
        accept = True
        types = dict(list(zip(rdd.columns, rdd.dtype)))
        for key, values in expected_dtype.items():
            if not isinstance(values, (tuple, list)):
                values = [values]
            accept = accept and types[key] in values
        return accept

    if not isinstance(expected_dtype, (tuple, list)):
        expected_dtype = [expected_dtype]

    return rdd.dtype in expected_dtype


Instruction:
Implement a Python 3 function for
learning a list of feature name - > indices mappings.

Response:
def fit(self, Z):
        """Learn a list of feature name -> indices mappings.

        Parameters
        ----------
        Z : DictRDD with column 'X'
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        self
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

        """Create vocabulary
        """
        class SetAccum(AccumulatorParam):

            def zero(self, initialValue):
                return set(initialValue)

            def addInPlace(self, v1, v2):
                v1 |= v2
                return v1

        accum = X.context.accumulator(set(), SetAccum())

        def mapper(X, separator=self.separator):
            feature_names = []
            for x in X:
                for f, v in six.iteritems(x):
                    if isinstance(v, six.string_types):
                        f = "%s%s%s" % (f, self.separator, v)
                    feature_names.append(f)
            accum.add(set(feature_names))

        X.foreach(mapper)  # init vocabulary
        feature_names = list(accum.value)

        if self.sort:
            feature_names.sort()

        vocab = dict((f, i) for i, f in enumerate(feature_names))

        self.feature_names_ = feature_names
        self.vocabulary_ = vocab

        return self


Instruction:
Write a Python 3 function that can
transform ArrayRDDs or DictRDD s X column s feature - > value dicts to array or sparse matrix.

Response:
def transform(self, Z):
        """Transform ArrayRDD's (or DictRDD's 'X' column's) feature->value dicts
        to array or sparse matrix.
        Named features not encountered during fit or fit_transform will be
        silently ignored.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD with column 'X' containing Mapping or
            iterable over Mappings, length = n_samples
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        Z : transformed, containing {array, sparse matrix}
            Feature vectors; always 2-d.
        """
        mapper = self.broadcast(super(SparkDictVectorizer, self).transform,
                                Z.context)
        dtype = sp.spmatrix if self.sparse else np.ndarray
        return Z.transform(mapper, column='X', dtype=dtype)


Instruction:
Create a Python 3 function to
learn empirical variances from X.

Response:
def fit(self, Z):
        """Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self
        """

        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        def mapper(X):
            """Calculate statistics for every numpy or scipy blocks."""
            X = check_array(X, ('csr', 'csc'), dtype=np.float64)
            if hasattr(X, "toarray"):   # sparse matrix
                mean, var = mean_variance_axis(X, axis=0)
            else:
                mean, var = np.mean(X, axis=0), np.var(X, axis=0)
            return X.shape[0], mean, var

        def reducer(a, b):
            """Calculate the combined statistics."""
            n_a, mean_a, var_a = a
            n_b, mean_b, var_b = b
            n_ab = n_a + n_b
            mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab
            var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) + \
                     ((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** 2)
            return (n_ab, mean_ab, var_ab)

        _, _, self.variances_ = X.map(mapper).treeReduce(reducer)

        if np.all(self.variances_ <= self.threshold):
            msg = "No feature in X meets the variance threshold {0:.5f}"
            if X.shape[0] == 1:
                msg += " (X contains only one sample)"
            raise ValueError(msg.format(self.threshold))

        return self


Instruction:
Here you have a function in Python 3, explain what it does
def svd(blocked_rdd, k):

    # compute the covariance matrix (without mean subtraction)
    # TODO use one func for this (with mean subtraction as an option?)
    c = blocked_rdd.map(lambda x: (x.T.dot(x), x.shape[0]))
    prod, n = c.reduce(lambda x, y: (x[0] + y[0], x[1] + y[1]))

    # do local eigendecomposition
    w, v = ln.eig(prod / n)
    w = np.real(w)
    v = np.real(v)
    inds = np.argsort(w)[::-1]
    s = np.sqrt(w[inds[0:k]]) * np.sqrt(n)
    v = v[:, inds[0:k]].T

    # project back into data, normalize by singular values
    u = blocked_rdd.map(lambda x: np.inner(x, v) / s)

    return u, s, v

Response:
Calculate the SVD of a blocked RDD directly returning only the leading k
    singular vectors.


Instruction:
Write a Python 3 script to
calculate the SVD of a blocked RDD using an expectation maximization algorithm that avoids explicitly computing the leading kometric subspace.

Response:
def svd_em(blocked_rdd, k, maxiter=20, tol=1e-6, compute_u=True, seed=None):
    """
    Calculate the SVD of a blocked RDD using an expectation maximization
    algorithm (from Roweis, NIPS, 1997) that avoids explicitly
    computing the covariance matrix, returning only the leading k
    singular vectors. Assumes n rows and d columns, does not require
    d^2 to fit into memory on a single machine.
    Parameters
    ----------
    blocked_rdd : ArrayRDD
        ArrayRDD with data points in numpy array blocks
    k : Int
        Number of singular vectors to return
    maxiter : Int, optional, default = 20
        Number of iterations to perform
    tol : Double, optional, default = 1e-5
        Tolerance for stopping iterative updates
    seed : Int, optional, default = None
        Seed for random number generator for initializing subspace
    Returns
    ----------
    u : RDD of blocks
        Left eigenvectors
    s : numpy array
        Singular values
    v : numpy array
        Right eigenvectors
    """

    n, m = blocked_rdd.shape[:2]
    sc = blocked_rdd._rdd.context

    def outerprod(x):
        return x.T.dot(x)

    # global run_sum

    # def accumsum(x):
    #     global run_sum
    #     run_sum += x

    # class MatrixAccum(AccumulatorParam):

    #     def zero(self, value):
    #         return np.zeros(np.shape(value))

    #     def addInPlace(self, val1, val2):
    #         val1 += val2
    #         return val1

    if seed is not None:
        rng = np.random.RandomState(seed)
        c = rng.randn(k, m)
    else:
        c = np.random.randn(k, m)

    iter = 0
    error = 100

    # iteratively update subspace using expectation maximization
    # e-step: x = (cc')^-1 c y
    # m-step: c = y x' (xx')^-1
    while (iter < maxiter) & (error > tol):
        c_old = c

        # pre compute (cc')^-1 c
        c_inv = np.dot(c.T, ln.inv(np.dot(c, c.T)))
        premult1 = sc.broadcast(c_inv)

        # compute (xx')^-1 through a map reduce
        xx = blocked_rdd.map(lambda x: outerprod(safe_sparse_dot(x, premult1.value))) \
                        .treeReduce(add)

        # compute (xx')^-1 using an accumulator
        # run_sum = sc.accumulator(np.zeros((k, k)), MatrixAccum())
        # blocked_rdd.map(lambda x: outerprod(safe_sparse_dot(x, premult1.value))) \
        #            .foreachPartition(lambda l: accumsum(sum(l)))
        # xx = run_sum.value
        xx_inv = ln.inv(xx)

        # pre compute (cc')^-1 c (xx')^-1
        premult2 = blocked_rdd.context.broadcast(np.dot(c_inv, xx_inv))

        # compute the new c through a map reduce
        c = blocked_rdd.map(lambda x: safe_sparse_dot(x.T, safe_sparse_dot(x, premult2.value))) \
                       .treeReduce(add)

        # compute the new c using an accumulator
        # run_sum = sc.accumulator(np.zeros((m, k)), MatrixAccum())
        # blocked_rdd.map(lambda x: safe_sparse_dot(x.T, safe_sparse_dot(x, premult2.value))) \
        #            .foreachPartition(lambda l: accumsum(sum(l)))
        # c = run_sum.value
        c = c.T

        error = np.sum((c - c_old) ** 2)
        iter += 1

    # project data into subspace spanned by columns of c
    # use standard eigendecomposition to recover an orthonormal basis
    c = ln.orth(c.T).T
    cov = blocked_rdd.map(lambda x: safe_sparse_dot(x, c.T)) \
                     .map(lambda x: outerprod(x)) \
                     .treeReduce(add)
    w, v = ln.eig(cov / n)
    w = np.real(w)
    v = np.real(v)
    inds = np.argsort(w)[::-1]
    s = np.sqrt(w[inds[0:k]]) * np.sqrt(n)
    v = np.dot(v[:, inds[0:k]].T, c)
    if compute_u:
        v_broadcasted = blocked_rdd.context.broadcast(v)
        u = blocked_rdd.map(
            lambda x: safe_sparse_dot(x, v_broadcasted.value.T) / s)
        return u, s, v
    else:
        return s, v


Instruction:
Write a Python 3 script for
fitting LSI model to X and perform dimensionality reduction on X.

Response:
def fit_transform(self, Z):
        """Fit LSI model to X and perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (sp.spmatrix, np.ndarray))
        if self.algorithm == "em":
            X = X.persist()  # boosting iterative svm
            Sigma, V = svd_em(X, k=self.n_components, maxiter=self.n_iter,
                              tol=self.tol, compute_u=False,
                              seed=self.random_state)
            self.components_ = V
            X.unpersist()
            return self.transform(Z)
        else:
            # TODO: raise warning non distributed
            return super(SparkTruncatedSVD, self).fit_transform(X.tosparse())


Instruction:
Write a Python 3 script for
performing dimensionality reduction on X.

Response:
def transform(self, Z):
        """Perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            New data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (sp.spmatrix, np.ndarray))

        mapper = self.broadcast(
            super(SparkTruncatedSVD, self).transform, Z.context)
        return Z.transform(mapper, column='X', dtype=np.ndarray)


Instruction:
Write a Python 3 function that can
pack rdd with a specific collection constructor.

Response:
def _block_collection(iterator, dtype, bsize=-1):
    """Pack rdd with a specific collection constructor."""
    i = 0
    accumulated = []
    for a in iterator:
        if (bsize > 0) and (i >= bsize):
            yield _pack_accumulated(accumulated, dtype)
            accumulated = []
            i = 0
        accumulated.append(a)
        i += 1
    if i > 0:
        yield _pack_accumulated(accumulated, dtype)


Instruction:
Implement a function in Python 3 to
pack rdd of tuples as tuples of arrays or scipy. sparse matrices.

Response:
def _block_tuple(iterator, dtypes, bsize=-1):
    """Pack rdd of tuples as tuples of arrays or scipy.sparse matrices."""
    i = 0
    blocked_tuple = None
    for tuple_i in iterator:
        if blocked_tuple is None:
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))

        if (bsize > 0) and (i >= bsize):
            yield tuple(_pack_accumulated(x, dtype)
                        for x, dtype in zip(blocked_tuple, dtypes))
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))
            i = 0

        for x_j, x in zip(tuple_i, blocked_tuple):
            x.append(x_j)
        i += 1
    if i > 0:
        yield tuple(_pack_accumulated(x, dtype)
                    for x, dtype in zip(blocked_tuple, dtypes))


Instruction:
Can you write a function in Python 3 where it
blocks an RDD of data points into a single block.

Response:
def block(rdd, bsize=-1, dtype=None):
    """Block an RDD

    Parameters
    ----------

    rdd : RDD
        RDD of data points to block into either numpy arrays,
        scipy sparse matrices, or pandas data frames.
        Type of data point will be automatically inferred
        and blocked accordingly.

    bsize : int, optional, default None
        Size of each block (number of elements), if None all data points
        from each partition will be combined in a block.

    Returns
    -------

    rdd : ArrayRDD or TupleRDD or DictRDD
        The transformed rdd with added functionality
    """
    try:
        entry = rdd.first()
    except IndexError:
        # empty RDD: do not block
        return rdd

    # do different kinds of block depending on the type
    if isinstance(entry, dict):
        rdd = rdd.map(lambda x: list(x.values()))
        return DictRDD(rdd, list(entry.keys()), bsize, dtype)
    elif isinstance(entry, tuple):
        return DictRDD(rdd, bsize=bsize, dtype=dtype)
    elif sp.issparse(entry):
        return SparseRDD(rdd, bsize)
    elif isinstance(entry, np.ndarray):
        return ArrayRDD(rdd, bsize)
    else:
        return BlockRDD(rdd, bsize, dtype)


Instruction:
Write a Python 3 function for
executing the blocking process on the given rdd.

Response:
def _block(self, rdd, bsize, dtype):
        """Execute the blocking process on the given rdd.

        Parameters
        ----------
        rdd : pyspark.rdd.RDD
            Distributed data to block
        bsize : int or None
            The desired size of the blocks

        Returns
        -------
        rdd : pyspark.rdd.RDD
            Blocked rdd.
        """
        return rdd.mapPartitions(
            lambda x: _block_collection(x, dtype, bsize))


Instruction:
Implement a function in Python 3 to
return the shape of the data.

Response:
def shape(self):
        """Returns the shape of the data."""
        # TODO cache
        first = self.first().shape
        shape = self._rdd.map(lambda x: x.shape[0]).sum()
        return (shape,) + first[1:]


Instruction:
Can you generate the documentation for the following Python 3 function
def toarray(self):
        rdd = self._rdd.map(lambda x: x.toarray())
        return np.concatenate(rdd.collect())

Response: Returns the data as numpy. array from each partition.


Instruction:
How would you explain what the following Python 3 function does
def _block(self, rdd, bsize, dtype):
        return rdd.mapPartitions(lambda x: _block_tuple(x, dtype, bsize))

Response: Execute the blocking process on the given rdd.


Instruction:
Can you tell what is the following Python 3 function doing
def transform(self, fn, column=None, dtype=None):
        dtypes = self.dtype
        if column is None:
            indices = list(range(len(self.columns)))
        else:
            if not type(column) in (list, tuple):
                column = [column]
            indices = [self.columns.index(c) for c in column]

        if dtype is not None:
            if not type(dtype) in (list, tuple):
                dtype = [dtype]
            dtypes = [dtype[indices.index(i)] if i in indices else t
                      for i, t in enumerate(self.dtype)]

        def mapper(values):
            result = fn(*[values[i] for i in indices])

            if len(indices) == 1:
                result = (result,)
            elif not isinstance(result, (tuple, list)):
                raise ValueError("Transformer function must return an"
                                 " iterable!")
            elif len(result) != len(indices):
                raise ValueError("Transformer result's length must be"
                                 " equal to the given columns length!")

            return tuple(result[indices.index(i)] if i in indices else v
                         for i, v in enumerate(values))

        return DictRDD(self._rdd.map(mapper),
                       columns=self.columns, dtype=dtypes,
                       bsize=self.bsize, noblock=True)

Response: Execute a transformation on a column or columns. Returns the modified DictRDD with transformed column names.


Instruction:
Can you write a function in Python 3 where it
returns a positive value if there are no permissions for a bit of the perm. Otherwise it returns a negative value.

Response:
def bitperm(s, perm, pos):
    """Returns zero if there are no permissions for a bit of the perm. of a file. Otherwise it returns a positive value

    :param os.stat_result s: os.stat(file) object
    :param str perm: R (Read) or W (Write) or X (eXecute)
    :param str pos: USR (USeR) or GRP (GRouP) or OTH (OTHer)
    :return: mask value
    :rtype: int
    """
    perm = perm.upper()
    pos = pos.upper()
    assert perm in ['R', 'W', 'X']
    assert pos in ['USR', 'GRP', 'OTH']
    return s.st_mode & getattr(stat, 'S_I{}{}'.format(perm, pos))