Instruction:
Explain what the following Python 3 code does
def is_char_device(self):
        try:
            return S_ISCHR(self.stat().st_mode)
        except OSError as e:
            if e.errno != ENOENT:
                raise
            # Path doesn't exist or is a broken symlink
            # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
            return False

Response: Return True if this path is a character device.


Instruction:
How would you implement a function in Python 3 that
returns a GridCoordinates object for the given upper left corner and lower right corner.

Response:
def grid_coords_from_corners(upper_left_corner, lower_right_corner, size):
    ''' Points are the outer edges of the UL and LR pixels. Size is rows, columns.
    GC projection type is taken from Points. '''
    assert upper_left_corner.wkt == lower_right_corner.wkt
    geotransform = np.array([upper_left_corner.lon, -(upper_left_corner.lon - lower_right_corner.lon) / float(size[1]), 0,
                            upper_left_corner.lat, 0, -(upper_left_corner.lat - lower_right_corner.lat) / float(size[0])])
    return GridCoordinates(geotransform=geotransform,
                               wkt=upper_left_corner.wkt,
                               y_size=size[0],
                               x_size=size[1])


Instruction:
How would you code a function in Python 3 to
return True if the GC s overlap.

Response:
def intersects(self, other_grid_coordinates):
        """ returns True if the GC's overlap. """
        ogc = other_grid_coordinates  # alias
        # for explanation: http://stackoverflow.com/questions/306316/determine-if-two-rectangles-overlap-each-other
        # Note the flipped y-coord in this coord system.
        ax1, ay1, ax2, ay2 = self.ULC.lon, self.ULC.lat, self.LRC.lon, self.LRC.lat
        bx1, by1, bx2, by2 = ogc.ULC.lon, ogc.ULC.lat, ogc.LRC.lon, ogc.LRC.lat
        if ((ax1 <= bx2) and (ax2 >= bx1) and (ay1 >= by2) and (ay2 <= by1)):
            return True
        else:
            return False


Instruction:
Explain what the following Python 3 code does
def unique_str(self):

        unique_str = "_".join(["%.3f" % f for f in self.geotransform] +
                              ["%d" % d for d in self.x_size, self.y_size]
                              )
        if self.date is not None:
            unique_str += '_' + str(self.date)
        if self.time is not None:
            unique_str += '_' + str(self.time)
        return unique_str.replace(':', '_')

Response: A string that uniquely represents this object.


Instruction:
How would you implement a function in Python 3 that
gets the x axis of the image.

Response:
def _get_x_axis(self):
        """See http://www.gdal.org/gdal_datamodel.html for details."""
        # 0,0 is top/left top top/left pixel. Actual x/y coord of that pixel are (.5,.5).
        x_centers = np.linspace(.5, self.x_size - .5, self.x_size)
        y_centers = x_centers * 0
        return (self.geotransform[0]
                + self.geotransform[1] * x_centers
                + self.geotransform[2] * y_centers)


Instruction:
Here you have a function in Python 3, explain what it does
def _get_y_axis(self):
        # 0,0 is top/left top top/left pixel. Actual x/y coord of that pixel are (.5,.5).
        y_centers = np.linspace(.5, self.y_size - .5, self.y_size)
        x_centers = y_centers * 0
        return (self.geotransform[3]
                + self.geotransform[4] * x_centers
                + self.geotransform[5] * y_centers)

Response: Get the y axis of the logarithmic data.


Instruction:
Write a Python 3 function for
converting pixel coordinates to projection coordinates.

Response:
def raster_to_projection_coords(self, pixel_x, pixel_y):
        """ Use pixel centers when appropriate.
        See documentation for the GDAL function GetGeoTransform for details. """
        h_px_py = np.array([1, pixel_x, pixel_y])
        gt = np.array([[1, 0, 0], self.geotransform[0:3], self.geotransform[3:6]])
        arr = np.inner(gt, h_px_py)
        return arr[2], arr[1]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def projection_to_raster_coords(self, lat, lon):
        r_px_py = np.array([1, lon, lat])
        tg = inv(np.array([[1, 0, 0], self.geotransform[0:3], self.geotransform[3:6]]))
        return np.inner(tg, r_px_py)[1:]

Response:
Returns pixel centers.
        See documentation for the GDAL function GetGeoTransform for details.


Instruction:
Create a Python 3 function to
replace masked - out elements in an array using an iterative image inpainting algorithm.

Response:
def inpaint(self):
        """ Replace masked-out elements in an array using an iterative image inpainting algorithm. """

        import inpaint
        filled = inpaint.replace_nans(np.ma.filled(self.raster_data, np.NAN).astype(np.float32), 3, 0.01, 2)
        self.raster_data = np.ma.masked_invalid(filled)


Instruction:
Given the following Python 3 function, write the documentation
def interp_value(self, lat, lon, indexed=False):
        (px, py) = self.grid_coordinates.projection_to_raster_coords(lat, lon)
        if indexed:
            return self.raster_data[round(py), round(px)]
        else:
#             from scipy.interpolate import interp2d
#             f_interp = interp2d(self.grid_coordinates.x_axis, self.grid_coordinates.y_axis, self.raster_data, bounds_error=True)
#             return f_interp(lon, lat)[0]
            from scipy.ndimage import map_coordinates
            ret = map_coordinates(self.raster_data, [[py], [px]], order=1)  # linear interp
            return ret[0]

Response: Interpolate the pixel value in the raster data.


Instruction:
Given the following Python 3 function, write the documentation
def get_connected_client(self):
        if self.__sem is not None:
            yield self.__sem.acquire()
        client = None
        newly_created, client = self._get_client_from_pool_or_make_it()
        if newly_created:
            res = yield client.connect()
            if not res:
                LOG.warning("can't connect to %s", client.title)
                raise tornado.gen.Return(
                    ClientError("can't connect to %s" % client.title))
        raise tornado.gen.Return(client)

Response: Gets a connected Client object.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_client_nowait(self):
        if self.__sem is not None:
            if self.__sem._value == 0:
                return None
            self.__sem.acquire()
        _, client = self._get_client_from_pool_or_make_it()
        return client

Response: Gets a Client object that is not yet available.


Instruction:
Can you create a Python 3 function that
returns a ContextManagerFuture that is yielded in a with statement.

Response:
def connected_client(self):
        """Returns a ContextManagerFuture to be yielded in a with statement.

        Returns:
            A ContextManagerFuture object.

        Examples:
            >>> with (yield pool.connected_client()) as client:
                    # client is a connected tornadis.Client instance
                    # it will be automatically released to the pool thanks to
                    # the "with" keyword
                    reply = yield client.call("PING")
        """
        future = self.get_connected_client()
        cb = functools.partial(self._connected_client_release_cb, future)
        return ContextManagerFuture(future, cb)


Instruction:
Make a summary of the following Python 3 code
def release_client(self, client):
        if isinstance(client, Client):
            if not self._is_expired_client(client):
                LOG.debug('Client is not expired. Adding back to pool')
                self.__pool.append(client)
            elif client.is_connected():
                LOG.debug('Client is expired and connected. Disconnecting')
                client.disconnect()
        if self.__sem is not None:
            self.__sem.release()

Response: Releases a client object to the pool.


Instruction:
Can you implement a function in Python 3 that
disconnects all pooled client objects.

Response:
def destroy(self):
        """Disconnects all pooled client objects."""
        while True:
            try:
                client = self.__pool.popleft()
                if isinstance(client, Client):
                    client.disconnect()
            except IndexError:
                break


Instruction:
Can you generate a brief explanation for the following Python 3 code
def preconnect(self, size=-1):
        if size == -1 and self.max_size == -1:
            raise ClientError("size=-1 not allowed with pool max_size=-1")
        limit = min(size, self.max_size) if size != -1 else self.max_size
        clients = yield [self.get_connected_client() for _ in range(0, limit)]
        for client in clients:
            self.release_client(client)

Response: Connects some or all redis clients inside the pool.


Instruction:
Make a summary of the following Python 3 code
def setup_path(invoke_minversion=None):
    # print("INVOKE.tasks: setup_path")
    if not os.path.isdir(TASKS_VENDOR_DIR):
        print("SKIP: TASKS_VENDOR_DIR=%s is missing" % TASKS_VENDOR_DIR)
        return
    elif os.path.abspath(TASKS_VENDOR_DIR) in sys.path:
        # -- SETUP ALREADY DONE:
        # return
        pass

    use_vendor_bundles = os.environ.get("INVOKE_TASKS_USE_VENDOR_BUNDLES", "no")
    if need_vendor_bundles(invoke_minversion):
        use_vendor_bundles = "yes"

    if use_vendor_bundles == "yes":
        syspath_insert(0, os.path.abspath(TASKS_VENDOR_DIR))
        if setup_path_for_bundle(INVOKE_BUNDLE, pos=1):
            import invoke
            bundle_path = os.path.relpath(INVOKE_BUNDLE, os.getcwd())
            print("USING: %s (version: %s)" % (bundle_path, invoke.__version__))
    else:
        # -- BEST-EFFORT: May rescue something
        syspath_append(os.path.abspath(TASKS_VENDOR_DIR))
        setup_path_for_bundle(INVOKE_BUNDLE, pos=len(sys.path))

    if DEBUG_SYSPATH:
        for index, p in enumerate(sys.path):
            print("  %d.  %s" % (index, p))

Response: Setup python search and add TASKS_VENDOR_DIR if available.


Instruction:
Can you tell what is the following Python 3 function doing
def require_invoke_minversion(min_version, verbose=False):
    # -- REQUIRES: sys.path is setup and contains invoke
    try:
        import invoke
        invoke_version = invoke.__version__
    except ImportError:
        invoke_version = "__NOT_INSTALLED"

    if invoke_version < min_version:
        message = "REQUIRE: invoke.version >= %s (but was: %s)" % \
                  (min_version, invoke_version)
        message += "\nUSE: pip install invoke>=%s" % min_version
        raise VersionRequirementError(message)

    INVOKE_VERSION = os.environ.get("INVOKE_VERSION", None)
    if verbose and not INVOKE_VERSION:
        os.environ["INVOKE_VERSION"] = invoke_version
        print("USING: invoke.version=%s" % invoke_version)

Response: Ensures that the current version of invoke is less than min_version.


Instruction:
Here you have a function in Python 3, explain what it does
def matches_section(section_name):
    section_names = section_name
    if isinstance(section_name, six.string_types):
        section_names = [section_name]
    elif not isinstance(section_name, (list, tuple)):
        raise ValueError("%r (expected: string, strings)" % section_name)

    def decorator(cls):
        class_section_names = getattr(cls, "section_names", None)
        if class_section_names is None:
            cls.section_names = list(section_names)
        else:
            # -- BETTER SUPPORT: For multiple decorators
            #   @matches_section("foo")
            #   @matches_section("bar.*")
            #   class Example(SectionSchema):
            #       pass
            #   assert Example.section_names == ["foo", "bar.*"]
            approved = [name for name in section_names
                        if name not in cls.section_names]
            cls.section_names = approved + cls.section_names
        return cls
    return decorator

Response: Decorator for SectionSchema classes to define the mapping between a config section schema class and one or more config sections with matching name.


Instruction:
Make a summary of the following Python 3 code
def assign_param_names(cls=None, param_class=None):
    if param_class is None:
        param_class = Param

    def decorate_class(cls):
        for name, value in select_params_from_section_schema(cls, param_class,
                                                             deep=True):
            # -- ANNOTATE PARAM: By assigning its name
            if not value.name:
                value.name = name
        return cls

    # -- DECORATOR LOGIC:
    if cls is None:
        # -- CASE: @assign_param_names
        # -- CASE: @assign_param_names(...)
        return decorate_class
    else:
        # -- CASE: @assign_param_names class X: ...
        # -- CASE: assign_param_names(my_class)
        # -- CASE: my_class = assign_param_names(my_class)
        return decorate_class(cls)

Response: Class decorator to assign parameter names to instances of Param.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def select_params_from_section_schema(section_schema, param_class=Param,
                                      deep=False):
    """Selects the parameters of a config section schema.

    :param section_schema:  Configuration file section schema to use.
    :return: Generator of params
    """
    # pylint: disable=invalid-name
    for name, value in inspect.getmembers(section_schema):
        if name.startswith("__") or value is None:
            continue    # pragma: no cover
        elif inspect.isclass(value) and deep:
            # -- CASE: class => SELF-CALL (recursively).
            # pylint: disable= bad-continuation
            cls = value
            for name, value in select_params_from_section_schema(cls,
                                            param_class=param_class, deep=True):
                yield (name, value)
        elif isinstance(value, param_class):
            yield (name, value)

Response: Selects the parameters of a config file section schema.


Instruction:
Make a summary of the following Python 3 code
def parse_config_section(config_section, section_schema):
    storage = {}
    for name, param in select_params_from_section_schema(section_schema):
        value = config_section.get(name, None)
        if value is None:
            if param.default is None:
                continue
            value = param.default
        else:
            value = param.parse(value)
        # -- DIAGNOSTICS:
        # print("  %s = %s" % (name, repr(value)))
        storage[name] = value
    return storage

Response: Parse a config file section by using its schema and description.


Instruction:
Create a Python 3 function to
generate all available configuration file names.

Response:
def generate_configfile_names(config_files, config_searchpath=None):
    """Generates all configuration file name combinations to read.

    .. sourcecode::

        # -- ALGORITHM:
        #    First basenames/directories are prefered and override other files.
        for config_path in reversed(config_searchpath):
            for config_basename in reversed(config_files):
                config_fname = os.path.join(config_path, config_basename)
                if os.path.isfile(config_fname):
                    yield config_fname

    :param config_files:        List of config file basenames.
    :param config_searchpath:   List of directories to look for config files.
    :return: List of available configuration file names (as generator)
    """
    if config_searchpath is None:
        config_searchpath = ["."]

    for config_path in reversed(config_searchpath):
        for config_basename in reversed(config_files):
            config_fname = os.path.join(config_path, config_basename)
            if os.path.isfile(config_fname):
                # MAYBE: yield os.path.normpath(config_fname)
                yield config_fname


Instruction:
Make a summary of the following Python 3 code
def select_config_sections(configfile_sections, desired_section_patterns):
    for section_name in configfile_sections:
        for desired_section_pattern in desired_section_patterns:
            if fnmatch(section_name, desired_section_pattern):
                yield section_name

Response:
Select a subset of the sections in a configuration file by using
    a list of section names of list of section name patters


Instruction:
Given the following Python 3 function, write the documentation
def matches_section(cls, section_name, supported_section_names=None):
        if supported_section_names is None:
            supported_section_names = getattr(cls, "section_names", None)

        # pylint: disable=invalid-name
        for supported_section_name_or_pattern in supported_section_names:
            if fnmatch(section_name, supported_section_name_or_pattern):
                return True
        # -- OTHERWISE:
        return False

Response: Indicates if this schema can be applied to a config section.


Instruction:
Can you create a Python 3 function that
derives support config section names from config section schemas.

Response:
def collect_config_sections_from_schemas(cls, config_section_schemas=None):
        # pylint: disable=invalid-name
        """Derive support config section names from config section schemas.
        If no :param:`config_section_schemas` are provided, the schemas from
        this class are used (normally defined in the DerivedClass).

        :param config_section_schemas:  List of config section schema classes.
        :return: List of config section names or name patterns (as string).
        """
        if config_section_schemas is None:
            config_section_schemas = cls.config_section_schemas

        collected = []
        for schema in config_section_schemas:
            collected.extend(schema.section_names)
            # -- MAYBE BETTER:
            # for name in schema.section_names:
            #    if name not in collected:
            #        collected.append(name)
        return collected


Instruction:
Implement a Python 3 function for
processing the config section and store the extracted data in the param storage.

Response:
def process_config_section(cls, config_section, storage):
        """Process the config section and store the extracted data in
        the param:`storage` (as outgoing param).
        """
        # -- CONCEPT:
        # if not storage:
        #     # -- INIT DATA: With default parts.
        #     storage.update(dict(_PERSONS={}))

        schema = cls.select_config_schema_for(config_section.name)
        if not schema:
            message = "No schema found for: section=%s"
            raise LookupError(message % config_section.name)

        # -- PARSE AND STORE CONFIG SECTION:
        section_storage = cls.select_storage_for(config_section.name, storage)
        section_data = parse_config_section(config_section, schema)
        section_storage.update(section_data)


Instruction:
Can you tell what is the following Python 3 function doing
def select_config_schema_for(cls, section_name):
        # pylint: disable=cell-var-from-loop, redefined-outer-name
        for section_schema in cls.config_section_schemas:
            schema_matches = getattr(section_schema, "matches_section", None)
            if schema_matches is None:
                # -- OTHER SCHEMA CLASS: Reuse SectionSchema functionality.
                schema_matches = lambda name: SectionSchema.matches_section(
                    name, section_schema.section_names)

            if schema_matches(section_name):
                return section_schema
        return None

Response: Select the config schema that matches the config section.


Instruction:
Write a Python 3 function for
selecting the data storage for a config section within the given storage.

Response:
def select_storage_for(cls, section_name, storage):
        """Selects the data storage for a config section within the
        :param:`storage`. The primary config section is normally merged into
        the :param:`storage`.

        :param section_name:    Config section (name) to process.
        :param storage:         Data storage to use.
        :return: :param:`storage` or a part of it (as section storage).
        """
        section_storage = storage
        storage_name = cls.get_storage_name_for(section_name)
        if storage_name:
            section_storage = storage.get(storage_name, None)
            if section_storage is None:
                section_storage = storage[storage_name] = dict()
        return section_storage


Instruction:
Write a Python 3 script for
cleaning up all the files and directories in the current order.

Response:
def clean_all(ctx, dry_run=False):
    """Clean up everything, even the precious stuff.
    NOTE: clean task is executed first.
    """
    cleanup_dirs(ctx.clean_all.directories or [], dry_run=dry_run)
    cleanup_dirs(ctx.clean_all.extra_directories or [], dry_run=dry_run)
    cleanup_files(ctx.clean_all.files or [], dry_run=dry_run)
    cleanup_files(ctx.clean_all.extra_files or [], dry_run=dry_run)
    execute_cleanup_tasks(ctx, cleanup_all_tasks, dry_run=dry_run)
    clean(ctx, dry_run=dry_run)


Instruction:
Write a Python 3 function that can
clean python related files and directories.

Response:
def clean_python(ctx, dry_run=False):
    """Cleanup python related files/dirs: *.pyc, *.pyo, ..."""
    # MAYBE NOT: "**/__pycache__"
    cleanup_dirs(["build", "dist", "*.egg-info", "**/__pycache__"],
                 dry_run=dry_run)
    if not dry_run:
        ctx.run("py.cleanup")
    cleanup_files(["**/*.pyc", "**/*.pyo", "**/*$py.class"], dry_run=dry_run)


Instruction:
Create a Python 3 function for
removing files or files selected by patterns.

Response:
def cleanup_files(patterns, dry_run=False, workdir="."):
    """Remove files or files selected by file patterns.
    Skips removal if file does not exist.

    :param patterns:    File patterns, like "**/*.pyc" (as list).
    :param dry_run:     Dry-run mode indicator (as bool).
    :param workdir:     Current work directory (default=".")
    """
    current_dir = Path(workdir)
    python_basedir = Path(Path(sys.executable).dirname()).joinpath("..").abspath()
    error_message = None
    error_count = 0
    for file_pattern in patterns:
        for file_ in path_glob(file_pattern, current_dir):
            if file_.abspath().startswith(python_basedir):
                # -- PROTECT CURRENTLY USED VIRTUAL ENVIRONMENT:
                continue

            if dry_run:
                print("REMOVE: %s (dry-run)" % file_)
            else:
                print("REMOVE: %s" % file_)
                try:
                    file_.remove_p()
                except os.error as e:
                    message = "%s: %s" % (e.__class__.__name__, e)
                    print(message + " basedir: "+ python_basedir)
                    error_count += 1
                    if not error_message:
                        error_message = message
    if False and error_message:
        class CleanupError(RuntimeError): pass
        raise CleanupError(error_message)


Instruction:
Explain what the following Python 3 code does
def path_glob(pattern, current_dir=None):
    if not current_dir:
        current_dir = pathlib.Path.cwd()
    elif not isinstance(current_dir, pathlib.Path):
        # -- CASE: string, path.Path (string-like)
        current_dir = pathlib.Path(str(current_dir))

    for p in current_dir.glob(pattern):
        yield Path(str(p))

Response: Use pathlib for ant - like patterns like. py


Instruction:
Can you generate a brief explanation for the following Python 3 code
def stack_call(self, *args):
        self.pipelined_args.append(args)
        self.number_of_stacked_calls = self.number_of_stacked_calls + 1

Response: Stack a redis command inside the object.


Instruction:
Given the following Python 3 function, write the documentation
def connect(self):
        if self.is_connected() or self.is_connecting():
            raise tornado.gen.Return(True)
        if self.unix_domain_socket is None:
            self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            if self.tcp_nodelay:
                self.__socket.setsockopt(socket.IPPROTO_TCP,
                                         socket.TCP_NODELAY, 1)
        else:
            if not os.path.exists(self.unix_domain_socket):
                LOG.warning("can't connect to %s, file does not exist",
                            self.unix_domain_socket)
                raise tornado.gen.Return(False)
            self.__socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        self.__socket.setblocking(0)
        self.__periodic_callback.start()
        try:
            LOG.debug("connecting to %s...", self._redis_server())
            self._state.set_connecting()
            if self.unix_domain_socket is None:
                self.__socket.connect((self.host, self.port))
            else:
                self.__socket.connect(self.unix_domain_socket)
        except socket.error as e:
            if (errno_from_exception(e) not in _ERRNO_INPROGRESS and
                    errno_from_exception(e) not in _ERRNO_WOULDBLOCK):
                self.disconnect()
                LOG.warning("can't connect to %s", self._redis_server())
                raise tornado.gen.Return(False)
            self.__socket_fileno = self.__socket.fileno()
            self._register_or_update_event_handler()
            yield self._state.get_changed_state_future()
            if not self.is_connected():
                LOG.warning("can't connect to %s", self._redis_server())
                raise tornado.gen.Return(False)
        else:
            LOG.debug("connected to %s", self._redis_server())
            self.__socket_fileno = self.__socket.fileno()
            self._state.set_connected()
            self._register_or_update_event_handler()
        raise tornado.gen.Return(True)

Response: Connects the object to the redis server.


Instruction:
Can you generate the documentation for the following Python 3 function
def disconnect(self):
        if not self.is_connected() and not self.is_connecting():
            return
        LOG.debug("disconnecting from %s...", self._redis_server())
        self.__periodic_callback.stop()
        try:
            self._ioloop.remove_handler(self.__socket_fileno)
            self._listened_events = 0
        except Exception:
            pass
        self.__socket_fileno = -1
        try:
            self.__socket.close()
        except Exception:
            pass
        self._state.set_disconnected()
        self._close_callback()
        LOG.debug("disconnected from %s", self._redis_server())

Response: Disconnects the object from the Redis server.


Instruction:
Create a Python 3 function to
write some data to the internal connection.

Response:
def write(self, data):
        """Buffers some data to be sent to the host:port in a non blocking way.

        So the data is always buffered and not sent on the socket in a
        synchronous way.

        You can give a WriteBuffer as parameter. The internal Connection
        WriteBuffer will be extended with this one (without copying).

        Args:
            data (str or WriteBuffer): string (or WriteBuffer) to write to
                the host:port.
        """
        if isinstance(data, WriteBuffer):
            self._write_buffer.append(data)
        else:
            if len(data) > 0:
                self._write_buffer.append(data)
        if self.aggressive_write:
            self._handle_write()
        if self._write_buffer._total_length > 0:
            self._register_or_update_event_handler(write=True)


Instruction:
How would you explain what the following Python 3 function does
def surrogate_escape(error):
    chars = error.object[error.start:error.end]
    assert len(chars) == 1
    val = ord(chars)
    val += 0xdc00
    return __builtin__.unichr(val), error.end

Response: Simulate the Python 3 surrogateescape handler but for Python 2 only.


Instruction:
Explain what the following Python 3 code does
def _always_unicode(cls, path):
        if PY3 or isinstance(path, text_type):
            return path
        return path.decode(sys.getfilesystemencoding(), 'surrogateescape')

Response: Ensure the path as retrieved from a Python API is a proper Unicode string.


Instruction:
Here you have a function in Python 3, explain what it does
def namebase(self):
        base, ext = self.module.splitext(self.name)
        return base

Response: The same as name but with one file extension stripped off.


Instruction:
Here you have a function in Python 3, explain what it does
def dirs(self, pattern=None):
        return [p for p in self.listdir(pattern) if p.isdir()]

Response: Returns a list of all directories in this directory.


Instruction:
Given the following Python 3 function, write the documentation
def files(self, pattern=None):

        return [p for p in self.listdir(pattern) if p.isfile()]

Response: D. files - > List of the files in this directory.


Instruction:
Make a summary of the following Python 3 code
def walkdirs(self, pattern=None, errors='strict'):
        if errors not in ('strict', 'warn', 'ignore'):
            raise ValueError("invalid errors parameter")

        try:
            dirs = self.dirs()
        except Exception:
            if errors == 'ignore':
                return
            elif errors == 'warn':
                warnings.warn(
                    "Unable to list directory '%s': %s"
                    % (self, sys.exc_info()[1]),
                    TreeWalkWarning)
                return
            else:
                raise

        for child in dirs:
            if pattern is None or child.fnmatch(pattern):
                yield child
            for subsubdir in child.walkdirs(pattern, errors):
                yield subsubdir

Response: A generator function that iterates over all directories in the directory and returns only directories whose names match the pattern.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def walkfiles(self, pattern=None, errors='strict'):
        if errors not in ('strict', 'warn', 'ignore'):
            raise ValueError("invalid errors parameter")

        try:
            childList = self.listdir()
        except Exception:
            if errors == 'ignore':
                return
            elif errors == 'warn':
                warnings.warn(
                    "Unable to list directory '%s': %s"
                    % (self, sys.exc_info()[1]),
                    TreeWalkWarning)
                return
            else:
                raise

        for child in childList:
            try:
                isfile = child.isfile()
                isdir = not isfile and child.isdir()
            except:
                if errors == 'ignore':
                    continue
                elif errors == 'warn':
                    warnings.warn(
                        "Unable to access '%s': %s"
                        % (self, sys.exc_info()[1]),
                        TreeWalkWarning)
                    continue
                else:
                    raise

            if isfile:
                if pattern is None or child.fnmatch(pattern):
                    yield child
            elif isdir:
                for f in child.walkfiles(pattern, errors):
                    yield f

Response: A generator function that returns all files in the directory and directories that match the pattern.


Instruction:
Can you tell what is the following Python 3 function doing
def open(self, *args, **kwargs):
        with io_error_compat():
            return io.open(self, *args, **kwargs)

Response: Open this file and return a corresponding file object.


Instruction:
Can you generate the documentation for the following Python 3 function
def write_lines(self, lines, encoding=None, errors='strict',
                    linesep=os.linesep, append=False):
        r""" Write the given lines of text to this file.

        By default this overwrites any existing file at this path.

        This puts a platform-specific newline sequence on every line.
        See `linesep` below.

            `lines` - A list of strings.

            `encoding` - A Unicode encoding to use.  This applies only if
                `lines` contains any Unicode strings.

            `errors` - How to handle errors in Unicode encoding.  This
                also applies only to Unicode strings.

            linesep - The desired line-ending.  This line-ending is
                applied to every line.  If a line already has any
                standard line ending (``'\r'``, ``'\n'``, ``'\r\n'``,
                ``u'\x85'``, ``u'\r\x85'``, ``u'\u2028'``), that will
                be stripped off and this will be used instead.  The
                default is os.linesep, which is platform-dependent
                (``'\r\n'`` on Windows, ``'\n'`` on Unix, etc.).
                Specify ``None`` to write the lines as-is, like
                :meth:`file.writelines`.

        Use the keyword argument ``append=True`` to append lines to the
        file.  The default is to overwrite the file.

        .. warning ::

            When you use this with Unicode data, if the encoding of the
            existing data in the file is different from the encoding
            you specify with the `encoding=` parameter, the result is
            mixed-encoding data, which can really confuse someone trying
            to read the file later.
        """
        with self.open('ab' if append else 'wb') as f:
            for l in lines:
                isUnicode = isinstance(l, text_type)
                if linesep is not None:
                    pattern = U_NL_END if isUnicode else NL_END
                    l = pattern.sub('', l) + linesep
                if isUnicode:
                    l = l.encode(encoding or sys.getdefaultencoding(), errors)
                f.write(l)

Response: r Write the given lines of text to the file.


Instruction:
Write a Python 3 script for
liking mkdir but does not raise an exception.

Response:
def mkdir_p(self, mode=0o777):
        """ Like :meth:`mkdir`, but does not raise an exception if the
        directory already exists. """
        try:
            self.mkdir(mode)
        except OSError:
            _, e, _ = sys.exc_info()
            if e.errno != errno.EEXIST:
                raise
        return self


Instruction:
Make a summary of the following Python 3 code
def makedirs_p(self, mode=0o777):
        try:
            self.makedirs(mode)
        except OSError:
            _, e, _ = sys.exc_info()
            if e.errno != errno.EEXIST:
                raise
        return self

Response: Like makedirs but does not raise an exception.


Instruction:
Explain what the following Python 3 code does
def rmdir_p(self):
        try:
            self.rmdir()
        except OSError:
            _, e, _ = sys.exc_info()
            if e.errno != errno.ENOTEMPTY and e.errno != errno.EEXIST:
                raise
        return self

Response: Like rmdir but does not raise an exception if the directory is not empty.


Instruction:
Can you generate the documentation for the following Python 3 function
def removedirs_p(self):
        try:
            self.removedirs()
        except OSError:
            _, e, _ = sys.exc_info()
            if e.errno != errno.ENOTEMPTY and e.errno != errno.EEXIST:
                raise
        return self

Response:
Like removedirs but does not raise an exception if the the
            directory is not empty.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def remove_p(self):
        try:
            self.unlink()
        except OSError:
            _, e, _ = sys.exc_info()
            if e.errno != errno.ENOENT:
                raise
        return self

Response:
Like remove but does not raise an exception if the the
            file does not exist.


Instruction:
How would you explain what the following Python 3 function does
def rmtree_p(self):
        try:
            self.rmtree()
        except OSError:
            _, e, _ = sys.exc_info()
            if e.errno != errno.ENOENT:
                raise
        return self

Response: Like rmtree but does not raise an exception if the directory does not exist.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def merge_tree(self, dst, symlinks=False, *args, **kwargs):
        update = kwargs.pop('update', False)
        with tempdir() as _temp_dir:
            # first copy the tree to a stage directory to support
            #  the parameters and behavior of copytree.
            stage = _temp_dir / str(hash(self))
            self.copytree(stage, symlinks, *args, **kwargs)
            # now copy everything from the stage directory using
            #  the semantics of dir_util.copy_tree
            dir_util.copy_tree(stage, dst, preserve_symlinks=symlinks,
                update=update)

Response:
Copy all contents of self to dst overwriting existing
        contents in dst.


Instruction:
Write a Python 3 script for
connecting the client object to redis.

Response:
def connect(self):
        """Connects the client object to redis.

        It's safe to use this method even if you are already connected.
        Note: this method is useless with autoconnect mode (default).

        Returns:
            a Future object with True as result if the connection was ok.
        """
        if self.is_connected():
            raise tornado.gen.Return(True)
        cb1 = self._read_callback
        cb2 = self._close_callback
        self.__callback_queue = collections.deque()
        self._reply_list = []
        self.__reader = hiredis.Reader(replyError=ClientError)
        kwargs = self.connection_kwargs
        self.__connection = Connection(cb1, cb2, **kwargs)
        connection_status = yield self.__connection.connect()
        if connection_status is not True:
            # nothing left to do here, return
            raise tornado.gen.Return(False)
        if self.password is not None:
            authentication_status = yield self._call('AUTH', self.password)
            if authentication_status != b'OK':
                # incorrect password, return back the result
                LOG.warning("impossible to connect: bad password")
                self.__connection.disconnect()
                raise tornado.gen.Return(False)
        if self.db != 0:
            db_status = yield self._call('SELECT', self.db)
            if db_status != b'OK':
                LOG.warning("can't select db %s", self.db)
                raise tornado.gen.Return(False)
        raise tornado.gen.Return(True)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _close_callback(self):
        while True:
            try:
                callback = self.__callback_queue.popleft()
                callback(ConnectionError("closed connection"))
            except IndexError:
                break
        if self.subscribed:
            # pubsub clients
            self._reply_list.append(ConnectionError("closed connection"))
            self._condition.notify_all()

Response: Callback called when redis closed the connection.


Instruction:
Can you tell what is the following Python 3 function doing
def _read_callback(self, data=None):
        try:
            if data is not None:
                self.__reader.feed(data)
                while True:
                    reply = self.__reader.gets()
                    if reply is not False:
                        try:
                            callback = self.__callback_queue.popleft()
                            # normal client (1 reply = 1 callback)
                            callback(reply)
                        except IndexError:
                            # pubsub clients
                            self._reply_list.append(reply)
                            self._condition.notify_all()
                    else:
                        break
        except hiredis.ProtocolError:
            # something nasty occured (corrupt stream => no way to recover)
            LOG.warning("corrupted stream => disconnect")
            self.disconnect()

Response: Callback called when some data is read on the socket.


Instruction:
Can you generate the documentation for the following Python 3 function
def call(self, *args, **kwargs):
        if not self.is_connected():
            if self.autoconnect:
                # We use this method only when we are not contected
                # to void performance penaly due to gen.coroutine decorator
                return self._call_with_autoconnect(*args, **kwargs)
            else:
                error = ConnectionError("you are not connected and "
                                        "autoconnect=False")
                return tornado.gen.maybe_future(error)
        return self._call(*args, **kwargs)

Response: Calls a redis command and returns a Future of the reply.


Instruction:
How would you code a function in Python 3 to
call a redis command and waits for the reply and call a callback.

Response:
def async_call(self, *args, **kwargs):
        """Calls a redis command, waits for the reply and call a callback.

        Following options are available (not part of the redis command itself):

        - callback
            Function called (with the result as argument) when the result
            is available. If not set, the reply is silently discarded. In
            case of errors, the callback is called with a
            TornadisException object as argument.

        Args:
            *args: full redis command as variable length argument list or
                a Pipeline object (as a single argument).
            **kwargs: options as keyword parameters.

        Examples:

            >>> def cb(result):
                    pass
            >>> client.async_call("HSET", "key", "field", "val", callback=cb)
        """
        def after_autoconnect_callback(future):
            if self.is_connected():
                self._call(*args, **kwargs)
            else:
                # FIXME
                pass

        if 'callback' not in kwargs:
            kwargs['callback'] = discard_reply_cb
        if not self.is_connected():
            if self.autoconnect:
                connect_future = self.connect()
                cb = after_autoconnect_callback
                self.__connection._ioloop.add_future(connect_future, cb)
            else:
                error = ConnectionError("you are not connected and "
                                        "autoconnect=False")
                kwargs['callback'](error)
        else:
            self._call(*args, **kwargs)


Instruction:
How would you implement a function in Python 3 that
formats arguments into the redis protocol

Response:
def format_args_in_redis_protocol(*args):
    """Formats arguments into redis protocol...

    This function makes and returns a string/buffer corresponding to
    given arguments formated with the redis protocol.

    integer, text, string or binary types are automatically converted
    (using utf8 if necessary).

    More informations about the protocol: http://redis.io/topics/protocol

    Args:
        *args: full redis command as variable length argument list

    Returns:
        binary string (arguments in redis protocol)

    Examples:
        >>> format_args_in_redis_protocol("HSET", "key", "field", "value")
        '*4\r\n$4\r\nHSET\r\n$3\r\nkey\r\n$5\r\nfield\r\n$5\r\nvalue\r\n'
    """
    buf = WriteBuffer()
    l = "*%d\r\n" % len(args)  # noqa: E741
    if six.PY2:
        buf.append(l)
    else:  # pragma: no cover
        buf.append(l.encode('utf-8'))
    for arg in args:
        if isinstance(arg, six.text_type):
            # it's a unicode string in Python2 or a standard (unicode)
            # string in Python3, let's encode it in utf-8 to get raw bytes
            arg = arg.encode('utf-8')
        elif isinstance(arg, six.string_types):
            # it's a basestring in Python2 => nothing to do
            pass
        elif isinstance(arg, six.binary_type):  # pragma: no cover
            # it's a raw bytes string in Python3 => nothing to do
            pass
        elif isinstance(arg, six.integer_types):
            tmp = "%d" % arg
            if six.PY2:
                arg = tmp
            else:  # pragma: no cover
                arg = tmp.encode('utf-8')
        elif isinstance(arg, WriteBuffer):
            # it's a WriteBuffer object => nothing to do
            pass
        else:
            raise Exception("don't know what to do with %s" % type(arg))
        l = "$%d\r\n" % len(arg)  # noqa: E741
        if six.PY2:
            buf.append(l)
        else:  # pragma: no cover
            buf.append(l.encode('utf-8'))
        buf.append(arg)
        buf.append(b"\r\n")
    return buf


Instruction:
Can you tell what is the following Python 3 function doing
def _done_callback(self, wrapped):
        if wrapped.exception():
            self.set_exception(wrapped.exception())
        else:
            self.set_result(wrapped.result())

Response: Internal done callback to set the result of the object.


Instruction:
Implement a function in Python 3 to
create the URL in the LyricWikia format

Response:
def create_url(artist, song):
    """Create the URL in the LyricWikia format"""
    return (__BASE_URL__ +
            '/wiki/{artist}:{song}'.format(artist=urlize(artist),
                                           song=urlize(song)))


Instruction:
Here you have a function in Python 3, explain what it does
def get_lyrics(artist, song, linesep='\n', timeout=None):
    return get_all_lyrics(artist, song, linesep, timeout)[0]

Response:
Retrieve the lyrics of the song and return the first one in case
    multiple versions are available.


Instruction:
Can you write a function in Python 3 where it
retrieves a list of all the lyrics versions of a song.

Response:
def get_all_lyrics(artist, song, linesep='\n', timeout=None):
    """Retrieve a list of all the lyrics versions of a song."""
    url = create_url(artist, song)
    response = _requests.get(url, timeout=timeout)
    soup = _BeautifulSoup(response.content, "html.parser")
    lyricboxes = soup.findAll('div', {'class': 'lyricbox'})

    if not lyricboxes:
        raise LyricsNotFound('Cannot download lyrics')

    for lyricbox in lyricboxes:
        for br in lyricbox.findAll('br'):
            br.replace_with(linesep)

    return [lyricbox.text.strip() for lyricbox in lyricboxes]


Instruction:
Write a Python 3 script for
opening an ARF file creating as necessary.

Response:
def open_file(name, mode=None, driver=None, libver=None, userblock_size=None, **kwargs):
    """Open an ARF file, creating as necessary.

    Use this instead of h5py.File to ensure that root-level attributes and group
    creation property lists are set correctly.

    """
    import sys
    import os
    from h5py import h5p
    from h5py._hl import files

    try:
        # If the byte string doesn't match the default
        # encoding, just pass it on as-is.  Note Unicode
        # objects can always be encoded.
        name = name.encode(sys.getfilesystemencoding())
    except (UnicodeError, LookupError):
        pass
    exists = os.path.exists(name)
    try:
        fcpl = h5p.create(h5p.FILE_CREATE)
        fcpl.set_link_creation_order(
            h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED)
    except AttributeError:
        # older version of h5py
        fp = files.File(name, mode=mode, driver=driver,
                        libver=libver, **kwargs)
    else:
        fapl = files.make_fapl(driver, libver, **kwargs)
        fp = files.File(files.make_fid(name, mode, userblock_size, fapl, fcpl))

    if not exists and fp.mode == 'r+':
        set_attributes(fp,
                       arf_library='python',
                       arf_library_version=__version__,
                       arf_version=spec_version)
    return fp


Instruction:
Write a Python 3 script to
create a new ARF entry under a group.

Response:
def create_entry(group, name, timestamp, **attributes):
    """Create a new ARF entry under group, setting required attributes.

    An entry is an abstract collection of data which all refer to the same time
    frame. Data can include physiological recordings, sound recordings, and
    derived data such as spike times and labels. See add_data() for information
    on how data are stored.

    name -- the name of the new entry. any valid python string.

    timestamp -- timestamp of entry (datetime object, or seconds since
               January 1, 1970). Can be an integer, a float, or a tuple
               of integers (seconds, microsceconds)

    Additional keyword arguments are set as attributes on created entry.

    Returns: newly created entry object

    """
    # create group using low-level interface to store creation order
    from h5py import h5p, h5g, _hl
    try:
        gcpl = h5p.create(h5p.GROUP_CREATE)
        gcpl.set_link_creation_order(
            h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED)
    except AttributeError:
        grp = group.create_group(name)
    else:
        name, lcpl = group._e(name, lcpl=True)
        grp = _hl.group.Group(h5g.create(group.id, name, lcpl=lcpl, gcpl=gcpl))
    set_uuid(grp, attributes.pop("uuid", None))
    set_attributes(grp,
                   timestamp=convert_timestamp(timestamp),
                   **attributes)
    return grp


Instruction:
Make a summary of the following Python 3 code
def create_dataset(group, name, data, units='', datatype=DataTypes.UNDEFINED,
                   chunks=True, maxshape=None, compression=None,
                   **attributes):
    """Create an ARF dataset under group, setting required attributes

    Required arguments:
    name --   the name of dataset in which to store the data
    data --   the data to store

    Data can be of the following types:

    * sampled data: an N-D numerical array of measurements
    * "simple" event data: a 1-D array of times
    * "complex" event data: a 1-D array of records, with field 'start' required

    Optional arguments:
    datatype --      a code defining the nature of the data in the channel
    units --         channel units (optional for sampled data, otherwise required)
    sampling_rate -- required for sampled data and event data with units=='samples'

    Arguments passed to h5py:
    maxshape --    make the node resizable up to this shape. Use None for axes that
                   need to be unlimited.
    chunks --      specify the chunk size. The optimal chunk size depends on the
                   intended use of the data. For single-channel sampled data the
                   auto-chunking (True) is probably best.
    compression -- compression strategy. Can be 'gzip', 'szip', 'lzf' or an integer
                   in range(10) specifying gzip(N).  Only gzip is really portable.

    Additional arguments are set as attributes on the created dataset

    Returns the created dataset
    """
    from numpy import asarray
    srate = attributes.get('sampling_rate', None)
    # check data validity before doing anything
    if not hasattr(data, 'dtype'):
        data = asarray(data)
        if data.dtype.kind in ('S', 'O', 'U'):
            raise ValueError(
                "data must be in array with numeric or compound type")
    if data.dtype.kind == 'V':
        if 'start' not in data.dtype.names:
            raise ValueError("complex event data requires 'start' field")
        if not isinstance(units, (list, tuple)):
            raise ValueError("complex event data requires sequence of units")
        if not len(units) == len(data.dtype.names):
            raise ValueError("number of units doesn't match number of fields")
    if units == '':
        if srate is None or not srate > 0:
            raise ValueError(
                "unitless data assumed time series and requires sampling_rate attribute")
    elif units == 'samples':
        if srate is None or not srate > 0:
            raise ValueError(
                "data with units of 'samples' requires sampling_rate attribute")
    # NB: can't really catch case where sampled data has units but doesn't
    # have sampling_rate attribute

    dset = group.create_dataset(
        name, data=data, maxshape=maxshape, chunks=chunks, compression=compression)
    set_attributes(dset, units=units, datatype=datatype, **attributes)
    return dset

Response: Create an ARF dataset under the specified group.


Instruction:
Can you generate the documentation for the following Python 3 function
def create_table(group, name, dtype, **attributes):
    dset = group.create_dataset(
        name, shape=(0,), dtype=dtype, maxshape=(None,))
    set_attributes(dset, **attributes)
    return dset

Response: Create a new array dataset under group with compound datatype and maxshape = None.


Instruction:
Explain what the following Python 3 code does
def append_data(dset, data):
    N = data.shape[0] if hasattr(data, 'shape') else 1
    if N == 0:
        return
    oldlen = dset.shape[0]
    newlen = oldlen + N
    dset.resize(newlen, axis=0)
    dset[oldlen:] = data

Response: Append data to dset along axis 0.


Instruction:
Implement a function in Python 3 to
check the ARF version attribute of file for compatibility.

Response:
def check_file_version(file):
    """Check the ARF version attribute of file for compatibility.

    Raises DeprecationWarning for backwards-incompatible files, FutureWarning
    for (potentially) forwards-incompatible files, and UserWarning for files
    that may not have been created by an ARF library.

    Returns the version for the file

    """
    from distutils.version import StrictVersion as Version
    try:
        ver = file.attrs.get('arf_version', None)
        if ver is None:
            ver = file.attrs['arf_library_version']
    except KeyError:
        raise UserWarning(
            "Unable to determine ARF version for {0.filename};"
            "created by another program?".format(file))
    try:
        # if the attribute is stored as a string, it's ascii-encoded
        ver = ver.decode("ascii")
    except (LookupError, AttributeError):
        pass
    # should be backwards compatible after 1.1
    file_version = Version(ver)
    if file_version < Version('1.1'):
        raise DeprecationWarning(
            "ARF library {} may have trouble reading file "
            "version {} (< 1.1)".format(version, file_version))
    elif file_version >= Version('3.0'):
        raise FutureWarning(
            "ARF library {} may be incompatible with file "
            "version {} (>= 3.0)".format(version, file_version))
    return file_version


Instruction:
Explain what the following Python 3 code does
def set_attributes(node, overwrite=True, **attributes):
    aset = node.attrs
    for k, v in attributes.items():
        if not overwrite and k in aset:
            pass
        elif v is None:
            if k in aset:
                del aset[k]
        else:
            aset[k] = v

Response: Set multiple attributes on a node.


Instruction:
Given the following Python 3 function, write the documentation
def keys_by_creation(group):
    from h5py import h5
    out = []
    try:
        group._id.links.iterate(
            out.append, idx_type=h5.INDEX_CRT_ORDER, order=h5.ITER_INC)
    except (AttributeError, RuntimeError):
        # pre 2.2 shim
        def f(name):
            if name.find(b'/', 1) == -1:
                out.append(name)
        group._id.links.visit(
            f, idx_type=h5.INDEX_CRT_ORDER, order=h5.ITER_INC)
    return map(group._d, out)

Response: Returns a sequence of links in group in order of creation.


Instruction:
Can you write a function in Python 3 where it
converts an ARF timestamp from an object to an ARF timestamp.

Response:
def convert_timestamp(obj):
    """Make an ARF timestamp from an object.

    Argument can be a datetime.datetime object, a time.struct_time, an integer,
    a float, or a tuple of integers. The returned value is a numpy array with
    the integer number of seconds since the Epoch and any additional
    microseconds.

    Note that because floating point values are approximate, the conversion
    between float and integer tuple may not be reversible.

    """
    import numbers
    from datetime import datetime
    from time import mktime, struct_time
    from numpy import zeros

    out = zeros(2, dtype='int64')
    if isinstance(obj, datetime):
        out[0] = mktime(obj.timetuple())
        out[1] = obj.microsecond
    elif isinstance(obj, struct_time):
        out[0] = mktime(obj)
    elif isinstance(obj, numbers.Integral):
        out[0] = obj
    elif isinstance(obj, numbers.Real):
        out[0] = obj
        out[1] = (obj - out[0]) * 1e6
    else:
        try:
            out[:2] = obj[:2]
        except:
            raise TypeError("unable to convert %s to timestamp" % obj)
    return out


Instruction:
Create a Python 3 function for
converting an ARF timestamp to a datetime. datetime object

Response:
def timestamp_to_datetime(timestamp):
    """Convert an ARF timestamp to a datetime.datetime object (naive local time)"""
    from datetime import datetime, timedelta
    obj = datetime.fromtimestamp(timestamp[0])
    return obj + timedelta(microseconds=int(timestamp[1]))


Instruction:
Make a summary of the following Python 3 code
def set_uuid(obj, uuid=None):
    from uuid import uuid4, UUID
    if uuid is None:
        uuid = uuid4()
    elif isinstance(uuid, bytes):
        if len(uuid) == 16:
            uuid = UUID(bytes=uuid)
        else:
            uuid = UUID(hex=uuid)

    if "uuid" in obj.attrs:
        del obj.attrs["uuid"]
    obj.attrs.create("uuid", str(uuid).encode('ascii'), dtype="|S36")

Response: Set the uuid attribute of an HDF5 object.


Instruction:
Can you tell what is the following Python 3 function doing
def get_uuid(obj):
    # TODO: deprecate null uuid ret val
    from uuid import UUID
    try:
        uuid = obj.attrs['uuid']
    except KeyError:
        return UUID(int=0)
    # convert to unicode for python 3
    try:
        uuid = uuid.decode('ascii')
    except (LookupError, AttributeError):
        pass
    return UUID(uuid)

Response: Return the uuid for obj or null uuid if none is set


Instruction:
How would you explain what the following Python 3 function does
def count_children(obj, type=None):
    if type is None:
        return len(obj)
    else:
        # there doesn't appear to be any hdf5 function for getting this
        # information without inspecting each child, which makes this somewhat
        # slow
        return sum(1 for x in obj if obj.get(x, getclass=True) is type)

Response: Return the number of children of obj optionally restricting by class


Instruction:
How would you implement a function in Python 3 that
generates a dict keyed by value

Response:
def _todict(cls):
        """ generate a dict keyed by value """
        return dict((getattr(cls, attr), attr) for attr in dir(cls) if not attr.startswith('_'))


Instruction:
How would you explain what the following Python 3 function does
def get_template(template_name,fields=None):
    '''get_template will return a template in the template folder,
    with some substitutions (eg, {'{{ graph | safe }}':"fill this in!"}
    '''
    template = None
    if not template_name.endswith('.html'):
        template_name = "%s.html" %(template_name)
    here = "%s/cli/app/templates" %(get_installdir())
    template_path = "%s/%s" %(here,template_name)
    if os.path.exists(template_path):
        template = ''.join(read_file(template_path))
    if fields is not None:
        for tag,sub in fields.items():
            template = template.replace(tag,sub)
    return template

Response: get_template will return a template in the template folder with some substitutions


Instruction:
Explain what the following Python 3 code does
def container_similarity_vector(container1=None,packages_set=None,by=None):
    '''container similarity_vector is similar to compare_packages, but intended
    to compare a container object (singularity image or singularity hub container)
    to a list of packages. If packages_set is not provided, the default used is
    'docker-os'. This can be changed to 'docker-library', or if the user wants a custom
    list, should define custom_set.
    :param container1: singularity image or singularity hub container.
    :param packages_set: a name of a package set, provided are docker-os and docker-library
    :by: metrics to compare by (files.txt and or folders.txt)
    '''

    if by == None:
        by = ['files.txt']

    if not isinstance(by,list):
        by = [by]
    if not isinstance(packages_set,list):
        packages_set = [packages_set]

    comparisons = dict()

    for b in by:
        bot.debug("Starting comparisons for %s" %b)
        df = pandas.DataFrame(columns=packages_set)
        for package2 in packages_set:
            sim = calculate_similarity(container1=container1,
                                       image_package2=package2,
                                       by=b)[b]

            name1 = os.path.basename(package2).replace('.img.zip','')
            bot.debug("container vs. %s: %s" %(name1,sim))
            df.loc["container",package2] = sim
        df.columns = [os.path.basename(x).replace('.img.zip','') for x in df.columns.tolist()]
        comparisons[b] = df
    return comparisons

Response:
container similarity_vector is similar to compare_packages but intended
    to compare a container object.


Instruction:
Make a summary of the following Python 3 code
def compare_containers(container1=None, container2=None):

    '''compare_containers will generate a data structure with common and unique files to
    two images. If environmental variable SINGULARITY_HUB is set, will use container
    database objects.
    :param container1: first container for comparison
    :param container2: second container for comparison if either not defined must include
    default compares just files
    '''

    # Get files and folders for each
    container1_guts = get_container_contents(split_delim="\n",
                                             container=container1)['all']
    container2_guts = get_container_contents(split_delim="\n",
                                             container=container2)['all']

    # Do the comparison for each metric
    return compare_lists(container1_guts, container2_guts)

Response:
This function will generate a data structure with common and unique files to
    two images.


Instruction:
Explain what the following Python 3 code does
def compare_lists(list1,list2):
    '''compare lists is the lowest level that drives compare_containers and
    compare_packages. It returns a comparison object (dict) with the unique,
    total, and intersecting things between two lists
    :param list1: the list for container1
    :param list2: the list for container2
    '''
    intersect = list(set(list1).intersection(list2))
    unique1 = list(set(list1).difference(list2))
    unique2 = list(set(list2).difference(list1))

    # Return data structure
    comparison = {"intersect":intersect,
                  "unique1": unique1,
                  "unique2": unique2,
                  "total1": len(list1),
                  "total2": len(list2)}
    return comparison

Response: compare lists is the lowest level that drives compare_containers and compare_packages. It returns a dict with the unique total and intersecting things between two lists


Instruction:
How would you implement a function in Python 3 that
packages a node into a container

Response:
def package_node(root=None, name=None):
    '''package node aims to package a (present working node) for a user into
    a container. This assumes that the node is a single partition.

    :param root: the root of the node to package, default is /
    :param name: the name for the image. If not specified, will use machine's

    psutil.disk_partitions()

    '''

    if name is None:
        name = platform.node()

    if root is None:
        root = "/"

    tmpdir = tempfile.mkdtemp()
    image = "%s/%s.tgz" %(tmpdir,name)

    print("Preparing to package root %s into %s" %(root,name))
    cmd = ["tar","--one-file-system","-czvSf", image, root,"--exclude",image]
    output = run_command(cmd)
    return image


Instruction:
Here you have a function in Python 3, explain what it does
def get_build_template(template_name,params=None,to_file=None):
    '''get_build template returns a string or file for a particular build template, which is
    intended to build a version of a Singularity image on a cloud resource.
    :param template_name: the name of the template to retrieve in build/scripts
    :param params: (if needed) a dictionary of parameters to substitute in the file
    :param to_file: if defined, will write to file. Default returns string.
    '''
    base = get_installdir()
    template_folder = "%s/build/scripts" %(base)
    template_file = "%s/%s" %(template_folder,template_name)
    if os.path.exists(template_file):
        bot.debug("Found template %s" %template_file)

        # Implement when needed - substitute params here
        # Will need to read in file instead of copying below
        # if params != None:

        if to_file is not None:
            shutil.copyfile(template_file,to_file)
            bot.debug("Template file saved to %s" %to_file)
            return to_file

        # If the user wants a string
        content = ''.join(read_file(template_file))
        return content

    else:
        bot.warning("Template %s not found." %template_file)

Response: get_build template returns a string or file for a particular build template


Instruction:
How would you explain what the following Python 3 function does
def sniff_extension(file_path,verbose=True):
    '''sniff_extension will attempt to determine the file type based on the extension,
    and return the proper mimetype
    :param file_path: the full path to the file to sniff
    :param verbose: print stuff out
    '''
    mime_types =    { "xls": 'application/vnd.ms-excel',
                      "xlsx": 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
                      "xml": 'text/xml',
                      "ods": 'application/vnd.oasis.opendocument.spreadsheet',
                      "csv": 'text/plain',
                      "tmpl": 'text/plain',
                      "pdf":  'application/pdf',
                      "php": 'application/x-httpd-php',
                      "jpg": 'image/jpeg',
                      "png": 'image/png',
                      "gif": 'image/gif',
                      "bmp": 'image/bmp',
                      "txt": 'text/plain',
                      "doc": 'application/msword',
                      "js": 'text/js',
                      "swf": 'application/x-shockwave-flash',
                      "mp3": 'audio/mpeg',
                      "zip": 'application/zip',
                      "simg": 'application/zip',
                      "rar": 'application/rar',
                      "tar": 'application/tar',
                      "arj": 'application/arj',
                      "cab": 'application/cab',
                      "html": 'text/html',
                      "htm": 'text/html',
                      "default": 'application/octet-stream',
                      "folder": 'application/vnd.google-apps.folder',
                      "img" : "application/octet-stream" }

    ext = os.path.basename(file_path).split('.')[-1]

    mime_type = mime_types.get(ext,None)

    if mime_type == None:
        mime_type = mime_types['txt']

    if verbose==True:
        bot.info("%s --> %s" %(file_path, mime_type))

    return mime_type

Response: sniff_extension will attempt to determine the file type based on the extension and return the proper mimetype


Instruction:
How would you explain what the following Python 3 function does
def zip_up(file_list,zip_name,output_folder=None):
    '''zip_up will zip up some list of files into a package (.zip)
    :param file_list: a list of files to include in the zip.
    :param output_folder: the output folder to create the zip in. If not
    :param zip_name: the name of the zipfile to return.
    specified, a temporary folder will be given.
    '''
    tmpdir = tempfile.mkdtemp()

    # Make a new archive
    output_zip = "%s/%s" %(tmpdir,zip_name)
    zf = zipfile.ZipFile(output_zip, "w", zipfile.ZIP_DEFLATED, allowZip64=True)

    # Write files to zip, depending on type
    for filename,content in file_list.items():

        bot.debug("Adding %s to package..." %filename)

        # If it's the files list, move files into the archive
        if filename.lower() == "files":
            if not isinstance(content,list):
                content = [content]
            for copyfile in content:
                zf.write(copyfile,os.path.basename(copyfile))
                os.remove(copyfile)

        else:

            output_file = "%s/%s" %(tmpdir, filename)

            # If it's a list, write to new file, and save
            if isinstance(content,list):
                write_file(output_file,"\n".join(content))

            # If it's a dict, save to json
            elif isinstance(content,dict):
                write_json(content,output_file)

            # If bytes, need to decode
            elif isinstance(content,bytes):
                write_file(output_file,content.decode('utf-8'))

            # String or other
            else:
                output_file = write_file(output_file,content)

            if os.path.exists(output_file):
                zf.write(output_file,filename)
                os.remove(output_file)

    # Close the zip file
    zf.close()

    if output_folder is not None:
        shutil.copyfile(output_zip,"%s/%s"%(output_folder,zip_name))
        shutil.rmtree(tmpdir)
        output_zip = "%s/%s"%(output_folder,zip_name)

    return output_zip

Response: This function will zip up some list of files into a single file in a folder.


Instruction:
Here you have a function in Python 3, explain what it does
def get_container_contents(container, split_delim=None):
    '''get_container_contents will return a list of folders and or files
    for a container. The environmental variable SINGULARITY_HUB being set
    means that container objects are referenced instead of packages
    :param container: the container to get content for
    :param gets: a list of file names to return, without parent folders
    :param split_delim: if defined, will split text by split delimiter
    '''

    # We will look for everything in guts, then return it
    guts = dict()

    SINGULARITY_HUB = os.environ.get('SINGULARITY_HUB',"False")

    # Visualization deployed local or elsewhere
    if SINGULARITY_HUB == "False":
        file_obj,tar = get_image_tar(container)
        guts = extract_guts(image_path=container, tar=tar)
        delete_image_tar(file_obj, tar)

    # Visualization deployed by singularity hub
    else:

        # user has provided a package, but not a container
        if container == None:
            guts = load_package(image_package,get=gets)

        # user has provided a container, but not a package
        else:
            for sfile in container.files:
                for gut_key in gets:
                    if os.path.basename(sfile['name']) == gut_key:
                        if split_delim == None:
                            guts[gut_key] = requests.get(sfile['mediaLink']).text
                        else:
                            guts[gut_key] = requests.get(sfile['mediaLink']).text.split(split_delim)

    return guts

Response:
get_container_contents will return a list of folders or files
    for a container


Instruction:
Given the following Python 3 function, write the documentation
def get_image_file_hash(image_path):
    '''get_image_hash will return an md5 hash of the file based on a criteria level.
    :param level: one of LOW, MEDIUM, HIGH
    :param image_path: full path to the singularity image
    '''
    hasher = hashlib.md5()
    with open(image_path, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hasher.update(chunk)
    return hasher.hexdigest()

Response: get_image_hash will return an md5 hash of the file based on a criteria level


Instruction:
Make a summary of the following Python 3 code
def container_difference(container=None,container_subtract=None,image_package=None,
                         image_package_subtract=None,comparison=None):
    '''container_difference will return a data structure to render an html
    tree (graph) of the differences between two images or packages. The second
    container is subtracted from the first
    :param container: the primary container object (to subtract from)
    :param container_subtract: the second container object to remove
    :param image_package: a zipped package for image 1, created with package
    :param image_package_subtract: a zipped package for subtraction image, created with package
    :param comparison: the comparison result object for the tree. If provided,
    will skip over function to obtain it.
    '''
    if comparison == None:
        comparison = compare_containers(container1=container,
                                        container2=container_subtract,
                                        image_package1=image_package,
                                        image_package2=image_package_subtract,
                                        by=['files.txt','folders.txt'])

    files = comparison["files.txt"]['unique1']
    folders = comparison['folders.txt']['unique1']
    tree = make_container_tree(folders=folders,
                               files=files)
    return tree

Response:
container_difference will return a data structure to render an html
    tree


Instruction:
Here you have a function in Python 3, explain what it does
def container_similarity(container1=None,container2=None,image_package1=None,
                         image_package2=None,comparison=None):
    '''container_sim will return a data structure to render an html tree
    (graph) of the intersection (commonalities) between two images or packages
    :param container1: the first container object
    :param container2: the second container object if either not defined, need
    :param image_package1: a packaged container1 (produced by package)
    :param image_package2: a packaged container2 (produced by package)
    :param comparison: the comparison result object for the tree. If provided,
    will skip over function to obtain it.
    '''
    if comparison == None:
        comparison = compare_containers(container1=container1,
                                        container2=container2,
                                        image_package1=image_package1,
                                        image_package2=image_package2,
                                        by=['files.txt','folders.txt'])
    files = comparison["files.txt"]['intersect']
    folders = comparison['folders.txt']['intersect']
    tree = make_container_tree(folders=folders,
                               files=files)
    return tree

Response: container_sim will return a data structure to render an html tree


Instruction:
Explain what the following Python 3 code does
def container_tree(container=None,image_package=None):
    '''tree will render an html tree (graph) of a container
    '''

    guts = get_container_contents(container=container,
                                  image_package=image_package,
                                  split_delim="\n")

    # Make the tree and return it
    tree = make_container_tree(folders = guts["folders.txt"],
                               files = guts['files.txt'])
    return tree

Response: will render an html tree of a container


Instruction:
Can you generate a brief explanation for the following Python 3 code
def make_container_tree(folders,files,path_delim="/",parse_files=True):
    '''make_container_tree will convert a list of folders and files into a json structure that represents a graph.
    :param folders: a list of folders in the image
    :param files: a list of files in the folder
    :param parse_files: return 'files' lookup in result, to associate ID of node with files (default True)
    :param path_delim: the path delimiter, default is '/'
    '''
    nodes = {}  # first we will make a list of nodes
    lookup = {}
    count = 1   # count will hold an id for nodes
    max_depth = 0
    for folder in folders:
        if folder != ".":
            folder = re.sub("^[.]/","",folder)
            path_components = folder.split(path_delim)
            for p in range(len(path_components)):
                path_component = path_components[p]
                fullpath = path_delim.join(path_components[0:p+1])
                # Have we created the node yet?
                if fullpath not in lookup:
                    lookup[fullpath] = count
                    node = {"id":count,"name":path_component,"path":fullpath,"level":p,"children":[]}
                    count +=1
                    # Did we find a deeper level?
                    if p > max_depth:
                        max_depth = p
                    # Does the node have a parent?
                    if p==0: # base node, no parent
                        parent_id = 0
                    else: # look up the parent id
                        parent_path = path_delim.join(path_components[0:p])
                        parent_id = lookup[parent_path]
                    node["parent"] = parent_id
                    nodes[node['id']] = node

    # Now make the graph, we simply append children to their parents
    seen = []
    graph = []
    iters = list(range(max_depth+1)) # 0,1,2,3...
    iters.reverse()            # ...3,2,1,0
    iters.pop()                # remove 0
    for level in iters:
        children = {x:y for x,y in nodes.items() if y['level'] == level}
        seen = seen + [y['id'] for x,y in children.items()]
        nodes = {x:y for x,y in nodes.items() if y['id'] not in seen}
        for node_id,child_node in children.items():
            if node_id == 0: #base node
                graph[node_id] = child_node
            else:
                parent_id = child_node['parent']
                nodes[parent_id]["children"].append(child_node)

    # Now add the parents to graph, with name as main lookup
    for parent,parent_info in nodes.items():
        graph.append(parent_info)
    graph = {"name":"base","children":graph}
    result = {"graph":graph,"lookup":lookup,"depth":max_depth+1}

    # Parse files to include in tree
    if parse_files == True:
        file_lookup = {}
        for filey in files:
            filey = re.sub("^[.]/","",filey)
            filepath,filename = os.path.split(filey)
            if filepath in lookup:
                folder_id = lookup[filepath]
                if folder_id in file_lookup:
                    file_lookup[folder_id].append(filename)
                else:
                    file_lookup[folder_id] = [filename]
            elif filepath == '': # base folder
                if 0 in file_lookup:
                    file_lookup[0].append(filename)
                else:
                    file_lookup[0] = [filename]
        result['files'] = file_lookup

    return result

Response: This function will convert a list of folders and files into a json structure that represents a graph.


Instruction:
Make a summary of the following Python 3 code
def make_package_tree(matrix=None,labels=None,width=25,height=10,title=None,font_size=None):
    '''make package tree will make a dendrogram comparing a matrix of packages
    :param matrix: a pandas df of packages, with names in index and columns
    :param labels: a list of labels corresponding to row names, will be
    pulled from rows if not defined
    :param title: a title for the plot, if not defined, will be left out.
    :returns a plot that can be saved with savefig
    '''
    from matplotlib import pyplot as plt
    from scipy.cluster.hierarchy import (
        dendrogram,
        linkage
    )

    if font_size is None:
        font_size = 8.

    from scipy.cluster.hierarchy import cophenet
    from scipy.spatial.distance import pdist

    if not isinstance(matrix,pandas.DataFrame):
        bot.info("No pandas DataFrame (matrix) of similarities defined, will use default.")
        matrix = compare_packages()['files.txt']
        title = 'Docker Library Similarity to Base OS'

    Z = linkage(matrix, 'ward')
    c, coph_dists = cophenet(Z, pdist(matrix))

    if labels == None:
        labels = matrix.index.tolist()

    plt.figure(figsize=(width, height))

    if title != None:
        plt.title(title)

    plt.xlabel('image index')
    plt.ylabel('distance')
    dendrogram(Z,
               leaf_rotation=90.,  # rotates the x axis labels
               leaf_font_size=font_size,  # font size for the x axis labels
               labels=labels)
    return plt

Response: make a package tree


Instruction:
Explain what the following Python 3 code does
def make_interactive_tree(matrix=None,labels=None):
    '''make interactive tree will return complete html for an interactive tree
    :param title: a title for the plot, if not defined, will be left out.
    '''
    from scipy.cluster.hierarchy import (
        dendrogram,
        linkage,
        to_tree
    )

    d3 = None
    from scipy.cluster.hierarchy import cophenet
    from scipy.spatial.distance import pdist

    if isinstance(matrix,pandas.DataFrame):
        Z = linkage(matrix, 'ward') # clusters
        T = to_tree(Z, rd=False)

        if labels == None:
            labels = matrix.index.tolist()
        lookup = dict(zip(range(len(labels)), labels))

        # Create a dendrogram object without plotting
        dend = dendrogram(Z,no_plot=True,
                      orientation="right",
                      leaf_rotation=90.,  # rotates the x axis labels
                      leaf_font_size=8.,  # font size for the x axis labels
                      labels=labels)

        d3 = dict(children=[], name="root")
        add_node(T, d3)
        label_tree(d3["children"][0],lookup)
    else:
        bot.warning('Please provide data as pandas Data Frame.')
    return d3

Response: make an interactive tree


Instruction:
Can you tell what is the following Python 3 function doing
def add_node(node, parent):
    '''add_node will add a node to it's parent
    '''
    newNode = dict(node_id=node.id, children=[])
    parent["children"].append(newNode)
    if node.left: add_node(node.left, newNode)
    if node.right: add_node(node.right, newNode)

Response: add_node will add a node to it s parent


Instruction:
Can you tell what is the following Python 3 function doing
def label_tree(n,lookup):
    '''label tree will again recursively label the tree
    :param n: the root node, usually d3['children'][0]
    :param lookup: the node/id lookup
    '''
    if len(n["children"]) == 0:
        leaves = [lookup[n["node_id"]]]
    else:
        leaves = reduce(lambda ls, c: ls + label_tree(c,lookup), n["children"], [])
    del n["node_id"]
    n["name"] = name = "|||".join(sorted(map(str, leaves)))
    return leaves

Response: label tree will recursively label the tree


Instruction:
Implement a function in Python 3 to
extract app will extract metadata for one or more apps

Response:
def extract_apps(image, app_names):
    ''' extract app will extract metadata for one or more apps
        Parameters
        ==========
        image: the absolute path to the image
        app_name: the name of the app under /scif/apps

    '''
    apps = dict()

    if isinstance(app_names, tuple):
        app_names = list(app_names)
    if not isinstance(app_names, list):
        app_names = [app_names]
    if len(app_names) == 0:
        return apps

    for app_name in app_names:
        metadata = dict()

        # Inspect: labels, env, runscript, tests, help
        try:
            inspection = json.loads(Client.inspect(image, app=app_name))
            del inspection['data']['attributes']['deffile']
            metadata['inspect'] = inspection

        # If illegal characters prevent load, not much we can do
        except:
            pass
        apps[app_name] = metadata
    return apps


Instruction:
Can you generate the documentation for the following Python 3 function
def download_repo(repo_url, destination, commit=None):
    '''download_repo
    :param repo_url: the url of the repo to clone from
    :param destination: the full path to the destination for the repo
    '''
    command = "git clone %s %s" % (repo_url, destination)
    os.system(command)
    return destination

Response: downloads the repo from the specified url to the specified destination


Instruction:
Can you write a function in Python 3 where it
gets tags will return a list of tags that describe the software in an image

Response:
def get_tags(container=None,
             search_folders=None,
             file_list=None,
             return_unique=True):

    '''get tags will return a list of tags that describe the software in an image,
    meaning inside of a paricular folder. If search_folder is not defined, uses lib
    :param container: if provided, will use container as image. Can also provide
    :param image_package: if provided, can be used instead of container
    :param search_folders: specify one or more folders to look for tags
    :param file_list: the list of files
    :param return_unique: return unique files in folders. Default True.
    Default is 'bin'

    ::notes

    The algorithm works as follows:
      1) first compare package to set of base OS (provided with shub)
      2) subtract the most similar os from image, leaving "custom" files
      3) organize custom files into dict based on folder name
      4) return search_folders as tags

    '''
    if file_list is None:
        file_list = get_container_contents(container, split_delim='\n')['all']

    if search_folders == None:
        search_folders = 'bin'

    if not isinstance(search_folders,list):
        search_folders = [search_folders]

    tags = []
    for search_folder in search_folders:
        for file_name in file_list:
            if search_folder in file_name:
                tags.append(file_name)

    if return_unique == True:
        tags = list(set(tags))
    return tags


Instruction:
Implement a Python 3 function for
filing counts will return a list of files that match one or more regular expressions.

Response:
def file_counts(container=None,
                patterns=None,
                image_package=None,
                file_list=None):

    '''file counts will return a list of files that match one or more regular expressions.
    if no patterns is defined, a default of readme is used. All patterns and files are made
    case insensitive.

    Parameters
    ==========
    :param container: if provided, will use container as image. Can also provide
    :param image_package: if provided, can be used instead of container
    :param patterns: one or more patterns (str or list) of files to search for.
    :param diff: the difference between a container and it's parent OS from get_diff
    if not provided, will be generated.

    '''
    if file_list is None:
        file_list = get_container_contents(container, split_delim='\n')['all']

    if patterns == None:
        patterns = 'readme'

    if not isinstance(patterns,list):
        patterns = [patterns]

    count = 0
    for pattern in patterns:
        count += len([x for x in file_list if re.search(pattern.lower(),x.lower())])
    bot.info("Total files matching patterns is %s" %count)
    return count


Instruction:
Can you generate the documentation for the following Python 3 function
def extension_counts(container=None, file_list=None, return_counts=True):
    '''extension counts will return a dictionary with counts of file extensions for
    an image.
    :param container: if provided, will use container as image. Can also provide
    :param image_package: if provided, can be used instead of container
    :param file_list: the complete list of files
    :param return_counts: return counts over dict with files. Default True
    '''
    if file_list is None:
        file_list = get_container_contents(container, split_delim='\n')['all']

    extensions = dict()
    for item in file_list:
        filename,ext = os.path.splitext(item)
        if ext == '':
            if return_counts == False:
                extensions = update_dict(extensions,'no-extension',item)
            else:
                extensions = update_dict_sum(extensions,'no-extension')
        else:
            if return_counts == False:
                extensions = update_dict(extensions,ext,item)
            else:
                extensions = update_dict_sum(extensions,ext)

    return extensions

Response: returns a dictionary with counts of file extensions for a container and an image.


Instruction:
Can you create a Python 3 function that
assesses the differences between two images on each level of returns a dictionary with files that are the same different and an overall score.

Response:
def assess_differences(image_file1,
                       image_file2,
                       levels=None,
                       version=None,
                       size_heuristic=False,
                       guts1=None,
                       guts2=None):

    '''assess_differences will compare two images on each level of
    reproducibility, returning for each level a dictionary with files
    that are the same, different, and an overall score.
    :param size_heuristic: if True, assess root owned files based on size
    :param guts1,guts2: the result (dict with sizes,roots,etc) from get_content_hashes
    '''
    if levels is None:
        levels = get_levels(version=version)

    reports = dict()
    scores = dict()

    for level_name, level_filter in levels.items():
        contenders = []
        different = []
        setdiff = []
        same = 0

        # Compare the dictionary of file:hash between two images, and get root owned lookup
        if guts1 is None:
            guts1 = get_content_hashes(image_path=image_file1,
                                       level_filter=level_filter)
                                       # tag_root=True
                                       # include_sizes=True

        if guts2 is None:
            guts2 = get_content_hashes(image_path=image_file2,
                                       level_filter=level_filter)

        print(level_name)
        files = list(set(list(guts1['hashes'].keys()) + list(guts2['hashes'].keys())))

        for file_name in files:

            # If it's not in one or the other
            if file_name not in guts1['hashes'] or file_name not in guts2['hashes']:
                setdiff.append(file_name)

            else:
                if guts1['hashes'][file_name] == guts2['hashes'][file_name]:
                    same+=1
                else:

                    # If the file is root owned, we compare based on size
                    if size_heuristic == True:
                        if guts1['root_owned'][file_name] or guts2['root_owned'][file_name]:
                            if guts1['sizes'][file_name] == guts2['sizes'][file_name]:
                                same+=1
                            else:
                                different.append(file_name)
                        else:
                            # Otherwise, we can assess the bytes content by reading it
                            contenders.append(file_name)
                    else:
                        contenders.append(file_name)

        # If the user wants identical (meaning extraction order and timestamps)
        if level_name == "IDENTICAL":
                different = different + contenders

        # Otherwise we need to check based on byte content
        else:
            if len(contenders) > 0:
                for rogue in contenders:
                    hashy1 = extract_content(image_file1, rogue, return_hash=True)
                    hashy2 = extract_content(image_file2, rogue, return_hash=True)

                    # If we can't compare, we use size as a heuristic
                    if hashy1 is None or hashy2 is None: # if one is symlink, could be None
                        different.append(file_name)
                    elif len(hashy1) == 0 or len(hashy2) == 0:
                        if guts1['sizes'][file_name] == guts2['sizes'][file_name]:
                            same+=1
                        else:
                            different.append(file_name)
                    elif hashy1 != hashy2:
                        different.append(rogue)
                    else:
                        same+=1

        # We use a similar Jacaard coefficient, twice the shared information in the numerator
        # (the intersection, same), as a proportion of the total summed files
        union = len(guts1['hashes']) + len(guts2['hashes'])

        report = {'difference': setdiff,
                  'intersect_different': different,
                  'same':same,
                  'union': union}

        if union == 0:
            scores[level_name] = 0
        else:
            scores[level_name] = 2*(same) / union
        reports[level_name] = report

    reports['scores'] = scores
    return reports


Instruction:
Here you have a function in Python 3, explain what it does
def include_file(member,file_filter):
    '''include_file will look at a path and determine
    if it matches a regular expression from a level
    '''
    member_path = member.name.replace('.','',1)

    if len(member_path) == 0:
        return False

    # Does the filter skip it explicitly?
    if "skip_files" in file_filter:
        if member_path in file_filter['skip_files']:
            return False

    # Include explicitly?
    if "include_files" in file_filter:
        if member_path in file_filter['include_files']:
            return True

    # Regular expression?
    if "regexp" in file_filter:
        if re.search(file_filter["regexp"],member_path):
            return True
    return False

Response:
Returns True if the member matches a regular expression from a level
   .


Instruction:
Can you generate the documentation for the following Python 3 function
def is_root_owned(member):
    '''assess if a file is root owned, meaning "root" or user/group
    id of 0'''
    if member.uid == 0 or member.gid == 0:
        return True
    elif member.uname == 'root' or member.gname == 'root':
        return True
    return False

Response:
assess if a file is root owned meaning root or user / group
    id of 0


Instruction:
Create a Python 3 function to
determine if the filter wants the file to be read for content.

Response:
def assess_content(member,file_filter):
    '''Determine if the filter wants the file to be read for content.
    In the case of yes, we would then want to add the content to the
    hash and not the file object.
    '''
    member_path = member.name.replace('.','',1)

    if len(member_path) == 0:
        return False

    # Does the filter skip it explicitly?
    if "skip_files" in file_filter:
        if member_path in file_filter['skip_files']:
            return False

    if "assess_content" in file_filter:
        if member_path in file_filter['assess_content']:
            return True
    return False


Instruction:
How would you explain what the following Python 3 function does
def get_custom_level(regexp=None,description=None,skip_files=None,include_files=None):
    '''get_custom_level will generate a custom level for the user,
    based on a regular expression. If used outside the context of tarsum, the user
    can generate their own named and described filters.
    :param regexp: must be defined, the file filter regular expression
    :param description: optional description
    '''
    if regexp == None:
        regexp = "."
    if description is None:
        description = "This is a custom filter generated by the user."

    custom = {"description":description,
              "regexp":regexp}

    # Include extra files?
    if include_files is not None:
        if not isinstance(include_files,set):
            include_files = set(include_files)
        custom['include_files'] = include_files

    # Skip files?
    if skip_files is not None:
        if not isinstance(skip_files,set):
            skip_files = set(skip_files)
        custom['skip_files'] = skip_files

    return custom

Response: This function returns a custom level for a user based on a regular expression.


Instruction:
Can you implement a function in Python 3 that
modifies level is intended to add or modify a content type.

Response:
def modify_level(level,field,values,append=True):
    '''modify level is intended to add / modify a content type.
    Default content type is list, meaning the entry is appended.
    If you set append to False, the content will be overwritten
    For any other content type, the entry is overwritten.
    '''
    field = field.lower()
    valid_fields = ['regexp','skip_files','include_files']
    if field not in valid_fields:
        bot.warning("%s is not a valid field, skipping. Choices are %s" %(field,",".join(valid_fields)))
        return level
    if append:
        if not isinstance(values,list):
            values = [values]
        if field in level:
            level[field] = level[field] + values
        else:
            level[field] = values
    else:
        level[field] = values

    level = make_level_set(level)

    return level


Instruction:
Can you tell what is the following Python 3 function doing
def get_levels(version=None):
    '''get_levels returns a dictionary of levels (key) and values (dictionaries with
    descriptions and regular expressions for files) for the user.
    :param version: the version of singularity to use (default is 2.2)
    :param include_files: files to add to the level, only relvant if
    '''
    valid_versions = ['2.3','2.2']

    if version is None:
        version = "2.3"
    version = str(version)

    if version not in valid_versions:
        bot.error("Unsupported version %s, valid versions are %s" %(version,
                                                                    ",".join(valid_versions)))

    levels_file = os.path.abspath(os.path.join(get_installdir(),
                                                           'analysis',
                                                           'reproduce',
                                                           'data',
                                                           'reproduce_levels.json'))
    levels = read_json(levels_file)
    if version == "2.2":
        # Labels not added until 2.3
        del levels['LABELS']

    levels = make_levels_set(levels)

    return levels

Response: get_levels returns a dictionary of levels descriptions and regular expressions for files for the user.