Instruction:
Explain what the following Python 3 code does
def create_closure_model(cls):
    meta_vals = {
        'unique_together':  (("parent", "child"),)
    }
    if getattr(cls._meta, 'db_table', None):
        meta_vals['db_table'] = '%sclosure' % getattr(cls._meta, 'db_table')
    model = type('%sClosure' % cls.__name__, (models.Model,), {
        'parent': models.ForeignKey(
            cls.__name__,
            related_name=cls.closure_parentref()
        ),
        'child': models.ForeignKey(
            cls.__name__,
            related_name=cls.closure_childref()
        ),
        'depth': models.IntegerField(),
        '__module__':   cls.__module__,
        '__unicode__': _closure_model_unicode,
        'Meta': type('Meta', (object,), meta_vals),
    })
    setattr(cls, "_closure_model", model)
    return model

Response: Creates a <Model >Closure model in the same module as the model.


Instruction:
Can you tell what is the following Python 3 function doing
def _toplevel(cls):
        superclasses = (
            list(set(ClosureModel.__subclasses__()) &
                 set(cls._meta.get_parent_list()))
        )
        return next(iter(superclasses)) if superclasses else cls

Response: Find the top level of the chain we re in.


Instruction:
Create a Python 3 function to
regenerate the entire closuretree.

Response:
def rebuildtable(cls):
        """Regenerate the entire closuretree."""
        cls._closure_model.objects.all().delete()
        cls._closure_model.objects.bulk_create([cls._closure_model(
            parent_id=x['pk'],
            child_id=x['pk'],
            depth=0
        ) for x in cls.objects.values("pk")])
        for node in cls.objects.all():
            node._closure_createlink()


Instruction:
Write a Python 3 script for
removing incorrect links from the closure tree.

Response:
def _closure_deletelink(self, oldparentpk):
        """Remove incorrect links from the closure tree."""
        self._closure_model.objects.filter(
            **{
                "parent__%s__child" % self._closure_parentref(): oldparentpk,
                "child__%s__parent" % self._closure_childref(): self.pk
            }
        ).delete()


Instruction:
Create a Python 3 function for
creating a link in the closure tree.

Response:
def _closure_createlink(self):
        """Create a link in the closure tree."""
        linkparents = self._closure_model.objects.filter(
            child__pk=self._closure_parent_pk
        ).values("parent", "depth")
        linkchildren = self._closure_model.objects.filter(
            parent__pk=self.pk
        ).values("child", "depth")
        newlinks = [self._closure_model(
            parent_id=p['parent'],
            child_id=c['child'],
            depth=p['depth']+c['depth']+1
        ) for p in linkparents for c in linkchildren]
        self._closure_model.objects.bulk_create(newlinks)


Instruction:
Write a Python 3 script for
returning all the ancestors of this object.

Response:
def get_ancestors(self, include_self=False, depth=None):
        """Return all the ancestors of this object."""
        if self.is_root_node():
            if not include_self:
                return self._toplevel().objects.none()
            else:
                # Filter on pk for efficiency.
                return self._toplevel().objects.filter(pk=self.pk)

        params = {"%s__child" % self._closure_parentref():self.pk}
        if depth is not None:
            params["%s__depth__lte" % self._closure_parentref()] = depth
        ancestors = self._toplevel().objects.filter(**params)
        if not include_self:
            ancestors = ancestors.exclude(pk=self.pk)
        return ancestors.order_by("%s__depth" % self._closure_parentref())


Instruction:
Write a Python 3 script to
return all the descendants of this object.

Response:
def get_descendants(self, include_self=False, depth=None):
        """Return all the descendants of this object."""
        params = {"%s__parent" % self._closure_childref():self.pk}
        if depth is not None:
            params["%s__depth__lte" % self._closure_childref()] = depth
        descendants = self._toplevel().objects.filter(**params)
        if not include_self:
            descendants = descendants.exclude(pk=self.pk)
        return descendants.order_by("%s__depth" % self._closure_childref())


Instruction:
Write a Python 3 function that can
return all the children of this object.

Response:
def get_children(self):
        """Return all the children of this object."""
        if hasattr(self, '_cached_children'):
            children = self._toplevel().objects.filter(
                pk__in=[n.pk for n in self._cached_children]
            )
            children._result_cache = self._cached_children
            return children
        else:
            return self.get_descendants(include_self=False, depth=1)


Instruction:
How would you implement a function in Python 3 that
returns the furthest ancestor of this node.

Response:
def get_root(self):
        """Return the furthest ancestor of this node."""
        if self.is_root_node():
            return self

        return self.get_ancestors().order_by(
            "-%s__depth" % self._closure_parentref()
        )[0]


Instruction:
Can you create a Python 3 function that
is this node a descendant of other?

Response:
def is_descendant_of(self, other, include_self=False):
        """Is this node a descendant of `other`?"""
        if other.pk == self.pk:
            return include_self

        return self._closure_model.objects.filter(
            parent=other,
            child=self
        ).exclude(pk=self.pk).exists()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_ancestor_of(self, other, include_self=False):
        return other.is_descendant_of(self, include_self=include_self)

Response: Is this node an ancestor of other?


Instruction:
Make a summary of the following Python 3 code
def quantize(number, digits=0, q=builtins.round):
    base, fraction = split(digits)

    # quantization beyond an order of magnitude results in a variable amount
    # of decimal digits depending on the lowest common multiple,
    # e.g. floor(1.2341234, 1.25) = 1.225 but floor(1.2341234, 1.5) = 1.20
    if fraction * 10 % 1 > 0:
        digits = base + 2
    else:
        digits = base + 1

    multiplier = 10 ** base * invert(fraction, default=1)
    quantized = q(number * multiplier) / multiplier

    # additional rounding step to get rid of floating point math wonkiness
    return builtins.round(quantized, digits)

Response: Quantize a number to a number of digits.


Instruction:
Can you tell what is the following Python 3 function doing
def vectorize(fn):

    @functools.wraps(fn)
    def vectorized_function(values, *vargs, **kwargs):
        return [fn(value, *vargs, **kwargs) for value in values]

    return vectorized_function

Response:
A method that accepts a list argument but internally deal only
    with a single item of that list.


Instruction:
Create a Python 3 function for
converting a number to engineering notation.

Response:
def engineering(value, precision=3, prefix=False, prefixes=SI):
    """ Convert a number to engineering notation. """

    display = decimal.Context(prec=precision)
    value = decimal.Decimal(value).normalize(context=display)
    string = value.to_eng_string()

    if prefix:
        prefixes = {e(exponent): prefix for exponent, prefix in prefixes.items()}
        return replace(string, prefixes)
    else:
        return string


Instruction:
Write a Python 3 script for
converting a list of numbers to the business notation appropriate to the reference point.

Response:
def business(values, precision=3, prefix=True, prefixes=SI, statistic=median, default=''):
    """
    Convert a list of numbers to the engineering notation appropriate to a
    reference point like the minimum, the median or the mean --
    think of it as "business notation".

    Any number will have at most the amount of significant digits of the
    reference point, that is, the function will round beyond the
    decimal point.

    For example, if the reference is `233K`, this function will turn
    1,175,125 into `1180K` and 11,234 into `11K` (instead of 1175K and
    11.2K respectively.) This can help enormously with readability.

    If the reference point is equal to or larger than E15 or
    equal to or smaller than E-15, E12 and E-12 become the
    reference point instead. (Petas and femtos are too
    unfamiliar to people to be easily comprehended.)
    """

    reference = statistic(values)

    if not reference:
        return upcast([''] * len(values), values)

    exponent = order(reference)
    e = bound(exponent - exponent % 3, -12, 12)
    # the amount of decimals is the precision minus the amount of digits
    # before the decimal point, which is one more than the relative order
    # of magnitude (for example, 10^5 can be represented as 100K, with
    # those three digits representing place values of 10^3, 10^4 and 10^5)
    d = precision - (1 + exponent - e)

    prefix = prefixes[e]

    strings = []
    for value in values:
        if isnan(value):
            strings.append('')
        else:
            normalized = value / 10.0 ** e
            # use `round` for rounding (beyond the decimal point if necessary)
            # use string formatting for padding to the right amount of decimals
            # and to hide decimals when necessary (by default, floats are always
            # displayed with a single decimal place, to distinguish them from
            # integers)
            relative_order = order(value) - exponent
            places = min(d - relative_order, d)
            normalized = round(normalized, places)
            strings.append('{0:,.{1}f}'.format(normalized, d) + prefix)

    return upcast(strings, values)


Instruction:
Write a Python 3 script for
yielding items from iterator in chunks of size chunksize.

Response:
def chunked(iterator, chunksize):
    """
    Yields items from 'iterator' in chunks of size 'chunksize'.

    >>> list(chunked([1, 2, 3, 4, 5], chunksize=2))
    [(1, 2), (3, 4), (5,)]
    """
    chunk = []
    for idx, item in enumerate(iterator, 1):
        chunk.append(item)
        if idx % chunksize == 0:
            yield chunk
            chunk = []
    if chunk:
        yield chunk


Instruction:
Given the following Python 3 function, write the documentation
def pre_start_check(self):
        try:
            sock = socket.socket()
            sock.connect((self.host, self.port))
            return True
        except (socket.error, socket.timeout):
            return False
        finally:
            # close socket manually for sake of PyPy
            sock.close()

Response: Check if process can accept TCP connections.


Instruction:
Can you generate the documentation for the following Python 3 function
def after_start_check(self):
        try:
            conn = HTTPConnection(self.host, self.port)

            conn.request('HEAD', self.url.path)
            status = str(conn.getresponse().status)

            if status == self.status or self.status_re.match(status):
                conn.close()
                return True

        except (HTTPException, socket.timeout, socket.error):
            return False

Response: Check if defined URL returns expected status to a HEAD request.


Instruction:
How would you code a function in Python 3 to
start the output process.

Response:
def start(self):
        """
        Start process.

        :returns: itself
        :rtype: OutputExecutor

        .. note::

            Process will be considered started, when defined banner will appear
            in process output.
        """
        super(OutputExecutor, self).start()

        # get a polling object
        self.poll_obj = select.poll()

        # register a file descriptor
        # POLLIN because we will wait for data to read
        self.poll_obj.register(self.output(), select.POLLIN)

        try:
            self.wait_for(self._wait_for_output)

            # unregister the file descriptor and delete the polling object
            self.poll_obj.unregister(self.output())
        finally:
            del self.poll_obj
        return self


Instruction:
Can you write a function in Python 3 where it
checks if output matches banner.

Response:
def _wait_for_output(self):
        """
        Check if output matches banner.

        .. warning::
            Waiting for I/O completion. It does not work on Windows. Sorry.
        """
        # Here we should get an empty list or list with a tuple [(fd, event)]
        # When we get list with a tuple we can use readline method on
        # the file descriptor.
        poll_result = self.poll_obj.poll(0)

        if poll_result:
            line = self.output().readline()
            if self._banner.match(line):
                return True

        return False


Instruction:
Can you create a Python 3 function that
constructs a new email object

Response:
def construct_mail(recipients=None, context=None, template_base='emailit/email', subject=None, message=None, site=None,
                   subject_templates=None, body_templates=None, html_templates=None, from_email=None, language=None,
                   **kwargs):
    """
    usage:
    construct_mail(['my@email.com'], {'my_obj': obj}, template_base='myapp/emails/my_obj_notification').send()
    :param recipients: recipient or list of recipients
    :param context: context for template rendering
    :param template_base: the base template. '.subject.txt', '.body.txt' and '.body.html' will be added
    :param subject: optional subject instead of rendering it through a template
    :param message: optional message (will be inserted into the base email template)
    :param site: the site this is on. uses current site by default
    :param subject_templates: override the subject template
    :param body_templates: override the body template
    :param html_templates: override the html body template
    :param from_email: defaults to settings.DEFAULT_FROM_EMAIL
    :param language: the language that should be active for this email. defaults to currently active lang
    :param kwargs: kwargs to pass into the Email class
    :return:
    """
    language = language or translation.get_language()
    with force_language(language):
        recipients = recipients or []
        if isinstance(recipients, basestring):
            recipients = [recipients]
        from_email = from_email or settings.DEFAULT_FROM_EMAIL
        subject_templates = subject_templates or get_template_names(language, template_base, 'subject', 'txt')
        body_templates = body_templates or get_template_names(language, template_base, 'body', 'txt')
        html_templates = html_templates or get_template_names(language, template_base, 'body', 'html')

        if not context:
            context = {}

        site = site or Site.objects.get_current()
        context['site'] = site
        context['site_name'] = site.name
        protocol = 'http'  # TODO: this should come from settings
        base_url = "%s://%s" % (protocol, site.domain)
        if message:
            context['message'] = message

        subject = subject or render_to_string(subject_templates, context)
        subject = subject.replace('\n', '').replace('\r', '').strip()
        context['subject'] = subject
        try:
            html = render_to_string(html_templates, context)
        except TemplateDoesNotExist:
            html = ''
        else:
            html = premailer.transform(html, base_url=base_url)
        try:
            body = render_to_string(body_templates, context)
        except TemplateDoesNotExist:
            body = ''

        mail = EmailMultiAlternatives(subject, body, from_email, recipients, **kwargs)

        if not (body or html):
            # this is so a meaningful exception can be raised
            render_to_string([html_templates], context)
            render_to_string([body_templates], context)

        if html:
            mail.attach_alternative(html, 'text/html')
    return mail


Instruction:
Can you tell what is the following Python 3 function doing
def cleanup_subprocesses():
    # pylint: disable=redefined-outer-name, reimported
    # atexit functions tends to loose global imports sometimes so reimport
    # everything what is needed again here:
    import os
    import errno
    from mirakuru.base_env import processes_with_env
    from mirakuru.compat import SIGKILL

    pids = processes_with_env(ENV_UUID, str(os.getpid()))
    for pid in pids:
        try:
            os.kill(pid, SIGKILL)
        except OSError as err:
            if err.errno != errno.ESRCH:
                print("Can not kill the", pid, "leaked process", err)

Response: On python exit kill all processes that are running.


Instruction:
Create a Python 3 function for
starting the countdown process.

Response:
def start(self):
        """
        Start defined process.

        After process gets started, timeout countdown begins as well.

        :returns: itself
        :rtype: SimpleExecutor

        .. note::
            We want to open ``stdin``, ``stdout`` and ``stderr`` as text
            streams in universal newlines mode, so we have to set
            ``universal_newlines`` to ``True``.
        """
        if self.process is None:
            command = self.command
            if not self._shell:
                command = self.command_parts

            env = os.environ.copy()
            # Trick with marking subprocesses with an environment variable.
            #
            # There is no easy way to recognize all subprocesses that were
            # spawned during lifetime of a certain subprocess so mirakuru does
            # this hack in order to mark who was the original parent. Even if
            # some subprocess got daemonized or changed original process group
            # mirakuru will be able to find it by this environment variable.
            #
            # There may be a situation when some subprocess will abandon
            # original envs from parents and then it won't be later found.
            env[ENV_UUID] = self._uuid
            popen_kwargs = {
                'shell': self._shell,
                'stdin': subprocess.PIPE,
                'stdout': subprocess.PIPE,
                'universal_newlines': True,
                'env': env,
            }
            if platform.system() != 'Windows':
                popen_kwargs['preexec_fn'] = os.setsid
            self.process = subprocess.Popen(
                command,
                **popen_kwargs
            )

        self._set_timeout()
        return self


Instruction:
Create a Python 3 function to
clear the process object.

Response:
def _clear_process(self):
        """
        Close stdin/stdout of subprocess.

        It is required because of ResourceWarning in Python 3.
        """
        if self.process:
            if self.process.stdin:
                self.process.stdin.close()
            if self.process.stdout:
                self.process.stdout.close()

            self.process = None

        self._endtime = None


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _kill_all_kids(self, sig):
        pids = processes_with_env(ENV_UUID, self._uuid)
        for pid in pids:
            log.debug("Killing process %d ...", pid)
            try:
                os.kill(pid, sig)
            except OSError as err:
                if err.errno in IGNORED_ERROR_CODES:
                    # the process has died before we tried to kill it.
                    pass
                else:
                    raise
            log.debug("Killed process %d.", pid)
        return pids

Response: Kill all processes that have started and have left.


Instruction:
How would you explain what the following Python 3 function does
def kill(self, wait=True, sig=None):
        if sig is None:
            sig = self._sig_kill
        if self.running():
            os.killpg(self.process.pid, sig)
            if wait:
                self.process.wait()

        self._kill_all_kids(sig)
        self._clear_process()
        return self

Response: Kill the process if running.


Instruction:
How would you implement a function in Python 3 that
waits for a callback to return True.

Response:
def wait_for(self, wait_for):
        """
        Wait for callback to return True.

        Simply returns if wait_for condition has been met,
        raises TimeoutExpired otherwise and kills the process.

        :param callback wait_for: callback to call
        :raises: mirakuru.exceptions.TimeoutExpired
        :returns: itself
        :rtype: SimpleExecutor
        """
        while self.check_timeout():
            if wait_for():
                return self
            time.sleep(self._sleep)

        self.kill()
        raise TimeoutExpired(self, timeout=self._timeout)


Instruction:
Can you tell what is the following Python 3 function doing
def start(self):
        if self.pre_start_check():
            # Some other executor (or process) is running with same config:
            raise AlreadyRunning(self)

        super(Executor, self).start()

        self.wait_for(self.check_subprocess)
        return self

Response: Start executor with additional checks.


Instruction:
Implement a Python 3 function for
checking the status of the main process and run the checks.

Response:
def check_subprocess(self):
        """
        Make sure the process didn't exit with an error and run the checks.

        :rtype: bool
        :return: the actual check status
        :raise ProcessExitedWithError: when the main process exits with
            an error
        """
        exit_code = self.process.poll()
        if exit_code is not None and exit_code != 0:
            # The main process exited with an error. Clean up the children
            # if any.
            self._kill_all_kids(self._sig_kill)
            self._clear_process()
            raise ProcessExitedWithError(self, exit_code)

        return self.after_start_check()


Instruction:
Write a Python 3 script to
find processes with environment variable matching given one.

Response:
def processes_with_env_psutil(env_name, env_value):
    """
    Find PIDs of processes having environment variable matching given one.

    Internally it uses `psutil` library.

    :param str env_name: name of environment variable to be found
    :param str env_value: environment variable value prefix
    :return: process identifiers (PIDs) of processes that have certain
             environment variable equal certain value
    :rtype: set
    """
    pids = set()

    for proc in psutil.process_iter():
        try:
            pinfo = proc.as_dict(attrs=['pid', 'environ'])
        except (psutil.NoSuchProcess, IOError):
            # can't do much if psutil is not able to get this process details
            pass
        else:
            penv = pinfo.get('environ')
            if penv and env_value in penv.get(env_name, ''):
                pids.add(pinfo['pid'])

    return pids


Instruction:
Write a Python 3 script for
finding processes with environment variable matching given one.

Response:
def processes_with_env_ps(env_name, env_value):
    """
    Find PIDs of processes having environment variable matching given one.

    It uses `$ ps xe -o pid,cmd` command so it works only on systems
    having such command available (Linux, MacOS). If not available function
    will just log error.

    :param str env_name: name of environment variable to be found
    :param str env_value: environment variable value prefix
    :return: process identifiers (PIDs) of processes that have certain
             environment variable equal certain value
    :rtype: set
    """
    pids = set()
    ps_xe = ''
    try:
        cmd = 'ps', 'xe', '-o', 'pid,cmd'
        ps_xe = subprocess.check_output(cmd).splitlines()
    except OSError as err:
        if err.errno == errno.ENOENT:
            log.error("`$ ps xe -o pid,cmd` command was called but it is not "
                      "available on this operating system. Mirakuru will not "
                      "be able to list the process tree and find if there are "
                      "any leftovers of the Executor.")
            return pids
    except subprocess.CalledProcessError:
        log.error("`$ ps xe -o pid,cmd` command exited with non-zero code.")

    env = '{0}={1}'.format(env_name, env_value)

    for line in ps_xe:
        line = str(line)
        if env in line:
            pids.add(int(PS_XE_PID_MATCH.match(line).group(1)))
    return pids


Instruction:
Can you write a function in Python 3 where it
computes the NCC of a 2D array.

Response:
def _ncc_c(x, y):
    """
    >>> _ncc_c([1,2,3,4], [1,2,3,4])
    array([ 0.13333333,  0.36666667,  0.66666667,  1.        ,  0.66666667,
            0.36666667,  0.13333333])
    >>> _ncc_c([1,1,1], [1,1,1])
    array([ 0.33333333,  0.66666667,  1.        ,  0.66666667,  0.33333333])
    >>> _ncc_c([1,2,3], [-1,-1,-1])
    array([-0.15430335, -0.46291005, -0.9258201 , -0.77151675, -0.46291005])
    """
    den = np.array(norm(x) * norm(y))
    den[den == 0] = np.Inf

    x_len = len(x)
    fft_size = 1 << (2*x_len-1).bit_length()
    cc = ifft(fft(x, fft_size) * np.conj(fft(y, fft_size)))
    cc = np.concatenate((cc[-(x_len-1):], cc[:x_len]))
    return np.real(cc) / den


Instruction:
Given the following Python 3 function, write the documentation
def _ncc_c_2dim(x, y):
    den = np.array(norm(x, axis=1) * norm(y))
    den[den == 0] = np.Inf
    x_len = x.shape[-1]
    fft_size = 1 << (2*x_len-1).bit_length()
    cc = ifft(fft(x, fft_size) * np.conj(fft(y, fft_size)))
    cc = np.concatenate((cc[:,-(x_len-1):], cc[:,:x_len]), axis=1)
    return np.real(cc) / den[:, np.newaxis]

Response:
Variant of NCCc that operates with 2 dimensional X arrays and 1 dimensional
    y vector
    Returns a 2 dimensional array of normalized fourier transforms


Instruction:
Make a summary of the following Python 3 code
def _ncc_c_3dim(x, y):
    den = norm(x, axis=1)[:, None] * norm(y, axis=1)
    den[den == 0] = np.Inf
    x_len = x.shape[-1]
    fft_size = 1 << (2*x_len-1).bit_length()
    cc = ifft(fft(x, fft_size) * np.conj(fft(y, fft_size))[:, None])
    cc = np.concatenate((cc[:,:,-(x_len-1):], cc[:,:,:x_len]), axis=2)
    return np.real(cc) / den.T[:, :, None]

Response:
Variant of NCCc that operates with 2 dimensional X arrays and 2 dimensional y vector
    Returns a 3 dimensional array of normalized fourier transforms


Instruction:
Make a summary of the following Python 3 code
def _sbd(x, y):
    ncc = _ncc_c(x, y)
    idx = ncc.argmax()
    dist = 1 - ncc[idx]
    yshift = roll_zeropad(y, (idx + 1) - max(len(x), len(y)))

    return dist, yshift

Response: Compute the SVD distance and yshift for a single node.


Instruction:
Can you generate the documentation for the following Python 3 function
def _extract_shape(idx, x, j, cur_center):
    _a = []
    for i in range(len(idx)):
        if idx[i] == j:
            if cur_center.sum() == 0:
                opt_x = x[i]
            else:
                _, opt_x = _sbd(cur_center, x[i])
            _a.append(opt_x)
    a = np.array(_a)

    if len(a) == 0:
        return np.zeros((1, x.shape[1]))
    columns = a.shape[1]
    y = zscore(a, axis=1, ddof=1)
    s = np.dot(y.transpose(), y)

    p = np.empty((columns, columns))
    p.fill(1.0/columns)
    p = np.eye(columns) - p

    m = np.dot(np.dot(p, s), p)
    _, vec = eigh(m)
    centroid = vec[:, -1]
    finddistance1 = math.sqrt(((a[0] - centroid) ** 2).sum())
    finddistance2 = math.sqrt(((a[0] + centroid) ** 2).sum())

    if finddistance1 >= finddistance2:
        centroid *= -1

    return zscore(centroid, ddof=1)

Response: Returns the shape of the array of the array of the array at the given index.


Instruction:
Can you tell what is the following Python 3 function doing
def _kshape(x, k):
    m = x.shape[0]
    idx = randint(0, k, size=m)
    centroids = np.zeros((k, x.shape[1]))
    distances = np.empty((m, k))

    for _ in range(100):
        old_idx = idx
        for j in range(k):
            centroids[j] = _extract_shape(idx, x, j, centroids[j])

        distances = (1 - _ncc_c_3dim(x, centroids).max(axis=2)).T

        idx = distances.argmin(1)
        if np.array_equal(old_idx, idx):
            break

    return idx, centroids

Response: Returns k - size of the randomized k - array


Instruction:
Write a Python 3 function that can
get the internal version ID be the version.

Response:
def get_version_by_version_id(version_id):
    """
    Get the internal version ID be the version.

    :param Tuple version_id: Major and minor version number
    :return: Internal version ID
    :rtype: Integer|None
    """
    for ver in registry.version_info:
        if ver.version_id == version_id:
            return ver.id

    return None


Instruction:
Write a Python 3 script for
getting the name of a protocol version by the internal version ID.

Response:
def get_version_name(version_id):
    """
    Get the name of a protocol version by the internal version ID.

    :param Integer version_id: Internal protocol version ID
    :return: Name of the version
    :rtype: String
    """
    ver = registry.version_info.get(version_id)
    if ver:
        return ver.name

    return 'unknown'


Instruction:
Write a Python 3 function that can
get a tuple with major and minor protocol version number.

Response:
def get_version_id(protocol_version):
    """
    Get a tuple with major and minor version number

    :param Integer protocol_version: Internal version ID
    :return: Tuple of major and minor protocol version
    :rtype: Tuple
    """
    ver = registry.version_info.get(protocol_version)
    if ver:
        return ver.version_id


Instruction:
How would you explain what the following Python 3 function does
def update(self, b):  # pylint: disable=method-hidden,invalid-name
        with self.__lock:
            output = compress_update(self.__ctx, b)
            if self.__write:
                self.__write(self.__header)
                self.__header = None
                self.__write(output)
                self.update = self.__updateNextWrite
            else:
                header = self.__header
                self.__header = None
                self.update = self.__updateNextReturn
                return header + output

Response: Compress data given in b returning compressed result either from this function or writing to fp.


Instruction:
How would you explain what the following Python 3 function does
def end(self):
        with self.__lock:
            if self.__write:
                self.__write(compress_end(self.__ctx))
            else:
                return compress_end(self.__ctx)

Response: Finalise lz4 frame by writing any remaining as return from this function.


Instruction:
Can you tell what is the following Python 3 function doing
def get_value_name(self, pretty=False):
        if pretty:
            return "%s (%x)" % (
                self.enums.get(self._value, "n/a"),
                self._value
            )

        return self.enums.get(self._value, "n/a")

Response: Get the name of the value of the n - A A


Instruction:
Explain what the following Python 3 code does
def set_value(self, value, force=False):
        if force:
            self._value = value
            return

        if value is None:
            self._value = value
            return

        if isinstance(value, six.integer_types):
            self._value = value
            return

        if isinstance(value, six.string_types):
            for v, n in self.enums.items():
                if n == value:
                    self._value = v
                    return
            raise ValueError("Unable to find value name in enum list")

        raise TypeError(
            "Value for '%s' must by of type String or Integer not '%s'" % (
                self.name,
                type(value)
            )
        )

Response: Set the value of the attribute.


Instruction:
Given the following Python 3 function, write the documentation
def dissect(self, data):

        size = struct.calcsize("B")
        if len(data) < size:
            raise NotEnoughData(
                "Not enough data to decode field '%s' value" % self.name
            )

        curve_type = struct.unpack("B", data[:size])[0]
        if curve_type == 0x03:
            self._value = ECParametersNamedCurveField("none")
            data = self._value.dissect(data)
        else:
            raise NotImplementedError(
                "Decoding of KeyExchange message for curve 0x%.2X not implemented" % curve_type
            )
        return data

Response: Dissect the field value from the given bytes.


Instruction:
Given the following Python 3 function, write the documentation
def compute_wcs(key, challenge):
    key = key.encode('utf8')
    challenge = challenge.encode('utf8')
    sig = hmac.new(key, challenge, hashlib.sha256).digest()
    return binascii.b2a_base64(sig).strip()

Response:
Compute an WAMP - CRA authentication signature from an authentication
    challenge and a derived key.


Instruction:
Implement a function in Python 3 to
register a procedure on a Client over the Router.

Response:
def _register_procedure(self, procedure_name, invocation_policy="single"):
        """ Register a "procedure" on a Client as callable over the Router.
        """
        options = {"invoke": invocation_policy}
        message = Register(procedure=procedure_name, options=options)
        request_id = message.request_id

        try:
            self.send_message(message)
        except ValueError:
            raise WampProtocolError(
                "failed to register callee: %s", procedure_name
            )

        self.request_ids[request_id] = procedure_name


Instruction:
Can you tell what is the following Python 3 function doing
def start(self):
        if self.started is True:
            raise WampyError("Router already started")

        # will attempt to connect or start up the CrossBar
        crossbar_config_path = self.config_path
        cbdir = self.crossbar_directory

        # starts the process from the root of the test namespace
        cmd = [
            'crossbar', 'start',
            '--cbdir', cbdir,
            '--config', crossbar_config_path,
        ]

        self.proc = subprocess.Popen(cmd, preexec_fn=os.setsid)

        self._wait_until_ready()
        logger.info(
            "Crosbar.io is ready for connections on %s (IPV%s)",
            self.url, self.ipv
        )

        self.started = True

Response: Start the Crossbar. io process.


Instruction:
Explain what the following Python 3 code does
def _get_handshake_headers(self, upgrade):
        headers = []
        # https://tools.ietf.org/html/rfc6455
        headers.append("GET {} HTTP/1.1".format(self.websocket_location))
        headers.append("Host: {}:{}".format(self.host, self.port))
        headers.append("Upgrade: websocket")
        headers.append("Connection: Upgrade")
        # Sec-WebSocket-Key header containing base64-encoded random bytes,
        # and the server replies with a hash of the key in the
        # Sec-WebSocket-Accept header. This is intended to prevent a caching
        # proxy from re-sending a previous WebSocket conversation and does not
        # provide any authentication, privacy or integrity
        headers.append("Sec-WebSocket-Key: {}".format(self.key))
        headers.append("Origin: ws://{}:{}".format(self.host, self.port))
        headers.append("Sec-WebSocket-Version: {}".format(WEBSOCKET_VERSION))

        if upgrade:
            headers.append("Sec-WebSocket-Protocol: {}".format(
                WEBSOCKET_SUBPROTOCOLS)
            )

        logger.debug("connection headers: %s", headers)

        return headers

Response: Return the HTTP upgrade handshake headers.


Instruction:
How would you code a function in Python 3 to
parse a URL into a set of attributes.

Response:
def parse_url(self):
        """ Parses a URL of the form:

        - ws://host[:port][path]
        - wss://host[:port][path]
        - ws+unix:///path/to/my.socket

        """
        self.scheme = None
        self.resource = None
        self.host = None
        self.port = None

        if self.url is None:
            return

        scheme, url = self.url.split(":", 1)
        parsed = urlsplit(url, scheme="http")
        if parsed.hostname:
            self.host = parsed.hostname
        elif '+unix' in scheme:
            self.host = 'localhost'
        else:
            raise ValueError("Invalid hostname from: %s", self.url)

        if parsed.port:
            self.port = parsed.port

        if scheme == "ws":
            if not self.port:
                self.port = 8080
        elif scheme == "wss":
            if not self.port:
                self.port = 443
        elif scheme in ('ws+unix', 'wss+unix'):
            pass
        else:
            raise ValueError("Invalid scheme: %s" % scheme)

        if parsed.path:
            resource = parsed.path
        else:
            resource = "/"

        if '+unix' in scheme:
            self.unix_socket_path = resource
            resource = '/'

        if parsed.query:
            resource += "?" + parsed.query

        self.scheme = scheme
        self.resource = resource


Instruction:
How would you code a function in Python 3 to
generate a mask for the given key and data.

Response:
def generate_mask(cls, mask_key, data):
        """ Mask data.

        :Parameters:
            mask_key: byte string
                4 byte string(byte), e.g. '\x10\xc6\xc4\x16'
            data: str
                data to mask

        """
        # Masking of WebSocket traffic from client to server is required
        # because of the unlikely chance that malicious code could cause
        # some broken proxies to do the wrong thing and use this as an
        # attack of some kind. Nobody has proved that this could actually
        # happen, but since the fact that it could happen was reason enough
        # for browser vendors to get twitchy, masking was added to remove
        # the possibility of it being used as an attack.
        if data is None:
            data = ""

        data = bytearray(data, 'utf-8')

        _m = array.array("B", mask_key)
        _d = array.array("B", data)

        for i in range(len(_d)):
            _d[i] ^= _m[i % 4]

        return _d.tostring()


Instruction:
Can you tell what is the following Python 3 function doing
def generate_bytes(cls, payload, fin_bit, opcode, mask_payload):
        # the first byte contains the FIN bit, the 3 RSV bits and the
        # 4 opcode bits and for a client will *always* be 1000 0001 (or 129).
        # so we want the first byte to look like...
        #
        #  1 0 0 0 0 0 0 1   (1 is a text frame)
        # +-+-+-+-+-------+
        # |F|R|R|R| opcode|
        # |I|S|S|S|       |
        # |N|V|V|V|       |
        # | |1|2|3|       |
        # +-+-+-+-+-------+
        # note that because all RSV bits are zero, we can ignore them

        # this shifts each bit into position and bitwise ORs them together,
        # using the struct module to pack them as incoming network bytes
        frame = pack(
            '!B', (
                (fin_bit << 7) | opcode
            )
        )

        # the second byte - and maybe the 7 after this, we'll use to tell
        # the server how long our payload is.

        #                 +-+-------------+-------------------------------+
        #                 |M| Payload len |    Extended payload length    |
        #                 |A|     (7)     |             (16/63)           |
        #                 |S|             |   (if payload len==126/127)   |
        #                 |K|             |                               |
        # +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - +
        # |     Extended payload length continued, if payload len == 127  |
        # + - - - - - - - - - - - - - - - +-------------------------------+

        # the mask is always included with client -> server, so the first bit
        # of the second byte is always 1 which flags that the data is masked,
        # i.e. encoded
        if mask_payload:
            mask_bit = 1 << 7
        else:
            mask_bit = 0 << 7
        # next we have to | this bit with the payload length.
        # note that we ensure that the payload is utf-8 encoded before we take
        # the length because unicode characters can be >1 bytes in length and
        # lead to bugs if we just do ``len(payload)``.
        length = len(payload.encode('utf-8'))

        if length >= Frame.MAX_LENGTH:
            raise WebsocktProtocolError("data is too long")

        # the second byte contains the payload length and mask
        if length < Frame.LENGTH_7:
            # we can simply represent payload length with first 7 bits
            frame += pack('!B', (mask_bit | length))
        elif length < Frame.LENGTH_16:
            frame += pack('!B', (mask_bit | 126)) + pack('!H', length)
        else:
            frame += pack('!B', (mask_bit | 127)) + pack('!Q', length)

        if mask_payload:
            # we always mask frames from the client to server
            # use a string of n random buffered_bytes for the mask
            mask_key = os.urandom(4)
            mask_data = cls.generate_mask(mask_key=mask_key, data=payload)
            mask = mask_key + mask_data
            frame += mask
        else:
            frame += bytearray(payload, 'utf-8')

        return bytearray(frame)

Response: Generate the bytes for the current state of the entry in the cache.


Instruction:
How would you code a function in Python 3 to
add an edge to the internal set.

Response:
def add_edge(self, edge):
        """
        Add edge (u, v) to the graph. Raises InvariantError if adding the edge
        would form a cycle.
        """
        u, v = edge

        both_exist = u in self.vertices and v in self.vertices
        # Using `is` because if they belong to the same component, they MUST
        # share the same set object!
        if both_exist and self.components[u] is self.components[v]:
            # Both vertices are part of the same connected component.
            raise InvariantError('Adding %r would form a cycle' % (edge,))
        if u == v:
            raise InvariantError('Cannot add loop: %r' % (edge,))

        # Ensure the vertices exist in the graph.
        self.add_vertex(u)
        self.add_vertex(v)

        # Add the edges to each other.
        self._vertices[u].add(v)
        self._vertices[v].add(u)

        # Add all of the smaller components to the bigger one.
        smaller_component, bigger_component = self.sort_components(u, v)
        for vertex in smaller_component:
            bigger_component.add(vertex)
            # And with this assignment, say bye-bye to the smaller component.
            self.components[vertex] = bigger_component


Instruction:
How would you explain what the following Python 3 function does
def edges(self):
        canonical_edges = set()
        for v1, neighbours in self._vertices.items():
            for v2 in neighbours:
                edge = self.canonical_order((v1, v2))
                canonical_edges.add(edge)
        return canonical_edges

Response: Returns a set of edges in canonical order.


Instruction:
How would you explain what the following Python 3 function does
def ordered_deduplicate(sequence):

    seen = set()
    # Micro optimization: each call to seen_add saves an extra attribute
    # lookup in most iterations of the loop.
    seen_add = seen.add

    return tuple(x for x in sequence if not (x in seen or seen_add(x)))

Response: Returns the sequence with the duplicates removed preserving input order.


Instruction:
How would you explain what the following Python 3 function does
def hash_parameters(words, minimize_indices=False):
    # Ensure that we have an indexable sequence.
    words = tuple(words)

    # Delegate to the hash builder.
    return CzechHashBuilder(words).hash_info

Response: Returns a CzechHashBuilder object that can be used to generate the hash parameters for the given set of words.


Instruction:
Write a Python 3 script for
creating a pickable hash function for the given sequence of words.

Response:
def make_pickable_hash(words, *args, **kwargs):
    """
    Creates an ordered, minimal perfect hash function for the given sequence
    of words.

    >>> hf = make_pickable_hash(['sun', 'mon', 'tue', 'wed', 'thu',
    ...                          'fri', 'sat'])
    >>> hf('fri')
    5
    >>> hf('sun')
    0
    """
    return PickableHash(CzechHashBuilder(words, *args, **kwargs)).czech_hash


Instruction:
How would you explain what the following Python 3 function does
def make_dict(name, words, *args, **kwargs):

    info = CzechHashBuilder(words, *args, **kwargs)

    # Create a docstring that at least describes where the class came from...
    doc = """
        Dictionary-like object that uses minimal perfect hashing, perserving
        original order. This class was generated by `%s.%s(%r, ...)`.
        """ % (__name__, make_dict.__name__, name)

    # Delegate to create_dict.
    return create_dict_subclass(name, info.hash_function, info.words, doc)

Response: Create a dictionary from a sequence of words.


Instruction:
Create a Python 3 function to
return the hash function proper for the given object.

Response:
def hash_function(self):
        """
        Returns the hash function proper. Ensures that `self` is not bound to
        the returned closure.
        """
        assert hasattr(self, 'f1') and hasattr(self, 'f2')

        # These are not just convenient aliases for the given
        # attributes; if `self` would creep into the returned closure,
        # that would ensure that a reference to this big, fat object
        # would be kept alive; hence, any hash function would carry
        # around all of the auxiliary state that was created during the
        # generation of the hash parameters.  Omitting `self` ensures
        # this object has a chance to be garbage collected.
        f1, f2, g = self.f1, self.f2, self.g

        def czech_hash(word):
            v1 = f1(word)
            v2 = f2(word)
            return g[v1] + g[v2]

        return czech_hash


Instruction:
Create a Python 3 function to
generate an acyclic graph for the given words.

Response:
def generate_acyclic_graph(self):
        """
        Generates an acyclic graph for the given words.
        Adds the graph, and a list of edge-word associations to the object.
        """

        # Maximum length of each table, respectively.
        # Hardcoded n = cm, where c = 3
        # There might be a good way to choose an appropriate C,
        # but [1] suggests the average amount of iterations needed
        # to generate an acyclic graph is sqrt(3).
        self.n = 3 * len(self.words)

        max_tries = len(self.words) ** 2
        for trial in range(max_tries):
            try:
                self.generate_or_fail()
            except forest.InvariantError:
                continue
            else:
                # Generated successfully!
                self.trials_taken = trial + 1
                return

        raise RuntimeError("Could not generate graph in "
                           "{} tries".format(max_tries))


Instruction:
Implement a function in Python 3 to
generate random tables for given word lists.

Response:
def generate_random_table(self):
        """
        Generates random tables for given word lists.
        """
        table = list(range(0, self.n))
        random.shuffle(table)
        return table


Instruction:
How would you explain what the following Python 3 function does
def generate_func(self, table):

        # Ensure that `self` isn't suddenly in the closure...
        n = self.n

        def func(word):
            return sum(x * ord(c) for x, c in zip(table, word)) % n

        return func

Response: Generates a random mini - hashing function.


Instruction:
Write a Python 3 function that can
create a dict subclass with the given name and hash_function and the given slots and documentation.

Response:
def create_dict_subclass(name, hash_func, slots, doc):
    """
    Creates a dict subclass named name, using the hash_function to index
    hash_length items. Doc should be any additional documentation added to the
    class.
    """

    hash_length = len(slots)

    # Returns array index -- raises a KeyError if the key does not match
    # its slot value.
    def index_or_key_error(key):
        index = hash_func(key)
        # Make sure the key is **exactly** the same.
        if key != slots[index]:
            raise KeyError(key)
        return index

    def init(self, *args, **kwargs):
        self._arr = [None] * hash_length
        self._len = 0

        # Delegate initialization to update provided by MutableMapping:
        self.update(*args, **kwargs)

    def getitem(self, key):
        index = index_or_key_error(key)
        if self._arr[index] is None:
            raise KeyError(key)
        return self._arr[index][1]

    def setitem(self, key, value):
        index = index_or_key_error(key)
        self._arr[index] = (key, value)

    def delitem(self, key):
        index = index_or_key_error(key)
        if self._arr[index] is None:
            raise KeyError(key)
        self._arr[index] = None

    def dict_iter(self):
        return (pair[0] for pair in self._arr if pair is not None)

    def dict_len(self):
        # TODO: Make this O(1) using auxiliary state?
        return sum(1 for _ in self)

    def dict_repr(self):
        arr_repr = (repr(pair) for pair in self._arr if pair is not None)
        return ''.join((name, '([', ', '.join(arr_repr), '])'))

    # Inheriting from MutableMapping gives us a whole whackload of methods for
    # free.
    bases = (collections.MutableMapping,)

    return type(name, bases, {
        '__init__': init,
        '__doc__': doc,

        '__getitem__': getitem,
        '__setitem__': setitem,
        '__delitem__': delitem,
        '__iter__': dict_iter,
        '__len__': dict_len,

        '__repr__': dict_repr,
    })


Instruction:
Write a Python 3 function for
checking that the geojson data is a feature collection and that all features are valid.

Response:
def validate(data, skiperrors=False, fixerrors=True):
    """Checks that the geojson data is a feature collection, that it
    contains a proper "features" attribute, and that all features are valid too.
    Returns True if all goes well.

    - skiperrors will throw away any features that fail to validate.
    - fixerrors will attempt to auto fix any minor errors without raising exceptions.
    """

    if not "type" in data:
        if fixerrors:
            data["type"] = "FeatureCollection"
        else:
            raise ValueError("The geojson data needs to have a type key")
    if not data["type"] == "FeatureCollection":
        if fixerrors:
            data["type"] = "FeatureCollection"
        else:
            raise ValueError("The geojson data needs to be a feature collection")
    if "features" in data:
        if not isinstance(data["features"], list):
            raise ValueError("The features property needs to be a list")
    else: raise ValueError("The FeatureCollection needs to contain a 'features' property")

    if skiperrors:
        for featuredict in data["features"]:
            feat = Feature(featuredict)
            try: feat.validate(fixerrors)
            except: data["features"].remove(featuredict)

    else:
        for featuredict in data["features"]:
            feat = Feature(featuredict)
            feat.validate(fixerrors)

    return True


Instruction:
How would you explain what the following Python 3 function does
def validate(self, fixerrors=True):

        # validate nullgeometry or has type and coordinates keys
        if not self._data:
            # null geometry, no further checking needed
            return True
        elif "type" not in self._data or "coordinates" not in self._data:
            raise Exception("A geometry dictionary or instance must have the type and coordinates entries")

        # first validate geometry type
        if not self.type in ("Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon"):
            if fixerrors:
                coretype = self.type.lower().replace("multi","")
                if coretype == "point":
                    newtype = "Point"
                elif coretype == "linestring":
                    newtype = "LineString"
                elif coretype == "polygon":
                    newtype = "Polygon"
                else:
                    raise Exception('Invalid geometry type. Must be one of: "Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon"')

                if self.type.lower().startswith("multi"):
                    newtype = "Multi" + newtype

                self.type = newtype
            else:
                raise Exception('Invalid geometry type. Must be one of: "Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon"')

        # then validate coordinate data type
        coords = self._data["coordinates"]
        if not isinstance(coords, (list,tuple)): raise Exception("Coordinates must be a list or tuple type")

        # then validate coordinate structures
        if self.type == "Point":
            if not len(coords) == 2: raise Exception("Point must be one coordinate pair")
        elif self.type in ("MultiPoint","LineString"):
            if not len(coords) > 1: raise Exception("MultiPoint and LineString must have more than one coordinates")
        elif self.type == "MultiLineString":
            for line in coords:
                if not len(line) > 1: raise Exception("All LineStrings in a MultiLineString must have more than one coordinate")
        elif self.type == "Polygon":
            for exterior_or_holes in coords:
                if not len(exterior_or_holes) >= 3: raise Exception("The exterior and all holes in a Polygon must have at least 3 coordinates")
        elif self.type == "MultiPolygon":
            for eachmulti in coords:
                for exterior_or_holes in eachmulti:
                    if not len(exterior_or_holes) >= 3: raise Exception("The exterior and all holes in all Polygons of a MultiPolygon must have at least 3 coordinates")

        # validation successful
        return True

Response: Validates that the geometry is correctly formatted according to the geometry type and coordinates.


Instruction:
Can you generate the documentation for the following Python 3 function
def validate(self, fixerrors=True):
        if not "type" in self._data or self._data["type"] != "Feature":
            if fixerrors:
                self._data["type"] = "Feature"
            else:
                raise Exception("A geojson feature dictionary must contain a type key and it must be named 'Feature'.")
        if not "geometry" in self._data:
            if fixerrors:
                self.geometry = Geometry() # nullgeometry
            else:
                raise Exception("A geojson feature dictionary must contain a geometry key.")
        if not "properties" in self._data or not isinstance(self.properties,dict):
            if fixerrors:
                self._data["properties"] = dict()
            else:
                raise Exception("A geojson feature dictionary must contain a properties key and it must be a dictionary type.")
        self.geometry.validate(fixerrors)
        return True

Response: Validates that the feature is correctly formatted.


Instruction:
Implement a function in Python 3 to
collect and return a list of all attributes and properties used in any of the features.

Response:
def all_attributes(self):
        """
        Collect and return a list of all attributes/properties/fields used in any of the features.
        """
        features = self._data["features"]
        if not features: return []
        elif len(features) == 1: return features[0]["properties"].keys()
        else:
            fields = set(features[0]["properties"].keys())
            for feature in features[1:]:
                fields.update(feature["properties"].keys())
            return list(fields)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def common_attributes(self):
        features = self._data["features"]
        if not features: return []
        elif len(features) == 1: return features[0]["properties"].keys()
        else:
            fields = set(features[0]["properties"].keys())
            for feature in features[1:]:
                fields.intersection_update(feature["properties"].keys())
            return list(fields)

Response: Collect and return a list of attributes and properties common to all features.


Instruction:
Explain what the following Python 3 code does
def add_feature(self, obj=None, geometry=None, properties=None):
        properties = properties or {}
        if isinstance(obj, Feature):
            # instead of creating copy, the original feat should reference the same one that was added here
            feat = obj._data
        elif isinstance(obj, dict):
            feat = obj.copy()
        else:
            feat = Feature(geometry=geometry, properties=properties).__geo_interface__
        self._data["features"].append(feat)

Response: Adds a given feature to the internal data structure.


Instruction:
How would you explain what the following Python 3 function does
def define_crs(self, type, name=None, link=None, link_type=None):
        if not type in ("name","link"): raise Exception("type must be either 'name' or 'link'")
        crs = self._data["crs"] = {"type":type, "properties":{} }
        if type == "name":
            if not name: raise Exception("name argument must be given")
            crs["properties"]["name"] = name
        elif type == "link":
            if not link: raise Exception("link argument must be given")
            crs["properties"]["href"] = link
            if link_type:
                crs["properties"]["type"] = link_type

Response: Define the coordinate reference system for the geojson file.


Instruction:
Given the following Python 3 function, write the documentation
def update_bbox(self):

        xmins, ymins, xmaxs, ymaxs = zip(*(feat.geometry.bbox for feat in self if feat.geometry.type != "Null"))
        bbox = [min(xmins), min(ymins), max(xmaxs), max(ymaxs)]
        self._data["bbox"] = bbox

Response: Recalculates the bbox region attribute for the entire file.


Instruction:
Explain what the following Python 3 code does
def add_unique_id(self):

        uid = 0
        for feature in self._data["features"]:
            if feature["properties"].get("id"):
                raise Exception("one of the features already had an id field")
            feature["properties"]["id"] = uid
            uid += 1

Response: Adds a unique id property to each feature.


Instruction:
Write a Python 3 function for
calculating and adds a bbox attribute to the geojson entry of all features in the archive.

Response:
def add_all_bboxes(self):
        """
        Calculates and adds a bbox attribute to the geojson entry of all feature geometries, updating any existing ones.
        """
        for feature in self:
            if feature.geometry.type != "Null":
                feature.geometry._data["bbox"] = Feature(feature).geometry.bbox


Instruction:
Write a Python 3 script for
saving the object to a file.

Response:
def save(self, savepath, **kwargs):
        """
        Saves the geojson instance to file. To save with a different text encoding use the 'encoding' argument.

        Parameters:

        - **savepath**: Filepath to save the file.
        """

        self.update_bbox()
        tempfile = open(savepath,"w")
        json.dump(self._data, tempfile, **kwargs)
        tempfile.close()


Instruction:
How would you explain what the following Python 3 function does
def _loadfilepath(self, filepath, **kwargs):
        with open(filepath, "r") as f:
            data = json.load(f, **kwargs)
        return data

Response: This loads a geojson file into a python dict using the json module.


Instruction:
Given the following Python 3 function, write the documentation
def _prepdata(self):

        # if missing, compute and add bbox
        if not self._data.get("bbox"):
            self.update_bbox()

        # if missing, set crs to default crs (WGS84), see http://geojson.org/geojson-spec.html
        if not self._data.get("crs"):
            self._data["crs"] = {"type":"name",
                               "properties":{"name":"urn:ogc:def:crs:OGC:2:84"}}

Response: Adds potentially missing items to the geojson dictionary


Instruction:
Can you generate a brief explanation for the following Python 3 code
def hash_parameters(keys, minimize=True, to_int=None):

    # If to_int is not assigned, simply use the identity function.
    if to_int is None:
        to_int = __identity

    key_to_original = {to_int(original): original for original in keys}

    # Create a set of all items to be hashed.
    items = list(key_to_original.keys())

    if minimize:
        offset = 0 - min(items)
        items = frozenset(x + offset for x in items)
    else:
        offset = 0

    # 1. Start with a square array (not stored) that is t units on each side.
    # Choose a t such that t * t >= max(S)
    t = choose_best_t(items)
    assert t * t > max(items) and t * t >= len(items)

    # 2. Place each key K in the square at location (x,y), where
    # x = K mod t, y = K / t.
    row_queue = place_items_in_square(items, t)

    # 3. Arrange rows so that they'll fit into one row and generate a
    # displacement vector.
    final_row, displacement_vector = arrange_rows(row_queue, t)

    # Translate the internal keys to their original items.
    slots = tuple(key_to_original[item - offset] if item is not None else None
                  for item in final_row)

    # Return the parameters
    return HashInfo(
        t=t,
        slots=slots,
        r=displacement_vector,
        offset=offset,
        to_int=to_int if to_int is not __identity else None
    )

Response: Calculates the parameters for a perfect hash.


Instruction:
Explain what the following Python 3 code does
def place_items_in_square(items, t):

    # A minheap (because that's all that heapq supports :/)
    # of the length of each row. Why this is important is because
    # we'll be popping the largest rows when figuring out row displacements.
    # Each item is a tuple of (t - |row|, y, [(xpos_1, item_1), ...]).
    # Until the call to heapq.heapify(), the rows are ordered in
    # increasing row number (y).
    rows = [(t, y, []) for y in range(t)]

    for item in items:
        # Calculate the cell the item should fall in.
        x = item % t
        y = item // t

        # Push the item to its corresponding row...
        inverse_length, _, row_contents = rows[y]
        heapq.heappush(row_contents, (x, item))

        # Ensure the heap key is kept intact.
        rows[y] = inverse_length - 1, y, row_contents

    assert all(inv_len == t - len(rows) for inv_len, _, rows in rows)

    heapq.heapify(rows)

    # Return only rows that are populated.
    return [row for row in rows if row[2]]

Response: Place items in a square of a tree.


Instruction:
How would you implement a function in Python 3 that
arranges the items from its conceptual square to one list.

Response:
def arrange_rows(row_queue, t):
    """
    Takes a priority queue as generated by place_items_in_square().
    Arranges the items from its conceptual square to one list.  Returns
    both the resultant vector, plus the displacement vector, to be used
    in the final output hash function.

    >>> rows = [(2, 1, [(0, 1), (1, 5)]), (3, 3, [(1, 7)])]
    >>> result, displacements = arrange_rows(rows, 4)
    >>> result
    (1, 5, 7)
    >>> displacements
    (None, 0, None, 1)

    >>> rows = [(1, 1, [(0, 1), (2, 7)]), (2, 2, [(1, 5)])]
    >>> result, displacements = arrange_rows(rows, 3)
    >>> result
    (1, 5, 7)
    >>> displacements
    (None, 0, 0)
    """

    # Create a set of all of the unoccupied columns.
    max_columns = t ** 2
    cols = ((x, True) for x in range(max_columns))
    unoccupied_columns = collections.OrderedDict(cols)

    # Create the resultant and displacement vectors.
    result = [None] * max_columns
    displacements = [None] * t

    while row_queue:
        # Get the next row to place.
        _inverse_length, y, row = heapq.heappop(row_queue)

        offset = find_first_fit(unoccupied_columns, row, max_columns)
        # Calculate the offset of the first item.
        first_item_x = row[0][0]

        displacements[y] = offset
        for x, item in row:
            actual_x = x + offset
            result[actual_x] = item
            del unoccupied_columns[actual_x]

    return tuple(trim_nones_from_right(result)), tuple(displacements)


Instruction:
Here you have a function in Python 3, explain what it does
def find_first_fit(unoccupied_columns, row, row_length):
    for free_col in unoccupied_columns:
        # The offset is that such that the first item goes in the free column.
        first_item_x = row[0][0]
        offset = free_col - first_item_x
        if check_columns_fit(unoccupied_columns, row, offset, row_length):
            return offset

    raise ValueError("Row cannot bossily fit in %r: %r"
                     % (list(unoccupied_columns.keys()), row))

Response: Find the first index that the row can fit in.


Instruction:
Can you tell what is the following Python 3 function doing
def check_columns_fit(unoccupied_columns, row, offset, row_length):
    for index, item in row:
        adjusted_index = (index + offset) % row_length

        # Check if the index is in the appropriate place.
        if adjusted_index not in unoccupied_columns:
            return False

    return True

Response: Checks if all the occupied columns in the row fit in the indices


Instruction:
Can you generate a brief explanation for the following Python 3 code
def print_square(row_queue, t):
    occupied_rows = {y: row for _, y, row in row_queue}

    empty_row = ', '.join('...' for _ in range(t))
    for y in range(t):
        print('|', end=' ')
        if y not in occupied_rows:
            print(empty_row, end=' ')
        else:
            row = dict(occupied_rows[y])
            all_cols = ('%3d' % row[x] if x in row else '...'
                        for x in range(t))
            print(', '.join(all_cols), end=' ')

        print("|")

Response: Prints a row queue as its conceptual square array.


Instruction:
Write a Python 3 function that can
return the list with all the Nones at the right end.

Response:
def trim_nones_from_right(xs):
    """
    Returns the list without all the Nones at the right end.

    >>> trim_nones_from_right([1, 2, None, 4, None, 5, None, None])
    [1, 2, None, 4, None, 5]

    """
    # Find the first element that does not contain none.
    for i, item in enumerate(reversed(xs)):
        if item is not None:
            break

    return xs[:-i]


Instruction:
Can you tell what is the following Python 3 function doing
def make_hash(keys, **kwargs):
    params = hash_parameters(keys, **kwargs)

    t = params.t
    r = params.r
    offset = params.offset
    to_int = params.to_int if params.to_int else __identity

    def perfect_hash(x):
        val = to_int(x) + offset
        x = val % t
        y = val // t
        return x + r[y]

    # Undocumented properties, but used in make_dict()...
    perfect_hash.length = len(params.slots)
    perfect_hash.slots = params.slots
    return perfect_hash

Response: Creates a hash function from the given keys.


Instruction:
Create a Python 3 function for
creating a dictionary - like mapping class that uses perfect hashing.

Response:
def make_dict(name, keys, **kwargs):
    """
    Creates a dictionary-like mapping class that uses perfect hashing.
    ``name`` is the proper class name of the returned class. See
    ``hash_parameters()`` for documentation on all arguments after
    ``name``.

    >>> MyDict = make_dict('MyDict', '+-<>[],.', to_int=ord)
    >>> d = MyDict([('+', 1), ('-', 2)])
    >>> d[','] = 3
    >>> d
    MyDict([('+', 1), (',', 3), ('-', 2)])
    >>> del d['+']
    >>> del d['.']
    Traceback (most recent call last):
    ...
    KeyError: '.'
    >>> len(d)
    2
    """
    hash_func = make_hash(keys, **kwargs)
    slots = hash_func.slots

    # Create a docstring that at least describes where the class came from...
    doc = """
        Dictionary-like object that uses perfect hashing. This class was
        generated by `%s.%s(%r, ...)`.
        """ % (__name__, make_dict.__name__, name)

    return create_dict_subclass(name, hash_func, slots, doc)


Instruction:
Implement a Python 3 function for
getting the timestamp from the timestamp file.

Response:
def _get_timestamp(dirname_full, remove):
    """
    Get the timestamp from the timestamp file.

    Optionally mark it for removal if we're going to write another one.
    """
    record_filename = os.path.join(dirname_full, RECORD_FILENAME)

    if not os.path.exists(record_filename):
        return None

    mtime = os.stat(record_filename).st_mtime
    mtime_str = datetime.fromtimestamp(mtime)
    print('Found timestamp {}:{}'.format(dirname_full, mtime_str))
    if Settings.record_timestamp and remove:
        OLD_TIMESTAMPS.add(record_filename)
    return mtime


Instruction:
Create a Python 3 function to
get the timestamp of a node in the cache or fill it if it doesn t exist

Response:
def _get_timestamp_cached(dirname_full, remove):
    """
    Get the timestamp from the cache or fill the cache
    Much quicker than reading the same files over and over
    """
    if dirname_full not in TIMESTAMP_CACHE:
        mtime = _get_timestamp(dirname_full, remove)
        TIMESTAMP_CACHE[dirname_full] = mtime
    return TIMESTAMP_CACHE[dirname_full]


Instruction:
Implement a function in Python 3 to
compare a timestamp file to one passed in. Get the max.

Response:
def _max_timestamps(dirname_full, remove, compare_tstamp):
    """Compare a timestamp file to one passed in. Get the max."""
    tstamp = _get_timestamp_cached(dirname_full, remove)
    return max_none((tstamp, compare_tstamp))


Instruction:
Can you tell what is the following Python 3 function doing
def _get_parent_timestamp(dirname, mtime):
    parent_pathname = os.path.dirname(dirname)

    # max between the parent timestamp the one passed in
    mtime = _max_timestamps(parent_pathname, False, mtime)

    if dirname != os.path.dirname(parent_pathname):
        # this is only called if we're not at the root
        mtime = _get_parent_timestamp(parent_pathname, mtime)

    return mtime

Response: Get the parent timestamp of the passed in directory tree.


Instruction:
Make a summary of the following Python 3 code
def get_walk_after(filename, optimize_after=None):
    if Settings.optimize_after is not None:
        return Settings.optimize_after

    dirname = os.path.dirname(filename)
    if optimize_after is None:
        optimize_after = _get_parent_timestamp(dirname, optimize_after)
    return _max_timestamps(dirname, True, optimize_after)

Response: Get the mtime of the file after the given file.


Instruction:
Explain what the following Python 3 code does
def record_timestamp(pathname_full):
    if Settings.test or Settings.list_only or not Settings.record_timestamp:
        return
    if not Settings.follow_symlinks and os.path.islink(pathname_full):
        if Settings.verbose:
            print('Not setting timestamp because not following symlinks')
        return
    if not os.path.isdir(pathname_full):
        if Settings.verbose:
            print('Not setting timestamp for a non-directory')
        return

    record_filename_full = os.path.join(pathname_full, RECORD_FILENAME)
    try:
        with open(record_filename_full, 'w'):
            os.utime(record_filename_full, None)
        if Settings.verbose:
            print("Set timestamp: {}".format(record_filename_full))
        for fname in OLD_TIMESTAMPS:
            if fname.startswith(pathname_full) and \
               fname != record_filename_full:
                # only remove timestamps below the curent path
                # but don't remove the timestamp we just set!
                os.remove(fname)
                if Settings.verbose:
                    print('Removed old timestamp: {}'.format(fname))
    except IOError:
        print("Could not set timestamp in {}".format(pathname_full))

Response: Record the timestamp of running in a dotfile.


Instruction:
Write a Python 3 script to
optimize a comic archive.

Response:
def walk_comic_archive(filename_full, image_format, optimize_after):
    """
    Optimize a comic archive.

    This is done mostly inline to use the master processes process pool
    for workers. And to avoid calling back up into walk from a dedicated
    module or format processor. It does mean that we block on uncompress
    and on waiting for the contents subprocesses to compress.
    """
    # uncompress archive
    tmp_dir, report_stats = comic.comic_archive_uncompress(filename_full,
                                                           image_format)
    if tmp_dir is None and report_stats:
        return Settings.pool.apply_async(_comic_archive_skip,
                                         args=report_stats)

    # optimize contents of archive
    archive_mtime = os.stat(filename_full).st_mtime
    result_set = walk_dir(tmp_dir, optimize_after, True, archive_mtime)

    # wait for archive contents to optimize before recompressing
    nag_about_gifs = False
    for result in result_set:
        res = result.get()
        nag_about_gifs = nag_about_gifs or res.nag_about_gifs

    # recompress archive
    args = (filename_full, image_format, Settings, nag_about_gifs)
    return Settings.pool.apply_async(comic.comic_archive_compress,
                                     args=(args,))


Instruction:
Write a Python 3 script for
handling things that are not optimizable files.

Response:
def _is_skippable(filename_full):
    """Handle things that are not optimizable files."""

    # File types
    if not Settings.follow_symlinks and os.path.islink(filename_full):
        return True
    if os.path.basename(filename_full) == timestamp.RECORD_FILENAME:
        return True
    if not os.path.exists(filename_full):
        if Settings.verbose:
            print(filename_full, 'was not found.')
        return True

    return False


Instruction:
Explain what the following Python 3 code does
def walk_file(filename, walk_after, recurse=None, archive_mtime=None):
    filename = os.path.normpath(filename)

    result_set = set()

    if _is_skippable(filename):
        return result_set

    walk_after = timestamp.get_walk_after(filename, walk_after)

    # File is a directory
    if os.path.isdir(filename):
        return walk_dir(filename, walk_after, recurse, archive_mtime)

    if _is_older_than_timestamp(filename, walk_after, archive_mtime):
        return result_set

    # Check image format
    try:
        image_format = detect_format.detect_file(filename)
    except Exception:
        res = Settings.pool.apply_async(stats.ReportStats,
                                        (filename,),
                                        {'error': "Detect Format"})
        result_set.add(res)
        image_format = False

    if not image_format:
        return result_set

    if Settings.list_only:
        # list only
        print("{}: {}".format(filename, image_format))
        return result_set

    if detect_format.is_format_selected(image_format, comic.FORMATS,
                                        comic.PROGRAMS):
        # comic archive
        result = walk_comic_archive(filename, image_format, walk_after)
    else:
        # regular image
        args = [filename, image_format, Settings]
        result = Settings.pool.apply_async(optimize.optimize_image,
                                           args=(args,))
    result_set.add(result)
    return result_set

Response: Optimize an individual file.


Instruction:
How would you explain what the following Python 3 function does
def walk_dir(dir_path, walk_after, recurse=None, archive_mtime=None):
    if recurse is None:
        recurse = Settings.recurse

    result_set = set()
    if not recurse:
        return result_set

    for root, _, filenames in os.walk(dir_path):
        for filename in filenames:
            filename_full = os.path.join(root, filename)
            try:
                results = walk_file(filename_full, walk_after, recurse,
                                    archive_mtime)
                result_set = result_set.union(results)
            except Exception:
                print("Error with file: {}".format(filename_full))
                raise

    return result_set

Response: Recursively optimize a directory.


Instruction:
Can you tell what is the following Python 3 function doing
def _walk_all_files():
    # Init records
    record_dirs = set()
    result_set = set()

    for filename in Settings.paths:
        # Record dirs to put timestamps in later
        filename_full = os.path.abspath(filename)
        if Settings.recurse and os.path.isdir(filename_full):
            record_dirs.add(filename_full)

        walk_after = timestamp.get_walk_after(filename_full)
        results = walk_file(filename_full, walk_after, Settings.recurse)
        result_set = result_set.union(results)

    bytes_in = 0
    bytes_out = 0
    nag_about_gifs = False
    errors = []
    for result in result_set:
        res = result.get()
        if res.error:
            errors += [(res.final_filename, res.error)]
            continue
        bytes_in += res.bytes_in
        bytes_out += res.bytes_out
        nag_about_gifs = nag_about_gifs or res.nag_about_gifs

    return record_dirs, bytes_in, bytes_out, nag_about_gifs, errors

Response: Walks all files in the current directory tree and returns a set of records.


Instruction:
Given the following Python 3 function, write the documentation
def run():
    # Setup Multiprocessing
    # manager = multiprocessing.Manager()
    Settings.pool = multiprocessing.Pool(Settings.jobs)

    # Optimize Files
    record_dirs, bytes_in, bytes_out, nag_about_gifs, errors = \
        _walk_all_files()

    # Shut down multiprocessing
    Settings.pool.close()
    Settings.pool.join()

    # Write timestamps
    for filename in record_dirs:
        timestamp.record_timestamp(filename)

    # Finish by reporting totals
    stats.report_totals(bytes_in, bytes_out, nag_about_gifs, errors)

Response: Use preconfigured settings to optimize files.


Instruction:
Here you have a function in Python 3, explain what it does
def replace_ext(filename, new_ext):
    filename_base = os.path.splitext(filename)[0]
    new_filename = '{}.{}'.format(filename_base, new_ext)
    return new_filename

Response: Replace the file extention.


Instruction:
Can you tell what is the following Python 3 function doing
def _cleanup_after_optimize_aux(filename, new_filename, old_format,
                                new_format):
    """
    Replace old file with better one or discard new wasteful file.
    """
    bytes_in = 0
    bytes_out = 0
    final_filename = filename
    try:
        bytes_in = os.stat(filename).st_size
        bytes_out = os.stat(new_filename).st_size
        if (bytes_out > 0) and ((bytes_out < bytes_in) or Settings.bigger):
            if old_format != new_format:
                final_filename = replace_ext(filename,
                                             new_format.lower())
            rem_filename = filename + REMOVE_EXT
            if not Settings.test:
                os.rename(filename, rem_filename)
                os.rename(new_filename, final_filename)
                os.remove(rem_filename)
            else:
                os.remove(new_filename)

        else:
            os.remove(new_filename)
            bytes_out = bytes_in
    except OSError as ex:
        print(ex)

    return final_filename, bytes_in, bytes_out

Response: Remove old file with better one or discard new wasteful file.


Instruction:
Make a summary of the following Python 3 code
def cleanup_after_optimize(filename, new_filename, old_format, new_format):
    final_filename, bytes_in, bytes_out = _cleanup_after_optimize_aux(
        filename, new_filename, old_format, new_format)
    return stats.ReportStats(final_filename, bytes_count=(bytes_in, bytes_out))

Response: Clean up the file after optimisation.


Instruction:
Create a Python 3 function to
parse setup requirements from a requirements. txt file.

Response:
def parse_reqs(filename):
    """Parse setup requirements from a requirements.txt file."""
    install_reqs = parse_requirements(filename, session=False)
    return [str(ir.req) for ir in install_reqs]


Instruction:
Can you generate the documentation for the following Python 3 function
def get_req_list():
    req_list = parse_reqs(REQUIREMENTS['prod'])
    if len(sys.argv) > 2 and sys.argv[2] == ('develop'):
        req_list += parse_reqs(REQUIREMENTS['dev'])
    return req_list

Response: Get the requirements by weather we re building develop or not.


Instruction:
Can you tell what is the following Python 3 function doing
def get_comic_format(filename):
    image_format = None
    filename_ext = os.path.splitext(filename)[-1].lower()
    if filename_ext in _COMIC_EXTS:
        if zipfile.is_zipfile(filename):
            image_format = _CBZ_FORMAT
        elif rarfile.is_rarfile(filename):
            image_format = _CBR_FORMAT
    return image_format

Response: Return the comic format if it is a comic archive.


Instruction:
Make a summary of the following Python 3 code
def _get_archive_tmp_dir(filename):
    head, tail = os.path.split(filename)
    return os.path.join(head, _ARCHIVE_TMP_DIR_TEMPLATE.format(tail))

Response: Get the name of the working dir to use for this filename.


Instruction:
Can you generate the documentation for the following Python 3 function
def comic_archive_uncompress(filename, image_format):
    if not Settings.comics:
        report = ['Skipping archive file: {}'.format(filename)]
        return None, ReportStats(filename, report=report)

    if Settings.verbose:
        truncated_filename = stats.truncate_cwd(filename)
        print("Extracting {}...".format(truncated_filename), end='')

    # create the tmpdir
    tmp_dir = _get_archive_tmp_dir(filename)
    if os.path.isdir(tmp_dir):
        shutil.rmtree(tmp_dir)
    os.mkdir(tmp_dir)

    # extract archvie into the tmpdir
    if image_format == _CBZ_FORMAT:
        with zipfile.ZipFile(filename, 'r') as zfile:
            zfile.extractall(tmp_dir)
    elif image_format == _CBR_FORMAT:
        with rarfile.RarFile(filename, 'r') as rfile:
            rfile.extractall(tmp_dir)
    else:
        report = '{} {} is not a good format'.format(filename, image_format)
        return None, ReportStats(filename, report=report)

    if Settings.verbose:
        print('done')

    return tmp_dir, None

Response: Uncompress a comic archive into a temporary directory.


Instruction:
Write a Python 3 script for
zipping up the files in the tempdir into the new filename.

Response:
def _comic_archive_write_zipfile(new_filename, tmp_dir):
    """Zip up the files in the tempdir into the new filename."""
    if Settings.verbose:
        print('Rezipping archive', end='')
    with zipfile.ZipFile(new_filename, 'w',
                         compression=zipfile.ZIP_DEFLATED) as new_zf:
        root_len = len(os.path.abspath(tmp_dir))
        for r_d_f in os.walk(tmp_dir):
            root = r_d_f[0]
            filenames = r_d_f[2]
            archive_root = os.path.abspath(root)[root_len:]
            for fname in filenames:
                fullpath = os.path.join(root, fname)
                archive_name = os.path.join(archive_root, fname)
                if Settings.verbose:
                    print('.', end='')
                new_zf.write(fullpath, archive_name, zipfile.ZIP_DEFLATED)


Instruction:
Write a Python 3 function for
calling back by every optimization inside a comic archive. When they're all done it creates the new archive and cleans up.

Response:
def comic_archive_compress(args):
    """
    Called back by every optimization inside a comic archive.

    When they're all done it creates the new archive and cleans up.
    """
    try:
        filename, old_format, settings, nag_about_gifs = args
        Settings.update(settings)
        tmp_dir = _get_archive_tmp_dir(filename)

        # archive into new filename
        new_filename = files.replace_ext(filename, _NEW_ARCHIVE_SUFFIX)

        _comic_archive_write_zipfile(new_filename, tmp_dir)

        # Cleanup tmpdir
        if os.path.isdir(tmp_dir):
            if Settings.verbose:
                print('.', end='')
            shutil.rmtree(tmp_dir)
        if Settings.verbose:
            print('done.')

        report_stats = files.cleanup_after_optimize(
            filename, new_filename, old_format, _CBZ_FORMAT)
        report_stats.nag_about_gifs = nag_about_gifs
        stats.report_saved(report_stats)
        return report_stats
    except Exception as exc:
        print(exc)
        traceback.print_exc(exc)
        raise exc


Instruction:
How would you explain what the following Python 3 function does
def mozjpeg(ext_args):
    args = copy.copy(_MOZJPEG_ARGS)
    if Settings.destroy_metadata:
        args += ["-copy", "none"]
    else:
        args += ["-copy", "all"]
    args += ['-outfile']
    args += [ext_args.new_filename, ext_args.old_filename]
    extern.run_ext(args)
    return _JPEG_FORMAT

Response: Create argument list for mozjpeg.


Instruction:
Explain what the following Python 3 code does
def jpegtran(ext_args):
    args = copy.copy(_JPEGTRAN_ARGS)
    if Settings.destroy_metadata:
        args += ["-copy", "none"]
    else:
        args += ["-copy", "all"]
    if Settings.jpegtran_prog:
        args += ["-progressive"]
    args += ['-outfile']
    args += [ext_args.new_filename, ext_args.old_filename]
    extern.run_ext(args)
    return _JPEG_FORMAT

Response: Create argument list for jpegtran.


Instruction:
Implement a function in Python 3 to
run the EXTERNAL program jpegrescan.

Response:
def jpegrescan(ext_args):
    """Run the EXTERNAL program jpegrescan."""
    args = copy.copy(_JPEGRESCAN_ARGS)
    if Settings.jpegrescan_multithread:
        args += ['-t']
    if Settings.destroy_metadata:
        args += ['-s']
    args += [ext_args.old_filename, ext_args.new_filename]
    extern.run_ext(args)
    return _JPEG_FORMAT


Instruction:
Write a Python 3 function that can
parse the command line arguments.

Response:
def get_arguments(args):
    """Parse the command line."""
    usage = "%(prog)s [arguments] [image files]"
    programs_str = ', '.join([prog.__name__ for prog in PROGRAMS])
    description = "Uses "+programs_str+" if they are on the path."
    parser = argparse.ArgumentParser(usage=usage, description=description)
    parser.add_argument("-r", "--recurse", action="store_true",
                        dest="recurse", default=0,
                        help="Recurse down through directories ignoring the"
                        "image file arguments on the command line")
    parser.add_argument("-v", "--verbose", action="count",
                        dest="verbose", default=0,
                        help="Display more output. -v (default) and -vv "
                        "(noisy)")
    parser.add_argument("-Q", "--quiet", action="store_const",
                        dest="verbose", const=-1,
                        help="Display little to no output")
    parser.add_argument("-a", "--enable_advpng", action="store_true",
                        dest="advpng", default=0,
                        help="Optimize with advpng (disabled by default)")
    parser.add_argument("-c", "--comics", action="store_true",
                        dest="comics", default=0,
                        help="Also optimize comic book archives (cbz & cbr)")
    parser.add_argument("-f", "--formats", action="store", dest="formats",
                        default=DEFAULT_FORMATS,
                        help="Only optimize images of the specifed '{}' "
                        "delimited formats from: {}".format(
                            FORMAT_DELIMETER,
                            ', '.join(sorted(ALL_FORMATS))))
    parser.add_argument("-O", "--disable_optipng", action="store_false",
                        dest="optipng", default=1,
                        help="Do not optimize with optipng")
    parser.add_argument("-P", "--disable_pngout", action="store_false",
                        dest="pngout", default=1,
                        help="Do not optimize with pngout")
    parser.add_argument("-J", "--disable_jpegrescan", action="store_false",
                        dest="jpegrescan", default=1,
                        help="Do not optimize with jpegrescan")
    parser.add_argument("-E", "--disable_progressive", action="store_false",
                        dest="jpegtran_prog", default=1,
                        help="Don't try to reduce size by making "
                        "progressive JPEGs with jpegtran")
    parser.add_argument("-Z", "--disable_mozjpeg", action="store_false",
                        dest="mozjpeg", default=1,
                        help="Do not optimize with mozjpeg")
    parser.add_argument("-T", "--disable_jpegtran", action="store_false",
                        dest="jpegtran", default=1,
                        help="Do not optimize with jpegtran")
    parser.add_argument("-G", "--disable_gifsicle", action="store_false",
                        dest="gifsicle", default=1,
                        help="disable optimizing animated GIFs")
    parser.add_argument("-Y", "--disable_convert_type", action="store_const",
                        dest="to_png_formats",
                        const=png.FORMATS,
                        default=png.CONVERTABLE_FORMATS,
                        help="Do not convert other lossless formats like "
                        " {} to PNG when optimizing. By default, {}"
                        " does convert these formats to PNG".format(
                            ', '.join(png.LOSSLESS_FORMATS),
                            PROGRAM_NAME))
    parser.add_argument("-S", "--disable_follow_symlinks",
                        action="store_false",
                        dest="follow_symlinks", default=1,
                        help="disable following symlinks for files and "
                        "directories")
    parser.add_argument("-b", "--bigger", action="store_true",
                        dest="bigger", default=0,
                        help="Save optimized files that are larger than "
                        "the originals")
    parser.add_argument("-t", "--record_timestamp", action="store_true",
                        dest="record_timestamp", default=0,
                        help="Store the time of the optimization of full "
                        "directories in directory local dotfiles.")
    parser.add_argument("-D", "--optimize_after", action="store",
                        dest="optimize_after", default=None,
                        help="only optimize files after the specified "
                        "timestamp. Supercedes -t")
    parser.add_argument("-N", "--noop", action="store_true",
                        dest="test", default=0,
                        help="Do not replace files with optimized versions")
    parser.add_argument("-l", "--list", action="store_true",
                        dest="list_only", default=0,
                        help="Only list files that would be optimized")
    parser.add_argument("-V", "--version", action="version",
                        version=__version__,
                        help="display the version number")
    parser.add_argument("-M", "--destroy_metadata", action="store_true",
                        dest="destroy_metadata", default=0,
                        help="*Destroy* metadata like EXIF and JFIF")
    parser.add_argument("paths", metavar="path", type=str, nargs="+",
                        help="File or directory paths to optimize")
    parser.add_argument("-j", "--jobs", type=int, action="store",
                        dest="jobs", default=multiprocessing.cpu_count(),
                        help="Number of parallel jobs to run simultaneously.")

    return parser.parse_args(args)


Instruction:
Implement a Python 3 function for
processing command line arguments and walk inputs.

Response:
def run(args):
    """Process command line arguments and walk inputs."""
    raw_arguments = get_arguments(args[1:])
    process_arguments(raw_arguments)
    walk.run()
    return True


Instruction:
Write a Python 3 function that can
test to see if the external programs can be run.

Response:
def does_external_program_run(prog, verbose):
    """Test to see if the external programs can be run."""
    try:
        with open('/dev/null') as null:
            subprocess.call([prog, '-h'], stdout=null, stderr=null)
        result = True
    except OSError:
        if verbose > 1:
            print("couldn't run {}".format(prog))
        result = False

    return result


Instruction:
Given the following Python 3 function, write the documentation
def _humanize_bytes(num_bytes, precision=1):
    if num_bytes == 0:
        return 'no bytes'
    if num_bytes == 1:
        return '1 byte'

    factored_bytes = 0
    factor_suffix = 'bytes'
    for factor, suffix in ABBREVS:
        if num_bytes >= factor:
            factored_bytes = num_bytes / factor
            factor_suffix = suffix
            break

    if factored_bytes == 1:
        precision = 0

    return '{:.{prec}f} {}'.format(factored_bytes, factor_suffix,
                                   prec=precision)

Response: Return a humanized string representation of a number of num_bytes.


Instruction:
Create a Python 3 function for
spitting out how much space the optimization saved.

Response:
def new_percent_saved(report_stats):
    """Spit out how much space the optimization saved."""
    size_in = report_stats.bytes_in
    if size_in > 0:
        size_out = report_stats.bytes_out
        ratio = size_out / size_in
        kb_saved = _humanize_bytes(size_in - size_out)
    else:
        ratio = 0
        kb_saved = 0
    percent_saved = (1 - ratio) * 100

    result = '{:.{prec}f}% ({})'.format(percent_saved, kb_saved, prec=2)
    return result


Instruction:
Given the following Python 3 function, write the documentation
def truncate_cwd(full_filename):
    if full_filename.startswith(os.getcwd()):
        truncated_filename = full_filename.split(os.getcwd(), 1)[1]
        truncated_filename = truncated_filename.split(os.sep, 1)[1]
    else:
        truncated_filename = full_filename
    return truncated_filename

Response: Remove the cwd from the full filename.


Instruction:
Create a Python 3 function for
recording the percent saved & print it.

Response:
def report_saved(report_stats):
    """Record the percent saved & print it."""
    if Settings.verbose:
        report = ''
        truncated_filename = truncate_cwd(report_stats.final_filename)

        report += '{}: '.format(truncated_filename)
        total = new_percent_saved(report_stats)
        if total:
            report += total
        else:
            report += '0%'
        if Settings.test:
            report += ' could be saved.'
        if Settings.verbose > 1:
            tools_report = ', '.join(report_stats.report_list)
            if tools_report:
                report += '\n\t' + tools_report
        print(report)


Instruction:
Can you implement a function in Python 3 that
reports the total number and percent of bytes saved.

Response:
def report_totals(bytes_in, bytes_out, nag_about_gifs, errors):
    """Report the total number and percent of bytes saved."""
    if bytes_in:
        bytes_saved = bytes_in - bytes_out
        percent_bytes_saved = bytes_saved / bytes_in * 100
        msg = ''
        if Settings.test:
            if percent_bytes_saved > 0:
                msg += "Could save"
            elif percent_bytes_saved == 0:
                msg += "Could even out for"
            else:
                msg += "Could lose"
        else:
            if percent_bytes_saved > 0:
                msg += "Saved"
            elif percent_bytes_saved == 0:
                msg += "Evened out"
            else:
                msg = "Lost"
        msg += " a total of {} or {:.{prec}f}%".format(
            _humanize_bytes(bytes_saved), percent_bytes_saved, prec=2)
        if Settings.verbose:
            print(msg)
            if Settings.test:
                print("Test run did not change any files.")

    else:
        if Settings.verbose:
            print("Didn't optimize any files.")

    if nag_about_gifs and Settings.verbose:
        print("Most animated GIFS would be better off converted to"
              " HTML5 video")

    if not errors:
        return

    print("Errors with the following files:")
    for error in errors:
        print("{}: {}".format(error[0], error[1]))


Instruction:
Implement a Python 3 function for
providing reporting statistics for a skipped file.

Response:
def skip(type_name, filename):
    """Provide reporting statistics for a skipped file."""
    report = ['Skipping {} file: {}'.format(type_name, filename)]
    report_stats = ReportStats(filename, report=report)
    return report_stats


Instruction:
Here you have a function in Python 3, explain what it does
def gifsicle(ext_args):
    args = _GIFSICLE_ARGS + [ext_args.new_filename]
    extern.run_ext(args)
    return _GIF_FORMAT

Response: Run the EXTERNAL program gifsicle.


Instruction:
Write a Python 3 script to
optimize the file with the external function.

Response:
def _optimize_image_external(filename, func, image_format, new_ext):
    """Optimize the file with the external function."""
    new_filename = filename + TMP_SUFFIX + new_ext
    new_filename = os.path.normpath(new_filename)
    shutil.copy2(filename, new_filename)

    ext_args = ExtArgs(filename, new_filename)
    new_image_format = func(ext_args)

    report_stats = files.cleanup_after_optimize(filename, new_filename,
                                                image_format,
                                                new_image_format)
    percent = stats.new_percent_saved(report_stats)
    if percent != 0:
        report = '{}: {}'.format(func.__name__, percent)
    else:
        report = ''
    report_stats.report_list.append(report)

    return report_stats