Instruction:
Can you generate the documentation for the following Python 3 function
def median_spectrum_from_rectified_image(self, slitlet2d_rect,
                                             sigma_gaussian_filtering=0,
                                             nwinwidth_initial=5,
                                             nwinwidth_refined=7,
                                             times_sigma_threshold=5,
                                             minimum_threshold=None,
                                             npix_avoid_border=0,
                                             nbrightlines=None):
        """Median spectrum and line peaks from rectified image.

        In order to avoid the line ghosts, the line peaks are identified
        independently in the upper and lower halves of the rectified
        image. The final peaks correspond to lines that appear in both
        spectra.

        Parameters
        ----------
        slitlet2d_rect : numpy array
            Rectified slitlet image.
        sigma_gaussian_filtering : float
            Sigma of the gaussian filter to be applied to the spectrum
            in order to avoid problems with saturated lines. This
            filtering is skipped when this parameter is <= 0.
        nwinwidth_initial : int
            Width of the window where each peak must be found using
            the initial method (approximate)
        nwinwidth_refined : int
            Width of the window where each peak location will be
            refined.
        times_sigma_threshold : float
            Times (robust) sigma above the median of the image to set
            the minimum threshold when searching for line peaks.
        minimum_threshold : float or None
            Minimum value of the threshold.
        npix_avoid_border : int
            Number of pixels at the borders of the spectrum where peaks
            are not considered. If zero, the actual number will be
            given by nwinwidth_initial.
        nbrightlines : list
            List with maximum number of brightest lines to be employed
            in the wavelength calibration. The length of the list
            indicates the number of equal-size subintervals in the
            wavelength calibration direction to be considered.
            If this value is [0] or None, all the detected lines will be
            employed.

        Returns
        -------
        sp0 : numpy array
            Median spectrum.
        fxpeaks : numpy array
            Refined location of arc lines (in array index scale).

        """

        # protections
        naxis2, naxis1 = slitlet2d_rect.shape
        if naxis1 != self.bb_nc2_orig - self.bb_nc1_orig + 1:
            raise ValueError("Unexpected slitlet2d_rect naxis1")
        if naxis2 != self.bb_ns2_orig - self.bb_ns1_orig + 1:
            raise ValueError("Unexpected slitlet2d_rect naxis2")

        # lower, middle and upper spectrum trails
        ylower_line = \
            self.list_spectrails[self.i_lower_spectrail].y_rectified
        ymiddle_line = \
            self.list_spectrails[self.i_middle_spectrail].y_rectified
        yupper_line = \
            self.list_spectrails[self.i_upper_spectrail].y_rectified

        ilower = int(ylower_line + 0.5) - self.bb_ns1_orig
        imiddle = int(ymiddle_line + 0.5) - self.bb_ns1_orig
        iupper = int(yupper_line + 0.5) - self.bb_ns1_orig

        # median spectra using different image regions
        sp0_ini = np.median(slitlet2d_rect[ilower:(iupper + 1), :], axis=0)
        sp1_ini = np.median(slitlet2d_rect[ilower:(imiddle + 1), :], axis=0)
        sp2_ini = np.median(slitlet2d_rect[imiddle:(iupper + 1), :], axis=0)

        # gaussian filtering when requested (to avoid line saturation)
        if sigma_gaussian_filtering > 0:
            sp0 = ndimage.filters.gaussian_filter(
                sp0_ini,
                sigma=sigma_gaussian_filtering
            )
            sp1 = ndimage.filters.gaussian_filter(
                sp1_ini,
                sigma=sigma_gaussian_filtering
            )
            sp2 = ndimage.filters.gaussian_filter(
                sp2_ini,
                sigma=sigma_gaussian_filtering
            )
        else:
            sp0 = np.copy(sp0_ini)
            sp1 = np.copy(sp1_ini)
            sp2 = np.copy(sp2_ini)

        # compute threshold
        q25, q50, q75 = np.percentile(sp0, q=[25.0, 50.0, 75.0])
        sigma_g = 0.7413 * (q75 - q25)  # robust standard deviation
        threshold = q50 + times_sigma_threshold * sigma_g
        if abs(self.debugplot) >= 10:
            print("median...........:", q50)
            print("robuts std.......:", sigma_g)
            print("threshold........:", threshold)
        if minimum_threshold is not None:
            if minimum_threshold > threshold:
                threshold = minimum_threshold
        if abs(self.debugplot) >= 10:
            print("minimum threshold:", minimum_threshold)
            print("final threshold..:", threshold)

        # initial location of the peaks (integer values)
        ixpeaks0 = find_peaks_spectrum(sp0, nwinwidth=nwinwidth_initial,
                                       threshold=threshold,
                                       debugplot=self.debugplot)

        # peaks in the lower and upper regions
        ixpeaks1 = find_peaks_spectrum(sp1, nwinwidth=nwinwidth_initial,
                                       threshold=threshold,
                                       debugplot=self.debugplot)
        ixpeaks2 = find_peaks_spectrum(sp2, nwinwidth=nwinwidth_initial,
                                       threshold=threshold,
                                       debugplot=self.debugplot)

        # the peaks are valid if the are also found in the lower and
        # upper regions (with a tolerance of +1 or -1 pixel)
        ixpeaks = []
        for ixpeak in ixpeaks0:
            l1 = ixpeak in np.concatenate((ixpeaks1, ixpeaks1+1, ixpeaks1-1))
            l2 = ixpeak in np.concatenate((ixpeaks2, ixpeaks2+1, ixpeaks2-1))
            if l1 and l2:
                ixpeaks.append(ixpeak)
        ixpeaks = np.array(ixpeaks)
        if abs(self.debugplot) >= 10:
            print("Merged initial list of peaks:\n", ixpeaks)

        # remove peaks too close to any of the borders of the spectrum
        if npix_avoid_border > 0:
            lok_ini = ixpeaks >= npix_avoid_border
            lok_end = ixpeaks <= len(sp0) - 1 - npix_avoid_border
            ixpeaks = ixpeaks[lok_ini * lok_end]

        # select a maximum number of brightest lines in each region
        if nbrightlines is None:
            pass
        elif len(nbrightlines) == 1 and nbrightlines[0] == 0:
            pass
        else:
            if abs(self.debugplot) >= 10:
                print('nbrightlines =', nbrightlines)
                print('ixpeaks in whole spectrum:\n', ixpeaks)
            region_size = (naxis1-1)/len(nbrightlines)
            ixpeaks_filtered = np.array([], dtype=int)
            for iregion, nlines_in_region in enumerate(nbrightlines):
                if nlines_in_region > 0:
                    imin = int(iregion * region_size)
                    imax = int((iregion + 1) * region_size)
                    if iregion > 0:
                        imin += 1
                    ixpeaks_region = \
                        ixpeaks[np.logical_and(ixpeaks >= imin,
                                               ixpeaks <= imax)]
                    if len(ixpeaks_region) > 0:
                        peak_fluxes = sp0[ixpeaks_region]
                        spos = peak_fluxes.argsort()
                        ixpeaks_tmp = ixpeaks_region[spos[-nlines_in_region:]]
                        ixpeaks_tmp.sort()  # in-place sort
                        if abs(self.debugplot) >= 10:
                            print('ixpeaks in region........:\n', ixpeaks_tmp)
                        ixpeaks_filtered = np.concatenate((ixpeaks_filtered,
                                                           ixpeaks_tmp))
            ixpeaks = ixpeaks_filtered
            if abs(self.debugplot) >= 10:
                print('ixpeaks filtered.........:\n', ixpeaks)

        # refined location of the peaks (float values)
        fxpeaks, sxpeaks = refine_peaks_spectrum(sp0, ixpeaks,
                                                 nwinwidth=nwinwidth_refined,
                                                 method="gaussian")

        if abs(self.debugplot) % 10 != 0:
            x = np.arange(self.bb_nc1_orig, self.bb_nc2_orig + 1)
            title = "Slitlet#" + str(self.islitlet) + " (median spectrum)"
            ax = ximplotxy(x, sp1, show=False, title=title,
                           xlabel='pixel coordinate (from 1 to NAXIS1)',
                           ylabel='number of counts',
                           **{'marker': ' ', 'label': 'lower region'})
            ax.plot(x, sp2, label='upper region')
            ax.plot(x, sp0, label='whole region')
            # mark peak location
            ax.plot(ixpeaks + self.bb_nc1_orig,
                    sp0[ixpeaks], 'o', label="initial location")
            ax.plot(fxpeaks + self.bb_nc1_orig,
                    sp0[ixpeaks], 'o', label="refined location")
            ax.legend()
            pause_debugplot(self.debugplot, pltshow=True, tight_layout=False)

        # return median spectrum and refined peak location
        return sp0, fxpeaks

Response: This function takes a rectified image and returns the median spectrum and line peaks from the image.


Instruction:
How would you code a function in Python 3 to
intersect slices with a region.

Response:
def clip_slices(r, region, scale=1):
    '''Intersect slices with a region.'''
    t = []
    for ch in r:
        a1 = intersection(ch[0], region[0], scale=scale)
        if a1 is None:
            continue
        a2 = intersection(ch[1], region[1], scale=scale)
        if a2 is None:
            continue

        t.append((a1, a2))

    return t


Instruction:
Here you have a function in Python 3, explain what it does
def set_base_headers(self, hdr):
        hdr = super(JoinDitheredImagesRecipe, self).set_base_headers(hdr)
        hdr['IMGOBBL'] = 0
        hdr['OBSMODE'] = 'DITHERED_IMAGE'
        return hdr

Response: Set metadata in FITS headers.


Instruction:
Here you have a function in Python 3, explain what it does
def _load_defaults(inventory_path=None, roles=None, extra_vars=None, tags=None,
                   basedir=False):
    """Load common defaults data structures.

    For factorization purpose."""

    extra_vars = extra_vars or {}
    tags = tags or []
    loader = DataLoader()
    if basedir:
        loader.set_basedir(basedir)

    inventory = EnosInventory(loader=loader,
                              sources=inventory_path, roles=roles)

    variable_manager = VariableManager(loader=loader,
                                       inventory=inventory)

    # seems mandatory to load group_vars variable
    if basedir:
        variable_manager.safe_basedir = True

    if extra_vars:
        variable_manager.extra_vars = extra_vars

    # NOTE(msimonin): The ansible api is "low level" in the
    # sense that we are redefining here all the default values
    # that are usually enforce by ansible called from the cli
    Options = namedtuple("Options", ["listtags", "listtasks",
                                     "listhosts", "syntax",
                                     "connection", "module_path",
                                     "forks", "private_key_file",
                                     "ssh_common_args",
                                     "ssh_extra_args",
                                     "sftp_extra_args",
                                     "scp_extra_args", "become",
                                     "become_method", "become_user",
                                     "remote_user", "verbosity",
                                     "check", "tags",
                                     "diff", "basedir"])

    options = Options(listtags=False, listtasks=False,
                      listhosts=False, syntax=False, connection="ssh",
                      module_path=None, forks=100,
                      private_key_file=None, ssh_common_args=None,
                      ssh_extra_args=None, sftp_extra_args=None,
                      scp_extra_args=None, become=None,
                      become_method="sudo", become_user="root",
                      remote_user=None, verbosity=2, check=False,
                      tags=tags, diff=None, basedir=basedir)

    return inventory, variable_manager, loader, options

Response: Load common defaults data structures.


Instruction:
Here you have a function in Python 3, explain what it does
def run_play(play_source, inventory_path=None, roles=None,
             extra_vars=None, on_error_continue=False):
    """Run a play.

    Args:
        pattern_hosts (str): pattern to describe ansible hosts to target.
            see https://docs.ansible.com/ansible/latest/intro_patterns.html
        play_source (dict): ansible task
        inventory_path (str): inventory to use
        extra_vars (dict): extra_vars to use
        on_error_continue(bool): Don't throw any exception in case a host is
            unreachable or the playbooks run with errors

    Raises:
        :py:class:`enoslib.errors.EnosFailedHostsError`: if a task returns an
            error on a host and ``on_error_continue==False``
        :py:class:`enoslib.errors.EnosUnreachableHostsError`: if a host is
            unreachable (through ssh) and ``on_error_continue==False``

    Returns:
        List of all the results
    """
    # NOTE(msimonin): inventory could be infered from a host list (maybe)
    results = []
    inventory, variable_manager, loader, options = _load_defaults(
        inventory_path=inventory_path,
        roles=roles,
        extra_vars=extra_vars)
    callback = _MyCallback(results)
    passwords = {}
    tqm = task_queue_manager.TaskQueueManager(
        inventory=inventory,
        variable_manager=variable_manager,
        loader=loader,
        options=options,
        passwords=passwords,
        stdout_callback=callback)

    # create play
    play_inst = play.Play().load(play_source,
                                 variable_manager=variable_manager,
                                 loader=loader)

    # actually run it
    try:
        tqm.run(play_inst)
    finally:
        tqm.cleanup()

    # Handling errors
    failed_hosts = []
    unreachable_hosts = []
    for r in results:
        if r.status == STATUS_UNREACHABLE:
            unreachable_hosts.append(r)
        if r.status == STATUS_FAILED:
            failed_hosts.append(r)

    if len(failed_hosts) > 0:
        logger.error("Failed hosts: %s" % failed_hosts)
        if not on_error_continue:
            raise EnosFailedHostsError(failed_hosts)
    if len(unreachable_hosts) > 0:
        logger.error("Unreachable hosts: %s" % unreachable_hosts)
        if not on_error_continue:
            raise EnosUnreachableHostsError(unreachable_hosts)

    return results

Response:
Run a play.

    Args:
        pattern_hosts (str): pattern to describe ansible hosts to target.
            see https://docs.ansible.com/ansible/latest/intro_patterns.html
        play_source (dict): ansible task
        inventory_path (str): inventory to use
        extra_vars (dict): extra_vars to use
        on_error_continue(bool): Don't throw any exception in case a host is
            unreachable or the playbooks run with errors

    Raises:
        :py:class:`enoslib.errors.EnosFailedHostsError`: if a task returns an
            error on a host and ``on_error_continue==False``
        :py:class:`enoslib.errors.EnosUnreachableHostsError`: if a host is
            unreachable (through ssh) and ``on_error_continue==False``

    Returns:
        List of all the results


Instruction:
Can you create a Python 3 function that
runs a command on some remote hosts.

Response:
def run_command(pattern_hosts, command, inventory_path=None, roles=None,
                extra_vars=None,
                on_error_continue=False):
    """Run a shell command on some remote hosts.

    Args:
        pattern_hosts (str): pattern to describe ansible hosts to target.
            see https://docs.ansible.com/ansible/latest/intro_patterns.html
        command (str): the command to run
        inventory_path (str): inventory to use
        extra_vars (dict): extra_vars to use
        on_error_continue(bool): Don't throw any exception in case a host is
            unreachable or the playbooks run with errors

    Raises:
        :py:class:`enoslib.errors.EnosFailedHostsError`: if a task returns an
            error on a host and ``on_error_continue==False``
        :py:class:`enoslib.errors.EnosUnreachableHostsError`: if a host is
            unreachable (through ssh) and ``on_error_continue==False``

    Returns:
        Dict combining the stdout and stderr of ok and failed hosts and every
        results of tasks executed (this may include the fact gathering tasks)

    Example:

    .. code-block:: python

        # Inventory
        [control1]
        enos-0
        [control2]
        enos-1

        # Python
        result = run_command("control*", "date", inventory)

        # Result
        {
            'failed': {},
            'ok':
            {
                u'enos-0':
                {
                    'stderr': u'',
                    'stdout': u'Tue Oct 31 04:53:04 GMT 2017'
                },
                u'enos-1':
                {
                    'stderr': u'',
                    'stdout': u'Tue Oct 31 04:53:05 GMT 2017'}
                },
            'results': [...]
        }

    If facts are gathers this is possible to use ansible templating

    .. code-block:: python

        result = run_command("control*", "ping -c 1
        {{hostvars['enos-1']['ansible_' + n1].ipv4.address}}", inventory)
    """

    def filter_results(results, status):
        _r = [r for r in results
              if r.status == status and r.task == COMMAND_NAME]
        s = dict([[r.host, {"stdout": r.payload.get("stdout"),
                            "stderr": r.payload.get("stderr")}]
                  for r in _r])
        return s

    play_source = {
        "hosts": pattern_hosts,
        "tasks": [{
            "name": COMMAND_NAME,
            "shell": command,
        }]
    }
    results = run_play(play_source,
                       inventory_path=inventory_path,
                       roles=roles,
                       extra_vars=extra_vars)
    ok = filter_results(results, STATUS_OK)
    failed = filter_results(results, STATUS_FAILED)
    return {"ok": ok, "failed": failed, "results": results}


Instruction:
Write a Python 3 function that can
run Ansible. Args: playbooks (list): list of paths to the playbooks to run inventory_path (str): path to the hosts file (inventory) extra_var (dict): extra vars to pass tags (list): list of tags to run on_error_continue(bool): Don't throw any exception in case a host is unreachable or the playbooks run with errors Raises: :py:class:`enoslib.errors.EnosFailedHostsError`: if a task returns an error on a host and ``on_error_continue==False`` :py:class:`enoslib.errors.EnosUnreachableHostsError`: if a host is unreachable (through ssh) and ``on_error_continue==False``

Response:
def run_ansible(playbooks, inventory_path=None, roles=None, extra_vars=None,
        tags=None, on_error_continue=False, basedir='.'):
    """Run Ansible.

    Args:
        playbooks (list): list of paths to the playbooks to run
        inventory_path (str): path to the hosts file (inventory)
        extra_var (dict): extra vars to pass
        tags (list): list of tags to run
        on_error_continue(bool): Don't throw any exception in case a host is
            unreachable or the playbooks run with errors

    Raises:
        :py:class:`enoslib.errors.EnosFailedHostsError`: if a task returns an
            error on a host and ``on_error_continue==False``
        :py:class:`enoslib.errors.EnosUnreachableHostsError`: if a host is
            unreachable (through ssh) and ``on_error_continue==False``
    """

    inventory, variable_manager, loader, options = _load_defaults(
        inventory_path=inventory_path,
        roles=roles,
        extra_vars=extra_vars,
        tags=tags,
        basedir=basedir
    )
    passwords = {}
    for path in playbooks:
        logger.info("Running playbook %s with vars:\n%s" % (path, extra_vars))
        pbex = PlaybookExecutor(
            playbooks=[path],
            inventory=inventory,
            variable_manager=variable_manager,
            loader=loader,
            options=options,
            passwords=passwords
        )

        code = pbex.run()
        stats = pbex._tqm._stats
        hosts = stats.processed.keys()
        result = [{h: stats.summarize(h)} for h in hosts]
        results = {"code": code, "result": result, "playbook": path}
        print(results)

        failed_hosts = []
        unreachable_hosts = []

        for h in hosts:
            t = stats.summarize(h)
            if t["failures"] > 0:
                failed_hosts.append(h)

            if t["unreachable"] > 0:
                unreachable_hosts.append(h)

        if len(failed_hosts) > 0:
            logger.error("Failed hosts: %s" % failed_hosts)
            if not on_error_continue:
                raise EnosFailedHostsError(failed_hosts)
        if len(unreachable_hosts) > 0:
            logger.error("Unreachable hosts: %s" % unreachable_hosts)
            if not on_error_continue:
                raise EnosUnreachableHostsError(unreachable_hosts)


Instruction:
Can you tell what is the following Python 3 function doing
def discover_networks(roles, networks, fake_interfaces=None,
                    fake_networks=None):
    """Checks the network interfaces on the nodes.

    This enables to auto-discover the mapping interface name <-> network role.
    Beware, this has a side effect on each Host in roles.

    Args:
        roles (dict): role->hosts mapping as returned by
            :py:meth:`enoslib.infra.provider.Provider.init`
        networks (list): network list as returned by
            :py:meth:`enoslib.infra.provider.Provider.init`
        fake_interfaces (list): names of optionnal dummy interfaces to create
        fake_networks (list): names of the roles to associate with the fake
            interfaces. Like reguilar network interfaces, the mapping will be
            added to the host vars. Internally this will be zipped with the
            fake_interfaces to produce the mapping.

    If the command is successful each host will be added some variables.
    Assuming that one network whose role is `mynetwork` has been declared, the
    following variables will be available through the ansible hostvars:

    - ``mynetwork=eth1``, `eth1` has been discovered has the interface in the
      network `mynetwork`.
    - ``mynetwork_dev=eth1``, same as above with a different accessor names
    - ``mynetwork_ip=192.168.42.42``, this indicates the ip in the network
      `mynetwork` for this node

    All of this variable can then be accessed by the other nodes through the
    hostvars: ``hostvars[remote_node]["mynetwork_ip"]``
    """

    def get_devices(facts):
        """Extract the network devices information from the facts."""
        devices = []
        for interface in facts['ansible_interfaces']:
            ansible_interface = 'ansible_' + interface
            # filter here (active/ name...)
            if 'ansible_' + interface in facts:
                interface = facts[ansible_interface]
                devices.append(interface)
        return devices

    wait_ssh(roles)
    tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME)
    _check_tmpdir(tmpdir)
    fake_interfaces = fake_interfaces or []
    fake_networks = fake_networks or []

    utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
    facts_file = os.path.join(tmpdir, 'facts.json')
    options = {
        'enos_action': 'check_network',
        'facts_file': facts_file,
        'fake_interfaces': fake_interfaces
    }
    run_ansible([utils_playbook], roles=roles,
        extra_vars=options,
        on_error_continue=False)

    # Read the file
    # Match provider networks to interface names for each host
    with open(facts_file) as f:
        facts = json.load(f)
        for _, host_facts in facts.items():
            host_nets = _map_device_on_host_networks(networks,
                                                    get_devices(host_facts))
            # Add the mapping : networks <-> nic name
            host_facts['networks'] = host_nets

    # Finally update the env with this information
    # generate the extra_mapping for the fake interfaces
    extra_mapping = dict(zip(fake_networks, fake_interfaces))
    _update_hosts(roles, facts, extra_mapping=extra_mapping)

Response: This function returns a list of hosts that are connected to the given networks.


Instruction:
How would you explain what the following Python 3 function does
def generate_inventory(roles, networks, inventory_path, check_networks=False,
                       fake_interfaces=None, fake_networks=None):
    """Generate an inventory file in the ini format.

    The inventory is generated using the ``roles`` in the ``ini`` format.  If
    ``check_network == True``, the function will try to discover which networks
    interfaces are available and map them to one network of the ``networks``
    parameters.  Note that this auto-discovery feature requires the servers to
    have their IP set.

    Args:
        roles (dict): role->hosts mapping as returned by
            :py:meth:`enoslib.infra.provider.Provider.init`
        networks (list): network list as returned by
            :py:meth:`enoslib.infra.provider.Provider.init`
        inventory_path (str): path to the inventory to generate
        check_networks (bool): True to enable the auto-discovery of the mapping
            interface name <-> network role
        fake_interfaces (list): names of optionnal dummy interfaces to create
            on the nodes
        fake_networks (list): names of the roles to associate with the fake
            interfaces. Like reguilar network interfaces, the mapping will be
            added to the host vars. Internally this will be zipped with the
            fake_interfaces to produce the mapping. """

    with open(inventory_path, "w") as f:
        f.write(_generate_inventory(roles))

    if check_networks:
        discover_networks(
            roles,
            networks,
            fake_interfaces=fake_interfaces,
            fake_networks=fake_networks
        )
        with open(inventory_path, "w") as f:
            f.write(_generate_inventory(roles))

Response: Generate an inventory file in the ini format.


Instruction:
How would you code a function in Python 3 to
emulate network links. Read ``network_constraints`` and apply ``tc`` rules on all the nodes. Constraints are applied between groups of machines. Theses groups are described in the ``network_constraints`` variable and must be found in the inventory file. The newtwork constraints support ``delay``, ``rate`` and ``loss``. Args: network_constraints (dict): network constraints to apply roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` inventory_path(string): path to an inventory extra_vars (dict): extra_vars to pass to ansible Examples: * Using defaults The following will apply the network constraints between every groups. For instance the constraints will be applied for the communication between "n1" and "n3" but not between "n1" and "n2". Note that using default leads to symetric constraints. .. code-block:: python roles = { "grp1": ["n1", "n2"], "grp2": ["n3", "n4"], "grp3": ["n3", "n4"], } tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", } emulate_network(roles, tc) If you want to control more precisely which groups need to be taken into account, you can use ``except`` or ``groups`` key .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "except": "grp3" } emulate_network(roles, tc) is equivalent to .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "groups": ["grp1", "grp2"] } emulate_network(roles, inventory, tc) * Using ``src`` and ``dst`` The following will enforce a symetric constraint between ``grp1`` and ``grp2``. .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "constraints": [{ "src": "grp1" "dst": "grp2" "delay": "10ms" "symetric": True }] } emulate_network(roles, inventory, tc)

Response:
def emulate_network(network_constraints,
                    roles=None,
                    inventory_path=None,
                    extra_vars=None):
    """Emulate network links.

    Read ``network_constraints`` and apply ``tc`` rules on all the nodes.
    Constraints are applied between groups of machines. Theses groups are
    described in the ``network_constraints`` variable and must be found in the
    inventory file. The newtwork constraints support ``delay``, ``rate`` and
    ``loss``.

    Args:
        network_constraints (dict): network constraints to apply
        roles (dict): role->hosts mapping as returned by
            :py:meth:`enoslib.infra.provider.Provider.init`
        inventory_path(string): path to an inventory
        extra_vars (dict): extra_vars to pass to ansible

    Examples:

        * Using defaults

        The following will apply the network constraints between every groups.
        For instance the constraints will be applied for the communication
        between "n1" and "n3" but not between "n1" and "n2". Note that using
        default leads to symetric constraints.

        .. code-block:: python

            roles = {
                "grp1": ["n1", "n2"],
                "grp2": ["n3", "n4"],
                "grp3": ["n3", "n4"],
            }

            tc = {
                "enable": True,
                "default_delay": "20ms",
                "default_rate": "1gbit",
            }
            emulate_network(roles, tc)

        If you want to control more precisely which groups need to be taken
        into account, you can use ``except`` or ``groups`` key

        .. code-block:: python

            tc = {
                "enable": True,
                "default_delay": "20ms",
                "default_rate": "1gbit",
                "except": "grp3"
            }
            emulate_network(roles, tc)

        is equivalent to

        .. code-block:: python

            tc = {
                "enable": True,
                "default_delay": "20ms",
                "default_rate": "1gbit",
                "groups": ["grp1", "grp2"]
            }
            emulate_network(roles, inventory, tc)

        * Using ``src`` and ``dst``

        The following will enforce a symetric constraint between ``grp1`` and
        ``grp2``.

        .. code-block:: python

            tc = {
                "enable": True,
                "default_delay": "20ms",
                "default_rate": "1gbit",
                "constraints": [{
                    "src": "grp1"
                    "dst": "grp2"
                    "delay": "10ms"
                    "symetric": True
                }]
            }
            emulate_network(roles, inventory, tc)

    """
    # 1) Retrieve the list of ips for all nodes (Ansible)
    # 2) Build all the constraints (Python)
    #    {source:src, target: ip_dest, device: if, rate:x,  delay:y}
    # 3) Enforce those constraints (Ansible)

    if not network_constraints:
        return

    if roles is None and inventory is None:
        raise ValueError("roles and inventory can't be None")

    if not extra_vars:
        extra_vars = {}

    # 1. getting  ips/devices information
    logger.debug('Getting the ips of all nodes')
    tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME)
    _check_tmpdir(tmpdir)
    utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
    ips_file = os.path.join(tmpdir, 'ips.txt')
    options = {'enos_action': 'tc_ips',
               'ips_file': ips_file}
    run_ansible([utils_playbook], roles=roles, extra_vars=options)

    # 2.a building the group constraints
    logger.debug('Building all the constraints')
    constraints = _build_grp_constraints(roles, network_constraints)
    # 2.b Building the ip/device level constaints
    with open(ips_file) as f:
        ips = yaml.safe_load(f)
        # will hold every single constraint
        ips_with_constraints = _build_ip_constraints(roles,
                                                     ips,
                                                     constraints)
        # dumping it for debugging purpose
        ips_with_constraints_file = os.path.join(tmpdir,
                                                 'ips_with_constraints.yml')
        with open(ips_with_constraints_file, 'w') as g:
            yaml.dump(ips_with_constraints, g)

    # 3. Enforcing those constraints
    logger.info('Enforcing the constraints')
    # enabling/disabling network constraints
    enable = network_constraints.setdefault('enable', True)
    utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
    options = {
        'enos_action': 'tc_apply',
        'ips_with_constraints': ips_with_constraints,
        'tc_enable': enable,
    }
    options.update(extra_vars)
    run_ansible([utils_playbook],
                roles=roles,
                inventory_path=inventory_path,
                extra_vars=options)


Instruction:
Write a Python 3 function that can
validate the network parameters.

Response:
def validate_network(roles=None,
                     inventory_path=None,
                     output_dir=None,
                     extra_vars=None):
    """Validate the network parameters (latency, bandwidth ...)

    Performs flent, ping tests to validate the constraints set by
    :py:func:`emulate_network`. Reports are available in the tmp directory used
    by enos.

    Args:
        roles (dict): role->hosts mapping as returned by
            :py:meth:`enoslib.infra.provider.Provider.init`
        inventory_path (str): path to an inventory
        output_dir (str): directory where validation files will be stored.
            Default to :py:const:`enoslib.constants.TMP_DIRNAME`.
    """
    logger.debug('Checking the constraints')
    if not output_dir:
        output_dir = os.path.join(os.getcwd(), TMP_DIRNAME)

    if not extra_vars:
        extra_vars = {}

    output_dir = os.path.abspath(output_dir)
    _check_tmpdir(output_dir)
    utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
    options = {'enos_action': 'tc_validate',
               'tc_output_dir': output_dir}

    options.update(extra_vars)
    run_ansible([utils_playbook],
                roles=roles,
                inventory_path=inventory_path,
                extra_vars=options)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def reset_network(roles, extra_vars=None):
    logger.debug('Reset the constraints')

    if not extra_vars:
        extra_vars = {}

    tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME)

    _check_tmpdir(tmpdir)
    utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
    options = {'enos_action': 'tc_reset',
               'tc_output_dir': tmpdir}
    options.update(extra_vars)
    run_ansible([utils_playbook], roles=roles, extra_vars=options)

Response: Reset the network constraints.


Instruction:
How would you explain what the following Python 3 function does
def wait_ssh(roles, retries=100, interval=30):
    utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
    options = {'enos_action': 'ping'}

    for i in range(0, retries):
        try:
            run_ansible([utils_playbook],
                        roles=roles,
                        extra_vars=options,
                        on_error_continue=False)
            break
        except EnosUnreachableHostsError as e:
            logger.info("Hosts unreachable: %s " % e.hosts)
            logger.info("Retrying... %s/%s" % (i + 1, retries))
            time.sleep(interval)
    else:
        raise EnosSSHNotReady('Maximum retries reached')

Response: Wait for all the machines to be ssh - reachable


Instruction:
Given the following Python 3 function, write the documentation
def expand_groups(grp):
    p = re.compile(r"(?P<name>.+)\[(?P<start>\d+)-(?P<end>\d+)\]")
    m = p.match(grp)
    if m is not None:
        s = int(m.group('start'))
        e = int(m.group('end'))
        n = m.group('name')
        return list(map(lambda x: n + str(x), range(s, e + 1)))
    else:
        return [grp]

Response:
Expand group names.

    Args:
        grp (string): group names to expand

    Returns:
        list of groups

    Examples:

        * grp[1-3] will be expanded to [grp1, grp2, grp3]
        * grp1 will be expanded to [grp1]


Instruction:
Given the following Python 3 function, write the documentation
def _expand_description(desc):
    srcs = expand_groups(desc['src'])
    dsts = expand_groups(desc['dst'])
    descs = []
    for src in srcs:
        for dst in dsts:
            local_desc = desc.copy()
            local_desc['src'] = src
            local_desc['dst'] = dst
            descs.append(local_desc)

    return descs

Response: Expand the description given the group names and patterns.


Instruction:
Explain what the following Python 3 code does
def _generate_default_grp_constraints(roles, network_constraints):
    default_delay = network_constraints.get('default_delay')
    default_rate = network_constraints.get('default_rate')
    default_loss = network_constraints.get('default_loss', 0)
    except_groups = network_constraints.get('except', [])
    grps = network_constraints.get('groups', roles.keys())
    # expand each groups
    grps = [expand_groups(g) for g in grps]
    # flatten
    grps = [x for expanded_group in grps for x in expanded_group]
    # building the default group constraints
    return [{'src': grp1,
             'dst': grp2,
             'delay': default_delay,
             'rate': default_rate,
             'loss': default_loss}
            for grp1 in grps for grp2 in grps
            if ((grp1 != grp2
                 or _src_equals_dst_in_constraints(network_constraints, grp1))
                and grp1 not in except_groups and grp2 not in except_groups)]

Response: Generate default symetric grp constraints.


Instruction:
Here you have a function in Python 3, explain what it does
def _generate_actual_grp_constraints(network_constraints):
    if 'constraints' not in network_constraints:
        return []

    constraints = network_constraints['constraints']
    actual = []
    for desc in constraints:
        descs = _expand_description(desc)
        for desc in descs:
            actual.append(desc)
            if 'symetric' in desc:
                sym = desc.copy()
                sym['src'] = desc['dst']
                sym['dst'] = desc['src']
                actual.append(sym)
    return actual

Response: Generate the user specified constraints


Instruction:
Explain what the following Python 3 code does
def _merge_constraints(constraints, overrides):
    for o in overrides:
        i = 0
        while i < len(constraints):
            c = constraints[i]
            if _same(o, c):
                constraints[i].update(o)
                break
            i = i + 1

Response:
Merge the constraints avoiding duplicates
    Change constraints in place.


Instruction:
Explain what the following Python 3 code does
def _build_grp_constraints(roles, network_constraints):
    # generate defaults constraints
    constraints = _generate_default_grp_constraints(roles,
                                                    network_constraints)
    # Updating the constraints if necessary
    if 'constraints' in network_constraints:
        actual = _generate_actual_grp_constraints(network_constraints)
        _merge_constraints(constraints, actual)

    return constraints

Response: Generate constraints at the group level and deal with symetric constraints.


Instruction:
Write a Python 3 function for
building the constraints at the ip level.

Response:
def _build_ip_constraints(roles, ips, constraints):
    """Generate the constraints at the ip/device level.
    Those constraints are those used by ansible to enforce tc/netem rules.
    """
    local_ips = copy.deepcopy(ips)
    for constraint in constraints:
        gsrc = constraint['src']
        gdst = constraint['dst']
        gdelay = constraint['delay']
        grate = constraint['rate']
        gloss = constraint['loss']
        for s in roles[gsrc]:
            # one possible source
            # Get all the active devices for this source
            active_devices = filter(lambda x: x["active"],
                                    local_ips[s.alias]['devices'])
            # Get only the devices specified in the network constraint
            if 'network' in constraint:
                active_devices = filter(
                    lambda x:
                    x['device'] == s.extra[constraint['network']],
                    active_devices)
            # Get only the name of the active devices
            sdevices = map(lambda x: x['device'], active_devices)
            for sdevice in sdevices:
                # one possible device
                for d in roles[gdst]:
                    # one possible destination
                    dallips = local_ips[d.alias]['all_ipv4_addresses']
                    # Let's keep docker bridge out of this
                    dallips = filter(lambda x: x != '172.17.0.1', dallips)
                    for dip in dallips:
                        local_ips[s.alias].setdefault('tc', []).append({
                            'source': s.alias,
                            'target': dip,
                            'device': sdevice,
                            'delay': gdelay,
                            'rate': grate,
                            'loss': gloss
                        })
    return local_ips


Instruction:
Make a summary of the following Python 3 code
def _map_device_on_host_networks(provider_nets, devices):
    networks = copy.deepcopy(provider_nets)
    for network in networks:
        for device in devices:
            network.setdefault('device', None)
            ip_set = IPSet([network['cidr']])
            if 'ipv4' not in device:
                continue
            ips = device['ipv4']
            if not isinstance(ips, list):
                ips = [ips]
            if len(ips) < 1:
                continue
            ip = IPAddress(ips[0]['address'])
            if ip in ip_set:
                network['device'] = device['device']
                continue
    return networks

Response: Decorate each network with the corresponding nic name.


Instruction:
Can you write a function in Python 3 where it
returns a list of dates in a frequency range.

Response:
def f_daterange(freq, tz='UTC', *args, **kwargs):
    """
    Use ``dateutil.rrule`` to create a range of dates. The frequency must be a
    string in the following list: YEARLY, MONTHLY, WEEKLY, DAILY, HOURLY,
    MINUTELY or SECONDLY.

    See `dateutil rrule`_ documentation for more detail.

    .. _dateutil rrule: https://dateutil.readthedocs.org/en/latest/rrule.html

    :param freq: One of the ``dateutil.rrule`` frequencies
    :type freq: str
    :param tz: One of the ``pytz`` timezones, defaults to UTC
    :type tz: str
    :param args: start date <datetime>, interval between each frequency <int>,
        max number of recurrences <int>, end date <datetime>
    :param kwargs: ``dtstart``, ``interval``, ``count``, ``until``
    :return: range of dates
    :rtype: list
    """
    tz = pytz.timezone(tz)
    freq = getattr(rrule, freq.upper())  # get frequency enumeration from rrule
    return [tz.localize(dt) for dt in rrule.rrule(freq, *args, **kwargs)]


Instruction:
Can you implement a function in Python 3 that
calculates the total energy accumulated from the end of each timestep between the given times.

Response:
def f_energy(ac_power, times):
    """
    Calculate the total energy accumulated from AC power at the end of each
    timestep between the given times.

    :param ac_power: AC Power [W]
    :param times: times
    :type times: np.datetime64[s]
    :return: energy [W*h] and energy times
    """
    dt = np.diff(times)  # calculate timesteps
    # convert timedeltas to quantities
    dt = dt.astype('timedelta64[s]').astype('float') / sc_const.hour
    # energy accumulate during timestep
    energy = dt * (ac_power[:-1] + ac_power[1:]) / 2
    return energy, times[1:]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def groupby_freq(items, times, freq, wkst='SU'):
    timeseries = zip(times, items)  # timeseries map of items
    # create a key lambda to group timeseries by
    if freq.upper() == 'DAILY':
        def key(ts_): return ts_[0].day
    elif freq.upper() == 'WEEKLY':
        weekday = getattr(rrule, wkst.upper())  # weekday start
        # generator that searches times for weekday start
        days = (day for day in times if day.weekday() == weekday.weekday)
        day0 = days.next()  # first weekday start of all times

        def key(ts_): return (ts_[0] - day0).days // 7
    else:
        def key(ts_): return getattr(ts_[0], freq.lower()[:-2])
    for k, ts in itertools.groupby(timeseries, key):
        yield k, ts

Response: Group items in a timeseries by frequency.


Instruction:
Write a Python 3 script for
using :func:`groupby_freq` to rollup items :param items: items in timeseries :param times: times corresponding to items :param freq: One of the ``dateutil.rrule`` frequency constants :type freq: str

Response:
def f_rollup(items, times, freq):
    """
    Use :func:`groupby_freq` to rollup items

    :param items: items in timeseries
    :param times: times corresponding to items
    :param freq: One of the ``dateutil.rrule`` frequency constants
    :type freq: str
    """
    rollup = [np.sum(item for __, item in ts)
              for _, ts in groupby_freq(items, times, freq)]
    return np.array(rollup)


Instruction:
Can you tell what is the following Python 3 function doing
def fileupdate(self, data):

        self.name = data["name"]
        add = self.__additional
        add["filetype"] = "other"
        for filetype in ("book", "image", "video", "audio", "archive"):
            if filetype in data:
                add["filetype"] = filetype
                break
        if add["filetype"] in ("image", "video", "audio"):
            add["thumb"] = data.get("thumb", dict())
        # checksum is md5
        add["checksum"] = data["checksum"]
        add["expire_time"] = data["expires"] / 1000
        add["size"] = data["size"]
        add["info"] = data.get(add["filetype"], dict())
        add["uploader"] = data["user"]
        if self.room.admin:
            add["info"].update({"room": data.get("room")})
            add["info"].update({"uploader_ip": data.get("uploader_ip")})
        self.updated = True

Response:
Method to update extra metadata fields with dict obtained
        through fileinfo


Instruction:
Write a Python 3 function that can
return the thumbnail s url for this image audio or video file.

Response:
def thumbnail(self):
        """Returns the thumbnail's url for this image, audio, or video file.
        Returns empty string if the file has no thumbnail"""

        if self.filetype not in ("video", "image", "audio"):
            raise RuntimeError("Only video, audio and image files can have thumbnails")
        thumb_srv = self.thumb.get("server")
        url = f"https://{thumb_srv}" if thumb_srv else None
        return f"{url}/asset/{self.fid}/thumb" if url else ""


Instruction:
Can you generate a brief explanation for the following Python 3 code
def duration(self):

        if self.filetype not in ("video", "audio"):
            raise RuntimeError("Only videos and audio have durations")
        return self.info.get("length") or self.info.get("duration")

Response: Returns the duration in seconds of this audio or video file


Instruction:
Given the following Python 3 function, write the documentation
def delete(self):
        self.room.check_owner()
        self.conn.make_call("deleteFiles", [self.fid])

Response: Remove this file from the room


Instruction:
Here you have a function in Python 3, explain what it does
def set_gid(self):
        if self.group:
            gid = getgrnam(self.group).gr_gid
            try:
                os.setgid(gid)
            except Exception:
                message = ("Unable to switch ownership to {0}:{1}. " +
                           "Did you start the daemon as root?")
                print(message.format(self.user, self.group))
                sys.exit(1)

Response: Change the group of the running process


Instruction:
Write a Python 3 function for
changing the user of the running process

Response:
def set_uid(self):
        """Change the user of the running process"""
        if self.user:
            uid = getpwnam(self.user).pw_uid
            try:
                os.setuid(uid)
            except Exception:
                message = ('Unable to switch ownership to {0}:{1}. ' +
                           'Did you start the daemon as root?')
                print(message.format(self.user, self.group))
                sys.exit(1)


Instruction:
Can you tell what is the following Python 3 function doing
def setup_logging(self):
        self.logger = logging.getLogger()

        if os.path.exists('/dev/log'):
            handler = SysLogHandler('/dev/log')
        else:
            handler = SysLogHandler()
        self.logger.addHandler(handler)

Response: Setup logging to syslog using the


Instruction:
Write a Python 3 script for
doing the UNIX double-fork magic, see Stevens' "Advanced Programming in the UNIX Environment" for details (ISBN 0201563177) http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16

Response:
def daemonize(self):
        """
        Do the UNIX double-fork magic, see Stevens' "Advanced
        Programming in the UNIX Environment" for details (ISBN 0201563177)
        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
        """
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError as e:
            sys.stderr.write('fork #1 failed: %d (%s)\n' %
                             (e.errno, e.strerror))
            sys.exit(1)
        os.chdir('/')
        os.setsid()
        os.umask(0)
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError as e:
            sys.stderr.write('fork #2 failed: %d (%s)\n' %
                             (e.errno, e.strerror))
            sys.exit(1)
        # to properly close the fd's, we need to use sys and os:
        # http://chromano.me/2011/05/23/starting-python-daemons-via-ssh.html
        sys.stdin.close()
        sys.stdout.close()
        sys.stderr.close()
        os.closerange(0, 3)

        pid = os.getpid()
        with open(self.pid_file, 'w+') as fp:
            fp.write("%s\n" % pid)

        def handler(*args):
            raise BaseException("SIGTERM was caught")
        signal.signal(signal.SIGTERM, handler)
        # atexit functions are "not called when the program is killed by a
        # signal not handled by Python". But since SIGTERM is now handled, the
        # atexit functions do get called.
        atexit.register(self._shutdown)


Instruction:
Create a Python 3 function for
determining the status of the daemon

Response:
def status(self):
        """Determine the status of the daemon"""
        my_name = os.path.basename(sys.argv[0])
        if self._already_running():
            message = "{0} (pid  {1}) is running...\n".format(my_name, self.pid)
            sys.stdout.write(message)
            return 0
        sys.stdout.write("{0} is stopped\n".format(my_name))
        return 3


Instruction:
Write a Python 3 function that can
retrieve embeddable resource from given uri.

Response:
def fetch_resources(uri, rel):
    """
    Retrieves embeddable resource from given ``uri``.

    For now only local resources (images, fonts) are supported.

    :param str uri: path or url to image or font resource
    :returns: path to local resource file.
    :rtype: str
    :raises: :exc:`~easy_pdf.exceptions.UnsupportedMediaPathException`
    """
    if settings.STATIC_URL and uri.startswith(settings.STATIC_URL):
        path = os.path.join(settings.STATIC_ROOT, uri.replace(settings.STATIC_URL, ""))
    elif settings.MEDIA_URL and uri.startswith(settings.MEDIA_URL):
        path = os.path.join(settings.MEDIA_ROOT, uri.replace(settings.MEDIA_URL, ""))
    else:
        path = os.path.join(settings.STATIC_ROOT, uri)

    if not os.path.isfile(path):
        raise UnsupportedMediaPathException(
            "media urls must start with {} or {}".format(
                settings.MEDIA_ROOT, settings.STATIC_ROOT
            )
        )

    return path.replace("\\", "/")


Instruction:
Explain what the following Python 3 code does
def html_to_pdf(content, encoding="utf-8",
                link_callback=fetch_resources, **kwargs):
    """
    Converts html ``content`` into PDF document.

    :param unicode content: html content
    :returns: PDF content
    :rtype: :class:`bytes`
    :raises: :exc:`~easy_pdf.exceptions.PDFRenderingError`
    """
    src = BytesIO(content.encode(encoding))
    dest = BytesIO()

    pdf = pisa.pisaDocument(src, dest, encoding=encoding,
                            link_callback=link_callback, **kwargs)
    if pdf.err:
        logger.error("Error rendering PDF document")
        for entry in pdf.log:
            if entry[0] == xhtml2pdf.default.PML_ERROR:
                logger_x2p.error("line %s, msg: %s, fragment: %s", entry[1], entry[2], entry[3])
        raise PDFRenderingError("Errors rendering PDF", content=content, log=pdf.log)

    if pdf.warn:
        for entry in pdf.log:
            if entry[0] == xhtml2pdf.default.PML_WARNING:
                logger_x2p.warning("line %s, msg: %s, fragment: %s", entry[1], entry[2], entry[3])

    return dest.getvalue()

Response: Converts html content into PDF document.


Instruction:
Explain what the following Python 3 code does
def make_response(content, filename=None, content_type="application/pdf"):
    response = HttpResponse(content, content_type=content_type)
    if filename is not None:
        response["Content-Disposition"] = "attachment; %s" % encode_filename(filename)
    return response

Response: Creates a response object with content.


Instruction:
Can you implement a function in Python 3 that
renders a Django html template and return a PDF document.

Response:
def render_to_pdf(template, context, using=None, request=None, encoding="utf-8", **kwargs):
    """
    Create PDF document from Django html template.

    :param str template: Path to Django template
    :param dict context: Template context
    :param using: Optional Django template engine
    :param request: Django HTTP request
    :type request: :class:`django.http.HttpRequest`

    :returns: rendered PDF
    :rtype: :class:`bytes`

    :raises: :exc:`~easy_pdf.exceptions.PDFRenderingError`, :exc:`~easy_pdf.exceptions.UnsupportedMediaPathException`
    """
    content = loader.render_to_string(template, context, request=request, using=using)
    return html_to_pdf(content, encoding, **kwargs)


Instruction:
Create a Python 3 function to
render a PDF response using given request template and context.

Response:
def render_to_pdf_response(request, template, context, using=None, filename=None,
                           encoding="utf-8", **kwargs):
    """
    Renders a PDF response using given ``request``, ``template`` and ``context``.

    If ``filename`` param is specified then the response ``Content-Disposition``
    header will be set to ``attachment`` making the browser display
    a "Save as.." dialog.

    :param request: Django HTTP request
    :type request: :class:`django.http.HttpRequest`
    :param str template: Path to Django template
    :param dict context: Template context
    :param using: Optional Django template engine
    :rtype: :class:`django.http.HttpResponse`
    """
    try:
        pdf = render_to_pdf(template, context, using=using, encoding=encoding, **kwargs)
        return make_response(pdf, filename)
    except PDFRenderingError as e:
        logger.exception(e.message)
        return HttpResponse(e.message)


Instruction:
Can you tell what is the following Python 3 function doing
def get_pdf_response(self, context, **response_kwargs):
        return render_to_pdf_response(
            request=self.request,
            template=self.get_template_names(),
            context=context,
            using=self.template_engine,
            filename=self.get_pdf_filename(),
            **self.get_pdf_kwargs()
        )

Response: Renders PDF document and prepares response.


Instruction:
Create a Python 3 function to
replace all strings in dictionary with compiled version of themselves and return dictionary.

Response:
def compile_patterns_in_dictionary(dictionary):
    """
    Replace all strings in dictionary with compiled
    version of themselves and return dictionary.
    """
    for key, value in dictionary.items():
        if isinstance(value, str):
            dictionary[key] = re.compile(value)
        elif isinstance(value, dict):
            compile_patterns_in_dictionary(value)
    return dictionary


Instruction:
How would you explain what the following Python 3 function does
def filter(self, source_file, encoding):  # noqa A001

        with codecs.open(source_file, 'r', encoding=encoding) as f:
            text = f.read()

        return [filters.SourceText(self._filter(text), source_file, encoding, 'context')]

Response: Parse file and return list of source text.


Instruction:
Can you generate the documentation for the following Python 3 function
def _filter(self, text):

        if self.line_endings:
            text = self.norm_nl(text)

        new_text = []
        index = 0
        last = 0
        end = len(text)
        while index < end:
            m = self.escapes.match(text, pos=index) if self.escapes else None
            if m:
                index = m.end(0)
                continue
            handled = False
            for delimiter in self.delimiters:
                m = delimiter[0].match(text, pos=index)
                if m:
                    if self.context_visible_first is True:
                        new_text.append(text[last:m.start(0)])
                    else:
                        new_text.append(m.group(delimiter[1]))
                    index = m.end(0)
                    last = index
                    handled = True
                    break
            if handled:
                continue
            index += 1
        if last < end and self.context_visible_first is True:
            new_text.append(text[last:end])

        return ' '.join(new_text)

Response: Filter the text for context delimiter.


Instruction:
How would you code a function in Python 3 to
create a new connection which can later be upgraded to use TLS.

Response:
def create_starttls_connection(
        loop,
        protocol_factory,
        host=None,
        port=None,
        *,
        sock=None,
        ssl_context_factory=None,
        use_starttls=False,
        local_addr=None,
        **kwargs):
    """
    Create a connection which can later be upgraded to use TLS.

    .. versionchanged:: 0.4

        The `local_addr` argument was added.

    :param loop: The event loop to use.
    :type loop: :class:`asyncio.BaseEventLoop`
    :param protocol_factory: Factory for the protocol for the connection
    :param host: The host name or address to connect to
    :type host: :class:`str` or :data:`None`
    :param port: The port to connect to
    :type port: :class:`int` or :data:`None`
    :param sock: A socket to wrap (conflicts with `host` and `port`)
    :type sock: :class:`socket.socket`
    :param ssl_context_factory: Function which returns a
        :class:`OpenSSL.SSL.Context` to use for TLS operations
    :param use_starttls: Flag to control whether TLS is negotiated right away
        or deferredly.
    :type use_starttls: :class:`bool`
    :param local_addr: Address to bind to

    This is roughly a copy of the asyncio implementation of
    :meth:`asyncio.BaseEventLoop.create_connection`. It returns a pair
    ``(transport, protocol)``, where `transport` is a newly created
    :class:`STARTTLSTransport` instance. Further keyword arguments are
    forwarded to the constructor of :class:`STARTTLSTransport`.

    `loop` must be a :class:`asyncio.BaseEventLoop`, with support for
    :meth:`asyncio.BaseEventLoop.add_reader` and the corresponding writer and
    removal functions for sockets. This is typically a selector type event
    loop.

    `protocol_factory` must be a callable which (without any arguments) returns
    a :class:`asyncio.Protocol` which will be connected to the STARTTLS
    transport.

    `host` and `port` must be a hostname and a port number, or both
    :data:`None`. Both must be :data:`None`, if and only if `sock` is not
    :data:`None`. In that case, `sock` is used instead of a newly created
    socket. `sock` is put into non-blocking mode and must be a stream socket.

    If `use_starttls` is :data:`True`, no TLS handshake will be performed
    initially. Instead, the connection is established without any
    transport-layer security. It is expected that the
    :meth:`STARTTLSTransport.starttls` method is used when the application
    protocol requires TLS. If `use_starttls` is :data:`False`, the TLS
    handshake is initiated right away.

    `local_addr` may be an address to bind this side of the socket to. If
    omitted or :data:`None`, the local address is assigned by the operating
    system.

    This coroutine returns when the stream is established. If `use_starttls` is
    :data:`False`, this means that the full TLS handshake has to be finished
    for this coroutine to return. Otherwise, no TLS handshake takes place. It
    must be invoked using the :meth:`STARTTLSTransport.starttls` coroutine.
    """

    if host is not None and port is not None:
        host_addrs = yield from loop.getaddrinfo(
            host, port,
            type=socket.SOCK_STREAM)

        exceptions = []

        for family, type, proto, cname, address in host_addrs:
            sock = None
            try:
                sock = socket.socket(family=family, type=type, proto=proto)
                sock.setblocking(False)
                if local_addr is not None:
                    sock.bind(local_addr)
                yield from loop.sock_connect(sock, address)
            except OSError as exc:
                if sock is not None:
                    sock.close()
                exceptions.append(exc)
            else:
                break
        else:
            if len(exceptions) == 1:
                raise exceptions[0]

            model = str(exceptions[0])
            if all(str(exc) == model for exc in exceptions):
                raise exceptions[0]

            try:
                from aioxmpp.errors import MultiOSError
            except ImportError:
                MultiOSError = OSError

            exc = MultiOSError(
                "could not connect to [{}]:{}".format(host, port),
                exceptions)
            raise exc
    elif sock is None:
        raise ValueError("sock must not be None if host and/or port are None")
    else:
        sock.setblocking(False)

    protocol = protocol_factory()
    waiter = asyncio.Future(loop=loop)
    transport = STARTTLSTransport(loop, sock, protocol,
                                  ssl_context_factory=ssl_context_factory,
                                  waiter=waiter,
                                  use_starttls=use_starttls,
                                  **kwargs)
    yield from waiter

    return transport, protocol


Instruction:
Can you generate the documentation for the following Python 3 function
def _call_connection_lost_and_clean_up(self, exc):

        self._state = _State.CLOSED
        try:
            self._protocol.connection_lost(exc)
        finally:
            self._rawsock.close()
            if self._tls_conn is not None:
                self._tls_conn.set_app_data(None)
                self._tls_conn = None
            self._rawsock = None
            self._protocol = None
            self._loop = None

Response: Clean up all resources and call the protocol connection lost method.


Instruction:
Create a Python 3 function to
abort the current session.

Response:
def abort(self):
        """
        Immediately close the stream, without sending remaining buffers or
        performing a proper shutdown.
        """
        if self._state == _State.CLOSED:
            self._invalid_state("abort() called")
            return

        self._force_close(None)


Instruction:
Can you implement a function in Python 3 that
closes the current session.

Response:
def close(self):
        """
        Close the stream. This performs a proper stream shutdown, except if the
        stream is currently performing a TLS handshake. In that case, calling
        :meth:`close` is equivalent to calling :meth:`abort`.

        Otherwise, the transport waits until all buffers are transmitted.
        """

        if self._state == _State.CLOSED:
            self._invalid_state("close() called")
            return

        if self._state == _State.TLS_HANDSHAKING:
            # hard-close
            self._force_close(None)
        elif self._state == _State.TLS_SHUTTING_DOWN:
            # shut down in progress, nothing to do
            pass
        elif self._buffer:
            # there is data to be send left, first wait for it to transmit ...
            self._closing = True
        elif self._state.tls_started:
            # normal TLS state, nothing left to transmit, shut down
            self._tls_shutdown()
        else:
            # normal non-TLS state, nothing left to transmit, close
            self._raw_shutdown()


Instruction:
Create a Python 3 function for
starting a TLS stream on top of the socket.

Response:
def starttls(self, ssl_context=None,
                 post_handshake_callback=None):
        """
        Start a TLS stream on top of the socket. This is an invalid operation
        if the stream is not in RAW_OPEN state.

        If `ssl_context` is set, it overrides the `ssl_context` passed to the
        constructor. If `post_handshake_callback` is set, it overrides the
        `post_handshake_callback` passed to the constructor.

        .. versionchanged:: 0.4

            This method is now a barrier with respect to reads and writes:
            before the handshake is completed (including the post handshake
            callback, if any), no data is received or sent.
        """
        if self._state != _State.RAW_OPEN or self._closing:
            raise self._invalid_state("starttls() called")

        if ssl_context is not None:
            self._ssl_context = ssl_context
            self._extra.update(
                sslcontext=ssl_context
            )
        else:
            self._ssl_context = self._ssl_context_factory(self)

        if post_handshake_callback is not None:
            self._tls_post_handshake_callback = post_handshake_callback

        self._waiter = asyncio.Future()
        self._waiter.add_done_callback(self._waiter_done)
        self._initiate_tls()
        try:
            yield from self._waiter
        finally:
            self._waiter = None


Instruction:
Write a Python 3 script to
write data to the transport.

Response:
def write(self, data):
        """
        Write data to the transport. This is an invalid operation if the stream
        is not writable, that is, if it is closed. During TLS negotiation, the
        data is buffered.
        """
        if not isinstance(data, (bytes, bytearray, memoryview)):
            raise TypeError('data argument must be byte-ish (%r)',
                            type(data))

        if not self._state.is_writable or self._closing:
            raise self._invalid_state("write() called")

        if not data:
            return

        if not self._buffer:
            self._loop.add_writer(self._raw_fd, self._write_ready)

        self._buffer.extend(data)


Instruction:
Can you tell what is the following Python 3 function doing
def explode(prefix: str):
    def _app(i, e=None):
        if i is not None:
            return {k: v for (k, v) in iter_fields(i)}, None
        return i, e

    def iter_fields(event_field: Union[dict, list]):
        if type(event_field) is dict:
            for key, val in event_field.items():
                yield (key, val)
        elif type(event_field) is list:
            for i, value in enumerate(event_field):
                for key, val in value.items():
                    if not i == 0:
                        yield ("{}_{}".format(key, i), val)
                    else:
                        yield (key, val)

    return compose(_app, add_column_prefix(prefix))

Response: given an array of objects de - normalized into fields


Instruction:
Implement a Python 3 function for
checking if the content contains XML encoding.

Response:
def _has_xml_encode(self, content):
        """Check XML encoding."""

        encode = None

        m = RE_XML_START.match(content)
        if m:
            if m.group(1):
                m2 = RE_XML_ENCODE.match(m.group(1))

                if m2:
                    enc = m2.group(2).decode('ascii')

                    try:
                        codecs.getencoder(enc)
                        encode = enc
                    except LookupError:
                        pass
            else:
                if m.group(2):
                    enc = 'utf-32-be'
                    text = m.group(2)
                elif m.group(3):
                    enc = 'utf-32-le'
                    text = m.group(3)
                elif m.group(4):
                    enc = 'utf-16-be'
                    text = m.group(4)
                elif m.group(5):
                    enc = 'utf-16-le'
                    text = m.group(5)
                try:
                    m2 = RE_XML_ENCODE_U.match(text.decode(enc))
                except Exception:  # pragma: no cover
                    m2 = None

                if m2:
                    enc = m2.group(2)

                    try:
                        codecs.getencoder(enc)
                        encode = enc
                    except Exception:
                        pass

        return encode


Instruction:
Here you have a function in Python 3, explain what it does
def is_break_tag(self, el):

        name = el.name
        return name in self.break_tags or name in self.user_break_tags

Response: Check if tag is an element we should break on.


Instruction:
Implement a Python 3 function for
storing the text as desired.

Response:
def store_blocks(self, el, blocks, text, force_root):
        """Store the text as desired."""

        if force_root or el.parent is None or self.is_break_tag(el):
            content = ''.join(text)
            if content:
                blocks.append((content, self.construct_selector(el)))
            text = []
        return text


Instruction:
Given the following Python 3 function, write the documentation
def construct_selector(self, el, attr=''):

        selector = deque()

        ancestor = el
        while ancestor and ancestor.parent:
            if ancestor is not el:
                selector.appendleft(ancestor.name)
            else:
                tag = ancestor.name
                prefix = ancestor.prefix
                sel = ''
                if prefix:
                    sel += prefix + '|'
                sel = tag
                if attr:
                    sel += '[%s]' % attr
                selector.appendleft(sel)
            ancestor = ancestor.parent
        return '>'.join(selector)

Response: Construct an selector for context.


Instruction:
Here you have a function in Python 3, explain what it does
def to_text(self, tree, force_root=False):

        self.extract_tag_metadata(tree)

        text = []
        attributes = []
        comments = []
        blocks = []

        if not (self.ignores.match(tree) if self.ignores else None):
            # The root of the document is the BeautifulSoup object
            capture = self.captures.match(tree) if self.captures is not None else None
            # Check attributes for normal tags
            if capture:
                for attr in self.attributes:
                    value = tree.attrs.get(attr, '').strip()
                    if value:
                        sel = self.construct_selector(tree, attr=attr)
                        attributes.append((value, sel))

            # Walk children
            for child in tree.children:
                string = str(child).strip()
                is_comment = isinstance(child, bs4.Comment)
                if isinstance(child, bs4.element.Tag):
                    t, b, a, c = self.to_text(child)
                    text.extend(t)
                    attributes.extend(a)
                    comments.extend(c)
                    blocks.extend(b)
                # Get content if not the root and not a comment (unless we want comments).
                elif not isinstance(child, NON_CONTENT) and (not is_comment or self.comments):
                    string = str(child).strip()
                    if string:
                        if is_comment:
                            sel = self.construct_selector(tree) + '<!--comment-->'
                            comments.append((string, sel))
                        elif capture:
                            text.append(string)
                            text.append(' ')
        elif self.comments:
            for child in tree.descendants:
                if isinstance(child, bs4.Comment):
                    string = str(child).strip()
                    if string:
                        sel = self.construct_selector(tree) + '<!--comment-->'
                        comments.append((string, sel))

        text = self.store_blocks(tree, blocks, text, force_root)

        if tree.parent is None or force_root:
            return blocks, attributes, comments
        else:
            return text, blocks, attributes, comments

Response: Extract text from tags and attributes.


Instruction:
Implement a function in Python 3 to
filter the source text.

Response:
def _filter(self, text, context, encoding):
        """Filter the source text."""

        content = []
        blocks, attributes, comments = self.to_text(bs4.BeautifulSoup(text, self.parser))
        if self.comments:
            for c, desc in comments:
                content.append(filters.SourceText(c, context + ': ' + desc, encoding, self.type + 'comment'))
        if self.attributes:
            for a, desc in attributes:
                content.append(filters.SourceText(a, context + ': ' + desc, encoding, self.type + 'attribute'))
        for b, desc in blocks:
            content.append(filters.SourceText(b, context + ': ' + desc, encoding, self.type + 'content'))
        return content


Instruction:
Implement a Python 3 function for
filtering the source text.

Response:
def sfilter(self, source):
        """Filter."""

        return self._filter(source.text, source.context, source.encoding)


Instruction:
Can you generate the documentation for the following Python 3 function
def lazy_import(func):
    try:
        f = sys._getframe(1)
    except Exception:  # pragma: no cover
        namespace = None
    else:
        namespace = f.f_locals
    return _LazyImport(func.__name__, func, namespace)

Response: Decorator for declaring a lazy import.


Instruction:
Write a Python 3 script to
convert csv to dict

Response:
def csv_to_map(fields, delimiter=','):
    """
    Convert csv to dict

    :param delimiter:
    :param fields:
    :return:
    """

    def _csv_to_list(csv_input):
        """
        Util function to overcome the use of files by in-memory io buffer

        :param csv_input:
        :return:
        """
        io_file = io.StringIO(csv_input)
        return next(csv.reader(io_file, delimiter=delimiter))

    def _app(current_tuple, e=None):
        if current_tuple is None or len(current_tuple) == 0:
            return None, "no input"
        csv_list = _csv_to_list(current_tuple)
        if len(csv_list) != len(fields):
            e = {"input": "unexpected number of fields {} obtained {} expected".format(len(csv_list), len(fields))}
            return None, e
        return {k: v for (k, v) in zip(fields, csv_list)}, e
    if fields is None or len(fields) == 0:
        return fixed_input(None, "no fields provided, cannot proceed without order")
    return _app


Instruction:
Can you create a Python 3 function that
converts dict to csv

Response:
def map_to_csv(fields, delimiter=","):
    """
    Convert dict to csv

    :param fields:
    :return:
    """

    def _list_to_csv(l):
        """
        Util function to overcome the use of files by in-memory io buffer

        :param l:
        :return:
        """
        io_file = io.StringIO()
        writer = csv.writer(io_file, quoting=csv.QUOTE_NONNUMERIC, lineterminator='', delimiter=delimiter)
        writer.writerow(l)
        return io_file.getvalue()

    def _app(current_tuple, e=None):
        if e is not None:
            return None, e
        csv_list = []
        for f in fields:
            if f in current_tuple:
                csv_list.append(current_tuple[f])
            else:
                e.update({"output": "expected field {} not found".format(f)})
                return None, e
        return _list_to_csv(csv_list), e
    if fields is None or len(fields) == 0:
        return fixed_input(None, "no fields provided, cannot proceed without order")
    return _app


Instruction:
Write a Python 3 function that can
make a function compatible with xarray. DataArray.

Response:
def xarrayfunc(func):
    """Make a function compatible with xarray.DataArray.

    This function is intended to be used as a decorator like::

        >>> @dc.xarrayfunc
        >>> def func(array):
        ...     # do something
        ...     return newarray
        >>>
        >>> result = func(array)

    Args:
        func (function): Function to be wrapped. The first argument
            of the function must be an array to be processed.

    Returns:
        wrapper (function): Wrapped function.
    """
    @wraps(func)
    def wrapper(*args, **kwargs):
        if any(isinstance(arg, xr.DataArray) for arg in args):
            newargs = []
            for arg in args:
                if isinstance(arg, xr.DataArray):
                    newargs.append(arg.values)
                else:
                    newargs.append(arg)

            return dc.full_like(args[0], func(*newargs, **kwargs))
        else:
            return func(*args, **kwargs)

    return wrapper


Instruction:
Create a Python 3 function for
making a function compatible with multicore chunk processing.

Response:
def chunk(*argnames, concatfunc=None):
    """Make a function compatible with multicore chunk processing.

    This function is intended to be used as a decorator like::

        >>> @dc.chunk('array')
        >>> def func(array):
        ...     # do something
        ...     return newarray
        >>>
        >>> result = func(array, timechunk=10)

    or you can set a global chunk parameter outside the function::

        >>> timechunk = 10
        >>> result = func(array)
    """
    def _chunk(func):
        depth = [s.function for s in stack()].index('<module>')
        f_globals = getframe(depth).f_globals

        # original (unwrapped) function
        orgname = '_original_' + func.__name__
        orgfunc = dc.utils.copy_function(func, orgname)
        f_globals[orgname] = orgfunc

        @wraps(func)
        def wrapper(*args, **kwargs):
            depth = [s.function for s in stack()].index('<module>')
            f_globals = getframe(depth).f_globals

            # parse args and kwargs
            params = signature(func).parameters
            for i, (key, val) in enumerate(params.items()):
                if not val.kind == Parameter.POSITIONAL_OR_KEYWORD:
                    break

                try:
                    kwargs.update({key: args[i]})
                except IndexError:
                    kwargs.setdefault(key, val.default)

            # n_chunks and n_processes
            n_chunks = DEFAULT_N_CHUNKS
            n_processes = MAX_WORKERS

            if argnames:
                length = len(kwargs[argnames[0]])

                if 'numchunk' in kwargs:
                    n_chunks = kwargs.pop('numchunk')
                elif 'timechunk' in kwargs:
                    n_chunks = round(length / kwargs.pop('timechunk'))
                elif 'numchunk' in f_globals:
                    n_chunks = f_globals['numchunk']
                elif 'timechunk' in f_globals:
                    n_chunks = round(length / f_globals['timechunk'])

                if 'n_processes' in kwargs:
                    n_processes = kwargs.pop('n_processes')
                elif 'n_processes' in f_globals:
                    n_processes = f_globals['n_processes']

            # make chunked args
            chunks = {}
            for name in argnames:
                arg = kwargs.pop(name)
                try:
                    chunks.update({name: np.array_split(arg, n_chunks)})
                except TypeError:
                    chunks.update({name: np.tile(arg, n_chunks)})

            # run the function
            futures = []
            results = []
            with dc.utils.one_thread_per_process(), Pool(n_processes) as p:
                for i in range(n_chunks):
                    chunk = {key: val[i] for key, val in chunks.items()}
                    futures.append(p.submit(orgfunc, **{**chunk, **kwargs}))

                for future in futures:
                    results.append(future.result())

            # make an output
            if concatfunc is not None:
                return concatfunc(results)

            try:
                return xr.concat(results, 't')
            except TypeError:
                return np.concatenate(results, 0)

        return wrapper

    return _chunk


Instruction:
Explain what the following Python 3 code does
def format_sec(s):

    prefixes = ["", "m", "u", "n"]
    unit = 0

    while s < 1 and unit + 1 < len(prefixes):
        s *= 1000
        unit += 1

    return "{:.1f} {}s".format(s, prefixes[unit])

Response:
Format seconds in a more human readable way. It supports units down to
    nanoseconds.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def create_entity(self, **kwargs):
        data = kwargs
        headers = kwargs.get('headers', {})
        if 'headers' in data:
            del data['headers']
        return self._create("/entities", data=data, coerce=self.get_entity, headers=headers)

Response: Create a new entity.


Instruction:
Explain what the following Python 3 code does
def make_gtfs(source_path, target_path, buffer, ndigits):
    pfeed = pf.read_protofeed(source_path)
    feed = m.build_feed(pfeed, buffer=buffer)
    gt.write_gtfs(feed, target_path, ndigits=ndigits)

Response: Create a GTFS feed from the source file and write it to the target file.


Instruction:
Explain what the following Python 3 code does
def psd(data, dt, ndivide=1, window=hanning, overlap_half=False):
    logger = getLogger('decode.utils.ndarray.psd')

    if overlap_half:
        step = int(len(data) / (ndivide + 1))
        size = step * 2
    else:
        step = int(len(data) / ndivide)
        size = step

    if bin(len(data)).count('1') != 1:
        logger.warning('warning: length of data is not power of 2: {}'.format(len(data)))
    size = int(len(data) / ndivide)
    if bin(size).count('1') != 1.:
        if overlap_half:
            logger.warning('warning: ((length of data) / (ndivide+1)) * 2 is not power of 2: {}'.format(size))
        else:
            logger.warning('warning: (length of data) / ndivide is not power of 2: {}'.format(size))
    psd = np.zeros(size)
    T   = (size - 1) * dt
    vs  = 1 / dt
    vk_ = fftfreq(size, dt)
    vk  = vk_[np.where(vk_ >= 0)]

    for i in range(ndivide):
        d = data[i * step:i * step + size]
        if window is None:
            w    = np.ones(size)
            corr = 1.0
        else:
            w    = window(size)
            corr = np.mean(w**2)
        psd = psd + 2 * (np.abs(fft(d * w)))**2 / size * dt / corr

    return vk, psd[:len(vk)] / ndivide

Response: Calculate the power spectrum density of a set of data.


Instruction:
How would you code a function in Python 3 to
calculate the Allan variance.

Response:
def allan_variance(data, dt, tmax=10):
    """Calculate Allan variance.

    Args:
        data (np.ndarray): Input data.
        dt (float): Time between each data.
        tmax (float): Maximum time.

    Returns:
        vk (np.ndarray): Frequency.
        allanvar (np.ndarray): Allan variance.
    """
    allanvar = []
    nmax = len(data) if len(data) < tmax / dt else int(tmax / dt)
    for i in range(1, nmax+1):
        databis = data[len(data) % i:]
        y = databis.reshape(len(data)//i, i).mean(axis=1)
        allanvar.append(((y[1:] - y[:-1])**2).mean() / 2)
    return dt * np.arange(1, nmax+1), np.array(allanvar)


Instruction:
Explain what the following Python 3 code does
def discover(cls, path, depth="0"):

        # Path should already be sanitized
        attributes = _get_attributes_from_path(path)

        try:
            if len(attributes) == 3:
                # If an item, create a collection for the item.
                item = attributes.pop()
                path = "/".join(attributes)
                collection = cls(path, _is_principal(path))
                yield collection.get(item)
                return

            collection = cls(path, _is_principal(path))
        except api.exceptions.DoesNotExist:
            return

        yield collection

        if depth == "0":
            return

        if len(attributes) == 0:
            yield cls(posixpath.join(path, cls.user), principal=True)
        elif len(attributes) == 1:
            for journal in cls.etesync.list():
                if journal.collection.TYPE in (api.AddressBook.TYPE, api.Calendar.TYPE, api.TaskList.TYPE):
                    yield cls(posixpath.join(path, journal.uid), principal=False)
        elif len(attributes) == 2:
            for item in collection.list():
                yield collection.get(item)

        elif len(attributes) > 2:
            raise RuntimeError("Found more than one attribute. Shouldn't happen")

Response: Discover a list of collections under the given path.


Instruction:
Write a Python 3 script to
encode as quoted - string ( see RFC 2616 ).

Response:
def etag(self):
        """Encoded as quoted-string (see RFC 2616)."""
        if self.is_fake:
            return

        entry = None
        for entry in self.journal.list():
            pass

        return entry.uid if entry is not None else hashlib.sha256(b"").hexdigest()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def create_collection(cls, href, collection=None, props=None):
        # Path should already be sanitized
        attributes = _get_attributes_from_path(href)
        if len(attributes) <= 1:
            raise PrincipalNotAllowedError

        # Try to infer tag
        if not props:
            props = {}
        if not props.get("tag") and collection:
            props["tag"] = collection[0].name

        # Try first getting the collection if exists, or create a new one otherwise.
        try:
            self = cls(href, principal=False, tag=props.get("tag"))
        except api.exceptions.DoesNotExist:
            user_path = posixpath.join('/', cls.user)
            collection_name = hashlib.sha256(str(time.time()).encode()).hexdigest()
            sane_path = posixpath.join(user_path, collection_name)

            if props.get("tag") == "VCALENDAR":
                inst = api.Calendar.create(cls.etesync, collection_name, None)
            elif props.get("tag") == "VADDRESSBOOK":
                inst = api.AddressBook.create(cls.etesync, collection_name, None)
            else:
                raise RuntimeError("Bad tag.")

            inst.save()
            self = cls(sane_path, principal=False)

        self.set_meta(props)

        if collection:
            if props.get("tag") == "VCALENDAR":
                collection, = collection
                items = []
                for content in ("vevent", "vtodo", "vjournal"):
                    items.extend(
                        getattr(collection, "%s_list" % content, []))
                items_by_uid = groupby(sorted(items, key=get_uid), get_uid)
                vobject_items = {}
                for uid, items in items_by_uid:
                    new_collection = vobject.iCalendar()
                    for item in items:
                        new_collection.add(item)
                    href = self._find_available_file_name(
                        vobject_items.get)
                    vobject_items[href] = new_collection
                self.upload_all_nonatomic(vobject_items)
            elif props.get("tag") == "VADDRESSBOOK":
                vobject_items = {}
                for card in collection:
                    href = self._find_available_file_name(
                        vobject_items.get)
                    vobject_items[href] = card
                self.upload_all_nonatomic(vobject_items)

        return self

Response: Create a new collection.


Instruction:
Can you implement a function in Python 3 that
gets the current sync token and changed items for synchronization.

Response:
def sync(self, old_token=None):
        """Get the current sync token and changed items for synchronization.

        ``old_token`` an old sync token which is used as the base of the
        delta update. If sync token is missing, all items are returned.
        ValueError is raised for invalid or old tokens.
        """
        # FIXME: Actually implement
        token = "http://radicale.org/ns/sync/%s" % self.etag.strip("\"")
        if old_token:
            raise ValueError("Sync token are not supported (you can ignore this warning)")
        return token, self.list()


Instruction:
How would you code a function in Python 3 to
fetch a single item.

Response:
def get(self, href):
        """Fetch a single item."""
        if self.is_fake:
            return

        uid = _trim_suffix(href, ('.ics', '.ical', '.vcf'))
        etesync_item = self.collection.get(uid)
        if etesync_item is None:
            return None

        try:
            item = vobject.readOne(etesync_item.content)
        except Exception as e:
            raise RuntimeError("Failed to parse item %r in %r" %
                               (href, self.path)) from e
        # FIXME: Make this sensible
        last_modified = time.strftime(
            "%a, %d %b %Y %H:%M:%S GMT",
            time.gmtime(time.time()))
        return EteSyncItem(self, item, href, last_modified=last_modified, etesync_item=etesync_item)


Instruction:
How would you implement a function in Python 3 that
uploads a new or replace an existing item.

Response:
def upload(self, href, vobject_item):
        """Upload a new or replace an existing item."""
        if self.is_fake:
            return

        content = vobject_item.serialize()
        try:
            item = self.get(href)
            etesync_item = item.etesync_item
            etesync_item.content = content
        except api.exceptions.DoesNotExist:
            etesync_item = self.collection.get_content_class().create(self.collection, content)

        etesync_item.save()

        return self.get(href)


Instruction:
Write a Python 3 script to
delete an item. When ``href`` is ``None``, delete the collection.

Response:
def delete(self, href=None):
        """Delete an item.

        When ``href`` is ``None``, delete the collection.

        """
        if self.is_fake:
            return

        if href is None:
            self.collection.delete()
            return

        item = self.get(href)
        if item is None:
            raise ComponentNotFoundError(href)

        item.etesync_item.delete()


Instruction:
Implement a function in Python 3 to
get metadata value for collection.

Response:
def get_meta(self, key=None):
        """Get metadata value for collection."""
        if self.is_fake:
            return {}

        if key == "tag":
            return self.tag
        elif key is None:
            ret = {}
            for key in self.journal.info.keys():
                ret[key] = self.meta_mappings.map_get(self.journal.info, key)[1]
            return ret
        else:
            key, value = self.meta_mappings.map_get(self.journal.info, key)
            return value


Instruction:
Here you have a function in Python 3, explain what it does
def set_meta(self, _props):
        if self.is_fake:
            return

        props = {}
        for key, value in _props.items():
            key, value = self.meta_mappings.map_set(key, value)
            props[key] = value

        # Pop out tag which we don't want
        props.pop("tag", None)

        self.journal.update_info({})
        self.journal.update_info(props)
        self.journal.save()

Response: Set metadata values for collection.


Instruction:
Can you generate the documentation for the following Python 3 function
def set_meta_all(self, props):
        delta_props = self.get_meta()
        for key in delta_props.keys():
            if key not in props:
                delta_props[key] = None
        delta_props.update(props)
        self.set_meta(delta_props)

Response: Set metadata values for all elements in the collection.


Instruction:
Implement a function in Python 3 to
get the HTTP - datetime of when the collection was modified.

Response:
def last_modified(self):
        """Get the HTTP-datetime of when the collection was modified."""
        # FIXME: Make this sensible
        last_modified = time.strftime(
            "%a, %d %b %Y %H:%M:%S GMT",
            time.gmtime(time.time()))
        return last_modified


Instruction:
Write a Python 3 function that can
get the unicode string representing the whole collection.

Response:
def serialize(self):
        """Get the unicode string representing the whole collection."""
        import datetime
        items = []
        time_begin = datetime.datetime.now()
        for href in self.list():
            items.append(self.get(href).item)
        time_end = datetime.datetime.now()
        self.logger.info(
            "Collection read %d items in %s sec from %s", len(items),
            (time_end - time_begin).total_seconds(), self.path)
        if self.get_meta("tag") == "VCALENDAR":
            collection = vobject.iCalendar()
            for item in items:
                for content in ("vevent", "vtodo", "vjournal"):
                    if content in item.contents:
                        for item_part in getattr(item, "%s_list" % content):
                            collection.add(item_part)
                        break
            return collection.serialize()
        elif self.get_meta("tag") == "VADDRESSBOOK":
            return "".join([item.serialize() for item in items])
        return ""


Instruction:
Can you write a function in Python 3 where it
acquires a lock for the entire storage.

Response:
def acquire_lock(cls, mode, user=None):
        """Set a context manager to lock the whole storage.

        ``mode`` must either be "r" for shared access or "w" for exclusive
        access.

        ``user`` is the name of the logged in user or empty.

        """
        if not user:
            return

        with EteSyncCache.lock:
            cls.user = user

            cls.etesync = cls._get_etesync_for_user(cls.user)

            if cls._should_sync():
                cls._mark_sync()
                cls.etesync.get_or_create_user_info(force_fetch=True)
                cls.etesync.sync_journal_list()
                for journal in cls.etesync.list():
                    cls.etesync.pull_journal(journal.uid)
            yield
            if cls.etesync.journal_list_is_dirty():
                cls.etesync.sync_journal_list()
            for journal in cls.etesync.list():
                if cls.etesync.journal_is_dirty(journal.uid):
                    cls.etesync.sync_journal(journal.uid)

            cls.etesync = None
            cls.user = None


Instruction:
Given the following Python 3 function, write the documentation
def setup(self):

        self.blocks = self.config['block_comments']
        self.lines = self.config['line_comments']
        self.group_comments = self.config['group_comments']
        self.prefix = self.config['prefix']
        self.generic_mode = self.config['generic_mode']
        self.strings = self.config['strings']
        self.trigraphs = self.config['trigraphs']
        self.decode_escapes = self.config['decode_escapes']
        self.charset_size = self.config['charset_size']
        self.wide_charset_size = self.config['wide_charset_size']
        self.exec_charset = self.get_encoding_name(self.config['exec_charset'])
        self.wide_exec_charset = self.get_encoding_name(self.config['wide_exec_charset'])
        self.string_types, self.wild_string_types = self.eval_string_type(self.config['string_types'])
        if not self.generic_mode:
            self.pattern = RE_CPP

Response: Initialize the object variables that are not yet set.


Instruction:
Given the following Python 3 function, write the documentation
def evaluate_inline_tail(self, groups):

        if self.lines:
            self.line_comments.append([groups['line'][2:].replace('\\\n', ''), self.line_num, self.current_encoding])

Response: Evaluate inline comments at the tail of source code.


Instruction:
Implement a function in Python 3 to
evaluate inline comments on their own lines.

Response:
def evaluate_inline(self, groups):
        """Evaluate inline comments on their own lines."""

        # Consecutive lines with only comments with same leading whitespace
        # will be captured as a single block.
        if self.lines:
            if (
                self.group_comments and
                self.line_num == self.prev_line + 1 and
                groups['leading_space'] == self.leading
            ):
                self.line_comments[-1][0] += '\n' + groups['line'][2:].replace('\\\n', '')
            else:
                self.line_comments.append(
                    [groups['line'][2:].replace('\\\n', ''), self.line_num, self.current_encoding]
                )
            self.leading = groups['leading_space']
            self.prev_line = self.line_num