Instruction:
Can you tell what is the following Python 3 function doing
def main():
    coloredlogs.install(syslog=True)
    # Command line option defaults.
    rotation_scheme = {}
    kw = dict(include_list=[], exclude_list=[])
    parallel = False
    use_sudo = False
    # Internal state.
    selected_locations = []
    # Parse the command line arguments.
    try:
        options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [
            'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=',
            'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed',
            'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=',
            'verbose', 'quiet', 'help',
        ])
        for option, value in options:
            if option in ('-M', '--minutely'):
                rotation_scheme['minutely'] = coerce_retention_period(value)
            elif option in ('-H', '--hourly'):
                rotation_scheme['hourly'] = coerce_retention_period(value)
            elif option in ('-d', '--daily'):
                rotation_scheme['daily'] = coerce_retention_period(value)
            elif option in ('-w', '--weekly'):
                rotation_scheme['weekly'] = coerce_retention_period(value)
            elif option in ('-m', '--monthly'):
                rotation_scheme['monthly'] = coerce_retention_period(value)
            elif option in ('-y', '--yearly'):
                rotation_scheme['yearly'] = coerce_retention_period(value)
            elif option in ('-I', '--include'):
                kw['include_list'].append(value)
            elif option in ('-x', '--exclude'):
                kw['exclude_list'].append(value)
            elif option in ('-j', '--parallel'):
                parallel = True
            elif option in ('-p', '--prefer-recent'):
                kw['prefer_recent'] = True
            elif option in ('-r', '--relaxed'):
                kw['strict'] = False
            elif option in ('-i', '--ionice'):
                value = validate_ionice_class(value.lower().strip())
                kw['io_scheduling_class'] = value
            elif option in ('-c', '--config'):
                kw['config_file'] = parse_path(value)
            elif option in ('-u', '--use-sudo'):
                use_sudo = True
            elif option in ('-n', '--dry-run'):
                logger.info("Performing a dry run (because of %s option) ..", option)
                kw['dry_run'] = True
            elif option in ('-C', '--removal-command'):
                removal_command = shlex.split(value)
                logger.info("Using custom removal command: %s", removal_command)
                kw['removal_command'] = removal_command
            elif option in ('-v', '--verbose'):
                coloredlogs.increase_verbosity()
            elif option in ('-q', '--quiet'):
                coloredlogs.decrease_verbosity()
            elif option in ('-h', '--help'):
                usage(__doc__)
                return
            else:
                assert False, "Unhandled option! (programming error)"
        if rotation_scheme:
            logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme)
        if arguments:
            # Rotation of the locations given on the command line.
            location_source = 'command line arguments'
            selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments)
        else:
            # Rotation of all configured locations.
            location_source = 'configuration file'
            selected_locations.extend(
                location for location, rotation_scheme, options
                in load_config_file(configuration_file=kw.get('config_file'), expand=True)
            )
        # Inform the user which location(s) will be rotated.
        if selected_locations:
            logger.verbose("Selected %s based on %s:",
                           pluralize(len(selected_locations), "location"),
                           location_source)
            for number, location in enumerate(selected_locations, start=1):
                logger.verbose(" %i. %s", number, location)
        else:
            # Show the usage message when no directories are given nor configured.
            logger.verbose("No location(s) to rotate selected.")
            usage(__doc__)
            return
    except Exception as e:
        logger.error("%s", e)
        sys.exit(1)
    # Rotate the backups in the selected directories.
    program = RotateBackups(rotation_scheme, **kw)
    if parallel:
        program.rotate_concurrent(*selected_locations)
    else:
        for location in selected_locations:
            program.rotate_backups(location)

Response: Command line interface for the rotate - backups program.


Instruction:
Can you tell what is the following Python 3 function doing
def get_battery_state(self, prop):
        with open(os.path.join(self.options['battery_path'], prop), 'r') as f:
                return f.readline().strip()

Response:
Return the first line from the file located at battery_path / prop as a
        string.


Instruction:
Given the following Python 3 function, write the documentation
def coerce_location(value, **options):
    # Location objects pass through untouched.
    if not isinstance(value, Location):
        # Other values are expected to be strings.
        if not isinstance(value, string_types):
            msg = "Expected Location object or string, got %s instead!"
            raise ValueError(msg % type(value))
        # Try to parse a remote location.
        ssh_alias, _, directory = value.partition(':')
        if ssh_alias and directory and '/' not in ssh_alias:
            options['ssh_alias'] = ssh_alias
        else:
            directory = value
        # Create the location object.
        value = Location(
            context=create_context(**options),
            directory=parse_path(directory),
        )
    return value

Response: Coerce a string to a Location object.


Instruction:
Make a summary of the following Python 3 code
def coerce_retention_period(value):
    # Numbers pass through untouched.
    if not isinstance(value, numbers.Number):
        # Other values are expected to be strings.
        if not isinstance(value, string_types):
            msg = "Expected string, got %s instead!"
            raise ValueError(msg % type(value))
        # Check for the literal string `always'.
        value = value.strip()
        if value.lower() == 'always':
            value = 'always'
        else:
            # Evaluate other strings as expressions.
            value = simple_eval(value)
            if not isinstance(value, numbers.Number):
                msg = "Expected numeric result, got %s instead!"
                raise ValueError(msg % type(value))
    return value

Response: Coerce a retention period to a Python value.


Instruction:
Write a Python 3 script for
loading a configuration file with backup directories and rotation schemes.

Response:
def load_config_file(configuration_file=None, expand=True):
    """
    Load a configuration file with backup directories and rotation schemes.

    :param configuration_file: Override the pathname of the configuration file
                               to load (a string or :data:`None`).
    :param expand: :data:`True` to expand filename patterns to their matches,
                   :data:`False` otherwise.
    :returns: A generator of tuples with four values each:

              1. An execution context created using :mod:`executor.contexts`.
              2. The pathname of a directory with backups (a string).
              3. A dictionary with the rotation scheme.
              4. A dictionary with additional options.
    :raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
             but doesn't exist or can't be loaded.

    This function is used by :class:`RotateBackups` to discover user defined
    rotation schemes and by :mod:`rotate_backups.cli` to discover directories
    for which backup rotation is configured. When `configuration_file` isn't
    given :class:`~update_dotdee.ConfigLoader` is used to search for
    configuration files in the following locations:

    - ``/etc/rotate-backups.ini`` and ``/etc/rotate-backups.d/*.ini``
    - ``~/.rotate-backups.ini`` and ``~/.rotate-backups.d/*.ini``
    - ``~/.config/rotate-backups.ini`` and ``~/.config/rotate-backups.d/*.ini``

    All of the available configuration files are loaded in the order given
    above, so that sections in user-specific configuration files override
    sections by the same name in system-wide configuration files.
    """
    expand_notice_given = False
    if configuration_file:
        loader = ConfigLoader(available_files=[configuration_file], strict=True)
    else:
        loader = ConfigLoader(program_name='rotate-backups', strict=False)
    for section in loader.section_names:
        items = dict(loader.get_options(section))
        context_options = {}
        if coerce_boolean(items.get('use-sudo')):
            context_options['sudo'] = True
        if items.get('ssh-user'):
            context_options['ssh_user'] = items['ssh-user']
        location = coerce_location(section, **context_options)
        rotation_scheme = dict((name, coerce_retention_period(items[name]))
                               for name in SUPPORTED_FREQUENCIES
                               if name in items)
        options = dict(include_list=split(items.get('include-list', '')),
                       exclude_list=split(items.get('exclude-list', '')),
                       io_scheduling_class=items.get('ionice'),
                       strict=coerce_boolean(items.get('strict', 'yes')),
                       prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
        # Don't override the value of the 'removal_command' property unless the
        # 'removal-command' configuration file option has a value set.
        if items.get('removal-command'):
            options['removal_command'] = shlex.split(items['removal-command'])
        # Expand filename patterns?
        if expand and location.have_wildcards:
            logger.verbose("Expanding filename pattern %s on %s ..", location.directory, location.context)
            if location.is_remote and not expand_notice_given:
                logger.notice("Expanding remote filename patterns (may be slow) ..")
                expand_notice_given = True
            for match in sorted(location.context.glob(location.directory)):
                if location.context.is_directory(match):
                    logger.verbose("Matched directory: %s", match)
                    expanded = Location(context=location.context, directory=match)
                    yield expanded, rotation_scheme, options
                else:
                    logger.verbose("Ignoring match (not a directory): %s", match)
        else:
            yield location, rotation_scheme, options


Instruction:
Implement a function in Python 3 to
rotate the backups in a directory according to a flexible rotation scheme.

Response:
def rotate_backups(directory, rotation_scheme, **options):
    """
    Rotate the backups in a directory according to a flexible rotation scheme.

    .. note:: This function exists to preserve backwards compatibility with
              older versions of the `rotate-backups` package where all of the
              logic was exposed as a single function. Please refer to the
              documentation of the :class:`RotateBackups` initializer and the
              :func:`~RotateBackups.rotate_backups()` method for an explanation
              of this function's parameters.
    """
    program = RotateBackups(rotation_scheme=rotation_scheme, **options)
    program.rotate_backups(directory)


Instruction:
Can you tell what is the following Python 3 function doing
def rotate_concurrent(self, *locations, **kw):
        timer = Timer()
        pool = CommandPool(concurrency=10)
        logger.info("Scanning %s ..", pluralize(len(locations), "backup location"))
        for location in locations:
            for cmd in self.rotate_backups(location, prepare=True, **kw):
                pool.add(cmd)
        if pool.num_commands > 0:
            backups = pluralize(pool.num_commands, "backup")
            logger.info("Preparing to rotate %s (in parallel) ..", backups)
            pool.run()
            logger.info("Successfully rotated %s in %s.", backups, timer)

Response: Rotate the backups in the given locations concurrently.


Instruction:
Implement a function in Python 3 to
rotate the backups in a given location according to a flexible rotation scheme.

Response:
def rotate_backups(self, location, load_config=True, prepare=False):
        """
        Rotate the backups in a directory according to a flexible rotation scheme.

        :param location: Any value accepted by :func:`coerce_location()`.
        :param load_config: If :data:`True` (so by default) the rotation scheme
                            and other options can be customized by the user in
                            a configuration file. In this case the caller's
                            arguments are only used when the configuration file
                            doesn't define a configuration for the location.
        :param prepare: If this is :data:`True` (not the default) then
                        :func:`rotate_backups()` will prepare the required
                        rotation commands without running them.
        :returns: A list with the rotation commands
                  (:class:`~executor.ExternalCommand` objects).
        :raises: :exc:`~exceptions.ValueError` when the given location doesn't
                 exist, isn't readable or isn't writable. The third check is
                 only performed when dry run isn't enabled.

        This function binds the main methods of the :class:`RotateBackups`
        class together to implement backup rotation with an easy to use Python
        API. If you're using `rotate-backups` as a Python API and the default
        behavior is not satisfactory, consider writing your own
        :func:`rotate_backups()` function based on the underlying
        :func:`collect_backups()`, :func:`group_backups()`,
        :func:`apply_rotation_scheme()` and
        :func:`find_preservation_criteria()` methods.
        """
        rotation_commands = []
        location = coerce_location(location)
        # Load configuration overrides by user?
        if load_config:
            location = self.load_config_file(location)
        # Collect the backups in the given directory.
        sorted_backups = self.collect_backups(location)
        if not sorted_backups:
            logger.info("No backups found in %s.", location)
            return
        # Make sure the directory is writable.
        if not self.dry_run:
            location.ensure_writable()
        most_recent_backup = sorted_backups[-1]
        # Group the backups by the rotation frequencies.
        backups_by_frequency = self.group_backups(sorted_backups)
        # Apply the user defined rotation scheme.
        self.apply_rotation_scheme(backups_by_frequency, most_recent_backup.timestamp)
        # Find which backups to preserve and why.
        backups_to_preserve = self.find_preservation_criteria(backups_by_frequency)
        # Apply the calculated rotation scheme.
        for backup in sorted_backups:
            friendly_name = backup.pathname
            if not location.is_remote:
                # Use human friendly pathname formatting for local backups.
                friendly_name = format_path(backup.pathname)
            if backup in backups_to_preserve:
                matching_periods = backups_to_preserve[backup]
                logger.info("Preserving %s (matches %s retention %s) ..",
                            friendly_name, concatenate(map(repr, matching_periods)),
                            "period" if len(matching_periods) == 1 else "periods")
            else:
                logger.info("Deleting %s ..", friendly_name)
                if not self.dry_run:
                    # Copy the list with the (possibly user defined) removal command.
                    removal_command = list(self.removal_command)
                    # Add the pathname of the backup as the final argument.
                    removal_command.append(backup.pathname)
                    # Construct the command object.
                    command = location.context.prepare(
                        command=removal_command,
                        group_by=(location.ssh_alias, location.mount_point),
                        ionice=self.io_scheduling_class,
                    )
                    rotation_commands.append(command)
                    if not prepare:
                        timer = Timer()
                        command.wait()
                        logger.verbose("Deleted %s in %s.", friendly_name, timer)
        if len(backups_to_preserve) == len(sorted_backups):
            logger.info("Nothing to do! (all backups preserved)")
        return rotation_commands


Instruction:
Can you write a function in Python 3 where it
loads a configuration file and options from a configuration file.

Response:
def load_config_file(self, location):
        """
        Load a rotation scheme and other options from a configuration file.

        :param location: Any value accepted by :func:`coerce_location()`.
        :returns: The configured or given :class:`Location` object.
        """
        location = coerce_location(location)
        for configured_location, rotation_scheme, options in load_config_file(self.config_file, expand=False):
            if configured_location.match(location):
                logger.verbose("Loading configuration for %s ..", location)
                if rotation_scheme:
                    self.rotation_scheme = rotation_scheme
                for name, value in options.items():
                    if value:
                        setattr(self, name, value)
                # Create a new Location object based on the directory of the
                # given location and the execution context of the configured
                # location, because:
                #
                # 1. The directory of the configured location may be a filename
                #    pattern whereas we are interested in the expanded name.
                #
                # 2. The execution context of the given location may lack some
                #    details of the configured location.
                return Location(
                    context=configured_location.context,
                    directory=location.directory,
                )
        logger.verbose("No configuration found for %s.", location)
        return location


Instruction:
Can you implement a function in Python 3 that
collects the backups at the given location.

Response:
def collect_backups(self, location):
        """
        Collect the backups at the given location.

        :param location: Any value accepted by :func:`coerce_location()`.
        :returns: A sorted :class:`list` of :class:`Backup` objects (the
                  backups are sorted by their date).
        :raises: :exc:`~exceptions.ValueError` when the given directory doesn't
                 exist or isn't readable.
        """
        backups = []
        location = coerce_location(location)
        logger.info("Scanning %s for backups ..", location)
        location.ensure_readable()
        for entry in natsort(location.context.list_entries(location.directory)):
            match = TIMESTAMP_PATTERN.search(entry)
            if match:
                if self.exclude_list and any(fnmatch.fnmatch(entry, p) for p in self.exclude_list):
                    logger.verbose("Excluded %s (it matched the exclude list).", entry)
                elif self.include_list and not any(fnmatch.fnmatch(entry, p) for p in self.include_list):
                    logger.verbose("Excluded %s (it didn't match the include list).", entry)
                else:
                    try:
                        backups.append(Backup(
                            pathname=os.path.join(location.directory, entry),
                            timestamp=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),
                        ))
                    except ValueError as e:
                        logger.notice("Ignoring %s due to invalid date (%s).", entry, e)
            else:
                logger.debug("Failed to match time stamp in filename: %s", entry)
        if backups:
            logger.info("Found %i timestamped backups in %s.", len(backups), location)
        return sorted(backups)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def group_backups(self, backups):
        backups_by_frequency = dict((frequency, collections.defaultdict(list)) for frequency in SUPPORTED_FREQUENCIES)
        for b in backups:
            backups_by_frequency['minutely'][(b.year, b.month, b.day, b.hour, b.minute)].append(b)
            backups_by_frequency['hourly'][(b.year, b.month, b.day, b.hour)].append(b)
            backups_by_frequency['daily'][(b.year, b.month, b.day)].append(b)
            backups_by_frequency['weekly'][(b.year, b.week)].append(b)
            backups_by_frequency['monthly'][(b.year, b.month)].append(b)
            backups_by_frequency['yearly'][b.year].append(b)
        return backups_by_frequency

Response: Groups the backups into a nested dictionary.


Instruction:
Given the following Python 3 function, write the documentation
def apply_rotation_scheme(self, backups_by_frequency, most_recent_backup):
        if not self.rotation_scheme:
            raise ValueError("Refusing to use empty rotation scheme! (all backups would be deleted)")
        for frequency, backups in backups_by_frequency.items():
            # Ignore frequencies not specified by the user.
            if frequency not in self.rotation_scheme:
                backups.clear()
            else:
                # Reduce the number of backups in each time slot of this
                # rotation frequency to a single backup (the oldest one or the
                # newest one).
                for period, backups_in_period in backups.items():
                    index = -1 if self.prefer_recent else 0
                    selected_backup = sorted(backups_in_period)[index]
                    backups[period] = [selected_backup]
                # Check if we need to rotate away backups in old periods.
                retention_period = self.rotation_scheme[frequency]
                if retention_period != 'always':
                    # Remove backups created before the minimum date of this
                    # rotation frequency? (relative to the most recent backup)
                    if self.strict:
                        minimum_date = most_recent_backup - SUPPORTED_FREQUENCIES[frequency] * retention_period
                        for period, backups_in_period in list(backups.items()):
                            for backup in backups_in_period:
                                if backup.timestamp < minimum_date:
                                    backups_in_period.remove(backup)
                            if not backups_in_period:
                                backups.pop(period)
                    # If there are more periods remaining than the user
                    # requested to be preserved we delete the oldest one(s).
                    items_to_preserve = sorted(backups.items())[-retention_period:]
                    backups_by_frequency[frequency] = dict(items_to_preserve)

Response: This method applies the user defined rotation scheme to the result of group_backups method.


Instruction:
Write a Python 3 script for
collecting the criteria used to decide which backups to preserve.

Response:
def find_preservation_criteria(self, backups_by_frequency):
        """
        Collect the criteria used to decide which backups to preserve.

        :param backups_by_frequency: A :class:`dict` in the format generated by
                                     :func:`group_backups()` which has been
                                     processed by :func:`apply_rotation_scheme()`.
        :returns: A :class:`dict` with :class:`Backup` objects as keys and
                  :class:`list` objects containing strings (rotation
                  frequencies) as values.
        """
        backups_to_preserve = collections.defaultdict(list)
        for frequency, delta in ORDERED_FREQUENCIES:
            for period in backups_by_frequency[frequency].values():
                for backup in period:
                    backups_to_preserve[backup].append(frequency)
        return backups_to_preserve


Instruction:
Can you implement a function in Python 3 that
returns the pathname of the mount point of the current user s virtual machine.

Response:
def mount_point(self):
        """
        The pathname of the mount point of :attr:`directory` (a string or :data:`None`).

        If the ``stat --format=%m ...`` command that is used to determine the
        mount point fails, the value of this property defaults to :data:`None`.
        This enables graceful degradation on e.g. Mac OS X whose ``stat``
        implementation is rather bare bones compared to GNU/Linux.
        """
        try:
            return self.context.capture('stat', '--format=%m', self.directory, silent=True)
        except ExternalCommandFailed:
            return None


Instruction:
Write a Python 3 function that can
make sure the location exists.

Response:
def ensure_exists(self):
        """Make sure the location exists."""
        if not self.context.is_directory(self.directory):
            # This can also happen when we don't have permission to one of the
            # parent directories so we'll point that out in the error message
            # when it seems applicable (so as not to confuse users).
            if self.context.have_superuser_privileges:
                msg = "The directory %s doesn't exist!"
                raise ValueError(msg % self)
            else:
                raise ValueError(compact("""
                    The directory {location} isn't accessible, most likely
                    because it doesn't exist or because of permissions. If
                    you're sure the directory exists you can use the
                    --use-sudo option.
                """, location=self))


Instruction:
Create a Python 3 function for
making sure the location exists and is readable.

Response:
def ensure_readable(self):
        """Make sure the location exists and is readable."""
        self.ensure_exists()
        if not self.context.is_readable(self.directory):
            if self.context.have_superuser_privileges:
                msg = "The directory %s isn't readable!"
                raise ValueError(msg % self)
            else:
                raise ValueError(compact("""
                    The directory {location} isn't readable, most likely
                    because of permissions. Consider using the --use-sudo
                    option.
                """, location=self))


Instruction:
Here you have a function in Python 3, explain what it does
def ensure_writable(self):
        self.ensure_exists()
        if not self.context.is_writable(self.directory):
            if self.context.have_superuser_privileges:
                msg = "The directory %s isn't writable!"
                raise ValueError(msg % self)
            else:
                raise ValueError(compact("""
                    The directory {location} isn't writable, most likely due
                    to permissions. Consider using the --use-sudo option.
                """, location=self))

Response: Make sure the directory exists and is writable.


Instruction:
Can you write a function in Python 3 where it
checks if the given location matches the current directory.

Response:
def match(self, location):
        """
        Check if the given location "matches".

        :param location: The :class:`Location` object to try to match.
        :returns: :data:`True` if the two locations are on the same system and
                  the :attr:`directory` can be matched as a filename pattern or
                  a literal match on the normalized pathname.
        """
        if self.ssh_alias != location.ssh_alias:
            # Never match locations on other systems.
            return False
        elif self.have_wildcards:
            # Match filename patterns using fnmatch().
            return fnmatch.fnmatch(location.directory, self.directory)
        else:
            # Compare normalized directory pathnames.
            self = os.path.normpath(self.directory)
            other = os.path.normpath(location.directory)
            return self == other


Instruction:
Given the following Python 3 function, write the documentation
def setup_file_logger(filename, formatting, log_level):
    logger = logging.getLogger()
    # If a stream handler has been attached, remove it.
    if logger.handlers:
        logger.removeHandler(logger.handlers[0])
    handler = logging.FileHandler(filename)
    logger.addHandler(handler)
    formatter = logging.Formatter(*formatting)
    handler.setFormatter(formatter)
    logger.setLevel(log_level)
    handler.setLevel(log_level)
    return logger

Response: A helper function for creating a file logger.


Instruction:
Implement a function in Python 3 to
output data to stdout without buffering.

Response:
def output_to_bar(self, message, comma=True):
        """
        Outputs data to stdout, without buffering.

        message: A string containing the data to be output.
        comma: Whether or not a comma should be placed at the end of the output.
        """
        if comma:
            message += ','
        sys.stdout.write(message + '\n')
        sys.stdout.flush()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def reload(self):
        logging.debug('Reloading config file as files have been modified.')
        self.config.plugin, self.config.general = self.config.reload()
        logging.debug('Reloading plugins as files have been modified.')
        self.loader = plugin_manager.PluginLoader(
            self._plugin_path, self.config.plugin)
        self._plugin_mod_time = os.path.getmtime(self._plugin_path)
        self._config_mod_time = os.path.getmtime(self._config_file_path)

Response:
Reload the installed plugins and the configuration file. This is called
        when either the plugins or config get updated.


Instruction:
Write a Python 3 script for
creating a thread for each plugin and lets the thread_manager handle it.

Response:
def run_plugins(self):
        """
        Creates a thread for each plugin and lets the thread_manager handle it.
        """
        for obj in self.loader.objects:
            # Reserve a slot in the output_dict in order to ensure that the
            # items are in the correct order.
            self.output_dict[obj.output_options['name']] = None
            self.thread_manager.add_thread(obj.main, obj.options['interval'])


Instruction:
How would you implement a function in Python 3 that
monitors if the config file or plugins are updated. Also outputs the JSON data generated by the plugins, without needing to poll the threads.

Response:
def run(self):
        """
        Monitors if the config file or plugins are updated. Also outputs the
        JSON data generated by the plugins, without needing to poll the threads.
        """
        self.run_plugins()
        while True:
            # Reload plugins and config if either the config file or plugin
            # directory are modified.
            if self._config_mod_time != os.path.getmtime(self._config_file_path) or \
                    self._plugin_mod_time != os.path.getmtime(self._plugin_path):
                self.thread_manager.kill_all_threads()
                self.output_dict.clear()
                self.reload()
                self.run_plugins()
            self.output_to_bar(json.dumps(self._remove_empty_output()))
            time.sleep(self.config.general['interval'])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _remove_empty_output(self):
        output = []
        for key in self.output_dict:
            if self.output_dict[key] is not None and 'full_text' in self.output_dict[key]:
                output.append(self.output_dict[key])
        return output

Response: Remove empty output from the output_dict.


Instruction:
Can you create a Python 3 function that
returns a dictionary of all fields in the given model that match the filter.

Response:
def field_dict(self, model):
        """
        Helper function that returns a dictionary of all fields in the given
        model. If self.field_filter is set, it only includes the fields that
        match the filter.
        """
        if self.field_filter:
            return dict(
                [(f.name, f) for f in model._meta.fields
                 if self.field_filter(f)]
            )
        else:
            return dict(
                [(f.name, f) for f in model._meta.fields
                 if not f.rel and
                    not f.primary_key and
                    not f.unique and
                    not isinstance(f, (models.AutoField, models.TextField))]
            )


Instruction:
Can you implement a function in Python 3 that
calls the main function of a plugin and mutates the output dict with its return value.

Response:
def run(self):
        """
        Calls the main function of a plugin and mutates the output dict
        with its return value. Provides an easy way to change the output
        whilst not needing to constantly poll a queue in another thread and
        allowing plugin's to manage their own intervals.
        """
        self.running = True
        while self.running:
            ret = self.func()
            self.output_dict[ret['name']] = ret
            time.sleep(self.interval)
        return


Instruction:
Can you generate the documentation for the following Python 3 function
def add_thread(self, func, interval):
        t = Thread(func, interval, self.output_dict)
        t.start()
        self._thread_pool.append(t)

Response: Adds a thread to the thread pool.


Instruction:
How would you explain what the following Python 3 function does
def _compile_files(self):
        for f in glob.glob(os.path.join(self.dir_path, '*.py')):
            # Check for compiled Python files that aren't in the __pycache__.
            if not os.path.isfile(os.path.join(self.dir_path, f + 'c')):
                compileall.compile_dir(self.dir_path, quiet=True)
                logging.debug('Compiled plugins as a new plugin has been added.')
                return
            # Recompile if there are newer plugins.
            elif os.path.getmtime(os.path.join(self.dir_path, f)) > os.path.getmtime(
                    os.path.join(self.dir_path, f + 'c')):
                compileall.compile_dir(self.dir_path, quiet=True)
                logging.debug('Compiled plugins as a plugin has been changed.')
                return

Response: Compile the python plugin files.


Instruction:
Can you implement a function in Python 3 that
loads a compiled plugin and returns a module object.

Response:
def _load_compiled(self, file_path):
        """
        Accepts a path to a compiled plugin and returns a module object.

        file_path: A string that represents a complete file path to a compiled
        plugin.
        """
        name = os.path.splitext(os.path.split(file_path)[-1])[0]
        plugin_directory = os.sep.join(os.path.split(file_path)[0:-1])
        compiled_directory = os.path.join(plugin_directory, '__pycache__')
        # Use glob to autocomplete the filename.
        compiled_file = glob.glob(os.path.join(compiled_directory, (name + '.*')))[0]
        plugin = imp.load_compiled(name, compiled_file)
        return plugin


Instruction:
Can you create a Python 3 function that
loads the objects based upon the plugins that have been specified in the config file.

Response:
def load_objects(self):
        """
        Matches the plugins that have been specified in the config file
        with the available plugins. Returns instantiated objects based upon
        the classes defined in the plugins.
        """
        objects = []
        for settings in self._config:
            if settings['plugin'] in self.plugins:
                module = self.plugins[settings['plugin']]
                # Trusts that the only item in __all__ is the name of the
                # plugin class.
                plugin_class = getattr(module, module.__all__)
                objects.append(plugin_class(settings))
                logging.debug('Loaded a plugin object based upon {0}'.format(
                    settings['plugin']))
            else:
                logging.critical('Missing plugin {0} was not found in {1}'.format(
                    settings['plugin'], self.dir_path))
                raise MissingPlugin('The plugin {0} was not found in {1}'.format(
                    settings['plugin'], self.dir_path))
        return objects


Instruction:
Can you implement a function in Python 3 that
refreshes the available plugins and creates a module object for each available plugin.

Response:
def refresh_files(self):
        """
        Discovers the available plugins and turns each into a module object.
        This is a seperate function to allow plugins to be updated
        dynamically by other parts of the application.
        """
        plugins = {}
        _plugin_files = glob.glob(os.path.join(self.dir_path, '[!_]*.pyc'))
        for f in glob.glob(os.path.join(self.dir_path, '[!_]*.py')):
            if not any(os.path.splitext(f)[0] == os.path.splitext(x)[0]
                    for x in _plugin_files):
                logging.debug('Adding plugin {0}'.format(f))
                _plugin_files.append(f)
        for f in _plugin_files:
            plugin = self._load_compiled(f)
            plugins[plugin.__name__] = plugin
            logging.debug('Loaded module object for plugin: {0}'.format(f))
        return plugins


Instruction:
Implement a Python 3 function for
handling main URL routing for the databrowse app.

Response:
def root(self, request, url):
        """
        Handles main URL routing for the databrowse app.

        `url` is the remainder of the URL -- e.g. 'objects/3'.
        """
        # Delegate to the appropriate method, based on the URL.
        if url is None:
            return self.main_view(request)
        try:
            plugin_name, rest_of_url = url.split('/', 1)
        except ValueError:  # need more than 1 value to unpack
            plugin_name, rest_of_url = url, None
        try:
            plugin = self.plugins[plugin_name]
        except KeyError:
            raise http.Http404('A plugin with the requested name '
                               'does not exist.')
        return plugin.model_view(request, self, rest_of_url)


Instruction:
How would you code a function in Python 3 to
register the given models with the given databrowse site.

Response:
def register(self, *model_list, **options):
        """
        Registers the given model(s) with the given databrowse site.

        The model(s) should be Model classes, not instances.

        If a databrowse class isn't given, it will use DefaultModelDatabrowse
        (the default databrowse options).

        If a model is already registered, this will raise AlreadyRegistered.
        """
        databrowse_class = options.pop('databrowse_class',
                                       DefaultModelDatabrowse)
        for model in model_list:
            if model in self.registry:
                raise AlreadyRegistered('The model %s is already registered' %
                                        model.__name__)
            self.registry[model] = databrowse_class


Instruction:
Can you generate the documentation for the following Python 3 function
def unregister(self, *model_list):
        for model in model_list:
            if model not in self.registry:
                raise NotRegistered('The model %s is not registered' %
                                    model.__name__)
            del self.registry[model]

Response: Unregisters the given models.


Instruction:
Create a Python 3 function to
handle main URL routing for the databrowse app.

Response:
def root(self, request, url):
        """
        Handles main URL routing for the databrowse app.

        `url` is the remainder of the URL -- e.g. 'comments/comment/'.
        """
        self.root_url = request.path[:len(request.path) - len(url)]
        url = url.rstrip('/')  # Trim trailing slash, if it exists.

        if url == '':
            return self.index(request)
        elif '/' in url:
            return self.model_page(request, *url.split('/', 2))

        raise http.Http404('The requested databrowse page does not exist.')


Instruction:
Write a Python 3 function for
returning a model - specific page.

Response:
def model_page(self, request, app_label, model_name, rest_of_url=None):
        """
        Handles the model-specific functionality of the databrowse site,
        delegating<to the appropriate ModelDatabrowse class.
        """
        try:
            model = get_model(app_label, model_name)
        except LookupError:
            model = None

        if model is None:
            raise http.Http404("App %r, model %r, not found." %
                               (app_label, model_name))
        try:
            databrowse_class = self.registry[model]
        except KeyError:
            raise http.Http404("This model exists but has not been registered "
                               "with databrowse.")
        return databrowse_class(model, self).root(request, rest_of_url)


Instruction:
Make a summary of the following Python 3 code
def urls(self):
        "Returns a list of (value, URL) tuples."
        # First, check the urls() method for each plugin.
        plugin_urls = []
        for plugin_name, plugin in \
                                self.model.model_databrowse().plugins.items():
            urls = plugin.urls(plugin_name, self)
            if urls is not None:
                #plugin_urls.append(urls)
                values = self.values()
                return zip(self.values(), urls)
        if self.field.rel:
            m = EasyModel(self.model.site, self.field.rel.to)
            if self.field.rel.to in self.model.model_list:
                lst = []
                for value in self.values():
                    if value is None:
                        continue
                    url = mark_safe('%s%s/%s/objects/%s/' %
                                            (self.model.site.root_url,
                                             m.model._meta.app_label,
                                             m.model._meta.model_name,
                                             iri_to_uri(value._get_pk_val())))
                    lst.append((smart_text(value), url))
            else:
                lst = [(value, None) for value in self.values()]
        elif self.field.choices:
            lst = []
            for value in self.values():
                url = mark_safe('%s%s/%s/fields/%s/%s/' %
                                        (self.model.site.root_url,
                                         self.model.model._meta.app_label,
                                         self.model.model._meta.model_name,
                                         self.field.name,
                                         iri_to_uri(self.raw_value)))
                lst.append((value, url))
        elif isinstance(self.field, models.URLField):
            val = self.values()[0]
            lst = [(val, iri_to_uri(val))]
        else:
            lst = [(self.values()[0], None)]
        return lst

Response: Returns a list of ( value URL ) tuples.


Instruction:
Here you have a function in Python 3, explain what it does
def field_dict(self, model):
        if self.field_names is None:
            return dict([(f.name, f) for f in model._meta.fields
                         if isinstance(f, models.DateField)])
        else:
            return dict([(f.name, f)
                         for f in model._meta.fields
                         if isinstance(f, models.DateField) and
                            (f.name in self.field_names)])

Response: Returns a dictionary of all DateFields or DateTimeFields in the given model.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def download_count_upload(job,
                          master_ip,
                          input_file,
                          output_file,
                          kmer_length,
                          spark_conf,
                          memory,
                          sudo):
    '''
    Runs k-mer counting.

    1. If the input file is located in S3, the file is copied into HDFS.
    2. If the input file is not in Parquet format, the file is converted into Parquet.
    3. The k-mers are counted and saved as text.
    4. If the output path is an S3 URL, the file is copied back to S3.

    :param job: Toil job
    :param input_file: URL/path to input file to count k-mers on
    :param output_file: URL/path to save k-mer counts at
    :param kmer_length: The length of k-mer substrings to count.
    :param spark_conf: Optional Spark configuration. If set, memory should \
    not be set.
    :param memory: Amount of memory to provided to Spark workers. Must be set \
    if spark_conf is not set.
    :param sudo: Whether or not to run Spark containers with sudo.

    :type job: toil.Job
    :type input_file: string
    :type output_file: string
    :type kmer_length: int or string
    :type spark_conf: list of string or None
    :type memory: int or None
    :type sudo: boolean
    '''

    if master_ip is not None:
        hdfs_dir = "hdfs://{0}:{1}/".format(master_ip, HDFS_MASTER_PORT)
    else:
        _log.warn('Master IP is not set. If default filesystem is not set, jobs may fail.')
        hdfs_dir = ""

    # if the file isn't already in hdfs, copy it in
    hdfs_input_file = hdfs_dir
    if input_file.startswith("s3://"):

        # append the s3 file name to our hdfs path
        hdfs_input_file += input_file.split("/")[-1]

        # run the download
        _log.info("Downloading input file %s to %s.", input_file, hdfs_input_file)
        call_conductor(job, master_ip, input_file, hdfs_input_file,
                       memory=memory, override_parameters=spark_conf)

    else:
        if not input_file.startswith("hdfs://"):
            _log.warn("If not in S3, input file (%s) expected to be in HDFS (%s).",
                      input_file, hdfs_dir)

    # where are we writing the output to? is it going to a location in hdfs or not?
    run_upload = True
    hdfs_output_file = hdfs_dir + "kmer_output.txt"
    if output_file.startswith(hdfs_dir):
        run_upload = False
        hdfs_output_file = output_file

    # do we need to convert to adam?
    if (hdfs_input_file.endswith('.bam') or
        hdfs_input_file.endswith('.sam') or
        hdfs_input_file.endswith('.fq') or
        hdfs_input_file.endswith('.fastq')):

        hdfs_tmp_file = hdfs_input_file

        # change the file extension to adam
        hdfs_input_file = '.'.join(hdfs_input_file.split('.')[:-1].append('adam'))

        # convert the file
        _log.info('Converting %s into ADAM format at %s.', hdfs_tmp_file, hdfs_input_file)
        call_adam(job, master_ip,
                  ['transform',
                   hdfs_tmp_file, hdfs_input_file],
                  memory=memory, override_parameters=spark_conf)

    # run k-mer counting
    _log.info('Counting %d-mers in %s, and saving to %s.',
              kmer_length, hdfs_input_file, hdfs_output_file)
    call_adam(job, master_ip,
              ['count_kmers',
               hdfs_input_file, hdfs_output_file,
               str(kmer_length)],
              memory=memory, override_parameters=spark_conf)

    # do we need to upload the file back? if so, run upload
    if run_upload:
        _log.info("Uploading output file %s to %s.", hdfs_output_file, output_file)
        call_conductor(job, master_ip, hdfs_output_file, output_file,
                       memory=memory, override_parameters=spark_conf)

Response: Download and upload k - mers from the master to the master.


Instruction:
How would you code a function in Python 3 to
set up command line parser for Toil/ADAM based k-mer counter, and launches k-mer counter with optional Spark cluster.

Response:
def main():
    '''
    Sets up command line parser for Toil/ADAM based k-mer counter, and launches
    k-mer counter with optional Spark cluster.
    '''

    parser = argparse.ArgumentParser()

    # add parser arguments
    parser.add_argument('--input_path',
                        help='The full path to the input SAM/BAM/ADAM/FASTQ file.')
    parser.add_argument('--output-path',
                        help='full path where final results will be output.')
    parser.add_argument('--kmer-length',
                        help='Length to use for k-mer counting. Defaults to 20.',
                        default=20,
                        type=int)
    parser.add_argument('--spark-conf',
                        help='Optional configuration to pass to Spark commands. Either this or --workers must be specified.',
                        default=None)
    parser.add_argument('--memory',
                        help='Optional memory configuration for Spark workers/driver. This must be specified if --workers is specified.',
                        default=None,
                        type=int)
    parser.add_argument('--cores',
                        help='Optional core configuration for Spark workers/driver. This must be specified if --workers is specified.',
                        default=None,
                        type=int)
    parser.add_argument('--workers',
                        help='Number of workers to spin up in Toil. Either this or --spark-conf must be specified. If this is specified, --memory and --cores must be specified.',
                        default=None,
                        type=int)
    parser.add_argument('--sudo',
                        help='Run docker containers with sudo. Defaults to False.',
                        default=False,
                        action='store_true')

    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    Job.Runner.startToil(Job.wrapJobFn(kmer_dag,
                                       args.kmer_length,
                                       args.input_path,
                                       args.output_path,
                                       args.spark_conf,
                                       args.workers,
                                       args.cores,
                                       args.memory,
                                       args.sudo,
                                       checkpoint=True), args)


Instruction:
Write a Python 3 function that can
run GATK germline pipeline for a cohort of samples.

Response:
def run_gatk_germline_pipeline(job, samples, config):
    """
    Downloads shared files and calls the GATK best practices germline pipeline for a cohort of samples

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param list[GermlineSample] samples: List of GermlineSample namedtuples
    :param Namespace config: Configuration options for pipeline
        Requires the following config attributes:
        config.preprocess_only      If True, then stops pipeline after preprocessing steps
        config.joint_genotype       If True, then joint genotypes cohort
        config.run_oncotator        If True, then adds Oncotator to pipeline
        Additional parameters are needed for downstream steps. Refer to pipeline README for more information.
    """
    # Determine the available disk space on a worker node before any jobs have been run.
    work_dir = job.fileStore.getLocalTempDir()
    st = os.statvfs(work_dir)
    config.available_disk = st.f_bavail * st.f_frsize

    # Check that there is a reasonable number of samples for joint genotyping
    num_samples = len(samples)
    if config.joint_genotype and not 30 < num_samples < 200:
        job.fileStore.logToMaster('WARNING: GATK recommends batches of '
                                  '30 to 200 samples for joint genotyping. '
                                  'The current cohort has %d samples.' % num_samples)

    shared_files = Job.wrapJobFn(download_shared_files, config).encapsulate()
    job.addChild(shared_files)

    if config.preprocess_only:
        for sample in samples:
            shared_files.addChildJobFn(prepare_bam,
                                       sample.uuid,
                                       sample.url,
                                       shared_files.rv(),
                                       paired_url=sample.paired_url,
                                       rg_line=sample.rg_line)
    else:
        run_pipeline = Job.wrapJobFn(gatk_germline_pipeline,
                                     samples,
                                     shared_files.rv()).encapsulate()
        shared_files.addChild(run_pipeline)

        if config.run_oncotator:
            annotate = Job.wrapJobFn(annotate_vcfs, run_pipeline.rv(), shared_files.rv())
            run_pipeline.addChild(annotate)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def gatk_germline_pipeline(job, samples, config):
    require(len(samples) > 0, 'No samples were provided!')

    # Get total size of genome reference files. This is used for configuring disk size.
    genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size

    # 0: Generate processed BAM and BAI files for each sample
    # group preprocessing and variant calling steps in empty Job instance
    group_bam_jobs = Job()
    gvcfs = {}
    for sample in samples:
        # 0: Generate processed BAM and BAI files for each sample
        get_bam = group_bam_jobs.addChildJobFn(prepare_bam,
                                               sample.uuid,
                                               sample.url,
                                               config,
                                               paired_url=sample.paired_url,
                                               rg_line=sample.rg_line)

        # 1: Generate per sample gvcfs {uuid: gvcf_id}
        # The HaplotypeCaller disk requirement depends on the input bam, bai, the genome reference
        # files, and the output GVCF file. The output GVCF is smaller than the input BAM file.
        hc_disk = PromisedRequirement(lambda bam, bai, ref_size:
                                      2 * bam.size + bai.size + ref_size,
                                      get_bam.rv(0),
                                      get_bam.rv(1),
                                      genome_ref_size)

        get_gvcf = get_bam.addFollowOnJobFn(gatk_haplotype_caller,
                                            get_bam.rv(0),
                                            get_bam.rv(1),
                                            config.genome_fasta, config.genome_fai, config.genome_dict,
                                            annotations=config.annotations,
                                            cores=config.cores,
                                            disk=hc_disk,
                                            memory=config.xmx,
                                            hc_output=config.hc_output)
        # Store cohort GVCFs in dictionary
        gvcfs[sample.uuid] = get_gvcf.rv()

        # Upload individual sample GVCF before genotyping to a sample specific output directory
        vqsr_name = '{}{}.g.vcf'.format(sample.uuid, config.suffix)
        get_gvcf.addChildJobFn(output_file_job,
                               vqsr_name,
                               get_gvcf.rv(),
                               os.path.join(config.output_dir, sample.uuid),
                               s3_key_path=config.ssec,
                               disk=PromisedRequirement(lambda x: x.size, get_gvcf.rv()))

    # VQSR requires many variants in order to train a decent model. GATK recommends a minimum of
    # 30 exomes or one large WGS sample:
    # https://software.broadinstitute.org/gatk/documentation/article?id=3225

    filtered_vcfs = {}
    if config.joint_genotype:
        # Need to configure joint genotype in a separate function to resolve promises
        filtered_vcfs = group_bam_jobs.addFollowOnJobFn(joint_genotype_and_filter,
                                                        gvcfs,
                                                        config).rv()

    # If not joint genotyping, then iterate over cohort and genotype and filter individually.
    else:
        for uuid, gvcf_id in gvcfs.iteritems():
            filtered_vcfs[uuid] = group_bam_jobs.addFollowOnJobFn(genotype_and_filter,
                                                                  {uuid: gvcf_id},
                                                                  config).rv()

    job.addChild(group_bam_jobs)
    return filtered_vcfs

Response: This function runs the GATK best practices pipeline for Germline SNP and INDEL discovery.


Instruction:
Create a Python 3 function for
checking for enough disk space for joint genotyping, then calls the genotype and filter pipeline function. :param JobFunctionWrappingJob job: passed automatically by Toil :param dict gvcfs: Dictionary of GVCFs {Sample ID: FileStoreID} :param Namespace config: Input parameters and reference FileStoreIDs Requires the following config attributes: config.genome_fasta FilesStoreID for reference genome fasta file config.genome_fai FilesStoreID for reference genome fasta index file config.genome_dict FilesStoreID for reference genome sequence dictionary file config.available_disk Total available disk space :returns: FileStoreID for the joint genotyped and filtered VCF file :rtype: str

Response:
def joint_genotype_and_filter(job, gvcfs, config):
    """
    Checks for enough disk space for joint genotyping, then calls the genotype and filter pipeline function.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param dict gvcfs: Dictionary of GVCFs {Sample ID: FileStoreID}
    :param Namespace config: Input parameters and reference FileStoreIDs
        Requires the following config attributes:
        config.genome_fasta         FilesStoreID for reference genome fasta file
        config.genome_fai           FilesStoreID for reference genome fasta index file
        config.genome_dict          FilesStoreID for reference genome sequence dictionary file
        config.available_disk       Total available disk space
    :returns: FileStoreID for the joint genotyped and filtered VCF file
    :rtype: str
    """
    # Get the total size of genome reference files
    genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size

    # Require at least 2.5x the sum of the individual GVCF files
    cohort_size = sum(gvcf.size for gvcf in gvcfs.values())
    require(int(2.5 * cohort_size + genome_ref_size) < config.available_disk,
            'There is not enough disk space to joint '
            'genotype samples:\n{}'.format('\n'.join(gvcfs.keys())))

    job.fileStore.logToMaster('Merging cohort into a single GVCF file')

    return job.addChildJobFn(genotype_and_filter, gvcfs, config).rv()


Instruction:
Given the following Python 3 function, write the documentation
def genotype_and_filter(job, gvcfs, config):
    # Get the total size of the genome reference
    genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size

    # GenotypeGVCF disk requirement depends on the input GVCF, the genome reference files, and
    # the output VCF file. The output VCF is smaller than the input GVCF.
    genotype_gvcf_disk = PromisedRequirement(lambda gvcf_ids, ref_size:
                                             2 * sum(gvcf_.size for gvcf_ in gvcf_ids) + ref_size,
                                             gvcfs.values(),
                                             genome_ref_size)

    genotype_gvcf = job.addChildJobFn(gatk_genotype_gvcfs,
                                      gvcfs,
                                      config.genome_fasta,
                                      config.genome_fai,
                                      config.genome_dict,
                                      annotations=config.annotations,
                                      unsafe_mode=config.unsafe_mode,
                                      cores=config.cores,
                                      disk=genotype_gvcf_disk,
                                      memory=config.xmx)

    # Determine if output GVCF has multiple samples
    if len(gvcfs) == 1:
        uuid = gvcfs.keys()[0]
    else:
        uuid = 'joint_genotyped'

    genotyped_filename = '%s.genotyped%s.vcf' % (uuid, config.suffix)
    genotype_gvcf.addChildJobFn(output_file_job,
                                genotyped_filename,
                                genotype_gvcf.rv(),
                                os.path.join(config.output_dir, uuid),
                                s3_key_path=config.ssec,
                                disk=PromisedRequirement(lambda x: x.size, genotype_gvcf.rv()))

    if config.run_vqsr:
        if not config.joint_genotype:
            job.fileStore.logToMaster('WARNING: Running VQSR without joint genotyping.')
        joint_genotype_vcf = genotype_gvcf.addFollowOnJobFn(vqsr_pipeline,
                                                            uuid,
                                                            genotype_gvcf.rv(),
                                                            config)
    else:
        joint_genotype_vcf = genotype_gvcf.addFollowOnJobFn(hard_filter_pipeline,
                                                            uuid,
                                                            genotype_gvcf.rv(),
                                                            config)
    return joint_genotype_vcf.rv()

Response: This function takes a dictionary of GVCFs and runs either the VQSR or hard filtering pipeline.


Instruction:
Given the following Python 3 function, write the documentation
def annotate_vcfs(job, vcfs, config):
    job.fileStore.logToMaster('Running Oncotator on the following samples:\n%s' % '\n'.join(vcfs.keys()))
    for uuid, vcf_id in vcfs.iteritems():
        # The Oncotator disk requirement depends on the input VCF, the Oncotator database
        # and the output VCF. The annotated VCF will be significantly larger than the input VCF.
        onco_disk = PromisedRequirement(lambda vcf, db: 3 * vcf.size + db.size,
                                        vcf_id,
                                        config.oncotator_db)

        annotated_vcf = job.addChildJobFn(run_oncotator,
                                          vcf_id,
                                          config.oncotator_db,
                                          disk=onco_disk,
                                          cores=config.cores,
                                          memory=config.xmx)

        output_dir = os.path.join(config.output_dir, uuid)
        filename = '{}.oncotator{}.vcf'.format(uuid, config.suffix)
        annotated_vcf.addChildJobFn(output_file_job,
                                    filename,
                                    annotated_vcf.rv(),
                                    output_dir,
                                    s3_key_path=config.ssec,
                                    disk=PromisedRequirement(lambda x: x.size, annotated_vcf.rv()))

Response: Runs Oncotator on a group of VCF files.


Instruction:
Can you tell what is the following Python 3 function doing
def parse_manifest(path_to_manifest):
    bam_re = r"^(?P<uuid>\S+)\s(?P<url>\S+[bsc][r]?am)"
    fq_re = r"^(?P<uuid>\S+)\s(?P<url>\S+)\s(?P<paired_url>\S+)?\s?(?P<rg_line>@RG\S+)"
    samples = []
    with open(path_to_manifest, 'r') as f:
        for line in f.readlines():
            line = line.strip()
            if line.startswith('#'):
                continue
            bam_match = re.match(bam_re, line)
            fastq_match = re.match(fq_re, line)
            if bam_match:
                uuid = bam_match.group('uuid')
                url = bam_match.group('url')
                paired_url = None
                rg_line = None
                require('.bam' in url.lower(),
                        'Expected .bam extension:\n{}:\t{}'.format(uuid, url))
            elif fastq_match:
                uuid = fastq_match.group('uuid')
                url = fastq_match.group('url')
                paired_url = fastq_match.group('paired_url')
                rg_line = fastq_match.group('rg_line')
                require('.fq' in url.lower() or '.fastq' in url.lower(),
                        'Expected .fq extension:\n{}:\t{}'.format(uuid, url))
            else:
                raise ValueError('Could not parse entry in manifest: %s\n%s' % (f.name, line))
            # Checks that URL has a scheme
            require(urlparse(url).scheme, 'Invalid URL passed for {}'.format(url))
            samples.append(GermlineSample(uuid, url, paired_url, rg_line))
    return samples

Response: Parses the manifest file for Toil Germline Pipeline and returns a list of namedtuples.


Instruction:
Implement a function in Python 3 to
download shared reference files for the given pipeline and returns updated config with shared files downloaded

Response:
def download_shared_files(job, config):
    """
    Downloads shared reference files for Toil Germline pipeline

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param Namespace config: Pipeline configuration options
    :return: Updated config with shared fileStoreIDS
    :rtype: Namespace
    """
    job.fileStore.logToMaster('Downloading shared reference files')
    shared_files = {'genome_fasta', 'genome_fai', 'genome_dict'}
    nonessential_files = {'genome_fai', 'genome_dict'}

    # Download necessary files for pipeline configuration
    if config.run_bwa:
        shared_files |= {'amb', 'ann', 'bwt', 'pac', 'sa', 'alt'}
        nonessential_files.add('alt')
    if config.preprocess:
        shared_files |= {'g1k_indel', 'mills', 'dbsnp'}
    if config.run_vqsr:
        shared_files |= {'g1k_snp', 'mills', 'dbsnp', 'hapmap', 'omni'}
    if config.run_oncotator:
        shared_files.add('oncotator_db')
    for name in shared_files:
        try:
            url = getattr(config, name, None)
            if url is None:
                continue
            setattr(config, name, job.addChildJobFn(download_url_job,
                                                    url,
                                                    name=name,
                                                    s3_key_path=config.ssec,
                                                    disk='15G'   # Estimated reference file size
                                                    ).rv())
        finally:
            if getattr(config, name, None) is None and name not in nonessential_files:
                raise ValueError("Necessary configuration parameter is missing:\n{}".format(name))
    return job.addFollowOnJobFn(reference_preprocessing, config).rv()


Instruction:
How would you code a function in Python 3 to
create a genome fasta index and sequence dictionary file if not already present in the pipeline config.

Response:
def reference_preprocessing(job, config):
    """
    Creates a genome fasta index and sequence dictionary file if not already present in the pipeline config.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param Namespace config: Pipeline configuration options and shared files.
                             Requires FileStoreID for genome fasta file as config.genome_fasta
    :return: Updated config with reference index files
    :rtype: Namespace
    """
    job.fileStore.logToMaster('Preparing Reference Files')
    genome_id = config.genome_fasta
    if getattr(config, 'genome_fai', None) is None:
        config.genome_fai = job.addChildJobFn(run_samtools_faidx,
                                              genome_id,
                                              cores=config.cores).rv()
    if getattr(config, 'genome_dict', None) is None:
        config.genome_dict = job.addChildJobFn(run_picard_create_sequence_dictionary,
                                               genome_id,
                                               cores=config.cores,
                                               memory=config.xmx).rv()
    return config


Instruction:
Can you write a function in Python 3 where it
prepares BAM file for GATK preprocessing.

Response:
def prepare_bam(job, uuid, url, config, paired_url=None, rg_line=None):
    """
    Prepares BAM file for Toil germline pipeline.

    Steps in pipeline
    0: Download and align BAM or FASTQ sample
    1: Sort BAM
    2: Index BAM
    3: Run GATK preprocessing pipeline (Optional)
        - Uploads preprocessed BAM to output directory

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str uuid: Unique identifier for the sample
    :param str url: URL or local path to BAM file or FASTQs
    :param Namespace config: Configuration options for pipeline
        Requires the following config attributes:
        config.genome_fasta         FilesStoreID for reference genome fasta file
        config.genome_fai           FilesStoreID for reference genome fasta index file
        config.genome_dict          FilesStoreID for reference genome sequence dictionary file
        config.g1k_indel            FileStoreID for 1000G INDEL resource file
        config.mills                FileStoreID for Mills resource file
        config.dbsnp                FileStoreID for dbSNP resource file
        config.suffix               Suffix added to output filename
        config.output_dir           URL or local path to output directory
        config.ssec                 Path to key file for SSE-C encryption
        config.cores                Number of cores for each job
        config.xmx                  Java heap size in bytes
    :param str|None paired_url: URL or local path to paired FASTQ file, default is None
    :param str|None rg_line: RG line for BWA alignment (i.e. @RG\tID:foo\tSM:bar), default is None
    :return: BAM and BAI FileStoreIDs
    :rtype: tuple
    """
    # 0: Align FASTQ or realign BAM
    if config.run_bwa:
        get_bam = job.wrapJobFn(setup_and_run_bwakit,
                                uuid,
                                url,
                                rg_line,
                                config,
                                paired_url=paired_url).encapsulate()

    # 0: Download BAM
    elif '.bam' in url.lower():
        job.fileStore.logToMaster("Downloading BAM: %s" % uuid)
        get_bam = job.wrapJobFn(download_url_job,
                                url,
                                name='toil.bam',
                                s3_key_path=config.ssec,
                                disk=config.file_size).encapsulate()
    else:
        raise ValueError('Could not generate BAM file for %s\n'
                         'Provide a FASTQ URL and set run-bwa or '
                         'provide a BAM URL that includes .bam extension.' % uuid)

    # 1: Sort BAM file if necessary
    # Realigning BAM file shuffles read order
    if config.sorted and not config.run_bwa:
        sorted_bam = get_bam

    else:
        # The samtools sort disk requirement depends on the input bam, the tmp files, and the
        # sorted output bam.
        sorted_bam_disk = PromisedRequirement(lambda bam: 3 * bam.size, get_bam.rv())
        sorted_bam = get_bam.addChildJobFn(run_samtools_sort,
                                           get_bam.rv(),
                                           cores=config.cores,
                                           disk=sorted_bam_disk)

    # 2: Index BAM
    # The samtools index disk requirement depends on the input bam and the output bam index
    index_bam_disk = PromisedRequirement(lambda bam: bam.size, sorted_bam.rv())
    index_bam = job.wrapJobFn(run_samtools_index, sorted_bam.rv(), disk=index_bam_disk)

    job.addChild(get_bam)
    sorted_bam.addChild(index_bam)

    if config.preprocess:
        preprocess = job.wrapJobFn(run_gatk_preprocessing,
                                   sorted_bam.rv(),
                                   index_bam.rv(),
                                   config.genome_fasta,
                                   config.genome_dict,
                                   config.genome_fai,
                                   config.g1k_indel,
                                   config.mills,
                                   config.dbsnp,
                                   memory=config.xmx,
                                   cores=config.cores).encapsulate()
        sorted_bam.addChild(preprocess)
        index_bam.addChild(preprocess)

        # Update output BAM promises
        output_bam_promise = preprocess.rv(0)
        output_bai_promise = preprocess.rv(1)

        # Save processed BAM
        output_dir = os.path.join(config.output_dir, uuid)
        filename = '{}.preprocessed{}.bam'.format(uuid, config.suffix)
        output_bam = job.wrapJobFn(output_file_job,
                                   filename,
                                   preprocess.rv(0),
                                   output_dir,
                                   s3_key_path=config.ssec)
        preprocess.addChild(output_bam)

    else:
        output_bam_promise = sorted_bam.rv()
        output_bai_promise = index_bam.rv()

    return output_bam_promise, output_bai_promise


Instruction:
Can you tell what is the following Python 3 function doing
def setup_and_run_bwakit(job, uuid, url, rg_line, config, paired_url=None):
    bwa_config = deepcopy(config)
    bwa_config.uuid = uuid
    bwa_config.rg_line = rg_line

    # bwa_alignment uses a different naming convention
    bwa_config.ref = config.genome_fasta
    bwa_config.fai = config.genome_fai

    # Determine if sample is a FASTQ or BAM file using the file extension
    basename, ext = os.path.splitext(url)
    ext = ext.lower()
    if ext == '.gz':
        _, ext = os.path.splitext(basename)
        ext = ext.lower()

    # The pipeline currently supports FASTQ and BAM files
    require(ext in ['.fq', '.fastq', '.bam'],
            'Please use .fq or .bam file extensions:\n%s' % url)

    # Download fastq files
    samples = []
    input1 = job.addChildJobFn(download_url_job,
                               url,
                               name='file1',
                               s3_key_path=config.ssec,
                               disk=config.file_size)

    samples.append(input1.rv())

    # If the extension is for a BAM  file, then configure bwakit to realign the BAM file.
    if ext == '.bam':
        bwa_config.bam = input1.rv()
    else:
        bwa_config.r1 = input1.rv()

    # Download the paired FASTQ URL
    if paired_url:
        input2 = job.addChildJobFn(download_url_job,
                                   paired_url,
                                   name='file2',
                                   s3_key_path=config.ssec,
                                   disk=config.file_size)
        samples.append(input2.rv())
        bwa_config.r2 = input2.rv()

    # The bwakit disk requirement depends on the size of the input files and the index
    # Take the sum of the input files and scale it by a factor of 4
    bwa_index_size = sum([getattr(config, index_file).size
                          for index_file in ['amb', 'ann', 'bwt', 'pac', 'sa', 'alt']
                          if getattr(config, index_file, None) is not None])

    bwakit_disk = PromisedRequirement(lambda lst, index_size:
                                      int(4 * sum(x.size for x in lst) + index_size),
                                      samples,
                                      bwa_index_size)

    return job.addFollowOnJobFn(run_bwakit,
                                bwa_config,
                                sort=False,         # BAM files are sorted later in the pipeline
                                trim=config.trim,
                                cores=config.cores,
                                disk=bwakit_disk).rv()

Response: Downloads and runs bwakit for the given sample identifier.


Instruction:
How would you implement a function in Python 3 that
uses GATK HaplotypeCaller to identify SNPs and INDELs. Outputs variants in a Genomic VCF file. :param JobFunctionWrappingJob job: passed automatically by Toil :param str bam: FileStoreID for BAM file :param str bai: FileStoreID for BAM index file :param str ref: FileStoreID for reference genome fasta file :param str ref_dict: FileStoreID for reference sequence dictionary file :param str fai: FileStoreID for reference fasta index file :param list[str] annotations: List of GATK variant annotations, default is None :param float emit_threshold: Minimum phred-scale confidence threshold for a variant to be emitted, default is 10.0 :param float call_threshold: Minimum phred-scale confidence threshold for a variant to be called, default is 30.0 :param bool unsafe_mode: If True, runs gatk UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY" :param str hc_output: URL or local path to pre-cooked VCF file, default is None :return: FileStoreID for GVCF file :rtype: str

Response:
def gatk_haplotype_caller(job,
                          bam, bai,
                          ref, fai, ref_dict,
                          annotations=None,
                          emit_threshold=10.0, call_threshold=30.0,
                          unsafe_mode=False,
                          hc_output=None):
    """
    Uses GATK HaplotypeCaller to identify SNPs and INDELs. Outputs variants in a Genomic VCF file.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str bam: FileStoreID for BAM file
    :param str bai: FileStoreID for BAM index file
    :param str ref: FileStoreID for reference genome fasta file
    :param str ref_dict: FileStoreID for reference sequence dictionary file
    :param str fai: FileStoreID for reference fasta index file
    :param list[str] annotations: List of GATK variant annotations, default is None
    :param float emit_threshold: Minimum phred-scale confidence threshold for a variant to be emitted, default is 10.0
    :param float call_threshold: Minimum phred-scale confidence threshold for a variant to be called, default is 30.0
    :param bool unsafe_mode: If True, runs gatk UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :param str hc_output: URL or local path to pre-cooked VCF file, default is None
    :return: FileStoreID for GVCF file
    :rtype: str
    """
    job.fileStore.logToMaster('Running GATK HaplotypeCaller')

    inputs = {'genome.fa': ref,
              'genome.fa.fai': fai,
              'genome.dict': ref_dict,
              'input.bam': bam,
              'input.bam.bai': bai}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    # Call GATK -- HaplotypeCaller with parameters to produce a genomic VCF file:
    # https://software.broadinstitute.org/gatk/documentation/article?id=2803
    command = ['-T', 'HaplotypeCaller',
               '-nct', str(job.cores),
               '-R', 'genome.fa',
               '-I', 'input.bam',
               '-o', 'output.g.vcf',
               '-stand_call_conf', str(call_threshold),
               '-stand_emit_conf', str(emit_threshold),
               '-variant_index_type', 'LINEAR',
               '-variant_index_parameter', '128000',
               '--genotyping_mode', 'Discovery',
               '--emitRefConfidence', 'GVCF']

    if unsafe_mode:
        command = ['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'] + command

    if annotations:
        for annotation in annotations:
            command.extend(['-A', annotation])

    # Uses docker_call mock mode to replace output with hc_output file
    outputs = {'output.g.vcf': hc_output}
    docker_call(job=job, work_dir=work_dir,
                env={'JAVA_OPTS': '-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)},
                parameters=command,
                tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
                inputs=inputs.keys(),
                outputs=outputs,
                mock=True if outputs['output.g.vcf'] else False)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.g.vcf'))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main():
    # Define Parser object and add to jobTree
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)

    # Generate subparsers
    subparsers = parser.add_subparsers(dest='command')
    subparsers.add_parser('generate-config',
                          help='Generates an editable config in the current working directory.')
    subparsers.add_parser('generate-manifest',
                          help='Generates an editable manifest in the current working directory.')
    subparsers.add_parser('generate',
                          help='Generates a config and manifest in the current working directory.')

    # Run subparser
    parser_run = subparsers.add_parser('run', help='Runs the GATK germline pipeline')
    parser_run.add_argument('--config',
                            required=True,
                            type=str,
                            help='Path to the (filled in) config file, generated with '
                                 '"generate-config".')
    parser_run.add_argument('--manifest',
                            type=str,
                            help='Path to the (filled in) manifest file, generated with '
                                 '"generate-manifest".\nDefault value: "%(default)s".')
    parser_run.add_argument('--sample',
                            default=None,
                            nargs=2,
                            type=str,
                            help='Input sample identifier and BAM file URL or local path')
    parser_run.add_argument('--output-dir',
                            default=None,
                            help='Path/URL to output directory')
    parser_run.add_argument('-s', '--suffix',
                            default=None,
                            help='Additional suffix to add to the names of the output files')
    parser_run.add_argument('--preprocess-only',
                            action='store_true',
                            help='Only runs preprocessing steps')

    Job.Runner.addToilOptions(parser_run)
    options = parser.parse_args()

    cwd = os.getcwd()
    if options.command == 'generate-config' or options.command == 'generate':
        generate_file(os.path.join(cwd, 'config-toil-germline.yaml'), generate_config)
    if options.command == 'generate-manifest' or options.command == 'generate':
        generate_file(os.path.join(cwd, 'manifest-toil-germline.tsv'), generate_manifest)
    elif options.command == 'run':
        # Program checks
        for program in ['curl', 'docker']:
            require(next(which(program)),
                    program + ' must be installed on every node.'.format(program))

        require(os.path.exists(options.config), '{} not found. Please run "generate-config"'.format(options.config))

        # Read sample manifest
        samples = []
        if options.manifest:
            samples.extend(parse_manifest(options.manifest))

        # Add BAM sample from command line
        if options.sample:
            uuid, url = options.sample
            # samples tuple: (uuid, url, paired_url, rg_line)
            # BAM samples should not have as paired URL or read group line
            samples.append(GermlineSample(uuid, url, None, None))

        require(len(samples) > 0,
                'No samples were detected in the manifest or on the command line')

        # Parse inputs
        inputs = {x.replace('-', '_'): y for x, y in
                  yaml.load(open(options.config).read()).iteritems()}

        required_fields = {'genome_fasta',
                           'output_dir',
                           'run_bwa',
                           'sorted',
                           'snp_filter_annotations',
                           'indel_filter_annotations',
                           'preprocess',
                           'preprocess_only',
                           'run_vqsr',
                           'joint_genotype',
                           'run_oncotator',
                           'cores',
                           'file_size',
                           'xmx',
                           'suffix'}

        input_fields = set(inputs.keys())
        require(input_fields > required_fields,
                'Missing config parameters:\n{}'.format(', '.join(required_fields - input_fields)))

        if inputs['output_dir'] is None:
            inputs['output_dir'] = options.output_dir

        require(inputs['output_dir'] is not None,
                'Missing output directory PATH/URL')

        if inputs['suffix'] is None:
            inputs['suffix'] = options.suffix if options.suffix else ''

        if inputs['preprocess_only'] is None:
            inputs['preprocess_only'] = options.preprocess_only

        if inputs['run_vqsr']:
            # Check that essential VQSR parameters are present
            vqsr_fields = {'g1k_snp', 'mills', 'dbsnp', 'hapmap', 'omni'}
            require(input_fields > vqsr_fields,
                    'Missing parameters for VQSR:\n{}'.format(', '.join(vqsr_fields - input_fields)))

        # Check that hard filtering parameters are present. If only running preprocessing steps, then we do
        # not need filtering information.
        elif not inputs['preprocess_only']:
            hard_filter_fields = {'snp_filter_name', 'snp_filter_expression',
                                  'indel_filter_name', 'indel_filter_expression'}
            require(input_fields > hard_filter_fields,
                    'Missing parameters for hard filtering:\n{}'.format(', '.join(hard_filter_fields - input_fields)))

            # Check for falsey hard filtering parameters
            for hard_filter_field in hard_filter_fields:
                require(inputs[hard_filter_field], 'Missing %s value for hard filtering, '
                                                   'got %s.' % (hard_filter_field, inputs[hard_filter_field]))

        # Set resource parameters
        inputs['xmx'] = human2bytes(inputs['xmx'])
        inputs['file_size'] = human2bytes(inputs['file_size'])
        inputs['cores'] = int(inputs['cores'])

        inputs['annotations'] = set(inputs['snp_filter_annotations'] + inputs['indel_filter_annotations'])

        # HaplotypeCaller test data for testing
        inputs['hc_output'] = inputs.get('hc_output', None)

        # It is a toil-scripts convention to store input parameters in a Namespace object
        config = argparse.Namespace(**inputs)

        root = Job.wrapJobFn(run_gatk_germline_pipeline, samples, config)
        Job.Runner.startToil(root, options)

Response: This function is the main function of the GATK germline pipeline. It is responsible for running the GATK germline pipeline with variant filtering and annotation.


Instruction:
Create a Python 3 function to
loop over the sample_ids in the manifest creating child jobs for each sample_id

Response:
def sample_loop(job, uuid_list, inputs):
  """
  Loops over the sample_ids (uuids) in the manifest, creating child jobs to process each
  """

  for uuid_rg in uuid_list:

    uuid_items = uuid_rg.split(',')
    uuid = uuid_items[0]
    rg_line = None
    if len(uuid_items) > 1:
        rg_line = uuid_items[1]

    job.addChildJobFn(static_dag, uuid, rg_line, inputs)


Instruction:
Create a Python 3 function to
prefer this here as it allows us to pull the job functions from other jobs without rewrapping the job functions back together. bwa_inputs: Input arguments to be passed to BWA. adam_inputs: Input arguments to be passed to ADAM. gatk_preprocess_inputs: Input arguments to be passed to GATK preprocessing. gatk_adam_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of ADAM preprocessing. gatk_gatk_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of GATK preprocessing.

Response:
def static_dag(job, uuid, rg_line, inputs):
    """
    Prefer this here as it allows us to pull the job functions from other jobs
    without rewrapping the job functions back together.

    bwa_inputs: Input arguments to be passed to BWA.
    adam_inputs: Input arguments to be passed to ADAM.
    gatk_preprocess_inputs: Input arguments to be passed to GATK preprocessing.
    gatk_adam_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of ADAM preprocessing.
    gatk_gatk_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of GATK preprocessing.
    """

    # get work directory
    work_dir = job.fileStore.getLocalTempDir()

    inputs.cpu_count = cpu_count()
    inputs.maxCores = sys.maxint
    args = {'uuid': uuid,
            's3_bucket': inputs.s3_bucket,
            'sequence_dir': inputs.sequence_dir,
            'dir_suffix': inputs.dir_suffix}

    # get head BWA alignment job function and encapsulate it
    inputs.rg_line = rg_line
    inputs.output_dir = 's3://{s3_bucket}/alignment{dir_suffix}'.format(**args)
    bwa = job.wrapJobFn(download_reference_files,
                        inputs,
                        [[uuid,
                         ['s3://{s3_bucket}/{sequence_dir}/{uuid}_1.fastq.gz'.format(**args),
                          's3://{s3_bucket}/{sequence_dir}/{uuid}_2.fastq.gz'.format(**args)]]]).encapsulate()

    # get head ADAM preprocessing job function and encapsulate it
    adam_preprocess = job.wrapJobFn(static_adam_preprocessing_dag,
                                    inputs,
                                    's3://{s3_bucket}/alignment{dir_suffix}/{uuid}.bam'.format(**args),
                                    's3://{s3_bucket}/analysis{dir_suffix}/{uuid}'.format(**args),
                                    suffix='.adam').encapsulate()

    # Configure options for Toil Germline pipeline. This function call only runs the preprocessing steps.
    gatk_preprocessing_inputs = copy.deepcopy(inputs)
    gatk_preprocessing_inputs.suffix = '.gatk'
    gatk_preprocessing_inputs.preprocess = True
    gatk_preprocessing_inputs.preprocess_only = True
    gatk_preprocessing_inputs.output_dir = 's3://{s3_bucket}/analysis{dir_suffix}'.format(**args)

    # get head GATK preprocessing job function and encapsulate it
    gatk_preprocess = job.wrapJobFn(run_gatk_germline_pipeline,
                                    GermlineSample(uuid,
                                                   's3://{s3_bucket}/alignment{dir_suffix}/{uuid}.bam'.format(**args),
                                                   None,    # Does not require second URL or RG_Line
                                                   None),
                                    gatk_preprocessing_inputs).encapsulate()

    # Configure options for Toil Germline pipeline for preprocessed ADAM BAM file.
    adam_call_inputs = inputs
    adam_call_inputs.suffix = '.adam'
    adam_call_inputs.sorted = True
    adam_call_inputs.preprocess = False
    adam_call_inputs.run_vqsr = False
    adam_call_inputs.joint_genotype = False
    adam_call_inputs.output_dir = 's3://{s3_bucket}/analysis{dir_suffix}'.format(**args)

    # get head GATK haplotype caller job function for the result of ADAM preprocessing and encapsulate it
    gatk_adam_call = job.wrapJobFn(run_gatk_germline_pipeline,
                                   GermlineSample(uuid,
                                                  's3://{s3_bucket}/analysis{dir_suffix}/{uuid}/{uuid}.adam.bam'.format(**args),
                                                  None,
                                                  None),
                                   adam_call_inputs).encapsulate()

    # Configure options for Toil Germline pipeline for preprocessed GATK BAM file.
    gatk_call_inputs = copy.deepcopy(inputs)
    gatk_call_inputs.sorted = True
    gatk_call_inputs.preprocess = False
    gatk_call_inputs.run_vqsr = False
    gatk_call_inputs.joint_genotype = False
    gatk_call_inputs.output_dir = 's3://{s3_bucket}/analysis{dir_suffix}'.format(**args)

    # get head GATK haplotype caller job function for the result of GATK preprocessing and encapsulate it
    gatk_gatk_call = job.wrapJobFn(run_gatk_germline_pipeline,
                                   GermlineSample(uuid,
                                                  'S3://{s3_bucket}/analysis{dir_suffix}/{uuid}/{uuid}.gatk.bam'.format(**args),
                                                  None, None),
                                   gatk_call_inputs).encapsulate()

    # wire up dag
    if not inputs.skip_alignment:
        job.addChild(bwa)

    if (inputs.pipeline_to_run == "adam" or
        inputs.pipeline_to_run == "both"):

        if inputs.skip_preprocessing:

            job.addChild(gatk_adam_call)
        else:
            if inputs.skip_alignment:
                job.addChild(adam_preprocess)
            else:
                bwa.addChild(adam_preprocess)

            adam_preprocess.addChild(gatk_adam_call)

    if (inputs.pipeline_to_run == "gatk" or
        inputs.pipeline_to_run == "both"):

        if inputs.skip_preprocessing:

            job.addChild(gatk_gatk_call)
        else:
            if inputs.skip_alignment:
                job.addChild(gatk_preprocess)
            else:
                bwa.addChild(gatk_preprocess)

            gatk_preprocess.addChild(gatk_gatk_call)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main():
    # Define Parser object and add to Toil
    if mock_mode():
        usage_msg = 'You have the TOIL_SCRIPTS_MOCK_MODE environment variable set, so this pipeline ' \
                    'will run in mock mode. To disable mock mode, set TOIL_SCRIPTS_MOCK_MODE=0'
    else:
        usage_msg = None

    parser = argparse.ArgumentParser(usage=usage_msg)
    subparsers = parser.add_subparsers(dest='command')
    subparsers.add_parser('generate-config', help='Generates an editable config in the current working directory.')
    subparsers.add_parser('generate-manifest', help='Generates an editable manifest in the current working directory.')
    subparsers.add_parser('generate', help='Generates a config and manifest in the current working directory.')
    # Run subparser
    parser_run = subparsers.add_parser('run', help='Runs the ADAM/GATK pipeline')
    default_config = 'adam-gatk-mock.config' if mock_mode() else 'adam-gatk.config'
    default_manifest = 'adam-gatk-mock-manifest.csv' if mock_mode() else 'adam-gatk-manifest.csv'
    parser_run.add_argument('--config', default=default_config, type=str,
                            help='Path to the (filled in) config file, generated with "generate-config".')
    parser_run.add_argument('--manifest', default=default_manifest,
                            type=str, help='Path to the (filled in) manifest file, generated with "generate-manifest". '
                                           '\nDefault value: "%(default)s".')
    Job.Runner.addToilOptions(parser_run)
    args = parser.parse_args()

    cwd = os.getcwd()
    if args.command == 'generate-config' or args.command == 'generate':
        generate_file(os.path.join(cwd, default_config), generate_config)
    if args.command == 'generate-manifest' or args.command == 'generate':
        generate_file(os.path.join(cwd, default_manifest), generate_manifest)
    # Pipeline execution
    elif args.command == 'run':
        require(os.path.exists(args.config), '{} not found. Please run '
                                             'generate-config'.format(args.config))
        if not hasattr(args, 'sample'):
            require(os.path.exists(args.manifest), '{} not found and no samples provided. Please '
                                                   'run "generate-manifest"'.format(args.manifest))
        # Parse config
        parsed_config = {x.replace('-', '_'): y for x, y in yaml.load(open(args.config).read()).iteritems()}
        inputs = argparse.Namespace(**parsed_config)

        # Parse manifest file
        uuid_list = []
        with open(args.manifest) as f_manifest:
            for line in f_manifest:
                if not line.isspace() and not line.startswith('#'):
                    uuid_list.append(line.strip())

        inputs.sort = False
        if not inputs.dir_suffix:
            inputs.dir_suffix = ''
        if not inputs.s3_bucket:
            inputs.s3_bucket = ''

        if inputs.master_ip and inputs.num_nodes:
            raise ValueError("Exactly one of master_ip (%s) and num_nodes (%d) must be provided." %
                             (inputs.master_ip, inputs.num_nodes))

        if not hasattr(inputs, 'master_ip') and inputs.num_nodes <= 1:
            raise ValueError('num_nodes allocates one Spark/HDFS master and n-1 workers, and thus must be greater '
                             'than 1. %d was passed.' % inputs.num_nodes)

        if (inputs.pipeline_to_run != "adam" and
            inputs.pipeline_to_run != "gatk" and
            inputs.pipeline_to_run != "both"):
            raise ValueError("pipeline_to_run must be either 'adam', 'gatk', or 'both'. %s was passed." % inputs.pipeline_to_run)

        Job.Runner.startToil(Job.wrapJobFn(sample_loop, uuid_list, inputs), args)

Response: This is the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function.