Instruction:
Given the following Python 3 function, write the documentation
def __get_placeholders(self):
        ret = True

        pattern = re.compile('(@[A-Za-z0-9_.]+(%(max-)?type)?@)')
        matches = pattern.findall(self._routine_source_code)

        placeholders = []

        if len(matches) != 0:
            for tmp in matches:
                placeholder = tmp[0]
                if placeholder.lower() not in self._replace_pairs:
                    raise LoaderException("Unknown placeholder '{0}' in file {1}".
                                          format(placeholder, self._source_filename))
                if placeholder not in placeholders:
                    placeholders.append(placeholder)

        for placeholder in placeholders:
            if placeholder not in self._replace:
                self._replace[placeholder] = self._replace_pairs[placeholder.lower()]

        return ret

Response: Extracts the placeholders from the stored routine source.


Instruction:
Given the following Python 3 function, write the documentation
def _get_designation_type(self):
        positions = self._get_specification_positions()
        if positions[0] != -1 and positions[1] != -1:
            pattern = re.compile(r'^\s*--\s+type\s*:\s*(\w+)\s*(.+)?\s*', re.IGNORECASE)
            for line_number in range(positions[0], positions[1] + 1):
                matches = pattern.findall(self._routine_source_code_lines[line_number])
                if matches:
                    self._designation_type = matches[0][0].lower()
                    tmp = str(matches[0][1])
                    if self._designation_type == 'bulk_insert':
                        n = re.compile(r'([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_,]+)', re.IGNORECASE)
                        info = n.findall(tmp)
                        if not info:
                            raise LoaderException('Expected: -- type: bulk_insert <table_name> <columns> in file {0}'.
                                                  format(self._source_filename))
                        self._table_name = info[0][0]
                        self._columns = str(info[0][1]).split(',')

                    elif self._designation_type == 'rows_with_key' or self._designation_type == 'rows_with_index':
                        self._columns = str(matches[0][1]).split(',')
                    else:
                        if matches[0][1]:
                            raise LoaderException('Expected: -- type: {}'.format(self._designation_type))

        if not self._designation_type:
            raise LoaderException("Unable to find the designation type of the stored routine in file {0}".
                                  format(self._source_filename))

Response: Extracts the designation type of the stored routine.


Instruction:
Make a summary of the following Python 3 code
def _get_specification_positions(self):
        start = -1
        for (i, line) in enumerate(self._routine_source_code_lines):
            if self._is_start_of_stored_routine(line):
                start = i

        end = -1
        for (i, line) in enumerate(self._routine_source_code_lines):
            if self._is_start_of_stored_routine_body(line):
                end = i - 1

        return start, end

Response: Returns a tuple with the start and end line numbers of the stored routine specification.


Instruction:
How would you explain what the following Python 3 function does
def __get_doc_block_lines(self):
        line1 = None
        line2 = None

        i = 0
        for line in self._routine_source_code_lines:
            if re.match(r'\s*/\*\*', line):
                line1 = i

            if re.match(r'\s*\*/', line):
                line2 = i

            if self._is_start_of_stored_routine(line):
                break

            i += 1

        return line1, line2

Response: Returns the start and end line of the DOcBlock of the stored routine code.


Instruction:
Here you have a function in Python 3, explain what it does
def __get_doc_block_parts_source(self):
        line1, line2 = self.__get_doc_block_lines()

        if line1 is not None and line2 is not None and line1 <= line2:
            doc_block = self._routine_source_code_lines[line1:line2 - line1 + 1]
        else:
            doc_block = list()

        reflection = DocBlockReflection(doc_block)

        self._doc_block_parts_source['description'] = reflection.get_description()

        self._doc_block_parts_source['parameters'] = list()
        for tag in reflection.get_tags('param'):
            parts = re.match(r'^(@param)\s+(\w+)\s*(.+)?', tag, re.DOTALL)
            if parts:
                self._doc_block_parts_source['parameters'].append({'name':        parts.group(2),
                                                                   'description': parts.group(3)})

Response: Extracts the DocBlock from the stored routine source.


Instruction:
Explain what the following Python 3 code does
def __get_doc_block_parts_wrapper(self):
        self.__get_doc_block_parts_source()

        helper = self._get_data_type_helper()

        parameters = list()
        for parameter_info in self._parameters:
            parameters.append(
                {'parameter_name':       parameter_info['name'],
                 'python_type':          helper.column_type_to_python_type(parameter_info),
                 'data_type_descriptor': parameter_info['data_type_descriptor'],
                 'description':          self.__get_parameter_doc_description(parameter_info['name'])})

        self._doc_block_parts_wrapper['description'] = self._doc_block_parts_source['description']
        self._doc_block_parts_wrapper['parameters'] = parameters

Response: Generates the DocBlock parts to be used by the wrapper generator.


Instruction:
Write a Python 3 script for
updating the metadata of the stored routine.

Response:
def _update_metadata(self):
        """
        Updates the metadata of the stored routine.
        """
        self._pystratum_metadata['routine_name'] = self._routine_name
        self._pystratum_metadata['designation'] = self._designation_type
        self._pystratum_metadata['table_name'] = self._table_name
        self._pystratum_metadata['parameters'] = self._parameters
        self._pystratum_metadata['columns'] = self._columns
        self._pystratum_metadata['fields'] = self._fields
        self._pystratum_metadata['column_types'] = self._columns_types
        self._pystratum_metadata['timestamp'] = self._m_time
        self._pystratum_metadata['replace'] = self._replace
        self._pystratum_metadata['pydoc'] = self._doc_block_parts_wrapper


Instruction:
Can you implement a function in Python 3 that
sets the magic constants to replace list.

Response:
def _set_magic_constants(self):
        """
        Adds magic constants to replace list.
        """
        real_path = os.path.realpath(self._source_filename)

        self._replace['__FILE__'] = "'%s'" % real_path
        self._replace['__ROUTINE__'] = "'%s'" % self._routine_name
        self._replace['__DIR__'] = "'%s'" % os.path.dirname(real_path)


Instruction:
Can you implement a function in Python 3 that
removes magic constants from current replace list.

Response:
def _unset_magic_constants(self):
        """
        Removes magic constants from current replace list.
        """
        if '__FILE__' in self._replace:
            del self._replace['__FILE__']

        if '__ROUTINE__' in self._replace:
            del self._replace['__ROUTINE__']

        if '__DIR__' in self._replace:
            del self._replace['__DIR__']

        if '__LINE__' in self._replace:
            del self._replace['__LINE__']


Instruction:
Given the following Python 3 function, write the documentation
def _print_sql_with_error(self, sql, error_line):
        if os.linesep in sql:
            lines = sql.split(os.linesep)
            digits = math.ceil(math.log(len(lines) + 1, 10))
            i = 1
            for line in lines:
                if i == error_line:
                    self._io.text('<error>{0:{width}} {1}</error>'.format(i, line, width=digits, ))
                else:
                    self._io.text('{0:{width}} {1}'.format(i, line, width=digits, ))
                i += 1
        else:
            self._io.text(sql)

Response: Writes a SQL statement with an error to the output.


Instruction:
Implement a function in Python 3 to
return a list of items according to filter parameters.

Response:
def list_filter(lst, startswith=None, notstartswith=None,
                contain=None, notcontain=None):
    """ Keep in list items according to filter parameters.

    :param lst: item list
    :param startswith: keep items starting with
    :param notstartswith: remove items starting with
    :return:
    """
    keeped = []
    for item in lst:
        keep = False
        if startswith is not None:
            if item.startswith(startswith):
                keep = True
        if notstartswith is not None:
            if not item.startswith(notstartswith):
                keep = True
        if contain is not None:
            if contain in item:
                keep = True
        if notcontain is not None:
            if not notcontain in item:
                keep = True

        if keep:
            keeped.append(item)
    return keeped


Instruction:
Write a Python 3 function for
splitting dict into two subdicts based on keys.

Response:
def split_dict(dct, keys):
    """
    Split dict into two subdicts based on keys.

    :param dct:
    :param keys:
    :return: dict_in, dict_out
    """
    if type(dct) == collections.OrderedDict:
        dict_in = collections.OrderedDict()
        dict_out = collections.OrderedDict()
    else:
        dict_in = {}
        dict_out = {}

    for key, value in dct.items:
        if key in keys:
            dict_in[key] = value
        else:
            dict_out[key] = value
    return dict_in, dict_out


Instruction:
Can you create a Python 3 function that
flattens dict with defined key join symbol.

Response:
def flatten_dict_join_keys(dct, join_symbol=" "):
    """ Flatten dict with defined key join symbol.

    :param dct: dict to flatten
    :param join_symbol: default value is " "
    :return:
    """
    return dict( flatten_dict(dct, join=lambda a,b:a+join_symbol+b) )


Instruction:
Here you have a function in Python 3, explain what it does
def list_contains(list_of_strings, substring, return_true_false_array=False):
    key_tf = [keyi.find(substring) != -1 for keyi in list_of_strings]
    if return_true_false_array:
        return key_tf
    keys_to_remove = list_of_strings[key_tf]
    return keys_to_remove

Response: Get strings in list which contains substring.


Instruction:
Can you tell what is the following Python 3 function doing
def df_drop_duplicates(df, ignore_key_pattern="time"):

    keys_to_remove = list_contains(df.keys(), ignore_key_pattern)
    #key_tf = [key.find(noinfo_key_pattern) != -1 for key in df.keys()]
    # keys_to_remove
    # remove duplicates
    ks = copy.copy(list(df.keys()))
    for key in keys_to_remove:
        ks.remove(key)

    df = df.drop_duplicates(ks)
    return df

Response: Drop duplicates from dataframe ignore columns with keys containing defined pattern.


Instruction:
Can you tell what is the following Python 3 function doing
def ndarray_to_list_in_structure(item, squeeze=True):
    tp = type(item)

    if tp == np.ndarray:
        if squeeze:
            item = item.squeeze()
        item = item.tolist()
    elif tp == list:
        for i in range(len(item)):
            item[i] = ndarray_to_list_in_structure(item[i])
    elif tp == dict:
        for lab in item:
            item[lab] = ndarray_to_list_in_structure(item[lab])

    return item

Response: Change ndarray in structure of lists and dicts into lists.


Instruction:
Write a Python 3 function for
finding first suitable key in dict.

Response:
def dict_find_key(dd, value):
    """ Find first suitable key in dict.

    :param dd:
    :param value:
    :return:
    """
    key = next(key for key, val in dd.items() if val == value)
    return key


Instruction:
Can you generate the documentation for the following Python 3 function
def sort_list_of_dicts(lst_of_dct, keys, reverse=False, **sort_args):

    if type(keys) != list:
        keys = [keys]
    # dcmdir = lst_of_dct[:]
    # lst_of_dct.sort(key=lambda x: [x[key] for key in keys], reverse=reverse, **sort_args)
    lst_of_dct.sort(key=lambda x: [((False, x[key]) if key in x else (True, 0)) for key in keys], reverse=reverse, **sort_args)
    return lst_of_dct

Response: Sort list of dicts by one or multiple keys.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ordered_dict_to_dict(config):

    if type(config) == collections.OrderedDict:
        config = dict(config)
    if type(config) == list:
        for i in range(0, len(config)):
            config[i] = ordered_dict_to_dict(config[i])
    elif type(config) == dict:
        for key in config:
            config[key] = ordered_dict_to_dict(config[key])

    return config

Response: Convert ordered dict to dict


Instruction:
Given the following Python 3 function, write the documentation
def run_command(self, config_file):
        config = configparser.ConfigParser()
        config.read(config_file)

        rdbms = config.get('database', 'rdbms').lower()

        wrapper = self.create_routine_wrapper_generator(rdbms)
        wrapper.main(config_file)

Response: Runs the neccesary command.


Instruction:
Make a summary of the following Python 3 code
def create_routine_wrapper_generator(self, rdbms):
        # Note: We load modules and classes dynamically such that on the end user's system only the required modules
        #       and other dependencies for the targeted RDBMS must be installed (and required modules and other
        #       dependencies for the other RDBMSs are not required).

        if rdbms == 'mysql':
            module = locate('pystratum_mysql.MySqlRoutineWrapperGenerator')
            return module.MySqlRoutineWrapperGenerator(self.output)

        if rdbms == 'mssql':
            module = locate('pystratum_mssql.MsSqlRoutineWrapperGenerator')
            return module.MsSqlRoutineWrapperGenerator(self.output)

        if rdbms == 'pgsql':
            module = locate('pystratum_pgsql.PgSqlRoutineWrapperGenerator')
            return module.PgSqlRoutineWrapperGenerator(self.output)

        raise Exception("Unknown RDBMS '{0!s}'.".format(rdbms))

Response: Returns a routine wrapper generator for the given RDBMS.


Instruction:
Write a Python 3 script for
reading an option for a configuration file.

Response:
def _get_option(config, supplement, section, option, fallback=None):
        """
        Reads an option for a configuration file.

        :param configparser.ConfigParser config: The main config file.
        :param configparser.ConfigParser supplement: The supplement config file.
        :param str section: The name of the section op the option.
        :param str option: The name of the option.
        :param str|None fallback: The fallback value of the option if it is not set in either configuration files.

        :rtype: str

        :raise KeyError:
        """
        if supplement:
            return_value = supplement.get(section, option, fallback=config.get(section, option, fallback=fallback))
        else:
            return_value = config.get(section, option, fallback=fallback)

        if fallback is None and return_value is None:
            raise KeyError("Option '{0!s}' is not found in section '{1!s}'.".format(option, section))

        return return_value


Instruction:
Create a Python 3 function for
reading the configuration file and returns the configparser and config_supplement object.

Response:
def _read_configuration(config_filename):
        """
        Checks the supplement file.

        :param str config_filename: The name of the configuration file.

        :rtype: (configparser.ConfigParser,configparser.ConfigParser)
        """
        config = ConfigParser()
        config.read(config_filename)

        if 'supplement' in config['database']:
            path = os.path.dirname(config_filename) + '/' + config.get('database', 'supplement')
            config_supplement = ConfigParser()
            config_supplement.read(path)
        else:
            config_supplement = None

        return config, config_supplement


Instruction:
Here you have a function in Python 3, explain what it does
def record_asciinema():
    '''a wrapper around generation of an asciinema.api.Api and a custom
       recorder to pull out the input arguments to the Record from argparse.
       The function generates a filename in advance and a return code
       so we can check the final status.
    '''

    import asciinema.config as aconfig
    from asciinema.api import Api

    # Load the API class

    cfg = aconfig.load()
    api = Api(cfg.api_url, os.environ.get("USER"), cfg.install_id)

    # Create dummy class to pass in as args
    recorder = HelpMeRecord(api)
    code = recorder.execute()

    if code == 0 and os.path.exists(recorder.filename):
        return recorder.filename
    print('Problem generating %s, return code %s' %(recorder.filename, code))

Response: a wrapper around generating an asciinema. api. Api and a custom helpmeRecord class to pull out the input arguments to the Record from argparse.


Instruction:
Given the following Python 3 function, write the documentation
def sendWakeOnLan(self, macAddress, lanInterfaceId=1, timeout=1):
        namespace = Fritz.getServiceType("sendWakeOnLan") + str(lanInterfaceId)
        uri = self.getControlURL(namespace)

        self.execute(uri, namespace, "X_AVM-DE_WakeOnLANByMACAddress", timeout=timeout,
                     NewMACAddress=macAddress)

Response: Send a wake up package to a device specified by its MAC address.


Instruction:
How would you implement a function in Python 3 that
does a software update of the Fritz Box if available.

Response:
def doUpdate(self, timeout=1):
        """Do a software update of the Fritz Box if available.

        :param float timeout: the timeout to wait for the action to be executed
        :return: a list of if an update was available and the update state (bool, str)
        :rtype: tuple(bool, str)
        """
        namespace = Fritz.getServiceType("doUpdate")
        uri = self.getControlURL(namespace)

        results = self.execute(uri, namespace, "X_AVM-DE_DoUpdate", timeout=timeout)

        return results["NewUpgradeAvailable"], results["NewX_AVM-DE_UpdateState"]


Instruction:
Can you generate the documentation for the following Python 3 function
def isOptimizedForIPTV(self, wifiInterfaceId=1, timeout=1):
        namespace = Fritz.getServiceType("isOptimizedForIPTV") + str(wifiInterfaceId)
        uri = self.getControlURL(namespace)

        results = self.execute(uri, namespace, "X_AVM-DE_GetIPTVOptimized", timeout=timeout)

        return bool(int(results["NewX_AVM-DE_IPTVoptimize"]))

Response: Execute X_AVM - DE - GetIPTVOptimized action to get the optimized status of the Wifi interface.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def setOptimizedForIPTV(self, status, wifiInterfaceId=1, timeout=1):
        namespace = Fritz.getServiceType("setOptimizedForIPTV") + str(wifiInterfaceId)
        uri = self.getControlURL(namespace)

        if status:
            setStatus = 1
        else:
            setStatus = 0

        arguments = {"timeout": timeout, "NewX_AVM-DE_IPTVoptimize": setStatus}

        self.execute(uri, namespace, "X_AVM-DE_SetIPTVOptimized", **arguments)

Response: Set if the Wifi interface is optimized for IP TV


Instruction:
Can you create a Python 3 function that
gets the list of phone calls made by the user.

Response:
def getCallList(self, timeout=1):
        """Get the list of phone calls made

        Example of a phone call result:

        ::

            [{'Count': None, 'Name': None, 'CalledNumber': '030868709971', 'Numbertype': 'sip', 'Duration': '0:01',
            'Caller': '015155255399', 'Called': 'SIP: 030868729971', 'Date': '02.01.14 13:14',
            'Device': 'Anrufbeantworter','Path': None, 'Port': '40', 'Type': '1', 'Id': '15'}]

        Types:

        * 1 - answered
        * 2 - missed
        * 3 - outgoing

        :param float timeout: the timeout to wait for the action to be executed
        :return: the list of made phone calls
        :rtype: list[dict[str: str]]
        """
        namespace = Fritz.getServiceType("getCallList")
        uri = self.getControlURL(namespace)

        results = self.execute(uri, namespace, "GetCallList")

        # setup proxies
        proxies = {}
        if self.httpProxy:
            proxies = {"https": self.httpProxy}

        if self.httpsProxy:
            proxies = {"http": self.httpsProxy}

        # get the content
        request = requests.get(results["NewCallListURL"], proxies=proxies, timeout=float(timeout))

        if request.status_code != 200:
            errorStr = DeviceTR64._extractErrorString(request)
            raise ValueError('Could not get CPE definitions "' + results["NewCallListURL"] + '" : ' +
                             str(request.status_code) + ' - ' + request.reason + " -- " + errorStr)

        # parse xml
        try:
            root = ET.fromstring(request.text.encode('utf-8'))
        except Exception as e:
            raise ValueError("Could not parse call list '" + results["NewCallListURL"] + "': " + str(e))

        calls = []

        for child in root.getchildren():
            if child.tag.lower() == "call":

                callParameters = {}

                for callChild in child.getchildren():
                    callParameters[callChild.tag] = callChild.text

                calls.append(callParameters)

        return calls


Instruction:
Make a summary of the following Python 3 code
def add_tcp_callback(port, callback, threaded_callback=False):
    _rpio.add_tcp_callback(port, callback, threaded_callback)

Response:
Adds a unix socket server callback which will be invoked when values
    arrive from a connected socket client.


Instruction:
Can you generate the documentation for the following Python 3 function
def add_interrupt_callback(gpio_id, callback, edge='both', \
        pull_up_down=PUD_OFF, threaded_callback=False, \
        debounce_timeout_ms=None):
    """
    Add a callback to be executed when the value on 'gpio_id' changes to
    the edge specified via the 'edge' parameter (default='both').

    `pull_up_down` can be set to `RPIO.PUD_UP`, `RPIO.PUD_DOWN`, and
    `RPIO.PUD_OFF`.

    If `threaded_callback` is True, the callback will be started
    inside a Thread.

    If debounce_timeout_ms is set, new interrupts will not be forwarded
    until after the specified amount of milliseconds.
    """
    _rpio.add_interrupt_callback(gpio_id, callback, edge, pull_up_down, \
            threaded_callback, debounce_timeout_ms)

Response: Add a callback to be executed when the value of gpio_id changes to the edge specified via the edge parameter.


Instruction:
Can you implement a function in Python 3 that
blocks loop to listen for GPIO interrupts and distribute them to the associated callbacks.

Response:
def wait_for_interrupts(threaded=False, epoll_timeout=1):
    """
    Blocking loop to listen for GPIO interrupts and distribute them to
    associated callbacks. epoll_timeout is an easy way to shutdown the
    blocking function. Per default the timeout is set to 1 second; if
    `_is_waiting_for_interrupts` is set to False the loop will exit.

    If an exception occurs while waiting for interrupts, the interrupt
    gpio interfaces will be cleaned up (/sys/class/gpio unexports). In
    this case all interrupts will be reset and you'd need to add the
    callbacks again before using `wait_for_interrupts(..)` again.

    If the argument `threaded` is True, wait_for_interrupts will be
    started in a daemon Thread. To quit it, call
    `RPIO.stop_waiting_for_interrupts()`.
    """
    if threaded:
        t = Thread(target=_rpio.wait_for_interrupts, args=(epoll_timeout,))
        t.daemon = True
        t.start()
    else:
        _rpio.wait_for_interrupts(epoll_timeout)


Instruction:
Can you write a function in Python 3 where it
builds source with Python 2. 7 and 3. 2 and tests.

Response:
def build_gpio():
    """ Builds source with Python 2.7 and 3.2, and tests import """
    with cd("/tmp/source/c_gpio"):
        test = "import _GPIO; print(_GPIO.VERSION_GPIO)"
        run("make gpio2.7 && cp build/_GPIO.so .")
        run('sudo python2.7 -c "%s"' % test)
        run("cp _GPIO.so ../RPIO/")
        run("cp _GPIO.so ../RPIO/_GPIO27.so")
        run("make gpio3.2 && cp build/_GPIO.so .")
        run('sudo python3.2 -c "%s"' % test)
        run("mv _GPIO.so ../RPIO/_GPIO32.so")


Instruction:
Given the following Python 3 function, write the documentation
def build_pwm():
    with cd("/tmp/source/c_pwm"):
        test = "import _PWM; print(_PWM.VERSION)"
        run("make py2.7")
        run('sudo python2.7 -c "%s"' % test)
        run("cp _PWM.so ../RPIO/PWM/")
        run("mv _PWM.so ../RPIO/PWM/_PWM27.so")
        run("make py3.2")
        run('python3.2 -c "%s"' % test)
        run("mv _PWM.so ../RPIO/PWM/_PWM32.so")

Response: Builds source with Python 2. 7 and 3. 2 and tests.


Instruction:
Can you generate the documentation for the following Python 3 function
def upload_to_pypi():
    # One more safety input and then we are ready to go :)
    x = prompt("Are you sure to upload the current version to pypi?")
    if not x or not x.lower() in ["y", "yes"]:
        return

    local("rm -rf dist")
    local("python setup.py sdist")
    version = _get_cur_version()
    fn = "RPIO-%s.tar.gz" % version
    put("dist/%s" % fn, "/tmp/")
    with cd("/tmp"):
        run("tar -xf /tmp/%s" % fn)
    with cd("/tmp/RPIO-%s" % version):
        run("python2.6 setup.py bdist_egg upload")
        run("python2.7 setup.py bdist_egg upload")
        run("python3.2 setup.py bdist_egg upload")
    local("python setup.py sdist upload")

Response: Upload sdist and bdist_eggs to pypi


Instruction:
Can you generate the documentation for the following Python 3 function
def add_tcp_callback(self, port, callback, threaded_callback=False):
        if not callback:
            raise AttributeError("No callback")

        serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        serversocket.bind((_TCP_SOCKET_HOST, port))
        serversocket.listen(1)
        serversocket.setblocking(0)
        self._epoll.register(serversocket.fileno(), select.EPOLLIN)

        # Prepare the callback (wrap in Thread if needed)
        cb = callback if not threaded_callback else \
                partial(_threaded_callback, callback)

        self._tcp_server_sockets[serversocket.fileno()] = (serversocket, cb)
        debug("Socket server started at port %s and callback added." % port)

Response:
Adds a unix socket server callback which will be invoked when values
        arrive from a connected socket client.


Instruction:
Implement a Python 3 function for
adding a callback to be executed when the value of a gpio_id changes to the specified edge.

Response:
def add_interrupt_callback(self, gpio_id, callback, edge='both',
            pull_up_down=_GPIO.PUD_OFF, threaded_callback=False,
            debounce_timeout_ms=None):
        """
        Add a callback to be executed when the value on 'gpio_id' changes to
        the edge specified via the 'edge' parameter (default='both').

        `pull_up_down` can be set to `RPIO.PUD_UP`, `RPIO.PUD_DOWN`, and
        `RPIO.PUD_OFF`.

        If `threaded_callback` is True, the callback will be started
        inside a Thread.
        """
        gpio_id = _GPIO.channel_to_gpio(gpio_id)
        debug("Adding callback for GPIO %s" % gpio_id)
        if not edge in ["falling", "rising", "both", "none"]:
            raise AttributeError("'%s' is not a valid edge." % edge)

        if not pull_up_down in [_GPIO.PUD_UP, _GPIO.PUD_DOWN, _GPIO.PUD_OFF]:
            raise AttributeError("'%s' is not a valid pull_up_down." % edge)

        # Make sure the gpio_id is valid
        if not gpio_id in set(chain(RPIO.GPIO_LIST_R1, RPIO.GPIO_LIST_R2, \
                              RPIO.GPIO_LIST_R3)):
            raise AttributeError("GPIO %s is not a valid gpio-id." % gpio_id)

        # Require INPUT pin setup; and set the correct PULL_UPDN
        if RPIO.gpio_function(int(gpio_id)) == RPIO.IN:
            RPIO.set_pullupdn(gpio_id, pull_up_down)
        else:
            debug("- changing gpio function from %s to INPUT" % \
                    (GPIO_FUNCTIONS[RPIO.gpio_function(int(gpio_id))]))
            RPIO.setup(gpio_id, RPIO.IN, pull_up_down)

        # Prepare the callback (wrap in Thread if needed)
        cb = callback if not threaded_callback else \
                partial(_threaded_callback, callback)

        # Prepare the /sys/class path of this gpio
        path_gpio = "%sgpio%s/" % (_SYS_GPIO_ROOT, gpio_id)

        # If initial callback for this GPIO then set everything up. Else make
        # sure the edge detection is the same.
        if gpio_id in self._map_gpioid_to_callbacks:
            with open(path_gpio + "edge", "r") as f:
                e = f.read().strip()
                if e != edge:
                    raise AttributeError(("Cannot add callback for gpio %s:"
                            " edge detection '%s' not compatible with existing"
                            " edge detection '%s'.") % (gpio_id, edge, e))

            # Check whether edge is the same, else throw Exception
            debug("- kernel interface already setup for GPIO %s" % gpio_id)
            self._map_gpioid_to_callbacks[gpio_id].append(cb)

        else:
            # If kernel interface already exists unexport first for clean setup
            if os.path.exists(path_gpio):
                if self._show_warnings:
                    warn("Kernel interface for GPIO %s already exists." % \
                            gpio_id)
                debug("- unexporting kernel interface for GPIO %s" % gpio_id)
                with open(_SYS_GPIO_ROOT + "unexport", "w") as f:
                    f.write("%s" % gpio_id)
                time.sleep(0.1)

            # Export kernel interface /sys/class/gpio/gpioN
            with open(_SYS_GPIO_ROOT + "export", "w") as f:
                f.write("%s" % gpio_id)
            self._gpio_kernel_interfaces_created.append(gpio_id)
            debug("- kernel interface exported for GPIO %s" % gpio_id)

            # Configure gpio as input
            with open(path_gpio + "direction", "w") as f:
                f.write("in")

            # Configure gpio edge detection
            with open(path_gpio + "edge", "w") as f:
                f.write(edge)

            debug(("- kernel interface configured for GPIO %s "
                    "(edge='%s', pullupdn=%s)") % (gpio_id, edge, \
                    _PULL_UPDN[pull_up_down]))

            # Open the gpio value stream and read the initial value
            f = open(path_gpio + "value", 'r')
            val_initial = f.read().strip()
            debug("- inital gpio value: %s" % val_initial)
            f.seek(0)

            # Add callback info to the mapping dictionaries
            self._map_fileno_to_file[f.fileno()] = f
            self._map_fileno_to_gpioid[f.fileno()] = gpio_id
            self._map_fileno_to_options[f.fileno()] = {
                    "debounce_timeout_s": debounce_timeout_ms / 1000.0 if \
                            debounce_timeout_ms else 0,
                    "interrupt_last": 0,
                    "edge": edge
                    }
            self._map_gpioid_to_fileno[gpio_id] = f.fileno()
            self._map_gpioid_to_callbacks[gpio_id] = [cb]

            # Add to epoll
            self._epoll.register(f.fileno(), select.EPOLLPRI | select.EPOLLERR)


Instruction:
Explain what the following Python 3 code does
def del_interrupt_callback(self, gpio_id):
        debug("- removing interrupts on gpio %s" % gpio_id)
        gpio_id = _GPIO.channel_to_gpio(gpio_id)
        fileno = self._map_gpioid_to_fileno[gpio_id]

        # 1. Remove from epoll
        self._epoll.unregister(fileno)

        # 2. Cache the file
        f = self._map_fileno_to_file[fileno]

        # 3. Remove from maps
        del self._map_fileno_to_file[fileno]
        del self._map_fileno_to_gpioid[fileno]
        del self._map_fileno_to_options[fileno]
        del self._map_gpioid_to_fileno[gpio_id]
        del self._map_gpioid_to_callbacks[gpio_id]

        # 4. Close file last in case of IOError
        f.close()

Response: Delete all interrupt callbacks from a certain gpio


Instruction:
Given the following Python 3 function, write the documentation
def _handle_interrupt(self, fileno, val):
        val = int(val)

        # Filter invalid edge values (sometimes 1 comes in when edge=falling)
        edge = self._map_fileno_to_options[fileno]["edge"]
        if (edge == 'rising' and val == 0) or (edge == 'falling' and val == 1):
            return

        # If user activated debounce for this callback, check timing now
        debounce = self._map_fileno_to_options[fileno]["debounce_timeout_s"]
        if debounce:
            t = time.time()
            t_last = self._map_fileno_to_options[fileno]["interrupt_last"]
            if t - t_last < debounce:
                debug("- don't start interrupt callback due to debouncing")
                return
            self._map_fileno_to_options[fileno]["interrupt_last"] = t

        # Start the callback(s) now
        gpio_id = self._map_fileno_to_gpioid[fileno]
        if gpio_id in self._map_gpioid_to_callbacks:
            for cb in self._map_gpioid_to_callbacks[gpio_id]:
                cb(gpio_id, val)

Response: Handles interrupt events from the specified fileno.


Instruction:
Given the following Python 3 function, write the documentation
def wait_for_interrupts(self, epoll_timeout=1):
        self._is_waiting_for_interrupts = True
        while self._is_waiting_for_interrupts:
            events = self._epoll.poll(epoll_timeout)
            for fileno, event in events:
                debug("- epoll event on fd %s: %s" % (fileno, event))
                if fileno in self._tcp_server_sockets:
                    # New client connection to socket server
                    serversocket, cb = self._tcp_server_sockets[fileno]
                    connection, address = serversocket.accept()
                    connection.setblocking(0)
                    f = connection.fileno()
                    self._epoll.register(f, select.EPOLLIN)
                    self._tcp_client_sockets[f] = (connection, cb)

                elif event & select.EPOLLIN:
                    # Input from TCP socket
                    socket, cb = self._tcp_client_sockets[fileno]
                    content = socket.recv(1024)
                    if not content or not content.strip():
                        # No content means quitting
                        self.close_tcp_client(fileno)
                    else:
                        sock, cb = self._tcp_client_sockets[fileno]
                        cb(self._tcp_client_sockets[fileno][0], \
                                content.strip())

                elif event & select.EPOLLHUP:
                    # TCP Socket Hangup
                    self.close_tcp_client(fileno)

                elif event & select.EPOLLPRI:
                    # GPIO interrupts
                    f = self._map_fileno_to_file[fileno]
                    # read() is workaround for not getting new values
                    # with read(1)
                    val = f.read().strip()
                    f.seek(0)
                    self._handle_interrupt(fileno, val)

Response: Wait for interrupts and distribute them to the interrupt related callbacks.


Instruction:
Write a Python 3 function that can
clean up the list of created GPIO interfaces that this script created.

Response:
def cleanup_interfaces(self):
        """
        Removes all /sys/class/gpio/gpioN interfaces that this script created,
        and deletes callback bindings. Should be used after using interrupts.
        """
        debug("Cleaning up interfaces...")
        for gpio_id in self._gpio_kernel_interfaces_created:
            # Close the value-file and remove interrupt bindings
            self.del_interrupt_callback(gpio_id)

            # Remove the kernel GPIO interface
            debug("- unexporting GPIO %s" % gpio_id)
            with open(_SYS_GPIO_ROOT + "unexport", "w") as f:
                f.write("%s" % gpio_id)

        # Reset list of created interfaces
        self._gpio_kernel_interfaces_created = []


Instruction:
Write a Python 3 script to
close all TCP connections and then the socket servers.

Response:
def cleanup_tcpsockets(self):
        """
        Closes all TCP connections and then the socket servers
        """
        for fileno in self._tcp_client_sockets.keys():
            self.close_tcp_client(fileno)
        for fileno, items in self._tcp_server_sockets.items():
            socket, cb = items
            debug("- _cleanup server socket connection (fd %s)" % fileno)
            self._epoll.unregister(fileno)
            socket.close()
        self._tcp_server_sockets = {}


Instruction:
Write a Python 3 script for
adding a pulse for a specific GPIO to a dma channel subcycle.

Response:
def add_channel_pulse(dma_channel, gpio, start, width):
    """
    Add a pulse for a specific GPIO to a dma channel subcycle. `start` and
    `width` are multiples of the pulse-width increment granularity.
    """
    return _PWM.add_channel_pulse(dma_channel, gpio, start, width)


Instruction:
Can you tell what is the following Python 3 function doing
def set_servo(self, gpio, pulse_width_us):
        # Make sure we can set the exact pulse_width_us
        _pulse_incr_us = _PWM.get_pulse_incr_us()
        if pulse_width_us % _pulse_incr_us:
            # No clean division possible
            raise AttributeError(("Pulse width increment granularity %sus "
                    "cannot divide a pulse-time of %sus") % (_pulse_incr_us,
                    pulse_width_us))

        # Initialize channel if not already done, else check subcycle time
        if _PWM.is_channel_initialized(self._dma_channel):
            _subcycle_us = _PWM.get_channel_subcycle_time_us(self._dma_channel)
            if _subcycle_us != self._subcycle_time_us:
                raise AttributeError(("Error: DMA channel %s is setup with a "
                        "subcycle_time of %sus (instead of %sus)") % \
                        (self._dma_channel, _subcycle_us,
                            self._subcycle_time_us))
        else:
            init_channel(self._dma_channel, self._subcycle_time_us)

        # Add pulse for this GPIO
        add_channel_pulse(self._dma_channel, gpio, 0, \
                int(pulse_width_us / _pulse_incr_us))

Response: Sets a pulse - width on a gpio to repeat every subcycle - time.


Instruction:
Here you have a function in Python 3, explain what it does
def unique(_list):
    ret = []

    for item in _list:
        if item not in ret:
            ret.append(item)

    return ret

Response: Returns a list of unique items in the order of the items in the list


Instruction:
Write a Python 3 script for
performing initial query cleanup

Response:
def preprocess_query(query):
    """
    Perform initial query cleanup

    :type query str
    :rtype str
    """
    # 1. remove aliases
    # FROM `dimension_wikis` `dw`
    # INNER JOIN `fact_wam_scores` `fwN`
    query = re.sub(r'(\s(FROM|JOIN)\s`[^`]+`)\s`[^`]+`', r'\1', query, flags=re.IGNORECASE)

    # 2. `database`.`table` notation -> database.table
    query = re.sub(r'`([^`]+)`\.`([^`]+)`', r'\1.\2', query)

    # 2. database.table notation -> table
    # query = re.sub(r'([a-z_0-9]+)\.([a-z_0-9]+)', r'\2', query, flags=re.IGNORECASE)

    return query


Instruction:
Can you generate the documentation for the following Python 3 function
def get_query_tokens(query):
    query = preprocess_query(query)
    parsed = sqlparse.parse(query)

    # handle empty queries (#12)
    if not parsed:
        return []

    tokens = TokenList(parsed[0].tokens).flatten()
    # print([(token.value, token.ttype) for token in tokens])

    return [token for token in tokens if token.ttype is not Whitespace]

Response: returns a list of SQL Token objects from a query string


Instruction:
Can you generate the documentation for the following Python 3 function
def get_query_columns(query):
    columns = []
    last_keyword = None
    last_token = None

    # print(preprocess_query(query))

    # these keywords should not change the state of a parser
    # and not "reset" previously found SELECT keyword
    keywords_ignored = ['AS', 'AND', 'OR', 'IN', 'IS', 'NOT', 'NOT NULL', 'LIKE', 'CASE', 'WHEN']

    # these function should be ignored
    # and not "reset" previously found SELECT keyword
    functions_ignored = ['COUNT', 'MIN', 'MAX', 'FROM_UNIXTIME', 'DATE_FORMAT', 'CAST', 'CONVERT']

    for token in get_query_tokens(query):
        if token.is_keyword and token.value.upper() not in keywords_ignored:
            # keep the name of the last keyword, e.g. SELECT, FROM, WHERE, (ORDER) BY
            last_keyword = token.value.upper()
            # print('keyword', last_keyword)
        elif token.ttype is Name:
            # analyze the name tokens, column names and where condition values
            if last_keyword in ['SELECT', 'WHERE', 'BY', 'ON'] \
                    and last_token.value.upper() not in ['AS']:
                # print(last_keyword, last_token, token.value)

                if token.value.upper() not in functions_ignored:
                    if str(last_token) == '.':
                        # print('DOT', last_token, columns[-1])

                        # we have table.column notation example
                        # append column name to the last entry of columns
                        # as it is a table name in fact
                        table_name = columns[-1]
                        columns[-1] = '{}.{}'.format(table_name, token)
                    else:
                        columns.append(str(token.value))
            elif last_keyword in ['INTO'] and last_token.ttype is Punctuation:
                # INSERT INTO `foo` (col1, `col2`) VALUES (..)
                #  print(last_keyword, token, last_token)
                columns.append(str(token.value).strip('`'))
        elif token.ttype is Wildcard:
            # handle * wildcard in SELECT part, but ignore count(*)
            # print(last_keyword, last_token, token.value)
            if last_keyword == 'SELECT' and last_token.value != '(':

                if str(last_token) == '.':
                    # handle SELECT foo.*
                    table_name = columns[-1]
                    columns[-1] = '{}.{}'.format(table_name, str(token))
                else:
                    columns.append(str(token.value))

        last_token = token

    return unique(columns)

Response: returns a list of column names for the given query


Instruction:
Explain what the following Python 3 code does
def get_query_tables(query):
    tables = []
    last_keyword = None
    last_token = None

    table_syntax_keywords = [
        # SELECT queries
        'FROM', 'WHERE', 'JOIN', 'INNER JOIN', 'LEFT JOIN', 'RIGHT JOIN', 'ON',
        # INSERT queries
        'INTO', 'VALUES',
        # UPDATE queries
        'UPDATE', 'SET',
        # Hive queries
        'TABLE',  # INSERT TABLE
    ]

    # print(query, get_query_tokens(query))

    for token in get_query_tokens(query):
        # print([token, token.ttype, last_token, last_keyword])
        if token.is_keyword and token.value.upper() in table_syntax_keywords:
            # keep the name of the last keyword, the next one can be a table name
            last_keyword = token.value.upper()
            # print('keyword', last_keyword)
        elif str(token) == '(':
            # reset the last_keyword for INSERT `foo` VALUES(id, bar) ...
            last_keyword = None
        elif token.is_keyword and str(token) in ['FORCE', 'ORDER']:
            # reset the last_keyword for "SELECT x FORCE INDEX" queries and "SELECT x ORDER BY"
            last_keyword = None
        elif token.is_keyword and str(token) == 'SELECT' and last_keyword in ['INTO', 'TABLE']:
            # reset the last_keyword for "INSERT INTO SELECT" and "INSERT TABLE SELECT" queries
            last_keyword = None
        elif token.ttype is Name or token.is_keyword:
            # print([last_keyword, last_token, token.value])
            # analyze the name tokens, column names and where condition values
            if last_keyword in ['FROM', 'JOIN', 'INNER JOIN', 'LEFT JOIN', 'RIGHT JOIN',
                                'INTO', 'UPDATE', 'TABLE'] \
                    and last_token not in ['AS'] \
                    and token.value not in ['AS', 'SELECT']:

                if last_token == '.':
                    # we have database.table notation example
                    # append table name to the last entry of tables
                    # as it is a database name in fact
                    database_name = tables[-1]
                    tables[-1] = '{}.{}'.format(database_name, token)
                    last_keyword = None
                elif last_token not in [',', last_keyword]:
                    # it's not a list of tables, e.g. SELECT * FROM foo, bar
                    # hence, it can be the case of alias without AS, e.g. SELECT * FROM foo bar
                    pass
                else:
                    table_name = str(token.value.strip('`'))
                    tables.append(table_name)

        last_token = token.value.upper()

    return unique(tables)

Response: returns a list of tables from a query string


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_query_limit_and_offset(query):
    limit = None
    offset = None
    last_keyword = None
    last_token = None

    # print(query)
    for token in get_query_tokens(query):
        # print([token, token.ttype, last_keyword])

        if token.is_keyword and token.value.upper() in ['LIMIT', 'OFFSET']:
            last_keyword = token.value.upper()
        elif token.ttype is Number.Integer:
            # print([token, last_keyword, last_token_was_integer])
            if last_keyword == 'LIMIT':
                # LIMIT <limit>
                limit = int(token.value)
                last_keyword = None
            elif last_keyword == 'OFFSET':
                # OFFSET <offset>
                offset = int(token.value)
                last_keyword = None
            elif last_token and last_token.ttype is Punctuation:
                # LIMIT <offset>,<limit>
                offset = limit
                limit = int(token.value)

        last_token = token

    if limit is None:
        return None

    return limit, offset or 0

Response: returns limit and offset from a query string


Instruction:
Here you have a function in Python 3, explain what it does
def normalize_likes(sql):
    sql = sql.replace('%', '')

    # LIKE '%bot'
    sql = re.sub(r"LIKE '[^\']+'", 'LIKE X', sql)

    # or all_groups LIKE X or all_groups LIKE X
    matches = re.finditer(r'(or|and) [^\s]+ LIKE X', sql, flags=re.IGNORECASE)
    matches = [match.group(0) for match in matches] if matches else None

    if matches:
        for match in set(matches):
            sql = re.sub(r'(\s?' + re.escape(match) + ')+', ' ' + match + ' ...', sql)

    return sql

Response: Normalize and wrap LIKE statements


Instruction:
How would you implement a function in Python 3 that
generalizes a SQL query into a single segment of the SQL query.

Response:
def generalize_sql(sql):
    """
    Removes most variables from an SQL query and replaces them with X or N for numbers.

    Based on Mediawiki's DatabaseBase::generalizeSQL

    :type sql str|None
    :rtype: str
    """
    if sql is None:
        return None

    # multiple spaces
    sql = re.sub(r'\s{2,}', ' ', sql)

    # MW comments
    # e.g. /* CategoryDataService::getMostVisited N.N.N.N */
    sql = remove_comments_from_sql(sql)

    # handle LIKE statements
    sql = normalize_likes(sql)

    sql = re.sub(r"\\\\", '', sql)
    sql = re.sub(r"\\'", '', sql)
    sql = re.sub(r'\\"', '', sql)
    sql = re.sub(r"'[^\']*'", 'X', sql)
    sql = re.sub(r'"[^\"]*"', 'X', sql)

    # All newlines, tabs, etc replaced by single space
    sql = re.sub(r'\s+', ' ', sql)

    # All numbers => N
    sql = re.sub(r'-?[0-9]+', 'N', sql)

    # WHERE foo IN ('880987','882618','708228','522330')
    sql = re.sub(r' (IN|VALUES)\s*\([^,]+,[^)]+\)', ' \\1 (XYZ)', sql, flags=re.IGNORECASE)

    return sql.strip()


Instruction:
Can you generate the documentation for the following Python 3 function
def deblur_system_call(params, input_fp):
    logger = logging.getLogger(__name__)
    logger.debug('[%s] deblur system call params %s, input_fp %s' %
                 (mp.current_process().name, params, input_fp))

    # construct command
    script_name = "deblur"
    script_subprogram = "workflow"
    command = [script_name,
               script_subprogram,
               '--seqs-fp', input_fp,
               '--is-worker-thread',
               '--keep-tmp-files']
    command.extend(params)

    logger.debug('[%s] running command %s' % (mp.current_process().name,
                                              command))
    return _system_call(command)

Response: Build deblur command for subprocess.


Instruction:
How would you explain what the following Python 3 function does
def run_functor(functor, *args, **kwargs):
    try:
        # This is where you do your actual work
        return functor(*args, **kwargs)
    except Exception:
        # Put all exception text into an exception and raise that
        raise Exception("".join(traceback.format_exception(*sys.exc_info())))

Response: Given a functor run it and return its result.


Instruction:
Here you have a function in Python 3, explain what it does
def parallel_deblur(inputs, params,
                    pos_ref_db_fp, neg_ref_dp_fp, jobs_to_start=1):
    """Dispatch execution over a pool of processors

    This code was adopted from the American Gut project:
    https://github.com/biocore/American-Gut/blob/master/americangut/parallel.py

    Parameters
    ----------
    inputs : iterable of str
        File paths to input per-sample sequence files
    params : list of str
        list of CLI parameters supplied to the deblur workflow
        (argv - first 2 are 'deblur','workflow' and are ignored)
    pos_ref_db_fp : list of str
        the indexed positive (16s) sortmerna database
        (created in the main thread)
    neg_ref_db_fp : list of str
        the indexed negative (artifacts) sortmerna database
        (created in the main thread)
    jobs_to_start : int, optional
        The number of processors on the local system to use

    Returns
    -------
    all_result_paths : list
        list of expected output files
    """
    logger = logging.getLogger(__name__)
    logger.info('parallel deblur started for %d inputs' % len(inputs))

    # remove the irrelevant parameters
    remove_param_list = ['-O', '--jobs-to-start', '--seqs-fp',
                         '--pos-ref-db-fp', '--neg-ref-db-fp']
    skipnext = False
    newparams = []
    for carg in params[2:]:
        if skipnext:
            skipnext = False
            continue
        if carg in remove_param_list:
            skipnext = True
            continue
        newparams.append(carg)

    # add the ref_db_fp (since it may be not present in the
    # original command parameters)
    if pos_ref_db_fp:
        new_pos_ref_db_fp = ','.join(pos_ref_db_fp)
        newparams.append('--pos-ref-db-fp')
        newparams.append(new_pos_ref_db_fp)
    if neg_ref_dp_fp:
        new_neg_ref_db_fp = ','.join(neg_ref_dp_fp)
        newparams.append('--neg-ref-db-fp')
        newparams.append(new_neg_ref_db_fp)

    logger.debug('ready for functor %s' % newparams)
    functor = partial(run_functor, deblur_system_call, newparams)
    logger.debug('ready for pool %d jobs' % jobs_to_start)
    pool = mp.Pool(processes=jobs_to_start)
    logger.debug('almost running...')
    for stdout, stderr, es in pool.map(functor, inputs):
        if es != 0:
            raise RuntimeError("stdout: %s\nstderr: %s\nexit: %d" % (stdout,
                                                                     stderr,
                                                                     es))

Response: Dispatch execution over a pool of processors


Instruction:
Write a Python 3 script for
yielding (id, sequence) from an input file Parameters ---------- input_fp : filepath A filepath, which can be any valid fasta or fastq file within the limitations of scikit-bio's IO registry. Notes ----- The use of this method is a stopgap to replicate the existing `parse_fasta` functionality while at the same time allowing for fastq support. Raises ------ skbio.io.FormatIdentificationWarning If the format of the input file cannot be determined. Returns ------- (str, str) The ID and sequence.

Response:
def sequence_generator(input_fp):
    """Yield (id, sequence) from an input file

    Parameters
    ----------
    input_fp : filepath
        A filepath, which can be any valid fasta or fastq file within the
        limitations of scikit-bio's IO registry.

    Notes
    -----
    The use of this method is a stopgap to replicate the existing `parse_fasta`
    functionality while at the same time allowing for fastq support.

    Raises
    ------
    skbio.io.FormatIdentificationWarning
        If the format of the input file cannot be determined.

    Returns
    -------
    (str, str)
        The ID and sequence.

    """
    logger = logging.getLogger(__name__)
    kw = {}
    if sniff_fasta(input_fp)[0]:
        format = 'fasta'
    elif sniff_fastq(input_fp)[0]:
        format = 'fastq'

        kw['variant'] = _get_fastq_variant(input_fp)
    else:
        # usually happens when the fasta file is empty
        # so need to return no sequences (and warn)
        msg = "input file %s does not appear to be FASTA or FASTQ" % input_fp
        logger.warn(msg)
        warnings.warn(msg, UserWarning)
        return

    # some of the test code is using file paths, some is using StringIO.
    if isinstance(input_fp, io.TextIOBase):
        input_fp.seek(0)

    for record in skbio.read(input_fp, format=format, **kw):
        yield (record.metadata['id'], str(record))


Instruction:
Explain what the following Python 3 code does
def trim_seqs(input_seqs, trim_len, left_trim_len):
    # counters for the number of trimmed and total sequences
    logger = logging.getLogger(__name__)

    okseqs = 0
    totseqs = 0

    if trim_len < -1:
        raise ValueError("Invalid trim_len: %d" % trim_len)

    for label, seq in input_seqs:
        totseqs += 1

        if trim_len == -1:
            okseqs += 1
            yield label, seq
        elif len(seq) >= trim_len:
            okseqs += 1
            yield label, seq[left_trim_len:trim_len]

    if okseqs < 0.01*totseqs:
        logger = logging.getLogger(__name__)
        errmsg = 'Vast majority of sequences (%d / %d) are shorter ' \
                 'than the trim length (%d). ' \
                 'Are you using the correct -t trim length?' \
                 % (totseqs-okseqs, totseqs, trim_len)
        logger.warn(errmsg)
        warnings.warn(errmsg, UserWarning)
    else:
        logger.debug('trimmed to length %d (%d / %d remaining)'
                     % (trim_len, okseqs, totseqs))

Response: Trim FASTA sequences to specified length.


Instruction:
Create a Python 3 function to
build a SortMeRNA index for all reference databases.

Response:
def build_index_sortmerna(ref_fp, working_dir):
    """Build a SortMeRNA index for all reference databases.

    Parameters
    ----------
    ref_fp: tuple
        filepaths to FASTA reference databases
    working_dir: string
        working directory path where to store the indexed database

    Returns
    -------
    all_db: tuple
        filepaths to SortMeRNA indexed reference databases
    """
    logger = logging.getLogger(__name__)
    logger.info('build_index_sortmerna files %s to'
                ' dir %s' % (ref_fp, working_dir))
    all_db = []
    for db in ref_fp:
        fasta_dir, fasta_filename = split(db)
        index_basename = splitext(fasta_filename)[0]
        db_output = join(working_dir, index_basename)
        logger.debug('processing file %s into location %s' % (db, db_output))
        params = ['indexdb_rna', '--ref', '%s,%s' %
                  (db, db_output), '--tmpdir', working_dir]
        sout, serr, res = _system_call(params)
        if not res == 0:
            logger.error('Problem running indexdb_rna on file %s to dir %s. '
                         'database not indexed' % (db, db_output))
            logger.debug('stdout: %s' % sout)
            logger.debug('stderr: %s' % serr)
            logger.critical('execution halted')
            raise RuntimeError('Cannot index database file %s' % db)
        logger.debug('file %s indexed' % db)
        all_db.append(db_output)
    return all_db


Instruction:
Can you implement a function in Python 3 that
filters samples from a biom table that have less than minreads reads

Response:
def filter_minreads_samples_from_table(table, minreads=1, inplace=True):
    """Filter samples from biom table that have less than
    minreads reads total

    Paraneters
    ----------
    table : biom.Table
        the biom table to filter
    minreads : int (optional)
        the minimal number of reads in a sample in order to keep it
    inplace : bool (optional)
        if True, filter the biom table in place, if false create a new copy

    Returns
    -------
    table : biom.Table
        the filtered biom table
    """
    logger = logging.getLogger(__name__)
    logger.debug('filter_minreads_started. minreads=%d' % minreads)
    samp_sum = table.sum(axis='sample')
    samp_ids = table.ids(axis='sample')
    bad_samples = samp_ids[samp_sum < minreads]
    if len(bad_samples) > 0:
        logger.warn('removed %d samples with reads per sample<%d'
                    % (len(bad_samples), minreads))
        table = table.filter(bad_samples, axis='sample',
                             inplace=inplace, invert=True)
    else:
        logger.debug('all samples contain > %d reads' % minreads)
    return table


Instruction:
Here you have a function in Python 3, explain what it does
def fasta_from_biom(table, fasta_file_name):
    '''Save sequences from a biom table to a fasta file

    Parameters
    ----------
    table : biom.Table
        The biom table containing the sequences
    fasta_file_name : str
        Name of the fasta output file
    '''
    logger = logging.getLogger(__name__)
    logger.debug('saving biom table sequences to fasta file %s' % fasta_file_name)

    with open(fasta_file_name, 'w') as f:
        for cseq in table.ids(axis='observation'):
            f.write('>%s\n%s\n' % (cseq, cseq))
    logger.info('saved biom table sequences to fasta file %s' % fasta_file_name)

Response: Save sequences from a biom table to a fasta file


Instruction:
Can you generate a brief explanation for the following Python 3 code
def remove_artifacts_from_biom_table(table_filename,
                                     fasta_filename,
                                     ref_fp,
                                     biom_table_dir,
                                     ref_db_fp,
                                     threads=1,
                                     verbose=False,
                                     sim_thresh=None,
                                     coverage_thresh=None):
    """Remove artifacts from a biom table using SortMeRNA

    Parameters
    ----------
    table : str
        name of the biom table file
    fasta_filename : str
        the fasta file containing all the sequences of the biom table

    Returns
    -------
    tmp_files : list of str
        The temp files created during the artifact removal step
    """
    logger = logging.getLogger(__name__)
    logger.info('getting 16s sequences from the biom table')

    # remove artifacts from the fasta file. output is in clean_fp fasta file
    clean_fp, num_seqs_left, tmp_files = remove_artifacts_seqs(fasta_filename, ref_fp,
                                                               working_dir=biom_table_dir,
                                                               ref_db_fp=ref_db_fp,
                                                               negate=False, threads=threads,
                                                               verbose=verbose,
                                                               sim_thresh=sim_thresh,
                                                               coverage_thresh=coverage_thresh)
    if clean_fp is None:
        logger.warn("No clean sequences in %s" % fasta_filename)
        return tmp_files

    logger.debug('removed artifacts from sequences input %s'
                 ' to output %s' % (fasta_filename, clean_fp))

    # read the clean fasta file
    good_seqs = {s for _, s in sequence_generator(clean_fp)}
    logger.debug('loaded %d sequences from cleaned biom table'
                 ' fasta file' % len(good_seqs))

    logger.debug('loading biom table %s' % table_filename)
    table = load_table(table_filename)

    # filter and save the artifact biom table
    artifact_table = table.filter(list(good_seqs),
                                  axis='observation', inplace=False,
                                  invert=True)
    # remove the samples with 0 reads
    filter_minreads_samples_from_table(artifact_table)
    output_nomatch_fp = join(biom_table_dir, 'reference-non-hit.biom')
    write_biom_table(artifact_table, output_nomatch_fp)
    logger.info('wrote artifact only filtered biom table to %s'
                % output_nomatch_fp)
    # and save the reference-non-hit fasta file
    output_nomatch_fasta_fp = join(biom_table_dir, 'reference-non-hit.seqs.fa')
    fasta_from_biom(artifact_table, output_nomatch_fasta_fp)

    # filter and save the only 16s biom table
    table.filter(list(good_seqs), axis='observation')
    # remove the samples with 0 reads
    filter_minreads_samples_from_table(table)
    output_fp = join(biom_table_dir, 'reference-hit.biom')
    write_biom_table(table, output_fp)
    logger.info('wrote 16s filtered biom table to %s' % output_fp)
    # and save the reference-non-hit fasta file
    output_match_fasta_fp = join(biom_table_dir, 'reference-hit.seqs.fa')
    fasta_from_biom(table, output_match_fasta_fp)

    # we also don't need the cleaned fasta file
    tmp_files.append(clean_fp)
    return tmp_files

Response: Remove artifacts from a biom table using SortMeRNA


Instruction:
Can you create a Python 3 function that
removes artifacts from a FASTA file using SortMeRNA.

Response:
def remove_artifacts_seqs(seqs_fp,
                          ref_fp,
                          working_dir,
                          ref_db_fp,
                          negate=False,
                          threads=1,
                          verbose=False,
                          sim_thresh=None,
                          coverage_thresh=None):
    """Remove artifacts from FASTA file using SortMeRNA.

    Parameters
    ----------
    seqs_fp: string
        file path to FASTA input sequence file
    ref_fp: tuple
        file path(s) to FASTA database file
    working_dir: string
        working directory path
    ref_db_fp: tuple
        file path(s) to indexed FASTA database
    negate: boolean, optional
        if True, discard all input sequences aligning
        to reference database
    threads: integer, optional
        number of threads to use for SortMeRNA
    verbose: boolean, optional
        If true, output SortMeRNA errors
    sim_thresh: float, optional
        The minimal similarity threshold (between 0 and 1)
        for keeping the sequence
        if None, the default values used are 0.65 for negate=False,
        0.95 for negate=True
    coverage_thresh: float, optional
        The minimal coverage threshold (between 0 and 1)
        for alignments for keeping the sequence
        if None, the default values used are 0.5 for negate=False,
        0.95 for negate=True

    Returns
    -------
    output_fp : str
        Name of the artifact removed fasta file
    okseqs : int
        The number of sequences left after artifact removal
    tmp_files : list of str
        Names of the tmp files created
    """
    logger = logging.getLogger(__name__)
    logger.info('remove_artifacts_seqs file %s' % seqs_fp)

    if stat(seqs_fp).st_size == 0:
        logger.warn('file %s has size 0, continuing' % seqs_fp)
        return None, 0, []

    if coverage_thresh is None:
        if negate:
            coverage_thresh = 0.95 * 100
        else:
            coverage_thresh = 0.5 * 100

    if sim_thresh is None:
        if negate:
            sim_thresh = 0.95 * 100
        else:
            sim_thresh = 0.65 * 100

    # the minimal average bitscore per nucleotide
    bitscore_thresh = 0.65

    output_fp = join(working_dir,
                     "%s.no_artifacts" % basename(seqs_fp))
    blast_output = join(working_dir,
                        '%s.sortmerna' % basename(seqs_fp))
    aligned_seq_ids = set()
    for i, db in enumerate(ref_fp):
        logger.debug('running on ref_fp %s working dir %s refdb_fp %s seqs %s'
                     % (db, working_dir, ref_db_fp[i], seqs_fp))
        # run SortMeRNA
        # we use -e 100 to remove E-value based filtering by sortmerna
        # since we use bitscore/identity/coverage filtering instead
        params = ['sortmerna', '--reads', seqs_fp, '--ref', '%s,%s' %
                  (db, ref_db_fp[i]),
                  '--aligned', blast_output, '--blast', '3', '--best', '1',
                  '--print_all_reads', '-v', '-e', '100']

        sout, serr, res = _system_call(params)
        if not res == 0:
            logger.error('sortmerna error on file %s' % seqs_fp)
            logger.error('stdout : %s' % sout)
            logger.error('stderr : %s' % serr)
            return output_fp, 0, []

        blast_output_filename = '%s.blast' % blast_output
        with open(blast_output_filename, 'r') as bfl:
            for line in bfl:
                line = line.strip().split('\t')
                # if * means no match
                if line[1] == '*':
                    continue
                # check if % identity[2] and coverage[13] are large enough
                if (float(line[2]) >= sim_thresh) and \
                   (float(line[13]) >= coverage_thresh) and \
                   (float(line[11]) >= bitscore_thresh * len(line[0])):
                    aligned_seq_ids.add(line[0])

    if negate:
        def op(x): return x not in aligned_seq_ids
    else:
        def op(x): return x in aligned_seq_ids

    # if negate = False, only output sequences
    # matching to at least one of the databases
    totalseqs = 0
    okseqs = 0
    badseqs = 0
    with open(output_fp, 'w') as out_f:
        for label, seq in sequence_generator(seqs_fp):
            totalseqs += 1
            label = label.split()[0]
            if op(label):
                out_f.write(">%s\n%s\n" % (label, seq))
                okseqs += 1
            else:
                badseqs += 1
    logger.info('total sequences %d, passing sequences %d, '
                'failing sequences %d' % (totalseqs, okseqs, badseqs))
    return output_fp, okseqs, [blast_output_filename]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def multiple_sequence_alignment(seqs_fp, threads=1):
    logger = logging.getLogger(__name__)
    logger.info('multiple_sequence_alignment seqs file %s' % seqs_fp)

    # for mafft we use -1 to denote all threads and not 0
    if threads == 0:
        threads = -1

    if stat(seqs_fp).st_size == 0:
        logger.warning('msa failed. file %s has no reads' % seqs_fp)
        return None
    msa_fp = seqs_fp + '.msa'
    params = ['mafft', '--quiet', '--preservecase', '--parttree', '--auto',
              '--thread', str(threads), seqs_fp]
    sout, serr, res = _system_call(params, stdoutfilename=msa_fp)
    if not res == 0:
        logger.info('msa failed for file %s (maybe only 1 read?)' % seqs_fp)
        logger.debug('stderr : %s' % serr)
        return None
    return msa_fp

Response: Perform multiple sequence alignment on a FASTA file using MAFFT.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def remove_chimeras_denovo_from_seqs(seqs_fp, working_dir, threads=1):
    logger = logging.getLogger(__name__)
    logger.info('remove_chimeras_denovo_from_seqs seqs file %s'
                'to working dir %s' % (seqs_fp, working_dir))

    output_fp = join(
        working_dir, "%s.no_chimeras" % basename(seqs_fp))

    # we use the parameters dn=0.000001, xn=1000, minh=10000000
    # so 1 mismatch in the A/B region will cancel it being labeled as chimera
    # and ~3 unique reads in each region will make it a chimera if
    # no mismatches
    params = ['vsearch', '--uchime_denovo', seqs_fp,
              '--nonchimeras', output_fp,
              '-dn', '0.000001', '-xn', '1000',
              '-minh', '10000000', '--mindiffs', '5',
              '--fasta_width', '0', '--threads', str(threads)]
    sout, serr, res = _system_call(params)
    if not res == 0:
        logger.error('problem with chimera removal for file %s' % seqs_fp)
        logger.debug('stdout : %s' % sout)
        logger.debug('stderr : %s' % serr)
    return output_fp

Response: Remove chimeras de novo from a set of sequences.


Instruction:
Create a Python 3 function to
split FASTA file on sample IDs and store them in a dict of dicts.

Response:
def split_sequence_file_on_sample_ids_to_files(seqs,
                                               outdir):
    """Split FASTA file on sample IDs.

    Parameters
    ----------
    seqs: file handler
        file handler to demultiplexed FASTA file
    outdir: string
        dirpath to output split FASTA files
    """
    logger = logging.getLogger(__name__)
    logger.info('split_sequence_file_on_sample_ids_to_files'
                ' for file %s into dir %s' % (seqs, outdir))

    outputs = {}

    for bits in sequence_generator(seqs):
        sample = sample_id_from_read_id(bits[0])

        if sample not in outputs:
            outputs[sample] = open(join(outdir, sample + '.fasta'), 'w')
        outputs[sample].write(">%s\n%s\n" % (bits[0], bits[1]))
    for sample in outputs:
        outputs[sample].close()
    logger.info('split to %d files' % len(outputs))


Instruction:
Can you implement a function in Python 3 that
writes a BIOM table to file.

Response:
def write_biom_table(table, biom_fp):
    """Write BIOM table to file.

    Parameters
    ----------
    table: biom.table
        an instance of a BIOM table
    biom_fp: string
        filepath to output BIOM table
    """
    logger = logging.getLogger(__name__)
    logger.debug('write_biom_table to file %s' % biom_fp)
    with biom_open(biom_fp, 'w') as f:
        table.to_hdf5(h5grp=f, generated_by="deblur")
        logger.debug('wrote to BIOM file %s' % biom_fp)


Instruction:
Here you have a function in Python 3, explain what it does
def get_files_for_table(input_dir,
                        file_end='.trim.derep.no_artifacts'
                        '.msa.deblur.no_chimeras'):
    """Get a list of files to add to the output table

    Parameters:
    -----------
    input_dir : string
        name of the directory containing the deblurred fasta files
    file_end : string
        the ending of all the fasta files to be added to the table
        (default '.fasta.trim.derep.no_artifacts.msa.deblur.no_chimeras')

    Returns
    -------
    names : list of tuples of (string,string)
        list of tuples of:
            name of fasta files to be added to the biom table
            sampleid (file names without the file_end and path)
    """
    logger = logging.getLogger(__name__)
    logger.debug('get_files_for_table input dir %s, '
                 'file-ending %s' % (input_dir, file_end))
    names = []
    for cfile in glob(join(input_dir, "*%s" % file_end)):
        if not isfile(cfile):
            continue
        sample_id = basename(cfile)[:-len(file_end)]
        sample_id = os.path.splitext(sample_id)[0]
        names.append((cfile, sample_id))

    logger.debug('found %d files' % len(names))
    return names

Response: Get a list of files to add to the biom table


Instruction:
Create a Python 3 function for
creating a BIOM table out of all samples in a directory and a list of deblurred sample ids and a list of sequences in a directory.

Response:
def create_otu_table(output_fp, deblurred_list,
                     outputfasta_fp=None, minreads=0):
    """Create a biom table out of all files in a directory

    Parameters
    ----------
    output_fp : string
        filepath to output BIOM table
    deblurred_list : list of (str, str)
        list of file names (including path), sampleid of all deblurred
        fasta files to add to the table
    outputfasta_fp : str, optional
        name of output fasta file (of all sequences in the table) or None
        to not write
    minreads : int, optional
        minimal number of reads per bacterial sequence in order to write
        it to the biom table and fasta file or 0 to write all
    """
    logger = logging.getLogger(__name__)
    logger.info('create_otu_table for %d samples, '
                'into output table %s' % (len(deblurred_list), output_fp))

    # the regexp for finding the number of reads of a sequence
    sizeregexp = re.compile('(?<=size=)\w+')
    seqdict = {}
    seqlist = []
    sampset = set()
    samplist = []
    # arbitrary size for the sparse results matrix so we won't run out of space
    obs = scipy.sparse.dok_matrix((int(1E9), len(deblurred_list)), dtype=np.double)

    # load the sequences from all samples into a sprase matrix
    sneaking_extensions = {'fasta', 'fastq', 'fna', 'fq', 'fa'}
    for (cfilename, csampleid) in deblurred_list:
        if csampleid.rsplit('.', 1)[-1] in sneaking_extensions:
            csampleid = csampleid.rsplit('.', 1)[0]

        # test if sample has already been processed
        if csampleid in sampset:
            warnings.warn('sample %s already in table!', UserWarning)
            logger.error('sample %s already in table!' % csampleid)
            continue
        sampset.add(csampleid)
        samplist.append(csampleid)
        csampidx = len(sampset)-1
        # read the fasta file and add to the matrix
        for chead, cseq in sequence_generator(cfilename):
            cseq = cseq.upper()
            if cseq not in seqdict:
                seqdict[cseq] = len(seqlist)
                seqlist.append(cseq)
            cseqidx = seqdict[cseq]
            cfreq = float(sizeregexp.search(chead).group(0))
            try:
                obs[cseqidx, csampidx] += cfreq
            except IndexError:
                # exception means we ran out of space - add more OTUs
                shape = obs.shape
                obs.resize((shape[0]*2,  shape[1]))
                obs[cseqidx, csampidx] = cfreq

    logger.info('for output biom table loaded %d samples, %d unique sequences'
                % (len(samplist), len(seqlist)))

    # and now make the sparse matrix the real size
    obs.resize((len(seqlist), len(samplist)))

    # do the minimal reads per otu filtering
    if minreads > 0:
        readsperotu = obs.sum(axis=1)
        keep = np.where(readsperotu >= minreads)[0]
        logger.info('keeping %d (out of %d sequences) with >=%d reads' %
                    (len(keep), len(seqlist), minreads))
        obs = obs[keep, :]
        seqlist = list(np.array(seqlist)[keep])
        logger.debug('filtering completed')

    # convert the matrix to a biom table
    table = Table(obs.tocsr(), seqlist, samplist,
                  observation_metadata=None,
                  sample_metadata=None, table_id=None,
                  generated_by="deblur",
                  create_date=datetime.now().isoformat())
    logger.debug('converted to biom table')

    # remove samples with 0 reads
    filter_minreads_samples_from_table(table)

    # save the merged otu table
    write_biom_table(table, output_fp)
    logger.info('saved to biom file %s' % output_fp)

    # and save the fasta file
    if outputfasta_fp is not None:
        logger.debug('saving fasta file')
        with open(outputfasta_fp, 'w') as f:
            for cseq in seqlist:
                f.write('>%s\n%s\n' % (cseq, cseq))
        logger.info('saved sequence fasta file to %s' % outputfasta_fp)


Instruction:
Can you write a function in Python 3 where it
launches a full deblur workflow for a single post split - libraries fasta file.

Response:
def launch_workflow(seqs_fp, working_dir, mean_error, error_dist,
                    indel_prob, indel_max, trim_length, left_trim_length,
                    min_size, ref_fp, ref_db_fp, threads_per_sample=1,
                    sim_thresh=None, coverage_thresh=None):
    """Launch full deblur workflow for a single post split-libraries fasta file

    Parameters
    ----------
    seqs_fp: string
        a post split library fasta file for debluring
    working_dir: string
        working directory path
    mean_error: float
        mean error for original sequence estimate
    error_dist: list
        list of error probabilities for each hamming distance
    indel_prob: float
        insertion/deletion (indel) probability
    indel_max: integer
        maximal indel number
    trim_length: integer
        sequence trim length
    left_trim_length: integer
        trim the first n reads
    min_size: integer
        upper limit on sequence abundance (discard sequences below limit)
    ref_fp: tuple
        filepath(s) to FASTA reference database for artifact removal
    ref_db_fp: tuple
        filepath(s) to SortMeRNA indexed database for artifact removal
    threads_per_sample: integer, optional
        number of threads to use for SortMeRNA/mafft/vsearch
        (0 for max available)
    sim_thresh: float, optional
        the minimal similarity for a sequence to the database.
        if None, take the defaults (0.65 for negate=False,
        0.95 for negate=True)
    coverage_thresh: float, optional
        the minimal coverage for alignment of a sequence to the database.
        if None, take the defaults (0.3 for negate=False, 0.95 for negate=True)

    Return
    ------
    output_no_chimers_fp : string
        filepath to fasta file with no chimeras of None if error encountered
    """
    logger = logging.getLogger(__name__)
    logger.info('--------------------------------------------------------')
    logger.info('launch_workflow for file %s' % seqs_fp)

    # Step 1: Trim sequences to specified length
    output_trim_fp = join(working_dir, "%s.trim" % basename(seqs_fp))
    with open(output_trim_fp, 'w') as out_f:
        for label, seq in trim_seqs(
                input_seqs=sequence_generator(seqs_fp),
                trim_len=trim_length,
                left_trim_len=left_trim_length):
            out_f.write(">%s\n%s\n" % (label, seq))
    # Step 2: Dereplicate sequences
    output_derep_fp = join(working_dir,
                           "%s.derep" % basename(output_trim_fp))
    dereplicate_seqs(seqs_fp=output_trim_fp,
                     output_fp=output_derep_fp,
                     min_size=min_size, threads=threads_per_sample)
    # Step 3: Remove artifacts
    output_artif_fp, num_seqs_left, _ = remove_artifacts_seqs(seqs_fp=output_derep_fp,
                                                              ref_fp=ref_fp,
                                                              working_dir=working_dir,
                                                              ref_db_fp=ref_db_fp,
                                                              negate=True,
                                                              threads=threads_per_sample,
                                                              sim_thresh=sim_thresh)
    if not output_artif_fp:
        warnings.warn('Problem removing artifacts from file %s' %
                      seqs_fp, UserWarning)
        logger.warning('remove artifacts failed, aborting')
        return None
    # Step 4: Multiple sequence alignment
    if num_seqs_left > 1:
        output_msa_fp = join(working_dir,
                             "%s.msa" % basename(output_artif_fp))
        alignment = multiple_sequence_alignment(seqs_fp=output_artif_fp,
                                                threads=threads_per_sample)
        if not alignment:
            warnings.warn('Problem performing multiple sequence alignment '
                          'on file %s' % seqs_fp, UserWarning)
            logger.warning('msa failed. aborting')
            return None
    elif num_seqs_left == 1:
        # only one sequence after remove artifacts (but could be many reads)
        # no need to run MSA - just use the pre-msa file as input for next step
        output_msa_fp = output_artif_fp
    else:
        err_msg = ('No sequences left after artifact removal in '
                   'file %s' % seqs_fp)
        warnings.warn(err_msg, UserWarning)
        logger.warning(err_msg)
        return None
    # Step 5: Launch deblur
    output_deblur_fp = join(working_dir,
                            "%s.deblur" % basename(output_msa_fp))
    with open(output_deblur_fp, 'w') as f:
        seqs = deblur(sequence_generator(output_msa_fp), mean_error,
                      error_dist, indel_prob, indel_max)
        if seqs is None:
            warnings.warn('multiple sequence alignment file %s contains '
                          'no sequences' % output_msa_fp, UserWarning)
            logger.warn('no sequences returned from deblur for file %s' %
                        output_msa_fp)
            return None
        for s in seqs:
            # remove '-' from aligned sequences
            s.sequence = s.sequence.replace('-', '')
            f.write(s.to_fasta())
    # Step 6: Chimera removal
    output_no_chimeras_fp = remove_chimeras_denovo_from_seqs(
        output_deblur_fp, working_dir, threads=threads_per_sample)
    logger.info('finished processing file')
    return output_no_chimeras_fp


Instruction:
How would you code a function in Python 3 to
start the logger for the run

Response:
def start_log(level=logging.DEBUG, filename=None):
    """start the logger for the run

    Parameters
    ----------
    level : int, optional
        logging.DEBUG, logging.INFO etc. for the log level (between 0-50).
    filename : str, optional
      name of the filename to save the log to or
      None (default) to use deblur.log.TIMESTAMP
    """
    if filename is None:
        tstr = time.ctime()
        tstr = tstr.replace(' ', '.')
        tstr = tstr.replace(':', '.')
        filename = 'deblur.log.%s' % tstr
    logging.basicConfig(filename=filename, level=level,
                        format='%(levelname)s(%(thread)d)'
                        '%(asctime)s:%(message)s')
    logger = logging.getLogger(__name__)
    logger.info('*************************')
    logger.info('deblurring started')


Instruction:
Create a Python 3 function for
executing the command cmd and return the standard output and standard error and exist status of the command.

Response:
def _system_call(cmd, stdoutfilename=None):
    """Execute the command `cmd`
    Parameters
    ----------
    cmd : str
        The string containing the command to be run.
    stdoutfilename : str
        Name of the file to save stdout to or None
        (default) to not save to file
    stderrfilename : str
        Name of the file to save stderr to or None
        (default) to not save to file

    Returns
    -------
    tuple of (str, str, int)
        The standard output, standard error and exist status of the
        executed command

    Notes
    -----
    This function is ported and modified from QIIME
    (http://www.qiime.org), previously named
    qiime_system_call. QIIME is a GPL project, but we obtained permission from
    the authors of this function to port it to Qiita and keep it under BSD
    license.
    """
    logger = logging.getLogger(__name__)
    logger.debug('system call: %s' % cmd)
    if stdoutfilename:
        with open(stdoutfilename, 'w') as f:
            proc = subprocess.Popen(cmd, universal_newlines=True,
                                    shell=False, stdout=f,
                                    stderr=subprocess.PIPE)
    else:
        proc = subprocess.Popen(cmd, universal_newlines=True,
                                shell=False, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
    # Communicate pulls all stdout/stderr from the PIPEs
    # This call blocks until the command is done
    stdout, stderr = proc.communicate()
    return_value = proc.returncode

    return stdout, stderr, return_value


Instruction:
Explain what the following Python 3 code does
def get_sequences(input_seqs):
    try:
        seqs = [Sequence(id, seq) for id, seq in input_seqs]
    except Exception:
        seqs = []

    if len(seqs) == 0:
        logger = logging.getLogger(__name__)
        logger.warn('No sequences found in fasta file!')
        return None

    # Check that all the sequence lengths (aligned and unaligned are the same)
    aligned_lengths = set(s.length for s in seqs)
    unaligned_lengths = set(s.unaligned_length for s in seqs)

    if len(aligned_lengths) != 1 or len(unaligned_lengths) != 1:
        raise ValueError(
            "Not all sequence have the same length. Aligned lengths: %s, "
            "sequence lengths: %s"
            % (", ".join(map(str, aligned_lengths)),
               ", ".join(map(str, unaligned_lengths))))

    seqs = sorted(seqs, key=attrgetter('frequency'), reverse=True)
    return seqs

Response: Returns a list of Sequences


Instruction:
Can you generate a brief explanation for the following Python 3 code
def to_fasta(self):
        prefix, suffix = re.split('(?<=size=)\w+', self.label, maxsplit=1)
        new_count = int(round(self.frequency))
        new_label = "%s%d%s" % (prefix, new_count, suffix)
        return ">%s\n%s\n" % (new_label, self.sequence)

Response: Returns a string with the sequence in fasta format


Instruction:
Implement a Python 3 function for
rendering options for select2.

Response:
def render_select2_options_code(self, options, id_):
        """Render options for select2."""
        output = []
        for key, value in options.items():
            if isinstance(value, (dict, list)):
                value = json.dumps(value)
            output.append("data-{name}='{value}'".format(
                    name=key,
                    value=mark_safe(value)))
        return mark_safe(' '.join(output))


Instruction:
Create a Python 3 function for
rendering html container for Select2 widget with options.

Response:
def render_js_code(self, id_, *args, **kwargs):
        """Render html container for Select2 widget with options."""
        if id_:
            options = self.render_select2_options_code(
                    dict(self.get_options()), id_)
            return mark_safe(self.html.format(id=id_, options=options))
        return u''


Instruction:
How would you implement a function in Python 3 that
extends base class s render method by appending javascript inline text to html output.

Response:
def render(self, name, value, attrs=None, **kwargs):
        """
        Extend base class's `render` method by appending
        javascript inline text to html output.
        """
        output = super(Select2Mixin, self).render(
            name, value, attrs=attrs, **kwargs)
        id_ = attrs['id']
        output += self.render_js_code(
            id_, name, value, attrs=attrs, **kwargs)
        return mark_safe(output)


Instruction:
How would you implement a function in Python 3 that
generates the console directive.

Response:
def visit_console_html(self, node):
    """Generate HTML for the console directive."""
    if self.builder.name in ('djangohtml', 'json') and node['win_console_text']:
        # Put a mark on the document object signaling the fact the directive
        # has been used on it.
        self.document._console_directive_used_flag = True
        uid = node['uid']
        self.body.append('''\
<div class="console-block" id="console-block-%(id)s">
<input class="c-tab-unix" id="c-tab-%(id)s-unix" type="radio" name="console-%(id)s" checked>
<label for="c-tab-%(id)s-unix" title="Linux/macOS">&#xf17c/&#xf179</label>
<input class="c-tab-win" id="c-tab-%(id)s-win" type="radio" name="console-%(id)s">
<label for="c-tab-%(id)s-win" title="Windows">&#xf17a</label>
<section class="c-content-unix" id="c-content-%(id)s-unix">\n''' % {'id': uid})
        try:
            self.visit_literal_block(node)
        except nodes.SkipNode:
            pass
        self.body.append('</section>\n')

        self.body.append('<section class="c-content-win" id="c-content-%(id)s-win">\n' % {'id': uid})
        win_text = node['win_console_text']
        highlight_args = {'force': True}
        if 'linenos' in node:
            linenos = node['linenos']
        else:
            linenos = win_text.count('\n') >= self.highlightlinenothreshold - 1

        def warner(msg):
            self.builder.warn(msg, (self.builder.current_docname, node.line))

        highlighted = self.highlighter.highlight_block(
            win_text, 'doscon', warn=warner, linenos=linenos, **highlight_args
        )
        self.body.append(highlighted)
        self.body.append('</section>\n')
        self.body.append('</div>\n')
        raise nodes.SkipNode
    else:
        self.visit_literal_block(node)


Instruction:
Can you generate the documentation for the following Python 3 function
def select2_modelform_meta(model,
                           meta_fields=None,
                           widgets=None,
                           attrs=None,
                           **kwargs):
    """
    Return `Meta` class with Select2-enabled widgets for fields
    with choices (e.g.  ForeignKey, CharField, etc) for use with
    ModelForm.

    Arguments:
        model - a model class to create `Meta` class for.
        meta_fields - dictionary with `Meta` class fields, for
            example, {'fields': ['id', 'name']}
        attrs - select2 widget attributes (width, for example),
            must be of type `dict`.
        **kwargs - will be merged with meta_fields.
    """
    widgets = widgets or {}
    meta_fields = meta_fields or {}

    # TODO: assert attrs is of type `dict`

    for field in model._meta.fields:
        if isinstance(field, ForeignKey) or field.choices:
            widgets.update({field.name: Select2(select2attrs=attrs)})

    for field in model._meta.many_to_many:
        widgets.update({field.name: Select2Multiple(select2attrs=attrs)})

    meta_fields.update({
        'model': model,
        'widgets': widgets,
    })
    if 'exclude' not in kwargs and 'fields' not in kwargs:
        meta_fields.update({'exclude': []})
    meta_fields.update(**kwargs)
    meta = type('Meta', (object,), meta_fields)

    return meta

Response:
Returns a new meta class with Select2 - enabled widgets for fields
racket with kwargs


Instruction:
Create a Python 3 function for
returning a modelform class for a model with select2 widgets.

Response:
def select2_modelform(
        model, attrs=None, form_class=es2_forms.FixedModelForm):
    """
    Return ModelForm class for model with select2 widgets.

    Arguments:
        attrs: select2 widget attributes (width, for example) of type `dict`.
        form_class: modelform base class, `forms.ModelForm` by default.

    ::

        SomeModelForm = select2_modelform(models.SomeModelBanner)

    is the same like::

        class SomeModelForm(forms.ModelForm):
            Meta = select2_modelform_meta(models.SomeModelForm)
    """
    classname = '%sForm' % model._meta.object_name
    meta = select2_modelform_meta(model, attrs=attrs)
    return type(classname, (form_class,), {'Meta': meta})


Instruction:
How would you implement a function in Python 3 that
yields (token_type, str_data) tokens. The last token will be (EOF, None) where EOF is the singleton object defined in this module.

Response:
def lex(self, text):
        """ Yield (token_type, str_data) tokens.

        The last token will be (EOF, None) where EOF is the singleton object
        defined in this module.
        """
        for match in self.regex.finditer(text):
            for name, _ in self.lexicon:
                m = match.group(name)
                if m is not None:
                    yield (name, m)
                    break
        yield (EOF, None)


Instruction:
Can you implement a function in Python 3 that
parses a string of SVG transform = " data.

Response:
def parse(self, text):
        """ Parse a string of SVG transform="" data.
        """
        gen = self.lexer.lex(text)
        next_val_fn = partial(next, *(gen,))

        commands = []
        token = next_val_fn()
        while token[0] is not EOF:
            command, token = self.rule_svg_transform(next_val_fn, token)
            commands.append(command)
        return commands


Instruction:
Implement a Python 3 function for
parsing a string of SVG <path > data.

Response:
def parse(self, text):
        """ Parse a string of SVG <path> data.
        """
        gen = self.lexer.lex(text)
        next_val_fn = partial(next, *(gen,))
        token = next_val_fn()
        return self.rule_svg_path(next_val_fn, token)


Instruction:
Can you generate the documentation for the following Python 3 function
def findElementsWithId(node, elems=None):
    if elems is None:
        elems = {}
    id = node.getAttribute('id')
    if id != '':
        elems[id] = node
    if node.hasChildNodes():
        for child in node.childNodes:
            # from http://www.w3.org/TR/DOM-Level-2-Core/idl-definitions.html
            # we are only really interested in nodes of type Element (1)
            if child.nodeType == Node.ELEMENT_NODE:
                findElementsWithId(child, elems)
    return elems

Response: Returns all elements with id attributes


Instruction:
Can you implement a function in Python 3 that
finds all elements that are referenced by a node.

Response:
def findReferencedElements(node, ids=None):
    """
    Returns IDs of all referenced elements
    - node is the node at which to start the search.
    - returns a map which has the id as key and
      each value is is a list of nodes

    Currently looks at 'xlink:href' and all attributes in 'referencingProps'
    """
    global referencingProps
    if ids is None:
        ids = {}
    # TODO: input argument ids is clunky here (see below how it is called)
    # GZ: alternative to passing dict, use **kwargs

    # if this node is a style element, parse its text into CSS
    if node.nodeName == 'style' and node.namespaceURI == NS['SVG']:
        # one stretch of text, please! (we could use node.normalize(), but
        # this actually modifies the node, and we don't want to keep
        # whitespace around if there's any)
        stylesheet = "".join([child.nodeValue for child in node.childNodes])
        if stylesheet != '':
            cssRules = parseCssString(stylesheet)
            for rule in cssRules:
                for propname in rule['properties']:
                    propval = rule['properties'][propname]
                    findReferencingProperty(node, propname, propval, ids)
        return ids

    # else if xlink:href is set, then grab the id
    href = node.getAttributeNS(NS['XLINK'], 'href')
    if href != '' and len(href) > 1 and href[0] == '#':
        # we remove the hash mark from the beginning of the id
        id = href[1:]
        if id in ids:
            ids[id].append(node)
        else:
            ids[id] = [node]

    # now get all style properties and the fill, stroke, filter attributes
    styles = node.getAttribute('style').split(';')

    for style in styles:
        propval = style.split(':')
        if len(propval) == 2:
            prop = propval[0].strip()
            val = propval[1].strip()
            findReferencingProperty(node, prop, val, ids)

    for attr in referencingProps:
        val = node.getAttribute(attr).strip()
        if not val:
            continue
        findReferencingProperty(node, attr, val, ids)

    if node.hasChildNodes():
        for child in node.childNodes:
            if child.nodeType == Node.ELEMENT_NODE:
                findReferencedElements(child, ids)
    return ids


Instruction:
Implement a function in Python 3 to
remove all unreferenced elements from the document.

Response:
def removeUnreferencedElements(doc, keepDefs):
    """
    Removes all unreferenced elements except for <svg>, <font>, <metadata>, <title>, and <desc>.
    Also vacuums the defs of any non-referenced renderable elements.

    Returns the number of unreferenced elements removed from the document.
    """
    global _num_elements_removed
    num = 0

    # Remove certain unreferenced elements outside of defs
    removeTags = ['linearGradient', 'radialGradient', 'pattern']
    identifiedElements = findElementsWithId(doc.documentElement)
    referencedIDs = findReferencedElements(doc.documentElement)

    for id in identifiedElements:
        if id not in referencedIDs:
            goner = identifiedElements[id]
            if (goner is not None and goner.nodeName in removeTags
                and goner.parentNode is not None
                    and goner.parentNode.tagName != 'defs'):
                goner.parentNode.removeChild(goner)
                num += 1
                _num_elements_removed += 1

    if not keepDefs:
        # Remove most unreferenced elements inside defs
        defs = doc.documentElement.getElementsByTagName('defs')
        for aDef in defs:
            elemsToRemove = removeUnusedDefs(doc, aDef)
            for elem in elemsToRemove:
                elem.parentNode.removeChild(elem)
                _num_elements_removed += 1
                num += 1
    return num