Instruction:
Can you generate the documentation for the following Python 3 function
def _get_lineage_table(self, tax_ids, merge_obsolete=True):

        try:
            with self.engine.connect() as con:
                # insert tax_ids into a temporary table

                temptab = self.prepend_schema(random_name(12))

                cmd = 'CREATE TEMPORARY TABLE "{tab}" (old_tax_id text)'.format(
                    tab=temptab)
                con.execute(cmd)

                log.info('inserting tax_ids into temporary table')
                # TODO: couldn't find an equivalent of "executemany" - does one exist?
                cmd = 'INSERT INTO "{tab}" VALUES ({x})'.format(
                    tab=temptab, x=self.placeholder)
                for tax_id in tax_ids:
                    con.execute(cmd, tax_id)

                log.info('executing recursive CTE')
                cmd = Template("""
                WITH RECURSIVE a AS (
                 SELECT tax_id as tid, 1 AS ord, tax_id, parent_id, rank
                  FROM {{ nodes }}
                  WHERE tax_id in (
                  {% if merge_obsolete %}
                  SELECT COALESCE(m.new_tax_id, "{{ temptab }}".old_tax_id)
                    FROM "{{ temptab }}" LEFT JOIN {{ merged }} m USING(old_tax_id)
                  {% else %}
                  SELECT * from "{{ temptab }}"
                  {% endif %}
                  )
                UNION ALL
                 SELECT a.tid, a.ord + 1, p.tax_id, p.parent_id, p.rank
                  FROM a JOIN {{ nodes }} p ON a.parent_id = p.tax_id
                )
                SELECT a.tid, a.tax_id, a.parent_id, a.rank, tax_name FROM a
                JOIN {{ names }} using(tax_id)
                WHERE names.is_primary
                ORDER BY tid, ord desc
                """).render(
                    temptab=temptab,
                    merge_obsolete=merge_obsolete,
                    merged=self.merged,
                    nodes=self.nodes,
                    names=self.names,
                )

                result = con.execute(cmd)
                rows = result.fetchall()

                con.execute('DROP TABLE "{}"'.format(temptab))
                log.info('returning lineages')
                if not rows:
                    raise ValueError('no tax_ids were found')
                else:
                    returned = {row[0] for row in rows}
                    # TODO: compare set membership, not lengths
                    if len(returned) < len(tax_ids):
                        msg = ('{} tax_ids were provided '
                               'but only {} were returned').format(
                                   len(tax_ids), len(returned))
                        log.error('Input tax_ids not represented in output:')
                        log.error(sorted(set(tax_ids) - returned))
                        raise ValueError(msg)

                return rows

        except sqlalchemy.exc.ResourceClosedError:
            raise ValueError('tax id "{}" not found'.format(tax_id))

Response: Return a list of tuples describing the lineage of tax_id and tax_name of tax_id.


Instruction:
Create a Python 3 function for
adding a row to table source if name does not exist. Returns a tuple of source_id and True if a new row is created.

Response:
def add_source(self, source_name, description=None):
        """Adds a row to table "source" if "name" does not
        exist. Returns (source_id, True) if a new row is created,
        (source_id, False) otherwise.

        """

        # TODO: shoud be able to do this inside a transaction

        if not source_name:
            raise ValueError('"source_name" may not be None or an empty string')

        sel = select([self.source.c.id], self.source.c.name == source_name).execute()
        result = sel.fetchone()
        if result:
            return result[0], False
        else:
            ins = self.source.insert().execute(
                name=source_name, description=description)
            return ins.inserted_primary_key[0], True


Instruction:
Write a Python 3 function that can
return a dict with keys id name description

Response:
def get_source(self, source_id=None, source_name=None):
        """Returns a dict with keys ['id', 'name', 'description'] or None if
        no match. The ``id`` field is guaranteed to be an int that
        exists in table source. Requires exactly one of ``source_id``
        or ``source_name``. A new source corresponding to
        ``source_name`` is created if necessary.

        """

        if not (bool(source_id) ^ bool(source_name)):
            raise ValueError('exactly one of source_id or source_name is required')

        if source_id:
            try:
                source_id = int(source_id)
            except (ValueError, TypeError):
                raise ValueError(
                    'source_id must be an int or a string representing one')

            sel = select([self.source], self.source.c.id == source_id).execute()
        else:
            sel = select([self.source], self.source.c.name == source_name).execute()

        result = sel.fetchone()
        if not result:
            raise ValueError(
                'there is no source with id {} or name {}'.format(
                    source_id, source_name))

        return dict(list(zip(list(sel.keys()), result)))


Instruction:
Explain what the following Python 3 code does
def verify_rank_integrity(self, tax_id, rank, parent_id, children):
        def _lower(n1, n2):
            return self.ranks.index(n1) < self.ranks.index(n2)

        if rank not in self.ranks:
            raise TaxonIntegrityError('rank "{}" is undefined'.format(rank))

        parent_rank = self.rank(parent_id)
        # undefined ranks can be placed anywhere in a lineage
        if not _lower(rank, parent_rank) and rank != self.NO_RANK:
            msg = ('New node "{}", rank "{}" has same or '
                   'higher rank than parent node "{}", rank "{}"')
            msg = msg.format(tax_id, rank, parent_id, parent_rank)
            raise TaxonIntegrityError(msg)

        for child in children:
            if not _lower(self.rank(child), rank):
                msg = 'Child node {} has same or lower rank as new node {}'
                msg = msg.format(tax_id, child)
                raise TaxonIntegrityError(msg)
        return True

Response: Confirm that the rank of a node is consistent with the parent and children ranks.


Instruction:
Here you have a function in Python 3, explain what it does
def add_node(self, tax_id, parent_id, rank, names, source_name, children=None,
                 is_valid=True, execute=True, **ignored):
        """Add a node to the taxonomy.

        ``source_name`` is added to table "source" if necessary.

        """

        if ignored:
            log.info('some arguments were ignored: {} '.format(str(ignored)))

        children = children or []
        self.verify_rank_integrity(tax_id, rank, parent_id, children)
        source_id, __ = self.add_source(source_name)
        assert isinstance(is_valid, bool)

        statements = []

        # add node
        statements.append(
            self.nodes.insert().values(
                tax_id=tax_id,
                parent_id=parent_id,
                rank=rank,
                source_id=source_id))

        # add names. Since this is a new node, at least one name must
        # be provided; if only one is provided, it is the primary
        # name. If more than one is primary, an error will be raised
        # from add_names()
        if len(names) == 1:
            names[0]['is_primary'] = True
        else:
            primary_names = [n['tax_name'] for n in names if n.get('is_primary')]
            if len(primary_names) != 1:
                raise ValueError(
                    '`is_primary` must be True for exactly one name in `names`')

        for namedict in names:
            namedict['source_id'] = source_id
            if 'source_name' in namedict:
                del namedict['source_name']

        statements.extend(self.add_names(tax_id, names, execute=False))

        # add children and update source_id
        for child in children:
            statements.append(self.nodes.update(
                whereclause=self.nodes.c.tax_id == child,
                values={'parent_id': tax_id, 'source_id': source_id}))

        if execute:
            self.execute(statements)
        else:
            return statements

Response: Add a node to the taxonomy.


Instruction:
Create a Python 3 function to
add a record to the names table corresponding to the tax_id.

Response:
def add_name(self, tax_id, tax_name, source_name=None, source_id=None,
                 name_class='synonym', is_primary=False, is_classified=None,
                 execute=True, **ignored):

        """Add a record to the names table corresponding to
        ``tax_id``. Arguments are as follows:

        - tax_id (string, required)
        - tax_name (string, required)

        *one* of the following are required:

        - source_id (int or string coercable to int)
        - source_name (string)

        ``source_id`` or ``source_name`` must identify an existing
        record in table "source".

        The following are optional:

        - name_class (string, default 'synonym')
        - is_primary (bool, see below)
        - is_classified (bool or None, default None)

        ``is_primary`` is optional and defaults to True if only one
        name is provided; otherwise is_primary must be True for
        exactly one name (and is optional in others).

        """

        assert isinstance(is_primary, bool)
        assert is_classified in {None, True, False}
        if ignored:
            log.info('some arguments were ignored: {} '.format(str(ignored)))

        source_id = self.get_source(source_id, source_name)['id']

        statements = []

        if is_primary:
            statements.append(self.names.update(
                whereclause=self.names.c.tax_id == tax_id,
                values={'is_primary': False}))

        statements.append(self.names.insert().values(
            tax_id=tax_id,
            tax_name=tax_name,
            source_id=source_id,
            is_primary=is_primary,
            name_class=name_class,
            is_classified=is_classified))

        if execute:
            self.execute(statements)
        else:
            return statements


Instruction:
Write a Python 3 function for
associating one or more names with tax_id.

Response:
def add_names(self, tax_id, names, execute=True):
        """Associate one or more names with ``tax_id``.

        ``names`` is a list of one or more dicts, with keys
        corresponding to the signature of ``self.add_name()``
        (excluding ``execute``).

        """

        primary_names = [n['tax_name'] for n in names if n.get('is_primary')]
        if len(primary_names) > 1:
            raise ValueError(
                '`is_primary` may be True for no more than one name in `names`')

        statements = []

        for namevals in names:
            if 'tax_id' in namevals:
                del namevals['tax_id']
            statements.extend(
                self.add_name(tax_id=tax_id, execute=False, **namevals))

        if execute:
            self.execute(statements)
        else:
            return statements


Instruction:
Can you generate the documentation for the following Python 3 function
def sibling_of(self, tax_id):
        if tax_id is None:
            return None
        parent_id, rank = self._node(tax_id)
        s = select([self.nodes.c.tax_id],
                   and_(self.nodes.c.parent_id == parent_id,
                        self.nodes.c.tax_id != tax_id,
                        self.nodes.c.rank == rank))
        res = s.execute()
        output = res.fetchone()
        if not output:
            msg = 'No sibling of tax_id {} with rank {} found in taxonomy'
            msg = msg.format(tax_id, rank)
            log.warning(msg)
            return None
        else:
            return output[0]

Response: Return None or a tax_id of a sibling of tax_id.


Instruction:
Create a Python 3 function for
returning all tax_ids in node table

Response:
def tax_ids(self):
        '''
        Return all tax_ids in node table
        '''
        fetch = select([self.nodes.c.tax_id]).execute().fetchall()
        ids = [t[0] for t in fetch]
        return ids


Instruction:
Can you generate the documentation for the following Python 3 function
def child_of(self, tax_id):
        if tax_id is None:
            return None
        parent_id, rank = self._node(tax_id)
        s = select([self.nodes.c.tax_id],
                   and_(self.nodes.c.parent_id == tax_id,
                        or_(*[self.nodes.c.rank == r
                              for r in self.ranks_below(rank)])))
        res = s.execute()
        output = res.fetchone()
        if not output:
            msg = ('No children of tax_id {} with '
                   'rank below {} found in database')
            msg = msg.format(tax_id, rank)
            log.warning(msg)
            return None
        else:
            r = output[0]
            assert self.is_ancestor_of(r, tax_id)
            return r

Response: Return None or a tax id of a child of tax_id.


Instruction:
How would you explain what the following Python 3 function does
def nary_subtree(self, tax_id, n=2):
        if tax_id is None:
            return None
        parent_id, rank = self._node(tax_id)
        if rank == 'species':
            return [tax_id]
        else:
            children = self.children_of(tax_id, 2)
            species_taxids = []
            for t in children:
                species_taxids.extend(self.nary_subtree(t, n))
            return species_taxids

Response: Return a list of species tax_ids under tax_id such that the node under tax_id is above the species and that the node under tax_id has two children.


Instruction:
Implement a function in Python 3 to
check if a DB is authorized and responding before executing the function

Response:
def validate_db(sqlalchemy_bind, is_enabled=ENABLE_DB):
    """ Checks if a DB is authorized and responding before executing the function """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            def is_db_responsive():
                try:
                    sqlalchemy_bind.session.query('1').first_or_404()
                except:
                    return False
                else:
                    return True
            if is_enabled and is_db_responsive():
                return func(*args, **kwargs)
            else:
                abort(HTTP_CODES.UNAUTHORIZED)
        return wrapper
    return decorator


Instruction:
Can you tell what is the following Python 3 function doing
def substitute(template, mapping=None):
    if mapping is None:
        mapping = {}
    templ = Template(template)
    return templ.substitute(mapping)

Response: Render the template with the given mapping.


Instruction:
Can you implement a function in Python 3 that
finds recursively and return all files and directories from the path * path*.

Response:
def dirtree(path):
    """
    Find recursively and return all files and directories from
    the path *path*.
    """
    results = []
    for name in glob.glob(os.path.join(path, '*')):
        results.append(name)
        if os.path.isdir(name):
            results.extend(dirtree(name))
    return results


Instruction:
How would you explain what the following Python 3 function does
def get_tornado_apps(context, debug=False):
    if context.config.app_settings_handler:
        app_settings_handler = import_object(
            context.config.app_settings_handler)
        settings = app_settings_handler(context)
    else:
        settings = {}

    apps = []
    for interface in context.config.interfaces:
        urls = interface.urls
        if not urls:
            urls = [tornado.web.URLSpec('/', NullHandler)]
        apps.append(
            tornado.web.Application(
                urls, debug=debug, context=context,
                interface=interface, **settings
            )
        )
    return apps

Response:
Create Tornado s application for all interfaces which are defined
    in the configuration.


Instruction:
Can you implement a function in Python 3 that
returns an iterator of dicts given a. csv - format file.

Response:
def get_new_nodes(fname):
    """
    Return an iterator of dicts given a .csv-format file.
    """

    with open(fname, 'rU') as infile:
        infile = (line for line in infile if not line.startswith('#'))
        reader = list(csv.DictReader(infile))
        rows = (d for d in reader if d['tax_id'])

    # for now, children are provided as a semicolon-delimited list
    # within a cell (yes, yuck). We need to convert thit into a list
    # if present.
    for d in rows:
        if 'children' in d:
            if d['children']:
                d['children'] = [x.strip() for x in d['children'].split(';')]
            else:
                del d['children']
        yield d


Instruction:
Can you tell what is the following Python 3 function doing
def getlines(fname):

    with open(fname, 'rU') as f:
        for line in f:
            if line.strip() and not line.startswith('#'):
                yield line.split('#', 1)[0].strip()

Response:
Returns iterator of whitespace - stripped lines in file fname omitting
Taxonomy blank lines beginning with '#' character.


Instruction:
Explain what the following Python 3 code does
def parse_raxml(handle):
    s = ''.join(handle.readlines())
    result = {}
    try_set_fields(result, r'(?P<program>RAxML version [0-9.]+)', s)
    try_set_fields(result, r'(?P<datatype>DNA|RNA|AA)', s)
    result['empirical_frequencies'] = (
        result['datatype'] != 'AA' or
        re.search('empirical base frequencies', s, re.IGNORECASE) is not None)
    try_set_fields(result, r'Substitution Matrix: (?P<subs_model>\w+)', s)
    rates = {}
    if result['datatype'] != 'AA':
        try_set_fields(rates,
                       (r"rates\[0\] ac ag at cg ct gt: "
                        r"(?P<ac>[0-9.]+) (?P<ag>[0-9.]+) (?P<at>[0-9.]+) "
                        r"(?P<cg>[0-9.]+) (?P<ct>[0-9.]+) (?P<gt>[0-9.]+)"),
                       s, hook=float)
        try_set_fields(rates, r'rate A <-> C: (?P<ac>[0-9.]+)', s, hook=float)
        try_set_fields(rates, r'rate A <-> G: (?P<ag>[0-9.]+)', s, hook=float)
        try_set_fields(rates, r'rate A <-> T: (?P<at>[0-9.]+)', s, hook=float)
        try_set_fields(rates, r'rate C <-> G: (?P<cg>[0-9.]+)', s, hook=float)
        try_set_fields(rates, r'rate C <-> T: (?P<ct>[0-9.]+)', s, hook=float)
        try_set_fields(rates, r'rate G <-> T: (?P<gt>[0-9.]+)', s, hook=float)
        if len(rates) > 0:
            result['subs_rates'] = rates
    result['gamma'] = {'n_cats': 4}
    try_set_fields(result['gamma'],
                   r"alpha[\[\]0-9]*: (?P<alpha>[0-9.]+)", s, hook=float)
    result['ras_model'] = 'gamma'
    return result

Response: Parse the RAxML summary output.


Instruction:
Implement a function in Python 3 to
parse an alignment a file.

Response:
def parse_stockholm(fobj):
    """Return a list of names from an Stockholm-format sequence alignment
    file. ``fobj`` is an open file or another object representing a
    sequence of lines.

    """

    names = OrderedDict()

    found_eof = False
    for line in fobj:
        line = line.strip()
        if line == '//':
            found_eof = True
        elif line.startswith('#') or not line.strip():
            continue
        else:
            name, __ = line.split(None, 1)
            names[name] = None

    if not found_eof:
        raise ValueError('Invalid Stockholm format: no file terminator')

    return list(names.keys())


Instruction:
Implement a function in Python 3 to
check if rppr binary exists in path

Response:
def has_rppr(rppr_name='rppr'):
    """
    Check for rppr binary in path
    """
    with open(os.devnull) as dn:
        try:
            subprocess.check_call([rppr_name], stdout=dn, stderr=dn)
        except OSError as e:
            if e.errno == os.errno.ENOENT:
                return False
            else:
                raise
        except subprocess.CalledProcessError as e:
            # rppr returns non-zero exit status with no arguments
            pass
    return True


Instruction:
Make a summary of the following Python 3 code
def add_database_args(parser):
    '''
    Add a standard set of database arguments for argparse
    '''
    parser.add_argument(
        'url',
        nargs='?',
        default='sqlite:///ncbi_taxonomy.db',
        type=sqlite_default(),
        help=('Database string URI or filename.  If no database scheme '
              'specified \"sqlite:///\" will be prepended. [%(default)s]'))
    db_parser = parser.add_argument_group(title='database options')

    # TODO: better description of what --schema does
    db_parser.add_argument(
        '--schema',
        help=('Name of SQL schema in database to query '
              '(if database flavor supports this).'))

    return parser

Response: Add a standard set of database arguments for argparse parser


Instruction:
Write a Python 3 script to
return a function that parses a sqlite database and returns the url of the database.

Response:
def sqlite_default():
    '''
    Prepend default scheme if none is specified. This helps provides backwards
    compatibility with old versions of taxtastic where sqlite was the automatic
    default database.
    '''
    def parse_url(url):
        # TODO: need separate option for a config file
        if url.endswith('.db') or url.endswith('.sqlite'):
            if not url.startswith('sqlite:///'):
                url = 'sqlite:///' + url
        elif url.endswith('.cfg') or url.endswith('.conf'):
            conf = configparser.SafeConfigParser(allow_no_value=True)
            conf.optionxform = str  # options are case-sensitive
            conf.read(url)
            url = conf.get('sqlalchemy', 'url')

        return url
    return parse_url


Instruction:
Can you implement a function in Python 3 that
strips non - current files and rollback information from a refpkg.

Response:
def action(args):
    """Strips non-current files and rollback information from a refpkg.

    *args* should be an argparse object with fields refpkg (giving the
    path to the refpkg to operate on).
    """
    log.info('loading reference package')

    refpkg.Refpkg(args.refpkg, create=False).strip()


Instruction:
Given the following Python 3 function, write the documentation
def get_conf_d_files(path):
    if not os.path.isdir(path):
        raise ValueError("'%s' is not a directory" % path)
    files_mask = os.path.join(path, "*.conf")
    return [f for f in sorted(glob.glob(files_mask)) if os.path.isfile(f)]

Response: Return alphabetical ordered list of the. conf files placed in the path.


Instruction:
Create a Python 3 function to
return a list of all configuration files.

Response:
def get_conf_files(filename):
    """
    Return :class:`list` of the all configuration files. *filename* is a
    path of the main configuration file.

    ::

        >>> get_conf_files('exampleapp.conf')
        ['exampleapp.conf', 'exampleapp.conf.d/10-database.conf']
    """
    if not os.path.isfile(filename):
        raise ValueError("'%s' is not a file" % filename)
    conf_d_path = "%s.d" % filename
    if not os.path.exists(conf_d_path):
        return [filename]
    return [filename] + get_conf_d_files(conf_d_path)


Instruction:
Can you generate the documentation for the following Python 3 function
def get_configparser(filename=''):
    filename = filename or os.environ.get('SHELTER_CONFIG_FILENAME', '')
    if not filename:
        raise ImproperlyConfiguredError(_(
            "Configuration file is not defined. You must either "
            "set 'SHELTER_CONFIG_FILENAME' environment variable or "
            "'-f/--config-file' command line argument."
        ))

    parser = six.moves.configparser.RawConfigParser()
    for conf_file in get_conf_files(filename):
        logger.info("Found config '%s'", conf_file)
        if not parser.read(conf_file):
            logger.warning("Error while parsing config '%s'", conf_file)
    return parser

Response:
Read main configuration file and all files from * conf. d * subdirectory
    and return parsed configuration as a ConfigParser instance.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def name(self):
        try:
            return self.config_parser.get('application', 'name')
        except CONFIGPARSER_EXC:
            return super(IniConfig, self).name

Response: Return the name of the application.


Instruction:
Can you generate the documentation for the following Python 3 function
def interfaces(self):
        if 'interfaces' not in self._cached_values:
            self._cached_values['interfaces'] = []
            for name, interface in six.iteritems(self.settings.INTERFACES):
                interface_name = 'interface_%s' % name
                # Hostname:port + unix socket
                try:
                    listen = self.config_parser.get(interface_name, 'Listen')
                except CONFIGPARSER_EXC:
                    listen = interface.get('LISTEN')
                try:
                    unix_socket = self.config_parser.get(
                        interface_name, 'UnixSocket')
                except CONFIGPARSER_EXC:
                    unix_socket = interface.get('UNIX_SOCKET')
                if not listen and not unix_socket:
                    raise ValueError(
                        'Interface MUST listen either on TCP '
                        'or UNIX socket or both')
                host, port = parse_host(listen) if listen else (None, None)
                # Processes
                try:
                    processes = self.config_parser.getint(
                        interface_name, 'Processes')
                except CONFIGPARSER_EXC:
                    processes = int(interface.get('PROCESSES', 1))
                # Urls
                try:
                    urls_obj_name = self.config_parser.get(
                        interface_name, 'Urls')
                except CONFIGPARSER_EXC:
                    urls_obj_name = interface.get('URLS', '')
                if urls_obj_name:
                    urls = import_object(urls_obj_name)
                else:
                    urls = ()

                self._cached_values['interfaces'].append(
                    self.Interface(
                        name, host, port, unix_socket, processes, urls)
                )
        return self._cached_values['interfaces']

Response: Return a list of all interfaces in the config file.


Instruction:
Can you implement a function in Python 3 that
opens a file from the zoneinfo subdir for reading.

Response:
def tz_file(name):
    """
    Open a timezone file from the zoneinfo subdir for reading.

    :param name: The name of the timezone.
    :type name: str

    :rtype: file
    """
    try:
        filepath = tz_path(name)

        return open(filepath, 'rb')
    except TimezoneNotFound:
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        try:
            from pkg_resources import resource_stream
        except ImportError:
            resource_stream = None

        if resource_stream is not None:
            try:
                return resource_stream(__name__, 'zoneinfo/' + name)
            except FileNotFoundError:
                return tz_path(name)

        raise


Instruction:
How would you explain what the following Python 3 function does
def tz_path(name):
    if not name:
        raise ValueError('Invalid timezone')

    name_parts = name.lstrip('/').split('/')

    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)

    filepath = os.path.join(_DIRECTORY, *name_parts)

    if not os.path.exists(filepath):
        raise TimezoneNotFound('Timezone {} not found at {}'.format(name, filepath))

    return filepath

Response: Return the path to a timezone file.


Instruction:
Can you write a function in Python 3 where it
gets the supported timezones.

Response:
def get_timezones():
    """
    Get the supported timezones.

    The list will be cached unless you set the "fresh" attribute to True.

    :param fresh: Whether to get a fresh list or not
    :type fresh: bool

    :rtype: tuple
    """
    base_dir = _DIRECTORY
    zones = ()

    for root, dirs, files in os.walk(base_dir):
        for basename in files:
            zone = os.path.join(root, basename)
            if os.path.isdir(zone):
                continue

            zone = os.path.relpath(zone, base_dir)

            with open(os.path.join(root, basename), 'rb') as fd:
                if fd.read(4) == b'TZif' and zone not in INVALID_ZONES:
                    zones = zones + (zone,)

    return tuple(sorted(zones))


Instruction:
Can you implement a function in Python 3 that
awards target the point value for key.

Response:
def award_points(target, key, reason="", source=None):
    """
    Awards target the point value for key.  If key is an integer then it's a
    one off assignment and should be interpreted as the actual point value.
    """
    point_value, points = get_points(key)

    if not ALLOW_NEGATIVE_TOTALS:
        total = points_awarded(target)
        if total + points < 0:
            reason = reason + "(floored from {0} to 0)".format(points)
            points = -total

    apv = AwardedPointValue(points=points, value=point_value, reason=reason)
    if isinstance(target, get_user_model()):
        apv.target_user = target
        lookup_params = {
            "target_user": target
        }
    else:
        apv.target_object = target
        lookup_params = {
            "target_content_type": apv.target_content_type,
            "target_object_id": apv.target_object_id,
        }

    if source is not None:
        if isinstance(source, get_user_model()):
            apv.source_user = source
        else:
            apv.source_object = source

    apv.save()

    if not TargetStat.update_points(points, lookup_params):
        try:
            sid = transaction.savepoint()
            TargetStat._default_manager.create(
                **dict(lookup_params, points=points)
            )
            transaction.savepoint_commit(sid)
        except IntegrityError:
            transaction.savepoint_rollback(sid)
            TargetStat.update_points(points, lookup_params)

    signals.points_awarded.send(
        sender=target.__class__,
        target=target,
        key=key,
        points=points,
        source=source
    )

    new_points = points_awarded(target)
    old_points = new_points - points

    TargetStat.update_positions((old_points, new_points))

    return apv


Instruction:
Explain what the following Python 3 code does
def points_awarded(target=None, source=None, since=None):

    lookup_params = {}

    if target is not None:
        if isinstance(target, get_user_model()):
            lookup_params["target_user"] = target
        else:
            lookup_params.update({
                "target_content_type": ContentType.objects.get_for_model(target),
                "target_object_id": target.pk,
            })
    if source is not None:
        if isinstance(source, get_user_model()):
            lookup_params["source_user"] = source
        else:
            lookup_params.update({
                "source_content_type": ContentType.objects.get_for_model(source),
                "source_object_id": source.pk,
            })

    if since is None:
        if target is not None and source is None:
            try:
                return TargetStat.objects.get(**lookup_params).points
            except TargetStat.DoesNotExist:
                return 0
        else:
            return AwardedPointValue.points_awarded(**lookup_params)
    else:
        lookup_params["timestamp__gte"] = since
        return AwardedPointValue.points_awarded(**lookup_params)

Response: Determines how many points the given target has received.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __validate_dates(start_date, end_date):
    try:
        start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
        end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
    except ValueError:
        raise ValueError("Incorrect data format, should be yyyy-mm-dd")
    if (end_date - start_date).days > 366:
        raise ValueError("The difference between start and end date " +
                         "should be less than or equal to 366 days.")
    if (end_date - start_date).days < 0:
        raise ValueError("End date cannot be before start date.")

Response: Validate if a date string is a date on yyyy - mm - dd format and it is less than or equal to a year.


Instruction:
Can you tell what is the following Python 3 function doing
def __yahoo_request(query):
    query = quote(query)
    url = 'https://query.yahooapis.com/v1/public/yql?q=' + query + \
        '&format=json&env=store://datatables.org/alltableswithkeys'

    response = urlopen(url).read()

    return json.loads(response.decode('utf-8'))['query']['results']

Response: Request Yahoo Finance information from YQL.


Instruction:
Can you generate the documentation for the following Python 3 function
def request_quotes(tickers_list, selected_columns=['*']):
    __validate_list(tickers_list)
    __validate_list(selected_columns)
    query = 'select {cols} from yahoo.finance.quotes where symbol in ({vals})'
    query = query.format(
        cols=', '.join(selected_columns),
        vals=', '.join('"{0}"'.format(s) for s in tickers_list)
    )

    response = __yahoo_request(query)

    if not response:
        raise RequestError('Unable to process the request. Check if the ' +
                           'columns selected are valid.')

    if not type(response['quote']) is list:
        return [response['quote']]
    return response['quote']

Response: Request Yahoo Finance recent quotes.


Instruction:
How would you explain what the following Python 3 function does
def request_historical(ticker, start_date, end_date):
    __validate_dates(start_date, end_date)

    cols = ['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Adj_Close']
    query = 'select {cols} from yahoo.finance.historicaldata ' + \
        'where symbol in ("{ticker}") and startDate = "{start_date}" ' + \
        'and endDate = "{end_date}"'
    query = query.format(
        cols=', '.join(cols),
        ticker=ticker,
        start_date=start_date,
        end_date=end_date
    )

    response = __yahoo_request(query)
    if not response:
        raise RequestError('Unable to process the request. Check if the ' +
                           'stock ticker used is a valid one.')

    if not type(response['quote']) is list:
        return [response['quote']]
    return response['quote']

Response: Request daily historical information for a given date range.


Instruction:
Create a Python 3 function to
download historical data from Yahoo Finance as CSV.

Response:
def download_historical(tickers_list, output_folder):
    """Download historical data from Yahoo Finance.

    Downloads full historical data from Yahoo Finance as CSV. The following
    fields are available: Adj Close, Close, High, Low, Open and Volume. Files
    will be saved to output_folder as <ticker>.csv.

    :param tickers_list: List of tickers that will be returned.
    :type tickers_list: list of strings
    :param output_folder: Output folder path
    :type output_folder: string
    """
    __validate_list(tickers_list)
    for ticker in tickers_list:
        file_name = os.path.join(output_folder, ticker + '.csv')
        with open(file_name, 'wb') as f:
            base_url = 'http://real-chart.finance.yahoo.com/table.csv?s='
            try:
                urlopen(base_url + ticker)
                urlretrieve(base_url + ticker, f.name)
            except:
                os.remove(file_name)
                raise RequestError('Unable to process the request. Check if ' +
                                   ticker + ' is a valid stock ticker')


Instruction:
Create a Python 3 function for
searching for files in the run - file directory and cellpy - files.

Response:
def search_for_files(run_name, raw_extension=None, cellpy_file_extension=None,
                     raw_file_dir=None, cellpy_file_dir=None, prm_filename=None,
                     file_name_format=None, cache=None):
    """Searches for files (raw-data files and cellpy-files).

    Args:
        run_name(str): run-file identification.
        raw_extension(str): optional, extension of run-files (without the '.').
        cellpy_file_extension(str): optional, extension for cellpy files
            (without the '.').
        raw_file_dir(path): optional, directory where to look for run-files
            (default: read prm-file)
        cellpy_file_dir(path): optional, directory where to look for
            cellpy-files (default: read prm-file)
        prm_filename(path): optional parameter file can be given.
        file_name_format(str): format of raw-file names or a glob pattern
            (default: YYYYMMDD_[name]EEE_CC_TT_RR).
        cache(list): list of cached file names to search through

    Returns:
        run-file names (list) and cellpy-file-name (path).
    """

    time_00 = time.time()
    cellpy_file_extension = "h5"
    res_extension = "res"
    version = 0.1
    # might include searching and removing "." in extensions
    # should include extension definitions in prm file (version 0.6)
    logger.debug(f"searching for {run_name}")
    if raw_extension is None:
        raw_extension = res_extension

    if cellpy_file_extension is None:
        cellpy_file_extension = cellpy_file_extension

    if prm_filename is not None:
        logging.debug("reading prm file disabled")

    if not all([raw_file_dir, cellpy_file_dir, file_name_format]):
        # import cellpy.parameters.prms as prms
        # prms = prmreader.read()
        logger.debug("using prms already set")

    if raw_file_dir is None:
        raw_file_dir = prms.Paths["rawdatadir"]

    if cellpy_file_dir is None:
        cellpy_file_dir = prms.Paths["cellpydatadir"]

    if file_name_format is None:
        try:
            # To be implemented in version 0.5:
            file_name_format = prms.file_name_format
        except AttributeError:
            file_name_format = "YYYYMMDD_[name]EEE_CC_TT_RR"
            if version >= 0.5:
                print("Could not read file_name_format "
                      "from _cellpy_prms_xxx.conf.")
                print("Using:")
                print("file_name_format:", file_name_format)
                file_format_explanation = "YYYYMMDD is date,"
                file_format_explanation += " EEE is electrode number"
                file_format_explanation += " CC is cell number,"
                file_format_explanation += " TT is cell_type, RR is run number."
                print(file_format_explanation)

    # check if raw_file_dir exists
    if not os.path.isdir(raw_file_dir):
        warnings.warn("your raw file directory cannot be accessed!")

    if file_name_format.upper() == "YYYYMMDD_[NAME]EEE_CC_TT_RR":
        glob_text_raw = "%s_*.%s" % (os.path.basename(run_name), raw_extension)
        reg_exp_raw = "xxx"
    else:
        glob_text_raw = file_name_format

    cellpy_file = "{0}.{1}".format(run_name, cellpy_file_extension)
    cellpy_file = os.path.join(cellpy_file_dir, cellpy_file)

    # TODO: @jepe - use pathlib

    if cache is None:

        use_pathlib_path = False
        return_as_str_list = True

        if use_pathlib_path:
            logger.debug("using pathlib.Path")
            if os.path.isdir(raw_file_dir):
                run_files = pathlib.Path(raw_file_dir).glob(glob_text_raw)
                if return_as_str_list:
                    run_files = [str(f.resolve()) for f in run_files]
                    run_files.sort()
            else:
                run_files = []

        else:
            if os.path.isdir(raw_file_dir):
                glob_text_raw_full = os.path.join(raw_file_dir, glob_text_raw)
                run_files = glob.glob(glob_text_raw_full)
                run_files.sort()
            else:
                run_files = []

        logger.debug(f"(dt: {(time.time() - time_00):4.2f}s)")
        return run_files, cellpy_file

    else:
        logger.debug("using cache in filefinder")
        if os.path.isdir(raw_file_dir):
            if len(cache) == 0:
                cache = os.listdir(raw_file_dir)
            run_files = [
                os.path.join(
                    raw_file_dir, x
                ) for x in cache if fnmatch.fnmatch(
                    x, glob_text_raw
                )
            ]
            run_files.sort()
        else:
            run_files = []

        logger.debug(f"(dt: {(time.time() - time_00):4.2f}s)")
        return run_files, cellpy_file, cache


Instruction:
Make a summary of the following Python 3 code
def squash_xml_to_text(elm, remove_namespaces=False):
    leading_text = elm.text and elm.text or ''
    result = [leading_text]

    for child in elm.getchildren():
        # Encoding is set to utf-8 because otherwise `ó` would
        # become `ó`
        child_value = etree.tostring(child, encoding='utf-8')
        # Decode to a string for later regexp and whitespace stripping
        child_value = child_value.decode('utf-8')
        result.append(child_value)

    if remove_namespaces:
        # Best way to remove the namespaces without having the parser complain
        # about producing invalid XML.
        result = [re.sub(' xmlns:?[^=]*="[^"]*"', '', v) for v in result]

    # Join the results and strip any surrounding whitespace
    result = u''.join(result).strip()
    return result

Response: Squash the given XML element as elm to a text containing XML.


Instruction:
Write a Python 3 function that can
load a raw data - file and inspect it

Response:
def load(self, file_name):
        """Load a raw data-file

        Args:
            file_name (path)

        Returns:
            loaded test
        """

        new_rundata = self.loader(file_name)
        new_rundata = self.inspect(new_rundata)
        return new_rundata


Instruction:
Here you have a function in Python 3, explain what it does
def datetime2ole(dt):
    delta = dt - OLE_TIME_ZERO
    delta_float = delta / datetime.timedelta(days=1)  # trick from SO
    return delta_float

Response: converts from datetime object to ole datetime float


Instruction:
Here you have a function in Python 3, explain what it does
def get_raw_limits():
        raw_limits = dict()
        raw_limits["current_hard"] = 0.0000000000001
        raw_limits["current_soft"] = 0.00001
        raw_limits["stable_current_hard"] = 2.0
        raw_limits["stable_current_soft"] = 4.0
        raw_limits["stable_voltage_hard"] = 2.0
        raw_limits["stable_voltage_soft"] = 4.0
        raw_limits["stable_charge_hard"] = 2.0
        raw_limits["stable_charge_soft"] = 5.0
        raw_limits["ir_change"] = 0.00001
        return raw_limits

Response:
Include the settings for how to decide what kind of
        step you are examining here.


Instruction:
Write a Python 3 function that can
load data from biologics. res files.

Response:
def loader(self, file_name, bad_steps=None, **kwargs):
        """Loads data from biologics .mpr files.

        Args:
            file_name (str): path to .res file.
            bad_steps (list of tuples): (c, s) tuples of steps s
             (in cycle c) to skip loading.

        Returns:
            new_tests (list of data objects)
        """
        new_tests = []
        if not os.path.isfile(file_name):
            self.logger.info("Missing file_\n   %s" % file_name)
            return None

        filesize = os.path.getsize(file_name)
        hfilesize = humanize_bytes(filesize)
        txt = "Filesize: %i (%s)" % (filesize, hfilesize)
        self.logger.debug(txt)

        # creating temporary file and connection
        temp_dir = tempfile.gettempdir()
        temp_filename = os.path.join(temp_dir, os.path.basename(file_name))
        shutil.copy2(file_name, temp_dir)

        self.logger.debug("tmp file: %s" % temp_filename)
        self.logger.debug("HERE WE LOAD THE DATA")

        data = DataSet()
        fid = FileID(file_name)

        # div parameters and information (probably load this last)
        test_no = 1
        data.test_no = test_no
        data.loaded_from = file_name

        # some overall prms
        data.channel_index = None
        data.channel_number = None
        data.creator = None
        data.item_ID = None
        data.schedule_file_name = None
        data.start_datetime = None
        data.test_ID = None
        data.test_name = None
        data.raw_data_files.append(fid)

        # --------- read raw-data (normal-data) -------------------------
        self.logger.debug("reading raw-data")
        self.mpr_data = None
        self.mpr_log = None
        self.mpr_settings = None

        self._load_mpr_data(temp_filename, bad_steps)
        length_of_test = self.mpr_data.shape[0]
        self.logger.debug(f"length of test: {length_of_test}")

        self.logger.debug("renaming columns")
        self._rename_headers()
        # ---------  stats-data (summary-data) -------------------------
        summary_df = self._create_summary_data()

        if summary_df.empty:
            txt = "\nCould not find any summary (stats-file)!"
            txt += " (summary_df.empty = True)"
            txt += "\n -> issue make_summary(use_cellpy_stat_file=False)"
            warnings.warn(txt)

        data.dfsummary = summary_df
        data.dfdata = self.mpr_data

        data.raw_data_files_length.append(length_of_test)
        new_tests.append(data)

        self._clean_up(temp_filename)
        return new_tests


Instruction:
Can you create a Python 3 function that
dumps data to csv

Response:
def csv_dumper(**kwargs):
    """dump data to csv"""
    logging.info("dumping to csv")
    barn = kwargs["barn"]
    farms = kwargs["farms"]
    experiments = kwargs["experiments"]
    for experiment, farm in zip(experiments, farms):
        name = experiment.journal.name
        project = experiment.journal.project
        project_dir, batch_dir, raw_dir = \
            experiment.journal.paginate()
        if batch_dir is None:
            logging.info("have to generate folder-name on the fly")
            out_data_dir, project_dir, batch_dir, raw_dir = \
                generate_folder_names(name, project)

        if barn == "batch_dir":
            out_dir = batch_dir
        elif barn == "project_dir":
            out_dir = project_dir
        elif barn == "raw_dir":
            out_dir = raw_dir
        else:
            out_dir = barn

        for animal in farm:
            file_name = os.path.join(
                out_dir, "summary_%s_%s.csv" % (
                    animal.name,
                    name
                )
            )
            logging.info(f"> {file_name}")
            animal.to_csv(file_name, sep=prms.Reader.sep)


Instruction:
Can you tell what is the following Python 3 function doing
def ram_dumper(**kwargs):
    logging.debug("trying to save stuff in memory")
    farms = kwargs["farms"]
    experiments = kwargs["experiments"]
    engine = kwargs["engine"]

    try:
        engine_name = engine.__name__
    except AttributeError:
        engine_name = engine.__dict__.__name__

    accepted_engines = ["summary_engine",]
    if engine_name in accepted_engines:
        logging.debug("found the engine that I will try to dump from: "
                      f"{engine_name}")

        for experiment, farm in zip(experiments, farms):
            name = experiment.journal.name
            project = experiment.journal.project
            experiment.memory_dumped[engine_name] = farm
            logging.debug(f"farm put into memory_dumped ({project}::{name})")

Response: Dump data to memory for later usage.


Instruction:
Can you generate the documentation for the following Python 3 function
def screen_dumper(**kwargs):
    farms = kwargs["farms"]
    engine = kwargs["engine"]
    logging.info("dumping to screen")

    print(f"\n[Screen dumper] ({engine})")
    try:
        if len(farms) == 1:
            print(f"You have one farm with little pandas.")

        else:
            print(f"You have {len(farms)} farms with little pandas.")
    except TypeError:
        print(" - your farm has burned to the ground.")
    else:
        for number, farm in enumerate(farms):
            print(f"[#{number+1}]You have {len(farm)} "
                  f"little pandas in this farm.")
            for animal in farm:
                print(80*"=")
                try:
                    print(animal.name)
                except AttributeError:
                    print("no-name")
                print(80*"-")
                print(animal.head(5))
                print()

Response: Dump data to screen.


Instruction:
Implement a Python 3 function for
creating more informative legends

Response:
def create_legend(info, c, option="clean", use_index=False):
    """creating more informative legends"""

    logging.debug("    - creating legends")
    mass, loading, label = info.loc[c, ["masses", "loadings", "labels"]]

    if use_index or not label:
        label = c.split("_")
        label = "_".join(label[1:])

    if option == "clean":
        return label

    if option == "mass":
        label = f"{label} ({mass:.2f} mg)"
    elif option == "loading":
        label = f"{label} ({loading:.2f} mg/cm2)"
    elif option == "all":
        label = f"{label} ({mass:.2f} mg) ({loading:.2f} mg/cm2)"

    return label


Instruction:
Given the following Python 3 function, write the documentation
def create_plot_option_dicts(info, marker_types=None, colors=None,
                             line_dash=None, size=None):
    """Create two dictionaries with plot-options.

    The first iterates colors (based on group-number), the second iterates
    through marker types.

    Returns: group_styles (dict), sub_group_styles (dict)
    """

    logging.debug("    - creating plot-options-dict (for bokeh)")

    # Current only works for bokeh

    if marker_types is None:
        marker_types = ["circle", "square", "triangle", "invertedtriangle",
                        "diamond", "cross", "asterix"]

    if line_dash is None:
        line_dash = [0, 0]

    if size is None:
        size = 10

    groups = info.groups.unique()
    number_of_groups = len(groups)
    if colors is None:
        if number_of_groups < 4:
            # print("using 3")
            colors = bokeh.palettes.brewer['YlGnBu'][3]
        else:
            # print(f"using {min(9, number_of_groups)}")
            colors = bokeh.palettes.brewer['YlGnBu'][min(9, number_of_groups)]

    sub_groups = info.sub_groups.unique()

    marker_it = itertools.cycle(marker_types)
    colors_it = itertools.cycle(colors)

    group_styles = dict()
    sub_group_styles = dict()

    for j in groups:
        color = next(colors_it)
        marker_options = {
            "line_color": color,
            "fill_color": color,
        }

        line_options = {
            "line_color": color,
        }
        group_styles[j] = {
            "marker": marker_options,
            "line": line_options,
        }

    for j in sub_groups:
        marker_type = next(marker_it)
        marker_options = {
            "marker": marker_type,
            "size": size,
        }

        line_options = {
            "line_dash": line_dash,
        }
        sub_group_styles[j] = {
            "marker": marker_options,
            "line": line_options,
        }
    return group_styles, sub_group_styles

Response: Create two dictionaries with plot - options.


Instruction:
Create a Python 3 function for
creating plots of summary data.

Response:
def summary_plotting_engine(**kwargs):
    """creates plots of summary data."""

    logging.debug(f"Using {prms.Batch.backend} for plotting")
    experiments = kwargs["experiments"]
    farms = kwargs["farms"]
    barn = None

    logging.debug("    - summary_plot_engine")
    farms = _preparing_data_and_plotting(
        experiments=experiments,
        farms=farms
    )

    return farms, barn


Instruction:
Can you tell what is the following Python 3 function doing
def run_dumper(self, dumper):

        logging.debug("start dumper::")
        dumper(
            experiments=self.experiments,
            farms=self.farms,
            barn=self.barn,
            engine=self.current_engine,
        )
        logging.debug("::dumper ended")

Response: run dumper once pr. engine


Instruction:
Can you write a function in Python 3 where it
returns a pandas. DataFrame containing the descriptor and hardware ID of each available COM port.

Response:
def _comports():
    '''
    Returns
    -------
    pandas.DataFrame
        Table containing descriptor, and hardware ID of each available COM
        port, indexed by port (e.g., "COM4").
    '''
    return (pd.DataFrame(list(map(list, serial.tools.list_ports.comports())),
                         columns=['port', 'descriptor', 'hardware_id'])
            .set_index('port'))


Instruction:
Given the following Python 3 function, write the documentation
def comports(vid_pid=None, include_all=False, check_available=True,
             only_available=False):
    '''
    .. versionchanged:: 0.9
        Add :data:`check_available` keyword argument to optionally check if
        each port is actually available by attempting to open a temporary
        connection.

        Add :data:`only_available` keyword argument to only include ports that
        are actually available for connection.

    Parameters
    ----------
    vid_pid : str or list, optional
        One or more USB vendor/product IDs to match.

        Each USB vendor/product must be in the form ``'<vid>:<pid>'``.
        For example, ``'2341:0010'``.
    include_all : bool, optional
        If ``True``, include all available serial ports, but sort rows such
        that ports matching specified USB vendor/product IDs come first.

        If ``False``, only include ports that match specified USB
        vendor/product IDs.
    check_available : bool, optional
        If ``True``, check if each port is actually available by attempting to
        open a temporary connection.
    only_available : bool, optional
        If ``True``, only include ports that are available.

    Returns
    -------
    pandas.DataFrame
        Table containing descriptor and hardware ID of each COM port, indexed
        by port (e.g., "COM4").

        .. versionchanged:: 0.9
            If :data:`check_available` is ``True``, add an ``available`` column
            to the table indicating whether each port accepted a connection.
    '''
    df_comports = _comports()

    # Extract USB product and vendor IDs from `hwid` entries of the form:
    #
    #     FTDIBUS\VID_0403+PID_6001+A60081GEA\0000
    df_hwid = (df_comports.hardware_id.str.lower().str
               .extract('vid_(?P<vid>[0-9a-f]+)\+pid_(?P<pid>[0-9a-f]+)',
                        expand=True))
    # Extract USB product and vendor IDs from `hwid` entries of the form:
    #
    #     USB VID:PID=16C0:0483 SNR=2145930
    no_id_mask = df_hwid.vid.isnull()
    df_hwid.loc[no_id_mask] = (df_comports.loc[no_id_mask, 'hardware_id']
                               .str.lower().str
                               .extract('vid:pid=(?P<vid>[0-9a-f]+):'
                                        '(?P<pid>[0-9a-f]+)', expand=True))
    df_comports = df_comports.join(df_hwid)

    if vid_pid is not None:
        if isinstance(vid_pid, six.string_types):
            # Single USB vendor/product ID specified.
            vid_pid = [vid_pid]

        # Mark ports that match specified USB vendor/product IDs.
        df_comports['include'] = (df_comports.vid + ':' +
                                  df_comports.pid).isin(map(str.lower,
                                                            vid_pid))

        if include_all:
            # All ports should be included, but sort rows such that ports
            # matching specified USB vendor/product IDs come first.
            df_comports = (df_comports.sort_values('include', ascending=False)
                           .drop('include', axis=1))
        else:
            # Only include ports that match specified USB vendor/product IDs.
            df_comports = (df_comports.loc[df_comports.include]
                           .drop('include', axis=1))

    if check_available or only_available:
        # Add `available` column indicating whether each port accepted a
        # connection.  A port may not, for example, accept a connection if the
        # port is already open.
        available = []

        for name_i, port_info_i in df_comports.iterrows():
            try:
                connection = serial.Serial(port=name_i)
                connection.close()
                available.append(True)
            except serial.SerialException:
                available.append(False)
        df_comports['available'] = available
        if only_available:
            df_comports = df_comports.loc[df_comports.available]
        if not check_available:
            del df_comports['available']
    return df_comports

Response: Return a table containing the descriptor and hardware ID of each COM port.


Instruction:
Implement a function in Python 3 to
get the serial ports available on Windows.

Response:
def _get_serial_ports_windows():
    '''
    Uses the Win32 registry to return a iterator of serial (COM) ports existing
    on this computer.

    See http://stackoverflow.com/questions/1205383/listing-serial-com-ports-on-windows
    '''
    import six.moves.winreg as winreg

    reg_path = 'HARDWARE\\DEVICEMAP\\SERIALCOMM'
    try:
        key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, reg_path)
    except WindowsError:
        # No serial ports. Return empty generator.
        return

    for i in itertools.count():
        try:
            val = winreg.EnumValue(key, i)
            yield str(val[1])
        except EnvironmentError:
            break


Instruction:
Explain what the following Python 3 code does
def get_port(self, baud_rate):
        '''
        Using the specified baud-rate, attempt to connect to each available
        serial port.  If the `test_connection()` method returns `True` for a
        port, update the `port` attribute and return the port.

        In the case where the `test_connection()` does not return `True` for
        any of the evaluated ports, raise a `ConnectionError`.
        '''
        self.port = None

        for test_port in get_serial_ports():
            if self.test_connection(test_port, baud_rate):
                self.port = test_port
                break
            sleep(0.1)

        if self.port is None:
            raise ConnectionError('Could not connect to serial device.')

        return self.port

Response: Attempts to connect to each available ArcGIS serial port. If the test_connection method returns True for the port and update the port attribute. If the test_connection method returns False for the port and raise a ConnectionError.


Instruction:
Can you implement a function in Python 3 that
reads the yml file

Response:
def _read(name):
    """read the yml file"""
    logging.debug("Reading config-file: %s" % name)
    try:
        with open(name, "r") as config_file:
            prm_dict = yaml.load(config_file)
    except yaml.YAMLError:
        raise yaml.YAMLErrorr
    else:
        return prm_dict


Instruction:
Can you tell what is the following Python 3 function doing
def simple_db_engine(reader=None, srnos=None):

    if reader is None:
        reader = dbreader.Reader()
        logger.debug("No reader provided. Creating one myself.")

    info_dict = dict()
    info_dict["filenames"] = [reader.get_cell_name(srno) for srno in srnos]
    info_dict["masses"] = [reader.get_mass(srno) for srno in srnos]
    info_dict["total_masses"] = [reader.get_total_mass(srno) for srno in srnos]
    info_dict["loadings"] = [reader.get_loading(srno) for srno in srnos]
    info_dict["fixed"] = [reader.inspect_hd5f_fixed(srno) for srno in srnos]
    info_dict["labels"] = [reader.get_label(srno) for srno in srnos]
    info_dict["cell_type"] = [reader.get_cell_type(srno) for srno in srnos]
    info_dict["raw_file_names"] = []
    info_dict["cellpy_file_names"] = []

    logger.debug("created info-dict")

    for key in list(info_dict.keys()):
        logger.debug("%s: %s" % (key, str(info_dict[key])))

    _groups = [reader.get_group(srno) for srno in srnos]
    logger.debug(">\ngroups: %s" % str(_groups))
    groups = helper.fix_groups(_groups)
    info_dict["groups"] = groups

    my_timer_start = time.time()
    filename_cache = []
    info_dict = helper.find_files(info_dict, filename_cache)
    my_timer_end = time.time()
    if (my_timer_end - my_timer_start) > 5.0:
        logger.info(
            "The function _find_files was very slow. "
            "Save your info_df so you don't have to run it again!"
        )

    info_df = pd.DataFrame(info_dict)
    info_df = info_df.sort_values(["groups", "filenames"])
    info_df = helper.make_unique_groups(info_df)

    info_df["labels"] = info_df["filenames"].apply(helper.create_labels)
    info_df.set_index("filenames", inplace=True)
    return info_df

Response: engine that gets values from the simple excel db


Instruction:
Can you create a Python 3 function that
overrides set and clear methods on event to call specified callback function after performing default behaviour.

Response:
def orify(event, changed_callback):
    '''
    Override ``set`` and ``clear`` methods on event to call specified callback
    function after performing default behaviour.

    Parameters
    ----------

    '''
    event.changed = changed_callback
    if not hasattr(event, '_set'):
        # `set`/`clear` methods have not been overridden on event yet.
        # Override methods to call `changed_callback` after performing default
        # action.
        event._set = event.set
        event._clear = event.clear
        event.set = lambda: or_set(event)
        event.clear = lambda: or_clear(event)


Instruction:
Implement a Python 3 function for
returning a new event that is set when any of the specified events have been set.

Response:
def OrEvent(*events):
    '''
    Parameters
    ----------
    events : list(threading.Event)
        List of events.

    Returns
    -------
    threading.Event
        Event that is set when **at least one** of the events in :data:`events`
        is set.
    '''
    or_event = threading.Event()

    def changed():
        '''
        Set ``or_event`` if any of the specified events have been set.
        '''
        bools = [event_i.is_set() for event_i in events]
        if any(bools):
            or_event.set()
        else:
            or_event.clear()
    for event_i in events:
        # Override ``set`` and ``clear`` methods on event to update state of
        # `or_event` after performing default behaviour.
        orify(event_i, changed)

    # Set initial state of `or_event`.
    changed()
    return or_event


Instruction:
Given the following Python 3 function, write the documentation
def request(device, response_queue, payload, timeout_s=None, poll=POLL_QUEUES):
    '''
    Send payload to serial device and wait for response.

    Parameters
    ----------
    device : serial.Serial
        Serial instance.
    response_queue : Queue.Queue
        Queue to wait for response on.
    payload : str or bytes
        Payload to send.
    timeout_s : float, optional
        Maximum time to wait (in seconds) for response.

        By default, block until response is ready.
    poll : bool, optional
        If ``True``, poll response queue in a busy loop until response is
        ready (or timeout occurs).

        Polling is much more processor intensive, but (at least on Windows)
        results in faster response processing.  On Windows, polling is
        enabled by default.
    '''
    device.write(payload)
    if poll:
        # Polling enabled.  Wait for response in busy loop.
        start = dt.datetime.now()
        while not response_queue.qsize():
            if (dt.datetime.now() - start).total_seconds() > timeout_s:
                raise queue.Empty('No response received.')
        return response_queue.get()
    else:
        # Polling disabled.  Use blocking `Queue.get()` method to wait for
        # response.
        return response_queue.get(timeout=timeout_s)

Response: Send payload to serial device and wait for response.


Instruction:
How would you explain what the following Python 3 function does
def connection_made(self, transport):
        self.port = transport.serial.port
        logger.debug('connection_made: `%s` `%s`', self.port, transport)
        self.transport = transport
        self.connected.set()
        self.disconnected.clear()

Response: Called when the connection is made


Instruction:
Explain what the following Python 3 code does
def connection_lost(self, exception):
        if isinstance(exception, Exception):
            logger.debug('Connection to port `%s` lost: %s', self.port,
                         exception)
        else:
            logger.debug('Connection to port `%s` closed', self.port)
        self.connected.clear()
        self.disconnected.set()

Response: Called when the serial port is closed or the reader loop terminated.


Instruction:
Write a Python 3 function for
writing data to the serial port.

Response:
def write(self, data, timeout_s=None):
        '''
        Write to serial port.

        Waits for serial connection to be established before writing.

        Parameters
        ----------
        data : str or bytes
            Data to write to serial port.
        timeout_s : float, optional
            Maximum number of seconds to wait for serial connection to be
            established.

            By default, block until serial connection is ready.
        '''
        self.connected.wait(timeout_s)
        self.protocol.transport.write(data)


Instruction:
Can you tell what is the following Python 3 function doing
def request(self, response_queue, payload, timeout_s=None,
                poll=POLL_QUEUES):
        '''
        Send

        Parameters
        ----------
        device : serial.Serial
            Serial instance.
        response_queue : Queue.Queue
            Queue to wait for response on.
        payload : str or bytes
            Payload to send.
        timeout_s : float, optional
            Maximum time to wait (in seconds) for response.

            By default, block until response is ready.
        poll : bool, optional
            If ``True``, poll response queue in a busy loop until response is
            ready (or timeout occurs).

            Polling is much more processor intensive, but (at least on Windows)
            results in faster response processing.  On Windows, polling is
            enabled by default.
        '''
        self.connected.wait(timeout_s)
        return request(self, response_queue, payload, timeout_s=timeout_s,
                       poll=poll)

Response: Sends a message to the specified device.


Instruction:
Implement a function in Python 3 to
convert a column header from the MPT file into a tuple of canonical name and numpy dtype

Response:
def fieldname_to_dtype(fieldname):
    """Converts a column header from the MPT file into a tuple of
    canonical name and appropriate numpy dtype"""

    if fieldname == 'mode':
        return ('mode', np.uint8)
    elif fieldname in ("ox/red", "error", "control changes", "Ns changes",
                       "counter inc."):
        return (fieldname, np.bool_)
    elif fieldname in ("time/s", "P/W", "(Q-Qo)/mA.h", "x", "control/V",
                       "control/V/mA", "(Q-Qo)/C", "dQ/C", "freq/Hz",
                       "|Ewe|/V", "|I|/A", "Phase(Z)/deg", "|Z|/Ohm",
                       "Re(Z)/Ohm", "-Im(Z)/Ohm"):
        return (fieldname, np.float_)
    elif fieldname in ("cycle number", "I Range", "Ns", "half cycle"):
        return (fieldname, np.int_)
    elif fieldname in ("dq/mA.h", "dQ/mA.h"):
        return ("dQ/mA.h", np.float_)
    elif fieldname in ("I/mA", "<I>/mA"):
        return ("I/mA", np.float_)
    elif fieldname in ("Ewe/V", "<Ewe>/V"):
        return ("Ewe/V", np.float_)
    else:
        raise ValueError("Invalid column header: %s" % fieldname)


Instruction:
Given the following Python 3 function, write the documentation
def comma_converter(float_string):
    trans_table = maketrans(b',', b'.')
    return float(float_string.translate(trans_table))

Response: Convert numbers to floats whether the decimal point is. or.


Instruction:
Here you have a function in Python 3, explain what it does
def MPTfile(file_or_path):

    if isinstance(file_or_path, str):
        mpt_file = open(file_or_path, 'rb')
    else:
        mpt_file = file_or_path

    magic = next(mpt_file)
    if magic != b'EC-Lab ASCII FILE\r\n':
        raise ValueError("Bad first line for EC-Lab file: '%s'" % magic)

    nb_headers_match = re.match(b'Nb header lines : (\d+)\s*$', next(mpt_file))
    nb_headers = int(nb_headers_match.group(1))
    if nb_headers < 3:
        raise ValueError("Too few header lines: %d" % nb_headers)

    ## The 'magic number' line, the 'Nb headers' line and the column headers
    ## make three lines. Every additional line is a comment line.
    comments = [next(mpt_file) for i in range(nb_headers - 3)]

    fieldnames = str3(next(mpt_file)).strip().split('\t')
    record_type = np.dtype(list(map(fieldname_to_dtype, fieldnames)))

    ## Must be able to parse files where commas are used for decimal points
    converter_dict = dict(((i, comma_converter)
                           for i in range(len(fieldnames))))
    mpt_array = np.loadtxt(mpt_file, dtype=record_type,
                           converters=converter_dict)

    return mpt_array, comments

Response: Opens. mpt files as numpy record arrays and returns a numpy record array object and a list of comments


Instruction:
Create a Python 3 function to
read in the VMP module headers in the VMPmodule_hdr format. Yields a dict with the headers and offset for each module.

Response:
def read_VMP_modules(fileobj, read_module_data=True):
    """Reads in module headers in the VMPmodule_hdr format. Yields a dict with
    the headers and offset for each module.

    N.B. the offset yielded is the offset to the start of the data i.e. after
    the end of the header. The data runs from (offset) to (offset+length)"""
    while True:
        module_magic = fileobj.read(len(b'MODULE'))
        if len(module_magic) == 0:  # end of file
            raise StopIteration
        elif module_magic != b'MODULE':
            raise ValueError(
                "Found %r, expecting start of new VMP MODULE" % module_magic)

        hdr_bytes = fileobj.read(VMPmodule_hdr.itemsize)
        if len(hdr_bytes) < VMPmodule_hdr.itemsize:
            raise IOError("Unexpected end of file while reading module header")

        hdr = np.fromstring(hdr_bytes, dtype=VMPmodule_hdr, count=1)
        hdr_dict = dict(((n, hdr[n][0]) for n in VMPmodule_hdr.names))
        print("---hdr-dict---")
        pprint(hdr_dict)
        hdr_dict['offset'] = fileobj.tell()
        if read_module_data:
            hdr_dict['data'] = fileobj.read(hdr_dict['length'])
            if len(hdr_dict['data']) != hdr_dict['length']:
                raise IOError("""Unexpected end of file while reading data
                    current module: %s
                    length read: %d
                    length expected: %d""" % (hdr_dict['longname'],
                                              len(hdr_dict['data']),
                                              hdr_dict['length']))
            yield hdr_dict
        else:
            yield hdr_dict
            fileobj.seek(hdr_dict['offset'] + hdr_dict['length'], SEEK_SET)


Instruction:
Here you have a function in Python 3, explain what it does
def get_headers_global():
        headers = dict()
        # - global column headings (specific for Arbin)
        headers["applications_path_txt"] = 'Applications_Path'
        headers["channel_index_txt"] = 'Channel_Index'
        headers["channel_number_txt"] = 'Channel_Number'
        headers["channel_type_txt"] = 'Channel_Type'
        headers["comments_txt"] = 'Comments'
        headers["creator_txt"] = 'Creator'
        headers["daq_index_txt"] = 'DAQ_Index'
        headers["item_id_txt"] = 'Item_ID'
        headers["log_aux_data_flag_txt"] = 'Log_Aux_Data_Flag'
        headers["log_chanstat_data_flag_txt"] = 'Log_ChanStat_Data_Flag'
        headers["log_event_data_flag_txt"] = 'Log_Event_Data_Flag'
        headers["log_smart_battery_data_flag_txt"] = 'Log_Smart_Battery_Data_Flag'
        headers["mapped_aux_conc_cnumber_txt"] = 'Mapped_Aux_Conc_CNumber'
        headers["mapped_aux_di_cnumber_txt"] = 'Mapped_Aux_DI_CNumber'
        headers["mapped_aux_do_cnumber_txt"] = 'Mapped_Aux_DO_CNumber'
        headers["mapped_aux_flow_rate_cnumber_txt"] = 'Mapped_Aux_Flow_Rate_CNumber'
        headers["mapped_aux_ph_number_txt"] = 'Mapped_Aux_PH_Number'
        headers["mapped_aux_pressure_number_txt"] = 'Mapped_Aux_Pressure_Number'
        headers["mapped_aux_temperature_number_txt"] = 'Mapped_Aux_Temperature_Number'
        headers["mapped_aux_voltage_number_txt"] = 'Mapped_Aux_Voltage_Number'
        headers["schedule_file_name_txt"] = 'Schedule_File_Name'  # KEEP FOR CELLPY FILE FORMAT
        headers["start_datetime_txt"] = 'Start_DateTime'
        headers["test_id_txt"] = 'Test_ID'  # KEEP FOR CELLPY FILE FORMAT
        headers["test_name_txt"] = 'Test_Name'  # KEEP FOR CELLPY FILE FORMAT
        return headers

Response: Defines the so - called global column headings for Arbin. res - files


Instruction:
Implement a function in Python 3 to
inspect the file - > reports to log ( debug

Response:
def inspect(self, run_data):
        """Inspect the file -> reports to log (debug)"""

        # TODO: type checking
        if DEBUG_MODE:
            checked_rundata = []
            for data in run_data:
                new_cols = data.dfdata.columns
                for col in self.headers_normal:
                    if col not in new_cols:
                        logging.debug(f"Missing col: {col}")
                        # data.dfdata[col] = np.nan
                checked_rundata.append(data)
        else:
            checked_rundata = run_data

        return checked_rundata


Instruction:
Explain what the following Python 3 code does
def _iterdump(self, file_name, headers=None):
        if headers is None:
            headers = ["Discharge_Capacity", "Charge_Capacity"]

        step_txt = self.headers_normal['step_index_txt']
        point_txt = self.headers_normal['data_point_txt']
        cycle_txt = self.headers_normal['cycle_index_txt']

        self.logger.debug("iterating through file: %s" % file_name)
        if not os.path.isfile(file_name):
            print("Missing file_\n   %s" % file_name)

        filesize = os.path.getsize(file_name)
        hfilesize = humanize_bytes(filesize)
        txt = "Filesize: %i (%s)" % (filesize, hfilesize)
        self.logger.info(txt)

        table_name_global = TABLE_NAMES["global"]
        table_name_stats = TABLE_NAMES["statistic"]
        table_name_normal = TABLE_NAMES["normal"]

        # creating temporary file and connection

        temp_dir = tempfile.gettempdir()
        temp_filename = os.path.join(temp_dir, os.path.basename(file_name))
        shutil.copy2(file_name, temp_dir)
        constr = self.__get_res_connector(temp_filename)

        if use_ado:
            conn = dbloader.connect(constr)
        else:
            conn = dbloader.connect(constr, autocommit=True)

        self.logger.debug("tmp file: %s" % temp_filename)
        self.logger.debug("constr str: %s" % constr)

        # --------- read global-data ------------------------------------
        self.logger.debug("reading global data table")
        sql = "select * from %s" % table_name_global
        global_data_df = pd.read_sql_query(sql, conn)
        # col_names = list(global_data_df.columns.values)
        self.logger.debug("sql statement: %s" % sql)

        tests = global_data_df[self.headers_normal['test_id_txt']]
        number_of_sets = len(tests)
        self.logger.debug("number of datasets: %i" % number_of_sets)
        self.logger.debug("only selecting first test")
        test_no = 0
        self.logger.debug("setting data for test number %i" % test_no)
        loaded_from = file_name
        # fid = FileID(file_name)
        start_datetime = global_data_df[self.headers_global['start_datetime_txt']][test_no]
        test_ID = int(global_data_df[self.headers_normal['test_id_txt']][test_no])  # OBS
        test_name = global_data_df[self.headers_global['test_name_txt']][test_no]

        # --------- read raw-data (normal-data) -------------------------
        self.logger.debug("reading raw-data")

        columns = ["Data_Point", "Step_Index", "Cycle_Index"]
        columns.extend(headers)
        columns_txt = ", ".join(["%s"] * len(columns)) % tuple(columns)

        sql_1 = "select %s " % columns_txt
        sql_2 = "from %s " % table_name_normal
        sql_3 = "where %s=%s " % (self.headers_normal['test_id_txt'], test_ID)
        sql_5 = "order by %s" % self.headers_normal['data_point_txt']
        import time
        info_list = []
        info_header = ["cycle", "row_count", "start_point", "end_point"]
        info_header.extend(headers)
        self.logger.info(" ".join(info_header))
        self.logger.info("-------------------------------------------------")

        for cycle_number in range(1, 2000):
            t1 = time.time()
            self.logger.debug("picking cycle %i" % cycle_number)
            sql_4 = "AND %s=%i " % (cycle_txt, cycle_number)
            sql = sql_1 + sql_2 + sql_3 + sql_4 + sql_5
            self.logger.debug("sql statement: %s" % sql)
            normal_df = pd.read_sql_query(sql, conn)
            t2 = time.time()
            dt = t2 - t1
            self.logger.debug("time: %f" % dt)
            if normal_df.empty:
                self.logger.debug("reached the end")
                break
            row_count, _ = normal_df.shape
            start_point = normal_df[point_txt].min()
            end_point = normal_df[point_txt].max()
            last = normal_df.iloc[-1, :]

            step_list = [cycle_number, row_count, start_point, end_point]
            step_list.extend([last[x] for x in headers])
            info_list.append(step_list)

        self._clean_up_loadres(None, conn, temp_filename)
        info_dict = pd.DataFrame(info_list, columns=info_header)
        return info_dict

Response: Function for dumping values from a file.


Instruction:
Write a Python 3 function for
loading data from arbin. res files.

Response:
def loader(self, file_name, bad_steps=None, **kwargs):
        """Loads data from arbin .res files.

        Args:
            file_name (str): path to .res file.
            bad_steps (list of tuples): (c, s) tuples of steps s (in cycle c) to skip loading.

        Returns:
            new_tests (list of data objects)
        """
        # TODO: @jepe - insert kwargs - current chunk, only normal data, etc

        new_tests = []
        if not os.path.isfile(file_name):
            self.logger.info("Missing file_\n   %s" % file_name)
            return None

        self.logger.debug("in loader")
        self.logger.debug("filename: %s" % file_name)

        filesize = os.path.getsize(file_name)
        hfilesize = humanize_bytes(filesize)
        txt = "Filesize: %i (%s)" % (filesize, hfilesize)
        self.logger.debug(txt)
        if filesize > prms.Instruments["max_res_filesize"] and not prms.Reader["load_only_summary"]:
            error_message = "\nERROR (loader):\n"
            error_message += "%s > %s - File is too big!\n" % (
                hfilesize, humanize_bytes(prms.Instruments["max_res_filesize"]))
            error_message += "(edit prms.Instruments['max_res_filesize'])\n"
            print(error_message)
            return None

        table_name_global = TABLE_NAMES["global"]
        table_name_stats = TABLE_NAMES["statistic"]
        table_name_normal = TABLE_NAMES["normal"]

        # creating temporary file and connection

        temp_dir = tempfile.gettempdir()
        temp_filename = os.path.join(temp_dir, os.path.basename(file_name))
        shutil.copy2(file_name, temp_dir)
        self.logger.debug("tmp file: %s" % temp_filename)

        use_mdbtools = False
        if use_subprocess:
            use_mdbtools = True
        if is_posix:
            use_mdbtools = True

        # windows with same python bit as windows bit (the ideal case)
        if not use_mdbtools:
            constr = self.__get_res_connector(temp_filename)

            if use_ado:
                conn = dbloader.connect(constr)
            else:
                conn = dbloader.connect(constr, autocommit=True)
            self.logger.debug("constr str: %s" % constr)

            self.logger.debug("reading global data table")
            sql = "select * from %s" % table_name_global
            self.logger.debug("sql statement: %s" % sql)
            global_data_df = pd.read_sql_query(sql, conn)
            # col_names = list(global_data_df.columns.values)

        else:
            import subprocess
            if is_posix:
                if is_macos:
                    self.logger.debug("\nMAC OSX USING MDBTOOLS")
                else:
                    self.logger.debug("\nPOSIX USING MDBTOOLS")
            else:
                self.logger.debug("\nWINDOWS USING MDBTOOLS-WIN")

            # creating tmp-filenames
            temp_csv_filename_global = os.path.join(temp_dir, "global_tmp.csv")
            temp_csv_filename_normal = os.path.join(temp_dir, "normal_tmp.csv")
            temp_csv_filename_stats = os.path.join(temp_dir, "stats_tmp.csv")

            # making the cmds
            mdb_prms = [(table_name_global, temp_csv_filename_global), (table_name_normal, temp_csv_filename_normal),
                        (table_name_stats, temp_csv_filename_stats)]

            # executing cmds
            for table_name, tmp_file in mdb_prms:
                with open(tmp_file, "w") as f:
                    subprocess.call([sub_process_path, temp_filename, table_name], stdout=f)
                    self.logger.debug(f"ran mdb-export {str(f)} {table_name}")

            # use pandas to load in the data
            global_data_df = pd.read_csv(temp_csv_filename_global)

        tests = global_data_df[self.headers_normal['test_id_txt']]  # OBS
        number_of_sets = len(tests)
        self.logger.debug("number of datasets: %i" % number_of_sets)

        for counter, test_no in enumerate(range(number_of_sets)):
            if counter > 0:
                self.logger.warning("***MULTITEST-FILE (not recommended)")
                if not ALLOW_MULTI_TEST_FILE:
                    break
            data = DataSet()
            data.test_no = test_no
            data.loaded_from = file_name
            fid = FileID(file_name)
            # data.parent_filename = os.path.basename(file_name)# name of the .res file it is loaded from
            data.channel_index = int(global_data_df[self.headers_global['channel_index_txt']][test_no])
            data.channel_number = int(global_data_df[self.headers_global['channel_number_txt']][test_no])
            data.creator = global_data_df[self.headers_global['creator_txt']][test_no]
            data.item_ID = global_data_df[self.headers_global['item_id_txt']][test_no]
            data.schedule_file_name = global_data_df[self.headers_global['schedule_file_name_txt']][test_no]
            data.start_datetime = global_data_df[self.headers_global['start_datetime_txt']][test_no]
            data.test_ID = int(global_data_df[self.headers_normal['test_id_txt']][test_no])  # OBS
            data.test_name = global_data_df[self.headers_global['test_name_txt']][test_no]
            data.raw_data_files.append(fid)

            self.logger.debug("reading raw-data")
            if not use_mdbtools:
                # --------- read raw-data (normal-data) -------------------------
                length_of_test, normal_df = self._load_res_normal_table(conn, data.test_ID, bad_steps)
                # --------- read stats-data (summary-data) ----------------------
                sql = "select * from %s where %s=%s order by %s" % (table_name_stats,
                                                                    self.headers_normal['test_id_txt'],
                                                                    data.test_ID,
                                                                    self.headers_normal['data_point_txt'])
                summary_df = pd.read_sql_query(sql, conn)
                if counter > number_of_sets:
                    self._clean_up_loadres(None, conn, temp_filename)
            else:
                normal_df = pd.read_csv(temp_csv_filename_normal)
                # filter on test ID
                normal_df = normal_df[normal_df[self.headers_normal['test_id_txt']] == data.test_ID]
                # sort on data point
                if prms._sort_if_subprocess:
                    normal_df = normal_df.sort_values(self.headers_normal['data_point_txt'])
                length_of_test = normal_df.shape[0]
                summary_df = pd.read_csv(temp_csv_filename_stats)
                # clean up
                for f in [temp_filename, temp_csv_filename_stats, temp_csv_filename_normal,
                          temp_csv_filename_global]:
                    if os.path.isfile(f):
                        try:
                            os.remove(f)
                        except WindowsError as e:
                            self.logger.warning(
                                f"could not remove tmp-file\n{f} {e}"
                            )

            if summary_df.empty and prms.Reader.use_cellpy_stat_file:
                txt = "\nCould not find any summary (stats-file)!"
                txt += "\n -> issue make_summary(use_cellpy_stat_file=False)"
                logging.debug(txt)
            # normal_df = normal_df.set_index("Data_Point")
            data.dfsummary = summary_df
            data.dfdata = normal_df
            data.raw_data_files_length.append(length_of_test)
            new_tests.append(data)
        return new_tests


Instruction:
How would you explain what the following Python 3 function does
def make_df_from_batch(batch_name, batch_col="b01", reader=None, reader_label=None):

    batch_name = batch_name
    batch_col = batch_col
    logger.debug(f"batch_name, batch_col: {batch_name}, {batch_col}")
    if reader is None:
        reader_obj = get_db_reader(reader_label)
        reader = reader_obj()

    srnos = reader.select_batch(batch_name, batch_col)
    logger.debug("srnos:" + str(srnos))
    info_dict = _create_info_dict(reader, srnos)
    info_df = pd.DataFrame(info_dict)
    info_df = info_df.sort_values(["groups", "filenames"])
    info_df = _make_unique_groups(info_df)
    info_df["labels"] = info_df["filenames"].apply(create_labels)
    info_df.set_index("filenames", inplace=True)

    return info_df

Response:
Create a pandas DataFrame with the info needed for load
            the runs.


Instruction:
Create a Python 3 function for
reading and saves cellpy data from the given pandas. DataFrame.

Response:
def read_and_save_data(info_df, raw_dir, sep=";", force_raw=False,
                       force_cellpy=False,
                       export_cycles=False, shifted_cycles=False,
                       export_raw=True,
                       export_ica=False, save=True, use_cellpy_stat_file=False,
                       parent_level="CellpyData",
                       last_cycle=None,
                       ):
    """Reads and saves cell data defined by the info-DataFrame.

    The function iterates through the ``info_df`` and loads data from the runs.
    It saves individual data for each run (if selected), as well as returns a
    list of ``cellpy`` summary DataFrames, a list of the indexes (one for each
    run; same as used as index in the ``info_df``), as well as a list with
    indexes of runs (cells) where an error was encountered during loading.

    Args:
        use_cellpy_stat_file: use the stat file to perform the calculations.
        info_df: pandas.DataFrame with information about the runs.
        raw_dir: path to location where you want to save raw data.
        sep: delimiter to use when exporting to csv.
        force_raw: load raw data even-though cellpy-file is up-to-date.
        force_cellpy: load cellpy files even-though cellpy-file is not
            up-to-date.
        export_cycles: set to True for exporting cycles to csv.
        shifted_cycles: set to True for exporting the cycles with a cumulated
            shift.
        export_raw: set to True for exporting raw data to csv.
        export_ica: set to True for calculating and exporting dQ/dV to csv.
        save: set to False to prevent saving a cellpy-file.
        parent_level: optional, should use "cellpydata" for older hdf5-files and
            default for newer ones.

    Returns: frames (list of cellpy summary DataFrames), keys (list of indexes),
        errors (list of indexes that encountered errors).
    """
    no_export = False
    do_export_dqdv = export_ica
    keys = []
    frames = []
    number_of_runs = len(info_df)
    counter = 0
    errors = []

    for indx, row in info_df.iterrows():
        counter += 1
        h_txt = "[" + counter * "|" + (number_of_runs - counter) * "." + "]"
        l_txt = "starting to process file # %i (index=%s)" % (counter, indx)
        logger.debug(l_txt)
        print(h_txt)
        if not row.raw_file_names and not force_cellpy:
            logger.info("File(s) not found!")
            logger.info(indx)
            logger.debug("File(s) not found for index=%s" % indx)
            errors.append(indx)
            continue
        else:
            logger.info(f"Processing {indx}")
        cell_data = cellreader.CellpyData()
        if not force_cellpy:
            logger.info("setting cycle mode (%s)..." % row.cell_type)
            cell_data.set_cycle_mode(row.cell_type)

        logger.info("loading cell")
        if not force_cellpy:
            logger.info("not forcing")
            try:
                cell_data.loadcell(raw_files=row.raw_file_names,
                                   cellpy_file=row.cellpy_file_names,
                                   mass=row.masses, summary_on_raw=True,
                                   force_raw=force_raw,
                                   use_cellpy_stat_file=use_cellpy_stat_file)
            except Exception as e:
                logger.debug('Failed to load: ' + str(e))
                errors.append("loadcell:" + str(indx))
                continue
        else:
            logger.info("forcing")
            try:
                cell_data.load(row.cellpy_file_names, parent_level=parent_level)
            except Exception as e:
                logger.info(f"Critical exception encountered {type(e)} "
                            "- skipping this file")
                logger.debug('Failed to load. Error-message: ' + str(e))
                errors.append("load:" + str(indx))
                continue

        if not cell_data.check():
            logger.info("...not loaded...")
            logger.debug("Did not pass check(). Could not load cell!")
            errors.append("check:" + str(indx))
            continue

        logger.info("...loaded successfully...")
        keys.append(indx)

        summary_tmp = cell_data.dataset.dfsummary
        logger.info("Trying to get summary_data")
        if summary_tmp is None:
            logger.info("No existing summary made - running make_summary")
            cell_data.make_summary(find_end_voltage=True, find_ir=True)

        if summary_tmp.index.name == b"Cycle_Index":
            logger.debug("Strange: 'Cycle_Index' is a byte-string")
            summary_tmp.index.name = 'Cycle_Index'

        if not summary_tmp.index.name == "Cycle_Index":
            logger.debug("Setting index to Cycle_Index")
            # check if it is a byte-string
            if b"Cycle_Index" in summary_tmp.columns:
                logger.debug("Seems to be a byte-string in the column-headers")
                summary_tmp.rename(columns={b"Cycle_Index": 'Cycle_Index'},
                                   inplace=True)
            summary_tmp.set_index("Cycle_Index", inplace=True)

        frames.append(summary_tmp)

        if save:
            if not row.fixed:
                logger.info("saving cell to %s" % row.cellpy_file_names)
                cell_data.ensure_step_table = True
                cell_data.save(row.cellpy_file_names)
            else:
                logger.debug("saving cell skipped (set to 'fixed' in info_df)")

        if no_export:
            continue

        if export_raw:
            logger.info("exporting csv")
            cell_data.to_csv(raw_dir, sep=sep, cycles=export_cycles,
                             shifted=shifted_cycles, raw=export_raw,
                             last_cycle=last_cycle)

        if do_export_dqdv:
            logger.info("exporting dqdv")
            try:
                export_dqdv(cell_data, savedir=raw_dir, sep=sep,
                            last_cycle=last_cycle)
            except Exception as e:
                logging.error("Could not make/export dq/dv data")
                logger.debug("Failed to make/export "
                             "dq/dv data (%s): %s" % (indx, str(e)))
                errors.append("ica:" + str(indx))

    if len(errors) > 0:
        logger.error("Finished with errors!")
        logger.debug(errors)
    else:
        logger.info("Finished")

    return frames, keys, errors


Instruction:
How would you code a function in Python 3 to
write the selected summaries to csv - files for the current node.

Response:
def save_summaries(frames, keys, selected_summaries, batch_dir, batch_name):
    """Writes the summaries to csv-files

    Args:
        frames: list of ``cellpy`` summary DataFrames
        keys: list of indexes (typically run-names) for the different runs
        selected_summaries: list defining which summary data to save
        batch_dir: directory to save to
        batch_name: the batch name (will be used for making the file-name(s))

    Returns: a pandas DataFrame with your selected summaries.

    """
    if not frames:
        logger.info("Could save summaries - no summaries to save!")
        logger.info("You have no frames - aborting")
        return None
    if not keys:
        logger.info("Could save summaries - no summaries to save!")
        logger.info("You have no keys - aborting")
        return None

    selected_summaries_dict = create_selected_summaries_dict(selected_summaries)
    summary_df = pd.concat(frames, keys=keys, axis=1)
    # saving the selected summaries
    for key, value in selected_summaries_dict.items():
        _summary_file_name = os.path.join(batch_dir, "summary_%s_%s.csv" % (
        key, batch_name))
        _summary_df = summary_df.iloc[:,
                      summary_df.columns.get_level_values(1) == value]
        # include function to tweak headers here (need to learn MultiIndex)
        _header = _summary_df.columns
        _summary_df.to_csv(_summary_file_name, sep=";")
        logger.info(
            "saved summary (%s) to:\n       %s" % (key, _summary_file_name))
    logger.info("finished saving summaries")
    return summary_df


Instruction:
How would you explain what the following Python 3 function does
def pick_summary_data(key, summary_df, selected_summaries):

    selected_summaries_dict = create_selected_summaries_dict(selected_summaries)
    value = selected_summaries_dict[key]
    return summary_df.iloc[:, summary_df.columns.get_level_values(1) == value]

Response: picks the selected pandas. DataFrame


Instruction:
Can you implement a function in Python 3 that
creates a figure of the selected df - data in the given axes and returns a list of lines that match up the selected cell.

Response:
def plot_summary_data(ax, df, info_df, color_list, symbol_list, is_charge=False,
                      plot_style=None):
    """creates a plot of the selected df-data in the given axes.

    Typical usage:
        standard_fig, (ce_ax, cap_ax, ir_ax) = plt.subplots(nrows=3, ncols=1,
                                                            sharex=True)
        list_of_lines, plot_style = plot_summary_data(ce_ax, ce_df,
                                                      info_df=info_df,
                                                      color_list=color_list,
                                                      symbol_list=symbol_list,
                                                      is_charge=False,
                                                      plot_style=plot_style)

        the ce_df is a pandas.DataFrame with ce-values for all your selected
        cells. the color_list and the symbol_list are both list with colors and
        symbols to use when plotting to ensure that if you have several subplots
        (axes), then the lines and symbols match up for each given cell.

    Args:
        ax: the matplotlib axes to plot on
        df: DataFrame with the data to plot
        info_df: DataFrame with info for the data
        color_list: List of colors to use
        symbol_list: List of symbols to use
        is_charge: plots open symbols if True
        plot_style: selected style of the plot

    Returns: list of the matplotlib lines (convenient to have if you are adding
        a custom legend) the plot style (dictionary with matplotlib plotstyles)

    """

    logger.debug("trying to plot summary data")
    if plot_style is None:
        logger.debug("no plot_style given, using default")
        plot_style = DEFAULT_PLOT_STYLE
    else:
        logger.debug("plot_style given")

    list_of_lines = list()

    for datacol in df.columns:
        group = info_df.get_value(datacol[0], "groups")
        sub_group = info_df.get_value(datacol[0], "sub_groups")
        color = color_list[group - 1]
        marker = symbol_list[sub_group - 1]
        plot_style["marker"] = marker
        plot_style["markeredgecolor"] = color
        plot_style["color"] = color
        plot_style["markerfacecolor"] = 'none'
        logger.debug("selecting color for group: " + str(color))

        if not is_charge:
            plot_style["markerfacecolor"] = color
        lines = ax.plot(df[datacol], **plot_style)
        list_of_lines.extend(lines)

    return list_of_lines, plot_style