Instruction:
Explain what the following Python 3 code does
def resolve_blocks(template, context):
    '''
    Return a BlockContext instance of all the {% block %} tags in the template.

    If template is a string, it will be resolved through get_template
    '''
    try:
        blocks = context.render_context[BLOCK_CONTEXT_KEY]
    except KeyError:
        blocks = context.render_context[BLOCK_CONTEXT_KEY] = BlockContext()

    # If it's just the name, resolve into template
    if isinstance(template, six.string_types):
        template = get_template(template)

    # For Django 1.8 compatibility
    template = getattr(template, 'template', template)

    # Add this templates blocks as the first
    local_blocks = {
        block.name: block
        for block in template.nodelist.get_nodes_by_type(BlockNode)
    }
    blocks.add_blocks(local_blocks)

    # Do we extend a parent template?
    extends = template.nodelist.get_nodes_by_type(ExtendsNode)
    if extends:
        # Can only have one extends in a template
        extends_node = extends[0]

        # Get the parent, and recurse
        parent_template = extends_node.get_parent(context)
        resolve_blocks(parent_template, context)

    return blocks

Response: Resolves all the blocks in the template.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_widget_name(widget):
    '''
    Parse a alias:block_name string into separate parts.
    '''
    try:
        alias, block_name = widget.split(':', 1)
    except ValueError:
        raise template.TemplateSyntaxError('widget name must be "alias:block_name" - %s' % widget)

    return alias, block_name

Response: Parse a widget name into a alias and block name.


Instruction:
Write a Python 3 function that can
find the first matching block in the current block_context

Response:
def find_block(context, *names):
    '''
    Find the first matching block in the current block_context
    '''
    block_set = context.render_context[BLOCK_CONTEXT_KEY]
    for name in names:
        block = block_set.get_block(name)
        if block is not None:
            return block

    raise template.TemplateSyntaxError('No widget found for: %r' % (names,))


Instruction:
Make a summary of the following Python 3 code
def load_widgets(context, **kwargs):
    '''
    Load a series of widget libraries.
    '''
    _soft = kwargs.pop('_soft', False)

    try:
        widgets = context.render_context[WIDGET_CONTEXT_KEY]
    except KeyError:
        widgets = context.render_context[WIDGET_CONTEXT_KEY] = {}

    for alias, template_name in kwargs.items():
        if _soft and alias in widgets:
            continue

        with context.render_context.push({BLOCK_CONTEXT_KEY: BlockContext()}):
            blocks = resolve_blocks(template_name, context)
            widgets[alias] = blocks

    return ''

Response: Load a series of widget libraries.


Instruction:
Can you tell what is the following Python 3 function doing
def auto_widget(field):
    '''Return a list of widget names for the provided field.'''
    # Auto-detect
    info = {
        'widget': field.field.widget.__class__.__name__,
        'field': field.field.__class__.__name__,
        'name': field.name,
    }

    return [
        fmt.format(**info)
        for fmt in (
            '{field}_{widget}_{name}',
            '{field}_{name}',
            '{widget}_{name}',
            '{field}_{widget}',
            '{name}',
            '{widget}',
            '{field}',
        )
    ]

Response: Return a list of widget names for the provided field.


Instruction:
Write a Python 3 script for
allowing reuse of a block within a template.

Response:
def reuse(context, block_list, **kwargs):
    '''
    Allow reuse of a block within a template.

    {% reuse '_myblock' foo=bar %}

    If passed a list of block names, will use the first that matches:

    {% reuse list_of_block_names .... %}
    '''
    try:
        block_context = context.render_context[BLOCK_CONTEXT_KEY]
    except KeyError:
        block_context = BlockContext()

    if not isinstance(block_list, (list, tuple)):
        block_list = [block_list]

    for block in block_list:
        block = block_context.get_block(block)
        if block:
            break
    else:
        return ''

    with context.push(kwargs):
        return block.render(context)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def display(self):
        if not self.is_group():
            return self._display
        return ((force_text(k), v) for k, v in self._display)

Response: Return a list of key - value pairs that can be used to display the current value of the user.


Instruction:
Create a Python 3 function for
listing the items in the cache with the given key.

Response:
def _list_key(self, key):
        """
        boilerplate
        """
        ret = []
        for msg_json in self.client.lrange(key, 0, -1):
            ret.append(self._fromJSON(msg_json))
        return ret


Instruction:
How would you explain what the following Python 3 function does
def create_message(self, level, msg_text, extra_tags='', date=None, url=None):
        if not date:
            now = timezone.now()
        else:
            now = date
        r = now.isoformat()
        if now.microsecond:
            r = r[:23] + r[26:]
        if r.endswith('+00:00'):
            r = r[:-6] + 'Z'

        fingerprint = r + msg_text

        msg_id = hashlib.sha256(fingerprint.encode('ascii', 'ignore')).hexdigest()
        return Message(id=msg_id, message=msg_text, level=level, tags=extra_tags, date=r, url=url)

Response: Create a new message object.


Instruction:
Can you implement a function in Python 3 that
sends a message to a list of users without passing through django. contrib. messages

Response:
def add_message_for(users, level, message_text, extra_tags='', date=None, url=None, fail_silently=False):
    """
    Send a message to a list of users without passing through `django.contrib.messages`

    :param users: an iterable containing the recipients of the messages
    :param level: message level
    :param message_text: the string containing the message
    :param extra_tags: like the Django api, a string containing extra tags for the message
    :param date: a date, different than the default timezone.now
    :param url: an optional url
    :param fail_silently: not used at the moment
    """
    BackendClass = stored_messages_settings.STORAGE_BACKEND
    backend = BackendClass()
    m = backend.create_message(level, message_text, extra_tags, date, url)
    backend.archive_store(users, m)
    backend.inbox_store(users, m)


Instruction:
Write a Python 3 script for
sending a message to all users aka broadcast.

Response:
def broadcast_message(level, message_text, extra_tags='', date=None, url=None, fail_silently=False):
    """
    Send a message to all users aka broadcast.

    :param level: message level
    :param message_text: the string containing the message
    :param extra_tags: like the Django api, a string containing extra tags for the message
    :param date: a date, different than the default timezone.now
    :param url: an optional url
    :param fail_silently: not used at the moment
    """
    from django.contrib.auth import get_user_model
    users = get_user_model().objects.all()
    add_message_for(users, level, message_text, extra_tags=extra_tags, date=date, url=url, fail_silently=fail_silently)


Instruction:
Given the following Python 3 function, write the documentation
def mark_read(user, message):
    BackendClass = stored_messages_settings.STORAGE_BACKEND
    backend = BackendClass()
    backend.inbox_delete(user, message)

Response: Mark a message as read for a user.


Instruction:
Can you write a function in Python 3 where it
marks all message instances for a user as read.

Response:
def mark_all_read(user):
    """
    Mark all message instances for a user as read.

    :param user: user instance for the recipient
    """
    BackendClass = stored_messages_settings.STORAGE_BACKEND
    backend = BackendClass()
    backend.inbox_purge(user)


Instruction:
Implement a Python 3 function for
marking all messages as read

Response:
def mark_all_read(request):
    """
    Mark all messages as read (i.e. delete from inbox) for current logged in user
    """
    from .settings import stored_messages_settings
    backend = stored_messages_settings.STORAGE_BACKEND()
    backend.inbox_purge(request.user)
    return Response({"message": "All messages read"})


Instruction:
Create a Python 3 function to
mark the message as read

Response:
def read(self, request, pk=None):
        """
        Mark the message as read (i.e. delete from inbox)
        """
        from .settings import stored_messages_settings
        backend = stored_messages_settings.STORAGE_BACKEND()

        try:
            backend.inbox_delete(request.user, pk)
        except MessageDoesNotExist as e:
            return Response(e.message, status='404')

        return Response({'status': 'message marked as read'})


Instruction:
Write a Python 3 script for
returning a list of unread stored messages for the current user

Response:
def stored_messages_list(context, num_elements=10):
    """
    Renders a list of unread stored messages for the current user
    """
    if "user" in context:
        user = context["user"]
        if user.is_authenticated():
            qs = Inbox.objects.select_related("message").filter(user=user)
            return {
                "messages": qs[:num_elements],
                "count": qs.count(),
            }


Instruction:
Can you generate the documentation for the following Python 3 function
def stored_messages_count(context):
    if "user" in context:
        user = context["user"]
        if user.is_authenticated():
            return Inbox.objects.select_related("message").filter(user=user).count()

Response: Returns a list of unread stored messages for the current user


Instruction:
Explain what the following Python 3 code does
def stored_messages_archive(context, num_elements=10):
    if "user" in context:
        user = context["user"]
        if user.is_authenticated():
            qs = MessageArchive.objects.select_related("message").filter(user=user)
            return {
                "messages": qs[:num_elements],
                "count": qs.count(),
            }

Response: Returns a list of archived messages for the current user


Instruction:
Can you generate the documentation for the following Python 3 function
def _get(self, *args, **kwargs):
        messages, all_retrieved = super(StorageMixin, self)._get(*args, **kwargs)
        if self.user.is_authenticated():
            inbox_messages = self.backend.inbox_list(self.user)
        else:
            inbox_messages = []

        return messages + inbox_messages, all_retrieved

Response:
Retrieve unread messages for current user both from the inbox and
        from other storages


Instruction:
Given the following Python 3 function, write the documentation
def add(self, level, message, extra_tags=''):
        if not message:
            return
        # Check that the message level is not less than the recording level.
        level = int(level)
        if level < self.level:
            return
        # Check if the message doesn't have a level that needs to be persisted
        if level not in stored_messages_settings.STORE_LEVELS or self.user.is_anonymous():
            return super(StorageMixin, self).add(level, message, extra_tags)

        self.added_new = True
        m = self.backend.create_message(level, message, extra_tags)
        self.backend.archive_store([self.user], m)
        self._queued_messages.append(m)

Response: Add a new message to the database.


Instruction:
Create a Python 3 function for
storing the given messages in the database.

Response:
def _store(self, messages, response, *args, **kwargs):
        """
        persistent messages are already in the database inside the 'archive',
        so we can say they're already "stored".
        Here we put them in the inbox, or remove from the inbox in case the
        messages were iterated.

        messages contains only new msgs if self.used==True
        else contains both new and unread messages
        """
        contrib_messages = []
        if self.user.is_authenticated():
            if not messages:
                # erase inbox
                self.backend.inbox_purge(self.user)
            else:
                for m in messages:
                    try:
                        self.backend.inbox_store([self.user], m)
                    except MessageTypeNotSupported:
                        contrib_messages.append(m)

        super(StorageMixin, self)._store(contrib_messages, response, *args, **kwargs)


Instruction:
Make a summary of the following Python 3 code
def _prepare_messages(self, messages):
        for message in messages:
            if not self.backend.can_handle(message):
                message._prepare()

Response:
Prepare a list of messages for storage
       .


Instruction:
Write a Python 3 script to
initialize a base logger

Response:
def init(base_level=DEFAULT_BASE_LOGGING_LEVEL,
         verbose_level=DEFAULT_VERBOSE_LOGGING_LEVEL,
         logging_config=None):
    """initializes a base logger

    you can use this to init a logger in any of your files.
    this will use config.py's LOGGER param and logging.dictConfig to configure
    the logger for you.

    :param int|logging.LEVEL base_level: desired base logging level
    :param int|logging.LEVEL verbose_level: desired verbose logging level
    :param dict logging_dict: dictConfig based configuration.
     used to override the default configuration from config.py
    :rtype: `python logger`
    """
    if logging_config is None:
        logging_config = {}
    logging_config = logging_config or LOGGER
    # TODO: (IMPRV) only perform file related actions if file handler is
    # TODO: (IMPRV) defined.

    log_file = LOGGER['handlers']['file']['filename']
    log_dir = os.path.dirname(os.path.expanduser(log_file))
    if os.path.isfile(log_dir):
        sys.exit('file {0} exists - log directory cannot be created '
                 'there. please remove the file and try again.'
                 .format(log_dir))
    try:
        if not os.path.exists(log_dir) and not len(log_dir) == 0:
            os.makedirs(log_dir)
        dictconfig.dictConfig(logging_config)
        lgr = logging.getLogger('user')
        lgr.setLevel(base_level)
        return lgr
    except ValueError as e:
        sys.exit('could not initialize logger.'
                 ' verify your logger config'
                 ' and permissions to write to {0} ({1})'
                 .format(log_file, e))


Instruction:
Create a Python 3 function for
defaulting converter for the cfg:// protocol.

Response:
def cfg_convert(self, value):
        """Default converter for the cfg:// protocol."""
        rest = value
        m = self.WORD_PATTERN.match(rest)
        if m is None:
            raise ValueError("Unable to convert %r" % value)
        else:
            rest = rest[m.end():]
            d = self.config[m.groups()[0]]
            # print d, rest
            while rest:
                m = self.DOT_PATTERN.match(rest)
                if m:
                    d = d[m.groups()[0]]
                else:
                    m = self.INDEX_PATTERN.match(rest)
                    if m:
                        idx = m.groups()[0]
                        if not self.DIGIT_PATTERN.match(idx):
                            d = d[idx]
                        else:
                            try:
                                n = int(idx)
                                d = d[n]
                            except TypeError:
                                d = d[idx]
                if m:
                    rest = rest[m.end():]
                else:
                    raise ValueError('Unable to convert '
                                     '%r at %r' % (value, rest))
        # rest should be empty
        return d


Instruction:
Can you generate a brief explanation for the following Python 3 code
def configure_custom(self, config):
        c = config.pop('()')
        if not hasattr(c, '__call__') and \
                hasattr(types, 'ClassType') and isinstance(c, types.ClassType):
            c = self.resolve(c)
        props = config.pop('.', None)
        # Check for valid identifiers
        kwargs = dict((k, config[k]) for k in config if valid_ident(k))
        result = c(**kwargs)
        if props:
            for name, value in props.items():
                setattr(result, name, value)
        return result

Response: Configure an object with a user - supplied factory.


Instruction:
Implement a Python 3 function for
setting the global verbosity level for console and the jocker_lgr logger.

Response:
def _set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the jocker_lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output
    # TODO: (IMPRV) only raise exceptions in verbose mode
    verbose_output = is_verbose_output
    if verbose_output:
        jocker_lgr.setLevel(logging.DEBUG)
    else:
        jocker_lgr.setLevel(logging.INFO)


Instruction:
How would you explain what the following Python 3 function does
def _import_config(config_file):
    # get config file path
    jocker_lgr.debug('config file is: {0}'.format(config_file))
    # append to path for importing
    try:
        jocker_lgr.debug('importing config...')
        with open(config_file, 'r') as c:
            return yaml.safe_load(c.read())
    except IOError as ex:
        jocker_lgr.error(str(ex))
        raise RuntimeError('cannot access config file')
    except yaml.parser.ParserError as ex:
        jocker_lgr.error('invalid yaml file: {0}'.format(ex))
        raise RuntimeError('invalid yaml file')

Response: returns a configuration object from a config file


Instruction:
Implement a Python 3 function for
generating a Dockerfile and builds an image and pushes it to DockerHub.

Response:
def execute(varsfile, templatefile, outputfile=None, configfile=None,
            dryrun=False, build=False, push=False, verbose=False):
    """generates a Dockerfile, builds an image and pushes it to DockerHub

    A `Dockerfile` will be generated by Jinja2 according to the `varsfile`
    imported. If build is true, an image will be generated from the
    `outputfile` which is the generated Dockerfile and committed to the
    image:tag string supplied to `build`.
    If push is true, a build will be triggered and the produced image
    will be pushed to DockerHub upon completion.

    :param string varsfile: path to file with variables.
    :param string templatefile: path to template file to use.
    :param string outputfile: path to output Dockerfile.
    :param string configfile: path to yaml file with docker-py config.
    :param bool dryrun: mock run.
    :param build: False or the image:tag to build to.
    :param push: False or the image:tag to build to. (triggers build)
    :param bool verbose: verbose output.
    """
    if dryrun and (build or push):
        jocker_lgr.error('dryrun requested, cannot build.')
        sys.exit(100)

    _set_global_verbosity_level(verbose)
    j = Jocker(varsfile, templatefile, outputfile, configfile, dryrun,
               build, push)
    formatted_text = j.generate()
    if dryrun:
        g = j.dryrun(formatted_text)
    if build or push:
        j.build_image()
    if push:
        j.push_image()
    if dryrun:
        return g


Instruction:
Implement a Python 3 function for
parsing the output of a dumb push process

Response:
def _parse_dumb_push_output(self, string):
        """since the push process outputs a single unicode string consisting of
        multiple JSON formatted "status" lines, we need to parse it so that it
        can be read as multiple strings.

        This will receive the string as an input, count curly braces and ignore
        any newlines. When the curly braces stack is 0, it will append the
        entire string it has read up until then to a list and so forth.

        :param string: the string to parse
        :rtype: list of JSON's
        """
        stack = 0
        json_list = []
        tmp_json = ''
        for char in string:
            if not char == '\r' and not char == '\n':
                tmp_json += char
            if char == '{':
                stack += 1
            elif char == '}':
                stack -= 1
            if stack == 0:
                if not len(tmp_json) == 0:
                    json_list.append(tmp_json)
                tmp_json = ''
        return json_list


Instruction:
Write a Python 3 script for
uploading an image file to Imgur

Response:
def upload_gif(gif):
    """Uploads an image file to Imgur"""

    client_id = os.environ.get('IMGUR_API_ID')
    client_secret = os.environ.get('IMGUR_API_SECRET')

    if client_id is None or client_secret is None:
        click.echo('Cannot upload - could not find IMGUR_API_ID or IMGUR_API_SECRET environment variables')
        return

    client = ImgurClient(client_id, client_secret)

    click.echo('Uploading file {}'.format(click.format_filename(gif)))

    response = client.upload_from_path(gif)

    click.echo('File uploaded - see your gif at {}'.format(response['link']))


Instruction:
Can you create a Python 3 function that
returns True if the IP address is in dotted decimal notation.

Response:
def is_dot(ip):
    """Return true if the IP address is in dotted decimal notation."""
    octets = str(ip).split('.')
    if len(octets) != 4:
        return False
    for i in octets:
        try:
            val = int(i)
        except ValueError:
            return False
        if val > 255 or val < 0:
            return False
    return True


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_bin(ip):
    try:
        ip = str(ip)
        if len(ip) != 32:
            return False
        dec = int(ip, 2)
    except (TypeError, ValueError):
        return False
    if dec > 4294967295 or dec < 0:
        return False
    return True

Response: Return True if the IP address is in binary notation.


Instruction:
Can you create a Python 3 function that
returns True if the IP address is in octal notation.

Response:
def is_oct(ip):
    """Return true if the IP address is in octal notation."""
    try:
        dec = int(str(ip), 8)
    except (TypeError, ValueError):
        return False
    if dec > 0o37777777777 or dec < 0:
        return False
    return True


Instruction:
Write a Python 3 script for
returning True if the IP address is in decimal notation.

Response:
def is_dec(ip):
    """Return true if the IP address is in decimal notation."""
    try:
        dec = int(str(ip))
    except ValueError:
        return False
    if dec > 4294967295 or dec < 0:
        return False
    return True


Instruction:
Can you tell what is the following Python 3 function doing
def _check_nm(nm, notation):
    # Convert to decimal, and check if it's in the list of valid netmasks.
    _NM_CHECK_FUNCT = {
        NM_DOT: _dot_to_dec,
        NM_HEX: _hex_to_dec,
        NM_BIN: _bin_to_dec,
        NM_OCT: _oct_to_dec,
        NM_DEC: _dec_to_dec_long}
    try:
        dec = _NM_CHECK_FUNCT[notation](nm, check=True)
    except ValueError:
        return False
    if dec in _NETMASKS_VALUES:
        return True
    return False

Response:
Function internally used to check if the given netmask
    is of the specified notation.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_bits_nm(nm):
    try:
        bits = int(str(nm))
    except ValueError:
        return False
    if bits > 32 or bits < 0:
        return False
    return True

Response: Return true if the netmask is in bits notatation.


Instruction:
Can you implement a function in Python 3 that
returns True if the netmask is in wildcard bits notatation.

Response:
def is_wildcard_nm(nm):
    """Return true if the netmask is in wildcard bits notatation."""
    try:
        dec = 0xFFFFFFFF - _dot_to_dec(nm, check=True)
    except ValueError:
        return False
    if dec in _NETMASKS_VALUES:
        return True
    return False


Instruction:
Can you tell what is the following Python 3 function doing
def _dot_to_dec(ip, check=True):
    if check and not is_dot(ip):
        raise ValueError('_dot_to_dec: invalid IP: "%s"' % ip)
    octets = str(ip).split('.')
    dec = 0
    dec |= int(octets[0]) << 24
    dec |= int(octets[1]) << 16
    dec |= int(octets[2]) << 8
    dec |= int(octets[3])
    return dec

Response: Dotted decimal notation to decimal conversion.


Instruction:
Can you tell what is the following Python 3 function doing
def _dec_to_dot(ip):
    first = int((ip >> 24) & 255)
    second = int((ip >> 16) & 255)
    third = int((ip >> 8) & 255)
    fourth = int(ip & 255)
    return '%d.%d.%d.%d' % (first, second, third, fourth)

Response: Decimal to dotted decimal notation conversion.


Instruction:
Can you generate the documentation for the following Python 3 function
def _BYTES_TO_BITS():
    the_table = 256*[None]
    bits_per_byte = list(range(7, -1, -1))
    for n in range(256):
        l = n
        bits = 8*[None]
        for i in bits_per_byte:
            bits[i] = '01'[n & 1]
            n >>= 1
        the_table[l] = ''.join(bits)
    return the_table

Response: Generate a table to convert a whole byte to binary.


Instruction:
How would you explain what the following Python 3 function does
def _dec_to_bin(ip):
    bits = []
    while ip:
        bits.append(_BYTES_TO_BITS[ip & 255])
        ip >>= 8
    bits.reverse()
    return ''.join(bits) or 32*'0'

Response: Decimal to binary conversion.


Instruction:
Here you have a function in Python 3, explain what it does
def _wildcard_to_dec(nm, check=False):
    if check and not is_wildcard_nm(nm):
        raise ValueError('_wildcard_to_dec: invalid netmask: "%s"' % nm)
    return 0xFFFFFFFF - _dot_to_dec(nm, check=False)

Response: Wildcard bits to decimal conversion.


Instruction:
How would you explain what the following Python 3 function does
def _is_notation(ip, notation, _isnm):
    notation_orig = notation
    notation = _get_notation(notation)
    if notation not in _CHECK_FUNCT_KEYS:
        raise ValueError('_is_notation: unkown notation: "%s"' % notation_orig)
    return _CHECK_FUNCT[notation][_isnm](ip)

Response: Internally used to check if an IP is in the given notation.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _detect(ip, _isnm):
    ip = str(ip)
    if len(ip) > 1:
        if ip[0:2] == '0x':
            if _CHECK_FUNCT[IP_HEX][_isnm](ip):
                return IP_HEX
        elif ip[0] == '0':
            if _CHECK_FUNCT[IP_OCT][_isnm](ip):
                return IP_OCT
    if _CHECK_FUNCT[IP_DOT][_isnm](ip):
        return IP_DOT
    elif _isnm and _CHECK_FUNCT[NM_BITS][_isnm](ip):
        return NM_BITS
    elif _CHECK_FUNCT[IP_DEC][_isnm](ip):
        return IP_DEC
    elif _isnm and _CHECK_FUNCT[NM_WILDCARD][_isnm](ip):
        return NM_WILDCARD
    elif _CHECK_FUNCT[IP_BIN][_isnm](ip):
        return IP_BIN
    return IP_UNKNOWN

Response:
Function internally used to detect the notation of the
    given IP or netmask.


Instruction:
Can you tell what is the following Python 3 function doing
def convert(ip, notation=IP_DOT, inotation=IP_UNKNOWN, check=True):
    return _convert(ip, notation, inotation, _check=check, _isnm=False)

Response: Convert a given IP address to another notation.


Instruction:
Can you create a Python 3 function that
converts a netmask to another notation.

Response:
def convert_nm(nm, notation=IP_DOT, inotation=IP_UNKNOWN, check=True):
    """Convert a netmask to another notation."""
    return _convert(nm, notation, inotation, _check=check, _isnm=True)


Instruction:
Write a Python 3 function that can
set the IP address and netmask.

Response:
def set(self, ip, notation=IP_UNKNOWN):
        """Set the IP address/netmask."""
        self._ip_dec = int(_convert(ip, notation=IP_DEC, inotation=notation,
                                    _check=True, _isnm=self._isnm))
        self._ip = _convert(self._ip_dec, notation=IP_DOT, inotation=IP_DEC,
                            _check=False, _isnm=self._isnm)


Instruction:
How would you implement a function in Python 3 that
returns the hexadecimal notation of the address or netmask.

Response:
def get_hex(self):
        """Return the hexadecimal notation of the address/netmask."""
        return _convert(self._ip_dec, notation=IP_HEX,
                        inotation=IP_DEC, _check=False, _isnm=self._isnm)


Instruction:
Can you implement a function in Python 3 that
returns the binary notation of the address and netmask.

Response:
def get_bin(self):
        """Return the binary notation of the address/netmask."""
        return _convert(self._ip_dec, notation=IP_BIN,
                        inotation=IP_DEC, _check=False, _isnm=self._isnm)


Instruction:
Write a Python 3 function for
returning the octal notation of the address or netmask.

Response:
def get_oct(self):
        """Return the octal notation of the address/netmask."""
        return _convert(self._ip_dec, notation=IP_OCT,
                        inotation=IP_DEC, _check=False, _isnm=self._isnm)


Instruction:
Can you write a function in Python 3 where it
prepares the item to be compared with this address / netmask.

Response:
def _cmp_prepare(self, other):
        """Prepare the item to be compared with this address/netmask."""
        if isinstance(other, self.__class__):
            return other._ip_dec
        elif isinstance(other, int):
            # NOTE: this hides the fact that "other" can be a non valid IP/nm.
            return other
        return self.__class__(other)._ip_dec


Instruction:
Write a Python 3 script for
summing two IP addresses.

Response:
def _add(self, other):
        """Sum two IP addresses."""
        if isinstance(other, self.__class__):
            sum_ = self._ip_dec + other._ip_dec
        elif isinstance(other, int):
            sum_ = self._ip_dec + other
        else:
            other = self.__class__(other)
            sum_ = self._ip_dec + other._ip_dec
        return sum_


Instruction:
Can you tell what is the following Python 3 function doing
def _sub(self, other):
        if isinstance(other, self.__class__):
            sub = self._ip_dec - other._ip_dec
        if isinstance(other, int):
            sub = self._ip_dec - other
        else:
            other = self.__class__(other)
            sub = self._ip_dec - other._ip_dec
        return sub

Response: Subtract two IP addresses.


Instruction:
Write a Python 3 function that can
return the bits notation of the netmask.

Response:
def get_bits(self):
        """Return the bits notation of the netmask."""
        return _convert(self._ip, notation=NM_BITS,
                        inotation=IP_DOT, _check=False, _isnm=self._isnm)


Instruction:
Make a summary of the following Python 3 code
def get_wildcard(self):
        return _convert(self._ip, notation=NM_WILDCARD,
                        inotation=IP_DOT, _check=False, _isnm=self._isnm)

Response: Return the wildcard bits notation of the netmask.


Instruction:
Here you have a function in Python 3, explain what it does
def set(self, ip, netmask=None):
        if isinstance(ip, str) and netmask is None:
            ipnm = ip.split('/')
            if len(ipnm) != 2:
                raise ValueError('set: invalid CIDR: "%s"' % ip)
            ip = ipnm[0]
            netmask = ipnm[1]
        if isinstance(ip, IPv4Address):
            self._ip = ip
        else:
            self._ip = IPv4Address(ip)
        if isinstance(netmask, IPv4NetMask):
            self._nm = netmask
        else:
            self._nm = IPv4NetMask(netmask)
        ipl = int(self._ip)
        nml = int(self._nm)
        base_add = ipl & nml
        self._ip_num = 0xFFFFFFFF - 1 - nml
        # NOTE: quite a mess.
        #      This's here to handle /32 (-1) and /31 (0) netmasks.
        if self._ip_num in (-1, 0):
            if self._ip_num == -1:
                self._ip_num = 1
            else:
                self._ip_num = 2
            self._net_ip = None
            self._bc_ip = None
            self._first_ip_dec = base_add
            self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)
            if self._ip_num == 1:
                last_ip_dec = self._first_ip_dec
            else:
                last_ip_dec = self._first_ip_dec + 1
            self._last_ip = IPv4Address(last_ip_dec, notation=IP_DEC)
            return
        self._net_ip = IPv4Address(base_add, notation=IP_DEC)
        self._bc_ip = IPv4Address(base_add + self._ip_num + 1, notation=IP_DEC)
        self._first_ip_dec = base_add + 1
        self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)
        self._last_ip = IPv4Address(base_add + self._ip_num, notation=IP_DEC)

Response: Set the IP address and netmask.


Instruction:
Implement a function in Python 3 to
change the current IP.

Response:
def set_ip(self, ip):
        """Change the current IP."""
        self.set(ip=ip, netmask=self._nm)


Instruction:
Can you generate the documentation for the following Python 3 function
def set_netmask(self, netmask):
        self.set(ip=self._ip, netmask=netmask)

Response: Change the netmask of the current log entry.


Instruction:
Implement a Python 3 function for
returning true if the given address in amongst the usable addresses and is contained in this one.

Response:
def is_valid_ip(self, ip):
        """Return true if the given address in amongst the usable addresses,
        or if the given CIDR is contained in this one."""
        if not isinstance(ip, (IPv4Address, CIDR)):
            if str(ip).find('/') == -1:
                ip = IPv4Address(ip)
            else:
                # Support for CIDR strings/objects, an idea of Nicola Novello.
                ip = CIDR(ip)
        if isinstance(ip, IPv4Address):
            if ip < self._first_ip or ip > self._last_ip:
                return False
        elif isinstance(ip, CIDR):
            # NOTE: manage /31 networks; 127.0.0.1/31 is considered to
            #       be included in 127.0.0.1/8.
            if ip._nm._ip_dec == 0xFFFFFFFE \
                    and self._nm._ip_dec != 0xFFFFFFFE:
                compare_to_first = self._net_ip._ip_dec
                compare_to_last = self._bc_ip._ip_dec
            else:
                compare_to_first = self._first_ip._ip_dec
                compare_to_last = self._last_ip._ip_dec
            if ip._first_ip._ip_dec < compare_to_first or \
                    ip._last_ip._ip_dec > compare_to_last:
                return False
        return True


Instruction:
Implement a Python 3 function for
uploading a file to S3 possibly using the multi - part uploader

Response:
async def upload_file(self, bucket, file, uploadpath=None, key=None,
                          ContentType=None, **kw):
        """Upload a file to S3 possibly using the multi-part uploader
        Return the key uploaded
        """
        is_filename = False

        if hasattr(file, 'read'):
            if hasattr(file, 'seek'):
                file.seek(0)
            file = file.read()
            size = len(file)
        elif key:
            size = len(file)
        else:
            is_filename = True
            size = os.stat(file).st_size
            key = os.path.basename(file)

        assert key, 'key not available'

        if not ContentType:
            ContentType, _ = mimetypes.guess_type(key)

        if uploadpath:
            if not uploadpath.endswith('/'):
                uploadpath = '%s/' % uploadpath
            key = '%s%s' % (uploadpath, key)

        params = dict(Bucket=bucket, Key=key)

        if not ContentType:
            ContentType = 'application/octet-stream'

        params['ContentType'] = ContentType

        if size > MULTI_PART_SIZE and is_filename:
            resp = await _multipart(self, file, params)
        elif is_filename:
            with open(file, 'rb') as fp:
                params['Body'] = fp.read()
            resp = await self.put_object(**params)
        else:
            params['Body'] = file
            resp = await self.put_object(**params)
        if 'Key' not in resp:
            resp['Key'] = key
        if 'Bucket' not in resp:
            resp['Bucket'] = bucket
        return resp


Instruction:
Make a summary of the following Python 3 code
async def copy_storage_object(self, source_bucket, source_key,
                                  bucket, key):
        """Copy a file from one bucket into another
        """
        info = await self.head_object(Bucket=source_bucket, Key=source_key)
        size = info['ContentLength']

        if size > MULTI_PART_SIZE:
            result = await _multipart_copy(self, source_bucket, source_key,
                                           bucket, key, size)
        else:
            result = await self.copy_object(
                Bucket=bucket, Key=key,
                CopySource=_source_string(source_bucket, source_key)
            )
        return result

Response: Copy a file from one bucket into another.


Instruction:
Here you have a function in Python 3, explain what it does
async def _upload_file(self, full_path):
        rel_path = os.path.relpath(full_path, self.folder)
        key = s3_key(os.path.join(self.key, rel_path))
        ct = self.content_types.get(key.split('.')[-1])
        with open(full_path, 'rb') as fp:
            file = fp.read()
        try:
            await self.botocore.upload_file(self.bucket, file, key=key,
                                            ContentType=ct)
        except Exception as exc:
            LOGGER.error('Could not upload "%s": %s', key, exc)
            self.failures[key] = self.all.pop(full_path)
            return
        size = self.all.pop(full_path)
        self.success[key] = size
        self.total_size += size
        percentage = 100*(1 - len(self.all)/self.total_files)
        message = '{0:.0f}% completed - uploaded "{1}" - {2}'.format(
            percentage, key, convert_bytes(size))
        LOGGER.info(message)

Response: Coroutine for uploading a single file to S3.


Instruction:
How would you code a function in Python 3 to
create a paginator for an operation.

Response:
def get_paginator(self, operation_name):
        """Create a paginator for an operation.
        :type operation_name: string
        :param operation_name: The operation name.  This is the same name
            as the method name on the client.  For example, if the
            method name is ``create_foo``, and you'd normally invoke the
            operation as ``client.create_foo(**kwargs)``, if the
            ``create_foo`` operation can be paginated, you can use the
            call ``client.get_paginator("create_foo")``.
        :raise OperationNotPageableError: Raised if the operation is not
            pageable.  You can use the ``client.can_paginate`` method to
            check if an operation is pageable.
        :rtype: L{botocore.paginate.Paginator}
        :return: A paginator object.
        """
        if not self.can_paginate(operation_name):
            raise OperationNotPageableError(operation_name=operation_name)
        else:
            actual_operation_name = self._PY_TO_OP_NAME[operation_name]
            # substitute iterator with async one
            Paginator.PAGE_ITERATOR_CLS = AsyncPageIterator
            paginator = Paginator(
                getattr(self, operation_name),
                self._cache['page_config'][actual_operation_name])
            return paginator


Instruction:
Implement a function in Python 3 to
trigger an event on this channel

Response:
async def trigger(self, event, data=None, socket_id=None):
        '''Trigger an ``event`` on this channel
        '''
        json_data = json.dumps(data, cls=self.pusher.encoder)
        query_string = self.signed_query(event, json_data, socket_id)
        signed_path = "%s?%s" % (self.path, query_string)
        pusher = self.pusher
        absolute_url = pusher.get_absolute_path(signed_path)
        response = await pusher.http.post(
            absolute_url, data=json_data,
            headers=[('Content-Type', 'application/json')])
        response.raise_for_status()
        return response.status_code == 202


Instruction:
Can you generate the documentation for the following Python 3 function
async def connect(self):
        '''Connect to a Pusher websocket
        '''
        if not self._consumer:
            waiter = self._waiter = asyncio.Future()
            try:
                address = self._websocket_host()
                self.logger.info('Connect to %s', address)
                self._consumer = await self.http.get(address)
                if self._consumer.status_code != 101:
                    raise PusherError("Could not connect to websocket")
            except Exception as exc:
                waiter.set_exception(exc)
                raise
            else:
                await waiter
        return self._consumer

Response: Connect to a Pusher websocket


Instruction:
Create a Python 3 function to
handle incoming messages from the pusher.

Response:
def on_message(self, websocket, message):
        '''Handle websocket incoming messages
        '''
        waiter = self._waiter
        self._waiter = None
        encoded = json.loads(message)
        event = encoded.get('event')
        channel = encoded.get('channel')
        data = json.loads(encoded.get('data'))
        try:
            if event == PUSHER_ERROR:
                raise PusherError(data['message'], data['code'])
            elif event == PUSHER_CONNECTION:
                self.socket_id = data.get('socket_id')
                self.logger.info('Succesfully connected on socket %s',
                                 self.socket_id)
                waiter.set_result(self.socket_id)
            elif event == PUSHER_SUBSCRIBED:
                self.logger.info('Succesfully subscribed to %s',
                                 encoded.get('channel'))
            elif channel:
                self[channel]._event(event, data)
        except Exception as exc:
            if waiter:
                waiter.set_exception(exc)
            else:
                self.logger.exception('pusher error')


Instruction:
Make a summary of the following Python 3 code
def urlsafe_nopadding_b64decode(data):
    '''URL safe Base64 decode without padding (=)'''

    padding = len(data) % 4
    if padding != 0:
        padding = 4 - padding
    padding = '=' * padding
    data = data + padding
    return urlsafe_b64decode(data)

Response: URL safe Base64 decode without padding


Instruction:
Write a Python 3 function for
decoding a limited set of HTML entities.

Response:
def decode_html_entities(html):
    """
    Decodes a limited set of HTML entities.
    """
    if not html:
        return html

    for entity, char in six.iteritems(html_entity_map):
        html = html.replace(entity, char)

    return html


Instruction:
Here you have a function in Python 3, explain what it does
def set_signature_passphrases(self, signature_passphrases):
        '''Set signature passphrases'''
        self.signature_passphrases = self._update_dict(signature_passphrases,
                                                       {}, replace_data=True)

Response: Set the signature passphrases for this resource.


Instruction:
Create a Python 3 function for
setting the set of encryption passphrases for this resource.

Response:
def set_encryption_passphrases(self, encryption_passphrases):
        '''Set encryption passphrases'''
        self.encryption_passphrases = self._update_dict(encryption_passphrases,
                                                        {}, replace_data=True)


Instruction:
How would you explain what the following Python 3 function does
def set_algorithms(self, signature=None, encryption=None,
                       serialization=None, compression=None):
        '''Set algorithms used for sealing. Defaults can not be overridden.'''

        self.signature_algorithms = \
            self._update_dict(signature, self.DEFAULT_SIGNATURE)

        self.encryption_algorithms = \
            self._update_dict(encryption, self.DEFAULT_ENCRYPTION)

        self.serialization_algorithms = \
            self._update_dict(serialization, self.DEFAULT_SERIALIZATION)

        self.compression_algorithms = \
            self._update_dict(compression, self.DEFAULT_COMPRESSION)

Response: Sets the algorithms used for sealing. Defaults can not be overridden.


Instruction:
Can you implement a function in Python 3 that
gets algorithms used for sealing

Response:
def get_algorithms(self):
        '''Get algorithms used for sealing'''

        return {
            'signature': self.signature_algorithms,
            'encryption': self.encryption_algorithms,
            'serialization': self.serialization_algorithms,
            'compression': self.compression_algorithms,
        }


Instruction:
Can you generate a brief explanation for the following Python 3 code
def verify_signature(self, data):
        '''Verify sealed data signature'''

        data = self._remove_magic(data)
        data = urlsafe_nopadding_b64decode(data)
        options = self._read_header(data)
        data = self._add_magic(data)
        self._unsign_data(data, options)

Response: Verify sealed data signature


Instruction:
Write a Python 3 script to
encode data with specific algorithm

Response:
def _encode(self, data, algorithm, key=None):
        '''Encode data with specific algorithm'''

        if algorithm['type'] == 'hmac':
            return data + self._hmac_generate(data, algorithm, key)
        elif algorithm['type'] == 'aes':
            return self._aes_encrypt(data, algorithm, key)
        elif algorithm['type'] == 'no-serialization':
            return data
        elif algorithm['type'] == 'json':
            return json.dumps(data)
        elif algorithm['type'] == 'no-compression':
            return data
        elif algorithm['type'] == 'gzip':
            return self._zlib_compress(data, algorithm)
        else:
            raise Exception('Algorithm not supported: %s' % algorithm['type'])


Instruction:
Explain what the following Python 3 code does
def _decode(self, data, algorithm, key=None):
        '''Decode data with specific algorithm'''

        if algorithm['type'] == 'hmac':
            verify_signature = data[-algorithm['hash_size']:]
            data = data[:-algorithm['hash_size']]
            signature = self._hmac_generate(data, algorithm, key)
            if not const_equal(verify_signature, signature):
                raise Exception('Invalid signature')
            return data
        elif algorithm['type'] == 'aes':
            return self._aes_decrypt(data, algorithm, key)
        elif algorithm['type'] == 'no-serialization':
            return data
        elif algorithm['type'] == 'json':
            return json.loads(data)
        elif algorithm['type'] == 'no-compression':
            return data
        elif algorithm['type'] == 'gzip':
            return self._zlib_decompress(data, algorithm)
        else:
            raise Exception('Algorithm not supported: %s' % algorithm['type'])

Response: Decode data with specific algorithm


Instruction:
How would you explain what the following Python 3 function does
def _sign_data(self, data, options):
        '''Add signature to data'''

        if options['signature_algorithm_id'] not in self.signature_algorithms:
            raise Exception('Unknown signature algorithm id: %d'
                            % options['signature_algorithm_id'])

        signature_algorithm = \
            self.signature_algorithms[options['signature_algorithm_id']]

        algorithm = self._get_algorithm_info(signature_algorithm)

        key_salt = get_random_bytes(algorithm['salt_size'])
        key = self._generate_key(options['signature_passphrase_id'],
                            self.signature_passphrases, key_salt, algorithm)

        data = self._encode(data, algorithm, key)

        return data + key_salt

Response: Add signature to data


Instruction:
Can you generate the documentation for the following Python 3 function
def _unsign_data(self, data, options):
        '''Verify and remove signature'''

        if options['signature_algorithm_id'] not in self.signature_algorithms:
            raise Exception('Unknown signature algorithm id: %d'
                            % options['signature_algorithm_id'])

        signature_algorithm = \
            self.signature_algorithms[options['signature_algorithm_id']]

        algorithm = self._get_algorithm_info(signature_algorithm)

        key_salt = ''
        if algorithm['salt_size']:
            key_salt = data[-algorithm['salt_size']:]
            data = data[:-algorithm['salt_size']]

        key = self._generate_key(options['signature_passphrase_id'],
                            self.signature_passphrases, key_salt, algorithm)

        data = self._decode(data, algorithm, key)

        return data

Response: Verify and remove signature


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _encrypt_data(self, data, options):
        '''Encrypt data'''

        if options['encryption_algorithm_id'] not in self.encryption_algorithms:
            raise Exception('Unknown encryption algorithm id: %d'
                            % options['encryption_algorithm_id'])

        encryption_algorithm = \
            self.encryption_algorithms[options['encryption_algorithm_id']]

        algorithm = self._get_algorithm_info(encryption_algorithm)

        key_salt = get_random_bytes(algorithm['salt_size'])
        key = self._generate_key(options['encryption_passphrase_id'],
                            self.encryption_passphrases, key_salt, algorithm)

        data = self._encode(data, algorithm, key)

        return data + key_salt

Response: Encrypt data with the key and passphrase


Instruction:
Can you tell what is the following Python 3 function doing
def _remove_magic(self, data):
        '''Verify and remove magic'''

        if not self.magic:
            return data

        magic_size = len(self.magic)
        magic = data[:magic_size]
        if magic != self.magic:
            raise Exception('Invalid magic')
        data = data[magic_size:]

        return data

Response: Verify and remove magic from data


Instruction:
Can you tell what is the following Python 3 function doing
def _add_header(self, data, options):
        '''Add header to data'''

        # pylint: disable=W0142

        version_info = self._get_version_info(options['version'])

        flags = options['flags']

        header_flags = dict(
            (i, str(int(j))) for i, j in options['flags'].iteritems())
        header_flags = ''.join(version_info['flags'](**header_flags))
        header_flags = int(header_flags, 2)
        options['flags'] = header_flags

        header = version_info['header']
        header = header(**options)
        header = pack(version_info['header_format'], *header)

        if 'timestamp' in flags and flags['timestamp']:
            timestamp = long(time())
            timestamp = pack(version_info['timestamp_format'], timestamp)
            header = header + timestamp

        return header + data

Response: Add header to data


Instruction:
How would you explain what the following Python 3 function does
def _read_header(self, data):
        '''Read header from data'''

        # pylint: disable=W0212

        version = self._read_version(data)
        version_info = self._get_version_info(version)
        header_data = data[:version_info['header_size']]
        header = version_info['header']
        header = header._make(
            unpack(version_info['header_format'], header_data))
        header = dict(header._asdict())

        flags = list("{0:0>8b}".format(header['flags']))
        flags = dict(version_info['flags']._make(flags)._asdict())
        flags = dict((i, bool(int(j))) for i, j in flags.iteritems())
        header['flags'] = flags

        timestamp = None
        if flags['timestamp']:
            ts_start = version_info['header_size']
            ts_end = ts_start + version_info['timestamp_size']
            timestamp_data = data[ts_start:ts_end]
            timestamp = unpack(
                version_info['timestamp_format'], timestamp_data)[0]
        header['info'] = {'timestamp': timestamp}

        return header

Response: Read the header from data


Instruction:
Can you implement a function in Python 3 that
removes the header from the data

Response:
def _remove_header(self, data, options):
        '''Remove header from data'''

        version_info = self._get_version_info(options['version'])
        header_size = version_info['header_size']

        if options['flags']['timestamp']:
            header_size += version_info['timestamp_size']

        data = data[header_size:]

        return data


Instruction:
Can you tell what is the following Python 3 function doing
def _read_version(self, data):
        '''Read header version from data'''

        version = ord(data[0])
        if version not in self.VERSIONS:
            raise Exception('Version not defined: %d' % version)
        return version

Response: Read the version from data


Instruction:
Make a summary of the following Python 3 code
def _generate_key(pass_id, passphrases, salt, algorithm):
        '''Generate and return PBKDF2 key'''

        if pass_id not in passphrases:
            raise Exception('Passphrase not defined for id: %d' % pass_id)

        passphrase = passphrases[pass_id]

        if len(passphrase) < 32:
            raise Exception('Passphrase less than 32 characters long')

        digestmod = EncryptedPickle._get_hashlib(algorithm['pbkdf2_algorithm'])

        encoder = PBKDF2(passphrase, salt,
                         iterations=algorithm['pbkdf2_iterations'],
                         digestmodule=digestmod)

        return encoder.read(algorithm['key_size'])

Response: Generate and return PBKDF2 key


Instruction:
Explain what the following Python 3 code does
def _update_dict(data, default_data, replace_data=False):
        '''Update algorithm definition type dictionaries'''

        if not data:
            data = default_data.copy()
            return data

        if not isinstance(data, dict):
            raise TypeError('Value not dict type')
        if len(data) > 255:
            raise ValueError('More than 255 values defined')
        for i in data.keys():
            if not isinstance(i, int):
                raise TypeError('Index not int type')
            if i < 0 or i > 255:
                raise ValueError('Index value out of range')

        if not replace_data:
            data.update(default_data)

        return data

Response: Update algorithm definition type dictionaries


Instruction:
Here you have a function in Python 3, explain what it does
def _get_hashlib(digestmode):
        '''Generate HMAC hash'''
        if digestmode == 'sha1':
            return SHA
        if digestmode == 'sha256':
            return SHA256
        elif digestmode  == 'sha384':
            return SHA384
        elif digestmode == 'sha512':
            return SHA512
        else:
            raise Exception('digestmode not supported: %s'
                            % digestmode)

Response: Generate HMAC hash from digestmode


Instruction:
Here you have a function in Python 3, explain what it does
def getTableOfContents(self):

        self.directory_size = self.getDirectorySize()
        if self.directory_size > 65536:
            self.directory_size += 2
            self.requestContentDirectory()

        # and find the offset from start of file where it can be found
        directory_start = unpack("i", self.raw_bytes[self.directory_end + 16: self.directory_end + 20])[0]

        # find the data in the raw_bytes
        self.raw_bytes = self.raw_bytes
        current_start = directory_start - self.start
        filestart = 0
        compressedsize = 0
        tableOfContents = []

        try:
            while True:
                # get file name size (n), extra len (m) and comm len (k)
                zip_n = unpack("H", self.raw_bytes[current_start + 28: current_start + 28 + 2])[0]
                zip_m = unpack("H", self.raw_bytes[current_start + 30: current_start + 30 + 2])[0]
                zip_k = unpack("H", self.raw_bytes[current_start + 32: current_start + 32 + 2])[0]

                filename = self.raw_bytes[current_start + 46: current_start + 46 + zip_n]

                # check if this is the index file
                filestart = unpack("I", self.raw_bytes[current_start + 42: current_start + 42 + 4])[0]
                compressedsize = unpack("I", self.raw_bytes[current_start + 20: current_start + 20 + 4])[0]
                uncompressedsize = unpack("I", self.raw_bytes[current_start + 24: current_start + 24 + 4])[0]
                tableItem = {
                    'filename': filename,
                    'compressedsize': compressedsize,
                    'uncompressedsize': uncompressedsize,
                    'filestart': filestart
                }
                tableOfContents.append(tableItem)

                # not this file, move along
                current_start = current_start + 46 + zip_n + zip_m + zip_k
        except:
            pass

        self.tableOfContents = tableOfContents
        return tableOfContents

Response: This function returns the contents of the internal tableOfContents list. It will raise an exception if the server does not support ranged requests.


Instruction:
How would you explain what the following Python 3 function does
def make_postcard(self, npix=300, shape=(1070, 1132), buffer_size=15):

        source = self.kic
        client = kplr.API()
        targ = client.target(source)

        channel = [targ.params['Channel_0'], targ.params['Channel_1'],
                    targ.params['Channel_2'], targ.params['Channel_3']]

        col = [targ.params['Column_0'], targ.params['Column_1'],
                    targ.params['Column_2'], targ.params['Column_3']]

        row = [targ.params['Row_0'], targ.params['Row_1'],
                    targ.params['Row_2'], targ.params['Row_3']]

        if None in row:
            raise ValueError('Star not on detector all quarters!')

        if None in col:
            raise ValueError('Star not on detector all quarters!')

        center = np.array([npix/2, npix/2])

        # If star close to edge, shift frame so that we have the full npix by npix
        # In this case, postcard will not be centered on target star
        if (np.min(col) < npix/2):
            jump = npix/2 - np.min(col) + buffer_size
            col += jump
            center[1] -= jump

        if (np.min(row) < npix/2):
            jump = npix/2 - np.min(row) + buffer_size
            row += jump
            center[0] -= jump

        if (np.max(row) > shape[0] - npix/2):
            jump = shape[0]-npix/2 - np.max(row) - buffer_size
            row += jump
            center[0] -= jump

        if (np.max(col) > shape[1] - npix/2):
            jump = shape[1]-npix/2 - np.max(col) - buffer_size
            col += jump
            center[1] -= jump

        fin_arr = np.zeros((len(self.times), npix, npix))

        for icount, iname in enumerate(self.obs_filenames):
            a = fits.open(self.ffi_dir+iname)

            quarter = a[0].header['quarter']

            if int(quarter) == 0:
                season = 3
            else:
                season = (int(quarter) - 2) % 4

            #season_arr[icount] = season
            img = a[channel[season]].data
            img -= np.median(img)

            ymin = int(max([int(row[season])-npix/2,0]))
            ymax = int(min([int(row[season])+npix/2,img.shape[0]]))
            xmin = int(max([int(col[season])-npix/2,0]))
            xmax = int(min([int(col[season])+npix/2,img.shape[1]]))

            pimg = img[ymin:ymax,xmin:xmax]
            fin_arr[icount,:,:] = pimg

        self.postcard = fin_arr
        self.integrated_postcard = np.sum(self.postcard, axis=0)
        self.center = center

Response: This function will make a postcard region around the target star.


Instruction:
Explain what the following Python 3 code does
def find_other_sources(self,  edge_lim = 0.015, min_val = 5000,
                          ntargets = 250, extend_region_size=3, remove_excess=4,
                          plot_flag = False, plot_window=15):
        """
        Identify apertures for all sources on the postcard, both for the
        target and potential reference stars

        Args:
            edge_lim: The initial limit for the creation of apertures. The aperture will be a region of
                contiguous pixels with flux values larger than the product of ``edge_lim`` and the brightest
                pixel value for this star, as long as that product is larger than ``min_val`` (default: ``0.015``)
            min_val: Threshold for the minimum flux value in the ``integrated_postcard`` for a pixel to be included
                in the default apertures (default: ``5000``)
            ntargets: The maximum number of potential reference stars to be included in the analysis (default: ``250``)
            extend_region_size: After the initial apertures are generated, they will be optionally extended an
                additional number of pixels following this flag. Safe practice for reasonable apertures is to
                leave ``min_val`` at a value well above the noise and then extend apertures via this flag until
                they are of suitable size (default: ``3``)
            remove_excess: Stars with apertures that touch will be combined into a single aperture.
                This is done by iterating through the starlist; this flag represents the number of times the
                list will be iterated through to delete redundant apertures (default: ``4``)
            plot_flag: If true, a series of diagnostic plots will appear while this function runs to observe
                apertures for the target star and other stars.
                (default: ``False``)
            plot_window: If ``plot_flag`` is ``True``, the size of the region to be plotted around the target star
                to show the drawn aperture for visualization purposes only (default: ``15``)

        """
        j,i = self.center

        region = self.integrated_postcard + 0.0
        if plot_flag == True:
            ff = plt.imshow(self.integrated_postcard, interpolation='nearest', cmap='gray',
                            vmax = np.percentile(region, 99.6))
            plt.colorbar(ff)
            plt.show()
        targets = np.zeros_like(self.integrated_postcard)
        sizeimg = np.shape(targets)[0]

        jj = j + 0
        ii = i + 0

        edge = edge_lim

        lim = max(min_val, self.integrated_postcard[int(j), int(i)]*edge)

        maxpt = np.percentile(self.integrated_postcard, 94)

        bin_img = (region > lim)
        lab_img, n_features = label(bin_img)

        key_targ = (lab_img == (lab_img[int(j), int(i)]))
        tot = np.sum(key_targ)

        targets[key_targ] = 1
        region[key_targ] = 0.0

        lim = np.zeros(ntargets)
        for peaks in range(1,ntargets):
            k = np.argmax(region)
            j,i = np.unravel_index(k, region.shape)
            lim[peaks] = max(maxpt, edge*region[j,i])

            bin_img = (region >= lim[peaks])
            lab_img, n_features = label(bin_img)

            key_targ = (lab_img == (lab_img[j,i]))
            targets[key_targ] = peaks + 1
            region[key_targ] = 0.0

        lab_img, n_features = label(targets)
        for i in range(1, ntargets+1):
            for j in range(extend_region_size):
                border= mh.labeled.border(targets, 0, i)

                targets[border*(region < (10)*lim[peaks])] = i
        for i in range(2, ntargets+1):
            for j in range(2, ntargets+1):
                if i != j:
                    border = mh.labeled.border(targets, i, j)
                    if np.sum(border) != 0:
                        targets[targets == j] = i

        targets = mh.labeled.remove_bordering(targets)
        for k in range(remove_excess):
            for i in range(ntargets):
                if np.sum(self.integrated_postcard[targets == i]) < 0.01:
                    targets[targets > i] -= 1

        self.targets = targets

        if plot_flag == True:
            plt.imshow(self.targets, interpolation='nearest')
            plt.show()

            plt.imshow(((targets == 1)*self.integrated_postcard + (targets == 1)*100000)
                       [jj-plot_window:jj+plot_window,ii-plot_window:ii+plot_window],
                       interpolation='nearest', cmap='gray', vmax=np.percentile(self.integrated_postcard, 99.6))
            plt.show()

            plt.imshow((np.ceil(targets/100.0)*self.integrated_postcard+np.ceil(targets/500.0)*3500000),
                       interpolation='nearest', cmap='gray', vmax=np.percentile(self.integrated_postcard, 99.99))
            plt.show()

Response: Find other sources on the postcard.


Instruction:
Can you create a Python 3 function that
does photometry and estimates uncertainties by calculating the scatter around a linear fit to the data.

Response:
def do_photometry(self):
        """
        Does photometry and estimates uncertainties by calculating the scatter around a linear fit to the data
        in each orientation. This function is called by other functions and generally the user will not need
        to interact with it directly.
        """

        std_f = np.zeros(4)
        data_save = np.zeros_like(self.postcard)
        self.obs_flux = np.zeros_like(self.reference_flux)

        for i in range(4):
            g = np.where(self.qs == i)[0]
            wh = np.where(self.times[g] > 54947)

            data_save[g] = np.roll(self.postcard[g], int(self.roll_best[i,0]), axis=1)
            data_save[g] = np.roll(data_save[g], int(self.roll_best[i,1]), axis=2)

            self.target_flux_pixels = data_save[:,self.targets == 1]
            self.target_flux = np.sum(self.target_flux_pixels, axis=1)

            self.obs_flux[g] = self.target_flux[g] / self.reference_flux[g]
            self.obs_flux[g] /= np.median(self.obs_flux[g[wh]])

            fitline = np.polyfit(self.times[g][wh], self.obs_flux[g][wh], 1)
            std_f[i] = np.max([np.std(self.obs_flux[g][wh]/(fitline[0]*self.times[g][wh]+fitline[1])), 0.001])

        self.flux_uncert = std_f


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_panel(self, img):
        plt.figure(figsize=(14,6))
        ax = plt.gca()
        fig = plt.gcf()
        plt.subplot(122)

        data_save = np.zeros_like(self.postcard)
        self.roll_best = np.zeros((4,2))

        for i in range(4):
            g = np.where(self.qs == i)[0]
            wh = np.where(self.times[g] > 54947)

            self.roll_best[i] = self.do_rolltest(g, wh)

        self.do_photometry()
        for i in range(4):
            g = np.where(self.qs == i)[0]
            plt.errorbar(self.times[g], self.obs_flux[g], yerr=self.flux_uncert[i], fmt=fmt[i])

        plt.xlabel('Time', fontsize=20)
        plt.ylabel('Relative Flux', fontsize=20)

        plt.subplot(121)
        implot = plt.imshow(img, interpolation='nearest', cmap='gray', vmin=98000*52, vmax=104000*52)
        cid = fig.canvas.mpl_connect('button_press_event', self.onclick)

        plt.show(block=True)

Response: Generates the figure shown in adjust_aperture for visualization purposes.


Instruction:
How would you explain what the following Python 3 function does
def adjust_aperture(self, image_region=15, ignore_bright=0):
        self.ignore_bright = ignore_bright
        self.calc_fluxes()

        self.coordsx = []
        self.coordsy = []

        jj, ii = self.center
        jj, ii = int(jj), int(ii)  # Indices must be integer
        plt.ion()
        img = np.sum(((self.targets == 1)*self.postcard + (self.targets == 1)*100000)
                     [:,jj-image_region:jj+image_region,ii-image_region:ii+image_region], axis=0)
        self.generate_panel(img)

        while len(self.coordsx) != 0:
            for i in range(len(self.coordsx)):
                if self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] != 1:
                    self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] = 1
                elif self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] == 1:
                    self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] = 0
                if self.coordsy[i] == 0:
                    thiscol = np.where(self.targets[:,self.coordsx[i]+ii-image_region] == 1)
                    self.targets[thiscol,self.coordsx[i]+ii-image_region] = 0
                if self.coordsx[i] == 0:
                    thiscol = np.where(self.targets[self.coordsy[i]+jj-image_region,:] == 1)
                    self.targets[self.coordsy[i]+jj-image_region, thiscol] = 0

            self.coordsx = []
            self.coordsy = []
            img = np.sum(((self.targets == 1)*self.postcard +
                          (self.targets == 1)*100000)[:,jj-image_region:jj+image_region,ii-image_region:ii+image_region],
                         axis=0)
            self.generate_panel(img)

Response: This function updates the aperture and the light curve in the aperture panel.


Instruction:
Create a Python 3 function to
calculate the normalized photometry for the target aperture.

Response:
def data_for_target(self, do_roll=True, ignore_bright=0):
        """
        Determine the normalized photometry, accounting for effects shared by reference stars. Does not provide
        the opportunity to adjust the aperture

        Args:
            image_region: If ``True`` allow the aperture to be shifted up to one pixel in both the x and y
                directions to account for differential velocity aberration (default: ``True``)
            ignore_bright: The number of brightest stars to be ignored in the determination of the flux from
                reference stars. If there is reason to believe (for example) that saturated stars may behave
                differently than the target star, they can be avoided with this flag (default: ``0``)

        """
        self.ignore_bright = ignore_bright
        self.calc_fluxes()
        self.roll_best = np.zeros((4,2))

        if do_roll == True:
            for i in range(4):
                g = np.where(self.qs == i)[0]
                wh = np.where(self.times[g] > 54947)

                self.roll_best[i] = self.do_rolltest(g, wh)

        self.do_photometry()


Instruction:
How would you implement a function in Python 3 that
calculates the total flux in each object in each epoch and returns a numpy array of the fluxes in each object.

Response:
def calc_fluxes(self, min_flux = 5000, outlier_iterations=5,
                       max_outlier_obs=4, outlier_limit=1.7):
        """
        Determine the suitable reference stars, and then the total flux in those stars and
        in the target star in each epoch

        Args:
            min_flux: The size of the region around the target star to be plotted. Images will be a square
                with side length ``image_region`` (default: ``5000``)
            outlier_iterations: The number of iterations to remove outliers from the reference star sample
                (stars at epochs with more than ``max_outlier_obs`` observations more than ``outlier_limit`` standard
                deviations from the median value for all stars after normalization) (default: ``5``)
            max_outlier_obs: The maximum number of epochs at which a star is allowed to be more than ``outlier_limit``
                standard deviations from the median value for all stars before it is removed as a suitable
                reference star (default: ``4``)
            outlier_limit: The level of deviation (measured in standard deviations) which a target is allowed
                to be discrepant from the median. If it is this discrepant at more than ``max_outlier_obs``
                epochs, it is removed from consideration (default: ``1.7``)

        """

        jj, ii = self.center

        numer = np.zeros(len(self.times))
        denom = np.zeros(len(self.times))
        factr = np.zeros(len(self.times))

        numer_pix = self.postcard[:,self.targets == 1]
        numer = np.sum(numer_pix, axis=1)

        tar_vals = np.zeros((len(self.times), int(np.max(self.targets)+1-2-self.ignore_bright)))

        for i in range(2+self.ignore_bright,int(np.max(self.targets)+1)):
            tval = np.sum(self.postcard[:,self.targets == i], axis=1)
            #denom += tval/np.median(tval)
            tar_vals[:,i-2-self.ignore_bright] = tval #/ np.median(tval)

        for i in range(len(self.obs_filenames)):
            if np.max(tar_vals[i]) < min_flux:
                tar_vals[self.qs == self.qs[i]] = 0.0

        all_tar = np.zeros((len(self.times), int(np.max(self.targets)-self.ignore_bright)))
        all_tar[:,0] = numer
        all_tar[:,1:] = tar_vals

        self.photometry_array = all_tar

        for i in range(len(tar_vals[0])):
            for j in range(4):
                g = np.where(self.qs == j)[0]
                tar_vals[g,i] /= (np.median(tar_vals[g,i])+1e-15)

        tar_vals_old = tar_vals + 0.0

        for i in range(outlier_iterations):
            nonzeros = np.where(tar_vals[0,:] != 0)[0]
            med = np.median(tar_vals[:,nonzeros], axis=1)
            std = np.std(tar_vals[:,nonzeros], axis=1)

            if np.sum(tar_vals) != 0.0:
                tar_vals_old = tar_vals + 0.0

            for k in range(len(tar_vals[0])):
                h = np.where((np.abs(med-tar_vals[:,k])/std) > outlier_limit)[0]
                if len(h) >= max_outlier_obs:
                    tar_vals[:,k] = 0

        if np.sum(tar_vals) == 0.0:
            tar_vals = tar_vals_old + 0.0

        denom = np.sum(tar_vals, axis=1)
        self.target_flux_pixels = numer_pix
        self.reference_flux = denom


Instruction:
Write a Python 3 function that can
calculate the centroid positions for the target star at all epochs.

Response:
def calc_centroids(self):
        """
        Identify the centroid positions for the target star at all epochs. Useful for verifying that there is
        no correlation between flux and position, as might be expected for high proper motion stars.
        """
        self.cm = np.zeros((len(self.postcard), 2))
        for i in range(len(self.postcard)):
            target = self.postcard[i]
            target[self.targets != 1] = 0.0
            self.cm[i] = center_of_mass(target)


Instruction:
Implement a Python 3 function for
defining the spotsignal for each single cadence object.

Response:
def define_spotsignal(self):
        """
        Identify the "expected" flux value at the time of each observation based on the
        Kepler long-cadence data, to ensure variations observed are not the effects of a single
        large starspot. Only works if the target star was targeted for long or short cadence
        observations during the primary mission.
        """
        client = kplr.API()
        star = client.star(self.kic)

        lcs = star.get_light_curves(short_cadence=False)
        time, flux, ferr, qual = [], [], [], []
        for lc in lcs:
            with lc.open() as f:
                hdu_data = f[1].data
                time.append(hdu_data["time"])
                flux.append(hdu_data["pdcsap_flux"])
                ferr.append(hdu_data["pdcsap_flux_err"])
                qual.append(hdu_data["sap_quality"])
            tout = np.array([])
            fout = np.array([])
            eout = np.array([])
            for i in range(len(flux)):
                t = time[i][qual[i] == 0]
                f = flux[i][qual[i] == 0]
                e = ferr[i][qual[i] == 0]

                t = t[np.isfinite(f)]
                e = e[np.isfinite(f)]
                f = f[np.isfinite(f)]

                e /= np.median(f)
                f /= np.median(f)
                tout = np.append(tout, t[50:]+54833)
                fout = np.append(fout, f[50:])
                eout = np.append(eout, e[50:])

            self.spot_signal = np.zeros(52)

            for i in range(len(self.times)):
                if self.times[i] < 55000:
                    self.spot_signal[i] = 1.0
                else:
                    self.spot_signal[i] = fout[np.abs(self.times[i] - tout) == np.min(np.abs(self.times[i] - tout))]


Instruction:
Make a summary of the following Python 3 code
def model_uncert(self):
        Y = self.photometry_array.T
        Y /= np.median(Y, axis=1)[:, None]
        C = np.median(Y, axis=0)

        nstars, nobs = np.shape(Y)

        Z = np.empty((nstars, 4))

        qs = self.qs.astype(int)

        for s in range(4):
            Z[:, s] = np.median((Y / C)[:, qs == s], axis=1)

        resid2 = (Y - Z[:, qs] * C)**2
        z = Z[:, qs]
        trend = z * C[None, :]

        lnS = np.log(np.nanmedian(resid2, axis=0))
        jitter = np.log(0.1*np.nanmedian(np.abs(np.diff(Y, axis=1))))

        cal_ferr = np.sqrt(np.exp(2*(jitter/trend))+z**2*np.exp(lnS)[None, :])

        self.modeled_uncert = cal_ferr
        self.target_uncert = cal_ferr[0]

Response: Estimate the photometric uncertainties on each data point following Equation A. 2 of The Paper.


Instruction:
Create a Python 3 function for
appending line to internal list.

Response:
def _print(self, line=''):
        """Append line to internal list.
        Uses self.tabs to format indents.

        Keyword arguments:
        line -- line to append
        """
        self.lines.append('{}{}'.format('\t'*self.tabs , line))


Instruction:
Create a Python 3 function for
dumping single enum type.

Response:
def _dump_enum(self, e, top=''):
        """Dump single enum type.

        Keyword arguments:
        top -- top namespace
        """
        self._print()
        self._print('enum {} {{'.format(e.name))
        self.defines.append('{}.{}'.format(top,e.name))

        self.tabs+=1
        for v in e.value:
            self._print('{} = {};'.format(v.name, v.number))
        self.tabs-=1
        self._print('}')


Instruction:
Write a Python 3 script to
dump a single field.

Response:
def _dump_field(self, fd):
        """Dump single field.
        """
        v = {}
        v['label'] = Pbd.LABELS[fd.label]
        v['type'] = fd.type_name if len(fd.type_name) > 0 else Pbd.TYPES[fd.type]
        v['name'] = fd.name
        v['number'] = fd.number
        v['default'] = '[default = {}]'.format(fd.default_value) if len(fd.default_value) > 0 else ''

        f = '{label} {type} {name} = {number} {default};'.format(**v)
        f = ' '.join(f.split())
        self._print(f)

        if len(fd.type_name) > 0:
            self.uses.append(fd.type_name)


Instruction:
Here you have a function in Python 3, explain what it does
def _dump_message(self, m, top=''):
        self._print()
        self._print('message {} {{'.format(m.name))
        self.defines.append('{}.{}'.format(top, m.name))
        self.tabs+=1

        for f in m.field:
            self._dump_field(f)

        for e in m.enum_type:
            self._dump_enum(e, top='{}.{}'.format(top, m.name))

        for n in m.nested_type:
            self._dump_message(n, top='{}.{}'.format(top, m.name))

        self.tabs-=1
        self._print('}')

Response: Dump a single message type.


Instruction:
Make a summary of the following Python 3 code
def _walk(self, fd):
        top = '.{}'.format(fd.package) if len(fd.package) > 0 else ''

        for e in fd.enum_type: self._dump_enum(e, top)
        for m in fd.message_type: self. _dump_message(m, top)

Response: Walk and dump the class metadata.


Instruction:
Implement a Python 3 function for
disassembling serialized protocol buffers file.

Response:
def disassemble(self):
        """Disassemble serialized protocol buffers file.
        """
        ser_pb = open(self.input_file, 'rb').read()  # Read serialized pb file

        fd = FileDescriptorProto()
        fd.ParseFromString(ser_pb)
        self.name = fd.name

        self._print('// Reversed by pbd (https://github.com/rsc-dev/pbd)')
        self._print('syntax = "proto2";')
        self._print('')

        if len(fd.package) > 0:
            self._print('package {};'.format(fd.package))
            self.package = fd.package
        else:
            self._print('// Package not defined')

        self._walk(fd)


Instruction:
How would you implement a function in Python 3 that
dumps the proto file to given directory.

Response:
def dump(self, out_dir='.'):
        """Dump proto file to given directory.

        Keyword arguments:
        out_dir -- dump directory. Default='.'
        """
        uri = out_dir + os.sep + self.name
        with open(uri, 'w') as fh:
            fh.write('\n'.join(self.lines))


Instruction:
Can you implement a function in Python 3 that
finds all missing imports in list of Pbd instances.

Response:
def find_imports(self, pbds):
        """Find all missing imports in list of Pbd instances.
        """
        # List of types used, but not defined
        imports = list(set(self.uses).difference(set(self.defines)))

        # Clumpsy, but enought for now
        for imp in imports:
            for p in pbds:
                if imp in p.defines:
                    self.imports.append(p.name)
                    break

        self.imports = list(set(self.imports))

        for import_file in self.imports:
            self.lines.insert(2, 'import "{}";'.format(import_file))


Instruction:
Write a Python 3 script for
returning a dict from a fasta file and the number of sequences as the second return value.

Response:
def fasta_file_to_dict(fasta_file, id=True, header=False, seq=False):
    """Returns a dict from a fasta file and the number of sequences as the second return value.
    fasta_file can be a string path or a file object.
    The key of fasta_dict can be set using the keyword arguments and
    results in a combination of id, header, sequence, in that order. joined with '||'. (default: id)
    Duplicate keys are checked and a warning is logged if found.
    The value of fasta_dict is a python dict with 3 keys: header, id and seq

    Changelog:
    2014/11/17:
    * Added support for url escaped id
    """
    fasta_file_f = fasta_file
    if isinstance(fasta_file, str):
        fasta_file_f = open(fasta_file, 'rb')

    fasta_dict = OrderedDict()
    keys = ['id', 'header', 'seq']
    flags = dict([('id', id), ('header', header), ('seq', seq)])
    entry = dict([('id', ''), ('header', ''), ('seq', '')])
    count = 0
    line_num = 0

    for line in fasta_file_f:
        line = line.strip()
        if line and line[0] == '>':
            count += 1
            key = '||'.join([entry[i] for i in keys if flags[i]])
            if key: # key != ''
                if key in fasta_dict: # check for duplicate key
                    logger.warning('%s : Line %d : Duplicate %s [%s] : ID = [%s].', fasta_file_f.name, line_num, '||'.join([i for i in keys if flags[i]]), key[:25] + (key[25:] and '..'), entry['id'])
                entry['seq'] = ''.join(entry['seq'])
                fasta_dict[key] = entry
                # check for url escaped id
                if id:
                    unescaped_id = unquote(entry['id'])
                    if id != unescaped_id:
                        key = '||'.join([unescaped_id] + [entry[i] for i in keys if i != 'id' and flags[i]])
                        entry['unescaped_id'] = unescaped_id
                        fasta_dict[key] = entry
                entry = dict()
            entry['header'] = line
            entry['id'] = line.split()[0][1:]
            entry['seq'] = []
        else:
            entry['seq'].append(line.upper())
        line_num += 1

    if isinstance(fasta_file, str):
        fasta_file_f.close()

    key = '||'.join([entry[i] for i in keys if flags[i]])
    if key: # key != ''
        if key in fasta_dict:
            logger.warning('%s : Line %d : Duplicate %s [%s] : ID = [%s].', fasta_file_f.name, line_num, '||'.join([i for i in keys if flags[i]]), key[:25] + (key[25:] and '..'), entry['id'])
        entry['seq'] = ''.join(entry['seq'])
        fasta_dict[key] = entry
        # check for url escaped id
        if id:
            unescaped_id = unquote(entry['id'])
            if id != unescaped_id:
                key = '||'.join([unescaped_id] + [entry[i] for i in keys if i != 'id' and flags[i]])
                entry['unescaped_id'] = unescaped_id
                fasta_dict[key] = entry

    return fasta_dict, count


Instruction:
Make a summary of the following Python 3 code
def fasta_dict_to_file(fasta_dict, fasta_file, line_char_limit=None):
    fasta_fp = fasta_file
    if isinstance(fasta_file, str):
        fasta_fp = open(fasta_file, 'wb')

    for key in fasta_dict:
        seq = fasta_dict[key]['seq']
        if line_char_limit:
            seq = '\n'.join([seq[i:i+line_char_limit] for i in range(0, len(seq), line_char_limit)])
        fasta_fp.write(u'{0:s}\n{1:s}\n'.format(fasta_dict[key]['header'], seq))

Response: Write a dictionary to a fasta file


Instruction:
How would you code a function in Python 3 to
check whether child features are within the coordinate boundaries of parent features.

Response:
def check_parent_boundary(self):
        """
        checks whether child features are within the coordinate boundaries of parent features

        :return:
        """
        for line in self.lines:
            for parent_feature in line['parents']:
                ok = False
                for parent_line in parent_feature:
                    if parent_line['start'] <= line['start'] and line['end'] <= parent_line['end']:
                        ok = True
                        break
                if not ok:
                    self.add_line_error(line, {'message': 'This feature is not contained within the feature boundaries of parent: {0:s}: {1:s}'.format(
                        parent_feature[0]['attributes']['ID'],
                        ','.join(['({0:s}, {1:d}, {2:d})'.format(line['seqid'], line['start'], line['end']) for line in parent_feature])
                    ), 'error_type': 'BOUNDS', 'location': 'parent_boundary'})


Instruction:
Write a Python 3 function that can
check if the phase of the current node is correct

Response:
def check_phase(self):
        """
        1. get a list of CDS with the same parent
        2. sort according to strand
        3. calculate and validate phase
        """
        plus_minus = set(['+', '-'])
        for k, g in groupby(sorted([line for line in self.lines if  line['line_type'] == 'feature' and line['type'] == 'CDS' and 'Parent' in line['attributes']], key=lambda x: x['attributes']['Parent']), key=lambda x: x['attributes']['Parent']):
            cds_list = list(g)
            strand_set = list(set([line['strand'] for line in cds_list]))
            if len(strand_set) != 1:
                for line in cds_list:
                    self.add_line_error(line, {'message': 'Inconsistent CDS strand with parent: {0:s}'.format(k), 'error_type': 'STRAND'})
                continue
            if len(cds_list) == 1:
                if cds_list[0]['phase'] != 0:
                    self.add_line_error(cds_list[0], {'message': 'Wrong phase {0:d}, should be {1:d}'.format(cds_list[0]['phase'], 0), 'error_type': 'PHASE'})
                continue
            strand = strand_set[0]
            if strand not in plus_minus:
                # don't process unknown strands
                continue
            if strand == '-':
                # sort end descending
                sorted_cds_list = sorted(cds_list, key=lambda x: x['end'], reverse=True)
            else:
                sorted_cds_list = sorted(cds_list, key=lambda x: x['start'])
            phase = 0
            for line in sorted_cds_list:
                if line['phase'] != phase:
                    self.add_line_error(line, {'message': 'Wrong phase {0:d}, should be {1:d}'.format(line['phase'], phase), 'error_type': 'PHASE'})
                phase = (3 - ((line['end'] - line['start'] + 1 - phase) % 3)) % 3


Instruction:
Can you generate a brief explanation for the following Python 3 code
def check_reference(self, sequence_region=False, fasta_embedded=False, fasta_external=False, check_bounds=True, check_n=True, allowed_num_of_n=0, feature_types=('CDS',)):
        # collect lines with errors in this set
        error_lines = set()
        # check if we have a parsed gff3
        if not self.lines:
            self.logger.debug('.parse(gff_file) before calling .check_bounds()')
            return error_lines
        # setup default line_types
        check_n_feature_types = set(feature_types)
        if len(check_n_feature_types) == 0:
            check_n_feature_types.add('CDS')
        # compile regex
        n_segments_finditer = re.compile(r'[Nn]+').finditer
        # check_all_sources mode
        check_all_sources = True
        if sequence_region or fasta_embedded or fasta_external:
            check_all_sources = False
        # get a list of line_data with valid start and end coordinates and unescape the seqid
        start_end_error_locations = set(('start', 'end', 'start,end'))
        valid_line_data_seqid = [(line_data, unquote(line_data['seqid'])) for line_data in self.lines if line_data['line_type'] == 'feature' and line_data['seqid'] != '.' and (not line_data['line_errors'] or not [error_info for error_info in line_data['line_errors'] if 'location' in error_info and error_info['location'] in start_end_error_locations])]
        checked_at_least_one_source = False
        # check directive
        # don't use any directives with errors
        valid_sequence_regions = dict([(unquote(line_data['seqid']), line_data) for line_data in self.lines if line_data['directive'] == '##sequence-region' and not line_data['line_errors']])
        unresolved_seqid = set()
        if (check_all_sources or sequence_region) and valid_sequence_regions:
            checked_at_least_one_source = True
            for line_data, seqid in valid_line_data_seqid:
                if seqid not in valid_sequence_regions and seqid not in unresolved_seqid:
                    unresolved_seqid.add(seqid)
                    error_lines.add(line_data['line_index'])
                    self.add_line_error(line_data, {'message': u'Seqid not found in any ##sequence-region: {0:s}'.format(
                        seqid), 'error_type': 'BOUNDS', 'location': 'sequence_region'})
                    continue
                if line_data['start'] < valid_sequence_regions[seqid]['start']:
                    error_lines.add(line_data['line_index'])
                    self.add_line_error(line_data, {'message': 'Start is less than the ##sequence-region start: %d' % valid_sequence_regions[seqid]['start'], 'error_type': 'BOUNDS', 'location': 'sequence_region'})
                if line_data['end'] > valid_sequence_regions[seqid]['end']:
                    error_lines.add(line_data['line_index'])
                    self.add_line_error(line_data, {'message': 'End is greater than the ##sequence-region end: %d' % valid_sequence_regions[seqid]['end'], 'error_type': 'BOUNDS', 'location': 'sequence_region'})
        elif sequence_region:
            self.logger.debug('##sequence-region not found in GFF3')
        # check fasta_embedded
        unresolved_seqid = set()
        if (check_all_sources or fasta_embedded) and self.fasta_embedded:
            checked_at_least_one_source = True
            for line_data, seqid in valid_line_data_seqid:
                if seqid not in self.fasta_embedded and seqid not in unresolved_seqid:
                    unresolved_seqid.add(seqid)
                    error_lines.add(line_data['line_index'])
                    self.add_line_error(line_data, {'message': 'Seqid not found in the embedded ##FASTA: %s' % seqid, 'error_type': 'BOUNDS', 'location': 'fasta_embedded'})
                    continue
                # check bounds
                if line_data['end'] > len(self.fasta_embedded[seqid]['seq']):
                    error_lines.add(line_data['line_index'])
                    self.add_line_error(line_data, {'message': 'End is greater than the embedded ##FASTA sequence length: %d' % len(self.fasta_embedded[seqid]['seq']), 'error_type': 'BOUNDS', 'location': 'fasta_embedded'})
                # check n
                if check_n and line_data['type'] in check_n_feature_types:
                    """
                    >>> timeit("a.lower().count('n')", "import re; a = ('ASDKADSJHFIUDNNNNNNNnnnnSHFD'*50)")
                    5.540903252684302
                    >>> timeit("a.count('n'); a.count('N')", "import re; a = ('ASDKADSJHFIUDNNNNNNNnnnnSHFD'*50)")
                    2.3504867946058425
                    >>> timeit("re.findall('[Nn]+', a)", "import re; a = ('ASDKADSJHFIUDNNNNNNNnnnnSHFD'*50)")
                    30.60731204915959
                    """
                    n_count = self.fasta_embedded[seqid]['seq'].count('N', line_data['start'] - 1, line_data['end']) + self.fasta_embedded[seqid]['seq'].count('n', line_data['start'] - 1, line_data['end'])
                    if n_count > allowed_num_of_n:
                        # get detailed segments info
                        n_segments = [(m.start(), m.end() - m.start()) for m in n_segments_finditer(self.fasta_embedded[seqid]['seq'], line_data['start'] - 1, line_data['end'])]
                        n_segments_str = ['(%d, %d)' % (m[0], m[1]) for m in n_segments]
                        error_lines.add(line_data['line_index'])
                        self.add_line_error(line_data, {'message': 'Found %d Ns in %s feature of length %d using the embedded ##FASTA, consists of %d segment (start, length): %s' % (n_count, line_data['type'], line_data['end'] - line_data['start'], len(n_segments), ', '.join(n_segments_str)), 'error_type': 'N_COUNT', 'n_segments': n_segments, 'location': 'fasta_embedded'})
        elif fasta_embedded:
            self.logger.debug('Embedded ##FASTA not found in GFF3')
        # check fasta_external
        unresolved_seqid = set()
        if (check_all_sources or fasta_external) and self.fasta_external:
            checked_at_least_one_source = True
            for line_data, seqid in valid_line_data_seqid:
                if seqid not in self.fasta_external and seqid not in unresolved_seqid:
                    unresolved_seqid.add(seqid)
                    error_lines.add(line_data['line_index'])
                    self.add_line_error(line_data, {'message': 'Seqid not found in the external FASTA file: %s' % seqid, 'error_type': 'BOUNDS', 'location': 'fasta_external'})
                    continue
                # check bounds
                if line_data['end'] > len(self.fasta_external[seqid]['seq']):
                    error_lines.add(line_data['line_index'])
                    self.add_line_error(line_data, {'message': 'End is greater than the external FASTA sequence length: %d' % len(self.fasta_external[seqid]['seq']), 'error_type': 'BOUNDS', 'location': 'fasta_external'})
                # check n
                if check_n and line_data['type'] in check_n_feature_types:
                    n_count = self.fasta_external[seqid]['seq'].count('N', line_data['start'] - 1, line_data['end']) + self.fasta_external[seqid]['seq'].count('n', line_data['start'] - 1, line_data['end'])
                    if n_count > allowed_num_of_n:
                        # get detailed segments info
                        n_segments = [(m.start(), m.end() - m.start()) for m in n_segments_finditer(self.fasta_external[seqid]['seq'], line_data['start'] - 1, line_data['end'])]
                        n_segments_str = ['(%d, %d)' % (m[0], m[1]) for m in n_segments]
                        error_lines.add(line_data['line_index'])
                        self.add_line_error(line_data, {'message': 'Found %d Ns in %s feature of length %d using the external FASTA, consists of %d segment (start, length): %s' % (n_count, line_data['type'], line_data['end'] - line_data['start'], len(n_segments), ', '.join(n_segments_str)), 'error_type': 'N_COUNT', 'n_segments': n_segments, 'location': 'fasta_external'})
        elif fasta_external:
            self.logger.debug('External FASTA file not given')
        if check_all_sources and not checked_at_least_one_source:
            self.logger.debug('Unable to perform bounds check, requires at least one of the following sources: ##sequence-region, embedded ##FASTA, or external FASTA file')
        return error_lines

Response: Checks the validity of the reference sequence.