Instruction:
How would you implement a function in Python 3 that
returns a dict of cache entries for a given destination address.

Response:
def route(self, dst=None, dev=None, verbose=conf.verb):
        """
        Provide best route to IPv6 destination address, based on Scapy
        internal routing table content.

        When a set of address is passed (e.g. 2001:db8:cafe:*::1-5) an address
        of the set is used. Be aware of that behavior when using wildcards in
        upper parts of addresses !

        If 'dst' parameter is a FQDN, name resolution is performed and result
        is used.

        if optional 'dev' parameter is provided a specific interface, filtering
        is performed to limit search to route associated to that interface.
        """
        dst = dst or "::/0"  # Enable route(None) to return default route
        # Transform "2001:db8:cafe:*::1-5:0/120" to one IPv6 address of the set
        dst = dst.split("/")[0]
        savedst = dst  # In case following inet_pton() fails
        dst = dst.replace("*", "0")
        idx = dst.find("-")
        while idx >= 0:
            m = (dst[idx:] + ":").find(":")
            dst = dst[:idx] + dst[idx + m:]
            idx = dst.find("-")

        try:
            inet_pton(socket.AF_INET6, dst)
        except socket.error:
            dst = socket.getaddrinfo(savedst, None, socket.AF_INET6)[0][-1][0]
            # TODO : Check if name resolution went well

        # Use the default interface while dealing with link-local addresses
        if dev is None and (in6_islladdr(dst) or in6_ismlladdr(dst)):
            dev = conf.iface

        # Deal with dev-specific request for cache search
        k = dst
        if dev is not None:
            k = dst + "%%" + (dev if isinstance(dev, six.string_types) else dev.pcap_name)  # noqa: E501
        if k in self.cache:
            return self.cache[k]

        paths = []

        # TODO : review all kinds of addresses (scope and *cast) to see
        #        if we are able to cope with everything possible. I'm convinced
        #        it's not the case.
        # -- arnaud
        for p, plen, gw, iface, cset, me in self.routes:
            if dev is not None and iface != dev:
                continue
            if in6_isincluded(dst, p, plen):
                paths.append((plen, me, (iface, cset, gw)))
            elif (in6_ismlladdr(dst) and in6_islladdr(p) and in6_islladdr(cset[0])):  # noqa: E501
                paths.append((plen, me, (iface, cset, gw)))

        if not paths:
            if verbose:
                warning("No route found for IPv6 destination %s "
                        "(no default route?)", dst)
            return (scapy.consts.LOOPBACK_INTERFACE, "::", "::")

        # Sort with longest prefix first then use metrics as a tie-breaker
        paths.sort(key=lambda x: (-x[0], x[1]))

        best_plen = (paths[0][0], paths[0][1])
        paths = [x for x in paths if (x[0], x[1]) == best_plen]

        res = []
        for p in paths:  # Here we select best source address for every route
            tmp = p[2]
            srcaddr = get_source_addr_from_candidate_set(dst, tmp[1])
            if srcaddr is not None:
                res.append((p[0], p[1], (tmp[0], srcaddr, tmp[2])))

        if res == []:
            warning("Found a route for IPv6 destination '%s', but no possible source address.", dst)  # noqa: E501
            return (scapy.consts.LOOPBACK_INTERFACE, "::", "::")

        # Symptom  : 2 routes with same weight (our weight is plen)
        # Solution :
        #  - dst is unicast global. Check if it is 6to4 and we have a source
        #    6to4 address in those available
        #  - dst is link local (unicast or multicast) and multiple output
        #    interfaces are available. Take main one (conf.iface6)
        #  - if none of the previous or ambiguity persists, be lazy and keep
        #    first one

        if len(res) > 1:
            tmp = []
            if in6_isgladdr(dst) and in6_isaddr6to4(dst):
                # TODO : see if taking the longest match between dst and
                #        every source addresses would provide better results
                tmp = [x for x in res if in6_isaddr6to4(x[2][1])]
            elif in6_ismaddr(dst) or in6_islladdr(dst):
                # TODO : I'm sure we are not covering all addresses. Check that
                tmp = [x for x in res if x[2][0] == conf.iface6]

            if tmp:
                res = tmp

        # Fill the cache (including dev-specific request)
        k = dst
        if dev is not None:
            k = dst + "%%" + (dev if isinstance(dev, six.string_types) else dev.pcap_name)  # noqa: E501
        self.cache[k] = res[0][2]

        return res[0][2]


Instruction:
Can you write a function in Python 3 where it
compresses a TLSPlaintext instance using a compression algorithm instance alg and return a TLSCompressed instance.

Response:
def _tls_compress(alg, p):
    """
    Compress p (a TLSPlaintext instance) using compression algorithm instance
    alg and return a TLSCompressed instance.
    """
    c = TLSCompressed()
    c.type = p.type
    c.version = p.version
    c.data = alg.compress(p.data)
    c.len = len(c.data)
    return c


Instruction:
Write a Python 3 function for
decompressing a TLSCompressed instance using the compression algorithm and return a TLSPlaintext instance.

Response:
def _tls_decompress(alg, c):
    """
    Decompress c (a TLSCompressed instance) using compression algorithm
    instance alg and return a TLSPlaintext instance.
    """
    p = TLSPlaintext()
    p.type = c.type
    p.version = c.version
    p.data = alg.decompress(c.data)
    p.len = len(p.data)
    return p


Instruction:
Here you have a function in Python 3, explain what it does
def _tls_mac_add(alg, c, write_seq_num):
    write_seq_num = struct.pack("!Q", write_seq_num)
    h = alg.digest(write_seq_num + bytes(c))
    c.data += h
    c.len += alg.hash_len

Response:
Compute the MAC using the provided MAC alg instance over TLSCiphertext c using
    current write sequence number write_seq_num.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _tls_mac_verify(alg, p, read_seq_num):
    h_size = alg.hash_len
    if p.len < h_size:
        return False
    received_h = p.data[-h_size:]
    p.len -= h_size
    p.data = p.data[:-h_size]

    read_seq_num = struct.pack("!Q", read_seq_num)
    h = alg.digest(read_seq_num + bytes(p))
    return h == received_h

Response: Verify that the MAC in the provided message is valid.


Instruction:
How would you implement a function in Python 3 that
provides with cipher block size parameter and current TLSCompressed packet p (after MAC addition), the function adds required, deterministic padding to p.data before encryption step, as it is defined for TLS (i.e. not SSL and its allowed random padding). The function has no return value.

Response:
def _tls_add_pad(p, block_size):
    """
    Provided with cipher block size parameter and current TLSCompressed packet
    p (after MAC addition), the function adds required, deterministic padding
    to p.data before encryption step, as it is defined for TLS (i.e. not
    SSL and its allowed random padding). The function has no return value.
    """
    padlen = -p.len % block_size
    padding = chb(padlen) * (padlen + 1)
    p.len += len(padding)
    p.data += padding


Instruction:
Can you generate the documentation for the following Python 3 function
def _tls_del_pad(p):

    if p.len < 1:
        warning("Message format is invalid (padding)")
        return False

    padlen = orb(p.data[-1])
    padsize = padlen + 1

    if p.len < padsize:
        warning("Invalid padding length")
        return False

    if p.data[-padsize:] != chb(padlen) * padsize:
        warning("Padding content is invalid %s", repr(p.data[-padsize:]))
        return False

    p.data = p.data[:-padsize]
    p.len -= padsize

    return True

Response: This function is used to remove the trailing padding found in p. data.


Instruction:
Explain what the following Python 3 code does
def _tls_encrypt(alg, p):
    c = TLSCiphertext()
    c.type = p.type
    c.version = p.version
    c.data = alg.encrypt(p.data)
    c.len = len(c.data)
    return c

Response: This function encrypts a TLSCompressed packet and returns a TLSCiphertext object.


Instruction:
Implement a function in Python 3 to
provide with a TLSCiphertext instance c, and a stream or block cipher alg, the function decrypts c.data and returns a newly created TLSPlaintext.

Response:
def _tls_decrypt(alg, c):
    """
    Provided with a TLSCiphertext instance c, and a stream or block cipher alg,
    the function decrypts c.data and returns a newly created TLSPlaintext.
    """
    p = TLSPlaintext()
    p.type = c.type
    p.version = c.version
    p.data = alg.decrypt(c.data)
    p.len = len(p.data)
    return p


Instruction:
Explain what the following Python 3 code does
def _tls_aead_auth_encrypt(alg, p, write_seq_num):
    P = bytes(p)
    write_seq_num = struct.pack("!Q", write_seq_num)
    A = write_seq_num + P[:5]

    c = TLSCiphertext()
    c.type = p.type
    c.version = p.version
    c.data = alg.auth_encrypt(P, A, write_seq_num)
    c.len = len(c.data)
    return c

Response: This function applies an AEAD cipher to p. data and returns a TLSCiphertext instance with the encrypted data.


Instruction:
Can you generate the documentation for the following Python 3 function
def _extcap_call(prog, args, keyword, values):
    p = subprocess.Popen(
        [prog] + args,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    data, err = p.communicate()
    if p.returncode != 0:
        raise OSError("%s returned with error code %s: %s" % (prog,
                                                              p.returncode,
                                                              err))
    data = plain_str(data)
    res = []
    for ifa in data.split("\n"):
        ifa = ifa.strip()
        if not ifa.startswith(keyword):
            continue
        res.append(tuple([re.search(r"{%s=([^}]*)}" % val, ifa).group(1)
                   for val in values]))
    return res

Response: Function used to call a program using the extcap format and parse the results


Instruction:
How would you implement a function in Python 3 that
builds a list of _HCSINullField with numbered "Reserved" names.

Response:
def _hcsi_null_range(*args, **kwargs):
    """Builds a list of _HCSINullField with numbered "Reserved" names.

    Takes the same arguments as the ``range`` built-in.

    :returns: list[HCSINullField]
    """
    return [
        HCSINullField('Reserved{:02d}'.format(x))
        for x in range(*args, **kwargs)
    ]


Instruction:
Write a Python 3 function that can
convert internal value to machine value

Response:
def i2m(self, pkt, x):
        """Convert internal value to machine value"""
        if x is None:
            # Try to return zero if undefined
            x = self.h2i(pkt, 0)
        return x


Instruction:
Create a Python 3 function for
returning the main unicast address associated with provided interface in human readable form.

Response:
def get_if_addr6(iff):
    """
    Returns the main global unicast address associated with provided
    interface, in human readable form. If no global address is found,
    None is returned.
    """
    return next((x[0] for x in in6_getifaddr()
                 if x[2] == iff and x[1] == IPV6_ADDR_GLOBAL), None)


Instruction:
Can you implement a function in Python 3 that
returns the main unicast address associated with provided interface in network format.

Response:
def get_if_raw_addr6(iff):
    """
    Returns the main global unicast address associated with provided
    interface, in network format. If no global address is found, None
    is returned.
    """
    ip6 = get_if_addr6(iff)
    if ip6 is not None:
        return inet_pton(socket.AF_INET6, ip6)

    return None


Instruction:
Create a Python 3 function for
getting representation name of a pnio frame ID.

Response:
def i2s_frameid(x):
    """ Get representation name of a pnio frame ID

    :param x: a key of the PNIO_FRAME_IDS dictionary
    :returns: str
    """
    try:
        return PNIO_FRAME_IDS[x]
    except KeyError:
        pass
    if 0x0100 <= x < 0x1000:
        return "RT_CLASS_3 (%4x)" % x
    if 0x8000 <= x < 0xC000:
        return "RT_CLASS_1 (%4x)" % x
    if 0xC000 <= x < 0xFC00:
        return "RT_CLASS_UDP (%4x)" % x
    if 0xFF80 <= x < 0xFF90:
        return "FragmentationFrameID (%4x)" % x
    return x


Instruction:
Can you tell what is the following Python 3 function doing
def s2i_frameid(x):
    try:
        return {
            "RT_CLASS_3": 0x0100,
            "RT_CLASS_1": 0x8000,
            "RT_CLASS_UDP": 0xC000,
            "FragmentationFrameID": 0xFF80,
        }[x]
    except KeyError:
        pass
    try:
        return next(key for key, value in six.iteritems(PNIO_FRAME_IDS)
                    if value == x)
    except StopIteration:
        pass
    return x

Response: Get pnio frame ID from a representation name


Instruction:
Explain what the following Python 3 code does
def dispatch_hook(cls, _pkt=None, *args, **kargs):
        '''
          Attribution of correct type depending on version and pdu_type
        '''
        if _pkt and len(_pkt) >= 2:
            version = orb(_pkt[0])
            pdu_type = orb(_pkt[1])
            if version == 0:
                return PDU_CLASS_VERSION_0[pdu_type]
            elif version == 1:
                return PDU_CLASS_VERSION_1[pdu_type]
        return Raw

Response: This method is used to dispatch the event from the backend to the appropriate handler.


Instruction:
Make a summary of the following Python 3 code
def parse(self, data):

        ASNlist = []
        for line in data.splitlines()[1:]:
            line = plain_str(line)
            if "|" not in line:
                continue
            asn, ip, desc = [elt.strip() for elt in line.split('|')]
            if asn == "NA":
                continue
            asn = "AS%s" % asn
            ASNlist.append((ip, asn, desc))
        return ASNlist

Response: Parse bulk cymru data


Instruction:
How would you implement a function in Python 3 that
encodes the mrcode value to its IGMPv3 encoded time value if needed.

Response:
def encode_maxrespcode(self):
        """Encode and replace the mrcode value to its IGMPv3 encoded time value if needed,  # noqa: E501
        as specified in rfc3376#section-4.1.1.

        If value < 128, return the value specified. If >= 128, encode as a floating  # noqa: E501
        point value. Value can be 0 - 31744.
        """
        value = self.mrcode
        if value < 128:
            code = value
        elif value > 31743:
            code = 255
        else:
            exp = 0
            value >>= 3
            while(value > 31):
                exp += 1
                value >>= 1
            exp <<= 4
            code = 0x80 | exp | (value & 0x0F)
        self.mrcode = code


Instruction:
Can you generate the documentation for the following Python 3 function
def _register_lltd_specific_class(*attr_types):
    def _register(cls):
        for attr_type in attr_types:
            SPECIFIC_CLASSES[attr_type] = cls
        type_fld = LLTDAttribute.fields_desc[0].copy()
        type_fld.default = attr_types[0]
        cls.fields_desc = [type_fld] + cls.fields_desc
        return cls
    return _register

Response: This is a class decorator that registers LLTDAttributeSpecific with the given list of attr_types.


Instruction:
Write a Python 3 function that can
update the builder using the provided plist. plist can be either a Packet or PacketList.

Response:
def parse(self, plist):
        """Update the builder using the provided `plist`. `plist` can
        be either a Packet() or a PacketList().

        """
        if not isinstance(plist, PacketList):
            plist = PacketList(plist)
        for pkt in plist[LLTD]:
            if LLTDQueryLargeTlv in pkt:
                key = "%s:%s:%d" % (pkt.real_dst, pkt.real_src, pkt.seq)
                self.types_offsets[key] = (pkt[LLTDQueryLargeTlv].type,
                                           pkt[LLTDQueryLargeTlv].offset)
            elif LLTDQueryLargeTlvResp in pkt:
                try:
                    key = "%s:%s:%d" % (pkt.real_src, pkt.real_dst, pkt.seq)
                    content, offset = self.types_offsets[key]
                except KeyError:
                    continue
                loc = slice(offset, offset + pkt[LLTDQueryLargeTlvResp].len)
                key = "%s > %s [%s]" % (
                    pkt.real_src, pkt.real_dst,
                    LLTDQueryLargeTlv.fields_desc[0].i2s.get(content, content),
                )
                data = self.data.setdefault(key, array("B"))
                datalen = len(data)
                if datalen < loc.stop:
                    data.extend(array("B", b"\x00" * (loc.stop - datalen)))
                data[loc] = array("B", pkt[LLTDQueryLargeTlvResp].value)


Instruction:
Create a Python 3 function to
return a dictionary object containing the content type and the content type of the content.

Response:
def get_data(self):
        """Returns a dictionary object, keys are strings "source >
        destincation [content type]", and values are the content
        fetched, also as a string.

        """
        return {key: "".join(chr(byte) for byte in data)
                for key, data in six.iteritems(self.data)}


Instruction:
How would you implement a function in Python 3 that
converts an IPv6 address from text representation into binary form.

Response:
def _inet6_pton(addr):
    """Convert an IPv6 address from text representation into binary form,
used when socket.inet_pton is not available.

    """
    joker_pos = None
    result = b""
    addr = plain_str(addr)
    if addr == '::':
        return b'\x00' * 16
    if addr.startswith('::'):
        addr = addr[1:]
    if addr.endswith('::'):
        addr = addr[:-1]
    parts = addr.split(":")
    nparts = len(parts)
    for i, part in enumerate(parts):
        if not part:
            # "::" indicates one or more groups of 2 null bytes
            if joker_pos is None:
                joker_pos = len(result)
            else:
                # Wildcard is only allowed once
                raise _INET6_PTON_EXC
        elif i + 1 == nparts and '.' in part:
            # The last part of an IPv6 address can be an IPv4 address
            if part.count('.') != 3:
                # we have to do this since socket.inet_aton('1.2') ==
                # b'\x01\x00\x00\x02'
                raise _INET6_PTON_EXC
            try:
                result += socket.inet_aton(part)
            except socket.error:
                raise _INET6_PTON_EXC
        else:
            # Each part must be 16bit. Add missing zeroes before decoding.
            try:
                result += hex_bytes(part.rjust(4, "0"))
            except (binascii.Error, TypeError):
                raise _INET6_PTON_EXC
    # If there's a wildcard, fill up with zeros to reach 128bit (16 bytes)
    if joker_pos is not None:
        if len(result) == 16:
            raise _INET6_PTON_EXC
        result = (result[:joker_pos] + b"\x00" * (16 - len(result)) +
                  result[joker_pos:])
    if len(result) != 16:
        raise _INET6_PTON_EXC
    return result


Instruction:
Can you write a function in Python 3 where it
converts an IP address from text representation into binary form.

Response:
def inet_pton(af, addr):
    """Convert an IP address from text representation into binary form."""
    # Will replace Net/Net6 objects
    addr = plain_str(addr)
    # Use inet_pton if available
    try:
        return socket.inet_pton(af, addr)
    except AttributeError:
        try:
            return _INET_PTON[af](addr)
        except KeyError:
            raise socket.error("Address family not supported by protocol")


Instruction:
How would you explain what the following Python 3 function does
def _inet6_ntop(addr):
    # IPv6 addresses have 128bits (16 bytes)
    if len(addr) != 16:
        raise ValueError("invalid length of packed IP address string")

    # Decode to hex representation
    address = ":".join(plain_str(bytes_hex(addr[idx:idx + 2])).lstrip('0') or '0'  # noqa: E501
                       for idx in range(0, 16, 2))

    try:
        # Get the longest set of zero blocks. We need to take a look
        # at group 1 regarding the length, as 0:0:1:0:0:2:3:4 would
        # have two matches: 0:0: and :0:0: where the latter is longer,
        # though the first one should be taken. Group 1 is in both
        # cases 0:0.
        match = max(_IP6_ZEROS.finditer(address),
                    key=lambda m: m.end(1) - m.start(1))
        return '{}::{}'.format(address[:match.start()], address[match.end():])
    except ValueError:
        return address

Response: Convert an IPv6 address from binary form into text representation.


Instruction:
Create a Python 3 function to
convert an IP address from binary form into text representation.

Response:
def inet_ntop(af, addr):
    """Convert an IP address from binary form into text representation."""
    # Use inet_ntop if available
    addr = bytes_encode(addr)
    try:
        return socket.inet_ntop(af, addr)
    except AttributeError:
        try:
            return _INET_NTOP[af](addr)
        except KeyError:
            raise ValueError("unknown address family %d" % af)


Instruction:
Can you generate the documentation for the following Python 3 function
def recv_raw(self, x=MTU):
        ll = self.ins.datalink()
        if ll in conf.l2types:
            cls = conf.l2types[ll]
        else:
            cls = conf.default_l2
            warning("Unable to guess datalink type (interface=%s linktype=%i). Using %s",  # noqa: E501
                    self.iface, ll, cls.name)

        pkt = None
        while pkt is None:
            pkt = self.ins.next()
            if pkt is not None:
                ts, pkt = pkt
            if pkt is None and scapy.consts.WINDOWS:
                raise TimeoutElapsed  # To understand this behavior, have a look at L2pcapListenSocket's note  # noqa: E501
            if pkt is None:
                return None, None, None
        return cls, pkt, ts

Response: Receives a packet then returns a tuple containing cls pkt_data time


Instruction:
How would you explain what the following Python 3 function does
def nonblock_recv(self):
        self.ins.setnonblock(1)
        p = self.recv(MTU)
        self.ins.setnonblock(0)
        return p

Response: Receives and dissects a packet in non - blocking mode.


Instruction:
Explain what the following Python 3 code does
def delt(self, *args, **kargs):
        self.invalidate_cache()
        route = self.make_route(*args, **kargs)
        try:
            i = self.routes.index(route)
            del(self.routes[i])
        except ValueError:
            warning("no matching route found")

Response: delt - Delete a single entry from the cache


Instruction:
Implement a Python 3 function for
returning the IPv4 routes to a destination host.

Response:
def route(self, dst=None, verbose=conf.verb):
        """Returns the IPv4 routes to a host.
        parameters:
         - dst: the IPv4 of the destination host

        returns: (iface, output_ip, gateway_ip)
         - iface: the interface used to connect to the host
         - output_ip: the outgoing IP that will be used
         - gateway_ip: the gateway IP that will be used
        """
        dst = dst or "0.0.0.0"  # Enable route(None) to return default route
        if isinstance(dst, bytes):
            try:
                dst = plain_str(dst)
            except UnicodeDecodeError:
                raise TypeError("Unknown IP address input (bytes)")
        if dst in self.cache:
            return self.cache[dst]
        # Transform "192.168.*.1-5" to one IP of the set
        _dst = dst.split("/")[0].replace("*", "0")
        while True:
            idx = _dst.find("-")
            if idx < 0:
                break
            m = (_dst[idx:] + ".").find(".")
            _dst = _dst[:idx] + _dst[idx + m:]

        atol_dst = atol(_dst)
        paths = []
        for d, m, gw, i, a, me in self.routes:
            if not a:  # some interfaces may not currently be connected
                continue
            aa = atol(a)
            if aa == atol_dst:
                paths.append(
                    (0xffffffff, 1, (scapy.consts.LOOPBACK_INTERFACE, a, "0.0.0.0"))  # noqa: E501
                )
            if (atol_dst & m) == (d & m):
                paths.append((m, me, (i, a, gw)))

        if not paths:
            if verbose:
                warning("No route found (no default route?)")
            return scapy.consts.LOOPBACK_INTERFACE, "0.0.0.0", "0.0.0.0"
        # Choose the more specific route
        # Sort by greatest netmask and use metrics as a tie-breaker
        paths.sort(key=lambda x: (-x[0], x[1]))
        # Return interface
        ret = paths[0][2]
        self.cache[dst] = ret
        return ret


Instruction:
Create a Python 3 function to
update the internal state of the TLS session.

Response:
def tls_session_update(self, msg_str):
        """
        Either for parsing or building, we store the client_random
        along with the raw string representing this handshake message.
        """
        super(TLSClientHello, self).tls_session_update(msg_str)

        self.tls_session.advertised_tls_version = self.version
        self.random_bytes = msg_str[10:38]
        self.tls_session.client_random = (struct.pack('!I',
                                                      self.gmt_unix_time) +
                                          self.random_bytes)
        if self.ext:
            for e in self.ext:
                if isinstance(e, TLS_Ext_SupportedVersions):
                    if self.tls_session.tls13_early_secret is None:
                        # this is not recomputed if there was a TLS 1.3 HRR
                        self.tls_session.compute_tls13_early_secrets()
                    break


Instruction:
Write a Python 3 script for
updating the TLS session with the message string.

Response:
def tls_session_update(self, msg_str):
        """
        Either for parsing or building, we store the server_random along with
        the raw string representing this handshake message. We also store the
        cipher suite (if recognized), and finally we instantiate the write and
        read connection states.
        """
        super(TLSClientHello, self).tls_session_update(msg_str)

        s = self.tls_session
        s.tls_version = self.version
        s.server_random = self.random_bytes

        cs_cls = None
        if self.cipher:
            cs_val = self.cipher
            if cs_val not in _tls_cipher_suites_cls:
                warning("Unknown cipher suite %d from ServerHello" % cs_val)
                # we do not try to set a default nor stop the execution
            else:
                cs_cls = _tls_cipher_suites_cls[cs_val]

        connection_end = s.connection_end
        s.pwcs = writeConnState(ciphersuite=cs_cls,
                                connection_end=connection_end,
                                tls_version=self.version)
        s.triggered_pwcs_commit = True
        s.prcs = readConnState(ciphersuite=cs_cls,
                               connection_end=connection_end,
                               tls_version=self.version)
        s.triggered_prcs_commit = True

        if self.tls_session.tls13_early_secret is None:
            # In case the connState was not pre-initialized, we could not
            # compute the early secrets at the ClientHello, so we do it here.
            self.tls_session.compute_tls13_early_secrets()
        s.compute_tls13_handshake_secrets()


Instruction:
Implement a Python 3 function for
building the message with the appropriate parameters.

Response:
def build(self, *args, **kargs):
        """
        We overload build() method in order to provide a valid default value
        for params based on TLS session if not provided. This cannot be done by
        overriding i2m() because the method is called on a copy of the packet.

        The 'params' field is built according to key_exchange.server_kx_msg_cls
        which should have been set after receiving a cipher suite in a
        previous ServerHello. Usual cases are:
        - None: for RSA encryption or fixed FF/ECDH. This should never happen,
          as no ServerKeyExchange should be generated in the first place.
        - ServerDHParams: for ephemeral FFDH. In that case, the parameter to
          server_kx_msg_cls does not matter.
        - ServerECDH*Params: for ephemeral ECDH. There are actually three
          classes, which are dispatched by _tls_server_ecdh_cls_guess on
          the first byte retrieved. The default here is b"\03", which
          corresponds to ServerECDHNamedCurveParams (implicit curves).

        When the Server*DHParams are built via .fill_missing(), the session
        server_kx_privkey will be updated accordingly.
        """
        fval = self.getfieldval("params")
        if fval is None:
            s = self.tls_session
            if s.pwcs:
                if s.pwcs.key_exchange.export:
                    cls = ServerRSAParams(tls_session=s)
                else:
                    cls = s.pwcs.key_exchange.server_kx_msg_cls(b"\x03")
                    cls = cls(tls_session=s)
                try:
                    cls.fill_missing()
                except Exception:
                    if conf.debug_dissector:
                        raise
                    pass
            else:
                cls = Raw()
            self.params = cls

        fval = self.getfieldval("sig")
        if fval is None:
            s = self.tls_session
            if s.pwcs:
                if not s.pwcs.key_exchange.anonymous:
                    p = self.params
                    if p is None:
                        p = b""
                    m = s.client_random + s.server_random + raw(p)
                    cls = _TLSSignature(tls_session=s)
                    cls._update_sig(m, s.server_key)
                else:
                    cls = Raw()
            else:
                cls = Raw()
            self.sig = cls

        return _TLSHandshake.build(self, *args, **kargs)


Instruction:
Can you write a function in Python 3 where it
checks if the server_kx_pubkey is valid and verify the server_kx_pubkey.

Response:
def post_dissection(self, pkt):
        """
        While previously dissecting Server*DHParams, the session
        server_kx_pubkey should have been updated.

        XXX Add a 'fixed_dh' OR condition to the 'anonymous' test.
        """
        s = self.tls_session
        if s.prcs and s.prcs.key_exchange.no_ske:
            pkt_info = pkt.firstlayer().summary()
            log_runtime.info("TLS: useless ServerKeyExchange [%s]", pkt_info)
        if (s.prcs and
            not s.prcs.key_exchange.anonymous and
            s.client_random and s.server_random and
                s.server_certs and len(s.server_certs) > 0):
            m = s.client_random + s.server_random + raw(self.params)
            sig_test = self.sig._verify_sig(m, s.server_certs[0])
            if not sig_test:
                pkt_info = pkt.firstlayer().summary()
                log_runtime.info("TLS: invalid ServerKeyExchange signature [%s]", pkt_info)


Instruction:
Can you tell what is the following Python 3 function doing
def m2i(self, pkt, m):
        tmp_len = self.length_from(pkt)
        tbd, rem = m[:tmp_len], m[tmp_len:]

        s = pkt.tls_session
        cls = None

        if s.prcs and s.prcs.key_exchange:
            cls = s.prcs.key_exchange.client_kx_msg_cls

        if cls is None:
            return Raw(tbd) / Padding(rem)

        return cls(tbd, tls_session=s) / Padding(rem)

Response: Convert a client_kx_msg to an internal representation.


Instruction:
Explain what the following Python 3 code does
def nmap_fp(target, oport=80, cport=81):
    sigs = nmap_sig(target, oport, cport)
    return nmap_search(sigs)

Response:
nmap fingerprinting
nmap_fp - return a list of best guesses with accuracy


Instruction:
Given the following Python 3 function, write the documentation
def process_kwargs(obj, kwargs):
    '''
    Convenience wrapper around binwalk.core.module.Modules.kwargs.

    @obj    - The class object (an instance of a sub-class of binwalk.core.module.Module).
    @kwargs - The kwargs provided to the object's __init__ method.

    Returns None.
    '''
    with Modules() as m:
        kwargs = m.kwargs(obj, kwargs)
    return kwargs

Response: Wrapper around binwalk. core. module. Modules. kwargs.


Instruction:
How would you explain what the following Python 3 function does
def show_help(fd=sys.stdout):
    '''
    Convenience wrapper around binwalk.core.module.Modules.help.

    @fd - An object with a write method (e.g., sys.stdout, sys.stderr, etc).

    Returns None.
    '''
    with Modules() as m:
        fd.write(m.help())

Response: A convenience wrapper around binwalk. core. module. Modules. help. show_help


Instruction:
Implement a Python 3 function for
getting the next file to be scanned.

Response:
def next_file(self, close_previous=True):
        '''
        Gets the next file to be scanned (including pending extracted files, if applicable).
        Also re/initializes self.status.
        All modules should access the target file list through this method.
        '''
        fp = None

        # Ensure files are close to prevent IOError (too many open files)
        if close_previous:
            try:
                self.previous_next_file_fp.close()
            except KeyboardInterrupt as e:
                raise e
            except Exception:
                pass

        # Add any pending extracted files to the target_files list and reset
        # the extractor's pending file list
        self.target_file_list += self.extractor.pending

        # Reset all dependencies prior to continuing with another file.
        # This is particularly important for the extractor module, which must be reset
        # in order to reset its base output directory path for each file, and the
        # list of pending files.
        self.reset_dependencies()

        while self.target_file_list:
            next_target_file = self.target_file_list.pop(0)

            # Values in self.target_file_list are either already open files (BlockFile instances), or paths
            # to files that need to be opened for scanning.
            if isinstance(next_target_file, str):
                fp = self.config.open_file(next_target_file)
            else:
                fp = next_target_file

            if not fp:
                break
            else:
                if (self.config.file_name_filter(fp) == False or
                        self._plugins_load_file(fp) == False):
                    fp.close()
                    fp = None
                    continue
                else:
                    self.status.clear()
                    self.status.total = fp.length
                    break

        if fp is not None:
            self.current_target_file_name = fp.path
            self.status.fp = fp
        else:
            self.current_target_file_name = None
            self.status.fp = None

        self.previous_next_file_fp = fp

        self._plugins_new_file(fp)

        return fp


Instruction:
Can you create a Python 3 function that
clears the results and errors lists.

Response:
def clear(self, results=True, errors=True):
        '''
        Clears results and errors lists.
        '''
        if results:
            self.results = []
        if errors:
            self.errors = []


Instruction:
Explain what the following Python 3 code does
def result(self, r=None, **kwargs):
        '''
        Validates a result, stores it in self.results and prints it.
        Accepts the same kwargs as the binwalk.core.module.Result class.

        @r - An existing instance of binwalk.core.module.Result.

        Returns an instance of binwalk.core.module.Result.
        '''
        if r is None:
            r = Result(**kwargs)

        # Add the name of the current module to the result
        r.module = self.__class__.__name__

        # Any module that is reporting results, valid or not, should be marked
        # as enabled
        if not self.enabled:
            self.enabled = True

        self.validate(r)
        self._plugins_result(r)

        # Update the progress status automatically if it is not being done
        # manually by the module
        if r.offset and r.file and self.AUTO_UPDATE_STATUS:
            self.status.total = r.file.length
            self.status.completed = r.offset
            self.status.fp = r.file

        for dependency in self.dependencies:
            try:
                getattr(self, dependency.attribute).callback(r)
            except AttributeError:
                continue

        if r.valid:
            self.results.append(r)

            if r.display:
                display_args = self._build_display_args(r)
                if display_args:
                    self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)
                    self.config.display.result(*display_args)

        return r

Response: Validates a result stores it in self. results and prints it.


Instruction:
Write a Python 3 function that can
store the specified error in self. errors. A new error is created and the specified kwargs are passed to the Error class.

Response:
def error(self, **kwargs):
        '''
        Stores the specified error in self.errors.

        Accepts the same kwargs as the binwalk.core.module.Error class.

        Returns None.
        '''
        exception_header_width = 100

        e = Error(**kwargs)
        e.module = self.__class__.__name__

        self.errors.append(e)

        if e.exception:
            sys.stderr.write("\n" + e.module + " Exception: " + str(e.exception) + "\n")
            sys.stderr.write("-" * exception_header_width + "\n")
            traceback.print_exc(file=sys.stderr)
            sys.stderr.write("-" * exception_header_width + "\n\n")
        elif e.description:
            sys.stderr.write("\n" + e.module + " Error: " + e.description + "\n\n")


Instruction:
Make a summary of the following Python 3 code
def header(self):
        '''
        Displays the scan header, as defined by self.HEADER and self.HEADER_FORMAT.

        Returns None.
        '''
        self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)
        self.config.display.add_custom_header(self.VERBOSE_FORMAT, self.VERBOSE)

        if type(self.HEADER) == type([]):
            self.config.display.header(*self.HEADER, file_name=self.current_target_file_name)
        elif self.HEADER:
            self.config.display.header(self.HEADER, file_name=self.current_target_file_name)

Response:
Displays the scan header as defined by self. HEADER and self. HEADER_FORMAT.
        Returns None.


Instruction:
Can you tell what is the following Python 3 function doing
def main(self):
        '''
        Responsible for calling self.init, initializing self.config.display, and calling self.run.

        Returns the value returned from self.run.
        '''
        self.status = self.parent.status
        self.modules = self.parent.executed_modules

        # A special exception for the extractor module, which should be allowed to
        # override the verbose setting, e.g., if --matryoshka has been
        # specified
        if hasattr(self, "extractor") and self.extractor.config.verbose:
            self.config.verbose = self.config.display.verbose = True

        if not self.config.files:
            binwalk.core.common.debug("No target files specified, module %s terminated" % self.name)
            return False

        self.reset_dependencies()

        try:
            self.init()
        except KeyboardInterrupt as e:
            raise e
        except Exception as e:
            self.error(exception=e)
            return False

        try:
            self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)
        except KeyboardInterrupt as e:
            raise e
        except Exception as e:
            self.error(exception=e)
            return False

        self._plugins_pre_scan()

        try:
            retval = self.run()
        except KeyboardInterrupt as e:
            raise e
        except Exception as e:
            self.error(exception=e)
            return False

        self._plugins_post_scan()

        return retval

Response: Main entry point for the base class.


Instruction:
How would you explain what the following Python 3 function does
def list(self, attribute="run"):
        '''
        Finds all modules with the specified attribute.

        @attribute - The desired module attribute.

        Returns a list of modules that contain the specified attribute, in the order they should be executed.
        '''
        import binwalk.modules
        modules = {}

        for (name, module) in inspect.getmembers(binwalk.modules):
            if inspect.isclass(module) and hasattr(module, attribute):
                modules[module] = module.PRIORITY

        # user-defined modules
        import imp
        user_modules = binwalk.core.settings.Settings().user.modules
        for file_name in os.listdir(user_modules):
            if not file_name.endswith('.py'):
                continue
            module_name = file_name[:-3]
            try:
                user_module = imp.load_source(module_name, os.path.join(user_modules, file_name))
            except KeyboardInterrupt as e:
                raise e
            except Exception as e:
                binwalk.core.common.warning("Error loading module '%s': %s" % (file_name, str(e)))

            for (name, module) in inspect.getmembers(user_module):
                if inspect.isclass(module) and hasattr(module, attribute):
                    modules[module] = module.PRIORITY

        return sorted(modules, key=modules.get, reverse=True)

Response: Returns a list of modules that contain the specified attribute.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def help(self):
        '''
        Generates formatted help output.

        Returns the help string.
        '''
        modules = {}
        help_string = "\n"
        help_string += "Binwalk v%s\n" % binwalk.__version__
        help_string += "Craig Heffner, ReFirmLabs\n"
        help_string += "https://github.com/ReFirmLabs/binwalk\n"
        help_string += "\n"
        help_string += "Usage: binwalk [OPTIONS] [FILE1] [FILE2] [FILE3] ...\n"

        # Build a dictionary of modules and their ORDER attributes.
        # This makes it easy to sort modules by their ORDER attribute for
        # display.
        for module in self.list(attribute="CLI"):
            if module.CLI:
                modules[module] = module.ORDER

        for module in sorted(modules, key=modules.get, reverse=True):
            help_string += "\n%s Options:\n" % module.TITLE

            for module_option in module.CLI:
                if module_option.long and not module_option.hidden:
                    long_opt = '--' + module_option.long

                    if module_option.dtype:
                        optargs = "=<%s>" % module_option.dtype
                    else:
                        optargs = ""

                    if module_option.short:
                        short_opt = "-" + module_option.short + ","
                    else:
                        short_opt = "   "

                    fmt = "    %%s %%s%%-%ds%%s\n" % (25 - len(long_opt))
                    help_string += fmt % (short_opt, long_opt, optargs, module_option.description)

        return help_string + "\n"

Response: Generates formatted help output.


Instruction:
Make a summary of the following Python 3 code
def execute(self, *args, **kwargs):
        '''
        Executes all appropriate modules according to the options specified in args/kwargs.

        Returns a list of executed module objects.
        '''
        run_modules = []
        orig_arguments = self.arguments

        if args or kwargs:
            self._set_arguments(list(args), kwargs)

        # Run all modules
        for module in self.list():
            obj = self.run(module)

        # Add all loaded modules that marked themselves as enabled to the
        # run_modules list
        for (module, obj) in iterator(self.executed_modules):
            # Report the results if the module is enabled and if it is a
            # primary module or if it reported any results/errors
            if obj.enabled and (obj.PRIMARY or obj.results or obj.errors):
                run_modules.append(obj)

        self.arguments = orig_arguments

        return run_modules

Response: Executes all appropriate modules according to the options specified in args or kwargs. Returns a list of module objects that were executed.


Instruction:
Can you write a function in Python 3 where it
runs a specific module.

Response:
def run(self, module, dependency=False, kwargs={}):
        '''
        Runs a specific module.
        '''
        try:
            obj = self.load(module, kwargs)

            if isinstance(obj, binwalk.core.module.Module) and obj.enabled:
                obj.main()
                self.status.clear()

            # If the module is not being loaded as a dependency, add it to the executed modules dictionary.
            # This is used later in self.execute to determine which objects
            # should be returned.
            if not dependency:
                self.executed_modules[module] = obj

                # The unload method tells the module that we're done with it, and gives it a chance to do
                # any cleanup operations that may be necessary. We still retain
                # the object instance in self.executed_modules.
                obj._unload_dependencies()
                obj.unload()
        except KeyboardInterrupt as e:
            # Tell the status server to shut down, and give it time to clean
            # up.
            if self.status.running:
                self.status.shutdown = True
                while not self.status.finished:
                    time.sleep(0.1)
            raise e

        return obj


Instruction:
Write a Python 3 script for
processing the command line arguments for any specific module.

Response:
def argv(self, module, argv=sys.argv[1:]):
        '''
        Processes argv for any options specific to the specified module.

        @module - The module to process argv for.
        @argv   - A list of command line arguments (excluding argv[0]).

        Returns a dictionary of kwargs for the specified module.
        '''
        kwargs = {'enabled': False}
        last_priority = {}
        parser = argparse.ArgumentParser(add_help=False)
        # Hack: This allows the ListActionParser class to correllate short options to long options.
        # There is probably a built-in way to do this in the
        # argparse.ArgumentParser class?
        parser.short_to_long = {}

        # Must build arguments from all modules so that:
        #
        #    1) Any conflicting arguments will raise an exception
        #    2) The only unknown arguments will be the target files, making them
        #       easy to identify
        for m in self.list(attribute="CLI"):

            for module_option in m.CLI:

                parser_args = []
                parser_kwargs = {}

                if not module_option.long:
                    continue

                if module_option.short:
                    parser_args.append('-' + module_option.short)
                parser_args.append('--' + module_option.long)
                parser_kwargs['dest'] = module_option.long

                if module_option.type is None:
                    parser_kwargs['action'] = 'store_true'
                elif module_option.type is list:
                    parser_kwargs['action'] = 'append'
                    parser.short_to_long[
                        module_option.short] = module_option.long

                parser.add_argument(*parser_args, **parser_kwargs)

        args, unknown = parser.parse_known_args(argv)
        args = args.__dict__

        # Only add parsed options pertinent to the requested module
        for module_option in module.CLI:

            if module_option.type == binwalk.core.common.BlockFile:

                for k in get_keys(module_option.kwargs):
                    kwargs[k] = []
                    for unk in unknown:
                        kwargs[k].append(unk)

            elif has_key(args, module_option.long) and args[module_option.long] not in [None, False]:

                # Loop through all the kwargs for this command line option
                for (name, default_value) in iterator(module_option.kwargs):

                    # If this kwarg has not been previously processed, or if its priority is equal to or
                    # greater than the previously processed kwarg's priority,
                    # then let's process it.
                    if not has_key(last_priority, name) or last_priority[name] <= module_option.priority:

                        # Track the priority for future iterations that may
                        # process the same kwarg name
                        last_priority[name] = module_option.priority

                        try:
                            kwargs[name] = module_option.convert(args[module_option.long], default_value)
                        except KeyboardInterrupt as e:
                            raise e
                        except Exception as e:
                            raise ModuleException("Invalid usage: %s" % str(e))

        binwalk.core.common.debug("%s :: %s => %s" % (module.TITLE, str(argv), str(kwargs)))
        return kwargs


Instruction:
How would you code a function in Python 3 to
process a module s kwargs. All modules should use this for kwarg processing.

Response:
def kwargs(self, obj, kwargs):
        '''
        Processes a module's kwargs. All modules should use this for kwarg processing.

        @obj    - An instance of the module (e.g., self)
        @kwargs - The kwargs passed to the module

        Returns None.
        '''
        if hasattr(obj, "KWARGS"):
            for module_argument in obj.KWARGS:
                if has_key(kwargs, module_argument.name):
                    arg_value = kwargs[module_argument.name]
                else:
                    arg_value = copy(module_argument.default)

                setattr(obj, module_argument.name, arg_value)

            for (k, v) in iterator(kwargs):
                if not hasattr(obj, k):
                    setattr(obj, k, v)
        else:
            raise Exception("binwalk.core.module.Modules.process_kwargs: %s has no attribute 'KWARGS'" % str(obj))


Instruction:
How would you implement a function in Python 3 that
starts the status server on the specified port.

Response:
def status_server(self, port):
        '''
        Starts the progress bar TCP service on the specified port.
        This service will only be started once per instance, regardless of the
        number of times this method is invoked.

        Failure to start the status service is considered non-critical; that is,
        a warning will be displayed to the user, but normal operation will proceed.
        '''
        if self.status_server_started == False:
            self.status_server_started = True
            try:
                self.status_service = binwalk.core.statuserver.StatusServer(port, self)
            except Exception as e:
                binwalk.core.common.warning("Failed to start status server on port %d: %s" % (port, str(e)))


Instruction:
Here you have a function in Python 3, explain what it does
def _make_short(self, data, endianness):
        data = binwalk.core.compat.str2bytes(data)
        return struct.unpack('%sH' % endianness, data)[0]

Response: Returns a 2 byte integer.


Instruction:
Here you have a function in Python 3, explain what it does
def _get_fname_len(self, bufflen=128):
        buff = self.meta.peek(bufflen)
        strlen = buff.find('\0')
        for i, b in enumerate(buff[strlen:]):
            if b != '\0':
                return strlen+i
        return bufflen

Response: Returns the number of bytes designated for the filename.


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_node(self):
        data = self.meta.read(self.node_size)
        return PFSNode(data, self.endianness)

Response: Reads a chunk of meta data from file and returns a PFSNode.


Instruction:
Can you create a Python 3 function that
returns an iterator over the file meta - data entries one by one.

Response:
def entries(self):
        """Returns file meta-data entries one by one."""
        self.meta.seek(self.file_list_start)
        for i in range(0, self.num_files):
            yield self._get_node()


Instruction:
Write a Python 3 script for
extracting the actual string from the available bytes.

Response:
def _decode_fname(self):
        """Extracts the actual string from the available bytes."""
        self.fname = self.fname[:self.fname.find('\0')]
        self.fname = self.fname.replace('\\', '/')


Instruction:
How would you explain what the following Python 3 function does
def _generate_regex(self, line):
        '''
        Generates a regular expression from the magic bytes of a signature.
        The regex is used by Magic._analyze.

        @line - The first SignatureLine object of the signature.

        Returns a compile regular expression.
        '''
        restr = ""

        # Strings and single byte signatures are taken at face value;
        # multi-byte integer values are turned into regex strings based
        # on their data type size and endianness.
        if line.type == 'regex':
            # Regex types are already compiled expressions.
            # Note that since re.finditer is used, unless the specified
            # regex accounts for it, overlapping signatures will be ignored.
            return line.value
        if line.type == 'string':
            restr = line.value
        elif line.size == 1:
            restr = chr(line.value)
        elif line.size == 2:
            if line.endianness == '<':
                restr = chr(line.value & 0xFF) + chr(line.value >> 8)
            elif line.endianness == '>':
                restr = chr(line.value >> 8) + chr(line.value & 0xFF)
        elif line.size == 4:
            if line.endianness == '<':
                restr = (chr(line.value & 0xFF) +
                         chr((line.value >> 8) & 0xFF) +
                         chr((line.value >> 16) & 0xFF) +
                         chr(line.value >> 24))
            elif line.endianness == '>':
                restr = (chr(line.value >> 24) +
                         chr((line.value >> 16) & 0xFF) +
                         chr((line.value >> 8) & 0xFF) +
                         chr(line.value & 0xFF))
        elif line.size == 8:
            if line.endianness == '<':
                restr = (chr(line.value & 0xFF) +
                         chr((line.value >> 8) & 0xFF) +
                         chr((line.value >> 16) & 0xFF) +
                         chr((line.value >> 24) & 0xFF) +
                         chr((line.value >> 32) & 0xFF) +
                         chr((line.value >> 40) & 0xFF) +
                         chr((line.value >> 48) & 0xFF) +
                         chr(line.value >> 56))
            elif line.endianness == '>':
                restr = (chr(line.value >> 56) +
                         chr((line.value >> 48) & 0xFF) +
                         chr((line.value >> 40) & 0xFF) +
                         chr((line.value >> 32) & 0xFF) +
                         chr((line.value >> 24) & 0xFF) +
                         chr((line.value >> 16) & 0xFF) +
                         chr((line.value >> 8) & 0xFF) +
                         chr(line.value & 0xFF))

        # Since re.finditer is used on a per-signature basis, signatures should be crafted carefully
        # to ensure that they aren't potentially self-overlapping (e.g., a signature of "ABCDAB" could
        # be confused by the byte sequence "ABCDABCDAB"). The longer the signature, the less likely an
        # unintentional overlap is, although files could still be maliciously crafted to cause false
        # negative results.
        #
        # Thus, unless a signature has been explicitly marked as knowingly overlapping ('{overlap}'),
        # spit out a warning about any self-overlapping signatures.
        if not binwalk.core.compat.has_key(line.tags, 'overlap'):
            for i in range(1, line.size):
                if restr[i:] == restr[0:(line.size - i)]:
                    binwalk.core.common.warning("Signature '%s' is a self-overlapping signature!" % line.text)
                    break

        return re.compile(re.escape(restr))

Response: Generates a regular expression from the magic bytes of a signature.


Instruction:
How would you explain what the following Python 3 function does
def _filtered(self, text):
        '''
        Tests if a string should be filtered out or not.

        @text - The string to check against filter rules.

        Returns True if the string should be filtered out, i.e., not displayed.
        Returns False if the string should be displayed.
        '''
        filtered = None
        # Text is converted to lower case first, partially for historical
        # purposes, but also because it simplifies writing filter rules
        # (e.g., don't have to worry about case sensitivity).
        text = text.lower()

        for include in self.includes:
            if include.search(text):
                filtered = False
                break

        # If exclusive include filters have been specified and did
        # not match the text, then the text should be filtered out.
        if self.includes and filtered == None:
            return True

        for exclude in self.excludes:
            if exclude.search(text):
                filtered = True
                break

        # If no explicit exclude filters were matched, then the
        # text should *not* be filtered.
        if filtered == None:
            filtered = False

        return filtered

Response: Tests if a string should be filtered out or not.


Instruction:
Here you have a function in Python 3, explain what it does
def _do_math(self, offset, expression):
        '''
        Parses and evaluates complex expressions, e.g., "(4.l+12)", "(6*32)", etc.

        @offset      - The offset inside self.data that the current signature starts at.
        @expressions - The expression to evaluate.

        Returns an integer value that is the result of the evaluated expression.
        '''
        # Does the expression contain an offset (e.g., "(4.l+12)")?
        if '.' in expression and '(' in expression:
            replacements = {}

            for period in [match.start() for match in self.period.finditer(expression)]:
                # Separate the offset field into the integer offset and type
                # values (o and t respsectively)
                s = expression[:period].rfind('(') + 1
                # The offset address may be an evaluatable expression, such as '(4+0.L)', typically the result
                # of the original offset being something like '(&0.L)'.
                o = binwalk.core.common.MathExpression(expression[s:period]).value
                t = expression[period + 1]

                # Re-build just the parsed offset portion of the expression
                text = "%s.%c" % (expression[s:period], t)

                # Have we already evaluated this offset expression? If so, skip
                # it.
                if binwalk.core.common.has_key(replacements, text):
                    continue

                # The offset specified in the expression is relative to the
                # starting offset inside self.data
                o += offset

                # Read the value from self.data at the specified offset
                try:
                    # Big and little endian byte format
                    if t in ['b', 'B']:
                        v = struct.unpack('b', binwalk.core.compat.str2bytes(self.data[o:o + 1]))[0]
                    # Little endian short format
                    elif t == 's':
                        v = struct.unpack('<h', binwalk.core.compat.str2bytes(self.data[o:o + 2]))[0]
                    # Little endian long format
                    elif t == 'l':
                        v = struct.unpack('<i', binwalk.core.compat.str2bytes(self.data[o:o + 4]))[0]
                    # Big endian short format
                    elif t == 'S':
                        v = struct.unpack('>h', binwalk.core.compat.str2bytes(self.data[o:o + 2]))[0]
                    # Bit endian long format
                    elif t == 'L':
                        v = struct.unpack('>i', binwalk.core.compat.str2bytes(self.data[o:o + 4]))[0]
                # struct.error is thrown if there is not enough bytes in
                # self.data for the specified format type
                except struct.error as e:
                    v = 0

                # Keep track of all the recovered values from self.data
                replacements[text] = v

            # Finally, replace all offset expressions with their corresponding
            # text value
            v = expression
            for (text, value) in binwalk.core.common.iterator(replacements):
                v = v.replace(text, "%d" % value)

        # If no offset, then it's just an evaluatable math expression (e.g.,
        # "(32+0x20)")
        else:
            v = expression

        # Evaluate the final expression
        value = binwalk.core.common.MathExpression(v).value

        return value

Response: Parse and evaluate a complex expression and return the value of the expression.


Instruction:
How would you explain what the following Python 3 function does
def _analyze(self, signature, offset):
        '''
        Analyzes self.data for the specified signature data at the specified offset .

        @signature - The signature to apply to the data.
        @offset    - The offset in self.data to apply the signature to.

        Returns a dictionary of tags parsed from the data.
        '''
        description = []
        max_line_level = 0
        previous_line_end = 0
        tags = {'id': signature.id, 'offset':
                offset, 'invalid': False, 'once': False}

        # Apply each line of the signature to self.data, starting at the
        # specified offset
        for n in range(0, len(signature.lines)):
            line = signature.lines[n]

            # Ignore indentation levels above the current max indent level
            if line.level <= max_line_level:
                # If the relative offset of this signature line is just an
                # integer value, use it
                if isinstance(line.offset, int):
                    line_offset = line.offset
                # Else, evaluate the complex expression
                else:
                    # Format the previous_line_end value into a string. Add the '+' sign to explicitly
                    # state that this value is to be added to any subsequent values in the expression
                    # (e.g., '&0' becomes '4+0').
                    ple = '%d+' % previous_line_end
                    # Allow users to use either the '&0' (libmagic) or '&+0' (explcit addition) sytaxes;
                    # replace both with the ple text.
                    line_offset_text = line.offset.replace('&+', ple).replace('&', ple)
                    # Evaluate the expression
                    line_offset = self._do_math(offset, line_offset_text)

                # Sanity check
                if not isinstance(line_offset, int):
                    raise ParserException("Failed to convert offset '%s' to a number: '%s'" % (line.offset, line.text))

                # The start of the data needed by this line is at offset + line_offset.
                # The end of the data will be line.size bytes later.
                start = offset + line_offset
                end = start + line.size

                # If the line has a packed format string, unpack it
                if line.pkfmt:
                    try:
                        dvalue = struct.unpack(line.pkfmt, binwalk.core.compat.str2bytes(self.data[start:end]))[0]
                    # Not enough bytes left in self.data for the specified
                    # format size
                    except struct.error as e:
                        dvalue = 0
                # Else, this is a string
                else:
                    # Wildcard strings have line.value == None
                    if line.value is None:
                        # Check to see if this is a string whose size is known and has been specified on a previous
                        # signature line.
                        if binwalk.core.compat.has_key(tags, 'strlen') and binwalk.core.compat.has_key(line.tags, 'string'):
                            dvalue = self.data[start:(start + tags['strlen'])]
                        # Else, just terminate the string at the first newline,
                        # carriage return, or NULL byte
                        else:
                            dvalue = self.data[start:end].split('\x00')[0].split('\r')[0].split('\n')[0]
                    # Non-wildcard strings have a known length, specified in
                    # the signature line
                    else:
                        dvalue = self.data[start:end]

                # Some integer values have special operations that need to be performed on them
                # before comparison (e.g., "belong&0x0000FFFF"). Complex math expressions are
                # supported here as well.
                # if isinstance(dvalue, int) and line.operator:
                if line.operator:
                    try:
                        # If the operator value of this signature line is just
                        # an integer value, use it
                        if isinstance(line.opvalue, int) or isinstance(line.opvalue, long):
                            opval = line.opvalue
                        # Else, evaluate the complex expression
                        else:
                            opval = self._do_math(offset, line.opvalue)

                        # Perform the specified operation
                        if line.operator == '&':
                            dvalue &= opval
                        elif line.operator == '|':
                            dvalue |= opval
                        elif line.operator == '*':
                            dvalue *= opval
                        elif line.operator == '+':
                            dvalue += opval
                        elif line.operator == '-':
                            dvalue -= opval
                        elif line.operator == '/':
                            dvalue /= opval
                        elif line.operator == '~':
                            dvalue = ~opval
                        elif line.operator == '^':
                            dvalue ^= opval
                    except KeyboardInterrupt as e:
                        raise e
                    except Exception as e:
                        raise ParserException("Operation '" +
                                              str(dvalue) +
                                              " " +
                                              str(line.operator) +
                                              "= " +
                                              str(line.opvalue) +
                                              "' failed: " + str(e))

                # Does the data (dvalue) match the specified comparison?
                if ((line.value is None) or
                    (line.regex and line.value.match(dvalue)) or
                    (line.condition == '=' and dvalue == line.value) or
                    (line.condition == '>' and dvalue > line.value) or
                    (line.condition == '<' and dvalue < line.value) or
                    (line.condition == '!' and dvalue != line.value) or
                    (line.condition == '~' and (dvalue == ~line.value)) or
                    (line.condition == '^' and (dvalue ^ line.value)) or
                    (line.condition == '&' and (dvalue & line.value)) or
                        (line.condition == '|' and (dvalue | line.value))):

                    # Up until this point, date fields are treated as integer values,
                    # but we want to display them as nicely formatted strings.
                    if line.type == 'date':
                        try:
                            ts = datetime.datetime.utcfromtimestamp(dvalue)
                            dvalue = ts.strftime("%Y-%m-%d %H:%M:%S")
                        except KeyboardInterrupt as e:
                            raise e
                        except Exception:
                            dvalue = "invalid timestamp"

                    # Generate the tuple for the format string
                    dvalue_tuple = ()
                    for x in self.fmtstr.finditer(line.format):
                        dvalue_tuple += (dvalue,)

                    # Format the description string
                    desc = line.format % dvalue_tuple

                    # If there was any description string, append it to the
                    # list of description string parts
                    if desc:
                        description.append(desc)

                    # Process tag keywords specified in the signature line. These have already been parsed out of the
                    # original format string so that they can be processed
                    # separately from the printed description string.
                    for (tag_name, tag_value) in binwalk.core.compat.iterator(line.tags):
                        # If the tag value is a string, try to format it
                        if isinstance(tag_value, str):
                            # Generate the tuple for the format string
                            dvalue_tuple = ()
                            for x in self.fmtstr.finditer(tag_value):
                                dvalue_tuple += (dvalue,)

                            # Format the tag string
                            tags[tag_name] = tag_value % dvalue_tuple
                        # Else, just use the raw tag value
                        else:
                            tags[tag_name] = tag_value

                        # Some tag values are intended to be integer values, so
                        # try to convert them as such
                        try:
                            tags[tag_name] = int(tags[tag_name], 0)
                        except KeyboardInterrupt as e:
                            raise e
                        except Exception as e:
                            pass

                    # Abort processing soon as this signature is marked invalid, unless invalid results
                    # were explicitly requested. This means that the sooner invalid checks are made in a
                    # given signature, the faster the scan can filter out false
                    # positives.
                    if not self.show_invalid and tags['invalid']:
                        break

                    # Look ahead to the next line in the signature; if its indent level is greater than
                    # that of the current line, then track the end of data for the current line. This is
                    # so that subsequent lines can use the '>>&0' offset syntax to specify relative offsets
                    # from previous lines.
                    try:
                        next_line = signature.lines[n + 1]
                        if next_line.level > line.level:
                            if line.type == 'string':
                                previous_line_end = line_offset + len(dvalue)
                            else:
                                previous_line_end = line_offset + line.size
                    except IndexError as e:
                        pass

                    # If this line satisfied its comparison, +1 the max
                    # indentation level
                    max_line_level = line.level + 1
                else:
                    # No match on the first line, abort
                    if line.level == 0:
                        break
                    else:
                        # If this line did not satisfy its comparison, then higher
                        # indentation levels will not be accepted.
                        max_line_level = line.level

        # Join the formatted description strings and remove backspace
        # characters (plus the preceeding character as well)
        tags['description'] = self.bspace.sub('', " ".join(description))

        # This should never happen
        if not tags['description']:
            tags['display'] = False
            tags['invalid'] = True

        # If the formatted string contains non-printable characters, consider
        # it invalid
        if self.printable.match(tags['description']).group() != tags['description']:
            tags['invalid'] = True

        return tags

Response: Analyzes the data for the specified signature at the specified offset.


Instruction:
How would you explain what the following Python 3 function does
def scan(self, data, dlen=None):
        '''
        Scan a data block for matching signatures.

        @data - A string of data to scan.
        @dlen - If specified, signatures at offsets larger than dlen will be ignored.

        Returns a list of SignatureResult objects.
        '''
        results = []
        matched_offsets = set()

        # Since data can potentially be quite a large string, make it available to other
        # methods via a class attribute so that it doesn't need to be passed around to
        # different methods over and over again.
        self.data = data

        # If dlen wasn't specified, search all of self.data
        if dlen is None:
            dlen = len(data)

        for signature in self.signatures:
            # Use regex to search the data block for potential signature
            # matches (fast)
            for match in signature.regex.finditer(data):
                # Take the offset of the start of the signature into account
                offset = match.start() - signature.offset

                # Signatures are ordered based on the length of their magic bytes (largest first).
                # If this offset has already been matched to a previous signature, ignore it unless
                # self.show_invalid has been specified. Also ignore obviously invalid offsets (<0)
                # as well as those outside the specified self.data range (dlen).
                if (offset not in matched_offsets or self.show_invalid) and offset >= 0 and offset < dlen:
                # if offset >= 0 and offset < dlen:
                    # Analyze the data at this offset using the current
                    # signature rule
                    tags = self._analyze(signature, offset)

                    # Generate a SignatureResult object and append it to the results list if the
                    # signature is valid, or if invalid results were requested.
                    if (not tags['invalid'] or self.show_invalid) and not self._filtered(tags['description']):
                        # Only display results with the 'once' tag once.
                        if tags['once']:
                            if signature.title in self.display_once:
                                continue
                            else:
                                self.display_once.add(signature.title)

                        # Append the result to the results list
                        results.append(SignatureResult(**tags))

                        # Add this offset to the matched_offsets set, so that it can be ignored by
                        # subsequent loops.
                        matched_offsets.add(offset)

        # Sort results by offset
        results.sort(key=lambda x: x.offset, reverse=False)

        return results

Response: Scan a data block for matching signatures.


Instruction:
Can you implement a function in Python 3 that
loads signatures from a file.

Response:
def load(self, fname):
        '''
        Load signatures from a file.

        @fname - Path to signature file.

        Returns None.
        '''
        # Magic files must be ASCII, else encoding issues can arise.
        fp = open(fname, "r")
        lines = fp.readlines()
        self.parse(lines)
        fp.close()


Instruction:
Create a Python 3 function for
parsing the signature file lines.

Response:
def parse(self, lines):
        '''
        Parse signature file lines.

        @lines - A list of lines from a signature file.

        Returns None.
        '''
        signature = None

        for line in lines:
            # Split at the first comment delimiter (if any) and strip the
            # result
            line = line.split('#')[0].strip()
            # Ignore blank lines and lines that are nothing but comments.
            # We also don't support the '!mime' style line entries.
            if line and line[0] != '!':
                # Parse this signature line
                sigline = SignatureLine(line)
                # Level 0 means the first line of a signature entry
                if sigline.level == 0:
                    # If there is an existing signature, append it to the signature list,
                    # unless the text in its title field has been filtered by user-defined
                    # filter rules.
                    if signature and not self._filtered(signature.title):
                        self.signatures.append(signature)

                    # Create a new signature object; use the size of self.signatures to
                    # assign each signature a unique ID.
                    signature = Signature(len(self.signatures), sigline)
                # Else, just append this line to the existing signature
                elif signature:
                    # signature.append(sigline)
                    signature.lines.append(sigline)
                # If this is not the first line of a signature entry and there is no other
                # existing signature entry, something is very wrong with the
                # signature file.
                else:
                    raise ParserException("Invalid signature line: '%s'" % line)

        # Add the final signature to the signature list
        if signature:
            if not self._filtered(signature.lines[0].format):
                self.signatures.append(signature)

        # Sort signatures by confidence (aka, length of their magic bytes),
        # largest first
        self.signatures.sort(key=lambda x: x.confidence, reverse=True)


Instruction:
Can you generate the documentation for the following Python 3 function
def _set_verbosity(self):
        '''
        Sets the appropriate verbosity.
        Must be called after self._test_target_files so that self.target_files is properly set.
        '''
        # If more than one target file was specified, enable verbose mode; else, there is
        # nothing in some outputs to indicate which scan corresponds to which
        # file.
        if len(self.target_files) > 1 and not self.verbose:
            self.verbose = True

Response: Sets the appropriate verbosity.


Instruction:
Implement a function in Python 3 to
check to see if a file should be scanned based on file name include and exclude filters.

Response:
def file_name_filter(self, fp):
        '''
        Checks to see if a file should be scanned based on file name include/exclude filters.
        Most useful for matryoshka scans where only certian files are desired.

        @fp - An instances of binwalk.common.BlockFile

        Returns True if the file should be scanned, False if not.
        '''
        if self.file_name_include_regex and not self.file_name_include_regex.search(fp.name):
            return False
        if self.file_name_exclude_regex and self.file_name_exclude_regex.search(fp.name):
            return False

        return True


Instruction:
How would you implement a function in Python 3 that
opens the specified file with all pertinent configuration settings.

Response:
def open_file(self, fname, length=None, offset=None, swap=None, block=None, peek=None):
        '''
        Opens the specified file with all pertinent configuration settings.
        '''
        if length is None:
            length = self.length
        if offset is None:
            offset = self.offset
        if swap is None:
            swap = self.swap_size

        return binwalk.core.common.BlockFile(fname,
                                             subclass=self.subclass,
                                             length=length,
                                             offset=offset,
                                             swap=swap,
                                             block=block,
                                             peek=peek)


Instruction:
How would you explain what the following Python 3 function does
def _open_target_files(self):
        '''
        Checks if the target files can be opened.
        Any files that cannot be opened are removed from the self.target_files list.
        '''
        # Validate the target files listed in target_files
        for tfile in self.files:
            # Ignore directories.
            if not self.subclass == io.FileIO or not os.path.isdir(tfile):
                # Make sure we can open the target files
                try:
                    fp = self.open_file(tfile)
                    fp.close()
                    self.target_files.append(tfile)
                except KeyboardInterrupt as e:
                    raise e
                except Exception as e:
                    self.error(description="Cannot open file %s (CWD: %s) : %s" % (tfile, os.getcwd(), str(e)))

Response: Checks if the target files can be opened.


Instruction:
How would you implement a function in Python 3 that
validates the result of a single entry.

Response:
def validate(self, r):
        '''
        Called automatically by self.result.
        '''
        if self.show_invalid:
            r.valid = True
        elif r.valid:
            if not r.description:
                r.valid = False

            if r.size and (r.size + r.offset) > r.file.size:
                r.valid = False

            if r.jump and (r.jump + r.offset) > r.file.size:
                r.valid = False

            if hasattr(r, "location") and (r.location != r.offset):
                r.valid = False

        if r.valid:
            # Don't keep displaying signatures that repeat a bunch of times
            # (e.g., JFFS2 nodes)
            if r.id == self.one_of_many:
                r.display = False
            elif r.many:
                self.one_of_many = r.id
            else:
                self.one_of_many = None


Instruction:
Implement a function in Python 3 to
obtain a list of all user and system plugin modules.

Response:
def list_plugins(self):
        '''
        Obtain a list of all user and system plugin modules.

        Returns a dictionary of:

            {
                'user'      : {
                            'modules'       : [list, of, module, names],
                            'descriptions'  : {'module_name' : 'module pydoc string'},
                            'enabled'       : {'module_name' : True},
                            'path'          : "path/to/module/plugin/directory"
                },
                'system'    : {
                            'modules'       : [list, of, module, names],
                            'descriptions'  : {'module_name' : 'module pydoc string'},
                            'enabled'       : {'module_name' : True},
                            'path'          : "path/to/module/plugin/directory"
                }
            }
        '''

        plugins = {
            'user': {
                'modules': [],
                'descriptions': {},
                'enabled': {},
                'path': None,
            },
            'system': {
                'modules': [],
                'descriptions': {},
                'enabled': {},
                'path': None,
            }
        }

        for key in plugins.keys():
            if key == 'user':
                plugins[key]['path'] = self.settings.user.plugins
            else:
                plugins[key]['path'] = self.settings.system.plugins

            if plugins[key]['path']:
                for file_name in os.listdir(plugins[key]['path']):
                    if file_name.endswith(self.MODULE_EXTENSION):
                        module = file_name[:-len(self.MODULE_EXTENSION)]

                        try:
                            plugin = imp.load_source(module, os.path.join(plugins[key]['path'], file_name))
                            plugin_class = self._find_plugin_class(plugin)

                            plugins[key]['enabled'][module] = True
                            plugins[key]['modules'].append(module)
                        except KeyboardInterrupt as e:
                            raise e
                        # Python files in the plugins directory that are not
                        # actually binwalk plugins will generate a TypeError
                        # about converting an object to a string implicitly.
                        # Don't need to warn about these.
                        except TypeError:
                            pass
                        except Exception as e:
                            binwalk.core.common.warning("Error loading plugin '%s': %s" % (file_name, str(e)))
                            plugins[key]['enabled'][module] = False

                        try:
                            plugins[key]['descriptions'][
                                module] = plugin_class.__doc__.strip().split('\n')[0]
                        except KeyboardInterrupt as e:
                            raise e
                        except Exception as e:
                            plugins[key]['descriptions'][
                                module] = 'No description'
        return plugins


Instruction:
Can you create a Python 3 function that
calculates Shannon entropy on a given block of data.

Response:
def shannon(self, data):
        '''
        Performs a Shannon entropy analysis on a given block of data.
        '''
        entropy = 0

        if data:
            length = len(data)

            seen = dict(((chr(x), 0) for x in range(0, 256)))
            for byte in data:
                seen[byte] += 1

            for x in range(0, 256):
                p_x = float(seen[chr(x)]) / length
                if p_x > 0:
                    entropy -= p_x * math.log(p_x, 2)

        return (entropy / 8)


Instruction:
Make a summary of the following Python 3 code
def gzip(self, data, truncate=True):
        '''
        Performs an entropy analysis based on zlib compression ratio.
        This is faster than the shannon entropy analysis, but not as accurate.
        '''
        # Entropy is a simple ratio of: <zlib compressed size> / <original
        # size>
        e = float(float(len(zlib.compress(str2bytes(data), 9))) / float(len(data)))

        if truncate and e > 1.0:
            e = 1.0

        return e

Response: Calculates the entropy of a file.


Instruction:
Make a summary of the following Python 3 code
def nts(self, s):
        # Use the string up to the first null char.
        p = s.find("\0")
        if p == -1:
            return s
        return s[:p]

Response: Convert a null - terminated string field to a python string.


Instruction:
Given the following Python 3 function, write the documentation
def nti(self, s):
        # There are two possible encodings for a number field, see
        # itn() below.
        if s[0] != chr(0x80):
            try:
                n = int(self.nts(s) or "0", 8)
            except ValueError:
                raise ValueError("invalid tar header")
        else:
            n = 0
            for i in xrange(len(s) - 1):
                n <<= 8
                n += ord(s[i + 1])
        return n

Response: Convert a number field to a python number.


Instruction:
Here you have a function in Python 3, explain what it does
def debug(msg):
    '''
    Displays debug messages to stderr only if the Python interpreter was invoked with the -O flag.
    '''
    if DEBUG:
        sys.stderr.write("DEBUG: " + msg + "\n")
        sys.stderr.flush()

Response: Displays debug messages to stderr only if DEBUG is set to True.


Instruction:
Write a Python 3 script for
generating an MD5 hash of the specified file.

Response:
def file_md5(file_name):
    '''
    Generate an MD5 hash of the specified file.

    @file_name - The file to hash.

    Returns an MD5 hex digest string.
    '''
    md5 = hashlib.md5()

    with open(file_name, 'rb') as f:
        for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
            md5.update(chunk)

    return md5.hexdigest()


Instruction:
Make a summary of the following Python 3 code
def file_size(filename):
    '''
    Obtains the size of a given file.

    @filename - Path to the file.

    Returns the size of the file.
    '''
    # Using open/lseek works on both regular files and block devices
    fd = os.open(filename, os.O_RDONLY)
    try:
        return os.lseek(fd, 0, os.SEEK_END)
    except KeyboardInterrupt as e:
        raise e
    except Exception as e:
        raise Exception(
            "file_size failed to obtain the size of '%s': %s" % (filename, str(e)))
    finally:
        os.close(fd)

Response: Returns the size of a given file.


Instruction:
Implement a Python 3 function for
returning a string comprised of all data in between double quotes.

Response:
def get_quoted_strings(quoted_string):
    '''
    Returns a string comprised of all data in between double quotes.

    @quoted_string - String to get quoted data from.

    Returns a string of quoted data on success.
    Returns a blank string if no quoted data is present.
    '''
    try:
        # This regex grabs all quoted data from string.
        # Note that this gets everything in between the first and last double quote.
        # This is intentional, as printed (and quoted) strings from a target file may contain
        # double quotes, and this function should ignore those. However, it also means that any
        # data between two quoted strings (ex: '"quote 1" non-quoted data
        # "quote 2"') will also be included.
        return re.findall(r'\"(.*)\"', quoted_string)[0]
    except KeyboardInterrupt as e:
        raise e
    except Exception:
        return ''


Instruction:
Can you create a Python 3 function that
returns a unique file name based on the specified base name and extension.

Response:
def unique_file_name(base_name, extension=''):
    '''
    Creates a unique file name based on the specified base name.

    @base_name - The base name to use for the unique file name.
    @extension - The file extension to use for the unique file name.

    Returns a unique file string.
    '''
    idcount = 0

    if extension and not extension.startswith('.'):
        extension = '.%s' % extension

    fname = base_name + extension

    while os.path.exists(fname):
        fname = "%s-%d%s" % (base_name, idcount, extension)
        idcount += 1

    return fname


Instruction:
Can you generate a brief explanation for the following Python 3 code
def strings(filename, minimum=4):
    '''
    A strings generator, similar to the Unix strings utility.

    @filename - The file to search for strings in.
    @minimum  - The minimum string length to search for.

    Yeilds printable ASCII strings from filename.
    '''
    result = ""

    with BlockFile(filename) as f:
        while True:
            (data, dlen) = f.read_block()
            if dlen < 1:
                break

            for c in data:
                if c in string.printable:
                    result += c
                    continue
                elif len(result) >= minimum:
                    yield result
                    result = ""
                else:
                    result = ""

Response: A generator that returns a list of strings from a file.


Instruction:
Make a summary of the following Python 3 code
def bytes2str(bs):
    '''
    For cross compatibility between Python 2 and Python 3 strings.
    '''
    if isinstance(bs, type(b'')) and PY_MAJOR_VERSION > 2:
        return bs.decode('latin1')
    else:
        return bs

Response: Convert a byte string to a unicode string.


Instruction:
Write a Python 3 function that can
create a set of rules for the current locale.

Response:
def create_rule(self, txtrule=None, regex=None, extension=None, cmd=None, codes=[0, None], recurse=True):
        '''
        Adds a set of rules to the extraction rule list.

        @txtrule   - Rule string, or list of rule strings, in the format <regular expression>:<file extension>[:<command to run>]
        @regex     - If rule string is not specified, this is the regular expression string to use.
        @extension - If rule string is not specified, this is the file extension to use.
        @cmd       - If rule string is not specified, this is the command to run.
                     Alternatively a callable object may be specified, which will be passed one argument: the path to the file to extract.
        @codes     - A list of valid return codes for the extractor.
        @recurse   - If False, extracted directories will not be recursed into when the matryoshka option is enabled.

        Returns None.
        '''
        rules = []
        created_rules = []
        match = False
        r = {
            'extension': '',
            'cmd': '',
            'regex': None,
            'codes': codes,
            'recurse': recurse,
        }

        # Process single explicitly specified rule
        if not txtrule and regex and extension:
            r['extension'] = extension
            r['regex'] = re.compile(regex)
            if cmd:
                r['cmd'] = cmd

            return [r]

        # Process rule string, or list of rule strings
        if not isinstance(txtrule, type([])):
            rules = [txtrule]
        else:
            rules = txtrule

        for rule in rules:
            r['cmd'] = ''
            r['extension'] = ''

            try:
                values = self._parse_rule(rule)
                match = values[0]
                r['regex'] = re.compile(values[0])
                r['extension'] = values[1]
                r['cmd'] = values[2]
                r['codes'] = values[3]
                r['recurse'] = values[4]
            except KeyboardInterrupt as e:
                raise e
            except Exception:
                pass

            # Verify that the match string was retrieved.
            if match:
                created_rules.append(r)

        return created_rules


Instruction:
Can you generate the documentation for the following Python 3 function
def remove_rules(self, description):
        '''
        Remove all rules that match a specified description.

        @description - The description to match against.

        Returns the number of rules removed.
        '''
        rm = []
        description = description.lower()

        for i in range(0, len(self.extract_rules)):
            if self.extract_rules[i]['regex'].search(description):
                rm.append(i)

        for i in rm:
            self.extract_rules.pop(i)

        return len(rm)

Response: Removes all rules that match a specified description.


Instruction:
Write a Python 3 function that can
edit all rules that match a specified description.

Response:
def edit_rules(self, description, key, value):
        '''
        Edit all rules that match a specified description.

        @description - The description to match against.
        @key         - The key to change for each matching rule.
        @value       - The new key value for each matching rule.

        Returns the number of rules modified.
        '''
        count = 0
        description = description.lower()

        for i in range(0, len(self.extract_rules)):
            if self.extract_rules[i]['regex'].search(description):
                if has_key(self.extract_rules[i], key):
                    self.extract_rules[i][key] = value
                    count += 1

        return count


Instruction:
Can you tell what is the following Python 3 function doing
def get_rules(self, description=None):
        '''
        Returns a list of extraction rules that match a given description.

        @description - The description to match against.

        Returns a list of extraction rules that match the given description.
        If no description is provided, a list of all rules are returned.
        '''
        if description:
            rules = []
            description = description.lower()
            for i in range(0, len(self.extract_rules)):
                if self.extract_rules[i]['regex'].search(description):
                    rules.append(self.extract_rules[i])
        else:
            rules = self.extract_rules

        return rules

Response: Returns a list of extraction rules that match a given description.


Instruction:
Implement a function in Python 3 to
load extraction rules from the specified file.

Response:
def load_from_file(self, fname):
        '''
        Loads extraction rules from the specified file.

        @fname - Path to the extraction rule file.

        Returns None.
        '''
        try:
            # Process each line from the extract file, ignoring comments
            with open(fname, 'r') as f:
                for rule in f.readlines():
                    self.add_rule(rule.split(self.COMMENT_DELIM, 1)[0])
        except KeyboardInterrupt as e:
            raise e
        except Exception as e:
            raise Exception("Extractor.load_from_file failed to load file '%s': %s" % (fname, str(e)))