Instruction:
Create a Python 3 function to
group Tox environments for Travis CI builds and unit tests and linting together.

Response:
def group_envs(envlist):
    """Group Tox environments for Travis CI builds

    Separate by Python version so that they can go in different Travis jobs:

    >>> group_envs('py37-int-snappy', 'py36-int')
    [('py36', 'int', ['py36-int']), ('py37', 'int', ['py37-int-snappy'])]

    Group unit tests and linting together:

    >>> group_envs(['py27-unit', 'py27-lint'])
    [('py27', 'unit', ['py27-unit', 'py27-lint'])]
    """
    groups = {}
    for env in envlist:
        envpy, category = env.split('-')[0:2]

        if category == 'lint':
            category = 'unit'

        try:
            groups[envpy, category].append(env)
        except KeyError:
            groups[envpy, category] = [env]

    return sorted((envpy, category, envs) for (envpy, category), envs in groups.items())


Instruction:
Make a summary of the following Python 3 code
def create_message(payload, key=None):
    assert payload is None or isinstance(payload, bytes), 'payload={!r} should be bytes or None'.format(payload)
    assert key is None or isinstance(key, bytes), 'key={!r} should be bytes or None'.format(key)
    return Message(0, 0, key, payload)

Response: Create a new message in the KAFKA node.


Instruction:
Make a summary of the following Python 3 code
def create_gzip_message(message_set):
    encoded_message_set = KafkaCodec._encode_message_set(message_set)

    gzipped = gzip_encode(encoded_message_set)
    return Message(0, CODEC_GZIP, None, gzipped)

Response: Construct a gzip - compressed message containing multiple messages


Instruction:
Can you tell what is the following Python 3 function doing
def create_snappy_message(message_set):
    encoded_message_set = KafkaCodec._encode_message_set(message_set)
    snapped = snappy_encode(encoded_message_set)
    return Message(0, CODEC_SNAPPY, None, snapped)

Response: Construct a Snappy - compressed message containing multiple messages


Instruction:
Can you generate a brief explanation for the following Python 3 code
def create_message_set(requests, codec=CODEC_NONE):
    msglist = []
    for req in requests:
        msglist.extend([create_message(m, key=req.key) for m in req.messages])

    if codec == CODEC_NONE:
        return msglist
    elif codec == CODEC_GZIP:
        return [create_gzip_message(msglist)]
    elif codec == CODEC_SNAPPY:
        return [create_snappy_message(msglist)]
    else:
        raise UnsupportedCodecError("Codec 0x%02x unsupported" % codec)

Response: Create a message set from a list of requests.


Instruction:
Can you generate the documentation for the following Python 3 function
def _encode_message_header(cls, client_id, correlation_id, request_key,
                               api_version=0):
        """
        Encode the common request envelope
        """
        return (struct.pack('>hhih',
                            request_key,          # ApiKey
                            api_version,          # ApiVersion
                            correlation_id,       # CorrelationId
                            len(client_id)) +     # ClientId size
                client_id)

Response: Encode the message header.


Instruction:
How would you code a function in Python 3 to
encode a list of messages into a message set.

Response:
def _encode_message_set(cls, messages, offset=None):
        """
        Encode a MessageSet. Unlike other arrays in the protocol,
        MessageSets are not length-prefixed.  Format::

            MessageSet => [Offset MessageSize Message]
              Offset => int64
              MessageSize => int32
        """
        message_set = []
        incr = 1
        if offset is None:
            incr = 0
            offset = 0
        for message in messages:
            encoded_message = KafkaCodec._encode_message(message)
            message_set.append(struct.pack('>qi', offset, len(encoded_message)))
            message_set.append(encoded_message)
            offset += incr
        return b''.join(message_set)


Instruction:
Can you create a Python 3 function that
encodes a single message.

Response:
def _encode_message(cls, message):
        """
        Encode a single message.

        The magic number of a message is a format version number.  The only
        supported magic number right now is zero.  Format::

            Message => Crc MagicByte Attributes Key Value
              Crc => int32
              MagicByte => int8
              Attributes => int8
              Key => bytes
              Value => bytes
        """
        if message.magic == 0:
            msg = struct.pack('>BB', message.magic, message.attributes)
            msg += write_int_string(message.key)
            msg += write_int_string(message.value)
            crc = zlib.crc32(msg) & 0xffffffff  # Ensure unsigned
            msg = struct.pack('>I', crc) + msg
        else:
            raise ProtocolError("Unexpected magic number: %d" % message.magic)
        return msg


Instruction:
Can you write a function in Python 3 where it
encodes some ProduceRequest structs :param bytes client_id: :param int correlation_id: :param list payloads: list of ProduceRequest :param int acks: How "acky" you want the request to be: 0: immediate response 1: written to disk by the leader 2+: waits for this many number of replicas to sync -1: waits for all replicas to be in sync :param int timeout: Maximum time the server will wait for acks from replicas. This is _not_ a socket timeout.

Response:
def encode_produce_request(cls, client_id, correlation_id,
                               payloads=None, acks=1,
                               timeout=DEFAULT_REPLICAS_ACK_TIMEOUT_MSECS):
        """
        Encode some ProduceRequest structs

        :param bytes client_id:
        :param int correlation_id:
        :param list payloads: list of ProduceRequest
        :param int acks:

            How "acky" you want the request to be:

            0: immediate response
            1: written to disk by the leader
            2+: waits for this many number of replicas to sync
            -1: waits for all replicas to be in sync

        :param int timeout:
            Maximum time the server will wait for acks from replicas.  This is
            _not_ a socket timeout.
        """
        if not isinstance(client_id, bytes):
            raise TypeError('client_id={!r} should be bytes'.format(client_id))
        payloads = [] if payloads is None else payloads
        grouped_payloads = group_by_topic_and_partition(payloads)

        message = cls._encode_message_header(client_id, correlation_id,
                                             KafkaCodec.PRODUCE_KEY)

        message += struct.pack('>hii', acks, timeout, len(grouped_payloads))

        for topic, topic_payloads in grouped_payloads.items():
            message += write_short_ascii(topic)

            message += struct.pack('>i', len(topic_payloads))
            for partition, payload in topic_payloads.items():
                msg_set = KafkaCodec._encode_message_set(payload.messages)
                message += struct.pack('>ii', partition, len(msg_set))
                message += msg_set

        return message


Instruction:
Write a Python 3 function for
decoding the bytes to a ProduceResponse object.

Response:
def decode_produce_response(cls, data):
        """
        Decode bytes to a ProduceResponse

        :param bytes data: bytes to decode
        :returns: iterable of `afkak.common.ProduceResponse`
        """
        ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0)

        for _i in range(num_topics):
            topic, cur = read_short_ascii(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)
            for _i in range(num_partitions):
                ((partition, error, offset), cur) = relative_unpack('>ihq', data, cur)

                yield ProduceResponse(topic, partition, error, offset)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def encode_fetch_request(cls, client_id, correlation_id, payloads=None,
                             max_wait_time=100, min_bytes=4096):
        """
        Encodes some FetchRequest structs

        :param bytes client_id:
        :param int correlation_id:
        :param list payloads: list of :class:`FetchRequest`
        :param int max_wait_time: how long to block waiting on min_bytes of data
        :param int min_bytes:
            the minimum number of bytes to accumulate before returning the
            response
        """
        payloads = [] if payloads is None else payloads
        grouped_payloads = group_by_topic_and_partition(payloads)

        message = cls._encode_message_header(client_id, correlation_id,
                                             KafkaCodec.FETCH_KEY)

        assert isinstance(max_wait_time, int)

        # -1 is the replica id
        message += struct.pack('>iiii', -1, max_wait_time, min_bytes,
                               len(grouped_payloads))

        for topic, topic_payloads in grouped_payloads.items():
            message += write_short_ascii(topic)
            message += struct.pack('>i', len(topic_payloads))
            for partition, payload in topic_payloads.items():
                message += struct.pack('>iqi', partition, payload.offset,
                                       payload.max_bytes)

        return message

Response: Encodes some FetchRequest structs into a byte string.


Instruction:
How would you implement a function in Python 3 that
decodes bytes to a FetchResponse object.

Response:
def decode_fetch_response(cls, data):
        """
        Decode bytes to a FetchResponse

        :param bytes data: bytes to decode
        """
        ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0)

        for _i in range(num_topics):
            (topic, cur) = read_short_ascii(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)

            for _i in range(num_partitions):
                ((partition, error, highwater_mark_offset), cur) = \
                    relative_unpack('>ihq', data, cur)

                (message_set, cur) = read_int_string(data, cur)

                yield FetchResponse(
                    topic, partition, error,
                    highwater_mark_offset,
                    KafkaCodec._decode_message_set_iter(message_set))


Instruction:
Can you implement a function in Python 3 that
decodes a byte string to an iterable of OffsetResponse objects.

Response:
def decode_offset_response(cls, data):
        """
        Decode bytes to an :class:`OffsetResponse`

        :param bytes data: bytes to decode
        """
        ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0)

        for _i in range(num_topics):
            (topic, cur) = read_short_ascii(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)

            for _i in range(num_partitions):
                ((partition, error, num_offsets), cur) = \
                    relative_unpack('>ihi', data, cur)

                offsets = []
                for _i in range(num_offsets):
                    ((offset,), cur) = relative_unpack('>q', data, cur)
                    offsets.append(offset)

                yield OffsetResponse(topic, partition, error, tuple(offsets))


Instruction:
How would you explain what the following Python 3 function does
def encode_metadata_request(cls, client_id, correlation_id, topics=None):
        topics = [] if topics is None else topics
        message = [
            cls._encode_message_header(client_id, correlation_id,
                                       KafkaCodec.METADATA_KEY),
            struct.pack('>i', len(topics)),
        ]
        for topic in topics:
            message.append(write_short_ascii(topic))
        return b''.join(message)

Response: Encode a MetadataRequest object.


Instruction:
Can you implement a function in Python 3 that
decodes the metadata response from the byte stream into a MetadataResponse object.

Response:
def decode_metadata_response(cls, data):
        """
        Decode bytes to a MetadataResponse

        :param bytes data: bytes to decode
        """
        ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0)

        # In testing, I saw this routine swap my machine to death when
        # passed bad data. So, some checks are in order...
        if numbrokers > MAX_BROKERS:
            raise InvalidMessageError(
                "Brokers:{} exceeds max:{}".format(numbrokers, MAX_BROKERS))

        # Broker info
        brokers = {}
        for _i in range(numbrokers):
            ((nodeId, ), cur) = relative_unpack('>i', data, cur)
            (host, cur) = read_short_ascii(data, cur)
            ((port,), cur) = relative_unpack('>i', data, cur)
            brokers[nodeId] = BrokerMetadata(nodeId, nativeString(host), port)

        # Topic info
        ((num_topics,), cur) = relative_unpack('>i', data, cur)
        topic_metadata = {}

        for _i in range(num_topics):
            ((topic_error,), cur) = relative_unpack('>h', data, cur)
            (topic_name, cur) = read_short_ascii(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)
            partition_metadata = {}

            for _j in range(num_partitions):
                ((partition_error_code, partition, leader, numReplicas),
                 cur) = relative_unpack('>hiii', data, cur)

                (replicas, cur) = relative_unpack(
                    '>%di' % numReplicas, data, cur)

                ((num_isr,), cur) = relative_unpack('>i', data, cur)
                (isr, cur) = relative_unpack('>%di' % num_isr, data, cur)

                partition_metadata[partition] = \
                    PartitionMetadata(
                        topic_name, partition, partition_error_code, leader,
                        replicas, isr)

            topic_metadata[topic_name] = TopicMetadata(
                topic_name, topic_error, partition_metadata)

        return brokers, topic_metadata


Instruction:
Implement a function in Python 3 to
encode a ConsumerMetadataRequest object.

Response:
def encode_consumermetadata_request(cls, client_id, correlation_id,
                                        consumer_group):
        """
        Encode a ConsumerMetadataRequest

        :param bytes client_id: string
        :param int correlation_id: int
        :param str consumer_group: string
        """
        message = cls._encode_message_header(client_id, correlation_id,
                                             KafkaCodec.CONSUMER_METADATA_KEY)
        message += write_short_ascii(consumer_group)
        return message


Instruction:
Write a Python 3 script to
decode a byte array into a ConsumerMetadataResponse object.

Response:
def decode_consumermetadata_response(cls, data):
        """
        Decode bytes to a ConsumerMetadataResponse

        :param bytes data: bytes to decode
        """
        (correlation_id, error_code, node_id), cur = \
            relative_unpack('>ihi', data, 0)
        host, cur = read_short_ascii(data, cur)
        (port,), cur = relative_unpack('>i', data, cur)

        return ConsumerMetadataResponse(
            error_code, node_id, nativeString(host), port)


Instruction:
Write a Python 3 script to
encode the OffsetCommitRequest struct.

Response:
def encode_offset_commit_request(cls, client_id, correlation_id,
                                     group, group_generation_id, consumer_id,
                                     payloads):
        """
        Encode some OffsetCommitRequest structs (v1)

        :param bytes client_id: string
        :param int correlation_id: int
        :param str group: the consumer group to which you are committing offsets
        :param int group_generation_id: int32, generation ID of the group
        :param str consumer_id: string, Identifier for the consumer
        :param list payloads: list of :class:`OffsetCommitRequest`
        """
        grouped_payloads = group_by_topic_and_partition(payloads)

        message = cls._encode_message_header(
            client_id, correlation_id, KafkaCodec.OFFSET_COMMIT_KEY,
            api_version=1,
        )

        message += write_short_ascii(group)
        message += struct.pack('>i', group_generation_id)
        message += write_short_ascii(consumer_id)
        message += struct.pack('>i', len(grouped_payloads))

        for topic, topic_payloads in grouped_payloads.items():
            message += write_short_ascii(topic)
            message += struct.pack('>i', len(topic_payloads))

            for partition, payload in topic_payloads.items():
                message += struct.pack('>iqq', partition, payload.offset,
                                       payload.timestamp)
                message += write_short_bytes(payload.metadata)

        return message


Instruction:
Explain what the following Python 3 code does
def decode_offset_commit_response(cls, data):
        ((correlation_id,), cur) = relative_unpack('>i', data, 0)
        ((num_topics,), cur) = relative_unpack('>i', data, cur)

        for _i in range(num_topics):
            (topic, cur) = read_short_ascii(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)

            for _i in range(num_partitions):
                ((partition, error), cur) = relative_unpack('>ih', data, cur)
                yield OffsetCommitResponse(topic, partition, error)

Response: Decode bytes to an OffsetCommitResponse object.


Instruction:
Make a summary of the following Python 3 code
def encode_offset_fetch_request(cls, client_id, correlation_id,
                                    group, payloads):
        """
        Encode some OffsetFetchRequest structs

        :param bytes client_id: string
        :param int correlation_id: int
        :param bytes group: string, the consumer group you are fetching offsets for
        :param list payloads: list of :class:`OffsetFetchRequest`
        """
        grouped_payloads = group_by_topic_and_partition(payloads)
        message = cls._encode_message_header(
            client_id, correlation_id, KafkaCodec.OFFSET_FETCH_KEY,
            api_version=1)

        message += write_short_ascii(group)
        message += struct.pack('>i', len(grouped_payloads))

        for topic, topic_payloads in grouped_payloads.items():
            message += write_short_ascii(topic)
            message += struct.pack('>i', len(topic_payloads))

            for partition in topic_payloads:
                message += struct.pack('>i', partition)

        return message

Response: Encode some OffsetFetchRequest structs that are used to fetch the offsets for a specific consumer group.


Instruction:
Here you have a function in Python 3, explain what it does
def decode_offset_fetch_response(cls, data):

        ((correlation_id,), cur) = relative_unpack('>i', data, 0)
        ((num_topics,), cur) = relative_unpack('>i', data, cur)

        for _i in range(num_topics):
            (topic, cur) = read_short_ascii(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)

            for _i in range(num_partitions):
                ((partition, offset), cur) = relative_unpack('>iq', data, cur)
                (metadata, cur) = read_short_bytes(data, cur)
                ((error,), cur) = relative_unpack('>h', data, cur)

                yield OffsetFetchResponse(topic, partition, offset,
                                          metadata, error)

Response: Decode bytes to an OffsetFetchResponse object.


Instruction:
Create a Python 3 function for
handling a response from the broker.

Response:
def stringReceived(self, response):
        """
        Handle a response from the broker.
        """
        correlation_id = response[0:4]
        try:
            d = self._pending.pop(correlation_id)
        except KeyError:
            self._log.warn((
                "Response has unknown correlation ID {correlation_id!r}."
                " Dropping connection to {peer}."
            ), correlation_id=correlation_id, peer=self.transport.getPeer())
            self.transport.loseConnection()
        else:
            d.callback(response)


Instruction:
Can you generate the documentation for the following Python 3 function
def connectionLost(self, reason=connectionDone):
        self._failed = reason
        pending, self._pending = self._pending, None
        for d in pending.values():
            d.errback(reason)

Response: Mark the protocol as failed and fail all pending operations.


Instruction:
Write a Python 3 script to
send a request to the Kafka broker.

Response:
def request(self, request):
        """
        Send a request to the Kafka broker.

        :param bytes request:
            The bytes of a Kafka `RequestMessage`_ structure. It must have
            a unique (to this connection) correlation ID.

        :returns:
            `Deferred` which will:

              - Succeed with the bytes of a Kafka `ResponseMessage`_
              - Fail when the connection terminates

        .. _RequestMessage:: https://kafka.apache.org/protocol.html#protocol_messages

        """
        if self._failed is not None:
            return fail(self._failed)
        correlation_id = request[4:8]
        assert correlation_id not in self._pending
        d = Deferred()
        self.sendString(request)
        self._pending[correlation_id] = d
        return d


Instruction:
Can you generate a brief explanation for the following Python 3 code
def send_messages(self, topic, key=None, msgs=()):
        try:
            topic = _coerce_topic(topic)
            if key is not None and not isinstance(key, bytes):
                raise TypeError('key={!r} must be bytes or None'.format(key))

            if not msgs:
                raise ValueError("msgs must be a non-empty sequence")

            msg_cnt = len(msgs)
            byte_cnt = 0
            for index, m in enumerate(msgs):
                if m is None:
                    continue

                if not isinstance(m, bytes):
                    raise TypeError('Message {} to topic {} ({!r:.100}) has type {}, but must have type {}'.format(
                        index, topic, m, type(m).__name__, type(bytes).__name__))

                byte_cnt += len(m)
        except Exception:
            return fail()

        d = Deferred(self._cancel_send_messages)
        self._batch_reqs.append(SendRequest(topic, key, msgs, d))
        self._waitingMsgCount += msg_cnt
        self._waitingByteCount += byte_cnt

        # Add request to list of outstanding reqs' callback to remove
        self._outstanding.append(d)
        d.addBoth(self._remove_from_outstanding, d)
        # See if we have enough messages in the batch to do a send.
        self._check_send_batch()
        return d

Response: Sends a list of messages to Kafka.


Instruction:
Write a Python 3 script for
stopping any outstanding requests and returns a Deferred which fires when fully stopped.

Response:
def stop(self):
        """
        Terminate any outstanding requests.

        :returns: :class:``Deferred` which fires when fully stopped.
        """
        self.stopping = True
        # Cancel any outstanding request to our client
        if self._batch_send_d:
            self._batch_send_d.cancel()
        # Do we have to worry about our looping call?
        if self.batch_every_t is not None:
            # Stop our looping call, and wait for the deferred to be called
            if self._sendLooper is not None:
                self._sendLooper.stop()
        # Make sure requests that wasn't cancelled above are now
        self._cancel_outstanding()
        return self._sendLooperD or succeed(None)


Instruction:
Can you create a Python 3 function that
calls by the LoopingCall when the send_batch function failed.

Response:
def _send_timer_failed(self, fail):
        """
        Our _send_batch() function called by the LoopingCall failed. Some
        error probably came back from Kafka and _check_error() raised the
        exception
        For now, just log the failure and restart the loop
        """
        log.warning('_send_timer_failed:%r: %s', fail,
                    fail.getBriefTraceback())
        self._sendLooperD = self._sendLooper.start(
            self.batch_every_t, now=False)


Instruction:
Explain what the following Python 3 code does
def _send_timer_stopped(self, lCall):
        if self._sendLooper is not lCall:
            log.warning('commitTimerStopped with wrong timer:%s not:%s',
                        lCall, self._sendLooper)
        else:
            self._sendLooper = None
            self._sendLooperD = None

Response: Called when a timer is stopped.


Instruction:
How would you explain what the following Python 3 function does
def _next_partition(self, topic, key=None):
        # check if the client has metadata for the topic
        while self.client.metadata_error_for_topic(topic):
            # client doesn't have good metadata for topic. ask to fetch...
            # check if we have request attempts left
            if self._req_attempts >= self._max_attempts:
                # No, no attempts left, so raise the error
                _check_error(self.client.metadata_error_for_topic(topic))
            yield self.client.load_metadata_for_topics(topic)
            if not self.client.metadata_error_for_topic(topic):
                break
            self._req_attempts += 1
            d = Deferred()
            self.client.reactor.callLater(
                self._retry_interval, d.callback, True)
            self._retry_interval *= self.RETRY_INTERVAL_FACTOR
            yield d

        # Ok, should be safe to get the partitions now...
        partitions = self.client.topic_partitions[topic]
        # Do we have a partitioner for this topic already?
        if topic not in self.partitioners:
            # No, create a new paritioner for topic, partitions
            self.partitioners[topic] = \
                self.partitioner_class(topic, partitions)
        # Lookup the next partition
        partition = self.partitioners[topic].partition(key, partitions)
        returnValue(partition)

Response: Get the next partition to which to publish for the given topic.


Instruction:
Given the following Python 3 function, write the documentation
def _send_requests(self, parts_results, requests):
        # We use these dictionaries to be able to combine all the messages
        # destined to the same topic/partition into one request
        # the messages & deferreds, both by topic+partition
        reqsByTopicPart = defaultdict(list)
        payloadsByTopicPart = defaultdict(list)
        deferredsByTopicPart = defaultdict(list)

        # We now have a list of (succeeded/failed, partition/None) tuples
        # for the partition lookups we did on each message group, zipped with
        # the requests
        for (success, part_or_failure), req in zip(parts_results, requests):
            if req.deferred.called:
                # Submitter cancelled the request while we were waiting for
                # the topic/partition, skip it
                continue
            if not success:
                # We failed to get a partition for this request, errback to the
                # caller with the failure. Maybe this should retry? However,
                # since this failure is likely to affect an entire Topic, there
                # should be no issues with ordering of messages within a
                # partition of a topic getting out of order. Let the caller
                # retry the particular request if they like, or they could
                # cancel all their outstanding requests in
                req.deferred.errback(part_or_failure)
                continue
            # Ok, we now have a partition for this request, we can add the
            # request for this topic/partition to reqsByTopicPart, and the
            # caller's deferred to deferredsByTopicPart
            topicPart = TopicAndPartition(req.topic, part_or_failure)
            reqsByTopicPart[topicPart].append(req)
            deferredsByTopicPart[topicPart].append(req.deferred)

        # Build list of payloads grouped by topic/partition
        # That is, we bundle all the messages destined for a given
        # topic/partition, even if they were submitted by different
        # requests into a single 'payload', and then we submit all the
        # payloads as a list to the client for sending to the various
        # brokers. The finest granularity of success/failure is at the
        # payload (topic/partition) level.
        payloads = []
        for (topic, partition), reqs in reqsByTopicPart.items():
            msgSet = create_message_set(reqs, self.codec)
            req = ProduceRequest(topic, partition, msgSet)
            topicPart = TopicAndPartition(topic, partition)
            payloads.append(req)
            payloadsByTopicPart[topicPart] = req
        # Make sure we have some payloads to send
        if not payloads:
            return
        # send the request
        d = self.client.send_produce_request(
            payloads, acks=self.req_acks, timeout=self.ack_timeout,
            fail_on_error=False)
        self._req_attempts += 1
        # add our handlers
        d.addBoth(self._handle_send_response, payloadsByTopicPart,
                  deferredsByTopicPart)
        return d

Response: Send the requests to the specified topic and partition.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _complete_batch_send(self, resp):
        self._batch_send_d = None
        self._req_attempts = 0
        self._retry_interval = self._init_retry_interval
        if isinstance(resp, Failure) and not resp.check(tid_CancelledError,
                                                        CancelledError):
            log.error("Failure detected in _complete_batch_send: %r\n%r",
                      resp, resp.getTraceback())
        return

Response: Complete the processing of our current batch send operation.


Instruction:
Create a Python 3 function to
check if we have enough messages to send

Response:
def _check_send_batch(self, result=None):
        """Check if we have enough messages/bytes to send
        Since this can be called from the callback chain, we
        pass through our first (non-self) arg
        """
        if (
            (self.batch_every_n and self.batch_every_n <= self._waitingMsgCount) or
            (self.batch_every_b and self.batch_every_b <= self._waitingByteCount)
        ):
            self._send_batch()
        return result


Instruction:
How would you explain what the following Python 3 function does
def _send_batch(self):
        # We can be triggered by the LoopingCall, and have nothing to send...
        # Or, we've got SendRequest(s) to send, but are still processing the
        # previous batch...
        if (not self._batch_reqs) or self._batch_send_d:
            return

        # Save a local copy, and clear the global list & metrics
        requests, self._batch_reqs = self._batch_reqs, []
        self._waitingByteCount = 0
        self._waitingMsgCount = 0

        # Iterate over them, fetching the partition for each message batch
        d_list = []
        for req in requests:
            # For each request, we get the topic & key and use that to lookup
            # the next partition on which we should produce
            d_list.append(self._next_partition(req.topic, req.key))
        d = self._batch_send_d = Deferred()
        # Since DeferredList doesn't propagate cancel() calls to deferreds it
        # might be waiting on for a result, we need to use this structure,
        # rather than just using the DeferredList directly
        d.addCallback(lambda r: DeferredList(d_list, consumeErrors=True))
        d.addCallback(self._send_requests, requests)
        # Once we finish fully processing the current batch, clear the
        # _batch_send_d and check if any more requests piled up when we
        # were busy.
        d.addBoth(self._complete_batch_send)
        d.addBoth(self._check_send_batch)
        # Fire off the callback to start processing...
        d.callback(None)

Response: Send the messages in the current batch and return the result.


Instruction:
How would you explain what the following Python 3 function does
def _cancel_send_messages(self, d):
        # Is the request in question in an unsent batch?
        for req in self._batch_reqs:
            if req.deferred == d:
                # Found the request, remove it and return.
                msgs = req.messages
                self._waitingMsgCount -= len(msgs)
                for m in (_m for _m in msgs if _m is not None):
                    self._waitingByteCount -= len(m)
                # This _should_ be safe as we abort the iteration upon removal
                self._batch_reqs.remove(req)
                d.errback(CancelledError(request_sent=False))
                return

        # If it wasn't found in the unsent batch. We just rely on the
        # downstream processing of the request to check if the deferred
        # has been called and skip further processing for this request
        # Errback the deferred with whether or not we sent the request
        # to Kafka already
        d.errback(
            CancelledError(request_sent=(self._batch_send_d is not None)))
        return

Response: Cancel a send_messages request.


Instruction:
Explain what the following Python 3 code does
def _handle_send_response(self, result, payloadsByTopicPart,
                              deferredsByTopicPart):
        """Handle the response from our client to our send_produce_request

        This is a bit complex. Failures can happen in a few ways:

          1. The client sent an empty list, False, None or some similar thing
             as the result, but we were expecting real responses.
          2. The client had a failure before it even tried sending any requests
             to any brokers.

               a. Kafka error: See if we can retry the whole request
               b. Non-kafka: Figure it's a programming error, fail all deferreds

          3. The client sent all the requests (it's all or none) to the brokers
             but one or more request failed (timed out before receiving a
             response, or the brokerclient threw some sort of exception on send
             In this case, the client throws FailedPayloadsError, and attaches
             the responses (NOTE: some can have errors!), and the payloads
             where the send itself failed to the exception.
          4. The client sent all the requests, all responses were received, but
             the Kafka broker indicated an error with servicing the request on
             some of the responses.
        """

        def _deliver_result(d_list, result=None):
            """Possibly callback each deferred in a list with single result"""
            for d in d_list:
                if not isinstance(d, Deferred):
                    # nested list...
                    _deliver_result(d, result)
                else:
                    # We check d.called since the request could have been
                    # cancelled while we waited for the response
                    if not d.called:
                        d.callback(result)

        def _do_retry(payloads):
            # We use 'fail_on_error=False' because we want our client to
            # process every response that comes back from the brokers so
            # we can determine which requests were successful, and which
            # failed for retry
            d = self.client.send_produce_request(
                payloads, acks=self.req_acks, timeout=self.ack_timeout,
                fail_on_error=False)
            self._req_attempts += 1
            # add our handlers
            d.addBoth(self._handle_send_response, payloadsByTopicPart,
                      deferredsByTopicPart)
            return d

        def _cancel_retry(failure, dc):
            # Cancel the retry callLater and pass-thru the failure
            dc.cancel()
            # cancel all the top-level deferreds associated with the request
            _deliver_result(deferredsByTopicPart.values(), failure)
            return failure

        def _check_retry_payloads(failed_payloads_with_errs):
            """Check our retry count and retry after a delay or errback

            If we have more retries to try, create a deferred that will fire
            with the result of delayed retry. If not, errback the remaining
            deferreds with failure

            Params:
            failed_payloads - list of (payload, failure) tuples
            """
            # Do we have retries left?
            if self._req_attempts >= self._max_attempts:
                # No, no retries left, fail each failed_payload with its
                # associated failure
                for p, f in failed_payloads_with_errs:
                    t_and_p = TopicAndPartition(p.topic, p.partition)
                    _deliver_result(deferredsByTopicPart[t_and_p], f)
                return
            # Retries remain!  Schedule one...
            d = Deferred()
            dc = self.client.reactor.callLater(
                self._retry_interval, d.callback, [p for p, f in
                                                   failed_payloads])
            self._retry_interval *= self.RETRY_INTERVAL_FACTOR
            # Cancel the callLater when request is cancelled before it fires
            d.addErrback(_cancel_retry, dc)

            # Reset the topic metadata for all topics which had failed_requests
            # where the failures were of the kind UnknownTopicOrPartitionError
            # or NotLeaderForPartitionError, since those indicate our client's
            # metadata is out of date.
            reset_topics = set()
            for payload, e in failed_payloads:
                if (isinstance(e, NotLeaderForPartitionError) or
                        isinstance(e, UnknownTopicOrPartitionError)):
                    reset_topics.add(payload.topic)
            if reset_topics:
                self.client.reset_topic_metadata(*reset_topics)

            d.addCallback(_do_retry)
            return d

        # The payloads we need to retry, if we still can..
        failed_payloads = []
        # In the case we are sending requests without requiring acks, the
        # brokerclient will immediately callback() the deferred upon send with
        # None. In that case, we just iterate over all the deferreds in
        # deferredsByTopicPart and callback them with None
        # If we are expecting responses/acks, and we get an empty result, we
        # callback with a Failure of NoResponseError
        if not result:
            # Success, but no results, is that what we're expecting?
            if self.req_acks == PRODUCER_ACK_NOT_REQUIRED:
                result = None
            else:
                # We got no result, but we were expecting one? Fail everything!
                result = Failure(NoResponseError())
            _deliver_result(deferredsByTopicPart.values(), result)
            return
        elif isinstance(result, Failure):
            # Failure!  Was it total, or partial?
            if not result.check(FailedPayloadsError):
                # Total failure of some sort!
                # The client was unable to send the request at all. If it's
                # a KafkaError (probably Leader/Partition unavailable), retry
                if result.check(KafkaError):
                    # Yep, a kafak error. Set failed_payloads, and we'll retry
                    # them all below. Set failure for errback to callers if we
                    # are all out of retries
                    failure, result = result, []  # no succesful results, retry
                    failed_payloads = [(p, failure) for p in
                                       payloadsByTopicPart.values()]
                else:
                    # Was the request cancelled?
                    if not result.check(tid_CancelledError):
                        # Uh Oh, programming error? Log it!
                        log.error("Unexpected failure: %r in "
                                  "_handle_send_response", result)
                    # Cancelled, or programming error, we fail the requests
                    _deliver_result(deferredsByTopicPart.values(), result)
                    return
            else:
                # FailedPayloadsError: This means that some/all of the
                # requests to a/some brokerclients failed to send.
                # Pull the successful responses and the failed_payloads off
                # the exception and handle them below. Preserve the
                # FailedPayloadsError as 'failure'
                failure = result
                result = failure.value.args[0]
                failed_payloads = failure.value.args[1]

        # Do we have results? Iterate over them and if the response indicates
        # success, then callback the associated deferred. If the response
        # indicates an error, then setup that request for retry.
        # NOTE: In this case, each failed_payload get it's own error...
        for res in result:
            t_and_p = TopicAndPartition(res.topic, res.partition)
            t_and_p_err = _check_error(res, raiseException=False)
            if not t_and_p_err:
                # Success for this topic/partition
                d_list = deferredsByTopicPart[t_and_p]
                _deliver_result(d_list, res)
            else:
                p = payloadsByTopicPart[t_and_p]
                failed_payloads.append((p, t_and_p_err))

        # Were there any failed requests to possibly retry?
        if failed_payloads:
            return _check_retry_payloads(failed_payloads)
        return

Response:
Handle the response from our client to our send_produce_request
COOKIE


Instruction:
Write a Python 3 function that can
cancel all of our outstanding requests

Response:
def _cancel_outstanding(self):
        """Cancel all of our outstanding requests"""
        for d in list(self._outstanding):
            d.addErrback(lambda _: None)  # Eat any uncaught errors
            d.cancel()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def reset_consumer_group_metadata(self, *groups):
        groups = tuple(_coerce_consumer_group(g) for g in groups)
        for group in groups:
            if group in self.consumer_group_to_brokers:
                del self.consumer_group_to_brokers[group]

Response: Reset the cache of what broker manages the offset for a given set of consumer groups.


Instruction:
Implement a function in Python 3 to
clear all cached metadata

Response:
def reset_all_metadata(self):
        """Clear all cached metadata

        Metadata will be re-fetched as required to satisfy requests.
        """
        self.topics_to_brokers.clear()
        self.topic_partitions.clear()
        self.topic_errors.clear()
        self.consumer_group_to_brokers.clear()


Instruction:
Write a Python 3 function that can
determine if the given topic is fully replicated according to the currently known cluster metadata.

Response:
def topic_fully_replicated(self, topic):
        """
        Determine if the given topic is fully replicated according to the
        currently known cluster metadata.

        .. note::

            This relies on cached cluster metadata. You may call
            :meth:`load_metadata_for_topics()` first to refresh this cache.

        :param str topic: Topic name

        :returns:
            A boolean indicating that:

            1. The number of partitions in the topic is non-zero.
            2. For each partition, all replicas are in the in-sync replica
               (ISR) set.
        :rtype: :class:`bool`
        """
        topic = _coerce_topic(topic)
        if topic not in self.topic_partitions:
            return False
        if not self.topic_partitions[topic]:
            # Don't consider an empty partition list 'fully replicated'
            return False
        return all(
            self.partition_fully_replicated(TopicAndPartition(topic, p))
            for p in self.topic_partitions[topic]
        )


Instruction:
Can you tell what is the following Python 3 function doing
def load_metadata_for_topics(self, *topics):
        topics = tuple(_coerce_topic(t) for t in topics)
        log.debug("%r: load_metadata_for_topics(%s)", self, ', '.join(repr(t) for t in topics))
        fetch_all_metadata = not topics

        # create the request
        requestId = self._next_id()
        request = KafkaCodec.encode_metadata_request(self._clientIdBytes,
                                                     requestId, topics)

        # Callbacks for the request deferred...
        def _handleMetadataResponse(response):
            # Decode the response
            brokers, topics = KafkaCodec.decode_metadata_response(response)
            log.debug("%r: got metadata brokers=%r topics=%r", self, brokers, topics)

            # If we fetched the metadata for all topics, then store away the
            # received metadata for diagnostics.
            if fetch_all_metadata:
                self._brokers = brokers
                self._topics = topics

            # Iff we were fetching for all topics, and we got at least one
            # broker back, then remove brokers when we update our brokers
            ok_to_remove = (fetch_all_metadata and len(brokers))
            # Take the metadata we got back, update our self.clients, and
            # if needed disconnect or connect from/to old/new brokers
            self._update_brokers(brokers.values(), remove=ok_to_remove)

            # Now loop through all the topics/partitions in the response
            # and setup our cache/data-structures
            for topic, topic_metadata in topics.items():
                _, topic_error, partitions = topic_metadata
                self.reset_topic_metadata(topic)
                self.topic_errors[topic] = topic_error
                if not partitions:
                    log.warning('No partitions for %s, Err:%d',
                                topic, topic_error)
                    continue

                self.topic_partitions[topic] = []
                for partition, meta in partitions.items():
                    self.topic_partitions[topic].append(partition)
                    topic_part = TopicAndPartition(topic, partition)
                    self.partition_meta[topic_part] = meta
                    if meta.leader == -1:
                        log.warning('No leader for topic %s partition %s',
                                    topic, partition)
                        self.topics_to_brokers[topic_part] = None
                    else:
                        self.topics_to_brokers[
                            topic_part] = brokers[meta.leader]
                self.topic_partitions[topic] = sorted(
                    self.topic_partitions[topic])
            return True

        def _handleMetadataErr(err):
            # This should maybe do more cleanup?
            if err.check(t_CancelledError, CancelledError):
                # Eat the error
                # XXX Shouldn't this return False? The success branch
                # returns True.
                return None
            log.error("Failed to retrieve metadata:%s", err)
            raise KafkaUnavailableError(
                "Unable to load metadata from configured "
                "hosts: {!r}".format(err))

        # Send the request, add the handlers
        d = self._send_broker_unaware_request(requestId, request)
        d.addCallbacks(_handleMetadataResponse, _handleMetadataErr)
        return d

Response: Discover topic metadata and brokers and topics and return the metadata for that topic.


Instruction:
Make a summary of the following Python 3 code
def load_consumer_metadata_for_group(self, group):
        group = _coerce_consumer_group(group)
        log.debug("%r: load_consumer_metadata_for_group(%r)", self, group)

        # If we are already loading the metadata for this group, then
        # just return the outstanding deferred
        if group in self.coordinator_fetches:
            d = defer.Deferred()
            self.coordinator_fetches[group][1].append(d)
            return d

        # No outstanding request, create a new one
        requestId = self._next_id()
        request = KafkaCodec.encode_consumermetadata_request(
            self._clientIdBytes, requestId, group)

        # Callbacks for the request deferred...
        def _handleConsumerMetadataResponse(response_bytes):
            # Decode the response (returns ConsumerMetadataResponse)
            response = KafkaCodec.decode_consumermetadata_response(response_bytes)
            log.debug("%r: load_consumer_metadata_for_group(%r) -> %r", self, group, response)
            if response.error:
                raise BrokerResponseError.errnos.get(response.error, UnknownError)(response)

            bm = BrokerMetadata(response.node_id, response.host, response.port)
            self.consumer_group_to_brokers[group] = bm
            self._update_brokers([bm])
            return True

        def _handleConsumerMetadataErr(err):
            log.error("Failed to retrieve consumer metadata for group %r", group,
                      exc_info=(err.type, err.value, err.getTracebackObject()))
            # Clear any stored value for the group's coordinator
            self.reset_consumer_group_metadata(group)
            # FIXME: This exception should chain from err.
            raise ConsumerCoordinatorNotAvailableError(
                "Coordinator for group {!r} not available".format(group),
            )

        def _propagate(result):
            [_, ds] = self.coordinator_fetches.pop(group, None)
            for d in ds:
                d.callback(result)

        # Send the request, add the handlers
        request_d = self._send_broker_unaware_request(requestId, request)
        d = defer.Deferred()
        # Save the deferred under the fetches for this group
        self.coordinator_fetches[group] = (request_d, [d])
        request_d.addCallback(_handleConsumerMetadataResponse)
        request_d.addErrback(_handleConsumerMetadataErr)
        request_d.addBoth(_propagate)
        return d

Response: Load the consumer metadata for the specified group.


Instruction:
Write a Python 3 script for
sending a ProduceRequest to the broker.

Response:
def send_produce_request(self, payloads=None, acks=1,
                             timeout=DEFAULT_REPLICAS_ACK_MSECS,
                             fail_on_error=True, callback=None):
        """
        Encode and send some ProduceRequests

        ProduceRequests will be grouped by (topic, partition) and then
        sent to a specific broker. Output is a list of responses in the
        same order as the list of payloads specified

        Parameters
        ----------
        payloads:
            list of ProduceRequest
        acks:
            How many Kafka broker replicas need to write before
            the leader replies with a response
        timeout:
            How long the server has to receive the acks from the
            replicas before returning an error.
        fail_on_error:
            boolean, should we raise an Exception if we encounter an API error?
        callback:
            function, instead of returning the ProduceResponse,
            first pass it through this function

        Return
        ------
        a deferred which callbacks with a list of ProduceResponse

        Raises
        ------
        FailedPayloadsError, LeaderUnavailableError, PartitionUnavailableError
        """

        encoder = partial(
            KafkaCodec.encode_produce_request,
            acks=acks,
            timeout=timeout)

        if acks == 0:
            decoder = None
        else:
            decoder = KafkaCodec.decode_produce_response

        resps = yield self._send_broker_aware_request(
            payloads, encoder, decoder)

        returnValue(self._handle_responses(resps, fail_on_error, callback))


Instruction:
Implement a function in Python 3 to
send FetchRequest to all brokers.

Response:
def send_fetch_request(self, payloads=None, fail_on_error=True,
                           callback=None,
                           max_wait_time=DEFAULT_FETCH_SERVER_WAIT_MSECS,
                           min_bytes=DEFAULT_FETCH_MIN_BYTES):
        """
        Encode and send a FetchRequest

        Payloads are grouped by topic and partition so they can be pipelined
        to the same brokers.

        Raises
        ======
        FailedPayloadsError, LeaderUnavailableError, PartitionUnavailableError
        """
        if (max_wait_time / 1000) > (self.timeout - 0.1):
            raise ValueError(
                "%r: max_wait_time: %d must be less than client.timeout by "
                "at least 100 milliseconds.", self, max_wait_time)

        encoder = partial(KafkaCodec.encode_fetch_request,
                          max_wait_time=max_wait_time,
                          min_bytes=min_bytes)

        # resps is a list of FetchResponse() objects, each of which can hold
        # 1-n messages.
        resps = yield self._send_broker_aware_request(
            payloads, encoder,
            KafkaCodec.decode_fetch_response)

        returnValue(self._handle_responses(resps, fail_on_error, callback))


Instruction:
Here you have a function in Python 3, explain what it does
def send_offset_fetch_request(self, group, payloads=None,
                                  fail_on_error=True, callback=None):
        """
        Takes a group (string) and list of OffsetFetchRequest and returns
        a list of OffsetFetchResponse objects
        """
        encoder = partial(KafkaCodec.encode_offset_fetch_request,
                          group=group)
        decoder = KafkaCodec.decode_offset_fetch_response
        resps = yield self._send_broker_aware_request(
            payloads, encoder, decoder, consumer_group=group)

        returnValue(self._handle_responses(
            resps, fail_on_error, callback, group))

Response: Sends an OffsetFetchRequest to the broker.


Instruction:
Write a Python 3 script to
send a list of OffsetCommitRequest objects to the Kafka broker for the given consumer group.

Response:
def send_offset_commit_request(self, group, payloads=None,
                                   fail_on_error=True, callback=None,
                                   group_generation_id=-1,
                                   consumer_id=''):
        """Send a list of OffsetCommitRequests to the Kafka broker for the
        given consumer group.

        Args:
          group (str): The consumer group to which to commit the offsets
          payloads ([OffsetCommitRequest]): List of topic, partition, offsets
            to commit.
          fail_on_error (bool): Whether to raise an exception if a response
            from the Kafka broker indicates an error
          callback (callable): a function to call with each of the responses
            before returning the returned value to the caller.
          group_generation_id (int): Must currently always be -1
          consumer_id (str): Must currently always be empty string
        Returns:
          [OffsetCommitResponse]: List of OffsetCommitResponse objects.
          Will raise KafkaError for failed requests if fail_on_error is True
        """
        group = _coerce_consumer_group(group)
        encoder = partial(KafkaCodec.encode_offset_commit_request,
                          group=group, group_generation_id=group_generation_id,
                          consumer_id=consumer_id)
        decoder = KafkaCodec.decode_offset_commit_response
        resps = yield self._send_broker_aware_request(
            payloads, encoder, decoder, consumer_group=group)

        returnValue(self._handle_responses(
            resps, fail_on_error, callback, group))


Instruction:
Given the following Python 3 function, write the documentation
def _get_brokerclient(self, node_id):
        if self._closing:
            raise ClientError("Cannot get broker client for node_id={}: {} has been closed".format(node_id, self))
        if node_id not in self.clients:
            broker_metadata = self._brokers[node_id]
            log.debug("%r: creating client for %s", self, broker_metadata)
            self.clients[node_id] = _KafkaBrokerClient(
                self.reactor, self._endpoint_factory,
                broker_metadata, self.clientId, self._retry_policy,
            )
        return self.clients[node_id]

Response: Get a broker client.


Instruction:
How would you explain what the following Python 3 function does
def _close_brokerclients(self, clients):
        def _log_close_failure(failure, brokerclient):
            log.debug(
                'BrokerClient: %s close result: %s: %s', brokerclient,
                failure.type.__name__, failure.getErrorMessage())

        def _clean_close_dlist(result, close_dlist):
            # If there aren't any other outstanding closings going on, then
            # close_dlist == self.close_dlist, and we can reset it.
            if close_dlist == self.close_dlist:
                self.close_dlist = None

        if not self.close_dlist:
            dList = []
        else:
            log.debug("%r: _close_brokerclients has nested deferredlist: %r",
                      self, self.close_dlist)
            dList = [self.close_dlist]
        for brokerClient in clients:
            log.debug("Calling close on: %r", brokerClient)
            d = brokerClient.close().addErrback(_log_close_failure, brokerClient)
            dList.append(d)
        self.close_dlist = DeferredList(dList)
        self.close_dlist.addBoth(_clean_close_dlist, self.close_dlist)

Response: Close the given broker clients.


Instruction:
Can you tell what is the following Python 3 function doing
def _update_brokers(self, brokers, remove=False):
        log.debug("%r: _update_brokers(%r, remove=%r)",
                  self, brokers, remove)
        brokers_by_id = {bm.node_id: bm for bm in brokers}
        self._brokers.update(brokers_by_id)

        # Update the metadata of broker clients that already exist.
        for node_id, broker_meta in brokers_by_id.items():
            if node_id not in self.clients:
                continue
            self.clients[node_id].updateMetadata(broker_meta)

        # Remove any clients for brokers which no longer exist.
        if remove:
            to_close = [
                self.clients.pop(node_id)
                for node_id in set(self.clients) - set(brokers_by_id)
            ]

            if to_close:
                self._close_brokerclients(to_close)

Response: Update our self. brokers and self. clients based on brokers in received metadata.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_leader_for_partition(self, topic, partition):

        key = TopicAndPartition(topic, partition)
        # reload metadata whether the partition is not available
        # or has no leader (broker is None)
        if self.topics_to_brokers.get(key) is None:
            yield self.load_metadata_for_topics(topic)

        if key not in self.topics_to_brokers:
            raise PartitionUnavailableError("%s not available" % str(key))

        returnValue(self.topics_to_brokers[key])

Response: Get the leader for a topic and partition.


Instruction:
How would you explain what the following Python 3 function does
def _get_coordinator_for_group(self, consumer_group):
        if self.consumer_group_to_brokers.get(consumer_group) is None:
            yield self.load_consumer_metadata_for_group(consumer_group)

        returnValue(self.consumer_group_to_brokers.get(consumer_group))

Response: Returns the broker for a given consumer group


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _make_request_to_broker(self, broker, requestId, request, **kwArgs):
        def _timeout_request(broker, requestId):
            """The time we allotted for the request expired, cancel it."""
            try:
                # FIXME: This should be done by calling .cancel() on the Deferred
                # returned by the broker client.
                broker.cancelRequest(requestId, reason=RequestTimedOutError(
                    'Request: {} cancelled due to timeout'.format(requestId)))
            except KeyError:  # pragma: no cover This should never happen...
                log.exception('ERROR: Failed to find key for timed-out '
                              'request. Broker: %r Req: %d',
                              broker, requestId)
                raise
            if self._disconnect_on_timeout:
                broker.disconnect()

        def _alert_blocked_reactor(timeout, start):
            """Complain if this timer didn't fire before the timeout elapsed"""
            now = self.reactor.seconds()
            if now >= (start + timeout):
                log.warning('Reactor was starved for %r seconds', now - start)

        def _cancel_timeout(result, dc):
            """Request completed/cancelled, cancel the timeout delayedCall."""
            if dc.active():
                dc.cancel()
            return result

        # Make the request to the specified broker
        log.debug('_mrtb: sending request: %d to broker: %r',
                  requestId, broker)
        d = broker.makeRequest(requestId, request, **kwArgs)
        # Set a delayedCall to fire if we don't get a reply in time
        dc = self.reactor.callLater(
            self.timeout, _timeout_request, broker, requestId)
        # Set a delayedCall to complain if the reactor has been blocked
        rc = self.reactor.callLater(
            (self.timeout * 0.9), _alert_blocked_reactor, self.timeout,
            self.reactor.seconds())
        # Setup a callback on the request deferred to cancel both callLater
        d.addBoth(_cancel_timeout, dc)
        d.addBoth(_cancel_timeout, rc)
        return d

Response: Send a request to the specified broker.


Instruction:
Create a Python 3 function for
sending a broker - agnostic request to all known brokers.

Response:
def _send_broker_unaware_request(self, requestId, request):
        """
        Attempt to send a broker-agnostic request to one of the known brokers:

        1. Try each connected broker (in random order)
        2. Try each known but unconnected broker (in random order)
        3. Try each of the bootstrap hosts (in random order)

        :param bytes request:
            The bytes of a Kafka `RequestMessage`_ structure. It must have
            a unique (to this connection) correlation ID.

        :returns: API response message for *request*
        :rtype: Deferred[bytes]

        :raises:
            `KafkaUnavailableError` when making the request of all known hosts
            has failed.
        """
        node_ids = list(self._brokers.keys())
        # Randomly shuffle the brokers to distribute the load
        random.shuffle(node_ids)

        # Prioritize connected brokers
        def connected(node_id):
            try:
                return self.clients[node_id].connected()
            except KeyError:
                return False

        node_ids.sort(reverse=True, key=connected)

        for node_id in node_ids:
            broker = self._get_brokerclient(node_id)
            try:
                log.debug('_sbur: sending request %d to broker %r', requestId, broker)
                d = self._make_request_to_broker(broker, requestId, request)
                resp = yield d
                returnValue(resp)
            except KafkaError as e:
                log.warning((
                    "Will try next server after request with correlationId=%d"
                    " failed against server %s:%i. Error: %s"
                ), requestId, broker.host, broker.port, e)

        # The request was not handled, likely because no broker metadata has
        # loaded yet (or all broker connections have failed). Fall back to
        # boostrapping.
        returnValue((yield self._send_bootstrap_request(request)))


Instruction:
Implement a function in Python 3 to
send a request to all configured hosts and return the response.

Response:
def _send_bootstrap_request(self, request):
        """Make a request using an ephemeral broker connection

        This routine is used to make broker-unaware requests to get the initial
        cluster metadata. It cycles through the configured hosts, trying to
        connect and send the request to each in turn. This temporary connection
        is closed once a response is received.

        Note that most Kafka APIs require requests be sent to a specific
        broker. This method will only function for broker-agnostic requests
        like:

          * `Metadata <https://kafka.apache.org/protocol.html#The_Messages_Metadata>`_
          * `FindCoordinator <https://kafka.apache.org/protocol.html#The_Messages_FindCoordinator>`_

        :param bytes request:
            The bytes of a Kafka `RequestMessage`_ structure. It must have
            a unique (to this connection) correlation ID.

        :returns: API response message for *request*
        :rtype: Deferred[bytes]

        :raises:
            - `KafkaUnavailableError` when making the request of all known hosts
               has failed.
            - `twisted.internet.defer.TimeoutError` when connecting or making
               a request exceeds the timeout.
        """
        hostports = list(self._bootstrap_hosts)
        random.shuffle(hostports)
        for host, port in hostports:
            ep = self._endpoint_factory(self.reactor, host, port)
            try:
                protocol = yield ep.connect(_bootstrapFactory)
            except Exception as e:
                log.debug("%s: bootstrap connect to %s:%s -> %s", self, host, port, e)
                continue

            try:
                response = yield protocol.request(request).addTimeout(self.timeout, self.reactor)
            except Exception:
                log.debug("%s: bootstrap request to %s:%s failed", self, host, port, exc_info=True)
            else:
                returnValue(response)
            finally:
                protocol.transport.loseConnection()

        raise KafkaUnavailableError("Failed to bootstrap from hosts {}".format(hostports))


Instruction:
Here you have a function in Python 3, explain what it does
def _send_broker_aware_request(self, payloads, encoder_fn, decode_fn,
                                   consumer_group=None):
        """
        Group a list of request payloads by topic+partition and send them to
        the leader broker for that partition using the supplied encode/decode
        functions

        Params
        ======
        payloads: list of object-like entities with a topic and
                  partition attribute. payloads must be grouped by
                  (topic, partition) tuples.
        encode_fn: a method to encode the list of payloads to a request body,
                   must accept client_id, correlation_id, and payloads as
                   keyword arguments
        decode_fn: a method to decode a response body into response objects.
                   The response objects must be object-like and have topic
                   and partition attributes
        consumer_group: [string], optional. Indicates the request should be
                   directed to the Offset Coordinator for the specified
                   consumer_group.

        Return
        ======
        deferred yielding a list of response objects in the same order
        as the supplied payloads, or None if decode_fn is None.

        Raises
        ======
        FailedPayloadsError, LeaderUnavailableError, PartitionUnavailableError,

        """

        # Calling this without payloads is nonsensical
        if not payloads:
            raise ValueError("Payloads parameter is empty")

        # Group the requests by topic+partition
        original_keys = []
        payloads_by_broker = collections.defaultdict(list)

        # Go through all the payloads, lookup the leader/coordinator for that
        # payload's topic/partition or consumer group. If there's no
        # leader/coordinator (broker), raise. For each broker, keep
        # a list of the payloads to be sent to it. Also, for each payload in
        # the list of payloads, make a corresponding list (original_keys) with
        # the topic/partition in the same order, so we can lookup the returned
        # result(s) by that topic/partition key in the set of returned results
        # and return them in a list the same order the payloads were supplied
        for payload in payloads:
            # get leader/coordinator, depending on consumer_group
            if consumer_group is None:
                leader = yield self._get_leader_for_partition(
                    payload.topic, payload.partition)
                if leader is None:
                    raise LeaderUnavailableError(
                        "Leader not available for topic %s partition %s" %
                        (payload.topic, payload.partition))
            else:
                leader = yield self._get_coordinator_for_group(consumer_group)
                if leader is None:
                    raise ConsumerCoordinatorNotAvailableError(
                        "Coordinator not available for group: %s" %
                        (consumer_group))

            payloads_by_broker[leader].append(payload)
            original_keys.append((payload.topic, payload.partition))

        # Accumulate the responses in a dictionary
        acc = {}

        # The kafka server doesn't send replies to produce requests
        # with acks=0. In that case, our decode_fn will be
        # None, and we need to let the brokerclient know not
        # to expect a reply. makeRequest() returns a deferred
        # regardless, but in the expectResponse=False case, it will
        # fire as soon as the request is sent, and it can errBack()
        # due to being cancelled prior to the broker being able to
        # send the request.
        expectResponse = decode_fn is not None

        # keep a list of payloads that were failed to be sent to brokers
        failed_payloads = []

        # Keep track of outstanding requests in a list of deferreds
        inFlight = []
        # and the payloads that go along with them
        payloadsList = []
        # For each broker, send the list of request payloads,
        for broker_meta, payloads in payloads_by_broker.items():
            broker = self._get_brokerclient(broker_meta.node_id)
            requestId = self._next_id()
            request = encoder_fn(client_id=self._clientIdBytes,
                                 correlation_id=requestId, payloads=payloads)

            # Make the request
            d = self._make_request_to_broker(broker, requestId, request,
                                             expectResponse=expectResponse)
            inFlight.append(d)
            payloadsList.append(payloads)

        # Wait for all the responses to come back, or the requests to fail
        results = yield DeferredList(inFlight, consumeErrors=True)
        # We now have a list of (succeeded, response/Failure) tuples. Check 'em
        for (success, response), payloads in zip(results, payloadsList):
            if not success:
                # The brokerclient deferred was errback()'d:
                #   The send failed, or this request was cancelled (by timeout)
                log.debug("%r: request:%r to broker failed: %r", self,
                          payloads, response)
                failed_payloads.extend([(p, response) for p in payloads])
                continue
            if not expectResponse:
                continue
            # Successful request/response. Decode it and store by topic/part
            for response in decode_fn(response):
                acc[(response.topic, response.partition)] = response

        # Order the accumulated responses by the original key order
        # Note that this scheme will throw away responses which we did
        # not request.  See test_send_fetch_request, where the response
        # includes an error, but for a topic/part we didn't request.
        # Since that topic/partition isn't in original_keys, we don't pass
        # it back from here and it doesn't error out.
        # If any of the payloads failed, fail
        responses = [acc[k] for k in original_keys if k in acc] if acc else []
        if failed_payloads:
            self.reset_all_metadata()
            raise FailedPayloadsError(responses, failed_payloads)

        returnValue(responses)

Response: Send a request to the broker for the specified topic + partition.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_description(expression, options=None):
    descripter = ExpressionDescriptor(expression, options)
    return descripter.get_description(DescriptionTypeEnum.FULL)

Response: Generates a human readable string for the Cron Expression


Instruction:
Can you write a function in Python 3 where it
generates a human readable string for the Cron Expression

Response:
def get_description(self, description_type=DescriptionTypeEnum.FULL):
        """Generates a human readable string for the Cron Expression

        Args:
            description_type: Which part(s) of the expression to describe
        Returns:
            The cron expression description
        Raises:
            Exception: if throw_exception_on_parse_error is True

        """
        try:
            if self._parsed is False:
                parser = ExpressionParser(self._expression, self._options)
                self._expression_parts = parser.parse()
                self._parsed = True

            choices = {
                DescriptionTypeEnum.FULL: self.get_full_description,
                DescriptionTypeEnum.TIMEOFDAY: self.get_time_of_day_description,
                DescriptionTypeEnum.HOURS: self.get_hours_description,
                DescriptionTypeEnum.MINUTES: self.get_minutes_description,
                DescriptionTypeEnum.SECONDS: self.get_seconds_description,
                DescriptionTypeEnum.DAYOFMONTH: self.get_day_of_month_description,
                DescriptionTypeEnum.MONTH: self.get_month_description,
                DescriptionTypeEnum.DAYOFWEEK: self.get_day_of_week_description,
                DescriptionTypeEnum.YEAR: self.get_year_description,
            }

            description = choices.get(description_type, self.get_seconds_description)()

        except Exception as ex:
            if self._options.throw_exception_on_parse_error:
                raise
            else:
                description = str(ex)
        return description


Instruction:
How would you code a function in Python 3 to
generate the FULL description of the current entry in the cron expression

Response:
def get_full_description(self):
        """Generates the FULL description

        Returns:
            The FULL description
        Raises:
            FormatException: if formating fails and throw_exception_on_parse_error is True

        """

        try:
            time_segment = self.get_time_of_day_description()
            day_of_month_desc = self.get_day_of_month_description()
            month_desc = self.get_month_description()
            day_of_week_desc = self.get_day_of_week_description()
            year_desc = self.get_year_description()

            description = "{0}{1}{2}{3}{4}".format(
                time_segment,
                day_of_month_desc,
                day_of_week_desc,
                month_desc,
                year_desc)

            description = self.transform_verbosity(
                description, self._options.verbose)
            description = self.transform_case(
                description,
                self._options.casing_type)
        except Exception:
            description = _(
                "An error occured when generating the expression description.  Check the cron expression syntax.")
            if self._options.throw_exception_on_parse_error:
                raise FormatException(description)

        return description


Instruction:
Write a Python 3 script for
generating a description for only the TIMEOFDAY portion of the expression.

Response:
def get_time_of_day_description(self):
        """Generates a description for only the TIMEOFDAY portion of the expression

        Returns:
            The TIMEOFDAY description

        """
        seconds_expression = self._expression_parts[0]
        minute_expression = self._expression_parts[1]
        hour_expression = self._expression_parts[2]

        description = StringBuilder()

        # handle special cases first
        if any(exp in minute_expression for exp in self._special_characters) is False and \
            any(exp in hour_expression for exp in self._special_characters) is False and \
                any(exp in seconds_expression for exp in self._special_characters) is False:
            # specific time of day (i.e. 10 14)
            description.append(_("At "))
            description.append(
                self.format_time(
                    hour_expression,
                    minute_expression,
                    seconds_expression))
        elif "-" in minute_expression and \
            "," not in minute_expression and \
                any(exp in hour_expression for exp in self._special_characters) is False:
            # minute range in single hour (i.e. 0-10 11)
            minute_parts = minute_expression.split('-')
            description.append(_("Every minute between {0} and {1}").format(
                self.format_time(hour_expression, minute_parts[0]), self.format_time(hour_expression, minute_parts[1])))
        elif "," in hour_expression and "-" not in hour_expression and \
                any(exp in minute_expression for exp in self._special_characters) is False:
            # hours list with single minute (o.e. 30 6,14,16)
            hour_parts = hour_expression.split(',')
            description.append(_("At"))
            for i, hour_part in enumerate(hour_parts):
                description.append(" ")
                description.append(
                    self.format_time(hour_part, minute_expression))

                if i < (len(hour_parts) - 2):
                    description.append(",")

                if i == len(hour_parts) - 2:
                    description.append(_(" and"))
        else:
            # default time description
            seconds_description = self.get_seconds_description()
            minutes_description = self.get_minutes_description()
            hours_description = self.get_hours_description()

            description.append(seconds_description)

            if description:
                description.append(", ")

            description.append(minutes_description)

            if description:
                description.append(", ")

            description.append(hours_description)
        return str(description)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_seconds_description(self):

        return self.get_segment_description(
            self._expression_parts[0],
            _("every second"),
            lambda s: s,
            lambda s: _("every {0} seconds").format(s),
            lambda s: _("seconds {0} through {1} past the minute"),
            lambda s: _("at {0} seconds past the minute")
        )

Response: Generates a description for only the SECONDS portion of the expression


Instruction:
Can you generate the documentation for the following Python 3 function
def get_minutes_description(self):

        return self.get_segment_description(
            self._expression_parts[1],
            _("every minute"),
            lambda s: s,
            lambda s: _("every {0} minutes").format(s),
            lambda s: _("minutes {0} through {1} past the hour"),
            lambda s: '' if s == "0" else _("at {0} minutes past the hour")
        )

Response: Generates a description for only the MINUTE portion of the expression


Instruction:
Make a summary of the following Python 3 code
def get_hours_description(self):
        expression = self._expression_parts[2]
        return self.get_segment_description(
            expression,
            _("every hour"),
            lambda s: self.format_time(s, "0"),
            lambda s: _("every {0} hours").format(s),
            lambda s: _("between {0} and {1}"),
            lambda s: _("at {0}")
        )

Response: Generates a description for only the HOUR portion of the expression


Instruction:
Given the following Python 3 function, write the documentation
def get_day_of_week_description(self):

        if self._expression_parts[5] == "*" and self._expression_parts[3] != "*":
            # DOM is specified and DOW is * so to prevent contradiction like "on day 1 of the month, every day"
            # we will not specified a DOW description.
            return ""

        def get_day_name(s):
            exp = s
            if "#" in s:
                exp, useless = s.split("#", 2)
            elif "L" in s:
                exp = exp.replace("L", '')
            return self.number_to_day(int(exp))

        def get_format(s):
            if "#" in s:
                day_of_week_of_month = s[s.find("#") + 1:]

                try:
                    day_of_week_of_month_number = int(day_of_week_of_month)
                    choices = {
                        1: _("first"),
                        2: _("second"),
                        3: _("third"),
                        4: _("forth"),
                        5: _("fifth"),
                    }
                    day_of_week_of_month_description = choices.get(day_of_week_of_month_number, '')
                except ValueError:
                    day_of_week_of_month_description = ''

                formated = "{}{}{}".format(_(", on the "),
                                           day_of_week_of_month_description, _(" {0} of the month"))
            elif "L" in s:
                formated = _(", on the last {0} of the month")
            else:
                formated = _(", only on {0}")

            return formated

        return self.get_segment_description(
            self._expression_parts[5],
            _(", every day"),
            lambda s: get_day_name(s),
            lambda s: _(", every {0} days of the week").format(s),
            lambda s: _(", {0} through {1}"),
            lambda s: get_format(s)
        )

Response: Generates a description for only the DAYOFWEEK portion of the expression.


Instruction:
Can you tell what is the following Python 3 function doing
def get_month_description(self):
        return self.get_segment_description(
            self._expression_parts[4],
            '',
            lambda s: datetime.date(datetime.date.today().year, int(s), 1).strftime("%B"),
            lambda s: _(", every {0} months").format(s),
            lambda s: _(", {0} through {1}"),
            lambda s: _(", only in {0}")
        )

Response: Generates a description for only the MONTH portion of the expression


Instruction:
Can you generate the documentation for the following Python 3 function
def get_day_of_month_description(self):
        expression = self._expression_parts[3]
        expression = expression.replace("?", "*")

        if expression == "L":
            description = _(", on the last day of the month")
        elif expression == "LW" or expression == "WL":
            description = _(", on the last weekday of the month")
        else:
            regex = re.compile("(\\d{1,2}W)|(W\\d{1,2})")
            if regex.match(expression):
                m = regex.match(expression)
                day_number = int(m.group().replace("W", ""))

                day_string = _("first weekday") if day_number == 1 else _("weekday nearest day {0}").format(
                    day_number)
                description = _(", on the {0} of the month").format(
                    day_string)
            else:
                description = self.get_segment_description(
                    expression,
                    _(", every day"),
                    lambda s: s,
                    lambda s: _(", every day") if s == "1" else _(", every {0} days"),
                    lambda s: _(", between day {0} and {1} of the month"),
                    lambda s: _(", on day {0} of the month")
                )

        return description

Response: Generates a description for only the DAYOFMONTH portion of the expression


Instruction:
Create a Python 3 function for
generating a description for only the YEAR portion of the expression

Response:
def get_year_description(self):
        """Generates a description for only the YEAR portion of the expression

        Returns:
            The YEAR description

        """

        def format_year(s):
            regex = re.compile(r"^\d+$")
            if regex.match(s):
                year_int = int(s)
                if year_int < 1900:
                    return year_int
                return datetime.date(year_int, 1, 1).strftime("%Y")
            else:
                return s

        return self.get_segment_description(
            self._expression_parts[6],
            '',
            lambda s: format_year(s),
            lambda s: _(", every {0} years").format(s),
            lambda s: _(", {0} through {1}"),
            lambda s: _(", only in {0}")
        )


Instruction:
Can you write a function in Python 3 where it
returns the string representation of the current entry in the language of the item in the archive.

Response:
def get_segment_description(
        self,
        expression,
        all_description,
        get_single_item_description,
        get_interval_description_format,
        get_between_description_format,
        get_description_format
    ):
        """Returns segment description
        Args:
            expression: Segment to descript
            all_description: *
            get_single_item_description: 1
            get_interval_description_format: 1/2
            get_between_description_format: 1-2
            get_description_format: format get_single_item_description
        Returns:
            segment description

        """
        description = None
        if expression is None or expression == '':
            description = ''
        elif expression == "*":
            description = all_description
        elif any(ext in expression for ext in ['/', '-', ',']) is False:
            description = get_description_format(expression).format(
                get_single_item_description(expression))
        elif "/" in expression:
            segments = expression.split('/')
            description = get_interval_description_format(
                segments[1]).format(get_single_item_description(segments[1]))

            # interval contains 'between' piece (i.e. 2-59/3 )
            if "-" in segments[0]:
                between_segment_description = self.generate_between_segment_description(
                    segments[0], get_between_description_format, get_single_item_description)
                if not between_segment_description.startswith(", "):
                    description += ", "
                description += between_segment_description
            elif any(ext in segments[0] for ext in ['*', ',']) is False:
                range_item_description = get_description_format(segments[0]).format(
                    get_single_item_description(segments[0])
                )
                range_item_description = range_item_description.replace(", ", "")

                description += _(", starting {0}").format(range_item_description)
        elif "," in expression:
            segments = expression.split(',')

            description_content = ''
            for i, segment in enumerate(segments):
                if i > 0 and len(segments) > 2:
                    description_content += ","

                    if i < len(segments) - 1:
                        description_content += " "

                if i > 0 and len(segments) > 1 and (i == len(segments) - 1 or len(segments) == 2):
                    description_content += _(" and ")

                if "-" in segment:
                    between_description = self.generate_between_segment_description(
                        segment,
                        lambda s: _(", {0} through {1}"),
                        get_single_item_description
                    )

                    between_description = between_description.replace(", ", "")

                    description_content += between_description
                else:
                    description_content += get_single_item_description(segment)

            description = get_description_format(
                expression).format(
                    description_content)
        elif "-" in expression:
            description = self.generate_between_segment_description(
                expression, get_between_description_format, get_single_item_description)

        return description


Instruction:
Implement a function in Python 3 to
generate the between segment description

Response:
def generate_between_segment_description(
            self,
            between_expression,
            get_between_description_format,
            get_single_item_description
    ):
        """
        Generates the between segment description
        :param between_expression:
        :param get_between_description_format:
        :param get_single_item_description:
        :return: The between segment description
        """
        description = ""
        between_segments = between_expression.split('-')
        between_segment_1_description = get_single_item_description(between_segments[0])
        between_segment_2_description = get_single_item_description(between_segments[1])
        between_segment_2_description = between_segment_2_description.replace(
            ":00", ":59")

        between_description_format = get_between_description_format(between_expression)
        description += between_description_format.format(between_segment_1_description, between_segment_2_description)

        return description


Instruction:
Can you write a function in Python 3 where it
gives time parts, will contruct a formatted time description Args: hour_expression: Hours part minute_expression: Minutes part second_expression: Seconds part Returns: Formatted time description

Response:
def format_time(
        self,
        hour_expression,
        minute_expression,
        second_expression=''
    ):
        """Given time parts, will contruct a formatted time description
        Args:
            hour_expression: Hours part
            minute_expression: Minutes part
            second_expression: Seconds part
        Returns:
            Formatted time description

        """
        hour = int(hour_expression)

        period = ''
        if self._options.use_24hour_time_format is False:
            period = " PM" if (hour >= 12) else " AM"
            if hour > 12:
                hour -= 12

        minute = str(int(minute_expression))  # !FIXME WUT ???
        second = ''
        if second_expression is not None and second_expression:
            second = "{}{}".format(":", str(int(second_expression)).zfill(2))

        return "{0}:{1}{2}{3}".format(str(hour).zfill(2), minute.zfill(2), second, period)


Instruction:
Given the following Python 3 function, write the documentation
def transform_verbosity(self, description, use_verbose_format):
        if use_verbose_format is False:
            description = description.replace(
                _(", every minute"), '')
            description = description.replace(_(", every hour"), '')
            description = description.replace(_(", every day"), '')
        return description

Response: Transforms the verbosity of the expression description by stripping the verbosity from original description


Instruction:
Can you implement a function in Python 3 that
transforms the case of the expression description based on options

Response:
def transform_case(self, description, case_type):
        """Transforms the case of the expression description, based on options
        Args:
            description: The description to transform
            case_type: The casing type that controls the output casing
            second_expression: Seconds part
        Returns:
            The transformed description with proper casing

        """
        if case_type == CasingTypeEnum.Sentence:
            description = "{}{}".format(
                description[0].upper(),
                description[1:])
        elif case_type == CasingTypeEnum.Title:
            description = description.title()
        else:
            description = description.lower()
        return description


Instruction:
Write a Python 3 script to
return localized day name by its CRON number

Response:
def number_to_day(self, day_number):
        """Returns localized day name by its CRON number

        Args:
            day_number: Number of a day
        Returns:
            Day corresponding to day_number
        Raises:
            IndexError: When day_number is not found
        """
        return [
            calendar.day_name[6],
            calendar.day_name[0],
            calendar.day_name[1],
            calendar.day_name[2],
            calendar.day_name[3],
            calendar.day_name[4],
            calendar.day_name[5]
        ][day_number]


Instruction:
How would you explain what the following Python 3 function does
def start(self, start_offset):
        # Have we been started already, and not stopped?
        if self._start_d is not None:
            raise RestartError("Start called on already-started consumer")

        # Keep track of state for debugging
        self._state = '[started]'

        # Create and return a deferred for alerting on errors/stoppage
        start_d = self._start_d = Deferred()

        # Start a new fetch request, possibly just for the starting offset
        self._fetch_offset = start_offset
        self._do_fetch()

        # Set up the auto-commit timer, if needed
        if self.consumer_group and self.auto_commit_every_s:
            self._commit_looper = LoopingCall(self._auto_commit)
            self._commit_looper.clock = self.client.reactor
            self._commit_looper_d = self._commit_looper.start(
                self.auto_commit_every_s, now=False)
            self._commit_looper_d.addCallbacks(self._commit_timer_stopped,
                                               self._commit_timer_failed)
        return start_d

Response: Starts fetching messages from Kafka and deliver them to Kafka.


Instruction:
Can you generate the documentation for the following Python 3 function
def shutdown(self):
        def _handle_shutdown_commit_success(result):
            """Handle the result of the commit attempted by shutdown"""
            self._shutdown_d, d = None, self._shutdown_d
            self.stop()
            self._shuttingdown = False  # Shutdown complete
            d.callback((self._last_processed_offset,
                       self._last_committed_offset))

        def _handle_shutdown_commit_failure(failure):
            """Handle failure of commit() attempted by shutdown"""
            if failure.check(OperationInProgress):
                failure.value.deferred.addCallback(_commit_and_stop)
                return

            self._shutdown_d, d = None, self._shutdown_d
            self.stop()
            self._shuttingdown = False  # Shutdown complete
            d.errback(failure)

        def _commit_and_stop(result):
            """Commit the current offsets (if needed) and stop the consumer"""
            if not self.consumer_group:  # No consumer group, no committing
                return _handle_shutdown_commit_success(None)

            # Need to commit prior to stopping
            self.commit().addCallbacks(_handle_shutdown_commit_success,
                                       _handle_shutdown_commit_failure)

        # If we're not running, return an failure
        if self._start_d is None:
            return fail(Failure(
                RestopError("Shutdown called on non-running consumer")))
        # If we're called multiple times, return a failure
        if self._shutdown_d:
            return fail(Failure(
                RestopError("Shutdown called more than once.")))
        # Set our _shuttingdown flag, so our _process_message routine will stop
        # feeding new messages to the processor, and fetches won't be retried
        self._shuttingdown = True
        # Keep track of state for debugging
        self._state = '[shutting down]'

        # Create a deferred to track the shutdown
        self._shutdown_d = d = Deferred()

        # Are we waiting for the processor to complete? If so, when it's done,
        # commit our offsets and stop.
        if self._processor_d:
            self._processor_d.addCallback(_commit_and_stop)
        else:
            # No need to wait for the processor, we can commit and stop now
            _commit_and_stop(None)

        # return the deferred
        return d

Response: Gracefully shutdown the consumer and return a deferred which will be called with a tuple of the last processed offset last committed offset and the state of the current consumer.