Instruction:
How would you code a function in Python 3 to
collect a comma - separated set of hosts and optionally randomize the returned list.

Response:
def collect_hosts(hosts, randomize=True):
    """
    Collects a comma-separated set of hosts (host:port) and optionally
    randomize the returned list.
    """

    if isinstance(hosts, six.string_types):
        hosts = hosts.strip().split(',')

    result = []
    afi = socket.AF_INET
    for host_port in hosts:

        host, port, afi = get_ip_port_afi(host_port)

        if port < 0:
            port = DEFAULT_KAFKA_PORT

        result.append((host, port, afi))

    if randomize:
        shuffle(result)

    return result


Instruction:
Explain what the following Python 3 code does
def dns_lookup(host, port, afi=socket.AF_UNSPEC):
    # XXX: all DNS functions in Python are blocking. If we really
    # want to be non-blocking here, we need to use a 3rd-party
    # library like python-adns, or move resolution onto its
    # own thread. This will be subject to the default libc
    # name resolution timeout (5s on most Linux boxes)
    try:
        return list(filter(is_inet_4_or_6,
                           socket.getaddrinfo(host, port, afi,
                                              socket.SOCK_STREAM)))
    except socket.gaierror as ex:
        log.warning('DNS lookup failed for %s:%d,'
                    ' exception was %s. Is your'
                    ' advertised.listeners (called'
                    ' advertised.host.name before Kafka 9)'
                    ' correct and resolvable?',
                    host, port, ex)
        return []

Response: Returns a list of getaddrinfo structs optionally filtered to an AFI.


Instruction:
Implement a Python 3 function for
attempting to connect and returns ConnectionState

Response:
def connect(self):
        """Attempt to connect and return ConnectionState"""
        if self.state is ConnectionStates.DISCONNECTED and not self.blacked_out():
            self.last_attempt = time.time()
            next_lookup = self._next_afi_sockaddr()
            if not next_lookup:
                self.close(Errors.KafkaConnectionError('DNS failure'))
                return self.state
            else:
                log.debug('%s: creating new socket', self)
                assert self._sock is None
                self._sock_afi, self._sock_addr = next_lookup
                self._sock = socket.socket(self._sock_afi, socket.SOCK_STREAM)

            for option in self.config['socket_options']:
                log.debug('%s: setting socket option %s', self, option)
                self._sock.setsockopt(*option)

            self._sock.setblocking(False)
            self.state = ConnectionStates.CONNECTING
            self.config['state_change_callback'](self.node_id, self._sock, self)
            log.info('%s: connecting to %s:%d [%s %s]', self, self.host,
                     self.port, self._sock_addr, AFI_NAMES[self._sock_afi])

        if self.state is ConnectionStates.CONNECTING:
            # in non-blocking mode, use repeated calls to socket.connect_ex
            # to check connection status
            ret = None
            try:
                ret = self._sock.connect_ex(self._sock_addr)
            except socket.error as err:
                ret = err.errno

            # Connection succeeded
            if not ret or ret == errno.EISCONN:
                log.debug('%s: established TCP connection', self)

                if self.config['security_protocol'] in ('SSL', 'SASL_SSL'):
                    log.debug('%s: initiating SSL handshake', self)
                    self.state = ConnectionStates.HANDSHAKE
                    self.config['state_change_callback'](self.node_id, self._sock, self)
                    # _wrap_ssl can alter the connection state -- disconnects on failure
                    self._wrap_ssl()

                elif self.config['security_protocol'] == 'SASL_PLAINTEXT':
                    log.debug('%s: initiating SASL authentication', self)
                    self.state = ConnectionStates.AUTHENTICATING
                    self.config['state_change_callback'](self.node_id, self._sock, self)

                else:
                    # security_protocol PLAINTEXT
                    log.info('%s: Connection complete.', self)
                    self.state = ConnectionStates.CONNECTED
                    self._reset_reconnect_backoff()
                    self.config['state_change_callback'](self.node_id, self._sock, self)

            # Connection failed
            # WSAEINVAL == 10022, but errno.WSAEINVAL is not available on non-win systems
            elif ret not in (errno.EINPROGRESS, errno.EALREADY, errno.EWOULDBLOCK, 10022):
                log.error('Connect attempt to %s returned error %s.'
                          ' Disconnecting.', self, ret)
                errstr = errno.errorcode.get(ret, 'UNKNOWN')
                self.close(Errors.KafkaConnectionError('{} {}'.format(ret, errstr)))
                return self.state

            # Needs retry
            else:
                pass

        if self.state is ConnectionStates.HANDSHAKE:
            if self._try_handshake():
                log.debug('%s: completed SSL handshake.', self)
                if self.config['security_protocol'] == 'SASL_SSL':
                    log.debug('%s: initiating SASL authentication', self)
                    self.state = ConnectionStates.AUTHENTICATING
                else:
                    log.info('%s: Connection complete.', self)
                    self.state = ConnectionStates.CONNECTED
                    self._reset_reconnect_backoff()
                self.config['state_change_callback'](self.node_id, self._sock, self)

        if self.state is ConnectionStates.AUTHENTICATING:
            assert self.config['security_protocol'] in ('SASL_PLAINTEXT', 'SASL_SSL')
            if self._try_authenticate():
                # _try_authenticate has side-effects: possibly disconnected on socket errors
                if self.state is ConnectionStates.AUTHENTICATING:
                    log.info('%s: Connection complete.', self)
                    self.state = ConnectionStates.CONNECTED
                    self._reset_reconnect_backoff()
                    self.config['state_change_callback'](self.node_id, self._sock, self)

        if self.state not in (ConnectionStates.CONNECTED,
                              ConnectionStates.DISCONNECTED):
            # Connection timed out
            request_timeout = self.config['request_timeout_ms'] / 1000.0
            if time.time() > request_timeout + self.last_attempt:
                log.error('Connection attempt to %s timed out', self)
                self.close(Errors.KafkaConnectionError('timeout'))
                return self.state

        return self.state


Instruction:
How would you code a function in Python 3 to
return a string representation of the OPTIONAL key - value pairs that can be sent with an OAUTHBEARER initial request.

Response:
def _token_extensions(self):
        """
        Return a string representation of the OPTIONAL key-value pairs that can be sent with an OAUTHBEARER
        initial request.
        """
        token_provider = self.config['sasl_oauth_token_provider']

        # Only run if the #extensions() method is implemented by the clients Token Provider class
        # Builds up a string separated by \x01 via a dict of key value pairs
        if callable(getattr(token_provider, "extensions", None)) and len(token_provider.extensions()) > 0:
            msg = "\x01".join(["{}={}".format(k, v) for k, v in token_provider.extensions().items()])
            return "\x01" + msg
        else:
            return ""


Instruction:
How would you explain what the following Python 3 function does
def blacked_out(self):
        if self.state is ConnectionStates.DISCONNECTED:
            if time.time() < self.last_attempt + self._reconnect_backoff:
                return True
        return False

Response: Return True if we are disconnected from the given node and can t re - establish a connection yet.


Instruction:
Make a summary of the following Python 3 code
def connection_delay(self):
        time_waited = time.time() - (self.last_attempt or 0)
        if self.state is ConnectionStates.DISCONNECTED:
            return max(self._reconnect_backoff - time_waited, 0) * 1000
        elif self.connecting():
            return 0
        else:
            return float('inf')

Response: Returns the number of milliseconds to wait for the connection to finish.


Instruction:
Can you implement a function in Python 3 that
returns True if the connection is still connecting.

Response:
def connecting(self):
        """Returns True if still connecting (this may encompass several
        different states, such as SSL handshake, authorization, etc)."""
        return self.state in (ConnectionStates.CONNECTING,
                              ConnectionStates.HANDSHAKE,
                              ConnectionStates.AUTHENTICATING)


Instruction:
Can you generate the documentation for the following Python 3 function
def close(self, error=None):
        if self.state is ConnectionStates.DISCONNECTED:
            return
        with self._lock:
            if self.state is ConnectionStates.DISCONNECTED:
                return
            log.info('%s: Closing connection. %s', self, error or '')
            self._update_reconnect_backoff()
            self._sasl_auth_future = None
            self._protocol = KafkaProtocol(
                client_id=self.config['client_id'],
                api_version=self.config['api_version'])
            if error is None:
                error = Errors.Cancelled(str(self))
            ifrs = list(self.in_flight_requests.items())
            self.in_flight_requests.clear()
            self.state = ConnectionStates.DISCONNECTED
            # To avoid race conditions and/or deadlocks
            # keep a reference to the socket but leave it
            # open until after the state_change_callback
            # This should give clients a change to deregister
            # the socket fd from selectors cleanly.
            sock = self._sock
            self._sock = None

        # drop lock before state change callback and processing futures
        self.config['state_change_callback'](self.node_id, sock, self)
        sock.close()
        for (_correlation_id, (future, _timestamp)) in ifrs:
            future.failure(error)

Response: Closes the connection and fails all pending in - flight requests.


Instruction:
Can you implement a function in Python 3 that
queues a request for async network send

Response:
def send(self, request, blocking=True):
        """Queue request for async network send, return Future()"""
        future = Future()
        if self.connecting():
            return future.failure(Errors.NodeNotReadyError(str(self)))
        elif not self.connected():
            return future.failure(Errors.KafkaConnectionError(str(self)))
        elif not self.can_send_more():
            return future.failure(Errors.TooManyInFlightRequests(str(self)))
        return self._send(request, blocking=blocking)


Instruction:
Given the following Python 3 function, write the documentation
def send_pending_requests(self):
        try:
            with self._lock:
                if not self._can_send_recv():
                    return Errors.NodeNotReadyError(str(self))
                # In the future we might manage an internal write buffer
                # and send bytes asynchronously. For now, just block
                # sending each request payload
                data = self._protocol.send_bytes()
                total_bytes = self._send_bytes_blocking(data)
            if self._sensors:
                self._sensors.bytes_sent.record(total_bytes)
            return total_bytes
        except ConnectionError as e:
            log.exception("Error sending request data to %s", self)
            error = Errors.KafkaConnectionError("%s: %s" % (self, e))
            self.close(error=error)
            return error

Response: Send pending requests to the broker.


Instruction:
Can you generate the documentation for the following Python 3 function
def _recv(self):
        recvd = []
        self._lock.acquire()
        if not self._can_send_recv():
            log.warning('%s cannot recv: socket not connected', self)
            self._lock.release()
            return ()

        while len(recvd) < self.config['sock_chunk_buffer_count']:
            try:
                data = self._sock.recv(self.config['sock_chunk_bytes'])
                # We expect socket.recv to raise an exception if there are no
                # bytes available to read from the socket in non-blocking mode.
                # but if the socket is disconnected, we will get empty data
                # without an exception raised
                if not data:
                    log.error('%s: socket disconnected', self)
                    self._lock.release()
                    self.close(error=Errors.KafkaConnectionError('socket disconnected'))
                    return []
                else:
                    recvd.append(data)

            except SSLWantReadError:
                break
            except ConnectionError as e:
                if six.PY2 and e.errno == errno.EWOULDBLOCK:
                    break
                log.exception('%s: Error receiving network data'
                              ' closing socket', self)
                self._lock.release()
                self.close(error=Errors.KafkaConnectionError(e))
                return []
            except BlockingIOError:
                if six.PY3:
                    break
                self._lock.release()
                raise

        recvd_data = b''.join(recvd)
        if self._sensors:
            self._sensors.bytes_received.record(len(recvd_data))

        try:
            responses = self._protocol.receive_bytes(recvd_data)
        except Errors.KafkaProtocolError as e:
            self._lock.release()
            self.close(e)
            return []
        else:
            self._lock.release()
            return responses

Response: Take all available bytes from socket return list of any responses from parser


Instruction:
Can you create a Python 3 function that
attempts to guess the broker version.

Response:
def check_version(self, timeout=2, strict=False, topics=[]):
        """Attempt to guess the broker version.

        Note: This is a blocking call.

        Returns: version tuple, i.e. (0, 10), (0, 9), (0, 8, 2), ...
        """
        timeout_at = time.time() + timeout
        log.info('Probing node %s broker version', self.node_id)
        # Monkeypatch some connection configurations to avoid timeouts
        override_config = {
            'request_timeout_ms': timeout * 1000,
            'max_in_flight_requests_per_connection': 5
        }
        stashed = {}
        for key in override_config:
            stashed[key] = self.config[key]
            self.config[key] = override_config[key]

        # kafka kills the connection when it doesn't recognize an API request
        # so we can send a test request and then follow immediately with a
        # vanilla MetadataRequest. If the server did not recognize the first
        # request, both will be failed with a ConnectionError that wraps
        # socket.error (32, 54, or 104)
        from kafka.protocol.admin import ApiVersionRequest, ListGroupsRequest
        from kafka.protocol.commit import OffsetFetchRequest, GroupCoordinatorRequest

        test_cases = [
            # All cases starting from 0.10 will be based on ApiVersionResponse
            ((0, 10), ApiVersionRequest[0]()),
            ((0, 9), ListGroupsRequest[0]()),
            ((0, 8, 2), GroupCoordinatorRequest[0]('kafka-python-default-group')),
            ((0, 8, 1), OffsetFetchRequest[0]('kafka-python-default-group', [])),
            ((0, 8, 0), MetadataRequest[0](topics)),
        ]

        for version, request in test_cases:
            if not self.connect_blocking(timeout_at - time.time()):
                raise Errors.NodeNotReadyError()
            f = self.send(request)
            # HACK: sleeping to wait for socket to send bytes
            time.sleep(0.1)
            # when broker receives an unrecognized request API
            # it abruptly closes our socket.
            # so we attempt to send a second request immediately
            # that we believe it will definitely recognize (metadata)
            # the attempt to write to a disconnected socket should
            # immediately fail and allow us to infer that the prior
            # request was unrecognized
            mr = self.send(MetadataRequest[0](topics))

            selector = self.config['selector']()
            selector.register(self._sock, selectors.EVENT_READ)
            while not (f.is_done and mr.is_done):
                selector.select(1)
                for response, future in self.recv():
                    future.success(response)
            selector.close()

            if f.succeeded():
                if isinstance(request, ApiVersionRequest[0]):
                    # Starting from 0.10 kafka broker we determine version
                    # by looking at ApiVersionResponse
                    api_versions = self._handle_api_version_response(f.value)
                    version = self._infer_broker_version_from_api_versions(api_versions)
                log.info('Broker version identifed as %s', '.'.join(map(str, version)))
                log.info('Set configuration api_version=%s to skip auto'
                         ' check_version requests on startup', version)
                break

            # Only enable strict checking to verify that we understand failure
            # modes. For most users, the fact that the request failed should be
            # enough to rule out a particular broker version.
            if strict:
                # If the socket flush hack did not work (which should force the
                # connection to close and fail all pending requests), then we
                # get a basic Request Timeout. This is not ideal, but we'll deal
                if isinstance(f.exception, Errors.RequestTimedOutError):
                    pass

                # 0.9 brokers do not close the socket on unrecognized api
                # requests (bug...). In this case we expect to see a correlation
                # id mismatch
                elif (isinstance(f.exception, Errors.CorrelationIdError) and
                      version == (0, 10)):
                    pass
                elif six.PY2:
                    assert isinstance(f.exception.args[0], socket.error)
                    assert f.exception.args[0].errno in (32, 54, 104)
                else:
                    assert isinstance(f.exception.args[0], ConnectionError)
            log.info("Broker is not v%s -- it did not recognize %s",
                     version, request.__class__.__name__)
        else:
            raise Errors.UnrecognizedBrokerVersion()

        for key in stashed:
            self.config[key] = stashed[key]
        return version


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(self):
        log.debug("Starting Kafka producer I/O thread.")

        # main loop, runs until close is called
        while self._running:
            try:
                self.run_once()
            except Exception:
                log.exception("Uncaught error in kafka producer I/O thread")

        log.debug("Beginning shutdown of Kafka producer I/O thread, sending"
                  " remaining records.")

        # okay we stopped accepting requests but there may still be
        # requests in the accumulator or waiting for acknowledgment,
        # wait until these are completed.
        while (not self._force_close
               and (self._accumulator.has_unsent()
                    or self._client.in_flight_request_count() > 0)):
            try:
                self.run_once()
            except Exception:
                log.exception("Uncaught error in kafka producer I/O thread")

        if self._force_close:
            # We need to fail all the incomplete batches and wake up the
            # threads waiting on the futures.
            self._accumulator.abort_incomplete_batches()

        try:
            self._client.close()
        except Exception:
            log.exception("Failed to close network client")

        log.debug("Shutdown of Kafka producer I/O thread has completed.")

Response: The main loop for the sender thread.


Instruction:
How would you explain what the following Python 3 function does
def run_once(self):
        while self._topics_to_add:
            self._client.add_topic(self._topics_to_add.pop())

        # get the list of partitions with data ready to send
        result = self._accumulator.ready(self._metadata)
        ready_nodes, next_ready_check_delay, unknown_leaders_exist = result

        # if there are any partitions whose leaders are not known yet, force
        # metadata update
        if unknown_leaders_exist:
            log.debug('Unknown leaders exist, requesting metadata update')
            self._metadata.request_update()

        # remove any nodes we aren't ready to send to
        not_ready_timeout = float('inf')
        for node in list(ready_nodes):
            if not self._client.is_ready(node):
                log.debug('Node %s not ready; delaying produce of accumulated batch', node)
                self._client.maybe_connect(node, wakeup=False)
                ready_nodes.remove(node)
                not_ready_timeout = min(not_ready_timeout,
                                        self._client.connection_delay(node))

        # create produce requests
        batches_by_node = self._accumulator.drain(
            self._metadata, ready_nodes, self.config['max_request_size'])

        if self.config['guarantee_message_order']:
            # Mute all the partitions drained
            for batch_list in six.itervalues(batches_by_node):
                for batch in batch_list:
                    self._accumulator.muted.add(batch.topic_partition)

        expired_batches = self._accumulator.abort_expired_batches(
            self.config['request_timeout_ms'], self._metadata)
        for expired_batch in expired_batches:
            self._sensors.record_errors(expired_batch.topic_partition.topic, expired_batch.record_count)

        self._sensors.update_produce_request_metrics(batches_by_node)
        requests = self._create_produce_requests(batches_by_node)
        # If we have any nodes that are ready to send + have sendable data,
        # poll with 0 timeout so this can immediately loop and try sending more
        # data. Otherwise, the timeout is determined by nodes that have
        # partitions with data that isn't yet sendable (e.g. lingering, backing
        # off). Note that this specifically does not include nodes with
        # sendable data that aren't ready to send since they would cause busy
        # looping.
        poll_timeout_ms = min(next_ready_check_delay * 1000, not_ready_timeout)
        if ready_nodes:
            log.debug("Nodes with data ready to send: %s", ready_nodes) # trace
            log.debug("Created %d produce requests: %s", len(requests), requests) # trace
            poll_timeout_ms = 0

        for node_id, request in six.iteritems(requests):
            batches = batches_by_node[node_id]
            log.debug('Sending Produce Request: %r', request)
            (self._client.send(node_id, request, wakeup=False)
                 .add_callback(
                     self._handle_produce_response, node_id, time.time(), batches)
                 .add_errback(
                     self._failed_produce, batches, node_id))

        # if some partitions are already ready to be sent, the select time
        # would be 0; otherwise if some partition already has some data
        # accumulated but not ready yet, the select time will be the time
        # difference between now and its linger expiry time; otherwise the
        # select time will be the time difference between now and the
        # metadata expiry time
        self._client.poll(poll_timeout_ms)

Response: Run a single iteration of sending.


Instruction:
Write a Python 3 function that can
start closing the sender.

Response:
def initiate_close(self):
        """Start closing the sender (won't complete until all data is sent)."""
        self._running = False
        self._accumulator.close()
        self.wakeup()


Instruction:
How would you explain what the following Python 3 function does
def _handle_produce_response(self, node_id, send_time, batches, response):
        # if we have a response, parse it
        log.debug('Parsing produce response: %r', response)
        if response:
            batches_by_partition = dict([(batch.topic_partition, batch)
                                         for batch in batches])

            for topic, partitions in response.topics:
                for partition_info in partitions:
                    if response.API_VERSION < 2:
                        partition, error_code, offset = partition_info
                        ts = None
                    else:
                        partition, error_code, offset, ts = partition_info
                    tp = TopicPartition(topic, partition)
                    error = Errors.for_code(error_code)
                    batch = batches_by_partition[tp]
                    self._complete_batch(batch, error, offset, ts)

            if response.API_VERSION > 0:
                self._sensors.record_throttle_time(response.throttle_time_ms, node=node_id)

        else:
            # this is the acks = 0 case, just complete all requests
            for batch in batches:
                self._complete_batch(batch, None, -1, None)

Response: Handle a produce response.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _complete_batch(self, batch, error, base_offset, timestamp_ms=None):
        # Standardize no-error to None
        if error is Errors.NoError:
            error = None

        if error is not None and self._can_retry(batch, error):
            # retry
            log.warning("Got error produce response on topic-partition %s,"
                        " retrying (%d attempts left). Error: %s",
                        batch.topic_partition,
                        self.config['retries'] - batch.attempts - 1,
                        error)
            self._accumulator.reenqueue(batch)
            self._sensors.record_retries(batch.topic_partition.topic, batch.record_count)
        else:
            if error is Errors.TopicAuthorizationFailedError:
                error = error(batch.topic_partition.topic)

            # tell the user the result of their request
            batch.done(base_offset, timestamp_ms, error)
            self._accumulator.deallocate(batch)
            if error is not None:
                self._sensors.record_errors(batch.topic_partition.topic, batch.record_count)

        if getattr(error, 'invalid_metadata', False):
            self._metadata.request_update()

        # Unmute the completed partition.
        if self.config['guarantee_message_order']:
            self._accumulator.muted.remove(batch.topic_partition)

Response: Complete the given batch of records.


Instruction:
Can you tell what is the following Python 3 function doing
def _can_retry(self, batch, error):
        return (batch.attempts < self.config['retries']
                and getattr(error, 'retriable', False))

Response: Return whether we can retry a send if the error is transient or not.


Instruction:
Implement a function in Python 3 to
transfer the record batches into a list of produce requests on a node basis.

Response:
def _create_produce_requests(self, collated):
        """
        Transfer the record batches into a list of produce requests on a
        per-node basis.

        Arguments:
            collated: {node_id: [RecordBatch]}

        Returns:
            dict: {node_id: ProduceRequest} (version depends on api_version)
        """
        requests = {}
        for node_id, batches in six.iteritems(collated):
            requests[node_id] = self._produce_request(
                node_id, self.config['acks'],
                self.config['request_timeout_ms'], batches)
        return requests


Instruction:
Can you write a function in Python 3 where it
creates a ProduceRequest object from the given record batches.

Response:
def _produce_request(self, node_id, acks, timeout, batches):
        """Create a produce request from the given record batches.

        Returns:
            ProduceRequest (version depends on api_version)
        """
        produce_records_by_partition = collections.defaultdict(dict)
        for batch in batches:
            topic = batch.topic_partition.topic
            partition = batch.topic_partition.partition

            buf = batch.records.buffer()
            produce_records_by_partition[topic][partition] = buf

        kwargs = {}
        if self.config['api_version'] >= (0, 11):
            version = 3
            kwargs = dict(transactional_id=None)
        elif self.config['api_version'] >= (0, 10):
            version = 2
        elif self.config['api_version'] == (0, 9):
            version = 1
        else:
            version = 0
        return ProduceRequest[version](
            required_acks=acks,
            timeout=timeout,
            topics=[(topic, list(partition_info.items()))
                    for topic, partition_info
                    in six.iteritems(produce_records_by_partition)],
            **kwargs
        )


Instruction:
Create a Python 3 function for
encoding the given data with snappy compression.

Response:
def snappy_encode(payload, xerial_compatible=True, xerial_blocksize=32*1024):
    """Encodes the given data with snappy compression.

    If xerial_compatible is set then the stream is encoded in a fashion
    compatible with the xerial snappy library.

    The block size (xerial_blocksize) controls how frequent the blocking occurs
    32k is the default in the xerial library.

    The format winds up being:

        +-------------+------------+--------------+------------+--------------+
        |   Header    | Block1 len | Block1 data  | Blockn len | Blockn data  |
        +-------------+------------+--------------+------------+--------------+
        |  16 bytes   |  BE int32  | snappy bytes |  BE int32  | snappy bytes |
        +-------------+------------+--------------+------------+--------------+

    It is important to note that the blocksize is the amount of uncompressed
    data presented to snappy at each block, whereas the blocklen is the number
    of bytes that will be present in the stream; so the length will always be
    <= blocksize.

    """

    if not has_snappy():
        raise NotImplementedError("Snappy codec is not available")

    if not xerial_compatible:
        return snappy.compress(payload)

    out = io.BytesIO()
    for fmt, dat in zip(_XERIAL_V1_FORMAT, _XERIAL_V1_HEADER):
        out.write(struct.pack('!' + fmt, dat))

    # Chunk through buffers to avoid creating intermediate slice copies
    if PYPY:
        # on pypy, snappy.compress() on a sliced buffer consumes the entire
        # buffer... likely a python-snappy bug, so just use a slice copy
        chunker = lambda payload, i, size: payload[i:size+i]

    elif six.PY2:
        # Sliced buffer avoids additional copies
        # pylint: disable-msg=undefined-variable
        chunker = lambda payload, i, size: buffer(payload, i, size)
    else:
        # snappy.compress does not like raw memoryviews, so we have to convert
        # tobytes, which is a copy... oh well. it's the thought that counts.
        # pylint: disable-msg=undefined-variable
        chunker = lambda payload, i, size: memoryview(payload)[i:size+i].tobytes()

    for chunk in (chunker(payload, i, xerial_blocksize)
                  for i in range(0, len(payload), xerial_blocksize)):

        block = snappy.compress(chunk)
        block_size = len(block)
        out.write(struct.pack('!i', block_size))
        out.write(block)

    return out.getvalue()


Instruction:
Can you implement a function in Python 3 that
decodes payload using interoperable LZ4 framing. Requires Kafka >= 0. 10

Response:
def lz4f_decode(payload):
    """Decode payload using interoperable LZ4 framing. Requires Kafka >= 0.10"""
    # pylint: disable-msg=no-member
    ctx = lz4f.createDecompContext()
    data = lz4f.decompressFrame(payload, ctx)
    lz4f.freeDecompContext(ctx)

    # lz4f python module does not expose how much of the payload was
    # actually read if the decompression was only partial.
    if data['next'] != 0:
        raise RuntimeError('lz4f unable to decompress full payload')
    return data['decomp']


Instruction:
Can you create a Python 3 function that
encodes payload for 0. 8 / 0. 9 brokers -- requires an incorrect header checksum.

Response:
def lz4_encode_old_kafka(payload):
    """Encode payload for 0.8/0.9 brokers -- requires an incorrect header checksum."""
    assert xxhash is not None
    data = lz4_encode(payload)
    header_size = 7
    flg = data[4]
    if not isinstance(flg, int):
        flg = ord(flg)

    content_size_bit = ((flg >> 3) & 1)
    if content_size_bit:
        # Old kafka does not accept the content-size field
        # so we need to discard it and reset the header flag
        flg -= 8
        data = bytearray(data)
        data[4] = flg
        data = bytes(data)
        payload = data[header_size+8:]
    else:
        payload = data[header_size:]

    # This is the incorrect hc
    hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1]  # pylint: disable-msg=no-member

    return b''.join([
        data[0:header_size-1],
        hc,
        payload
    ])


Instruction:
Write a Python 3 function that can
get all BrokerMetadata objects in this cluster

Response:
def brokers(self):
        """Get all BrokerMetadata

        Returns:
            set: {BrokerMetadata, ...}
        """
        return set(self._brokers.values()) or set(self._bootstrap_brokers.values())


Instruction:
Can you tell what is the following Python 3 function doing
def broker_metadata(self, broker_id):
        return self._brokers.get(broker_id) or self._bootstrap_brokers.get(broker_id)

Response: Get the broker metadata for a specific broker


Instruction:
Given the following Python 3 function, write the documentation
def partitions_for_topic(self, topic):
        if topic not in self._partitions:
            return None
        return set(self._partitions[topic].keys())

Response: Returns set of all partitions for a topic


Instruction:
How would you explain what the following Python 3 function does
def available_partitions_for_topic(self, topic):
        if topic not in self._partitions:
            return None
        return set([partition for partition, metadata
                              in six.iteritems(self._partitions[topic])
                              if metadata.leader != -1])

Response: Returns set of partitions with known leaders for topic.


Instruction:
Write a Python 3 script to
return node_id of leader for given partition.

Response:
def leader_for_partition(self, partition):
        """Return node_id of leader, -1 unavailable, None if unknown."""
        if partition.topic not in self._partitions:
            return None
        elif partition.partition not in self._partitions[partition.topic]:
            return None
        return self._partitions[partition.topic][partition.partition].leader


Instruction:
Explain what the following Python 3 code does
def ttl(self):
        now = time.time() * 1000
        if self._need_update:
            ttl = 0
        else:
            metadata_age = now - self._last_successful_refresh_ms
            ttl = self.config['metadata_max_age_ms'] - metadata_age

        retry_age = now - self._last_refresh_ms
        next_retry = self.config['retry_backoff_ms'] - retry_age

        return max(ttl, next_retry, 0)

Response: Milliseconds until metadata should be refreshed


Instruction:
Implement a function in Python 3 to
request metadata for update

Response:
def request_update(self):
        """Flags metadata for update, return Future()

        Actual update must be handled separately. This method will only
        change the reported ttl()

        Returns:
            kafka.future.Future (value will be the cluster object after update)
        """
        with self._lock:
            self._need_update = True
            if not self._future or self._future.is_done:
              self._future = Future()
            return self._future


Instruction:
How would you implement a function in Python 3 that
gets set of known topics.

Response:
def topics(self, exclude_internal_topics=True):
        """Get set of known topics.

        Arguments:
            exclude_internal_topics (bool): Whether records from internal topics
                (such as offsets) should be exposed to the consumer. If set to
                True the only way to receive records from an internal topic is
                subscribing to it. Default True

        Returns:
            set: {topic (str), ...}
        """
        topics = set(self._partitions.keys())
        if exclude_internal_topics:
            return topics - self.internal_topics
        else:
            return topics


Instruction:
Write a Python 3 script for
updating the state of the cluster with a failed MetadataRequest.

Response:
def failed_update(self, exception):
        """Update cluster state given a failed MetadataRequest."""
        f = None
        with self._lock:
            if self._future:
                f = self._future
                self._future = None
        if f:
            f.failure(exception)
        self._last_refresh_ms = time.time() * 1000


Instruction:
Write a Python 3 function for
updating the state of the cluster given a MetadataResponse.

Response:
def update_metadata(self, metadata):
        """Update cluster state given a MetadataResponse.

        Arguments:
            metadata (MetadataResponse): broker response to a metadata request

        Returns: None
        """
        # In the common case where we ask for a single topic and get back an
        # error, we should fail the future
        if len(metadata.topics) == 1 and metadata.topics[0][0] != 0:
            error_code, topic = metadata.topics[0][:2]
            error = Errors.for_code(error_code)(topic)
            return self.failed_update(error)

        if not metadata.brokers:
            log.warning("No broker metadata found in MetadataResponse -- ignoring.")
            return self.failed_update(Errors.MetadataEmptyBrokerList(metadata))

        _new_brokers = {}
        for broker in metadata.brokers:
            if metadata.API_VERSION == 0:
                node_id, host, port = broker
                rack = None
            else:
                node_id, host, port, rack = broker
            _new_brokers.update({
                node_id: BrokerMetadata(node_id, host, port, rack)
            })

        if metadata.API_VERSION == 0:
            _new_controller = None
        else:
            _new_controller = _new_brokers.get(metadata.controller_id)

        _new_partitions = {}
        _new_broker_partitions = collections.defaultdict(set)
        _new_unauthorized_topics = set()
        _new_internal_topics = set()

        for topic_data in metadata.topics:
            if metadata.API_VERSION == 0:
                error_code, topic, partitions = topic_data
                is_internal = False
            else:
                error_code, topic, is_internal, partitions = topic_data
            if is_internal:
                _new_internal_topics.add(topic)
            error_type = Errors.for_code(error_code)
            if error_type is Errors.NoError:
                _new_partitions[topic] = {}
                for p_error, partition, leader, replicas, isr in partitions:
                    _new_partitions[topic][partition] = PartitionMetadata(
                        topic=topic, partition=partition, leader=leader,
                        replicas=replicas, isr=isr, error=p_error)
                    if leader != -1:
                        _new_broker_partitions[leader].add(
                            TopicPartition(topic, partition))

            elif error_type is Errors.LeaderNotAvailableError:
                log.warning("Topic %s is not available during auto-create"
                            " initialization", topic)
            elif error_type is Errors.UnknownTopicOrPartitionError:
                log.error("Topic %s not found in cluster metadata", topic)
            elif error_type is Errors.TopicAuthorizationFailedError:
                log.error("Topic %s is not authorized for this client", topic)
                _new_unauthorized_topics.add(topic)
            elif error_type is Errors.InvalidTopicError:
                log.error("'%s' is not a valid topic name", topic)
            else:
                log.error("Error fetching metadata for topic %s: %s",
                          topic, error_type)

        with self._lock:
            self._brokers = _new_brokers
            self.controller = _new_controller
            self._partitions = _new_partitions
            self._broker_partitions = _new_broker_partitions
            self.unauthorized_topics = _new_unauthorized_topics
            self.internal_topics = _new_internal_topics
            f = None
            if self._future:
                f = self._future
            self._future = None
            self._need_update = False

        now = time.time() * 1000
        self._last_refresh_ms = now
        self._last_successful_refresh_ms = now

        if f:
            f.success(self)
        log.debug("Updated cluster metadata to %s", self)

        for listener in self._listeners:
            listener(self)

        if self.need_all_topic_metadata:
            # the listener may change the interested topics,
            # which could cause another metadata refresh.
            # If we have already fetched all topics, however,
            # another fetch should be unnecessary.
            self._need_update = False


Instruction:
Make a summary of the following Python 3 code
def add_group_coordinator(self, group, response):
        log.debug("Updating coordinator for %s: %s", group, response)
        error_type = Errors.for_code(response.error_code)
        if error_type is not Errors.NoError:
            log.error("GroupCoordinatorResponse error: %s", error_type)
            self._groups[group] = -1
            return False

        node_id = response.coordinator_id
        coordinator = BrokerMetadata(
            response.coordinator_id,
            response.host,
            response.port,
            None)

        # Assume that group coordinators are just brokers
        # (this is true now, but could diverge in future)
        if node_id not in self._brokers:
            self._brokers[node_id] = coordinator

        # If this happens, either brokers have moved without
        # changing IDs, or our assumption above is wrong
        else:
            node = self._brokers[node_id]
            if coordinator.host != node.host or coordinator.port != node.port:
                log.error("GroupCoordinator metadata conflicts with existing"
                          " broker metadata. Coordinator: %s, Broker: %s",
                          coordinator, node)
                self._groups[group] = node_id
                return False

        log.info("Group coordinator for %s is %s", group, coordinator)
        self._groups[group] = node_id
        return True

Response: Update with metadata for a group coordinator


Instruction:
Write a Python 3 function that can
return a copy of this cluster metadata with partitions added

Response:
def with_partitions(self, partitions_to_add):
        """Returns a copy of cluster metadata with partitions added"""
        new_metadata = ClusterMetadata(**self.config)
        new_metadata._brokers = copy.deepcopy(self._brokers)
        new_metadata._partitions = copy.deepcopy(self._partitions)
        new_metadata._broker_partitions = copy.deepcopy(self._broker_partitions)
        new_metadata._groups = copy.deepcopy(self._groups)
        new_metadata.internal_topics = copy.deepcopy(self.internal_topics)
        new_metadata.unauthorized_topics = copy.deepcopy(self.unauthorized_topics)

        for partition in partitions_to_add:
            new_metadata._partitions[partition.topic][partition.partition] = partition

            if partition.leader is not None and partition.leader != -1:
                new_metadata._broker_partitions[partition.leader].add(
                    TopicPartition(partition.topic, partition.partition))

        return new_metadata


Instruction:
Can you tell what is the following Python 3 function doing
def assign(self, partitions):
        self._subscription.assign_from_user(partitions)
        self._client.set_topics([tp.topic for tp in partitions])

Response: Manually assign a list of TopicPartitions to this consumer.


Instruction:
Create a Python 3 function for
closing the consumer and all associated resources.

Response:
def close(self, autocommit=True):
        """Close the consumer, waiting indefinitely for any needed cleanup.

        Keyword Arguments:
            autocommit (bool): If auto-commit is configured for this consumer,
                this optional flag causes the consumer to attempt to commit any
                pending consumed offsets prior to close. Default: True
        """
        if self._closed:
            return
        log.debug("Closing the KafkaConsumer.")
        self._closed = True
        self._coordinator.close(autocommit=autocommit)
        self._metrics.close()
        self._client.close()
        try:
            self.config['key_deserializer'].close()
        except AttributeError:
            pass
        try:
            self.config['value_deserializer'].close()
        except AttributeError:
            pass
        log.debug("The KafkaConsumer has closed.")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def commit_async(self, offsets=None, callback=None):
        assert self.config['api_version'] >= (0, 8, 1), 'Requires >= Kafka 0.8.1'
        assert self.config['group_id'] is not None, 'Requires group_id'
        if offsets is None:
            offsets = self._subscription.all_consumed_offsets()
        log.debug("Committing offsets: %s", offsets)
        future = self._coordinator.commit_offsets_async(
            offsets, callback=callback)
        return future

Response: Commits the current consumed offsets to Kafka asynchronously.


Instruction:
How would you explain what the following Python 3 function does
def commit(self, offsets=None):
        assert self.config['api_version'] >= (0, 8, 1), 'Requires >= Kafka 0.8.1'
        assert self.config['group_id'] is not None, 'Requires group_id'
        if offsets is None:
            offsets = self._subscription.all_consumed_offsets()
        self._coordinator.commit_offsets_sync(offsets)

Response: Commits the current consumed offsets to Kafka.


Instruction:
How would you explain what the following Python 3 function does
def committed(self, partition):
        assert self.config['api_version'] >= (0, 8, 1), 'Requires >= Kafka 0.8.1'
        assert self.config['group_id'] is not None, 'Requires group_id'
        if not isinstance(partition, TopicPartition):
            raise TypeError('partition must be a TopicPartition namedtuple')
        if self._subscription.is_assigned(partition):
            committed = self._subscription.assignment[partition].committed
            if committed is None:
                self._coordinator.refresh_committed_offsets_if_needed()
                committed = self._subscription.assignment[partition].committed
        else:
            commit_map = self._coordinator.fetch_committed_offsets([partition])
            if partition in commit_map:
                committed = commit_map[partition].offset
            else:
                committed = None
        return committed

Response: Get the last committed offset for the given partition.


Instruction:
Write a Python 3 script to
get all topics the user is authorized to view.

Response:
def topics(self):
        """Get all topics the user is authorized to view.

        Returns:
            set: topics
        """
        cluster = self._client.cluster
        if self._client._metadata_refresh_in_progress and self._client._topics:
            future = cluster.request_update()
            self._client.poll(future=future)
        stash = cluster.need_all_topic_metadata
        cluster.need_all_topic_metadata = True
        future = cluster.request_update()
        self._client.poll(future=future)
        cluster.need_all_topic_metadata = stash
        return cluster.topics()


Instruction:
Given the following Python 3 function, write the documentation
def poll(self, timeout_ms=0, max_records=None):
        assert timeout_ms >= 0, 'Timeout must not be negative'
        if max_records is None:
            max_records = self.config['max_poll_records']
        assert isinstance(max_records, int), 'max_records must be an integer'
        assert max_records > 0, 'max_records must be positive'

        # Poll for new data until the timeout expires
        start = time.time()
        remaining = timeout_ms
        while True:
            records = self._poll_once(remaining, max_records)
            if records:
                return records

            elapsed_ms = (time.time() - start) * 1000
            remaining = timeout_ms - elapsed_ms

            if remaining <= 0:
                return {}

Response: Poll for new data from assigned topics and partitions.


Instruction:
Write a Python 3 function that can
do one round of polling.

Response:
def _poll_once(self, timeout_ms, max_records):
        """Do one round of polling. In addition to checking for new data, this does
        any needed heart-beating, auto-commits, and offset updates.

        Arguments:
            timeout_ms (int): The maximum time in milliseconds to block.

        Returns:
            dict: Map of topic to list of records (may be empty).
        """
        self._coordinator.poll()

        # Fetch positions if we have partitions we're subscribed to that we
        # don't know the offset for
        if not self._subscription.has_all_fetch_positions():
            self._update_fetch_positions(self._subscription.missing_fetch_positions())

        # If data is available already, e.g. from a previous network client
        # poll() call to commit, then just return it immediately
        records, partial = self._fetcher.fetched_records(max_records)
        if records:
            # Before returning the fetched records, we can send off the
            # next round of fetches and avoid block waiting for their
            # responses to enable pipelining while the user is handling the
            # fetched records.
            if not partial:
                self._fetcher.send_fetches()
            return records

        # Send any new fetches (won't resend pending fetches)
        self._fetcher.send_fetches()

        timeout_ms = min(timeout_ms, self._coordinator.time_to_next_poll() * 1000)
        self._client.poll(timeout_ms=timeout_ms)
        # after the long poll, we should check whether the group needs to rebalance
        # prior to returning data so that the group can stabilize faster
        if self._coordinator.need_rejoin():
            return {}

        records, _ = self._fetcher.fetched_records(max_records)
        return records


Instruction:
Create a Python 3 function for
getting the offset of the next record that will be fetched by the given partition

Response:
def position(self, partition):
        """Get the offset of the next record that will be fetched

        Arguments:
            partition (TopicPartition): Partition to check

        Returns:
            int: Offset
        """
        if not isinstance(partition, TopicPartition):
            raise TypeError('partition must be a TopicPartition namedtuple')
        assert self._subscription.is_assigned(partition), 'Partition is not assigned'
        offset = self._subscription.assignment[partition].position
        if offset is None:
            self._update_fetch_positions([partition])
            offset = self._subscription.assignment[partition].position
        return offset


Instruction:
Explain what the following Python 3 code does
def highwater(self, partition):
        if not isinstance(partition, TopicPartition):
            raise TypeError('partition must be a TopicPartition namedtuple')
        assert self._subscription.is_assigned(partition), 'Partition is not assigned'
        return self._subscription.assignment[partition].highwater

Response: Returns the last known highwater offset for a topic partition.


Instruction:
How would you explain what the following Python 3 function does
def pause(self, *partitions):
        if not all([isinstance(p, TopicPartition) for p in partitions]):
            raise TypeError('partitions must be TopicPartition namedtuples')
        for partition in partitions:
            log.debug("Pausing partition %s", partition)
            self._subscription.pause(partition)

Response: Suspend fetching records from the requested partitions.


Instruction:
Implement a function in Python 3 to
seek to a specific offset for a topic partition.

Response:
def seek(self, partition, offset):
        """Manually specify the fetch offset for a TopicPartition.

        Overrides the fetch offsets that the consumer will use on the next
        :meth:`~kafka.KafkaConsumer.poll`. If this API is invoked for the same
        partition more than once, the latest offset will be used on the next
        :meth:`~kafka.KafkaConsumer.poll`.

        Note: You may lose data if this API is arbitrarily used in the middle of
        consumption to reset the fetch offsets.

        Arguments:
            partition (TopicPartition): Partition for seek operation
            offset (int): Message offset in partition

        Raises:
            AssertionError: If offset is not an int >= 0; or if partition is not
                currently assigned.
        """
        if not isinstance(partition, TopicPartition):
            raise TypeError('partition must be a TopicPartition namedtuple')
        assert isinstance(offset, int) and offset >= 0, 'Offset must be >= 0'
        assert partition in self._subscription.assigned_partitions(), 'Unassigned partition'
        log.debug("Seeking to offset %s for partition %s", offset, partition)
        self._subscription.assignment[partition].seek(offset)


Instruction:
Write a Python 3 script for
seeking to the oldest available offset for partitions.

Response:
def seek_to_beginning(self, *partitions):
        """Seek to the oldest available offset for partitions.

        Arguments:
            *partitions: Optionally provide specific TopicPartitions, otherwise
                default to all assigned partitions.

        Raises:
            AssertionError: If any partition is not currently assigned, or if
                no partitions are assigned.
        """
        if not all([isinstance(p, TopicPartition) for p in partitions]):
            raise TypeError('partitions must be TopicPartition namedtuples')
        if not partitions:
            partitions = self._subscription.assigned_partitions()
            assert partitions, 'No partitions are currently assigned'
        else:
            for p in partitions:
                assert p in self._subscription.assigned_partitions(), 'Unassigned partition'

        for tp in partitions:
            log.debug("Seeking to beginning of partition %s", tp)
            self._subscription.need_offset_reset(tp, OffsetResetStrategy.EARLIEST)


Instruction:
How would you implement a function in Python 3 that
seeks to the most recent available offset for partitions.

Response:
def seek_to_end(self, *partitions):
        """Seek to the most recent available offset for partitions.

        Arguments:
            *partitions: Optionally provide specific TopicPartitions, otherwise
                default to all assigned partitions.

        Raises:
            AssertionError: If any partition is not currently assigned, or if
                no partitions are assigned.
        """
        if not all([isinstance(p, TopicPartition) for p in partitions]):
            raise TypeError('partitions must be TopicPartition namedtuples')
        if not partitions:
            partitions = self._subscription.assigned_partitions()
            assert partitions, 'No partitions are currently assigned'
        else:
            for p in partitions:
                assert p in self._subscription.assigned_partitions(), 'Unassigned partition'

        for tp in partitions:
            log.debug("Seeking to end of partition %s", tp)
            self._subscription.need_offset_reset(tp, OffsetResetStrategy.LATEST)


Instruction:
How would you code a function in Python 3 to
subscribe to a list of topics or a regex pattern.

Response:
def subscribe(self, topics=(), pattern=None, listener=None):
        """Subscribe to a list of topics, or a topic regex pattern.

        Partitions will be dynamically assigned via a group coordinator.
        Topic subscriptions are not incremental: this list will replace the
        current assignment (if there is one).

        This method is incompatible with :meth:`~kafka.KafkaConsumer.assign`.

        Arguments:
            topics (list): List of topics for subscription.
            pattern (str): Pattern to match available topics. You must provide
                either topics or pattern, but not both.
            listener (ConsumerRebalanceListener): Optionally include listener
                callback, which will be called before and after each rebalance
                operation.

                As part of group management, the consumer will keep track of the
                list of consumers that belong to a particular group and will
                trigger a rebalance operation if one of the following events
                trigger:

                * Number of partitions change for any of the subscribed topics
                * Topic is created or deleted
                * An existing member of the consumer group dies
                * A new member is added to the consumer group

                When any of these events are triggered, the provided listener
                will be invoked first to indicate that the consumer's assignment
                has been revoked, and then again when the new assignment has
                been received. Note that this listener will immediately override
                any listener set in a previous call to subscribe. It is
                guaranteed, however, that the partitions revoked/assigned
                through this interface are from topics subscribed in this call.

        Raises:
            IllegalStateError: If called after previously calling
                :meth:`~kafka.KafkaConsumer.assign`.
            AssertionError: If neither topics or pattern is provided.
            TypeError: If listener is not a ConsumerRebalanceListener.
        """
        # SubscriptionState handles error checking
        self._subscription.subscribe(topics=topics,
                                     pattern=pattern,
                                     listener=listener)

        # Regex will need all topic metadata
        if pattern is not None:
            self._client.cluster.need_all_topic_metadata = True
            self._client.set_topics([])
            self._client.cluster.request_update()
            log.debug("Subscribed to topic pattern: %s", pattern)
        else:
            self._client.cluster.need_all_topic_metadata = False
            self._client.set_topics(self._subscription.group_subscription())
            log.debug("Subscribed to topic(s): %s", topics)


Instruction:
Here you have a function in Python 3, explain what it does
def unsubscribe(self):
        self._subscription.unsubscribe()
        self._coordinator.close()
        self._client.cluster.need_all_topic_metadata = False
        self._client.set_topics([])
        log.debug("Unsubscribed all topics or patterns and assigned partitions")

Response: Unsubscribe from all topics and clear all assigned partitions.


Instruction:
How would you explain what the following Python 3 function does
def offsets_for_times(self, timestamps):
        if self.config['api_version'] <= (0, 10, 0):
            raise UnsupportedVersionError(
                "offsets_for_times API not supported for cluster version {}"
                .format(self.config['api_version']))
        for tp, ts in six.iteritems(timestamps):
            timestamps[tp] = int(ts)
            if ts < 0:
                raise ValueError(
                    "The target time for partition {} is {}. The target time "
                    "cannot be negative.".format(tp, ts))
        return self._fetcher.get_offsets_by_times(
            timestamps, self.config['request_timeout_ms'])

Response: Look up the offsets for the given timestamps.


Instruction:
Given the following Python 3 function, write the documentation
def beginning_offsets(self, partitions):
        offsets = self._fetcher.beginning_offsets(
            partitions, self.config['request_timeout_ms'])
        return offsets

Response: Get the earliest available offsets for the given partitions.


Instruction:
Can you implement a function in Python 3 that
gets the last offset for the given partitions.

Response:
def end_offsets(self, partitions):
        """Get the last offset for the given partitions. The last offset of a
        partition is the offset of the upcoming message, i.e. the offset of the
        last available message + 1.

        This method does not change the current consumer position of the
        partitions.

        Note:
            This method may block indefinitely if the partition does not exist.

        Arguments:
            partitions (list): List of TopicPartition instances to fetch
                offsets for.

        Returns:
            ``{TopicPartition: int}``: The end offsets for the given partitions.

        Raises:
            UnsupportedVersionError: If the broker does not support looking
                up the offsets by timestamp.
            KafkaTimeoutError: If fetch failed in request_timeout_ms
        """
        offsets = self._fetcher.end_offsets(
            partitions, self.config['request_timeout_ms'])
        return offsets


Instruction:
Write a Python 3 function that can
return True iff this consumer can join a broker - coordinated group.

Response:
def _use_consumer_group(self):
        """Return True iff this consumer can/should join a broker-coordinated group."""
        if self.config['api_version'] < (0, 9):
            return False
        elif self.config['group_id'] is None:
            return False
        elif not self._subscription.partitions_auto_assigned():
            return False
        return True


Instruction:
Write a Python 3 script for
updating the fetch positions for the given partitions.

Response:
def _update_fetch_positions(self, partitions):
        """Set the fetch position to the committed position (if there is one)
        or reset it using the offset reset policy the user has configured.

        Arguments:
            partitions (List[TopicPartition]): The partitions that need
                updating fetch positions.

        Raises:
            NoOffsetForPartitionError: If no offset is stored for a given
                partition and no offset reset policy is defined.
        """
        # Lookup any positions for partitions which are awaiting reset (which may be the
        # case if the user called :meth:`seek_to_beginning` or :meth:`seek_to_end`. We do
        # this check first to avoid an unnecessary lookup of committed offsets (which
        # typically occurs when the user is manually assigning partitions and managing
        # their own offsets).
        self._fetcher.reset_offsets_if_needed(partitions)

        if not self._subscription.has_all_fetch_positions():
            # if we still don't have offsets for all partitions, then we should either seek
            # to the last committed position or reset using the auto reset policy
            if (self.config['api_version'] >= (0, 8, 1) and
                self.config['group_id'] is not None):
                # first refresh commits for all assigned partitions
                self._coordinator.refresh_committed_offsets_if_needed()

            # Then, do any offset lookups in case some positions are not known
            self._fetcher.update_fetch_positions(partitions)


Instruction:
Can you write a function in Python 3 where it
returns a nested dictionary snapshot of all metrics and their values at this time.

Response:
def snapshot(self):
        """
        Return a nested dictionary snapshot of all metrics and their
        values at this time. Example:
        {
            'category': {
                'metric1_name': 42.0,
                'metric2_name': 'foo'
            }
        }
        """
        return dict((category, dict((name, metric.value())
                                    for name, metric in list(metrics.items())))
                    for category, metrics in
                    list(self._store.items()))


Instruction:
Here you have a function in Python 3, explain what it does
def get_category(self, metric):
        tags = ','.join('%s=%s' % (k, v) for k, v in
                        sorted(metric.metric_name.tags.items()))
        return '.'.join(x for x in
                        [self._prefix, metric.metric_name.group, tags] if x)

Response: Returns a string category for the metric.


Instruction:
How would you implement a function in Python 3 that
checks if a node is ready to be connected to the broker.

Response:
def maybe_connect(self, node_id, wakeup=True):
        """Queues a node for asynchronous connection during the next .poll()"""
        if self._can_connect(node_id):
            self._connecting.add(node_id)
            # Wakeup signal is useful in case another thread is
            # blocked waiting for incoming network traffic while holding
            # the client lock in poll().
            if wakeup:
                self.wakeup()
            return True
        return False


Instruction:
Can you generate the documentation for the following Python 3 function
def _maybe_connect(self, node_id):
        with self._lock:
            conn = self._conns.get(node_id)

            if conn is None:
                broker = self.cluster.broker_metadata(node_id)
                assert broker, 'Broker id %s not in current metadata' % (node_id,)

                log.debug("Initiating connection to node %s at %s:%s",
                          node_id, broker.host, broker.port)
                host, port, afi = get_ip_port_afi(broker.host)
                cb = WeakMethod(self._conn_state_change)
                conn = BrokerConnection(host, broker.port, afi,
                                        state_change_callback=cb,
                                        node_id=node_id,
                                        **self.config)
                self._conns[node_id] = conn

            # Check if existing connection should be recreated because host/port changed
            elif self._should_recycle_connection(conn):
                self._conns.pop(node_id)
                return False

            elif conn.connected():
                return True

            conn.connect()
            return conn.connected()

Response: Idempotent non - blocking connection attempt to the given node id.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ready(self, node_id, metadata_priority=True):
        self.maybe_connect(node_id)
        return self.is_ready(node_id, metadata_priority=metadata_priority)

Response: Check whether a node is ready to send more requests.


Instruction:
Write a Python 3 function for
returning True iff the node_id is connected.

Response:
def connected(self, node_id):
        """Return True iff the node_id is connected."""
        conn = self._conns.get(node_id)
        if conn is None:
            return False
        return conn.connected()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def close(self, node_id=None):
        with self._lock:
            if node_id is None:
                self._close()
                conns = list(self._conns.values())
                self._conns.clear()
                for conn in conns:
                    conn.close()
            elif node_id in self._conns:
                self._conns.pop(node_id).close()
            else:
                log.warning("Node %s not found in current connection list; skipping", node_id)
                return

Response: Closes one or all broker connections.


Instruction:
Explain what the following Python 3 code does
def is_disconnected(self, node_id):
        conn = self._conns.get(node_id)
        if conn is None:
            return False
        return conn.disconnected()

Response: Checks whether the node connection has been disconnected or failed.


Instruction:
Implement a function in Python 3 to
return the number of milliseconds to wait for the connection to finish.

Response:
def connection_delay(self, node_id):
        """
        Return the number of milliseconds to wait, based on the connection
        state, before attempting to send data. When disconnected, this respects
        the reconnect backoff time. When connecting, returns 0 to allow
        non-blocking connect to finish. When connected, returns a very large
        number to handle slow/stalled connections.

        Arguments:
            node_id (int): The id of the node to check

        Returns:
            int: The number of milliseconds to wait.
        """
        conn = self._conns.get(node_id)
        if conn is None:
            return 0
        return conn.connection_delay()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_ready(self, node_id, metadata_priority=True):
        if not self._can_send_request(node_id):
            return False

        # if we need to update our metadata now declare all requests unready to
        # make metadata requests first priority
        if metadata_priority:
            if self._metadata_refresh_in_progress:
                return False
            if self.cluster.ttl() == 0:
                return False
        return True

Response: Check whether a node is ready to send more requests.


Instruction:
Write a Python 3 script to
send a request to a specific node.

Response:
def send(self, node_id, request, wakeup=True):
        """Send a request to a specific node. Bytes are placed on an
        internal per-connection send-queue. Actual network I/O will be
        triggered in a subsequent call to .poll()

        Arguments:
            node_id (int): destination node
            request (Struct): request object (not-encoded)
            wakeup (bool): optional flag to disable thread-wakeup

        Raises:
            AssertionError: if node_id is not in current cluster metadata

        Returns:
            Future: resolves to Response struct or Error
        """
        conn = self._conns.get(node_id)
        if not conn or not self._can_send_request(node_id):
            self.maybe_connect(node_id, wakeup=wakeup)
            return Future().failure(Errors.NodeNotReadyError(node_id))

        # conn.send will queue the request internally
        # we will need to call send_pending_requests()
        # to trigger network I/O
        future = conn.send(request, blocking=False)

        # Wakeup signal is useful in case another thread is
        # blocked waiting for incoming network traffic while holding
        # the client lock in poll().
        if wakeup:
            self.wakeup()

        return future


Instruction:
Can you generate the documentation for the following Python 3 function
def poll(self, timeout_ms=None, future=None):
        if future is not None:
            timeout_ms = 100
        elif timeout_ms is None:
            timeout_ms = self.config['request_timeout_ms']
        elif not isinstance(timeout_ms, (int, float)):
            raise TypeError('Invalid type for timeout: %s' % type(timeout_ms))

        # Loop for futures, break after first loop if None
        responses = []
        while True:
            with self._lock:
                if self._closed:
                    break

                # Attempt to complete pending connections
                for node_id in list(self._connecting):
                    self._maybe_connect(node_id)

                # Send a metadata request if needed
                metadata_timeout_ms = self._maybe_refresh_metadata()

                # If we got a future that is already done, don't block in _poll
                if future is not None and future.is_done:
                    timeout = 0
                else:
                    idle_connection_timeout_ms = self._idle_expiry_manager.next_check_ms()
                    timeout = min(
                        timeout_ms,
                        metadata_timeout_ms,
                        idle_connection_timeout_ms,
                        self.config['request_timeout_ms'])
                    timeout = max(0, timeout / 1000)  # avoid negative timeouts

                self._poll(timeout)

                responses.extend(self._fire_pending_completed_requests())

            # If all we had was a timeout (future is None) - only do one poll
            # If we do have a future, we keep looping until it is done
            if future is None or future.is_done:
                break

        return responses

Response: Try to read and write to sockets.


Instruction:
Here you have a function in Python 3, explain what it does
def in_flight_request_count(self, node_id=None):
        if node_id is not None:
            conn = self._conns.get(node_id)
            if conn is None:
                return 0
            return len(conn.in_flight_requests)
        else:
            return sum([len(conn.in_flight_requests)
                        for conn in list(self._conns.values())])

Response: Get the number of in - flight requests for a specific node or all nodes.


Instruction:
Write a Python 3 script to
choose the least loaded node with few outstanding requests with fallbacks.

Response:
def least_loaded_node(self):
        """Choose the node with fewest outstanding requests, with fallbacks.

        This method will prefer a node with an existing connection and no
        in-flight-requests. If no such node is found, a node will be chosen
        randomly from disconnected nodes that are not "blacked out" (i.e.,
        are not subject to a reconnect backoff). If no node metadata has been
        obtained, will return a bootstrap node (subject to exponential backoff).

        Returns:
            node_id or None if no suitable node was found
        """
        nodes = [broker.nodeId for broker in self.cluster.brokers()]
        random.shuffle(nodes)

        inflight = float('inf')
        found = None
        for node_id in nodes:
            conn = self._conns.get(node_id)
            connected = conn is not None and conn.connected()
            blacked_out = conn is not None and conn.blacked_out()
            curr_inflight = len(conn.in_flight_requests) if conn is not None else 0
            if connected and curr_inflight == 0:
                # if we find an established connection
                # with no in-flight requests, we can stop right away
                return node_id
            elif not blacked_out and curr_inflight < inflight:
                # otherwise if this is the best we have found so far, record that
                inflight = curr_inflight
                found = node_id

        if found is not None:
            return found

        return None


Instruction:
Can you implement a function in Python 3 that
sets specific topics to track for metadata.

Response:
def set_topics(self, topics):
        """Set specific topics to track for metadata.

        Arguments:
            topics (list of str): topics to check for metadata

        Returns:
            Future: resolves after metadata request/response
        """
        if set(topics).difference(self._topics):
            future = self.cluster.request_update()
        else:
            future = Future().success(set(topics))
        self._topics = set(topics)
        return future


Instruction:
Write a Python 3 script to
add a topic to the list of topics tracked via metadata.

Response:
def add_topic(self, topic):
        """Add a topic to the list of topics tracked via metadata.

        Arguments:
            topic (str): topic to track

        Returns:
            Future: resolves after metadata request/response
        """
        if topic in self._topics:
            return Future().success(set(self._topics))

        self._topics.add(topic)
        return self.cluster.request_update()


Instruction:
How would you implement a function in Python 3 that
sends a metadata request if needed.

Response:
def _maybe_refresh_metadata(self, wakeup=False):
        """Send a metadata request if needed.

        Returns:
            int: milliseconds until next refresh
        """
        ttl = self.cluster.ttl()
        wait_for_in_progress_ms = self.config['request_timeout_ms'] if self._metadata_refresh_in_progress else 0
        metadata_timeout = max(ttl, wait_for_in_progress_ms)

        if metadata_timeout > 0:
            return metadata_timeout

        # Beware that the behavior of this method and the computation of
        # timeouts for poll() are highly dependent on the behavior of
        # least_loaded_node()
        node_id = self.least_loaded_node()
        if node_id is None:
            log.debug("Give up sending metadata request since no node is available");
            return self.config['reconnect_backoff_ms']

        if self._can_send_request(node_id):
            topics = list(self._topics)
            if not topics and self.cluster.is_bootstrap(node_id):
                topics = list(self.config['bootstrap_topics_filter'])

            if self.cluster.need_all_topic_metadata or not topics:
                topics = [] if self.config['api_version'] < (0, 10) else None
            api_version = 0 if self.config['api_version'] < (0, 10) else 1
            request = MetadataRequest[api_version](topics)
            log.debug("Sending metadata request %s to node %s", request, node_id)
            future = self.send(node_id, request, wakeup=wakeup)
            future.add_callback(self.cluster.update_metadata)
            future.add_errback(self.cluster.failed_update)

            self._metadata_refresh_in_progress = True
            def refresh_done(val_or_error):
                self._metadata_refresh_in_progress = False
            future.add_callback(refresh_done)
            future.add_errback(refresh_done)
            return self.config['request_timeout_ms']

        # If there's any connection establishment underway, wait until it completes. This prevents
        # the client from unnecessarily connecting to additional nodes while a previous connection
        # attempt has not been completed.
        if self._connecting:
            return self.config['reconnect_backoff_ms']

        if self.maybe_connect(node_id, wakeup=wakeup):
            log.debug("Initializing connection to node %s for metadata request", node_id)
            return self.config['reconnect_backoff_ms']

        # connected but can't send more, OR connecting
        # In either case we just need to wait for a network event
        # to let us know the selected connection might be usable again.
        return float('inf')


Instruction:
Make a summary of the following Python 3 code
def check_version(self, node_id=None, timeout=2, strict=False):
        self._lock.acquire()
        end = time.time() + timeout
        while time.time() < end:

            # It is possible that least_loaded_node falls back to bootstrap,
            # which can block for an increasing backoff period
            try_node = node_id or self.least_loaded_node()
            if try_node is None:
                self._lock.release()
                raise Errors.NoBrokersAvailable()
            self._maybe_connect(try_node)
            conn = self._conns[try_node]

            # We will intentionally cause socket failures
            # These should not trigger metadata refresh
            self._refresh_on_disconnects = False
            try:
                remaining = end - time.time()
                version = conn.check_version(timeout=remaining, strict=strict, topics=list(self.config['bootstrap_topics_filter']))
                if version >= (0, 10, 0):
                    # cache the api versions map if it's available (starting
                    # in 0.10 cluster version)
                    self._api_versions = conn.get_api_versions()
                self._lock.release()
                return version
            except Errors.NodeNotReadyError:
                # Only raise to user if this is a node-specific request
                if node_id is not None:
                    self._lock.release()
                    raise
            finally:
                self._refresh_on_disconnects = True

        # Timeout
        else:
            self._lock.release()
            raise Errors.NoBrokersAvailable()

Response: Attempt to guess the version of a Kafka broker.


Instruction:
Make a summary of the following Python 3 code
def window_size(self, config, now):
        # purge old samples before we compute the window size
        self._stat.purge_obsolete_samples(config, now)

        """
        Here we check the total amount of time elapsed since the oldest
        non-obsolete window. This give the total window_size of the batch
        which is the time used for Rate computation. However, there is
        an issue if we do not have sufficient data for e.g. if only
        1 second has elapsed in a 30 second window, the measured rate
        will be very high. Hence we assume that the elapsed time is
        always N-1 complete windows plus whatever fraction of the final
        window is complete.

        Note that we could simply count the amount of time elapsed in
        the current window and add n-1 windows to get the total time,
        but this approach does not account for sleeps. AbstractSampledStat
        only creates samples whenever record is called, if no record is
        called for a period of time that time is not accounted for in
        window_size and produces incorrect results.
        """
        total_elapsed_time_ms = now - self._stat.oldest(now).last_window_ms
        # Check how many full windows of data we have currently retained
        num_full_windows = int(total_elapsed_time_ms / config.time_window_ms)
        min_full_windows = config.samples - 1

        # If the available windows are less than the minimum required,
        # add the difference to the totalElapsedTime
        if num_full_windows < min_full_windows:
            total_elapsed_time_ms += ((min_full_windows - num_full_windows) *
                                      config.time_window_ms)

        return total_elapsed_time_ms

Response: Compute the total size of the batch record for this time window.


Instruction:
How would you explain what the following Python 3 function does
def send_messages(self, topic, partition, *msg):
        return self._send_messages(topic, partition, *msg)

Response: This method sends a message to the broker.


Instruction:
How would you implement a function in Python 3 that
stops the producer (async mode). Blocks until async thread completes.

Response:
def stop(self, timeout=None):
        """
        Stop the producer (async mode). Blocks until async thread completes.
        """
        if timeout is not None:
            log.warning('timeout argument to stop() is deprecated - '
                        'it will be removed in future release')

        if not self.async_send:
            log.warning('producer.stop() called, but producer is not async')
            return

        if self.stopped:
            log.warning('producer.stop() called, but producer is already stopped')
            return

        if self.async_send:
            self.queue.put((STOP_ASYNC_PRODUCER, None, None))
            self.thread_stop_event.set()
            self.thread.join()

        if hasattr(self, '_cleanup_func'):
            # Remove cleanup handler now that we've stopped

            # py3 supports unregistering
            if hasattr(atexit, 'unregister'):
                atexit.unregister(self._cleanup_func)  # pylint: disable=no-member

            # py2 requires removing from private attribute...
            else:

                # ValueError on list.remove() if the exithandler no longer exists
                # but that is fine here
                try:
                    atexit._exithandlers.remove(  # pylint: disable=no-member
                        (self._cleanup_func, (self,), {}))
                except ValueError:
                    pass

            del self._cleanup_func

        self.stopped = True


Instruction:
Make a summary of the following Python 3 code
def append(self, offset, timestamp, key, value, headers=None):
        assert not headers, "Headers not supported in v0/v1"
        # Check types
        if type(offset) != int:
            raise TypeError(offset)
        if self._magic == 0:
            timestamp = self.NO_TIMESTAMP
        elif timestamp is None:
            timestamp = int(time.time() * 1000)
        elif type(timestamp) != int:
            raise TypeError(
                "`timestamp` should be int, but {} provided".format(
                    type(timestamp)))
        if not (key is None or
                isinstance(key, (bytes, bytearray, memoryview))):
            raise TypeError(
                "Not supported type for key: {}".format(type(key)))
        if not (value is None or
                isinstance(value, (bytes, bytearray, memoryview))):
            raise TypeError(
                "Not supported type for value: {}".format(type(value)))

        # Check if we have room for another message
        pos = len(self._buffer)
        size = self.size_in_bytes(offset, timestamp, key, value)
        # We always allow at least one record to be appended
        if offset != 0 and pos + size >= self._batch_size:
            return None

        # Allocate proper buffer length
        self._buffer.extend(bytearray(size))

        # Encode message
        crc = self._encode_msg(pos, offset, timestamp, key, value)

        return LegacyRecordMetadata(offset, crc, size, timestamp)

Response: Append a new record to the batch.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _encode_msg(self, start_pos, offset, timestamp, key, value,
                    attributes=0):
        """ Encode msg data into the `msg_buffer`, which should be allocated
            to at least the size of this message.
        """
        magic = self._magic
        buf = self._buffer
        pos = start_pos

        # Write key and value
        pos += self.KEY_OFFSET_V0 if magic == 0 else self.KEY_OFFSET_V1

        if key is None:
            struct.pack_into(">i", buf, pos, -1)
            pos += self.KEY_LENGTH
        else:
            key_size = len(key)
            struct.pack_into(">i", buf, pos, key_size)
            pos += self.KEY_LENGTH
            buf[pos: pos + key_size] = key
            pos += key_size

        if value is None:
            struct.pack_into(">i", buf, pos, -1)
            pos += self.VALUE_LENGTH
        else:
            value_size = len(value)
            struct.pack_into(">i", buf, pos, value_size)
            pos += self.VALUE_LENGTH
            buf[pos: pos + value_size] = value
            pos += value_size
        length = (pos - start_pos) - self.LOG_OVERHEAD

        # Write msg header. Note, that Crc will be updated later
        if magic == 0:
            self.HEADER_STRUCT_V0.pack_into(
                buf, start_pos,
                offset, length, 0, magic, attributes)
        else:
            self.HEADER_STRUCT_V1.pack_into(
                buf, start_pos,
                offset, length, 0, magic, attributes, timestamp)

        # Calculate CRC for msg
        crc_data = memoryview(buf)[start_pos + self.MAGIC_OFFSET:]
        crc = calc_crc32(crc_data)
        struct.pack_into(">I", buf, start_pos + self.CRC_OFFSET, crc)
        return crc

Response: Encode the message into the msg_buffer.


Instruction:
Write a Python 3 function that can
return the size of the record in bytes.

Response:
def size_in_bytes(self, offset, timestamp, key, value, headers=None):
        """ Actual size of message to add
        """
        assert not headers, "Headers not supported in v0/v1"
        magic = self._magic
        return self.LOG_OVERHEAD + self.record_size(magic, key, value)


Instruction:
Given the following Python 3 function, write the documentation
def estimate_size_in_bytes(cls, magic, compression_type, key, value):
        assert magic in [0, 1], "Not supported magic"
        # In case of compression we may need another overhead for inner msg
        if compression_type:
            return (
                cls.LOG_OVERHEAD + cls.record_overhead(magic) +
                cls.record_size(magic, key, value)
            )
        return cls.LOG_OVERHEAD + cls.record_size(magic, key, value)

Response: Estimate the size of a record in bytes.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def allocate(self, size, max_time_to_block_ms):
        with self._lock:
            # check if we have a free buffer of the right size pooled
            if self._free:
                return self._free.popleft()

            elif self._poolable_size == 0:
                return io.BytesIO()

            else:
                # we are out of buffers and will have to block
                buf = None
                more_memory = threading.Condition(self._lock)
                self._waiters.append(more_memory)
                # loop over and over until we have a buffer or have reserved
                # enough memory to allocate one
                while buf is None:
                    start_wait = time.time()
                    more_memory.wait(max_time_to_block_ms / 1000.0)
                    end_wait = time.time()
                    if self.wait_time:
                        self.wait_time.record(end_wait - start_wait)

                    if self._free:
                        buf = self._free.popleft()
                    else:
                        self._waiters.remove(more_memory)
                        raise Errors.KafkaTimeoutError(
                            "Failed to allocate memory within the configured"
                            " max blocking time")

                # remove the condition for this thread to let the next thread
                # in line start getting memory
                removed = self._waiters.popleft()
                assert removed is more_memory, 'Wrong condition'

                # signal any additional waiters if there is more memory left
                # over for them
                if self._free and self._waiters:
                    self._waiters[0].notify()

                # unlock and return the buffer
                return buf

Response: Allocate a buffer of the given size. This method blocks if there is not enough memory and the buffer pool is configured with blocking mode.


Instruction:
Can you tell what is the following Python 3 function doing
def deallocate(self, buf):
        with self._lock:
            # BytesIO.truncate here makes the pool somewhat pointless
            # but we stick with the BufferPool API until migrating to
            # bytesarray / memoryview. The buffer we return must not
            # expose any prior data on read().
            buf.truncate(0)
            self._free.append(buf)
            if self._waiters:
                self._waiters[0].notify()

Response: Deallocate a buffer from the pool.


Instruction:
Implement a function in Python 3 to
decode a message set into a list of items.

Response:
def decode(cls, data, bytes_to_read=None):
        """Compressed messages should pass in bytes_to_read (via message size)
        otherwise, we decode from data as Int32
        """
        if isinstance(data, bytes):
            data = io.BytesIO(data)
        if bytes_to_read is None:
            bytes_to_read = Int32.decode(data)

        # if FetchRequest max_bytes is smaller than the available message set
        # the server returns partial data for the final message
        # So create an internal buffer to avoid over-reading
        raw = io.BytesIO(data.read(bytes_to_read))

        items = []
        while bytes_to_read:
            try:
                offset = Int64.decode(raw)
                msg_bytes = Bytes.decode(raw)
                bytes_to_read -= 8 + 4 + len(msg_bytes)
                items.append((offset, len(msg_bytes), Message.decode(msg_bytes)))
            except ValueError:
                # PartialMessage to signal that max_bytes may be too small
                items.append((None, None, PartialMessage()))
                break
        return items


Instruction:
Write a Python 3 script for
encoding an integer to a varint presentation.

Response:
def encode_varint(value, write):
    """ Encode an integer to a varint presentation. See
    https://developers.google.com/protocol-buffers/docs/encoding?csw=1#varints
    on how those can be produced.

        Arguments:
            value (int): Value to encode
            write (function): Called per byte that needs to be writen

        Returns:
            int: Number of bytes written
    """
    value = (value << 1) ^ (value >> 63)

    if value <= 0x7f:  # 1 byte
        write(value)
        return 1
    if value <= 0x3fff:  # 2 bytes
        write(0x80 | (value & 0x7f))
        write(value >> 7)
        return 2
    if value <= 0x1fffff:  # 3 bytes
        write(0x80 | (value & 0x7f))
        write(0x80 | ((value >> 7) & 0x7f))
        write(value >> 14)
        return 3
    if value <= 0xfffffff:  # 4 bytes
        write(0x80 | (value & 0x7f))
        write(0x80 | ((value >> 7) & 0x7f))
        write(0x80 | ((value >> 14) & 0x7f))
        write(value >> 21)
        return 4
    if value <= 0x7ffffffff:  # 5 bytes
        write(0x80 | (value & 0x7f))
        write(0x80 | ((value >> 7) & 0x7f))
        write(0x80 | ((value >> 14) & 0x7f))
        write(0x80 | ((value >> 21) & 0x7f))
        write(value >> 28)
        return 5
    else:
        # Return to general algorithm
        bits = value & 0x7f
        value >>= 7
        i = 0
        while value:
            write(0x80 | bits)
            bits = value & 0x7f
            value >>= 7
            i += 1
    write(bits)
    return i


Instruction:
Create a Python 3 function to
return the number of bytes needed to encode an integer in variable - length format.

Response:
def size_of_varint(value):
    """ Number of bytes needed to encode an integer in variable-length format.
    """
    value = (value << 1) ^ (value >> 63)
    if value <= 0x7f:
        return 1
    if value <= 0x3fff:
        return 2
    if value <= 0x1fffff:
        return 3
    if value <= 0xfffffff:
        return 4
    if value <= 0x7ffffffff:
        return 5
    if value <= 0x3ffffffffff:
        return 6
    if value <= 0x1ffffffffffff:
        return 7
    if value <= 0xffffffffffffff:
        return 8
    if value <= 0x7fffffffffffffff:
        return 9
    return 10


Instruction:
Can you generate a brief explanation for the following Python 3 code
def decode_varint(buffer, pos=0):
    result = buffer[pos]
    if not (result & 0x81):
        return (result >> 1), pos + 1
    if not (result & 0x80):
        return (result >> 1) ^ (~0), pos + 1

    result &= 0x7f
    pos += 1
    shift = 7
    while 1:
        b = buffer[pos]
        result |= ((b & 0x7f) << shift)
        pos += 1
        if not (b & 0x80):
            return ((result >> 1) ^ -(result & 1), pos)
        shift += 7
        if shift >= 64:
            raise ValueError("Out of int64 range")

Response: Decodes an integer from a varint presentation.


Instruction:
Here you have a function in Python 3, explain what it does
def send_request(self, request, correlation_id=None):
        log.debug('Sending request %s', request)
        if correlation_id is None:
            correlation_id = self._next_correlation_id()
        header = RequestHeader(request,
                               correlation_id=correlation_id,
                               client_id=self._client_id)
        message = b''.join([header.encode(), request.encode()])
        size = Int32.encode(len(message))
        data = size + message
        self.bytes_to_send.append(data)
        if request.expect_response():
            ifr = (correlation_id, request)
            self.in_flight_requests.append(ifr)
        return correlation_id

Response: Encode and queue a kafka api request for sending.


Instruction:
Create a Python 3 function for
retrieving all pending bytes to send on the network

Response:
def send_bytes(self):
        """Retrieve all pending bytes to send on the network"""
        data = b''.join(self.bytes_to_send)
        self.bytes_to_send = []
        return data